1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pagewalk.h> 3 #include <linux/mm_inline.h> 4 #include <linux/hugetlb.h> 5 #include <linux/huge_mm.h> 6 #include <linux/mount.h> 7 #include <linux/ksm.h> 8 #include <linux/seq_file.h> 9 #include <linux/highmem.h> 10 #include <linux/ptrace.h> 11 #include <linux/slab.h> 12 #include <linux/pagemap.h> 13 #include <linux/mempolicy.h> 14 #include <linux/rmap.h> 15 #include <linux/swap.h> 16 #include <linux/sched/mm.h> 17 #include <linux/leafops.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/page_idle.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/uaccess.h> 22 #include <linux/pkeys.h> 23 #include <linux/minmax.h> 24 #include <linux/overflow.h> 25 #include <linux/buildid.h> 26 27 #include <asm/elf.h> 28 #include <asm/tlb.h> 29 #include <asm/tlbflush.h> 30 #include "internal.h" 31 32 #define SENTINEL_VMA_END -1 33 #define SENTINEL_VMA_GATE -2 34 35 #define SEQ_PUT_DEC(str, val) \ 36 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) 37 void task_mem(struct seq_file *m, struct mm_struct *mm) 38 { 39 unsigned long text, lib, swap, anon, file, shmem; 40 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; 41 42 anon = get_mm_counter_sum(mm, MM_ANONPAGES); 43 file = get_mm_counter_sum(mm, MM_FILEPAGES); 44 shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES); 45 46 /* 47 * Note: to minimize their overhead, mm maintains hiwater_vm and 48 * hiwater_rss only when about to *lower* total_vm or rss. Any 49 * collector of these hiwater stats must therefore get total_vm 50 * and rss too, which will usually be the higher. Barriers? not 51 * worth the effort, such snapshots can always be inconsistent. 52 */ 53 hiwater_vm = total_vm = mm->total_vm; 54 if (hiwater_vm < mm->hiwater_vm) 55 hiwater_vm = mm->hiwater_vm; 56 hiwater_rss = total_rss = anon + file + shmem; 57 if (hiwater_rss < mm->hiwater_rss) 58 hiwater_rss = mm->hiwater_rss; 59 60 /* split executable areas between text and lib */ 61 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); 62 text = min(text, mm->exec_vm << PAGE_SHIFT); 63 lib = (mm->exec_vm << PAGE_SHIFT) - text; 64 65 swap = get_mm_counter_sum(mm, MM_SWAPENTS); 66 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); 67 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); 68 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); 69 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); 70 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); 71 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); 72 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); 73 SEQ_PUT_DEC(" kB\nRssFile:\t", file); 74 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem); 75 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm); 76 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm); 77 seq_put_decimal_ull_width(m, 78 " kB\nVmExe:\t", text >> 10, 8); 79 seq_put_decimal_ull_width(m, 80 " kB\nVmLib:\t", lib >> 10, 8); 81 seq_put_decimal_ull_width(m, 82 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8); 83 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap); 84 seq_puts(m, " kB\n"); 85 hugetlb_report_usage(m, mm); 86 } 87 #undef SEQ_PUT_DEC 88 89 unsigned long task_vsize(struct mm_struct *mm) 90 { 91 return PAGE_SIZE * mm->total_vm; 92 } 93 94 unsigned long task_statm(struct mm_struct *mm, 95 unsigned long *shared, unsigned long *text, 96 unsigned long *data, unsigned long *resident) 97 { 98 *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + 99 get_mm_counter_sum(mm, MM_SHMEMPAGES); 100 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) 101 >> PAGE_SHIFT; 102 *data = mm->data_vm + mm->stack_vm; 103 *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); 104 return mm->total_vm; 105 } 106 107 #ifdef CONFIG_NUMA 108 /* 109 * Save get_task_policy() for show_numa_map(). 110 */ 111 static void hold_task_mempolicy(struct proc_maps_private *priv) 112 { 113 struct task_struct *task = priv->task; 114 115 task_lock(task); 116 priv->task_mempolicy = get_task_policy(task); 117 mpol_get(priv->task_mempolicy); 118 task_unlock(task); 119 } 120 static void release_task_mempolicy(struct proc_maps_private *priv) 121 { 122 mpol_put(priv->task_mempolicy); 123 } 124 #else 125 static void hold_task_mempolicy(struct proc_maps_private *priv) 126 { 127 } 128 static void release_task_mempolicy(struct proc_maps_private *priv) 129 { 130 } 131 #endif 132 133 #ifdef CONFIG_PER_VMA_LOCK 134 135 static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 136 { 137 int ret = mmap_read_lock_killable(lock_ctx->mm); 138 139 if (!ret) 140 lock_ctx->mmap_locked = true; 141 142 return ret; 143 } 144 145 static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 146 { 147 mmap_read_unlock(lock_ctx->mm); 148 lock_ctx->mmap_locked = false; 149 } 150 151 static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) 152 { 153 lock_ctx->locked_vma = NULL; 154 lock_ctx->mmap_locked = false; 155 } 156 157 static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) 158 { 159 if (lock_ctx->locked_vma) { 160 vma_end_read(lock_ctx->locked_vma); 161 lock_ctx->locked_vma = NULL; 162 } 163 } 164 165 static inline bool lock_vma_range(struct seq_file *m, 166 struct proc_maps_locking_ctx *lock_ctx) 167 { 168 rcu_read_lock(); 169 reset_lock_ctx(lock_ctx); 170 171 return true; 172 } 173 174 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) 175 { 176 if (lock_ctx->mmap_locked) { 177 unlock_ctx_mm(lock_ctx); 178 } else { 179 unlock_ctx_vma(lock_ctx); 180 rcu_read_unlock(); 181 } 182 } 183 184 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, 185 loff_t last_pos) 186 { 187 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 188 struct vm_area_struct *vma; 189 190 if (lock_ctx->mmap_locked) 191 return vma_next(&priv->iter); 192 193 unlock_ctx_vma(lock_ctx); 194 vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); 195 if (!IS_ERR_OR_NULL(vma)) 196 lock_ctx->locked_vma = vma; 197 198 return vma; 199 } 200 201 static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, 202 loff_t pos) 203 { 204 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 205 206 if (lock_ctx->mmap_locked) 207 return false; 208 209 rcu_read_unlock(); 210 mmap_read_lock(lock_ctx->mm); 211 /* Reinitialize the iterator after taking mmap_lock */ 212 vma_iter_set(&priv->iter, pos); 213 lock_ctx->mmap_locked = true; 214 215 return true; 216 } 217 218 static inline void drop_rcu(struct proc_maps_private *priv) 219 { 220 if (priv->lock_ctx.mmap_locked) 221 return; 222 223 rcu_read_unlock(); 224 } 225 226 static inline void reacquire_rcu(struct proc_maps_private *priv) 227 { 228 if (priv->lock_ctx.mmap_locked) 229 return; 230 231 rcu_read_lock(); 232 /* Reinitialize the iterator. */ 233 vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); 234 } 235 236 #else /* CONFIG_PER_VMA_LOCK */ 237 238 static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 239 { 240 return mmap_read_lock_killable(lock_ctx->mm); 241 } 242 243 static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 244 { 245 mmap_read_unlock(lock_ctx->mm); 246 } 247 248 static inline bool lock_vma_range(struct seq_file *m, 249 struct proc_maps_locking_ctx *lock_ctx) 250 { 251 return lock_ctx_mm(lock_ctx) == 0; 252 } 253 254 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) 255 { 256 unlock_ctx_mm(lock_ctx); 257 } 258 259 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, 260 loff_t last_pos) 261 { 262 return vma_next(&priv->iter); 263 } 264 265 static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, 266 loff_t pos) 267 { 268 return false; 269 } 270 271 static inline void drop_rcu(struct proc_maps_private *priv) {} 272 static inline void reacquire_rcu(struct proc_maps_private *priv) {} 273 274 #endif /* CONFIG_PER_VMA_LOCK */ 275 276 static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) 277 { 278 struct proc_maps_private *priv = m->private; 279 struct vm_area_struct *vma; 280 281 retry: 282 vma = get_next_vma(priv, *ppos); 283 /* EINTR of EAGAIN is possible */ 284 if (IS_ERR(vma)) { 285 if (PTR_ERR(vma) == -EAGAIN && fallback_to_mmap_lock(priv, *ppos)) 286 goto retry; 287 288 return vma; 289 } 290 291 /* Store previous position to be able to restart if needed */ 292 priv->last_pos = *ppos; 293 if (vma) { 294 /* 295 * Track the end of the reported vma to ensure position changes 296 * even if previous vma was merged with the next vma and we 297 * found the extended vma with the same vm_start. 298 */ 299 *ppos = vma->vm_end; 300 } else { 301 *ppos = SENTINEL_VMA_GATE; 302 vma = get_gate_vma(priv->lock_ctx.mm); 303 } 304 305 return vma; 306 } 307 308 static void *m_start(struct seq_file *m, loff_t *ppos) 309 { 310 struct proc_maps_private *priv = m->private; 311 struct proc_maps_locking_ctx *lock_ctx; 312 loff_t last_addr = *ppos; 313 struct mm_struct *mm; 314 315 /* See m_next(). Zero at the start or after lseek. */ 316 if (last_addr == SENTINEL_VMA_END) 317 return NULL; 318 319 priv->task = get_proc_task(priv->inode); 320 if (!priv->task) 321 return ERR_PTR(-ESRCH); 322 323 lock_ctx = &priv->lock_ctx; 324 mm = lock_ctx->mm; 325 if (!mm || !mmget_not_zero(mm)) { 326 put_task_struct(priv->task); 327 priv->task = NULL; 328 return NULL; 329 } 330 331 if (!lock_vma_range(m, lock_ctx)) { 332 mmput(mm); 333 put_task_struct(priv->task); 334 priv->task = NULL; 335 return ERR_PTR(-EINTR); 336 } 337 338 /* 339 * Reset current position if last_addr was set before 340 * and it's not a sentinel. 341 */ 342 if (last_addr > 0) 343 *ppos = last_addr = priv->last_pos; 344 vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); 345 hold_task_mempolicy(priv); 346 if (last_addr == SENTINEL_VMA_GATE) 347 return get_gate_vma(mm); 348 349 return proc_get_vma(m, ppos); 350 } 351 352 static void *m_next(struct seq_file *m, void *v, loff_t *ppos) 353 { 354 if (*ppos == SENTINEL_VMA_GATE) { 355 *ppos = SENTINEL_VMA_END; 356 return NULL; 357 } 358 return proc_get_vma(m, ppos); 359 } 360 361 static void m_stop(struct seq_file *m, void *v) 362 { 363 struct proc_maps_private *priv = m->private; 364 struct mm_struct *mm = priv->lock_ctx.mm; 365 366 if (!priv->task) 367 return; 368 369 release_task_mempolicy(priv); 370 unlock_vma_range(&priv->lock_ctx); 371 mmput(mm); 372 put_task_struct(priv->task); 373 priv->task = NULL; 374 } 375 376 static int proc_maps_open(struct inode *inode, struct file *file, 377 const struct seq_operations *ops, int psize) 378 { 379 struct proc_maps_private *priv = __seq_open_private(file, ops, psize); 380 381 if (!priv) 382 return -ENOMEM; 383 384 priv->inode = inode; 385 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 386 if (IS_ERR(priv->lock_ctx.mm)) { 387 int err = PTR_ERR(priv->lock_ctx.mm); 388 389 seq_release_private(inode, file); 390 return err; 391 } 392 393 return 0; 394 } 395 396 static int proc_map_release(struct inode *inode, struct file *file) 397 { 398 struct seq_file *seq = file->private_data; 399 struct proc_maps_private *priv = seq->private; 400 401 if (priv->lock_ctx.mm) 402 mmdrop(priv->lock_ctx.mm); 403 404 return seq_release_private(inode, file); 405 } 406 407 static int do_maps_open(struct inode *inode, struct file *file, 408 const struct seq_operations *ops) 409 { 410 return proc_maps_open(inode, file, ops, 411 sizeof(struct proc_maps_private)); 412 } 413 414 static void get_vma_name(struct vm_area_struct *vma, 415 const struct path **path, 416 const char **name, 417 const char **name_fmt) 418 { 419 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL; 420 421 *name = NULL; 422 *path = NULL; 423 *name_fmt = NULL; 424 425 /* 426 * Print the dentry name for named mappings, and a 427 * special [heap] marker for the heap: 428 */ 429 if (vma->vm_file) { 430 /* 431 * If user named this anon shared memory via 432 * prctl(PR_SET_VMA ..., use the provided name. 433 */ 434 if (anon_name) { 435 *name_fmt = "[anon_shmem:%s]"; 436 *name = anon_name->name; 437 } else { 438 *path = file_user_path(vma->vm_file); 439 } 440 return; 441 } 442 443 if (vma->vm_ops && vma->vm_ops->name) { 444 *name = vma->vm_ops->name(vma); 445 if (*name) 446 return; 447 } 448 449 *name = arch_vma_name(vma); 450 if (*name) 451 return; 452 453 if (!vma->vm_mm) { 454 *name = "[vdso]"; 455 return; 456 } 457 458 if (vma_is_initial_heap(vma)) { 459 *name = "[heap]"; 460 return; 461 } 462 463 if (vma_is_initial_stack(vma)) { 464 *name = "[stack]"; 465 return; 466 } 467 468 if (anon_name) { 469 *name_fmt = "[anon:%s]"; 470 *name = anon_name->name; 471 return; 472 } 473 } 474 475 static void show_vma_header_prefix(struct seq_file *m, 476 unsigned long start, unsigned long end, 477 vm_flags_t flags, unsigned long long pgoff, 478 dev_t dev, u64 ino) 479 { 480 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); 481 seq_put_hex_ll(m, NULL, start, 8); 482 seq_put_hex_ll(m, "-", end, 8); 483 seq_putc(m, ' '); 484 seq_putc(m, flags & VM_READ ? 'r' : '-'); 485 seq_putc(m, flags & VM_WRITE ? 'w' : '-'); 486 seq_putc(m, flags & VM_EXEC ? 'x' : '-'); 487 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p'); 488 seq_put_hex_ll(m, " ", pgoff, 8); 489 seq_put_hex_ll(m, " ", MAJOR(dev), 2); 490 seq_put_hex_ll(m, ":", MINOR(dev), 2); 491 seq_put_decimal_ull(m, " ", ino); 492 seq_putc(m, ' '); 493 } 494 495 static void 496 show_map_vma(struct seq_file *m, struct vm_area_struct *vma) 497 { 498 const struct path *path; 499 const char *name_fmt, *name; 500 vm_flags_t flags = vma->vm_flags; 501 u64 ino = 0; 502 unsigned long long pgoff = 0; 503 unsigned long start, end; 504 dev_t dev = 0; 505 506 if (vma->vm_file) { 507 const struct inode *inode = file_user_inode(vma->vm_file); 508 509 dev = inode->i_sb->s_dev; 510 ino = inode->i_ino; 511 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; 512 } 513 514 start = vma->vm_start; 515 end = vma->vm_end; 516 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino); 517 518 get_vma_name(vma, &path, &name, &name_fmt); 519 if (path) { 520 seq_pad(m, ' '); 521 seq_path(m, path, "\n"); 522 } else if (name_fmt) { 523 seq_pad(m, ' '); 524 seq_printf(m, name_fmt, name); 525 } else if (name) { 526 seq_pad(m, ' '); 527 seq_puts(m, name); 528 } 529 seq_putc(m, '\n'); 530 } 531 532 static int show_map(struct seq_file *m, void *v) 533 { 534 show_map_vma(m, v); 535 return 0; 536 } 537 538 static const struct seq_operations proc_pid_maps_op = { 539 .start = m_start, 540 .next = m_next, 541 .stop = m_stop, 542 .show = show_map 543 }; 544 545 static int pid_maps_open(struct inode *inode, struct file *file) 546 { 547 return do_maps_open(inode, file, &proc_pid_maps_op); 548 } 549 550 #define PROCMAP_QUERY_VMA_FLAGS ( \ 551 PROCMAP_QUERY_VMA_READABLE | \ 552 PROCMAP_QUERY_VMA_WRITABLE | \ 553 PROCMAP_QUERY_VMA_EXECUTABLE | \ 554 PROCMAP_QUERY_VMA_SHARED \ 555 ) 556 557 #define PROCMAP_QUERY_VALID_FLAGS_MASK ( \ 558 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \ 559 PROCMAP_QUERY_FILE_BACKED_VMA | \ 560 PROCMAP_QUERY_VMA_FLAGS \ 561 ) 562 563 #ifdef CONFIG_PER_VMA_LOCK 564 565 static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) 566 { 567 reset_lock_ctx(lock_ctx); 568 569 return 0; 570 } 571 572 static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) 573 { 574 if (lock_ctx->mmap_locked) 575 unlock_ctx_mm(lock_ctx); 576 else 577 unlock_ctx_vma(lock_ctx); 578 } 579 580 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, 581 unsigned long addr) 582 { 583 struct mm_struct *mm = lock_ctx->mm; 584 struct vm_area_struct *vma; 585 struct vma_iterator vmi; 586 587 if (lock_ctx->mmap_locked) 588 return find_vma(mm, addr); 589 590 /* Unlock previously locked VMA and find the next one under RCU */ 591 unlock_ctx_vma(lock_ctx); 592 rcu_read_lock(); 593 vma_iter_init(&vmi, mm, addr); 594 vma = lock_next_vma(mm, &vmi, addr); 595 rcu_read_unlock(); 596 597 if (!vma) 598 return NULL; 599 600 if (!IS_ERR(vma)) { 601 lock_ctx->locked_vma = vma; 602 return vma; 603 } 604 605 if (PTR_ERR(vma) == -EAGAIN) { 606 /* Fallback to mmap_lock on vma->vm_refcnt overflow */ 607 mmap_read_lock(mm); 608 vma = find_vma(mm, addr); 609 lock_ctx->mmap_locked = true; 610 } 611 612 return vma; 613 } 614 615 #else /* CONFIG_PER_VMA_LOCK */ 616 617 static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) 618 { 619 return mmap_read_lock_killable(lock_ctx->mm); 620 } 621 622 static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) 623 { 624 mmap_read_unlock(lock_ctx->mm); 625 } 626 627 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, 628 unsigned long addr) 629 { 630 return find_vma(lock_ctx->mm, addr); 631 } 632 633 #endif /* CONFIG_PER_VMA_LOCK */ 634 635 static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, 636 unsigned long addr, u32 flags) 637 { 638 struct vm_area_struct *vma; 639 640 next_vma: 641 vma = query_vma_find_by_addr(lock_ctx, addr); 642 if (IS_ERR(vma)) 643 return vma; 644 645 if (!vma) 646 goto no_vma; 647 648 /* user requested only file-backed VMA, keep iterating */ 649 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file) 650 goto skip_vma; 651 652 /* VMA permissions should satisfy query flags */ 653 if (flags & PROCMAP_QUERY_VMA_FLAGS) { 654 u32 perm = 0; 655 656 if (flags & PROCMAP_QUERY_VMA_READABLE) 657 perm |= VM_READ; 658 if (flags & PROCMAP_QUERY_VMA_WRITABLE) 659 perm |= VM_WRITE; 660 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE) 661 perm |= VM_EXEC; 662 if (flags & PROCMAP_QUERY_VMA_SHARED) 663 perm |= VM_MAYSHARE; 664 665 if ((vma->vm_flags & perm) != perm) 666 goto skip_vma; 667 } 668 669 /* found covering VMA or user is OK with the matching next VMA */ 670 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) 671 return vma; 672 673 skip_vma: 674 /* 675 * If the user needs closest matching VMA, keep iterating. 676 */ 677 addr = vma->vm_end; 678 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) 679 goto next_vma; 680 681 no_vma: 682 return ERR_PTR(-ENOENT); 683 } 684 685 static int do_procmap_query(struct mm_struct *mm, void __user *uarg) 686 { 687 struct proc_maps_locking_ctx lock_ctx = { .mm = mm }; 688 struct procmap_query karg; 689 struct vm_area_struct *vma; 690 struct file *vm_file = NULL; 691 const char *name = NULL; 692 char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; 693 __u64 usize; 694 int err; 695 696 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize))) 697 return -EFAULT; 698 /* argument struct can never be that large, reject abuse */ 699 if (usize > PAGE_SIZE) 700 return -E2BIG; 701 /* argument struct should have at least query_flags and query_addr fields */ 702 if (usize < offsetofend(struct procmap_query, query_addr)) 703 return -EINVAL; 704 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); 705 if (err) 706 return err; 707 708 /* reject unknown flags */ 709 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK) 710 return -EINVAL; 711 /* either both buffer address and size are set, or both should be zero */ 712 if (!!karg.vma_name_size != !!karg.vma_name_addr) 713 return -EINVAL; 714 if (!!karg.build_id_size != !!karg.build_id_addr) 715 return -EINVAL; 716 717 if (!mm || !mmget_not_zero(mm)) 718 return -ESRCH; 719 720 err = query_vma_setup(&lock_ctx); 721 if (err) { 722 mmput(mm); 723 return err; 724 } 725 726 vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); 727 if (IS_ERR(vma)) { 728 err = PTR_ERR(vma); 729 vma = NULL; 730 goto out; 731 } 732 733 karg.vma_start = vma->vm_start; 734 karg.vma_end = vma->vm_end; 735 736 karg.vma_flags = 0; 737 if (vma->vm_flags & VM_READ) 738 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE; 739 if (vma->vm_flags & VM_WRITE) 740 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE; 741 if (vma->vm_flags & VM_EXEC) 742 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE; 743 if (vma->vm_flags & VM_MAYSHARE) 744 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED; 745 746 karg.vma_page_size = vma_kernel_pagesize(vma); 747 748 if (vma->vm_file) { 749 const struct inode *inode = file_user_inode(vma->vm_file); 750 751 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; 752 karg.dev_major = MAJOR(inode->i_sb->s_dev); 753 karg.dev_minor = MINOR(inode->i_sb->s_dev); 754 karg.inode = inode->i_ino; 755 } else { 756 karg.vma_offset = 0; 757 karg.dev_major = 0; 758 karg.dev_minor = 0; 759 karg.inode = 0; 760 } 761 762 if (karg.vma_name_size) { 763 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); 764 const struct path *path; 765 const char *name_fmt; 766 size_t name_sz = 0; 767 768 get_vma_name(vma, &path, &name, &name_fmt); 769 770 if (path || name_fmt || name) { 771 name_buf = kmalloc(name_buf_sz, GFP_KERNEL); 772 if (!name_buf) { 773 err = -ENOMEM; 774 goto out; 775 } 776 } 777 if (path) { 778 name = d_path(path, name_buf, name_buf_sz); 779 if (IS_ERR(name)) { 780 err = PTR_ERR(name); 781 goto out; 782 } 783 name_sz = name_buf + name_buf_sz - name; 784 } else if (name || name_fmt) { 785 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name); 786 name = name_buf; 787 } 788 if (name_sz > name_buf_sz) { 789 err = -ENAMETOOLONG; 790 goto out; 791 } 792 karg.vma_name_size = name_sz; 793 } 794 795 if (karg.build_id_size && vma->vm_file) 796 vm_file = get_file(vma->vm_file); 797 798 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ 799 query_vma_teardown(&lock_ctx); 800 mmput(mm); 801 802 if (karg.build_id_size) { 803 __u32 build_id_sz; 804 805 if (vm_file) 806 err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz); 807 else 808 err = -ENOENT; 809 if (err) { 810 karg.build_id_size = 0; 811 } else { 812 if (karg.build_id_size < build_id_sz) { 813 err = -ENAMETOOLONG; 814 goto out_file; 815 } 816 karg.build_id_size = build_id_sz; 817 } 818 } 819 820 if (vm_file) 821 fput(vm_file); 822 823 if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), 824 name, karg.vma_name_size)) { 825 kfree(name_buf); 826 return -EFAULT; 827 } 828 kfree(name_buf); 829 830 if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr), 831 build_id_buf, karg.build_id_size)) 832 return -EFAULT; 833 834 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize))) 835 return -EFAULT; 836 837 return 0; 838 839 out: 840 query_vma_teardown(&lock_ctx); 841 mmput(mm); 842 out_file: 843 if (vm_file) 844 fput(vm_file); 845 kfree(name_buf); 846 return err; 847 } 848 849 static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 850 { 851 struct seq_file *seq = file->private_data; 852 struct proc_maps_private *priv = seq->private; 853 854 switch (cmd) { 855 case PROCMAP_QUERY: 856 /* priv->lock_ctx.mm is set during file open operation */ 857 return do_procmap_query(priv->lock_ctx.mm, (void __user *)arg); 858 default: 859 return -ENOIOCTLCMD; 860 } 861 } 862 863 const struct file_operations proc_pid_maps_operations = { 864 .open = pid_maps_open, 865 .read = seq_read, 866 .llseek = seq_lseek, 867 .release = proc_map_release, 868 .unlocked_ioctl = procfs_procmap_ioctl, 869 .compat_ioctl = compat_ptr_ioctl, 870 }; 871 872 /* 873 * Proportional Set Size(PSS): my share of RSS. 874 * 875 * PSS of a process is the count of pages it has in memory, where each 876 * page is divided by the number of processes sharing it. So if a 877 * process has 1000 pages all to itself, and 1000 shared with one other 878 * process, its PSS will be 1500. 879 * 880 * To keep (accumulated) division errors low, we adopt a 64bit 881 * fixed-point pss counter to minimize division errors. So (pss >> 882 * PSS_SHIFT) would be the real byte count. 883 * 884 * A shift of 12 before division means (assuming 4K page size): 885 * - 1M 3-user-pages add up to 8KB errors; 886 * - supports mapcount up to 2^24, or 16M; 887 * - supports PSS up to 2^52 bytes, or 4PB. 888 */ 889 #define PSS_SHIFT 12 890 891 #ifdef CONFIG_PROC_PAGE_MONITOR 892 struct mem_size_stats { 893 unsigned long resident; 894 unsigned long shared_clean; 895 unsigned long shared_dirty; 896 unsigned long private_clean; 897 unsigned long private_dirty; 898 unsigned long referenced; 899 unsigned long anonymous; 900 unsigned long lazyfree; 901 unsigned long anonymous_thp; 902 unsigned long shmem_thp; 903 unsigned long file_thp; 904 unsigned long swap; 905 unsigned long shared_hugetlb; 906 unsigned long private_hugetlb; 907 unsigned long ksm; 908 u64 pss; 909 u64 pss_anon; 910 u64 pss_file; 911 u64 pss_shmem; 912 u64 pss_dirty; 913 u64 pss_locked; 914 u64 swap_pss; 915 }; 916 917 static void smaps_page_accumulate(struct mem_size_stats *mss, 918 struct folio *folio, unsigned long size, unsigned long pss, 919 bool dirty, bool locked, bool private) 920 { 921 mss->pss += pss; 922 923 if (folio_test_anon(folio)) 924 mss->pss_anon += pss; 925 else if (folio_test_swapbacked(folio)) 926 mss->pss_shmem += pss; 927 else 928 mss->pss_file += pss; 929 930 if (locked) 931 mss->pss_locked += pss; 932 933 if (dirty || folio_test_dirty(folio)) { 934 mss->pss_dirty += pss; 935 if (private) 936 mss->private_dirty += size; 937 else 938 mss->shared_dirty += size; 939 } else { 940 if (private) 941 mss->private_clean += size; 942 else 943 mss->shared_clean += size; 944 } 945 } 946 947 static void smaps_account(struct mem_size_stats *mss, struct page *page, 948 bool compound, bool young, bool dirty, bool locked, 949 bool present) 950 { 951 struct folio *folio = page_folio(page); 952 int i, nr = compound ? compound_nr(page) : 1; 953 unsigned long size = nr * PAGE_SIZE; 954 bool exclusive; 955 int mapcount; 956 957 /* 958 * First accumulate quantities that depend only on |size| and the type 959 * of the compound page. 960 */ 961 if (folio_test_anon(folio)) { 962 mss->anonymous += size; 963 if (!folio_test_swapbacked(folio) && !dirty && 964 !folio_test_dirty(folio)) 965 mss->lazyfree += size; 966 } 967 968 if (folio_test_ksm(folio)) 969 mss->ksm += size; 970 971 mss->resident += size; 972 /* Accumulate the size in pages that have been accessed. */ 973 if (young || folio_test_young(folio) || folio_test_referenced(folio)) 974 mss->referenced += size; 975 976 /* 977 * Then accumulate quantities that may depend on sharing, or that may 978 * differ page-by-page. 979 * 980 * refcount == 1 for present entries guarantees that the folio is mapped 981 * exactly once. For large folios this implies that exactly one 982 * PTE/PMD/... maps (a part of) this folio. 983 * 984 * Treat all non-present entries (where relying on the mapcount and 985 * refcount doesn't make sense) as "maybe shared, but not sure how 986 * often". We treat device private entries as being fake-present. 987 * 988 * Note that it would not be safe to read the mapcount especially for 989 * pages referenced by migration entries, even with the PTL held. 990 */ 991 if (folio_ref_count(folio) == 1 || !present) { 992 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, 993 dirty, locked, present); 994 return; 995 } 996 997 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 998 mapcount = folio_average_page_mapcount(folio); 999 exclusive = !folio_maybe_mapped_shared(folio); 1000 } 1001 1002 /* 1003 * We obtain a snapshot of the mapcount. Without holding the folio lock 1004 * this snapshot can be slightly wrong as we cannot always read the 1005 * mapcount atomically. 1006 */ 1007 for (i = 0; i < nr; i++, page++) { 1008 unsigned long pss = PAGE_SIZE << PSS_SHIFT; 1009 1010 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { 1011 mapcount = folio_precise_page_mapcount(folio, page); 1012 exclusive = mapcount < 2; 1013 } 1014 1015 if (mapcount >= 2) 1016 pss /= mapcount; 1017 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, 1018 dirty, locked, exclusive); 1019 } 1020 } 1021 1022 #ifdef CONFIG_SHMEM 1023 static int smaps_pte_hole(unsigned long addr, unsigned long end, 1024 __always_unused int depth, struct mm_walk *walk) 1025 { 1026 struct mem_size_stats *mss = walk->private; 1027 struct vm_area_struct *vma = walk->vma; 1028 1029 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, 1030 linear_page_index(vma, addr), 1031 linear_page_index(vma, end)); 1032 1033 return 0; 1034 } 1035 #else 1036 #define smaps_pte_hole NULL 1037 #endif /* CONFIG_SHMEM */ 1038 1039 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) 1040 { 1041 #ifdef CONFIG_SHMEM 1042 if (walk->ops->pte_hole) { 1043 /* depth is not used */ 1044 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); 1045 } 1046 #endif 1047 } 1048 1049 static void smaps_pte_entry(pte_t *pte, unsigned long addr, 1050 struct mm_walk *walk) 1051 { 1052 struct mem_size_stats *mss = walk->private; 1053 struct vm_area_struct *vma = walk->vma; 1054 bool locked = !!(vma->vm_flags & VM_LOCKED); 1055 struct page *page = NULL; 1056 bool present = false, young = false, dirty = false; 1057 pte_t ptent = ptep_get(pte); 1058 1059 if (pte_present(ptent)) { 1060 page = vm_normal_page(vma, addr, ptent); 1061 young = pte_young(ptent); 1062 dirty = pte_dirty(ptent); 1063 present = true; 1064 } else if (pte_none(ptent)) { 1065 smaps_pte_hole_lookup(addr, walk); 1066 } else { 1067 const softleaf_t entry = softleaf_from_pte(ptent); 1068 1069 if (softleaf_is_swap(entry)) { 1070 int mapcount; 1071 1072 mss->swap += PAGE_SIZE; 1073 mapcount = swp_swapcount(entry); 1074 if (mapcount >= 2) { 1075 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; 1076 1077 do_div(pss_delta, mapcount); 1078 mss->swap_pss += pss_delta; 1079 } else { 1080 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 1081 } 1082 } else if (softleaf_has_pfn(entry)) { 1083 if (softleaf_is_device_private(entry)) 1084 present = true; 1085 page = softleaf_to_page(entry); 1086 } 1087 } 1088 1089 if (!page) 1090 return; 1091 1092 smaps_account(mss, page, false, young, dirty, locked, present); 1093 } 1094 1095 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1096 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1097 struct mm_walk *walk) 1098 { 1099 struct mem_size_stats *mss = walk->private; 1100 struct vm_area_struct *vma = walk->vma; 1101 bool locked = !!(vma->vm_flags & VM_LOCKED); 1102 struct page *page = NULL; 1103 bool present = false; 1104 struct folio *folio; 1105 1106 if (pmd_none(*pmd)) 1107 return; 1108 if (pmd_present(*pmd)) { 1109 page = vm_normal_page_pmd(vma, addr, *pmd); 1110 present = true; 1111 } else if (unlikely(thp_migration_supported())) { 1112 const softleaf_t entry = softleaf_from_pmd(*pmd); 1113 1114 if (softleaf_has_pfn(entry)) 1115 page = softleaf_to_page(entry); 1116 } 1117 if (IS_ERR_OR_NULL(page)) 1118 return; 1119 folio = page_folio(page); 1120 if (folio_test_anon(folio)) 1121 mss->anonymous_thp += HPAGE_PMD_SIZE; 1122 else if (folio_test_swapbacked(folio)) 1123 mss->shmem_thp += HPAGE_PMD_SIZE; 1124 else if (folio_is_zone_device(folio)) 1125 /* pass */; 1126 else 1127 mss->file_thp += HPAGE_PMD_SIZE; 1128 1129 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), 1130 locked, present); 1131 } 1132 #else 1133 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1134 struct mm_walk *walk) 1135 { 1136 } 1137 #endif 1138 1139 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 1140 struct mm_walk *walk) 1141 { 1142 struct vm_area_struct *vma = walk->vma; 1143 pte_t *pte; 1144 spinlock_t *ptl; 1145 1146 ptl = pmd_trans_huge_lock(pmd, vma); 1147 if (ptl) { 1148 smaps_pmd_entry(pmd, addr, walk); 1149 spin_unlock(ptl); 1150 goto out; 1151 } 1152 1153 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1154 if (!pte) { 1155 walk->action = ACTION_AGAIN; 1156 return 0; 1157 } 1158 for (; addr != end; pte++, addr += PAGE_SIZE) 1159 smaps_pte_entry(pte, addr, walk); 1160 pte_unmap_unlock(pte - 1, ptl); 1161 out: 1162 cond_resched(); 1163 return 0; 1164 } 1165 1166 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) 1167 { 1168 /* 1169 * Don't forget to update Documentation/ on changes. 1170 * 1171 * The length of the second argument of mnemonics[] 1172 * needs to be 3 instead of previously set 2 1173 * (i.e. from [BITS_PER_LONG][2] to [BITS_PER_LONG][3]) 1174 * to avoid spurious 1175 * -Werror=unterminated-string-initialization warning 1176 * with GCC 15 1177 */ 1178 static const char mnemonics[BITS_PER_LONG][3] = { 1179 /* 1180 * In case if we meet a flag we don't know about. 1181 */ 1182 [0 ... (BITS_PER_LONG-1)] = "??", 1183 1184 [ilog2(VM_READ)] = "rd", 1185 [ilog2(VM_WRITE)] = "wr", 1186 [ilog2(VM_EXEC)] = "ex", 1187 [ilog2(VM_SHARED)] = "sh", 1188 [ilog2(VM_MAYREAD)] = "mr", 1189 [ilog2(VM_MAYWRITE)] = "mw", 1190 [ilog2(VM_MAYEXEC)] = "me", 1191 [ilog2(VM_MAYSHARE)] = "ms", 1192 [ilog2(VM_GROWSDOWN)] = "gd", 1193 [ilog2(VM_PFNMAP)] = "pf", 1194 [ilog2(VM_MAYBE_GUARD)] = "gu", 1195 [ilog2(VM_LOCKED)] = "lo", 1196 [ilog2(VM_IO)] = "io", 1197 [ilog2(VM_SEQ_READ)] = "sr", 1198 [ilog2(VM_RAND_READ)] = "rr", 1199 [ilog2(VM_DONTCOPY)] = "dc", 1200 [ilog2(VM_DONTEXPAND)] = "de", 1201 [ilog2(VM_LOCKONFAULT)] = "lf", 1202 [ilog2(VM_ACCOUNT)] = "ac", 1203 [ilog2(VM_NORESERVE)] = "nr", 1204 [ilog2(VM_HUGETLB)] = "ht", 1205 [ilog2(VM_SYNC)] = "sf", 1206 [ilog2(VM_ARCH_1)] = "ar", 1207 [ilog2(VM_WIPEONFORK)] = "wf", 1208 [ilog2(VM_DONTDUMP)] = "dd", 1209 #ifdef CONFIG_ARM64_BTI 1210 [ilog2(VM_ARM64_BTI)] = "bt", 1211 #endif 1212 #ifdef CONFIG_MEM_SOFT_DIRTY 1213 [ilog2(VM_SOFTDIRTY)] = "sd", 1214 #endif 1215 [ilog2(VM_MIXEDMAP)] = "mm", 1216 [ilog2(VM_HUGEPAGE)] = "hg", 1217 [ilog2(VM_NOHUGEPAGE)] = "nh", 1218 [ilog2(VM_MERGEABLE)] = "mg", 1219 [ilog2(VM_UFFD_MISSING)]= "um", 1220 [ilog2(VM_UFFD_WP)] = "uw", 1221 #ifdef CONFIG_ARM64_MTE 1222 [ilog2(VM_MTE)] = "mt", 1223 [ilog2(VM_MTE_ALLOWED)] = "", 1224 #endif 1225 #ifdef CONFIG_ARCH_HAS_PKEYS 1226 /* These come out via ProtectionKey: */ 1227 [ilog2(VM_PKEY_BIT0)] = "", 1228 [ilog2(VM_PKEY_BIT1)] = "", 1229 [ilog2(VM_PKEY_BIT2)] = "", 1230 #if CONFIG_ARCH_PKEY_BITS > 3 1231 [ilog2(VM_PKEY_BIT3)] = "", 1232 #endif 1233 #if CONFIG_ARCH_PKEY_BITS > 4 1234 [ilog2(VM_PKEY_BIT4)] = "", 1235 #endif 1236 #endif /* CONFIG_ARCH_HAS_PKEYS */ 1237 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1238 [ilog2(VM_UFFD_MINOR)] = "ui", 1239 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 1240 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK 1241 [ilog2(VM_SHADOW_STACK)] = "ss", 1242 #endif 1243 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) 1244 [ilog2(VM_DROPPABLE)] = "dp", 1245 #endif 1246 #ifdef CONFIG_64BIT 1247 [ilog2(VM_SEALED)] = "sl", 1248 #endif 1249 }; 1250 size_t i; 1251 1252 seq_puts(m, "VmFlags: "); 1253 for (i = 0; i < BITS_PER_LONG; i++) { 1254 if (!mnemonics[i][0]) 1255 continue; 1256 if (vma->vm_flags & (1UL << i)) 1257 seq_printf(m, "%s ", mnemonics[i]); 1258 } 1259 seq_putc(m, '\n'); 1260 } 1261 1262 #ifdef CONFIG_HUGETLB_PAGE 1263 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, 1264 unsigned long addr, unsigned long end, 1265 struct mm_walk *walk) 1266 { 1267 struct mem_size_stats *mss = walk->private; 1268 struct vm_area_struct *vma = walk->vma; 1269 struct folio *folio = NULL; 1270 bool present = false; 1271 spinlock_t *ptl; 1272 pte_t ptent; 1273 1274 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); 1275 ptent = huge_ptep_get(walk->mm, addr, pte); 1276 if (pte_present(ptent)) { 1277 folio = page_folio(pte_page(ptent)); 1278 present = true; 1279 } else { 1280 const softleaf_t entry = softleaf_from_pte(ptent); 1281 1282 if (softleaf_has_pfn(entry)) 1283 folio = softleaf_to_folio(entry); 1284 } 1285 1286 if (folio) { 1287 /* We treat non-present entries as "maybe shared". */ 1288 if (!present || folio_maybe_mapped_shared(folio) || 1289 hugetlb_pmd_shared(pte)) 1290 mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); 1291 else 1292 mss->private_hugetlb += huge_page_size(hstate_vma(vma)); 1293 } 1294 spin_unlock(ptl); 1295 return 0; 1296 } 1297 #else 1298 #define smaps_hugetlb_range NULL 1299 #endif /* HUGETLB_PAGE */ 1300 1301 static const struct mm_walk_ops smaps_walk_ops = { 1302 .pmd_entry = smaps_pte_range, 1303 .hugetlb_entry = smaps_hugetlb_range, 1304 .walk_lock = PGWALK_RDLOCK, 1305 }; 1306 1307 static const struct mm_walk_ops smaps_shmem_walk_ops = { 1308 .pmd_entry = smaps_pte_range, 1309 .hugetlb_entry = smaps_hugetlb_range, 1310 .pte_hole = smaps_pte_hole, 1311 .walk_lock = PGWALK_RDLOCK, 1312 }; 1313 1314 #ifdef CONFIG_PER_VMA_LOCK 1315 1316 static const struct mm_walk_ops smaps_walk_vma_lock_ops = { 1317 .pmd_entry = smaps_pte_range, 1318 .hugetlb_entry = smaps_hugetlb_range, 1319 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1320 }; 1321 1322 static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { 1323 .pmd_entry = smaps_pte_range, 1324 .hugetlb_entry = smaps_hugetlb_range, 1325 .pte_hole = smaps_pte_hole, 1326 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1327 }; 1328 1329 static inline const struct mm_walk_ops * 1330 get_smaps_walk_ops(struct proc_maps_private *priv) 1331 { 1332 if (priv->lock_ctx.mmap_locked) 1333 return &smaps_walk_ops; 1334 return &smaps_walk_vma_lock_ops; 1335 } 1336 1337 static inline const struct mm_walk_ops * 1338 get_smaps_shmem_walk_ops(struct proc_maps_private *priv) 1339 { 1340 if (priv->lock_ctx.mmap_locked) 1341 return &smaps_shmem_walk_ops; 1342 return &smaps_shmem_walk_vma_lock_ops; 1343 } 1344 1345 #else /* CONFIG_PER_VMA_LOCK */ 1346 1347 static inline const struct mm_walk_ops * 1348 get_smaps_walk_ops(struct proc_maps_private *priv) 1349 { 1350 return &smaps_walk_ops; 1351 } 1352 1353 static inline const struct mm_walk_ops * 1354 get_smaps_shmem_walk_ops(struct proc_maps_private *priv) 1355 { 1356 return &smaps_shmem_walk_ops; 1357 } 1358 1359 #endif /* CONFIG_PER_VMA_LOCK */ 1360 1361 /* 1362 * Gather mem stats from @vma with the indicated beginning 1363 * address @start, and keep them in @mss. 1364 * 1365 * Use vm_start of @vma as the beginning address if @start is 0. 1366 */ 1367 static void smap_gather_stats(struct proc_maps_private *priv, 1368 struct vm_area_struct *vma, 1369 struct mem_size_stats *mss, unsigned long start) 1370 { 1371 const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); 1372 1373 /* Invalid start */ 1374 if (start >= vma->vm_end) 1375 return; 1376 1377 if (vma == get_gate_vma(priv->lock_ctx.mm)) 1378 return; 1379 1380 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 1381 drop_rcu(priv); 1382 1383 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { 1384 /* 1385 * For shared or readonly shmem mappings we know that all 1386 * swapped out pages belong to the shmem object, and we can 1387 * obtain the swap value much more efficiently. For private 1388 * writable mappings, we might have COW pages that are 1389 * not affected by the parent swapped out pages of the shmem 1390 * object, so we have to distinguish them during the page walk. 1391 * Unless we know that the shmem object (or the part mapped by 1392 * our VMA) has no swapped out pages at all. 1393 */ 1394 unsigned long shmem_swapped = shmem_swap_usage(vma); 1395 1396 if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || 1397 !(vma->vm_flags & VM_WRITE))) { 1398 mss->swap += shmem_swapped; 1399 } else { 1400 ops = get_smaps_shmem_walk_ops(priv); 1401 } 1402 } 1403 1404 if (!start) 1405 walk_page_vma(vma, ops, mss); 1406 else 1407 walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); 1408 1409 reacquire_rcu(priv); 1410 } 1411 1412 #define SEQ_PUT_DEC(str, val) \ 1413 seq_put_decimal_ull_width(m, str, (val) >> 10, 8) 1414 1415 /* Show the contents common for smaps and smaps_rollup */ 1416 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, 1417 bool rollup_mode) 1418 { 1419 SEQ_PUT_DEC("Rss: ", mss->resident); 1420 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT); 1421 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT); 1422 if (rollup_mode) { 1423 /* 1424 * These are meaningful only for smaps_rollup, otherwise two of 1425 * them are zero, and the other one is the same as Pss. 1426 */ 1427 SEQ_PUT_DEC(" kB\nPss_Anon: ", 1428 mss->pss_anon >> PSS_SHIFT); 1429 SEQ_PUT_DEC(" kB\nPss_File: ", 1430 mss->pss_file >> PSS_SHIFT); 1431 SEQ_PUT_DEC(" kB\nPss_Shmem: ", 1432 mss->pss_shmem >> PSS_SHIFT); 1433 } 1434 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean); 1435 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty); 1436 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean); 1437 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty); 1438 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced); 1439 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous); 1440 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm); 1441 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree); 1442 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp); 1443 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp); 1444 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp); 1445 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb); 1446 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ", 1447 mss->private_hugetlb >> 10, 7); 1448 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap); 1449 SEQ_PUT_DEC(" kB\nSwapPss: ", 1450 mss->swap_pss >> PSS_SHIFT); 1451 SEQ_PUT_DEC(" kB\nLocked: ", 1452 mss->pss_locked >> PSS_SHIFT); 1453 seq_puts(m, " kB\n"); 1454 } 1455 1456 static int show_smap(struct seq_file *m, void *v) 1457 { 1458 struct proc_maps_private *priv = m->private; 1459 struct vm_area_struct *vma = v; 1460 struct mem_size_stats mss = {}; 1461 1462 smap_gather_stats(priv, vma, &mss, 0); 1463 1464 show_map_vma(m, vma); 1465 1466 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); 1467 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); 1468 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); 1469 seq_puts(m, " kB\n"); 1470 1471 __show_smap(m, &mss, false); 1472 1473 seq_printf(m, "THPeligible: %8u\n", 1474 !!thp_vma_allowable_orders(vma, vma->vm_flags, TVA_SMAPS, 1475 THP_ORDERS_ALL)); 1476 1477 if (arch_pkeys_enabled()) 1478 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); 1479 show_smap_vma_flags(m, vma); 1480 1481 return 0; 1482 } 1483 1484 static int show_smaps_rollup(struct seq_file *m, void *v) 1485 { 1486 struct proc_maps_private *priv = m->private; 1487 struct mem_size_stats mss = {}; 1488 struct mm_struct *mm = priv->lock_ctx.mm; 1489 struct vm_area_struct *vma; 1490 unsigned long vma_start = 0, last_vma_end = 0; 1491 int ret = 0; 1492 VMA_ITERATOR(vmi, mm, 0); 1493 1494 priv->task = get_proc_task(priv->inode); 1495 if (!priv->task) 1496 return -ESRCH; 1497 1498 if (!mm || !mmget_not_zero(mm)) { 1499 ret = -ESRCH; 1500 goto out_put_task; 1501 } 1502 1503 ret = lock_ctx_mm(&priv->lock_ctx); 1504 if (ret) 1505 goto out_put_mm; 1506 1507 hold_task_mempolicy(priv); 1508 vma = vma_next(&vmi); 1509 1510 if (unlikely(!vma)) 1511 goto empty_set; 1512 1513 vma_start = vma->vm_start; 1514 do { 1515 smap_gather_stats(priv, vma, &mss, 0); 1516 last_vma_end = vma->vm_end; 1517 1518 /* 1519 * Release mmap_lock temporarily if someone wants to 1520 * access it for write request. 1521 */ 1522 if (mmap_lock_is_contended(mm)) { 1523 vma_iter_invalidate(&vmi); 1524 unlock_ctx_mm(&priv->lock_ctx); 1525 ret = lock_ctx_mm(&priv->lock_ctx); 1526 if (ret) { 1527 release_task_mempolicy(priv); 1528 goto out_put_mm; 1529 } 1530 1531 /* 1532 * After dropping the lock, there are four cases to 1533 * consider. See the following example for explanation. 1534 * 1535 * +------+------+-----------+ 1536 * | VMA1 | VMA2 | VMA3 | 1537 * +------+------+-----------+ 1538 * | | | | 1539 * 4k 8k 16k 400k 1540 * 1541 * Suppose we drop the lock after reading VMA2 due to 1542 * contention, then we get: 1543 * 1544 * last_vma_end = 16k 1545 * 1546 * 1) VMA2 is freed, but VMA3 exists: 1547 * 1548 * vma_next(vmi) will return VMA3. 1549 * In this case, just continue from VMA3. 1550 * 1551 * 2) VMA2 still exists: 1552 * 1553 * vma_next(vmi) will return VMA3. 1554 * In this case, just continue from VMA3. 1555 * 1556 * 3) No more VMAs can be found: 1557 * 1558 * vma_next(vmi) will return NULL. 1559 * No more things to do, just break. 1560 * 1561 * 4) (last_vma_end - 1) is the middle of a vma (VMA'): 1562 * 1563 * vma_next(vmi) will return VMA' whose range 1564 * contains last_vma_end. 1565 * Iterate VMA' from last_vma_end. 1566 */ 1567 vma = vma_next(&vmi); 1568 /* Case 3 above */ 1569 if (!vma) 1570 break; 1571 1572 /* Case 1 and 2 above */ 1573 if (vma->vm_start >= last_vma_end) { 1574 smap_gather_stats(priv, vma, &mss, 0); 1575 last_vma_end = vma->vm_end; 1576 continue; 1577 } 1578 1579 /* Case 4 above */ 1580 if (vma->vm_end > last_vma_end) { 1581 smap_gather_stats(priv, vma, &mss, last_vma_end); 1582 last_vma_end = vma->vm_end; 1583 } 1584 } 1585 } for_each_vma(vmi, vma); 1586 1587 empty_set: 1588 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); 1589 seq_pad(m, ' '); 1590 seq_puts(m, "[rollup]\n"); 1591 1592 __show_smap(m, &mss, true); 1593 1594 release_task_mempolicy(priv); 1595 unlock_ctx_mm(&priv->lock_ctx); 1596 1597 out_put_mm: 1598 mmput(mm); 1599 out_put_task: 1600 put_task_struct(priv->task); 1601 priv->task = NULL; 1602 1603 return ret; 1604 } 1605 #undef SEQ_PUT_DEC 1606 1607 static const struct seq_operations proc_pid_smaps_op = { 1608 .start = m_start, 1609 .next = m_next, 1610 .stop = m_stop, 1611 .show = show_smap 1612 }; 1613 1614 static int pid_smaps_open(struct inode *inode, struct file *file) 1615 { 1616 return do_maps_open(inode, file, &proc_pid_smaps_op); 1617 } 1618 1619 static int smaps_rollup_open(struct inode *inode, struct file *file) 1620 { 1621 int ret; 1622 struct proc_maps_private *priv; 1623 1624 priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT); 1625 if (!priv) 1626 return -ENOMEM; 1627 1628 ret = single_open(file, show_smaps_rollup, priv); 1629 if (ret) 1630 goto out_free; 1631 1632 priv->inode = inode; 1633 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 1634 if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) { 1635 ret = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH; 1636 1637 single_release(inode, file); 1638 goto out_free; 1639 } 1640 1641 return 0; 1642 1643 out_free: 1644 kfree(priv); 1645 return ret; 1646 } 1647 1648 static int smaps_rollup_release(struct inode *inode, struct file *file) 1649 { 1650 struct seq_file *seq = file->private_data; 1651 struct proc_maps_private *priv = seq->private; 1652 1653 if (priv->lock_ctx.mm) 1654 mmdrop(priv->lock_ctx.mm); 1655 1656 kfree(priv); 1657 return single_release(inode, file); 1658 } 1659 1660 const struct file_operations proc_pid_smaps_operations = { 1661 .open = pid_smaps_open, 1662 .read = seq_read, 1663 .llseek = seq_lseek, 1664 .release = proc_map_release, 1665 }; 1666 1667 const struct file_operations proc_pid_smaps_rollup_operations = { 1668 .open = smaps_rollup_open, 1669 .read = seq_read, 1670 .llseek = seq_lseek, 1671 .release = smaps_rollup_release, 1672 }; 1673 1674 enum clear_refs_types { 1675 CLEAR_REFS_ALL = 1, 1676 CLEAR_REFS_ANON, 1677 CLEAR_REFS_MAPPED, 1678 CLEAR_REFS_SOFT_DIRTY, 1679 CLEAR_REFS_MM_HIWATER_RSS, 1680 CLEAR_REFS_LAST, 1681 }; 1682 1683 struct clear_refs_private { 1684 enum clear_refs_types type; 1685 }; 1686 1687 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1688 { 1689 struct folio *folio; 1690 1691 if (!pte_write(pte)) 1692 return false; 1693 if (!is_cow_mapping(vma->vm_flags)) 1694 return false; 1695 if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))) 1696 return false; 1697 folio = vm_normal_folio(vma, addr, pte); 1698 if (!folio) 1699 return false; 1700 return folio_maybe_dma_pinned(folio); 1701 } 1702 1703 static inline void clear_soft_dirty(struct vm_area_struct *vma, 1704 unsigned long addr, pte_t *pte) 1705 { 1706 if (!pgtable_supports_soft_dirty()) 1707 return; 1708 /* 1709 * The soft-dirty tracker uses #PF-s to catch writes 1710 * to pages, so write-protect the pte as well. See the 1711 * Documentation/admin-guide/mm/soft-dirty.rst for full description 1712 * of how soft-dirty works. 1713 */ 1714 pte_t ptent = ptep_get(pte); 1715 1716 if (pte_none(ptent)) 1717 return; 1718 1719 if (pte_present(ptent)) { 1720 pte_t old_pte; 1721 1722 if (pte_is_pinned(vma, addr, ptent)) 1723 return; 1724 old_pte = ptep_modify_prot_start(vma, addr, pte); 1725 ptent = pte_wrprotect(old_pte); 1726 ptent = pte_clear_soft_dirty(ptent); 1727 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 1728 } else { 1729 ptent = pte_swp_clear_soft_dirty(ptent); 1730 set_pte_at(vma->vm_mm, addr, pte, ptent); 1731 } 1732 } 1733 1734 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) 1735 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1736 unsigned long addr, pmd_t *pmdp) 1737 { 1738 pmd_t old, pmd = *pmdp; 1739 1740 if (!pgtable_supports_soft_dirty()) 1741 return; 1742 1743 if (pmd_present(pmd)) { 1744 /* See comment in change_huge_pmd() */ 1745 old = pmdp_invalidate(vma, addr, pmdp); 1746 if (pmd_dirty(old)) 1747 pmd = pmd_mkdirty(pmd); 1748 if (pmd_young(old)) 1749 pmd = pmd_mkyoung(pmd); 1750 1751 pmd = pmd_wrprotect(pmd); 1752 pmd = pmd_clear_soft_dirty(pmd); 1753 1754 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1755 } else if (pmd_is_migration_entry(pmd)) { 1756 pmd = pmd_swp_clear_soft_dirty(pmd); 1757 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1758 } 1759 } 1760 #else 1761 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1762 unsigned long addr, pmd_t *pmdp) 1763 { 1764 } 1765 #endif 1766 1767 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, 1768 unsigned long end, struct mm_walk *walk) 1769 { 1770 struct clear_refs_private *cp = walk->private; 1771 struct vm_area_struct *vma = walk->vma; 1772 pte_t *pte, ptent; 1773 spinlock_t *ptl; 1774 struct folio *folio; 1775 1776 ptl = pmd_trans_huge_lock(pmd, vma); 1777 if (ptl) { 1778 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1779 clear_soft_dirty_pmd(vma, addr, pmd); 1780 goto out; 1781 } 1782 1783 if (!pmd_present(*pmd)) 1784 goto out; 1785 1786 folio = pmd_folio(*pmd); 1787 1788 /* Clear accessed and referenced bits. */ 1789 pmdp_test_and_clear_young(vma, addr, pmd); 1790 folio_test_clear_young(folio); 1791 folio_clear_referenced(folio); 1792 out: 1793 spin_unlock(ptl); 1794 return 0; 1795 } 1796 1797 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1798 if (!pte) { 1799 walk->action = ACTION_AGAIN; 1800 return 0; 1801 } 1802 for (; addr != end; pte++, addr += PAGE_SIZE) { 1803 ptent = ptep_get(pte); 1804 1805 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1806 clear_soft_dirty(vma, addr, pte); 1807 continue; 1808 } 1809 1810 if (!pte_present(ptent)) 1811 continue; 1812 1813 folio = vm_normal_folio(vma, addr, ptent); 1814 if (!folio) 1815 continue; 1816 1817 /* Clear accessed and referenced bits. */ 1818 ptep_test_and_clear_young(vma, addr, pte); 1819 folio_test_clear_young(folio); 1820 folio_clear_referenced(folio); 1821 } 1822 pte_unmap_unlock(pte - 1, ptl); 1823 cond_resched(); 1824 return 0; 1825 } 1826 1827 static int clear_refs_test_walk(unsigned long start, unsigned long end, 1828 struct mm_walk *walk) 1829 { 1830 struct clear_refs_private *cp = walk->private; 1831 struct vm_area_struct *vma = walk->vma; 1832 1833 if (vma->vm_flags & VM_PFNMAP) 1834 return 1; 1835 1836 /* 1837 * Writing 1 to /proc/pid/clear_refs affects all pages. 1838 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. 1839 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages. 1840 * Writing 4 to /proc/pid/clear_refs affects all pages. 1841 */ 1842 if (cp->type == CLEAR_REFS_ANON && vma->vm_file) 1843 return 1; 1844 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file) 1845 return 1; 1846 return 0; 1847 } 1848 1849 static const struct mm_walk_ops clear_refs_walk_ops = { 1850 .pmd_entry = clear_refs_pte_range, 1851 .test_walk = clear_refs_test_walk, 1852 .walk_lock = PGWALK_WRLOCK, 1853 }; 1854 1855 static ssize_t clear_refs_write(struct file *file, const char __user *buf, 1856 size_t count, loff_t *ppos) 1857 { 1858 struct task_struct *task; 1859 char buffer[PROC_NUMBUF] = {}; 1860 struct mm_struct *mm; 1861 struct vm_area_struct *vma; 1862 enum clear_refs_types type; 1863 int itype; 1864 int rv; 1865 1866 if (count > sizeof(buffer) - 1) 1867 count = sizeof(buffer) - 1; 1868 if (copy_from_user(buffer, buf, count)) 1869 return -EFAULT; 1870 rv = kstrtoint(strstrip(buffer), 10, &itype); 1871 if (rv < 0) 1872 return rv; 1873 type = (enum clear_refs_types)itype; 1874 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) 1875 return -EINVAL; 1876 1877 task = get_proc_task(file_inode(file)); 1878 if (!task) 1879 return -ESRCH; 1880 mm = get_task_mm(task); 1881 if (mm) { 1882 VMA_ITERATOR(vmi, mm, 0); 1883 struct mmu_notifier_range range; 1884 struct clear_refs_private cp = { 1885 .type = type, 1886 }; 1887 1888 if (mmap_write_lock_killable(mm)) { 1889 count = -EINTR; 1890 goto out_mm; 1891 } 1892 if (type == CLEAR_REFS_MM_HIWATER_RSS) { 1893 /* 1894 * Writing 5 to /proc/pid/clear_refs resets the peak 1895 * resident set size to this mm's current rss value. 1896 */ 1897 reset_mm_hiwater_rss(mm); 1898 goto out_unlock; 1899 } 1900 1901 if (type == CLEAR_REFS_SOFT_DIRTY) { 1902 for_each_vma(vmi, vma) { 1903 if (!(vma->vm_flags & VM_SOFTDIRTY)) 1904 continue; 1905 vm_flags_clear(vma, VM_SOFTDIRTY); 1906 vma_set_page_prot(vma); 1907 } 1908 1909 inc_tlb_flush_pending(mm); 1910 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, 1911 0, mm, 0, -1UL); 1912 mmu_notifier_invalidate_range_start(&range); 1913 } 1914 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); 1915 if (type == CLEAR_REFS_SOFT_DIRTY) { 1916 mmu_notifier_invalidate_range_end(&range); 1917 flush_tlb_mm(mm); 1918 dec_tlb_flush_pending(mm); 1919 } 1920 out_unlock: 1921 mmap_write_unlock(mm); 1922 out_mm: 1923 mmput(mm); 1924 } 1925 put_task_struct(task); 1926 1927 return count; 1928 } 1929 1930 const struct file_operations proc_clear_refs_operations = { 1931 .write = clear_refs_write, 1932 .llseek = noop_llseek, 1933 }; 1934 1935 typedef struct { 1936 u64 pme; 1937 } pagemap_entry_t; 1938 1939 struct pagemapread { 1940 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ 1941 pagemap_entry_t *buffer; 1942 bool show_pfn; 1943 }; 1944 1945 #define PAGEMAP_WALK_SIZE (PMD_SIZE) 1946 #define PAGEMAP_WALK_MASK (PMD_MASK) 1947 1948 #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) 1949 #define PM_PFRAME_BITS 55 1950 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) 1951 #define PM_SOFT_DIRTY BIT_ULL(55) 1952 #define PM_MMAP_EXCLUSIVE BIT_ULL(56) 1953 #define PM_UFFD_WP BIT_ULL(57) 1954 #define PM_GUARD_REGION BIT_ULL(58) 1955 #define PM_FILE BIT_ULL(61) 1956 #define PM_SWAP BIT_ULL(62) 1957 #define PM_PRESENT BIT_ULL(63) 1958 1959 #define PM_END_OF_BUFFER 1 1960 1961 static inline pagemap_entry_t make_pme(u64 frame, u64 flags) 1962 { 1963 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; 1964 } 1965 1966 static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) 1967 { 1968 pm->buffer[pm->pos++] = *pme; 1969 if (pm->pos >= pm->len) 1970 return PM_END_OF_BUFFER; 1971 return 0; 1972 } 1973 1974 static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) 1975 { 1976 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1977 return folio_precise_page_mapcount(folio, page) == 1; 1978 return !folio_maybe_mapped_shared(folio); 1979 } 1980 1981 static int pagemap_pte_hole(unsigned long start, unsigned long end, 1982 __always_unused int depth, struct mm_walk *walk) 1983 { 1984 struct pagemapread *pm = walk->private; 1985 unsigned long addr = start; 1986 int err = 0; 1987 1988 while (addr < end) { 1989 struct vm_area_struct *vma = find_vma(walk->mm, addr); 1990 pagemap_entry_t pme = make_pme(0, 0); 1991 /* End of address space hole, which we mark as non-present. */ 1992 unsigned long hole_end; 1993 1994 if (vma) 1995 hole_end = min(end, vma->vm_start); 1996 else 1997 hole_end = end; 1998 1999 for (; addr < hole_end; addr += PAGE_SIZE) { 2000 err = add_to_pagemap(&pme, pm); 2001 if (err) 2002 goto out; 2003 } 2004 2005 if (!vma) 2006 break; 2007 2008 /* Addresses in the VMA. */ 2009 if (vma->vm_flags & VM_SOFTDIRTY) 2010 pme = make_pme(0, PM_SOFT_DIRTY); 2011 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { 2012 err = add_to_pagemap(&pme, pm); 2013 if (err) 2014 goto out; 2015 } 2016 } 2017 out: 2018 return err; 2019 } 2020 2021 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 2022 struct vm_area_struct *vma, unsigned long addr, pte_t pte) 2023 { 2024 u64 frame = 0, flags = 0; 2025 struct page *page = NULL; 2026 struct folio *folio; 2027 2028 if (pte_none(pte)) 2029 goto out; 2030 2031 if (pte_present(pte)) { 2032 if (pm->show_pfn) 2033 frame = pte_pfn(pte); 2034 flags |= PM_PRESENT; 2035 page = vm_normal_page(vma, addr, pte); 2036 if (pte_soft_dirty(pte)) 2037 flags |= PM_SOFT_DIRTY; 2038 if (pte_uffd_wp(pte)) 2039 flags |= PM_UFFD_WP; 2040 } else { 2041 softleaf_t entry; 2042 2043 if (pte_swp_soft_dirty(pte)) 2044 flags |= PM_SOFT_DIRTY; 2045 if (pte_swp_uffd_wp(pte)) 2046 flags |= PM_UFFD_WP; 2047 entry = softleaf_from_pte(pte); 2048 if (pm->show_pfn) { 2049 pgoff_t offset; 2050 2051 /* 2052 * For PFN swap offsets, keeping the offset field 2053 * to be PFN only to be compatible with old smaps. 2054 */ 2055 if (softleaf_has_pfn(entry)) 2056 offset = softleaf_to_pfn(entry); 2057 else 2058 offset = swp_offset(entry); 2059 frame = swp_type(entry) | 2060 (offset << MAX_SWAPFILES_SHIFT); 2061 } 2062 flags |= PM_SWAP; 2063 if (softleaf_has_pfn(entry)) 2064 page = softleaf_to_page(entry); 2065 if (softleaf_is_uffd_wp_marker(entry)) 2066 flags |= PM_UFFD_WP; 2067 if (softleaf_is_guard_marker(entry)) 2068 flags |= PM_GUARD_REGION; 2069 } 2070 2071 if (page) { 2072 folio = page_folio(page); 2073 if (!folio_test_anon(folio)) 2074 flags |= PM_FILE; 2075 if ((flags & PM_PRESENT) && 2076 __folio_page_mapped_exclusively(folio, page)) 2077 flags |= PM_MMAP_EXCLUSIVE; 2078 } 2079 2080 out: 2081 if (vma->vm_flags & VM_SOFTDIRTY) 2082 flags |= PM_SOFT_DIRTY; 2083 2084 return make_pme(frame, flags); 2085 } 2086 2087 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2088 static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, 2089 unsigned long end, struct vm_area_struct *vma, 2090 struct pagemapread *pm) 2091 { 2092 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 2093 u64 flags = 0, frame = 0; 2094 pmd_t pmd = *pmdp; 2095 struct page *page = NULL; 2096 struct folio *folio = NULL; 2097 int err = 0; 2098 2099 if (vma->vm_flags & VM_SOFTDIRTY) 2100 flags |= PM_SOFT_DIRTY; 2101 2102 if (pmd_none(pmd)) 2103 goto populate_pagemap; 2104 2105 if (pmd_present(pmd)) { 2106 page = pmd_page(pmd); 2107 2108 flags |= PM_PRESENT; 2109 if (pmd_soft_dirty(pmd)) 2110 flags |= PM_SOFT_DIRTY; 2111 if (pmd_uffd_wp(pmd)) 2112 flags |= PM_UFFD_WP; 2113 if (pm->show_pfn) 2114 frame = pmd_pfn(pmd) + idx; 2115 } else if (thp_migration_supported()) { 2116 const softleaf_t entry = softleaf_from_pmd(pmd); 2117 unsigned long offset; 2118 2119 if (pm->show_pfn) { 2120 if (softleaf_has_pfn(entry)) 2121 offset = softleaf_to_pfn(entry) + idx; 2122 else 2123 offset = swp_offset(entry) + idx; 2124 frame = swp_type(entry) | 2125 (offset << MAX_SWAPFILES_SHIFT); 2126 } 2127 flags |= PM_SWAP; 2128 if (pmd_swp_soft_dirty(pmd)) 2129 flags |= PM_SOFT_DIRTY; 2130 if (pmd_swp_uffd_wp(pmd)) 2131 flags |= PM_UFFD_WP; 2132 page = softleaf_to_page(entry); 2133 } 2134 2135 if (page) { 2136 folio = page_folio(page); 2137 if (!folio_test_anon(folio)) 2138 flags |= PM_FILE; 2139 } 2140 2141 populate_pagemap: 2142 for (; addr != end; addr += PAGE_SIZE, idx++) { 2143 u64 cur_flags = flags; 2144 pagemap_entry_t pme; 2145 2146 if (folio && (flags & PM_PRESENT) && 2147 __folio_page_mapped_exclusively(folio, page)) 2148 cur_flags |= PM_MMAP_EXCLUSIVE; 2149 2150 pme = make_pme(frame, cur_flags); 2151 err = add_to_pagemap(&pme, pm); 2152 if (err) 2153 break; 2154 if (pm->show_pfn) { 2155 if (flags & PM_PRESENT) 2156 frame++; 2157 else if (flags & PM_SWAP) 2158 frame += (1 << MAX_SWAPFILES_SHIFT); 2159 } 2160 } 2161 return err; 2162 } 2163 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2164 2165 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, 2166 struct mm_walk *walk) 2167 { 2168 struct vm_area_struct *vma = walk->vma; 2169 struct pagemapread *pm = walk->private; 2170 spinlock_t *ptl; 2171 pte_t *pte, *orig_pte; 2172 int err = 0; 2173 2174 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2175 ptl = pmd_trans_huge_lock(pmdp, vma); 2176 if (ptl) { 2177 err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); 2178 spin_unlock(ptl); 2179 return err; 2180 } 2181 #endif 2182 2183 /* 2184 * We can assume that @vma always points to a valid one and @end never 2185 * goes beyond vma->vm_end. 2186 */ 2187 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); 2188 if (!pte) { 2189 walk->action = ACTION_AGAIN; 2190 return err; 2191 } 2192 for (; addr < end; pte++, addr += PAGE_SIZE) { 2193 pagemap_entry_t pme; 2194 2195 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); 2196 err = add_to_pagemap(&pme, pm); 2197 if (err) 2198 break; 2199 } 2200 pte_unmap_unlock(orig_pte, ptl); 2201 2202 cond_resched(); 2203 2204 return err; 2205 } 2206 2207 #ifdef CONFIG_HUGETLB_PAGE 2208 /* This function walks within one hugetlb entry in the single call */ 2209 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, 2210 unsigned long addr, unsigned long end, 2211 struct mm_walk *walk) 2212 { 2213 struct pagemapread *pm = walk->private; 2214 struct vm_area_struct *vma = walk->vma; 2215 u64 flags = 0, frame = 0; 2216 spinlock_t *ptl; 2217 int err = 0; 2218 pte_t pte; 2219 2220 if (vma->vm_flags & VM_SOFTDIRTY) 2221 flags |= PM_SOFT_DIRTY; 2222 2223 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, ptep); 2224 pte = huge_ptep_get(walk->mm, addr, ptep); 2225 if (pte_present(pte)) { 2226 struct folio *folio = page_folio(pte_page(pte)); 2227 2228 if (!folio_test_anon(folio)) 2229 flags |= PM_FILE; 2230 2231 if (!folio_maybe_mapped_shared(folio) && 2232 !hugetlb_pmd_shared(ptep)) 2233 flags |= PM_MMAP_EXCLUSIVE; 2234 2235 if (huge_pte_uffd_wp(pte)) 2236 flags |= PM_UFFD_WP; 2237 2238 flags |= PM_PRESENT; 2239 if (pm->show_pfn) 2240 frame = pte_pfn(pte) + 2241 ((addr & ~hmask) >> PAGE_SHIFT); 2242 } else if (pte_swp_uffd_wp_any(pte)) { 2243 flags |= PM_UFFD_WP; 2244 } 2245 2246 for (; addr != end; addr += PAGE_SIZE) { 2247 pagemap_entry_t pme = make_pme(frame, flags); 2248 2249 err = add_to_pagemap(&pme, pm); 2250 if (err) 2251 break; 2252 if (pm->show_pfn && (flags & PM_PRESENT)) 2253 frame++; 2254 } 2255 2256 spin_unlock(ptl); 2257 cond_resched(); 2258 2259 return err; 2260 } 2261 #else 2262 #define pagemap_hugetlb_range NULL 2263 #endif /* HUGETLB_PAGE */ 2264 2265 static const struct mm_walk_ops pagemap_ops = { 2266 .pmd_entry = pagemap_pmd_range, 2267 .pte_hole = pagemap_pte_hole, 2268 .hugetlb_entry = pagemap_hugetlb_range, 2269 .walk_lock = PGWALK_RDLOCK, 2270 }; 2271 2272 /* 2273 * /proc/pid/pagemap - an array mapping virtual pages to pfns 2274 * 2275 * For each page in the address space, this file contains one 64-bit entry 2276 * consisting of the following: 2277 * 2278 * Bits 0-54 page frame number (PFN) if present 2279 * Bits 0-4 swap type if swapped 2280 * Bits 5-54 swap offset if swapped 2281 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) 2282 * Bit 56 page exclusively mapped 2283 * Bit 57 pte is uffd-wp write-protected 2284 * Bit 58 pte is a guard region 2285 * Bits 59-60 zero 2286 * Bit 61 page is file-page or shared-anon 2287 * Bit 62 page swapped 2288 * Bit 63 page present 2289 * 2290 * If the page is not present but in swap, then the PFN contains an 2291 * encoding of the swap file number and the page's offset into the 2292 * swap. Unmapped pages return a null PFN. This allows determining 2293 * precisely which pages are mapped (or in swap) and comparing mapped 2294 * pages between processes. 2295 * 2296 * Efficient users of this interface will use /proc/pid/maps to 2297 * determine which areas of memory are actually mapped and llseek to 2298 * skip over unmapped regions. 2299 */ 2300 static ssize_t pagemap_read(struct file *file, char __user *buf, 2301 size_t count, loff_t *ppos) 2302 { 2303 struct mm_struct *mm = file->private_data; 2304 struct pagemapread pm; 2305 unsigned long src; 2306 unsigned long svpfn; 2307 unsigned long start_vaddr; 2308 unsigned long end_vaddr; 2309 int ret = 0, copied = 0; 2310 2311 if (!mm || !mmget_not_zero(mm)) 2312 goto out; 2313 2314 ret = -EINVAL; 2315 /* file position must be aligned */ 2316 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES)) 2317 goto out_mm; 2318 2319 ret = 0; 2320 if (!count) 2321 goto out_mm; 2322 2323 /* do not disclose physical addresses: attack vector */ 2324 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); 2325 2326 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); 2327 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); 2328 ret = -ENOMEM; 2329 if (!pm.buffer) 2330 goto out_mm; 2331 2332 src = *ppos; 2333 svpfn = src / PM_ENTRY_BYTES; 2334 end_vaddr = mm->task_size; 2335 2336 /* watch out for wraparound */ 2337 start_vaddr = end_vaddr; 2338 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { 2339 unsigned long end; 2340 2341 ret = mmap_read_lock_killable(mm); 2342 if (ret) 2343 goto out_free; 2344 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); 2345 mmap_read_unlock(mm); 2346 2347 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); 2348 if (end >= start_vaddr && end < mm->task_size) 2349 end_vaddr = end; 2350 } 2351 2352 /* Ensure the address is inside the task */ 2353 if (start_vaddr > mm->task_size) 2354 start_vaddr = end_vaddr; 2355 2356 ret = 0; 2357 while (count && (start_vaddr < end_vaddr)) { 2358 int len; 2359 unsigned long end; 2360 2361 pm.pos = 0; 2362 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; 2363 /* overflow ? */ 2364 if (end < start_vaddr || end > end_vaddr) 2365 end = end_vaddr; 2366 ret = mmap_read_lock_killable(mm); 2367 if (ret) 2368 goto out_free; 2369 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); 2370 mmap_read_unlock(mm); 2371 start_vaddr = end; 2372 2373 len = min(count, PM_ENTRY_BYTES * pm.pos); 2374 if (copy_to_user(buf, pm.buffer, len)) { 2375 ret = -EFAULT; 2376 goto out_free; 2377 } 2378 copied += len; 2379 buf += len; 2380 count -= len; 2381 } 2382 *ppos += copied; 2383 if (!ret || ret == PM_END_OF_BUFFER) 2384 ret = copied; 2385 2386 out_free: 2387 kfree(pm.buffer); 2388 out_mm: 2389 mmput(mm); 2390 out: 2391 return ret; 2392 } 2393 2394 static int pagemap_open(struct inode *inode, struct file *file) 2395 { 2396 struct mm_struct *mm; 2397 2398 mm = proc_mem_open(inode, PTRACE_MODE_READ); 2399 if (IS_ERR_OR_NULL(mm)) 2400 return mm ? PTR_ERR(mm) : -ESRCH; 2401 file->private_data = mm; 2402 return 0; 2403 } 2404 2405 static int pagemap_release(struct inode *inode, struct file *file) 2406 { 2407 struct mm_struct *mm = file->private_data; 2408 2409 if (mm) 2410 mmdrop(mm); 2411 return 0; 2412 } 2413 2414 #define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 2415 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 2416 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 2417 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ 2418 PAGE_IS_GUARD) 2419 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) 2420 2421 struct pagemap_scan_private { 2422 struct pm_scan_arg arg; 2423 unsigned long masks_of_interest, cur_vma_category; 2424 struct page_region *vec_buf; 2425 unsigned long vec_buf_len, vec_buf_index, found_pages; 2426 struct page_region __user *vec_out; 2427 }; 2428 2429 static unsigned long pagemap_page_category(struct pagemap_scan_private *p, 2430 struct vm_area_struct *vma, 2431 unsigned long addr, pte_t pte) 2432 { 2433 unsigned long categories; 2434 2435 if (pte_none(pte)) { 2436 /* 2437 * An unpopulated pte carries no uffd-wp marker, i.e. it is not 2438 * write-protected, the same condition under which the present 2439 * and swap cases below report PAGE_IS_WRITTEN. Report it here 2440 * too so this generic path agrees with the PAGE_IS_WRITTEN fast 2441 * path in pagemap_scan_pmd_entry(), which reports pte_none as 2442 * written and, under PM_SCAN_WP_MATCHING, arms a marker. The 2443 * fast path applies no VMA test, so neither does this. 2444 */ 2445 return PAGE_IS_WRITTEN; 2446 } 2447 2448 if (pte_present(pte)) { 2449 struct page *page; 2450 2451 categories = PAGE_IS_PRESENT; 2452 2453 if (!pte_uffd_wp(pte)) 2454 categories |= PAGE_IS_WRITTEN; 2455 2456 if (p->masks_of_interest & PAGE_IS_FILE) { 2457 page = vm_normal_page(vma, addr, pte); 2458 if (page && !PageAnon(page)) 2459 categories |= PAGE_IS_FILE; 2460 } 2461 2462 if (is_zero_pfn(pte_pfn(pte))) 2463 categories |= PAGE_IS_PFNZERO; 2464 if (pte_soft_dirty(pte)) 2465 categories |= PAGE_IS_SOFT_DIRTY; 2466 } else { 2467 softleaf_t entry; 2468 2469 categories = PAGE_IS_SWAPPED; 2470 2471 if (!pte_swp_uffd_wp_any(pte)) 2472 categories |= PAGE_IS_WRITTEN; 2473 2474 entry = softleaf_from_pte(pte); 2475 if (softleaf_is_guard_marker(entry)) 2476 categories |= PAGE_IS_GUARD; 2477 else if ((p->masks_of_interest & PAGE_IS_FILE) && 2478 softleaf_has_pfn(entry) && 2479 !folio_test_anon(softleaf_to_folio(entry))) 2480 categories |= PAGE_IS_FILE; 2481 2482 if (pte_swp_soft_dirty(pte)) 2483 categories |= PAGE_IS_SOFT_DIRTY; 2484 } 2485 2486 return categories; 2487 } 2488 2489 static void make_uffd_wp_pte(struct vm_area_struct *vma, 2490 unsigned long addr, pte_t *pte, pte_t ptent) 2491 { 2492 if (pte_present(ptent)) { 2493 pte_t old_pte; 2494 2495 old_pte = ptep_modify_prot_start(vma, addr, pte); 2496 ptent = pte_mkuffd_wp(old_pte); 2497 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 2498 } else if (pte_none(ptent)) { 2499 set_pte_at(vma->vm_mm, addr, pte, 2500 make_pte_marker(PTE_MARKER_UFFD_WP)); 2501 } else { 2502 ptent = pte_swp_mkuffd_wp(ptent); 2503 set_pte_at(vma->vm_mm, addr, pte, ptent); 2504 } 2505 } 2506 2507 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2508 static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, 2509 struct vm_area_struct *vma, 2510 unsigned long addr, pmd_t pmd) 2511 { 2512 unsigned long categories = PAGE_IS_HUGE; 2513 2514 if (pmd_none(pmd)) 2515 return categories; 2516 2517 if (pmd_present(pmd)) { 2518 struct page *page; 2519 2520 categories |= PAGE_IS_PRESENT; 2521 if (!pmd_uffd_wp(pmd)) 2522 categories |= PAGE_IS_WRITTEN; 2523 2524 if (p->masks_of_interest & PAGE_IS_FILE) { 2525 page = vm_normal_page_pmd(vma, addr, pmd); 2526 if (page && !PageAnon(page)) 2527 categories |= PAGE_IS_FILE; 2528 } 2529 2530 if (is_huge_zero_pmd(pmd)) 2531 categories |= PAGE_IS_PFNZERO; 2532 if (pmd_soft_dirty(pmd)) 2533 categories |= PAGE_IS_SOFT_DIRTY; 2534 } else { 2535 categories |= PAGE_IS_SWAPPED; 2536 if (!pmd_swp_uffd_wp(pmd)) 2537 categories |= PAGE_IS_WRITTEN; 2538 if (pmd_swp_soft_dirty(pmd)) 2539 categories |= PAGE_IS_SOFT_DIRTY; 2540 2541 if (p->masks_of_interest & PAGE_IS_FILE) { 2542 const softleaf_t entry = softleaf_from_pmd(pmd); 2543 2544 if (softleaf_has_pfn(entry) && 2545 !folio_test_anon(softleaf_to_folio(entry))) 2546 categories |= PAGE_IS_FILE; 2547 } 2548 } 2549 2550 return categories; 2551 } 2552 2553 static void make_uffd_wp_pmd(struct vm_area_struct *vma, 2554 unsigned long addr, pmd_t *pmdp) 2555 { 2556 pmd_t old, pmd = *pmdp; 2557 2558 if (pmd_present(pmd)) { 2559 old = pmdp_invalidate_ad(vma, addr, pmdp); 2560 pmd = pmd_mkuffd_wp(old); 2561 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2562 } else if (pmd_is_migration_entry(pmd)) { 2563 pmd = pmd_swp_mkuffd_wp(pmd); 2564 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2565 } 2566 } 2567 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2568 2569 #ifdef CONFIG_HUGETLB_PAGE 2570 static unsigned long pagemap_hugetlb_category(pte_t pte) 2571 { 2572 unsigned long categories = PAGE_IS_HUGE; 2573 2574 if (pte_none(pte)) 2575 return categories; 2576 2577 /* 2578 * According to pagemap_hugetlb_range(), file-backed HugeTLB 2579 * page cannot be swapped. So PAGE_IS_FILE is not checked for 2580 * swapped pages. 2581 */ 2582 if (pte_present(pte)) { 2583 categories |= PAGE_IS_PRESENT; 2584 2585 if (!huge_pte_uffd_wp(pte)) 2586 categories |= PAGE_IS_WRITTEN; 2587 if (!PageAnon(pte_page(pte))) 2588 categories |= PAGE_IS_FILE; 2589 if (is_zero_pfn(pte_pfn(pte))) 2590 categories |= PAGE_IS_PFNZERO; 2591 if (pte_soft_dirty(pte)) 2592 categories |= PAGE_IS_SOFT_DIRTY; 2593 } else { 2594 categories |= PAGE_IS_SWAPPED; 2595 2596 if (!pte_swp_uffd_wp_any(pte)) 2597 categories |= PAGE_IS_WRITTEN; 2598 if (pte_swp_soft_dirty(pte)) 2599 categories |= PAGE_IS_SOFT_DIRTY; 2600 } 2601 2602 return categories; 2603 } 2604 2605 static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, 2606 unsigned long addr, pte_t *ptep, 2607 pte_t ptent) 2608 { 2609 const unsigned long psize = huge_page_size(hstate_vma(vma)); 2610 softleaf_t entry; 2611 2612 if (huge_pte_none(ptent)) { 2613 set_huge_pte_at(vma->vm_mm, addr, ptep, 2614 make_pte_marker(PTE_MARKER_UFFD_WP), psize); 2615 return; 2616 } 2617 2618 entry = softleaf_from_pte(ptent); 2619 if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) 2620 return; 2621 2622 if (softleaf_is_migration(entry)) { 2623 set_huge_pte_at(vma->vm_mm, addr, ptep, 2624 pte_swp_mkuffd_wp(ptent), psize); 2625 } else { 2626 pte_t old_pte, new_pte; 2627 2628 old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); 2629 new_pte = huge_pte_mkuffd_wp(old_pte); 2630 huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); 2631 } 2632 } 2633 #endif /* CONFIG_HUGETLB_PAGE */ 2634 2635 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 2636 static void pagemap_scan_backout_range(struct pagemap_scan_private *p, 2637 unsigned long addr, unsigned long end) 2638 { 2639 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2640 2641 if (!p->vec_buf) 2642 return; 2643 2644 if (cur_buf->start != addr) 2645 cur_buf->end = addr; 2646 else 2647 cur_buf->start = cur_buf->end = 0; 2648 2649 p->found_pages -= (end - addr) / PAGE_SIZE; 2650 } 2651 #endif 2652 2653 static bool pagemap_scan_is_interesting_page(unsigned long categories, 2654 const struct pagemap_scan_private *p) 2655 { 2656 categories ^= p->arg.category_inverted; 2657 if ((categories & p->arg.category_mask) != p->arg.category_mask) 2658 return false; 2659 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) 2660 return false; 2661 2662 return true; 2663 } 2664 2665 static bool pagemap_scan_is_interesting_vma(unsigned long categories, 2666 const struct pagemap_scan_private *p) 2667 { 2668 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; 2669 2670 categories ^= p->arg.category_inverted; 2671 if ((categories & required) != required) 2672 return false; 2673 2674 return true; 2675 } 2676 2677 static int pagemap_scan_test_walk(unsigned long start, unsigned long end, 2678 struct mm_walk *walk) 2679 { 2680 struct pagemap_scan_private *p = walk->private; 2681 struct vm_area_struct *vma = walk->vma; 2682 unsigned long vma_category = 0; 2683 bool wp_allowed = userfaultfd_wp_async(vma) && 2684 userfaultfd_wp_use_markers(vma); 2685 2686 if (!wp_allowed) { 2687 /* User requested explicit failure over wp-async capability */ 2688 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) 2689 return -EPERM; 2690 /* 2691 * User requires wr-protect, and allows silently skipping 2692 * unsupported vmas. 2693 */ 2694 if (p->arg.flags & PM_SCAN_WP_MATCHING) 2695 return 1; 2696 /* 2697 * Then the request doesn't involve wr-protects at all, 2698 * fall through to the rest checks, and allow vma walk. 2699 */ 2700 } 2701 2702 if (vma->vm_flags & VM_PFNMAP) 2703 return 1; 2704 2705 if (wp_allowed) 2706 vma_category |= PAGE_IS_WPALLOWED; 2707 2708 if (vma->vm_flags & VM_SOFTDIRTY) 2709 vma_category |= PAGE_IS_SOFT_DIRTY; 2710 2711 if (!pagemap_scan_is_interesting_vma(vma_category, p)) 2712 return 1; 2713 2714 p->cur_vma_category = vma_category; 2715 2716 return 0; 2717 } 2718 2719 static bool pagemap_scan_push_range(unsigned long categories, 2720 struct pagemap_scan_private *p, 2721 unsigned long addr, unsigned long end) 2722 { 2723 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2724 2725 /* 2726 * When there is no output buffer provided at all, the sentinel values 2727 * won't match here. There is no other way for `cur_buf->end` to be 2728 * non-zero other than it being non-empty. 2729 */ 2730 if (addr == cur_buf->end && categories == cur_buf->categories) { 2731 cur_buf->end = end; 2732 return true; 2733 } 2734 2735 if (cur_buf->end) { 2736 if (p->vec_buf_index >= p->vec_buf_len - 1) 2737 return false; 2738 2739 cur_buf = &p->vec_buf[++p->vec_buf_index]; 2740 } 2741 2742 cur_buf->start = addr; 2743 cur_buf->end = end; 2744 cur_buf->categories = categories; 2745 2746 return true; 2747 } 2748 2749 static int pagemap_scan_output(unsigned long categories, 2750 struct pagemap_scan_private *p, 2751 unsigned long addr, unsigned long *end) 2752 { 2753 unsigned long n_pages, total_pages; 2754 int ret = 0; 2755 2756 if (!p->vec_buf) 2757 return 0; 2758 2759 categories &= p->arg.return_mask; 2760 2761 n_pages = (*end - addr) / PAGE_SIZE; 2762 if (check_add_overflow(p->found_pages, n_pages, &total_pages) || 2763 total_pages > p->arg.max_pages) { 2764 size_t n_too_much = total_pages - p->arg.max_pages; 2765 *end -= n_too_much * PAGE_SIZE; 2766 n_pages -= n_too_much; 2767 ret = -ENOSPC; 2768 } 2769 2770 if (!pagemap_scan_push_range(categories, p, addr, *end)) { 2771 *end = addr; 2772 n_pages = 0; 2773 ret = -ENOSPC; 2774 } 2775 2776 p->found_pages += n_pages; 2777 if (ret) 2778 p->arg.walk_end = *end; 2779 2780 return ret; 2781 } 2782 2783 static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, 2784 unsigned long end, struct mm_walk *walk) 2785 { 2786 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2787 struct pagemap_scan_private *p = walk->private; 2788 struct vm_area_struct *vma = walk->vma; 2789 unsigned long categories; 2790 spinlock_t *ptl; 2791 int ret = 0; 2792 2793 ptl = pmd_trans_huge_lock(pmd, vma); 2794 if (!ptl) 2795 return -ENOENT; 2796 2797 categories = p->cur_vma_category | 2798 pagemap_thp_category(p, vma, start, *pmd); 2799 2800 if (!pagemap_scan_is_interesting_page(categories, p)) 2801 goto out_unlock; 2802 2803 ret = pagemap_scan_output(categories, p, start, &end); 2804 if (start == end) 2805 goto out_unlock; 2806 2807 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2808 goto out_unlock; 2809 if (~categories & PAGE_IS_WRITTEN) 2810 goto out_unlock; 2811 2812 /* 2813 * Break huge page into small pages if the WP operation 2814 * needs to be performed on a portion of the huge page. 2815 */ 2816 if (end != start + HPAGE_SIZE) { 2817 spin_unlock(ptl); 2818 split_huge_pmd(vma, pmd, start); 2819 pagemap_scan_backout_range(p, start, end); 2820 /* Report as if there was no THP */ 2821 return -ENOENT; 2822 } 2823 2824 make_uffd_wp_pmd(vma, start, pmd); 2825 flush_tlb_range(vma, start, end); 2826 out_unlock: 2827 spin_unlock(ptl); 2828 return ret; 2829 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 2830 return -ENOENT; 2831 #endif 2832 } 2833 2834 static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, 2835 unsigned long end, struct mm_walk *walk) 2836 { 2837 struct pagemap_scan_private *p = walk->private; 2838 struct vm_area_struct *vma = walk->vma; 2839 unsigned long addr, flush_end = 0; 2840 pte_t *pte, *start_pte; 2841 spinlock_t *ptl; 2842 int ret; 2843 2844 ret = pagemap_scan_thp_entry(pmd, start, end, walk); 2845 if (ret != -ENOENT) 2846 return ret; 2847 2848 ret = 0; 2849 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); 2850 if (!pte) { 2851 walk->action = ACTION_AGAIN; 2852 return 0; 2853 } 2854 2855 lazy_mmu_mode_enable(); 2856 2857 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { 2858 /* Fast path for performing exclusive WP */ 2859 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2860 pte_t ptent = ptep_get(pte); 2861 2862 if ((pte_present(ptent) && pte_uffd_wp(ptent)) || 2863 pte_swp_uffd_wp_any(ptent)) 2864 continue; 2865 make_uffd_wp_pte(vma, addr, pte, ptent); 2866 if (!flush_end) 2867 start = addr; 2868 flush_end = addr + PAGE_SIZE; 2869 } 2870 goto flush_and_return; 2871 } 2872 2873 if (!p->arg.category_anyof_mask && !p->arg.category_inverted && 2874 p->arg.category_mask == PAGE_IS_WRITTEN && 2875 p->arg.return_mask == PAGE_IS_WRITTEN) { 2876 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { 2877 unsigned long next = addr + PAGE_SIZE; 2878 pte_t ptent = ptep_get(pte); 2879 2880 if ((pte_present(ptent) && pte_uffd_wp(ptent)) || 2881 pte_swp_uffd_wp_any(ptent)) 2882 continue; 2883 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, 2884 p, addr, &next); 2885 if (next == addr) 2886 break; 2887 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2888 continue; 2889 make_uffd_wp_pte(vma, addr, pte, ptent); 2890 if (!flush_end) 2891 start = addr; 2892 flush_end = next; 2893 } 2894 goto flush_and_return; 2895 } 2896 2897 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2898 pte_t ptent = ptep_get(pte); 2899 unsigned long categories = p->cur_vma_category | 2900 pagemap_page_category(p, vma, addr, ptent); 2901 unsigned long next = addr + PAGE_SIZE; 2902 2903 if (!pagemap_scan_is_interesting_page(categories, p)) 2904 continue; 2905 2906 ret = pagemap_scan_output(categories, p, addr, &next); 2907 if (next == addr) 2908 break; 2909 2910 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2911 continue; 2912 if (~categories & PAGE_IS_WRITTEN) 2913 continue; 2914 2915 make_uffd_wp_pte(vma, addr, pte, ptent); 2916 if (!flush_end) 2917 start = addr; 2918 flush_end = next; 2919 } 2920 2921 flush_and_return: 2922 if (flush_end) 2923 flush_tlb_range(vma, start, addr); 2924 2925 lazy_mmu_mode_disable(); 2926 pte_unmap_unlock(start_pte, ptl); 2927 2928 cond_resched(); 2929 return ret; 2930 } 2931 2932 #ifdef CONFIG_HUGETLB_PAGE 2933 static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, 2934 unsigned long start, unsigned long end, 2935 struct mm_walk *walk) 2936 { 2937 struct pagemap_scan_private *p = walk->private; 2938 struct vm_area_struct *vma = walk->vma; 2939 unsigned long categories; 2940 spinlock_t *ptl; 2941 int ret = 0; 2942 pte_t pte; 2943 2944 if (~p->arg.flags & PM_SCAN_WP_MATCHING) { 2945 /* Go the short route when not write-protecting pages. */ 2946 2947 pte = huge_ptep_get(walk->mm, start, ptep); 2948 categories = p->cur_vma_category | pagemap_hugetlb_category(pte); 2949 2950 if (!pagemap_scan_is_interesting_page(categories, p)) 2951 return 0; 2952 2953 return pagemap_scan_output(categories, p, start, &end); 2954 } 2955 2956 i_mmap_lock_write(vma->vm_file->f_mapping); 2957 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); 2958 2959 pte = huge_ptep_get(walk->mm, start, ptep); 2960 categories = p->cur_vma_category | pagemap_hugetlb_category(pte); 2961 2962 if (!pagemap_scan_is_interesting_page(categories, p)) 2963 goto out_unlock; 2964 2965 ret = pagemap_scan_output(categories, p, start, &end); 2966 if (start == end) 2967 goto out_unlock; 2968 2969 if (~categories & PAGE_IS_WRITTEN) 2970 goto out_unlock; 2971 2972 if (end != start + huge_page_size(hstate_vma(vma))) { 2973 /* Partial HugeTLB page WP isn't possible. */ 2974 pagemap_scan_backout_range(p, start, end); 2975 p->arg.walk_end = start; 2976 ret = 0; 2977 goto out_unlock; 2978 } 2979 2980 make_uffd_wp_huge_pte(vma, start, ptep, pte); 2981 flush_hugetlb_tlb_range(vma, start, end); 2982 2983 out_unlock: 2984 spin_unlock(ptl); 2985 i_mmap_unlock_write(vma->vm_file->f_mapping); 2986 2987 return ret; 2988 } 2989 2990 /* 2991 * Write-protect the unpopulated hugetlb entries covering [addr, end) by 2992 * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() 2993 * does for populated entries. 2994 * 2995 * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the 2996 * loop normally runs a single iteration; it is written to cover the full range 2997 * in case the walker ever coalesces adjacent holes. 2998 * 2999 * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- 3000 * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table 3001 * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the 3002 * same VMA, so the scanning thread would deadlock against itself. PMD sharing 3003 * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so 3004 * the vma lock guards nothing that matters for these entries anyway. 3005 */ 3006 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 3007 unsigned long addr, unsigned long end) 3008 { 3009 struct hstate *h = hstate_vma(vma); 3010 unsigned long psize = huge_page_size(h); 3011 struct mm_struct *mm = vma->vm_mm; 3012 spinlock_t *ptl; 3013 pte_t *ptep; 3014 pte_t pte; 3015 3016 for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { 3017 ptep = huge_pte_alloc(mm, vma, addr, psize); 3018 if (!ptep) 3019 return -ENOMEM; 3020 3021 i_mmap_lock_write(vma->vm_file->f_mapping); 3022 ptl = huge_pte_lock(h, mm, ptep); 3023 pte = huge_ptep_get(mm, addr, ptep); 3024 make_uffd_wp_huge_pte(vma, addr, ptep, pte); 3025 /* 3026 * A none entry has no cached translation, so installing the 3027 * marker needs no TLB flush. Flush only if a fault populated 3028 * the entry between huge_pte_alloc() and the page table lock. 3029 */ 3030 if (!huge_pte_none(pte)) 3031 flush_hugetlb_tlb_range(vma, addr, addr + psize); 3032 spin_unlock(ptl); 3033 i_mmap_unlock_write(vma->vm_file->f_mapping); 3034 } 3035 3036 return 0; 3037 } 3038 #else 3039 #define pagemap_scan_hugetlb_entry NULL 3040 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 3041 unsigned long addr, unsigned long end) 3042 { 3043 return 0; 3044 } 3045 #endif 3046 3047 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, 3048 int depth, struct mm_walk *walk) 3049 { 3050 struct pagemap_scan_private *p = walk->private; 3051 struct vm_area_struct *vma = walk->vma; 3052 unsigned long categories; 3053 int ret, err; 3054 3055 if (!vma) 3056 return 0; 3057 3058 /* 3059 * In a uffd-wp VMA an unpopulated range is treated as written: 3060 * uffd-wp registration populates page tables and installs markers 3061 * with WP_UNPOPULATED, so a missing marker means the range was 3062 * zapped. See the pte_none() handling in pagemap_page_category(). 3063 * 3064 * hugetlb differs, see pagemap_hugetlb_category(). 3065 */ 3066 categories = p->cur_vma_category; 3067 if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma)) 3068 categories |= PAGE_IS_WRITTEN; 3069 3070 if (!pagemap_scan_is_interesting_page(categories, p)) 3071 return 0; 3072 3073 ret = pagemap_scan_output(categories, p, addr, &end); 3074 if (addr == end) 3075 return ret; 3076 3077 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 3078 return ret; 3079 3080 if (is_vm_hugetlb_page(vma)) 3081 err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); 3082 else 3083 err = uffd_wp_range(vma, addr, end - addr, true); 3084 if (err < 0) 3085 ret = err; 3086 3087 return ret; 3088 } 3089 3090 static const struct mm_walk_ops pagemap_scan_ops = { 3091 .test_walk = pagemap_scan_test_walk, 3092 .pmd_entry = pagemap_scan_pmd_entry, 3093 .pte_hole = pagemap_scan_pte_hole, 3094 .hugetlb_entry = pagemap_scan_hugetlb_entry, 3095 }; 3096 3097 static int pagemap_scan_get_args(struct pm_scan_arg *arg, 3098 unsigned long uarg) 3099 { 3100 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) 3101 return -EFAULT; 3102 3103 if (arg->size != sizeof(struct pm_scan_arg)) 3104 return -EINVAL; 3105 3106 /* Validate requested features */ 3107 if (arg->flags & ~PM_SCAN_FLAGS) 3108 return -EINVAL; 3109 if ((arg->category_inverted | arg->category_mask | 3110 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) 3111 return -EINVAL; 3112 3113 arg->start = untagged_addr((unsigned long)arg->start); 3114 arg->end = untagged_addr((unsigned long)arg->end); 3115 arg->vec = untagged_addr((unsigned long)arg->vec); 3116 3117 /* Validate memory pointers */ 3118 if (!IS_ALIGNED(arg->start, PAGE_SIZE)) 3119 return -EINVAL; 3120 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) 3121 return -EFAULT; 3122 if (!arg->vec && arg->vec_len) 3123 return -EINVAL; 3124 if (UINT_MAX == SIZE_MAX && arg->vec_len > SIZE_MAX) 3125 return -EINVAL; 3126 if (arg->vec && !access_ok((void __user *)(long)arg->vec, 3127 size_mul(arg->vec_len, sizeof(struct page_region)))) 3128 return -EFAULT; 3129 3130 /* Fixup default values */ 3131 arg->end = ALIGN(arg->end, PAGE_SIZE); 3132 arg->walk_end = 0; 3133 if (!arg->max_pages) 3134 arg->max_pages = ULONG_MAX; 3135 3136 return 0; 3137 } 3138 3139 static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, 3140 unsigned long uargl) 3141 { 3142 struct pm_scan_arg __user *uarg = (void __user *)uargl; 3143 3144 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) 3145 return -EFAULT; 3146 3147 return 0; 3148 } 3149 3150 static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) 3151 { 3152 if (!p->arg.vec_len) 3153 return 0; 3154 3155 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, 3156 p->arg.vec_len); 3157 p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); 3158 if (!p->vec_buf) 3159 return -ENOMEM; 3160 3161 p->vec_buf->start = p->vec_buf->end = 0; 3162 p->vec_out = (struct page_region __user *)(long)p->arg.vec; 3163 3164 return 0; 3165 } 3166 3167 static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) 3168 { 3169 const struct page_region *buf = p->vec_buf; 3170 long n = p->vec_buf_index; 3171 3172 if (!p->vec_buf) 3173 return 0; 3174 3175 if (buf[n].end != buf[n].start) 3176 n++; 3177 3178 if (!n) 3179 return 0; 3180 3181 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) 3182 return -EFAULT; 3183 3184 p->arg.vec_len -= n; 3185 p->vec_out += n; 3186 3187 p->vec_buf_index = 0; 3188 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); 3189 p->vec_buf->start = p->vec_buf->end = 0; 3190 3191 return n; 3192 } 3193 3194 static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) 3195 { 3196 struct pagemap_scan_private p = {0}; 3197 unsigned long walk_start; 3198 size_t n_ranges_out = 0; 3199 int ret; 3200 3201 ret = pagemap_scan_get_args(&p.arg, uarg); 3202 if (ret) 3203 return ret; 3204 3205 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | 3206 p.arg.return_mask; 3207 ret = pagemap_scan_init_bounce_buffer(&p); 3208 if (ret) 3209 return ret; 3210 3211 for (walk_start = p.arg.start; walk_start < p.arg.end; 3212 walk_start = p.arg.walk_end) { 3213 struct mmu_notifier_range range; 3214 long n_out; 3215 3216 if (fatal_signal_pending(current)) { 3217 ret = -EINTR; 3218 break; 3219 } 3220 3221 ret = mmap_read_lock_killable(mm); 3222 if (ret) 3223 break; 3224 3225 /* Protection change for the range is going to happen. */ 3226 if (p.arg.flags & PM_SCAN_WP_MATCHING) { 3227 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, 3228 mm, walk_start, p.arg.end); 3229 mmu_notifier_invalidate_range_start(&range); 3230 } 3231 3232 ret = walk_page_range(mm, walk_start, p.arg.end, 3233 &pagemap_scan_ops, &p); 3234 3235 if (p.arg.flags & PM_SCAN_WP_MATCHING) 3236 mmu_notifier_invalidate_range_end(&range); 3237 3238 mmap_read_unlock(mm); 3239 3240 n_out = pagemap_scan_flush_buffer(&p); 3241 if (n_out < 0) 3242 ret = n_out; 3243 else 3244 n_ranges_out += n_out; 3245 3246 if (ret != -ENOSPC) 3247 break; 3248 3249 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) 3250 break; 3251 } 3252 3253 /* ENOSPC signifies early stop (buffer full) from the walk. */ 3254 if (!ret || ret == -ENOSPC) 3255 ret = n_ranges_out; 3256 3257 /* The walk_end isn't set when ret is zero */ 3258 if (!p.arg.walk_end) 3259 p.arg.walk_end = p.arg.end; 3260 if (pagemap_scan_writeback_args(&p.arg, uarg)) 3261 ret = -EFAULT; 3262 3263 kfree(p.vec_buf); 3264 return ret; 3265 } 3266 3267 static long do_pagemap_cmd(struct file *file, unsigned int cmd, 3268 unsigned long arg) 3269 { 3270 struct mm_struct *mm = file->private_data; 3271 3272 switch (cmd) { 3273 case PAGEMAP_SCAN: 3274 return do_pagemap_scan(mm, arg); 3275 3276 default: 3277 return -EINVAL; 3278 } 3279 } 3280 3281 const struct file_operations proc_pagemap_operations = { 3282 .llseek = mem_lseek, /* borrow this */ 3283 .read = pagemap_read, 3284 .open = pagemap_open, 3285 .release = pagemap_release, 3286 .unlocked_ioctl = do_pagemap_cmd, 3287 .compat_ioctl = do_pagemap_cmd, 3288 }; 3289 #endif /* CONFIG_PROC_PAGE_MONITOR */ 3290 3291 #ifdef CONFIG_NUMA 3292 3293 struct numa_maps { 3294 unsigned long pages; 3295 unsigned long anon; 3296 unsigned long active; 3297 unsigned long writeback; 3298 unsigned long mapcount_max; 3299 unsigned long dirty; 3300 unsigned long swapcache; 3301 unsigned long node[MAX_NUMNODES]; 3302 }; 3303 3304 struct numa_maps_private { 3305 struct proc_maps_private proc_maps; 3306 struct numa_maps md; 3307 }; 3308 3309 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, 3310 unsigned long nr_pages) 3311 { 3312 struct folio *folio = page_folio(page); 3313 int count; 3314 3315 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 3316 count = folio_precise_page_mapcount(folio, page); 3317 else 3318 count = folio_average_page_mapcount(folio); 3319 3320 md->pages += nr_pages; 3321 if (pte_dirty || folio_test_dirty(folio)) 3322 md->dirty += nr_pages; 3323 3324 if (folio_test_swapcache(folio)) 3325 md->swapcache += nr_pages; 3326 3327 if (folio_test_active(folio) || folio_test_unevictable(folio)) 3328 md->active += nr_pages; 3329 3330 if (folio_test_writeback(folio)) 3331 md->writeback += nr_pages; 3332 3333 if (folio_test_anon(folio)) 3334 md->anon += nr_pages; 3335 3336 if (count > md->mapcount_max) 3337 md->mapcount_max = count; 3338 3339 md->node[folio_nid(folio)] += nr_pages; 3340 } 3341 3342 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, 3343 unsigned long addr) 3344 { 3345 struct page *page; 3346 int nid; 3347 3348 if (!pte_present(pte)) 3349 return NULL; 3350 3351 page = vm_normal_page(vma, addr, pte); 3352 if (!page || is_zone_device_page(page)) 3353 return NULL; 3354 3355 if (PageReserved(page)) 3356 return NULL; 3357 3358 nid = page_to_nid(page); 3359 if (!node_isset(nid, node_states[N_MEMORY])) 3360 return NULL; 3361 3362 return page; 3363 } 3364 3365 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3366 static struct page *can_gather_numa_stats_pmd(pmd_t pmd, 3367 struct vm_area_struct *vma, 3368 unsigned long addr) 3369 { 3370 struct page *page; 3371 int nid; 3372 3373 if (!pmd_present(pmd)) 3374 return NULL; 3375 3376 page = vm_normal_page_pmd(vma, addr, pmd); 3377 if (!page) 3378 return NULL; 3379 3380 if (PageReserved(page)) 3381 return NULL; 3382 3383 nid = page_to_nid(page); 3384 if (!node_isset(nid, node_states[N_MEMORY])) 3385 return NULL; 3386 3387 return page; 3388 } 3389 #endif 3390 3391 static int gather_pte_stats(pmd_t *pmd, unsigned long addr, 3392 unsigned long end, struct mm_walk *walk) 3393 { 3394 struct numa_maps *md = walk->private; 3395 struct vm_area_struct *vma = walk->vma; 3396 spinlock_t *ptl; 3397 pte_t *orig_pte; 3398 pte_t *pte; 3399 3400 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3401 ptl = pmd_trans_huge_lock(pmd, vma); 3402 if (ptl) { 3403 struct page *page; 3404 3405 page = can_gather_numa_stats_pmd(*pmd, vma, addr); 3406 if (page) 3407 gather_stats(page, md, pmd_dirty(*pmd), 3408 HPAGE_PMD_SIZE/PAGE_SIZE); 3409 spin_unlock(ptl); 3410 return 0; 3411 } 3412 #endif 3413 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 3414 if (!pte) { 3415 walk->action = ACTION_AGAIN; 3416 return 0; 3417 } 3418 do { 3419 pte_t ptent = ptep_get(pte); 3420 struct page *page = can_gather_numa_stats(ptent, vma, addr); 3421 if (!page) 3422 continue; 3423 gather_stats(page, md, pte_dirty(ptent), 1); 3424 3425 } while (pte++, addr += PAGE_SIZE, addr != end); 3426 pte_unmap_unlock(orig_pte, ptl); 3427 cond_resched(); 3428 return 0; 3429 } 3430 #ifdef CONFIG_HUGETLB_PAGE 3431 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3432 unsigned long addr, unsigned long end, struct mm_walk *walk) 3433 { 3434 pte_t huge_pte; 3435 struct numa_maps *md; 3436 struct page *page; 3437 spinlock_t *ptl; 3438 3439 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); 3440 huge_pte = huge_ptep_get(walk->mm, addr, pte); 3441 if (!pte_present(huge_pte)) 3442 goto out; 3443 3444 page = pte_page(huge_pte); 3445 3446 md = walk->private; 3447 gather_stats(page, md, pte_dirty(huge_pte), 1); 3448 out: 3449 spin_unlock(ptl); 3450 return 0; 3451 } 3452 3453 #else 3454 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3455 unsigned long addr, unsigned long end, struct mm_walk *walk) 3456 { 3457 return 0; 3458 } 3459 #endif 3460 3461 static const struct mm_walk_ops show_numa_ops = { 3462 .hugetlb_entry = gather_hugetlb_stats, 3463 .pmd_entry = gather_pte_stats, 3464 .walk_lock = PGWALK_RDLOCK, 3465 }; 3466 3467 #ifdef CONFIG_PER_VMA_LOCK 3468 static const struct mm_walk_ops show_numa_vma_lock_ops = { 3469 .hugetlb_entry = gather_hugetlb_stats, 3470 .pmd_entry = gather_pte_stats, 3471 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 3472 }; 3473 3474 static inline const struct mm_walk_ops * 3475 get_show_numa_ops(struct proc_maps_private *priv) 3476 { 3477 if (priv->lock_ctx.mmap_locked) 3478 return &show_numa_ops; 3479 return &show_numa_vma_lock_ops; 3480 } 3481 3482 #else /* CONFIG_PER_VMA_LOCK */ 3483 3484 static inline const struct mm_walk_ops * 3485 get_show_numa_ops(struct proc_maps_private *priv) 3486 { 3487 return &show_numa_ops; 3488 } 3489 3490 #endif /* CONFIG_PER_VMA_LOCK */ 3491 3492 /* 3493 * Display pages allocated per node and memory policy via /proc. 3494 */ 3495 static int show_numa_map(struct seq_file *m, void *v) 3496 { 3497 struct numa_maps_private *numa_priv = m->private; 3498 struct proc_maps_private *proc_priv = &numa_priv->proc_maps; 3499 struct vm_area_struct *vma = v; 3500 struct numa_maps *md = &numa_priv->md; 3501 struct file *file = vma->vm_file; 3502 struct mm_struct *mm = vma->vm_mm; 3503 char buffer[64]; 3504 struct mempolicy *pol; 3505 pgoff_t ilx; 3506 int nid; 3507 3508 if (!mm) 3509 return 0; 3510 3511 /* Ensure we start with an empty set of numa_maps statistics. */ 3512 memset(md, 0, sizeof(*md)); 3513 3514 pol = __get_vma_policy(vma, vma->vm_start, &ilx); 3515 if (pol) { 3516 mpol_to_str(buffer, sizeof(buffer), pol); 3517 mpol_cond_put(pol); 3518 } else { 3519 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy); 3520 } 3521 3522 seq_printf(m, "%08lx %s", vma->vm_start, buffer); 3523 3524 if (file) { 3525 seq_puts(m, " file="); 3526 seq_path(m, file_user_path(file), "\n\t= "); 3527 } else if (vma_is_initial_heap(vma)) { 3528 seq_puts(m, " heap"); 3529 } else if (vma_is_initial_stack(vma)) { 3530 seq_puts(m, " stack"); 3531 } 3532 3533 if (is_vm_hugetlb_page(vma)) 3534 seq_puts(m, " huge"); 3535 3536 /* Skip walking pages if gate VMA */ 3537 if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) { 3538 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 3539 drop_rcu(proc_priv); 3540 walk_page_vma(vma, get_show_numa_ops(proc_priv), md); 3541 reacquire_rcu(proc_priv); 3542 } 3543 3544 if (!md->pages) 3545 goto out; 3546 3547 if (md->anon) 3548 seq_printf(m, " anon=%lu", md->anon); 3549 3550 if (md->dirty) 3551 seq_printf(m, " dirty=%lu", md->dirty); 3552 3553 if (md->pages != md->anon && md->pages != md->dirty) 3554 seq_printf(m, " mapped=%lu", md->pages); 3555 3556 if (md->mapcount_max > 1) 3557 seq_printf(m, " mapmax=%lu", md->mapcount_max); 3558 3559 if (md->swapcache) 3560 seq_printf(m, " swapcache=%lu", md->swapcache); 3561 3562 if (md->active < md->pages && !is_vm_hugetlb_page(vma)) 3563 seq_printf(m, " active=%lu", md->active); 3564 3565 if (md->writeback) 3566 seq_printf(m, " writeback=%lu", md->writeback); 3567 3568 for_each_node_state(nid, N_MEMORY) 3569 if (md->node[nid]) 3570 seq_printf(m, " N%d=%lu", nid, md->node[nid]); 3571 3572 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); 3573 out: 3574 seq_putc(m, '\n'); 3575 return 0; 3576 } 3577 3578 static const struct seq_operations proc_pid_numa_maps_op = { 3579 .start = m_start, 3580 .next = m_next, 3581 .stop = m_stop, 3582 .show = show_numa_map, 3583 }; 3584 3585 static int pid_numa_maps_open(struct inode *inode, struct file *file) 3586 { 3587 return proc_maps_open(inode, file, &proc_pid_numa_maps_op, 3588 sizeof(struct numa_maps_private)); 3589 } 3590 3591 const struct file_operations proc_pid_numa_maps_operations = { 3592 .open = pid_numa_maps_open, 3593 .read = seq_read, 3594 .llseek = seq_lseek, 3595 .release = proc_map_release, 3596 }; 3597 3598 #endif /* CONFIG_NUMA */ 3599