1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pagewalk.h> 3 #include <linux/mm_inline.h> 4 #include <linux/hugetlb.h> 5 #include <linux/huge_mm.h> 6 #include <linux/mount.h> 7 #include <linux/ksm.h> 8 #include <linux/seq_file.h> 9 #include <linux/highmem.h> 10 #include <linux/ptrace.h> 11 #include <linux/slab.h> 12 #include <linux/pagemap.h> 13 #include <linux/mempolicy.h> 14 #include <linux/rmap.h> 15 #include <linux/swap.h> 16 #include <linux/sched/mm.h> 17 #include <linux/leafops.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/page_idle.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/uaccess.h> 22 #include <linux/pkeys.h> 23 #include <linux/minmax.h> 24 #include <linux/overflow.h> 25 #include <linux/buildid.h> 26 27 #include <asm/elf.h> 28 #include <asm/tlb.h> 29 #include <asm/tlbflush.h> 30 #include "internal.h" 31 32 #define SENTINEL_VMA_END -1 33 #define SENTINEL_VMA_GATE -2 34 35 #define SEQ_PUT_DEC(str, val) \ 36 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) 37 void task_mem(struct seq_file *m, struct mm_struct *mm) 38 { 39 unsigned long text, lib, swap, anon, file, shmem; 40 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; 41 42 anon = get_mm_counter_sum(mm, MM_ANONPAGES); 43 file = get_mm_counter_sum(mm, MM_FILEPAGES); 44 shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES); 45 46 /* 47 * Note: to minimize their overhead, mm maintains hiwater_vm and 48 * hiwater_rss only when about to *lower* total_vm or rss. Any 49 * collector of these hiwater stats must therefore get total_vm 50 * and rss too, which will usually be the higher. Barriers? not 51 * worth the effort, such snapshots can always be inconsistent. 52 */ 53 hiwater_vm = total_vm = mm->total_vm; 54 if (hiwater_vm < mm->hiwater_vm) 55 hiwater_vm = mm->hiwater_vm; 56 hiwater_rss = total_rss = anon + file + shmem; 57 if (hiwater_rss < mm->hiwater_rss) 58 hiwater_rss = mm->hiwater_rss; 59 60 /* split executable areas between text and lib */ 61 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); 62 text = min(text, mm->exec_vm << PAGE_SHIFT); 63 lib = (mm->exec_vm << PAGE_SHIFT) - text; 64 65 swap = get_mm_counter_sum(mm, MM_SWAPENTS); 66 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); 67 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); 68 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); 69 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); 70 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); 71 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); 72 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); 73 SEQ_PUT_DEC(" kB\nRssFile:\t", file); 74 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem); 75 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm); 76 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm); 77 seq_put_decimal_ull_width(m, 78 " kB\nVmExe:\t", text >> 10, 8); 79 seq_put_decimal_ull_width(m, 80 " kB\nVmLib:\t", lib >> 10, 8); 81 seq_put_decimal_ull_width(m, 82 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8); 83 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap); 84 seq_puts(m, " kB\n"); 85 hugetlb_report_usage(m, mm); 86 } 87 #undef SEQ_PUT_DEC 88 89 unsigned long task_vsize(struct mm_struct *mm) 90 { 91 return PAGE_SIZE * mm->total_vm; 92 } 93 94 unsigned long task_statm(struct mm_struct *mm, 95 unsigned long *shared, unsigned long *text, 96 unsigned long *data, unsigned long *resident) 97 { 98 *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + 99 get_mm_counter_sum(mm, MM_SHMEMPAGES); 100 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) 101 >> PAGE_SHIFT; 102 *data = mm->data_vm + mm->stack_vm; 103 *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); 104 return mm->total_vm; 105 } 106 107 #ifdef CONFIG_NUMA 108 /* 109 * Save get_task_policy() for show_numa_map(). 110 */ 111 static void hold_task_mempolicy(struct proc_maps_private *priv) 112 { 113 struct task_struct *task = priv->task; 114 115 task_lock(task); 116 priv->task_mempolicy = get_task_policy(task); 117 mpol_get(priv->task_mempolicy); 118 task_unlock(task); 119 } 120 static void release_task_mempolicy(struct proc_maps_private *priv) 121 { 122 mpol_put(priv->task_mempolicy); 123 } 124 #else 125 static void hold_task_mempolicy(struct proc_maps_private *priv) 126 { 127 } 128 static void release_task_mempolicy(struct proc_maps_private *priv) 129 { 130 } 131 #endif 132 133 static void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 134 { 135 mmap_read_unlock(lock_ctx->mm); 136 lock_ctx->mmap_locked = false; 137 } 138 139 static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) 140 { 141 if (lock_ctx->locked_vma) { 142 vma_end_read(lock_ctx->locked_vma); 143 lock_ctx->locked_vma = NULL; 144 } 145 } 146 147 static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) 148 { 149 lock_ctx->locked_vma = NULL; 150 lock_ctx->mmap_locked = false; 151 } 152 153 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, 154 loff_t last_pos) 155 { 156 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 157 struct vm_area_struct *vma; 158 159 if (lock_ctx->mmap_locked) 160 return vma_next(&priv->iter); 161 162 unlock_ctx_vma(lock_ctx); 163 vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); 164 if (!IS_ERR_OR_NULL(vma)) 165 lock_ctx->locked_vma = vma; 166 167 return vma; 168 } 169 170 static bool fallback_to_mmap_lock(struct proc_maps_private *priv, 171 loff_t pos) 172 { 173 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 174 175 if (lock_ctx->mmap_locked) 176 return false; 177 178 rcu_read_unlock(); 179 mmap_read_lock(lock_ctx->mm); 180 /* Reinitialize the iterator after taking mmap_lock */ 181 vma_iter_set(&priv->iter, pos); 182 lock_ctx->mmap_locked = true; 183 184 return true; 185 } 186 187 #if defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA) 188 static void drop_rcu(struct proc_maps_private *priv) 189 { 190 if (priv->lock_ctx.mmap_locked) 191 return; 192 193 rcu_read_unlock(); 194 } 195 196 static void reacquire_rcu(struct proc_maps_private *priv) 197 { 198 if (priv->lock_ctx.mmap_locked) 199 return; 200 201 rcu_read_lock(); 202 /* Reinitialize the iterator. */ 203 vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); 204 } 205 #endif 206 207 static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) 208 { 209 struct proc_maps_private *priv = m->private; 210 struct vm_area_struct *vma; 211 212 retry: 213 vma = get_next_vma(priv, *ppos); 214 /* EINTR of EAGAIN is possible */ 215 if (IS_ERR(vma)) { 216 if (PTR_ERR(vma) == -EAGAIN && fallback_to_mmap_lock(priv, *ppos)) 217 goto retry; 218 219 return vma; 220 } 221 222 /* Store previous position to be able to restart if needed */ 223 priv->last_pos = *ppos; 224 if (vma) { 225 /* 226 * Track the end of the reported vma to ensure position changes 227 * even if previous vma was merged with the next vma and we 228 * found the extended vma with the same vm_start. 229 */ 230 *ppos = vma->vm_end; 231 } 232 233 return vma; 234 } 235 236 static void *m_start(struct seq_file *m, loff_t *ppos) 237 { 238 struct proc_maps_private *priv = m->private; 239 struct proc_maps_locking_ctx *lock_ctx; 240 struct vm_area_struct *vma; 241 loff_t last_addr = *ppos; 242 struct mm_struct *mm; 243 244 /* See m_next(). Zero at the start or after lseek. */ 245 if (last_addr == SENTINEL_VMA_END) 246 return NULL; 247 248 priv->task = get_proc_task(priv->inode); 249 if (!priv->task) 250 return ERR_PTR(-ESRCH); 251 252 lock_ctx = &priv->lock_ctx; 253 mm = lock_ctx->mm; 254 if (!mm || !mmget_not_zero(mm)) { 255 put_task_struct(priv->task); 256 priv->task = NULL; 257 return NULL; 258 } 259 260 rcu_read_lock(); 261 reset_lock_ctx(lock_ctx); 262 /* 263 * Reset current position if last_addr was set before 264 * and it's not a sentinel. 265 */ 266 if (last_addr > 0) 267 *ppos = last_addr = priv->last_pos; 268 vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); 269 hold_task_mempolicy(priv); 270 /* 271 * If seq_file had to flush its collected data right after m_next() set 272 * position to SENTINEL_VMA_GATE, m_start() will get that sentinel and 273 * should return gate_vma without calling proc_get_vma(). 274 */ 275 if (last_addr == SENTINEL_VMA_GATE) 276 return get_gate_vma(mm); 277 278 vma = proc_get_vma(m, ppos); 279 if (vma) 280 return vma; 281 282 /* Return gate VMA at the end */ 283 *ppos = SENTINEL_VMA_GATE; 284 return get_gate_vma(mm); 285 } 286 287 static void *m_next(struct seq_file *m, void *v, loff_t *ppos) 288 { 289 struct proc_maps_private *priv = m->private; 290 struct vm_area_struct *vma; 291 292 if (*ppos == SENTINEL_VMA_GATE) { 293 *ppos = SENTINEL_VMA_END; 294 return NULL; 295 } 296 vma = proc_get_vma(m, ppos); 297 if (vma) 298 return vma; 299 300 /* Return gate VMA at the end */ 301 *ppos = SENTINEL_VMA_GATE; 302 return get_gate_vma(priv->lock_ctx.mm); 303 } 304 305 static void m_stop(struct seq_file *m, void *v) 306 { 307 struct proc_maps_private *priv = m->private; 308 struct mm_struct *mm = priv->lock_ctx.mm; 309 310 if (!priv->task) 311 return; 312 313 release_task_mempolicy(priv); 314 if (priv->lock_ctx.mmap_locked) { 315 unlock_ctx_mm(&priv->lock_ctx); 316 } else { 317 unlock_ctx_vma(&priv->lock_ctx); 318 rcu_read_unlock(); 319 } 320 mmput(mm); 321 put_task_struct(priv->task); 322 priv->task = NULL; 323 } 324 325 static int proc_maps_open(struct inode *inode, struct file *file, 326 const struct seq_operations *ops, int psize) 327 { 328 struct proc_maps_private *priv = __seq_open_private(file, ops, psize); 329 330 if (!priv) 331 return -ENOMEM; 332 333 priv->inode = inode; 334 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 335 if (IS_ERR(priv->lock_ctx.mm)) { 336 int err = PTR_ERR(priv->lock_ctx.mm); 337 338 seq_release_private(inode, file); 339 return err; 340 } 341 342 return 0; 343 } 344 345 static int proc_map_release(struct inode *inode, struct file *file) 346 { 347 struct seq_file *seq = file->private_data; 348 struct proc_maps_private *priv = seq->private; 349 350 if (priv->lock_ctx.mm) 351 mmdrop(priv->lock_ctx.mm); 352 353 return seq_release_private(inode, file); 354 } 355 356 static int do_maps_open(struct inode *inode, struct file *file, 357 const struct seq_operations *ops) 358 { 359 return proc_maps_open(inode, file, ops, 360 sizeof(struct proc_maps_private)); 361 } 362 363 static void get_vma_name(struct vm_area_struct *vma, 364 const struct path **path, 365 const char **name, 366 const char **name_fmt) 367 { 368 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL; 369 370 *name = NULL; 371 *path = NULL; 372 *name_fmt = NULL; 373 374 /* 375 * Print the dentry name for named mappings, and a 376 * special [heap] marker for the heap: 377 */ 378 if (vma->vm_file) { 379 /* 380 * If user named this anon shared memory via 381 * prctl(PR_SET_VMA ..., use the provided name. 382 */ 383 if (anon_name) { 384 *name_fmt = "[anon_shmem:%s]"; 385 *name = anon_name->name; 386 } else { 387 *path = file_user_path(vma->vm_file); 388 } 389 return; 390 } 391 392 if (vma->vm_ops && vma->vm_ops->name) { 393 *name = vma->vm_ops->name(vma); 394 if (*name) 395 return; 396 } 397 398 *name = arch_vma_name(vma); 399 if (*name) 400 return; 401 402 if (!vma->vm_mm) { 403 *name = "[vdso]"; 404 return; 405 } 406 407 if (vma_is_initial_heap(vma)) { 408 *name = "[heap]"; 409 return; 410 } 411 412 if (vma_is_initial_stack(vma)) { 413 *name = "[stack]"; 414 return; 415 } 416 417 if (anon_name) { 418 *name_fmt = "[anon:%s]"; 419 *name = anon_name->name; 420 return; 421 } 422 } 423 424 static void show_vma_header_prefix(struct seq_file *m, 425 unsigned long start, unsigned long end, 426 vm_flags_t flags, unsigned long long pgoff, 427 dev_t dev, u64 ino) 428 { 429 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); 430 seq_put_hex_ll(m, NULL, start, 8); 431 seq_put_hex_ll(m, "-", end, 8); 432 seq_putc(m, ' '); 433 seq_putc(m, flags & VM_READ ? 'r' : '-'); 434 seq_putc(m, flags & VM_WRITE ? 'w' : '-'); 435 seq_putc(m, flags & VM_EXEC ? 'x' : '-'); 436 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p'); 437 seq_put_hex_ll(m, " ", pgoff, 8); 438 seq_put_hex_ll(m, " ", MAJOR(dev), 2); 439 seq_put_hex_ll(m, ":", MINOR(dev), 2); 440 seq_put_decimal_ull(m, " ", ino); 441 seq_putc(m, ' '); 442 } 443 444 static void 445 show_map_vma(struct seq_file *m, struct vm_area_struct *vma) 446 { 447 const struct path *path; 448 const char *name_fmt, *name; 449 vm_flags_t flags = vma->vm_flags; 450 u64 ino = 0; 451 unsigned long long pgoff = 0; 452 unsigned long start, end; 453 dev_t dev = 0; 454 455 if (vma->vm_file) { 456 const struct inode *inode = file_user_inode(vma->vm_file); 457 458 dev = inode->i_sb->s_dev; 459 ino = inode->i_ino; 460 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; 461 } 462 463 start = vma->vm_start; 464 end = vma->vm_end; 465 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino); 466 467 get_vma_name(vma, &path, &name, &name_fmt); 468 if (path) { 469 seq_pad(m, ' '); 470 seq_path(m, path, "\n"); 471 } else if (name_fmt) { 472 seq_pad(m, ' '); 473 seq_printf(m, name_fmt, name); 474 } else if (name) { 475 seq_pad(m, ' '); 476 seq_puts(m, name); 477 } 478 seq_putc(m, '\n'); 479 } 480 481 static int show_map(struct seq_file *m, void *v) 482 { 483 show_map_vma(m, v); 484 return 0; 485 } 486 487 static const struct seq_operations proc_pid_maps_op = { 488 .start = m_start, 489 .next = m_next, 490 .stop = m_stop, 491 .show = show_map 492 }; 493 494 static int pid_maps_open(struct inode *inode, struct file *file) 495 { 496 return do_maps_open(inode, file, &proc_pid_maps_op); 497 } 498 499 #define PROCMAP_QUERY_VMA_FLAGS ( \ 500 PROCMAP_QUERY_VMA_READABLE | \ 501 PROCMAP_QUERY_VMA_WRITABLE | \ 502 PROCMAP_QUERY_VMA_EXECUTABLE | \ 503 PROCMAP_QUERY_VMA_SHARED \ 504 ) 505 506 #define PROCMAP_QUERY_VALID_FLAGS_MASK ( \ 507 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \ 508 PROCMAP_QUERY_FILE_BACKED_VMA | \ 509 PROCMAP_QUERY_VMA_FLAGS \ 510 ) 511 512 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, 513 unsigned long addr) 514 { 515 struct mm_struct *mm = lock_ctx->mm; 516 struct vm_area_struct *vma; 517 struct vma_iterator vmi; 518 519 if (lock_ctx->mmap_locked) 520 return find_vma(mm, addr); 521 522 /* Unlock previously locked VMA and find the next one under RCU */ 523 unlock_ctx_vma(lock_ctx); 524 rcu_read_lock(); 525 vma_iter_init(&vmi, mm, addr); 526 vma = lock_next_vma(mm, &vmi, addr); 527 rcu_read_unlock(); 528 529 if (!vma) 530 return NULL; 531 532 if (!IS_ERR(vma)) { 533 lock_ctx->locked_vma = vma; 534 return vma; 535 } 536 537 if (PTR_ERR(vma) == -EAGAIN) { 538 /* Fallback to mmap_lock on vma->vm_refcnt overflow */ 539 mmap_read_lock(mm); 540 vma = find_vma(mm, addr); 541 lock_ctx->mmap_locked = true; 542 } 543 544 return vma; 545 } 546 547 static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, 548 unsigned long addr, u32 flags) 549 { 550 struct vm_area_struct *vma; 551 552 next_vma: 553 vma = query_vma_find_by_addr(lock_ctx, addr); 554 if (IS_ERR(vma)) 555 return vma; 556 557 if (!vma) 558 goto no_vma; 559 560 /* user requested only file-backed VMA, keep iterating */ 561 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file) 562 goto skip_vma; 563 564 /* VMA permissions should satisfy query flags */ 565 if (flags & PROCMAP_QUERY_VMA_FLAGS) { 566 u32 perm = 0; 567 568 if (flags & PROCMAP_QUERY_VMA_READABLE) 569 perm |= VM_READ; 570 if (flags & PROCMAP_QUERY_VMA_WRITABLE) 571 perm |= VM_WRITE; 572 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE) 573 perm |= VM_EXEC; 574 if (flags & PROCMAP_QUERY_VMA_SHARED) 575 perm |= VM_MAYSHARE; 576 577 if ((vma->vm_flags & perm) != perm) 578 goto skip_vma; 579 } 580 581 /* found covering VMA or user is OK with the matching next VMA */ 582 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) 583 return vma; 584 585 skip_vma: 586 /* 587 * If the user needs closest matching VMA, keep iterating. 588 */ 589 addr = vma->vm_end; 590 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) 591 goto next_vma; 592 593 no_vma: 594 return ERR_PTR(-ENOENT); 595 } 596 597 static int do_procmap_query(struct mm_struct *mm, void __user *uarg) 598 { 599 struct proc_maps_locking_ctx lock_ctx = { .mm = mm }; 600 struct procmap_query karg; 601 struct vm_area_struct *vma; 602 struct file *vm_file = NULL; 603 const char *name = NULL; 604 char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; 605 __u64 usize; 606 int err; 607 608 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize))) 609 return -EFAULT; 610 /* argument struct can never be that large, reject abuse */ 611 if (usize > PAGE_SIZE) 612 return -E2BIG; 613 /* argument struct should have at least query_flags and query_addr fields */ 614 if (usize < offsetofend(struct procmap_query, query_addr)) 615 return -EINVAL; 616 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); 617 if (err) 618 return err; 619 620 /* reject unknown flags */ 621 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK) 622 return -EINVAL; 623 /* either both buffer address and size are set, or both should be zero */ 624 if (!!karg.vma_name_size != !!karg.vma_name_addr) 625 return -EINVAL; 626 if (!!karg.build_id_size != !!karg.build_id_addr) 627 return -EINVAL; 628 629 if (!mm || !mmget_not_zero(mm)) 630 return -ESRCH; 631 632 reset_lock_ctx(&lock_ctx); 633 vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); 634 if (IS_ERR(vma)) { 635 err = PTR_ERR(vma); 636 vma = NULL; 637 goto out; 638 } 639 640 karg.vma_start = vma->vm_start; 641 karg.vma_end = vma->vm_end; 642 643 karg.vma_flags = 0; 644 if (vma->vm_flags & VM_READ) 645 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE; 646 if (vma->vm_flags & VM_WRITE) 647 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE; 648 if (vma->vm_flags & VM_EXEC) 649 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE; 650 if (vma->vm_flags & VM_MAYSHARE) 651 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED; 652 653 karg.vma_page_size = vma_kernel_pagesize(vma); 654 655 if (vma->vm_file) { 656 const struct inode *inode = file_user_inode(vma->vm_file); 657 658 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; 659 karg.dev_major = MAJOR(inode->i_sb->s_dev); 660 karg.dev_minor = MINOR(inode->i_sb->s_dev); 661 karg.inode = inode->i_ino; 662 } else { 663 karg.vma_offset = 0; 664 karg.dev_major = 0; 665 karg.dev_minor = 0; 666 karg.inode = 0; 667 } 668 669 if (karg.vma_name_size) { 670 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); 671 const struct path *path; 672 const char *name_fmt; 673 size_t name_sz = 0; 674 675 get_vma_name(vma, &path, &name, &name_fmt); 676 677 if (path || name_fmt || name) { 678 name_buf = kmalloc(name_buf_sz, GFP_KERNEL); 679 if (!name_buf) { 680 err = -ENOMEM; 681 goto out; 682 } 683 } 684 if (path) { 685 name = d_path(path, name_buf, name_buf_sz); 686 if (IS_ERR(name)) { 687 err = PTR_ERR(name); 688 goto out; 689 } 690 name_sz = name_buf + name_buf_sz - name; 691 } else if (name || name_fmt) { 692 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name); 693 name = name_buf; 694 } 695 if (name_sz > name_buf_sz) { 696 err = -ENAMETOOLONG; 697 goto out; 698 } 699 karg.vma_name_size = name_sz; 700 } 701 702 if (karg.build_id_size && vma->vm_file) 703 vm_file = get_file(vma->vm_file); 704 705 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ 706 if (lock_ctx.mmap_locked) 707 unlock_ctx_mm(&lock_ctx); 708 else 709 unlock_ctx_vma(&lock_ctx); 710 mmput(mm); 711 712 if (karg.build_id_size) { 713 __u32 build_id_sz; 714 715 if (vm_file) 716 err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz); 717 else 718 err = -ENOENT; 719 if (err) { 720 karg.build_id_size = 0; 721 } else { 722 if (karg.build_id_size < build_id_sz) { 723 err = -ENAMETOOLONG; 724 goto out_file; 725 } 726 karg.build_id_size = build_id_sz; 727 } 728 } 729 730 if (vm_file) 731 fput(vm_file); 732 733 if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), 734 name, karg.vma_name_size)) { 735 kfree(name_buf); 736 return -EFAULT; 737 } 738 kfree(name_buf); 739 740 if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr), 741 build_id_buf, karg.build_id_size)) 742 return -EFAULT; 743 744 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize))) 745 return -EFAULT; 746 747 return 0; 748 749 out: 750 if (lock_ctx.mmap_locked) 751 unlock_ctx_mm(&lock_ctx); 752 else 753 unlock_ctx_vma(&lock_ctx); 754 mmput(mm); 755 out_file: 756 if (vm_file) 757 fput(vm_file); 758 kfree(name_buf); 759 return err; 760 } 761 762 static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 763 { 764 struct seq_file *seq = file->private_data; 765 struct proc_maps_private *priv = seq->private; 766 767 switch (cmd) { 768 case PROCMAP_QUERY: 769 /* priv->lock_ctx.mm is set during file open operation */ 770 return do_procmap_query(priv->lock_ctx.mm, (void __user *)arg); 771 default: 772 return -ENOIOCTLCMD; 773 } 774 } 775 776 const struct file_operations proc_pid_maps_operations = { 777 .open = pid_maps_open, 778 .read = seq_read, 779 .llseek = seq_lseek, 780 .release = proc_map_release, 781 .unlocked_ioctl = procfs_procmap_ioctl, 782 .compat_ioctl = compat_ptr_ioctl, 783 }; 784 785 /* 786 * Proportional Set Size(PSS): my share of RSS. 787 * 788 * PSS of a process is the count of pages it has in memory, where each 789 * page is divided by the number of processes sharing it. So if a 790 * process has 1000 pages all to itself, and 1000 shared with one other 791 * process, its PSS will be 1500. 792 * 793 * To keep (accumulated) division errors low, we adopt a 64bit 794 * fixed-point pss counter to minimize division errors. So (pss >> 795 * PSS_SHIFT) would be the real byte count. 796 * 797 * A shift of 12 before division means (assuming 4K page size): 798 * - 1M 3-user-pages add up to 8KB errors; 799 * - supports mapcount up to 2^24, or 16M; 800 * - supports PSS up to 2^52 bytes, or 4PB. 801 */ 802 #define PSS_SHIFT 12 803 804 #ifdef CONFIG_PROC_PAGE_MONITOR 805 struct mem_size_stats { 806 unsigned long resident; 807 unsigned long shared_clean; 808 unsigned long shared_dirty; 809 unsigned long private_clean; 810 unsigned long private_dirty; 811 unsigned long referenced; 812 unsigned long anonymous; 813 unsigned long lazyfree; 814 unsigned long anonymous_thp; 815 unsigned long shmem_thp; 816 unsigned long file_thp; 817 unsigned long swap; 818 unsigned long shared_hugetlb; 819 unsigned long private_hugetlb; 820 unsigned long ksm; 821 u64 pss; 822 u64 pss_anon; 823 u64 pss_file; 824 u64 pss_shmem; 825 u64 pss_dirty; 826 u64 pss_locked; 827 u64 swap_pss; 828 }; 829 830 static void smaps_page_accumulate(struct mem_size_stats *mss, 831 struct folio *folio, unsigned long size, unsigned long pss, 832 bool dirty, bool locked, bool private) 833 { 834 mss->pss += pss; 835 836 if (folio_test_anon(folio)) 837 mss->pss_anon += pss; 838 else if (folio_test_swapbacked(folio)) 839 mss->pss_shmem += pss; 840 else 841 mss->pss_file += pss; 842 843 if (locked) 844 mss->pss_locked += pss; 845 846 if (dirty || folio_test_dirty(folio)) { 847 mss->pss_dirty += pss; 848 if (private) 849 mss->private_dirty += size; 850 else 851 mss->shared_dirty += size; 852 } else { 853 if (private) 854 mss->private_clean += size; 855 else 856 mss->shared_clean += size; 857 } 858 } 859 860 static void smaps_account(struct mem_size_stats *mss, struct page *page, 861 bool compound, bool young, bool dirty, bool locked, 862 bool present) 863 { 864 struct folio *folio = page_folio(page); 865 int i, nr = compound ? compound_nr(page) : 1; 866 unsigned long size = nr * PAGE_SIZE; 867 bool exclusive; 868 int mapcount; 869 870 /* 871 * First accumulate quantities that depend only on |size| and the type 872 * of the compound page. 873 */ 874 if (folio_test_anon(folio)) { 875 mss->anonymous += size; 876 if (!folio_test_swapbacked(folio) && !dirty && 877 !folio_test_dirty(folio)) 878 mss->lazyfree += size; 879 } 880 881 if (folio_test_ksm(folio)) 882 mss->ksm += size; 883 884 mss->resident += size; 885 /* Accumulate the size in pages that have been accessed. */ 886 if (young || folio_test_young(folio) || folio_test_referenced(folio)) 887 mss->referenced += size; 888 889 /* 890 * Then accumulate quantities that may depend on sharing, or that may 891 * differ page-by-page. 892 * 893 * refcount == 1 for present entries guarantees that the folio is mapped 894 * exactly once. For large folios this implies that exactly one 895 * PTE/PMD/... maps (a part of) this folio. 896 * 897 * Treat all non-present entries (where relying on the mapcount and 898 * refcount doesn't make sense) as "maybe shared, but not sure how 899 * often". We treat device private entries as being fake-present. 900 * 901 * Note that it would not be safe to read the mapcount especially for 902 * pages referenced by migration entries, even with the PTL held. 903 */ 904 if (folio_ref_count(folio) == 1 || !present) { 905 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, 906 dirty, locked, present); 907 return; 908 } 909 910 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 911 mapcount = folio_average_page_mapcount(folio); 912 exclusive = !folio_maybe_mapped_shared(folio); 913 } 914 915 /* 916 * We obtain a snapshot of the mapcount. Without holding the folio lock 917 * this snapshot can be slightly wrong as we cannot always read the 918 * mapcount atomically. 919 */ 920 for (i = 0; i < nr; i++, page++) { 921 unsigned long pss = PAGE_SIZE << PSS_SHIFT; 922 923 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { 924 mapcount = folio_precise_page_mapcount(folio, page); 925 exclusive = mapcount < 2; 926 } 927 928 if (mapcount >= 2) 929 pss /= mapcount; 930 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, 931 dirty, locked, exclusive); 932 } 933 } 934 935 #ifdef CONFIG_SHMEM 936 static int smaps_pte_hole(unsigned long addr, unsigned long end, 937 __always_unused int depth, struct mm_walk *walk) 938 { 939 struct mem_size_stats *mss = walk->private; 940 struct vm_area_struct *vma = walk->vma; 941 942 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, 943 linear_page_index(vma, addr), 944 linear_page_index(vma, end)); 945 946 return 0; 947 } 948 #else 949 #define smaps_pte_hole NULL 950 #endif /* CONFIG_SHMEM */ 951 952 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) 953 { 954 #ifdef CONFIG_SHMEM 955 if (walk->ops->pte_hole) { 956 /* depth is not used */ 957 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); 958 } 959 #endif 960 } 961 962 static void smaps_pte_entry(pte_t *pte, unsigned long addr, 963 struct mm_walk *walk) 964 { 965 struct mem_size_stats *mss = walk->private; 966 struct vm_area_struct *vma = walk->vma; 967 bool locked = !!(vma->vm_flags & VM_LOCKED); 968 struct page *page = NULL; 969 bool present = false, young = false, dirty = false; 970 pte_t ptent = ptep_get(pte); 971 972 if (pte_present(ptent)) { 973 page = vm_normal_page(vma, addr, ptent); 974 young = pte_young(ptent); 975 dirty = pte_dirty(ptent); 976 present = true; 977 } else if (pte_none(ptent)) { 978 smaps_pte_hole_lookup(addr, walk); 979 } else { 980 const softleaf_t entry = softleaf_from_pte(ptent); 981 982 if (softleaf_is_swap(entry)) { 983 int mapcount; 984 985 mss->swap += PAGE_SIZE; 986 mapcount = swp_swapcount(entry); 987 if (mapcount >= 2) { 988 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; 989 990 do_div(pss_delta, mapcount); 991 mss->swap_pss += pss_delta; 992 } else { 993 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 994 } 995 } else if (softleaf_has_pfn(entry)) { 996 if (softleaf_is_device_private(entry)) 997 present = true; 998 page = softleaf_to_page(entry); 999 } 1000 } 1001 1002 if (!page) 1003 return; 1004 1005 smaps_account(mss, page, false, young, dirty, locked, present); 1006 } 1007 1008 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1009 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1010 struct mm_walk *walk) 1011 { 1012 struct mem_size_stats *mss = walk->private; 1013 struct vm_area_struct *vma = walk->vma; 1014 bool locked = !!(vma->vm_flags & VM_LOCKED); 1015 struct page *page = NULL; 1016 bool present = false; 1017 struct folio *folio; 1018 1019 if (pmd_none(*pmd)) 1020 return; 1021 if (pmd_present(*pmd)) { 1022 page = vm_normal_page_pmd(vma, addr, *pmd); 1023 present = true; 1024 } else if (unlikely(thp_migration_supported())) { 1025 const softleaf_t entry = softleaf_from_pmd(*pmd); 1026 1027 if (softleaf_has_pfn(entry)) 1028 page = softleaf_to_page(entry); 1029 } 1030 if (IS_ERR_OR_NULL(page)) 1031 return; 1032 folio = page_folio(page); 1033 if (folio_test_anon(folio)) 1034 mss->anonymous_thp += HPAGE_PMD_SIZE; 1035 else if (folio_test_swapbacked(folio)) 1036 mss->shmem_thp += HPAGE_PMD_SIZE; 1037 else if (folio_is_zone_device(folio)) 1038 /* pass */; 1039 else 1040 mss->file_thp += HPAGE_PMD_SIZE; 1041 1042 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), 1043 locked, present); 1044 } 1045 #else 1046 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1047 struct mm_walk *walk) 1048 { 1049 } 1050 #endif 1051 1052 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 1053 struct mm_walk *walk) 1054 { 1055 struct vm_area_struct *vma = walk->vma; 1056 pte_t *pte; 1057 spinlock_t *ptl; 1058 1059 ptl = pmd_trans_huge_lock(pmd, vma); 1060 if (ptl) { 1061 smaps_pmd_entry(pmd, addr, walk); 1062 spin_unlock(ptl); 1063 goto out; 1064 } 1065 1066 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1067 if (!pte) { 1068 walk->action = ACTION_AGAIN; 1069 return 0; 1070 } 1071 for (; addr != end; pte++, addr += PAGE_SIZE) 1072 smaps_pte_entry(pte, addr, walk); 1073 pte_unmap_unlock(pte - 1, ptl); 1074 out: 1075 cond_resched(); 1076 return 0; 1077 } 1078 1079 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) 1080 { 1081 /* 1082 * Don't forget to update Documentation/ on changes. 1083 * 1084 * The length of the second argument of mnemonics[] 1085 * needs to be 3 instead of previously set 2 1086 * (i.e. from [BITS_PER_LONG][2] to [BITS_PER_LONG][3]) 1087 * to avoid spurious 1088 * -Werror=unterminated-string-initialization warning 1089 * with GCC 15 1090 */ 1091 static const char mnemonics[BITS_PER_LONG][3] = { 1092 /* 1093 * In case if we meet a flag we don't know about. 1094 */ 1095 [0 ... (BITS_PER_LONG-1)] = "??", 1096 1097 [ilog2(VM_READ)] = "rd", 1098 [ilog2(VM_WRITE)] = "wr", 1099 [ilog2(VM_EXEC)] = "ex", 1100 [ilog2(VM_SHARED)] = "sh", 1101 [ilog2(VM_MAYREAD)] = "mr", 1102 [ilog2(VM_MAYWRITE)] = "mw", 1103 [ilog2(VM_MAYEXEC)] = "me", 1104 [ilog2(VM_MAYSHARE)] = "ms", 1105 [ilog2(VM_GROWSDOWN)] = "gd", 1106 [ilog2(VM_PFNMAP)] = "pf", 1107 [ilog2(VM_MAYBE_GUARD)] = "gu", 1108 [ilog2(VM_LOCKED)] = "lo", 1109 [ilog2(VM_IO)] = "io", 1110 [ilog2(VM_SEQ_READ)] = "sr", 1111 [ilog2(VM_RAND_READ)] = "rr", 1112 [ilog2(VM_DONTCOPY)] = "dc", 1113 [ilog2(VM_DONTEXPAND)] = "de", 1114 [ilog2(VM_LOCKONFAULT)] = "lf", 1115 [ilog2(VM_ACCOUNT)] = "ac", 1116 [ilog2(VM_NORESERVE)] = "nr", 1117 [ilog2(VM_HUGETLB)] = "ht", 1118 [ilog2(VM_SYNC)] = "sf", 1119 [ilog2(VM_ARCH_1)] = "ar", 1120 [ilog2(VM_WIPEONFORK)] = "wf", 1121 [ilog2(VM_DONTDUMP)] = "dd", 1122 #ifdef CONFIG_ARM64_BTI 1123 [ilog2(VM_ARM64_BTI)] = "bt", 1124 #endif 1125 #ifdef CONFIG_MEM_SOFT_DIRTY 1126 [ilog2(VM_SOFTDIRTY)] = "sd", 1127 #endif 1128 [ilog2(VM_MIXEDMAP)] = "mm", 1129 [ilog2(VM_HUGEPAGE)] = "hg", 1130 [ilog2(VM_NOHUGEPAGE)] = "nh", 1131 [ilog2(VM_MERGEABLE)] = "mg", 1132 [ilog2(VM_UFFD_MISSING)]= "um", 1133 [ilog2(VM_UFFD_WP)] = "uw", 1134 #ifdef CONFIG_ARM64_MTE 1135 [ilog2(VM_MTE)] = "mt", 1136 [ilog2(VM_MTE_ALLOWED)] = "", 1137 #endif 1138 #ifdef CONFIG_ARCH_HAS_PKEYS 1139 /* These come out via ProtectionKey: */ 1140 [ilog2(VM_PKEY_BIT0)] = "", 1141 [ilog2(VM_PKEY_BIT1)] = "", 1142 [ilog2(VM_PKEY_BIT2)] = "", 1143 #if CONFIG_ARCH_PKEY_BITS > 3 1144 [ilog2(VM_PKEY_BIT3)] = "", 1145 #endif 1146 #if CONFIG_ARCH_PKEY_BITS > 4 1147 [ilog2(VM_PKEY_BIT4)] = "", 1148 #endif 1149 #endif /* CONFIG_ARCH_HAS_PKEYS */ 1150 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1151 [ilog2(VM_UFFD_MINOR)] = "ui", 1152 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 1153 #ifdef CONFIG_USERFAULTFD_RWP 1154 [ilog2(VM_UFFD_RWP)] = "ur", 1155 #endif 1156 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK 1157 [ilog2(VM_SHADOW_STACK)] = "ss", 1158 #endif 1159 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) 1160 [ilog2(VM_DROPPABLE)] = "dp", 1161 #endif 1162 #ifdef CONFIG_64BIT 1163 [ilog2(VM_SEALED)] = "sl", 1164 #endif 1165 }; 1166 size_t i; 1167 1168 seq_puts(m, "VmFlags: "); 1169 for (i = 0; i < BITS_PER_LONG; i++) { 1170 if (!mnemonics[i][0]) 1171 continue; 1172 if (vma->vm_flags & (1UL << i)) 1173 seq_printf(m, "%s ", mnemonics[i]); 1174 } 1175 seq_putc(m, '\n'); 1176 } 1177 1178 #ifdef CONFIG_HUGETLB_PAGE 1179 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, 1180 unsigned long addr, unsigned long end, 1181 struct mm_walk *walk) 1182 { 1183 struct mem_size_stats *mss = walk->private; 1184 struct vm_area_struct *vma = walk->vma; 1185 struct folio *folio = NULL; 1186 bool present = false; 1187 spinlock_t *ptl; 1188 pte_t ptent; 1189 1190 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); 1191 ptent = huge_ptep_get(walk->mm, addr, pte); 1192 if (pte_present(ptent)) { 1193 folio = page_folio(pte_page(ptent)); 1194 present = true; 1195 } else { 1196 const softleaf_t entry = softleaf_from_pte(ptent); 1197 1198 if (softleaf_has_pfn(entry)) 1199 folio = softleaf_to_folio(entry); 1200 } 1201 1202 if (folio) { 1203 /* We treat non-present entries as "maybe shared". */ 1204 if (!present || folio_maybe_mapped_shared(folio) || 1205 hugetlb_pmd_shared(pte)) 1206 mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); 1207 else 1208 mss->private_hugetlb += huge_page_size(hstate_vma(vma)); 1209 } 1210 spin_unlock(ptl); 1211 return 0; 1212 } 1213 #else 1214 #define smaps_hugetlb_range NULL 1215 #endif /* HUGETLB_PAGE */ 1216 1217 static const struct mm_walk_ops smaps_walk_ops = { 1218 .pmd_entry = smaps_pte_range, 1219 .hugetlb_entry = smaps_hugetlb_range, 1220 .walk_lock = PGWALK_RDLOCK, 1221 }; 1222 1223 static const struct mm_walk_ops smaps_shmem_walk_ops = { 1224 .pmd_entry = smaps_pte_range, 1225 .hugetlb_entry = smaps_hugetlb_range, 1226 .pte_hole = smaps_pte_hole, 1227 .walk_lock = PGWALK_RDLOCK, 1228 }; 1229 1230 static const struct mm_walk_ops smaps_walk_vma_lock_ops = { 1231 .pmd_entry = smaps_pte_range, 1232 .hugetlb_entry = smaps_hugetlb_range, 1233 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1234 }; 1235 1236 static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { 1237 .pmd_entry = smaps_pte_range, 1238 .hugetlb_entry = smaps_hugetlb_range, 1239 .pte_hole = smaps_pte_hole, 1240 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1241 }; 1242 1243 static const struct mm_walk_ops * 1244 get_smaps_walk_ops(struct proc_maps_private *priv) 1245 { 1246 if (priv->lock_ctx.mmap_locked) 1247 return &smaps_walk_ops; 1248 return &smaps_walk_vma_lock_ops; 1249 } 1250 1251 static const struct mm_walk_ops * 1252 get_smaps_shmem_walk_ops(struct proc_maps_private *priv) 1253 { 1254 if (priv->lock_ctx.mmap_locked) 1255 return &smaps_shmem_walk_ops; 1256 return &smaps_shmem_walk_vma_lock_ops; 1257 } 1258 1259 /** 1260 * smap_gather_stats_range() - Gather mem stats from a portion of the @vma. 1261 * @priv: proc maps private state. 1262 * @vma: The VMA to gather stats for. 1263 * @mss: The accumulated stats. 1264 * @start: The address from which to start. 1265 * 1266 * This gathers stats for the portion of the VMA starting at the @start 1267 * address. 1268 */ 1269 static void smap_gather_stats_range(struct proc_maps_private *priv, 1270 struct vm_area_struct *vma, 1271 struct mem_size_stats *mss, 1272 unsigned long start) 1273 { 1274 const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); 1275 const bool is_partial = start > vma->vm_start; 1276 1277 /* Invalid start */ 1278 if (start < vma->vm_start || start >= vma->vm_end) 1279 return; 1280 1281 if (vma == get_gate_vma(priv->lock_ctx.mm)) 1282 return; 1283 1284 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 1285 drop_rcu(priv); 1286 1287 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { 1288 /* 1289 * CoW mappings might map anon folios that do not belong to 1290 * shmem. Perform a less efficient page table walk in this 1291 * situation, unless we know that the shmem object (or the 1292 * part mapped by our VMA) has no swapped out pages at all. 1293 */ 1294 const unsigned long shmem_swapped = shmem_swap_usage(vma); 1295 const bool is_cow = vma_is_cow_mapping(vma); 1296 1297 if (is_partial || (shmem_swapped && is_cow)) 1298 ops = get_smaps_shmem_walk_ops(priv); 1299 else 1300 mss->swap += shmem_swapped; 1301 } 1302 1303 walk_page_range_vma(vma, start, vma->vm_end, ops, mss); 1304 1305 reacquire_rcu(priv); 1306 } 1307 1308 /** 1309 * smap_gather_stats() - Gather mem stats from the entire @vma. 1310 * @priv: proc maps private state. 1311 * @vma: The VMA to gather stats for. 1312 * @mss: The accumulated stats. 1313 * 1314 * This gathers stats for the whole of the VMA. 1315 */ 1316 static void smap_gather_stats(struct proc_maps_private *priv, 1317 struct vm_area_struct *vma, struct mem_size_stats *mss) 1318 { 1319 smap_gather_stats_range(priv, vma, mss, vma->vm_start); 1320 } 1321 1322 #define SEQ_PUT_DEC(str, val) \ 1323 seq_put_decimal_ull_width(m, str, (val) >> 10, 8) 1324 1325 /* Show the contents common for smaps and smaps_rollup */ 1326 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, 1327 bool rollup_mode) 1328 { 1329 SEQ_PUT_DEC("Rss: ", mss->resident); 1330 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT); 1331 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT); 1332 if (rollup_mode) { 1333 /* 1334 * These are meaningful only for smaps_rollup, otherwise two of 1335 * them are zero, and the other one is the same as Pss. 1336 */ 1337 SEQ_PUT_DEC(" kB\nPss_Anon: ", 1338 mss->pss_anon >> PSS_SHIFT); 1339 SEQ_PUT_DEC(" kB\nPss_File: ", 1340 mss->pss_file >> PSS_SHIFT); 1341 SEQ_PUT_DEC(" kB\nPss_Shmem: ", 1342 mss->pss_shmem >> PSS_SHIFT); 1343 } 1344 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean); 1345 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty); 1346 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean); 1347 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty); 1348 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced); 1349 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous); 1350 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm); 1351 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree); 1352 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp); 1353 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp); 1354 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp); 1355 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb); 1356 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ", 1357 mss->private_hugetlb >> 10, 7); 1358 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap); 1359 SEQ_PUT_DEC(" kB\nSwapPss: ", 1360 mss->swap_pss >> PSS_SHIFT); 1361 SEQ_PUT_DEC(" kB\nLocked: ", 1362 mss->pss_locked >> PSS_SHIFT); 1363 seq_puts(m, " kB\n"); 1364 } 1365 1366 static int show_smap(struct seq_file *m, void *v) 1367 { 1368 struct proc_maps_private *priv = m->private; 1369 struct vm_area_struct *vma = v; 1370 struct mem_size_stats mss = {}; 1371 1372 smap_gather_stats(priv, vma, &mss); 1373 1374 show_map_vma(m, vma); 1375 1376 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); 1377 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); 1378 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); 1379 seq_puts(m, " kB\n"); 1380 1381 __show_smap(m, &mss, false); 1382 1383 seq_printf(m, "THPeligible: %8u\n", 1384 !!thp_vma_allowable_orders(vma, vma->vm_flags, TVA_SMAPS, 1385 THP_ORDERS_ALL)); 1386 1387 if (arch_pkeys_enabled()) 1388 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); 1389 show_smap_vma_flags(m, vma); 1390 1391 return 0; 1392 } 1393 1394 static int show_smaps_rollup(struct seq_file *m, void *v) 1395 { 1396 struct proc_maps_private *priv = m->private; 1397 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 1398 struct mm_struct *mm = lock_ctx->mm; 1399 struct mem_size_stats mss = {}; 1400 unsigned long last_vma_end = 0; 1401 unsigned long vma_start = 0; 1402 struct vm_area_struct *vma; 1403 loff_t pos = 0; 1404 int ret = 0; 1405 1406 priv->task = get_proc_task(priv->inode); 1407 if (!priv->task) 1408 return -ESRCH; 1409 1410 if (!mm || !mmget_not_zero(mm)) { 1411 ret = -ESRCH; 1412 goto out_put_task; 1413 } 1414 1415 hold_task_mempolicy(priv); 1416 rcu_read_lock(); 1417 reset_lock_ctx(lock_ctx); 1418 1419 vma_iter_init(&priv->iter, mm, 0); 1420 vma = proc_get_vma(m, &pos); 1421 if (unlikely(!vma)) 1422 goto empty_set; 1423 1424 if (!IS_ERR(vma)) 1425 vma_start = vma->vm_start; 1426 1427 while (vma) { 1428 if (IS_ERR(vma)) { 1429 ret = PTR_ERR(vma); 1430 goto out_unlock; 1431 } 1432 1433 if (vma->vm_start < last_vma_end) { 1434 /* 1435 * After retaking the lock, already reported VMA grew 1436 * or got merged with the next one and we found it 1437 * again. Gather stats for the remaining portion by 1438 * starting at last_vma_end. 1439 */ 1440 smap_gather_stats_range(priv, vma, &mss, last_vma_end); 1441 } else { 1442 /* Found next unreported VMA, start from its beginning */ 1443 smap_gather_stats(priv, vma, &mss); 1444 } 1445 last_vma_end = vma->vm_end; 1446 1447 /* 1448 * If the VMA lock is not taken, we hold the often contended 1449 * mmap lock. This can happen if we had to fall back to the 1450 * mmap lock. 1451 * 1452 * To relieve pressure, check if it is indeed contended, then 1453 * temporarily release it. 1454 */ 1455 if (lock_ctx->mmap_locked && 1456 mmap_lock_is_contended(lock_ctx->mm)) { 1457 unlock_ctx_mm(lock_ctx); 1458 /* 1459 * Even though we previously fell back to mmap lock, 1460 * we try taking VMA lock for the next VMA, since it 1461 * might not be under modification. In the worst case 1462 * we will fall back to mmap lock again. 1463 */ 1464 rcu_read_lock(); 1465 reset_lock_ctx(lock_ctx); 1466 /* Resume from the last position. */ 1467 pos = last_vma_end; 1468 vma_iter_init(&priv->iter, mm, pos); 1469 } 1470 vma = proc_get_vma(m, &pos); 1471 } 1472 1473 empty_set: 1474 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); 1475 seq_pad(m, ' '); 1476 seq_puts(m, "[rollup]\n"); 1477 1478 __show_smap(m, &mss, true); 1479 1480 out_unlock: 1481 if (lock_ctx->mmap_locked) { 1482 unlock_ctx_mm(lock_ctx); 1483 } else { 1484 unlock_ctx_vma(lock_ctx); 1485 rcu_read_unlock(); 1486 } 1487 release_task_mempolicy(priv); 1488 mmput(mm); 1489 out_put_task: 1490 put_task_struct(priv->task); 1491 priv->task = NULL; 1492 1493 return ret; 1494 } 1495 #undef SEQ_PUT_DEC 1496 1497 static const struct seq_operations proc_pid_smaps_op = { 1498 .start = m_start, 1499 .next = m_next, 1500 .stop = m_stop, 1501 .show = show_smap 1502 }; 1503 1504 static int pid_smaps_open(struct inode *inode, struct file *file) 1505 { 1506 return do_maps_open(inode, file, &proc_pid_smaps_op); 1507 } 1508 1509 static int smaps_rollup_open(struct inode *inode, struct file *file) 1510 { 1511 int ret; 1512 struct proc_maps_private *priv; 1513 1514 priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT); 1515 if (!priv) 1516 return -ENOMEM; 1517 1518 ret = single_open(file, show_smaps_rollup, priv); 1519 if (ret) 1520 goto out_free; 1521 1522 priv->inode = inode; 1523 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 1524 if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) { 1525 ret = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH; 1526 1527 single_release(inode, file); 1528 goto out_free; 1529 } 1530 1531 return 0; 1532 1533 out_free: 1534 kfree(priv); 1535 return ret; 1536 } 1537 1538 static int smaps_rollup_release(struct inode *inode, struct file *file) 1539 { 1540 struct seq_file *seq = file->private_data; 1541 struct proc_maps_private *priv = seq->private; 1542 1543 if (priv->lock_ctx.mm) 1544 mmdrop(priv->lock_ctx.mm); 1545 1546 kfree(priv); 1547 return single_release(inode, file); 1548 } 1549 1550 const struct file_operations proc_pid_smaps_operations = { 1551 .open = pid_smaps_open, 1552 .read = seq_read, 1553 .llseek = seq_lseek, 1554 .release = proc_map_release, 1555 }; 1556 1557 const struct file_operations proc_pid_smaps_rollup_operations = { 1558 .open = smaps_rollup_open, 1559 .read = seq_read, 1560 .llseek = seq_lseek, 1561 .release = smaps_rollup_release, 1562 }; 1563 1564 enum clear_refs_types { 1565 CLEAR_REFS_ALL = 1, 1566 CLEAR_REFS_ANON, 1567 CLEAR_REFS_MAPPED, 1568 CLEAR_REFS_SOFT_DIRTY, 1569 CLEAR_REFS_MM_HIWATER_RSS, 1570 CLEAR_REFS_LAST, 1571 }; 1572 1573 struct clear_refs_private { 1574 enum clear_refs_types type; 1575 }; 1576 1577 static bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1578 { 1579 struct folio *folio; 1580 1581 if (!pte_write(pte)) 1582 return false; 1583 if (!vma_is_cow_mapping(vma)) 1584 return false; 1585 if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))) 1586 return false; 1587 folio = vm_normal_folio(vma, addr, pte); 1588 if (!folio) 1589 return false; 1590 return folio_maybe_dma_pinned(folio); 1591 } 1592 1593 static void clear_soft_dirty(struct vm_area_struct *vma, unsigned long addr, 1594 pte_t *pte) 1595 { 1596 if (!pgtable_supports_soft_dirty()) 1597 return; 1598 /* 1599 * The soft-dirty tracker uses #PF-s to catch writes 1600 * to pages, so write-protect the pte as well. See the 1601 * Documentation/admin-guide/mm/soft-dirty.rst for full description 1602 * of how soft-dirty works. 1603 */ 1604 pte_t ptent = ptep_get(pte); 1605 1606 if (pte_none(ptent)) 1607 return; 1608 1609 if (pte_present(ptent)) { 1610 pte_t old_pte; 1611 1612 if (pte_is_pinned(vma, addr, ptent)) 1613 return; 1614 old_pte = ptep_modify_prot_start(vma, addr, pte); 1615 ptent = pte_wrprotect(old_pte); 1616 ptent = pte_clear_soft_dirty(ptent); 1617 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 1618 } else { 1619 ptent = pte_swp_clear_soft_dirty(ptent); 1620 set_pte_at(vma->vm_mm, addr, pte, ptent); 1621 } 1622 } 1623 1624 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) 1625 static void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1626 unsigned long addr, pmd_t *pmdp) 1627 { 1628 pmd_t old, pmd = *pmdp; 1629 1630 if (!pgtable_supports_soft_dirty()) 1631 return; 1632 1633 if (pmd_present(pmd)) { 1634 /* See comment in change_huge_pmd() */ 1635 old = pmdp_invalidate(vma, addr, pmdp); 1636 if (pmd_dirty(old)) 1637 pmd = pmd_mkdirty(pmd); 1638 if (pmd_young(old)) 1639 pmd = pmd_mkyoung(pmd); 1640 1641 pmd = pmd_wrprotect(pmd); 1642 pmd = pmd_clear_soft_dirty(pmd); 1643 1644 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1645 } else if (pmd_is_migration_entry(pmd)) { 1646 pmd = pmd_swp_clear_soft_dirty(pmd); 1647 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1648 } 1649 } 1650 #else 1651 static void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1652 unsigned long addr, pmd_t *pmdp) 1653 { 1654 } 1655 #endif 1656 1657 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, 1658 unsigned long end, struct mm_walk *walk) 1659 { 1660 struct clear_refs_private *cp = walk->private; 1661 struct vm_area_struct *vma = walk->vma; 1662 pte_t *pte, ptent; 1663 spinlock_t *ptl; 1664 struct folio *folio; 1665 1666 ptl = pmd_trans_huge_lock(pmd, vma); 1667 if (ptl) { 1668 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1669 clear_soft_dirty_pmd(vma, addr, pmd); 1670 goto out; 1671 } 1672 1673 if (!pmd_present(*pmd)) 1674 goto out; 1675 1676 folio = vm_normal_folio_pmd(vma, addr, *pmd); 1677 if (!folio) 1678 goto out; 1679 1680 /* Clear accessed and referenced bits. */ 1681 pmdp_test_and_clear_young(vma, addr, pmd); 1682 folio_test_clear_young(folio); 1683 folio_clear_referenced(folio); 1684 out: 1685 spin_unlock(ptl); 1686 return 0; 1687 } 1688 1689 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1690 if (!pte) { 1691 walk->action = ACTION_AGAIN; 1692 return 0; 1693 } 1694 for (; addr != end; pte++, addr += PAGE_SIZE) { 1695 ptent = ptep_get(pte); 1696 1697 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1698 clear_soft_dirty(vma, addr, pte); 1699 continue; 1700 } 1701 1702 if (!pte_present(ptent)) 1703 continue; 1704 1705 folio = vm_normal_folio(vma, addr, ptent); 1706 if (!folio) 1707 continue; 1708 1709 /* Clear accessed and referenced bits. */ 1710 ptep_test_and_clear_young(vma, addr, pte); 1711 folio_test_clear_young(folio); 1712 folio_clear_referenced(folio); 1713 } 1714 pte_unmap_unlock(pte - 1, ptl); 1715 cond_resched(); 1716 return 0; 1717 } 1718 1719 static int clear_refs_test_walk(unsigned long start, unsigned long end, 1720 struct mm_walk *walk) 1721 { 1722 struct clear_refs_private *cp = walk->private; 1723 struct vm_area_struct *vma = walk->vma; 1724 1725 if (vma->vm_flags & VM_PFNMAP) 1726 return 1; 1727 1728 /* 1729 * Writing 1 to /proc/pid/clear_refs affects all pages. 1730 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. 1731 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages. 1732 * Writing 4 to /proc/pid/clear_refs affects all pages. 1733 */ 1734 if (cp->type == CLEAR_REFS_ANON && vma->vm_file) 1735 return 1; 1736 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file) 1737 return 1; 1738 return 0; 1739 } 1740 1741 static const struct mm_walk_ops clear_refs_walk_ops = { 1742 .pmd_entry = clear_refs_pte_range, 1743 .test_walk = clear_refs_test_walk, 1744 .walk_lock = PGWALK_WRLOCK, 1745 }; 1746 1747 static ssize_t clear_refs_write(struct file *file, const char __user *buf, 1748 size_t count, loff_t *ppos) 1749 { 1750 struct task_struct *task; 1751 char buffer[PROC_NUMBUF] = {}; 1752 struct mm_struct *mm; 1753 struct vm_area_struct *vma; 1754 enum clear_refs_types type; 1755 int itype; 1756 int rv; 1757 1758 if (count > sizeof(buffer) - 1) 1759 count = sizeof(buffer) - 1; 1760 if (copy_from_user(buffer, buf, count)) 1761 return -EFAULT; 1762 rv = kstrtoint(strstrip(buffer), 10, &itype); 1763 if (rv < 0) 1764 return rv; 1765 type = (enum clear_refs_types)itype; 1766 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) 1767 return -EINVAL; 1768 1769 task = get_proc_task(file_inode(file)); 1770 if (!task) 1771 return -ESRCH; 1772 mm = get_task_mm(task); 1773 if (mm) { 1774 VMA_ITERATOR(vmi, mm, 0); 1775 struct mmu_notifier_range range; 1776 struct clear_refs_private cp = { 1777 .type = type, 1778 }; 1779 1780 if (mmap_write_lock_killable(mm)) { 1781 count = -EINTR; 1782 goto out_mm; 1783 } 1784 if (type == CLEAR_REFS_MM_HIWATER_RSS) { 1785 /* 1786 * Writing 5 to /proc/pid/clear_refs resets the peak 1787 * resident set size to this mm's current rss value. 1788 */ 1789 reset_mm_hiwater_rss(mm); 1790 goto out_unlock; 1791 } 1792 1793 if (type == CLEAR_REFS_SOFT_DIRTY) { 1794 for_each_vma(vmi, vma) { 1795 if (!(vma->vm_flags & VM_SOFTDIRTY)) 1796 continue; 1797 vm_flags_clear(vma, VM_SOFTDIRTY); 1798 vma_set_page_prot(vma); 1799 } 1800 1801 inc_tlb_flush_pending(mm); 1802 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, 1803 0, mm, 0, -1UL); 1804 mmu_notifier_invalidate_range_start(&range); 1805 } 1806 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); 1807 if (type == CLEAR_REFS_SOFT_DIRTY) { 1808 mmu_notifier_invalidate_range_end(&range); 1809 flush_tlb_mm(mm); 1810 dec_tlb_flush_pending(mm); 1811 } 1812 out_unlock: 1813 mmap_write_unlock(mm); 1814 out_mm: 1815 mmput(mm); 1816 } 1817 put_task_struct(task); 1818 1819 return count; 1820 } 1821 1822 const struct file_operations proc_clear_refs_operations = { 1823 .write = clear_refs_write, 1824 .llseek = noop_llseek, 1825 }; 1826 1827 typedef struct { 1828 u64 pme; 1829 } pagemap_entry_t; 1830 1831 struct pagemapread { 1832 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ 1833 pagemap_entry_t *buffer; 1834 bool show_pfn; 1835 }; 1836 1837 #define PAGEMAP_WALK_SIZE (PMD_SIZE) 1838 #define PAGEMAP_WALK_MASK (PMD_MASK) 1839 1840 #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) 1841 #define PM_PFRAME_BITS 55 1842 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) 1843 #define PM_SOFT_DIRTY BIT_ULL(55) 1844 #define PM_MMAP_EXCLUSIVE BIT_ULL(56) 1845 #define PM_UFFD_WP BIT_ULL(57) 1846 #define PM_GUARD_REGION BIT_ULL(58) 1847 #define PM_FILE BIT_ULL(61) 1848 #define PM_SWAP BIT_ULL(62) 1849 #define PM_PRESENT BIT_ULL(63) 1850 1851 #define PM_END_OF_BUFFER 1 1852 1853 static pagemap_entry_t make_pme(u64 frame, u64 flags) 1854 { 1855 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; 1856 } 1857 1858 static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) 1859 { 1860 pm->buffer[pm->pos++] = *pme; 1861 if (pm->pos >= pm->len) 1862 return PM_END_OF_BUFFER; 1863 return 0; 1864 } 1865 1866 static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) 1867 { 1868 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1869 return folio_precise_page_mapcount(folio, page) == 1; 1870 return !folio_maybe_mapped_shared(folio); 1871 } 1872 1873 static int pagemap_pte_hole(unsigned long start, unsigned long end, 1874 __always_unused int depth, struct mm_walk *walk) 1875 { 1876 struct pagemapread *pm = walk->private; 1877 unsigned long addr = start; 1878 int err = 0; 1879 1880 while (addr < end) { 1881 struct vm_area_struct *vma = find_vma(walk->mm, addr); 1882 pagemap_entry_t pme = make_pme(0, 0); 1883 /* End of address space hole, which we mark as non-present. */ 1884 unsigned long hole_end; 1885 1886 if (vma) 1887 hole_end = min(end, vma->vm_start); 1888 else 1889 hole_end = end; 1890 1891 for (; addr < hole_end; addr += PAGE_SIZE) { 1892 err = add_to_pagemap(&pme, pm); 1893 if (err) 1894 goto out; 1895 } 1896 1897 if (!vma) 1898 break; 1899 1900 /* Addresses in the VMA. */ 1901 if (vma->vm_flags & VM_SOFTDIRTY) 1902 pme = make_pme(0, PM_SOFT_DIRTY); 1903 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { 1904 err = add_to_pagemap(&pme, pm); 1905 if (err) 1906 goto out; 1907 } 1908 } 1909 out: 1910 return err; 1911 } 1912 1913 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 1914 struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1915 { 1916 u64 frame = 0, flags = 0; 1917 struct page *page = NULL; 1918 struct folio *folio; 1919 1920 if (pte_none(pte)) 1921 goto out; 1922 1923 if (pte_present(pte)) { 1924 if (pm->show_pfn) 1925 frame = pte_pfn(pte); 1926 flags |= PM_PRESENT; 1927 page = vm_normal_page(vma, addr, pte); 1928 if (pte_soft_dirty(pte)) 1929 flags |= PM_SOFT_DIRTY; 1930 if (pte_uffd(pte)) 1931 flags |= PM_UFFD_WP; 1932 } else { 1933 softleaf_t entry; 1934 1935 if (pte_swp_soft_dirty(pte)) 1936 flags |= PM_SOFT_DIRTY; 1937 if (pte_swp_uffd(pte)) 1938 flags |= PM_UFFD_WP; 1939 entry = softleaf_from_pte(pte); 1940 if (pm->show_pfn) { 1941 pgoff_t offset; 1942 1943 /* 1944 * For PFN swap offsets, keeping the offset field 1945 * to be PFN only to be compatible with old smaps. 1946 */ 1947 if (softleaf_has_pfn(entry)) 1948 offset = softleaf_to_pfn(entry); 1949 else 1950 offset = swp_offset(entry); 1951 frame = swp_type(entry) | 1952 (offset << MAX_SWAPFILES_SHIFT); 1953 } 1954 flags |= PM_SWAP; 1955 if (softleaf_has_pfn(entry)) 1956 page = softleaf_to_page(entry); 1957 if (softleaf_is_uffd_wp_marker(entry)) 1958 flags |= PM_UFFD_WP; 1959 if (softleaf_is_guard_marker(entry)) 1960 flags |= PM_GUARD_REGION; 1961 } 1962 1963 if (page) { 1964 folio = page_folio(page); 1965 if (!folio_test_anon(folio)) 1966 flags |= PM_FILE; 1967 if ((flags & PM_PRESENT) && 1968 __folio_page_mapped_exclusively(folio, page)) 1969 flags |= PM_MMAP_EXCLUSIVE; 1970 } 1971 1972 out: 1973 if (vma->vm_flags & VM_SOFTDIRTY) 1974 flags |= PM_SOFT_DIRTY; 1975 1976 return make_pme(frame, flags); 1977 } 1978 1979 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1980 static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, 1981 unsigned long end, struct vm_area_struct *vma, 1982 struct pagemapread *pm) 1983 { 1984 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 1985 u64 flags = 0, frame = 0; 1986 pmd_t pmd = *pmdp; 1987 struct page *page = NULL; 1988 struct folio *folio = NULL; 1989 int err = 0; 1990 1991 if (vma->vm_flags & VM_SOFTDIRTY) 1992 flags |= PM_SOFT_DIRTY; 1993 1994 if (pmd_none(pmd)) 1995 goto populate_pagemap; 1996 1997 if (pmd_present(pmd)) { 1998 page = vm_normal_page_pmd(vma, addr, pmd); 1999 2000 flags |= PM_PRESENT; 2001 if (pmd_soft_dirty(pmd)) 2002 flags |= PM_SOFT_DIRTY; 2003 if (pmd_uffd(pmd)) 2004 flags |= PM_UFFD_WP; 2005 if (pm->show_pfn) 2006 frame = pmd_pfn(pmd) + idx; 2007 } else if (thp_migration_supported()) { 2008 const softleaf_t entry = softleaf_from_pmd(pmd); 2009 unsigned long offset; 2010 2011 if (pm->show_pfn) { 2012 if (softleaf_has_pfn(entry)) 2013 offset = softleaf_to_pfn(entry) + idx; 2014 else 2015 offset = swp_offset(entry) + idx; 2016 frame = swp_type(entry) | 2017 (offset << MAX_SWAPFILES_SHIFT); 2018 } 2019 flags |= PM_SWAP; 2020 if (pmd_swp_soft_dirty(pmd)) 2021 flags |= PM_SOFT_DIRTY; 2022 if (pmd_swp_uffd(pmd)) 2023 flags |= PM_UFFD_WP; 2024 if (softleaf_has_pfn(entry)) 2025 page = softleaf_to_page(entry); 2026 } 2027 2028 if (page) { 2029 folio = page_folio(page); 2030 if (!folio_test_anon(folio)) 2031 flags |= PM_FILE; 2032 } 2033 2034 populate_pagemap: 2035 for (; addr != end; addr += PAGE_SIZE, idx++) { 2036 u64 cur_flags = flags; 2037 pagemap_entry_t pme; 2038 2039 if (folio && (flags & PM_PRESENT) && 2040 __folio_page_mapped_exclusively(folio, page)) 2041 cur_flags |= PM_MMAP_EXCLUSIVE; 2042 2043 pme = make_pme(frame, cur_flags); 2044 err = add_to_pagemap(&pme, pm); 2045 if (err) 2046 break; 2047 if (pm->show_pfn) { 2048 if (flags & PM_PRESENT) 2049 frame++; 2050 else if (flags & PM_SWAP) 2051 frame += (1 << MAX_SWAPFILES_SHIFT); 2052 } 2053 } 2054 return err; 2055 } 2056 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2057 2058 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, 2059 struct mm_walk *walk) 2060 { 2061 struct vm_area_struct *vma = walk->vma; 2062 struct pagemapread *pm = walk->private; 2063 spinlock_t *ptl; 2064 pte_t *pte, *orig_pte; 2065 int err = 0; 2066 2067 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2068 ptl = pmd_trans_huge_lock(pmdp, vma); 2069 if (ptl) { 2070 err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); 2071 spin_unlock(ptl); 2072 return err; 2073 } 2074 #endif 2075 2076 /* 2077 * We can assume that @vma always points to a valid one and @end never 2078 * goes beyond vma->vm_end. 2079 */ 2080 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); 2081 if (!pte) { 2082 walk->action = ACTION_AGAIN; 2083 return err; 2084 } 2085 for (; addr < end; pte++, addr += PAGE_SIZE) { 2086 pagemap_entry_t pme; 2087 2088 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); 2089 err = add_to_pagemap(&pme, pm); 2090 if (err) 2091 break; 2092 } 2093 pte_unmap_unlock(orig_pte, ptl); 2094 2095 cond_resched(); 2096 2097 return err; 2098 } 2099 2100 #ifdef CONFIG_HUGETLB_PAGE 2101 /* This function walks within one hugetlb entry in the single call */ 2102 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, 2103 unsigned long addr, unsigned long end, 2104 struct mm_walk *walk) 2105 { 2106 struct pagemapread *pm = walk->private; 2107 struct vm_area_struct *vma = walk->vma; 2108 u64 flags = 0, frame = 0; 2109 spinlock_t *ptl; 2110 int err = 0; 2111 pte_t pte; 2112 2113 if (vma->vm_flags & VM_SOFTDIRTY) 2114 flags |= PM_SOFT_DIRTY; 2115 2116 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, ptep); 2117 pte = huge_ptep_get(walk->mm, addr, ptep); 2118 if (pte_present(pte)) { 2119 struct folio *folio = page_folio(pte_page(pte)); 2120 2121 if (!folio_test_anon(folio)) 2122 flags |= PM_FILE; 2123 2124 if (!folio_maybe_mapped_shared(folio) && 2125 !hugetlb_pmd_shared(ptep)) 2126 flags |= PM_MMAP_EXCLUSIVE; 2127 2128 if (huge_pte_uffd(pte)) 2129 flags |= PM_UFFD_WP; 2130 2131 flags |= PM_PRESENT; 2132 if (pm->show_pfn) 2133 frame = pte_pfn(pte) + 2134 ((addr & ~hmask) >> PAGE_SHIFT); 2135 } else if (pte_swp_uffd_any(pte)) { 2136 flags |= PM_UFFD_WP; 2137 } 2138 2139 for (; addr != end; addr += PAGE_SIZE) { 2140 pagemap_entry_t pme = make_pme(frame, flags); 2141 2142 err = add_to_pagemap(&pme, pm); 2143 if (err) 2144 break; 2145 if (pm->show_pfn && (flags & PM_PRESENT)) 2146 frame++; 2147 } 2148 2149 spin_unlock(ptl); 2150 cond_resched(); 2151 2152 return err; 2153 } 2154 #else 2155 #define pagemap_hugetlb_range NULL 2156 #endif /* HUGETLB_PAGE */ 2157 2158 static const struct mm_walk_ops pagemap_ops = { 2159 .pmd_entry = pagemap_pmd_range, 2160 .pte_hole = pagemap_pte_hole, 2161 .hugetlb_entry = pagemap_hugetlb_range, 2162 .walk_lock = PGWALK_RDLOCK, 2163 }; 2164 2165 /* 2166 * /proc/pid/pagemap - an array mapping virtual pages to pfns 2167 * 2168 * For each page in the address space, this file contains one 64-bit entry 2169 * consisting of the following: 2170 * 2171 * Bits 0-54 page frame number (PFN) if present 2172 * Bits 0-4 swap type if swapped 2173 * Bits 5-54 swap offset if swapped 2174 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) 2175 * Bit 56 page exclusively mapped 2176 * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP) 2177 * Bit 58 pte is a guard region 2178 * Bits 59-60 zero 2179 * Bit 61 page is file-page or shared-anon 2180 * Bit 62 page swapped 2181 * Bit 63 page present 2182 * 2183 * If the page is not present but in swap, then the PFN contains an 2184 * encoding of the swap file number and the page's offset into the 2185 * swap. Unmapped pages return a null PFN. This allows determining 2186 * precisely which pages are mapped (or in swap) and comparing mapped 2187 * pages between processes. 2188 * 2189 * Efficient users of this interface will use /proc/pid/maps to 2190 * determine which areas of memory are actually mapped and llseek to 2191 * skip over unmapped regions. 2192 */ 2193 static ssize_t pagemap_read(struct file *file, char __user *buf, 2194 size_t count, loff_t *ppos) 2195 { 2196 struct mm_struct *mm = file->private_data; 2197 struct pagemapread pm; 2198 unsigned long src; 2199 unsigned long svpfn; 2200 unsigned long start_vaddr; 2201 unsigned long end_vaddr; 2202 int ret = 0, copied = 0; 2203 2204 if (!mm || !mmget_not_zero(mm)) 2205 goto out; 2206 2207 ret = -EINVAL; 2208 /* file position must be aligned */ 2209 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES)) 2210 goto out_mm; 2211 2212 ret = 0; 2213 if (!count) 2214 goto out_mm; 2215 2216 /* do not disclose physical addresses: attack vector */ 2217 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); 2218 2219 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); 2220 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); 2221 ret = -ENOMEM; 2222 if (!pm.buffer) 2223 goto out_mm; 2224 2225 src = *ppos; 2226 svpfn = src / PM_ENTRY_BYTES; 2227 end_vaddr = mm->task_size; 2228 2229 /* watch out for wraparound */ 2230 start_vaddr = end_vaddr; 2231 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { 2232 unsigned long end; 2233 2234 ret = mmap_read_lock_killable(mm); 2235 if (ret) 2236 goto out_free; 2237 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); 2238 mmap_read_unlock(mm); 2239 2240 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); 2241 if (end >= start_vaddr && end < mm->task_size) 2242 end_vaddr = end; 2243 } 2244 2245 /* Ensure the address is inside the task */ 2246 if (start_vaddr > mm->task_size) 2247 start_vaddr = end_vaddr; 2248 2249 ret = 0; 2250 while (count && (start_vaddr < end_vaddr)) { 2251 int len; 2252 unsigned long end; 2253 2254 pm.pos = 0; 2255 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; 2256 /* overflow ? */ 2257 if (end < start_vaddr || end > end_vaddr) 2258 end = end_vaddr; 2259 ret = mmap_read_lock_killable(mm); 2260 if (ret) 2261 goto out_free; 2262 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); 2263 mmap_read_unlock(mm); 2264 start_vaddr = end; 2265 2266 len = min(count, PM_ENTRY_BYTES * pm.pos); 2267 if (copy_to_user(buf, pm.buffer, len)) { 2268 ret = -EFAULT; 2269 goto out_free; 2270 } 2271 copied += len; 2272 buf += len; 2273 count -= len; 2274 } 2275 *ppos += copied; 2276 if (!ret || ret == PM_END_OF_BUFFER) 2277 ret = copied; 2278 2279 out_free: 2280 kfree(pm.buffer); 2281 out_mm: 2282 mmput(mm); 2283 out: 2284 return ret; 2285 } 2286 2287 static int pagemap_open(struct inode *inode, struct file *file) 2288 { 2289 struct mm_struct *mm; 2290 2291 mm = proc_mem_open(inode, PTRACE_MODE_READ); 2292 if (IS_ERR_OR_NULL(mm)) 2293 return mm ? PTR_ERR(mm) : -ESRCH; 2294 file->private_data = mm; 2295 return 0; 2296 } 2297 2298 static int pagemap_release(struct inode *inode, struct file *file) 2299 { 2300 struct mm_struct *mm = file->private_data; 2301 2302 if (mm) 2303 mmdrop(mm); 2304 return 0; 2305 } 2306 2307 #define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 2308 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 2309 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 2310 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ 2311 PAGE_IS_GUARD | PAGE_IS_ACCESSED) 2312 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) 2313 2314 struct pagemap_scan_private { 2315 struct pm_scan_arg arg; 2316 unsigned long masks_of_interest, cur_vma_category; 2317 struct page_region *vec_buf; 2318 unsigned long vec_buf_len, vec_buf_index, found_pages; 2319 struct page_region __user *vec_out; 2320 }; 2321 2322 static unsigned long pagemap_page_category(struct pagemap_scan_private *p, 2323 struct vm_area_struct *vma, 2324 unsigned long addr, pte_t pte) 2325 { 2326 unsigned long categories; 2327 2328 if (pte_none(pte)) { 2329 /* 2330 * An unpopulated pte carries no uffd bit, i.e. it is not 2331 * write-protected. The PAGE_IS_WRITTEN fast path in 2332 * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA; 2333 * gate the pte_none report here the same way so the two paths 2334 * still agree. RWP has no such fast path and an unpopulated 2335 * page is not part of the RWP working set, so it is reported as 2336 * neither. 2337 */ 2338 if (userfaultfd_wp(vma)) 2339 return PAGE_IS_WRITTEN; 2340 return 0; 2341 } 2342 2343 if (pte_present(pte)) { 2344 struct page *page; 2345 2346 categories = PAGE_IS_PRESENT; 2347 2348 if (!pte_uffd(pte)) { 2349 if (userfaultfd_wp(vma)) 2350 categories |= PAGE_IS_WRITTEN; 2351 if (userfaultfd_rwp(vma)) 2352 categories |= PAGE_IS_ACCESSED; 2353 } 2354 2355 if (p->masks_of_interest & PAGE_IS_FILE) { 2356 page = vm_normal_page(vma, addr, pte); 2357 if (page && !PageAnon(page)) 2358 categories |= PAGE_IS_FILE; 2359 } 2360 2361 if (is_zero_pfn(pte_pfn(pte))) 2362 categories |= PAGE_IS_PFNZERO; 2363 if (pte_soft_dirty(pte)) 2364 categories |= PAGE_IS_SOFT_DIRTY; 2365 } else { 2366 softleaf_t entry; 2367 2368 categories = PAGE_IS_SWAPPED; 2369 2370 if (!pte_swp_uffd_any(pte)) { 2371 if (userfaultfd_wp(vma)) 2372 categories |= PAGE_IS_WRITTEN; 2373 if (userfaultfd_rwp(vma)) 2374 categories |= PAGE_IS_ACCESSED; 2375 } 2376 2377 entry = softleaf_from_pte(pte); 2378 if (softleaf_is_guard_marker(entry)) 2379 categories |= PAGE_IS_GUARD; 2380 else if ((p->masks_of_interest & PAGE_IS_FILE) && 2381 softleaf_has_pfn(entry) && 2382 !folio_test_anon(softleaf_to_folio(entry))) 2383 categories |= PAGE_IS_FILE; 2384 2385 if (pte_swp_soft_dirty(pte)) 2386 categories |= PAGE_IS_SOFT_DIRTY; 2387 } 2388 2389 return categories; 2390 } 2391 2392 static void make_uffd_wp_pte(struct vm_area_struct *vma, 2393 unsigned long addr, pte_t *pte, pte_t ptent) 2394 { 2395 if (pte_present(ptent)) { 2396 pte_t old_pte; 2397 2398 old_pte = ptep_modify_prot_start(vma, addr, pte); 2399 ptent = pte_mkuffd(old_pte); 2400 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 2401 } else if (pte_none(ptent)) { 2402 set_pte_at(vma->vm_mm, addr, pte, 2403 make_pte_marker(PTE_MARKER_UFFD_WP)); 2404 } else { 2405 ptent = pte_swp_mkuffd(ptent); 2406 set_pte_at(vma->vm_mm, addr, pte, ptent); 2407 } 2408 } 2409 2410 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2411 static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, 2412 struct vm_area_struct *vma, 2413 unsigned long addr, pmd_t pmd) 2414 { 2415 unsigned long categories = PAGE_IS_HUGE; 2416 2417 if (pmd_none(pmd)) 2418 return categories; 2419 2420 if (pmd_present(pmd)) { 2421 struct page *page; 2422 2423 categories |= PAGE_IS_PRESENT; 2424 if (!pmd_uffd(pmd)) { 2425 if (userfaultfd_wp(vma)) 2426 categories |= PAGE_IS_WRITTEN; 2427 if (userfaultfd_rwp(vma)) 2428 categories |= PAGE_IS_ACCESSED; 2429 } 2430 2431 if (p->masks_of_interest & PAGE_IS_FILE) { 2432 page = vm_normal_page_pmd(vma, addr, pmd); 2433 if (page && !PageAnon(page)) 2434 categories |= PAGE_IS_FILE; 2435 } 2436 2437 if (is_huge_zero_pmd(pmd)) 2438 categories |= PAGE_IS_PFNZERO; 2439 if (pmd_soft_dirty(pmd)) 2440 categories |= PAGE_IS_SOFT_DIRTY; 2441 } else { 2442 categories |= PAGE_IS_SWAPPED; 2443 if (!pmd_swp_uffd(pmd)) { 2444 if (userfaultfd_wp(vma)) 2445 categories |= PAGE_IS_WRITTEN; 2446 if (userfaultfd_rwp(vma)) 2447 categories |= PAGE_IS_ACCESSED; 2448 } 2449 if (pmd_swp_soft_dirty(pmd)) 2450 categories |= PAGE_IS_SOFT_DIRTY; 2451 2452 if (p->masks_of_interest & PAGE_IS_FILE) { 2453 const softleaf_t entry = softleaf_from_pmd(pmd); 2454 2455 if (softleaf_has_pfn(entry) && 2456 !folio_test_anon(softleaf_to_folio(entry))) 2457 categories |= PAGE_IS_FILE; 2458 } 2459 } 2460 2461 return categories; 2462 } 2463 2464 static void make_uffd_wp_pmd(struct vm_area_struct *vma, 2465 unsigned long addr, pmd_t *pmdp) 2466 { 2467 pmd_t old, pmd = *pmdp; 2468 2469 if (pmd_present(pmd)) { 2470 old = pmdp_invalidate_ad(vma, addr, pmdp); 2471 pmd = pmd_mkuffd(old); 2472 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2473 } else if (pmd_is_migration_entry(pmd)) { 2474 pmd = pmd_swp_mkuffd(pmd); 2475 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2476 } 2477 } 2478 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2479 2480 #ifdef CONFIG_HUGETLB_PAGE 2481 static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma, 2482 pte_t pte) 2483 { 2484 unsigned long categories = PAGE_IS_HUGE; 2485 2486 if (pte_none(pte)) 2487 return categories; 2488 2489 /* 2490 * According to pagemap_hugetlb_range(), file-backed HugeTLB 2491 * page cannot be swapped. So PAGE_IS_FILE is not checked for 2492 * swapped pages. 2493 */ 2494 if (pte_present(pte)) { 2495 categories |= PAGE_IS_PRESENT; 2496 2497 if (!huge_pte_uffd(pte)) { 2498 if (userfaultfd_wp(vma)) 2499 categories |= PAGE_IS_WRITTEN; 2500 if (userfaultfd_rwp(vma)) 2501 categories |= PAGE_IS_ACCESSED; 2502 } 2503 if (!PageAnon(pte_page(pte))) 2504 categories |= PAGE_IS_FILE; 2505 if (is_zero_pfn(pte_pfn(pte))) 2506 categories |= PAGE_IS_PFNZERO; 2507 if (pte_soft_dirty(pte)) 2508 categories |= PAGE_IS_SOFT_DIRTY; 2509 } else { 2510 categories |= PAGE_IS_SWAPPED; 2511 2512 if (!pte_swp_uffd_any(pte)) { 2513 if (userfaultfd_wp(vma)) 2514 categories |= PAGE_IS_WRITTEN; 2515 if (userfaultfd_rwp(vma)) 2516 categories |= PAGE_IS_ACCESSED; 2517 } 2518 if (pte_swp_soft_dirty(pte)) 2519 categories |= PAGE_IS_SOFT_DIRTY; 2520 } 2521 2522 return categories; 2523 } 2524 2525 static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, 2526 unsigned long addr, pte_t *ptep, 2527 pte_t ptent) 2528 { 2529 const unsigned long psize = huge_page_size(hstate_vma(vma)); 2530 softleaf_t entry; 2531 2532 if (huge_pte_none(ptent)) { 2533 set_huge_pte_at(vma->vm_mm, addr, ptep, 2534 make_pte_marker(PTE_MARKER_UFFD_WP), psize); 2535 return; 2536 } 2537 2538 entry = softleaf_from_pte(ptent); 2539 if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) 2540 return; 2541 2542 if (softleaf_is_migration(entry)) { 2543 set_huge_pte_at(vma->vm_mm, addr, ptep, 2544 pte_swp_mkuffd(ptent), psize); 2545 } else { 2546 pte_t old_pte, new_pte; 2547 2548 old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); 2549 new_pte = huge_pte_mkuffd(old_pte); 2550 huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); 2551 } 2552 } 2553 #endif /* CONFIG_HUGETLB_PAGE */ 2554 2555 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 2556 static void pagemap_scan_backout_range(struct pagemap_scan_private *p, 2557 unsigned long addr, unsigned long end) 2558 { 2559 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2560 2561 if (!p->vec_buf) 2562 return; 2563 2564 if (cur_buf->start != addr) 2565 cur_buf->end = addr; 2566 else 2567 cur_buf->start = cur_buf->end = 0; 2568 2569 p->found_pages -= (end - addr) / PAGE_SIZE; 2570 } 2571 #endif 2572 2573 static bool pagemap_scan_is_interesting_page(unsigned long categories, 2574 const struct pagemap_scan_private *p) 2575 { 2576 categories ^= p->arg.category_inverted; 2577 if ((categories & p->arg.category_mask) != p->arg.category_mask) 2578 return false; 2579 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) 2580 return false; 2581 2582 return true; 2583 } 2584 2585 static bool pagemap_scan_is_interesting_vma(unsigned long categories, 2586 const struct pagemap_scan_private *p) 2587 { 2588 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; 2589 2590 categories ^= p->arg.category_inverted; 2591 if ((categories & required) != required) 2592 return false; 2593 2594 return true; 2595 } 2596 2597 static int pagemap_scan_test_walk(unsigned long start, unsigned long end, 2598 struct mm_walk *walk) 2599 { 2600 struct pagemap_scan_private *p = walk->private; 2601 struct vm_area_struct *vma = walk->vma; 2602 unsigned long vma_category = 0; 2603 bool wp_allowed = userfaultfd_wp_async(vma) && 2604 userfaultfd_wp_use_markers(vma); 2605 2606 if (!wp_allowed) { 2607 /* User requested explicit failure over wp-async capability */ 2608 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) 2609 return -EPERM; 2610 /* 2611 * User requires wr-protect, and allows silently skipping 2612 * unsupported vmas. 2613 */ 2614 if (p->arg.flags & PM_SCAN_WP_MATCHING) 2615 return 1; 2616 /* 2617 * Then the request doesn't involve wr-protects at all, 2618 * fall through to the rest checks, and allow vma walk. 2619 */ 2620 } 2621 2622 if (vma->vm_flags & VM_PFNMAP) 2623 return 1; 2624 2625 if (wp_allowed) 2626 vma_category |= PAGE_IS_WPALLOWED; 2627 2628 if (vma->vm_flags & VM_SOFTDIRTY) 2629 vma_category |= PAGE_IS_SOFT_DIRTY; 2630 2631 if (!pagemap_scan_is_interesting_vma(vma_category, p)) 2632 return 1; 2633 2634 p->cur_vma_category = vma_category; 2635 2636 return 0; 2637 } 2638 2639 static bool pagemap_scan_push_range(unsigned long categories, 2640 struct pagemap_scan_private *p, 2641 unsigned long addr, unsigned long end) 2642 { 2643 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2644 2645 /* 2646 * When there is no output buffer provided at all, the sentinel values 2647 * won't match here. There is no other way for `cur_buf->end` to be 2648 * non-zero other than it being non-empty. 2649 */ 2650 if (addr == cur_buf->end && categories == cur_buf->categories) { 2651 cur_buf->end = end; 2652 return true; 2653 } 2654 2655 if (cur_buf->end) { 2656 if (p->vec_buf_index >= p->vec_buf_len - 1) 2657 return false; 2658 2659 cur_buf = &p->vec_buf[++p->vec_buf_index]; 2660 } 2661 2662 cur_buf->start = addr; 2663 cur_buf->end = end; 2664 cur_buf->categories = categories; 2665 2666 return true; 2667 } 2668 2669 static int pagemap_scan_output(unsigned long categories, 2670 struct pagemap_scan_private *p, 2671 unsigned long addr, unsigned long *end) 2672 { 2673 unsigned long n_pages, total_pages; 2674 int ret = 0; 2675 2676 if (!p->vec_buf) 2677 return 0; 2678 2679 categories &= p->arg.return_mask; 2680 2681 n_pages = (*end - addr) / PAGE_SIZE; 2682 if (check_add_overflow(p->found_pages, n_pages, &total_pages) || 2683 total_pages > p->arg.max_pages) { 2684 size_t n_too_much = total_pages - p->arg.max_pages; 2685 *end -= n_too_much * PAGE_SIZE; 2686 n_pages -= n_too_much; 2687 ret = -ENOSPC; 2688 } 2689 2690 if (!pagemap_scan_push_range(categories, p, addr, *end)) { 2691 *end = addr; 2692 n_pages = 0; 2693 ret = -ENOSPC; 2694 } 2695 2696 p->found_pages += n_pages; 2697 if (ret) 2698 p->arg.walk_end = *end; 2699 2700 return ret; 2701 } 2702 2703 static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, 2704 unsigned long end, struct mm_walk *walk) 2705 { 2706 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2707 struct pagemap_scan_private *p = walk->private; 2708 struct vm_area_struct *vma = walk->vma; 2709 unsigned long categories; 2710 spinlock_t *ptl; 2711 int ret = 0; 2712 2713 ptl = pmd_trans_huge_lock(pmd, vma); 2714 if (!ptl) 2715 return -ENOENT; 2716 2717 categories = p->cur_vma_category | 2718 pagemap_thp_category(p, vma, start, *pmd); 2719 2720 if (!pagemap_scan_is_interesting_page(categories, p)) 2721 goto out_unlock; 2722 2723 ret = pagemap_scan_output(categories, p, start, &end); 2724 if (start == end) 2725 goto out_unlock; 2726 2727 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2728 goto out_unlock; 2729 if (~categories & PAGE_IS_WRITTEN) 2730 goto out_unlock; 2731 2732 /* 2733 * Break huge page into small pages if the WP operation 2734 * needs to be performed on a portion of the huge page. 2735 */ 2736 if (end != start + HPAGE_SIZE) { 2737 spin_unlock(ptl); 2738 split_huge_pmd(vma, pmd, start); 2739 pagemap_scan_backout_range(p, start, end); 2740 /* Report as if there was no THP */ 2741 return -ENOENT; 2742 } 2743 2744 make_uffd_wp_pmd(vma, start, pmd); 2745 flush_tlb_range(vma, start, end); 2746 out_unlock: 2747 spin_unlock(ptl); 2748 return ret; 2749 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 2750 return -ENOENT; 2751 #endif 2752 } 2753 2754 static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, 2755 unsigned long end, struct mm_walk *walk) 2756 { 2757 struct pagemap_scan_private *p = walk->private; 2758 struct vm_area_struct *vma = walk->vma; 2759 unsigned long addr, flush_end = 0; 2760 pte_t *pte, *start_pte; 2761 spinlock_t *ptl; 2762 int ret; 2763 2764 ret = pagemap_scan_thp_entry(pmd, start, end, walk); 2765 if (ret != -ENOENT) 2766 return ret; 2767 2768 ret = 0; 2769 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); 2770 if (!pte) { 2771 walk->action = ACTION_AGAIN; 2772 return 0; 2773 } 2774 2775 lazy_mmu_mode_enable(); 2776 2777 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { 2778 /* Fast path for performing exclusive WP */ 2779 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2780 pte_t ptent = ptep_get(pte); 2781 2782 if ((pte_present(ptent) && pte_uffd(ptent)) || 2783 pte_swp_uffd_any(ptent)) 2784 continue; 2785 make_uffd_wp_pte(vma, addr, pte, ptent); 2786 if (!flush_end) 2787 start = addr; 2788 flush_end = addr + PAGE_SIZE; 2789 } 2790 goto flush_and_return; 2791 } 2792 2793 if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask && 2794 !p->arg.category_inverted && 2795 p->arg.category_mask == PAGE_IS_WRITTEN && 2796 p->arg.return_mask == PAGE_IS_WRITTEN) { 2797 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { 2798 unsigned long next = addr + PAGE_SIZE; 2799 pte_t ptent = ptep_get(pte); 2800 2801 if ((pte_present(ptent) && pte_uffd(ptent)) || 2802 pte_swp_uffd_any(ptent)) 2803 continue; 2804 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, 2805 p, addr, &next); 2806 if (next == addr) 2807 break; 2808 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2809 continue; 2810 make_uffd_wp_pte(vma, addr, pte, ptent); 2811 if (!flush_end) 2812 start = addr; 2813 flush_end = next; 2814 } 2815 goto flush_and_return; 2816 } 2817 2818 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2819 pte_t ptent = ptep_get(pte); 2820 unsigned long categories = p->cur_vma_category | 2821 pagemap_page_category(p, vma, addr, ptent); 2822 unsigned long next = addr + PAGE_SIZE; 2823 2824 if (!pagemap_scan_is_interesting_page(categories, p)) 2825 continue; 2826 2827 ret = pagemap_scan_output(categories, p, addr, &next); 2828 if (next == addr) 2829 break; 2830 2831 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2832 continue; 2833 if (~categories & PAGE_IS_WRITTEN) 2834 continue; 2835 2836 make_uffd_wp_pte(vma, addr, pte, ptent); 2837 if (!flush_end) 2838 start = addr; 2839 flush_end = next; 2840 } 2841 2842 flush_and_return: 2843 if (flush_end) 2844 flush_tlb_range(vma, start, addr); 2845 2846 lazy_mmu_mode_disable(); 2847 pte_unmap_unlock(start_pte, ptl); 2848 2849 cond_resched(); 2850 return ret; 2851 } 2852 2853 #ifdef CONFIG_HUGETLB_PAGE 2854 static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, 2855 unsigned long start, unsigned long end, 2856 struct mm_walk *walk) 2857 { 2858 struct pagemap_scan_private *p = walk->private; 2859 struct vm_area_struct *vma = walk->vma; 2860 unsigned long categories; 2861 spinlock_t *ptl; 2862 int ret = 0; 2863 pte_t pte; 2864 2865 if (~p->arg.flags & PM_SCAN_WP_MATCHING) { 2866 /* Go the short route when not write-protecting pages. */ 2867 2868 pte = huge_ptep_get(walk->mm, start, ptep); 2869 categories = p->cur_vma_category | 2870 pagemap_hugetlb_category(vma, pte); 2871 2872 if (!pagemap_scan_is_interesting_page(categories, p)) 2873 return 0; 2874 2875 return pagemap_scan_output(categories, p, start, &end); 2876 } 2877 2878 i_mmap_lock_write(vma->vm_file->f_mapping); 2879 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); 2880 2881 pte = huge_ptep_get(walk->mm, start, ptep); 2882 categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte); 2883 2884 if (!pagemap_scan_is_interesting_page(categories, p)) 2885 goto out_unlock; 2886 2887 ret = pagemap_scan_output(categories, p, start, &end); 2888 if (start == end) 2889 goto out_unlock; 2890 2891 if (~categories & PAGE_IS_WRITTEN) 2892 goto out_unlock; 2893 2894 if (end != start + huge_page_size(hstate_vma(vma))) { 2895 /* Partial HugeTLB page WP isn't possible. */ 2896 pagemap_scan_backout_range(p, start, end); 2897 p->arg.walk_end = start; 2898 ret = 0; 2899 goto out_unlock; 2900 } 2901 2902 make_uffd_wp_huge_pte(vma, start, ptep, pte); 2903 flush_hugetlb_tlb_range(vma, start, end); 2904 2905 out_unlock: 2906 spin_unlock(ptl); 2907 i_mmap_unlock_write(vma->vm_file->f_mapping); 2908 2909 return ret; 2910 } 2911 2912 /* 2913 * Write-protect the unpopulated hugetlb entries covering [addr, end) by 2914 * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() 2915 * does for populated entries. 2916 * 2917 * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the 2918 * loop normally runs a single iteration; it is written to cover the full range 2919 * in case the walker ever coalesces adjacent holes. 2920 * 2921 * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- 2922 * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table 2923 * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the 2924 * same VMA, so the scanning thread would deadlock against itself. PMD sharing 2925 * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so 2926 * the vma lock guards nothing that matters for these entries anyway. 2927 */ 2928 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 2929 unsigned long addr, unsigned long end) 2930 { 2931 struct hstate *h = hstate_vma(vma); 2932 unsigned long psize = huge_page_size(h); 2933 struct mm_struct *mm = vma->vm_mm; 2934 spinlock_t *ptl; 2935 pte_t *ptep; 2936 pte_t pte; 2937 2938 for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { 2939 ptep = huge_pte_alloc(mm, vma, addr, psize); 2940 if (!ptep) 2941 return -ENOMEM; 2942 2943 i_mmap_lock_write(vma->vm_file->f_mapping); 2944 ptl = huge_pte_lock(h, mm, ptep); 2945 pte = huge_ptep_get(mm, addr, ptep); 2946 make_uffd_wp_huge_pte(vma, addr, ptep, pte); 2947 /* 2948 * A none entry has no cached translation, so installing the 2949 * marker needs no TLB flush. Flush only if a fault populated 2950 * the entry between huge_pte_alloc() and the page table lock. 2951 */ 2952 if (!huge_pte_none(pte)) 2953 flush_hugetlb_tlb_range(vma, addr, addr + psize); 2954 spin_unlock(ptl); 2955 i_mmap_unlock_write(vma->vm_file->f_mapping); 2956 } 2957 2958 return 0; 2959 } 2960 #else 2961 #define pagemap_scan_hugetlb_entry NULL 2962 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 2963 unsigned long addr, unsigned long end) 2964 { 2965 return 0; 2966 } 2967 #endif 2968 2969 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, 2970 int depth, struct mm_walk *walk) 2971 { 2972 struct pagemap_scan_private *p = walk->private; 2973 struct vm_area_struct *vma = walk->vma; 2974 unsigned long categories; 2975 int ret, err; 2976 2977 if (!vma) 2978 return 0; 2979 2980 /* 2981 * In a uffd-wp VMA an unpopulated range is treated as written: 2982 * uffd-wp registration populates page tables and installs markers 2983 * with WP_UNPOPULATED, so a missing marker means the range was 2984 * zapped. See the pte_none() handling in pagemap_page_category(). 2985 * 2986 * hugetlb differs, see pagemap_hugetlb_category(). 2987 */ 2988 categories = p->cur_vma_category; 2989 if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma)) 2990 categories |= PAGE_IS_WRITTEN; 2991 2992 if (!pagemap_scan_is_interesting_page(categories, p)) 2993 return 0; 2994 2995 ret = pagemap_scan_output(categories, p, addr, &end); 2996 if (addr == end) 2997 return ret; 2998 2999 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 3000 return ret; 3001 3002 if (is_vm_hugetlb_page(vma)) 3003 err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); 3004 else 3005 err = uffd_wp_range(vma, addr, end - addr, true); 3006 if (err < 0) 3007 ret = err; 3008 3009 return ret; 3010 } 3011 3012 static const struct mm_walk_ops pagemap_scan_ops = { 3013 .test_walk = pagemap_scan_test_walk, 3014 .pmd_entry = pagemap_scan_pmd_entry, 3015 .pte_hole = pagemap_scan_pte_hole, 3016 .hugetlb_entry = pagemap_scan_hugetlb_entry, 3017 }; 3018 3019 static int pagemap_scan_get_args(struct pm_scan_arg *arg, 3020 unsigned long uarg) 3021 { 3022 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) 3023 return -EFAULT; 3024 3025 if (arg->size != sizeof(struct pm_scan_arg)) 3026 return -EINVAL; 3027 3028 /* Validate requested features */ 3029 if (arg->flags & ~PM_SCAN_FLAGS) 3030 return -EINVAL; 3031 if ((arg->category_inverted | arg->category_mask | 3032 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) 3033 return -EINVAL; 3034 3035 arg->start = untagged_addr((unsigned long)arg->start); 3036 arg->end = untagged_addr((unsigned long)arg->end); 3037 arg->vec = untagged_addr((unsigned long)arg->vec); 3038 3039 /* Validate memory pointers */ 3040 if (!IS_ALIGNED(arg->start, PAGE_SIZE)) 3041 return -EINVAL; 3042 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) 3043 return -EFAULT; 3044 if (!arg->vec && arg->vec_len) 3045 return -EINVAL; 3046 if (UINT_MAX == SIZE_MAX && arg->vec_len > SIZE_MAX) 3047 return -EINVAL; 3048 if (arg->vec && !access_ok((void __user *)(long)arg->vec, 3049 size_mul(arg->vec_len, sizeof(struct page_region)))) 3050 return -EFAULT; 3051 3052 /* Fixup default values */ 3053 arg->end = ALIGN(arg->end, PAGE_SIZE); 3054 arg->walk_end = 0; 3055 if (!arg->max_pages) 3056 arg->max_pages = ULONG_MAX; 3057 3058 return 0; 3059 } 3060 3061 static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, 3062 unsigned long uargl) 3063 { 3064 struct pm_scan_arg __user *uarg = (void __user *)uargl; 3065 3066 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) 3067 return -EFAULT; 3068 3069 return 0; 3070 } 3071 3072 static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) 3073 { 3074 if (!p->arg.vec_len) 3075 return 0; 3076 3077 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, 3078 p->arg.vec_len); 3079 p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); 3080 if (!p->vec_buf) 3081 return -ENOMEM; 3082 3083 p->vec_buf->start = p->vec_buf->end = 0; 3084 p->vec_out = (struct page_region __user *)(long)p->arg.vec; 3085 3086 return 0; 3087 } 3088 3089 static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) 3090 { 3091 const struct page_region *buf = p->vec_buf; 3092 long n = p->vec_buf_index; 3093 3094 if (!p->vec_buf) 3095 return 0; 3096 3097 if (buf[n].end != buf[n].start) 3098 n++; 3099 3100 if (!n) 3101 return 0; 3102 3103 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) 3104 return -EFAULT; 3105 3106 p->arg.vec_len -= n; 3107 p->vec_out += n; 3108 3109 p->vec_buf_index = 0; 3110 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); 3111 p->vec_buf->start = p->vec_buf->end = 0; 3112 3113 return n; 3114 } 3115 3116 static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) 3117 { 3118 struct pagemap_scan_private p = {0}; 3119 unsigned long walk_start; 3120 size_t n_ranges_out = 0; 3121 int ret; 3122 3123 ret = pagemap_scan_get_args(&p.arg, uarg); 3124 if (ret) 3125 return ret; 3126 3127 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | 3128 p.arg.return_mask; 3129 ret = pagemap_scan_init_bounce_buffer(&p); 3130 if (ret) 3131 return ret; 3132 3133 for (walk_start = p.arg.start; walk_start < p.arg.end; 3134 walk_start = p.arg.walk_end) { 3135 struct mmu_notifier_range range; 3136 long n_out; 3137 3138 if (fatal_signal_pending(current)) { 3139 ret = -EINTR; 3140 break; 3141 } 3142 3143 ret = mmap_read_lock_killable(mm); 3144 if (ret) 3145 break; 3146 3147 /* Protection change for the range is going to happen. */ 3148 if (p.arg.flags & PM_SCAN_WP_MATCHING) { 3149 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, 3150 mm, walk_start, p.arg.end); 3151 mmu_notifier_invalidate_range_start(&range); 3152 } 3153 3154 ret = walk_page_range(mm, walk_start, p.arg.end, 3155 &pagemap_scan_ops, &p); 3156 3157 if (p.arg.flags & PM_SCAN_WP_MATCHING) 3158 mmu_notifier_invalidate_range_end(&range); 3159 3160 mmap_read_unlock(mm); 3161 3162 n_out = pagemap_scan_flush_buffer(&p); 3163 if (n_out < 0) 3164 ret = n_out; 3165 else 3166 n_ranges_out += n_out; 3167 3168 if (ret != -ENOSPC) 3169 break; 3170 3171 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) 3172 break; 3173 } 3174 3175 /* ENOSPC signifies early stop (buffer full) from the walk. */ 3176 if (!ret || ret == -ENOSPC) 3177 ret = n_ranges_out; 3178 3179 /* The walk_end isn't set when ret is zero */ 3180 if (!p.arg.walk_end) 3181 p.arg.walk_end = p.arg.end; 3182 if (pagemap_scan_writeback_args(&p.arg, uarg)) 3183 ret = -EFAULT; 3184 3185 kfree(p.vec_buf); 3186 return ret; 3187 } 3188 3189 static long do_pagemap_cmd(struct file *file, unsigned int cmd, 3190 unsigned long arg) 3191 { 3192 struct mm_struct *mm = file->private_data; 3193 3194 switch (cmd) { 3195 case PAGEMAP_SCAN: 3196 return do_pagemap_scan(mm, arg); 3197 3198 default: 3199 return -EINVAL; 3200 } 3201 } 3202 3203 const struct file_operations proc_pagemap_operations = { 3204 .llseek = mem_lseek, /* borrow this */ 3205 .read = pagemap_read, 3206 .open = pagemap_open, 3207 .release = pagemap_release, 3208 .unlocked_ioctl = do_pagemap_cmd, 3209 .compat_ioctl = do_pagemap_cmd, 3210 }; 3211 #endif /* CONFIG_PROC_PAGE_MONITOR */ 3212 3213 #ifdef CONFIG_NUMA 3214 3215 struct numa_maps { 3216 unsigned long pages; 3217 unsigned long anon; 3218 unsigned long active; 3219 unsigned long writeback; 3220 unsigned long mapcount_max; 3221 unsigned long dirty; 3222 unsigned long swapcache; 3223 unsigned long node[MAX_NUMNODES]; 3224 }; 3225 3226 struct numa_maps_private { 3227 struct proc_maps_private proc_maps; 3228 struct numa_maps md; 3229 }; 3230 3231 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, 3232 unsigned long nr_pages) 3233 { 3234 struct folio *folio = page_folio(page); 3235 int count; 3236 3237 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 3238 count = folio_precise_page_mapcount(folio, page); 3239 else 3240 count = folio_average_page_mapcount(folio); 3241 3242 md->pages += nr_pages; 3243 if (pte_dirty || folio_test_dirty(folio)) 3244 md->dirty += nr_pages; 3245 3246 if (folio_test_swapcache(folio)) 3247 md->swapcache += nr_pages; 3248 3249 if (folio_test_active(folio) || folio_test_unevictable(folio)) 3250 md->active += nr_pages; 3251 3252 if (folio_test_writeback(folio)) 3253 md->writeback += nr_pages; 3254 3255 if (folio_test_anon(folio)) 3256 md->anon += nr_pages; 3257 3258 if (count > md->mapcount_max) 3259 md->mapcount_max = count; 3260 3261 md->node[folio_nid(folio)] += nr_pages; 3262 } 3263 3264 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, 3265 unsigned long addr) 3266 { 3267 struct page *page; 3268 int nid; 3269 3270 if (!pte_present(pte)) 3271 return NULL; 3272 3273 page = vm_normal_page(vma, addr, pte); 3274 if (!page || is_zone_device_page(page)) 3275 return NULL; 3276 3277 if (PageReserved(page)) 3278 return NULL; 3279 3280 nid = page_to_nid(page); 3281 if (!node_isset(nid, node_states[N_MEMORY])) 3282 return NULL; 3283 3284 return page; 3285 } 3286 3287 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3288 static struct page *can_gather_numa_stats_pmd(pmd_t pmd, 3289 struct vm_area_struct *vma, 3290 unsigned long addr) 3291 { 3292 struct page *page; 3293 int nid; 3294 3295 if (!pmd_present(pmd)) 3296 return NULL; 3297 3298 page = vm_normal_page_pmd(vma, addr, pmd); 3299 if (!page) 3300 return NULL; 3301 3302 if (PageReserved(page)) 3303 return NULL; 3304 3305 nid = page_to_nid(page); 3306 if (!node_isset(nid, node_states[N_MEMORY])) 3307 return NULL; 3308 3309 return page; 3310 } 3311 #endif 3312 3313 static int gather_pte_stats(pmd_t *pmd, unsigned long addr, 3314 unsigned long end, struct mm_walk *walk) 3315 { 3316 struct numa_maps *md = walk->private; 3317 struct vm_area_struct *vma = walk->vma; 3318 spinlock_t *ptl; 3319 pte_t *orig_pte; 3320 pte_t *pte; 3321 3322 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3323 ptl = pmd_trans_huge_lock(pmd, vma); 3324 if (ptl) { 3325 struct page *page; 3326 3327 page = can_gather_numa_stats_pmd(*pmd, vma, addr); 3328 if (page) 3329 gather_stats(page, md, pmd_dirty(*pmd), 3330 HPAGE_PMD_SIZE/PAGE_SIZE); 3331 spin_unlock(ptl); 3332 return 0; 3333 } 3334 #endif 3335 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 3336 if (!pte) { 3337 walk->action = ACTION_AGAIN; 3338 return 0; 3339 } 3340 do { 3341 pte_t ptent = ptep_get(pte); 3342 struct page *page = can_gather_numa_stats(ptent, vma, addr); 3343 if (!page) 3344 continue; 3345 gather_stats(page, md, pte_dirty(ptent), 1); 3346 3347 } while (pte++, addr += PAGE_SIZE, addr != end); 3348 pte_unmap_unlock(orig_pte, ptl); 3349 cond_resched(); 3350 return 0; 3351 } 3352 #ifdef CONFIG_HUGETLB_PAGE 3353 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3354 unsigned long addr, unsigned long end, struct mm_walk *walk) 3355 { 3356 pte_t huge_pte; 3357 struct numa_maps *md; 3358 struct page *page; 3359 spinlock_t *ptl; 3360 3361 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); 3362 huge_pte = huge_ptep_get(walk->mm, addr, pte); 3363 if (!pte_present(huge_pte)) 3364 goto out; 3365 3366 page = pte_page(huge_pte); 3367 3368 md = walk->private; 3369 gather_stats(page, md, pte_dirty(huge_pte), 1); 3370 out: 3371 spin_unlock(ptl); 3372 return 0; 3373 } 3374 3375 #else 3376 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3377 unsigned long addr, unsigned long end, struct mm_walk *walk) 3378 { 3379 return 0; 3380 } 3381 #endif 3382 3383 static const struct mm_walk_ops show_numa_ops = { 3384 .hugetlb_entry = gather_hugetlb_stats, 3385 .pmd_entry = gather_pte_stats, 3386 .walk_lock = PGWALK_RDLOCK, 3387 }; 3388 3389 static const struct mm_walk_ops show_numa_vma_lock_ops = { 3390 .hugetlb_entry = gather_hugetlb_stats, 3391 .pmd_entry = gather_pte_stats, 3392 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 3393 }; 3394 3395 static const struct mm_walk_ops * 3396 get_show_numa_ops(struct proc_maps_private *priv) 3397 { 3398 if (priv->lock_ctx.mmap_locked) 3399 return &show_numa_ops; 3400 return &show_numa_vma_lock_ops; 3401 } 3402 3403 /* 3404 * Display pages allocated per node and memory policy via /proc. 3405 */ 3406 static int show_numa_map(struct seq_file *m, void *v) 3407 { 3408 struct numa_maps_private *numa_priv = m->private; 3409 struct proc_maps_private *proc_priv = &numa_priv->proc_maps; 3410 struct vm_area_struct *vma = v; 3411 struct numa_maps *md = &numa_priv->md; 3412 struct file *file = vma->vm_file; 3413 struct mm_struct *mm = vma->vm_mm; 3414 char buffer[64]; 3415 struct mempolicy *pol; 3416 pgoff_t ilx; 3417 int nid; 3418 3419 if (!mm) 3420 return 0; 3421 3422 /* Ensure we start with an empty set of numa_maps statistics. */ 3423 memset(md, 0, sizeof(*md)); 3424 3425 pol = __get_vma_policy(vma, vma->vm_start, &ilx); 3426 if (pol) { 3427 mpol_to_str(buffer, sizeof(buffer), pol); 3428 mpol_cond_put(pol); 3429 } else { 3430 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy); 3431 } 3432 3433 seq_printf(m, "%08lx %s", vma->vm_start, buffer); 3434 3435 if (file) { 3436 seq_puts(m, " file="); 3437 seq_path(m, file_user_path(file), "\n\t= "); 3438 } else if (vma_is_initial_heap(vma)) { 3439 seq_puts(m, " heap"); 3440 } else if (vma_is_initial_stack(vma)) { 3441 seq_puts(m, " stack"); 3442 } 3443 3444 if (is_vm_hugetlb_page(vma)) 3445 seq_puts(m, " huge"); 3446 3447 /* Skip walking pages if gate VMA */ 3448 if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) { 3449 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 3450 drop_rcu(proc_priv); 3451 walk_page_vma(vma, get_show_numa_ops(proc_priv), md); 3452 reacquire_rcu(proc_priv); 3453 } 3454 3455 if (!md->pages) 3456 goto out; 3457 3458 if (md->anon) 3459 seq_printf(m, " anon=%lu", md->anon); 3460 3461 if (md->dirty) 3462 seq_printf(m, " dirty=%lu", md->dirty); 3463 3464 if (md->pages != md->anon && md->pages != md->dirty) 3465 seq_printf(m, " mapped=%lu", md->pages); 3466 3467 if (md->mapcount_max > 1) 3468 seq_printf(m, " mapmax=%lu", md->mapcount_max); 3469 3470 if (md->swapcache) 3471 seq_printf(m, " swapcache=%lu", md->swapcache); 3472 3473 if (md->active < md->pages && !is_vm_hugetlb_page(vma)) 3474 seq_printf(m, " active=%lu", md->active); 3475 3476 if (md->writeback) 3477 seq_printf(m, " writeback=%lu", md->writeback); 3478 3479 for_each_node_state(nid, N_MEMORY) 3480 if (md->node[nid]) 3481 seq_printf(m, " N%d=%lu", nid, md->node[nid]); 3482 3483 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); 3484 out: 3485 seq_putc(m, '\n'); 3486 return 0; 3487 } 3488 3489 static const struct seq_operations proc_pid_numa_maps_op = { 3490 .start = m_start, 3491 .next = m_next, 3492 .stop = m_stop, 3493 .show = show_numa_map, 3494 }; 3495 3496 static int pid_numa_maps_open(struct inode *inode, struct file *file) 3497 { 3498 return proc_maps_open(inode, file, &proc_pid_numa_maps_op, 3499 sizeof(struct numa_maps_private)); 3500 } 3501 3502 const struct file_operations proc_pid_numa_maps_operations = { 3503 .open = pid_numa_maps_open, 3504 .read = seq_read, 3505 .llseek = seq_lseek, 3506 .release = proc_map_release, 3507 }; 3508 3509 #endif /* CONFIG_NUMA */ 3510