1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <cl@gentwo.org> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13 #include <linux/fs.h> 14 #include <linux/mm.h> 15 #include <linux/err.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/cpumask.h> 20 #include <linux/vmstat.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/debugfs.h> 24 #include <linux/sched.h> 25 #include <linux/math64.h> 26 #include <linux/writeback.h> 27 #include <linux/compaction.h> 28 #include <linux/mm_inline.h> 29 #include <linux/page_owner.h> 30 #include <linux/sched/isolation.h> 31 32 #include "internal.h" 33 #include "page_alloc.h" 34 35 #ifdef CONFIG_PROC_FS 36 #ifdef CONFIG_NUMA 37 #define ENABLE_NUMA_STAT 1 38 static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 39 40 /* zero numa counters within a zone */ 41 static void zero_zone_numa_counters(struct zone *zone) 42 { 43 int item, cpu; 44 45 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 46 atomic_long_set(&zone->vm_numa_event[item], 0); 47 for_each_online_cpu(cpu) { 48 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 49 = 0; 50 } 51 } 52 } 53 54 /* zero numa counters of all the populated zones */ 55 static void zero_zones_numa_counters(void) 56 { 57 struct zone *zone; 58 59 for_each_populated_zone(zone) 60 zero_zone_numa_counters(zone); 61 } 62 63 /* zero global numa counters */ 64 static void zero_global_numa_counters(void) 65 { 66 int item; 67 68 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 69 atomic_long_set(&vm_numa_event[item], 0); 70 } 71 72 static void invalid_numa_statistics(void) 73 { 74 zero_zones_numa_counters(); 75 zero_global_numa_counters(); 76 } 77 78 static DEFINE_MUTEX(vm_numa_stat_lock); 79 80 static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, 81 void *buffer, size_t *length, loff_t *ppos) 82 { 83 int ret, oldval; 84 85 mutex_lock(&vm_numa_stat_lock); 86 if (write) 87 oldval = sysctl_vm_numa_stat; 88 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 89 if (ret || !write) 90 goto out; 91 92 if (oldval == sysctl_vm_numa_stat) 93 goto out; 94 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 95 static_branch_enable(&vm_numa_stat_key); 96 pr_info("enable numa statistics\n"); 97 } else { 98 static_branch_disable(&vm_numa_stat_key); 99 invalid_numa_statistics(); 100 pr_info("disable numa statistics, and clear numa counters\n"); 101 } 102 103 out: 104 mutex_unlock(&vm_numa_stat_lock); 105 return ret; 106 } 107 #endif 108 #endif /* CONFIG_PROC_FS */ 109 110 #ifdef CONFIG_VM_EVENT_COUNTERS 111 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 112 EXPORT_PER_CPU_SYMBOL(vm_event_states); 113 114 static void sum_vm_events(unsigned long *ret) 115 { 116 int cpu; 117 int i; 118 119 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 120 121 for_each_online_cpu(cpu) { 122 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 123 124 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 125 ret[i] += this->event[i]; 126 } 127 } 128 129 /* 130 * Accumulate the vm event counters across all CPUs. 131 * The result is unavoidably approximate - it can change 132 * during and after execution of this function. 133 */ 134 void all_vm_events(unsigned long *ret) 135 { 136 cpus_read_lock(); 137 sum_vm_events(ret); 138 cpus_read_unlock(); 139 } 140 EXPORT_SYMBOL_GPL(all_vm_events); 141 142 /* 143 * Fold the foreign cpu events into our own. 144 * 145 * This is adding to the events on one processor 146 * but keeps the global counts constant. 147 */ 148 void vm_events_fold_cpu(int cpu) 149 { 150 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 151 int i; 152 153 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 154 count_vm_events(i, fold_state->event[i]); 155 fold_state->event[i] = 0; 156 } 157 } 158 159 #endif /* CONFIG_VM_EVENT_COUNTERS */ 160 161 /* 162 * Manage combined zone based / global counters 163 * 164 * vm_stat contains the global counters 165 */ 166 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 167 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 168 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 169 EXPORT_SYMBOL(vm_zone_stat); 170 EXPORT_SYMBOL(vm_node_stat); 171 172 #ifdef CONFIG_NUMA 173 static void fold_vm_zone_numa_events(struct zone *zone) 174 { 175 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 176 int cpu; 177 enum numa_stat_item item; 178 179 for_each_online_cpu(cpu) { 180 struct per_cpu_zonestat *pzstats; 181 182 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 183 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 184 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 185 } 186 187 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 188 zone_numa_event_add(zone_numa_events[item], zone, item); 189 } 190 191 void fold_vm_numa_events(void) 192 { 193 struct zone *zone; 194 195 for_each_populated_zone(zone) 196 fold_vm_zone_numa_events(zone); 197 } 198 #endif 199 200 #ifdef CONFIG_SMP 201 202 int calculate_pressure_threshold(struct zone *zone) 203 { 204 int threshold; 205 int watermark_distance; 206 207 /* 208 * As vmstats are not up to date, there is drift between the estimated 209 * and real values. For high thresholds and a high number of CPUs, it 210 * is possible for the min watermark to be breached while the estimated 211 * value looks fine. The pressure threshold is a reduced value such 212 * that even the maximum amount of drift will not accidentally breach 213 * the min watermark 214 */ 215 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 216 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 217 218 /* 219 * Maximum threshold is 125 220 */ 221 threshold = min(125, threshold); 222 223 return threshold; 224 } 225 226 int calculate_normal_threshold(struct zone *zone) 227 { 228 int threshold; 229 int mem; /* memory in 128 MB units */ 230 231 /* 232 * The threshold scales with the number of processors and the amount 233 * of memory per zone. More memory means that we can defer updates for 234 * longer, more processors could lead to more contention. 235 * fls() is used to have a cheap way of logarithmic scaling. 236 * 237 * Some sample thresholds: 238 * 239 * Threshold Processors (fls) Zonesize fls(mem)+1 240 * ------------------------------------------------------------------ 241 * 8 1 1 0.9-1 GB 4 242 * 16 2 2 0.9-1 GB 4 243 * 20 2 2 1-2 GB 5 244 * 24 2 2 2-4 GB 6 245 * 28 2 2 4-8 GB 7 246 * 32 2 2 8-16 GB 8 247 * 4 2 2 <128M 1 248 * 30 4 3 2-4 GB 5 249 * 48 4 3 8-16 GB 8 250 * 32 8 4 1-2 GB 4 251 * 32 8 4 0.9-1GB 4 252 * 10 16 5 <128M 1 253 * 40 16 5 900M 4 254 * 70 64 7 2-4 GB 5 255 * 84 64 7 4-8 GB 6 256 * 108 512 9 4-8 GB 6 257 * 125 1024 10 8-16 GB 8 258 * 125 1024 10 16-32 GB 9 259 */ 260 261 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 262 263 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 264 265 /* 266 * Maximum threshold is 125 267 */ 268 threshold = min(125, threshold); 269 270 return threshold; 271 } 272 273 /* 274 * Refresh the thresholds for each zone. 275 */ 276 void refresh_zone_stat_thresholds(void) 277 { 278 struct pglist_data *pgdat; 279 struct zone *zone; 280 int cpu; 281 int threshold; 282 283 /* Zero current pgdat thresholds */ 284 for_each_online_pgdat(pgdat) { 285 for_each_online_cpu(cpu) { 286 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 287 } 288 } 289 290 for_each_populated_zone(zone) { 291 struct pglist_data *pgdat = zone->zone_pgdat; 292 unsigned long max_drift, tolerate_drift; 293 294 threshold = calculate_normal_threshold(zone); 295 296 for_each_online_cpu(cpu) { 297 int pgdat_threshold; 298 299 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 300 = threshold; 301 302 /* Base nodestat threshold on the largest populated zone. */ 303 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 304 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 305 = max(threshold, pgdat_threshold); 306 } 307 308 /* 309 * Only set percpu_drift_mark if there is a danger that 310 * NR_FREE_PAGES reports the low watermark is ok when in fact 311 * the min watermark could be breached by an allocation 312 */ 313 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 314 max_drift = num_online_cpus() * threshold; 315 if (max_drift > tolerate_drift) 316 zone->percpu_drift_mark = high_wmark_pages(zone) + 317 max_drift; 318 } 319 } 320 321 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 322 int (*calculate_pressure)(struct zone *)) 323 { 324 struct zone *zone; 325 int cpu; 326 int threshold; 327 int i; 328 329 for (i = 0; i < pgdat->nr_zones; i++) { 330 zone = &pgdat->node_zones[i]; 331 if (!zone->percpu_drift_mark) 332 continue; 333 334 threshold = (*calculate_pressure)(zone); 335 for_each_online_cpu(cpu) 336 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 337 = threshold; 338 } 339 } 340 341 /* 342 * For use when we know that interrupts are disabled, 343 * or when we know that preemption is disabled and that 344 * particular counter cannot be updated from interrupt context. 345 */ 346 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 347 long delta) 348 { 349 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 350 s8 __percpu *p = pcp->vm_stat_diff + item; 351 long x; 352 long t; 353 354 /* 355 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 356 * atomicity is provided by IRQs being disabled -- either explicitly 357 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 358 * CPU migrations and preemption potentially corrupts a counter so 359 * disable preemption. 360 */ 361 preempt_disable_nested(); 362 363 x = delta + __this_cpu_read(*p); 364 365 t = __this_cpu_read(pcp->stat_threshold); 366 367 if (unlikely(abs(x) > t)) { 368 zone_page_state_add(x, zone, item); 369 x = 0; 370 } 371 __this_cpu_write(*p, x); 372 373 preempt_enable_nested(); 374 } 375 EXPORT_SYMBOL(__mod_zone_page_state); 376 377 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 378 long delta) 379 { 380 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 381 s8 __percpu *p = pcp->vm_node_stat_diff + item; 382 long x; 383 long t; 384 385 if (vmstat_item_in_bytes(item)) { 386 /* 387 * Only cgroups use subpage accounting right now; at 388 * the global level, these items still change in 389 * multiples of whole pages. Store them as pages 390 * internally to keep the per-cpu counters compact. 391 */ 392 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 393 delta >>= PAGE_SHIFT; 394 } 395 396 /* See __mod_zone_page_state() */ 397 preempt_disable_nested(); 398 399 x = delta + __this_cpu_read(*p); 400 401 t = __this_cpu_read(pcp->stat_threshold); 402 403 if (unlikely(abs(x) > t)) { 404 node_page_state_add(x, pgdat, item); 405 x = 0; 406 } 407 __this_cpu_write(*p, x); 408 409 preempt_enable_nested(); 410 } 411 EXPORT_SYMBOL(__mod_node_page_state); 412 413 /* 414 * Optimized increment and decrement functions. 415 * 416 * These are only for a single page and therefore can take a struct page * 417 * argument instead of struct zone *. This allows the inclusion of the code 418 * generated for page_zone(page) into the optimized functions. 419 * 420 * No overflow check is necessary and therefore the differential can be 421 * incremented or decremented in place which may allow the compilers to 422 * generate better code. 423 * The increment or decrement is known and therefore one boundary check can 424 * be omitted. 425 * 426 * NOTE: These functions are very performance sensitive. Change only 427 * with care. 428 * 429 * Some processors have inc/dec instructions that are atomic vs an interrupt. 430 * However, the code must first determine the differential location in a zone 431 * based on the processor number and then inc/dec the counter. There is no 432 * guarantee without disabling preemption that the processor will not change 433 * in between and therefore the atomicity vs. interrupt cannot be exploited 434 * in a useful way here. 435 */ 436 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 437 { 438 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 439 s8 __percpu *p = pcp->vm_stat_diff + item; 440 s8 v, t; 441 442 /* See __mod_zone_page_state() */ 443 preempt_disable_nested(); 444 445 v = __this_cpu_inc_return(*p); 446 t = __this_cpu_read(pcp->stat_threshold); 447 if (unlikely(v > t)) { 448 s8 overstep = t >> 1; 449 450 zone_page_state_add(v + overstep, zone, item); 451 __this_cpu_write(*p, -overstep); 452 } 453 454 preempt_enable_nested(); 455 } 456 457 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 458 { 459 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 460 s8 __percpu *p = pcp->vm_node_stat_diff + item; 461 s8 v, t; 462 463 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 464 465 /* See __mod_zone_page_state() */ 466 preempt_disable_nested(); 467 468 v = __this_cpu_inc_return(*p); 469 t = __this_cpu_read(pcp->stat_threshold); 470 if (unlikely(v > t)) { 471 s8 overstep = t >> 1; 472 473 node_page_state_add(v + overstep, pgdat, item); 474 __this_cpu_write(*p, -overstep); 475 } 476 477 preempt_enable_nested(); 478 } 479 480 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 481 { 482 __inc_zone_state(page_zone(page), item); 483 } 484 EXPORT_SYMBOL(__inc_zone_page_state); 485 486 void __inc_node_page_state(struct page *page, enum node_stat_item item) 487 { 488 __inc_node_state(page_pgdat(page), item); 489 } 490 EXPORT_SYMBOL(__inc_node_page_state); 491 492 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 493 { 494 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 495 s8 __percpu *p = pcp->vm_stat_diff + item; 496 s8 v, t; 497 498 /* See __mod_zone_page_state() */ 499 preempt_disable_nested(); 500 501 v = __this_cpu_dec_return(*p); 502 t = __this_cpu_read(pcp->stat_threshold); 503 if (unlikely(v < - t)) { 504 s8 overstep = t >> 1; 505 506 zone_page_state_add(v - overstep, zone, item); 507 __this_cpu_write(*p, overstep); 508 } 509 510 preempt_enable_nested(); 511 } 512 513 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 514 { 515 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 516 s8 __percpu *p = pcp->vm_node_stat_diff + item; 517 s8 v, t; 518 519 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 520 521 /* See __mod_zone_page_state() */ 522 preempt_disable_nested(); 523 524 v = __this_cpu_dec_return(*p); 525 t = __this_cpu_read(pcp->stat_threshold); 526 if (unlikely(v < - t)) { 527 s8 overstep = t >> 1; 528 529 node_page_state_add(v - overstep, pgdat, item); 530 __this_cpu_write(*p, overstep); 531 } 532 533 preempt_enable_nested(); 534 } 535 536 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 537 { 538 __dec_zone_state(page_zone(page), item); 539 } 540 EXPORT_SYMBOL(__dec_zone_page_state); 541 542 void __dec_node_page_state(struct page *page, enum node_stat_item item) 543 { 544 __dec_node_state(page_pgdat(page), item); 545 } 546 EXPORT_SYMBOL(__dec_node_page_state); 547 548 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 549 /* 550 * If we have cmpxchg_local support then we do not need to incur the overhead 551 * that comes with local_irq_save/restore if we use this_cpu_try_cmpxchg(). 552 * 553 * mod_state() modifies the zone counter state through atomic per cpu 554 * operations. 555 * 556 * Overstep mode specifies how overstep should handled: 557 * 0 No overstepping 558 * 1 Overstepping half of threshold 559 * -1 Overstepping minus half of threshold 560 */ 561 static inline void mod_zone_state(struct zone *zone, 562 enum zone_stat_item item, long delta, int overstep_mode) 563 { 564 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 565 s8 __percpu *p = pcp->vm_stat_diff + item; 566 long n, t, z; 567 s8 o; 568 569 o = this_cpu_read(*p); 570 do { 571 z = 0; /* overflow to zone counters */ 572 573 /* 574 * The fetching of the stat_threshold is racy. We may apply 575 * a counter threshold to the wrong the cpu if we get 576 * rescheduled while executing here. However, the next 577 * counter update will apply the threshold again and 578 * therefore bring the counter under the threshold again. 579 * 580 * Most of the time the thresholds are the same anyways 581 * for all cpus in a zone. 582 */ 583 t = this_cpu_read(pcp->stat_threshold); 584 585 n = delta + (long)o; 586 587 if (abs(n) > t) { 588 int os = overstep_mode * (t >> 1) ; 589 590 /* Overflow must be added to zone counters */ 591 z = n + os; 592 n = -os; 593 } 594 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 595 596 if (z) 597 zone_page_state_add(z, zone, item); 598 } 599 600 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 601 long delta) 602 { 603 mod_zone_state(zone, item, delta, 0); 604 } 605 EXPORT_SYMBOL(mod_zone_page_state); 606 607 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 608 { 609 mod_zone_state(page_zone(page), item, 1, 1); 610 } 611 EXPORT_SYMBOL(inc_zone_page_state); 612 613 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 614 { 615 mod_zone_state(page_zone(page), item, -1, -1); 616 } 617 EXPORT_SYMBOL(dec_zone_page_state); 618 619 static inline void mod_node_state(struct pglist_data *pgdat, 620 enum node_stat_item item, int delta, int overstep_mode) 621 { 622 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 623 s8 __percpu *p = pcp->vm_node_stat_diff + item; 624 long n, t, z; 625 s8 o; 626 627 if (vmstat_item_in_bytes(item)) { 628 /* 629 * Only cgroups use subpage accounting right now; at 630 * the global level, these items still change in 631 * multiples of whole pages. Store them as pages 632 * internally to keep the per-cpu counters compact. 633 */ 634 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 635 delta >>= PAGE_SHIFT; 636 } 637 638 o = this_cpu_read(*p); 639 do { 640 z = 0; /* overflow to node counters */ 641 642 /* 643 * The fetching of the stat_threshold is racy. We may apply 644 * a counter threshold to the wrong the cpu if we get 645 * rescheduled while executing here. However, the next 646 * counter update will apply the threshold again and 647 * therefore bring the counter under the threshold again. 648 * 649 * Most of the time the thresholds are the same anyways 650 * for all cpus in a node. 651 */ 652 t = this_cpu_read(pcp->stat_threshold); 653 654 n = delta + (long)o; 655 656 if (abs(n) > t) { 657 int os = overstep_mode * (t >> 1) ; 658 659 /* Overflow must be added to node counters */ 660 z = n + os; 661 n = -os; 662 } 663 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 664 665 if (z) 666 node_page_state_add(z, pgdat, item); 667 } 668 669 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 670 long delta) 671 { 672 mod_node_state(pgdat, item, delta, 0); 673 } 674 EXPORT_SYMBOL(mod_node_page_state); 675 676 void inc_node_page_state(struct page *page, enum node_stat_item item) 677 { 678 mod_node_state(page_pgdat(page), item, 1, 1); 679 } 680 EXPORT_SYMBOL(inc_node_page_state); 681 682 void dec_node_page_state(struct page *page, enum node_stat_item item) 683 { 684 mod_node_state(page_pgdat(page), item, -1, -1); 685 } 686 EXPORT_SYMBOL(dec_node_page_state); 687 #else 688 /* 689 * Use interrupt disable to serialize counter updates 690 */ 691 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 692 long delta) 693 { 694 unsigned long flags; 695 696 local_irq_save(flags); 697 __mod_zone_page_state(zone, item, delta); 698 local_irq_restore(flags); 699 } 700 EXPORT_SYMBOL(mod_zone_page_state); 701 702 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 703 { 704 unsigned long flags; 705 struct zone *zone; 706 707 zone = page_zone(page); 708 local_irq_save(flags); 709 __inc_zone_state(zone, item); 710 local_irq_restore(flags); 711 } 712 EXPORT_SYMBOL(inc_zone_page_state); 713 714 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 715 { 716 unsigned long flags; 717 718 local_irq_save(flags); 719 __dec_zone_page_state(page, item); 720 local_irq_restore(flags); 721 } 722 EXPORT_SYMBOL(dec_zone_page_state); 723 724 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 725 long delta) 726 { 727 unsigned long flags; 728 729 local_irq_save(flags); 730 __mod_node_page_state(pgdat, item, delta); 731 local_irq_restore(flags); 732 } 733 EXPORT_SYMBOL(mod_node_page_state); 734 735 void inc_node_page_state(struct page *page, enum node_stat_item item) 736 { 737 unsigned long flags; 738 struct pglist_data *pgdat; 739 740 pgdat = page_pgdat(page); 741 local_irq_save(flags); 742 __inc_node_state(pgdat, item); 743 local_irq_restore(flags); 744 } 745 EXPORT_SYMBOL(inc_node_page_state); 746 747 void dec_node_page_state(struct page *page, enum node_stat_item item) 748 { 749 unsigned long flags; 750 751 local_irq_save(flags); 752 __dec_node_page_state(page, item); 753 local_irq_restore(flags); 754 } 755 EXPORT_SYMBOL(dec_node_page_state); 756 #endif 757 758 /* 759 * Fold a differential into the global counters. 760 * Returns whether counters were updated. 761 */ 762 static int fold_diff(int *zone_diff, int *node_diff) 763 { 764 int i; 765 bool changed = false; 766 767 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 768 if (zone_diff[i]) { 769 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 770 changed = true; 771 } 772 } 773 774 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 775 if (node_diff[i]) { 776 atomic_long_add(node_diff[i], &vm_node_stat[i]); 777 changed = true; 778 } 779 } 780 781 return changed; 782 } 783 784 /* 785 * Update the zone counters for the current cpu. 786 * 787 * Note that refresh_cpu_vm_stats strives to only access 788 * node local memory. The per cpu pagesets on remote zones are placed 789 * in the memory local to the processor using that pageset. So the 790 * loop over all zones will access a series of cachelines local to 791 * the processor. 792 * 793 * The call to zone_page_state_add updates the cachelines with the 794 * statistics in the remote zone struct as well as the global cachelines 795 * with the global counters. These could cause remote node cache line 796 * bouncing and will have to be only done when necessary. 797 * 798 * The function returns whether global counters were updated. 799 */ 800 static bool refresh_cpu_vm_stats(bool do_pagesets) 801 { 802 struct pglist_data *pgdat; 803 struct zone *zone; 804 int i; 805 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 806 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 807 bool changed = false; 808 809 for_each_populated_zone(zone) { 810 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 811 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 812 813 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 814 int v; 815 816 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 817 if (v) { 818 819 atomic_long_add(v, &zone->vm_stat[i]); 820 global_zone_diff[i] += v; 821 #ifdef CONFIG_NUMA 822 /* 3 seconds idle till flush */ 823 __this_cpu_write(pcp->expire, 3); 824 #endif 825 } 826 } 827 828 if (do_pagesets) { 829 cond_resched(); 830 831 if (decay_pcp_high(zone, this_cpu_ptr(pcp))) 832 changed = true; 833 #ifdef CONFIG_NUMA 834 /* 835 * Deal with draining the remote pageset of this 836 * processor 837 * 838 * Check if there are pages remaining in this pageset 839 * if not then there is nothing to expire. 840 */ 841 if (!__this_cpu_read(pcp->expire) || 842 !__this_cpu_read(pcp->count)) 843 continue; 844 845 /* 846 * We never drain zones local to this processor. 847 */ 848 if (zone_to_nid(zone) == numa_node_id()) { 849 __this_cpu_write(pcp->expire, 0); 850 continue; 851 } 852 853 if (__this_cpu_dec_return(pcp->expire)) { 854 changed = true; 855 continue; 856 } 857 858 if (__this_cpu_read(pcp->count)) { 859 drain_zone_pages(zone, this_cpu_ptr(pcp)); 860 changed = true; 861 } 862 #endif 863 } 864 } 865 866 for_each_online_pgdat(pgdat) { 867 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 868 869 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 870 int v; 871 872 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 873 if (v) { 874 atomic_long_add(v, &pgdat->vm_stat[i]); 875 global_node_diff[i] += v; 876 } 877 } 878 } 879 880 if (fold_diff(global_zone_diff, global_node_diff)) 881 changed = true; 882 return changed; 883 } 884 885 /* 886 * Fold the data for an offline cpu into the global array. 887 * There cannot be any access by the offline cpu and therefore 888 * synchronization is simplified. 889 */ 890 void cpu_vm_stats_fold(int cpu) 891 { 892 struct pglist_data *pgdat; 893 struct zone *zone; 894 int i; 895 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 896 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 897 898 for_each_populated_zone(zone) { 899 struct per_cpu_zonestat *pzstats; 900 901 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 902 903 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 904 if (pzstats->vm_stat_diff[i]) { 905 int v; 906 907 v = pzstats->vm_stat_diff[i]; 908 pzstats->vm_stat_diff[i] = 0; 909 atomic_long_add(v, &zone->vm_stat[i]); 910 global_zone_diff[i] += v; 911 } 912 } 913 #ifdef CONFIG_NUMA 914 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 915 if (pzstats->vm_numa_event[i]) { 916 unsigned long v; 917 918 v = pzstats->vm_numa_event[i]; 919 pzstats->vm_numa_event[i] = 0; 920 zone_numa_event_add(v, zone, i); 921 } 922 } 923 #endif 924 } 925 926 for_each_online_pgdat(pgdat) { 927 struct per_cpu_nodestat *p; 928 929 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 930 931 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 932 if (p->vm_node_stat_diff[i]) { 933 int v; 934 935 v = p->vm_node_stat_diff[i]; 936 p->vm_node_stat_diff[i] = 0; 937 atomic_long_add(v, &pgdat->vm_stat[i]); 938 global_node_diff[i] += v; 939 } 940 } 941 942 fold_diff(global_zone_diff, global_node_diff); 943 } 944 945 /* 946 * this is only called if !populated_zone(zone), which implies no other users of 947 * pset->vm_stat_diff[] exist. 948 */ 949 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 950 { 951 unsigned long v; 952 int i; 953 954 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 955 if (pzstats->vm_stat_diff[i]) { 956 v = pzstats->vm_stat_diff[i]; 957 pzstats->vm_stat_diff[i] = 0; 958 zone_page_state_add(v, zone, i); 959 } 960 } 961 962 #ifdef CONFIG_NUMA 963 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 964 if (pzstats->vm_numa_event[i]) { 965 v = pzstats->vm_numa_event[i]; 966 pzstats->vm_numa_event[i] = 0; 967 zone_numa_event_add(v, zone, i); 968 } 969 } 970 #endif 971 } 972 #endif 973 974 #ifdef CONFIG_NUMA 975 /* 976 * Determine the per node value of a stat item. This function 977 * is called frequently in a NUMA machine, so try to be as 978 * frugal as possible. 979 */ 980 unsigned long sum_zone_node_page_state(int node, 981 enum zone_stat_item item) 982 { 983 struct zone *zones = NODE_DATA(node)->node_zones; 984 int i; 985 unsigned long count = 0; 986 987 for (i = 0; i < MAX_NR_ZONES; i++) 988 count += zone_page_state(zones + i, item); 989 990 return count; 991 } 992 993 /* Determine the per node value of a numa stat item. */ 994 unsigned long sum_zone_numa_event_state(int node, 995 enum numa_stat_item item) 996 { 997 struct zone *zones = NODE_DATA(node)->node_zones; 998 unsigned long count = 0; 999 int i; 1000 1001 for (i = 0; i < MAX_NR_ZONES; i++) 1002 count += zone_numa_event_state(zones + i, item); 1003 1004 return count; 1005 } 1006 1007 /* 1008 * Determine the per node value of a stat item. 1009 */ 1010 unsigned long node_page_state_pages(struct pglist_data *pgdat, 1011 enum node_stat_item item) 1012 { 1013 long x = atomic_long_read(&pgdat->vm_stat[item]); 1014 #ifdef CONFIG_SMP 1015 if (x < 0) 1016 x = 0; 1017 #endif 1018 return x; 1019 } 1020 1021 unsigned long node_page_state(struct pglist_data *pgdat, 1022 enum node_stat_item item) 1023 { 1024 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1025 1026 return node_page_state_pages(pgdat, item); 1027 } 1028 #endif 1029 1030 /* 1031 * Count number of pages "struct page" and "struct page_ext" consume. 1032 * nr_memmap_boot_pages: # of pages allocated by boot allocator 1033 * nr_memmap_pages: # of pages that were allocated by buddy allocator 1034 */ 1035 static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); 1036 static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); 1037 1038 void memmap_boot_pages_add(long delta) 1039 { 1040 atomic_long_add(delta, &nr_memmap_boot_pages); 1041 } 1042 1043 void memmap_pages_add(long delta) 1044 { 1045 atomic_long_add(delta, &nr_memmap_pages); 1046 } 1047 1048 #ifdef CONFIG_COMPACTION 1049 1050 struct contig_page_info { 1051 unsigned long free_pages; 1052 unsigned long free_blocks_total; 1053 unsigned long free_blocks_suitable; 1054 }; 1055 1056 /* 1057 * Calculate the number of free pages in a zone, how many contiguous 1058 * pages are free and how many are large enough to satisfy an allocation of 1059 * the target size. Note that this function makes no attempt to estimate 1060 * how many suitable free blocks there *might* be if MOVABLE pages were 1061 * migrated. Calculating that is possible, but expensive and can be 1062 * figured out from userspace 1063 */ 1064 static void fill_contig_page_info(struct zone *zone, 1065 unsigned int suitable_order, 1066 struct contig_page_info *info) 1067 { 1068 unsigned int order; 1069 1070 info->free_pages = 0; 1071 info->free_blocks_total = 0; 1072 info->free_blocks_suitable = 0; 1073 1074 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1075 unsigned long blocks; 1076 1077 /* 1078 * Count number of free blocks. 1079 * 1080 * Access to nr_free is lockless as nr_free is used only for 1081 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1082 */ 1083 blocks = data_race(zone->free_area[order].nr_free); 1084 info->free_blocks_total += blocks; 1085 1086 /* Count free base pages */ 1087 info->free_pages += blocks << order; 1088 1089 /* Count the suitable free blocks */ 1090 if (order >= suitable_order) 1091 info->free_blocks_suitable += blocks << 1092 (order - suitable_order); 1093 } 1094 } 1095 1096 /* 1097 * A fragmentation index only makes sense if an allocation of a requested 1098 * size would fail. If that is true, the fragmentation index indicates 1099 * whether external fragmentation or a lack of memory was the problem. 1100 * The value can be used to determine if page reclaim or compaction 1101 * should be used 1102 */ 1103 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1104 { 1105 unsigned long requested = 1UL << order; 1106 1107 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1108 return 0; 1109 1110 if (!info->free_blocks_total) 1111 return 0; 1112 1113 /* Fragmentation index only makes sense when a request would fail */ 1114 if (info->free_blocks_suitable) 1115 return -1000; 1116 1117 /* 1118 * Index is between 0 and 1 so return within 3 decimal places 1119 * 1120 * 0 => allocation would fail due to lack of memory 1121 * 1 => allocation would fail due to fragmentation 1122 */ 1123 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1124 } 1125 1126 /* 1127 * Calculates external fragmentation within a zone wrt the given order. 1128 * It is defined as the percentage of pages found in blocks of size 1129 * less than 1 << order. It returns values in range [0, 100]. 1130 */ 1131 unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1132 { 1133 struct contig_page_info info; 1134 1135 fill_contig_page_info(zone, order, &info); 1136 if (info.free_pages == 0) 1137 return 0; 1138 1139 return div_u64((info.free_pages - 1140 (info.free_blocks_suitable << order)) * 100, 1141 info.free_pages); 1142 } 1143 1144 /* Same as __fragmentation index but allocs contig_page_info on stack */ 1145 int fragmentation_index(struct zone *zone, unsigned int order) 1146 { 1147 struct contig_page_info info; 1148 1149 fill_contig_page_info(zone, order, &info); 1150 return __fragmentation_index(order, &info); 1151 } 1152 #endif 1153 1154 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1155 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1156 #ifdef CONFIG_ZONE_DMA 1157 #define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma", 1158 #else 1159 #define TEXT_FOR_DMA(xx, yy) 1160 #endif 1161 1162 #ifdef CONFIG_ZONE_DMA32 1163 #define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32", 1164 #else 1165 #define TEXT_FOR_DMA32(xx, yy) 1166 #endif 1167 1168 #ifdef CONFIG_HIGHMEM 1169 #define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high", 1170 #else 1171 #define TEXT_FOR_HIGHMEM(xx, yy) 1172 #endif 1173 1174 #ifdef CONFIG_ZONE_DEVICE 1175 #define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device", 1176 #else 1177 #define TEXT_FOR_DEVICE(xx, yy) 1178 #endif 1179 1180 #define TEXTS_FOR_ZONES(xx, yy) \ 1181 TEXT_FOR_DMA(xx, yy) \ 1182 TEXT_FOR_DMA32(xx, yy) \ 1183 [xx##_NORMAL] = yy "_normal", \ 1184 TEXT_FOR_HIGHMEM(xx, yy) \ 1185 [xx##_MOVABLE] = yy "_movable", \ 1186 TEXT_FOR_DEVICE(xx, yy) 1187 1188 const char * const vmstat_text[] = { 1189 /* enum zone_stat_item counters */ 1190 #define I(x) (x) 1191 [I(NR_FREE_PAGES)] = "nr_free_pages", 1192 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks", 1193 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon", 1194 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon", 1195 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file", 1196 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file", 1197 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable", 1198 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending", 1199 [I(NR_MLOCK)] = "nr_mlock", 1200 #if IS_ENABLED(CONFIG_ZSMALLOC) 1201 [I(NR_ZSPAGES)] = "nr_zspages", 1202 #endif 1203 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma", 1204 #ifdef CONFIG_UNACCEPTED_MEMORY 1205 [I(NR_UNACCEPTED)] = "nr_unaccepted", 1206 #endif 1207 #undef I 1208 1209 /* enum numa_stat_item counters */ 1210 #define I(x) (NR_VM_ZONE_STAT_ITEMS + x) 1211 #ifdef CONFIG_NUMA 1212 [I(NUMA_HIT)] = "numa_hit", 1213 [I(NUMA_MISS)] = "numa_miss", 1214 [I(NUMA_FOREIGN)] = "numa_foreign", 1215 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave", 1216 [I(NUMA_LOCAL)] = "numa_local", 1217 [I(NUMA_OTHER)] = "numa_other", 1218 #endif 1219 #undef I 1220 1221 /* enum node_stat_item counters */ 1222 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x) 1223 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon", 1224 [I(NR_ACTIVE_ANON)] = "nr_active_anon", 1225 [I(NR_INACTIVE_FILE)] = "nr_inactive_file", 1226 [I(NR_ACTIVE_FILE)] = "nr_active_file", 1227 [I(NR_UNEVICTABLE)] = "nr_unevictable", 1228 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable", 1229 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable", 1230 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon", 1231 [I(NR_ISOLATED_FILE)] = "nr_isolated_file", 1232 [I(WORKINGSET_NODES)] = "workingset_nodes", 1233 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon", 1234 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file", 1235 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon", 1236 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file", 1237 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon", 1238 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file", 1239 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim", 1240 [I(NR_ANON_MAPPED)] = "nr_anon_pages", 1241 [I(NR_FILE_MAPPED)] = "nr_mapped", 1242 [I(NR_FILE_PAGES)] = "nr_file_pages", 1243 [I(NR_FILE_DIRTY)] = "nr_dirty", 1244 [I(NR_WRITEBACK)] = "nr_writeback", 1245 [I(NR_SHMEM)] = "nr_shmem", 1246 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages", 1247 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped", 1248 [I(NR_FILE_THPS)] = "nr_file_hugepages", 1249 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped", 1250 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages", 1251 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write", 1252 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim", 1253 [I(NR_DIRTIED)] = "nr_dirtied", 1254 [I(NR_WRITTEN)] = "nr_written", 1255 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written", 1256 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable", 1257 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired", 1258 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released", 1259 [I(NR_VMALLOC)] = "nr_vmalloc", 1260 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack", 1261 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1262 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack", 1263 #endif 1264 [I(NR_PAGETABLE)] = "nr_page_table_pages", 1265 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages", 1266 #ifdef CONFIG_IOMMU_SUPPORT 1267 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages", 1268 #endif 1269 #ifdef CONFIG_SWAP 1270 [I(NR_SWAPCACHE)] = "nr_swapcached", 1271 #endif 1272 #ifdef CONFIG_NUMA_BALANCING 1273 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success", 1274 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate", 1275 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl", 1276 #endif 1277 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd", 1278 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct", 1279 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged", 1280 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive", 1281 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd", 1282 [I(PGSTEAL_DIRECT)] = "pgsteal_direct", 1283 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged", 1284 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive", 1285 [I(PGSTEAL_ANON)] = "pgsteal_anon", 1286 [I(PGSTEAL_FILE)] = "pgsteal_file", 1287 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd", 1288 [I(PGSCAN_DIRECT)] = "pgscan_direct", 1289 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged", 1290 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", 1291 [I(PGSCAN_ANON)] = "pgscan_anon", 1292 [I(PGSCAN_FILE)] = "pgscan_file", 1293 [I(PGREFILL)] = "pgrefill", 1294 #ifdef CONFIG_HUGETLB_PAGE 1295 [I(NR_HUGETLB)] = "nr_hugetlb", 1296 #endif 1297 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages", 1298 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages", 1299 [I(NR_GPU_ACTIVE)] = "nr_gpu_active", 1300 [I(NR_GPU_RECLAIM)] = "nr_gpu_reclaim", 1301 #undef I 1302 1303 /* system-wide enum vm_stat_item counters */ 1304 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1305 NR_VM_NODE_STAT_ITEMS + x) 1306 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold", 1307 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold", 1308 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages", 1309 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages", 1310 #undef I 1311 1312 #if defined(CONFIG_VM_EVENT_COUNTERS) 1313 /* enum vm_event_item counters */ 1314 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1315 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x) 1316 1317 [I(PGPGIN)] = "pgpgin", 1318 [I(PGPGOUT)] = "pgpgout", 1319 [I(PSWPIN)] = "pswpin", 1320 [I(PSWPOUT)] = "pswpout", 1321 1322 #define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1323 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS) 1324 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc") 1325 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall") 1326 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip") 1327 #undef OFF 1328 1329 [I(PGFREE)] = "pgfree", 1330 [I(PGACTIVATE)] = "pgactivate", 1331 [I(PGDEACTIVATE)] = "pgdeactivate", 1332 [I(PGLAZYFREE)] = "pglazyfree", 1333 1334 [I(PGFAULT)] = "pgfault", 1335 [I(PGMAJFAULT)] = "pgmajfault", 1336 [I(PGLAZYFREED)] = "pglazyfreed", 1337 1338 [I(PGREUSE)] = "pgreuse", 1339 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle", 1340 1341 #ifdef CONFIG_NUMA 1342 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success", 1343 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed", 1344 #endif 1345 [I(PGINODESTEAL)] = "pginodesteal", 1346 [I(SLABS_SCANNED)] = "slabs_scanned", 1347 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal", 1348 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly", 1349 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly", 1350 [I(PAGEOUTRUN)] = "pageoutrun", 1351 1352 [I(PGROTATED)] = "pgrotated", 1353 1354 [I(DROP_PAGECACHE)] = "drop_pagecache", 1355 [I(DROP_SLAB)] = "drop_slab", 1356 [I(OOM_KILL)] = "oom_kill", 1357 1358 #ifdef CONFIG_NUMA_BALANCING 1359 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates", 1360 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates", 1361 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults", 1362 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local", 1363 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated", 1364 #endif 1365 #ifdef CONFIG_MIGRATION 1366 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success", 1367 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail", 1368 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success", 1369 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail", 1370 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split", 1371 #endif 1372 #ifdef CONFIG_COMPACTION 1373 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned", 1374 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned", 1375 [I(COMPACTISOLATED)] = "compact_isolated", 1376 [I(COMPACTSTALL)] = "compact_stall", 1377 [I(COMPACTFAIL)] = "compact_fail", 1378 [I(COMPACTSUCCESS)] = "compact_success", 1379 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake", 1380 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned", 1381 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned", 1382 #endif 1383 1384 #ifdef CONFIG_HUGETLB_PAGE 1385 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success", 1386 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail", 1387 #endif 1388 #ifdef CONFIG_CMA 1389 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success", 1390 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail", 1391 #endif 1392 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled", 1393 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned", 1394 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued", 1395 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked", 1396 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked", 1397 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared", 1398 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded", 1399 1400 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1401 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc", 1402 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback", 1403 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge", 1404 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc", 1405 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed", 1406 [I(THP_FILE_ALLOC)] = "thp_file_alloc", 1407 [I(THP_FILE_FALLBACK)] = "thp_file_fallback", 1408 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge", 1409 [I(THP_FILE_MAPPED)] = "thp_file_mapped", 1410 [I(THP_SPLIT_PAGE)] = "thp_split_page", 1411 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed", 1412 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page", 1413 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page", 1414 [I(THP_SPLIT_PMD)] = "thp_split_pmd", 1415 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte", 1416 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte", 1417 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte", 1418 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1419 [I(THP_SPLIT_PUD)] = "thp_split_pud", 1420 #endif 1421 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc", 1422 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed", 1423 [I(THP_SWPOUT)] = "thp_swpout", 1424 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback", 1425 #endif 1426 #ifdef CONFIG_BALLOON 1427 [I(BALLOON_INFLATE)] = "balloon_inflate", 1428 [I(BALLOON_DEFLATE)] = "balloon_deflate", 1429 #ifdef CONFIG_BALLOON_MIGRATION 1430 [I(BALLOON_MIGRATE)] = "balloon_migrate", 1431 #endif /* CONFIG_BALLOON_MIGRATION */ 1432 #endif /* CONFIG_BALLOON */ 1433 #ifdef CONFIG_DEBUG_TLBFLUSH 1434 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush", 1435 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received", 1436 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all", 1437 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one", 1438 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1439 1440 #ifdef CONFIG_SWAP 1441 [I(SWAP_RA)] = "swap_ra", 1442 [I(SWAP_RA_HIT)] = "swap_ra_hit", 1443 [I(SWPIN_ZERO)] = "swpin_zero", 1444 [I(SWPOUT_ZERO)] = "swpout_zero", 1445 #ifdef CONFIG_KSM 1446 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy", 1447 #endif 1448 #endif 1449 #ifdef CONFIG_KSM 1450 [I(COW_KSM)] = "cow_ksm", 1451 #endif 1452 #ifdef CONFIG_ZSWAP 1453 [I(ZSWPIN)] = "zswpin", 1454 [I(ZSWPOUT)] = "zswpout", 1455 [I(ZSWPWB)] = "zswpwb", 1456 #endif 1457 #ifdef CONFIG_X86 1458 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits", 1459 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits", 1460 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses", 1461 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses", 1462 #endif 1463 #ifdef CONFIG_PER_VMA_LOCK_STATS 1464 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success", 1465 [I(VMA_LOCK_ABORT)] = "vma_lock_abort", 1466 [I(VMA_LOCK_RETRY)] = "vma_lock_retry", 1467 [I(VMA_LOCK_MISS)] = "vma_lock_miss", 1468 #endif 1469 #ifdef CONFIG_DEBUG_STACK_USAGE 1470 [I(KSTACK_1K)] = "kstack_1k", 1471 #if THREAD_SIZE > 1024 1472 [I(KSTACK_2K)] = "kstack_2k", 1473 #endif 1474 #if THREAD_SIZE > 2048 1475 [I(KSTACK_4K)] = "kstack_4k", 1476 #endif 1477 #if THREAD_SIZE > 4096 1478 [I(KSTACK_8K)] = "kstack_8k", 1479 #endif 1480 #if THREAD_SIZE > 8192 1481 [I(KSTACK_16K)] = "kstack_16k", 1482 #endif 1483 #if THREAD_SIZE > 16384 1484 [I(KSTACK_32K)] = "kstack_32k", 1485 #endif 1486 #if THREAD_SIZE > 32768 1487 [I(KSTACK_64K)] = "kstack_64k", 1488 #endif 1489 #if THREAD_SIZE > 65536 1490 [I(KSTACK_REST)] = "kstack_rest", 1491 #endif 1492 #endif 1493 #undef I 1494 #endif /* CONFIG_VM_EVENT_COUNTERS */ 1495 }; 1496 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1497 1498 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1499 defined(CONFIG_PROC_FS) 1500 static void *frag_start(struct seq_file *m, loff_t *pos) 1501 { 1502 pg_data_t *pgdat; 1503 loff_t node = *pos; 1504 1505 for (pgdat = first_online_pgdat(); 1506 pgdat && node; 1507 pgdat = next_online_pgdat(pgdat)) 1508 --node; 1509 1510 return pgdat; 1511 } 1512 1513 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1514 { 1515 pg_data_t *pgdat = (pg_data_t *)arg; 1516 1517 (*pos)++; 1518 return next_online_pgdat(pgdat); 1519 } 1520 1521 static void frag_stop(struct seq_file *m, void *arg) 1522 { 1523 } 1524 1525 /* 1526 * Walk zones in a node and print using a callback. 1527 * If @assert_populated is true, only use callback for zones that are populated. 1528 */ 1529 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1530 bool assert_populated, bool nolock, 1531 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1532 { 1533 struct zone *zone; 1534 struct zone *node_zones = pgdat->node_zones; 1535 unsigned long flags; 1536 1537 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1538 if (assert_populated && !populated_zone(zone)) 1539 continue; 1540 1541 if (!nolock) 1542 spin_lock_irqsave(&zone->lock, flags); 1543 print(m, pgdat, zone); 1544 if (!nolock) 1545 spin_unlock_irqrestore(&zone->lock, flags); 1546 } 1547 } 1548 #endif 1549 1550 #ifdef CONFIG_PROC_FS 1551 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1552 struct zone *zone) 1553 { 1554 int order; 1555 1556 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1557 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1558 /* 1559 * Access to nr_free is lockless as nr_free is used only for 1560 * printing purposes. Use data_race to avoid KCSAN warning. 1561 */ 1562 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1563 seq_putc(m, '\n'); 1564 } 1565 1566 /* 1567 * This walks the free areas for each zone. 1568 */ 1569 static int frag_show(struct seq_file *m, void *arg) 1570 { 1571 pg_data_t *pgdat = (pg_data_t *)arg; 1572 walk_zones_in_node(m, pgdat, true, true, frag_show_print); 1573 return 0; 1574 } 1575 1576 static void pagetypeinfo_showfree_print(struct seq_file *m, 1577 pg_data_t *pgdat, struct zone *zone) 1578 { 1579 int order, mtype; 1580 1581 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1582 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1583 pgdat->node_id, 1584 zone->name, 1585 migratetype_names[mtype]); 1586 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1587 unsigned long freecount = 0; 1588 struct free_area *area; 1589 struct list_head *curr; 1590 bool overflow = false; 1591 1592 area = &(zone->free_area[order]); 1593 1594 list_for_each(curr, &area->free_list[mtype]) { 1595 /* 1596 * Cap the free_list iteration because it might 1597 * be really large and we are under a spinlock 1598 * so a long time spent here could trigger a 1599 * hard lockup detector. Anyway this is a 1600 * debugging tool so knowing there is a handful 1601 * of pages of this order should be more than 1602 * sufficient. 1603 */ 1604 if (++freecount >= 100000) { 1605 overflow = true; 1606 break; 1607 } 1608 } 1609 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1610 spin_unlock_irq(&zone->lock); 1611 cond_resched(); 1612 spin_lock_irq(&zone->lock); 1613 } 1614 seq_putc(m, '\n'); 1615 } 1616 } 1617 1618 /* Print out the free pages at each order for each migratetype */ 1619 static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1620 { 1621 int order; 1622 pg_data_t *pgdat = (pg_data_t *)arg; 1623 1624 /* Print header */ 1625 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1626 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1627 seq_printf(m, "%6d ", order); 1628 seq_putc(m, '\n'); 1629 1630 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1631 } 1632 1633 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1634 pg_data_t *pgdat, struct zone *zone) 1635 { 1636 int mtype; 1637 unsigned long pfn; 1638 unsigned long start_pfn = zone->zone_start_pfn; 1639 unsigned long end_pfn = zone_end_pfn(zone); 1640 unsigned long count[MIGRATE_TYPES] = { 0, }; 1641 1642 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1643 struct page *page; 1644 1645 page = pfn_to_online_page(pfn); 1646 if (!page) 1647 continue; 1648 1649 if (page_zone(page) != zone) 1650 continue; 1651 1652 mtype = get_pageblock_migratetype(page); 1653 1654 if (mtype < MIGRATE_TYPES) 1655 count[mtype]++; 1656 } 1657 1658 /* Print counts */ 1659 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1660 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1661 seq_printf(m, "%12lu ", count[mtype]); 1662 seq_putc(m, '\n'); 1663 } 1664 1665 /* Print out the number of pageblocks for each migratetype */ 1666 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1667 { 1668 int mtype; 1669 pg_data_t *pgdat = (pg_data_t *)arg; 1670 1671 seq_printf(m, "\n%-23s", "Number of blocks type "); 1672 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1673 seq_printf(m, "%12s ", migratetype_names[mtype]); 1674 seq_putc(m, '\n'); 1675 walk_zones_in_node(m, pgdat, true, false, 1676 pagetypeinfo_showblockcount_print); 1677 } 1678 1679 /* 1680 * Print out the number of pageblocks for each migratetype that contain pages 1681 * of other types. This gives an indication of how well fallbacks are being 1682 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1683 * to determine what is going on 1684 */ 1685 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1686 { 1687 #ifdef CONFIG_PAGE_OWNER 1688 int mtype; 1689 1690 if (!static_branch_unlikely(&page_owner_inited)) 1691 return; 1692 1693 drain_all_pages(NULL); 1694 1695 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1696 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1697 seq_printf(m, "%12s ", migratetype_names[mtype]); 1698 seq_putc(m, '\n'); 1699 1700 walk_zones_in_node(m, pgdat, true, true, 1701 pagetypeinfo_showmixedcount_print); 1702 #endif /* CONFIG_PAGE_OWNER */ 1703 } 1704 1705 /* 1706 * This prints out statistics in relation to grouping pages by mobility. 1707 * It is expensive to collect so do not constantly read the file. 1708 */ 1709 static int pagetypeinfo_show(struct seq_file *m, void *arg) 1710 { 1711 pg_data_t *pgdat = (pg_data_t *)arg; 1712 1713 /* check memoryless node */ 1714 if (!node_state(pgdat->node_id, N_MEMORY)) 1715 return 0; 1716 1717 seq_printf(m, "Page block order: %d\n", pageblock_order); 1718 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1719 seq_putc(m, '\n'); 1720 pagetypeinfo_showfree(m, pgdat); 1721 pagetypeinfo_showblockcount(m, pgdat); 1722 pagetypeinfo_showmixedcount(m, pgdat); 1723 1724 return 0; 1725 } 1726 1727 static const struct seq_operations fragmentation_op = { 1728 .start = frag_start, 1729 .next = frag_next, 1730 .stop = frag_stop, 1731 .show = frag_show, 1732 }; 1733 1734 static const struct seq_operations pagetypeinfo_op = { 1735 .start = frag_start, 1736 .next = frag_next, 1737 .stop = frag_stop, 1738 .show = pagetypeinfo_show, 1739 }; 1740 1741 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1742 { 1743 int zid; 1744 1745 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1746 struct zone *compare = &pgdat->node_zones[zid]; 1747 1748 if (populated_zone(compare)) 1749 return zone == compare; 1750 } 1751 1752 return false; 1753 } 1754 1755 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1756 struct zone *zone) 1757 { 1758 int i; 1759 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1760 if (is_zone_first_populated(pgdat, zone)) { 1761 seq_printf(m, "\n per-node stats"); 1762 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1763 unsigned long pages = node_page_state_pages(pgdat, i); 1764 1765 if (vmstat_item_print_in_thp(i)) 1766 pages /= HPAGE_PMD_NR; 1767 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1768 pages); 1769 } 1770 } 1771 seq_printf(m, 1772 "\n pages free %lu" 1773 "\n boost %lu" 1774 "\n min %lu" 1775 "\n low %lu" 1776 "\n high %lu" 1777 "\n promo %lu" 1778 "\n spanned %lu" 1779 "\n present %lu" 1780 "\n managed %lu" 1781 "\n cma %lu", 1782 zone_page_state(zone, NR_FREE_PAGES), 1783 zone->watermark_boost, 1784 min_wmark_pages(zone), 1785 low_wmark_pages(zone), 1786 high_wmark_pages(zone), 1787 promo_wmark_pages(zone), 1788 zone->spanned_pages, 1789 zone->present_pages, 1790 zone_managed_pages(zone), 1791 zone_cma_pages(zone)); 1792 1793 seq_printf(m, 1794 "\n protection: (%ld", 1795 zone->lowmem_reserve[0]); 1796 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1797 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1798 seq_putc(m, ')'); 1799 1800 /* If unpopulated, no other information is useful */ 1801 if (!populated_zone(zone)) { 1802 seq_putc(m, '\n'); 1803 return; 1804 } 1805 1806 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1807 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1808 zone_page_state(zone, i)); 1809 1810 #ifdef CONFIG_NUMA 1811 fold_vm_zone_numa_events(zone); 1812 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1813 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1814 zone_numa_event_state(zone, i)); 1815 #endif 1816 1817 seq_printf(m, "\n pagesets"); 1818 for_each_online_cpu(i) { 1819 struct per_cpu_pages *pcp; 1820 struct per_cpu_zonestat __maybe_unused *pzstats; 1821 1822 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1823 seq_printf(m, 1824 "\n cpu: %i" 1825 "\n count: %i" 1826 "\n high: %i" 1827 "\n batch: %i" 1828 "\n high_min: %i" 1829 "\n high_max: %i", 1830 i, 1831 pcp->count, 1832 pcp->high, 1833 pcp->batch, 1834 pcp->high_min, 1835 pcp->high_max); 1836 #ifdef CONFIG_SMP 1837 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1838 seq_printf(m, "\n vm stats threshold: %d", 1839 pzstats->stat_threshold); 1840 #endif 1841 } 1842 seq_printf(m, 1843 "\n node_unreclaimable: %u" 1844 "\n start_pfn: %lu" 1845 "\n reserved_highatomic: %lu" 1846 "\n free_highatomic: %lu", 1847 kswapd_test_hopeless(pgdat), 1848 zone->zone_start_pfn, 1849 zone->nr_reserved_highatomic, 1850 zone->nr_free_highatomic); 1851 seq_putc(m, '\n'); 1852 } 1853 1854 /* 1855 * Output information about zones in @pgdat. All zones are printed regardless 1856 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1857 * set of all zones and userspace would not be aware of such zones if they are 1858 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1859 */ 1860 static int zoneinfo_show(struct seq_file *m, void *arg) 1861 { 1862 pg_data_t *pgdat = (pg_data_t *)arg; 1863 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1864 return 0; 1865 } 1866 1867 static const struct seq_operations zoneinfo_op = { 1868 .start = frag_start, /* iterate over all zones. The same as in 1869 * fragmentation. */ 1870 .next = frag_next, 1871 .stop = frag_stop, 1872 .show = zoneinfo_show, 1873 }; 1874 1875 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1876 NR_VM_NUMA_EVENT_ITEMS + \ 1877 NR_VM_NODE_STAT_ITEMS + \ 1878 NR_VM_STAT_ITEMS + \ 1879 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1880 NR_VM_EVENT_ITEMS : 0)) 1881 1882 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1883 { 1884 unsigned long *v; 1885 int i; 1886 1887 if (*pos >= NR_VMSTAT_ITEMS) 1888 return NULL; 1889 1890 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS); 1891 fold_vm_numa_events(); 1892 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1893 m->private = v; 1894 if (!v) 1895 return ERR_PTR(-ENOMEM); 1896 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1897 v[i] = global_zone_page_state(i); 1898 v += NR_VM_ZONE_STAT_ITEMS; 1899 1900 #ifdef CONFIG_NUMA 1901 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1902 v[i] = global_numa_event_state(i); 1903 v += NR_VM_NUMA_EVENT_ITEMS; 1904 #endif 1905 1906 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1907 v[i] = global_node_page_state_pages(i); 1908 if (vmstat_item_print_in_thp(i)) 1909 v[i] /= HPAGE_PMD_NR; 1910 } 1911 v += NR_VM_NODE_STAT_ITEMS; 1912 1913 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1914 v + NR_DIRTY_THRESHOLD); 1915 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); 1916 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); 1917 v += NR_VM_STAT_ITEMS; 1918 1919 #ifdef CONFIG_VM_EVENT_COUNTERS 1920 all_vm_events(v); 1921 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1922 v[PGPGOUT] /= 2; 1923 #endif 1924 return (unsigned long *)m->private + *pos; 1925 } 1926 1927 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1928 { 1929 (*pos)++; 1930 if (*pos >= NR_VMSTAT_ITEMS) 1931 return NULL; 1932 return (unsigned long *)m->private + *pos; 1933 } 1934 1935 static int vmstat_show(struct seq_file *m, void *arg) 1936 { 1937 unsigned long *l = arg; 1938 unsigned long off = l - (unsigned long *)m->private; 1939 1940 seq_puts(m, vmstat_text[off]); 1941 seq_put_decimal_ull(m, " ", *l); 1942 seq_putc(m, '\n'); 1943 1944 if (off == NR_VMSTAT_ITEMS - 1) { 1945 /* 1946 * We've come to the end - add any deprecated counters to avoid 1947 * breaking userspace which might depend on them being present. 1948 */ 1949 seq_puts(m, "nr_unstable 0\n"); 1950 } 1951 return 0; 1952 } 1953 1954 static void vmstat_stop(struct seq_file *m, void *arg) 1955 { 1956 kfree(m->private); 1957 m->private = NULL; 1958 } 1959 1960 static const struct seq_operations vmstat_op = { 1961 .start = vmstat_start, 1962 .next = vmstat_next, 1963 .stop = vmstat_stop, 1964 .show = vmstat_show, 1965 }; 1966 #endif /* CONFIG_PROC_FS */ 1967 1968 #ifdef CONFIG_SMP 1969 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1970 static int sysctl_stat_interval __read_mostly = HZ; 1971 static int vmstat_late_init_done; 1972 1973 #ifdef CONFIG_PROC_FS 1974 static void refresh_vm_stats(struct work_struct *work) 1975 { 1976 refresh_cpu_vm_stats(true); 1977 } 1978 1979 static int vmstat_refresh(const struct ctl_table *table, int write, 1980 void *buffer, size_t *lenp, loff_t *ppos) 1981 { 1982 long val; 1983 int err; 1984 int i; 1985 1986 /* 1987 * The regular update, every sysctl_stat_interval, may come later 1988 * than expected: leaving a significant amount in per_cpu buckets. 1989 * This is particularly misleading when checking a quantity of HUGE 1990 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 1991 * which can equally be echo'ed to or cat'ted from (by root), 1992 * can be used to update the stats just before reading them. 1993 * 1994 * Oh, and since global_zone_page_state() etc. are so careful to hide 1995 * transiently negative values, report an error here if any of 1996 * the stats is negative, so we know to go looking for imbalance. 1997 */ 1998 err = schedule_on_each_cpu(refresh_vm_stats); 1999 if (err) 2000 return err; 2001 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 2002 /* 2003 * Skip checking stats known to go negative occasionally. 2004 */ 2005 switch (i) { 2006 case NR_ZONE_WRITE_PENDING: 2007 case NR_FREE_CMA_PAGES: 2008 continue; 2009 } 2010 val = atomic_long_read(&vm_zone_stat[i]); 2011 if (val < 0) { 2012 pr_warn("%s: %s %ld\n", 2013 __func__, zone_stat_name(i), val); 2014 } 2015 } 2016 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 2017 /* 2018 * Skip checking stats known to go negative occasionally. 2019 */ 2020 switch (i) { 2021 case NR_WRITEBACK: 2022 continue; 2023 } 2024 val = atomic_long_read(&vm_node_stat[i]); 2025 if (val < 0) { 2026 pr_warn("%s: %s %ld\n", 2027 __func__, node_stat_name(i), val); 2028 } 2029 } 2030 if (write) 2031 *ppos += *lenp; 2032 else 2033 *lenp = 0; 2034 return 0; 2035 } 2036 #endif /* CONFIG_PROC_FS */ 2037 2038 static void vmstat_update(struct work_struct *w) 2039 { 2040 if (refresh_cpu_vm_stats(true)) { 2041 /* 2042 * Counters were updated so we expect more updates 2043 * to occur in the future. Keep on running the 2044 * update worker thread. 2045 */ 2046 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 2047 this_cpu_ptr(&vmstat_work), 2048 round_jiffies_relative(sysctl_stat_interval)); 2049 } 2050 } 2051 2052 /* 2053 * Check if the diffs for a certain cpu indicate that 2054 * an update is needed. 2055 */ 2056 static bool need_update(int cpu) 2057 { 2058 pg_data_t *last_pgdat = NULL; 2059 struct zone *zone; 2060 2061 for_each_populated_zone(zone) { 2062 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 2063 struct per_cpu_nodestat *n; 2064 2065 /* 2066 * The fast way of checking if there are any vmstat diffs. 2067 */ 2068 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 2069 return true; 2070 2071 if (last_pgdat == zone->zone_pgdat) 2072 continue; 2073 last_pgdat = zone->zone_pgdat; 2074 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 2075 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 2076 return true; 2077 } 2078 return false; 2079 } 2080 2081 /* 2082 * Switch off vmstat processing and then fold all the remaining differentials 2083 * until the diffs stay at zero. The function is used by NOHZ and can only be 2084 * invoked when tick processing is not active. 2085 */ 2086 void quiet_vmstat(void) 2087 { 2088 if (system_state != SYSTEM_RUNNING) 2089 return; 2090 2091 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2092 return; 2093 2094 if (!need_update(smp_processor_id())) 2095 return; 2096 2097 /* 2098 * Just refresh counters and do not care about the pending delayed 2099 * vmstat_update. It doesn't fire that often to matter and canceling 2100 * it would be too expensive from this path. 2101 * vmstat_shepherd will take care about that for us. 2102 */ 2103 refresh_cpu_vm_stats(false); 2104 } 2105 2106 /* 2107 * Shepherd worker thread that checks the 2108 * differentials of processors that have their worker 2109 * threads for vm statistics updates disabled because of 2110 * inactivity. 2111 */ 2112 static void vmstat_shepherd(struct work_struct *w); 2113 2114 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2115 2116 void vmstat_flush_workqueue(void) 2117 { 2118 flush_workqueue(mm_percpu_wq); 2119 } 2120 2121 static void vmstat_shepherd(struct work_struct *w) 2122 { 2123 int cpu; 2124 2125 cpus_read_lock(); 2126 /* Check processors whose vmstat worker threads have been disabled */ 2127 for_each_online_cpu(cpu) { 2128 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2129 2130 /* 2131 * In kernel users of vmstat counters either require the precise value and 2132 * they are using zone_page_state_snapshot interface or they can live with 2133 * an imprecision as the regular flushing can happen at arbitrary time and 2134 * cumulative error can grow (see calculate_normal_threshold). 2135 * 2136 * From that POV the regular flushing can be postponed for CPUs that have 2137 * been isolated from the kernel interference without critical 2138 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2139 * for all isolated CPUs to avoid interference with the isolated workload. 2140 */ 2141 scoped_guard(rcu) { 2142 if (cpu_is_isolated(cpu)) 2143 continue; 2144 2145 if (!work_busy(&dw->work) && need_update(cpu)) 2146 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2147 } 2148 2149 cond_resched(); 2150 } 2151 cpus_read_unlock(); 2152 2153 schedule_delayed_work(&shepherd, 2154 round_jiffies_relative(sysctl_stat_interval)); 2155 } 2156 2157 static void __init start_shepherd_timer(void) 2158 { 2159 int cpu; 2160 2161 for_each_possible_cpu(cpu) { 2162 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2163 vmstat_update); 2164 2165 /* 2166 * For secondary CPUs during CPU hotplug scenarios, 2167 * vmstat_cpu_online() will enable the work. 2168 * mm/vmstat:online enables and disables vmstat_work 2169 * symmetrically during CPU hotplug events. 2170 */ 2171 if (!cpu_online(cpu)) 2172 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2173 } 2174 2175 schedule_delayed_work(&shepherd, 2176 round_jiffies_relative(sysctl_stat_interval)); 2177 } 2178 2179 static void __init init_cpu_node_state(void) 2180 { 2181 int node; 2182 2183 for_each_online_node(node) { 2184 if (!cpumask_empty(cpumask_of_node(node))) 2185 node_set_state(node, N_CPU); 2186 } 2187 } 2188 2189 static int vmstat_cpu_online(unsigned int cpu) 2190 { 2191 if (vmstat_late_init_done) 2192 refresh_zone_stat_thresholds(); 2193 2194 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2195 node_set_state(cpu_to_node(cpu), N_CPU); 2196 } 2197 enable_delayed_work(&per_cpu(vmstat_work, cpu)); 2198 2199 return 0; 2200 } 2201 2202 static int vmstat_cpu_down_prep(unsigned int cpu) 2203 { 2204 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2205 return 0; 2206 } 2207 2208 static int vmstat_cpu_dead(unsigned int cpu) 2209 { 2210 const struct cpumask *node_cpus; 2211 int node; 2212 2213 node = cpu_to_node(cpu); 2214 2215 refresh_zone_stat_thresholds(); 2216 node_cpus = cpumask_of_node(node); 2217 if (!cpumask_empty(node_cpus)) 2218 return 0; 2219 2220 node_clear_state(node, N_CPU); 2221 2222 return 0; 2223 } 2224 2225 static int __init vmstat_late_init(void) 2226 { 2227 refresh_zone_stat_thresholds(); 2228 vmstat_late_init_done = 1; 2229 2230 return 0; 2231 } 2232 late_initcall(vmstat_late_init); 2233 #endif 2234 2235 #ifdef CONFIG_PROC_FS 2236 static const struct ctl_table vmstat_table[] = { 2237 #ifdef CONFIG_SMP 2238 { 2239 .procname = "stat_interval", 2240 .data = &sysctl_stat_interval, 2241 .maxlen = sizeof(sysctl_stat_interval), 2242 .mode = 0644, 2243 .proc_handler = proc_dointvec_jiffies, 2244 }, 2245 { 2246 .procname = "stat_refresh", 2247 .data = NULL, 2248 .maxlen = 0, 2249 .mode = 0600, 2250 .proc_handler = vmstat_refresh, 2251 }, 2252 #endif 2253 #ifdef CONFIG_NUMA 2254 { 2255 .procname = "numa_stat", 2256 .data = &sysctl_vm_numa_stat, 2257 .maxlen = sizeof(int), 2258 .mode = 0644, 2259 .proc_handler = sysctl_vm_numa_stat_handler, 2260 .extra1 = SYSCTL_ZERO, 2261 .extra2 = SYSCTL_ONE, 2262 }, 2263 #endif 2264 }; 2265 #endif 2266 2267 struct workqueue_struct *mm_percpu_wq; 2268 2269 void __init init_mm_internals(void) 2270 { 2271 int ret __maybe_unused; 2272 2273 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", 2274 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2275 2276 #ifdef CONFIG_SMP 2277 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2278 NULL, vmstat_cpu_dead); 2279 if (ret < 0) 2280 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2281 2282 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2283 vmstat_cpu_online, 2284 vmstat_cpu_down_prep); 2285 if (ret < 0) 2286 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2287 2288 cpus_read_lock(); 2289 init_cpu_node_state(); 2290 cpus_read_unlock(); 2291 2292 start_shepherd_timer(); 2293 #endif 2294 #ifdef CONFIG_PROC_FS 2295 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2296 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2297 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2298 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2299 register_sysctl_init("vm", vmstat_table); 2300 #endif 2301 } 2302 2303 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2304 2305 /* 2306 * Return an index indicating how much of the available free memory is 2307 * unusable for an allocation of the requested size. 2308 */ 2309 static int unusable_free_index(unsigned int order, 2310 struct contig_page_info *info) 2311 { 2312 /* No free memory is interpreted as all free memory is unusable */ 2313 if (info->free_pages == 0) 2314 return 1000; 2315 2316 /* 2317 * Index should be a value between 0 and 1. Return a value to 3 2318 * decimal places. 2319 * 2320 * 0 => no fragmentation 2321 * 1 => high fragmentation 2322 */ 2323 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2324 2325 } 2326 2327 static void unusable_show_print(struct seq_file *m, 2328 pg_data_t *pgdat, struct zone *zone) 2329 { 2330 unsigned int order; 2331 int index; 2332 struct contig_page_info info; 2333 2334 seq_printf(m, "Node %d, zone %8s ", 2335 pgdat->node_id, 2336 zone->name); 2337 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2338 fill_contig_page_info(zone, order, &info); 2339 index = unusable_free_index(order, &info); 2340 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2341 } 2342 2343 seq_putc(m, '\n'); 2344 } 2345 2346 /* 2347 * Display unusable free space index 2348 * 2349 * The unusable free space index measures how much of the available free 2350 * memory cannot be used to satisfy an allocation of a given size and is a 2351 * value between 0 and 1. The higher the value, the more of free memory is 2352 * unusable and by implication, the worse the external fragmentation is. This 2353 * can be expressed as a percentage by multiplying by 100. 2354 */ 2355 static int unusable_show(struct seq_file *m, void *arg) 2356 { 2357 pg_data_t *pgdat = (pg_data_t *)arg; 2358 2359 /* check memoryless node */ 2360 if (!node_state(pgdat->node_id, N_MEMORY)) 2361 return 0; 2362 2363 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2364 2365 return 0; 2366 } 2367 2368 static const struct seq_operations unusable_sops = { 2369 .start = frag_start, 2370 .next = frag_next, 2371 .stop = frag_stop, 2372 .show = unusable_show, 2373 }; 2374 2375 DEFINE_SEQ_ATTRIBUTE(unusable); 2376 2377 static void extfrag_show_print(struct seq_file *m, 2378 pg_data_t *pgdat, struct zone *zone) 2379 { 2380 unsigned int order; 2381 int index; 2382 2383 /* Alloc on stack as interrupts are disabled for zone walk */ 2384 struct contig_page_info info; 2385 2386 seq_printf(m, "Node %d, zone %8s ", 2387 pgdat->node_id, 2388 zone->name); 2389 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2390 fill_contig_page_info(zone, order, &info); 2391 index = __fragmentation_index(order, &info); 2392 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2393 } 2394 2395 seq_putc(m, '\n'); 2396 } 2397 2398 /* 2399 * Display fragmentation index for orders that allocations would fail for 2400 */ 2401 static int extfrag_show(struct seq_file *m, void *arg) 2402 { 2403 pg_data_t *pgdat = (pg_data_t *)arg; 2404 2405 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2406 2407 return 0; 2408 } 2409 2410 static const struct seq_operations extfrag_sops = { 2411 .start = frag_start, 2412 .next = frag_next, 2413 .stop = frag_stop, 2414 .show = extfrag_show, 2415 }; 2416 2417 DEFINE_SEQ_ATTRIBUTE(extfrag); 2418 2419 static int __init extfrag_debug_init(void) 2420 { 2421 struct dentry *extfrag_debug_root; 2422 2423 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2424 2425 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2426 &unusable_fops); 2427 2428 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2429 &extfrag_fops); 2430 2431 return 0; 2432 } 2433 2434 module_init(extfrag_debug_init); 2435 2436 #endif 2437