1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <cl@gentwo.org> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13 #include <linux/fs.h> 14 #include <linux/mm.h> 15 #include <linux/err.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/cpumask.h> 20 #include <linux/vmstat.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/debugfs.h> 24 #include <linux/sched.h> 25 #include <linux/math64.h> 26 #include <linux/writeback.h> 27 #include <linux/compaction.h> 28 #include <linux/mm_inline.h> 29 #include <linux/page_owner.h> 30 #include <linux/sched/isolation.h> 31 32 #include "internal.h" 33 34 #ifdef CONFIG_PROC_FS 35 #ifdef CONFIG_NUMA 36 #define ENABLE_NUMA_STAT 1 37 static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 38 39 /* zero numa counters within a zone */ 40 static void zero_zone_numa_counters(struct zone *zone) 41 { 42 int item, cpu; 43 44 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 45 atomic_long_set(&zone->vm_numa_event[item], 0); 46 for_each_online_cpu(cpu) { 47 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 48 = 0; 49 } 50 } 51 } 52 53 /* zero numa counters of all the populated zones */ 54 static void zero_zones_numa_counters(void) 55 { 56 struct zone *zone; 57 58 for_each_populated_zone(zone) 59 zero_zone_numa_counters(zone); 60 } 61 62 /* zero global numa counters */ 63 static void zero_global_numa_counters(void) 64 { 65 int item; 66 67 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 68 atomic_long_set(&vm_numa_event[item], 0); 69 } 70 71 static void invalid_numa_statistics(void) 72 { 73 zero_zones_numa_counters(); 74 zero_global_numa_counters(); 75 } 76 77 static DEFINE_MUTEX(vm_numa_stat_lock); 78 79 static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, 80 void *buffer, size_t *length, loff_t *ppos) 81 { 82 int ret, oldval; 83 84 mutex_lock(&vm_numa_stat_lock); 85 if (write) 86 oldval = sysctl_vm_numa_stat; 87 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 88 if (ret || !write) 89 goto out; 90 91 if (oldval == sysctl_vm_numa_stat) 92 goto out; 93 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 94 static_branch_enable(&vm_numa_stat_key); 95 pr_info("enable numa statistics\n"); 96 } else { 97 static_branch_disable(&vm_numa_stat_key); 98 invalid_numa_statistics(); 99 pr_info("disable numa statistics, and clear numa counters\n"); 100 } 101 102 out: 103 mutex_unlock(&vm_numa_stat_lock); 104 return ret; 105 } 106 #endif 107 #endif /* CONFIG_PROC_FS */ 108 109 #ifdef CONFIG_VM_EVENT_COUNTERS 110 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 111 EXPORT_PER_CPU_SYMBOL(vm_event_states); 112 113 static void sum_vm_events(unsigned long *ret) 114 { 115 int cpu; 116 int i; 117 118 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 119 120 for_each_online_cpu(cpu) { 121 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 122 123 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 124 ret[i] += this->event[i]; 125 } 126 } 127 128 /* 129 * Accumulate the vm event counters across all CPUs. 130 * The result is unavoidably approximate - it can change 131 * during and after execution of this function. 132 */ 133 void all_vm_events(unsigned long *ret) 134 { 135 cpus_read_lock(); 136 sum_vm_events(ret); 137 cpus_read_unlock(); 138 } 139 EXPORT_SYMBOL_GPL(all_vm_events); 140 141 /* 142 * Fold the foreign cpu events into our own. 143 * 144 * This is adding to the events on one processor 145 * but keeps the global counts constant. 146 */ 147 void vm_events_fold_cpu(int cpu) 148 { 149 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 150 int i; 151 152 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 153 count_vm_events(i, fold_state->event[i]); 154 fold_state->event[i] = 0; 155 } 156 } 157 158 #endif /* CONFIG_VM_EVENT_COUNTERS */ 159 160 /* 161 * Manage combined zone based / global counters 162 * 163 * vm_stat contains the global counters 164 */ 165 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 166 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 167 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 168 EXPORT_SYMBOL(vm_zone_stat); 169 EXPORT_SYMBOL(vm_node_stat); 170 171 #ifdef CONFIG_NUMA 172 static void fold_vm_zone_numa_events(struct zone *zone) 173 { 174 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 175 int cpu; 176 enum numa_stat_item item; 177 178 for_each_online_cpu(cpu) { 179 struct per_cpu_zonestat *pzstats; 180 181 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 182 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 183 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 184 } 185 186 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 187 zone_numa_event_add(zone_numa_events[item], zone, item); 188 } 189 190 void fold_vm_numa_events(void) 191 { 192 struct zone *zone; 193 194 for_each_populated_zone(zone) 195 fold_vm_zone_numa_events(zone); 196 } 197 #endif 198 199 #ifdef CONFIG_SMP 200 201 int calculate_pressure_threshold(struct zone *zone) 202 { 203 int threshold; 204 int watermark_distance; 205 206 /* 207 * As vmstats are not up to date, there is drift between the estimated 208 * and real values. For high thresholds and a high number of CPUs, it 209 * is possible for the min watermark to be breached while the estimated 210 * value looks fine. The pressure threshold is a reduced value such 211 * that even the maximum amount of drift will not accidentally breach 212 * the min watermark 213 */ 214 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 215 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 216 217 /* 218 * Maximum threshold is 125 219 */ 220 threshold = min(125, threshold); 221 222 return threshold; 223 } 224 225 int calculate_normal_threshold(struct zone *zone) 226 { 227 int threshold; 228 int mem; /* memory in 128 MB units */ 229 230 /* 231 * The threshold scales with the number of processors and the amount 232 * of memory per zone. More memory means that we can defer updates for 233 * longer, more processors could lead to more contention. 234 * fls() is used to have a cheap way of logarithmic scaling. 235 * 236 * Some sample thresholds: 237 * 238 * Threshold Processors (fls) Zonesize fls(mem)+1 239 * ------------------------------------------------------------------ 240 * 8 1 1 0.9-1 GB 4 241 * 16 2 2 0.9-1 GB 4 242 * 20 2 2 1-2 GB 5 243 * 24 2 2 2-4 GB 6 244 * 28 2 2 4-8 GB 7 245 * 32 2 2 8-16 GB 8 246 * 4 2 2 <128M 1 247 * 30 4 3 2-4 GB 5 248 * 48 4 3 8-16 GB 8 249 * 32 8 4 1-2 GB 4 250 * 32 8 4 0.9-1GB 4 251 * 10 16 5 <128M 1 252 * 40 16 5 900M 4 253 * 70 64 7 2-4 GB 5 254 * 84 64 7 4-8 GB 6 255 * 108 512 9 4-8 GB 6 256 * 125 1024 10 8-16 GB 8 257 * 125 1024 10 16-32 GB 9 258 */ 259 260 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 261 262 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 263 264 /* 265 * Maximum threshold is 125 266 */ 267 threshold = min(125, threshold); 268 269 return threshold; 270 } 271 272 /* 273 * Refresh the thresholds for each zone. 274 */ 275 void refresh_zone_stat_thresholds(void) 276 { 277 struct pglist_data *pgdat; 278 struct zone *zone; 279 int cpu; 280 int threshold; 281 282 /* Zero current pgdat thresholds */ 283 for_each_online_pgdat(pgdat) { 284 for_each_online_cpu(cpu) { 285 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 286 } 287 } 288 289 for_each_populated_zone(zone) { 290 struct pglist_data *pgdat = zone->zone_pgdat; 291 unsigned long max_drift, tolerate_drift; 292 293 threshold = calculate_normal_threshold(zone); 294 295 for_each_online_cpu(cpu) { 296 int pgdat_threshold; 297 298 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 299 = threshold; 300 301 /* Base nodestat threshold on the largest populated zone. */ 302 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 303 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 304 = max(threshold, pgdat_threshold); 305 } 306 307 /* 308 * Only set percpu_drift_mark if there is a danger that 309 * NR_FREE_PAGES reports the low watermark is ok when in fact 310 * the min watermark could be breached by an allocation 311 */ 312 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 313 max_drift = num_online_cpus() * threshold; 314 if (max_drift > tolerate_drift) 315 zone->percpu_drift_mark = high_wmark_pages(zone) + 316 max_drift; 317 } 318 } 319 320 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 321 int (*calculate_pressure)(struct zone *)) 322 { 323 struct zone *zone; 324 int cpu; 325 int threshold; 326 int i; 327 328 for (i = 0; i < pgdat->nr_zones; i++) { 329 zone = &pgdat->node_zones[i]; 330 if (!zone->percpu_drift_mark) 331 continue; 332 333 threshold = (*calculate_pressure)(zone); 334 for_each_online_cpu(cpu) 335 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 336 = threshold; 337 } 338 } 339 340 /* 341 * For use when we know that interrupts are disabled, 342 * or when we know that preemption is disabled and that 343 * particular counter cannot be updated from interrupt context. 344 */ 345 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 346 long delta) 347 { 348 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 349 s8 __percpu *p = pcp->vm_stat_diff + item; 350 long x; 351 long t; 352 353 /* 354 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 355 * atomicity is provided by IRQs being disabled -- either explicitly 356 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 357 * CPU migrations and preemption potentially corrupts a counter so 358 * disable preemption. 359 */ 360 preempt_disable_nested(); 361 362 x = delta + __this_cpu_read(*p); 363 364 t = __this_cpu_read(pcp->stat_threshold); 365 366 if (unlikely(abs(x) > t)) { 367 zone_page_state_add(x, zone, item); 368 x = 0; 369 } 370 __this_cpu_write(*p, x); 371 372 preempt_enable_nested(); 373 } 374 EXPORT_SYMBOL(__mod_zone_page_state); 375 376 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 377 long delta) 378 { 379 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 380 s8 __percpu *p = pcp->vm_node_stat_diff + item; 381 long x; 382 long t; 383 384 if (vmstat_item_in_bytes(item)) { 385 /* 386 * Only cgroups use subpage accounting right now; at 387 * the global level, these items still change in 388 * multiples of whole pages. Store them as pages 389 * internally to keep the per-cpu counters compact. 390 */ 391 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 392 delta >>= PAGE_SHIFT; 393 } 394 395 /* See __mod_zone_page_state() */ 396 preempt_disable_nested(); 397 398 x = delta + __this_cpu_read(*p); 399 400 t = __this_cpu_read(pcp->stat_threshold); 401 402 if (unlikely(abs(x) > t)) { 403 node_page_state_add(x, pgdat, item); 404 x = 0; 405 } 406 __this_cpu_write(*p, x); 407 408 preempt_enable_nested(); 409 } 410 EXPORT_SYMBOL(__mod_node_page_state); 411 412 /* 413 * Optimized increment and decrement functions. 414 * 415 * These are only for a single page and therefore can take a struct page * 416 * argument instead of struct zone *. This allows the inclusion of the code 417 * generated for page_zone(page) into the optimized functions. 418 * 419 * No overflow check is necessary and therefore the differential can be 420 * incremented or decremented in place which may allow the compilers to 421 * generate better code. 422 * The increment or decrement is known and therefore one boundary check can 423 * be omitted. 424 * 425 * NOTE: These functions are very performance sensitive. Change only 426 * with care. 427 * 428 * Some processors have inc/dec instructions that are atomic vs an interrupt. 429 * However, the code must first determine the differential location in a zone 430 * based on the processor number and then inc/dec the counter. There is no 431 * guarantee without disabling preemption that the processor will not change 432 * in between and therefore the atomicity vs. interrupt cannot be exploited 433 * in a useful way here. 434 */ 435 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 436 { 437 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 438 s8 __percpu *p = pcp->vm_stat_diff + item; 439 s8 v, t; 440 441 /* See __mod_zone_page_state() */ 442 preempt_disable_nested(); 443 444 v = __this_cpu_inc_return(*p); 445 t = __this_cpu_read(pcp->stat_threshold); 446 if (unlikely(v > t)) { 447 s8 overstep = t >> 1; 448 449 zone_page_state_add(v + overstep, zone, item); 450 __this_cpu_write(*p, -overstep); 451 } 452 453 preempt_enable_nested(); 454 } 455 456 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 457 { 458 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 459 s8 __percpu *p = pcp->vm_node_stat_diff + item; 460 s8 v, t; 461 462 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 463 464 /* See __mod_zone_page_state() */ 465 preempt_disable_nested(); 466 467 v = __this_cpu_inc_return(*p); 468 t = __this_cpu_read(pcp->stat_threshold); 469 if (unlikely(v > t)) { 470 s8 overstep = t >> 1; 471 472 node_page_state_add(v + overstep, pgdat, item); 473 __this_cpu_write(*p, -overstep); 474 } 475 476 preempt_enable_nested(); 477 } 478 479 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 480 { 481 __inc_zone_state(page_zone(page), item); 482 } 483 EXPORT_SYMBOL(__inc_zone_page_state); 484 485 void __inc_node_page_state(struct page *page, enum node_stat_item item) 486 { 487 __inc_node_state(page_pgdat(page), item); 488 } 489 EXPORT_SYMBOL(__inc_node_page_state); 490 491 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 492 { 493 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 494 s8 __percpu *p = pcp->vm_stat_diff + item; 495 s8 v, t; 496 497 /* See __mod_zone_page_state() */ 498 preempt_disable_nested(); 499 500 v = __this_cpu_dec_return(*p); 501 t = __this_cpu_read(pcp->stat_threshold); 502 if (unlikely(v < - t)) { 503 s8 overstep = t >> 1; 504 505 zone_page_state_add(v - overstep, zone, item); 506 __this_cpu_write(*p, overstep); 507 } 508 509 preempt_enable_nested(); 510 } 511 512 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 513 { 514 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 515 s8 __percpu *p = pcp->vm_node_stat_diff + item; 516 s8 v, t; 517 518 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 519 520 /* See __mod_zone_page_state() */ 521 preempt_disable_nested(); 522 523 v = __this_cpu_dec_return(*p); 524 t = __this_cpu_read(pcp->stat_threshold); 525 if (unlikely(v < - t)) { 526 s8 overstep = t >> 1; 527 528 node_page_state_add(v - overstep, pgdat, item); 529 __this_cpu_write(*p, overstep); 530 } 531 532 preempt_enable_nested(); 533 } 534 535 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 536 { 537 __dec_zone_state(page_zone(page), item); 538 } 539 EXPORT_SYMBOL(__dec_zone_page_state); 540 541 void __dec_node_page_state(struct page *page, enum node_stat_item item) 542 { 543 __dec_node_state(page_pgdat(page), item); 544 } 545 EXPORT_SYMBOL(__dec_node_page_state); 546 547 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 548 /* 549 * If we have cmpxchg_local support then we do not need to incur the overhead 550 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 551 * 552 * mod_state() modifies the zone counter state through atomic per cpu 553 * operations. 554 * 555 * Overstep mode specifies how overstep should handled: 556 * 0 No overstepping 557 * 1 Overstepping half of threshold 558 * -1 Overstepping minus half of threshold 559 */ 560 static inline void mod_zone_state(struct zone *zone, 561 enum zone_stat_item item, long delta, int overstep_mode) 562 { 563 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 564 s8 __percpu *p = pcp->vm_stat_diff + item; 565 long n, t, z; 566 s8 o; 567 568 o = this_cpu_read(*p); 569 do { 570 z = 0; /* overflow to zone counters */ 571 572 /* 573 * The fetching of the stat_threshold is racy. We may apply 574 * a counter threshold to the wrong the cpu if we get 575 * rescheduled while executing here. However, the next 576 * counter update will apply the threshold again and 577 * therefore bring the counter under the threshold again. 578 * 579 * Most of the time the thresholds are the same anyways 580 * for all cpus in a zone. 581 */ 582 t = this_cpu_read(pcp->stat_threshold); 583 584 n = delta + (long)o; 585 586 if (abs(n) > t) { 587 int os = overstep_mode * (t >> 1) ; 588 589 /* Overflow must be added to zone counters */ 590 z = n + os; 591 n = -os; 592 } 593 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 594 595 if (z) 596 zone_page_state_add(z, zone, item); 597 } 598 599 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 600 long delta) 601 { 602 mod_zone_state(zone, item, delta, 0); 603 } 604 EXPORT_SYMBOL(mod_zone_page_state); 605 606 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 607 { 608 mod_zone_state(page_zone(page), item, 1, 1); 609 } 610 EXPORT_SYMBOL(inc_zone_page_state); 611 612 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 613 { 614 mod_zone_state(page_zone(page), item, -1, -1); 615 } 616 EXPORT_SYMBOL(dec_zone_page_state); 617 618 static inline void mod_node_state(struct pglist_data *pgdat, 619 enum node_stat_item item, int delta, int overstep_mode) 620 { 621 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 622 s8 __percpu *p = pcp->vm_node_stat_diff + item; 623 long n, t, z; 624 s8 o; 625 626 if (vmstat_item_in_bytes(item)) { 627 /* 628 * Only cgroups use subpage accounting right now; at 629 * the global level, these items still change in 630 * multiples of whole pages. Store them as pages 631 * internally to keep the per-cpu counters compact. 632 */ 633 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 634 delta >>= PAGE_SHIFT; 635 } 636 637 o = this_cpu_read(*p); 638 do { 639 z = 0; /* overflow to node counters */ 640 641 /* 642 * The fetching of the stat_threshold is racy. We may apply 643 * a counter threshold to the wrong the cpu if we get 644 * rescheduled while executing here. However, the next 645 * counter update will apply the threshold again and 646 * therefore bring the counter under the threshold again. 647 * 648 * Most of the time the thresholds are the same anyways 649 * for all cpus in a node. 650 */ 651 t = this_cpu_read(pcp->stat_threshold); 652 653 n = delta + (long)o; 654 655 if (abs(n) > t) { 656 int os = overstep_mode * (t >> 1) ; 657 658 /* Overflow must be added to node counters */ 659 z = n + os; 660 n = -os; 661 } 662 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 663 664 if (z) 665 node_page_state_add(z, pgdat, item); 666 } 667 668 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 669 long delta) 670 { 671 mod_node_state(pgdat, item, delta, 0); 672 } 673 EXPORT_SYMBOL(mod_node_page_state); 674 675 void inc_node_page_state(struct page *page, enum node_stat_item item) 676 { 677 mod_node_state(page_pgdat(page), item, 1, 1); 678 } 679 EXPORT_SYMBOL(inc_node_page_state); 680 681 void dec_node_page_state(struct page *page, enum node_stat_item item) 682 { 683 mod_node_state(page_pgdat(page), item, -1, -1); 684 } 685 EXPORT_SYMBOL(dec_node_page_state); 686 #else 687 /* 688 * Use interrupt disable to serialize counter updates 689 */ 690 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 691 long delta) 692 { 693 unsigned long flags; 694 695 local_irq_save(flags); 696 __mod_zone_page_state(zone, item, delta); 697 local_irq_restore(flags); 698 } 699 EXPORT_SYMBOL(mod_zone_page_state); 700 701 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 702 { 703 unsigned long flags; 704 struct zone *zone; 705 706 zone = page_zone(page); 707 local_irq_save(flags); 708 __inc_zone_state(zone, item); 709 local_irq_restore(flags); 710 } 711 EXPORT_SYMBOL(inc_zone_page_state); 712 713 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 714 { 715 unsigned long flags; 716 717 local_irq_save(flags); 718 __dec_zone_page_state(page, item); 719 local_irq_restore(flags); 720 } 721 EXPORT_SYMBOL(dec_zone_page_state); 722 723 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 724 long delta) 725 { 726 unsigned long flags; 727 728 local_irq_save(flags); 729 __mod_node_page_state(pgdat, item, delta); 730 local_irq_restore(flags); 731 } 732 EXPORT_SYMBOL(mod_node_page_state); 733 734 void inc_node_page_state(struct page *page, enum node_stat_item item) 735 { 736 unsigned long flags; 737 struct pglist_data *pgdat; 738 739 pgdat = page_pgdat(page); 740 local_irq_save(flags); 741 __inc_node_state(pgdat, item); 742 local_irq_restore(flags); 743 } 744 EXPORT_SYMBOL(inc_node_page_state); 745 746 void dec_node_page_state(struct page *page, enum node_stat_item item) 747 { 748 unsigned long flags; 749 750 local_irq_save(flags); 751 __dec_node_page_state(page, item); 752 local_irq_restore(flags); 753 } 754 EXPORT_SYMBOL(dec_node_page_state); 755 #endif 756 757 /* 758 * Fold a differential into the global counters. 759 * Returns whether counters were updated. 760 */ 761 static int fold_diff(int *zone_diff, int *node_diff) 762 { 763 int i; 764 bool changed = false; 765 766 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 767 if (zone_diff[i]) { 768 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 769 changed = true; 770 } 771 } 772 773 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 774 if (node_diff[i]) { 775 atomic_long_add(node_diff[i], &vm_node_stat[i]); 776 changed = true; 777 } 778 } 779 780 return changed; 781 } 782 783 /* 784 * Update the zone counters for the current cpu. 785 * 786 * Note that refresh_cpu_vm_stats strives to only access 787 * node local memory. The per cpu pagesets on remote zones are placed 788 * in the memory local to the processor using that pageset. So the 789 * loop over all zones will access a series of cachelines local to 790 * the processor. 791 * 792 * The call to zone_page_state_add updates the cachelines with the 793 * statistics in the remote zone struct as well as the global cachelines 794 * with the global counters. These could cause remote node cache line 795 * bouncing and will have to be only done when necessary. 796 * 797 * The function returns whether global counters were updated. 798 */ 799 static bool refresh_cpu_vm_stats(bool do_pagesets) 800 { 801 struct pglist_data *pgdat; 802 struct zone *zone; 803 int i; 804 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 805 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 806 bool changed = false; 807 808 for_each_populated_zone(zone) { 809 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 810 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 811 812 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 813 int v; 814 815 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 816 if (v) { 817 818 atomic_long_add(v, &zone->vm_stat[i]); 819 global_zone_diff[i] += v; 820 #ifdef CONFIG_NUMA 821 /* 3 seconds idle till flush */ 822 __this_cpu_write(pcp->expire, 3); 823 #endif 824 } 825 } 826 827 if (do_pagesets) { 828 cond_resched(); 829 830 if (decay_pcp_high(zone, this_cpu_ptr(pcp))) 831 changed = true; 832 #ifdef CONFIG_NUMA 833 /* 834 * Deal with draining the remote pageset of this 835 * processor 836 * 837 * Check if there are pages remaining in this pageset 838 * if not then there is nothing to expire. 839 */ 840 if (!__this_cpu_read(pcp->expire) || 841 !__this_cpu_read(pcp->count)) 842 continue; 843 844 /* 845 * We never drain zones local to this processor. 846 */ 847 if (zone_to_nid(zone) == numa_node_id()) { 848 __this_cpu_write(pcp->expire, 0); 849 continue; 850 } 851 852 if (__this_cpu_dec_return(pcp->expire)) { 853 changed = true; 854 continue; 855 } 856 857 if (__this_cpu_read(pcp->count)) { 858 drain_zone_pages(zone, this_cpu_ptr(pcp)); 859 changed = true; 860 } 861 #endif 862 } 863 } 864 865 for_each_online_pgdat(pgdat) { 866 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 867 868 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 869 int v; 870 871 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 872 if (v) { 873 atomic_long_add(v, &pgdat->vm_stat[i]); 874 global_node_diff[i] += v; 875 } 876 } 877 } 878 879 if (fold_diff(global_zone_diff, global_node_diff)) 880 changed = true; 881 return changed; 882 } 883 884 /* 885 * Fold the data for an offline cpu into the global array. 886 * There cannot be any access by the offline cpu and therefore 887 * synchronization is simplified. 888 */ 889 void cpu_vm_stats_fold(int cpu) 890 { 891 struct pglist_data *pgdat; 892 struct zone *zone; 893 int i; 894 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 895 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 896 897 for_each_populated_zone(zone) { 898 struct per_cpu_zonestat *pzstats; 899 900 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 901 902 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 903 if (pzstats->vm_stat_diff[i]) { 904 int v; 905 906 v = pzstats->vm_stat_diff[i]; 907 pzstats->vm_stat_diff[i] = 0; 908 atomic_long_add(v, &zone->vm_stat[i]); 909 global_zone_diff[i] += v; 910 } 911 } 912 #ifdef CONFIG_NUMA 913 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 914 if (pzstats->vm_numa_event[i]) { 915 unsigned long v; 916 917 v = pzstats->vm_numa_event[i]; 918 pzstats->vm_numa_event[i] = 0; 919 zone_numa_event_add(v, zone, i); 920 } 921 } 922 #endif 923 } 924 925 for_each_online_pgdat(pgdat) { 926 struct per_cpu_nodestat *p; 927 928 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 929 930 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 931 if (p->vm_node_stat_diff[i]) { 932 int v; 933 934 v = p->vm_node_stat_diff[i]; 935 p->vm_node_stat_diff[i] = 0; 936 atomic_long_add(v, &pgdat->vm_stat[i]); 937 global_node_diff[i] += v; 938 } 939 } 940 941 fold_diff(global_zone_diff, global_node_diff); 942 } 943 944 /* 945 * this is only called if !populated_zone(zone), which implies no other users of 946 * pset->vm_stat_diff[] exist. 947 */ 948 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 949 { 950 unsigned long v; 951 int i; 952 953 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 954 if (pzstats->vm_stat_diff[i]) { 955 v = pzstats->vm_stat_diff[i]; 956 pzstats->vm_stat_diff[i] = 0; 957 zone_page_state_add(v, zone, i); 958 } 959 } 960 961 #ifdef CONFIG_NUMA 962 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 963 if (pzstats->vm_numa_event[i]) { 964 v = pzstats->vm_numa_event[i]; 965 pzstats->vm_numa_event[i] = 0; 966 zone_numa_event_add(v, zone, i); 967 } 968 } 969 #endif 970 } 971 #endif 972 973 #ifdef CONFIG_NUMA 974 /* 975 * Determine the per node value of a stat item. This function 976 * is called frequently in a NUMA machine, so try to be as 977 * frugal as possible. 978 */ 979 unsigned long sum_zone_node_page_state(int node, 980 enum zone_stat_item item) 981 { 982 struct zone *zones = NODE_DATA(node)->node_zones; 983 int i; 984 unsigned long count = 0; 985 986 for (i = 0; i < MAX_NR_ZONES; i++) 987 count += zone_page_state(zones + i, item); 988 989 return count; 990 } 991 992 /* Determine the per node value of a numa stat item. */ 993 unsigned long sum_zone_numa_event_state(int node, 994 enum numa_stat_item item) 995 { 996 struct zone *zones = NODE_DATA(node)->node_zones; 997 unsigned long count = 0; 998 int i; 999 1000 for (i = 0; i < MAX_NR_ZONES; i++) 1001 count += zone_numa_event_state(zones + i, item); 1002 1003 return count; 1004 } 1005 1006 /* 1007 * Determine the per node value of a stat item. 1008 */ 1009 unsigned long node_page_state_pages(struct pglist_data *pgdat, 1010 enum node_stat_item item) 1011 { 1012 long x = atomic_long_read(&pgdat->vm_stat[item]); 1013 #ifdef CONFIG_SMP 1014 if (x < 0) 1015 x = 0; 1016 #endif 1017 return x; 1018 } 1019 1020 unsigned long node_page_state(struct pglist_data *pgdat, 1021 enum node_stat_item item) 1022 { 1023 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1024 1025 return node_page_state_pages(pgdat, item); 1026 } 1027 #endif 1028 1029 /* 1030 * Count number of pages "struct page" and "struct page_ext" consume. 1031 * nr_memmap_boot_pages: # of pages allocated by boot allocator 1032 * nr_memmap_pages: # of pages that were allocated by buddy allocator 1033 */ 1034 static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); 1035 static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); 1036 1037 void memmap_boot_pages_add(long delta) 1038 { 1039 atomic_long_add(delta, &nr_memmap_boot_pages); 1040 } 1041 1042 void memmap_pages_add(long delta) 1043 { 1044 atomic_long_add(delta, &nr_memmap_pages); 1045 } 1046 1047 #ifdef CONFIG_COMPACTION 1048 1049 struct contig_page_info { 1050 unsigned long free_pages; 1051 unsigned long free_blocks_total; 1052 unsigned long free_blocks_suitable; 1053 }; 1054 1055 /* 1056 * Calculate the number of free pages in a zone, how many contiguous 1057 * pages are free and how many are large enough to satisfy an allocation of 1058 * the target size. Note that this function makes no attempt to estimate 1059 * how many suitable free blocks there *might* be if MOVABLE pages were 1060 * migrated. Calculating that is possible, but expensive and can be 1061 * figured out from userspace 1062 */ 1063 static void fill_contig_page_info(struct zone *zone, 1064 unsigned int suitable_order, 1065 struct contig_page_info *info) 1066 { 1067 unsigned int order; 1068 1069 info->free_pages = 0; 1070 info->free_blocks_total = 0; 1071 info->free_blocks_suitable = 0; 1072 1073 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1074 unsigned long blocks; 1075 1076 /* 1077 * Count number of free blocks. 1078 * 1079 * Access to nr_free is lockless as nr_free is used only for 1080 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1081 */ 1082 blocks = data_race(zone->free_area[order].nr_free); 1083 info->free_blocks_total += blocks; 1084 1085 /* Count free base pages */ 1086 info->free_pages += blocks << order; 1087 1088 /* Count the suitable free blocks */ 1089 if (order >= suitable_order) 1090 info->free_blocks_suitable += blocks << 1091 (order - suitable_order); 1092 } 1093 } 1094 1095 /* 1096 * A fragmentation index only makes sense if an allocation of a requested 1097 * size would fail. If that is true, the fragmentation index indicates 1098 * whether external fragmentation or a lack of memory was the problem. 1099 * The value can be used to determine if page reclaim or compaction 1100 * should be used 1101 */ 1102 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1103 { 1104 unsigned long requested = 1UL << order; 1105 1106 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1107 return 0; 1108 1109 if (!info->free_blocks_total) 1110 return 0; 1111 1112 /* Fragmentation index only makes sense when a request would fail */ 1113 if (info->free_blocks_suitable) 1114 return -1000; 1115 1116 /* 1117 * Index is between 0 and 1 so return within 3 decimal places 1118 * 1119 * 0 => allocation would fail due to lack of memory 1120 * 1 => allocation would fail due to fragmentation 1121 */ 1122 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1123 } 1124 1125 /* 1126 * Calculates external fragmentation within a zone wrt the given order. 1127 * It is defined as the percentage of pages found in blocks of size 1128 * less than 1 << order. It returns values in range [0, 100]. 1129 */ 1130 unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1131 { 1132 struct contig_page_info info; 1133 1134 fill_contig_page_info(zone, order, &info); 1135 if (info.free_pages == 0) 1136 return 0; 1137 1138 return div_u64((info.free_pages - 1139 (info.free_blocks_suitable << order)) * 100, 1140 info.free_pages); 1141 } 1142 1143 /* Same as __fragmentation index but allocs contig_page_info on stack */ 1144 int fragmentation_index(struct zone *zone, unsigned int order) 1145 { 1146 struct contig_page_info info; 1147 1148 fill_contig_page_info(zone, order, &info); 1149 return __fragmentation_index(order, &info); 1150 } 1151 #endif 1152 1153 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1154 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1155 #ifdef CONFIG_ZONE_DMA 1156 #define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma", 1157 #else 1158 #define TEXT_FOR_DMA(xx, yy) 1159 #endif 1160 1161 #ifdef CONFIG_ZONE_DMA32 1162 #define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32", 1163 #else 1164 #define TEXT_FOR_DMA32(xx, yy) 1165 #endif 1166 1167 #ifdef CONFIG_HIGHMEM 1168 #define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high", 1169 #else 1170 #define TEXT_FOR_HIGHMEM(xx, yy) 1171 #endif 1172 1173 #ifdef CONFIG_ZONE_DEVICE 1174 #define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device", 1175 #else 1176 #define TEXT_FOR_DEVICE(xx, yy) 1177 #endif 1178 1179 #define TEXTS_FOR_ZONES(xx, yy) \ 1180 TEXT_FOR_DMA(xx, yy) \ 1181 TEXT_FOR_DMA32(xx, yy) \ 1182 [xx##_NORMAL] = yy "_normal", \ 1183 TEXT_FOR_HIGHMEM(xx, yy) \ 1184 [xx##_MOVABLE] = yy "_movable", \ 1185 TEXT_FOR_DEVICE(xx, yy) 1186 1187 const char * const vmstat_text[] = { 1188 /* enum zone_stat_item counters */ 1189 #define I(x) (x) 1190 [I(NR_FREE_PAGES)] = "nr_free_pages", 1191 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks", 1192 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon", 1193 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon", 1194 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file", 1195 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file", 1196 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable", 1197 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending", 1198 [I(NR_MLOCK)] = "nr_mlock", 1199 #if IS_ENABLED(CONFIG_ZSMALLOC) 1200 [I(NR_ZSPAGES)] = "nr_zspages", 1201 #endif 1202 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma", 1203 #ifdef CONFIG_UNACCEPTED_MEMORY 1204 [I(NR_UNACCEPTED)] = "nr_unaccepted", 1205 #endif 1206 #undef I 1207 1208 /* enum numa_stat_item counters */ 1209 #define I(x) (NR_VM_ZONE_STAT_ITEMS + x) 1210 #ifdef CONFIG_NUMA 1211 [I(NUMA_HIT)] = "numa_hit", 1212 [I(NUMA_MISS)] = "numa_miss", 1213 [I(NUMA_FOREIGN)] = "numa_foreign", 1214 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave", 1215 [I(NUMA_LOCAL)] = "numa_local", 1216 [I(NUMA_OTHER)] = "numa_other", 1217 #endif 1218 #undef I 1219 1220 /* enum node_stat_item counters */ 1221 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x) 1222 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon", 1223 [I(NR_ACTIVE_ANON)] = "nr_active_anon", 1224 [I(NR_INACTIVE_FILE)] = "nr_inactive_file", 1225 [I(NR_ACTIVE_FILE)] = "nr_active_file", 1226 [I(NR_UNEVICTABLE)] = "nr_unevictable", 1227 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable", 1228 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable", 1229 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon", 1230 [I(NR_ISOLATED_FILE)] = "nr_isolated_file", 1231 [I(WORKINGSET_NODES)] = "workingset_nodes", 1232 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon", 1233 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file", 1234 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon", 1235 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file", 1236 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon", 1237 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file", 1238 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim", 1239 [I(NR_ANON_MAPPED)] = "nr_anon_pages", 1240 [I(NR_FILE_MAPPED)] = "nr_mapped", 1241 [I(NR_FILE_PAGES)] = "nr_file_pages", 1242 [I(NR_FILE_DIRTY)] = "nr_dirty", 1243 [I(NR_WRITEBACK)] = "nr_writeback", 1244 [I(NR_SHMEM)] = "nr_shmem", 1245 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages", 1246 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped", 1247 [I(NR_FILE_THPS)] = "nr_file_hugepages", 1248 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped", 1249 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages", 1250 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write", 1251 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim", 1252 [I(NR_DIRTIED)] = "nr_dirtied", 1253 [I(NR_WRITTEN)] = "nr_written", 1254 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written", 1255 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable", 1256 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired", 1257 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released", 1258 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack", 1259 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1260 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack", 1261 #endif 1262 [I(NR_PAGETABLE)] = "nr_page_table_pages", 1263 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages", 1264 #ifdef CONFIG_IOMMU_SUPPORT 1265 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages", 1266 #endif 1267 #ifdef CONFIG_SWAP 1268 [I(NR_SWAPCACHE)] = "nr_swapcached", 1269 #endif 1270 #ifdef CONFIG_NUMA_BALANCING 1271 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success", 1272 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate", 1273 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl", 1274 #endif 1275 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd", 1276 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct", 1277 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged", 1278 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive", 1279 #ifdef CONFIG_HUGETLB_PAGE 1280 [I(NR_HUGETLB)] = "nr_hugetlb", 1281 #endif 1282 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages", 1283 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages", 1284 #undef I 1285 1286 /* system-wide enum vm_stat_item counters */ 1287 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1288 NR_VM_NODE_STAT_ITEMS + x) 1289 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold", 1290 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold", 1291 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages", 1292 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages", 1293 #undef I 1294 1295 #if defined(CONFIG_VM_EVENT_COUNTERS) 1296 /* enum vm_event_item counters */ 1297 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1298 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x) 1299 1300 [I(PGPGIN)] = "pgpgin", 1301 [I(PGPGOUT)] = "pgpgout", 1302 [I(PSWPIN)] = "pswpin", 1303 [I(PSWPOUT)] = "pswpout", 1304 1305 #define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1306 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS) 1307 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc") 1308 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall") 1309 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip") 1310 #undef OFF 1311 1312 [I(PGFREE)] = "pgfree", 1313 [I(PGACTIVATE)] = "pgactivate", 1314 [I(PGDEACTIVATE)] = "pgdeactivate", 1315 [I(PGLAZYFREE)] = "pglazyfree", 1316 1317 [I(PGFAULT)] = "pgfault", 1318 [I(PGMAJFAULT)] = "pgmajfault", 1319 [I(PGLAZYFREED)] = "pglazyfreed", 1320 1321 [I(PGREFILL)] = "pgrefill", 1322 [I(PGREUSE)] = "pgreuse", 1323 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd", 1324 [I(PGSTEAL_DIRECT)] = "pgsteal_direct", 1325 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged", 1326 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive", 1327 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd", 1328 [I(PGSCAN_DIRECT)] = "pgscan_direct", 1329 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged", 1330 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", 1331 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle", 1332 [I(PGSCAN_ANON)] = "pgscan_anon", 1333 [I(PGSCAN_FILE)] = "pgscan_file", 1334 [I(PGSTEAL_ANON)] = "pgsteal_anon", 1335 [I(PGSTEAL_FILE)] = "pgsteal_file", 1336 1337 #ifdef CONFIG_NUMA 1338 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success", 1339 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed", 1340 #endif 1341 [I(PGINODESTEAL)] = "pginodesteal", 1342 [I(SLABS_SCANNED)] = "slabs_scanned", 1343 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal", 1344 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly", 1345 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly", 1346 [I(PAGEOUTRUN)] = "pageoutrun", 1347 1348 [I(PGROTATED)] = "pgrotated", 1349 1350 [I(DROP_PAGECACHE)] = "drop_pagecache", 1351 [I(DROP_SLAB)] = "drop_slab", 1352 [I(OOM_KILL)] = "oom_kill", 1353 1354 #ifdef CONFIG_NUMA_BALANCING 1355 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates", 1356 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates", 1357 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults", 1358 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local", 1359 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated", 1360 #endif 1361 #ifdef CONFIG_MIGRATION 1362 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success", 1363 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail", 1364 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success", 1365 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail", 1366 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split", 1367 #endif 1368 #ifdef CONFIG_COMPACTION 1369 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned", 1370 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned", 1371 [I(COMPACTISOLATED)] = "compact_isolated", 1372 [I(COMPACTSTALL)] = "compact_stall", 1373 [I(COMPACTFAIL)] = "compact_fail", 1374 [I(COMPACTSUCCESS)] = "compact_success", 1375 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake", 1376 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned", 1377 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned", 1378 #endif 1379 1380 #ifdef CONFIG_HUGETLB_PAGE 1381 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success", 1382 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail", 1383 #endif 1384 #ifdef CONFIG_CMA 1385 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success", 1386 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail", 1387 #endif 1388 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled", 1389 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned", 1390 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued", 1391 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked", 1392 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked", 1393 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared", 1394 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded", 1395 1396 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1397 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc", 1398 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback", 1399 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge", 1400 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc", 1401 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed", 1402 [I(THP_FILE_ALLOC)] = "thp_file_alloc", 1403 [I(THP_FILE_FALLBACK)] = "thp_file_fallback", 1404 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge", 1405 [I(THP_FILE_MAPPED)] = "thp_file_mapped", 1406 [I(THP_SPLIT_PAGE)] = "thp_split_page", 1407 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed", 1408 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page", 1409 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page", 1410 [I(THP_SPLIT_PMD)] = "thp_split_pmd", 1411 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte", 1412 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte", 1413 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte", 1414 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1415 [I(THP_SPLIT_PUD)] = "thp_split_pud", 1416 #endif 1417 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc", 1418 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed", 1419 [I(THP_SWPOUT)] = "thp_swpout", 1420 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback", 1421 #endif 1422 #ifdef CONFIG_BALLOON 1423 [I(BALLOON_INFLATE)] = "balloon_inflate", 1424 [I(BALLOON_DEFLATE)] = "balloon_deflate", 1425 #ifdef CONFIG_BALLOON_MIGRATION 1426 [I(BALLOON_MIGRATE)] = "balloon_migrate", 1427 #endif /* CONFIG_BALLOON_MIGRATION */ 1428 #endif /* CONFIG_BALLOON */ 1429 #ifdef CONFIG_DEBUG_TLBFLUSH 1430 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush", 1431 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received", 1432 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all", 1433 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one", 1434 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1435 1436 #ifdef CONFIG_SWAP 1437 [I(SWAP_RA)] = "swap_ra", 1438 [I(SWAP_RA_HIT)] = "swap_ra_hit", 1439 [I(SWPIN_ZERO)] = "swpin_zero", 1440 [I(SWPOUT_ZERO)] = "swpout_zero", 1441 #ifdef CONFIG_KSM 1442 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy", 1443 #endif 1444 #endif 1445 #ifdef CONFIG_KSM 1446 [I(COW_KSM)] = "cow_ksm", 1447 #endif 1448 #ifdef CONFIG_ZSWAP 1449 [I(ZSWPIN)] = "zswpin", 1450 [I(ZSWPOUT)] = "zswpout", 1451 [I(ZSWPWB)] = "zswpwb", 1452 #endif 1453 #ifdef CONFIG_X86 1454 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits", 1455 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits", 1456 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses", 1457 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses", 1458 #endif 1459 #ifdef CONFIG_PER_VMA_LOCK_STATS 1460 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success", 1461 [I(VMA_LOCK_ABORT)] = "vma_lock_abort", 1462 [I(VMA_LOCK_RETRY)] = "vma_lock_retry", 1463 [I(VMA_LOCK_MISS)] = "vma_lock_miss", 1464 #endif 1465 #ifdef CONFIG_DEBUG_STACK_USAGE 1466 [I(KSTACK_1K)] = "kstack_1k", 1467 #if THREAD_SIZE > 1024 1468 [I(KSTACK_2K)] = "kstack_2k", 1469 #endif 1470 #if THREAD_SIZE > 2048 1471 [I(KSTACK_4K)] = "kstack_4k", 1472 #endif 1473 #if THREAD_SIZE > 4096 1474 [I(KSTACK_8K)] = "kstack_8k", 1475 #endif 1476 #if THREAD_SIZE > 8192 1477 [I(KSTACK_16K)] = "kstack_16k", 1478 #endif 1479 #if THREAD_SIZE > 16384 1480 [I(KSTACK_32K)] = "kstack_32k", 1481 #endif 1482 #if THREAD_SIZE > 32768 1483 [I(KSTACK_64K)] = "kstack_64k", 1484 #endif 1485 #if THREAD_SIZE > 65536 1486 [I(KSTACK_REST)] = "kstack_rest", 1487 #endif 1488 #endif 1489 #undef I 1490 #endif /* CONFIG_VM_EVENT_COUNTERS */ 1491 }; 1492 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1493 1494 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1495 defined(CONFIG_PROC_FS) 1496 static void *frag_start(struct seq_file *m, loff_t *pos) 1497 { 1498 pg_data_t *pgdat; 1499 loff_t node = *pos; 1500 1501 for (pgdat = first_online_pgdat(); 1502 pgdat && node; 1503 pgdat = next_online_pgdat(pgdat)) 1504 --node; 1505 1506 return pgdat; 1507 } 1508 1509 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1510 { 1511 pg_data_t *pgdat = (pg_data_t *)arg; 1512 1513 (*pos)++; 1514 return next_online_pgdat(pgdat); 1515 } 1516 1517 static void frag_stop(struct seq_file *m, void *arg) 1518 { 1519 } 1520 1521 /* 1522 * Walk zones in a node and print using a callback. 1523 * If @assert_populated is true, only use callback for zones that are populated. 1524 */ 1525 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1526 bool assert_populated, bool nolock, 1527 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1528 { 1529 struct zone *zone; 1530 struct zone *node_zones = pgdat->node_zones; 1531 unsigned long flags; 1532 1533 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1534 if (assert_populated && !populated_zone(zone)) 1535 continue; 1536 1537 if (!nolock) 1538 spin_lock_irqsave(&zone->lock, flags); 1539 print(m, pgdat, zone); 1540 if (!nolock) 1541 spin_unlock_irqrestore(&zone->lock, flags); 1542 } 1543 } 1544 #endif 1545 1546 #ifdef CONFIG_PROC_FS 1547 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1548 struct zone *zone) 1549 { 1550 int order; 1551 1552 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1553 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1554 /* 1555 * Access to nr_free is lockless as nr_free is used only for 1556 * printing purposes. Use data_race to avoid KCSAN warning. 1557 */ 1558 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1559 seq_putc(m, '\n'); 1560 } 1561 1562 /* 1563 * This walks the free areas for each zone. 1564 */ 1565 static int frag_show(struct seq_file *m, void *arg) 1566 { 1567 pg_data_t *pgdat = (pg_data_t *)arg; 1568 walk_zones_in_node(m, pgdat, true, false, frag_show_print); 1569 return 0; 1570 } 1571 1572 static void pagetypeinfo_showfree_print(struct seq_file *m, 1573 pg_data_t *pgdat, struct zone *zone) 1574 { 1575 int order, mtype; 1576 1577 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1578 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1579 pgdat->node_id, 1580 zone->name, 1581 migratetype_names[mtype]); 1582 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1583 unsigned long freecount = 0; 1584 struct free_area *area; 1585 struct list_head *curr; 1586 bool overflow = false; 1587 1588 area = &(zone->free_area[order]); 1589 1590 list_for_each(curr, &area->free_list[mtype]) { 1591 /* 1592 * Cap the free_list iteration because it might 1593 * be really large and we are under a spinlock 1594 * so a long time spent here could trigger a 1595 * hard lockup detector. Anyway this is a 1596 * debugging tool so knowing there is a handful 1597 * of pages of this order should be more than 1598 * sufficient. 1599 */ 1600 if (++freecount >= 100000) { 1601 overflow = true; 1602 break; 1603 } 1604 } 1605 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1606 spin_unlock_irq(&zone->lock); 1607 cond_resched(); 1608 spin_lock_irq(&zone->lock); 1609 } 1610 seq_putc(m, '\n'); 1611 } 1612 } 1613 1614 /* Print out the free pages at each order for each migratetype */ 1615 static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1616 { 1617 int order; 1618 pg_data_t *pgdat = (pg_data_t *)arg; 1619 1620 /* Print header */ 1621 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1622 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1623 seq_printf(m, "%6d ", order); 1624 seq_putc(m, '\n'); 1625 1626 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1627 } 1628 1629 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1630 pg_data_t *pgdat, struct zone *zone) 1631 { 1632 int mtype; 1633 unsigned long pfn; 1634 unsigned long start_pfn = zone->zone_start_pfn; 1635 unsigned long end_pfn = zone_end_pfn(zone); 1636 unsigned long count[MIGRATE_TYPES] = { 0, }; 1637 1638 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1639 struct page *page; 1640 1641 page = pfn_to_online_page(pfn); 1642 if (!page) 1643 continue; 1644 1645 if (page_zone(page) != zone) 1646 continue; 1647 1648 mtype = get_pageblock_migratetype(page); 1649 1650 if (mtype < MIGRATE_TYPES) 1651 count[mtype]++; 1652 } 1653 1654 /* Print counts */ 1655 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1656 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1657 seq_printf(m, "%12lu ", count[mtype]); 1658 seq_putc(m, '\n'); 1659 } 1660 1661 /* Print out the number of pageblocks for each migratetype */ 1662 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1663 { 1664 int mtype; 1665 pg_data_t *pgdat = (pg_data_t *)arg; 1666 1667 seq_printf(m, "\n%-23s", "Number of blocks type "); 1668 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1669 seq_printf(m, "%12s ", migratetype_names[mtype]); 1670 seq_putc(m, '\n'); 1671 walk_zones_in_node(m, pgdat, true, false, 1672 pagetypeinfo_showblockcount_print); 1673 } 1674 1675 /* 1676 * Print out the number of pageblocks for each migratetype that contain pages 1677 * of other types. This gives an indication of how well fallbacks are being 1678 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1679 * to determine what is going on 1680 */ 1681 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1682 { 1683 #ifdef CONFIG_PAGE_OWNER 1684 int mtype; 1685 1686 if (!static_branch_unlikely(&page_owner_inited)) 1687 return; 1688 1689 drain_all_pages(NULL); 1690 1691 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1692 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1693 seq_printf(m, "%12s ", migratetype_names[mtype]); 1694 seq_putc(m, '\n'); 1695 1696 walk_zones_in_node(m, pgdat, true, true, 1697 pagetypeinfo_showmixedcount_print); 1698 #endif /* CONFIG_PAGE_OWNER */ 1699 } 1700 1701 /* 1702 * This prints out statistics in relation to grouping pages by mobility. 1703 * It is expensive to collect so do not constantly read the file. 1704 */ 1705 static int pagetypeinfo_show(struct seq_file *m, void *arg) 1706 { 1707 pg_data_t *pgdat = (pg_data_t *)arg; 1708 1709 /* check memoryless node */ 1710 if (!node_state(pgdat->node_id, N_MEMORY)) 1711 return 0; 1712 1713 seq_printf(m, "Page block order: %d\n", pageblock_order); 1714 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1715 seq_putc(m, '\n'); 1716 pagetypeinfo_showfree(m, pgdat); 1717 pagetypeinfo_showblockcount(m, pgdat); 1718 pagetypeinfo_showmixedcount(m, pgdat); 1719 1720 return 0; 1721 } 1722 1723 static const struct seq_operations fragmentation_op = { 1724 .start = frag_start, 1725 .next = frag_next, 1726 .stop = frag_stop, 1727 .show = frag_show, 1728 }; 1729 1730 static const struct seq_operations pagetypeinfo_op = { 1731 .start = frag_start, 1732 .next = frag_next, 1733 .stop = frag_stop, 1734 .show = pagetypeinfo_show, 1735 }; 1736 1737 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1738 { 1739 int zid; 1740 1741 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1742 struct zone *compare = &pgdat->node_zones[zid]; 1743 1744 if (populated_zone(compare)) 1745 return zone == compare; 1746 } 1747 1748 return false; 1749 } 1750 1751 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1752 struct zone *zone) 1753 { 1754 int i; 1755 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1756 if (is_zone_first_populated(pgdat, zone)) { 1757 seq_printf(m, "\n per-node stats"); 1758 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1759 unsigned long pages = node_page_state_pages(pgdat, i); 1760 1761 if (vmstat_item_print_in_thp(i)) 1762 pages /= HPAGE_PMD_NR; 1763 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1764 pages); 1765 } 1766 } 1767 seq_printf(m, 1768 "\n pages free %lu" 1769 "\n boost %lu" 1770 "\n min %lu" 1771 "\n low %lu" 1772 "\n high %lu" 1773 "\n promo %lu" 1774 "\n spanned %lu" 1775 "\n present %lu" 1776 "\n managed %lu" 1777 "\n cma %lu", 1778 zone_page_state(zone, NR_FREE_PAGES), 1779 zone->watermark_boost, 1780 min_wmark_pages(zone), 1781 low_wmark_pages(zone), 1782 high_wmark_pages(zone), 1783 promo_wmark_pages(zone), 1784 zone->spanned_pages, 1785 zone->present_pages, 1786 zone_managed_pages(zone), 1787 zone_cma_pages(zone)); 1788 1789 seq_printf(m, 1790 "\n protection: (%ld", 1791 zone->lowmem_reserve[0]); 1792 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1793 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1794 seq_putc(m, ')'); 1795 1796 /* If unpopulated, no other information is useful */ 1797 if (!populated_zone(zone)) { 1798 seq_putc(m, '\n'); 1799 return; 1800 } 1801 1802 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1803 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1804 zone_page_state(zone, i)); 1805 1806 #ifdef CONFIG_NUMA 1807 fold_vm_zone_numa_events(zone); 1808 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1809 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1810 zone_numa_event_state(zone, i)); 1811 #endif 1812 1813 seq_printf(m, "\n pagesets"); 1814 for_each_online_cpu(i) { 1815 struct per_cpu_pages *pcp; 1816 struct per_cpu_zonestat __maybe_unused *pzstats; 1817 1818 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1819 seq_printf(m, 1820 "\n cpu: %i" 1821 "\n count: %i" 1822 "\n high: %i" 1823 "\n batch: %i" 1824 "\n high_min: %i" 1825 "\n high_max: %i", 1826 i, 1827 pcp->count, 1828 pcp->high, 1829 pcp->batch, 1830 pcp->high_min, 1831 pcp->high_max); 1832 #ifdef CONFIG_SMP 1833 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1834 seq_printf(m, "\n vm stats threshold: %d", 1835 pzstats->stat_threshold); 1836 #endif 1837 } 1838 seq_printf(m, 1839 "\n node_unreclaimable: %u" 1840 "\n start_pfn: %lu" 1841 "\n reserved_highatomic: %lu" 1842 "\n free_highatomic: %lu", 1843 kswapd_test_hopeless(pgdat), 1844 zone->zone_start_pfn, 1845 zone->nr_reserved_highatomic, 1846 zone->nr_free_highatomic); 1847 seq_putc(m, '\n'); 1848 } 1849 1850 /* 1851 * Output information about zones in @pgdat. All zones are printed regardless 1852 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1853 * set of all zones and userspace would not be aware of such zones if they are 1854 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1855 */ 1856 static int zoneinfo_show(struct seq_file *m, void *arg) 1857 { 1858 pg_data_t *pgdat = (pg_data_t *)arg; 1859 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1860 return 0; 1861 } 1862 1863 static const struct seq_operations zoneinfo_op = { 1864 .start = frag_start, /* iterate over all zones. The same as in 1865 * fragmentation. */ 1866 .next = frag_next, 1867 .stop = frag_stop, 1868 .show = zoneinfo_show, 1869 }; 1870 1871 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1872 NR_VM_NUMA_EVENT_ITEMS + \ 1873 NR_VM_NODE_STAT_ITEMS + \ 1874 NR_VM_STAT_ITEMS + \ 1875 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1876 NR_VM_EVENT_ITEMS : 0)) 1877 1878 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1879 { 1880 unsigned long *v; 1881 int i; 1882 1883 if (*pos >= NR_VMSTAT_ITEMS) 1884 return NULL; 1885 1886 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS); 1887 fold_vm_numa_events(); 1888 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1889 m->private = v; 1890 if (!v) 1891 return ERR_PTR(-ENOMEM); 1892 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1893 v[i] = global_zone_page_state(i); 1894 v += NR_VM_ZONE_STAT_ITEMS; 1895 1896 #ifdef CONFIG_NUMA 1897 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1898 v[i] = global_numa_event_state(i); 1899 v += NR_VM_NUMA_EVENT_ITEMS; 1900 #endif 1901 1902 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1903 v[i] = global_node_page_state_pages(i); 1904 if (vmstat_item_print_in_thp(i)) 1905 v[i] /= HPAGE_PMD_NR; 1906 } 1907 v += NR_VM_NODE_STAT_ITEMS; 1908 1909 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1910 v + NR_DIRTY_THRESHOLD); 1911 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); 1912 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); 1913 v += NR_VM_STAT_ITEMS; 1914 1915 #ifdef CONFIG_VM_EVENT_COUNTERS 1916 all_vm_events(v); 1917 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1918 v[PGPGOUT] /= 2; 1919 #endif 1920 return (unsigned long *)m->private + *pos; 1921 } 1922 1923 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1924 { 1925 (*pos)++; 1926 if (*pos >= NR_VMSTAT_ITEMS) 1927 return NULL; 1928 return (unsigned long *)m->private + *pos; 1929 } 1930 1931 static int vmstat_show(struct seq_file *m, void *arg) 1932 { 1933 unsigned long *l = arg; 1934 unsigned long off = l - (unsigned long *)m->private; 1935 1936 seq_puts(m, vmstat_text[off]); 1937 seq_put_decimal_ull(m, " ", *l); 1938 seq_putc(m, '\n'); 1939 1940 if (off == NR_VMSTAT_ITEMS - 1) { 1941 /* 1942 * We've come to the end - add any deprecated counters to avoid 1943 * breaking userspace which might depend on them being present. 1944 */ 1945 seq_puts(m, "nr_unstable 0\n"); 1946 } 1947 return 0; 1948 } 1949 1950 static void vmstat_stop(struct seq_file *m, void *arg) 1951 { 1952 kfree(m->private); 1953 m->private = NULL; 1954 } 1955 1956 static const struct seq_operations vmstat_op = { 1957 .start = vmstat_start, 1958 .next = vmstat_next, 1959 .stop = vmstat_stop, 1960 .show = vmstat_show, 1961 }; 1962 #endif /* CONFIG_PROC_FS */ 1963 1964 #ifdef CONFIG_SMP 1965 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1966 static int sysctl_stat_interval __read_mostly = HZ; 1967 static int vmstat_late_init_done; 1968 1969 #ifdef CONFIG_PROC_FS 1970 static void refresh_vm_stats(struct work_struct *work) 1971 { 1972 refresh_cpu_vm_stats(true); 1973 } 1974 1975 static int vmstat_refresh(const struct ctl_table *table, int write, 1976 void *buffer, size_t *lenp, loff_t *ppos) 1977 { 1978 long val; 1979 int err; 1980 int i; 1981 1982 /* 1983 * The regular update, every sysctl_stat_interval, may come later 1984 * than expected: leaving a significant amount in per_cpu buckets. 1985 * This is particularly misleading when checking a quantity of HUGE 1986 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 1987 * which can equally be echo'ed to or cat'ted from (by root), 1988 * can be used to update the stats just before reading them. 1989 * 1990 * Oh, and since global_zone_page_state() etc. are so careful to hide 1991 * transiently negative values, report an error here if any of 1992 * the stats is negative, so we know to go looking for imbalance. 1993 */ 1994 err = schedule_on_each_cpu(refresh_vm_stats); 1995 if (err) 1996 return err; 1997 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 1998 /* 1999 * Skip checking stats known to go negative occasionally. 2000 */ 2001 switch (i) { 2002 case NR_ZONE_WRITE_PENDING: 2003 case NR_FREE_CMA_PAGES: 2004 continue; 2005 } 2006 val = atomic_long_read(&vm_zone_stat[i]); 2007 if (val < 0) { 2008 pr_warn("%s: %s %ld\n", 2009 __func__, zone_stat_name(i), val); 2010 } 2011 } 2012 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 2013 /* 2014 * Skip checking stats known to go negative occasionally. 2015 */ 2016 switch (i) { 2017 case NR_WRITEBACK: 2018 continue; 2019 } 2020 val = atomic_long_read(&vm_node_stat[i]); 2021 if (val < 0) { 2022 pr_warn("%s: %s %ld\n", 2023 __func__, node_stat_name(i), val); 2024 } 2025 } 2026 if (write) 2027 *ppos += *lenp; 2028 else 2029 *lenp = 0; 2030 return 0; 2031 } 2032 #endif /* CONFIG_PROC_FS */ 2033 2034 static void vmstat_update(struct work_struct *w) 2035 { 2036 if (refresh_cpu_vm_stats(true)) { 2037 /* 2038 * Counters were updated so we expect more updates 2039 * to occur in the future. Keep on running the 2040 * update worker thread. 2041 */ 2042 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 2043 this_cpu_ptr(&vmstat_work), 2044 round_jiffies_relative(sysctl_stat_interval)); 2045 } 2046 } 2047 2048 /* 2049 * Check if the diffs for a certain cpu indicate that 2050 * an update is needed. 2051 */ 2052 static bool need_update(int cpu) 2053 { 2054 pg_data_t *last_pgdat = NULL; 2055 struct zone *zone; 2056 2057 for_each_populated_zone(zone) { 2058 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 2059 struct per_cpu_nodestat *n; 2060 2061 /* 2062 * The fast way of checking if there are any vmstat diffs. 2063 */ 2064 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 2065 return true; 2066 2067 if (last_pgdat == zone->zone_pgdat) 2068 continue; 2069 last_pgdat = zone->zone_pgdat; 2070 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 2071 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 2072 return true; 2073 } 2074 return false; 2075 } 2076 2077 /* 2078 * Switch off vmstat processing and then fold all the remaining differentials 2079 * until the diffs stay at zero. The function is used by NOHZ and can only be 2080 * invoked when tick processing is not active. 2081 */ 2082 void quiet_vmstat(void) 2083 { 2084 if (system_state != SYSTEM_RUNNING) 2085 return; 2086 2087 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2088 return; 2089 2090 if (!need_update(smp_processor_id())) 2091 return; 2092 2093 /* 2094 * Just refresh counters and do not care about the pending delayed 2095 * vmstat_update. It doesn't fire that often to matter and canceling 2096 * it would be too expensive from this path. 2097 * vmstat_shepherd will take care about that for us. 2098 */ 2099 refresh_cpu_vm_stats(false); 2100 } 2101 2102 /* 2103 * Shepherd worker thread that checks the 2104 * differentials of processors that have their worker 2105 * threads for vm statistics updates disabled because of 2106 * inactivity. 2107 */ 2108 static void vmstat_shepherd(struct work_struct *w); 2109 2110 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2111 2112 void vmstat_flush_workqueue(void) 2113 { 2114 flush_workqueue(mm_percpu_wq); 2115 } 2116 2117 static void vmstat_shepherd(struct work_struct *w) 2118 { 2119 int cpu; 2120 2121 cpus_read_lock(); 2122 /* Check processors whose vmstat worker threads have been disabled */ 2123 for_each_online_cpu(cpu) { 2124 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2125 2126 /* 2127 * In kernel users of vmstat counters either require the precise value and 2128 * they are using zone_page_state_snapshot interface or they can live with 2129 * an imprecision as the regular flushing can happen at arbitrary time and 2130 * cumulative error can grow (see calculate_normal_threshold). 2131 * 2132 * From that POV the regular flushing can be postponed for CPUs that have 2133 * been isolated from the kernel interference without critical 2134 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2135 * for all isolated CPUs to avoid interference with the isolated workload. 2136 */ 2137 scoped_guard(rcu) { 2138 if (cpu_is_isolated(cpu)) 2139 continue; 2140 2141 if (!delayed_work_pending(dw) && need_update(cpu)) 2142 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2143 } 2144 2145 cond_resched(); 2146 } 2147 cpus_read_unlock(); 2148 2149 schedule_delayed_work(&shepherd, 2150 round_jiffies_relative(sysctl_stat_interval)); 2151 } 2152 2153 static void __init start_shepherd_timer(void) 2154 { 2155 int cpu; 2156 2157 for_each_possible_cpu(cpu) { 2158 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2159 vmstat_update); 2160 2161 /* 2162 * For secondary CPUs during CPU hotplug scenarios, 2163 * vmstat_cpu_online() will enable the work. 2164 * mm/vmstat:online enables and disables vmstat_work 2165 * symmetrically during CPU hotplug events. 2166 */ 2167 if (!cpu_online(cpu)) 2168 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2169 } 2170 2171 schedule_delayed_work(&shepherd, 2172 round_jiffies_relative(sysctl_stat_interval)); 2173 } 2174 2175 static void __init init_cpu_node_state(void) 2176 { 2177 int node; 2178 2179 for_each_online_node(node) { 2180 if (!cpumask_empty(cpumask_of_node(node))) 2181 node_set_state(node, N_CPU); 2182 } 2183 } 2184 2185 static int vmstat_cpu_online(unsigned int cpu) 2186 { 2187 if (vmstat_late_init_done) 2188 refresh_zone_stat_thresholds(); 2189 2190 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2191 node_set_state(cpu_to_node(cpu), N_CPU); 2192 } 2193 enable_delayed_work(&per_cpu(vmstat_work, cpu)); 2194 2195 return 0; 2196 } 2197 2198 static int vmstat_cpu_down_prep(unsigned int cpu) 2199 { 2200 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2201 return 0; 2202 } 2203 2204 static int vmstat_cpu_dead(unsigned int cpu) 2205 { 2206 const struct cpumask *node_cpus; 2207 int node; 2208 2209 node = cpu_to_node(cpu); 2210 2211 refresh_zone_stat_thresholds(); 2212 node_cpus = cpumask_of_node(node); 2213 if (!cpumask_empty(node_cpus)) 2214 return 0; 2215 2216 node_clear_state(node, N_CPU); 2217 2218 return 0; 2219 } 2220 2221 static int __init vmstat_late_init(void) 2222 { 2223 refresh_zone_stat_thresholds(); 2224 vmstat_late_init_done = 1; 2225 2226 return 0; 2227 } 2228 late_initcall(vmstat_late_init); 2229 #endif 2230 2231 #ifdef CONFIG_PROC_FS 2232 static const struct ctl_table vmstat_table[] = { 2233 #ifdef CONFIG_SMP 2234 { 2235 .procname = "stat_interval", 2236 .data = &sysctl_stat_interval, 2237 .maxlen = sizeof(sysctl_stat_interval), 2238 .mode = 0644, 2239 .proc_handler = proc_dointvec_jiffies, 2240 }, 2241 { 2242 .procname = "stat_refresh", 2243 .data = NULL, 2244 .maxlen = 0, 2245 .mode = 0600, 2246 .proc_handler = vmstat_refresh, 2247 }, 2248 #endif 2249 #ifdef CONFIG_NUMA 2250 { 2251 .procname = "numa_stat", 2252 .data = &sysctl_vm_numa_stat, 2253 .maxlen = sizeof(int), 2254 .mode = 0644, 2255 .proc_handler = sysctl_vm_numa_stat_handler, 2256 .extra1 = SYSCTL_ZERO, 2257 .extra2 = SYSCTL_ONE, 2258 }, 2259 #endif 2260 }; 2261 #endif 2262 2263 struct workqueue_struct *mm_percpu_wq; 2264 2265 void __init init_mm_internals(void) 2266 { 2267 int ret __maybe_unused; 2268 2269 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", 2270 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2271 2272 #ifdef CONFIG_SMP 2273 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2274 NULL, vmstat_cpu_dead); 2275 if (ret < 0) 2276 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2277 2278 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2279 vmstat_cpu_online, 2280 vmstat_cpu_down_prep); 2281 if (ret < 0) 2282 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2283 2284 cpus_read_lock(); 2285 init_cpu_node_state(); 2286 cpus_read_unlock(); 2287 2288 start_shepherd_timer(); 2289 #endif 2290 #ifdef CONFIG_PROC_FS 2291 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2292 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2293 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2294 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2295 register_sysctl_init("vm", vmstat_table); 2296 #endif 2297 } 2298 2299 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2300 2301 /* 2302 * Return an index indicating how much of the available free memory is 2303 * unusable for an allocation of the requested size. 2304 */ 2305 static int unusable_free_index(unsigned int order, 2306 struct contig_page_info *info) 2307 { 2308 /* No free memory is interpreted as all free memory is unusable */ 2309 if (info->free_pages == 0) 2310 return 1000; 2311 2312 /* 2313 * Index should be a value between 0 and 1. Return a value to 3 2314 * decimal places. 2315 * 2316 * 0 => no fragmentation 2317 * 1 => high fragmentation 2318 */ 2319 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2320 2321 } 2322 2323 static void unusable_show_print(struct seq_file *m, 2324 pg_data_t *pgdat, struct zone *zone) 2325 { 2326 unsigned int order; 2327 int index; 2328 struct contig_page_info info; 2329 2330 seq_printf(m, "Node %d, zone %8s ", 2331 pgdat->node_id, 2332 zone->name); 2333 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2334 fill_contig_page_info(zone, order, &info); 2335 index = unusable_free_index(order, &info); 2336 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2337 } 2338 2339 seq_putc(m, '\n'); 2340 } 2341 2342 /* 2343 * Display unusable free space index 2344 * 2345 * The unusable free space index measures how much of the available free 2346 * memory cannot be used to satisfy an allocation of a given size and is a 2347 * value between 0 and 1. The higher the value, the more of free memory is 2348 * unusable and by implication, the worse the external fragmentation is. This 2349 * can be expressed as a percentage by multiplying by 100. 2350 */ 2351 static int unusable_show(struct seq_file *m, void *arg) 2352 { 2353 pg_data_t *pgdat = (pg_data_t *)arg; 2354 2355 /* check memoryless node */ 2356 if (!node_state(pgdat->node_id, N_MEMORY)) 2357 return 0; 2358 2359 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2360 2361 return 0; 2362 } 2363 2364 static const struct seq_operations unusable_sops = { 2365 .start = frag_start, 2366 .next = frag_next, 2367 .stop = frag_stop, 2368 .show = unusable_show, 2369 }; 2370 2371 DEFINE_SEQ_ATTRIBUTE(unusable); 2372 2373 static void extfrag_show_print(struct seq_file *m, 2374 pg_data_t *pgdat, struct zone *zone) 2375 { 2376 unsigned int order; 2377 int index; 2378 2379 /* Alloc on stack as interrupts are disabled for zone walk */ 2380 struct contig_page_info info; 2381 2382 seq_printf(m, "Node %d, zone %8s ", 2383 pgdat->node_id, 2384 zone->name); 2385 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2386 fill_contig_page_info(zone, order, &info); 2387 index = __fragmentation_index(order, &info); 2388 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2389 } 2390 2391 seq_putc(m, '\n'); 2392 } 2393 2394 /* 2395 * Display fragmentation index for orders that allocations would fail for 2396 */ 2397 static int extfrag_show(struct seq_file *m, void *arg) 2398 { 2399 pg_data_t *pgdat = (pg_data_t *)arg; 2400 2401 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2402 2403 return 0; 2404 } 2405 2406 static const struct seq_operations extfrag_sops = { 2407 .start = frag_start, 2408 .next = frag_next, 2409 .stop = frag_stop, 2410 .show = extfrag_show, 2411 }; 2412 2413 DEFINE_SEQ_ATTRIBUTE(extfrag); 2414 2415 static int __init extfrag_debug_init(void) 2416 { 2417 struct dentry *extfrag_debug_root; 2418 2419 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2420 2421 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2422 &unusable_fops); 2423 2424 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2425 &extfrag_fops); 2426 2427 return 0; 2428 } 2429 2430 module_init(extfrag_debug_init); 2431 2432 #endif 2433