1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <cl@gentwo.org> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13 #include <linux/fs.h> 14 #include <linux/mm.h> 15 #include <linux/err.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/cpumask.h> 20 #include <linux/vmstat.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/debugfs.h> 24 #include <linux/sched.h> 25 #include <linux/math64.h> 26 #include <linux/writeback.h> 27 #include <linux/compaction.h> 28 #include <linux/mm_inline.h> 29 #include <linux/page_owner.h> 30 #include <linux/sched/isolation.h> 31 32 #include "internal.h" 33 34 #ifdef CONFIG_PROC_FS 35 #ifdef CONFIG_NUMA 36 #define ENABLE_NUMA_STAT 1 37 static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 38 39 /* zero numa counters within a zone */ 40 static void zero_zone_numa_counters(struct zone *zone) 41 { 42 int item, cpu; 43 44 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 45 atomic_long_set(&zone->vm_numa_event[item], 0); 46 for_each_online_cpu(cpu) { 47 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 48 = 0; 49 } 50 } 51 } 52 53 /* zero numa counters of all the populated zones */ 54 static void zero_zones_numa_counters(void) 55 { 56 struct zone *zone; 57 58 for_each_populated_zone(zone) 59 zero_zone_numa_counters(zone); 60 } 61 62 /* zero global numa counters */ 63 static void zero_global_numa_counters(void) 64 { 65 int item; 66 67 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 68 atomic_long_set(&vm_numa_event[item], 0); 69 } 70 71 static void invalid_numa_statistics(void) 72 { 73 zero_zones_numa_counters(); 74 zero_global_numa_counters(); 75 } 76 77 static DEFINE_MUTEX(vm_numa_stat_lock); 78 79 static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, 80 void *buffer, size_t *length, loff_t *ppos) 81 { 82 int ret, oldval; 83 84 mutex_lock(&vm_numa_stat_lock); 85 if (write) 86 oldval = sysctl_vm_numa_stat; 87 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 88 if (ret || !write) 89 goto out; 90 91 if (oldval == sysctl_vm_numa_stat) 92 goto out; 93 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 94 static_branch_enable(&vm_numa_stat_key); 95 pr_info("enable numa statistics\n"); 96 } else { 97 static_branch_disable(&vm_numa_stat_key); 98 invalid_numa_statistics(); 99 pr_info("disable numa statistics, and clear numa counters\n"); 100 } 101 102 out: 103 mutex_unlock(&vm_numa_stat_lock); 104 return ret; 105 } 106 #endif 107 #endif /* CONFIG_PROC_FS */ 108 109 #ifdef CONFIG_VM_EVENT_COUNTERS 110 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 111 EXPORT_PER_CPU_SYMBOL(vm_event_states); 112 113 static void sum_vm_events(unsigned long *ret) 114 { 115 int cpu; 116 int i; 117 118 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 119 120 for_each_online_cpu(cpu) { 121 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 122 123 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 124 ret[i] += this->event[i]; 125 } 126 } 127 128 /* 129 * Accumulate the vm event counters across all CPUs. 130 * The result is unavoidably approximate - it can change 131 * during and after execution of this function. 132 */ 133 void all_vm_events(unsigned long *ret) 134 { 135 cpus_read_lock(); 136 sum_vm_events(ret); 137 cpus_read_unlock(); 138 } 139 EXPORT_SYMBOL_GPL(all_vm_events); 140 141 /* 142 * Fold the foreign cpu events into our own. 143 * 144 * This is adding to the events on one processor 145 * but keeps the global counts constant. 146 */ 147 void vm_events_fold_cpu(int cpu) 148 { 149 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 150 int i; 151 152 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 153 count_vm_events(i, fold_state->event[i]); 154 fold_state->event[i] = 0; 155 } 156 } 157 158 #endif /* CONFIG_VM_EVENT_COUNTERS */ 159 160 /* 161 * Manage combined zone based / global counters 162 * 163 * vm_stat contains the global counters 164 */ 165 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 166 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 167 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 168 EXPORT_SYMBOL(vm_zone_stat); 169 EXPORT_SYMBOL(vm_node_stat); 170 171 #ifdef CONFIG_NUMA 172 static void fold_vm_zone_numa_events(struct zone *zone) 173 { 174 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 175 int cpu; 176 enum numa_stat_item item; 177 178 for_each_online_cpu(cpu) { 179 struct per_cpu_zonestat *pzstats; 180 181 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 182 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 183 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 184 } 185 186 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 187 zone_numa_event_add(zone_numa_events[item], zone, item); 188 } 189 190 void fold_vm_numa_events(void) 191 { 192 struct zone *zone; 193 194 for_each_populated_zone(zone) 195 fold_vm_zone_numa_events(zone); 196 } 197 #endif 198 199 #ifdef CONFIG_SMP 200 201 int calculate_pressure_threshold(struct zone *zone) 202 { 203 int threshold; 204 int watermark_distance; 205 206 /* 207 * As vmstats are not up to date, there is drift between the estimated 208 * and real values. For high thresholds and a high number of CPUs, it 209 * is possible for the min watermark to be breached while the estimated 210 * value looks fine. The pressure threshold is a reduced value such 211 * that even the maximum amount of drift will not accidentally breach 212 * the min watermark 213 */ 214 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 215 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 216 217 /* 218 * Maximum threshold is 125 219 */ 220 threshold = min(125, threshold); 221 222 return threshold; 223 } 224 225 int calculate_normal_threshold(struct zone *zone) 226 { 227 int threshold; 228 int mem; /* memory in 128 MB units */ 229 230 /* 231 * The threshold scales with the number of processors and the amount 232 * of memory per zone. More memory means that we can defer updates for 233 * longer, more processors could lead to more contention. 234 * fls() is used to have a cheap way of logarithmic scaling. 235 * 236 * Some sample thresholds: 237 * 238 * Threshold Processors (fls) Zonesize fls(mem)+1 239 * ------------------------------------------------------------------ 240 * 8 1 1 0.9-1 GB 4 241 * 16 2 2 0.9-1 GB 4 242 * 20 2 2 1-2 GB 5 243 * 24 2 2 2-4 GB 6 244 * 28 2 2 4-8 GB 7 245 * 32 2 2 8-16 GB 8 246 * 4 2 2 <128M 1 247 * 30 4 3 2-4 GB 5 248 * 48 4 3 8-16 GB 8 249 * 32 8 4 1-2 GB 4 250 * 32 8 4 0.9-1GB 4 251 * 10 16 5 <128M 1 252 * 40 16 5 900M 4 253 * 70 64 7 2-4 GB 5 254 * 84 64 7 4-8 GB 6 255 * 108 512 9 4-8 GB 6 256 * 125 1024 10 8-16 GB 8 257 * 125 1024 10 16-32 GB 9 258 */ 259 260 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 261 262 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 263 264 /* 265 * Maximum threshold is 125 266 */ 267 threshold = min(125, threshold); 268 269 return threshold; 270 } 271 272 /* 273 * Refresh the thresholds for each zone. 274 */ 275 void refresh_zone_stat_thresholds(void) 276 { 277 struct pglist_data *pgdat; 278 struct zone *zone; 279 int cpu; 280 int threshold; 281 282 /* Zero current pgdat thresholds */ 283 for_each_online_pgdat(pgdat) { 284 for_each_online_cpu(cpu) { 285 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 286 } 287 } 288 289 for_each_populated_zone(zone) { 290 struct pglist_data *pgdat = zone->zone_pgdat; 291 unsigned long max_drift, tolerate_drift; 292 293 threshold = calculate_normal_threshold(zone); 294 295 for_each_online_cpu(cpu) { 296 int pgdat_threshold; 297 298 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 299 = threshold; 300 301 /* Base nodestat threshold on the largest populated zone. */ 302 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 303 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 304 = max(threshold, pgdat_threshold); 305 } 306 307 /* 308 * Only set percpu_drift_mark if there is a danger that 309 * NR_FREE_PAGES reports the low watermark is ok when in fact 310 * the min watermark could be breached by an allocation 311 */ 312 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 313 max_drift = num_online_cpus() * threshold; 314 if (max_drift > tolerate_drift) 315 zone->percpu_drift_mark = high_wmark_pages(zone) + 316 max_drift; 317 } 318 } 319 320 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 321 int (*calculate_pressure)(struct zone *)) 322 { 323 struct zone *zone; 324 int cpu; 325 int threshold; 326 int i; 327 328 for (i = 0; i < pgdat->nr_zones; i++) { 329 zone = &pgdat->node_zones[i]; 330 if (!zone->percpu_drift_mark) 331 continue; 332 333 threshold = (*calculate_pressure)(zone); 334 for_each_online_cpu(cpu) 335 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 336 = threshold; 337 } 338 } 339 340 /* 341 * For use when we know that interrupts are disabled, 342 * or when we know that preemption is disabled and that 343 * particular counter cannot be updated from interrupt context. 344 */ 345 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 346 long delta) 347 { 348 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 349 s8 __percpu *p = pcp->vm_stat_diff + item; 350 long x; 351 long t; 352 353 /* 354 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 355 * atomicity is provided by IRQs being disabled -- either explicitly 356 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 357 * CPU migrations and preemption potentially corrupts a counter so 358 * disable preemption. 359 */ 360 preempt_disable_nested(); 361 362 x = delta + __this_cpu_read(*p); 363 364 t = __this_cpu_read(pcp->stat_threshold); 365 366 if (unlikely(abs(x) > t)) { 367 zone_page_state_add(x, zone, item); 368 x = 0; 369 } 370 __this_cpu_write(*p, x); 371 372 preempt_enable_nested(); 373 } 374 EXPORT_SYMBOL(__mod_zone_page_state); 375 376 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 377 long delta) 378 { 379 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 380 s8 __percpu *p = pcp->vm_node_stat_diff + item; 381 long x; 382 long t; 383 384 if (vmstat_item_in_bytes(item)) { 385 /* 386 * Only cgroups use subpage accounting right now; at 387 * the global level, these items still change in 388 * multiples of whole pages. Store them as pages 389 * internally to keep the per-cpu counters compact. 390 */ 391 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 392 delta >>= PAGE_SHIFT; 393 } 394 395 /* See __mod_zone_page_state() */ 396 preempt_disable_nested(); 397 398 x = delta + __this_cpu_read(*p); 399 400 t = __this_cpu_read(pcp->stat_threshold); 401 402 if (unlikely(abs(x) > t)) { 403 node_page_state_add(x, pgdat, item); 404 x = 0; 405 } 406 __this_cpu_write(*p, x); 407 408 preempt_enable_nested(); 409 } 410 EXPORT_SYMBOL(__mod_node_page_state); 411 412 /* 413 * Optimized increment and decrement functions. 414 * 415 * These are only for a single page and therefore can take a struct page * 416 * argument instead of struct zone *. This allows the inclusion of the code 417 * generated for page_zone(page) into the optimized functions. 418 * 419 * No overflow check is necessary and therefore the differential can be 420 * incremented or decremented in place which may allow the compilers to 421 * generate better code. 422 * The increment or decrement is known and therefore one boundary check can 423 * be omitted. 424 * 425 * NOTE: These functions are very performance sensitive. Change only 426 * with care. 427 * 428 * Some processors have inc/dec instructions that are atomic vs an interrupt. 429 * However, the code must first determine the differential location in a zone 430 * based on the processor number and then inc/dec the counter. There is no 431 * guarantee without disabling preemption that the processor will not change 432 * in between and therefore the atomicity vs. interrupt cannot be exploited 433 * in a useful way here. 434 */ 435 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 436 { 437 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 438 s8 __percpu *p = pcp->vm_stat_diff + item; 439 s8 v, t; 440 441 /* See __mod_zone_page_state() */ 442 preempt_disable_nested(); 443 444 v = __this_cpu_inc_return(*p); 445 t = __this_cpu_read(pcp->stat_threshold); 446 if (unlikely(v > t)) { 447 s8 overstep = t >> 1; 448 449 zone_page_state_add(v + overstep, zone, item); 450 __this_cpu_write(*p, -overstep); 451 } 452 453 preempt_enable_nested(); 454 } 455 456 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 457 { 458 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 459 s8 __percpu *p = pcp->vm_node_stat_diff + item; 460 s8 v, t; 461 462 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 463 464 /* See __mod_zone_page_state() */ 465 preempt_disable_nested(); 466 467 v = __this_cpu_inc_return(*p); 468 t = __this_cpu_read(pcp->stat_threshold); 469 if (unlikely(v > t)) { 470 s8 overstep = t >> 1; 471 472 node_page_state_add(v + overstep, pgdat, item); 473 __this_cpu_write(*p, -overstep); 474 } 475 476 preempt_enable_nested(); 477 } 478 479 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 480 { 481 __inc_zone_state(page_zone(page), item); 482 } 483 EXPORT_SYMBOL(__inc_zone_page_state); 484 485 void __inc_node_page_state(struct page *page, enum node_stat_item item) 486 { 487 __inc_node_state(page_pgdat(page), item); 488 } 489 EXPORT_SYMBOL(__inc_node_page_state); 490 491 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 492 { 493 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 494 s8 __percpu *p = pcp->vm_stat_diff + item; 495 s8 v, t; 496 497 /* See __mod_zone_page_state() */ 498 preempt_disable_nested(); 499 500 v = __this_cpu_dec_return(*p); 501 t = __this_cpu_read(pcp->stat_threshold); 502 if (unlikely(v < - t)) { 503 s8 overstep = t >> 1; 504 505 zone_page_state_add(v - overstep, zone, item); 506 __this_cpu_write(*p, overstep); 507 } 508 509 preempt_enable_nested(); 510 } 511 512 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 513 { 514 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 515 s8 __percpu *p = pcp->vm_node_stat_diff + item; 516 s8 v, t; 517 518 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 519 520 /* See __mod_zone_page_state() */ 521 preempt_disable_nested(); 522 523 v = __this_cpu_dec_return(*p); 524 t = __this_cpu_read(pcp->stat_threshold); 525 if (unlikely(v < - t)) { 526 s8 overstep = t >> 1; 527 528 node_page_state_add(v - overstep, pgdat, item); 529 __this_cpu_write(*p, overstep); 530 } 531 532 preempt_enable_nested(); 533 } 534 535 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 536 { 537 __dec_zone_state(page_zone(page), item); 538 } 539 EXPORT_SYMBOL(__dec_zone_page_state); 540 541 void __dec_node_page_state(struct page *page, enum node_stat_item item) 542 { 543 __dec_node_state(page_pgdat(page), item); 544 } 545 EXPORT_SYMBOL(__dec_node_page_state); 546 547 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 548 /* 549 * If we have cmpxchg_local support then we do not need to incur the overhead 550 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 551 * 552 * mod_state() modifies the zone counter state through atomic per cpu 553 * operations. 554 * 555 * Overstep mode specifies how overstep should handled: 556 * 0 No overstepping 557 * 1 Overstepping half of threshold 558 * -1 Overstepping minus half of threshold 559 */ 560 static inline void mod_zone_state(struct zone *zone, 561 enum zone_stat_item item, long delta, int overstep_mode) 562 { 563 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 564 s8 __percpu *p = pcp->vm_stat_diff + item; 565 long n, t, z; 566 s8 o; 567 568 o = this_cpu_read(*p); 569 do { 570 z = 0; /* overflow to zone counters */ 571 572 /* 573 * The fetching of the stat_threshold is racy. We may apply 574 * a counter threshold to the wrong the cpu if we get 575 * rescheduled while executing here. However, the next 576 * counter update will apply the threshold again and 577 * therefore bring the counter under the threshold again. 578 * 579 * Most of the time the thresholds are the same anyways 580 * for all cpus in a zone. 581 */ 582 t = this_cpu_read(pcp->stat_threshold); 583 584 n = delta + (long)o; 585 586 if (abs(n) > t) { 587 int os = overstep_mode * (t >> 1) ; 588 589 /* Overflow must be added to zone counters */ 590 z = n + os; 591 n = -os; 592 } 593 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 594 595 if (z) 596 zone_page_state_add(z, zone, item); 597 } 598 599 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 600 long delta) 601 { 602 mod_zone_state(zone, item, delta, 0); 603 } 604 EXPORT_SYMBOL(mod_zone_page_state); 605 606 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 607 { 608 mod_zone_state(page_zone(page), item, 1, 1); 609 } 610 EXPORT_SYMBOL(inc_zone_page_state); 611 612 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 613 { 614 mod_zone_state(page_zone(page), item, -1, -1); 615 } 616 EXPORT_SYMBOL(dec_zone_page_state); 617 618 static inline void mod_node_state(struct pglist_data *pgdat, 619 enum node_stat_item item, int delta, int overstep_mode) 620 { 621 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 622 s8 __percpu *p = pcp->vm_node_stat_diff + item; 623 long n, t, z; 624 s8 o; 625 626 if (vmstat_item_in_bytes(item)) { 627 /* 628 * Only cgroups use subpage accounting right now; at 629 * the global level, these items still change in 630 * multiples of whole pages. Store them as pages 631 * internally to keep the per-cpu counters compact. 632 */ 633 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 634 delta >>= PAGE_SHIFT; 635 } 636 637 o = this_cpu_read(*p); 638 do { 639 z = 0; /* overflow to node counters */ 640 641 /* 642 * The fetching of the stat_threshold is racy. We may apply 643 * a counter threshold to the wrong the cpu if we get 644 * rescheduled while executing here. However, the next 645 * counter update will apply the threshold again and 646 * therefore bring the counter under the threshold again. 647 * 648 * Most of the time the thresholds are the same anyways 649 * for all cpus in a node. 650 */ 651 t = this_cpu_read(pcp->stat_threshold); 652 653 n = delta + (long)o; 654 655 if (abs(n) > t) { 656 int os = overstep_mode * (t >> 1) ; 657 658 /* Overflow must be added to node counters */ 659 z = n + os; 660 n = -os; 661 } 662 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 663 664 if (z) 665 node_page_state_add(z, pgdat, item); 666 } 667 668 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 669 long delta) 670 { 671 mod_node_state(pgdat, item, delta, 0); 672 } 673 EXPORT_SYMBOL(mod_node_page_state); 674 675 void inc_node_page_state(struct page *page, enum node_stat_item item) 676 { 677 mod_node_state(page_pgdat(page), item, 1, 1); 678 } 679 EXPORT_SYMBOL(inc_node_page_state); 680 681 void dec_node_page_state(struct page *page, enum node_stat_item item) 682 { 683 mod_node_state(page_pgdat(page), item, -1, -1); 684 } 685 EXPORT_SYMBOL(dec_node_page_state); 686 #else 687 /* 688 * Use interrupt disable to serialize counter updates 689 */ 690 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 691 long delta) 692 { 693 unsigned long flags; 694 695 local_irq_save(flags); 696 __mod_zone_page_state(zone, item, delta); 697 local_irq_restore(flags); 698 } 699 EXPORT_SYMBOL(mod_zone_page_state); 700 701 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 702 { 703 unsigned long flags; 704 struct zone *zone; 705 706 zone = page_zone(page); 707 local_irq_save(flags); 708 __inc_zone_state(zone, item); 709 local_irq_restore(flags); 710 } 711 EXPORT_SYMBOL(inc_zone_page_state); 712 713 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 714 { 715 unsigned long flags; 716 717 local_irq_save(flags); 718 __dec_zone_page_state(page, item); 719 local_irq_restore(flags); 720 } 721 EXPORT_SYMBOL(dec_zone_page_state); 722 723 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 724 long delta) 725 { 726 unsigned long flags; 727 728 local_irq_save(flags); 729 __mod_node_page_state(pgdat, item, delta); 730 local_irq_restore(flags); 731 } 732 EXPORT_SYMBOL(mod_node_page_state); 733 734 void inc_node_page_state(struct page *page, enum node_stat_item item) 735 { 736 unsigned long flags; 737 struct pglist_data *pgdat; 738 739 pgdat = page_pgdat(page); 740 local_irq_save(flags); 741 __inc_node_state(pgdat, item); 742 local_irq_restore(flags); 743 } 744 EXPORT_SYMBOL(inc_node_page_state); 745 746 void dec_node_page_state(struct page *page, enum node_stat_item item) 747 { 748 unsigned long flags; 749 750 local_irq_save(flags); 751 __dec_node_page_state(page, item); 752 local_irq_restore(flags); 753 } 754 EXPORT_SYMBOL(dec_node_page_state); 755 #endif 756 757 /* 758 * Fold a differential into the global counters. 759 * Returns whether counters were updated. 760 */ 761 static int fold_diff(int *zone_diff, int *node_diff) 762 { 763 int i; 764 bool changed = false; 765 766 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 767 if (zone_diff[i]) { 768 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 769 changed = true; 770 } 771 } 772 773 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 774 if (node_diff[i]) { 775 atomic_long_add(node_diff[i], &vm_node_stat[i]); 776 changed = true; 777 } 778 } 779 780 return changed; 781 } 782 783 /* 784 * Update the zone counters for the current cpu. 785 * 786 * Note that refresh_cpu_vm_stats strives to only access 787 * node local memory. The per cpu pagesets on remote zones are placed 788 * in the memory local to the processor using that pageset. So the 789 * loop over all zones will access a series of cachelines local to 790 * the processor. 791 * 792 * The call to zone_page_state_add updates the cachelines with the 793 * statistics in the remote zone struct as well as the global cachelines 794 * with the global counters. These could cause remote node cache line 795 * bouncing and will have to be only done when necessary. 796 * 797 * The function returns whether global counters were updated. 798 */ 799 static bool refresh_cpu_vm_stats(bool do_pagesets) 800 { 801 struct pglist_data *pgdat; 802 struct zone *zone; 803 int i; 804 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 805 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 806 bool changed = false; 807 808 for_each_populated_zone(zone) { 809 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 810 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 811 812 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 813 int v; 814 815 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 816 if (v) { 817 818 atomic_long_add(v, &zone->vm_stat[i]); 819 global_zone_diff[i] += v; 820 #ifdef CONFIG_NUMA 821 /* 3 seconds idle till flush */ 822 __this_cpu_write(pcp->expire, 3); 823 #endif 824 } 825 } 826 827 if (do_pagesets) { 828 cond_resched(); 829 830 if (decay_pcp_high(zone, this_cpu_ptr(pcp))) 831 changed = true; 832 #ifdef CONFIG_NUMA 833 /* 834 * Deal with draining the remote pageset of this 835 * processor 836 * 837 * Check if there are pages remaining in this pageset 838 * if not then there is nothing to expire. 839 */ 840 if (!__this_cpu_read(pcp->expire) || 841 !__this_cpu_read(pcp->count)) 842 continue; 843 844 /* 845 * We never drain zones local to this processor. 846 */ 847 if (zone_to_nid(zone) == numa_node_id()) { 848 __this_cpu_write(pcp->expire, 0); 849 continue; 850 } 851 852 if (__this_cpu_dec_return(pcp->expire)) { 853 changed = true; 854 continue; 855 } 856 857 if (__this_cpu_read(pcp->count)) { 858 drain_zone_pages(zone, this_cpu_ptr(pcp)); 859 changed = true; 860 } 861 #endif 862 } 863 } 864 865 for_each_online_pgdat(pgdat) { 866 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 867 868 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 869 int v; 870 871 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 872 if (v) { 873 atomic_long_add(v, &pgdat->vm_stat[i]); 874 global_node_diff[i] += v; 875 } 876 } 877 } 878 879 if (fold_diff(global_zone_diff, global_node_diff)) 880 changed = true; 881 return changed; 882 } 883 884 /* 885 * Fold the data for an offline cpu into the global array. 886 * There cannot be any access by the offline cpu and therefore 887 * synchronization is simplified. 888 */ 889 void cpu_vm_stats_fold(int cpu) 890 { 891 struct pglist_data *pgdat; 892 struct zone *zone; 893 int i; 894 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 895 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 896 897 for_each_populated_zone(zone) { 898 struct per_cpu_zonestat *pzstats; 899 900 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 901 902 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 903 if (pzstats->vm_stat_diff[i]) { 904 int v; 905 906 v = pzstats->vm_stat_diff[i]; 907 pzstats->vm_stat_diff[i] = 0; 908 atomic_long_add(v, &zone->vm_stat[i]); 909 global_zone_diff[i] += v; 910 } 911 } 912 #ifdef CONFIG_NUMA 913 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 914 if (pzstats->vm_numa_event[i]) { 915 unsigned long v; 916 917 v = pzstats->vm_numa_event[i]; 918 pzstats->vm_numa_event[i] = 0; 919 zone_numa_event_add(v, zone, i); 920 } 921 } 922 #endif 923 } 924 925 for_each_online_pgdat(pgdat) { 926 struct per_cpu_nodestat *p; 927 928 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 929 930 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 931 if (p->vm_node_stat_diff[i]) { 932 int v; 933 934 v = p->vm_node_stat_diff[i]; 935 p->vm_node_stat_diff[i] = 0; 936 atomic_long_add(v, &pgdat->vm_stat[i]); 937 global_node_diff[i] += v; 938 } 939 } 940 941 fold_diff(global_zone_diff, global_node_diff); 942 } 943 944 /* 945 * this is only called if !populated_zone(zone), which implies no other users of 946 * pset->vm_stat_diff[] exist. 947 */ 948 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 949 { 950 unsigned long v; 951 int i; 952 953 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 954 if (pzstats->vm_stat_diff[i]) { 955 v = pzstats->vm_stat_diff[i]; 956 pzstats->vm_stat_diff[i] = 0; 957 zone_page_state_add(v, zone, i); 958 } 959 } 960 961 #ifdef CONFIG_NUMA 962 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 963 if (pzstats->vm_numa_event[i]) { 964 v = pzstats->vm_numa_event[i]; 965 pzstats->vm_numa_event[i] = 0; 966 zone_numa_event_add(v, zone, i); 967 } 968 } 969 #endif 970 } 971 #endif 972 973 #ifdef CONFIG_NUMA 974 /* 975 * Determine the per node value of a stat item. This function 976 * is called frequently in a NUMA machine, so try to be as 977 * frugal as possible. 978 */ 979 unsigned long sum_zone_node_page_state(int node, 980 enum zone_stat_item item) 981 { 982 struct zone *zones = NODE_DATA(node)->node_zones; 983 int i; 984 unsigned long count = 0; 985 986 for (i = 0; i < MAX_NR_ZONES; i++) 987 count += zone_page_state(zones + i, item); 988 989 return count; 990 } 991 992 /* Determine the per node value of a numa stat item. */ 993 unsigned long sum_zone_numa_event_state(int node, 994 enum numa_stat_item item) 995 { 996 struct zone *zones = NODE_DATA(node)->node_zones; 997 unsigned long count = 0; 998 int i; 999 1000 for (i = 0; i < MAX_NR_ZONES; i++) 1001 count += zone_numa_event_state(zones + i, item); 1002 1003 return count; 1004 } 1005 1006 /* 1007 * Determine the per node value of a stat item. 1008 */ 1009 unsigned long node_page_state_pages(struct pglist_data *pgdat, 1010 enum node_stat_item item) 1011 { 1012 long x = atomic_long_read(&pgdat->vm_stat[item]); 1013 #ifdef CONFIG_SMP 1014 if (x < 0) 1015 x = 0; 1016 #endif 1017 return x; 1018 } 1019 1020 unsigned long node_page_state(struct pglist_data *pgdat, 1021 enum node_stat_item item) 1022 { 1023 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1024 1025 return node_page_state_pages(pgdat, item); 1026 } 1027 #endif 1028 1029 /* 1030 * Count number of pages "struct page" and "struct page_ext" consume. 1031 * nr_memmap_boot_pages: # of pages allocated by boot allocator 1032 * nr_memmap_pages: # of pages that were allocated by buddy allocator 1033 */ 1034 static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); 1035 static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); 1036 1037 void memmap_boot_pages_add(long delta) 1038 { 1039 atomic_long_add(delta, &nr_memmap_boot_pages); 1040 } 1041 1042 void memmap_pages_add(long delta) 1043 { 1044 atomic_long_add(delta, &nr_memmap_pages); 1045 } 1046 1047 #ifdef CONFIG_COMPACTION 1048 1049 struct contig_page_info { 1050 unsigned long free_pages; 1051 unsigned long free_blocks_total; 1052 unsigned long free_blocks_suitable; 1053 }; 1054 1055 /* 1056 * Calculate the number of free pages in a zone, how many contiguous 1057 * pages are free and how many are large enough to satisfy an allocation of 1058 * the target size. Note that this function makes no attempt to estimate 1059 * how many suitable free blocks there *might* be if MOVABLE pages were 1060 * migrated. Calculating that is possible, but expensive and can be 1061 * figured out from userspace 1062 */ 1063 static void fill_contig_page_info(struct zone *zone, 1064 unsigned int suitable_order, 1065 struct contig_page_info *info) 1066 { 1067 unsigned int order; 1068 1069 info->free_pages = 0; 1070 info->free_blocks_total = 0; 1071 info->free_blocks_suitable = 0; 1072 1073 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1074 unsigned long blocks; 1075 1076 /* 1077 * Count number of free blocks. 1078 * 1079 * Access to nr_free is lockless as nr_free is used only for 1080 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1081 */ 1082 blocks = data_race(zone->free_area[order].nr_free); 1083 info->free_blocks_total += blocks; 1084 1085 /* Count free base pages */ 1086 info->free_pages += blocks << order; 1087 1088 /* Count the suitable free blocks */ 1089 if (order >= suitable_order) 1090 info->free_blocks_suitable += blocks << 1091 (order - suitable_order); 1092 } 1093 } 1094 1095 /* 1096 * A fragmentation index only makes sense if an allocation of a requested 1097 * size would fail. If that is true, the fragmentation index indicates 1098 * whether external fragmentation or a lack of memory was the problem. 1099 * The value can be used to determine if page reclaim or compaction 1100 * should be used 1101 */ 1102 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1103 { 1104 unsigned long requested = 1UL << order; 1105 1106 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1107 return 0; 1108 1109 if (!info->free_blocks_total) 1110 return 0; 1111 1112 /* Fragmentation index only makes sense when a request would fail */ 1113 if (info->free_blocks_suitable) 1114 return -1000; 1115 1116 /* 1117 * Index is between 0 and 1 so return within 3 decimal places 1118 * 1119 * 0 => allocation would fail due to lack of memory 1120 * 1 => allocation would fail due to fragmentation 1121 */ 1122 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1123 } 1124 1125 /* 1126 * Calculates external fragmentation within a zone wrt the given order. 1127 * It is defined as the percentage of pages found in blocks of size 1128 * less than 1 << order. It returns values in range [0, 100]. 1129 */ 1130 unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1131 { 1132 struct contig_page_info info; 1133 1134 fill_contig_page_info(zone, order, &info); 1135 if (info.free_pages == 0) 1136 return 0; 1137 1138 return div_u64((info.free_pages - 1139 (info.free_blocks_suitable << order)) * 100, 1140 info.free_pages); 1141 } 1142 1143 /* Same as __fragmentation index but allocs contig_page_info on stack */ 1144 int fragmentation_index(struct zone *zone, unsigned int order) 1145 { 1146 struct contig_page_info info; 1147 1148 fill_contig_page_info(zone, order, &info); 1149 return __fragmentation_index(order, &info); 1150 } 1151 #endif 1152 1153 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1154 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1155 #ifdef CONFIG_ZONE_DMA 1156 #define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma", 1157 #else 1158 #define TEXT_FOR_DMA(xx, yy) 1159 #endif 1160 1161 #ifdef CONFIG_ZONE_DMA32 1162 #define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32", 1163 #else 1164 #define TEXT_FOR_DMA32(xx, yy) 1165 #endif 1166 1167 #ifdef CONFIG_HIGHMEM 1168 #define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high", 1169 #else 1170 #define TEXT_FOR_HIGHMEM(xx, yy) 1171 #endif 1172 1173 #ifdef CONFIG_ZONE_DEVICE 1174 #define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device", 1175 #else 1176 #define TEXT_FOR_DEVICE(xx, yy) 1177 #endif 1178 1179 #define TEXTS_FOR_ZONES(xx, yy) \ 1180 TEXT_FOR_DMA(xx, yy) \ 1181 TEXT_FOR_DMA32(xx, yy) \ 1182 [xx##_NORMAL] = yy "_normal", \ 1183 TEXT_FOR_HIGHMEM(xx, yy) \ 1184 [xx##_MOVABLE] = yy "_movable", \ 1185 TEXT_FOR_DEVICE(xx, yy) 1186 1187 const char * const vmstat_text[] = { 1188 /* enum zone_stat_item counters */ 1189 #define I(x) (x) 1190 [I(NR_FREE_PAGES)] = "nr_free_pages", 1191 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks", 1192 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon", 1193 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon", 1194 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file", 1195 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file", 1196 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable", 1197 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending", 1198 [I(NR_MLOCK)] = "nr_mlock", 1199 #if IS_ENABLED(CONFIG_ZSMALLOC) 1200 [I(NR_ZSPAGES)] = "nr_zspages", 1201 #endif 1202 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma", 1203 #ifdef CONFIG_UNACCEPTED_MEMORY 1204 [I(NR_UNACCEPTED)] = "nr_unaccepted", 1205 #endif 1206 #undef I 1207 1208 /* enum numa_stat_item counters */ 1209 #define I(x) (NR_VM_ZONE_STAT_ITEMS + x) 1210 #ifdef CONFIG_NUMA 1211 [I(NUMA_HIT)] = "numa_hit", 1212 [I(NUMA_MISS)] = "numa_miss", 1213 [I(NUMA_FOREIGN)] = "numa_foreign", 1214 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave", 1215 [I(NUMA_LOCAL)] = "numa_local", 1216 [I(NUMA_OTHER)] = "numa_other", 1217 #endif 1218 #undef I 1219 1220 /* enum node_stat_item counters */ 1221 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x) 1222 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon", 1223 [I(NR_ACTIVE_ANON)] = "nr_active_anon", 1224 [I(NR_INACTIVE_FILE)] = "nr_inactive_file", 1225 [I(NR_ACTIVE_FILE)] = "nr_active_file", 1226 [I(NR_UNEVICTABLE)] = "nr_unevictable", 1227 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable", 1228 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable", 1229 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon", 1230 [I(NR_ISOLATED_FILE)] = "nr_isolated_file", 1231 [I(WORKINGSET_NODES)] = "workingset_nodes", 1232 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon", 1233 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file", 1234 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon", 1235 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file", 1236 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon", 1237 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file", 1238 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim", 1239 [I(NR_ANON_MAPPED)] = "nr_anon_pages", 1240 [I(NR_FILE_MAPPED)] = "nr_mapped", 1241 [I(NR_FILE_PAGES)] = "nr_file_pages", 1242 [I(NR_FILE_DIRTY)] = "nr_dirty", 1243 [I(NR_WRITEBACK)] = "nr_writeback", 1244 [I(NR_SHMEM)] = "nr_shmem", 1245 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages", 1246 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped", 1247 [I(NR_FILE_THPS)] = "nr_file_hugepages", 1248 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped", 1249 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages", 1250 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write", 1251 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim", 1252 [I(NR_DIRTIED)] = "nr_dirtied", 1253 [I(NR_WRITTEN)] = "nr_written", 1254 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written", 1255 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable", 1256 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired", 1257 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released", 1258 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack", 1259 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1260 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack", 1261 #endif 1262 [I(NR_PAGETABLE)] = "nr_page_table_pages", 1263 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages", 1264 #ifdef CONFIG_IOMMU_SUPPORT 1265 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages", 1266 #endif 1267 #ifdef CONFIG_SWAP 1268 [I(NR_SWAPCACHE)] = "nr_swapcached", 1269 #endif 1270 #ifdef CONFIG_NUMA_BALANCING 1271 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success", 1272 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate", 1273 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl", 1274 #endif 1275 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd", 1276 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct", 1277 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged", 1278 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive", 1279 #ifdef CONFIG_HUGETLB_PAGE 1280 [I(NR_HUGETLB)] = "nr_hugetlb", 1281 #endif 1282 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages", 1283 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages", 1284 [I(NR_GPU_ACTIVE)] = "nr_gpu_active", 1285 [I(NR_GPU_RECLAIM)] = "nr_gpu_reclaim", 1286 #undef I 1287 1288 /* system-wide enum vm_stat_item counters */ 1289 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1290 NR_VM_NODE_STAT_ITEMS + x) 1291 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold", 1292 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold", 1293 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages", 1294 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages", 1295 #undef I 1296 1297 #if defined(CONFIG_VM_EVENT_COUNTERS) 1298 /* enum vm_event_item counters */ 1299 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1300 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x) 1301 1302 [I(PGPGIN)] = "pgpgin", 1303 [I(PGPGOUT)] = "pgpgout", 1304 [I(PSWPIN)] = "pswpin", 1305 [I(PSWPOUT)] = "pswpout", 1306 1307 #define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1308 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS) 1309 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc") 1310 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall") 1311 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip") 1312 #undef OFF 1313 1314 [I(PGFREE)] = "pgfree", 1315 [I(PGACTIVATE)] = "pgactivate", 1316 [I(PGDEACTIVATE)] = "pgdeactivate", 1317 [I(PGLAZYFREE)] = "pglazyfree", 1318 1319 [I(PGFAULT)] = "pgfault", 1320 [I(PGMAJFAULT)] = "pgmajfault", 1321 [I(PGLAZYFREED)] = "pglazyfreed", 1322 1323 [I(PGREFILL)] = "pgrefill", 1324 [I(PGREUSE)] = "pgreuse", 1325 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd", 1326 [I(PGSTEAL_DIRECT)] = "pgsteal_direct", 1327 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged", 1328 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive", 1329 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd", 1330 [I(PGSCAN_DIRECT)] = "pgscan_direct", 1331 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged", 1332 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", 1333 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle", 1334 [I(PGSCAN_ANON)] = "pgscan_anon", 1335 [I(PGSCAN_FILE)] = "pgscan_file", 1336 [I(PGSTEAL_ANON)] = "pgsteal_anon", 1337 [I(PGSTEAL_FILE)] = "pgsteal_file", 1338 1339 #ifdef CONFIG_NUMA 1340 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success", 1341 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed", 1342 #endif 1343 [I(PGINODESTEAL)] = "pginodesteal", 1344 [I(SLABS_SCANNED)] = "slabs_scanned", 1345 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal", 1346 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly", 1347 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly", 1348 [I(PAGEOUTRUN)] = "pageoutrun", 1349 1350 [I(PGROTATED)] = "pgrotated", 1351 1352 [I(DROP_PAGECACHE)] = "drop_pagecache", 1353 [I(DROP_SLAB)] = "drop_slab", 1354 [I(OOM_KILL)] = "oom_kill", 1355 1356 #ifdef CONFIG_NUMA_BALANCING 1357 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates", 1358 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates", 1359 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults", 1360 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local", 1361 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated", 1362 #endif 1363 #ifdef CONFIG_MIGRATION 1364 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success", 1365 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail", 1366 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success", 1367 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail", 1368 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split", 1369 #endif 1370 #ifdef CONFIG_COMPACTION 1371 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned", 1372 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned", 1373 [I(COMPACTISOLATED)] = "compact_isolated", 1374 [I(COMPACTSTALL)] = "compact_stall", 1375 [I(COMPACTFAIL)] = "compact_fail", 1376 [I(COMPACTSUCCESS)] = "compact_success", 1377 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake", 1378 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned", 1379 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned", 1380 #endif 1381 1382 #ifdef CONFIG_HUGETLB_PAGE 1383 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success", 1384 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail", 1385 #endif 1386 #ifdef CONFIG_CMA 1387 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success", 1388 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail", 1389 #endif 1390 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled", 1391 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned", 1392 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued", 1393 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked", 1394 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked", 1395 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared", 1396 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded", 1397 1398 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1399 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc", 1400 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback", 1401 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge", 1402 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc", 1403 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed", 1404 [I(THP_FILE_ALLOC)] = "thp_file_alloc", 1405 [I(THP_FILE_FALLBACK)] = "thp_file_fallback", 1406 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge", 1407 [I(THP_FILE_MAPPED)] = "thp_file_mapped", 1408 [I(THP_SPLIT_PAGE)] = "thp_split_page", 1409 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed", 1410 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page", 1411 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page", 1412 [I(THP_SPLIT_PMD)] = "thp_split_pmd", 1413 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte", 1414 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte", 1415 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte", 1416 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1417 [I(THP_SPLIT_PUD)] = "thp_split_pud", 1418 #endif 1419 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc", 1420 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed", 1421 [I(THP_SWPOUT)] = "thp_swpout", 1422 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback", 1423 #endif 1424 #ifdef CONFIG_BALLOON 1425 [I(BALLOON_INFLATE)] = "balloon_inflate", 1426 [I(BALLOON_DEFLATE)] = "balloon_deflate", 1427 #ifdef CONFIG_BALLOON_MIGRATION 1428 [I(BALLOON_MIGRATE)] = "balloon_migrate", 1429 #endif /* CONFIG_BALLOON_MIGRATION */ 1430 #endif /* CONFIG_BALLOON */ 1431 #ifdef CONFIG_DEBUG_TLBFLUSH 1432 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush", 1433 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received", 1434 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all", 1435 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one", 1436 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1437 1438 #ifdef CONFIG_SWAP 1439 [I(SWAP_RA)] = "swap_ra", 1440 [I(SWAP_RA_HIT)] = "swap_ra_hit", 1441 [I(SWPIN_ZERO)] = "swpin_zero", 1442 [I(SWPOUT_ZERO)] = "swpout_zero", 1443 #ifdef CONFIG_KSM 1444 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy", 1445 #endif 1446 #endif 1447 #ifdef CONFIG_KSM 1448 [I(COW_KSM)] = "cow_ksm", 1449 #endif 1450 #ifdef CONFIG_ZSWAP 1451 [I(ZSWPIN)] = "zswpin", 1452 [I(ZSWPOUT)] = "zswpout", 1453 [I(ZSWPWB)] = "zswpwb", 1454 #endif 1455 #ifdef CONFIG_X86 1456 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits", 1457 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits", 1458 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses", 1459 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses", 1460 #endif 1461 #ifdef CONFIG_PER_VMA_LOCK_STATS 1462 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success", 1463 [I(VMA_LOCK_ABORT)] = "vma_lock_abort", 1464 [I(VMA_LOCK_RETRY)] = "vma_lock_retry", 1465 [I(VMA_LOCK_MISS)] = "vma_lock_miss", 1466 #endif 1467 #ifdef CONFIG_DEBUG_STACK_USAGE 1468 [I(KSTACK_1K)] = "kstack_1k", 1469 #if THREAD_SIZE > 1024 1470 [I(KSTACK_2K)] = "kstack_2k", 1471 #endif 1472 #if THREAD_SIZE > 2048 1473 [I(KSTACK_4K)] = "kstack_4k", 1474 #endif 1475 #if THREAD_SIZE > 4096 1476 [I(KSTACK_8K)] = "kstack_8k", 1477 #endif 1478 #if THREAD_SIZE > 8192 1479 [I(KSTACK_16K)] = "kstack_16k", 1480 #endif 1481 #if THREAD_SIZE > 16384 1482 [I(KSTACK_32K)] = "kstack_32k", 1483 #endif 1484 #if THREAD_SIZE > 32768 1485 [I(KSTACK_64K)] = "kstack_64k", 1486 #endif 1487 #if THREAD_SIZE > 65536 1488 [I(KSTACK_REST)] = "kstack_rest", 1489 #endif 1490 #endif 1491 #undef I 1492 #endif /* CONFIG_VM_EVENT_COUNTERS */ 1493 }; 1494 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1495 1496 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1497 defined(CONFIG_PROC_FS) 1498 static void *frag_start(struct seq_file *m, loff_t *pos) 1499 { 1500 pg_data_t *pgdat; 1501 loff_t node = *pos; 1502 1503 for (pgdat = first_online_pgdat(); 1504 pgdat && node; 1505 pgdat = next_online_pgdat(pgdat)) 1506 --node; 1507 1508 return pgdat; 1509 } 1510 1511 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1512 { 1513 pg_data_t *pgdat = (pg_data_t *)arg; 1514 1515 (*pos)++; 1516 return next_online_pgdat(pgdat); 1517 } 1518 1519 static void frag_stop(struct seq_file *m, void *arg) 1520 { 1521 } 1522 1523 /* 1524 * Walk zones in a node and print using a callback. 1525 * If @assert_populated is true, only use callback for zones that are populated. 1526 */ 1527 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1528 bool assert_populated, bool nolock, 1529 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1530 { 1531 struct zone *zone; 1532 struct zone *node_zones = pgdat->node_zones; 1533 unsigned long flags; 1534 1535 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1536 if (assert_populated && !populated_zone(zone)) 1537 continue; 1538 1539 if (!nolock) 1540 spin_lock_irqsave(&zone->lock, flags); 1541 print(m, pgdat, zone); 1542 if (!nolock) 1543 spin_unlock_irqrestore(&zone->lock, flags); 1544 } 1545 } 1546 #endif 1547 1548 #ifdef CONFIG_PROC_FS 1549 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1550 struct zone *zone) 1551 { 1552 int order; 1553 1554 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1555 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1556 /* 1557 * Access to nr_free is lockless as nr_free is used only for 1558 * printing purposes. Use data_race to avoid KCSAN warning. 1559 */ 1560 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1561 seq_putc(m, '\n'); 1562 } 1563 1564 /* 1565 * This walks the free areas for each zone. 1566 */ 1567 static int frag_show(struct seq_file *m, void *arg) 1568 { 1569 pg_data_t *pgdat = (pg_data_t *)arg; 1570 walk_zones_in_node(m, pgdat, true, false, frag_show_print); 1571 return 0; 1572 } 1573 1574 static void pagetypeinfo_showfree_print(struct seq_file *m, 1575 pg_data_t *pgdat, struct zone *zone) 1576 { 1577 int order, mtype; 1578 1579 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1580 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1581 pgdat->node_id, 1582 zone->name, 1583 migratetype_names[mtype]); 1584 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1585 unsigned long freecount = 0; 1586 struct free_area *area; 1587 struct list_head *curr; 1588 bool overflow = false; 1589 1590 area = &(zone->free_area[order]); 1591 1592 list_for_each(curr, &area->free_list[mtype]) { 1593 /* 1594 * Cap the free_list iteration because it might 1595 * be really large and we are under a spinlock 1596 * so a long time spent here could trigger a 1597 * hard lockup detector. Anyway this is a 1598 * debugging tool so knowing there is a handful 1599 * of pages of this order should be more than 1600 * sufficient. 1601 */ 1602 if (++freecount >= 100000) { 1603 overflow = true; 1604 break; 1605 } 1606 } 1607 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1608 spin_unlock_irq(&zone->lock); 1609 cond_resched(); 1610 spin_lock_irq(&zone->lock); 1611 } 1612 seq_putc(m, '\n'); 1613 } 1614 } 1615 1616 /* Print out the free pages at each order for each migratetype */ 1617 static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1618 { 1619 int order; 1620 pg_data_t *pgdat = (pg_data_t *)arg; 1621 1622 /* Print header */ 1623 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1624 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1625 seq_printf(m, "%6d ", order); 1626 seq_putc(m, '\n'); 1627 1628 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1629 } 1630 1631 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1632 pg_data_t *pgdat, struct zone *zone) 1633 { 1634 int mtype; 1635 unsigned long pfn; 1636 unsigned long start_pfn = zone->zone_start_pfn; 1637 unsigned long end_pfn = zone_end_pfn(zone); 1638 unsigned long count[MIGRATE_TYPES] = { 0, }; 1639 1640 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1641 struct page *page; 1642 1643 page = pfn_to_online_page(pfn); 1644 if (!page) 1645 continue; 1646 1647 if (page_zone(page) != zone) 1648 continue; 1649 1650 mtype = get_pageblock_migratetype(page); 1651 1652 if (mtype < MIGRATE_TYPES) 1653 count[mtype]++; 1654 } 1655 1656 /* Print counts */ 1657 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1658 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1659 seq_printf(m, "%12lu ", count[mtype]); 1660 seq_putc(m, '\n'); 1661 } 1662 1663 /* Print out the number of pageblocks for each migratetype */ 1664 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1665 { 1666 int mtype; 1667 pg_data_t *pgdat = (pg_data_t *)arg; 1668 1669 seq_printf(m, "\n%-23s", "Number of blocks type "); 1670 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1671 seq_printf(m, "%12s ", migratetype_names[mtype]); 1672 seq_putc(m, '\n'); 1673 walk_zones_in_node(m, pgdat, true, false, 1674 pagetypeinfo_showblockcount_print); 1675 } 1676 1677 /* 1678 * Print out the number of pageblocks for each migratetype that contain pages 1679 * of other types. This gives an indication of how well fallbacks are being 1680 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1681 * to determine what is going on 1682 */ 1683 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1684 { 1685 #ifdef CONFIG_PAGE_OWNER 1686 int mtype; 1687 1688 if (!static_branch_unlikely(&page_owner_inited)) 1689 return; 1690 1691 drain_all_pages(NULL); 1692 1693 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1694 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1695 seq_printf(m, "%12s ", migratetype_names[mtype]); 1696 seq_putc(m, '\n'); 1697 1698 walk_zones_in_node(m, pgdat, true, true, 1699 pagetypeinfo_showmixedcount_print); 1700 #endif /* CONFIG_PAGE_OWNER */ 1701 } 1702 1703 /* 1704 * This prints out statistics in relation to grouping pages by mobility. 1705 * It is expensive to collect so do not constantly read the file. 1706 */ 1707 static int pagetypeinfo_show(struct seq_file *m, void *arg) 1708 { 1709 pg_data_t *pgdat = (pg_data_t *)arg; 1710 1711 /* check memoryless node */ 1712 if (!node_state(pgdat->node_id, N_MEMORY)) 1713 return 0; 1714 1715 seq_printf(m, "Page block order: %d\n", pageblock_order); 1716 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1717 seq_putc(m, '\n'); 1718 pagetypeinfo_showfree(m, pgdat); 1719 pagetypeinfo_showblockcount(m, pgdat); 1720 pagetypeinfo_showmixedcount(m, pgdat); 1721 1722 return 0; 1723 } 1724 1725 static const struct seq_operations fragmentation_op = { 1726 .start = frag_start, 1727 .next = frag_next, 1728 .stop = frag_stop, 1729 .show = frag_show, 1730 }; 1731 1732 static const struct seq_operations pagetypeinfo_op = { 1733 .start = frag_start, 1734 .next = frag_next, 1735 .stop = frag_stop, 1736 .show = pagetypeinfo_show, 1737 }; 1738 1739 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1740 { 1741 int zid; 1742 1743 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1744 struct zone *compare = &pgdat->node_zones[zid]; 1745 1746 if (populated_zone(compare)) 1747 return zone == compare; 1748 } 1749 1750 return false; 1751 } 1752 1753 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1754 struct zone *zone) 1755 { 1756 int i; 1757 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1758 if (is_zone_first_populated(pgdat, zone)) { 1759 seq_printf(m, "\n per-node stats"); 1760 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1761 unsigned long pages = node_page_state_pages(pgdat, i); 1762 1763 if (vmstat_item_print_in_thp(i)) 1764 pages /= HPAGE_PMD_NR; 1765 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1766 pages); 1767 } 1768 } 1769 seq_printf(m, 1770 "\n pages free %lu" 1771 "\n boost %lu" 1772 "\n min %lu" 1773 "\n low %lu" 1774 "\n high %lu" 1775 "\n promo %lu" 1776 "\n spanned %lu" 1777 "\n present %lu" 1778 "\n managed %lu" 1779 "\n cma %lu", 1780 zone_page_state(zone, NR_FREE_PAGES), 1781 zone->watermark_boost, 1782 min_wmark_pages(zone), 1783 low_wmark_pages(zone), 1784 high_wmark_pages(zone), 1785 promo_wmark_pages(zone), 1786 zone->spanned_pages, 1787 zone->present_pages, 1788 zone_managed_pages(zone), 1789 zone_cma_pages(zone)); 1790 1791 seq_printf(m, 1792 "\n protection: (%ld", 1793 zone->lowmem_reserve[0]); 1794 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1795 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1796 seq_putc(m, ')'); 1797 1798 /* If unpopulated, no other information is useful */ 1799 if (!populated_zone(zone)) { 1800 seq_putc(m, '\n'); 1801 return; 1802 } 1803 1804 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1805 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1806 zone_page_state(zone, i)); 1807 1808 #ifdef CONFIG_NUMA 1809 fold_vm_zone_numa_events(zone); 1810 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1811 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1812 zone_numa_event_state(zone, i)); 1813 #endif 1814 1815 seq_printf(m, "\n pagesets"); 1816 for_each_online_cpu(i) { 1817 struct per_cpu_pages *pcp; 1818 struct per_cpu_zonestat __maybe_unused *pzstats; 1819 1820 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1821 seq_printf(m, 1822 "\n cpu: %i" 1823 "\n count: %i" 1824 "\n high: %i" 1825 "\n batch: %i" 1826 "\n high_min: %i" 1827 "\n high_max: %i", 1828 i, 1829 pcp->count, 1830 pcp->high, 1831 pcp->batch, 1832 pcp->high_min, 1833 pcp->high_max); 1834 #ifdef CONFIG_SMP 1835 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1836 seq_printf(m, "\n vm stats threshold: %d", 1837 pzstats->stat_threshold); 1838 #endif 1839 } 1840 seq_printf(m, 1841 "\n node_unreclaimable: %u" 1842 "\n start_pfn: %lu" 1843 "\n reserved_highatomic: %lu" 1844 "\n free_highatomic: %lu", 1845 kswapd_test_hopeless(pgdat), 1846 zone->zone_start_pfn, 1847 zone->nr_reserved_highatomic, 1848 zone->nr_free_highatomic); 1849 seq_putc(m, '\n'); 1850 } 1851 1852 /* 1853 * Output information about zones in @pgdat. All zones are printed regardless 1854 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1855 * set of all zones and userspace would not be aware of such zones if they are 1856 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1857 */ 1858 static int zoneinfo_show(struct seq_file *m, void *arg) 1859 { 1860 pg_data_t *pgdat = (pg_data_t *)arg; 1861 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1862 return 0; 1863 } 1864 1865 static const struct seq_operations zoneinfo_op = { 1866 .start = frag_start, /* iterate over all zones. The same as in 1867 * fragmentation. */ 1868 .next = frag_next, 1869 .stop = frag_stop, 1870 .show = zoneinfo_show, 1871 }; 1872 1873 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1874 NR_VM_NUMA_EVENT_ITEMS + \ 1875 NR_VM_NODE_STAT_ITEMS + \ 1876 NR_VM_STAT_ITEMS + \ 1877 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1878 NR_VM_EVENT_ITEMS : 0)) 1879 1880 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1881 { 1882 unsigned long *v; 1883 int i; 1884 1885 if (*pos >= NR_VMSTAT_ITEMS) 1886 return NULL; 1887 1888 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS); 1889 fold_vm_numa_events(); 1890 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1891 m->private = v; 1892 if (!v) 1893 return ERR_PTR(-ENOMEM); 1894 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1895 v[i] = global_zone_page_state(i); 1896 v += NR_VM_ZONE_STAT_ITEMS; 1897 1898 #ifdef CONFIG_NUMA 1899 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1900 v[i] = global_numa_event_state(i); 1901 v += NR_VM_NUMA_EVENT_ITEMS; 1902 #endif 1903 1904 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1905 v[i] = global_node_page_state_pages(i); 1906 if (vmstat_item_print_in_thp(i)) 1907 v[i] /= HPAGE_PMD_NR; 1908 } 1909 v += NR_VM_NODE_STAT_ITEMS; 1910 1911 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1912 v + NR_DIRTY_THRESHOLD); 1913 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); 1914 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); 1915 v += NR_VM_STAT_ITEMS; 1916 1917 #ifdef CONFIG_VM_EVENT_COUNTERS 1918 all_vm_events(v); 1919 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1920 v[PGPGOUT] /= 2; 1921 #endif 1922 return (unsigned long *)m->private + *pos; 1923 } 1924 1925 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1926 { 1927 (*pos)++; 1928 if (*pos >= NR_VMSTAT_ITEMS) 1929 return NULL; 1930 return (unsigned long *)m->private + *pos; 1931 } 1932 1933 static int vmstat_show(struct seq_file *m, void *arg) 1934 { 1935 unsigned long *l = arg; 1936 unsigned long off = l - (unsigned long *)m->private; 1937 1938 seq_puts(m, vmstat_text[off]); 1939 seq_put_decimal_ull(m, " ", *l); 1940 seq_putc(m, '\n'); 1941 1942 if (off == NR_VMSTAT_ITEMS - 1) { 1943 /* 1944 * We've come to the end - add any deprecated counters to avoid 1945 * breaking userspace which might depend on them being present. 1946 */ 1947 seq_puts(m, "nr_unstable 0\n"); 1948 } 1949 return 0; 1950 } 1951 1952 static void vmstat_stop(struct seq_file *m, void *arg) 1953 { 1954 kfree(m->private); 1955 m->private = NULL; 1956 } 1957 1958 static const struct seq_operations vmstat_op = { 1959 .start = vmstat_start, 1960 .next = vmstat_next, 1961 .stop = vmstat_stop, 1962 .show = vmstat_show, 1963 }; 1964 #endif /* CONFIG_PROC_FS */ 1965 1966 #ifdef CONFIG_SMP 1967 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1968 static int sysctl_stat_interval __read_mostly = HZ; 1969 static int vmstat_late_init_done; 1970 1971 #ifdef CONFIG_PROC_FS 1972 static void refresh_vm_stats(struct work_struct *work) 1973 { 1974 refresh_cpu_vm_stats(true); 1975 } 1976 1977 static int vmstat_refresh(const struct ctl_table *table, int write, 1978 void *buffer, size_t *lenp, loff_t *ppos) 1979 { 1980 long val; 1981 int err; 1982 int i; 1983 1984 /* 1985 * The regular update, every sysctl_stat_interval, may come later 1986 * than expected: leaving a significant amount in per_cpu buckets. 1987 * This is particularly misleading when checking a quantity of HUGE 1988 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 1989 * which can equally be echo'ed to or cat'ted from (by root), 1990 * can be used to update the stats just before reading them. 1991 * 1992 * Oh, and since global_zone_page_state() etc. are so careful to hide 1993 * transiently negative values, report an error here if any of 1994 * the stats is negative, so we know to go looking for imbalance. 1995 */ 1996 err = schedule_on_each_cpu(refresh_vm_stats); 1997 if (err) 1998 return err; 1999 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 2000 /* 2001 * Skip checking stats known to go negative occasionally. 2002 */ 2003 switch (i) { 2004 case NR_ZONE_WRITE_PENDING: 2005 case NR_FREE_CMA_PAGES: 2006 continue; 2007 } 2008 val = atomic_long_read(&vm_zone_stat[i]); 2009 if (val < 0) { 2010 pr_warn("%s: %s %ld\n", 2011 __func__, zone_stat_name(i), val); 2012 } 2013 } 2014 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 2015 /* 2016 * Skip checking stats known to go negative occasionally. 2017 */ 2018 switch (i) { 2019 case NR_WRITEBACK: 2020 continue; 2021 } 2022 val = atomic_long_read(&vm_node_stat[i]); 2023 if (val < 0) { 2024 pr_warn("%s: %s %ld\n", 2025 __func__, node_stat_name(i), val); 2026 } 2027 } 2028 if (write) 2029 *ppos += *lenp; 2030 else 2031 *lenp = 0; 2032 return 0; 2033 } 2034 #endif /* CONFIG_PROC_FS */ 2035 2036 static void vmstat_update(struct work_struct *w) 2037 { 2038 if (refresh_cpu_vm_stats(true)) { 2039 /* 2040 * Counters were updated so we expect more updates 2041 * to occur in the future. Keep on running the 2042 * update worker thread. 2043 */ 2044 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 2045 this_cpu_ptr(&vmstat_work), 2046 round_jiffies_relative(sysctl_stat_interval)); 2047 } 2048 } 2049 2050 /* 2051 * Check if the diffs for a certain cpu indicate that 2052 * an update is needed. 2053 */ 2054 static bool need_update(int cpu) 2055 { 2056 pg_data_t *last_pgdat = NULL; 2057 struct zone *zone; 2058 2059 for_each_populated_zone(zone) { 2060 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 2061 struct per_cpu_nodestat *n; 2062 2063 /* 2064 * The fast way of checking if there are any vmstat diffs. 2065 */ 2066 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 2067 return true; 2068 2069 if (last_pgdat == zone->zone_pgdat) 2070 continue; 2071 last_pgdat = zone->zone_pgdat; 2072 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 2073 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 2074 return true; 2075 } 2076 return false; 2077 } 2078 2079 /* 2080 * Switch off vmstat processing and then fold all the remaining differentials 2081 * until the diffs stay at zero. The function is used by NOHZ and can only be 2082 * invoked when tick processing is not active. 2083 */ 2084 void quiet_vmstat(void) 2085 { 2086 if (system_state != SYSTEM_RUNNING) 2087 return; 2088 2089 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2090 return; 2091 2092 if (!need_update(smp_processor_id())) 2093 return; 2094 2095 /* 2096 * Just refresh counters and do not care about the pending delayed 2097 * vmstat_update. It doesn't fire that often to matter and canceling 2098 * it would be too expensive from this path. 2099 * vmstat_shepherd will take care about that for us. 2100 */ 2101 refresh_cpu_vm_stats(false); 2102 } 2103 2104 /* 2105 * Shepherd worker thread that checks the 2106 * differentials of processors that have their worker 2107 * threads for vm statistics updates disabled because of 2108 * inactivity. 2109 */ 2110 static void vmstat_shepherd(struct work_struct *w); 2111 2112 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2113 2114 void vmstat_flush_workqueue(void) 2115 { 2116 flush_workqueue(mm_percpu_wq); 2117 } 2118 2119 static void vmstat_shepherd(struct work_struct *w) 2120 { 2121 int cpu; 2122 2123 cpus_read_lock(); 2124 /* Check processors whose vmstat worker threads have been disabled */ 2125 for_each_online_cpu(cpu) { 2126 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2127 2128 /* 2129 * In kernel users of vmstat counters either require the precise value and 2130 * they are using zone_page_state_snapshot interface or they can live with 2131 * an imprecision as the regular flushing can happen at arbitrary time and 2132 * cumulative error can grow (see calculate_normal_threshold). 2133 * 2134 * From that POV the regular flushing can be postponed for CPUs that have 2135 * been isolated from the kernel interference without critical 2136 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2137 * for all isolated CPUs to avoid interference with the isolated workload. 2138 */ 2139 scoped_guard(rcu) { 2140 if (cpu_is_isolated(cpu)) 2141 continue; 2142 2143 if (!delayed_work_pending(dw) && need_update(cpu)) 2144 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2145 } 2146 2147 cond_resched(); 2148 } 2149 cpus_read_unlock(); 2150 2151 schedule_delayed_work(&shepherd, 2152 round_jiffies_relative(sysctl_stat_interval)); 2153 } 2154 2155 static void __init start_shepherd_timer(void) 2156 { 2157 int cpu; 2158 2159 for_each_possible_cpu(cpu) { 2160 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2161 vmstat_update); 2162 2163 /* 2164 * For secondary CPUs during CPU hotplug scenarios, 2165 * vmstat_cpu_online() will enable the work. 2166 * mm/vmstat:online enables and disables vmstat_work 2167 * symmetrically during CPU hotplug events. 2168 */ 2169 if (!cpu_online(cpu)) 2170 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2171 } 2172 2173 schedule_delayed_work(&shepherd, 2174 round_jiffies_relative(sysctl_stat_interval)); 2175 } 2176 2177 static void __init init_cpu_node_state(void) 2178 { 2179 int node; 2180 2181 for_each_online_node(node) { 2182 if (!cpumask_empty(cpumask_of_node(node))) 2183 node_set_state(node, N_CPU); 2184 } 2185 } 2186 2187 static int vmstat_cpu_online(unsigned int cpu) 2188 { 2189 if (vmstat_late_init_done) 2190 refresh_zone_stat_thresholds(); 2191 2192 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2193 node_set_state(cpu_to_node(cpu), N_CPU); 2194 } 2195 enable_delayed_work(&per_cpu(vmstat_work, cpu)); 2196 2197 return 0; 2198 } 2199 2200 static int vmstat_cpu_down_prep(unsigned int cpu) 2201 { 2202 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2203 return 0; 2204 } 2205 2206 static int vmstat_cpu_dead(unsigned int cpu) 2207 { 2208 const struct cpumask *node_cpus; 2209 int node; 2210 2211 node = cpu_to_node(cpu); 2212 2213 refresh_zone_stat_thresholds(); 2214 node_cpus = cpumask_of_node(node); 2215 if (!cpumask_empty(node_cpus)) 2216 return 0; 2217 2218 node_clear_state(node, N_CPU); 2219 2220 return 0; 2221 } 2222 2223 static int __init vmstat_late_init(void) 2224 { 2225 refresh_zone_stat_thresholds(); 2226 vmstat_late_init_done = 1; 2227 2228 return 0; 2229 } 2230 late_initcall(vmstat_late_init); 2231 #endif 2232 2233 #ifdef CONFIG_PROC_FS 2234 static const struct ctl_table vmstat_table[] = { 2235 #ifdef CONFIG_SMP 2236 { 2237 .procname = "stat_interval", 2238 .data = &sysctl_stat_interval, 2239 .maxlen = sizeof(sysctl_stat_interval), 2240 .mode = 0644, 2241 .proc_handler = proc_dointvec_jiffies, 2242 }, 2243 { 2244 .procname = "stat_refresh", 2245 .data = NULL, 2246 .maxlen = 0, 2247 .mode = 0600, 2248 .proc_handler = vmstat_refresh, 2249 }, 2250 #endif 2251 #ifdef CONFIG_NUMA 2252 { 2253 .procname = "numa_stat", 2254 .data = &sysctl_vm_numa_stat, 2255 .maxlen = sizeof(int), 2256 .mode = 0644, 2257 .proc_handler = sysctl_vm_numa_stat_handler, 2258 .extra1 = SYSCTL_ZERO, 2259 .extra2 = SYSCTL_ONE, 2260 }, 2261 #endif 2262 }; 2263 #endif 2264 2265 struct workqueue_struct *mm_percpu_wq; 2266 2267 void __init init_mm_internals(void) 2268 { 2269 int ret __maybe_unused; 2270 2271 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", 2272 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2273 2274 #ifdef CONFIG_SMP 2275 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2276 NULL, vmstat_cpu_dead); 2277 if (ret < 0) 2278 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2279 2280 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2281 vmstat_cpu_online, 2282 vmstat_cpu_down_prep); 2283 if (ret < 0) 2284 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2285 2286 cpus_read_lock(); 2287 init_cpu_node_state(); 2288 cpus_read_unlock(); 2289 2290 start_shepherd_timer(); 2291 #endif 2292 #ifdef CONFIG_PROC_FS 2293 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2294 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2295 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2296 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2297 register_sysctl_init("vm", vmstat_table); 2298 #endif 2299 } 2300 2301 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2302 2303 /* 2304 * Return an index indicating how much of the available free memory is 2305 * unusable for an allocation of the requested size. 2306 */ 2307 static int unusable_free_index(unsigned int order, 2308 struct contig_page_info *info) 2309 { 2310 /* No free memory is interpreted as all free memory is unusable */ 2311 if (info->free_pages == 0) 2312 return 1000; 2313 2314 /* 2315 * Index should be a value between 0 and 1. Return a value to 3 2316 * decimal places. 2317 * 2318 * 0 => no fragmentation 2319 * 1 => high fragmentation 2320 */ 2321 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2322 2323 } 2324 2325 static void unusable_show_print(struct seq_file *m, 2326 pg_data_t *pgdat, struct zone *zone) 2327 { 2328 unsigned int order; 2329 int index; 2330 struct contig_page_info info; 2331 2332 seq_printf(m, "Node %d, zone %8s ", 2333 pgdat->node_id, 2334 zone->name); 2335 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2336 fill_contig_page_info(zone, order, &info); 2337 index = unusable_free_index(order, &info); 2338 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2339 } 2340 2341 seq_putc(m, '\n'); 2342 } 2343 2344 /* 2345 * Display unusable free space index 2346 * 2347 * The unusable free space index measures how much of the available free 2348 * memory cannot be used to satisfy an allocation of a given size and is a 2349 * value between 0 and 1. The higher the value, the more of free memory is 2350 * unusable and by implication, the worse the external fragmentation is. This 2351 * can be expressed as a percentage by multiplying by 100. 2352 */ 2353 static int unusable_show(struct seq_file *m, void *arg) 2354 { 2355 pg_data_t *pgdat = (pg_data_t *)arg; 2356 2357 /* check memoryless node */ 2358 if (!node_state(pgdat->node_id, N_MEMORY)) 2359 return 0; 2360 2361 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2362 2363 return 0; 2364 } 2365 2366 static const struct seq_operations unusable_sops = { 2367 .start = frag_start, 2368 .next = frag_next, 2369 .stop = frag_stop, 2370 .show = unusable_show, 2371 }; 2372 2373 DEFINE_SEQ_ATTRIBUTE(unusable); 2374 2375 static void extfrag_show_print(struct seq_file *m, 2376 pg_data_t *pgdat, struct zone *zone) 2377 { 2378 unsigned int order; 2379 int index; 2380 2381 /* Alloc on stack as interrupts are disabled for zone walk */ 2382 struct contig_page_info info; 2383 2384 seq_printf(m, "Node %d, zone %8s ", 2385 pgdat->node_id, 2386 zone->name); 2387 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2388 fill_contig_page_info(zone, order, &info); 2389 index = __fragmentation_index(order, &info); 2390 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2391 } 2392 2393 seq_putc(m, '\n'); 2394 } 2395 2396 /* 2397 * Display fragmentation index for orders that allocations would fail for 2398 */ 2399 static int extfrag_show(struct seq_file *m, void *arg) 2400 { 2401 pg_data_t *pgdat = (pg_data_t *)arg; 2402 2403 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2404 2405 return 0; 2406 } 2407 2408 static const struct seq_operations extfrag_sops = { 2409 .start = frag_start, 2410 .next = frag_next, 2411 .stop = frag_stop, 2412 .show = extfrag_show, 2413 }; 2414 2415 DEFINE_SEQ_ATTRIBUTE(extfrag); 2416 2417 static int __init extfrag_debug_init(void) 2418 { 2419 struct dentry *extfrag_debug_root; 2420 2421 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2422 2423 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2424 &unusable_fops); 2425 2426 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2427 &extfrag_fops); 2428 2429 return 0; 2430 } 2431 2432 module_init(extfrag_debug_init); 2433 2434 #endif 2435