1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <cl@gentwo.org> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13 #include <linux/fs.h> 14 #include <linux/mm.h> 15 #include <linux/err.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/cpumask.h> 20 #include <linux/vmstat.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/debugfs.h> 24 #include <linux/sched.h> 25 #include <linux/math64.h> 26 #include <linux/writeback.h> 27 #include <linux/compaction.h> 28 #include <linux/mm_inline.h> 29 #include <linux/page_owner.h> 30 #include <linux/sched/isolation.h> 31 32 #include "internal.h" 33 #include "page_alloc.h" 34 35 #ifdef CONFIG_PROC_FS 36 #ifdef CONFIG_NUMA 37 #define ENABLE_NUMA_STAT 1 38 static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 39 40 /* zero numa counters within a zone */ 41 static void zero_zone_numa_counters(struct zone *zone) 42 { 43 int item, cpu; 44 45 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 46 atomic_long_set(&zone->vm_numa_event[item], 0); 47 for_each_online_cpu(cpu) { 48 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 49 = 0; 50 } 51 } 52 } 53 54 /* zero numa counters of all the populated zones */ 55 static void zero_zones_numa_counters(void) 56 { 57 struct zone *zone; 58 59 for_each_populated_zone(zone) 60 zero_zone_numa_counters(zone); 61 } 62 63 /* zero global numa counters */ 64 static void zero_global_numa_counters(void) 65 { 66 int item; 67 68 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 69 atomic_long_set(&vm_numa_event[item], 0); 70 } 71 72 static void invalid_numa_statistics(void) 73 { 74 zero_zones_numa_counters(); 75 zero_global_numa_counters(); 76 } 77 78 static DEFINE_MUTEX(vm_numa_stat_lock); 79 80 static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, 81 void *buffer, size_t *length, loff_t *ppos) 82 { 83 int ret, oldval; 84 85 mutex_lock(&vm_numa_stat_lock); 86 if (write) 87 oldval = sysctl_vm_numa_stat; 88 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 89 if (ret || !write) 90 goto out; 91 92 if (oldval == sysctl_vm_numa_stat) 93 goto out; 94 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 95 static_branch_enable(&vm_numa_stat_key); 96 pr_info("enable numa statistics\n"); 97 } else { 98 static_branch_disable(&vm_numa_stat_key); 99 invalid_numa_statistics(); 100 pr_info("disable numa statistics, and clear numa counters\n"); 101 } 102 103 out: 104 mutex_unlock(&vm_numa_stat_lock); 105 return ret; 106 } 107 #endif 108 #endif /* CONFIG_PROC_FS */ 109 110 #ifdef CONFIG_VM_EVENT_COUNTERS 111 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 112 EXPORT_PER_CPU_SYMBOL(vm_event_states); 113 114 static void sum_vm_events(unsigned long *ret) 115 { 116 int cpu; 117 int i; 118 119 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 120 121 for_each_online_cpu(cpu) { 122 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 123 124 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 125 ret[i] += this->event[i]; 126 } 127 } 128 129 /* 130 * Accumulate the vm event counters across all CPUs. 131 * The result is unavoidably approximate - it can change 132 * during and after execution of this function. 133 */ 134 void all_vm_events(unsigned long *ret) 135 { 136 cpus_read_lock(); 137 sum_vm_events(ret); 138 cpus_read_unlock(); 139 } 140 EXPORT_SYMBOL_GPL(all_vm_events); 141 142 /* 143 * Fold the foreign cpu events into our own. 144 * 145 * This is adding to the events on one processor 146 * but keeps the global counts constant. 147 */ 148 void vm_events_fold_cpu(int cpu) 149 { 150 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 151 int i; 152 153 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 154 count_vm_events(i, fold_state->event[i]); 155 fold_state->event[i] = 0; 156 } 157 } 158 159 #endif /* CONFIG_VM_EVENT_COUNTERS */ 160 161 /* 162 * Manage combined zone based / global counters 163 * 164 * vm_stat contains the global counters 165 */ 166 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 167 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 168 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 169 EXPORT_SYMBOL(vm_zone_stat); 170 EXPORT_SYMBOL(vm_node_stat); 171 172 #ifdef CONFIG_NUMA 173 static void fold_vm_zone_numa_events(struct zone *zone) 174 { 175 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 176 int cpu; 177 enum numa_stat_item item; 178 179 for_each_online_cpu(cpu) { 180 struct per_cpu_zonestat *pzstats; 181 182 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 183 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 184 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 185 } 186 187 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 188 zone_numa_event_add(zone_numa_events[item], zone, item); 189 } 190 191 void fold_vm_numa_events(void) 192 { 193 struct zone *zone; 194 195 for_each_populated_zone(zone) 196 fold_vm_zone_numa_events(zone); 197 } 198 #endif 199 200 #ifdef CONFIG_SMP 201 202 int calculate_pressure_threshold(struct zone *zone) 203 { 204 int threshold; 205 int watermark_distance; 206 207 /* 208 * As vmstats are not up to date, there is drift between the estimated 209 * and real values. For high thresholds and a high number of CPUs, it 210 * is possible for the min watermark to be breached while the estimated 211 * value looks fine. The pressure threshold is a reduced value such 212 * that even the maximum amount of drift will not accidentally breach 213 * the min watermark 214 */ 215 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 216 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 217 218 /* 219 * Maximum threshold is 125 220 */ 221 threshold = min(125, threshold); 222 223 return threshold; 224 } 225 226 int calculate_normal_threshold(struct zone *zone) 227 { 228 int threshold; 229 int mem; /* memory in 128 MB units */ 230 231 /* 232 * The threshold scales with the number of processors and the amount 233 * of memory per zone. More memory means that we can defer updates for 234 * longer, more processors could lead to more contention. 235 * fls() is used to have a cheap way of logarithmic scaling. 236 * 237 * Some sample thresholds: 238 * 239 * Threshold Processors (fls) Zonesize fls(mem)+1 240 * ------------------------------------------------------------------ 241 * 8 1 1 0.9-1 GB 4 242 * 16 2 2 0.9-1 GB 4 243 * 20 2 2 1-2 GB 5 244 * 24 2 2 2-4 GB 6 245 * 28 2 2 4-8 GB 7 246 * 32 2 2 8-16 GB 8 247 * 4 2 2 <128M 1 248 * 30 4 3 2-4 GB 5 249 * 48 4 3 8-16 GB 8 250 * 32 8 4 1-2 GB 4 251 * 32 8 4 0.9-1GB 4 252 * 10 16 5 <128M 1 253 * 40 16 5 900M 4 254 * 70 64 7 2-4 GB 5 255 * 84 64 7 4-8 GB 6 256 * 108 512 9 4-8 GB 6 257 * 125 1024 10 8-16 GB 8 258 * 125 1024 10 16-32 GB 9 259 */ 260 261 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 262 263 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 264 265 /* 266 * Maximum threshold is 125 267 */ 268 threshold = min(125, threshold); 269 270 return threshold; 271 } 272 273 /* 274 * Refresh the thresholds for each zone. 275 */ 276 void refresh_zone_stat_thresholds(void) 277 { 278 struct pglist_data *pgdat; 279 struct zone *zone; 280 int cpu; 281 int threshold; 282 283 /* Zero current pgdat thresholds */ 284 for_each_online_pgdat(pgdat) { 285 for_each_online_cpu(cpu) { 286 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 287 } 288 } 289 290 for_each_populated_zone(zone) { 291 struct pglist_data *pgdat = zone->zone_pgdat; 292 unsigned long max_drift, tolerate_drift; 293 294 threshold = calculate_normal_threshold(zone); 295 296 for_each_online_cpu(cpu) { 297 int pgdat_threshold; 298 299 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 300 = threshold; 301 302 /* Base nodestat threshold on the largest populated zone. */ 303 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 304 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 305 = max(threshold, pgdat_threshold); 306 } 307 308 /* 309 * Only set percpu_drift_mark if there is a danger that 310 * NR_FREE_PAGES reports the low watermark is ok when in fact 311 * the min watermark could be breached by an allocation 312 */ 313 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 314 max_drift = num_online_cpus() * threshold; 315 if (max_drift > tolerate_drift) 316 zone->percpu_drift_mark = high_wmark_pages(zone) + 317 max_drift; 318 } 319 } 320 321 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 322 int (*calculate_pressure)(struct zone *)) 323 { 324 struct zone *zone; 325 int cpu; 326 int threshold; 327 int i; 328 329 for (i = 0; i < pgdat->nr_zones; i++) { 330 zone = &pgdat->node_zones[i]; 331 if (!zone->percpu_drift_mark) 332 continue; 333 334 threshold = (*calculate_pressure)(zone); 335 for_each_online_cpu(cpu) 336 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 337 = threshold; 338 } 339 } 340 341 /* 342 * For use when we know that interrupts are disabled, 343 * or when we know that preemption is disabled and that 344 * particular counter cannot be updated from interrupt context. 345 */ 346 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 347 long delta) 348 { 349 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 350 s8 __percpu *p = pcp->vm_stat_diff + item; 351 long x; 352 long t; 353 354 /* 355 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 356 * atomicity is provided by IRQs being disabled -- either explicitly 357 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 358 * CPU migrations and preemption potentially corrupts a counter so 359 * disable preemption. 360 */ 361 preempt_disable_nested(); 362 363 x = delta + __this_cpu_read(*p); 364 365 t = __this_cpu_read(pcp->stat_threshold); 366 367 if (unlikely(abs(x) > t)) { 368 zone_page_state_add(x, zone, item); 369 x = 0; 370 } 371 __this_cpu_write(*p, x); 372 373 preempt_enable_nested(); 374 } 375 EXPORT_SYMBOL(__mod_zone_page_state); 376 377 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 378 long delta) 379 { 380 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 381 s8 __percpu *p = pcp->vm_node_stat_diff + item; 382 long x; 383 long t; 384 385 if (vmstat_item_in_bytes(item)) { 386 /* 387 * Only cgroups use subpage accounting right now; at 388 * the global level, these items still change in 389 * multiples of whole pages. Store them as pages 390 * internally to keep the per-cpu counters compact. 391 */ 392 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 393 delta >>= PAGE_SHIFT; 394 } 395 396 /* See __mod_zone_page_state() */ 397 preempt_disable_nested(); 398 399 x = delta + __this_cpu_read(*p); 400 401 t = __this_cpu_read(pcp->stat_threshold); 402 403 if (unlikely(abs(x) > t)) { 404 node_page_state_add(x, pgdat, item); 405 x = 0; 406 } 407 __this_cpu_write(*p, x); 408 409 preempt_enable_nested(); 410 } 411 EXPORT_SYMBOL(__mod_node_page_state); 412 413 /* 414 * Optimized increment and decrement functions. 415 * 416 * These are only for a single page and therefore can take a struct page * 417 * argument instead of struct zone *. This allows the inclusion of the code 418 * generated for page_zone(page) into the optimized functions. 419 * 420 * No overflow check is necessary and therefore the differential can be 421 * incremented or decremented in place which may allow the compilers to 422 * generate better code. 423 * The increment or decrement is known and therefore one boundary check can 424 * be omitted. 425 * 426 * NOTE: These functions are very performance sensitive. Change only 427 * with care. 428 * 429 * Some processors have inc/dec instructions that are atomic vs an interrupt. 430 * However, the code must first determine the differential location in a zone 431 * based on the processor number and then inc/dec the counter. There is no 432 * guarantee without disabling preemption that the processor will not change 433 * in between and therefore the atomicity vs. interrupt cannot be exploited 434 * in a useful way here. 435 */ 436 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 437 { 438 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 439 s8 __percpu *p = pcp->vm_stat_diff + item; 440 s8 v, t; 441 442 /* See __mod_zone_page_state() */ 443 preempt_disable_nested(); 444 445 v = __this_cpu_inc_return(*p); 446 t = __this_cpu_read(pcp->stat_threshold); 447 if (unlikely(v > t)) { 448 s8 overstep = t >> 1; 449 450 zone_page_state_add(v + overstep, zone, item); 451 __this_cpu_write(*p, -overstep); 452 } 453 454 preempt_enable_nested(); 455 } 456 457 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 458 { 459 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 460 s8 __percpu *p = pcp->vm_node_stat_diff + item; 461 s8 v, t; 462 463 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 464 465 /* See __mod_zone_page_state() */ 466 preempt_disable_nested(); 467 468 v = __this_cpu_inc_return(*p); 469 t = __this_cpu_read(pcp->stat_threshold); 470 if (unlikely(v > t)) { 471 s8 overstep = t >> 1; 472 473 node_page_state_add(v + overstep, pgdat, item); 474 __this_cpu_write(*p, -overstep); 475 } 476 477 preempt_enable_nested(); 478 } 479 480 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 481 { 482 __inc_zone_state(page_zone(page), item); 483 } 484 EXPORT_SYMBOL(__inc_zone_page_state); 485 486 void __inc_node_page_state(struct page *page, enum node_stat_item item) 487 { 488 __inc_node_state(page_pgdat(page), item); 489 } 490 EXPORT_SYMBOL(__inc_node_page_state); 491 492 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 493 { 494 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 495 s8 __percpu *p = pcp->vm_stat_diff + item; 496 s8 v, t; 497 498 /* See __mod_zone_page_state() */ 499 preempt_disable_nested(); 500 501 v = __this_cpu_dec_return(*p); 502 t = __this_cpu_read(pcp->stat_threshold); 503 if (unlikely(v < - t)) { 504 s8 overstep = t >> 1; 505 506 zone_page_state_add(v - overstep, zone, item); 507 __this_cpu_write(*p, overstep); 508 } 509 510 preempt_enable_nested(); 511 } 512 513 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 514 { 515 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 516 s8 __percpu *p = pcp->vm_node_stat_diff + item; 517 s8 v, t; 518 519 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 520 521 /* See __mod_zone_page_state() */ 522 preempt_disable_nested(); 523 524 v = __this_cpu_dec_return(*p); 525 t = __this_cpu_read(pcp->stat_threshold); 526 if (unlikely(v < - t)) { 527 s8 overstep = t >> 1; 528 529 node_page_state_add(v - overstep, pgdat, item); 530 __this_cpu_write(*p, overstep); 531 } 532 533 preempt_enable_nested(); 534 } 535 536 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 537 { 538 __dec_zone_state(page_zone(page), item); 539 } 540 EXPORT_SYMBOL(__dec_zone_page_state); 541 542 void __dec_node_page_state(struct page *page, enum node_stat_item item) 543 { 544 __dec_node_state(page_pgdat(page), item); 545 } 546 EXPORT_SYMBOL(__dec_node_page_state); 547 548 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 549 /* 550 * If we have cmpxchg_local support then we do not need to incur the overhead 551 * that comes with local_irq_save/restore if we use this_cpu_try_cmpxchg(). 552 * 553 * mod_state() modifies the zone counter state through atomic per cpu 554 * operations. 555 * 556 * Overstep mode specifies how overstep should handled: 557 * 0 No overstepping 558 * 1 Overstepping half of threshold 559 * -1 Overstepping minus half of threshold 560 */ 561 static inline void mod_zone_state(struct zone *zone, 562 enum zone_stat_item item, long delta, int overstep_mode) 563 { 564 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 565 s8 __percpu *p = pcp->vm_stat_diff + item; 566 long n, t, z; 567 s8 o; 568 569 o = this_cpu_read(*p); 570 do { 571 z = 0; /* overflow to zone counters */ 572 573 /* 574 * The fetching of the stat_threshold is racy. We may apply 575 * a counter threshold to the wrong the cpu if we get 576 * rescheduled while executing here. However, the next 577 * counter update will apply the threshold again and 578 * therefore bring the counter under the threshold again. 579 * 580 * Most of the time the thresholds are the same anyways 581 * for all cpus in a zone. 582 */ 583 t = this_cpu_read(pcp->stat_threshold); 584 585 n = delta + (long)o; 586 587 if (abs(n) > t) { 588 int os = overstep_mode * (t >> 1) ; 589 590 /* Overflow must be added to zone counters */ 591 z = n + os; 592 n = -os; 593 } 594 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 595 596 if (z) 597 zone_page_state_add(z, zone, item); 598 } 599 600 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 601 long delta) 602 { 603 mod_zone_state(zone, item, delta, 0); 604 } 605 EXPORT_SYMBOL(mod_zone_page_state); 606 607 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 608 { 609 mod_zone_state(page_zone(page), item, 1, 1); 610 } 611 EXPORT_SYMBOL(inc_zone_page_state); 612 613 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 614 { 615 mod_zone_state(page_zone(page), item, -1, -1); 616 } 617 EXPORT_SYMBOL(dec_zone_page_state); 618 619 static inline void mod_node_state(struct pglist_data *pgdat, 620 enum node_stat_item item, int delta, int overstep_mode) 621 { 622 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 623 s8 __percpu *p = pcp->vm_node_stat_diff + item; 624 long n, t, z; 625 s8 o; 626 627 if (vmstat_item_in_bytes(item)) { 628 /* 629 * Only cgroups use subpage accounting right now; at 630 * the global level, these items still change in 631 * multiples of whole pages. Store them as pages 632 * internally to keep the per-cpu counters compact. 633 */ 634 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 635 delta >>= PAGE_SHIFT; 636 } 637 638 o = this_cpu_read(*p); 639 do { 640 z = 0; /* overflow to node counters */ 641 642 /* 643 * The fetching of the stat_threshold is racy. We may apply 644 * a counter threshold to the wrong the cpu if we get 645 * rescheduled while executing here. However, the next 646 * counter update will apply the threshold again and 647 * therefore bring the counter under the threshold again. 648 * 649 * Most of the time the thresholds are the same anyways 650 * for all cpus in a node. 651 */ 652 t = this_cpu_read(pcp->stat_threshold); 653 654 n = delta + (long)o; 655 656 if (abs(n) > t) { 657 int os = overstep_mode * (t >> 1) ; 658 659 /* Overflow must be added to node counters */ 660 z = n + os; 661 n = -os; 662 } 663 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 664 665 if (z) 666 node_page_state_add(z, pgdat, item); 667 } 668 669 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 670 long delta) 671 { 672 mod_node_state(pgdat, item, delta, 0); 673 } 674 EXPORT_SYMBOL(mod_node_page_state); 675 676 void inc_node_page_state(struct page *page, enum node_stat_item item) 677 { 678 mod_node_state(page_pgdat(page), item, 1, 1); 679 } 680 EXPORT_SYMBOL(inc_node_page_state); 681 682 void dec_node_page_state(struct page *page, enum node_stat_item item) 683 { 684 mod_node_state(page_pgdat(page), item, -1, -1); 685 } 686 EXPORT_SYMBOL(dec_node_page_state); 687 #else 688 /* 689 * Use interrupt disable to serialize counter updates 690 */ 691 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 692 long delta) 693 { 694 unsigned long flags; 695 696 local_irq_save(flags); 697 __mod_zone_page_state(zone, item, delta); 698 local_irq_restore(flags); 699 } 700 EXPORT_SYMBOL(mod_zone_page_state); 701 702 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 703 { 704 unsigned long flags; 705 struct zone *zone; 706 707 zone = page_zone(page); 708 local_irq_save(flags); 709 __inc_zone_state(zone, item); 710 local_irq_restore(flags); 711 } 712 EXPORT_SYMBOL(inc_zone_page_state); 713 714 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 715 { 716 unsigned long flags; 717 718 local_irq_save(flags); 719 __dec_zone_page_state(page, item); 720 local_irq_restore(flags); 721 } 722 EXPORT_SYMBOL(dec_zone_page_state); 723 724 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 725 long delta) 726 { 727 unsigned long flags; 728 729 local_irq_save(flags); 730 __mod_node_page_state(pgdat, item, delta); 731 local_irq_restore(flags); 732 } 733 EXPORT_SYMBOL(mod_node_page_state); 734 735 void inc_node_page_state(struct page *page, enum node_stat_item item) 736 { 737 unsigned long flags; 738 struct pglist_data *pgdat; 739 740 pgdat = page_pgdat(page); 741 local_irq_save(flags); 742 __inc_node_state(pgdat, item); 743 local_irq_restore(flags); 744 } 745 EXPORT_SYMBOL(inc_node_page_state); 746 747 void dec_node_page_state(struct page *page, enum node_stat_item item) 748 { 749 unsigned long flags; 750 751 local_irq_save(flags); 752 __dec_node_page_state(page, item); 753 local_irq_restore(flags); 754 } 755 EXPORT_SYMBOL(dec_node_page_state); 756 #endif 757 758 /* 759 * Fold a differential into the global counters. 760 * Returns whether counters were updated. 761 */ 762 static int fold_diff(int *zone_diff, int *node_diff) 763 { 764 int i; 765 bool changed = false; 766 767 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 768 if (zone_diff[i]) { 769 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 770 changed = true; 771 } 772 } 773 774 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 775 if (node_diff[i]) { 776 atomic_long_add(node_diff[i], &vm_node_stat[i]); 777 changed = true; 778 } 779 } 780 781 return changed; 782 } 783 784 /* 785 * Update the zone counters for the current cpu. 786 * 787 * Note that refresh_cpu_vm_stats strives to only access 788 * node local memory. The per cpu pagesets on remote zones are placed 789 * in the memory local to the processor using that pageset. So the 790 * loop over all zones will access a series of cachelines local to 791 * the processor. 792 * 793 * The call to zone_page_state_add updates the cachelines with the 794 * statistics in the remote zone struct as well as the global cachelines 795 * with the global counters. These could cause remote node cache line 796 * bouncing and will have to be only done when necessary. 797 * 798 * The function returns whether global counters were updated. 799 */ 800 static bool refresh_cpu_vm_stats(bool do_pagesets) 801 { 802 struct pglist_data *pgdat; 803 struct zone *zone; 804 int i; 805 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 806 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 807 bool changed = false; 808 809 for_each_populated_zone(zone) { 810 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 811 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 812 813 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 814 int v; 815 816 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 817 if (v) { 818 819 atomic_long_add(v, &zone->vm_stat[i]); 820 global_zone_diff[i] += v; 821 #ifdef CONFIG_NUMA 822 /* 3 seconds idle till flush */ 823 __this_cpu_write(pcp->expire, 3); 824 #endif 825 } 826 } 827 828 if (do_pagesets) { 829 cond_resched(); 830 831 if (decay_pcp_high(zone, this_cpu_ptr(pcp))) 832 changed = true; 833 #ifdef CONFIG_NUMA 834 /* 835 * Deal with draining the remote pageset of this 836 * processor 837 * 838 * Check if there are pages remaining in this pageset 839 * if not then there is nothing to expire. 840 */ 841 if (!__this_cpu_read(pcp->expire) || 842 !__this_cpu_read(pcp->count)) 843 continue; 844 845 /* 846 * We never drain zones local to this processor. 847 */ 848 if (zone_to_nid(zone) == numa_node_id()) { 849 __this_cpu_write(pcp->expire, 0); 850 continue; 851 } 852 853 if (__this_cpu_dec_return(pcp->expire)) { 854 changed = true; 855 continue; 856 } 857 858 if (__this_cpu_read(pcp->count)) { 859 drain_zone_pages(zone, this_cpu_ptr(pcp)); 860 changed = true; 861 } 862 #endif 863 } 864 } 865 866 for_each_online_pgdat(pgdat) { 867 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 868 869 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 870 int v; 871 872 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 873 if (v) { 874 atomic_long_add(v, &pgdat->vm_stat[i]); 875 global_node_diff[i] += v; 876 } 877 } 878 } 879 880 if (fold_diff(global_zone_diff, global_node_diff)) 881 changed = true; 882 return changed; 883 } 884 885 /* 886 * Fold the data for an offline cpu into the global array. 887 * There cannot be any access by the offline cpu and therefore 888 * synchronization is simplified. 889 */ 890 void cpu_vm_stats_fold(int cpu) 891 { 892 struct pglist_data *pgdat; 893 struct zone *zone; 894 int i; 895 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 896 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 897 898 for_each_populated_zone(zone) { 899 struct per_cpu_zonestat *pzstats; 900 901 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 902 903 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 904 if (pzstats->vm_stat_diff[i]) { 905 int v; 906 907 v = pzstats->vm_stat_diff[i]; 908 pzstats->vm_stat_diff[i] = 0; 909 atomic_long_add(v, &zone->vm_stat[i]); 910 global_zone_diff[i] += v; 911 } 912 } 913 #ifdef CONFIG_NUMA 914 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 915 if (pzstats->vm_numa_event[i]) { 916 unsigned long v; 917 918 v = pzstats->vm_numa_event[i]; 919 pzstats->vm_numa_event[i] = 0; 920 zone_numa_event_add(v, zone, i); 921 } 922 } 923 #endif 924 } 925 926 for_each_online_pgdat(pgdat) { 927 struct per_cpu_nodestat *p; 928 929 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 930 931 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 932 if (p->vm_node_stat_diff[i]) { 933 int v; 934 935 v = p->vm_node_stat_diff[i]; 936 p->vm_node_stat_diff[i] = 0; 937 atomic_long_add(v, &pgdat->vm_stat[i]); 938 global_node_diff[i] += v; 939 } 940 } 941 942 fold_diff(global_zone_diff, global_node_diff); 943 } 944 945 /* 946 * this is only called if !populated_zone(zone), which implies no other users of 947 * pset->vm_stat_diff[] exist. 948 */ 949 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 950 { 951 unsigned long v; 952 int i; 953 954 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 955 if (pzstats->vm_stat_diff[i]) { 956 v = pzstats->vm_stat_diff[i]; 957 pzstats->vm_stat_diff[i] = 0; 958 zone_page_state_add(v, zone, i); 959 } 960 } 961 962 #ifdef CONFIG_NUMA 963 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 964 if (pzstats->vm_numa_event[i]) { 965 v = pzstats->vm_numa_event[i]; 966 pzstats->vm_numa_event[i] = 0; 967 zone_numa_event_add(v, zone, i); 968 } 969 } 970 #endif 971 } 972 #endif 973 974 #ifdef CONFIG_NUMA 975 /* 976 * Determine the per node value of a stat item. This function 977 * is called frequently in a NUMA machine, so try to be as 978 * frugal as possible. 979 */ 980 unsigned long sum_zone_node_page_state(int node, 981 enum zone_stat_item item) 982 { 983 struct zone *zones = NODE_DATA(node)->node_zones; 984 int i; 985 unsigned long count = 0; 986 987 for (i = 0; i < MAX_NR_ZONES; i++) 988 count += zone_page_state(zones + i, item); 989 990 return count; 991 } 992 993 /* Determine the per node value of a numa stat item. */ 994 unsigned long sum_zone_numa_event_state(int node, 995 enum numa_stat_item item) 996 { 997 struct zone *zones = NODE_DATA(node)->node_zones; 998 unsigned long count = 0; 999 int i; 1000 1001 for (i = 0; i < MAX_NR_ZONES; i++) 1002 count += zone_numa_event_state(zones + i, item); 1003 1004 return count; 1005 } 1006 1007 /* 1008 * Determine the per node value of a stat item. 1009 */ 1010 unsigned long node_page_state_pages(struct pglist_data *pgdat, 1011 enum node_stat_item item) 1012 { 1013 long x = atomic_long_read(&pgdat->vm_stat[item]); 1014 #ifdef CONFIG_SMP 1015 if (x < 0) 1016 x = 0; 1017 #endif 1018 return x; 1019 } 1020 1021 unsigned long node_page_state(struct pglist_data *pgdat, 1022 enum node_stat_item item) 1023 { 1024 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1025 1026 return node_page_state_pages(pgdat, item); 1027 } 1028 1029 /* 1030 * Non-clamping variant of node_page_state() intended for callers that 1031 * snapshot a monotonically-incremented counter and subtract two samples. 1032 * See global_node_page_state_monotonic() for the rationale. 1033 */ 1034 unsigned long node_page_state_monotonic(struct pglist_data *pgdat, 1035 enum node_stat_item item) 1036 { 1037 return (unsigned long)atomic_long_read(&pgdat->vm_stat[item]); 1038 } 1039 #endif 1040 1041 /* 1042 * Count number of pages "struct page" and "struct page_ext" consume. 1043 * nr_memmap_boot_pages: # of pages allocated by boot allocator 1044 * nr_memmap_pages: # of pages that were allocated by buddy allocator 1045 */ 1046 static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); 1047 static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); 1048 1049 void memmap_boot_pages_add(long delta) 1050 { 1051 atomic_long_add(delta, &nr_memmap_boot_pages); 1052 } 1053 1054 void memmap_pages_add(long delta) 1055 { 1056 atomic_long_add(delta, &nr_memmap_pages); 1057 } 1058 1059 #ifdef CONFIG_COMPACTION 1060 1061 struct contig_page_info { 1062 unsigned long free_pages; 1063 unsigned long free_blocks_total; 1064 unsigned long free_blocks_suitable; 1065 }; 1066 1067 /* 1068 * Calculate the number of free pages in a zone, how many contiguous 1069 * pages are free and how many are large enough to satisfy an allocation of 1070 * the target size. Note that this function makes no attempt to estimate 1071 * how many suitable free blocks there *might* be if MOVABLE pages were 1072 * migrated. Calculating that is possible, but expensive and can be 1073 * figured out from userspace 1074 */ 1075 static void fill_contig_page_info(struct zone *zone, 1076 unsigned int suitable_order, 1077 struct contig_page_info *info) 1078 { 1079 unsigned int order; 1080 1081 info->free_pages = 0; 1082 info->free_blocks_total = 0; 1083 info->free_blocks_suitable = 0; 1084 1085 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1086 unsigned long blocks; 1087 1088 /* 1089 * Count number of free blocks. 1090 * 1091 * Access to nr_free is lockless as nr_free is used only for 1092 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1093 */ 1094 blocks = data_race(zone->free_area[order].nr_free); 1095 info->free_blocks_total += blocks; 1096 1097 /* Count free base pages */ 1098 info->free_pages += blocks << order; 1099 1100 /* Count the suitable free blocks */ 1101 if (order >= suitable_order) 1102 info->free_blocks_suitable += blocks << 1103 (order - suitable_order); 1104 } 1105 } 1106 1107 /* 1108 * A fragmentation index only makes sense if an allocation of a requested 1109 * size would fail. If that is true, the fragmentation index indicates 1110 * whether external fragmentation or a lack of memory was the problem. 1111 * The value can be used to determine if page reclaim or compaction 1112 * should be used 1113 */ 1114 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1115 { 1116 unsigned long requested = 1UL << order; 1117 1118 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1119 return 0; 1120 1121 if (!info->free_blocks_total) 1122 return 0; 1123 1124 /* Fragmentation index only makes sense when a request would fail */ 1125 if (info->free_blocks_suitable) 1126 return -1000; 1127 1128 /* 1129 * Index is between 0 and 1 so return within 3 decimal places 1130 * 1131 * 0 => allocation would fail due to lack of memory 1132 * 1 => allocation would fail due to fragmentation 1133 */ 1134 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1135 } 1136 1137 /* 1138 * Calculates external fragmentation within a zone wrt the given order. 1139 * It is defined as the percentage of pages found in blocks of size 1140 * less than 1 << order. It returns values in range [0, 100]. 1141 */ 1142 unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1143 { 1144 struct contig_page_info info; 1145 1146 fill_contig_page_info(zone, order, &info); 1147 if (info.free_pages == 0) 1148 return 0; 1149 1150 return div_u64((info.free_pages - 1151 (info.free_blocks_suitable << order)) * 100, 1152 info.free_pages); 1153 } 1154 1155 /* Same as __fragmentation index but allocs contig_page_info on stack */ 1156 int fragmentation_index(struct zone *zone, unsigned int order) 1157 { 1158 struct contig_page_info info; 1159 1160 fill_contig_page_info(zone, order, &info); 1161 return __fragmentation_index(order, &info); 1162 } 1163 #endif 1164 1165 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1166 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1167 #ifdef CONFIG_ZONE_DMA 1168 #define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma", 1169 #else 1170 #define TEXT_FOR_DMA(xx, yy) 1171 #endif 1172 1173 #ifdef CONFIG_ZONE_DMA32 1174 #define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32", 1175 #else 1176 #define TEXT_FOR_DMA32(xx, yy) 1177 #endif 1178 1179 #ifdef CONFIG_HIGHMEM 1180 #define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high", 1181 #else 1182 #define TEXT_FOR_HIGHMEM(xx, yy) 1183 #endif 1184 1185 #ifdef CONFIG_ZONE_DEVICE 1186 #define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device", 1187 #else 1188 #define TEXT_FOR_DEVICE(xx, yy) 1189 #endif 1190 1191 #define TEXTS_FOR_ZONES(xx, yy) \ 1192 TEXT_FOR_DMA(xx, yy) \ 1193 TEXT_FOR_DMA32(xx, yy) \ 1194 [xx##_NORMAL] = yy "_normal", \ 1195 TEXT_FOR_HIGHMEM(xx, yy) \ 1196 [xx##_MOVABLE] = yy "_movable", \ 1197 TEXT_FOR_DEVICE(xx, yy) 1198 1199 const char * const vmstat_text[] = { 1200 /* enum zone_stat_item counters */ 1201 #define I(x) (x) 1202 [I(NR_FREE_PAGES)] = "nr_free_pages", 1203 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks", 1204 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon", 1205 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon", 1206 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file", 1207 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file", 1208 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable", 1209 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending", 1210 [I(NR_MLOCK)] = "nr_mlock", 1211 #if IS_ENABLED(CONFIG_ZSMALLOC) 1212 [I(NR_ZSPAGES)] = "nr_zspages", 1213 #endif 1214 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma", 1215 #ifdef CONFIG_UNACCEPTED_MEMORY 1216 [I(NR_UNACCEPTED)] = "nr_unaccepted", 1217 #endif 1218 #undef I 1219 1220 /* enum numa_stat_item counters */ 1221 #define I(x) (NR_VM_ZONE_STAT_ITEMS + x) 1222 #ifdef CONFIG_NUMA 1223 [I(NUMA_HIT)] = "numa_hit", 1224 [I(NUMA_MISS)] = "numa_miss", 1225 [I(NUMA_FOREIGN)] = "numa_foreign", 1226 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave", 1227 [I(NUMA_LOCAL)] = "numa_local", 1228 [I(NUMA_OTHER)] = "numa_other", 1229 #endif 1230 #undef I 1231 1232 /* enum node_stat_item counters */ 1233 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x) 1234 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon", 1235 [I(NR_ACTIVE_ANON)] = "nr_active_anon", 1236 [I(NR_INACTIVE_FILE)] = "nr_inactive_file", 1237 [I(NR_ACTIVE_FILE)] = "nr_active_file", 1238 [I(NR_UNEVICTABLE)] = "nr_unevictable", 1239 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable", 1240 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable", 1241 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon", 1242 [I(NR_ISOLATED_FILE)] = "nr_isolated_file", 1243 [I(WORKINGSET_NODES)] = "workingset_nodes", 1244 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon", 1245 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file", 1246 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon", 1247 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file", 1248 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon", 1249 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file", 1250 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim", 1251 [I(NR_ANON_MAPPED)] = "nr_anon_pages", 1252 [I(NR_FILE_MAPPED)] = "nr_mapped", 1253 [I(NR_FILE_PAGES)] = "nr_file_pages", 1254 [I(NR_FILE_DIRTY)] = "nr_dirty", 1255 [I(NR_WRITEBACK)] = "nr_writeback", 1256 [I(NR_SHMEM)] = "nr_shmem", 1257 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages", 1258 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped", 1259 [I(NR_FILE_THPS)] = "nr_file_hugepages", 1260 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped", 1261 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages", 1262 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write", 1263 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim", 1264 [I(NR_DIRTIED)] = "nr_dirtied", 1265 [I(NR_WRITTEN)] = "nr_written", 1266 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written", 1267 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable", 1268 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired", 1269 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released", 1270 [I(NR_VMALLOC)] = "nr_vmalloc", 1271 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack", 1272 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1273 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack", 1274 #endif 1275 [I(NR_PAGETABLE)] = "nr_page_table_pages", 1276 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages", 1277 #ifdef CONFIG_IOMMU_SUPPORT 1278 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages", 1279 #endif 1280 #ifdef CONFIG_SWAP 1281 [I(NR_SWAPCACHE)] = "nr_swapcached", 1282 #endif 1283 #ifdef CONFIG_NUMA_BALANCING 1284 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success", 1285 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate", 1286 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl", 1287 #endif 1288 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd", 1289 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct", 1290 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged", 1291 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive", 1292 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd", 1293 [I(PGSTEAL_DIRECT)] = "pgsteal_direct", 1294 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged", 1295 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive", 1296 [I(PGSTEAL_ANON)] = "pgsteal_anon", 1297 [I(PGSTEAL_FILE)] = "pgsteal_file", 1298 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd", 1299 [I(PGSCAN_DIRECT)] = "pgscan_direct", 1300 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged", 1301 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", 1302 [I(PGSCAN_ANON)] = "pgscan_anon", 1303 [I(PGSCAN_FILE)] = "pgscan_file", 1304 [I(PGROTATE_ANON)] = "pgrotate_anon", 1305 [I(PGROTATE_FILE)] = "pgrotate_file", 1306 [I(PGREFILL)] = "pgrefill", 1307 #ifdef CONFIG_HUGETLB_PAGE 1308 [I(NR_HUGETLB)] = "nr_hugetlb", 1309 #endif 1310 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages", 1311 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages", 1312 [I(NR_GPU_ACTIVE)] = "nr_gpu_active", 1313 [I(NR_GPU_RECLAIM)] = "nr_gpu_reclaim", 1314 #undef I 1315 1316 /* system-wide enum vm_stat_item counters */ 1317 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1318 NR_VM_NODE_STAT_ITEMS + x) 1319 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold", 1320 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold", 1321 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages", 1322 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages", 1323 #undef I 1324 1325 #if defined(CONFIG_VM_EVENT_COUNTERS) 1326 /* enum vm_event_item counters */ 1327 #define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1328 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x) 1329 1330 [I(PGPGIN)] = "pgpgin", 1331 [I(PGPGOUT)] = "pgpgout", 1332 [I(PSWPIN)] = "pswpin", 1333 [I(PSWPOUT)] = "pswpout", 1334 1335 #define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1336 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS) 1337 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc") 1338 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall") 1339 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip") 1340 #undef OFF 1341 1342 [I(PGFREE)] = "pgfree", 1343 [I(PGACTIVATE)] = "pgactivate", 1344 [I(PGDEACTIVATE)] = "pgdeactivate", 1345 [I(PGLAZYFREE)] = "pglazyfree", 1346 1347 [I(PGFAULT)] = "pgfault", 1348 [I(PGMAJFAULT)] = "pgmajfault", 1349 [I(PGLAZYFREED)] = "pglazyfreed", 1350 1351 [I(PGREUSE)] = "pgreuse", 1352 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle", 1353 1354 #ifdef CONFIG_NUMA 1355 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success", 1356 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed", 1357 #endif 1358 [I(PGINODESTEAL)] = "pginodesteal", 1359 [I(SLABS_SCANNED)] = "slabs_scanned", 1360 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal", 1361 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly", 1362 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly", 1363 [I(PAGEOUTRUN)] = "pageoutrun", 1364 1365 [I(PGROTATED)] = "pgrotated", 1366 1367 [I(DROP_PAGECACHE)] = "drop_pagecache", 1368 [I(DROP_SLAB)] = "drop_slab", 1369 [I(OOM_KILL)] = "oom_kill", 1370 1371 #ifdef CONFIG_NUMA_BALANCING 1372 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates", 1373 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates", 1374 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults", 1375 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local", 1376 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated", 1377 #endif 1378 #ifdef CONFIG_MIGRATION 1379 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success", 1380 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail", 1381 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success", 1382 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail", 1383 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split", 1384 #endif 1385 #ifdef CONFIG_COMPACTION 1386 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned", 1387 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned", 1388 [I(COMPACTISOLATED)] = "compact_isolated", 1389 [I(COMPACTSTALL)] = "compact_stall", 1390 [I(COMPACTFAIL)] = "compact_fail", 1391 [I(COMPACTSUCCESS)] = "compact_success", 1392 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake", 1393 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned", 1394 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned", 1395 #endif 1396 1397 #ifdef CONFIG_HUGETLB_PAGE 1398 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success", 1399 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail", 1400 #endif 1401 #ifdef CONFIG_CMA 1402 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success", 1403 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail", 1404 #endif 1405 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled", 1406 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned", 1407 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued", 1408 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked", 1409 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked", 1410 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared", 1411 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded", 1412 1413 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1414 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc", 1415 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback", 1416 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge", 1417 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc", 1418 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed", 1419 [I(THP_FILE_ALLOC)] = "thp_file_alloc", 1420 [I(THP_FILE_FALLBACK)] = "thp_file_fallback", 1421 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge", 1422 [I(THP_FILE_MAPPED)] = "thp_file_mapped", 1423 [I(THP_SPLIT_PAGE)] = "thp_split_page", 1424 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed", 1425 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page", 1426 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page", 1427 [I(THP_SPLIT_PMD)] = "thp_split_pmd", 1428 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte", 1429 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte", 1430 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte", 1431 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1432 [I(THP_SPLIT_PUD)] = "thp_split_pud", 1433 #endif 1434 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc", 1435 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed", 1436 [I(THP_SWPOUT)] = "thp_swpout", 1437 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback", 1438 #endif 1439 #ifdef CONFIG_BALLOON 1440 [I(BALLOON_INFLATE)] = "balloon_inflate", 1441 [I(BALLOON_DEFLATE)] = "balloon_deflate", 1442 #ifdef CONFIG_BALLOON_MIGRATION 1443 [I(BALLOON_MIGRATE)] = "balloon_migrate", 1444 #endif /* CONFIG_BALLOON_MIGRATION */ 1445 #endif /* CONFIG_BALLOON */ 1446 #ifdef CONFIG_DEBUG_TLBFLUSH 1447 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush", 1448 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received", 1449 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all", 1450 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one", 1451 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1452 1453 #ifdef CONFIG_SWAP 1454 [I(SWAP_RA)] = "swap_ra", 1455 [I(SWAP_RA_HIT)] = "swap_ra_hit", 1456 [I(SWPIN_ZERO)] = "swpin_zero", 1457 [I(SWPOUT_ZERO)] = "swpout_zero", 1458 #ifdef CONFIG_KSM 1459 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy", 1460 #endif 1461 #endif 1462 #ifdef CONFIG_KSM 1463 [I(COW_KSM)] = "cow_ksm", 1464 #endif 1465 #ifdef CONFIG_ZSWAP 1466 [I(ZSWPIN)] = "zswpin", 1467 [I(ZSWPOUT)] = "zswpout", 1468 [I(ZSWPWB)] = "zswpwb", 1469 #endif 1470 #ifdef CONFIG_X86 1471 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits", 1472 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits", 1473 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses", 1474 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses", 1475 #endif 1476 #ifdef CONFIG_PER_VMA_LOCK_STATS 1477 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success", 1478 [I(VMA_LOCK_ABORT)] = "vma_lock_abort", 1479 [I(VMA_LOCK_RETRY)] = "vma_lock_retry", 1480 [I(VMA_LOCK_MISS)] = "vma_lock_miss", 1481 #endif 1482 #ifdef CONFIG_DEBUG_STACK_USAGE 1483 [I(KSTACK_1K)] = "kstack_1k", 1484 #if THREAD_SIZE > 1024 1485 [I(KSTACK_2K)] = "kstack_2k", 1486 #endif 1487 #if THREAD_SIZE > 2048 1488 [I(KSTACK_4K)] = "kstack_4k", 1489 #endif 1490 #if THREAD_SIZE > 4096 1491 [I(KSTACK_8K)] = "kstack_8k", 1492 #endif 1493 #if THREAD_SIZE > 8192 1494 [I(KSTACK_16K)] = "kstack_16k", 1495 #endif 1496 #if THREAD_SIZE > 16384 1497 [I(KSTACK_32K)] = "kstack_32k", 1498 #endif 1499 #if THREAD_SIZE > 32768 1500 [I(KSTACK_64K)] = "kstack_64k", 1501 #endif 1502 #if THREAD_SIZE > 65536 1503 [I(KSTACK_REST)] = "kstack_rest", 1504 #endif 1505 #endif /* CONFIG_DEBUG_STACK_USAGE */ 1506 #ifdef CONFIG_SWAP 1507 [I(NRSWPIN)] = "nrswpin", 1508 [I(NRSWPOUT)] = "nrswpout", 1509 #endif /* CONFIG_SWAP */ 1510 #undef I 1511 #endif /* CONFIG_VM_EVENT_COUNTERS */ 1512 }; 1513 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1514 1515 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1516 defined(CONFIG_PROC_FS) 1517 static void *frag_start(struct seq_file *m, loff_t *pos) 1518 { 1519 pg_data_t *pgdat; 1520 loff_t node = *pos; 1521 1522 for (pgdat = first_online_pgdat(); 1523 pgdat && node; 1524 pgdat = next_online_pgdat(pgdat)) 1525 --node; 1526 1527 return pgdat; 1528 } 1529 1530 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1531 { 1532 pg_data_t *pgdat = (pg_data_t *)arg; 1533 1534 (*pos)++; 1535 return next_online_pgdat(pgdat); 1536 } 1537 1538 static void frag_stop(struct seq_file *m, void *arg) 1539 { 1540 } 1541 1542 /* 1543 * Walk zones in a node and print using a callback. 1544 * If @assert_populated is true, only use callback for zones that are populated. 1545 */ 1546 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1547 bool assert_populated, bool nolock, 1548 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1549 { 1550 struct zone *zone; 1551 struct zone *node_zones = pgdat->node_zones; 1552 unsigned long flags; 1553 1554 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1555 if (assert_populated && !populated_zone(zone)) 1556 continue; 1557 1558 if (!nolock) 1559 spin_lock_irqsave(&zone->lock, flags); 1560 print(m, pgdat, zone); 1561 if (!nolock) 1562 spin_unlock_irqrestore(&zone->lock, flags); 1563 } 1564 } 1565 #endif 1566 1567 #ifdef CONFIG_PROC_FS 1568 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1569 struct zone *zone) 1570 { 1571 int order; 1572 1573 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1574 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1575 /* 1576 * Access to nr_free is lockless as nr_free is used only for 1577 * printing purposes. Use data_race to avoid KCSAN warning. 1578 */ 1579 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1580 seq_putc(m, '\n'); 1581 } 1582 1583 /* 1584 * This walks the free areas for each zone. 1585 */ 1586 static int frag_show(struct seq_file *m, void *arg) 1587 { 1588 pg_data_t *pgdat = (pg_data_t *)arg; 1589 walk_zones_in_node(m, pgdat, true, true, frag_show_print); 1590 return 0; 1591 } 1592 1593 static void pagetypeinfo_showfree_print(struct seq_file *m, 1594 pg_data_t *pgdat, struct zone *zone) 1595 { 1596 int order, mtype; 1597 1598 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1599 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1600 pgdat->node_id, 1601 zone->name, 1602 migratetype_names[mtype]); 1603 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1604 unsigned long freecount = 0; 1605 struct free_area *area; 1606 struct list_head *curr; 1607 bool overflow = false; 1608 1609 area = &(zone->free_area[order]); 1610 1611 list_for_each(curr, &area->free_list[mtype]) { 1612 /* 1613 * Cap the free_list iteration because it might 1614 * be really large and we are under a spinlock 1615 * so a long time spent here could trigger a 1616 * hard lockup detector. Anyway this is a 1617 * debugging tool so knowing there is a handful 1618 * of pages of this order should be more than 1619 * sufficient. 1620 */ 1621 if (++freecount >= 100000) { 1622 overflow = true; 1623 break; 1624 } 1625 } 1626 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1627 spin_unlock_irq(&zone->lock); 1628 cond_resched(); 1629 spin_lock_irq(&zone->lock); 1630 } 1631 seq_putc(m, '\n'); 1632 } 1633 } 1634 1635 /* Print out the free pages at each order for each migratetype */ 1636 static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1637 { 1638 int order; 1639 pg_data_t *pgdat = (pg_data_t *)arg; 1640 1641 /* Print header */ 1642 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1643 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1644 seq_printf(m, "%6d ", order); 1645 seq_putc(m, '\n'); 1646 1647 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1648 } 1649 1650 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1651 pg_data_t *pgdat, struct zone *zone) 1652 { 1653 int mtype; 1654 unsigned long pfn; 1655 unsigned long start_pfn = zone->zone_start_pfn; 1656 unsigned long end_pfn = zone_end_pfn(zone); 1657 unsigned long count[MIGRATE_TYPES] = { 0, }; 1658 1659 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1660 struct page *page; 1661 1662 page = pfn_to_online_page(pfn); 1663 if (!page) 1664 continue; 1665 1666 if (page_zone(page) != zone) 1667 continue; 1668 1669 mtype = get_pageblock_migratetype(page); 1670 1671 if (mtype < MIGRATE_TYPES) 1672 count[mtype]++; 1673 } 1674 1675 /* Print counts */ 1676 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1677 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1678 seq_printf(m, "%12lu ", count[mtype]); 1679 seq_putc(m, '\n'); 1680 } 1681 1682 /* Print out the number of pageblocks for each migratetype */ 1683 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1684 { 1685 int mtype; 1686 pg_data_t *pgdat = (pg_data_t *)arg; 1687 1688 seq_printf(m, "\n%-23s", "Number of blocks type "); 1689 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1690 seq_printf(m, "%12s ", migratetype_names[mtype]); 1691 seq_putc(m, '\n'); 1692 walk_zones_in_node(m, pgdat, true, false, 1693 pagetypeinfo_showblockcount_print); 1694 } 1695 1696 /* 1697 * Print out the number of pageblocks for each migratetype that contain pages 1698 * of other types. This gives an indication of how well fallbacks are being 1699 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1700 * to determine what is going on 1701 */ 1702 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1703 { 1704 #ifdef CONFIG_PAGE_OWNER 1705 int mtype; 1706 1707 if (!static_branch_unlikely(&page_owner_inited)) 1708 return; 1709 1710 drain_all_pages(NULL); 1711 1712 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1713 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1714 seq_printf(m, "%12s ", migratetype_names[mtype]); 1715 seq_putc(m, '\n'); 1716 1717 walk_zones_in_node(m, pgdat, true, true, 1718 pagetypeinfo_showmixedcount_print); 1719 #endif /* CONFIG_PAGE_OWNER */ 1720 } 1721 1722 /* 1723 * This prints out statistics in relation to grouping pages by mobility. 1724 * It is expensive to collect so do not constantly read the file. 1725 */ 1726 static int pagetypeinfo_show(struct seq_file *m, void *arg) 1727 { 1728 pg_data_t *pgdat = (pg_data_t *)arg; 1729 1730 /* check memoryless node */ 1731 if (!node_state(pgdat->node_id, N_MEMORY)) 1732 return 0; 1733 1734 seq_printf(m, "Page block order: %d\n", pageblock_order); 1735 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1736 seq_putc(m, '\n'); 1737 pagetypeinfo_showfree(m, pgdat); 1738 pagetypeinfo_showblockcount(m, pgdat); 1739 pagetypeinfo_showmixedcount(m, pgdat); 1740 1741 return 0; 1742 } 1743 1744 static const struct seq_operations fragmentation_op = { 1745 .start = frag_start, 1746 .next = frag_next, 1747 .stop = frag_stop, 1748 .show = frag_show, 1749 }; 1750 1751 static const struct seq_operations pagetypeinfo_op = { 1752 .start = frag_start, 1753 .next = frag_next, 1754 .stop = frag_stop, 1755 .show = pagetypeinfo_show, 1756 }; 1757 1758 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1759 { 1760 int zid; 1761 1762 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1763 struct zone *compare = &pgdat->node_zones[zid]; 1764 1765 if (populated_zone(compare)) 1766 return zone == compare; 1767 } 1768 1769 return false; 1770 } 1771 1772 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1773 struct zone *zone) 1774 { 1775 int i; 1776 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1777 if (is_zone_first_populated(pgdat, zone)) { 1778 seq_printf(m, "\n per-node stats"); 1779 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1780 unsigned long pages = node_page_state_pages(pgdat, i); 1781 1782 if (vmstat_item_print_in_thp(i)) 1783 pages /= HPAGE_PMD_NR; 1784 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1785 pages); 1786 } 1787 } 1788 seq_printf(m, 1789 "\n pages free %lu" 1790 "\n boost %lu" 1791 "\n min %lu" 1792 "\n low %lu" 1793 "\n high %lu" 1794 "\n promo %lu" 1795 "\n spanned %lu" 1796 "\n present %lu" 1797 "\n managed %lu" 1798 "\n cma %lu", 1799 zone_page_state(zone, NR_FREE_PAGES), 1800 zone->watermark_boost, 1801 min_wmark_pages(zone), 1802 low_wmark_pages(zone), 1803 high_wmark_pages(zone), 1804 promo_wmark_pages(zone), 1805 zone->spanned_pages, 1806 zone->present_pages, 1807 zone_managed_pages(zone), 1808 zone_cma_pages(zone)); 1809 1810 seq_printf(m, 1811 "\n protection: (%ld", 1812 zone->lowmem_reserve[0]); 1813 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1814 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1815 seq_putc(m, ')'); 1816 1817 /* If unpopulated, no other information is useful */ 1818 if (!populated_zone(zone)) { 1819 seq_putc(m, '\n'); 1820 return; 1821 } 1822 1823 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1824 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1825 zone_page_state(zone, i)); 1826 1827 #ifdef CONFIG_NUMA 1828 fold_vm_zone_numa_events(zone); 1829 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1830 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1831 zone_numa_event_state(zone, i)); 1832 #endif 1833 1834 seq_printf(m, "\n pagesets"); 1835 for_each_online_cpu(i) { 1836 struct per_cpu_pages *pcp; 1837 struct per_cpu_zonestat __maybe_unused *pzstats; 1838 1839 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1840 seq_printf(m, 1841 "\n cpu: %i" 1842 "\n count: %i" 1843 "\n high: %i" 1844 "\n batch: %i" 1845 "\n high_min: %i" 1846 "\n high_max: %i", 1847 i, 1848 pcp->count, 1849 pcp->high, 1850 pcp->batch, 1851 pcp->high_min, 1852 pcp->high_max); 1853 #ifdef CONFIG_SMP 1854 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1855 seq_printf(m, "\n vm stats threshold: %d", 1856 pzstats->stat_threshold); 1857 #endif 1858 } 1859 seq_printf(m, 1860 "\n node_unreclaimable: %u" 1861 "\n start_pfn: %lu" 1862 "\n reserved_highatomic: %lu" 1863 "\n free_highatomic: %lu", 1864 kswapd_test_hopeless(pgdat), 1865 zone->zone_start_pfn, 1866 zone->nr_reserved_highatomic, 1867 zone->nr_free_highatomic); 1868 seq_putc(m, '\n'); 1869 } 1870 1871 /* 1872 * Output information about zones in @pgdat. All zones are printed regardless 1873 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1874 * set of all zones and userspace would not be aware of such zones if they are 1875 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1876 */ 1877 static int zoneinfo_show(struct seq_file *m, void *arg) 1878 { 1879 pg_data_t *pgdat = (pg_data_t *)arg; 1880 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1881 return 0; 1882 } 1883 1884 static const struct seq_operations zoneinfo_op = { 1885 .start = frag_start, /* iterate over all zones. The same as in 1886 * fragmentation. */ 1887 .next = frag_next, 1888 .stop = frag_stop, 1889 .show = zoneinfo_show, 1890 }; 1891 1892 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1893 NR_VM_NUMA_EVENT_ITEMS + \ 1894 NR_VM_NODE_STAT_ITEMS + \ 1895 NR_VM_STAT_ITEMS + \ 1896 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1897 NR_VM_EVENT_ITEMS : 0)) 1898 1899 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1900 { 1901 unsigned long *v; 1902 int i; 1903 1904 if (*pos >= NR_VMSTAT_ITEMS) 1905 return NULL; 1906 1907 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS); 1908 fold_vm_numa_events(); 1909 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1910 m->private = v; 1911 if (!v) 1912 return ERR_PTR(-ENOMEM); 1913 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1914 v[i] = global_zone_page_state(i); 1915 v += NR_VM_ZONE_STAT_ITEMS; 1916 1917 #ifdef CONFIG_NUMA 1918 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1919 v[i] = global_numa_event_state(i); 1920 v += NR_VM_NUMA_EVENT_ITEMS; 1921 #endif 1922 1923 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1924 v[i] = global_node_page_state_pages(i); 1925 if (vmstat_item_print_in_thp(i)) 1926 v[i] /= HPAGE_PMD_NR; 1927 } 1928 v += NR_VM_NODE_STAT_ITEMS; 1929 1930 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1931 v + NR_DIRTY_THRESHOLD); 1932 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); 1933 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); 1934 v += NR_VM_STAT_ITEMS; 1935 1936 #ifdef CONFIG_VM_EVENT_COUNTERS 1937 all_vm_events(v); 1938 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1939 v[PGPGOUT] /= 2; 1940 #endif 1941 return (unsigned long *)m->private + *pos; 1942 } 1943 1944 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1945 { 1946 (*pos)++; 1947 if (*pos >= NR_VMSTAT_ITEMS) 1948 return NULL; 1949 return (unsigned long *)m->private + *pos; 1950 } 1951 1952 static int vmstat_show(struct seq_file *m, void *arg) 1953 { 1954 unsigned long *l = arg; 1955 unsigned long off = l - (unsigned long *)m->private; 1956 1957 seq_puts(m, vmstat_text[off]); 1958 seq_put_decimal_ull(m, " ", *l); 1959 seq_putc(m, '\n'); 1960 1961 if (off == NR_VMSTAT_ITEMS - 1) { 1962 /* 1963 * We've come to the end - add any deprecated counters to avoid 1964 * breaking userspace which might depend on them being present. 1965 */ 1966 seq_puts(m, "nr_unstable 0\n"); 1967 } 1968 return 0; 1969 } 1970 1971 static void vmstat_stop(struct seq_file *m, void *arg) 1972 { 1973 kfree(m->private); 1974 m->private = NULL; 1975 } 1976 1977 static const struct seq_operations vmstat_op = { 1978 .start = vmstat_start, 1979 .next = vmstat_next, 1980 .stop = vmstat_stop, 1981 .show = vmstat_show, 1982 }; 1983 #endif /* CONFIG_PROC_FS */ 1984 1985 #ifdef CONFIG_SMP 1986 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1987 static int sysctl_stat_interval __read_mostly = HZ; 1988 static int vmstat_late_init_done; 1989 1990 #ifdef CONFIG_PROC_FS 1991 static void refresh_vm_stats(struct work_struct *work) 1992 { 1993 refresh_cpu_vm_stats(true); 1994 } 1995 1996 static int vmstat_refresh(const struct ctl_table *table, int write, 1997 void *buffer, size_t *lenp, loff_t *ppos) 1998 { 1999 long val; 2000 int err; 2001 int i; 2002 2003 /* 2004 * The regular update, every sysctl_stat_interval, may come later 2005 * than expected: leaving a significant amount in per_cpu buckets. 2006 * This is particularly misleading when checking a quantity of HUGE 2007 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 2008 * which can equally be echo'ed to or cat'ted from (by root), 2009 * can be used to update the stats just before reading them. 2010 * 2011 * Oh, and since global_zone_page_state() etc. are so careful to hide 2012 * transiently negative values, report an error here if any of 2013 * the stats is negative, so we know to go looking for imbalance. 2014 */ 2015 err = schedule_on_each_cpu(refresh_vm_stats); 2016 if (err) 2017 return err; 2018 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 2019 /* 2020 * Skip checking stats known to go negative occasionally. 2021 */ 2022 switch (i) { 2023 case NR_ZONE_WRITE_PENDING: 2024 case NR_FREE_CMA_PAGES: 2025 continue; 2026 } 2027 val = atomic_long_read(&vm_zone_stat[i]); 2028 if (val < 0) { 2029 pr_warn("%s: %s %ld\n", 2030 __func__, zone_stat_name(i), val); 2031 } 2032 } 2033 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 2034 /* 2035 * Skip checking stats known to go negative occasionally. 2036 */ 2037 switch (i) { 2038 case NR_WRITEBACK: 2039 continue; 2040 } 2041 val = atomic_long_read(&vm_node_stat[i]); 2042 if (val < 0) { 2043 pr_warn("%s: %s %ld\n", 2044 __func__, node_stat_name(i), val); 2045 } 2046 } 2047 if (write) 2048 *ppos += *lenp; 2049 else 2050 *lenp = 0; 2051 return 0; 2052 } 2053 #endif /* CONFIG_PROC_FS */ 2054 2055 static void vmstat_update(struct work_struct *w) 2056 { 2057 if (refresh_cpu_vm_stats(true)) { 2058 /* 2059 * Counters were updated so we expect more updates 2060 * to occur in the future. Keep on running the 2061 * update worker thread. 2062 */ 2063 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 2064 this_cpu_ptr(&vmstat_work), 2065 round_jiffies_relative(sysctl_stat_interval)); 2066 } 2067 } 2068 2069 /* 2070 * Check if the diffs for a certain cpu indicate that 2071 * an update is needed. 2072 */ 2073 static bool need_update(int cpu) 2074 { 2075 pg_data_t *last_pgdat = NULL; 2076 struct zone *zone; 2077 2078 for_each_populated_zone(zone) { 2079 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 2080 struct per_cpu_nodestat *n; 2081 2082 /* 2083 * The fast way of checking if there are any vmstat diffs. 2084 */ 2085 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 2086 return true; 2087 2088 if (last_pgdat == zone->zone_pgdat) 2089 continue; 2090 last_pgdat = zone->zone_pgdat; 2091 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 2092 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 2093 return true; 2094 } 2095 return false; 2096 } 2097 2098 /* 2099 * Switch off vmstat processing and then fold all the remaining differentials 2100 * until the diffs stay at zero. The function is used by NOHZ and can only be 2101 * invoked when tick processing is not active. 2102 */ 2103 void quiet_vmstat(void) 2104 { 2105 if (system_state != SYSTEM_RUNNING) 2106 return; 2107 2108 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2109 return; 2110 2111 if (!need_update(smp_processor_id())) 2112 return; 2113 2114 /* 2115 * Just refresh counters and do not care about the pending delayed 2116 * vmstat_update. It doesn't fire that often to matter and canceling 2117 * it would be too expensive from this path. 2118 * vmstat_shepherd will take care about that for us. 2119 */ 2120 refresh_cpu_vm_stats(false); 2121 } 2122 2123 /* 2124 * Shepherd worker thread that checks the 2125 * differentials of processors that have their worker 2126 * threads for vm statistics updates disabled because of 2127 * inactivity. 2128 */ 2129 static void vmstat_shepherd(struct work_struct *w); 2130 2131 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2132 2133 void vmstat_flush_workqueue(void) 2134 { 2135 flush_workqueue(mm_percpu_wq); 2136 } 2137 2138 static void vmstat_shepherd(struct work_struct *w) 2139 { 2140 int cpu; 2141 2142 cpus_read_lock(); 2143 /* Check processors whose vmstat worker threads have been disabled */ 2144 for_each_online_cpu(cpu) { 2145 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2146 2147 /* 2148 * In kernel users of vmstat counters either require the precise value and 2149 * they are using zone_page_state_snapshot interface or they can live with 2150 * an imprecision as the regular flushing can happen at arbitrary time and 2151 * cumulative error can grow (see calculate_normal_threshold). 2152 * 2153 * From that POV the regular flushing can be postponed for CPUs that have 2154 * been isolated from the kernel interference without critical 2155 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2156 * for all isolated CPUs to avoid interference with the isolated workload. 2157 */ 2158 scoped_guard(rcu) { 2159 if (cpu_is_isolated(cpu)) 2160 continue; 2161 2162 if (!work_busy(&dw->work) && need_update(cpu)) 2163 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2164 } 2165 2166 cond_resched(); 2167 } 2168 cpus_read_unlock(); 2169 2170 schedule_delayed_work(&shepherd, 2171 round_jiffies_relative(sysctl_stat_interval)); 2172 } 2173 2174 static void __init start_shepherd_timer(void) 2175 { 2176 int cpu; 2177 2178 for_each_possible_cpu(cpu) { 2179 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2180 vmstat_update); 2181 2182 /* 2183 * For secondary CPUs during CPU hotplug scenarios, 2184 * vmstat_cpu_online() will enable the work. 2185 * mm/vmstat:online enables and disables vmstat_work 2186 * symmetrically during CPU hotplug events. 2187 */ 2188 if (!cpu_online(cpu)) 2189 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2190 } 2191 2192 schedule_delayed_work(&shepherd, 2193 round_jiffies_relative(sysctl_stat_interval)); 2194 } 2195 2196 static void __init init_cpu_node_state(void) 2197 { 2198 int node; 2199 2200 for_each_online_node(node) { 2201 if (!cpumask_empty(cpumask_of_node(node))) 2202 node_set_state(node, N_CPU); 2203 } 2204 } 2205 2206 static int vmstat_cpu_online(unsigned int cpu) 2207 { 2208 if (vmstat_late_init_done) 2209 refresh_zone_stat_thresholds(); 2210 2211 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2212 node_set_state(cpu_to_node(cpu), N_CPU); 2213 } 2214 enable_delayed_work(&per_cpu(vmstat_work, cpu)); 2215 2216 return 0; 2217 } 2218 2219 static int vmstat_cpu_down_prep(unsigned int cpu) 2220 { 2221 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2222 return 0; 2223 } 2224 2225 static int vmstat_cpu_dead(unsigned int cpu) 2226 { 2227 const struct cpumask *node_cpus; 2228 int node; 2229 2230 node = cpu_to_node(cpu); 2231 2232 refresh_zone_stat_thresholds(); 2233 node_cpus = cpumask_of_node(node); 2234 if (!cpumask_empty(node_cpus)) 2235 return 0; 2236 2237 node_clear_state(node, N_CPU); 2238 2239 return 0; 2240 } 2241 2242 static int __init vmstat_late_init(void) 2243 { 2244 refresh_zone_stat_thresholds(); 2245 vmstat_late_init_done = 1; 2246 2247 return 0; 2248 } 2249 late_initcall(vmstat_late_init); 2250 #endif 2251 2252 #ifdef CONFIG_PROC_FS 2253 static const struct ctl_table vmstat_table[] = { 2254 #ifdef CONFIG_SMP 2255 { 2256 .procname = "stat_interval", 2257 .data = &sysctl_stat_interval, 2258 .maxlen = sizeof(sysctl_stat_interval), 2259 .mode = 0644, 2260 .proc_handler = proc_dointvec_jiffies, 2261 }, 2262 { 2263 .procname = "stat_refresh", 2264 .data = NULL, 2265 .maxlen = 0, 2266 .mode = 0600, 2267 .proc_handler = vmstat_refresh, 2268 }, 2269 #endif 2270 #ifdef CONFIG_NUMA 2271 { 2272 .procname = "numa_stat", 2273 .data = &sysctl_vm_numa_stat, 2274 .maxlen = sizeof(int), 2275 .mode = 0644, 2276 .proc_handler = sysctl_vm_numa_stat_handler, 2277 .extra1 = SYSCTL_ZERO, 2278 .extra2 = SYSCTL_ONE, 2279 }, 2280 #endif 2281 }; 2282 #endif 2283 2284 struct workqueue_struct *mm_percpu_wq; 2285 2286 void __init init_mm_internals(void) 2287 { 2288 int ret __maybe_unused; 2289 2290 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", 2291 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2292 2293 #ifdef CONFIG_SMP 2294 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2295 NULL, vmstat_cpu_dead); 2296 if (ret < 0) 2297 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2298 2299 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2300 vmstat_cpu_online, 2301 vmstat_cpu_down_prep); 2302 if (ret < 0) 2303 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2304 2305 cpus_read_lock(); 2306 init_cpu_node_state(); 2307 cpus_read_unlock(); 2308 2309 start_shepherd_timer(); 2310 #endif 2311 #ifdef CONFIG_PROC_FS 2312 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2313 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2314 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2315 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2316 register_sysctl_init("vm", vmstat_table); 2317 #endif 2318 } 2319 2320 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2321 2322 /* 2323 * Return an index indicating how much of the available free memory is 2324 * unusable for an allocation of the requested size. 2325 */ 2326 static int unusable_free_index(unsigned int order, 2327 struct contig_page_info *info) 2328 { 2329 /* No free memory is interpreted as all free memory is unusable */ 2330 if (info->free_pages == 0) 2331 return 1000; 2332 2333 /* 2334 * Index should be a value between 0 and 1. Return a value to 3 2335 * decimal places. 2336 * 2337 * 0 => no fragmentation 2338 * 1 => high fragmentation 2339 */ 2340 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2341 2342 } 2343 2344 static void unusable_show_print(struct seq_file *m, 2345 pg_data_t *pgdat, struct zone *zone) 2346 { 2347 unsigned int order; 2348 int index; 2349 struct contig_page_info info; 2350 2351 seq_printf(m, "Node %d, zone %8s ", 2352 pgdat->node_id, 2353 zone->name); 2354 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2355 fill_contig_page_info(zone, order, &info); 2356 index = unusable_free_index(order, &info); 2357 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2358 } 2359 2360 seq_putc(m, '\n'); 2361 } 2362 2363 /* 2364 * Display unusable free space index 2365 * 2366 * The unusable free space index measures how much of the available free 2367 * memory cannot be used to satisfy an allocation of a given size and is a 2368 * value between 0 and 1. The higher the value, the more of free memory is 2369 * unusable and by implication, the worse the external fragmentation is. This 2370 * can be expressed as a percentage by multiplying by 100. 2371 */ 2372 static int unusable_show(struct seq_file *m, void *arg) 2373 { 2374 pg_data_t *pgdat = (pg_data_t *)arg; 2375 2376 /* check memoryless node */ 2377 if (!node_state(pgdat->node_id, N_MEMORY)) 2378 return 0; 2379 2380 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2381 2382 return 0; 2383 } 2384 2385 static const struct seq_operations unusable_sops = { 2386 .start = frag_start, 2387 .next = frag_next, 2388 .stop = frag_stop, 2389 .show = unusable_show, 2390 }; 2391 2392 DEFINE_SEQ_ATTRIBUTE(unusable); 2393 2394 static void extfrag_show_print(struct seq_file *m, 2395 pg_data_t *pgdat, struct zone *zone) 2396 { 2397 unsigned int order; 2398 int index; 2399 2400 /* Alloc on stack as interrupts are disabled for zone walk */ 2401 struct contig_page_info info; 2402 2403 seq_printf(m, "Node %d, zone %8s ", 2404 pgdat->node_id, 2405 zone->name); 2406 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2407 fill_contig_page_info(zone, order, &info); 2408 index = __fragmentation_index(order, &info); 2409 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2410 } 2411 2412 seq_putc(m, '\n'); 2413 } 2414 2415 /* 2416 * Display fragmentation index for orders that allocations would fail for 2417 */ 2418 static int extfrag_show(struct seq_file *m, void *arg) 2419 { 2420 pg_data_t *pgdat = (pg_data_t *)arg; 2421 2422 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2423 2424 return 0; 2425 } 2426 2427 static const struct seq_operations extfrag_sops = { 2428 .start = frag_start, 2429 .next = frag_next, 2430 .stop = frag_stop, 2431 .show = extfrag_show, 2432 }; 2433 2434 DEFINE_SEQ_ATTRIBUTE(extfrag); 2435 2436 static int __init extfrag_debug_init(void) 2437 { 2438 struct dentry *extfrag_debug_root; 2439 2440 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2441 2442 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2443 &unusable_fops); 2444 2445 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2446 &extfrag_fops); 2447 2448 return 0; 2449 } 2450 2451 module_init(extfrag_debug_init); 2452 2453 #endif 2454