1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <christoph@lameter.com> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13 #include <linux/fs.h> 14 #include <linux/mm.h> 15 #include <linux/err.h> 16 #include <linux/module.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/cpumask.h> 20 #include <linux/vmstat.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/debugfs.h> 24 #include <linux/sched.h> 25 #include <linux/math64.h> 26 #include <linux/writeback.h> 27 #include <linux/compaction.h> 28 #include <linux/mm_inline.h> 29 #include <linux/page_owner.h> 30 #include <linux/sched/isolation.h> 31 32 #include "internal.h" 33 34 #ifdef CONFIG_NUMA 35 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 36 37 /* zero numa counters within a zone */ 38 static void zero_zone_numa_counters(struct zone *zone) 39 { 40 int item, cpu; 41 42 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 43 atomic_long_set(&zone->vm_numa_event[item], 0); 44 for_each_online_cpu(cpu) { 45 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 46 = 0; 47 } 48 } 49 } 50 51 /* zero numa counters of all the populated zones */ 52 static void zero_zones_numa_counters(void) 53 { 54 struct zone *zone; 55 56 for_each_populated_zone(zone) 57 zero_zone_numa_counters(zone); 58 } 59 60 /* zero global numa counters */ 61 static void zero_global_numa_counters(void) 62 { 63 int item; 64 65 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 66 atomic_long_set(&vm_numa_event[item], 0); 67 } 68 69 static void invalid_numa_statistics(void) 70 { 71 zero_zones_numa_counters(); 72 zero_global_numa_counters(); 73 } 74 75 static DEFINE_MUTEX(vm_numa_stat_lock); 76 77 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write, 78 void *buffer, size_t *length, loff_t *ppos) 79 { 80 int ret, oldval; 81 82 mutex_lock(&vm_numa_stat_lock); 83 if (write) 84 oldval = sysctl_vm_numa_stat; 85 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 86 if (ret || !write) 87 goto out; 88 89 if (oldval == sysctl_vm_numa_stat) 90 goto out; 91 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 92 static_branch_enable(&vm_numa_stat_key); 93 pr_info("enable numa statistics\n"); 94 } else { 95 static_branch_disable(&vm_numa_stat_key); 96 invalid_numa_statistics(); 97 pr_info("disable numa statistics, and clear numa counters\n"); 98 } 99 100 out: 101 mutex_unlock(&vm_numa_stat_lock); 102 return ret; 103 } 104 #endif 105 106 #ifdef CONFIG_VM_EVENT_COUNTERS 107 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 108 EXPORT_PER_CPU_SYMBOL(vm_event_states); 109 110 static void sum_vm_events(unsigned long *ret) 111 { 112 int cpu; 113 int i; 114 115 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 116 117 for_each_online_cpu(cpu) { 118 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 119 120 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 121 ret[i] += this->event[i]; 122 } 123 } 124 125 /* 126 * Accumulate the vm event counters across all CPUs. 127 * The result is unavoidably approximate - it can change 128 * during and after execution of this function. 129 */ 130 void all_vm_events(unsigned long *ret) 131 { 132 cpus_read_lock(); 133 sum_vm_events(ret); 134 cpus_read_unlock(); 135 } 136 EXPORT_SYMBOL_GPL(all_vm_events); 137 138 /* 139 * Fold the foreign cpu events into our own. 140 * 141 * This is adding to the events on one processor 142 * but keeps the global counts constant. 143 */ 144 void vm_events_fold_cpu(int cpu) 145 { 146 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 147 int i; 148 149 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 150 count_vm_events(i, fold_state->event[i]); 151 fold_state->event[i] = 0; 152 } 153 } 154 155 #endif /* CONFIG_VM_EVENT_COUNTERS */ 156 157 /* 158 * Manage combined zone based / global counters 159 * 160 * vm_stat contains the global counters 161 */ 162 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 163 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 164 atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 165 EXPORT_SYMBOL(vm_zone_stat); 166 EXPORT_SYMBOL(vm_node_stat); 167 168 #ifdef CONFIG_NUMA 169 static void fold_vm_zone_numa_events(struct zone *zone) 170 { 171 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 172 int cpu; 173 enum numa_stat_item item; 174 175 for_each_online_cpu(cpu) { 176 struct per_cpu_zonestat *pzstats; 177 178 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 179 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 180 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 181 } 182 183 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 184 zone_numa_event_add(zone_numa_events[item], zone, item); 185 } 186 187 void fold_vm_numa_events(void) 188 { 189 struct zone *zone; 190 191 for_each_populated_zone(zone) 192 fold_vm_zone_numa_events(zone); 193 } 194 #endif 195 196 #ifdef CONFIG_SMP 197 198 int calculate_pressure_threshold(struct zone *zone) 199 { 200 int threshold; 201 int watermark_distance; 202 203 /* 204 * As vmstats are not up to date, there is drift between the estimated 205 * and real values. For high thresholds and a high number of CPUs, it 206 * is possible for the min watermark to be breached while the estimated 207 * value looks fine. The pressure threshold is a reduced value such 208 * that even the maximum amount of drift will not accidentally breach 209 * the min watermark 210 */ 211 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 212 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 213 214 /* 215 * Maximum threshold is 125 216 */ 217 threshold = min(125, threshold); 218 219 return threshold; 220 } 221 222 int calculate_normal_threshold(struct zone *zone) 223 { 224 int threshold; 225 int mem; /* memory in 128 MB units */ 226 227 /* 228 * The threshold scales with the number of processors and the amount 229 * of memory per zone. More memory means that we can defer updates for 230 * longer, more processors could lead to more contention. 231 * fls() is used to have a cheap way of logarithmic scaling. 232 * 233 * Some sample thresholds: 234 * 235 * Threshold Processors (fls) Zonesize fls(mem)+1 236 * ------------------------------------------------------------------ 237 * 8 1 1 0.9-1 GB 4 238 * 16 2 2 0.9-1 GB 4 239 * 20 2 2 1-2 GB 5 240 * 24 2 2 2-4 GB 6 241 * 28 2 2 4-8 GB 7 242 * 32 2 2 8-16 GB 8 243 * 4 2 2 <128M 1 244 * 30 4 3 2-4 GB 5 245 * 48 4 3 8-16 GB 8 246 * 32 8 4 1-2 GB 4 247 * 32 8 4 0.9-1GB 4 248 * 10 16 5 <128M 1 249 * 40 16 5 900M 4 250 * 70 64 7 2-4 GB 5 251 * 84 64 7 4-8 GB 6 252 * 108 512 9 4-8 GB 6 253 * 125 1024 10 8-16 GB 8 254 * 125 1024 10 16-32 GB 9 255 */ 256 257 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 258 259 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 260 261 /* 262 * Maximum threshold is 125 263 */ 264 threshold = min(125, threshold); 265 266 return threshold; 267 } 268 269 /* 270 * Refresh the thresholds for each zone. 271 */ 272 void refresh_zone_stat_thresholds(void) 273 { 274 struct pglist_data *pgdat; 275 struct zone *zone; 276 int cpu; 277 int threshold; 278 279 /* Zero current pgdat thresholds */ 280 for_each_online_pgdat(pgdat) { 281 for_each_online_cpu(cpu) { 282 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 283 } 284 } 285 286 for_each_populated_zone(zone) { 287 struct pglist_data *pgdat = zone->zone_pgdat; 288 unsigned long max_drift, tolerate_drift; 289 290 threshold = calculate_normal_threshold(zone); 291 292 for_each_online_cpu(cpu) { 293 int pgdat_threshold; 294 295 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 296 = threshold; 297 298 /* Base nodestat threshold on the largest populated zone. */ 299 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 300 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 301 = max(threshold, pgdat_threshold); 302 } 303 304 /* 305 * Only set percpu_drift_mark if there is a danger that 306 * NR_FREE_PAGES reports the low watermark is ok when in fact 307 * the min watermark could be breached by an allocation 308 */ 309 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 310 max_drift = num_online_cpus() * threshold; 311 if (max_drift > tolerate_drift) 312 zone->percpu_drift_mark = high_wmark_pages(zone) + 313 max_drift; 314 } 315 } 316 317 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 318 int (*calculate_pressure)(struct zone *)) 319 { 320 struct zone *zone; 321 int cpu; 322 int threshold; 323 int i; 324 325 for (i = 0; i < pgdat->nr_zones; i++) { 326 zone = &pgdat->node_zones[i]; 327 if (!zone->percpu_drift_mark) 328 continue; 329 330 threshold = (*calculate_pressure)(zone); 331 for_each_online_cpu(cpu) 332 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 333 = threshold; 334 } 335 } 336 337 /* 338 * For use when we know that interrupts are disabled, 339 * or when we know that preemption is disabled and that 340 * particular counter cannot be updated from interrupt context. 341 */ 342 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 343 long delta) 344 { 345 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 346 s8 __percpu *p = pcp->vm_stat_diff + item; 347 long x; 348 long t; 349 350 /* 351 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 352 * atomicity is provided by IRQs being disabled -- either explicitly 353 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 354 * CPU migrations and preemption potentially corrupts a counter so 355 * disable preemption. 356 */ 357 preempt_disable_nested(); 358 359 x = delta + __this_cpu_read(*p); 360 361 t = __this_cpu_read(pcp->stat_threshold); 362 363 if (unlikely(abs(x) > t)) { 364 zone_page_state_add(x, zone, item); 365 x = 0; 366 } 367 __this_cpu_write(*p, x); 368 369 preempt_enable_nested(); 370 } 371 EXPORT_SYMBOL(__mod_zone_page_state); 372 373 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 374 long delta) 375 { 376 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 377 s8 __percpu *p = pcp->vm_node_stat_diff + item; 378 long x; 379 long t; 380 381 if (vmstat_item_in_bytes(item)) { 382 /* 383 * Only cgroups use subpage accounting right now; at 384 * the global level, these items still change in 385 * multiples of whole pages. Store them as pages 386 * internally to keep the per-cpu counters compact. 387 */ 388 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 389 delta >>= PAGE_SHIFT; 390 } 391 392 /* See __mod_node_page_state */ 393 preempt_disable_nested(); 394 395 x = delta + __this_cpu_read(*p); 396 397 t = __this_cpu_read(pcp->stat_threshold); 398 399 if (unlikely(abs(x) > t)) { 400 node_page_state_add(x, pgdat, item); 401 x = 0; 402 } 403 __this_cpu_write(*p, x); 404 405 preempt_enable_nested(); 406 } 407 EXPORT_SYMBOL(__mod_node_page_state); 408 409 /* 410 * Optimized increment and decrement functions. 411 * 412 * These are only for a single page and therefore can take a struct page * 413 * argument instead of struct zone *. This allows the inclusion of the code 414 * generated for page_zone(page) into the optimized functions. 415 * 416 * No overflow check is necessary and therefore the differential can be 417 * incremented or decremented in place which may allow the compilers to 418 * generate better code. 419 * The increment or decrement is known and therefore one boundary check can 420 * be omitted. 421 * 422 * NOTE: These functions are very performance sensitive. Change only 423 * with care. 424 * 425 * Some processors have inc/dec instructions that are atomic vs an interrupt. 426 * However, the code must first determine the differential location in a zone 427 * based on the processor number and then inc/dec the counter. There is no 428 * guarantee without disabling preemption that the processor will not change 429 * in between and therefore the atomicity vs. interrupt cannot be exploited 430 * in a useful way here. 431 */ 432 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 433 { 434 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 435 s8 __percpu *p = pcp->vm_stat_diff + item; 436 s8 v, t; 437 438 /* See __mod_node_page_state */ 439 preempt_disable_nested(); 440 441 v = __this_cpu_inc_return(*p); 442 t = __this_cpu_read(pcp->stat_threshold); 443 if (unlikely(v > t)) { 444 s8 overstep = t >> 1; 445 446 zone_page_state_add(v + overstep, zone, item); 447 __this_cpu_write(*p, -overstep); 448 } 449 450 preempt_enable_nested(); 451 } 452 453 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 454 { 455 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 456 s8 __percpu *p = pcp->vm_node_stat_diff + item; 457 s8 v, t; 458 459 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 460 461 /* See __mod_node_page_state */ 462 preempt_disable_nested(); 463 464 v = __this_cpu_inc_return(*p); 465 t = __this_cpu_read(pcp->stat_threshold); 466 if (unlikely(v > t)) { 467 s8 overstep = t >> 1; 468 469 node_page_state_add(v + overstep, pgdat, item); 470 __this_cpu_write(*p, -overstep); 471 } 472 473 preempt_enable_nested(); 474 } 475 476 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 477 { 478 __inc_zone_state(page_zone(page), item); 479 } 480 EXPORT_SYMBOL(__inc_zone_page_state); 481 482 void __inc_node_page_state(struct page *page, enum node_stat_item item) 483 { 484 __inc_node_state(page_pgdat(page), item); 485 } 486 EXPORT_SYMBOL(__inc_node_page_state); 487 488 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 489 { 490 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 491 s8 __percpu *p = pcp->vm_stat_diff + item; 492 s8 v, t; 493 494 /* See __mod_node_page_state */ 495 preempt_disable_nested(); 496 497 v = __this_cpu_dec_return(*p); 498 t = __this_cpu_read(pcp->stat_threshold); 499 if (unlikely(v < - t)) { 500 s8 overstep = t >> 1; 501 502 zone_page_state_add(v - overstep, zone, item); 503 __this_cpu_write(*p, overstep); 504 } 505 506 preempt_enable_nested(); 507 } 508 509 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 510 { 511 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 512 s8 __percpu *p = pcp->vm_node_stat_diff + item; 513 s8 v, t; 514 515 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 516 517 /* See __mod_node_page_state */ 518 preempt_disable_nested(); 519 520 v = __this_cpu_dec_return(*p); 521 t = __this_cpu_read(pcp->stat_threshold); 522 if (unlikely(v < - t)) { 523 s8 overstep = t >> 1; 524 525 node_page_state_add(v - overstep, pgdat, item); 526 __this_cpu_write(*p, overstep); 527 } 528 529 preempt_enable_nested(); 530 } 531 532 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 533 { 534 __dec_zone_state(page_zone(page), item); 535 } 536 EXPORT_SYMBOL(__dec_zone_page_state); 537 538 void __dec_node_page_state(struct page *page, enum node_stat_item item) 539 { 540 __dec_node_state(page_pgdat(page), item); 541 } 542 EXPORT_SYMBOL(__dec_node_page_state); 543 544 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 545 /* 546 * If we have cmpxchg_local support then we do not need to incur the overhead 547 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 548 * 549 * mod_state() modifies the zone counter state through atomic per cpu 550 * operations. 551 * 552 * Overstep mode specifies how overstep should handled: 553 * 0 No overstepping 554 * 1 Overstepping half of threshold 555 * -1 Overstepping minus half of threshold 556 */ 557 static inline void mod_zone_state(struct zone *zone, 558 enum zone_stat_item item, long delta, int overstep_mode) 559 { 560 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 561 s8 __percpu *p = pcp->vm_stat_diff + item; 562 long n, t, z; 563 s8 o; 564 565 o = this_cpu_read(*p); 566 do { 567 z = 0; /* overflow to zone counters */ 568 569 /* 570 * The fetching of the stat_threshold is racy. We may apply 571 * a counter threshold to the wrong the cpu if we get 572 * rescheduled while executing here. However, the next 573 * counter update will apply the threshold again and 574 * therefore bring the counter under the threshold again. 575 * 576 * Most of the time the thresholds are the same anyways 577 * for all cpus in a zone. 578 */ 579 t = this_cpu_read(pcp->stat_threshold); 580 581 n = delta + (long)o; 582 583 if (abs(n) > t) { 584 int os = overstep_mode * (t >> 1) ; 585 586 /* Overflow must be added to zone counters */ 587 z = n + os; 588 n = -os; 589 } 590 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 591 592 if (z) 593 zone_page_state_add(z, zone, item); 594 } 595 596 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 597 long delta) 598 { 599 mod_zone_state(zone, item, delta, 0); 600 } 601 EXPORT_SYMBOL(mod_zone_page_state); 602 603 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 604 { 605 mod_zone_state(page_zone(page), item, 1, 1); 606 } 607 EXPORT_SYMBOL(inc_zone_page_state); 608 609 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 610 { 611 mod_zone_state(page_zone(page), item, -1, -1); 612 } 613 EXPORT_SYMBOL(dec_zone_page_state); 614 615 static inline void mod_node_state(struct pglist_data *pgdat, 616 enum node_stat_item item, int delta, int overstep_mode) 617 { 618 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 619 s8 __percpu *p = pcp->vm_node_stat_diff + item; 620 long n, t, z; 621 s8 o; 622 623 if (vmstat_item_in_bytes(item)) { 624 /* 625 * Only cgroups use subpage accounting right now; at 626 * the global level, these items still change in 627 * multiples of whole pages. Store them as pages 628 * internally to keep the per-cpu counters compact. 629 */ 630 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 631 delta >>= PAGE_SHIFT; 632 } 633 634 o = this_cpu_read(*p); 635 do { 636 z = 0; /* overflow to node counters */ 637 638 /* 639 * The fetching of the stat_threshold is racy. We may apply 640 * a counter threshold to the wrong the cpu if we get 641 * rescheduled while executing here. However, the next 642 * counter update will apply the threshold again and 643 * therefore bring the counter under the threshold again. 644 * 645 * Most of the time the thresholds are the same anyways 646 * for all cpus in a node. 647 */ 648 t = this_cpu_read(pcp->stat_threshold); 649 650 n = delta + (long)o; 651 652 if (abs(n) > t) { 653 int os = overstep_mode * (t >> 1) ; 654 655 /* Overflow must be added to node counters */ 656 z = n + os; 657 n = -os; 658 } 659 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 660 661 if (z) 662 node_page_state_add(z, pgdat, item); 663 } 664 665 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 666 long delta) 667 { 668 mod_node_state(pgdat, item, delta, 0); 669 } 670 EXPORT_SYMBOL(mod_node_page_state); 671 672 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 673 { 674 mod_node_state(pgdat, item, 1, 1); 675 } 676 677 void inc_node_page_state(struct page *page, enum node_stat_item item) 678 { 679 mod_node_state(page_pgdat(page), item, 1, 1); 680 } 681 EXPORT_SYMBOL(inc_node_page_state); 682 683 void dec_node_page_state(struct page *page, enum node_stat_item item) 684 { 685 mod_node_state(page_pgdat(page), item, -1, -1); 686 } 687 EXPORT_SYMBOL(dec_node_page_state); 688 #else 689 /* 690 * Use interrupt disable to serialize counter updates 691 */ 692 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 693 long delta) 694 { 695 unsigned long flags; 696 697 local_irq_save(flags); 698 __mod_zone_page_state(zone, item, delta); 699 local_irq_restore(flags); 700 } 701 EXPORT_SYMBOL(mod_zone_page_state); 702 703 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 704 { 705 unsigned long flags; 706 struct zone *zone; 707 708 zone = page_zone(page); 709 local_irq_save(flags); 710 __inc_zone_state(zone, item); 711 local_irq_restore(flags); 712 } 713 EXPORT_SYMBOL(inc_zone_page_state); 714 715 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 716 { 717 unsigned long flags; 718 719 local_irq_save(flags); 720 __dec_zone_page_state(page, item); 721 local_irq_restore(flags); 722 } 723 EXPORT_SYMBOL(dec_zone_page_state); 724 725 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 726 { 727 unsigned long flags; 728 729 local_irq_save(flags); 730 __inc_node_state(pgdat, item); 731 local_irq_restore(flags); 732 } 733 EXPORT_SYMBOL(inc_node_state); 734 735 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 736 long delta) 737 { 738 unsigned long flags; 739 740 local_irq_save(flags); 741 __mod_node_page_state(pgdat, item, delta); 742 local_irq_restore(flags); 743 } 744 EXPORT_SYMBOL(mod_node_page_state); 745 746 void inc_node_page_state(struct page *page, enum node_stat_item item) 747 { 748 unsigned long flags; 749 struct pglist_data *pgdat; 750 751 pgdat = page_pgdat(page); 752 local_irq_save(flags); 753 __inc_node_state(pgdat, item); 754 local_irq_restore(flags); 755 } 756 EXPORT_SYMBOL(inc_node_page_state); 757 758 void dec_node_page_state(struct page *page, enum node_stat_item item) 759 { 760 unsigned long flags; 761 762 local_irq_save(flags); 763 __dec_node_page_state(page, item); 764 local_irq_restore(flags); 765 } 766 EXPORT_SYMBOL(dec_node_page_state); 767 #endif 768 769 /* 770 * Fold a differential into the global counters. 771 * Returns the number of counters updated. 772 */ 773 static int fold_diff(int *zone_diff, int *node_diff) 774 { 775 int i; 776 int changes = 0; 777 778 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 779 if (zone_diff[i]) { 780 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 781 changes++; 782 } 783 784 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 785 if (node_diff[i]) { 786 atomic_long_add(node_diff[i], &vm_node_stat[i]); 787 changes++; 788 } 789 return changes; 790 } 791 792 /* 793 * Update the zone counters for the current cpu. 794 * 795 * Note that refresh_cpu_vm_stats strives to only access 796 * node local memory. The per cpu pagesets on remote zones are placed 797 * in the memory local to the processor using that pageset. So the 798 * loop over all zones will access a series of cachelines local to 799 * the processor. 800 * 801 * The call to zone_page_state_add updates the cachelines with the 802 * statistics in the remote zone struct as well as the global cachelines 803 * with the global counters. These could cause remote node cache line 804 * bouncing and will have to be only done when necessary. 805 * 806 * The function returns the number of global counters updated. 807 */ 808 static int refresh_cpu_vm_stats(bool do_pagesets) 809 { 810 struct pglist_data *pgdat; 811 struct zone *zone; 812 int i; 813 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 814 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 815 int changes = 0; 816 817 for_each_populated_zone(zone) { 818 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 819 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 820 821 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 822 int v; 823 824 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 825 if (v) { 826 827 atomic_long_add(v, &zone->vm_stat[i]); 828 global_zone_diff[i] += v; 829 #ifdef CONFIG_NUMA 830 /* 3 seconds idle till flush */ 831 __this_cpu_write(pcp->expire, 3); 832 #endif 833 } 834 } 835 836 if (do_pagesets) { 837 cond_resched(); 838 839 changes += decay_pcp_high(zone, this_cpu_ptr(pcp)); 840 #ifdef CONFIG_NUMA 841 /* 842 * Deal with draining the remote pageset of this 843 * processor 844 * 845 * Check if there are pages remaining in this pageset 846 * if not then there is nothing to expire. 847 */ 848 if (!__this_cpu_read(pcp->expire) || 849 !__this_cpu_read(pcp->count)) 850 continue; 851 852 /* 853 * We never drain zones local to this processor. 854 */ 855 if (zone_to_nid(zone) == numa_node_id()) { 856 __this_cpu_write(pcp->expire, 0); 857 continue; 858 } 859 860 if (__this_cpu_dec_return(pcp->expire)) { 861 changes++; 862 continue; 863 } 864 865 if (__this_cpu_read(pcp->count)) { 866 drain_zone_pages(zone, this_cpu_ptr(pcp)); 867 changes++; 868 } 869 #endif 870 } 871 } 872 873 for_each_online_pgdat(pgdat) { 874 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 875 876 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 877 int v; 878 879 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 880 if (v) { 881 atomic_long_add(v, &pgdat->vm_stat[i]); 882 global_node_diff[i] += v; 883 } 884 } 885 } 886 887 changes += fold_diff(global_zone_diff, global_node_diff); 888 return changes; 889 } 890 891 /* 892 * Fold the data for an offline cpu into the global array. 893 * There cannot be any access by the offline cpu and therefore 894 * synchronization is simplified. 895 */ 896 void cpu_vm_stats_fold(int cpu) 897 { 898 struct pglist_data *pgdat; 899 struct zone *zone; 900 int i; 901 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 902 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 903 904 for_each_populated_zone(zone) { 905 struct per_cpu_zonestat *pzstats; 906 907 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 908 909 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 910 if (pzstats->vm_stat_diff[i]) { 911 int v; 912 913 v = pzstats->vm_stat_diff[i]; 914 pzstats->vm_stat_diff[i] = 0; 915 atomic_long_add(v, &zone->vm_stat[i]); 916 global_zone_diff[i] += v; 917 } 918 } 919 #ifdef CONFIG_NUMA 920 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 921 if (pzstats->vm_numa_event[i]) { 922 unsigned long v; 923 924 v = pzstats->vm_numa_event[i]; 925 pzstats->vm_numa_event[i] = 0; 926 zone_numa_event_add(v, zone, i); 927 } 928 } 929 #endif 930 } 931 932 for_each_online_pgdat(pgdat) { 933 struct per_cpu_nodestat *p; 934 935 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 936 937 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 938 if (p->vm_node_stat_diff[i]) { 939 int v; 940 941 v = p->vm_node_stat_diff[i]; 942 p->vm_node_stat_diff[i] = 0; 943 atomic_long_add(v, &pgdat->vm_stat[i]); 944 global_node_diff[i] += v; 945 } 946 } 947 948 fold_diff(global_zone_diff, global_node_diff); 949 } 950 951 /* 952 * this is only called if !populated_zone(zone), which implies no other users of 953 * pset->vm_stat_diff[] exist. 954 */ 955 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 956 { 957 unsigned long v; 958 int i; 959 960 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 961 if (pzstats->vm_stat_diff[i]) { 962 v = pzstats->vm_stat_diff[i]; 963 pzstats->vm_stat_diff[i] = 0; 964 zone_page_state_add(v, zone, i); 965 } 966 } 967 968 #ifdef CONFIG_NUMA 969 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 970 if (pzstats->vm_numa_event[i]) { 971 v = pzstats->vm_numa_event[i]; 972 pzstats->vm_numa_event[i] = 0; 973 zone_numa_event_add(v, zone, i); 974 } 975 } 976 #endif 977 } 978 #endif 979 980 #ifdef CONFIG_NUMA 981 /* 982 * Determine the per node value of a stat item. This function 983 * is called frequently in a NUMA machine, so try to be as 984 * frugal as possible. 985 */ 986 unsigned long sum_zone_node_page_state(int node, 987 enum zone_stat_item item) 988 { 989 struct zone *zones = NODE_DATA(node)->node_zones; 990 int i; 991 unsigned long count = 0; 992 993 for (i = 0; i < MAX_NR_ZONES; i++) 994 count += zone_page_state(zones + i, item); 995 996 return count; 997 } 998 999 /* Determine the per node value of a numa stat item. */ 1000 unsigned long sum_zone_numa_event_state(int node, 1001 enum numa_stat_item item) 1002 { 1003 struct zone *zones = NODE_DATA(node)->node_zones; 1004 unsigned long count = 0; 1005 int i; 1006 1007 for (i = 0; i < MAX_NR_ZONES; i++) 1008 count += zone_numa_event_state(zones + i, item); 1009 1010 return count; 1011 } 1012 1013 /* 1014 * Determine the per node value of a stat item. 1015 */ 1016 unsigned long node_page_state_pages(struct pglist_data *pgdat, 1017 enum node_stat_item item) 1018 { 1019 long x = atomic_long_read(&pgdat->vm_stat[item]); 1020 #ifdef CONFIG_SMP 1021 if (x < 0) 1022 x = 0; 1023 #endif 1024 return x; 1025 } 1026 1027 unsigned long node_page_state(struct pglist_data *pgdat, 1028 enum node_stat_item item) 1029 { 1030 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1031 1032 return node_page_state_pages(pgdat, item); 1033 } 1034 #endif 1035 1036 #ifdef CONFIG_COMPACTION 1037 1038 struct contig_page_info { 1039 unsigned long free_pages; 1040 unsigned long free_blocks_total; 1041 unsigned long free_blocks_suitable; 1042 }; 1043 1044 /* 1045 * Calculate the number of free pages in a zone, how many contiguous 1046 * pages are free and how many are large enough to satisfy an allocation of 1047 * the target size. Note that this function makes no attempt to estimate 1048 * how many suitable free blocks there *might* be if MOVABLE pages were 1049 * migrated. Calculating that is possible, but expensive and can be 1050 * figured out from userspace 1051 */ 1052 static void fill_contig_page_info(struct zone *zone, 1053 unsigned int suitable_order, 1054 struct contig_page_info *info) 1055 { 1056 unsigned int order; 1057 1058 info->free_pages = 0; 1059 info->free_blocks_total = 0; 1060 info->free_blocks_suitable = 0; 1061 1062 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1063 unsigned long blocks; 1064 1065 /* 1066 * Count number of free blocks. 1067 * 1068 * Access to nr_free is lockless as nr_free is used only for 1069 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1070 */ 1071 blocks = data_race(zone->free_area[order].nr_free); 1072 info->free_blocks_total += blocks; 1073 1074 /* Count free base pages */ 1075 info->free_pages += blocks << order; 1076 1077 /* Count the suitable free blocks */ 1078 if (order >= suitable_order) 1079 info->free_blocks_suitable += blocks << 1080 (order - suitable_order); 1081 } 1082 } 1083 1084 /* 1085 * A fragmentation index only makes sense if an allocation of a requested 1086 * size would fail. If that is true, the fragmentation index indicates 1087 * whether external fragmentation or a lack of memory was the problem. 1088 * The value can be used to determine if page reclaim or compaction 1089 * should be used 1090 */ 1091 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1092 { 1093 unsigned long requested = 1UL << order; 1094 1095 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1096 return 0; 1097 1098 if (!info->free_blocks_total) 1099 return 0; 1100 1101 /* Fragmentation index only makes sense when a request would fail */ 1102 if (info->free_blocks_suitable) 1103 return -1000; 1104 1105 /* 1106 * Index is between 0 and 1 so return within 3 decimal places 1107 * 1108 * 0 => allocation would fail due to lack of memory 1109 * 1 => allocation would fail due to fragmentation 1110 */ 1111 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1112 } 1113 1114 /* 1115 * Calculates external fragmentation within a zone wrt the given order. 1116 * It is defined as the percentage of pages found in blocks of size 1117 * less than 1 << order. It returns values in range [0, 100]. 1118 */ 1119 unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1120 { 1121 struct contig_page_info info; 1122 1123 fill_contig_page_info(zone, order, &info); 1124 if (info.free_pages == 0) 1125 return 0; 1126 1127 return div_u64((info.free_pages - 1128 (info.free_blocks_suitable << order)) * 100, 1129 info.free_pages); 1130 } 1131 1132 /* Same as __fragmentation index but allocs contig_page_info on stack */ 1133 int fragmentation_index(struct zone *zone, unsigned int order) 1134 { 1135 struct contig_page_info info; 1136 1137 fill_contig_page_info(zone, order, &info); 1138 return __fragmentation_index(order, &info); 1139 } 1140 #endif 1141 1142 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1143 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1144 #ifdef CONFIG_ZONE_DMA 1145 #define TEXT_FOR_DMA(xx) xx "_dma", 1146 #else 1147 #define TEXT_FOR_DMA(xx) 1148 #endif 1149 1150 #ifdef CONFIG_ZONE_DMA32 1151 #define TEXT_FOR_DMA32(xx) xx "_dma32", 1152 #else 1153 #define TEXT_FOR_DMA32(xx) 1154 #endif 1155 1156 #ifdef CONFIG_HIGHMEM 1157 #define TEXT_FOR_HIGHMEM(xx) xx "_high", 1158 #else 1159 #define TEXT_FOR_HIGHMEM(xx) 1160 #endif 1161 1162 #ifdef CONFIG_ZONE_DEVICE 1163 #define TEXT_FOR_DEVICE(xx) xx "_device", 1164 #else 1165 #define TEXT_FOR_DEVICE(xx) 1166 #endif 1167 1168 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ 1169 TEXT_FOR_HIGHMEM(xx) xx "_movable", \ 1170 TEXT_FOR_DEVICE(xx) 1171 1172 const char * const vmstat_text[] = { 1173 /* enum zone_stat_item counters */ 1174 "nr_free_pages", 1175 "nr_zone_inactive_anon", 1176 "nr_zone_active_anon", 1177 "nr_zone_inactive_file", 1178 "nr_zone_active_file", 1179 "nr_zone_unevictable", 1180 "nr_zone_write_pending", 1181 "nr_mlock", 1182 "nr_bounce", 1183 #if IS_ENABLED(CONFIG_ZSMALLOC) 1184 "nr_zspages", 1185 #endif 1186 "nr_free_cma", 1187 #ifdef CONFIG_UNACCEPTED_MEMORY 1188 "nr_unaccepted", 1189 #endif 1190 1191 /* enum numa_stat_item counters */ 1192 #ifdef CONFIG_NUMA 1193 "numa_hit", 1194 "numa_miss", 1195 "numa_foreign", 1196 "numa_interleave", 1197 "numa_local", 1198 "numa_other", 1199 #endif 1200 1201 /* enum node_stat_item counters */ 1202 "nr_inactive_anon", 1203 "nr_active_anon", 1204 "nr_inactive_file", 1205 "nr_active_file", 1206 "nr_unevictable", 1207 "nr_slab_reclaimable", 1208 "nr_slab_unreclaimable", 1209 "nr_isolated_anon", 1210 "nr_isolated_file", 1211 "workingset_nodes", 1212 "workingset_refault_anon", 1213 "workingset_refault_file", 1214 "workingset_activate_anon", 1215 "workingset_activate_file", 1216 "workingset_restore_anon", 1217 "workingset_restore_file", 1218 "workingset_nodereclaim", 1219 "nr_anon_pages", 1220 "nr_mapped", 1221 "nr_file_pages", 1222 "nr_dirty", 1223 "nr_writeback", 1224 "nr_writeback_temp", 1225 "nr_shmem", 1226 "nr_shmem_hugepages", 1227 "nr_shmem_pmdmapped", 1228 "nr_file_hugepages", 1229 "nr_file_pmdmapped", 1230 "nr_anon_transparent_hugepages", 1231 "nr_vmscan_write", 1232 "nr_vmscan_immediate_reclaim", 1233 "nr_dirtied", 1234 "nr_written", 1235 "nr_throttled_written", 1236 "nr_kernel_misc_reclaimable", 1237 "nr_foll_pin_acquired", 1238 "nr_foll_pin_released", 1239 "nr_kernel_stack", 1240 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1241 "nr_shadow_call_stack", 1242 #endif 1243 "nr_page_table_pages", 1244 "nr_sec_page_table_pages", 1245 #ifdef CONFIG_IOMMU_SUPPORT 1246 "nr_iommu_pages", 1247 #endif 1248 #ifdef CONFIG_SWAP 1249 "nr_swapcached", 1250 #endif 1251 #ifdef CONFIG_NUMA_BALANCING 1252 "pgpromote_success", 1253 "pgpromote_candidate", 1254 #endif 1255 "pgdemote_kswapd", 1256 "pgdemote_direct", 1257 "pgdemote_khugepaged", 1258 1259 /* enum writeback_stat_item counters */ 1260 "nr_dirty_threshold", 1261 "nr_dirty_background_threshold", 1262 1263 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG) 1264 /* enum vm_event_item counters */ 1265 "pgpgin", 1266 "pgpgout", 1267 "pswpin", 1268 "pswpout", 1269 1270 TEXTS_FOR_ZONES("pgalloc") 1271 TEXTS_FOR_ZONES("allocstall") 1272 TEXTS_FOR_ZONES("pgskip") 1273 1274 "pgfree", 1275 "pgactivate", 1276 "pgdeactivate", 1277 "pglazyfree", 1278 1279 "pgfault", 1280 "pgmajfault", 1281 "pglazyfreed", 1282 1283 "pgrefill", 1284 "pgreuse", 1285 "pgsteal_kswapd", 1286 "pgsteal_direct", 1287 "pgsteal_khugepaged", 1288 "pgscan_kswapd", 1289 "pgscan_direct", 1290 "pgscan_khugepaged", 1291 "pgscan_direct_throttle", 1292 "pgscan_anon", 1293 "pgscan_file", 1294 "pgsteal_anon", 1295 "pgsteal_file", 1296 1297 #ifdef CONFIG_NUMA 1298 "zone_reclaim_failed", 1299 #endif 1300 "pginodesteal", 1301 "slabs_scanned", 1302 "kswapd_inodesteal", 1303 "kswapd_low_wmark_hit_quickly", 1304 "kswapd_high_wmark_hit_quickly", 1305 "pageoutrun", 1306 1307 "pgrotated", 1308 1309 "drop_pagecache", 1310 "drop_slab", 1311 "oom_kill", 1312 1313 #ifdef CONFIG_NUMA_BALANCING 1314 "numa_pte_updates", 1315 "numa_huge_pte_updates", 1316 "numa_hint_faults", 1317 "numa_hint_faults_local", 1318 "numa_pages_migrated", 1319 #endif 1320 #ifdef CONFIG_MIGRATION 1321 "pgmigrate_success", 1322 "pgmigrate_fail", 1323 "thp_migration_success", 1324 "thp_migration_fail", 1325 "thp_migration_split", 1326 #endif 1327 #ifdef CONFIG_COMPACTION 1328 "compact_migrate_scanned", 1329 "compact_free_scanned", 1330 "compact_isolated", 1331 "compact_stall", 1332 "compact_fail", 1333 "compact_success", 1334 "compact_daemon_wake", 1335 "compact_daemon_migrate_scanned", 1336 "compact_daemon_free_scanned", 1337 #endif 1338 1339 #ifdef CONFIG_HUGETLB_PAGE 1340 "htlb_buddy_alloc_success", 1341 "htlb_buddy_alloc_fail", 1342 #endif 1343 #ifdef CONFIG_CMA 1344 "cma_alloc_success", 1345 "cma_alloc_fail", 1346 #endif 1347 "unevictable_pgs_culled", 1348 "unevictable_pgs_scanned", 1349 "unevictable_pgs_rescued", 1350 "unevictable_pgs_mlocked", 1351 "unevictable_pgs_munlocked", 1352 "unevictable_pgs_cleared", 1353 "unevictable_pgs_stranded", 1354 1355 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1356 "thp_fault_alloc", 1357 "thp_fault_fallback", 1358 "thp_fault_fallback_charge", 1359 "thp_collapse_alloc", 1360 "thp_collapse_alloc_failed", 1361 "thp_file_alloc", 1362 "thp_file_fallback", 1363 "thp_file_fallback_charge", 1364 "thp_file_mapped", 1365 "thp_split_page", 1366 "thp_split_page_failed", 1367 "thp_deferred_split_page", 1368 "thp_split_pmd", 1369 "thp_scan_exceed_none_pte", 1370 "thp_scan_exceed_swap_pte", 1371 "thp_scan_exceed_share_pte", 1372 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1373 "thp_split_pud", 1374 #endif 1375 "thp_zero_page_alloc", 1376 "thp_zero_page_alloc_failed", 1377 "thp_swpout", 1378 "thp_swpout_fallback", 1379 #endif 1380 #ifdef CONFIG_MEMORY_BALLOON 1381 "balloon_inflate", 1382 "balloon_deflate", 1383 #ifdef CONFIG_BALLOON_COMPACTION 1384 "balloon_migrate", 1385 #endif 1386 #endif /* CONFIG_MEMORY_BALLOON */ 1387 #ifdef CONFIG_DEBUG_TLBFLUSH 1388 "nr_tlb_remote_flush", 1389 "nr_tlb_remote_flush_received", 1390 "nr_tlb_local_flush_all", 1391 "nr_tlb_local_flush_one", 1392 #endif /* CONFIG_DEBUG_TLBFLUSH */ 1393 1394 #ifdef CONFIG_SWAP 1395 "swap_ra", 1396 "swap_ra_hit", 1397 #ifdef CONFIG_KSM 1398 "ksm_swpin_copy", 1399 #endif 1400 #endif 1401 #ifdef CONFIG_KSM 1402 "cow_ksm", 1403 #endif 1404 #ifdef CONFIG_ZSWAP 1405 "zswpin", 1406 "zswpout", 1407 "zswpwb", 1408 #endif 1409 #ifdef CONFIG_X86 1410 "direct_map_level2_splits", 1411 "direct_map_level3_splits", 1412 #endif 1413 #ifdef CONFIG_PER_VMA_LOCK_STATS 1414 "vma_lock_success", 1415 "vma_lock_abort", 1416 "vma_lock_retry", 1417 "vma_lock_miss", 1418 #endif 1419 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ 1420 }; 1421 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1422 1423 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1424 defined(CONFIG_PROC_FS) 1425 static void *frag_start(struct seq_file *m, loff_t *pos) 1426 { 1427 pg_data_t *pgdat; 1428 loff_t node = *pos; 1429 1430 for (pgdat = first_online_pgdat(); 1431 pgdat && node; 1432 pgdat = next_online_pgdat(pgdat)) 1433 --node; 1434 1435 return pgdat; 1436 } 1437 1438 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1439 { 1440 pg_data_t *pgdat = (pg_data_t *)arg; 1441 1442 (*pos)++; 1443 return next_online_pgdat(pgdat); 1444 } 1445 1446 static void frag_stop(struct seq_file *m, void *arg) 1447 { 1448 } 1449 1450 /* 1451 * Walk zones in a node and print using a callback. 1452 * If @assert_populated is true, only use callback for zones that are populated. 1453 */ 1454 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1455 bool assert_populated, bool nolock, 1456 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1457 { 1458 struct zone *zone; 1459 struct zone *node_zones = pgdat->node_zones; 1460 unsigned long flags; 1461 1462 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1463 if (assert_populated && !populated_zone(zone)) 1464 continue; 1465 1466 if (!nolock) 1467 spin_lock_irqsave(&zone->lock, flags); 1468 print(m, pgdat, zone); 1469 if (!nolock) 1470 spin_unlock_irqrestore(&zone->lock, flags); 1471 } 1472 } 1473 #endif 1474 1475 #ifdef CONFIG_PROC_FS 1476 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1477 struct zone *zone) 1478 { 1479 int order; 1480 1481 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1482 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1483 /* 1484 * Access to nr_free is lockless as nr_free is used only for 1485 * printing purposes. Use data_race to avoid KCSAN warning. 1486 */ 1487 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1488 seq_putc(m, '\n'); 1489 } 1490 1491 /* 1492 * This walks the free areas for each zone. 1493 */ 1494 static int frag_show(struct seq_file *m, void *arg) 1495 { 1496 pg_data_t *pgdat = (pg_data_t *)arg; 1497 walk_zones_in_node(m, pgdat, true, false, frag_show_print); 1498 return 0; 1499 } 1500 1501 static void pagetypeinfo_showfree_print(struct seq_file *m, 1502 pg_data_t *pgdat, struct zone *zone) 1503 { 1504 int order, mtype; 1505 1506 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1507 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1508 pgdat->node_id, 1509 zone->name, 1510 migratetype_names[mtype]); 1511 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1512 unsigned long freecount = 0; 1513 struct free_area *area; 1514 struct list_head *curr; 1515 bool overflow = false; 1516 1517 area = &(zone->free_area[order]); 1518 1519 list_for_each(curr, &area->free_list[mtype]) { 1520 /* 1521 * Cap the free_list iteration because it might 1522 * be really large and we are under a spinlock 1523 * so a long time spent here could trigger a 1524 * hard lockup detector. Anyway this is a 1525 * debugging tool so knowing there is a handful 1526 * of pages of this order should be more than 1527 * sufficient. 1528 */ 1529 if (++freecount >= 100000) { 1530 overflow = true; 1531 break; 1532 } 1533 } 1534 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1535 spin_unlock_irq(&zone->lock); 1536 cond_resched(); 1537 spin_lock_irq(&zone->lock); 1538 } 1539 seq_putc(m, '\n'); 1540 } 1541 } 1542 1543 /* Print out the free pages at each order for each migatetype */ 1544 static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1545 { 1546 int order; 1547 pg_data_t *pgdat = (pg_data_t *)arg; 1548 1549 /* Print header */ 1550 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1551 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1552 seq_printf(m, "%6d ", order); 1553 seq_putc(m, '\n'); 1554 1555 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1556 } 1557 1558 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1559 pg_data_t *pgdat, struct zone *zone) 1560 { 1561 int mtype; 1562 unsigned long pfn; 1563 unsigned long start_pfn = zone->zone_start_pfn; 1564 unsigned long end_pfn = zone_end_pfn(zone); 1565 unsigned long count[MIGRATE_TYPES] = { 0, }; 1566 1567 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1568 struct page *page; 1569 1570 page = pfn_to_online_page(pfn); 1571 if (!page) 1572 continue; 1573 1574 if (page_zone(page) != zone) 1575 continue; 1576 1577 mtype = get_pageblock_migratetype(page); 1578 1579 if (mtype < MIGRATE_TYPES) 1580 count[mtype]++; 1581 } 1582 1583 /* Print counts */ 1584 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1585 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1586 seq_printf(m, "%12lu ", count[mtype]); 1587 seq_putc(m, '\n'); 1588 } 1589 1590 /* Print out the number of pageblocks for each migratetype */ 1591 static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1592 { 1593 int mtype; 1594 pg_data_t *pgdat = (pg_data_t *)arg; 1595 1596 seq_printf(m, "\n%-23s", "Number of blocks type "); 1597 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1598 seq_printf(m, "%12s ", migratetype_names[mtype]); 1599 seq_putc(m, '\n'); 1600 walk_zones_in_node(m, pgdat, true, false, 1601 pagetypeinfo_showblockcount_print); 1602 } 1603 1604 /* 1605 * Print out the number of pageblocks for each migratetype that contain pages 1606 * of other types. This gives an indication of how well fallbacks are being 1607 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1608 * to determine what is going on 1609 */ 1610 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1611 { 1612 #ifdef CONFIG_PAGE_OWNER 1613 int mtype; 1614 1615 if (!static_branch_unlikely(&page_owner_inited)) 1616 return; 1617 1618 drain_all_pages(NULL); 1619 1620 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1621 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1622 seq_printf(m, "%12s ", migratetype_names[mtype]); 1623 seq_putc(m, '\n'); 1624 1625 walk_zones_in_node(m, pgdat, true, true, 1626 pagetypeinfo_showmixedcount_print); 1627 #endif /* CONFIG_PAGE_OWNER */ 1628 } 1629 1630 /* 1631 * This prints out statistics in relation to grouping pages by mobility. 1632 * It is expensive to collect so do not constantly read the file. 1633 */ 1634 static int pagetypeinfo_show(struct seq_file *m, void *arg) 1635 { 1636 pg_data_t *pgdat = (pg_data_t *)arg; 1637 1638 /* check memoryless node */ 1639 if (!node_state(pgdat->node_id, N_MEMORY)) 1640 return 0; 1641 1642 seq_printf(m, "Page block order: %d\n", pageblock_order); 1643 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1644 seq_putc(m, '\n'); 1645 pagetypeinfo_showfree(m, pgdat); 1646 pagetypeinfo_showblockcount(m, pgdat); 1647 pagetypeinfo_showmixedcount(m, pgdat); 1648 1649 return 0; 1650 } 1651 1652 static const struct seq_operations fragmentation_op = { 1653 .start = frag_start, 1654 .next = frag_next, 1655 .stop = frag_stop, 1656 .show = frag_show, 1657 }; 1658 1659 static const struct seq_operations pagetypeinfo_op = { 1660 .start = frag_start, 1661 .next = frag_next, 1662 .stop = frag_stop, 1663 .show = pagetypeinfo_show, 1664 }; 1665 1666 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1667 { 1668 int zid; 1669 1670 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1671 struct zone *compare = &pgdat->node_zones[zid]; 1672 1673 if (populated_zone(compare)) 1674 return zone == compare; 1675 } 1676 1677 return false; 1678 } 1679 1680 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1681 struct zone *zone) 1682 { 1683 int i; 1684 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1685 if (is_zone_first_populated(pgdat, zone)) { 1686 seq_printf(m, "\n per-node stats"); 1687 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1688 unsigned long pages = node_page_state_pages(pgdat, i); 1689 1690 if (vmstat_item_print_in_thp(i)) 1691 pages /= HPAGE_PMD_NR; 1692 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1693 pages); 1694 } 1695 } 1696 seq_printf(m, 1697 "\n pages free %lu" 1698 "\n boost %lu" 1699 "\n min %lu" 1700 "\n low %lu" 1701 "\n high %lu" 1702 "\n spanned %lu" 1703 "\n present %lu" 1704 "\n managed %lu" 1705 "\n cma %lu", 1706 zone_page_state(zone, NR_FREE_PAGES), 1707 zone->watermark_boost, 1708 min_wmark_pages(zone), 1709 low_wmark_pages(zone), 1710 high_wmark_pages(zone), 1711 zone->spanned_pages, 1712 zone->present_pages, 1713 zone_managed_pages(zone), 1714 zone_cma_pages(zone)); 1715 1716 seq_printf(m, 1717 "\n protection: (%ld", 1718 zone->lowmem_reserve[0]); 1719 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1720 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1721 seq_putc(m, ')'); 1722 1723 /* If unpopulated, no other information is useful */ 1724 if (!populated_zone(zone)) { 1725 seq_putc(m, '\n'); 1726 return; 1727 } 1728 1729 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1730 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1731 zone_page_state(zone, i)); 1732 1733 #ifdef CONFIG_NUMA 1734 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1735 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1736 zone_numa_event_state(zone, i)); 1737 #endif 1738 1739 seq_printf(m, "\n pagesets"); 1740 for_each_online_cpu(i) { 1741 struct per_cpu_pages *pcp; 1742 struct per_cpu_zonestat __maybe_unused *pzstats; 1743 1744 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1745 seq_printf(m, 1746 "\n cpu: %i" 1747 "\n count: %i" 1748 "\n high: %i" 1749 "\n batch: %i", 1750 i, 1751 pcp->count, 1752 pcp->high, 1753 pcp->batch); 1754 #ifdef CONFIG_SMP 1755 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1756 seq_printf(m, "\n vm stats threshold: %d", 1757 pzstats->stat_threshold); 1758 #endif 1759 } 1760 seq_printf(m, 1761 "\n node_unreclaimable: %u" 1762 "\n start_pfn: %lu", 1763 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES, 1764 zone->zone_start_pfn); 1765 seq_putc(m, '\n'); 1766 } 1767 1768 /* 1769 * Output information about zones in @pgdat. All zones are printed regardless 1770 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1771 * set of all zones and userspace would not be aware of such zones if they are 1772 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1773 */ 1774 static int zoneinfo_show(struct seq_file *m, void *arg) 1775 { 1776 pg_data_t *pgdat = (pg_data_t *)arg; 1777 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1778 return 0; 1779 } 1780 1781 static const struct seq_operations zoneinfo_op = { 1782 .start = frag_start, /* iterate over all zones. The same as in 1783 * fragmentation. */ 1784 .next = frag_next, 1785 .stop = frag_stop, 1786 .show = zoneinfo_show, 1787 }; 1788 1789 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1790 NR_VM_NUMA_EVENT_ITEMS + \ 1791 NR_VM_NODE_STAT_ITEMS + \ 1792 NR_VM_WRITEBACK_STAT_ITEMS + \ 1793 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1794 NR_VM_EVENT_ITEMS : 0)) 1795 1796 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1797 { 1798 unsigned long *v; 1799 int i; 1800 1801 if (*pos >= NR_VMSTAT_ITEMS) 1802 return NULL; 1803 1804 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS); 1805 fold_vm_numa_events(); 1806 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1807 m->private = v; 1808 if (!v) 1809 return ERR_PTR(-ENOMEM); 1810 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1811 v[i] = global_zone_page_state(i); 1812 v += NR_VM_ZONE_STAT_ITEMS; 1813 1814 #ifdef CONFIG_NUMA 1815 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1816 v[i] = global_numa_event_state(i); 1817 v += NR_VM_NUMA_EVENT_ITEMS; 1818 #endif 1819 1820 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1821 v[i] = global_node_page_state_pages(i); 1822 if (vmstat_item_print_in_thp(i)) 1823 v[i] /= HPAGE_PMD_NR; 1824 } 1825 v += NR_VM_NODE_STAT_ITEMS; 1826 1827 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1828 v + NR_DIRTY_THRESHOLD); 1829 v += NR_VM_WRITEBACK_STAT_ITEMS; 1830 1831 #ifdef CONFIG_VM_EVENT_COUNTERS 1832 all_vm_events(v); 1833 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1834 v[PGPGOUT] /= 2; 1835 #endif 1836 return (unsigned long *)m->private + *pos; 1837 } 1838 1839 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1840 { 1841 (*pos)++; 1842 if (*pos >= NR_VMSTAT_ITEMS) 1843 return NULL; 1844 return (unsigned long *)m->private + *pos; 1845 } 1846 1847 static int vmstat_show(struct seq_file *m, void *arg) 1848 { 1849 unsigned long *l = arg; 1850 unsigned long off = l - (unsigned long *)m->private; 1851 1852 seq_puts(m, vmstat_text[off]); 1853 seq_put_decimal_ull(m, " ", *l); 1854 seq_putc(m, '\n'); 1855 1856 if (off == NR_VMSTAT_ITEMS - 1) { 1857 /* 1858 * We've come to the end - add any deprecated counters to avoid 1859 * breaking userspace which might depend on them being present. 1860 */ 1861 seq_puts(m, "nr_unstable 0\n"); 1862 } 1863 return 0; 1864 } 1865 1866 static void vmstat_stop(struct seq_file *m, void *arg) 1867 { 1868 kfree(m->private); 1869 m->private = NULL; 1870 } 1871 1872 static const struct seq_operations vmstat_op = { 1873 .start = vmstat_start, 1874 .next = vmstat_next, 1875 .stop = vmstat_stop, 1876 .show = vmstat_show, 1877 }; 1878 #endif /* CONFIG_PROC_FS */ 1879 1880 #ifdef CONFIG_SMP 1881 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1882 int sysctl_stat_interval __read_mostly = HZ; 1883 1884 #ifdef CONFIG_PROC_FS 1885 static void refresh_vm_stats(struct work_struct *work) 1886 { 1887 refresh_cpu_vm_stats(true); 1888 } 1889 1890 int vmstat_refresh(struct ctl_table *table, int write, 1891 void *buffer, size_t *lenp, loff_t *ppos) 1892 { 1893 long val; 1894 int err; 1895 int i; 1896 1897 /* 1898 * The regular update, every sysctl_stat_interval, may come later 1899 * than expected: leaving a significant amount in per_cpu buckets. 1900 * This is particularly misleading when checking a quantity of HUGE 1901 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 1902 * which can equally be echo'ed to or cat'ted from (by root), 1903 * can be used to update the stats just before reading them. 1904 * 1905 * Oh, and since global_zone_page_state() etc. are so careful to hide 1906 * transiently negative values, report an error here if any of 1907 * the stats is negative, so we know to go looking for imbalance. 1908 */ 1909 err = schedule_on_each_cpu(refresh_vm_stats); 1910 if (err) 1911 return err; 1912 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 1913 /* 1914 * Skip checking stats known to go negative occasionally. 1915 */ 1916 switch (i) { 1917 case NR_ZONE_WRITE_PENDING: 1918 case NR_FREE_CMA_PAGES: 1919 continue; 1920 } 1921 val = atomic_long_read(&vm_zone_stat[i]); 1922 if (val < 0) { 1923 pr_warn("%s: %s %ld\n", 1924 __func__, zone_stat_name(i), val); 1925 } 1926 } 1927 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1928 /* 1929 * Skip checking stats known to go negative occasionally. 1930 */ 1931 switch (i) { 1932 case NR_WRITEBACK: 1933 continue; 1934 } 1935 val = atomic_long_read(&vm_node_stat[i]); 1936 if (val < 0) { 1937 pr_warn("%s: %s %ld\n", 1938 __func__, node_stat_name(i), val); 1939 } 1940 } 1941 if (write) 1942 *ppos += *lenp; 1943 else 1944 *lenp = 0; 1945 return 0; 1946 } 1947 #endif /* CONFIG_PROC_FS */ 1948 1949 static void vmstat_update(struct work_struct *w) 1950 { 1951 if (refresh_cpu_vm_stats(true)) { 1952 /* 1953 * Counters were updated so we expect more updates 1954 * to occur in the future. Keep on running the 1955 * update worker thread. 1956 */ 1957 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 1958 this_cpu_ptr(&vmstat_work), 1959 round_jiffies_relative(sysctl_stat_interval)); 1960 } 1961 } 1962 1963 /* 1964 * Check if the diffs for a certain cpu indicate that 1965 * an update is needed. 1966 */ 1967 static bool need_update(int cpu) 1968 { 1969 pg_data_t *last_pgdat = NULL; 1970 struct zone *zone; 1971 1972 for_each_populated_zone(zone) { 1973 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 1974 struct per_cpu_nodestat *n; 1975 1976 /* 1977 * The fast way of checking if there are any vmstat diffs. 1978 */ 1979 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 1980 return true; 1981 1982 if (last_pgdat == zone->zone_pgdat) 1983 continue; 1984 last_pgdat = zone->zone_pgdat; 1985 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 1986 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 1987 return true; 1988 } 1989 return false; 1990 } 1991 1992 /* 1993 * Switch off vmstat processing and then fold all the remaining differentials 1994 * until the diffs stay at zero. The function is used by NOHZ and can only be 1995 * invoked when tick processing is not active. 1996 */ 1997 void quiet_vmstat(void) 1998 { 1999 if (system_state != SYSTEM_RUNNING) 2000 return; 2001 2002 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2003 return; 2004 2005 if (!need_update(smp_processor_id())) 2006 return; 2007 2008 /* 2009 * Just refresh counters and do not care about the pending delayed 2010 * vmstat_update. It doesn't fire that often to matter and canceling 2011 * it would be too expensive from this path. 2012 * vmstat_shepherd will take care about that for us. 2013 */ 2014 refresh_cpu_vm_stats(false); 2015 } 2016 2017 /* 2018 * Shepherd worker thread that checks the 2019 * differentials of processors that have their worker 2020 * threads for vm statistics updates disabled because of 2021 * inactivity. 2022 */ 2023 static void vmstat_shepherd(struct work_struct *w); 2024 2025 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2026 2027 static void vmstat_shepherd(struct work_struct *w) 2028 { 2029 int cpu; 2030 2031 cpus_read_lock(); 2032 /* Check processors whose vmstat worker threads have been disabled */ 2033 for_each_online_cpu(cpu) { 2034 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2035 2036 /* 2037 * In kernel users of vmstat counters either require the precise value and 2038 * they are using zone_page_state_snapshot interface or they can live with 2039 * an imprecision as the regular flushing can happen at arbitrary time and 2040 * cumulative error can grow (see calculate_normal_threshold). 2041 * 2042 * From that POV the regular flushing can be postponed for CPUs that have 2043 * been isolated from the kernel interference without critical 2044 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2045 * for all isolated CPUs to avoid interference with the isolated workload. 2046 */ 2047 if (cpu_is_isolated(cpu)) 2048 continue; 2049 2050 if (!delayed_work_pending(dw) && need_update(cpu)) 2051 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2052 2053 cond_resched(); 2054 } 2055 cpus_read_unlock(); 2056 2057 schedule_delayed_work(&shepherd, 2058 round_jiffies_relative(sysctl_stat_interval)); 2059 } 2060 2061 static void __init start_shepherd_timer(void) 2062 { 2063 int cpu; 2064 2065 for_each_possible_cpu(cpu) 2066 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2067 vmstat_update); 2068 2069 schedule_delayed_work(&shepherd, 2070 round_jiffies_relative(sysctl_stat_interval)); 2071 } 2072 2073 static void __init init_cpu_node_state(void) 2074 { 2075 int node; 2076 2077 for_each_online_node(node) { 2078 if (!cpumask_empty(cpumask_of_node(node))) 2079 node_set_state(node, N_CPU); 2080 } 2081 } 2082 2083 static int vmstat_cpu_online(unsigned int cpu) 2084 { 2085 refresh_zone_stat_thresholds(); 2086 2087 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2088 node_set_state(cpu_to_node(cpu), N_CPU); 2089 } 2090 2091 return 0; 2092 } 2093 2094 static int vmstat_cpu_down_prep(unsigned int cpu) 2095 { 2096 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2097 return 0; 2098 } 2099 2100 static int vmstat_cpu_dead(unsigned int cpu) 2101 { 2102 const struct cpumask *node_cpus; 2103 int node; 2104 2105 node = cpu_to_node(cpu); 2106 2107 refresh_zone_stat_thresholds(); 2108 node_cpus = cpumask_of_node(node); 2109 if (!cpumask_empty(node_cpus)) 2110 return 0; 2111 2112 node_clear_state(node, N_CPU); 2113 2114 return 0; 2115 } 2116 2117 #endif 2118 2119 struct workqueue_struct *mm_percpu_wq; 2120 2121 void __init init_mm_internals(void) 2122 { 2123 int ret __maybe_unused; 2124 2125 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0); 2126 2127 #ifdef CONFIG_SMP 2128 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2129 NULL, vmstat_cpu_dead); 2130 if (ret < 0) 2131 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2132 2133 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2134 vmstat_cpu_online, 2135 vmstat_cpu_down_prep); 2136 if (ret < 0) 2137 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2138 2139 cpus_read_lock(); 2140 init_cpu_node_state(); 2141 cpus_read_unlock(); 2142 2143 start_shepherd_timer(); 2144 #endif 2145 #ifdef CONFIG_PROC_FS 2146 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2147 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2148 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2149 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2150 #endif 2151 } 2152 2153 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2154 2155 /* 2156 * Return an index indicating how much of the available free memory is 2157 * unusable for an allocation of the requested size. 2158 */ 2159 static int unusable_free_index(unsigned int order, 2160 struct contig_page_info *info) 2161 { 2162 /* No free memory is interpreted as all free memory is unusable */ 2163 if (info->free_pages == 0) 2164 return 1000; 2165 2166 /* 2167 * Index should be a value between 0 and 1. Return a value to 3 2168 * decimal places. 2169 * 2170 * 0 => no fragmentation 2171 * 1 => high fragmentation 2172 */ 2173 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2174 2175 } 2176 2177 static void unusable_show_print(struct seq_file *m, 2178 pg_data_t *pgdat, struct zone *zone) 2179 { 2180 unsigned int order; 2181 int index; 2182 struct contig_page_info info; 2183 2184 seq_printf(m, "Node %d, zone %8s ", 2185 pgdat->node_id, 2186 zone->name); 2187 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2188 fill_contig_page_info(zone, order, &info); 2189 index = unusable_free_index(order, &info); 2190 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2191 } 2192 2193 seq_putc(m, '\n'); 2194 } 2195 2196 /* 2197 * Display unusable free space index 2198 * 2199 * The unusable free space index measures how much of the available free 2200 * memory cannot be used to satisfy an allocation of a given size and is a 2201 * value between 0 and 1. The higher the value, the more of free memory is 2202 * unusable and by implication, the worse the external fragmentation is. This 2203 * can be expressed as a percentage by multiplying by 100. 2204 */ 2205 static int unusable_show(struct seq_file *m, void *arg) 2206 { 2207 pg_data_t *pgdat = (pg_data_t *)arg; 2208 2209 /* check memoryless node */ 2210 if (!node_state(pgdat->node_id, N_MEMORY)) 2211 return 0; 2212 2213 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2214 2215 return 0; 2216 } 2217 2218 static const struct seq_operations unusable_sops = { 2219 .start = frag_start, 2220 .next = frag_next, 2221 .stop = frag_stop, 2222 .show = unusable_show, 2223 }; 2224 2225 DEFINE_SEQ_ATTRIBUTE(unusable); 2226 2227 static void extfrag_show_print(struct seq_file *m, 2228 pg_data_t *pgdat, struct zone *zone) 2229 { 2230 unsigned int order; 2231 int index; 2232 2233 /* Alloc on stack as interrupts are disabled for zone walk */ 2234 struct contig_page_info info; 2235 2236 seq_printf(m, "Node %d, zone %8s ", 2237 pgdat->node_id, 2238 zone->name); 2239 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2240 fill_contig_page_info(zone, order, &info); 2241 index = __fragmentation_index(order, &info); 2242 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2243 } 2244 2245 seq_putc(m, '\n'); 2246 } 2247 2248 /* 2249 * Display fragmentation index for orders that allocations would fail for 2250 */ 2251 static int extfrag_show(struct seq_file *m, void *arg) 2252 { 2253 pg_data_t *pgdat = (pg_data_t *)arg; 2254 2255 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2256 2257 return 0; 2258 } 2259 2260 static const struct seq_operations extfrag_sops = { 2261 .start = frag_start, 2262 .next = frag_next, 2263 .stop = frag_stop, 2264 .show = extfrag_show, 2265 }; 2266 2267 DEFINE_SEQ_ATTRIBUTE(extfrag); 2268 2269 static int __init extfrag_debug_init(void) 2270 { 2271 struct dentry *extfrag_debug_root; 2272 2273 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2274 2275 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2276 &unusable_fops); 2277 2278 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2279 &extfrag_fops); 2280 2281 return 0; 2282 } 2283 2284 module_init(extfrag_debug_init); 2285 #endif 2286