1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_VMSTAT_H 3 #define _LINUX_VMSTAT_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 #include <linux/mmzone.h> 8 #include <linux/vm_event_item.h> 9 #include <linux/atomic.h> 10 #include <linux/static_key.h> 11 #include <linux/mmdebug.h> 12 13 #ifdef CONFIG_NUMA 14 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key); 15 #endif 16 17 struct reclaim_stat { 18 unsigned nr_dirty; 19 unsigned nr_unqueued_dirty; 20 unsigned nr_congested; 21 unsigned nr_writeback; 22 unsigned nr_immediate; 23 unsigned nr_pageout; 24 unsigned nr_activate[ANON_AND_FILE]; 25 unsigned nr_ref_keep; 26 unsigned nr_unmap_fail; 27 unsigned nr_lazyfree_fail; 28 unsigned nr_demoted; 29 }; 30 31 /* Stat data for system wide items */ 32 enum vm_stat_item { 33 NR_DIRTY_THRESHOLD, 34 NR_DIRTY_BG_THRESHOLD, 35 NR_MEMMAP_PAGES, /* page metadata allocated through buddy allocator */ 36 NR_MEMMAP_BOOT_PAGES, /* page metadata allocated through boot allocator */ 37 NR_VM_STAT_ITEMS, 38 }; 39 40 #ifdef CONFIG_VM_EVENT_COUNTERS 41 /* 42 * Light weight per cpu counter implementation. 43 * 44 * Counters should only be incremented and no critical kernel component 45 * should rely on the counter values. 46 * 47 * Counters are handled completely inline. On many platforms the code 48 * generated will simply be the increment of a global address. 49 */ 50 51 struct vm_event_state { 52 unsigned long event[NR_VM_EVENT_ITEMS]; 53 }; 54 55 DECLARE_PER_CPU(struct vm_event_state, vm_event_states); 56 57 /* 58 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the 59 * local_irq_disable overhead. 60 */ 61 static inline void __count_vm_event(enum vm_event_item item) 62 { 63 raw_cpu_inc(vm_event_states.event[item]); 64 } 65 66 static inline void count_vm_event(enum vm_event_item item) 67 { 68 this_cpu_inc(vm_event_states.event[item]); 69 } 70 71 static inline void __count_vm_events(enum vm_event_item item, long delta) 72 { 73 raw_cpu_add(vm_event_states.event[item], delta); 74 } 75 76 static inline void count_vm_events(enum vm_event_item item, long delta) 77 { 78 this_cpu_add(vm_event_states.event[item], delta); 79 } 80 81 extern void all_vm_events(unsigned long *); 82 83 extern void vm_events_fold_cpu(int cpu); 84 85 #else 86 87 /* Disable counters */ 88 static inline void count_vm_event(enum vm_event_item item) 89 { 90 } 91 static inline void count_vm_events(enum vm_event_item item, long delta) 92 { 93 } 94 static inline void __count_vm_event(enum vm_event_item item) 95 { 96 } 97 static inline void __count_vm_events(enum vm_event_item item, long delta) 98 { 99 } 100 static inline void all_vm_events(unsigned long *ret) 101 { 102 } 103 static inline void vm_events_fold_cpu(int cpu) 104 { 105 } 106 107 #endif /* CONFIG_VM_EVENT_COUNTERS */ 108 109 #ifdef CONFIG_NUMA_BALANCING 110 #define count_vm_numa_event(x) count_vm_event(x) 111 #define count_vm_numa_events(x, y) count_vm_events(x, y) 112 #else 113 #define count_vm_numa_event(x) do {} while (0) 114 #define count_vm_numa_events(x, y) do { (void)(y); } while (0) 115 #endif /* CONFIG_NUMA_BALANCING */ 116 117 #ifdef CONFIG_DEBUG_TLBFLUSH 118 #define count_vm_tlb_event(x) count_vm_event(x) 119 #define count_vm_tlb_events(x, y) count_vm_events(x, y) 120 #else 121 #define count_vm_tlb_event(x) do {} while (0) 122 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) 123 #endif 124 125 #ifdef CONFIG_PER_VMA_LOCK_STATS 126 #define count_vm_vma_lock_event(x) count_vm_event(x) 127 #else 128 #define count_vm_vma_lock_event(x) do {} while (0) 129 #endif 130 131 #define __count_zid_vm_events(item, zid, delta) \ 132 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) 133 134 /* 135 * Zone and node-based page accounting with per cpu differentials. 136 */ 137 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS]; 138 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS]; 139 extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; 140 141 #ifdef CONFIG_NUMA 142 static inline void zone_numa_event_add(long x, struct zone *zone, 143 enum numa_stat_item item) 144 { 145 atomic_long_add(x, &zone->vm_numa_event[item]); 146 atomic_long_add(x, &vm_numa_event[item]); 147 } 148 149 static inline unsigned long zone_numa_event_state(struct zone *zone, 150 enum numa_stat_item item) 151 { 152 return atomic_long_read(&zone->vm_numa_event[item]); 153 } 154 155 static inline unsigned long 156 global_numa_event_state(enum numa_stat_item item) 157 { 158 return atomic_long_read(&vm_numa_event[item]); 159 } 160 #endif /* CONFIG_NUMA */ 161 162 static inline void zone_page_state_add(long x, struct zone *zone, 163 enum zone_stat_item item) 164 { 165 atomic_long_add(x, &zone->vm_stat[item]); 166 atomic_long_add(x, &vm_zone_stat[item]); 167 } 168 169 static inline void node_page_state_add(long x, struct pglist_data *pgdat, 170 enum node_stat_item item) 171 { 172 atomic_long_add(x, &pgdat->vm_stat[item]); 173 atomic_long_add(x, &vm_node_stat[item]); 174 } 175 176 static inline unsigned long global_zone_page_state(enum zone_stat_item item) 177 { 178 long x = atomic_long_read(&vm_zone_stat[item]); 179 #ifdef CONFIG_SMP 180 if (x < 0) 181 x = 0; 182 #endif 183 return x; 184 } 185 186 static inline 187 unsigned long global_node_page_state_pages(enum node_stat_item item) 188 { 189 long x = atomic_long_read(&vm_node_stat[item]); 190 #ifdef CONFIG_SMP 191 if (x < 0) 192 x = 0; 193 #endif 194 return x; 195 } 196 197 static inline unsigned long global_node_page_state(enum node_stat_item item) 198 { 199 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 200 201 return global_node_page_state_pages(item); 202 } 203 204 static inline unsigned long zone_page_state(struct zone *zone, 205 enum zone_stat_item item) 206 { 207 long x = atomic_long_read(&zone->vm_stat[item]); 208 #ifdef CONFIG_SMP 209 if (x < 0) 210 x = 0; 211 #endif 212 return x; 213 } 214 215 /* 216 * More accurate version that also considers the currently pending 217 * deltas. For that we need to loop over all cpus to find the current 218 * deltas. There is no synchronization so the result cannot be 219 * exactly accurate either. 220 */ 221 static inline unsigned long zone_page_state_snapshot(struct zone *zone, 222 enum zone_stat_item item) 223 { 224 long x = atomic_long_read(&zone->vm_stat[item]); 225 226 #ifdef CONFIG_SMP 227 int cpu; 228 for_each_online_cpu(cpu) 229 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item]; 230 231 if (x < 0) 232 x = 0; 233 #endif 234 return x; 235 } 236 237 #ifdef CONFIG_NUMA 238 /* See __count_vm_event comment on why raw_cpu_inc is used. */ 239 static inline void 240 __count_numa_event(struct zone *zone, enum numa_stat_item item) 241 { 242 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 243 244 raw_cpu_inc(pzstats->vm_numa_event[item]); 245 } 246 247 static inline void 248 __count_numa_events(struct zone *zone, enum numa_stat_item item, long delta) 249 { 250 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 251 252 raw_cpu_add(pzstats->vm_numa_event[item], delta); 253 } 254 255 extern unsigned long sum_zone_node_page_state(int node, 256 enum zone_stat_item item); 257 extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item); 258 extern unsigned long node_page_state(struct pglist_data *pgdat, 259 enum node_stat_item item); 260 extern unsigned long node_page_state_pages(struct pglist_data *pgdat, 261 enum node_stat_item item); 262 extern void fold_vm_numa_events(void); 263 #else 264 #define sum_zone_node_page_state(node, item) global_zone_page_state(item) 265 #define node_page_state(node, item) global_node_page_state(item) 266 #define node_page_state_pages(node, item) global_node_page_state_pages(item) 267 static inline void fold_vm_numa_events(void) 268 { 269 } 270 #endif /* CONFIG_NUMA */ 271 272 #ifdef CONFIG_SMP 273 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long); 274 void __inc_zone_page_state(struct page *, enum zone_stat_item); 275 void __dec_zone_page_state(struct page *, enum zone_stat_item); 276 277 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long); 278 void __inc_node_page_state(struct page *, enum node_stat_item); 279 void __dec_node_page_state(struct page *, enum node_stat_item); 280 281 void mod_zone_page_state(struct zone *, enum zone_stat_item, long); 282 void inc_zone_page_state(struct page *, enum zone_stat_item); 283 void dec_zone_page_state(struct page *, enum zone_stat_item); 284 285 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long); 286 void inc_node_page_state(struct page *, enum node_stat_item); 287 void dec_node_page_state(struct page *, enum node_stat_item); 288 289 extern void __inc_zone_state(struct zone *, enum zone_stat_item); 290 extern void __inc_node_state(struct pglist_data *, enum node_stat_item); 291 extern void __dec_zone_state(struct zone *, enum zone_stat_item); 292 extern void __dec_node_state(struct pglist_data *, enum node_stat_item); 293 294 void quiet_vmstat(void); 295 void cpu_vm_stats_fold(int cpu); 296 void refresh_zone_stat_thresholds(void); 297 298 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *); 299 300 int calculate_pressure_threshold(struct zone *zone); 301 int calculate_normal_threshold(struct zone *zone); 302 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 303 int (*calculate_pressure)(struct zone *)); 304 void vmstat_flush_workqueue(void); 305 #else /* CONFIG_SMP */ 306 307 /* 308 * We do not maintain differentials in a single processor configuration. 309 * The functions directly modify the zone and global counters. 310 */ 311 static inline void __mod_zone_page_state(struct zone *zone, 312 enum zone_stat_item item, long delta) 313 { 314 zone_page_state_add(delta, zone, item); 315 } 316 317 static inline void __mod_node_page_state(struct pglist_data *pgdat, 318 enum node_stat_item item, int delta) 319 { 320 if (vmstat_item_in_bytes(item)) { 321 /* 322 * Only cgroups use subpage accounting right now; at 323 * the global level, these items still change in 324 * multiples of whole pages. Store them as pages 325 * internally to keep the per-cpu counters compact. 326 */ 327 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 328 delta >>= PAGE_SHIFT; 329 } 330 331 node_page_state_add(delta, pgdat, item); 332 } 333 334 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 335 { 336 atomic_long_inc(&zone->vm_stat[item]); 337 atomic_long_inc(&vm_zone_stat[item]); 338 } 339 340 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 341 { 342 atomic_long_inc(&pgdat->vm_stat[item]); 343 atomic_long_inc(&vm_node_stat[item]); 344 } 345 346 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 347 { 348 atomic_long_dec(&zone->vm_stat[item]); 349 atomic_long_dec(&vm_zone_stat[item]); 350 } 351 352 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 353 { 354 atomic_long_dec(&pgdat->vm_stat[item]); 355 atomic_long_dec(&vm_node_stat[item]); 356 } 357 358 static inline void __inc_zone_page_state(struct page *page, 359 enum zone_stat_item item) 360 { 361 __inc_zone_state(page_zone(page), item); 362 } 363 364 static inline void __inc_node_page_state(struct page *page, 365 enum node_stat_item item) 366 { 367 __inc_node_state(page_pgdat(page), item); 368 } 369 370 371 static inline void __dec_zone_page_state(struct page *page, 372 enum zone_stat_item item) 373 { 374 __dec_zone_state(page_zone(page), item); 375 } 376 377 static inline void __dec_node_page_state(struct page *page, 378 enum node_stat_item item) 379 { 380 __dec_node_state(page_pgdat(page), item); 381 } 382 383 384 /* 385 * We only use atomic operations to update counters. So there is no need to 386 * disable interrupts. 387 */ 388 #define inc_zone_page_state __inc_zone_page_state 389 #define dec_zone_page_state __dec_zone_page_state 390 #define mod_zone_page_state __mod_zone_page_state 391 392 #define inc_node_page_state __inc_node_page_state 393 #define dec_node_page_state __dec_node_page_state 394 #define mod_node_page_state __mod_node_page_state 395 396 #define set_pgdat_percpu_threshold(pgdat, callback) { } 397 398 static inline void refresh_zone_stat_thresholds(void) { } 399 static inline void cpu_vm_stats_fold(int cpu) { } 400 static inline void quiet_vmstat(void) { } 401 static inline void vmstat_flush_workqueue(void) { } 402 403 static inline void drain_zonestat(struct zone *zone, 404 struct per_cpu_zonestat *pzstats) { } 405 #endif /* CONFIG_SMP */ 406 407 static inline void __zone_stat_mod_folio(struct folio *folio, 408 enum zone_stat_item item, long nr) 409 { 410 __mod_zone_page_state(folio_zone(folio), item, nr); 411 } 412 413 static inline void __zone_stat_add_folio(struct folio *folio, 414 enum zone_stat_item item) 415 { 416 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio)); 417 } 418 419 static inline void __zone_stat_sub_folio(struct folio *folio, 420 enum zone_stat_item item) 421 { 422 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio)); 423 } 424 425 static inline void zone_stat_mod_folio(struct folio *folio, 426 enum zone_stat_item item, long nr) 427 { 428 mod_zone_page_state(folio_zone(folio), item, nr); 429 } 430 431 static inline void zone_stat_add_folio(struct folio *folio, 432 enum zone_stat_item item) 433 { 434 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio)); 435 } 436 437 static inline void zone_stat_sub_folio(struct folio *folio, 438 enum zone_stat_item item) 439 { 440 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio)); 441 } 442 443 static inline void __node_stat_mod_folio(struct folio *folio, 444 enum node_stat_item item, long nr) 445 { 446 __mod_node_page_state(folio_pgdat(folio), item, nr); 447 } 448 449 static inline void __node_stat_add_folio(struct folio *folio, 450 enum node_stat_item item) 451 { 452 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio)); 453 } 454 455 static inline void __node_stat_sub_folio(struct folio *folio, 456 enum node_stat_item item) 457 { 458 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); 459 } 460 461 static inline void node_stat_mod_folio(struct folio *folio, 462 enum node_stat_item item, long nr) 463 { 464 mod_node_page_state(folio_pgdat(folio), item, nr); 465 } 466 467 static inline void node_stat_add_folio(struct folio *folio, 468 enum node_stat_item item) 469 { 470 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio)); 471 } 472 473 static inline void node_stat_sub_folio(struct folio *folio, 474 enum node_stat_item item) 475 { 476 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); 477 } 478 479 extern const char * const vmstat_text[]; 480 481 static inline const char *zone_stat_name(enum zone_stat_item item) 482 { 483 return vmstat_text[item]; 484 } 485 486 #ifdef CONFIG_NUMA 487 static inline const char *numa_stat_name(enum numa_stat_item item) 488 { 489 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 490 item]; 491 } 492 #endif /* CONFIG_NUMA */ 493 494 static inline const char *node_stat_name(enum node_stat_item item) 495 { 496 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 497 NR_VM_NUMA_EVENT_ITEMS + 498 item]; 499 } 500 501 static inline const char *lru_list_name(enum lru_list lru) 502 { 503 return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_" 504 } 505 506 #if defined(CONFIG_VM_EVENT_COUNTERS) 507 static inline const char *vm_event_name(enum vm_event_item item) 508 { 509 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 510 NR_VM_NUMA_EVENT_ITEMS + 511 NR_VM_NODE_STAT_ITEMS + 512 NR_VM_STAT_ITEMS + 513 item]; 514 } 515 #endif /* CONFIG_VM_EVENT_COUNTERS */ 516 517 #ifdef CONFIG_MEMCG 518 519 void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 520 int val); 521 522 void lruvec_stat_mod_folio(struct folio *folio, 523 enum node_stat_item idx, int val); 524 525 static inline void mod_lruvec_page_state(struct page *page, 526 enum node_stat_item idx, int val) 527 { 528 lruvec_stat_mod_folio(page_folio(page), idx, val); 529 } 530 531 #else 532 533 static inline void mod_lruvec_state(struct lruvec *lruvec, 534 enum node_stat_item idx, int val) 535 { 536 mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 537 } 538 539 static inline void lruvec_stat_mod_folio(struct folio *folio, 540 enum node_stat_item idx, int val) 541 { 542 mod_node_page_state(folio_pgdat(folio), idx, val); 543 } 544 545 static inline void mod_lruvec_page_state(struct page *page, 546 enum node_stat_item idx, int val) 547 { 548 mod_node_page_state(page_pgdat(page), idx, val); 549 } 550 551 #endif /* CONFIG_MEMCG */ 552 553 static inline void lruvec_stat_add_folio(struct folio *folio, 554 enum node_stat_item idx) 555 { 556 lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio)); 557 } 558 559 static inline void lruvec_stat_sub_folio(struct folio *folio, 560 enum node_stat_item idx) 561 { 562 lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio)); 563 } 564 565 void memmap_boot_pages_add(long delta); 566 void memmap_pages_add(long delta); 567 #endif /* _LINUX_VMSTAT_H */ 568