1f6ac2354SChristoph Lameter /* 2f6ac2354SChristoph Lameter * linux/mm/vmstat.c 3f6ac2354SChristoph Lameter * 4f6ac2354SChristoph Lameter * Manages VM statistics 5f6ac2354SChristoph Lameter * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 62244b95aSChristoph Lameter * 72244b95aSChristoph Lameter * zoned VM statistics 82244b95aSChristoph Lameter * Copyright (C) 2006 Silicon Graphics, Inc., 92244b95aSChristoph Lameter * Christoph Lameter <christoph@lameter.com> 107cc36bbdSChristoph Lameter * Copyright (C) 2008-2014 Christoph Lameter 11f6ac2354SChristoph Lameter */ 128f32f7e5SAlexey Dobriyan #include <linux/fs.h> 13f6ac2354SChristoph Lameter #include <linux/mm.h> 144e950f6fSAlexey Dobriyan #include <linux/err.h> 152244b95aSChristoph Lameter #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 17df9ecabaSChristoph Lameter #include <linux/cpu.h> 187cc36bbdSChristoph Lameter #include <linux/cpumask.h> 19c748e134SAdrian Bunk #include <linux/vmstat.h> 203c486871SAndrew Morton #include <linux/proc_fs.h> 213c486871SAndrew Morton #include <linux/seq_file.h> 223c486871SAndrew Morton #include <linux/debugfs.h> 23e8edc6e0SAlexey Dobriyan #include <linux/sched.h> 24f1a5ab12SMel Gorman #include <linux/math64.h> 2579da826aSMichael Rubin #include <linux/writeback.h> 2636deb0beSNamhyung Kim #include <linux/compaction.h> 276e543d57SLisa Du #include <linux/mm_inline.h> 2848c96a36SJoonsoo Kim #include <linux/page_ext.h> 2948c96a36SJoonsoo Kim #include <linux/page_owner.h> 306e543d57SLisa Du 316e543d57SLisa Du #include "internal.h" 32f6ac2354SChristoph Lameter 331d90ca89SKemi Wang #define NUMA_STATS_THRESHOLD (U16_MAX - 2) 341d90ca89SKemi Wang 35f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 36f8891e5eSChristoph Lameter DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 37f8891e5eSChristoph Lameter EXPORT_PER_CPU_SYMBOL(vm_event_states); 38f8891e5eSChristoph Lameter 3931f961a8SMinchan Kim static void sum_vm_events(unsigned long *ret) 40f8891e5eSChristoph Lameter { 419eccf2a8SChristoph Lameter int cpu; 42f8891e5eSChristoph Lameter int i; 43f8891e5eSChristoph Lameter 44f8891e5eSChristoph Lameter memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 45f8891e5eSChristoph Lameter 4631f961a8SMinchan Kim for_each_online_cpu(cpu) { 47f8891e5eSChristoph Lameter struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 48f8891e5eSChristoph Lameter 49f8891e5eSChristoph Lameter for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 50f8891e5eSChristoph Lameter ret[i] += this->event[i]; 51f8891e5eSChristoph Lameter } 52f8891e5eSChristoph Lameter } 53f8891e5eSChristoph Lameter 54f8891e5eSChristoph Lameter /* 55f8891e5eSChristoph Lameter * Accumulate the vm event counters across all CPUs. 56f8891e5eSChristoph Lameter * The result is unavoidably approximate - it can change 57f8891e5eSChristoph Lameter * during and after execution of this function. 58f8891e5eSChristoph Lameter */ 59f8891e5eSChristoph Lameter void all_vm_events(unsigned long *ret) 60f8891e5eSChristoph Lameter { 61b5be1132SKOSAKI Motohiro get_online_cpus(); 6231f961a8SMinchan Kim sum_vm_events(ret); 63b5be1132SKOSAKI Motohiro put_online_cpus(); 64f8891e5eSChristoph Lameter } 6532dd66fcSHeiko Carstens EXPORT_SYMBOL_GPL(all_vm_events); 66f8891e5eSChristoph Lameter 67f8891e5eSChristoph Lameter /* 68f8891e5eSChristoph Lameter * Fold the foreign cpu events into our own. 69f8891e5eSChristoph Lameter * 70f8891e5eSChristoph Lameter * This is adding to the events on one processor 71f8891e5eSChristoph Lameter * but keeps the global counts constant. 72f8891e5eSChristoph Lameter */ 73f8891e5eSChristoph Lameter void vm_events_fold_cpu(int cpu) 74f8891e5eSChristoph Lameter { 75f8891e5eSChristoph Lameter struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 76f8891e5eSChristoph Lameter int i; 77f8891e5eSChristoph Lameter 78f8891e5eSChristoph Lameter for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 79f8891e5eSChristoph Lameter count_vm_events(i, fold_state->event[i]); 80f8891e5eSChristoph Lameter fold_state->event[i] = 0; 81f8891e5eSChristoph Lameter } 82f8891e5eSChristoph Lameter } 83f8891e5eSChristoph Lameter 84f8891e5eSChristoph Lameter #endif /* CONFIG_VM_EVENT_COUNTERS */ 85f8891e5eSChristoph Lameter 862244b95aSChristoph Lameter /* 872244b95aSChristoph Lameter * Manage combined zone based / global counters 882244b95aSChristoph Lameter * 892244b95aSChristoph Lameter * vm_stat contains the global counters 902244b95aSChristoph Lameter */ 9175ef7184SMel Gorman atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 923a321d2aSKemi Wang atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS] __cacheline_aligned_in_smp; 9375ef7184SMel Gorman atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 9475ef7184SMel Gorman EXPORT_SYMBOL(vm_zone_stat); 953a321d2aSKemi Wang EXPORT_SYMBOL(vm_numa_stat); 9675ef7184SMel Gorman EXPORT_SYMBOL(vm_node_stat); 972244b95aSChristoph Lameter 982244b95aSChristoph Lameter #ifdef CONFIG_SMP 992244b95aSChristoph Lameter 100b44129b3SMel Gorman int calculate_pressure_threshold(struct zone *zone) 10188f5acf8SMel Gorman { 10288f5acf8SMel Gorman int threshold; 10388f5acf8SMel Gorman int watermark_distance; 10488f5acf8SMel Gorman 10588f5acf8SMel Gorman /* 10688f5acf8SMel Gorman * As vmstats are not up to date, there is drift between the estimated 10788f5acf8SMel Gorman * and real values. For high thresholds and a high number of CPUs, it 10888f5acf8SMel Gorman * is possible for the min watermark to be breached while the estimated 10988f5acf8SMel Gorman * value looks fine. The pressure threshold is a reduced value such 11088f5acf8SMel Gorman * that even the maximum amount of drift will not accidentally breach 11188f5acf8SMel Gorman * the min watermark 11288f5acf8SMel Gorman */ 11388f5acf8SMel Gorman watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 11488f5acf8SMel Gorman threshold = max(1, (int)(watermark_distance / num_online_cpus())); 11588f5acf8SMel Gorman 11688f5acf8SMel Gorman /* 11788f5acf8SMel Gorman * Maximum threshold is 125 11888f5acf8SMel Gorman */ 11988f5acf8SMel Gorman threshold = min(125, threshold); 12088f5acf8SMel Gorman 12188f5acf8SMel Gorman return threshold; 12288f5acf8SMel Gorman } 12388f5acf8SMel Gorman 124b44129b3SMel Gorman int calculate_normal_threshold(struct zone *zone) 125df9ecabaSChristoph Lameter { 126df9ecabaSChristoph Lameter int threshold; 127df9ecabaSChristoph Lameter int mem; /* memory in 128 MB units */ 1282244b95aSChristoph Lameter 1292244b95aSChristoph Lameter /* 130df9ecabaSChristoph Lameter * The threshold scales with the number of processors and the amount 131df9ecabaSChristoph Lameter * of memory per zone. More memory means that we can defer updates for 132df9ecabaSChristoph Lameter * longer, more processors could lead to more contention. 133df9ecabaSChristoph Lameter * fls() is used to have a cheap way of logarithmic scaling. 1342244b95aSChristoph Lameter * 135df9ecabaSChristoph Lameter * Some sample thresholds: 136df9ecabaSChristoph Lameter * 137df9ecabaSChristoph Lameter * Threshold Processors (fls) Zonesize fls(mem+1) 138df9ecabaSChristoph Lameter * ------------------------------------------------------------------ 139df9ecabaSChristoph Lameter * 8 1 1 0.9-1 GB 4 140df9ecabaSChristoph Lameter * 16 2 2 0.9-1 GB 4 141df9ecabaSChristoph Lameter * 20 2 2 1-2 GB 5 142df9ecabaSChristoph Lameter * 24 2 2 2-4 GB 6 143df9ecabaSChristoph Lameter * 28 2 2 4-8 GB 7 144df9ecabaSChristoph Lameter * 32 2 2 8-16 GB 8 145df9ecabaSChristoph Lameter * 4 2 2 <128M 1 146df9ecabaSChristoph Lameter * 30 4 3 2-4 GB 5 147df9ecabaSChristoph Lameter * 48 4 3 8-16 GB 8 148df9ecabaSChristoph Lameter * 32 8 4 1-2 GB 4 149df9ecabaSChristoph Lameter * 32 8 4 0.9-1GB 4 150df9ecabaSChristoph Lameter * 10 16 5 <128M 1 151df9ecabaSChristoph Lameter * 40 16 5 900M 4 152df9ecabaSChristoph Lameter * 70 64 7 2-4 GB 5 153df9ecabaSChristoph Lameter * 84 64 7 4-8 GB 6 154df9ecabaSChristoph Lameter * 108 512 9 4-8 GB 6 155df9ecabaSChristoph Lameter * 125 1024 10 8-16 GB 8 156df9ecabaSChristoph Lameter * 125 1024 10 16-32 GB 9 1572244b95aSChristoph Lameter */ 158df9ecabaSChristoph Lameter 159b40da049SJiang Liu mem = zone->managed_pages >> (27 - PAGE_SHIFT); 160df9ecabaSChristoph Lameter 161df9ecabaSChristoph Lameter threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 162df9ecabaSChristoph Lameter 163df9ecabaSChristoph Lameter /* 164df9ecabaSChristoph Lameter * Maximum threshold is 125 165df9ecabaSChristoph Lameter */ 166df9ecabaSChristoph Lameter threshold = min(125, threshold); 167df9ecabaSChristoph Lameter 168df9ecabaSChristoph Lameter return threshold; 169df9ecabaSChristoph Lameter } 170df9ecabaSChristoph Lameter 171df9ecabaSChristoph Lameter /* 172df9ecabaSChristoph Lameter * Refresh the thresholds for each zone. 173df9ecabaSChristoph Lameter */ 174a6cccdc3SKOSAKI Motohiro void refresh_zone_stat_thresholds(void) 1752244b95aSChristoph Lameter { 17675ef7184SMel Gorman struct pglist_data *pgdat; 177df9ecabaSChristoph Lameter struct zone *zone; 178df9ecabaSChristoph Lameter int cpu; 179df9ecabaSChristoph Lameter int threshold; 180df9ecabaSChristoph Lameter 18175ef7184SMel Gorman /* Zero current pgdat thresholds */ 18275ef7184SMel Gorman for_each_online_pgdat(pgdat) { 18375ef7184SMel Gorman for_each_online_cpu(cpu) { 18475ef7184SMel Gorman per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 18575ef7184SMel Gorman } 18675ef7184SMel Gorman } 18775ef7184SMel Gorman 188ee99c71cSKOSAKI Motohiro for_each_populated_zone(zone) { 18975ef7184SMel Gorman struct pglist_data *pgdat = zone->zone_pgdat; 190aa454840SChristoph Lameter unsigned long max_drift, tolerate_drift; 191aa454840SChristoph Lameter 192b44129b3SMel Gorman threshold = calculate_normal_threshold(zone); 193df9ecabaSChristoph Lameter 19475ef7184SMel Gorman for_each_online_cpu(cpu) { 19575ef7184SMel Gorman int pgdat_threshold; 19675ef7184SMel Gorman 19799dcc3e5SChristoph Lameter per_cpu_ptr(zone->pageset, cpu)->stat_threshold 19899dcc3e5SChristoph Lameter = threshold; 1991d90ca89SKemi Wang 20075ef7184SMel Gorman /* Base nodestat threshold on the largest populated zone. */ 20175ef7184SMel Gorman pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 20275ef7184SMel Gorman per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 20375ef7184SMel Gorman = max(threshold, pgdat_threshold); 20475ef7184SMel Gorman } 20575ef7184SMel Gorman 206aa454840SChristoph Lameter /* 207aa454840SChristoph Lameter * Only set percpu_drift_mark if there is a danger that 208aa454840SChristoph Lameter * NR_FREE_PAGES reports the low watermark is ok when in fact 209aa454840SChristoph Lameter * the min watermark could be breached by an allocation 210aa454840SChristoph Lameter */ 211aa454840SChristoph Lameter tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 212aa454840SChristoph Lameter max_drift = num_online_cpus() * threshold; 213aa454840SChristoph Lameter if (max_drift > tolerate_drift) 214aa454840SChristoph Lameter zone->percpu_drift_mark = high_wmark_pages(zone) + 215aa454840SChristoph Lameter max_drift; 216df9ecabaSChristoph Lameter } 2172244b95aSChristoph Lameter } 2182244b95aSChristoph Lameter 219b44129b3SMel Gorman void set_pgdat_percpu_threshold(pg_data_t *pgdat, 220b44129b3SMel Gorman int (*calculate_pressure)(struct zone *)) 22188f5acf8SMel Gorman { 22288f5acf8SMel Gorman struct zone *zone; 22388f5acf8SMel Gorman int cpu; 22488f5acf8SMel Gorman int threshold; 22588f5acf8SMel Gorman int i; 22688f5acf8SMel Gorman 22788f5acf8SMel Gorman for (i = 0; i < pgdat->nr_zones; i++) { 22888f5acf8SMel Gorman zone = &pgdat->node_zones[i]; 22988f5acf8SMel Gorman if (!zone->percpu_drift_mark) 23088f5acf8SMel Gorman continue; 23188f5acf8SMel Gorman 232b44129b3SMel Gorman threshold = (*calculate_pressure)(zone); 2331d90ca89SKemi Wang for_each_online_cpu(cpu) 23488f5acf8SMel Gorman per_cpu_ptr(zone->pageset, cpu)->stat_threshold 23588f5acf8SMel Gorman = threshold; 23688f5acf8SMel Gorman } 23788f5acf8SMel Gorman } 23888f5acf8SMel Gorman 2392244b95aSChristoph Lameter /* 240bea04b07SJianyu Zhan * For use when we know that interrupts are disabled, 241bea04b07SJianyu Zhan * or when we know that preemption is disabled and that 242bea04b07SJianyu Zhan * particular counter cannot be updated from interrupt context. 2432244b95aSChristoph Lameter */ 2442244b95aSChristoph Lameter void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 2456cdb18adSHeiko Carstens long delta) 2462244b95aSChristoph Lameter { 24712938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 24812938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 2492244b95aSChristoph Lameter long x; 25012938a92SChristoph Lameter long t; 2512244b95aSChristoph Lameter 25212938a92SChristoph Lameter x = delta + __this_cpu_read(*p); 2532244b95aSChristoph Lameter 25412938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 25512938a92SChristoph Lameter 25612938a92SChristoph Lameter if (unlikely(x > t || x < -t)) { 2572244b95aSChristoph Lameter zone_page_state_add(x, zone, item); 2582244b95aSChristoph Lameter x = 0; 2592244b95aSChristoph Lameter } 26012938a92SChristoph Lameter __this_cpu_write(*p, x); 2612244b95aSChristoph Lameter } 2622244b95aSChristoph Lameter EXPORT_SYMBOL(__mod_zone_page_state); 2632244b95aSChristoph Lameter 26475ef7184SMel Gorman void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 26575ef7184SMel Gorman long delta) 26675ef7184SMel Gorman { 26775ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 26875ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 26975ef7184SMel Gorman long x; 27075ef7184SMel Gorman long t; 27175ef7184SMel Gorman 27275ef7184SMel Gorman x = delta + __this_cpu_read(*p); 27375ef7184SMel Gorman 27475ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 27575ef7184SMel Gorman 27675ef7184SMel Gorman if (unlikely(x > t || x < -t)) { 27775ef7184SMel Gorman node_page_state_add(x, pgdat, item); 27875ef7184SMel Gorman x = 0; 27975ef7184SMel Gorman } 28075ef7184SMel Gorman __this_cpu_write(*p, x); 28175ef7184SMel Gorman } 28275ef7184SMel Gorman EXPORT_SYMBOL(__mod_node_page_state); 28375ef7184SMel Gorman 2842244b95aSChristoph Lameter /* 2852244b95aSChristoph Lameter * Optimized increment and decrement functions. 2862244b95aSChristoph Lameter * 2872244b95aSChristoph Lameter * These are only for a single page and therefore can take a struct page * 2882244b95aSChristoph Lameter * argument instead of struct zone *. This allows the inclusion of the code 2892244b95aSChristoph Lameter * generated for page_zone(page) into the optimized functions. 2902244b95aSChristoph Lameter * 2912244b95aSChristoph Lameter * No overflow check is necessary and therefore the differential can be 2922244b95aSChristoph Lameter * incremented or decremented in place which may allow the compilers to 2932244b95aSChristoph Lameter * generate better code. 2942244b95aSChristoph Lameter * The increment or decrement is known and therefore one boundary check can 2952244b95aSChristoph Lameter * be omitted. 2962244b95aSChristoph Lameter * 297df9ecabaSChristoph Lameter * NOTE: These functions are very performance sensitive. Change only 298df9ecabaSChristoph Lameter * with care. 299df9ecabaSChristoph Lameter * 3002244b95aSChristoph Lameter * Some processors have inc/dec instructions that are atomic vs an interrupt. 3012244b95aSChristoph Lameter * However, the code must first determine the differential location in a zone 3022244b95aSChristoph Lameter * based on the processor number and then inc/dec the counter. There is no 3032244b95aSChristoph Lameter * guarantee without disabling preemption that the processor will not change 3042244b95aSChristoph Lameter * in between and therefore the atomicity vs. interrupt cannot be exploited 3052244b95aSChristoph Lameter * in a useful way here. 3062244b95aSChristoph Lameter */ 307c8785385SChristoph Lameter void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 3082244b95aSChristoph Lameter { 30912938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 31012938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 31112938a92SChristoph Lameter s8 v, t; 3122244b95aSChristoph Lameter 313908ee0f1SChristoph Lameter v = __this_cpu_inc_return(*p); 31412938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 31512938a92SChristoph Lameter if (unlikely(v > t)) { 31612938a92SChristoph Lameter s8 overstep = t >> 1; 3172244b95aSChristoph Lameter 31812938a92SChristoph Lameter zone_page_state_add(v + overstep, zone, item); 31912938a92SChristoph Lameter __this_cpu_write(*p, -overstep); 3202244b95aSChristoph Lameter } 3212244b95aSChristoph Lameter } 322ca889e6cSChristoph Lameter 32375ef7184SMel Gorman void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 32475ef7184SMel Gorman { 32575ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 32675ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 32775ef7184SMel Gorman s8 v, t; 32875ef7184SMel Gorman 32975ef7184SMel Gorman v = __this_cpu_inc_return(*p); 33075ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 33175ef7184SMel Gorman if (unlikely(v > t)) { 33275ef7184SMel Gorman s8 overstep = t >> 1; 33375ef7184SMel Gorman 33475ef7184SMel Gorman node_page_state_add(v + overstep, pgdat, item); 33575ef7184SMel Gorman __this_cpu_write(*p, -overstep); 33675ef7184SMel Gorman } 33775ef7184SMel Gorman } 33875ef7184SMel Gorman 339ca889e6cSChristoph Lameter void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 340ca889e6cSChristoph Lameter { 341ca889e6cSChristoph Lameter __inc_zone_state(page_zone(page), item); 342ca889e6cSChristoph Lameter } 3432244b95aSChristoph Lameter EXPORT_SYMBOL(__inc_zone_page_state); 3442244b95aSChristoph Lameter 34575ef7184SMel Gorman void __inc_node_page_state(struct page *page, enum node_stat_item item) 34675ef7184SMel Gorman { 34775ef7184SMel Gorman __inc_node_state(page_pgdat(page), item); 34875ef7184SMel Gorman } 34975ef7184SMel Gorman EXPORT_SYMBOL(__inc_node_page_state); 35075ef7184SMel Gorman 351c8785385SChristoph Lameter void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 3522244b95aSChristoph Lameter { 35312938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 35412938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 35512938a92SChristoph Lameter s8 v, t; 3562244b95aSChristoph Lameter 357908ee0f1SChristoph Lameter v = __this_cpu_dec_return(*p); 35812938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 35912938a92SChristoph Lameter if (unlikely(v < - t)) { 36012938a92SChristoph Lameter s8 overstep = t >> 1; 3612244b95aSChristoph Lameter 36212938a92SChristoph Lameter zone_page_state_add(v - overstep, zone, item); 36312938a92SChristoph Lameter __this_cpu_write(*p, overstep); 3642244b95aSChristoph Lameter } 3652244b95aSChristoph Lameter } 366c8785385SChristoph Lameter 36775ef7184SMel Gorman void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 36875ef7184SMel Gorman { 36975ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 37075ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 37175ef7184SMel Gorman s8 v, t; 37275ef7184SMel Gorman 37375ef7184SMel Gorman v = __this_cpu_dec_return(*p); 37475ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 37575ef7184SMel Gorman if (unlikely(v < - t)) { 37675ef7184SMel Gorman s8 overstep = t >> 1; 37775ef7184SMel Gorman 37875ef7184SMel Gorman node_page_state_add(v - overstep, pgdat, item); 37975ef7184SMel Gorman __this_cpu_write(*p, overstep); 38075ef7184SMel Gorman } 38175ef7184SMel Gorman } 38275ef7184SMel Gorman 383c8785385SChristoph Lameter void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 384c8785385SChristoph Lameter { 385c8785385SChristoph Lameter __dec_zone_state(page_zone(page), item); 386c8785385SChristoph Lameter } 3872244b95aSChristoph Lameter EXPORT_SYMBOL(__dec_zone_page_state); 3882244b95aSChristoph Lameter 38975ef7184SMel Gorman void __dec_node_page_state(struct page *page, enum node_stat_item item) 39075ef7184SMel Gorman { 39175ef7184SMel Gorman __dec_node_state(page_pgdat(page), item); 39275ef7184SMel Gorman } 39375ef7184SMel Gorman EXPORT_SYMBOL(__dec_node_page_state); 39475ef7184SMel Gorman 3954156153cSHeiko Carstens #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 3967c839120SChristoph Lameter /* 3977c839120SChristoph Lameter * If we have cmpxchg_local support then we do not need to incur the overhead 3987c839120SChristoph Lameter * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 3997c839120SChristoph Lameter * 4007c839120SChristoph Lameter * mod_state() modifies the zone counter state through atomic per cpu 4017c839120SChristoph Lameter * operations. 4027c839120SChristoph Lameter * 4037c839120SChristoph Lameter * Overstep mode specifies how overstep should handled: 4047c839120SChristoph Lameter * 0 No overstepping 4057c839120SChristoph Lameter * 1 Overstepping half of threshold 4067c839120SChristoph Lameter * -1 Overstepping minus half of threshold 4077c839120SChristoph Lameter */ 40875ef7184SMel Gorman static inline void mod_zone_state(struct zone *zone, 40975ef7184SMel Gorman enum zone_stat_item item, long delta, int overstep_mode) 4107c839120SChristoph Lameter { 4117c839120SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 4127c839120SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 4137c839120SChristoph Lameter long o, n, t, z; 4147c839120SChristoph Lameter 4157c839120SChristoph Lameter do { 4167c839120SChristoph Lameter z = 0; /* overflow to zone counters */ 4177c839120SChristoph Lameter 4187c839120SChristoph Lameter /* 4197c839120SChristoph Lameter * The fetching of the stat_threshold is racy. We may apply 4207c839120SChristoph Lameter * a counter threshold to the wrong the cpu if we get 421d3bc2367SChristoph Lameter * rescheduled while executing here. However, the next 422d3bc2367SChristoph Lameter * counter update will apply the threshold again and 423d3bc2367SChristoph Lameter * therefore bring the counter under the threshold again. 424d3bc2367SChristoph Lameter * 425d3bc2367SChristoph Lameter * Most of the time the thresholds are the same anyways 426d3bc2367SChristoph Lameter * for all cpus in a zone. 4277c839120SChristoph Lameter */ 4287c839120SChristoph Lameter t = this_cpu_read(pcp->stat_threshold); 4297c839120SChristoph Lameter 4307c839120SChristoph Lameter o = this_cpu_read(*p); 4317c839120SChristoph Lameter n = delta + o; 4327c839120SChristoph Lameter 4337c839120SChristoph Lameter if (n > t || n < -t) { 4347c839120SChristoph Lameter int os = overstep_mode * (t >> 1) ; 4357c839120SChristoph Lameter 4367c839120SChristoph Lameter /* Overflow must be added to zone counters */ 4377c839120SChristoph Lameter z = n + os; 4387c839120SChristoph Lameter n = -os; 4397c839120SChristoph Lameter } 4407c839120SChristoph Lameter } while (this_cpu_cmpxchg(*p, o, n) != o); 4417c839120SChristoph Lameter 4427c839120SChristoph Lameter if (z) 4437c839120SChristoph Lameter zone_page_state_add(z, zone, item); 4447c839120SChristoph Lameter } 4457c839120SChristoph Lameter 4467c839120SChristoph Lameter void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 4476cdb18adSHeiko Carstens long delta) 4487c839120SChristoph Lameter { 44975ef7184SMel Gorman mod_zone_state(zone, item, delta, 0); 4507c839120SChristoph Lameter } 4517c839120SChristoph Lameter EXPORT_SYMBOL(mod_zone_page_state); 4527c839120SChristoph Lameter 4537c839120SChristoph Lameter void inc_zone_page_state(struct page *page, enum zone_stat_item item) 4547c839120SChristoph Lameter { 45575ef7184SMel Gorman mod_zone_state(page_zone(page), item, 1, 1); 4567c839120SChristoph Lameter } 4577c839120SChristoph Lameter EXPORT_SYMBOL(inc_zone_page_state); 4587c839120SChristoph Lameter 4597c839120SChristoph Lameter void dec_zone_page_state(struct page *page, enum zone_stat_item item) 4607c839120SChristoph Lameter { 46175ef7184SMel Gorman mod_zone_state(page_zone(page), item, -1, -1); 4627c839120SChristoph Lameter } 4637c839120SChristoph Lameter EXPORT_SYMBOL(dec_zone_page_state); 46475ef7184SMel Gorman 46575ef7184SMel Gorman static inline void mod_node_state(struct pglist_data *pgdat, 46675ef7184SMel Gorman enum node_stat_item item, int delta, int overstep_mode) 46775ef7184SMel Gorman { 46875ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 46975ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 47075ef7184SMel Gorman long o, n, t, z; 47175ef7184SMel Gorman 47275ef7184SMel Gorman do { 47375ef7184SMel Gorman z = 0; /* overflow to node counters */ 47475ef7184SMel Gorman 47575ef7184SMel Gorman /* 47675ef7184SMel Gorman * The fetching of the stat_threshold is racy. We may apply 47775ef7184SMel Gorman * a counter threshold to the wrong the cpu if we get 47875ef7184SMel Gorman * rescheduled while executing here. However, the next 47975ef7184SMel Gorman * counter update will apply the threshold again and 48075ef7184SMel Gorman * therefore bring the counter under the threshold again. 48175ef7184SMel Gorman * 48275ef7184SMel Gorman * Most of the time the thresholds are the same anyways 48375ef7184SMel Gorman * for all cpus in a node. 48475ef7184SMel Gorman */ 48575ef7184SMel Gorman t = this_cpu_read(pcp->stat_threshold); 48675ef7184SMel Gorman 48775ef7184SMel Gorman o = this_cpu_read(*p); 48875ef7184SMel Gorman n = delta + o; 48975ef7184SMel Gorman 49075ef7184SMel Gorman if (n > t || n < -t) { 49175ef7184SMel Gorman int os = overstep_mode * (t >> 1) ; 49275ef7184SMel Gorman 49375ef7184SMel Gorman /* Overflow must be added to node counters */ 49475ef7184SMel Gorman z = n + os; 49575ef7184SMel Gorman n = -os; 49675ef7184SMel Gorman } 49775ef7184SMel Gorman } while (this_cpu_cmpxchg(*p, o, n) != o); 49875ef7184SMel Gorman 49975ef7184SMel Gorman if (z) 50075ef7184SMel Gorman node_page_state_add(z, pgdat, item); 50175ef7184SMel Gorman } 50275ef7184SMel Gorman 50375ef7184SMel Gorman void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 50475ef7184SMel Gorman long delta) 50575ef7184SMel Gorman { 50675ef7184SMel Gorman mod_node_state(pgdat, item, delta, 0); 50775ef7184SMel Gorman } 50875ef7184SMel Gorman EXPORT_SYMBOL(mod_node_page_state); 50975ef7184SMel Gorman 51075ef7184SMel Gorman void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 51175ef7184SMel Gorman { 51275ef7184SMel Gorman mod_node_state(pgdat, item, 1, 1); 51375ef7184SMel Gorman } 51475ef7184SMel Gorman 51575ef7184SMel Gorman void inc_node_page_state(struct page *page, enum node_stat_item item) 51675ef7184SMel Gorman { 51775ef7184SMel Gorman mod_node_state(page_pgdat(page), item, 1, 1); 51875ef7184SMel Gorman } 51975ef7184SMel Gorman EXPORT_SYMBOL(inc_node_page_state); 52075ef7184SMel Gorman 52175ef7184SMel Gorman void dec_node_page_state(struct page *page, enum node_stat_item item) 52275ef7184SMel Gorman { 52375ef7184SMel Gorman mod_node_state(page_pgdat(page), item, -1, -1); 52475ef7184SMel Gorman } 52575ef7184SMel Gorman EXPORT_SYMBOL(dec_node_page_state); 5267c839120SChristoph Lameter #else 5277c839120SChristoph Lameter /* 5287c839120SChristoph Lameter * Use interrupt disable to serialize counter updates 5297c839120SChristoph Lameter */ 5307c839120SChristoph Lameter void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 5316cdb18adSHeiko Carstens long delta) 5327c839120SChristoph Lameter { 5337c839120SChristoph Lameter unsigned long flags; 5347c839120SChristoph Lameter 5357c839120SChristoph Lameter local_irq_save(flags); 5367c839120SChristoph Lameter __mod_zone_page_state(zone, item, delta); 5377c839120SChristoph Lameter local_irq_restore(flags); 5387c839120SChristoph Lameter } 5397c839120SChristoph Lameter EXPORT_SYMBOL(mod_zone_page_state); 5407c839120SChristoph Lameter 5412244b95aSChristoph Lameter void inc_zone_page_state(struct page *page, enum zone_stat_item item) 5422244b95aSChristoph Lameter { 5432244b95aSChristoph Lameter unsigned long flags; 5442244b95aSChristoph Lameter struct zone *zone; 5452244b95aSChristoph Lameter 5462244b95aSChristoph Lameter zone = page_zone(page); 5472244b95aSChristoph Lameter local_irq_save(flags); 548ca889e6cSChristoph Lameter __inc_zone_state(zone, item); 5492244b95aSChristoph Lameter local_irq_restore(flags); 5502244b95aSChristoph Lameter } 5512244b95aSChristoph Lameter EXPORT_SYMBOL(inc_zone_page_state); 5522244b95aSChristoph Lameter 5532244b95aSChristoph Lameter void dec_zone_page_state(struct page *page, enum zone_stat_item item) 5542244b95aSChristoph Lameter { 5552244b95aSChristoph Lameter unsigned long flags; 5562244b95aSChristoph Lameter 5572244b95aSChristoph Lameter local_irq_save(flags); 558a302eb4eSChristoph Lameter __dec_zone_page_state(page, item); 5592244b95aSChristoph Lameter local_irq_restore(flags); 5602244b95aSChristoph Lameter } 5612244b95aSChristoph Lameter EXPORT_SYMBOL(dec_zone_page_state); 5622244b95aSChristoph Lameter 56375ef7184SMel Gorman void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 56475ef7184SMel Gorman { 56575ef7184SMel Gorman unsigned long flags; 56675ef7184SMel Gorman 56775ef7184SMel Gorman local_irq_save(flags); 56875ef7184SMel Gorman __inc_node_state(pgdat, item); 56975ef7184SMel Gorman local_irq_restore(flags); 57075ef7184SMel Gorman } 57175ef7184SMel Gorman EXPORT_SYMBOL(inc_node_state); 57275ef7184SMel Gorman 57375ef7184SMel Gorman void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 57475ef7184SMel Gorman long delta) 57575ef7184SMel Gorman { 57675ef7184SMel Gorman unsigned long flags; 57775ef7184SMel Gorman 57875ef7184SMel Gorman local_irq_save(flags); 57975ef7184SMel Gorman __mod_node_page_state(pgdat, item, delta); 58075ef7184SMel Gorman local_irq_restore(flags); 58175ef7184SMel Gorman } 58275ef7184SMel Gorman EXPORT_SYMBOL(mod_node_page_state); 58375ef7184SMel Gorman 58475ef7184SMel Gorman void inc_node_page_state(struct page *page, enum node_stat_item item) 58575ef7184SMel Gorman { 58675ef7184SMel Gorman unsigned long flags; 58775ef7184SMel Gorman struct pglist_data *pgdat; 58875ef7184SMel Gorman 58975ef7184SMel Gorman pgdat = page_pgdat(page); 59075ef7184SMel Gorman local_irq_save(flags); 59175ef7184SMel Gorman __inc_node_state(pgdat, item); 59275ef7184SMel Gorman local_irq_restore(flags); 59375ef7184SMel Gorman } 59475ef7184SMel Gorman EXPORT_SYMBOL(inc_node_page_state); 59575ef7184SMel Gorman 59675ef7184SMel Gorman void dec_node_page_state(struct page *page, enum node_stat_item item) 59775ef7184SMel Gorman { 59875ef7184SMel Gorman unsigned long flags; 59975ef7184SMel Gorman 60075ef7184SMel Gorman local_irq_save(flags); 60175ef7184SMel Gorman __dec_node_page_state(page, item); 60275ef7184SMel Gorman local_irq_restore(flags); 60375ef7184SMel Gorman } 60475ef7184SMel Gorman EXPORT_SYMBOL(dec_node_page_state); 60575ef7184SMel Gorman #endif 6067cc36bbdSChristoph Lameter 6077cc36bbdSChristoph Lameter /* 6087cc36bbdSChristoph Lameter * Fold a differential into the global counters. 6097cc36bbdSChristoph Lameter * Returns the number of counters updated. 6107cc36bbdSChristoph Lameter */ 6113a321d2aSKemi Wang #ifdef CONFIG_NUMA 6123a321d2aSKemi Wang static int fold_diff(int *zone_diff, int *numa_diff, int *node_diff) 6133a321d2aSKemi Wang { 6143a321d2aSKemi Wang int i; 6153a321d2aSKemi Wang int changes = 0; 6163a321d2aSKemi Wang 6173a321d2aSKemi Wang for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 6183a321d2aSKemi Wang if (zone_diff[i]) { 6193a321d2aSKemi Wang atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 6203a321d2aSKemi Wang changes++; 6213a321d2aSKemi Wang } 6223a321d2aSKemi Wang 6233a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) 6243a321d2aSKemi Wang if (numa_diff[i]) { 6253a321d2aSKemi Wang atomic_long_add(numa_diff[i], &vm_numa_stat[i]); 6263a321d2aSKemi Wang changes++; 6273a321d2aSKemi Wang } 6283a321d2aSKemi Wang 6293a321d2aSKemi Wang for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 6303a321d2aSKemi Wang if (node_diff[i]) { 6313a321d2aSKemi Wang atomic_long_add(node_diff[i], &vm_node_stat[i]); 6323a321d2aSKemi Wang changes++; 6333a321d2aSKemi Wang } 6343a321d2aSKemi Wang return changes; 6353a321d2aSKemi Wang } 6363a321d2aSKemi Wang #else 63775ef7184SMel Gorman static int fold_diff(int *zone_diff, int *node_diff) 6384edb0748SChristoph Lameter { 6394edb0748SChristoph Lameter int i; 6407cc36bbdSChristoph Lameter int changes = 0; 6414edb0748SChristoph Lameter 6424edb0748SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 64375ef7184SMel Gorman if (zone_diff[i]) { 64475ef7184SMel Gorman atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 64575ef7184SMel Gorman changes++; 64675ef7184SMel Gorman } 64775ef7184SMel Gorman 64875ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 64975ef7184SMel Gorman if (node_diff[i]) { 65075ef7184SMel Gorman atomic_long_add(node_diff[i], &vm_node_stat[i]); 6517cc36bbdSChristoph Lameter changes++; 6527cc36bbdSChristoph Lameter } 6537cc36bbdSChristoph Lameter return changes; 6544edb0748SChristoph Lameter } 6553a321d2aSKemi Wang #endif /* CONFIG_NUMA */ 6564edb0748SChristoph Lameter 6572244b95aSChristoph Lameter /* 6582bb921e5SChristoph Lameter * Update the zone counters for the current cpu. 659a7f75e25SChristoph Lameter * 6604037d452SChristoph Lameter * Note that refresh_cpu_vm_stats strives to only access 6614037d452SChristoph Lameter * node local memory. The per cpu pagesets on remote zones are placed 6624037d452SChristoph Lameter * in the memory local to the processor using that pageset. So the 6634037d452SChristoph Lameter * loop over all zones will access a series of cachelines local to 6644037d452SChristoph Lameter * the processor. 6654037d452SChristoph Lameter * 6664037d452SChristoph Lameter * The call to zone_page_state_add updates the cachelines with the 6674037d452SChristoph Lameter * statistics in the remote zone struct as well as the global cachelines 6684037d452SChristoph Lameter * with the global counters. These could cause remote node cache line 6694037d452SChristoph Lameter * bouncing and will have to be only done when necessary. 6707cc36bbdSChristoph Lameter * 6717cc36bbdSChristoph Lameter * The function returns the number of global counters updated. 6722244b95aSChristoph Lameter */ 6730eb77e98SChristoph Lameter static int refresh_cpu_vm_stats(bool do_pagesets) 6742244b95aSChristoph Lameter { 67575ef7184SMel Gorman struct pglist_data *pgdat; 6762244b95aSChristoph Lameter struct zone *zone; 6772244b95aSChristoph Lameter int i; 67875ef7184SMel Gorman int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 6793a321d2aSKemi Wang #ifdef CONFIG_NUMA 6803a321d2aSKemi Wang int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, }; 6813a321d2aSKemi Wang #endif 68275ef7184SMel Gorman int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 6837cc36bbdSChristoph Lameter int changes = 0; 6842244b95aSChristoph Lameter 685ee99c71cSKOSAKI Motohiro for_each_populated_zone(zone) { 686fbc2edb0SChristoph Lameter struct per_cpu_pageset __percpu *p = zone->pageset; 6872244b95aSChristoph Lameter 688fbc2edb0SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 689a7f75e25SChristoph Lameter int v; 690a7f75e25SChristoph Lameter 691fbc2edb0SChristoph Lameter v = this_cpu_xchg(p->vm_stat_diff[i], 0); 692fbc2edb0SChristoph Lameter if (v) { 693fbc2edb0SChristoph Lameter 694a7f75e25SChristoph Lameter atomic_long_add(v, &zone->vm_stat[i]); 69575ef7184SMel Gorman global_zone_diff[i] += v; 6964037d452SChristoph Lameter #ifdef CONFIG_NUMA 6974037d452SChristoph Lameter /* 3 seconds idle till flush */ 698fbc2edb0SChristoph Lameter __this_cpu_write(p->expire, 3); 6994037d452SChristoph Lameter #endif 7002244b95aSChristoph Lameter } 701fbc2edb0SChristoph Lameter } 7024037d452SChristoph Lameter #ifdef CONFIG_NUMA 7033a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) { 7043a321d2aSKemi Wang int v; 7053a321d2aSKemi Wang 7063a321d2aSKemi Wang v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0); 7073a321d2aSKemi Wang if (v) { 7083a321d2aSKemi Wang 7093a321d2aSKemi Wang atomic_long_add(v, &zone->vm_numa_stat[i]); 7103a321d2aSKemi Wang global_numa_diff[i] += v; 7113a321d2aSKemi Wang __this_cpu_write(p->expire, 3); 7123a321d2aSKemi Wang } 7133a321d2aSKemi Wang } 7143a321d2aSKemi Wang 7150eb77e98SChristoph Lameter if (do_pagesets) { 7160eb77e98SChristoph Lameter cond_resched(); 7174037d452SChristoph Lameter /* 7184037d452SChristoph Lameter * Deal with draining the remote pageset of this 7194037d452SChristoph Lameter * processor 7204037d452SChristoph Lameter * 7214037d452SChristoph Lameter * Check if there are pages remaining in this pageset 7224037d452SChristoph Lameter * if not then there is nothing to expire. 7234037d452SChristoph Lameter */ 724fbc2edb0SChristoph Lameter if (!__this_cpu_read(p->expire) || 725fbc2edb0SChristoph Lameter !__this_cpu_read(p->pcp.count)) 7264037d452SChristoph Lameter continue; 7274037d452SChristoph Lameter 7284037d452SChristoph Lameter /* 7294037d452SChristoph Lameter * We never drain zones local to this processor. 7304037d452SChristoph Lameter */ 7314037d452SChristoph Lameter if (zone_to_nid(zone) == numa_node_id()) { 732fbc2edb0SChristoph Lameter __this_cpu_write(p->expire, 0); 7334037d452SChristoph Lameter continue; 7344037d452SChristoph Lameter } 7354037d452SChristoph Lameter 736fbc2edb0SChristoph Lameter if (__this_cpu_dec_return(p->expire)) 7374037d452SChristoph Lameter continue; 7384037d452SChristoph Lameter 7397cc36bbdSChristoph Lameter if (__this_cpu_read(p->pcp.count)) { 7407c8e0181SChristoph Lameter drain_zone_pages(zone, this_cpu_ptr(&p->pcp)); 7417cc36bbdSChristoph Lameter changes++; 7427cc36bbdSChristoph Lameter } 7430eb77e98SChristoph Lameter } 7444037d452SChristoph Lameter #endif 7452244b95aSChristoph Lameter } 74675ef7184SMel Gorman 74775ef7184SMel Gorman for_each_online_pgdat(pgdat) { 74875ef7184SMel Gorman struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 74975ef7184SMel Gorman 75075ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 75175ef7184SMel Gorman int v; 75275ef7184SMel Gorman 75375ef7184SMel Gorman v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 75475ef7184SMel Gorman if (v) { 75575ef7184SMel Gorman atomic_long_add(v, &pgdat->vm_stat[i]); 75675ef7184SMel Gorman global_node_diff[i] += v; 75775ef7184SMel Gorman } 75875ef7184SMel Gorman } 75975ef7184SMel Gorman } 76075ef7184SMel Gorman 7613a321d2aSKemi Wang #ifdef CONFIG_NUMA 7623a321d2aSKemi Wang changes += fold_diff(global_zone_diff, global_numa_diff, 7633a321d2aSKemi Wang global_node_diff); 7643a321d2aSKemi Wang #else 76575ef7184SMel Gorman changes += fold_diff(global_zone_diff, global_node_diff); 7663a321d2aSKemi Wang #endif 7677cc36bbdSChristoph Lameter return changes; 7682244b95aSChristoph Lameter } 7692244b95aSChristoph Lameter 77040f4b1eaSCody P Schafer /* 7712bb921e5SChristoph Lameter * Fold the data for an offline cpu into the global array. 7722bb921e5SChristoph Lameter * There cannot be any access by the offline cpu and therefore 7732bb921e5SChristoph Lameter * synchronization is simplified. 7742bb921e5SChristoph Lameter */ 7752bb921e5SChristoph Lameter void cpu_vm_stats_fold(int cpu) 7762bb921e5SChristoph Lameter { 77775ef7184SMel Gorman struct pglist_data *pgdat; 7782bb921e5SChristoph Lameter struct zone *zone; 7792bb921e5SChristoph Lameter int i; 78075ef7184SMel Gorman int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 7813a321d2aSKemi Wang #ifdef CONFIG_NUMA 7823a321d2aSKemi Wang int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, }; 7833a321d2aSKemi Wang #endif 78475ef7184SMel Gorman int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 7852bb921e5SChristoph Lameter 7862bb921e5SChristoph Lameter for_each_populated_zone(zone) { 7872bb921e5SChristoph Lameter struct per_cpu_pageset *p; 7882bb921e5SChristoph Lameter 7892bb921e5SChristoph Lameter p = per_cpu_ptr(zone->pageset, cpu); 7902bb921e5SChristoph Lameter 7912bb921e5SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 7922bb921e5SChristoph Lameter if (p->vm_stat_diff[i]) { 7932bb921e5SChristoph Lameter int v; 7942bb921e5SChristoph Lameter 7952bb921e5SChristoph Lameter v = p->vm_stat_diff[i]; 7962bb921e5SChristoph Lameter p->vm_stat_diff[i] = 0; 7972bb921e5SChristoph Lameter atomic_long_add(v, &zone->vm_stat[i]); 79875ef7184SMel Gorman global_zone_diff[i] += v; 7992bb921e5SChristoph Lameter } 8003a321d2aSKemi Wang 8013a321d2aSKemi Wang #ifdef CONFIG_NUMA 8023a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) 8033a321d2aSKemi Wang if (p->vm_numa_stat_diff[i]) { 8043a321d2aSKemi Wang int v; 8053a321d2aSKemi Wang 8063a321d2aSKemi Wang v = p->vm_numa_stat_diff[i]; 8073a321d2aSKemi Wang p->vm_numa_stat_diff[i] = 0; 8083a321d2aSKemi Wang atomic_long_add(v, &zone->vm_numa_stat[i]); 8093a321d2aSKemi Wang global_numa_diff[i] += v; 8103a321d2aSKemi Wang } 8113a321d2aSKemi Wang #endif 8122bb921e5SChristoph Lameter } 8132bb921e5SChristoph Lameter 81475ef7184SMel Gorman for_each_online_pgdat(pgdat) { 81575ef7184SMel Gorman struct per_cpu_nodestat *p; 81675ef7184SMel Gorman 81775ef7184SMel Gorman p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 81875ef7184SMel Gorman 81975ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 82075ef7184SMel Gorman if (p->vm_node_stat_diff[i]) { 82175ef7184SMel Gorman int v; 82275ef7184SMel Gorman 82375ef7184SMel Gorman v = p->vm_node_stat_diff[i]; 82475ef7184SMel Gorman p->vm_node_stat_diff[i] = 0; 82575ef7184SMel Gorman atomic_long_add(v, &pgdat->vm_stat[i]); 82675ef7184SMel Gorman global_node_diff[i] += v; 82775ef7184SMel Gorman } 82875ef7184SMel Gorman } 82975ef7184SMel Gorman 8303a321d2aSKemi Wang #ifdef CONFIG_NUMA 8313a321d2aSKemi Wang fold_diff(global_zone_diff, global_numa_diff, global_node_diff); 8323a321d2aSKemi Wang #else 83375ef7184SMel Gorman fold_diff(global_zone_diff, global_node_diff); 8343a321d2aSKemi Wang #endif 8352bb921e5SChristoph Lameter } 8362bb921e5SChristoph Lameter 8372bb921e5SChristoph Lameter /* 83840f4b1eaSCody P Schafer * this is only called if !populated_zone(zone), which implies no other users of 83940f4b1eaSCody P Schafer * pset->vm_stat_diff[] exsist. 84040f4b1eaSCody P Schafer */ 8415a883813SMinchan Kim void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset) 8425a883813SMinchan Kim { 8435a883813SMinchan Kim int i; 8445a883813SMinchan Kim 8455a883813SMinchan Kim for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 8465a883813SMinchan Kim if (pset->vm_stat_diff[i]) { 8475a883813SMinchan Kim int v = pset->vm_stat_diff[i]; 8485a883813SMinchan Kim pset->vm_stat_diff[i] = 0; 8495a883813SMinchan Kim atomic_long_add(v, &zone->vm_stat[i]); 85075ef7184SMel Gorman atomic_long_add(v, &vm_zone_stat[i]); 8515a883813SMinchan Kim } 8523a321d2aSKemi Wang 8533a321d2aSKemi Wang #ifdef CONFIG_NUMA 8543a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) 8553a321d2aSKemi Wang if (pset->vm_numa_stat_diff[i]) { 8563a321d2aSKemi Wang int v = pset->vm_numa_stat_diff[i]; 8573a321d2aSKemi Wang 8583a321d2aSKemi Wang pset->vm_numa_stat_diff[i] = 0; 8593a321d2aSKemi Wang atomic_long_add(v, &zone->vm_numa_stat[i]); 8603a321d2aSKemi Wang atomic_long_add(v, &vm_numa_stat[i]); 8613a321d2aSKemi Wang } 8623a321d2aSKemi Wang #endif 8635a883813SMinchan Kim } 8642244b95aSChristoph Lameter #endif 8652244b95aSChristoph Lameter 866ca889e6cSChristoph Lameter #ifdef CONFIG_NUMA 8673a321d2aSKemi Wang void __inc_numa_state(struct zone *zone, 8683a321d2aSKemi Wang enum numa_stat_item item) 8693a321d2aSKemi Wang { 8703a321d2aSKemi Wang struct per_cpu_pageset __percpu *pcp = zone->pageset; 8711d90ca89SKemi Wang u16 __percpu *p = pcp->vm_numa_stat_diff + item; 8721d90ca89SKemi Wang u16 v; 8733a321d2aSKemi Wang 8743a321d2aSKemi Wang v = __this_cpu_inc_return(*p); 8753a321d2aSKemi Wang 8761d90ca89SKemi Wang if (unlikely(v > NUMA_STATS_THRESHOLD)) { 8771d90ca89SKemi Wang zone_numa_state_add(v, zone, item); 8781d90ca89SKemi Wang __this_cpu_write(*p, 0); 8793a321d2aSKemi Wang } 8803a321d2aSKemi Wang } 8813a321d2aSKemi Wang 882ca889e6cSChristoph Lameter /* 88375ef7184SMel Gorman * Determine the per node value of a stat item. This function 88475ef7184SMel Gorman * is called frequently in a NUMA machine, so try to be as 88575ef7184SMel Gorman * frugal as possible. 886c2d42c16SAndrew Morton */ 88775ef7184SMel Gorman unsigned long sum_zone_node_page_state(int node, 88875ef7184SMel Gorman enum zone_stat_item item) 889c2d42c16SAndrew Morton { 890c2d42c16SAndrew Morton struct zone *zones = NODE_DATA(node)->node_zones; 891e87d59f7SJoonsoo Kim int i; 892e87d59f7SJoonsoo Kim unsigned long count = 0; 893c2d42c16SAndrew Morton 894e87d59f7SJoonsoo Kim for (i = 0; i < MAX_NR_ZONES; i++) 895e87d59f7SJoonsoo Kim count += zone_page_state(zones + i, item); 896e87d59f7SJoonsoo Kim 897e87d59f7SJoonsoo Kim return count; 898c2d42c16SAndrew Morton } 899c2d42c16SAndrew Morton 90063803222SKemi Wang /* 90163803222SKemi Wang * Determine the per node value of a numa stat item. To avoid deviation, 90263803222SKemi Wang * the per cpu stat number in vm_numa_stat_diff[] is also included. 90363803222SKemi Wang */ 9043a321d2aSKemi Wang unsigned long sum_zone_numa_state(int node, 9053a321d2aSKemi Wang enum numa_stat_item item) 9063a321d2aSKemi Wang { 9073a321d2aSKemi Wang struct zone *zones = NODE_DATA(node)->node_zones; 9083a321d2aSKemi Wang int i; 9093a321d2aSKemi Wang unsigned long count = 0; 9103a321d2aSKemi Wang 9113a321d2aSKemi Wang for (i = 0; i < MAX_NR_ZONES; i++) 91263803222SKemi Wang count += zone_numa_state_snapshot(zones + i, item); 9133a321d2aSKemi Wang 9143a321d2aSKemi Wang return count; 9153a321d2aSKemi Wang } 9163a321d2aSKemi Wang 91775ef7184SMel Gorman /* 91875ef7184SMel Gorman * Determine the per node value of a stat item. 91975ef7184SMel Gorman */ 92075ef7184SMel Gorman unsigned long node_page_state(struct pglist_data *pgdat, 92175ef7184SMel Gorman enum node_stat_item item) 92275ef7184SMel Gorman { 92375ef7184SMel Gorman long x = atomic_long_read(&pgdat->vm_stat[item]); 92475ef7184SMel Gorman #ifdef CONFIG_SMP 92575ef7184SMel Gorman if (x < 0) 92675ef7184SMel Gorman x = 0; 92775ef7184SMel Gorman #endif 92875ef7184SMel Gorman return x; 92975ef7184SMel Gorman } 930ca889e6cSChristoph Lameter #endif 931ca889e6cSChristoph Lameter 932d7a5752cSMel Gorman #ifdef CONFIG_COMPACTION 93336deb0beSNamhyung Kim 934d7a5752cSMel Gorman struct contig_page_info { 935d7a5752cSMel Gorman unsigned long free_pages; 936d7a5752cSMel Gorman unsigned long free_blocks_total; 937d7a5752cSMel Gorman unsigned long free_blocks_suitable; 938d7a5752cSMel Gorman }; 939d7a5752cSMel Gorman 940d7a5752cSMel Gorman /* 941d7a5752cSMel Gorman * Calculate the number of free pages in a zone, how many contiguous 942d7a5752cSMel Gorman * pages are free and how many are large enough to satisfy an allocation of 943d7a5752cSMel Gorman * the target size. Note that this function makes no attempt to estimate 944d7a5752cSMel Gorman * how many suitable free blocks there *might* be if MOVABLE pages were 945d7a5752cSMel Gorman * migrated. Calculating that is possible, but expensive and can be 946d7a5752cSMel Gorman * figured out from userspace 947d7a5752cSMel Gorman */ 948d7a5752cSMel Gorman static void fill_contig_page_info(struct zone *zone, 949d7a5752cSMel Gorman unsigned int suitable_order, 950d7a5752cSMel Gorman struct contig_page_info *info) 951d7a5752cSMel Gorman { 952d7a5752cSMel Gorman unsigned int order; 953d7a5752cSMel Gorman 954d7a5752cSMel Gorman info->free_pages = 0; 955d7a5752cSMel Gorman info->free_blocks_total = 0; 956d7a5752cSMel Gorman info->free_blocks_suitable = 0; 957d7a5752cSMel Gorman 958d7a5752cSMel Gorman for (order = 0; order < MAX_ORDER; order++) { 959d7a5752cSMel Gorman unsigned long blocks; 960d7a5752cSMel Gorman 961d7a5752cSMel Gorman /* Count number of free blocks */ 962d7a5752cSMel Gorman blocks = zone->free_area[order].nr_free; 963d7a5752cSMel Gorman info->free_blocks_total += blocks; 964d7a5752cSMel Gorman 965d7a5752cSMel Gorman /* Count free base pages */ 966d7a5752cSMel Gorman info->free_pages += blocks << order; 967d7a5752cSMel Gorman 968d7a5752cSMel Gorman /* Count the suitable free blocks */ 969d7a5752cSMel Gorman if (order >= suitable_order) 970d7a5752cSMel Gorman info->free_blocks_suitable += blocks << 971d7a5752cSMel Gorman (order - suitable_order); 972d7a5752cSMel Gorman } 973d7a5752cSMel Gorman } 974f1a5ab12SMel Gorman 975f1a5ab12SMel Gorman /* 976f1a5ab12SMel Gorman * A fragmentation index only makes sense if an allocation of a requested 977f1a5ab12SMel Gorman * size would fail. If that is true, the fragmentation index indicates 978f1a5ab12SMel Gorman * whether external fragmentation or a lack of memory was the problem. 979f1a5ab12SMel Gorman * The value can be used to determine if page reclaim or compaction 980f1a5ab12SMel Gorman * should be used 981f1a5ab12SMel Gorman */ 98256de7263SMel Gorman static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 983f1a5ab12SMel Gorman { 984f1a5ab12SMel Gorman unsigned long requested = 1UL << order; 985f1a5ab12SMel Gorman 98688d6ac40SWen Yang if (WARN_ON_ONCE(order >= MAX_ORDER)) 98788d6ac40SWen Yang return 0; 98888d6ac40SWen Yang 989f1a5ab12SMel Gorman if (!info->free_blocks_total) 990f1a5ab12SMel Gorman return 0; 991f1a5ab12SMel Gorman 992f1a5ab12SMel Gorman /* Fragmentation index only makes sense when a request would fail */ 993f1a5ab12SMel Gorman if (info->free_blocks_suitable) 994f1a5ab12SMel Gorman return -1000; 995f1a5ab12SMel Gorman 996f1a5ab12SMel Gorman /* 997f1a5ab12SMel Gorman * Index is between 0 and 1 so return within 3 decimal places 998f1a5ab12SMel Gorman * 999f1a5ab12SMel Gorman * 0 => allocation would fail due to lack of memory 1000f1a5ab12SMel Gorman * 1 => allocation would fail due to fragmentation 1001f1a5ab12SMel Gorman */ 1002f1a5ab12SMel Gorman return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1003f1a5ab12SMel Gorman } 100456de7263SMel Gorman 100556de7263SMel Gorman /* Same as __fragmentation index but allocs contig_page_info on stack */ 100656de7263SMel Gorman int fragmentation_index(struct zone *zone, unsigned int order) 100756de7263SMel Gorman { 100856de7263SMel Gorman struct contig_page_info info; 100956de7263SMel Gorman 101056de7263SMel Gorman fill_contig_page_info(zone, order, &info); 101156de7263SMel Gorman return __fragmentation_index(order, &info); 101256de7263SMel Gorman } 1013d7a5752cSMel Gorman #endif 1014d7a5752cSMel Gorman 10150d6617c7SDavid Rientjes #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA) 1016fa25c503SKOSAKI Motohiro #ifdef CONFIG_ZONE_DMA 1017fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA(xx) xx "_dma", 1018fa25c503SKOSAKI Motohiro #else 1019fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA(xx) 1020fa25c503SKOSAKI Motohiro #endif 1021fa25c503SKOSAKI Motohiro 1022fa25c503SKOSAKI Motohiro #ifdef CONFIG_ZONE_DMA32 1023fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA32(xx) xx "_dma32", 1024fa25c503SKOSAKI Motohiro #else 1025fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA32(xx) 1026fa25c503SKOSAKI Motohiro #endif 1027fa25c503SKOSAKI Motohiro 1028fa25c503SKOSAKI Motohiro #ifdef CONFIG_HIGHMEM 1029fa25c503SKOSAKI Motohiro #define TEXT_FOR_HIGHMEM(xx) xx "_high", 1030fa25c503SKOSAKI Motohiro #else 1031fa25c503SKOSAKI Motohiro #define TEXT_FOR_HIGHMEM(xx) 1032fa25c503SKOSAKI Motohiro #endif 1033fa25c503SKOSAKI Motohiro 1034fa25c503SKOSAKI Motohiro #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ 1035fa25c503SKOSAKI Motohiro TEXT_FOR_HIGHMEM(xx) xx "_movable", 1036fa25c503SKOSAKI Motohiro 1037fa25c503SKOSAKI Motohiro const char * const vmstat_text[] = { 103809316c09SKonstantin Khlebnikov /* enum zone_stat_item countes */ 1039fa25c503SKOSAKI Motohiro "nr_free_pages", 104071c799f4SMinchan Kim "nr_zone_inactive_anon", 104171c799f4SMinchan Kim "nr_zone_active_anon", 104271c799f4SMinchan Kim "nr_zone_inactive_file", 104371c799f4SMinchan Kim "nr_zone_active_file", 104471c799f4SMinchan Kim "nr_zone_unevictable", 10455a1c84b4SMel Gorman "nr_zone_write_pending", 1046fa25c503SKOSAKI Motohiro "nr_mlock", 1047fa25c503SKOSAKI Motohiro "nr_page_table_pages", 1048fa25c503SKOSAKI Motohiro "nr_kernel_stack", 1049fa25c503SKOSAKI Motohiro "nr_bounce", 105091537feeSMinchan Kim #if IS_ENABLED(CONFIG_ZSMALLOC) 105191537feeSMinchan Kim "nr_zspages", 105291537feeSMinchan Kim #endif 10533a321d2aSKemi Wang "nr_free_cma", 10543a321d2aSKemi Wang 10553a321d2aSKemi Wang /* enum numa_stat_item counters */ 1056fa25c503SKOSAKI Motohiro #ifdef CONFIG_NUMA 1057fa25c503SKOSAKI Motohiro "numa_hit", 1058fa25c503SKOSAKI Motohiro "numa_miss", 1059fa25c503SKOSAKI Motohiro "numa_foreign", 1060fa25c503SKOSAKI Motohiro "numa_interleave", 1061fa25c503SKOSAKI Motohiro "numa_local", 1062fa25c503SKOSAKI Motohiro "numa_other", 1063fa25c503SKOSAKI Motohiro #endif 106409316c09SKonstantin Khlebnikov 1065599d0c95SMel Gorman /* Node-based counters */ 1066599d0c95SMel Gorman "nr_inactive_anon", 1067599d0c95SMel Gorman "nr_active_anon", 1068599d0c95SMel Gorman "nr_inactive_file", 1069599d0c95SMel Gorman "nr_active_file", 1070599d0c95SMel Gorman "nr_unevictable", 1071385386cfSJohannes Weiner "nr_slab_reclaimable", 1072385386cfSJohannes Weiner "nr_slab_unreclaimable", 1073599d0c95SMel Gorman "nr_isolated_anon", 1074599d0c95SMel Gorman "nr_isolated_file", 10751e6b1085SMel Gorman "workingset_refault", 10761e6b1085SMel Gorman "workingset_activate", 10771e6b1085SMel Gorman "workingset_nodereclaim", 107850658e2eSMel Gorman "nr_anon_pages", 107950658e2eSMel Gorman "nr_mapped", 108011fb9989SMel Gorman "nr_file_pages", 108111fb9989SMel Gorman "nr_dirty", 108211fb9989SMel Gorman "nr_writeback", 108311fb9989SMel Gorman "nr_writeback_temp", 108411fb9989SMel Gorman "nr_shmem", 108511fb9989SMel Gorman "nr_shmem_hugepages", 108611fb9989SMel Gorman "nr_shmem_pmdmapped", 108711fb9989SMel Gorman "nr_anon_transparent_hugepages", 108811fb9989SMel Gorman "nr_unstable", 1089c4a25635SMel Gorman "nr_vmscan_write", 1090c4a25635SMel Gorman "nr_vmscan_immediate_reclaim", 1091c4a25635SMel Gorman "nr_dirtied", 1092c4a25635SMel Gorman "nr_written", 1093599d0c95SMel Gorman 109409316c09SKonstantin Khlebnikov /* enum writeback_stat_item counters */ 1095fa25c503SKOSAKI Motohiro "nr_dirty_threshold", 1096fa25c503SKOSAKI Motohiro "nr_dirty_background_threshold", 1097fa25c503SKOSAKI Motohiro 1098fa25c503SKOSAKI Motohiro #ifdef CONFIG_VM_EVENT_COUNTERS 109909316c09SKonstantin Khlebnikov /* enum vm_event_item counters */ 1100fa25c503SKOSAKI Motohiro "pgpgin", 1101fa25c503SKOSAKI Motohiro "pgpgout", 1102fa25c503SKOSAKI Motohiro "pswpin", 1103fa25c503SKOSAKI Motohiro "pswpout", 1104fa25c503SKOSAKI Motohiro 1105fa25c503SKOSAKI Motohiro TEXTS_FOR_ZONES("pgalloc") 11067cc30fcfSMel Gorman TEXTS_FOR_ZONES("allocstall") 11077cc30fcfSMel Gorman TEXTS_FOR_ZONES("pgskip") 1108fa25c503SKOSAKI Motohiro 1109fa25c503SKOSAKI Motohiro "pgfree", 1110fa25c503SKOSAKI Motohiro "pgactivate", 1111fa25c503SKOSAKI Motohiro "pgdeactivate", 1112f7ad2a6cSShaohua Li "pglazyfree", 1113fa25c503SKOSAKI Motohiro 1114fa25c503SKOSAKI Motohiro "pgfault", 1115fa25c503SKOSAKI Motohiro "pgmajfault", 1116854e9ed0SMinchan Kim "pglazyfreed", 1117fa25c503SKOSAKI Motohiro 1118599d0c95SMel Gorman "pgrefill", 1119599d0c95SMel Gorman "pgsteal_kswapd", 1120599d0c95SMel Gorman "pgsteal_direct", 1121599d0c95SMel Gorman "pgscan_kswapd", 1122599d0c95SMel Gorman "pgscan_direct", 112368243e76SMel Gorman "pgscan_direct_throttle", 1124fa25c503SKOSAKI Motohiro 1125fa25c503SKOSAKI Motohiro #ifdef CONFIG_NUMA 1126fa25c503SKOSAKI Motohiro "zone_reclaim_failed", 1127fa25c503SKOSAKI Motohiro #endif 1128fa25c503SKOSAKI Motohiro "pginodesteal", 1129fa25c503SKOSAKI Motohiro "slabs_scanned", 1130fa25c503SKOSAKI Motohiro "kswapd_inodesteal", 1131fa25c503SKOSAKI Motohiro "kswapd_low_wmark_hit_quickly", 1132fa25c503SKOSAKI Motohiro "kswapd_high_wmark_hit_quickly", 1133fa25c503SKOSAKI Motohiro "pageoutrun", 1134fa25c503SKOSAKI Motohiro 1135fa25c503SKOSAKI Motohiro "pgrotated", 1136fa25c503SKOSAKI Motohiro 11375509a5d2SDave Hansen "drop_pagecache", 11385509a5d2SDave Hansen "drop_slab", 11398e675f7aSKonstantin Khlebnikov "oom_kill", 11405509a5d2SDave Hansen 114103c5a6e1SMel Gorman #ifdef CONFIG_NUMA_BALANCING 114203c5a6e1SMel Gorman "numa_pte_updates", 114372403b4aSMel Gorman "numa_huge_pte_updates", 114403c5a6e1SMel Gorman "numa_hint_faults", 114503c5a6e1SMel Gorman "numa_hint_faults_local", 114603c5a6e1SMel Gorman "numa_pages_migrated", 114703c5a6e1SMel Gorman #endif 11485647bc29SMel Gorman #ifdef CONFIG_MIGRATION 11495647bc29SMel Gorman "pgmigrate_success", 11505647bc29SMel Gorman "pgmigrate_fail", 11515647bc29SMel Gorman #endif 1152fa25c503SKOSAKI Motohiro #ifdef CONFIG_COMPACTION 1153397487dbSMel Gorman "compact_migrate_scanned", 1154397487dbSMel Gorman "compact_free_scanned", 1155397487dbSMel Gorman "compact_isolated", 1156fa25c503SKOSAKI Motohiro "compact_stall", 1157fa25c503SKOSAKI Motohiro "compact_fail", 1158fa25c503SKOSAKI Motohiro "compact_success", 1159698b1b30SVlastimil Babka "compact_daemon_wake", 11607f354a54SDavid Rientjes "compact_daemon_migrate_scanned", 11617f354a54SDavid Rientjes "compact_daemon_free_scanned", 1162fa25c503SKOSAKI Motohiro #endif 1163fa25c503SKOSAKI Motohiro 1164fa25c503SKOSAKI Motohiro #ifdef CONFIG_HUGETLB_PAGE 1165fa25c503SKOSAKI Motohiro "htlb_buddy_alloc_success", 1166fa25c503SKOSAKI Motohiro "htlb_buddy_alloc_fail", 1167fa25c503SKOSAKI Motohiro #endif 1168fa25c503SKOSAKI Motohiro "unevictable_pgs_culled", 1169fa25c503SKOSAKI Motohiro "unevictable_pgs_scanned", 1170fa25c503SKOSAKI Motohiro "unevictable_pgs_rescued", 1171fa25c503SKOSAKI Motohiro "unevictable_pgs_mlocked", 1172fa25c503SKOSAKI Motohiro "unevictable_pgs_munlocked", 1173fa25c503SKOSAKI Motohiro "unevictable_pgs_cleared", 1174fa25c503SKOSAKI Motohiro "unevictable_pgs_stranded", 1175fa25c503SKOSAKI Motohiro 1176fa25c503SKOSAKI Motohiro #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1177fa25c503SKOSAKI Motohiro "thp_fault_alloc", 1178fa25c503SKOSAKI Motohiro "thp_fault_fallback", 1179fa25c503SKOSAKI Motohiro "thp_collapse_alloc", 1180fa25c503SKOSAKI Motohiro "thp_collapse_alloc_failed", 118195ecedcdSKirill A. Shutemov "thp_file_alloc", 118295ecedcdSKirill A. Shutemov "thp_file_mapped", 1183122afea9SKirill A. Shutemov "thp_split_page", 1184122afea9SKirill A. Shutemov "thp_split_page_failed", 1185f9719a03SKirill A. Shutemov "thp_deferred_split_page", 1186122afea9SKirill A. Shutemov "thp_split_pmd", 1187ce9311cfSYisheng Xie #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1188ce9311cfSYisheng Xie "thp_split_pud", 1189ce9311cfSYisheng Xie #endif 1190d8a8e1f0SKirill A. Shutemov "thp_zero_page_alloc", 1191d8a8e1f0SKirill A. Shutemov "thp_zero_page_alloc_failed", 1192225311a4SHuang Ying "thp_swpout", 1193fe490cc0SHuang Ying "thp_swpout_fallback", 1194fa25c503SKOSAKI Motohiro #endif 119509316c09SKonstantin Khlebnikov #ifdef CONFIG_MEMORY_BALLOON 119609316c09SKonstantin Khlebnikov "balloon_inflate", 119709316c09SKonstantin Khlebnikov "balloon_deflate", 119809316c09SKonstantin Khlebnikov #ifdef CONFIG_BALLOON_COMPACTION 119909316c09SKonstantin Khlebnikov "balloon_migrate", 120009316c09SKonstantin Khlebnikov #endif 120109316c09SKonstantin Khlebnikov #endif /* CONFIG_MEMORY_BALLOON */ 1202ec659934SMel Gorman #ifdef CONFIG_DEBUG_TLBFLUSH 12036df46865SDave Hansen #ifdef CONFIG_SMP 12049824cf97SDave Hansen "nr_tlb_remote_flush", 12059824cf97SDave Hansen "nr_tlb_remote_flush_received", 1206ec659934SMel Gorman #endif /* CONFIG_SMP */ 12079824cf97SDave Hansen "nr_tlb_local_flush_all", 12089824cf97SDave Hansen "nr_tlb_local_flush_one", 1209ec659934SMel Gorman #endif /* CONFIG_DEBUG_TLBFLUSH */ 1210fa25c503SKOSAKI Motohiro 12114f115147SDavidlohr Bueso #ifdef CONFIG_DEBUG_VM_VMACACHE 12124f115147SDavidlohr Bueso "vmacache_find_calls", 12134f115147SDavidlohr Bueso "vmacache_find_hits", 1214f5f302e2SDavidlohr Bueso "vmacache_full_flushes", 12154f115147SDavidlohr Bueso #endif 1216cbc65df2SHuang Ying #ifdef CONFIG_SWAP 1217cbc65df2SHuang Ying "swap_ra", 1218cbc65df2SHuang Ying "swap_ra_hit", 1219cbc65df2SHuang Ying #endif 1220fa25c503SKOSAKI Motohiro #endif /* CONFIG_VM_EVENTS_COUNTERS */ 1221fa25c503SKOSAKI Motohiro }; 12220d6617c7SDavid Rientjes #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */ 1223fa25c503SKOSAKI Motohiro 12243c486871SAndrew Morton #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 12253c486871SAndrew Morton defined(CONFIG_PROC_FS) 12263c486871SAndrew Morton static void *frag_start(struct seq_file *m, loff_t *pos) 12273c486871SAndrew Morton { 12283c486871SAndrew Morton pg_data_t *pgdat; 12293c486871SAndrew Morton loff_t node = *pos; 12303c486871SAndrew Morton 12313c486871SAndrew Morton for (pgdat = first_online_pgdat(); 12323c486871SAndrew Morton pgdat && node; 12333c486871SAndrew Morton pgdat = next_online_pgdat(pgdat)) 12343c486871SAndrew Morton --node; 12353c486871SAndrew Morton 12363c486871SAndrew Morton return pgdat; 12373c486871SAndrew Morton } 12383c486871SAndrew Morton 12393c486871SAndrew Morton static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 12403c486871SAndrew Morton { 12413c486871SAndrew Morton pg_data_t *pgdat = (pg_data_t *)arg; 12423c486871SAndrew Morton 12433c486871SAndrew Morton (*pos)++; 12443c486871SAndrew Morton return next_online_pgdat(pgdat); 12453c486871SAndrew Morton } 12463c486871SAndrew Morton 12473c486871SAndrew Morton static void frag_stop(struct seq_file *m, void *arg) 12483c486871SAndrew Morton { 12493c486871SAndrew Morton } 12503c486871SAndrew Morton 1251b2bd8598SDavid Rientjes /* 1252b2bd8598SDavid Rientjes * Walk zones in a node and print using a callback. 1253b2bd8598SDavid Rientjes * If @assert_populated is true, only use callback for zones that are populated. 1254b2bd8598SDavid Rientjes */ 12553c486871SAndrew Morton static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1256727c080fSVinayak Menon bool assert_populated, bool nolock, 12573c486871SAndrew Morton void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 12583c486871SAndrew Morton { 12593c486871SAndrew Morton struct zone *zone; 12603c486871SAndrew Morton struct zone *node_zones = pgdat->node_zones; 12613c486871SAndrew Morton unsigned long flags; 12623c486871SAndrew Morton 12633c486871SAndrew Morton for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1264b2bd8598SDavid Rientjes if (assert_populated && !populated_zone(zone)) 12653c486871SAndrew Morton continue; 12663c486871SAndrew Morton 1267727c080fSVinayak Menon if (!nolock) 12683c486871SAndrew Morton spin_lock_irqsave(&zone->lock, flags); 12693c486871SAndrew Morton print(m, pgdat, zone); 1270727c080fSVinayak Menon if (!nolock) 12713c486871SAndrew Morton spin_unlock_irqrestore(&zone->lock, flags); 12723c486871SAndrew Morton } 12733c486871SAndrew Morton } 12743c486871SAndrew Morton #endif 12753c486871SAndrew Morton 1276d7a5752cSMel Gorman #ifdef CONFIG_PROC_FS 1277467c996cSMel Gorman static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1278467c996cSMel Gorman struct zone *zone) 1279467c996cSMel Gorman { 1280467c996cSMel Gorman int order; 1281467c996cSMel Gorman 1282f6ac2354SChristoph Lameter seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1283f6ac2354SChristoph Lameter for (order = 0; order < MAX_ORDER; ++order) 1284f6ac2354SChristoph Lameter seq_printf(m, "%6lu ", zone->free_area[order].nr_free); 1285f6ac2354SChristoph Lameter seq_putc(m, '\n'); 1286f6ac2354SChristoph Lameter } 1287467c996cSMel Gorman 1288467c996cSMel Gorman /* 1289467c996cSMel Gorman * This walks the free areas for each zone. 1290467c996cSMel Gorman */ 1291467c996cSMel Gorman static int frag_show(struct seq_file *m, void *arg) 1292467c996cSMel Gorman { 1293467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1294727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, false, frag_show_print); 1295467c996cSMel Gorman return 0; 1296467c996cSMel Gorman } 1297467c996cSMel Gorman 1298467c996cSMel Gorman static void pagetypeinfo_showfree_print(struct seq_file *m, 1299467c996cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1300467c996cSMel Gorman { 1301467c996cSMel Gorman int order, mtype; 1302467c996cSMel Gorman 1303467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1304467c996cSMel Gorman seq_printf(m, "Node %4d, zone %8s, type %12s ", 1305467c996cSMel Gorman pgdat->node_id, 1306467c996cSMel Gorman zone->name, 1307467c996cSMel Gorman migratetype_names[mtype]); 1308467c996cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 1309467c996cSMel Gorman unsigned long freecount = 0; 1310467c996cSMel Gorman struct free_area *area; 1311467c996cSMel Gorman struct list_head *curr; 1312467c996cSMel Gorman 1313467c996cSMel Gorman area = &(zone->free_area[order]); 1314467c996cSMel Gorman 1315467c996cSMel Gorman list_for_each(curr, &area->free_list[mtype]) 1316467c996cSMel Gorman freecount++; 1317467c996cSMel Gorman seq_printf(m, "%6lu ", freecount); 1318467c996cSMel Gorman } 1319467c996cSMel Gorman seq_putc(m, '\n'); 1320467c996cSMel Gorman } 1321467c996cSMel Gorman } 1322467c996cSMel Gorman 1323467c996cSMel Gorman /* Print out the free pages at each order for each migatetype */ 1324467c996cSMel Gorman static int pagetypeinfo_showfree(struct seq_file *m, void *arg) 1325467c996cSMel Gorman { 1326467c996cSMel Gorman int order; 1327467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1328467c996cSMel Gorman 1329467c996cSMel Gorman /* Print header */ 1330467c996cSMel Gorman seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1331467c996cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) 1332467c996cSMel Gorman seq_printf(m, "%6d ", order); 1333467c996cSMel Gorman seq_putc(m, '\n'); 1334467c996cSMel Gorman 1335727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1336467c996cSMel Gorman 1337467c996cSMel Gorman return 0; 1338467c996cSMel Gorman } 1339467c996cSMel Gorman 1340467c996cSMel Gorman static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1341467c996cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1342467c996cSMel Gorman { 1343467c996cSMel Gorman int mtype; 1344467c996cSMel Gorman unsigned long pfn; 1345467c996cSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 1346108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 1347467c996cSMel Gorman unsigned long count[MIGRATE_TYPES] = { 0, }; 1348467c996cSMel Gorman 1349467c996cSMel Gorman for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1350467c996cSMel Gorman struct page *page; 1351467c996cSMel Gorman 1352d336e94eSMichal Hocko page = pfn_to_online_page(pfn); 1353d336e94eSMichal Hocko if (!page) 1354467c996cSMel Gorman continue; 1355467c996cSMel Gorman 1356eb33575cSMel Gorman /* Watch for unexpected holes punched in the memmap */ 1357eb33575cSMel Gorman if (!memmap_valid_within(pfn, page, zone)) 1358e80d6a24SMel Gorman continue; 1359eb33575cSMel Gorman 1360a91c43c7SJoonsoo Kim if (page_zone(page) != zone) 1361a91c43c7SJoonsoo Kim continue; 1362a91c43c7SJoonsoo Kim 1363467c996cSMel Gorman mtype = get_pageblock_migratetype(page); 1364467c996cSMel Gorman 1365e80d6a24SMel Gorman if (mtype < MIGRATE_TYPES) 1366467c996cSMel Gorman count[mtype]++; 1367467c996cSMel Gorman } 1368467c996cSMel Gorman 1369467c996cSMel Gorman /* Print counts */ 1370467c996cSMel Gorman seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1371467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1372467c996cSMel Gorman seq_printf(m, "%12lu ", count[mtype]); 1373467c996cSMel Gorman seq_putc(m, '\n'); 1374467c996cSMel Gorman } 1375467c996cSMel Gorman 1376f113e641SSeongJae Park /* Print out the number of pageblocks for each migratetype */ 1377467c996cSMel Gorman static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1378467c996cSMel Gorman { 1379467c996cSMel Gorman int mtype; 1380467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1381467c996cSMel Gorman 1382467c996cSMel Gorman seq_printf(m, "\n%-23s", "Number of blocks type "); 1383467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1384467c996cSMel Gorman seq_printf(m, "%12s ", migratetype_names[mtype]); 1385467c996cSMel Gorman seq_putc(m, '\n'); 1386727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, false, 1387727c080fSVinayak Menon pagetypeinfo_showblockcount_print); 1388467c996cSMel Gorman 1389467c996cSMel Gorman return 0; 1390467c996cSMel Gorman } 1391467c996cSMel Gorman 139248c96a36SJoonsoo Kim /* 139348c96a36SJoonsoo Kim * Print out the number of pageblocks for each migratetype that contain pages 139448c96a36SJoonsoo Kim * of other types. This gives an indication of how well fallbacks are being 139548c96a36SJoonsoo Kim * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 139648c96a36SJoonsoo Kim * to determine what is going on 139748c96a36SJoonsoo Kim */ 139848c96a36SJoonsoo Kim static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 139948c96a36SJoonsoo Kim { 140048c96a36SJoonsoo Kim #ifdef CONFIG_PAGE_OWNER 140148c96a36SJoonsoo Kim int mtype; 140248c96a36SJoonsoo Kim 14037dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 140448c96a36SJoonsoo Kim return; 140548c96a36SJoonsoo Kim 140648c96a36SJoonsoo Kim drain_all_pages(NULL); 140748c96a36SJoonsoo Kim 140848c96a36SJoonsoo Kim seq_printf(m, "\n%-23s", "Number of mixed blocks "); 140948c96a36SJoonsoo Kim for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 141048c96a36SJoonsoo Kim seq_printf(m, "%12s ", migratetype_names[mtype]); 141148c96a36SJoonsoo Kim seq_putc(m, '\n'); 141248c96a36SJoonsoo Kim 1413727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, true, 1414727c080fSVinayak Menon pagetypeinfo_showmixedcount_print); 141548c96a36SJoonsoo Kim #endif /* CONFIG_PAGE_OWNER */ 141648c96a36SJoonsoo Kim } 141748c96a36SJoonsoo Kim 1418467c996cSMel Gorman /* 1419467c996cSMel Gorman * This prints out statistics in relation to grouping pages by mobility. 1420467c996cSMel Gorman * It is expensive to collect so do not constantly read the file. 1421467c996cSMel Gorman */ 1422467c996cSMel Gorman static int pagetypeinfo_show(struct seq_file *m, void *arg) 1423467c996cSMel Gorman { 1424467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1425467c996cSMel Gorman 142641b25a37SKOSAKI Motohiro /* check memoryless node */ 1427a47b53c5SLai Jiangshan if (!node_state(pgdat->node_id, N_MEMORY)) 142841b25a37SKOSAKI Motohiro return 0; 142941b25a37SKOSAKI Motohiro 1430467c996cSMel Gorman seq_printf(m, "Page block order: %d\n", pageblock_order); 1431467c996cSMel Gorman seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1432467c996cSMel Gorman seq_putc(m, '\n'); 1433467c996cSMel Gorman pagetypeinfo_showfree(m, pgdat); 1434467c996cSMel Gorman pagetypeinfo_showblockcount(m, pgdat); 143548c96a36SJoonsoo Kim pagetypeinfo_showmixedcount(m, pgdat); 1436467c996cSMel Gorman 1437f6ac2354SChristoph Lameter return 0; 1438f6ac2354SChristoph Lameter } 1439f6ac2354SChristoph Lameter 14408f32f7e5SAlexey Dobriyan static const struct seq_operations fragmentation_op = { 1441f6ac2354SChristoph Lameter .start = frag_start, 1442f6ac2354SChristoph Lameter .next = frag_next, 1443f6ac2354SChristoph Lameter .stop = frag_stop, 1444f6ac2354SChristoph Lameter .show = frag_show, 1445f6ac2354SChristoph Lameter }; 1446f6ac2354SChristoph Lameter 14478f32f7e5SAlexey Dobriyan static int fragmentation_open(struct inode *inode, struct file *file) 14488f32f7e5SAlexey Dobriyan { 14498f32f7e5SAlexey Dobriyan return seq_open(file, &fragmentation_op); 14508f32f7e5SAlexey Dobriyan } 14518f32f7e5SAlexey Dobriyan 14529d85e15fSAnshuman Khandual static const struct file_operations buddyinfo_file_operations = { 14538f32f7e5SAlexey Dobriyan .open = fragmentation_open, 14548f32f7e5SAlexey Dobriyan .read = seq_read, 14558f32f7e5SAlexey Dobriyan .llseek = seq_lseek, 14568f32f7e5SAlexey Dobriyan .release = seq_release, 14578f32f7e5SAlexey Dobriyan }; 14588f32f7e5SAlexey Dobriyan 145974e2e8e8SAlexey Dobriyan static const struct seq_operations pagetypeinfo_op = { 1460467c996cSMel Gorman .start = frag_start, 1461467c996cSMel Gorman .next = frag_next, 1462467c996cSMel Gorman .stop = frag_stop, 1463467c996cSMel Gorman .show = pagetypeinfo_show, 1464467c996cSMel Gorman }; 1465467c996cSMel Gorman 146674e2e8e8SAlexey Dobriyan static int pagetypeinfo_open(struct inode *inode, struct file *file) 146774e2e8e8SAlexey Dobriyan { 146874e2e8e8SAlexey Dobriyan return seq_open(file, &pagetypeinfo_op); 146974e2e8e8SAlexey Dobriyan } 147074e2e8e8SAlexey Dobriyan 14719d85e15fSAnshuman Khandual static const struct file_operations pagetypeinfo_file_operations = { 147274e2e8e8SAlexey Dobriyan .open = pagetypeinfo_open, 147374e2e8e8SAlexey Dobriyan .read = seq_read, 147474e2e8e8SAlexey Dobriyan .llseek = seq_lseek, 147574e2e8e8SAlexey Dobriyan .release = seq_release, 147674e2e8e8SAlexey Dobriyan }; 147774e2e8e8SAlexey Dobriyan 1478e2ecc8a7SMel Gorman static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1479e2ecc8a7SMel Gorman { 1480e2ecc8a7SMel Gorman int zid; 1481e2ecc8a7SMel Gorman 1482e2ecc8a7SMel Gorman for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1483e2ecc8a7SMel Gorman struct zone *compare = &pgdat->node_zones[zid]; 1484e2ecc8a7SMel Gorman 1485e2ecc8a7SMel Gorman if (populated_zone(compare)) 1486e2ecc8a7SMel Gorman return zone == compare; 1487e2ecc8a7SMel Gorman } 1488e2ecc8a7SMel Gorman 1489e2ecc8a7SMel Gorman return false; 1490e2ecc8a7SMel Gorman } 1491e2ecc8a7SMel Gorman 1492467c996cSMel Gorman static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1493467c996cSMel Gorman struct zone *zone) 1494f6ac2354SChristoph Lameter { 1495f6ac2354SChristoph Lameter int i; 1496f6ac2354SChristoph Lameter seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1497e2ecc8a7SMel Gorman if (is_zone_first_populated(pgdat, zone)) { 1498e2ecc8a7SMel Gorman seq_printf(m, "\n per-node stats"); 1499e2ecc8a7SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1500e2ecc8a7SMel Gorman seq_printf(m, "\n %-12s %lu", 15013a321d2aSKemi Wang vmstat_text[i + NR_VM_ZONE_STAT_ITEMS + 15023a321d2aSKemi Wang NR_VM_NUMA_STAT_ITEMS], 1503e2ecc8a7SMel Gorman node_page_state(pgdat, i)); 1504e2ecc8a7SMel Gorman } 1505e2ecc8a7SMel Gorman } 1506f6ac2354SChristoph Lameter seq_printf(m, 1507f6ac2354SChristoph Lameter "\n pages free %lu" 1508f6ac2354SChristoph Lameter "\n min %lu" 1509f6ac2354SChristoph Lameter "\n low %lu" 1510f6ac2354SChristoph Lameter "\n high %lu" 1511f6ac2354SChristoph Lameter "\n spanned %lu" 15129feedc9dSJiang Liu "\n present %lu" 15139feedc9dSJiang Liu "\n managed %lu", 151488f5acf8SMel Gorman zone_page_state(zone, NR_FREE_PAGES), 151541858966SMel Gorman min_wmark_pages(zone), 151641858966SMel Gorman low_wmark_pages(zone), 151741858966SMel Gorman high_wmark_pages(zone), 1518f6ac2354SChristoph Lameter zone->spanned_pages, 15199feedc9dSJiang Liu zone->present_pages, 15209feedc9dSJiang Liu zone->managed_pages); 15212244b95aSChristoph Lameter 1522f6ac2354SChristoph Lameter seq_printf(m, 15233484b2deSMel Gorman "\n protection: (%ld", 1524f6ac2354SChristoph Lameter zone->lowmem_reserve[0]); 1525f6ac2354SChristoph Lameter for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 15263484b2deSMel Gorman seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 15277dfb8bf3SDavid Rientjes seq_putc(m, ')'); 15287dfb8bf3SDavid Rientjes 15297dfb8bf3SDavid Rientjes /* If unpopulated, no other information is useful */ 15307dfb8bf3SDavid Rientjes if (!populated_zone(zone)) { 15317dfb8bf3SDavid Rientjes seq_putc(m, '\n'); 15327dfb8bf3SDavid Rientjes return; 15337dfb8bf3SDavid Rientjes } 15347dfb8bf3SDavid Rientjes 15357dfb8bf3SDavid Rientjes for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 15367dfb8bf3SDavid Rientjes seq_printf(m, "\n %-12s %lu", vmstat_text[i], 15377dfb8bf3SDavid Rientjes zone_page_state(zone, i)); 15387dfb8bf3SDavid Rientjes 15393a321d2aSKemi Wang #ifdef CONFIG_NUMA 15403a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) 15413a321d2aSKemi Wang seq_printf(m, "\n %-12s %lu", 15423a321d2aSKemi Wang vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], 154363803222SKemi Wang zone_numa_state_snapshot(zone, i)); 15443a321d2aSKemi Wang #endif 15453a321d2aSKemi Wang 15467dfb8bf3SDavid Rientjes seq_printf(m, "\n pagesets"); 1547f6ac2354SChristoph Lameter for_each_online_cpu(i) { 1548f6ac2354SChristoph Lameter struct per_cpu_pageset *pageset; 1549f6ac2354SChristoph Lameter 155099dcc3e5SChristoph Lameter pageset = per_cpu_ptr(zone->pageset, i); 1551f6ac2354SChristoph Lameter seq_printf(m, 15523dfa5721SChristoph Lameter "\n cpu: %i" 1553f6ac2354SChristoph Lameter "\n count: %i" 1554f6ac2354SChristoph Lameter "\n high: %i" 1555f6ac2354SChristoph Lameter "\n batch: %i", 15563dfa5721SChristoph Lameter i, 15573dfa5721SChristoph Lameter pageset->pcp.count, 15583dfa5721SChristoph Lameter pageset->pcp.high, 15593dfa5721SChristoph Lameter pageset->pcp.batch); 1560df9ecabaSChristoph Lameter #ifdef CONFIG_SMP 1561df9ecabaSChristoph Lameter seq_printf(m, "\n vm stats threshold: %d", 1562df9ecabaSChristoph Lameter pageset->stat_threshold); 1563df9ecabaSChristoph Lameter #endif 1564f6ac2354SChristoph Lameter } 1565f6ac2354SChristoph Lameter seq_printf(m, 1566599d0c95SMel Gorman "\n node_unreclaimable: %u" 1567*3a50d14dSAndrey Ryabinin "\n start_pfn: %lu", 1568c73322d0SJohannes Weiner pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES, 1569*3a50d14dSAndrey Ryabinin zone->zone_start_pfn); 1570f6ac2354SChristoph Lameter seq_putc(m, '\n'); 1571f6ac2354SChristoph Lameter } 1572467c996cSMel Gorman 1573467c996cSMel Gorman /* 1574b2bd8598SDavid Rientjes * Output information about zones in @pgdat. All zones are printed regardless 1575b2bd8598SDavid Rientjes * of whether they are populated or not: lowmem_reserve_ratio operates on the 1576b2bd8598SDavid Rientjes * set of all zones and userspace would not be aware of such zones if they are 1577b2bd8598SDavid Rientjes * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1578467c996cSMel Gorman */ 1579467c996cSMel Gorman static int zoneinfo_show(struct seq_file *m, void *arg) 1580467c996cSMel Gorman { 1581467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1582727c080fSVinayak Menon walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1583f6ac2354SChristoph Lameter return 0; 1584f6ac2354SChristoph Lameter } 1585f6ac2354SChristoph Lameter 15865c9fe628SAlexey Dobriyan static const struct seq_operations zoneinfo_op = { 1587f6ac2354SChristoph Lameter .start = frag_start, /* iterate over all zones. The same as in 1588f6ac2354SChristoph Lameter * fragmentation. */ 1589f6ac2354SChristoph Lameter .next = frag_next, 1590f6ac2354SChristoph Lameter .stop = frag_stop, 1591f6ac2354SChristoph Lameter .show = zoneinfo_show, 1592f6ac2354SChristoph Lameter }; 1593f6ac2354SChristoph Lameter 15945c9fe628SAlexey Dobriyan static int zoneinfo_open(struct inode *inode, struct file *file) 15955c9fe628SAlexey Dobriyan { 15965c9fe628SAlexey Dobriyan return seq_open(file, &zoneinfo_op); 15975c9fe628SAlexey Dobriyan } 15985c9fe628SAlexey Dobriyan 15999d85e15fSAnshuman Khandual static const struct file_operations zoneinfo_file_operations = { 16005c9fe628SAlexey Dobriyan .open = zoneinfo_open, 16015c9fe628SAlexey Dobriyan .read = seq_read, 16025c9fe628SAlexey Dobriyan .llseek = seq_lseek, 16035c9fe628SAlexey Dobriyan .release = seq_release, 16045c9fe628SAlexey Dobriyan }; 16055c9fe628SAlexey Dobriyan 160679da826aSMichael Rubin enum writeback_stat_item { 160779da826aSMichael Rubin NR_DIRTY_THRESHOLD, 160879da826aSMichael Rubin NR_DIRTY_BG_THRESHOLD, 160979da826aSMichael Rubin NR_VM_WRITEBACK_STAT_ITEMS, 161079da826aSMichael Rubin }; 161179da826aSMichael Rubin 1612f6ac2354SChristoph Lameter static void *vmstat_start(struct seq_file *m, loff_t *pos) 1613f6ac2354SChristoph Lameter { 16142244b95aSChristoph Lameter unsigned long *v; 161579da826aSMichael Rubin int i, stat_items_size; 1616f6ac2354SChristoph Lameter 1617f6ac2354SChristoph Lameter if (*pos >= ARRAY_SIZE(vmstat_text)) 1618f6ac2354SChristoph Lameter return NULL; 161979da826aSMichael Rubin stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) + 16203a321d2aSKemi Wang NR_VM_NUMA_STAT_ITEMS * sizeof(unsigned long) + 162175ef7184SMel Gorman NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) + 162279da826aSMichael Rubin NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long); 1623f6ac2354SChristoph Lameter 1624f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 162579da826aSMichael Rubin stat_items_size += sizeof(struct vm_event_state); 1626f8891e5eSChristoph Lameter #endif 162779da826aSMichael Rubin 162879da826aSMichael Rubin v = kmalloc(stat_items_size, GFP_KERNEL); 16292244b95aSChristoph Lameter m->private = v; 16302244b95aSChristoph Lameter if (!v) 1631f6ac2354SChristoph Lameter return ERR_PTR(-ENOMEM); 16322244b95aSChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1633c41f012aSMichal Hocko v[i] = global_zone_page_state(i); 163479da826aSMichael Rubin v += NR_VM_ZONE_STAT_ITEMS; 163579da826aSMichael Rubin 16363a321d2aSKemi Wang #ifdef CONFIG_NUMA 16373a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) 16383a321d2aSKemi Wang v[i] = global_numa_state(i); 16393a321d2aSKemi Wang v += NR_VM_NUMA_STAT_ITEMS; 16403a321d2aSKemi Wang #endif 16413a321d2aSKemi Wang 164275ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 164375ef7184SMel Gorman v[i] = global_node_page_state(i); 164475ef7184SMel Gorman v += NR_VM_NODE_STAT_ITEMS; 164575ef7184SMel Gorman 164679da826aSMichael Rubin global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 164779da826aSMichael Rubin v + NR_DIRTY_THRESHOLD); 164879da826aSMichael Rubin v += NR_VM_WRITEBACK_STAT_ITEMS; 164979da826aSMichael Rubin 1650f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 165179da826aSMichael Rubin all_vm_events(v); 165279da826aSMichael Rubin v[PGPGIN] /= 2; /* sectors -> kbytes */ 165379da826aSMichael Rubin v[PGPGOUT] /= 2; 1654f8891e5eSChristoph Lameter #endif 1655ff8b16d7SWu Fengguang return (unsigned long *)m->private + *pos; 1656f6ac2354SChristoph Lameter } 1657f6ac2354SChristoph Lameter 1658f6ac2354SChristoph Lameter static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1659f6ac2354SChristoph Lameter { 1660f6ac2354SChristoph Lameter (*pos)++; 1661f6ac2354SChristoph Lameter if (*pos >= ARRAY_SIZE(vmstat_text)) 1662f6ac2354SChristoph Lameter return NULL; 1663f6ac2354SChristoph Lameter return (unsigned long *)m->private + *pos; 1664f6ac2354SChristoph Lameter } 1665f6ac2354SChristoph Lameter 1666f6ac2354SChristoph Lameter static int vmstat_show(struct seq_file *m, void *arg) 1667f6ac2354SChristoph Lameter { 1668f6ac2354SChristoph Lameter unsigned long *l = arg; 1669f6ac2354SChristoph Lameter unsigned long off = l - (unsigned long *)m->private; 167068ba0326SAlexey Dobriyan 167168ba0326SAlexey Dobriyan seq_puts(m, vmstat_text[off]); 167275ba1d07SJoe Perches seq_put_decimal_ull(m, " ", *l); 167368ba0326SAlexey Dobriyan seq_putc(m, '\n'); 1674f6ac2354SChristoph Lameter return 0; 1675f6ac2354SChristoph Lameter } 1676f6ac2354SChristoph Lameter 1677f6ac2354SChristoph Lameter static void vmstat_stop(struct seq_file *m, void *arg) 1678f6ac2354SChristoph Lameter { 1679f6ac2354SChristoph Lameter kfree(m->private); 1680f6ac2354SChristoph Lameter m->private = NULL; 1681f6ac2354SChristoph Lameter } 1682f6ac2354SChristoph Lameter 1683b6aa44abSAlexey Dobriyan static const struct seq_operations vmstat_op = { 1684f6ac2354SChristoph Lameter .start = vmstat_start, 1685f6ac2354SChristoph Lameter .next = vmstat_next, 1686f6ac2354SChristoph Lameter .stop = vmstat_stop, 1687f6ac2354SChristoph Lameter .show = vmstat_show, 1688f6ac2354SChristoph Lameter }; 1689f6ac2354SChristoph Lameter 1690b6aa44abSAlexey Dobriyan static int vmstat_open(struct inode *inode, struct file *file) 1691b6aa44abSAlexey Dobriyan { 1692b6aa44abSAlexey Dobriyan return seq_open(file, &vmstat_op); 1693b6aa44abSAlexey Dobriyan } 1694b6aa44abSAlexey Dobriyan 16959d85e15fSAnshuman Khandual static const struct file_operations vmstat_file_operations = { 1696b6aa44abSAlexey Dobriyan .open = vmstat_open, 1697b6aa44abSAlexey Dobriyan .read = seq_read, 1698b6aa44abSAlexey Dobriyan .llseek = seq_lseek, 1699b6aa44abSAlexey Dobriyan .release = seq_release, 1700b6aa44abSAlexey Dobriyan }; 1701f6ac2354SChristoph Lameter #endif /* CONFIG_PROC_FS */ 1702f6ac2354SChristoph Lameter 1703df9ecabaSChristoph Lameter #ifdef CONFIG_SMP 1704d1187ed2SChristoph Lameter static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 170577461ab3SChristoph Lameter int sysctl_stat_interval __read_mostly = HZ; 1706d1187ed2SChristoph Lameter 170752b6f46bSHugh Dickins #ifdef CONFIG_PROC_FS 170852b6f46bSHugh Dickins static void refresh_vm_stats(struct work_struct *work) 170952b6f46bSHugh Dickins { 171052b6f46bSHugh Dickins refresh_cpu_vm_stats(true); 171152b6f46bSHugh Dickins } 171252b6f46bSHugh Dickins 171352b6f46bSHugh Dickins int vmstat_refresh(struct ctl_table *table, int write, 171452b6f46bSHugh Dickins void __user *buffer, size_t *lenp, loff_t *ppos) 171552b6f46bSHugh Dickins { 171652b6f46bSHugh Dickins long val; 171752b6f46bSHugh Dickins int err; 171852b6f46bSHugh Dickins int i; 171952b6f46bSHugh Dickins 172052b6f46bSHugh Dickins /* 172152b6f46bSHugh Dickins * The regular update, every sysctl_stat_interval, may come later 172252b6f46bSHugh Dickins * than expected: leaving a significant amount in per_cpu buckets. 172352b6f46bSHugh Dickins * This is particularly misleading when checking a quantity of HUGE 172452b6f46bSHugh Dickins * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 172552b6f46bSHugh Dickins * which can equally be echo'ed to or cat'ted from (by root), 172652b6f46bSHugh Dickins * can be used to update the stats just before reading them. 172752b6f46bSHugh Dickins * 1728c41f012aSMichal Hocko * Oh, and since global_zone_page_state() etc. are so careful to hide 172952b6f46bSHugh Dickins * transiently negative values, report an error here if any of 173052b6f46bSHugh Dickins * the stats is negative, so we know to go looking for imbalance. 173152b6f46bSHugh Dickins */ 173252b6f46bSHugh Dickins err = schedule_on_each_cpu(refresh_vm_stats); 173352b6f46bSHugh Dickins if (err) 173452b6f46bSHugh Dickins return err; 173552b6f46bSHugh Dickins for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 173675ef7184SMel Gorman val = atomic_long_read(&vm_zone_stat[i]); 173752b6f46bSHugh Dickins if (val < 0) { 173852b6f46bSHugh Dickins pr_warn("%s: %s %ld\n", 173952b6f46bSHugh Dickins __func__, vmstat_text[i], val); 174052b6f46bSHugh Dickins err = -EINVAL; 174152b6f46bSHugh Dickins } 174252b6f46bSHugh Dickins } 17433a321d2aSKemi Wang #ifdef CONFIG_NUMA 17443a321d2aSKemi Wang for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) { 17453a321d2aSKemi Wang val = atomic_long_read(&vm_numa_stat[i]); 17463a321d2aSKemi Wang if (val < 0) { 17473a321d2aSKemi Wang pr_warn("%s: %s %ld\n", 17483a321d2aSKemi Wang __func__, vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], val); 17493a321d2aSKemi Wang err = -EINVAL; 17503a321d2aSKemi Wang } 17513a321d2aSKemi Wang } 17523a321d2aSKemi Wang #endif 175352b6f46bSHugh Dickins if (err) 175452b6f46bSHugh Dickins return err; 175552b6f46bSHugh Dickins if (write) 175652b6f46bSHugh Dickins *ppos += *lenp; 175752b6f46bSHugh Dickins else 175852b6f46bSHugh Dickins *lenp = 0; 175952b6f46bSHugh Dickins return 0; 176052b6f46bSHugh Dickins } 176152b6f46bSHugh Dickins #endif /* CONFIG_PROC_FS */ 176252b6f46bSHugh Dickins 1763d1187ed2SChristoph Lameter static void vmstat_update(struct work_struct *w) 1764d1187ed2SChristoph Lameter { 17650eb77e98SChristoph Lameter if (refresh_cpu_vm_stats(true)) { 17667cc36bbdSChristoph Lameter /* 17677cc36bbdSChristoph Lameter * Counters were updated so we expect more updates 17687cc36bbdSChristoph Lameter * to occur in the future. Keep on running the 17697cc36bbdSChristoph Lameter * update worker thread. 17707cc36bbdSChristoph Lameter */ 1771ce612879SMichal Hocko queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 1772176bed1dSLinus Torvalds this_cpu_ptr(&vmstat_work), 177398f4ebb2SAnton Blanchard round_jiffies_relative(sysctl_stat_interval)); 1774f01f17d3SMichal Hocko } 1775d1187ed2SChristoph Lameter } 1776d1187ed2SChristoph Lameter 17777cc36bbdSChristoph Lameter /* 17780eb77e98SChristoph Lameter * Switch off vmstat processing and then fold all the remaining differentials 17790eb77e98SChristoph Lameter * until the diffs stay at zero. The function is used by NOHZ and can only be 17800eb77e98SChristoph Lameter * invoked when tick processing is not active. 17810eb77e98SChristoph Lameter */ 17820eb77e98SChristoph Lameter /* 17837cc36bbdSChristoph Lameter * Check if the diffs for a certain cpu indicate that 17847cc36bbdSChristoph Lameter * an update is needed. 17857cc36bbdSChristoph Lameter */ 17867cc36bbdSChristoph Lameter static bool need_update(int cpu) 1787d1187ed2SChristoph Lameter { 17887cc36bbdSChristoph Lameter struct zone *zone; 1789d1187ed2SChristoph Lameter 17907cc36bbdSChristoph Lameter for_each_populated_zone(zone) { 17917cc36bbdSChristoph Lameter struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu); 17927cc36bbdSChristoph Lameter 17937cc36bbdSChristoph Lameter BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1); 17943a321d2aSKemi Wang #ifdef CONFIG_NUMA 17951d90ca89SKemi Wang BUILD_BUG_ON(sizeof(p->vm_numa_stat_diff[0]) != 2); 17963a321d2aSKemi Wang #endif 179763803222SKemi Wang 17987cc36bbdSChristoph Lameter /* 17997cc36bbdSChristoph Lameter * The fast way of checking if there are any vmstat diffs. 18007cc36bbdSChristoph Lameter * This works because the diffs are byte sized items. 18017cc36bbdSChristoph Lameter */ 18027cc36bbdSChristoph Lameter if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS)) 18037cc36bbdSChristoph Lameter return true; 18043a321d2aSKemi Wang #ifdef CONFIG_NUMA 18053a321d2aSKemi Wang if (memchr_inv(p->vm_numa_stat_diff, 0, NR_VM_NUMA_STAT_ITEMS)) 18063a321d2aSKemi Wang return true; 18073a321d2aSKemi Wang #endif 18087cc36bbdSChristoph Lameter } 18097cc36bbdSChristoph Lameter return false; 18107cc36bbdSChristoph Lameter } 18117cc36bbdSChristoph Lameter 18127b8da4c7SChristoph Lameter /* 18137b8da4c7SChristoph Lameter * Switch off vmstat processing and then fold all the remaining differentials 18147b8da4c7SChristoph Lameter * until the diffs stay at zero. The function is used by NOHZ and can only be 18157b8da4c7SChristoph Lameter * invoked when tick processing is not active. 18167b8da4c7SChristoph Lameter */ 1817f01f17d3SMichal Hocko void quiet_vmstat(void) 1818f01f17d3SMichal Hocko { 1819f01f17d3SMichal Hocko if (system_state != SYSTEM_RUNNING) 1820f01f17d3SMichal Hocko return; 1821f01f17d3SMichal Hocko 18227b8da4c7SChristoph Lameter if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 1823f01f17d3SMichal Hocko return; 1824f01f17d3SMichal Hocko 1825f01f17d3SMichal Hocko if (!need_update(smp_processor_id())) 1826f01f17d3SMichal Hocko return; 1827f01f17d3SMichal Hocko 1828f01f17d3SMichal Hocko /* 1829f01f17d3SMichal Hocko * Just refresh counters and do not care about the pending delayed 1830f01f17d3SMichal Hocko * vmstat_update. It doesn't fire that often to matter and canceling 1831f01f17d3SMichal Hocko * it would be too expensive from this path. 1832f01f17d3SMichal Hocko * vmstat_shepherd will take care about that for us. 1833f01f17d3SMichal Hocko */ 1834f01f17d3SMichal Hocko refresh_cpu_vm_stats(false); 1835f01f17d3SMichal Hocko } 1836f01f17d3SMichal Hocko 18377cc36bbdSChristoph Lameter /* 18387cc36bbdSChristoph Lameter * Shepherd worker thread that checks the 18397cc36bbdSChristoph Lameter * differentials of processors that have their worker 18407cc36bbdSChristoph Lameter * threads for vm statistics updates disabled because of 18417cc36bbdSChristoph Lameter * inactivity. 18427cc36bbdSChristoph Lameter */ 18437cc36bbdSChristoph Lameter static void vmstat_shepherd(struct work_struct *w); 18447cc36bbdSChristoph Lameter 18450eb77e98SChristoph Lameter static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 18467cc36bbdSChristoph Lameter 18477cc36bbdSChristoph Lameter static void vmstat_shepherd(struct work_struct *w) 18487cc36bbdSChristoph Lameter { 18497cc36bbdSChristoph Lameter int cpu; 18507cc36bbdSChristoph Lameter 18517cc36bbdSChristoph Lameter get_online_cpus(); 18527cc36bbdSChristoph Lameter /* Check processors whose vmstat worker threads have been disabled */ 18537b8da4c7SChristoph Lameter for_each_online_cpu(cpu) { 1854f01f17d3SMichal Hocko struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 18557cc36bbdSChristoph Lameter 18567b8da4c7SChristoph Lameter if (!delayed_work_pending(dw) && need_update(cpu)) 1857ce612879SMichal Hocko queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 1858f01f17d3SMichal Hocko } 18597cc36bbdSChristoph Lameter put_online_cpus(); 18607cc36bbdSChristoph Lameter 18617cc36bbdSChristoph Lameter schedule_delayed_work(&shepherd, 18627cc36bbdSChristoph Lameter round_jiffies_relative(sysctl_stat_interval)); 18637cc36bbdSChristoph Lameter } 18647cc36bbdSChristoph Lameter 18657cc36bbdSChristoph Lameter static void __init start_shepherd_timer(void) 18667cc36bbdSChristoph Lameter { 18677cc36bbdSChristoph Lameter int cpu; 18687cc36bbdSChristoph Lameter 18697cc36bbdSChristoph Lameter for_each_possible_cpu(cpu) 1870ccde8bd4SMichal Hocko INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 18717cc36bbdSChristoph Lameter vmstat_update); 18727cc36bbdSChristoph Lameter 18737cc36bbdSChristoph Lameter schedule_delayed_work(&shepherd, 18747cc36bbdSChristoph Lameter round_jiffies_relative(sysctl_stat_interval)); 1875d1187ed2SChristoph Lameter } 1876d1187ed2SChristoph Lameter 187703e86dbaSTim Chen static void __init init_cpu_node_state(void) 187803e86dbaSTim Chen { 18794c501327SSebastian Andrzej Siewior int node; 188003e86dbaSTim Chen 18814c501327SSebastian Andrzej Siewior for_each_online_node(node) { 18824c501327SSebastian Andrzej Siewior if (cpumask_weight(cpumask_of_node(node)) > 0) 18834c501327SSebastian Andrzej Siewior node_set_state(node, N_CPU); 18844c501327SSebastian Andrzej Siewior } 188503e86dbaSTim Chen } 188603e86dbaSTim Chen 18875438da97SSebastian Andrzej Siewior static int vmstat_cpu_online(unsigned int cpu) 1888807a1bd2SToshi Kani { 18895ee28a44SKAMEZAWA Hiroyuki refresh_zone_stat_thresholds(); 1890ad596925SChristoph Lameter node_set_state(cpu_to_node(cpu), N_CPU); 18915438da97SSebastian Andrzej Siewior return 0; 1892df9ecabaSChristoph Lameter } 1893df9ecabaSChristoph Lameter 18945438da97SSebastian Andrzej Siewior static int vmstat_cpu_down_prep(unsigned int cpu) 18955438da97SSebastian Andrzej Siewior { 18965438da97SSebastian Andrzej Siewior cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 18975438da97SSebastian Andrzej Siewior return 0; 18985438da97SSebastian Andrzej Siewior } 18995438da97SSebastian Andrzej Siewior 19005438da97SSebastian Andrzej Siewior static int vmstat_cpu_dead(unsigned int cpu) 19015438da97SSebastian Andrzej Siewior { 19025438da97SSebastian Andrzej Siewior const struct cpumask *node_cpus; 19035438da97SSebastian Andrzej Siewior int node; 19045438da97SSebastian Andrzej Siewior 19055438da97SSebastian Andrzej Siewior node = cpu_to_node(cpu); 19065438da97SSebastian Andrzej Siewior 19075438da97SSebastian Andrzej Siewior refresh_zone_stat_thresholds(); 19085438da97SSebastian Andrzej Siewior node_cpus = cpumask_of_node(node); 19095438da97SSebastian Andrzej Siewior if (cpumask_weight(node_cpus) > 0) 19105438da97SSebastian Andrzej Siewior return 0; 19115438da97SSebastian Andrzej Siewior 19125438da97SSebastian Andrzej Siewior node_clear_state(node, N_CPU); 19135438da97SSebastian Andrzej Siewior return 0; 19145438da97SSebastian Andrzej Siewior } 19155438da97SSebastian Andrzej Siewior 19168f32f7e5SAlexey Dobriyan #endif 1917df9ecabaSChristoph Lameter 1918ce612879SMichal Hocko struct workqueue_struct *mm_percpu_wq; 1919ce612879SMichal Hocko 1920597b7305SMichal Hocko void __init init_mm_internals(void) 1921df9ecabaSChristoph Lameter { 1922ce612879SMichal Hocko int ret __maybe_unused; 19235438da97SSebastian Andrzej Siewior 192480d136e1SMichal Hocko mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0); 1925ce612879SMichal Hocko 1926ce612879SMichal Hocko #ifdef CONFIG_SMP 19275438da97SSebastian Andrzej Siewior ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 19285438da97SSebastian Andrzej Siewior NULL, vmstat_cpu_dead); 19295438da97SSebastian Andrzej Siewior if (ret < 0) 19305438da97SSebastian Andrzej Siewior pr_err("vmstat: failed to register 'dead' hotplug state\n"); 19315438da97SSebastian Andrzej Siewior 19325438da97SSebastian Andrzej Siewior ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 19335438da97SSebastian Andrzej Siewior vmstat_cpu_online, 19345438da97SSebastian Andrzej Siewior vmstat_cpu_down_prep); 19355438da97SSebastian Andrzej Siewior if (ret < 0) 19365438da97SSebastian Andrzej Siewior pr_err("vmstat: failed to register 'online' hotplug state\n"); 19375438da97SSebastian Andrzej Siewior 19385438da97SSebastian Andrzej Siewior get_online_cpus(); 193903e86dbaSTim Chen init_cpu_node_state(); 19405438da97SSebastian Andrzej Siewior put_online_cpus(); 1941d1187ed2SChristoph Lameter 19427cc36bbdSChristoph Lameter start_shepherd_timer(); 19438f32f7e5SAlexey Dobriyan #endif 19448f32f7e5SAlexey Dobriyan #ifdef CONFIG_PROC_FS 19459d85e15fSAnshuman Khandual proc_create("buddyinfo", 0444, NULL, &buddyinfo_file_operations); 19469d85e15fSAnshuman Khandual proc_create("pagetypeinfo", 0444, NULL, &pagetypeinfo_file_operations); 19479d85e15fSAnshuman Khandual proc_create("vmstat", 0444, NULL, &vmstat_file_operations); 19489d85e15fSAnshuman Khandual proc_create("zoneinfo", 0444, NULL, &zoneinfo_file_operations); 19498f32f7e5SAlexey Dobriyan #endif 1950df9ecabaSChristoph Lameter } 1951d7a5752cSMel Gorman 1952d7a5752cSMel Gorman #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 1953d7a5752cSMel Gorman 1954d7a5752cSMel Gorman /* 1955d7a5752cSMel Gorman * Return an index indicating how much of the available free memory is 1956d7a5752cSMel Gorman * unusable for an allocation of the requested size. 1957d7a5752cSMel Gorman */ 1958d7a5752cSMel Gorman static int unusable_free_index(unsigned int order, 1959d7a5752cSMel Gorman struct contig_page_info *info) 1960d7a5752cSMel Gorman { 1961d7a5752cSMel Gorman /* No free memory is interpreted as all free memory is unusable */ 1962d7a5752cSMel Gorman if (info->free_pages == 0) 1963d7a5752cSMel Gorman return 1000; 1964d7a5752cSMel Gorman 1965d7a5752cSMel Gorman /* 1966d7a5752cSMel Gorman * Index should be a value between 0 and 1. Return a value to 3 1967d7a5752cSMel Gorman * decimal places. 1968d7a5752cSMel Gorman * 1969d7a5752cSMel Gorman * 0 => no fragmentation 1970d7a5752cSMel Gorman * 1 => high fragmentation 1971d7a5752cSMel Gorman */ 1972d7a5752cSMel Gorman return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 1973d7a5752cSMel Gorman 1974d7a5752cSMel Gorman } 1975d7a5752cSMel Gorman 1976d7a5752cSMel Gorman static void unusable_show_print(struct seq_file *m, 1977d7a5752cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1978d7a5752cSMel Gorman { 1979d7a5752cSMel Gorman unsigned int order; 1980d7a5752cSMel Gorman int index; 1981d7a5752cSMel Gorman struct contig_page_info info; 1982d7a5752cSMel Gorman 1983d7a5752cSMel Gorman seq_printf(m, "Node %d, zone %8s ", 1984d7a5752cSMel Gorman pgdat->node_id, 1985d7a5752cSMel Gorman zone->name); 1986d7a5752cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 1987d7a5752cSMel Gorman fill_contig_page_info(zone, order, &info); 1988d7a5752cSMel Gorman index = unusable_free_index(order, &info); 1989d7a5752cSMel Gorman seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 1990d7a5752cSMel Gorman } 1991d7a5752cSMel Gorman 1992d7a5752cSMel Gorman seq_putc(m, '\n'); 1993d7a5752cSMel Gorman } 1994d7a5752cSMel Gorman 1995d7a5752cSMel Gorman /* 1996d7a5752cSMel Gorman * Display unusable free space index 1997d7a5752cSMel Gorman * 1998d7a5752cSMel Gorman * The unusable free space index measures how much of the available free 1999d7a5752cSMel Gorman * memory cannot be used to satisfy an allocation of a given size and is a 2000d7a5752cSMel Gorman * value between 0 and 1. The higher the value, the more of free memory is 2001d7a5752cSMel Gorman * unusable and by implication, the worse the external fragmentation is. This 2002d7a5752cSMel Gorman * can be expressed as a percentage by multiplying by 100. 2003d7a5752cSMel Gorman */ 2004d7a5752cSMel Gorman static int unusable_show(struct seq_file *m, void *arg) 2005d7a5752cSMel Gorman { 2006d7a5752cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 2007d7a5752cSMel Gorman 2008d7a5752cSMel Gorman /* check memoryless node */ 2009a47b53c5SLai Jiangshan if (!node_state(pgdat->node_id, N_MEMORY)) 2010d7a5752cSMel Gorman return 0; 2011d7a5752cSMel Gorman 2012727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2013d7a5752cSMel Gorman 2014d7a5752cSMel Gorman return 0; 2015d7a5752cSMel Gorman } 2016d7a5752cSMel Gorman 2017d7a5752cSMel Gorman static const struct seq_operations unusable_op = { 2018d7a5752cSMel Gorman .start = frag_start, 2019d7a5752cSMel Gorman .next = frag_next, 2020d7a5752cSMel Gorman .stop = frag_stop, 2021d7a5752cSMel Gorman .show = unusable_show, 2022d7a5752cSMel Gorman }; 2023d7a5752cSMel Gorman 2024d7a5752cSMel Gorman static int unusable_open(struct inode *inode, struct file *file) 2025d7a5752cSMel Gorman { 2026d7a5752cSMel Gorman return seq_open(file, &unusable_op); 2027d7a5752cSMel Gorman } 2028d7a5752cSMel Gorman 2029d7a5752cSMel Gorman static const struct file_operations unusable_file_ops = { 2030d7a5752cSMel Gorman .open = unusable_open, 2031d7a5752cSMel Gorman .read = seq_read, 2032d7a5752cSMel Gorman .llseek = seq_lseek, 2033d7a5752cSMel Gorman .release = seq_release, 2034d7a5752cSMel Gorman }; 2035d7a5752cSMel Gorman 2036f1a5ab12SMel Gorman static void extfrag_show_print(struct seq_file *m, 2037f1a5ab12SMel Gorman pg_data_t *pgdat, struct zone *zone) 2038f1a5ab12SMel Gorman { 2039f1a5ab12SMel Gorman unsigned int order; 2040f1a5ab12SMel Gorman int index; 2041f1a5ab12SMel Gorman 2042f1a5ab12SMel Gorman /* Alloc on stack as interrupts are disabled for zone walk */ 2043f1a5ab12SMel Gorman struct contig_page_info info; 2044f1a5ab12SMel Gorman 2045f1a5ab12SMel Gorman seq_printf(m, "Node %d, zone %8s ", 2046f1a5ab12SMel Gorman pgdat->node_id, 2047f1a5ab12SMel Gorman zone->name); 2048f1a5ab12SMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 2049f1a5ab12SMel Gorman fill_contig_page_info(zone, order, &info); 205056de7263SMel Gorman index = __fragmentation_index(order, &info); 2051f1a5ab12SMel Gorman seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2052f1a5ab12SMel Gorman } 2053f1a5ab12SMel Gorman 2054f1a5ab12SMel Gorman seq_putc(m, '\n'); 2055f1a5ab12SMel Gorman } 2056f1a5ab12SMel Gorman 2057f1a5ab12SMel Gorman /* 2058f1a5ab12SMel Gorman * Display fragmentation index for orders that allocations would fail for 2059f1a5ab12SMel Gorman */ 2060f1a5ab12SMel Gorman static int extfrag_show(struct seq_file *m, void *arg) 2061f1a5ab12SMel Gorman { 2062f1a5ab12SMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 2063f1a5ab12SMel Gorman 2064727c080fSVinayak Menon walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2065f1a5ab12SMel Gorman 2066f1a5ab12SMel Gorman return 0; 2067f1a5ab12SMel Gorman } 2068f1a5ab12SMel Gorman 2069f1a5ab12SMel Gorman static const struct seq_operations extfrag_op = { 2070f1a5ab12SMel Gorman .start = frag_start, 2071f1a5ab12SMel Gorman .next = frag_next, 2072f1a5ab12SMel Gorman .stop = frag_stop, 2073f1a5ab12SMel Gorman .show = extfrag_show, 2074f1a5ab12SMel Gorman }; 2075f1a5ab12SMel Gorman 2076f1a5ab12SMel Gorman static int extfrag_open(struct inode *inode, struct file *file) 2077f1a5ab12SMel Gorman { 2078f1a5ab12SMel Gorman return seq_open(file, &extfrag_op); 2079f1a5ab12SMel Gorman } 2080f1a5ab12SMel Gorman 2081f1a5ab12SMel Gorman static const struct file_operations extfrag_file_ops = { 2082f1a5ab12SMel Gorman .open = extfrag_open, 2083f1a5ab12SMel Gorman .read = seq_read, 2084f1a5ab12SMel Gorman .llseek = seq_lseek, 2085f1a5ab12SMel Gorman .release = seq_release, 2086f1a5ab12SMel Gorman }; 2087f1a5ab12SMel Gorman 2088d7a5752cSMel Gorman static int __init extfrag_debug_init(void) 2089d7a5752cSMel Gorman { 2090bde8bd8aSSasikantha babu struct dentry *extfrag_debug_root; 2091bde8bd8aSSasikantha babu 2092d7a5752cSMel Gorman extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2093d7a5752cSMel Gorman if (!extfrag_debug_root) 2094d7a5752cSMel Gorman return -ENOMEM; 2095d7a5752cSMel Gorman 2096d7a5752cSMel Gorman if (!debugfs_create_file("unusable_index", 0444, 2097d7a5752cSMel Gorman extfrag_debug_root, NULL, &unusable_file_ops)) 2098bde8bd8aSSasikantha babu goto fail; 2099d7a5752cSMel Gorman 2100f1a5ab12SMel Gorman if (!debugfs_create_file("extfrag_index", 0444, 2101f1a5ab12SMel Gorman extfrag_debug_root, NULL, &extfrag_file_ops)) 2102bde8bd8aSSasikantha babu goto fail; 2103f1a5ab12SMel Gorman 2104d7a5752cSMel Gorman return 0; 2105bde8bd8aSSasikantha babu fail: 2106bde8bd8aSSasikantha babu debugfs_remove_recursive(extfrag_debug_root); 2107bde8bd8aSSasikantha babu return -ENOMEM; 2108d7a5752cSMel Gorman } 2109d7a5752cSMel Gorman 2110d7a5752cSMel Gorman module_init(extfrag_debug_init); 2111d7a5752cSMel Gorman #endif 2112