1f6ac2354SChristoph Lameter /* 2f6ac2354SChristoph Lameter * linux/mm/vmstat.c 3f6ac2354SChristoph Lameter * 4f6ac2354SChristoph Lameter * Manages VM statistics 5f6ac2354SChristoph Lameter * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 62244b95aSChristoph Lameter * 72244b95aSChristoph Lameter * zoned VM statistics 82244b95aSChristoph Lameter * Copyright (C) 2006 Silicon Graphics, Inc., 92244b95aSChristoph Lameter * Christoph Lameter <christoph@lameter.com> 107cc36bbdSChristoph Lameter * Copyright (C) 2008-2014 Christoph Lameter 11f6ac2354SChristoph Lameter */ 128f32f7e5SAlexey Dobriyan #include <linux/fs.h> 13f6ac2354SChristoph Lameter #include <linux/mm.h> 144e950f6fSAlexey Dobriyan #include <linux/err.h> 152244b95aSChristoph Lameter #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 17df9ecabaSChristoph Lameter #include <linux/cpu.h> 187cc36bbdSChristoph Lameter #include <linux/cpumask.h> 19c748e134SAdrian Bunk #include <linux/vmstat.h> 203c486871SAndrew Morton #include <linux/proc_fs.h> 213c486871SAndrew Morton #include <linux/seq_file.h> 223c486871SAndrew Morton #include <linux/debugfs.h> 23e8edc6e0SAlexey Dobriyan #include <linux/sched.h> 24f1a5ab12SMel Gorman #include <linux/math64.h> 2579da826aSMichael Rubin #include <linux/writeback.h> 2636deb0beSNamhyung Kim #include <linux/compaction.h> 276e543d57SLisa Du #include <linux/mm_inline.h> 2848c96a36SJoonsoo Kim #include <linux/page_ext.h> 2948c96a36SJoonsoo Kim #include <linux/page_owner.h> 306e543d57SLisa Du 316e543d57SLisa Du #include "internal.h" 32f6ac2354SChristoph Lameter 33f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 34f8891e5eSChristoph Lameter DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 35f8891e5eSChristoph Lameter EXPORT_PER_CPU_SYMBOL(vm_event_states); 36f8891e5eSChristoph Lameter 3731f961a8SMinchan Kim static void sum_vm_events(unsigned long *ret) 38f8891e5eSChristoph Lameter { 399eccf2a8SChristoph Lameter int cpu; 40f8891e5eSChristoph Lameter int i; 41f8891e5eSChristoph Lameter 42f8891e5eSChristoph Lameter memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 43f8891e5eSChristoph Lameter 4431f961a8SMinchan Kim for_each_online_cpu(cpu) { 45f8891e5eSChristoph Lameter struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 46f8891e5eSChristoph Lameter 47f8891e5eSChristoph Lameter for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 48f8891e5eSChristoph Lameter ret[i] += this->event[i]; 49f8891e5eSChristoph Lameter } 50f8891e5eSChristoph Lameter } 51f8891e5eSChristoph Lameter 52f8891e5eSChristoph Lameter /* 53f8891e5eSChristoph Lameter * Accumulate the vm event counters across all CPUs. 54f8891e5eSChristoph Lameter * The result is unavoidably approximate - it can change 55f8891e5eSChristoph Lameter * during and after execution of this function. 56f8891e5eSChristoph Lameter */ 57f8891e5eSChristoph Lameter void all_vm_events(unsigned long *ret) 58f8891e5eSChristoph Lameter { 59b5be1132SKOSAKI Motohiro get_online_cpus(); 6031f961a8SMinchan Kim sum_vm_events(ret); 61b5be1132SKOSAKI Motohiro put_online_cpus(); 62f8891e5eSChristoph Lameter } 6332dd66fcSHeiko Carstens EXPORT_SYMBOL_GPL(all_vm_events); 64f8891e5eSChristoph Lameter 65f8891e5eSChristoph Lameter /* 66f8891e5eSChristoph Lameter * Fold the foreign cpu events into our own. 67f8891e5eSChristoph Lameter * 68f8891e5eSChristoph Lameter * This is adding to the events on one processor 69f8891e5eSChristoph Lameter * but keeps the global counts constant. 70f8891e5eSChristoph Lameter */ 71f8891e5eSChristoph Lameter void vm_events_fold_cpu(int cpu) 72f8891e5eSChristoph Lameter { 73f8891e5eSChristoph Lameter struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 74f8891e5eSChristoph Lameter int i; 75f8891e5eSChristoph Lameter 76f8891e5eSChristoph Lameter for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 77f8891e5eSChristoph Lameter count_vm_events(i, fold_state->event[i]); 78f8891e5eSChristoph Lameter fold_state->event[i] = 0; 79f8891e5eSChristoph Lameter } 80f8891e5eSChristoph Lameter } 81f8891e5eSChristoph Lameter 82f8891e5eSChristoph Lameter #endif /* CONFIG_VM_EVENT_COUNTERS */ 83f8891e5eSChristoph Lameter 842244b95aSChristoph Lameter /* 852244b95aSChristoph Lameter * Manage combined zone based / global counters 862244b95aSChristoph Lameter * 872244b95aSChristoph Lameter * vm_stat contains the global counters 882244b95aSChristoph Lameter */ 8975ef7184SMel Gorman atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 9075ef7184SMel Gorman atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 9175ef7184SMel Gorman EXPORT_SYMBOL(vm_zone_stat); 9275ef7184SMel Gorman EXPORT_SYMBOL(vm_node_stat); 932244b95aSChristoph Lameter 942244b95aSChristoph Lameter #ifdef CONFIG_SMP 952244b95aSChristoph Lameter 96b44129b3SMel Gorman int calculate_pressure_threshold(struct zone *zone) 9788f5acf8SMel Gorman { 9888f5acf8SMel Gorman int threshold; 9988f5acf8SMel Gorman int watermark_distance; 10088f5acf8SMel Gorman 10188f5acf8SMel Gorman /* 10288f5acf8SMel Gorman * As vmstats are not up to date, there is drift between the estimated 10388f5acf8SMel Gorman * and real values. For high thresholds and a high number of CPUs, it 10488f5acf8SMel Gorman * is possible for the min watermark to be breached while the estimated 10588f5acf8SMel Gorman * value looks fine. The pressure threshold is a reduced value such 10688f5acf8SMel Gorman * that even the maximum amount of drift will not accidentally breach 10788f5acf8SMel Gorman * the min watermark 10888f5acf8SMel Gorman */ 10988f5acf8SMel Gorman watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 11088f5acf8SMel Gorman threshold = max(1, (int)(watermark_distance / num_online_cpus())); 11188f5acf8SMel Gorman 11288f5acf8SMel Gorman /* 11388f5acf8SMel Gorman * Maximum threshold is 125 11488f5acf8SMel Gorman */ 11588f5acf8SMel Gorman threshold = min(125, threshold); 11688f5acf8SMel Gorman 11788f5acf8SMel Gorman return threshold; 11888f5acf8SMel Gorman } 11988f5acf8SMel Gorman 120b44129b3SMel Gorman int calculate_normal_threshold(struct zone *zone) 121df9ecabaSChristoph Lameter { 122df9ecabaSChristoph Lameter int threshold; 123df9ecabaSChristoph Lameter int mem; /* memory in 128 MB units */ 1242244b95aSChristoph Lameter 1252244b95aSChristoph Lameter /* 126df9ecabaSChristoph Lameter * The threshold scales with the number of processors and the amount 127df9ecabaSChristoph Lameter * of memory per zone. More memory means that we can defer updates for 128df9ecabaSChristoph Lameter * longer, more processors could lead to more contention. 129df9ecabaSChristoph Lameter * fls() is used to have a cheap way of logarithmic scaling. 1302244b95aSChristoph Lameter * 131df9ecabaSChristoph Lameter * Some sample thresholds: 132df9ecabaSChristoph Lameter * 133df9ecabaSChristoph Lameter * Threshold Processors (fls) Zonesize fls(mem+1) 134df9ecabaSChristoph Lameter * ------------------------------------------------------------------ 135df9ecabaSChristoph Lameter * 8 1 1 0.9-1 GB 4 136df9ecabaSChristoph Lameter * 16 2 2 0.9-1 GB 4 137df9ecabaSChristoph Lameter * 20 2 2 1-2 GB 5 138df9ecabaSChristoph Lameter * 24 2 2 2-4 GB 6 139df9ecabaSChristoph Lameter * 28 2 2 4-8 GB 7 140df9ecabaSChristoph Lameter * 32 2 2 8-16 GB 8 141df9ecabaSChristoph Lameter * 4 2 2 <128M 1 142df9ecabaSChristoph Lameter * 30 4 3 2-4 GB 5 143df9ecabaSChristoph Lameter * 48 4 3 8-16 GB 8 144df9ecabaSChristoph Lameter * 32 8 4 1-2 GB 4 145df9ecabaSChristoph Lameter * 32 8 4 0.9-1GB 4 146df9ecabaSChristoph Lameter * 10 16 5 <128M 1 147df9ecabaSChristoph Lameter * 40 16 5 900M 4 148df9ecabaSChristoph Lameter * 70 64 7 2-4 GB 5 149df9ecabaSChristoph Lameter * 84 64 7 4-8 GB 6 150df9ecabaSChristoph Lameter * 108 512 9 4-8 GB 6 151df9ecabaSChristoph Lameter * 125 1024 10 8-16 GB 8 152df9ecabaSChristoph Lameter * 125 1024 10 16-32 GB 9 1532244b95aSChristoph Lameter */ 154df9ecabaSChristoph Lameter 155b40da049SJiang Liu mem = zone->managed_pages >> (27 - PAGE_SHIFT); 156df9ecabaSChristoph Lameter 157df9ecabaSChristoph Lameter threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 158df9ecabaSChristoph Lameter 159df9ecabaSChristoph Lameter /* 160df9ecabaSChristoph Lameter * Maximum threshold is 125 161df9ecabaSChristoph Lameter */ 162df9ecabaSChristoph Lameter threshold = min(125, threshold); 163df9ecabaSChristoph Lameter 164df9ecabaSChristoph Lameter return threshold; 165df9ecabaSChristoph Lameter } 166df9ecabaSChristoph Lameter 167df9ecabaSChristoph Lameter /* 168df9ecabaSChristoph Lameter * Refresh the thresholds for each zone. 169df9ecabaSChristoph Lameter */ 170a6cccdc3SKOSAKI Motohiro void refresh_zone_stat_thresholds(void) 1712244b95aSChristoph Lameter { 17275ef7184SMel Gorman struct pglist_data *pgdat; 173df9ecabaSChristoph Lameter struct zone *zone; 174df9ecabaSChristoph Lameter int cpu; 175df9ecabaSChristoph Lameter int threshold; 176df9ecabaSChristoph Lameter 17775ef7184SMel Gorman /* Zero current pgdat thresholds */ 17875ef7184SMel Gorman for_each_online_pgdat(pgdat) { 17975ef7184SMel Gorman for_each_online_cpu(cpu) { 18075ef7184SMel Gorman per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 18175ef7184SMel Gorman } 18275ef7184SMel Gorman } 18375ef7184SMel Gorman 184ee99c71cSKOSAKI Motohiro for_each_populated_zone(zone) { 18575ef7184SMel Gorman struct pglist_data *pgdat = zone->zone_pgdat; 186aa454840SChristoph Lameter unsigned long max_drift, tolerate_drift; 187aa454840SChristoph Lameter 188b44129b3SMel Gorman threshold = calculate_normal_threshold(zone); 189df9ecabaSChristoph Lameter 19075ef7184SMel Gorman for_each_online_cpu(cpu) { 19175ef7184SMel Gorman int pgdat_threshold; 19275ef7184SMel Gorman 19399dcc3e5SChristoph Lameter per_cpu_ptr(zone->pageset, cpu)->stat_threshold 19499dcc3e5SChristoph Lameter = threshold; 195aa454840SChristoph Lameter 19675ef7184SMel Gorman /* Base nodestat threshold on the largest populated zone. */ 19775ef7184SMel Gorman pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 19875ef7184SMel Gorman per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 19975ef7184SMel Gorman = max(threshold, pgdat_threshold); 20075ef7184SMel Gorman } 20175ef7184SMel Gorman 202aa454840SChristoph Lameter /* 203aa454840SChristoph Lameter * Only set percpu_drift_mark if there is a danger that 204aa454840SChristoph Lameter * NR_FREE_PAGES reports the low watermark is ok when in fact 205aa454840SChristoph Lameter * the min watermark could be breached by an allocation 206aa454840SChristoph Lameter */ 207aa454840SChristoph Lameter tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 208aa454840SChristoph Lameter max_drift = num_online_cpus() * threshold; 209aa454840SChristoph Lameter if (max_drift > tolerate_drift) 210aa454840SChristoph Lameter zone->percpu_drift_mark = high_wmark_pages(zone) + 211aa454840SChristoph Lameter max_drift; 212df9ecabaSChristoph Lameter } 2132244b95aSChristoph Lameter } 2142244b95aSChristoph Lameter 215b44129b3SMel Gorman void set_pgdat_percpu_threshold(pg_data_t *pgdat, 216b44129b3SMel Gorman int (*calculate_pressure)(struct zone *)) 21788f5acf8SMel Gorman { 21888f5acf8SMel Gorman struct zone *zone; 21988f5acf8SMel Gorman int cpu; 22088f5acf8SMel Gorman int threshold; 22188f5acf8SMel Gorman int i; 22288f5acf8SMel Gorman 22388f5acf8SMel Gorman for (i = 0; i < pgdat->nr_zones; i++) { 22488f5acf8SMel Gorman zone = &pgdat->node_zones[i]; 22588f5acf8SMel Gorman if (!zone->percpu_drift_mark) 22688f5acf8SMel Gorman continue; 22788f5acf8SMel Gorman 228b44129b3SMel Gorman threshold = (*calculate_pressure)(zone); 229bb0b6dffSMel Gorman for_each_online_cpu(cpu) 23088f5acf8SMel Gorman per_cpu_ptr(zone->pageset, cpu)->stat_threshold 23188f5acf8SMel Gorman = threshold; 23288f5acf8SMel Gorman } 23388f5acf8SMel Gorman } 23488f5acf8SMel Gorman 2352244b95aSChristoph Lameter /* 236bea04b07SJianyu Zhan * For use when we know that interrupts are disabled, 237bea04b07SJianyu Zhan * or when we know that preemption is disabled and that 238bea04b07SJianyu Zhan * particular counter cannot be updated from interrupt context. 2392244b95aSChristoph Lameter */ 2402244b95aSChristoph Lameter void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 2416cdb18adSHeiko Carstens long delta) 2422244b95aSChristoph Lameter { 24312938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 24412938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 2452244b95aSChristoph Lameter long x; 24612938a92SChristoph Lameter long t; 2472244b95aSChristoph Lameter 24812938a92SChristoph Lameter x = delta + __this_cpu_read(*p); 2492244b95aSChristoph Lameter 25012938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 25112938a92SChristoph Lameter 25212938a92SChristoph Lameter if (unlikely(x > t || x < -t)) { 2532244b95aSChristoph Lameter zone_page_state_add(x, zone, item); 2542244b95aSChristoph Lameter x = 0; 2552244b95aSChristoph Lameter } 25612938a92SChristoph Lameter __this_cpu_write(*p, x); 2572244b95aSChristoph Lameter } 2582244b95aSChristoph Lameter EXPORT_SYMBOL(__mod_zone_page_state); 2592244b95aSChristoph Lameter 26075ef7184SMel Gorman void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 26175ef7184SMel Gorman long delta) 26275ef7184SMel Gorman { 26375ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 26475ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 26575ef7184SMel Gorman long x; 26675ef7184SMel Gorman long t; 26775ef7184SMel Gorman 26875ef7184SMel Gorman x = delta + __this_cpu_read(*p); 26975ef7184SMel Gorman 27075ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 27175ef7184SMel Gorman 27275ef7184SMel Gorman if (unlikely(x > t || x < -t)) { 27375ef7184SMel Gorman node_page_state_add(x, pgdat, item); 27475ef7184SMel Gorman x = 0; 27575ef7184SMel Gorman } 27675ef7184SMel Gorman __this_cpu_write(*p, x); 27775ef7184SMel Gorman } 27875ef7184SMel Gorman EXPORT_SYMBOL(__mod_node_page_state); 27975ef7184SMel Gorman 2802244b95aSChristoph Lameter /* 2812244b95aSChristoph Lameter * Optimized increment and decrement functions. 2822244b95aSChristoph Lameter * 2832244b95aSChristoph Lameter * These are only for a single page and therefore can take a struct page * 2842244b95aSChristoph Lameter * argument instead of struct zone *. This allows the inclusion of the code 2852244b95aSChristoph Lameter * generated for page_zone(page) into the optimized functions. 2862244b95aSChristoph Lameter * 2872244b95aSChristoph Lameter * No overflow check is necessary and therefore the differential can be 2882244b95aSChristoph Lameter * incremented or decremented in place which may allow the compilers to 2892244b95aSChristoph Lameter * generate better code. 2902244b95aSChristoph Lameter * The increment or decrement is known and therefore one boundary check can 2912244b95aSChristoph Lameter * be omitted. 2922244b95aSChristoph Lameter * 293df9ecabaSChristoph Lameter * NOTE: These functions are very performance sensitive. Change only 294df9ecabaSChristoph Lameter * with care. 295df9ecabaSChristoph Lameter * 2962244b95aSChristoph Lameter * Some processors have inc/dec instructions that are atomic vs an interrupt. 2972244b95aSChristoph Lameter * However, the code must first determine the differential location in a zone 2982244b95aSChristoph Lameter * based on the processor number and then inc/dec the counter. There is no 2992244b95aSChristoph Lameter * guarantee without disabling preemption that the processor will not change 3002244b95aSChristoph Lameter * in between and therefore the atomicity vs. interrupt cannot be exploited 3012244b95aSChristoph Lameter * in a useful way here. 3022244b95aSChristoph Lameter */ 303c8785385SChristoph Lameter void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 3042244b95aSChristoph Lameter { 30512938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 30612938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 30712938a92SChristoph Lameter s8 v, t; 3082244b95aSChristoph Lameter 309908ee0f1SChristoph Lameter v = __this_cpu_inc_return(*p); 31012938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 31112938a92SChristoph Lameter if (unlikely(v > t)) { 31212938a92SChristoph Lameter s8 overstep = t >> 1; 3132244b95aSChristoph Lameter 31412938a92SChristoph Lameter zone_page_state_add(v + overstep, zone, item); 31512938a92SChristoph Lameter __this_cpu_write(*p, -overstep); 3162244b95aSChristoph Lameter } 3172244b95aSChristoph Lameter } 318ca889e6cSChristoph Lameter 31975ef7184SMel Gorman void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 32075ef7184SMel Gorman { 32175ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 32275ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 32375ef7184SMel Gorman s8 v, t; 32475ef7184SMel Gorman 32575ef7184SMel Gorman v = __this_cpu_inc_return(*p); 32675ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 32775ef7184SMel Gorman if (unlikely(v > t)) { 32875ef7184SMel Gorman s8 overstep = t >> 1; 32975ef7184SMel Gorman 33075ef7184SMel Gorman node_page_state_add(v + overstep, pgdat, item); 33175ef7184SMel Gorman __this_cpu_write(*p, -overstep); 33275ef7184SMel Gorman } 33375ef7184SMel Gorman } 33475ef7184SMel Gorman 335ca889e6cSChristoph Lameter void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 336ca889e6cSChristoph Lameter { 337ca889e6cSChristoph Lameter __inc_zone_state(page_zone(page), item); 338ca889e6cSChristoph Lameter } 3392244b95aSChristoph Lameter EXPORT_SYMBOL(__inc_zone_page_state); 3402244b95aSChristoph Lameter 34175ef7184SMel Gorman void __inc_node_page_state(struct page *page, enum node_stat_item item) 34275ef7184SMel Gorman { 34375ef7184SMel Gorman __inc_node_state(page_pgdat(page), item); 34475ef7184SMel Gorman } 34575ef7184SMel Gorman EXPORT_SYMBOL(__inc_node_page_state); 34675ef7184SMel Gorman 347c8785385SChristoph Lameter void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 3482244b95aSChristoph Lameter { 34912938a92SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 35012938a92SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 35112938a92SChristoph Lameter s8 v, t; 3522244b95aSChristoph Lameter 353908ee0f1SChristoph Lameter v = __this_cpu_dec_return(*p); 35412938a92SChristoph Lameter t = __this_cpu_read(pcp->stat_threshold); 35512938a92SChristoph Lameter if (unlikely(v < - t)) { 35612938a92SChristoph Lameter s8 overstep = t >> 1; 3572244b95aSChristoph Lameter 35812938a92SChristoph Lameter zone_page_state_add(v - overstep, zone, item); 35912938a92SChristoph Lameter __this_cpu_write(*p, overstep); 3602244b95aSChristoph Lameter } 3612244b95aSChristoph Lameter } 362c8785385SChristoph Lameter 36375ef7184SMel Gorman void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 36475ef7184SMel Gorman { 36575ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 36675ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 36775ef7184SMel Gorman s8 v, t; 36875ef7184SMel Gorman 36975ef7184SMel Gorman v = __this_cpu_dec_return(*p); 37075ef7184SMel Gorman t = __this_cpu_read(pcp->stat_threshold); 37175ef7184SMel Gorman if (unlikely(v < - t)) { 37275ef7184SMel Gorman s8 overstep = t >> 1; 37375ef7184SMel Gorman 37475ef7184SMel Gorman node_page_state_add(v - overstep, pgdat, item); 37575ef7184SMel Gorman __this_cpu_write(*p, overstep); 37675ef7184SMel Gorman } 37775ef7184SMel Gorman } 37875ef7184SMel Gorman 379c8785385SChristoph Lameter void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 380c8785385SChristoph Lameter { 381c8785385SChristoph Lameter __dec_zone_state(page_zone(page), item); 382c8785385SChristoph Lameter } 3832244b95aSChristoph Lameter EXPORT_SYMBOL(__dec_zone_page_state); 3842244b95aSChristoph Lameter 38575ef7184SMel Gorman void __dec_node_page_state(struct page *page, enum node_stat_item item) 38675ef7184SMel Gorman { 38775ef7184SMel Gorman __dec_node_state(page_pgdat(page), item); 38875ef7184SMel Gorman } 38975ef7184SMel Gorman EXPORT_SYMBOL(__dec_node_page_state); 39075ef7184SMel Gorman 3914156153cSHeiko Carstens #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 3927c839120SChristoph Lameter /* 3937c839120SChristoph Lameter * If we have cmpxchg_local support then we do not need to incur the overhead 3947c839120SChristoph Lameter * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 3957c839120SChristoph Lameter * 3967c839120SChristoph Lameter * mod_state() modifies the zone counter state through atomic per cpu 3977c839120SChristoph Lameter * operations. 3987c839120SChristoph Lameter * 3997c839120SChristoph Lameter * Overstep mode specifies how overstep should handled: 4007c839120SChristoph Lameter * 0 No overstepping 4017c839120SChristoph Lameter * 1 Overstepping half of threshold 4027c839120SChristoph Lameter * -1 Overstepping minus half of threshold 4037c839120SChristoph Lameter */ 40475ef7184SMel Gorman static inline void mod_zone_state(struct zone *zone, 40575ef7184SMel Gorman enum zone_stat_item item, long delta, int overstep_mode) 4067c839120SChristoph Lameter { 4077c839120SChristoph Lameter struct per_cpu_pageset __percpu *pcp = zone->pageset; 4087c839120SChristoph Lameter s8 __percpu *p = pcp->vm_stat_diff + item; 4097c839120SChristoph Lameter long o, n, t, z; 4107c839120SChristoph Lameter 4117c839120SChristoph Lameter do { 4127c839120SChristoph Lameter z = 0; /* overflow to zone counters */ 4137c839120SChristoph Lameter 4147c839120SChristoph Lameter /* 4157c839120SChristoph Lameter * The fetching of the stat_threshold is racy. We may apply 4167c839120SChristoph Lameter * a counter threshold to the wrong the cpu if we get 417d3bc2367SChristoph Lameter * rescheduled while executing here. However, the next 418d3bc2367SChristoph Lameter * counter update will apply the threshold again and 419d3bc2367SChristoph Lameter * therefore bring the counter under the threshold again. 420d3bc2367SChristoph Lameter * 421d3bc2367SChristoph Lameter * Most of the time the thresholds are the same anyways 422d3bc2367SChristoph Lameter * for all cpus in a zone. 4237c839120SChristoph Lameter */ 4247c839120SChristoph Lameter t = this_cpu_read(pcp->stat_threshold); 4257c839120SChristoph Lameter 4267c839120SChristoph Lameter o = this_cpu_read(*p); 4277c839120SChristoph Lameter n = delta + o; 4287c839120SChristoph Lameter 4297c839120SChristoph Lameter if (n > t || n < -t) { 4307c839120SChristoph Lameter int os = overstep_mode * (t >> 1) ; 4317c839120SChristoph Lameter 4327c839120SChristoph Lameter /* Overflow must be added to zone counters */ 4337c839120SChristoph Lameter z = n + os; 4347c839120SChristoph Lameter n = -os; 4357c839120SChristoph Lameter } 4367c839120SChristoph Lameter } while (this_cpu_cmpxchg(*p, o, n) != o); 4377c839120SChristoph Lameter 4387c839120SChristoph Lameter if (z) 4397c839120SChristoph Lameter zone_page_state_add(z, zone, item); 4407c839120SChristoph Lameter } 4417c839120SChristoph Lameter 4427c839120SChristoph Lameter void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 4436cdb18adSHeiko Carstens long delta) 4447c839120SChristoph Lameter { 44575ef7184SMel Gorman mod_zone_state(zone, item, delta, 0); 4467c839120SChristoph Lameter } 4477c839120SChristoph Lameter EXPORT_SYMBOL(mod_zone_page_state); 4487c839120SChristoph Lameter 4497c839120SChristoph Lameter void inc_zone_page_state(struct page *page, enum zone_stat_item item) 4507c839120SChristoph Lameter { 45175ef7184SMel Gorman mod_zone_state(page_zone(page), item, 1, 1); 4527c839120SChristoph Lameter } 4537c839120SChristoph Lameter EXPORT_SYMBOL(inc_zone_page_state); 4547c839120SChristoph Lameter 4557c839120SChristoph Lameter void dec_zone_page_state(struct page *page, enum zone_stat_item item) 4567c839120SChristoph Lameter { 45775ef7184SMel Gorman mod_zone_state(page_zone(page), item, -1, -1); 4587c839120SChristoph Lameter } 4597c839120SChristoph Lameter EXPORT_SYMBOL(dec_zone_page_state); 46075ef7184SMel Gorman 46175ef7184SMel Gorman static inline void mod_node_state(struct pglist_data *pgdat, 46275ef7184SMel Gorman enum node_stat_item item, int delta, int overstep_mode) 46375ef7184SMel Gorman { 46475ef7184SMel Gorman struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 46575ef7184SMel Gorman s8 __percpu *p = pcp->vm_node_stat_diff + item; 46675ef7184SMel Gorman long o, n, t, z; 46775ef7184SMel Gorman 46875ef7184SMel Gorman do { 46975ef7184SMel Gorman z = 0; /* overflow to node counters */ 47075ef7184SMel Gorman 47175ef7184SMel Gorman /* 47275ef7184SMel Gorman * The fetching of the stat_threshold is racy. We may apply 47375ef7184SMel Gorman * a counter threshold to the wrong the cpu if we get 47475ef7184SMel Gorman * rescheduled while executing here. However, the next 47575ef7184SMel Gorman * counter update will apply the threshold again and 47675ef7184SMel Gorman * therefore bring the counter under the threshold again. 47775ef7184SMel Gorman * 47875ef7184SMel Gorman * Most of the time the thresholds are the same anyways 47975ef7184SMel Gorman * for all cpus in a node. 48075ef7184SMel Gorman */ 48175ef7184SMel Gorman t = this_cpu_read(pcp->stat_threshold); 48275ef7184SMel Gorman 48375ef7184SMel Gorman o = this_cpu_read(*p); 48475ef7184SMel Gorman n = delta + o; 48575ef7184SMel Gorman 48675ef7184SMel Gorman if (n > t || n < -t) { 48775ef7184SMel Gorman int os = overstep_mode * (t >> 1) ; 48875ef7184SMel Gorman 48975ef7184SMel Gorman /* Overflow must be added to node counters */ 49075ef7184SMel Gorman z = n + os; 49175ef7184SMel Gorman n = -os; 49275ef7184SMel Gorman } 49375ef7184SMel Gorman } while (this_cpu_cmpxchg(*p, o, n) != o); 49475ef7184SMel Gorman 49575ef7184SMel Gorman if (z) 49675ef7184SMel Gorman node_page_state_add(z, pgdat, item); 49775ef7184SMel Gorman } 49875ef7184SMel Gorman 49975ef7184SMel Gorman void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 50075ef7184SMel Gorman long delta) 50175ef7184SMel Gorman { 50275ef7184SMel Gorman mod_node_state(pgdat, item, delta, 0); 50375ef7184SMel Gorman } 50475ef7184SMel Gorman EXPORT_SYMBOL(mod_node_page_state); 50575ef7184SMel Gorman 50675ef7184SMel Gorman void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 50775ef7184SMel Gorman { 50875ef7184SMel Gorman mod_node_state(pgdat, item, 1, 1); 50975ef7184SMel Gorman } 51075ef7184SMel Gorman 51175ef7184SMel Gorman void inc_node_page_state(struct page *page, enum node_stat_item item) 51275ef7184SMel Gorman { 51375ef7184SMel Gorman mod_node_state(page_pgdat(page), item, 1, 1); 51475ef7184SMel Gorman } 51575ef7184SMel Gorman EXPORT_SYMBOL(inc_node_page_state); 51675ef7184SMel Gorman 51775ef7184SMel Gorman void dec_node_page_state(struct page *page, enum node_stat_item item) 51875ef7184SMel Gorman { 51975ef7184SMel Gorman mod_node_state(page_pgdat(page), item, -1, -1); 52075ef7184SMel Gorman } 52175ef7184SMel Gorman EXPORT_SYMBOL(dec_node_page_state); 5227c839120SChristoph Lameter #else 5237c839120SChristoph Lameter /* 5247c839120SChristoph Lameter * Use interrupt disable to serialize counter updates 5257c839120SChristoph Lameter */ 5267c839120SChristoph Lameter void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 5276cdb18adSHeiko Carstens long delta) 5287c839120SChristoph Lameter { 5297c839120SChristoph Lameter unsigned long flags; 5307c839120SChristoph Lameter 5317c839120SChristoph Lameter local_irq_save(flags); 5327c839120SChristoph Lameter __mod_zone_page_state(zone, item, delta); 5337c839120SChristoph Lameter local_irq_restore(flags); 5347c839120SChristoph Lameter } 5357c839120SChristoph Lameter EXPORT_SYMBOL(mod_zone_page_state); 5367c839120SChristoph Lameter 5372244b95aSChristoph Lameter void inc_zone_page_state(struct page *page, enum zone_stat_item item) 5382244b95aSChristoph Lameter { 5392244b95aSChristoph Lameter unsigned long flags; 5402244b95aSChristoph Lameter struct zone *zone; 5412244b95aSChristoph Lameter 5422244b95aSChristoph Lameter zone = page_zone(page); 5432244b95aSChristoph Lameter local_irq_save(flags); 544ca889e6cSChristoph Lameter __inc_zone_state(zone, item); 5452244b95aSChristoph Lameter local_irq_restore(flags); 5462244b95aSChristoph Lameter } 5472244b95aSChristoph Lameter EXPORT_SYMBOL(inc_zone_page_state); 5482244b95aSChristoph Lameter 5492244b95aSChristoph Lameter void dec_zone_page_state(struct page *page, enum zone_stat_item item) 5502244b95aSChristoph Lameter { 5512244b95aSChristoph Lameter unsigned long flags; 5522244b95aSChristoph Lameter 5532244b95aSChristoph Lameter local_irq_save(flags); 554a302eb4eSChristoph Lameter __dec_zone_page_state(page, item); 5552244b95aSChristoph Lameter local_irq_restore(flags); 5562244b95aSChristoph Lameter } 5572244b95aSChristoph Lameter EXPORT_SYMBOL(dec_zone_page_state); 5582244b95aSChristoph Lameter 55975ef7184SMel Gorman void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 56075ef7184SMel Gorman { 56175ef7184SMel Gorman unsigned long flags; 56275ef7184SMel Gorman 56375ef7184SMel Gorman local_irq_save(flags); 56475ef7184SMel Gorman __inc_node_state(pgdat, item); 56575ef7184SMel Gorman local_irq_restore(flags); 56675ef7184SMel Gorman } 56775ef7184SMel Gorman EXPORT_SYMBOL(inc_node_state); 56875ef7184SMel Gorman 56975ef7184SMel Gorman void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 57075ef7184SMel Gorman long delta) 57175ef7184SMel Gorman { 57275ef7184SMel Gorman unsigned long flags; 57375ef7184SMel Gorman 57475ef7184SMel Gorman local_irq_save(flags); 57575ef7184SMel Gorman __mod_node_page_state(pgdat, item, delta); 57675ef7184SMel Gorman local_irq_restore(flags); 57775ef7184SMel Gorman } 57875ef7184SMel Gorman EXPORT_SYMBOL(mod_node_page_state); 57975ef7184SMel Gorman 58075ef7184SMel Gorman void inc_node_page_state(struct page *page, enum node_stat_item item) 58175ef7184SMel Gorman { 58275ef7184SMel Gorman unsigned long flags; 58375ef7184SMel Gorman struct pglist_data *pgdat; 58475ef7184SMel Gorman 58575ef7184SMel Gorman pgdat = page_pgdat(page); 58675ef7184SMel Gorman local_irq_save(flags); 58775ef7184SMel Gorman __inc_node_state(pgdat, item); 58875ef7184SMel Gorman local_irq_restore(flags); 58975ef7184SMel Gorman } 59075ef7184SMel Gorman EXPORT_SYMBOL(inc_node_page_state); 59175ef7184SMel Gorman 59275ef7184SMel Gorman void dec_node_page_state(struct page *page, enum node_stat_item item) 59375ef7184SMel Gorman { 59475ef7184SMel Gorman unsigned long flags; 59575ef7184SMel Gorman 59675ef7184SMel Gorman local_irq_save(flags); 59775ef7184SMel Gorman __dec_node_page_state(page, item); 59875ef7184SMel Gorman local_irq_restore(flags); 59975ef7184SMel Gorman } 60075ef7184SMel Gorman EXPORT_SYMBOL(dec_node_page_state); 60175ef7184SMel Gorman #endif 6027cc36bbdSChristoph Lameter 6037cc36bbdSChristoph Lameter /* 6047cc36bbdSChristoph Lameter * Fold a differential into the global counters. 6057cc36bbdSChristoph Lameter * Returns the number of counters updated. 6067cc36bbdSChristoph Lameter */ 60775ef7184SMel Gorman static int fold_diff(int *zone_diff, int *node_diff) 6084edb0748SChristoph Lameter { 6094edb0748SChristoph Lameter int i; 6107cc36bbdSChristoph Lameter int changes = 0; 6114edb0748SChristoph Lameter 6124edb0748SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 61375ef7184SMel Gorman if (zone_diff[i]) { 61475ef7184SMel Gorman atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 61575ef7184SMel Gorman changes++; 61675ef7184SMel Gorman } 61775ef7184SMel Gorman 61875ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 61975ef7184SMel Gorman if (node_diff[i]) { 62075ef7184SMel Gorman atomic_long_add(node_diff[i], &vm_node_stat[i]); 6217cc36bbdSChristoph Lameter changes++; 6227cc36bbdSChristoph Lameter } 6237cc36bbdSChristoph Lameter return changes; 6244edb0748SChristoph Lameter } 6254edb0748SChristoph Lameter 6262244b95aSChristoph Lameter /* 6272bb921e5SChristoph Lameter * Update the zone counters for the current cpu. 628a7f75e25SChristoph Lameter * 6294037d452SChristoph Lameter * Note that refresh_cpu_vm_stats strives to only access 6304037d452SChristoph Lameter * node local memory. The per cpu pagesets on remote zones are placed 6314037d452SChristoph Lameter * in the memory local to the processor using that pageset. So the 6324037d452SChristoph Lameter * loop over all zones will access a series of cachelines local to 6334037d452SChristoph Lameter * the processor. 6344037d452SChristoph Lameter * 6354037d452SChristoph Lameter * The call to zone_page_state_add updates the cachelines with the 6364037d452SChristoph Lameter * statistics in the remote zone struct as well as the global cachelines 6374037d452SChristoph Lameter * with the global counters. These could cause remote node cache line 6384037d452SChristoph Lameter * bouncing and will have to be only done when necessary. 6397cc36bbdSChristoph Lameter * 6407cc36bbdSChristoph Lameter * The function returns the number of global counters updated. 6412244b95aSChristoph Lameter */ 6420eb77e98SChristoph Lameter static int refresh_cpu_vm_stats(bool do_pagesets) 6432244b95aSChristoph Lameter { 64475ef7184SMel Gorman struct pglist_data *pgdat; 6452244b95aSChristoph Lameter struct zone *zone; 6462244b95aSChristoph Lameter int i; 64775ef7184SMel Gorman int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 64875ef7184SMel Gorman int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 6497cc36bbdSChristoph Lameter int changes = 0; 6502244b95aSChristoph Lameter 651ee99c71cSKOSAKI Motohiro for_each_populated_zone(zone) { 652fbc2edb0SChristoph Lameter struct per_cpu_pageset __percpu *p = zone->pageset; 6532244b95aSChristoph Lameter 654fbc2edb0SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 655a7f75e25SChristoph Lameter int v; 656a7f75e25SChristoph Lameter 657fbc2edb0SChristoph Lameter v = this_cpu_xchg(p->vm_stat_diff[i], 0); 658fbc2edb0SChristoph Lameter if (v) { 659fbc2edb0SChristoph Lameter 660a7f75e25SChristoph Lameter atomic_long_add(v, &zone->vm_stat[i]); 66175ef7184SMel Gorman global_zone_diff[i] += v; 6624037d452SChristoph Lameter #ifdef CONFIG_NUMA 6634037d452SChristoph Lameter /* 3 seconds idle till flush */ 664fbc2edb0SChristoph Lameter __this_cpu_write(p->expire, 3); 6654037d452SChristoph Lameter #endif 6662244b95aSChristoph Lameter } 667fbc2edb0SChristoph Lameter } 6684037d452SChristoph Lameter #ifdef CONFIG_NUMA 6690eb77e98SChristoph Lameter if (do_pagesets) { 6700eb77e98SChristoph Lameter cond_resched(); 6714037d452SChristoph Lameter /* 6724037d452SChristoph Lameter * Deal with draining the remote pageset of this 6734037d452SChristoph Lameter * processor 6744037d452SChristoph Lameter * 6754037d452SChristoph Lameter * Check if there are pages remaining in this pageset 6764037d452SChristoph Lameter * if not then there is nothing to expire. 6774037d452SChristoph Lameter */ 678fbc2edb0SChristoph Lameter if (!__this_cpu_read(p->expire) || 679fbc2edb0SChristoph Lameter !__this_cpu_read(p->pcp.count)) 6804037d452SChristoph Lameter continue; 6814037d452SChristoph Lameter 6824037d452SChristoph Lameter /* 6834037d452SChristoph Lameter * We never drain zones local to this processor. 6844037d452SChristoph Lameter */ 6854037d452SChristoph Lameter if (zone_to_nid(zone) == numa_node_id()) { 686fbc2edb0SChristoph Lameter __this_cpu_write(p->expire, 0); 6874037d452SChristoph Lameter continue; 6884037d452SChristoph Lameter } 6894037d452SChristoph Lameter 690fbc2edb0SChristoph Lameter if (__this_cpu_dec_return(p->expire)) 6914037d452SChristoph Lameter continue; 6924037d452SChristoph Lameter 6937cc36bbdSChristoph Lameter if (__this_cpu_read(p->pcp.count)) { 6947c8e0181SChristoph Lameter drain_zone_pages(zone, this_cpu_ptr(&p->pcp)); 6957cc36bbdSChristoph Lameter changes++; 6967cc36bbdSChristoph Lameter } 6970eb77e98SChristoph Lameter } 6984037d452SChristoph Lameter #endif 6992244b95aSChristoph Lameter } 70075ef7184SMel Gorman 70175ef7184SMel Gorman for_each_online_pgdat(pgdat) { 70275ef7184SMel Gorman struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 70375ef7184SMel Gorman 70475ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 70575ef7184SMel Gorman int v; 70675ef7184SMel Gorman 70775ef7184SMel Gorman v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 70875ef7184SMel Gorman if (v) { 70975ef7184SMel Gorman atomic_long_add(v, &pgdat->vm_stat[i]); 71075ef7184SMel Gorman global_node_diff[i] += v; 71175ef7184SMel Gorman } 71275ef7184SMel Gorman } 71375ef7184SMel Gorman } 71475ef7184SMel Gorman 71575ef7184SMel Gorman changes += fold_diff(global_zone_diff, global_node_diff); 7167cc36bbdSChristoph Lameter return changes; 7172244b95aSChristoph Lameter } 7182244b95aSChristoph Lameter 71940f4b1eaSCody P Schafer /* 7202bb921e5SChristoph Lameter * Fold the data for an offline cpu into the global array. 7212bb921e5SChristoph Lameter * There cannot be any access by the offline cpu and therefore 7222bb921e5SChristoph Lameter * synchronization is simplified. 7232bb921e5SChristoph Lameter */ 7242bb921e5SChristoph Lameter void cpu_vm_stats_fold(int cpu) 7252bb921e5SChristoph Lameter { 72675ef7184SMel Gorman struct pglist_data *pgdat; 7272bb921e5SChristoph Lameter struct zone *zone; 7282bb921e5SChristoph Lameter int i; 72975ef7184SMel Gorman int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 73075ef7184SMel Gorman int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 7312bb921e5SChristoph Lameter 7322bb921e5SChristoph Lameter for_each_populated_zone(zone) { 7332bb921e5SChristoph Lameter struct per_cpu_pageset *p; 7342bb921e5SChristoph Lameter 7352bb921e5SChristoph Lameter p = per_cpu_ptr(zone->pageset, cpu); 7362bb921e5SChristoph Lameter 7372bb921e5SChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 7382bb921e5SChristoph Lameter if (p->vm_stat_diff[i]) { 7392bb921e5SChristoph Lameter int v; 7402bb921e5SChristoph Lameter 7412bb921e5SChristoph Lameter v = p->vm_stat_diff[i]; 7422bb921e5SChristoph Lameter p->vm_stat_diff[i] = 0; 7432bb921e5SChristoph Lameter atomic_long_add(v, &zone->vm_stat[i]); 74475ef7184SMel Gorman global_zone_diff[i] += v; 7452bb921e5SChristoph Lameter } 7462bb921e5SChristoph Lameter } 7472bb921e5SChristoph Lameter 74875ef7184SMel Gorman for_each_online_pgdat(pgdat) { 74975ef7184SMel Gorman struct per_cpu_nodestat *p; 75075ef7184SMel Gorman 75175ef7184SMel Gorman p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 75275ef7184SMel Gorman 75375ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 75475ef7184SMel Gorman if (p->vm_node_stat_diff[i]) { 75575ef7184SMel Gorman int v; 75675ef7184SMel Gorman 75775ef7184SMel Gorman v = p->vm_node_stat_diff[i]; 75875ef7184SMel Gorman p->vm_node_stat_diff[i] = 0; 75975ef7184SMel Gorman atomic_long_add(v, &pgdat->vm_stat[i]); 76075ef7184SMel Gorman global_node_diff[i] += v; 76175ef7184SMel Gorman } 76275ef7184SMel Gorman } 76375ef7184SMel Gorman 76475ef7184SMel Gorman fold_diff(global_zone_diff, global_node_diff); 7652bb921e5SChristoph Lameter } 7662bb921e5SChristoph Lameter 7672bb921e5SChristoph Lameter /* 76840f4b1eaSCody P Schafer * this is only called if !populated_zone(zone), which implies no other users of 76940f4b1eaSCody P Schafer * pset->vm_stat_diff[] exsist. 77040f4b1eaSCody P Schafer */ 7715a883813SMinchan Kim void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset) 7725a883813SMinchan Kim { 7735a883813SMinchan Kim int i; 7745a883813SMinchan Kim 7755a883813SMinchan Kim for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 7765a883813SMinchan Kim if (pset->vm_stat_diff[i]) { 7775a883813SMinchan Kim int v = pset->vm_stat_diff[i]; 7785a883813SMinchan Kim pset->vm_stat_diff[i] = 0; 7795a883813SMinchan Kim atomic_long_add(v, &zone->vm_stat[i]); 78075ef7184SMel Gorman atomic_long_add(v, &vm_zone_stat[i]); 7815a883813SMinchan Kim } 7825a883813SMinchan Kim } 7832244b95aSChristoph Lameter #endif 7842244b95aSChristoph Lameter 785ca889e6cSChristoph Lameter #ifdef CONFIG_NUMA 786ca889e6cSChristoph Lameter /* 78775ef7184SMel Gorman * Determine the per node value of a stat item. This function 78875ef7184SMel Gorman * is called frequently in a NUMA machine, so try to be as 78975ef7184SMel Gorman * frugal as possible. 790c2d42c16SAndrew Morton */ 79175ef7184SMel Gorman unsigned long sum_zone_node_page_state(int node, 79275ef7184SMel Gorman enum zone_stat_item item) 793c2d42c16SAndrew Morton { 794c2d42c16SAndrew Morton struct zone *zones = NODE_DATA(node)->node_zones; 795e87d59f7SJoonsoo Kim int i; 796e87d59f7SJoonsoo Kim unsigned long count = 0; 797c2d42c16SAndrew Morton 798e87d59f7SJoonsoo Kim for (i = 0; i < MAX_NR_ZONES; i++) 799e87d59f7SJoonsoo Kim count += zone_page_state(zones + i, item); 800e87d59f7SJoonsoo Kim 801e87d59f7SJoonsoo Kim return count; 802c2d42c16SAndrew Morton } 803c2d42c16SAndrew Morton 80475ef7184SMel Gorman /* 80575ef7184SMel Gorman * Determine the per node value of a stat item. 80675ef7184SMel Gorman */ 80775ef7184SMel Gorman unsigned long node_page_state(struct pglist_data *pgdat, 80875ef7184SMel Gorman enum node_stat_item item) 80975ef7184SMel Gorman { 81075ef7184SMel Gorman long x = atomic_long_read(&pgdat->vm_stat[item]); 81175ef7184SMel Gorman #ifdef CONFIG_SMP 81275ef7184SMel Gorman if (x < 0) 81375ef7184SMel Gorman x = 0; 81475ef7184SMel Gorman #endif 81575ef7184SMel Gorman return x; 81675ef7184SMel Gorman } 817ca889e6cSChristoph Lameter #endif 818ca889e6cSChristoph Lameter 819d7a5752cSMel Gorman #ifdef CONFIG_COMPACTION 82036deb0beSNamhyung Kim 821d7a5752cSMel Gorman struct contig_page_info { 822d7a5752cSMel Gorman unsigned long free_pages; 823d7a5752cSMel Gorman unsigned long free_blocks_total; 824d7a5752cSMel Gorman unsigned long free_blocks_suitable; 825d7a5752cSMel Gorman }; 826d7a5752cSMel Gorman 827d7a5752cSMel Gorman /* 828d7a5752cSMel Gorman * Calculate the number of free pages in a zone, how many contiguous 829d7a5752cSMel Gorman * pages are free and how many are large enough to satisfy an allocation of 830d7a5752cSMel Gorman * the target size. Note that this function makes no attempt to estimate 831d7a5752cSMel Gorman * how many suitable free blocks there *might* be if MOVABLE pages were 832d7a5752cSMel Gorman * migrated. Calculating that is possible, but expensive and can be 833d7a5752cSMel Gorman * figured out from userspace 834d7a5752cSMel Gorman */ 835d7a5752cSMel Gorman static void fill_contig_page_info(struct zone *zone, 836d7a5752cSMel Gorman unsigned int suitable_order, 837d7a5752cSMel Gorman struct contig_page_info *info) 838d7a5752cSMel Gorman { 839d7a5752cSMel Gorman unsigned int order; 840d7a5752cSMel Gorman 841d7a5752cSMel Gorman info->free_pages = 0; 842d7a5752cSMel Gorman info->free_blocks_total = 0; 843d7a5752cSMel Gorman info->free_blocks_suitable = 0; 844d7a5752cSMel Gorman 845d7a5752cSMel Gorman for (order = 0; order < MAX_ORDER; order++) { 846d7a5752cSMel Gorman unsigned long blocks; 847d7a5752cSMel Gorman 848d7a5752cSMel Gorman /* Count number of free blocks */ 849d7a5752cSMel Gorman blocks = zone->free_area[order].nr_free; 850d7a5752cSMel Gorman info->free_blocks_total += blocks; 851d7a5752cSMel Gorman 852d7a5752cSMel Gorman /* Count free base pages */ 853d7a5752cSMel Gorman info->free_pages += blocks << order; 854d7a5752cSMel Gorman 855d7a5752cSMel Gorman /* Count the suitable free blocks */ 856d7a5752cSMel Gorman if (order >= suitable_order) 857d7a5752cSMel Gorman info->free_blocks_suitable += blocks << 858d7a5752cSMel Gorman (order - suitable_order); 859d7a5752cSMel Gorman } 860d7a5752cSMel Gorman } 861f1a5ab12SMel Gorman 862f1a5ab12SMel Gorman /* 863f1a5ab12SMel Gorman * A fragmentation index only makes sense if an allocation of a requested 864f1a5ab12SMel Gorman * size would fail. If that is true, the fragmentation index indicates 865f1a5ab12SMel Gorman * whether external fragmentation or a lack of memory was the problem. 866f1a5ab12SMel Gorman * The value can be used to determine if page reclaim or compaction 867f1a5ab12SMel Gorman * should be used 868f1a5ab12SMel Gorman */ 86956de7263SMel Gorman static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 870f1a5ab12SMel Gorman { 871f1a5ab12SMel Gorman unsigned long requested = 1UL << order; 872f1a5ab12SMel Gorman 873f1a5ab12SMel Gorman if (!info->free_blocks_total) 874f1a5ab12SMel Gorman return 0; 875f1a5ab12SMel Gorman 876f1a5ab12SMel Gorman /* Fragmentation index only makes sense when a request would fail */ 877f1a5ab12SMel Gorman if (info->free_blocks_suitable) 878f1a5ab12SMel Gorman return -1000; 879f1a5ab12SMel Gorman 880f1a5ab12SMel Gorman /* 881f1a5ab12SMel Gorman * Index is between 0 and 1 so return within 3 decimal places 882f1a5ab12SMel Gorman * 883f1a5ab12SMel Gorman * 0 => allocation would fail due to lack of memory 884f1a5ab12SMel Gorman * 1 => allocation would fail due to fragmentation 885f1a5ab12SMel Gorman */ 886f1a5ab12SMel Gorman return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 887f1a5ab12SMel Gorman } 88856de7263SMel Gorman 88956de7263SMel Gorman /* Same as __fragmentation index but allocs contig_page_info on stack */ 89056de7263SMel Gorman int fragmentation_index(struct zone *zone, unsigned int order) 89156de7263SMel Gorman { 89256de7263SMel Gorman struct contig_page_info info; 89356de7263SMel Gorman 89456de7263SMel Gorman fill_contig_page_info(zone, order, &info); 89556de7263SMel Gorman return __fragmentation_index(order, &info); 89656de7263SMel Gorman } 897d7a5752cSMel Gorman #endif 898d7a5752cSMel Gorman 8990d6617c7SDavid Rientjes #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA) 900fa25c503SKOSAKI Motohiro #ifdef CONFIG_ZONE_DMA 901fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA(xx) xx "_dma", 902fa25c503SKOSAKI Motohiro #else 903fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA(xx) 904fa25c503SKOSAKI Motohiro #endif 905fa25c503SKOSAKI Motohiro 906fa25c503SKOSAKI Motohiro #ifdef CONFIG_ZONE_DMA32 907fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA32(xx) xx "_dma32", 908fa25c503SKOSAKI Motohiro #else 909fa25c503SKOSAKI Motohiro #define TEXT_FOR_DMA32(xx) 910fa25c503SKOSAKI Motohiro #endif 911fa25c503SKOSAKI Motohiro 912fa25c503SKOSAKI Motohiro #ifdef CONFIG_HIGHMEM 913fa25c503SKOSAKI Motohiro #define TEXT_FOR_HIGHMEM(xx) xx "_high", 914fa25c503SKOSAKI Motohiro #else 915fa25c503SKOSAKI Motohiro #define TEXT_FOR_HIGHMEM(xx) 916fa25c503SKOSAKI Motohiro #endif 917fa25c503SKOSAKI Motohiro 918fa25c503SKOSAKI Motohiro #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ 919fa25c503SKOSAKI Motohiro TEXT_FOR_HIGHMEM(xx) xx "_movable", 920fa25c503SKOSAKI Motohiro 921fa25c503SKOSAKI Motohiro const char * const vmstat_text[] = { 92209316c09SKonstantin Khlebnikov /* enum zone_stat_item countes */ 923fa25c503SKOSAKI Motohiro "nr_free_pages", 92471c799f4SMinchan Kim "nr_zone_inactive_anon", 92571c799f4SMinchan Kim "nr_zone_active_anon", 92671c799f4SMinchan Kim "nr_zone_inactive_file", 92771c799f4SMinchan Kim "nr_zone_active_file", 92871c799f4SMinchan Kim "nr_zone_unevictable", 9295a1c84b4SMel Gorman "nr_zone_write_pending", 930fa25c503SKOSAKI Motohiro "nr_mlock", 931fa25c503SKOSAKI Motohiro "nr_slab_reclaimable", 932fa25c503SKOSAKI Motohiro "nr_slab_unreclaimable", 933fa25c503SKOSAKI Motohiro "nr_page_table_pages", 934fa25c503SKOSAKI Motohiro "nr_kernel_stack", 935fa25c503SKOSAKI Motohiro "nr_bounce", 93691537feeSMinchan Kim #if IS_ENABLED(CONFIG_ZSMALLOC) 93791537feeSMinchan Kim "nr_zspages", 93891537feeSMinchan Kim #endif 939fa25c503SKOSAKI Motohiro #ifdef CONFIG_NUMA 940fa25c503SKOSAKI Motohiro "numa_hit", 941fa25c503SKOSAKI Motohiro "numa_miss", 942fa25c503SKOSAKI Motohiro "numa_foreign", 943fa25c503SKOSAKI Motohiro "numa_interleave", 944fa25c503SKOSAKI Motohiro "numa_local", 945fa25c503SKOSAKI Motohiro "numa_other", 946fa25c503SKOSAKI Motohiro #endif 947d1ce749aSBartlomiej Zolnierkiewicz "nr_free_cma", 94809316c09SKonstantin Khlebnikov 949599d0c95SMel Gorman /* Node-based counters */ 950599d0c95SMel Gorman "nr_inactive_anon", 951599d0c95SMel Gorman "nr_active_anon", 952599d0c95SMel Gorman "nr_inactive_file", 953599d0c95SMel Gorman "nr_active_file", 954599d0c95SMel Gorman "nr_unevictable", 955599d0c95SMel Gorman "nr_isolated_anon", 956599d0c95SMel Gorman "nr_isolated_file", 957599d0c95SMel Gorman "nr_pages_scanned", 9581e6b1085SMel Gorman "workingset_refault", 9591e6b1085SMel Gorman "workingset_activate", 9601e6b1085SMel Gorman "workingset_nodereclaim", 96150658e2eSMel Gorman "nr_anon_pages", 96250658e2eSMel Gorman "nr_mapped", 96311fb9989SMel Gorman "nr_file_pages", 96411fb9989SMel Gorman "nr_dirty", 96511fb9989SMel Gorman "nr_writeback", 96611fb9989SMel Gorman "nr_writeback_temp", 96711fb9989SMel Gorman "nr_shmem", 96811fb9989SMel Gorman "nr_shmem_hugepages", 96911fb9989SMel Gorman "nr_shmem_pmdmapped", 97011fb9989SMel Gorman "nr_anon_transparent_hugepages", 97111fb9989SMel Gorman "nr_unstable", 972c4a25635SMel Gorman "nr_vmscan_write", 973c4a25635SMel Gorman "nr_vmscan_immediate_reclaim", 974c4a25635SMel Gorman "nr_dirtied", 975c4a25635SMel Gorman "nr_written", 976599d0c95SMel Gorman 97709316c09SKonstantin Khlebnikov /* enum writeback_stat_item counters */ 978fa25c503SKOSAKI Motohiro "nr_dirty_threshold", 979fa25c503SKOSAKI Motohiro "nr_dirty_background_threshold", 980fa25c503SKOSAKI Motohiro 981fa25c503SKOSAKI Motohiro #ifdef CONFIG_VM_EVENT_COUNTERS 98209316c09SKonstantin Khlebnikov /* enum vm_event_item counters */ 983fa25c503SKOSAKI Motohiro "pgpgin", 984fa25c503SKOSAKI Motohiro "pgpgout", 985fa25c503SKOSAKI Motohiro "pswpin", 986fa25c503SKOSAKI Motohiro "pswpout", 987fa25c503SKOSAKI Motohiro 988fa25c503SKOSAKI Motohiro TEXTS_FOR_ZONES("pgalloc") 9897cc30fcfSMel Gorman TEXTS_FOR_ZONES("allocstall") 9907cc30fcfSMel Gorman TEXTS_FOR_ZONES("pgskip") 991fa25c503SKOSAKI Motohiro 992fa25c503SKOSAKI Motohiro "pgfree", 993fa25c503SKOSAKI Motohiro "pgactivate", 994fa25c503SKOSAKI Motohiro "pgdeactivate", 995fa25c503SKOSAKI Motohiro 996fa25c503SKOSAKI Motohiro "pgfault", 997fa25c503SKOSAKI Motohiro "pgmajfault", 998854e9ed0SMinchan Kim "pglazyfreed", 999fa25c503SKOSAKI Motohiro 1000599d0c95SMel Gorman "pgrefill", 1001599d0c95SMel Gorman "pgsteal_kswapd", 1002599d0c95SMel Gorman "pgsteal_direct", 1003599d0c95SMel Gorman "pgscan_kswapd", 1004599d0c95SMel Gorman "pgscan_direct", 100568243e76SMel Gorman "pgscan_direct_throttle", 1006fa25c503SKOSAKI Motohiro 1007fa25c503SKOSAKI Motohiro #ifdef CONFIG_NUMA 1008fa25c503SKOSAKI Motohiro "zone_reclaim_failed", 1009fa25c503SKOSAKI Motohiro #endif 1010fa25c503SKOSAKI Motohiro "pginodesteal", 1011fa25c503SKOSAKI Motohiro "slabs_scanned", 1012fa25c503SKOSAKI Motohiro "kswapd_inodesteal", 1013fa25c503SKOSAKI Motohiro "kswapd_low_wmark_hit_quickly", 1014fa25c503SKOSAKI Motohiro "kswapd_high_wmark_hit_quickly", 1015fa25c503SKOSAKI Motohiro "pageoutrun", 1016fa25c503SKOSAKI Motohiro 1017fa25c503SKOSAKI Motohiro "pgrotated", 1018fa25c503SKOSAKI Motohiro 10195509a5d2SDave Hansen "drop_pagecache", 10205509a5d2SDave Hansen "drop_slab", 10215509a5d2SDave Hansen 102203c5a6e1SMel Gorman #ifdef CONFIG_NUMA_BALANCING 102303c5a6e1SMel Gorman "numa_pte_updates", 102472403b4aSMel Gorman "numa_huge_pte_updates", 102503c5a6e1SMel Gorman "numa_hint_faults", 102603c5a6e1SMel Gorman "numa_hint_faults_local", 102703c5a6e1SMel Gorman "numa_pages_migrated", 102803c5a6e1SMel Gorman #endif 10295647bc29SMel Gorman #ifdef CONFIG_MIGRATION 10305647bc29SMel Gorman "pgmigrate_success", 10315647bc29SMel Gorman "pgmigrate_fail", 10325647bc29SMel Gorman #endif 1033fa25c503SKOSAKI Motohiro #ifdef CONFIG_COMPACTION 1034397487dbSMel Gorman "compact_migrate_scanned", 1035397487dbSMel Gorman "compact_free_scanned", 1036397487dbSMel Gorman "compact_isolated", 1037fa25c503SKOSAKI Motohiro "compact_stall", 1038fa25c503SKOSAKI Motohiro "compact_fail", 1039fa25c503SKOSAKI Motohiro "compact_success", 1040698b1b30SVlastimil Babka "compact_daemon_wake", 10417f354a54SDavid Rientjes "compact_daemon_migrate_scanned", 10427f354a54SDavid Rientjes "compact_daemon_free_scanned", 1043fa25c503SKOSAKI Motohiro #endif 1044fa25c503SKOSAKI Motohiro 1045fa25c503SKOSAKI Motohiro #ifdef CONFIG_HUGETLB_PAGE 1046fa25c503SKOSAKI Motohiro "htlb_buddy_alloc_success", 1047fa25c503SKOSAKI Motohiro "htlb_buddy_alloc_fail", 1048fa25c503SKOSAKI Motohiro #endif 1049fa25c503SKOSAKI Motohiro "unevictable_pgs_culled", 1050fa25c503SKOSAKI Motohiro "unevictable_pgs_scanned", 1051fa25c503SKOSAKI Motohiro "unevictable_pgs_rescued", 1052fa25c503SKOSAKI Motohiro "unevictable_pgs_mlocked", 1053fa25c503SKOSAKI Motohiro "unevictable_pgs_munlocked", 1054fa25c503SKOSAKI Motohiro "unevictable_pgs_cleared", 1055fa25c503SKOSAKI Motohiro "unevictable_pgs_stranded", 1056fa25c503SKOSAKI Motohiro 1057fa25c503SKOSAKI Motohiro #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1058fa25c503SKOSAKI Motohiro "thp_fault_alloc", 1059fa25c503SKOSAKI Motohiro "thp_fault_fallback", 1060fa25c503SKOSAKI Motohiro "thp_collapse_alloc", 1061fa25c503SKOSAKI Motohiro "thp_collapse_alloc_failed", 106295ecedcdSKirill A. Shutemov "thp_file_alloc", 106395ecedcdSKirill A. Shutemov "thp_file_mapped", 1064122afea9SKirill A. Shutemov "thp_split_page", 1065122afea9SKirill A. Shutemov "thp_split_page_failed", 1066f9719a03SKirill A. Shutemov "thp_deferred_split_page", 1067122afea9SKirill A. Shutemov "thp_split_pmd", 1068ce9311cfSYisheng Xie #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1069ce9311cfSYisheng Xie "thp_split_pud", 1070ce9311cfSYisheng Xie #endif 1071d8a8e1f0SKirill A. Shutemov "thp_zero_page_alloc", 1072d8a8e1f0SKirill A. Shutemov "thp_zero_page_alloc_failed", 1073fa25c503SKOSAKI Motohiro #endif 107409316c09SKonstantin Khlebnikov #ifdef CONFIG_MEMORY_BALLOON 107509316c09SKonstantin Khlebnikov "balloon_inflate", 107609316c09SKonstantin Khlebnikov "balloon_deflate", 107709316c09SKonstantin Khlebnikov #ifdef CONFIG_BALLOON_COMPACTION 107809316c09SKonstantin Khlebnikov "balloon_migrate", 107909316c09SKonstantin Khlebnikov #endif 108009316c09SKonstantin Khlebnikov #endif /* CONFIG_MEMORY_BALLOON */ 1081ec659934SMel Gorman #ifdef CONFIG_DEBUG_TLBFLUSH 10826df46865SDave Hansen #ifdef CONFIG_SMP 10839824cf97SDave Hansen "nr_tlb_remote_flush", 10849824cf97SDave Hansen "nr_tlb_remote_flush_received", 1085ec659934SMel Gorman #endif /* CONFIG_SMP */ 10869824cf97SDave Hansen "nr_tlb_local_flush_all", 10879824cf97SDave Hansen "nr_tlb_local_flush_one", 1088ec659934SMel Gorman #endif /* CONFIG_DEBUG_TLBFLUSH */ 1089fa25c503SKOSAKI Motohiro 10904f115147SDavidlohr Bueso #ifdef CONFIG_DEBUG_VM_VMACACHE 10914f115147SDavidlohr Bueso "vmacache_find_calls", 10924f115147SDavidlohr Bueso "vmacache_find_hits", 1093f5f302e2SDavidlohr Bueso "vmacache_full_flushes", 10944f115147SDavidlohr Bueso #endif 1095fa25c503SKOSAKI Motohiro #endif /* CONFIG_VM_EVENTS_COUNTERS */ 1096fa25c503SKOSAKI Motohiro }; 10970d6617c7SDavid Rientjes #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */ 1098fa25c503SKOSAKI Motohiro 1099fa25c503SKOSAKI Motohiro 11003c486871SAndrew Morton #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 11013c486871SAndrew Morton defined(CONFIG_PROC_FS) 11023c486871SAndrew Morton static void *frag_start(struct seq_file *m, loff_t *pos) 11033c486871SAndrew Morton { 11043c486871SAndrew Morton pg_data_t *pgdat; 11053c486871SAndrew Morton loff_t node = *pos; 11063c486871SAndrew Morton 11073c486871SAndrew Morton for (pgdat = first_online_pgdat(); 11083c486871SAndrew Morton pgdat && node; 11093c486871SAndrew Morton pgdat = next_online_pgdat(pgdat)) 11103c486871SAndrew Morton --node; 11113c486871SAndrew Morton 11123c486871SAndrew Morton return pgdat; 11133c486871SAndrew Morton } 11143c486871SAndrew Morton 11153c486871SAndrew Morton static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 11163c486871SAndrew Morton { 11173c486871SAndrew Morton pg_data_t *pgdat = (pg_data_t *)arg; 11183c486871SAndrew Morton 11193c486871SAndrew Morton (*pos)++; 11203c486871SAndrew Morton return next_online_pgdat(pgdat); 11213c486871SAndrew Morton } 11223c486871SAndrew Morton 11233c486871SAndrew Morton static void frag_stop(struct seq_file *m, void *arg) 11243c486871SAndrew Morton { 11253c486871SAndrew Morton } 11263c486871SAndrew Morton 11273c486871SAndrew Morton /* Walk all the zones in a node and print using a callback */ 11283c486871SAndrew Morton static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 11293c486871SAndrew Morton void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 11303c486871SAndrew Morton { 11313c486871SAndrew Morton struct zone *zone; 11323c486871SAndrew Morton struct zone *node_zones = pgdat->node_zones; 11333c486871SAndrew Morton unsigned long flags; 11343c486871SAndrew Morton 11353c486871SAndrew Morton for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 11363c486871SAndrew Morton if (!populated_zone(zone)) 11373c486871SAndrew Morton continue; 11383c486871SAndrew Morton 11393c486871SAndrew Morton spin_lock_irqsave(&zone->lock, flags); 11403c486871SAndrew Morton print(m, pgdat, zone); 11413c486871SAndrew Morton spin_unlock_irqrestore(&zone->lock, flags); 11423c486871SAndrew Morton } 11433c486871SAndrew Morton } 11443c486871SAndrew Morton #endif 11453c486871SAndrew Morton 1146d7a5752cSMel Gorman #ifdef CONFIG_PROC_FS 1147467c996cSMel Gorman static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1148467c996cSMel Gorman struct zone *zone) 1149467c996cSMel Gorman { 1150467c996cSMel Gorman int order; 1151467c996cSMel Gorman 1152f6ac2354SChristoph Lameter seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1153f6ac2354SChristoph Lameter for (order = 0; order < MAX_ORDER; ++order) 1154f6ac2354SChristoph Lameter seq_printf(m, "%6lu ", zone->free_area[order].nr_free); 1155f6ac2354SChristoph Lameter seq_putc(m, '\n'); 1156f6ac2354SChristoph Lameter } 1157467c996cSMel Gorman 1158467c996cSMel Gorman /* 1159467c996cSMel Gorman * This walks the free areas for each zone. 1160467c996cSMel Gorman */ 1161467c996cSMel Gorman static int frag_show(struct seq_file *m, void *arg) 1162467c996cSMel Gorman { 1163467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1164467c996cSMel Gorman walk_zones_in_node(m, pgdat, frag_show_print); 1165467c996cSMel Gorman return 0; 1166467c996cSMel Gorman } 1167467c996cSMel Gorman 1168467c996cSMel Gorman static void pagetypeinfo_showfree_print(struct seq_file *m, 1169467c996cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1170467c996cSMel Gorman { 1171467c996cSMel Gorman int order, mtype; 1172467c996cSMel Gorman 1173467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1174467c996cSMel Gorman seq_printf(m, "Node %4d, zone %8s, type %12s ", 1175467c996cSMel Gorman pgdat->node_id, 1176467c996cSMel Gorman zone->name, 1177467c996cSMel Gorman migratetype_names[mtype]); 1178467c996cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 1179467c996cSMel Gorman unsigned long freecount = 0; 1180467c996cSMel Gorman struct free_area *area; 1181467c996cSMel Gorman struct list_head *curr; 1182467c996cSMel Gorman 1183467c996cSMel Gorman area = &(zone->free_area[order]); 1184467c996cSMel Gorman 1185467c996cSMel Gorman list_for_each(curr, &area->free_list[mtype]) 1186467c996cSMel Gorman freecount++; 1187467c996cSMel Gorman seq_printf(m, "%6lu ", freecount); 1188467c996cSMel Gorman } 1189467c996cSMel Gorman seq_putc(m, '\n'); 1190467c996cSMel Gorman } 1191467c996cSMel Gorman } 1192467c996cSMel Gorman 1193467c996cSMel Gorman /* Print out the free pages at each order for each migatetype */ 1194467c996cSMel Gorman static int pagetypeinfo_showfree(struct seq_file *m, void *arg) 1195467c996cSMel Gorman { 1196467c996cSMel Gorman int order; 1197467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1198467c996cSMel Gorman 1199467c996cSMel Gorman /* Print header */ 1200467c996cSMel Gorman seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1201467c996cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) 1202467c996cSMel Gorman seq_printf(m, "%6d ", order); 1203467c996cSMel Gorman seq_putc(m, '\n'); 1204467c996cSMel Gorman 1205467c996cSMel Gorman walk_zones_in_node(m, pgdat, pagetypeinfo_showfree_print); 1206467c996cSMel Gorman 1207467c996cSMel Gorman return 0; 1208467c996cSMel Gorman } 1209467c996cSMel Gorman 1210467c996cSMel Gorman static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1211467c996cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1212467c996cSMel Gorman { 1213467c996cSMel Gorman int mtype; 1214467c996cSMel Gorman unsigned long pfn; 1215467c996cSMel Gorman unsigned long start_pfn = zone->zone_start_pfn; 1216108bcc96SCody P Schafer unsigned long end_pfn = zone_end_pfn(zone); 1217467c996cSMel Gorman unsigned long count[MIGRATE_TYPES] = { 0, }; 1218467c996cSMel Gorman 1219467c996cSMel Gorman for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1220467c996cSMel Gorman struct page *page; 1221467c996cSMel Gorman 1222467c996cSMel Gorman if (!pfn_valid(pfn)) 1223467c996cSMel Gorman continue; 1224467c996cSMel Gorman 1225467c996cSMel Gorman page = pfn_to_page(pfn); 1226eb33575cSMel Gorman 1227eb33575cSMel Gorman /* Watch for unexpected holes punched in the memmap */ 1228eb33575cSMel Gorman if (!memmap_valid_within(pfn, page, zone)) 1229e80d6a24SMel Gorman continue; 1230eb33575cSMel Gorman 1231a91c43c7SJoonsoo Kim if (page_zone(page) != zone) 1232a91c43c7SJoonsoo Kim continue; 1233a91c43c7SJoonsoo Kim 1234467c996cSMel Gorman mtype = get_pageblock_migratetype(page); 1235467c996cSMel Gorman 1236e80d6a24SMel Gorman if (mtype < MIGRATE_TYPES) 1237467c996cSMel Gorman count[mtype]++; 1238467c996cSMel Gorman } 1239467c996cSMel Gorman 1240467c996cSMel Gorman /* Print counts */ 1241467c996cSMel Gorman seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1242467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1243467c996cSMel Gorman seq_printf(m, "%12lu ", count[mtype]); 1244467c996cSMel Gorman seq_putc(m, '\n'); 1245467c996cSMel Gorman } 1246467c996cSMel Gorman 1247467c996cSMel Gorman /* Print out the free pages at each order for each migratetype */ 1248467c996cSMel Gorman static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1249467c996cSMel Gorman { 1250467c996cSMel Gorman int mtype; 1251467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1252467c996cSMel Gorman 1253467c996cSMel Gorman seq_printf(m, "\n%-23s", "Number of blocks type "); 1254467c996cSMel Gorman for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1255467c996cSMel Gorman seq_printf(m, "%12s ", migratetype_names[mtype]); 1256467c996cSMel Gorman seq_putc(m, '\n'); 1257467c996cSMel Gorman walk_zones_in_node(m, pgdat, pagetypeinfo_showblockcount_print); 1258467c996cSMel Gorman 1259467c996cSMel Gorman return 0; 1260467c996cSMel Gorman } 1261467c996cSMel Gorman 126248c96a36SJoonsoo Kim /* 126348c96a36SJoonsoo Kim * Print out the number of pageblocks for each migratetype that contain pages 126448c96a36SJoonsoo Kim * of other types. This gives an indication of how well fallbacks are being 126548c96a36SJoonsoo Kim * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 126648c96a36SJoonsoo Kim * to determine what is going on 126748c96a36SJoonsoo Kim */ 126848c96a36SJoonsoo Kim static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 126948c96a36SJoonsoo Kim { 127048c96a36SJoonsoo Kim #ifdef CONFIG_PAGE_OWNER 127148c96a36SJoonsoo Kim int mtype; 127248c96a36SJoonsoo Kim 12737dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 127448c96a36SJoonsoo Kim return; 127548c96a36SJoonsoo Kim 127648c96a36SJoonsoo Kim drain_all_pages(NULL); 127748c96a36SJoonsoo Kim 127848c96a36SJoonsoo Kim seq_printf(m, "\n%-23s", "Number of mixed blocks "); 127948c96a36SJoonsoo Kim for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 128048c96a36SJoonsoo Kim seq_printf(m, "%12s ", migratetype_names[mtype]); 128148c96a36SJoonsoo Kim seq_putc(m, '\n'); 128248c96a36SJoonsoo Kim 128348c96a36SJoonsoo Kim walk_zones_in_node(m, pgdat, pagetypeinfo_showmixedcount_print); 128448c96a36SJoonsoo Kim #endif /* CONFIG_PAGE_OWNER */ 128548c96a36SJoonsoo Kim } 128648c96a36SJoonsoo Kim 1287467c996cSMel Gorman /* 1288467c996cSMel Gorman * This prints out statistics in relation to grouping pages by mobility. 1289467c996cSMel Gorman * It is expensive to collect so do not constantly read the file. 1290467c996cSMel Gorman */ 1291467c996cSMel Gorman static int pagetypeinfo_show(struct seq_file *m, void *arg) 1292467c996cSMel Gorman { 1293467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1294467c996cSMel Gorman 129541b25a37SKOSAKI Motohiro /* check memoryless node */ 1296a47b53c5SLai Jiangshan if (!node_state(pgdat->node_id, N_MEMORY)) 129741b25a37SKOSAKI Motohiro return 0; 129841b25a37SKOSAKI Motohiro 1299467c996cSMel Gorman seq_printf(m, "Page block order: %d\n", pageblock_order); 1300467c996cSMel Gorman seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1301467c996cSMel Gorman seq_putc(m, '\n'); 1302467c996cSMel Gorman pagetypeinfo_showfree(m, pgdat); 1303467c996cSMel Gorman pagetypeinfo_showblockcount(m, pgdat); 130448c96a36SJoonsoo Kim pagetypeinfo_showmixedcount(m, pgdat); 1305467c996cSMel Gorman 1306f6ac2354SChristoph Lameter return 0; 1307f6ac2354SChristoph Lameter } 1308f6ac2354SChristoph Lameter 13098f32f7e5SAlexey Dobriyan static const struct seq_operations fragmentation_op = { 1310f6ac2354SChristoph Lameter .start = frag_start, 1311f6ac2354SChristoph Lameter .next = frag_next, 1312f6ac2354SChristoph Lameter .stop = frag_stop, 1313f6ac2354SChristoph Lameter .show = frag_show, 1314f6ac2354SChristoph Lameter }; 1315f6ac2354SChristoph Lameter 13168f32f7e5SAlexey Dobriyan static int fragmentation_open(struct inode *inode, struct file *file) 13178f32f7e5SAlexey Dobriyan { 13188f32f7e5SAlexey Dobriyan return seq_open(file, &fragmentation_op); 13198f32f7e5SAlexey Dobriyan } 13208f32f7e5SAlexey Dobriyan 13218f32f7e5SAlexey Dobriyan static const struct file_operations fragmentation_file_operations = { 13228f32f7e5SAlexey Dobriyan .open = fragmentation_open, 13238f32f7e5SAlexey Dobriyan .read = seq_read, 13248f32f7e5SAlexey Dobriyan .llseek = seq_lseek, 13258f32f7e5SAlexey Dobriyan .release = seq_release, 13268f32f7e5SAlexey Dobriyan }; 13278f32f7e5SAlexey Dobriyan 132874e2e8e8SAlexey Dobriyan static const struct seq_operations pagetypeinfo_op = { 1329467c996cSMel Gorman .start = frag_start, 1330467c996cSMel Gorman .next = frag_next, 1331467c996cSMel Gorman .stop = frag_stop, 1332467c996cSMel Gorman .show = pagetypeinfo_show, 1333467c996cSMel Gorman }; 1334467c996cSMel Gorman 133574e2e8e8SAlexey Dobriyan static int pagetypeinfo_open(struct inode *inode, struct file *file) 133674e2e8e8SAlexey Dobriyan { 133774e2e8e8SAlexey Dobriyan return seq_open(file, &pagetypeinfo_op); 133874e2e8e8SAlexey Dobriyan } 133974e2e8e8SAlexey Dobriyan 134074e2e8e8SAlexey Dobriyan static const struct file_operations pagetypeinfo_file_ops = { 134174e2e8e8SAlexey Dobriyan .open = pagetypeinfo_open, 134274e2e8e8SAlexey Dobriyan .read = seq_read, 134374e2e8e8SAlexey Dobriyan .llseek = seq_lseek, 134474e2e8e8SAlexey Dobriyan .release = seq_release, 134574e2e8e8SAlexey Dobriyan }; 134674e2e8e8SAlexey Dobriyan 1347e2ecc8a7SMel Gorman static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1348e2ecc8a7SMel Gorman { 1349e2ecc8a7SMel Gorman int zid; 1350e2ecc8a7SMel Gorman 1351e2ecc8a7SMel Gorman for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1352e2ecc8a7SMel Gorman struct zone *compare = &pgdat->node_zones[zid]; 1353e2ecc8a7SMel Gorman 1354e2ecc8a7SMel Gorman if (populated_zone(compare)) 1355e2ecc8a7SMel Gorman return zone == compare; 1356e2ecc8a7SMel Gorman } 1357e2ecc8a7SMel Gorman 1358e2ecc8a7SMel Gorman /* The zone must be somewhere! */ 1359e2ecc8a7SMel Gorman WARN_ON_ONCE(1); 1360e2ecc8a7SMel Gorman return false; 1361e2ecc8a7SMel Gorman } 1362e2ecc8a7SMel Gorman 1363467c996cSMel Gorman static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1364467c996cSMel Gorman struct zone *zone) 1365f6ac2354SChristoph Lameter { 1366f6ac2354SChristoph Lameter int i; 1367f6ac2354SChristoph Lameter seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1368e2ecc8a7SMel Gorman if (is_zone_first_populated(pgdat, zone)) { 1369e2ecc8a7SMel Gorman seq_printf(m, "\n per-node stats"); 1370e2ecc8a7SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1371e2ecc8a7SMel Gorman seq_printf(m, "\n %-12s %lu", 1372e2ecc8a7SMel Gorman vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], 1373e2ecc8a7SMel Gorman node_page_state(pgdat, i)); 1374e2ecc8a7SMel Gorman } 1375e2ecc8a7SMel Gorman } 1376f6ac2354SChristoph Lameter seq_printf(m, 1377f6ac2354SChristoph Lameter "\n pages free %lu" 1378f6ac2354SChristoph Lameter "\n min %lu" 1379f6ac2354SChristoph Lameter "\n low %lu" 1380f6ac2354SChristoph Lameter "\n high %lu" 1381599d0c95SMel Gorman "\n node_scanned %lu" 1382f6ac2354SChristoph Lameter "\n spanned %lu" 13839feedc9dSJiang Liu "\n present %lu" 13849feedc9dSJiang Liu "\n managed %lu", 138588f5acf8SMel Gorman zone_page_state(zone, NR_FREE_PAGES), 138641858966SMel Gorman min_wmark_pages(zone), 138741858966SMel Gorman low_wmark_pages(zone), 138841858966SMel Gorman high_wmark_pages(zone), 1389599d0c95SMel Gorman node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED), 1390f6ac2354SChristoph Lameter zone->spanned_pages, 13919feedc9dSJiang Liu zone->present_pages, 13929feedc9dSJiang Liu zone->managed_pages); 13932244b95aSChristoph Lameter 13942244b95aSChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 13952244b95aSChristoph Lameter seq_printf(m, "\n %-12s %lu", vmstat_text[i], 13962244b95aSChristoph Lameter zone_page_state(zone, i)); 13972244b95aSChristoph Lameter 1398f6ac2354SChristoph Lameter seq_printf(m, 13993484b2deSMel Gorman "\n protection: (%ld", 1400f6ac2354SChristoph Lameter zone->lowmem_reserve[0]); 1401f6ac2354SChristoph Lameter for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 14023484b2deSMel Gorman seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1403f6ac2354SChristoph Lameter seq_printf(m, 1404f6ac2354SChristoph Lameter ")" 1405f6ac2354SChristoph Lameter "\n pagesets"); 1406f6ac2354SChristoph Lameter for_each_online_cpu(i) { 1407f6ac2354SChristoph Lameter struct per_cpu_pageset *pageset; 1408f6ac2354SChristoph Lameter 140999dcc3e5SChristoph Lameter pageset = per_cpu_ptr(zone->pageset, i); 1410f6ac2354SChristoph Lameter seq_printf(m, 14113dfa5721SChristoph Lameter "\n cpu: %i" 1412f6ac2354SChristoph Lameter "\n count: %i" 1413f6ac2354SChristoph Lameter "\n high: %i" 1414f6ac2354SChristoph Lameter "\n batch: %i", 14153dfa5721SChristoph Lameter i, 14163dfa5721SChristoph Lameter pageset->pcp.count, 14173dfa5721SChristoph Lameter pageset->pcp.high, 14183dfa5721SChristoph Lameter pageset->pcp.batch); 1419df9ecabaSChristoph Lameter #ifdef CONFIG_SMP 1420df9ecabaSChristoph Lameter seq_printf(m, "\n vm stats threshold: %d", 1421df9ecabaSChristoph Lameter pageset->stat_threshold); 1422df9ecabaSChristoph Lameter #endif 1423f6ac2354SChristoph Lameter } 1424f6ac2354SChristoph Lameter seq_printf(m, 1425599d0c95SMel Gorman "\n node_unreclaimable: %u" 1426556adecbSRik van Riel "\n start_pfn: %lu" 1427599d0c95SMel Gorman "\n node_inactive_ratio: %u", 1428599d0c95SMel Gorman !pgdat_reclaimable(zone->zone_pgdat), 1429556adecbSRik van Riel zone->zone_start_pfn, 1430599d0c95SMel Gorman zone->zone_pgdat->inactive_ratio); 1431f6ac2354SChristoph Lameter seq_putc(m, '\n'); 1432f6ac2354SChristoph Lameter } 1433467c996cSMel Gorman 1434467c996cSMel Gorman /* 1435467c996cSMel Gorman * Output information about zones in @pgdat. 1436467c996cSMel Gorman */ 1437467c996cSMel Gorman static int zoneinfo_show(struct seq_file *m, void *arg) 1438467c996cSMel Gorman { 1439467c996cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1440467c996cSMel Gorman walk_zones_in_node(m, pgdat, zoneinfo_show_print); 1441f6ac2354SChristoph Lameter return 0; 1442f6ac2354SChristoph Lameter } 1443f6ac2354SChristoph Lameter 14445c9fe628SAlexey Dobriyan static const struct seq_operations zoneinfo_op = { 1445f6ac2354SChristoph Lameter .start = frag_start, /* iterate over all zones. The same as in 1446f6ac2354SChristoph Lameter * fragmentation. */ 1447f6ac2354SChristoph Lameter .next = frag_next, 1448f6ac2354SChristoph Lameter .stop = frag_stop, 1449f6ac2354SChristoph Lameter .show = zoneinfo_show, 1450f6ac2354SChristoph Lameter }; 1451f6ac2354SChristoph Lameter 14525c9fe628SAlexey Dobriyan static int zoneinfo_open(struct inode *inode, struct file *file) 14535c9fe628SAlexey Dobriyan { 14545c9fe628SAlexey Dobriyan return seq_open(file, &zoneinfo_op); 14555c9fe628SAlexey Dobriyan } 14565c9fe628SAlexey Dobriyan 14575c9fe628SAlexey Dobriyan static const struct file_operations proc_zoneinfo_file_operations = { 14585c9fe628SAlexey Dobriyan .open = zoneinfo_open, 14595c9fe628SAlexey Dobriyan .read = seq_read, 14605c9fe628SAlexey Dobriyan .llseek = seq_lseek, 14615c9fe628SAlexey Dobriyan .release = seq_release, 14625c9fe628SAlexey Dobriyan }; 14635c9fe628SAlexey Dobriyan 146479da826aSMichael Rubin enum writeback_stat_item { 146579da826aSMichael Rubin NR_DIRTY_THRESHOLD, 146679da826aSMichael Rubin NR_DIRTY_BG_THRESHOLD, 146779da826aSMichael Rubin NR_VM_WRITEBACK_STAT_ITEMS, 146879da826aSMichael Rubin }; 146979da826aSMichael Rubin 1470f6ac2354SChristoph Lameter static void *vmstat_start(struct seq_file *m, loff_t *pos) 1471f6ac2354SChristoph Lameter { 14722244b95aSChristoph Lameter unsigned long *v; 147379da826aSMichael Rubin int i, stat_items_size; 1474f6ac2354SChristoph Lameter 1475f6ac2354SChristoph Lameter if (*pos >= ARRAY_SIZE(vmstat_text)) 1476f6ac2354SChristoph Lameter return NULL; 147779da826aSMichael Rubin stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) + 147875ef7184SMel Gorman NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) + 147979da826aSMichael Rubin NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long); 1480f6ac2354SChristoph Lameter 1481f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 148279da826aSMichael Rubin stat_items_size += sizeof(struct vm_event_state); 1483f8891e5eSChristoph Lameter #endif 148479da826aSMichael Rubin 148579da826aSMichael Rubin v = kmalloc(stat_items_size, GFP_KERNEL); 14862244b95aSChristoph Lameter m->private = v; 14872244b95aSChristoph Lameter if (!v) 1488f6ac2354SChristoph Lameter return ERR_PTR(-ENOMEM); 14892244b95aSChristoph Lameter for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 14902244b95aSChristoph Lameter v[i] = global_page_state(i); 149179da826aSMichael Rubin v += NR_VM_ZONE_STAT_ITEMS; 149279da826aSMichael Rubin 149375ef7184SMel Gorman for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 149475ef7184SMel Gorman v[i] = global_node_page_state(i); 149575ef7184SMel Gorman v += NR_VM_NODE_STAT_ITEMS; 149675ef7184SMel Gorman 149779da826aSMichael Rubin global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 149879da826aSMichael Rubin v + NR_DIRTY_THRESHOLD); 149979da826aSMichael Rubin v += NR_VM_WRITEBACK_STAT_ITEMS; 150079da826aSMichael Rubin 1501f8891e5eSChristoph Lameter #ifdef CONFIG_VM_EVENT_COUNTERS 150279da826aSMichael Rubin all_vm_events(v); 150379da826aSMichael Rubin v[PGPGIN] /= 2; /* sectors -> kbytes */ 150479da826aSMichael Rubin v[PGPGOUT] /= 2; 1505f8891e5eSChristoph Lameter #endif 1506ff8b16d7SWu Fengguang return (unsigned long *)m->private + *pos; 1507f6ac2354SChristoph Lameter } 1508f6ac2354SChristoph Lameter 1509f6ac2354SChristoph Lameter static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1510f6ac2354SChristoph Lameter { 1511f6ac2354SChristoph Lameter (*pos)++; 1512f6ac2354SChristoph Lameter if (*pos >= ARRAY_SIZE(vmstat_text)) 1513f6ac2354SChristoph Lameter return NULL; 1514f6ac2354SChristoph Lameter return (unsigned long *)m->private + *pos; 1515f6ac2354SChristoph Lameter } 1516f6ac2354SChristoph Lameter 1517f6ac2354SChristoph Lameter static int vmstat_show(struct seq_file *m, void *arg) 1518f6ac2354SChristoph Lameter { 1519f6ac2354SChristoph Lameter unsigned long *l = arg; 1520f6ac2354SChristoph Lameter unsigned long off = l - (unsigned long *)m->private; 152168ba0326SAlexey Dobriyan 152268ba0326SAlexey Dobriyan seq_puts(m, vmstat_text[off]); 152375ba1d07SJoe Perches seq_put_decimal_ull(m, " ", *l); 152468ba0326SAlexey Dobriyan seq_putc(m, '\n'); 1525f6ac2354SChristoph Lameter return 0; 1526f6ac2354SChristoph Lameter } 1527f6ac2354SChristoph Lameter 1528f6ac2354SChristoph Lameter static void vmstat_stop(struct seq_file *m, void *arg) 1529f6ac2354SChristoph Lameter { 1530f6ac2354SChristoph Lameter kfree(m->private); 1531f6ac2354SChristoph Lameter m->private = NULL; 1532f6ac2354SChristoph Lameter } 1533f6ac2354SChristoph Lameter 1534b6aa44abSAlexey Dobriyan static const struct seq_operations vmstat_op = { 1535f6ac2354SChristoph Lameter .start = vmstat_start, 1536f6ac2354SChristoph Lameter .next = vmstat_next, 1537f6ac2354SChristoph Lameter .stop = vmstat_stop, 1538f6ac2354SChristoph Lameter .show = vmstat_show, 1539f6ac2354SChristoph Lameter }; 1540f6ac2354SChristoph Lameter 1541b6aa44abSAlexey Dobriyan static int vmstat_open(struct inode *inode, struct file *file) 1542b6aa44abSAlexey Dobriyan { 1543b6aa44abSAlexey Dobriyan return seq_open(file, &vmstat_op); 1544b6aa44abSAlexey Dobriyan } 1545b6aa44abSAlexey Dobriyan 1546b6aa44abSAlexey Dobriyan static const struct file_operations proc_vmstat_file_operations = { 1547b6aa44abSAlexey Dobriyan .open = vmstat_open, 1548b6aa44abSAlexey Dobriyan .read = seq_read, 1549b6aa44abSAlexey Dobriyan .llseek = seq_lseek, 1550b6aa44abSAlexey Dobriyan .release = seq_release, 1551b6aa44abSAlexey Dobriyan }; 1552f6ac2354SChristoph Lameter #endif /* CONFIG_PROC_FS */ 1553f6ac2354SChristoph Lameter 1554df9ecabaSChristoph Lameter #ifdef CONFIG_SMP 1555d1187ed2SChristoph Lameter static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 155677461ab3SChristoph Lameter int sysctl_stat_interval __read_mostly = HZ; 1557d1187ed2SChristoph Lameter 155852b6f46bSHugh Dickins #ifdef CONFIG_PROC_FS 155952b6f46bSHugh Dickins static void refresh_vm_stats(struct work_struct *work) 156052b6f46bSHugh Dickins { 156152b6f46bSHugh Dickins refresh_cpu_vm_stats(true); 156252b6f46bSHugh Dickins } 156352b6f46bSHugh Dickins 156452b6f46bSHugh Dickins int vmstat_refresh(struct ctl_table *table, int write, 156552b6f46bSHugh Dickins void __user *buffer, size_t *lenp, loff_t *ppos) 156652b6f46bSHugh Dickins { 156752b6f46bSHugh Dickins long val; 156852b6f46bSHugh Dickins int err; 156952b6f46bSHugh Dickins int i; 157052b6f46bSHugh Dickins 157152b6f46bSHugh Dickins /* 157252b6f46bSHugh Dickins * The regular update, every sysctl_stat_interval, may come later 157352b6f46bSHugh Dickins * than expected: leaving a significant amount in per_cpu buckets. 157452b6f46bSHugh Dickins * This is particularly misleading when checking a quantity of HUGE 157552b6f46bSHugh Dickins * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 157652b6f46bSHugh Dickins * which can equally be echo'ed to or cat'ted from (by root), 157752b6f46bSHugh Dickins * can be used to update the stats just before reading them. 157852b6f46bSHugh Dickins * 157952b6f46bSHugh Dickins * Oh, and since global_page_state() etc. are so careful to hide 158052b6f46bSHugh Dickins * transiently negative values, report an error here if any of 158152b6f46bSHugh Dickins * the stats is negative, so we know to go looking for imbalance. 158252b6f46bSHugh Dickins */ 158352b6f46bSHugh Dickins err = schedule_on_each_cpu(refresh_vm_stats); 158452b6f46bSHugh Dickins if (err) 158552b6f46bSHugh Dickins return err; 158652b6f46bSHugh Dickins for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 158775ef7184SMel Gorman val = atomic_long_read(&vm_zone_stat[i]); 158852b6f46bSHugh Dickins if (val < 0) { 158952b6f46bSHugh Dickins switch (i) { 159052b6f46bSHugh Dickins case NR_PAGES_SCANNED: 159152b6f46bSHugh Dickins /* 1592e6cbd7f2SMel Gorman * This is often seen to go negative in 159352b6f46bSHugh Dickins * recent kernels, but not to go permanently 159452b6f46bSHugh Dickins * negative. Whilst it would be nicer not to 159552b6f46bSHugh Dickins * have exceptions, rooting them out would be 159652b6f46bSHugh Dickins * another task, of rather low priority. 159752b6f46bSHugh Dickins */ 159852b6f46bSHugh Dickins break; 159952b6f46bSHugh Dickins default: 160052b6f46bSHugh Dickins pr_warn("%s: %s %ld\n", 160152b6f46bSHugh Dickins __func__, vmstat_text[i], val); 160252b6f46bSHugh Dickins err = -EINVAL; 160352b6f46bSHugh Dickins break; 160452b6f46bSHugh Dickins } 160552b6f46bSHugh Dickins } 160652b6f46bSHugh Dickins } 160752b6f46bSHugh Dickins if (err) 160852b6f46bSHugh Dickins return err; 160952b6f46bSHugh Dickins if (write) 161052b6f46bSHugh Dickins *ppos += *lenp; 161152b6f46bSHugh Dickins else 161252b6f46bSHugh Dickins *lenp = 0; 161352b6f46bSHugh Dickins return 0; 161452b6f46bSHugh Dickins } 161552b6f46bSHugh Dickins #endif /* CONFIG_PROC_FS */ 161652b6f46bSHugh Dickins 1617d1187ed2SChristoph Lameter static void vmstat_update(struct work_struct *w) 1618d1187ed2SChristoph Lameter { 16190eb77e98SChristoph Lameter if (refresh_cpu_vm_stats(true)) { 16207cc36bbdSChristoph Lameter /* 16217cc36bbdSChristoph Lameter * Counters were updated so we expect more updates 16227cc36bbdSChristoph Lameter * to occur in the future. Keep on running the 16237cc36bbdSChristoph Lameter * update worker thread. 16247cc36bbdSChristoph Lameter */ 1625*ce612879SMichal Hocko queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 1626176bed1dSLinus Torvalds this_cpu_ptr(&vmstat_work), 162798f4ebb2SAnton Blanchard round_jiffies_relative(sysctl_stat_interval)); 1628f01f17d3SMichal Hocko } 1629d1187ed2SChristoph Lameter } 1630d1187ed2SChristoph Lameter 16317cc36bbdSChristoph Lameter /* 16320eb77e98SChristoph Lameter * Switch off vmstat processing and then fold all the remaining differentials 16330eb77e98SChristoph Lameter * until the diffs stay at zero. The function is used by NOHZ and can only be 16340eb77e98SChristoph Lameter * invoked when tick processing is not active. 16350eb77e98SChristoph Lameter */ 16360eb77e98SChristoph Lameter /* 16377cc36bbdSChristoph Lameter * Check if the diffs for a certain cpu indicate that 16387cc36bbdSChristoph Lameter * an update is needed. 16397cc36bbdSChristoph Lameter */ 16407cc36bbdSChristoph Lameter static bool need_update(int cpu) 1641d1187ed2SChristoph Lameter { 16427cc36bbdSChristoph Lameter struct zone *zone; 1643d1187ed2SChristoph Lameter 16447cc36bbdSChristoph Lameter for_each_populated_zone(zone) { 16457cc36bbdSChristoph Lameter struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu); 16467cc36bbdSChristoph Lameter 16477cc36bbdSChristoph Lameter BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1); 16487cc36bbdSChristoph Lameter /* 16497cc36bbdSChristoph Lameter * The fast way of checking if there are any vmstat diffs. 16507cc36bbdSChristoph Lameter * This works because the diffs are byte sized items. 16517cc36bbdSChristoph Lameter */ 16527cc36bbdSChristoph Lameter if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS)) 16537cc36bbdSChristoph Lameter return true; 16547cc36bbdSChristoph Lameter 16557cc36bbdSChristoph Lameter } 16567cc36bbdSChristoph Lameter return false; 16577cc36bbdSChristoph Lameter } 16587cc36bbdSChristoph Lameter 16597b8da4c7SChristoph Lameter /* 16607b8da4c7SChristoph Lameter * Switch off vmstat processing and then fold all the remaining differentials 16617b8da4c7SChristoph Lameter * until the diffs stay at zero. The function is used by NOHZ and can only be 16627b8da4c7SChristoph Lameter * invoked when tick processing is not active. 16637b8da4c7SChristoph Lameter */ 1664f01f17d3SMichal Hocko void quiet_vmstat(void) 1665f01f17d3SMichal Hocko { 1666f01f17d3SMichal Hocko if (system_state != SYSTEM_RUNNING) 1667f01f17d3SMichal Hocko return; 1668f01f17d3SMichal Hocko 16697b8da4c7SChristoph Lameter if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 1670f01f17d3SMichal Hocko return; 1671f01f17d3SMichal Hocko 1672f01f17d3SMichal Hocko if (!need_update(smp_processor_id())) 1673f01f17d3SMichal Hocko return; 1674f01f17d3SMichal Hocko 1675f01f17d3SMichal Hocko /* 1676f01f17d3SMichal Hocko * Just refresh counters and do not care about the pending delayed 1677f01f17d3SMichal Hocko * vmstat_update. It doesn't fire that often to matter and canceling 1678f01f17d3SMichal Hocko * it would be too expensive from this path. 1679f01f17d3SMichal Hocko * vmstat_shepherd will take care about that for us. 1680f01f17d3SMichal Hocko */ 1681f01f17d3SMichal Hocko refresh_cpu_vm_stats(false); 1682f01f17d3SMichal Hocko } 1683f01f17d3SMichal Hocko 16847cc36bbdSChristoph Lameter /* 16857cc36bbdSChristoph Lameter * Shepherd worker thread that checks the 16867cc36bbdSChristoph Lameter * differentials of processors that have their worker 16877cc36bbdSChristoph Lameter * threads for vm statistics updates disabled because of 16887cc36bbdSChristoph Lameter * inactivity. 16897cc36bbdSChristoph Lameter */ 16907cc36bbdSChristoph Lameter static void vmstat_shepherd(struct work_struct *w); 16917cc36bbdSChristoph Lameter 16920eb77e98SChristoph Lameter static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 16937cc36bbdSChristoph Lameter 16947cc36bbdSChristoph Lameter static void vmstat_shepherd(struct work_struct *w) 16957cc36bbdSChristoph Lameter { 16967cc36bbdSChristoph Lameter int cpu; 16977cc36bbdSChristoph Lameter 16987cc36bbdSChristoph Lameter get_online_cpus(); 16997cc36bbdSChristoph Lameter /* Check processors whose vmstat worker threads have been disabled */ 17007b8da4c7SChristoph Lameter for_each_online_cpu(cpu) { 1701f01f17d3SMichal Hocko struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 17027cc36bbdSChristoph Lameter 17037b8da4c7SChristoph Lameter if (!delayed_work_pending(dw) && need_update(cpu)) 1704*ce612879SMichal Hocko queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 1705f01f17d3SMichal Hocko } 17067cc36bbdSChristoph Lameter put_online_cpus(); 17077cc36bbdSChristoph Lameter 17087cc36bbdSChristoph Lameter schedule_delayed_work(&shepherd, 17097cc36bbdSChristoph Lameter round_jiffies_relative(sysctl_stat_interval)); 17107cc36bbdSChristoph Lameter } 17117cc36bbdSChristoph Lameter 17127cc36bbdSChristoph Lameter static void __init start_shepherd_timer(void) 17137cc36bbdSChristoph Lameter { 17147cc36bbdSChristoph Lameter int cpu; 17157cc36bbdSChristoph Lameter 17167cc36bbdSChristoph Lameter for_each_possible_cpu(cpu) 1717ccde8bd4SMichal Hocko INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 17187cc36bbdSChristoph Lameter vmstat_update); 17197cc36bbdSChristoph Lameter 17207cc36bbdSChristoph Lameter schedule_delayed_work(&shepherd, 17217cc36bbdSChristoph Lameter round_jiffies_relative(sysctl_stat_interval)); 1722d1187ed2SChristoph Lameter } 1723d1187ed2SChristoph Lameter 172403e86dbaSTim Chen static void __init init_cpu_node_state(void) 172503e86dbaSTim Chen { 17264c501327SSebastian Andrzej Siewior int node; 172703e86dbaSTim Chen 17284c501327SSebastian Andrzej Siewior for_each_online_node(node) { 17294c501327SSebastian Andrzej Siewior if (cpumask_weight(cpumask_of_node(node)) > 0) 17304c501327SSebastian Andrzej Siewior node_set_state(node, N_CPU); 17314c501327SSebastian Andrzej Siewior } 173203e86dbaSTim Chen } 173303e86dbaSTim Chen 17345438da97SSebastian Andrzej Siewior static int vmstat_cpu_online(unsigned int cpu) 1735807a1bd2SToshi Kani { 17365ee28a44SKAMEZAWA Hiroyuki refresh_zone_stat_thresholds(); 1737ad596925SChristoph Lameter node_set_state(cpu_to_node(cpu), N_CPU); 17385438da97SSebastian Andrzej Siewior return 0; 1739df9ecabaSChristoph Lameter } 1740df9ecabaSChristoph Lameter 17415438da97SSebastian Andrzej Siewior static int vmstat_cpu_down_prep(unsigned int cpu) 17425438da97SSebastian Andrzej Siewior { 17435438da97SSebastian Andrzej Siewior cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 17445438da97SSebastian Andrzej Siewior return 0; 17455438da97SSebastian Andrzej Siewior } 17465438da97SSebastian Andrzej Siewior 17475438da97SSebastian Andrzej Siewior static int vmstat_cpu_dead(unsigned int cpu) 17485438da97SSebastian Andrzej Siewior { 17495438da97SSebastian Andrzej Siewior const struct cpumask *node_cpus; 17505438da97SSebastian Andrzej Siewior int node; 17515438da97SSebastian Andrzej Siewior 17525438da97SSebastian Andrzej Siewior node = cpu_to_node(cpu); 17535438da97SSebastian Andrzej Siewior 17545438da97SSebastian Andrzej Siewior refresh_zone_stat_thresholds(); 17555438da97SSebastian Andrzej Siewior node_cpus = cpumask_of_node(node); 17565438da97SSebastian Andrzej Siewior if (cpumask_weight(node_cpus) > 0) 17575438da97SSebastian Andrzej Siewior return 0; 17585438da97SSebastian Andrzej Siewior 17595438da97SSebastian Andrzej Siewior node_clear_state(node, N_CPU); 17605438da97SSebastian Andrzej Siewior return 0; 17615438da97SSebastian Andrzej Siewior } 17625438da97SSebastian Andrzej Siewior 17638f32f7e5SAlexey Dobriyan #endif 1764df9ecabaSChristoph Lameter 1765*ce612879SMichal Hocko struct workqueue_struct *mm_percpu_wq; 1766*ce612879SMichal Hocko 1767597b7305SMichal Hocko void __init init_mm_internals(void) 1768df9ecabaSChristoph Lameter { 1769*ce612879SMichal Hocko int ret __maybe_unused; 17705438da97SSebastian Andrzej Siewior 1771*ce612879SMichal Hocko mm_percpu_wq = alloc_workqueue("mm_percpu_wq", 1772*ce612879SMichal Hocko WQ_FREEZABLE|WQ_MEM_RECLAIM, 0); 1773*ce612879SMichal Hocko 1774*ce612879SMichal Hocko #ifdef CONFIG_SMP 17755438da97SSebastian Andrzej Siewior ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 17765438da97SSebastian Andrzej Siewior NULL, vmstat_cpu_dead); 17775438da97SSebastian Andrzej Siewior if (ret < 0) 17785438da97SSebastian Andrzej Siewior pr_err("vmstat: failed to register 'dead' hotplug state\n"); 17795438da97SSebastian Andrzej Siewior 17805438da97SSebastian Andrzej Siewior ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 17815438da97SSebastian Andrzej Siewior vmstat_cpu_online, 17825438da97SSebastian Andrzej Siewior vmstat_cpu_down_prep); 17835438da97SSebastian Andrzej Siewior if (ret < 0) 17845438da97SSebastian Andrzej Siewior pr_err("vmstat: failed to register 'online' hotplug state\n"); 17855438da97SSebastian Andrzej Siewior 17865438da97SSebastian Andrzej Siewior get_online_cpus(); 178703e86dbaSTim Chen init_cpu_node_state(); 17885438da97SSebastian Andrzej Siewior put_online_cpus(); 1789d1187ed2SChristoph Lameter 17907cc36bbdSChristoph Lameter start_shepherd_timer(); 17918f32f7e5SAlexey Dobriyan #endif 17928f32f7e5SAlexey Dobriyan #ifdef CONFIG_PROC_FS 17938f32f7e5SAlexey Dobriyan proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations); 179474e2e8e8SAlexey Dobriyan proc_create("pagetypeinfo", S_IRUGO, NULL, &pagetypeinfo_file_ops); 1795b6aa44abSAlexey Dobriyan proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations); 17965c9fe628SAlexey Dobriyan proc_create("zoneinfo", S_IRUGO, NULL, &proc_zoneinfo_file_operations); 17978f32f7e5SAlexey Dobriyan #endif 1798df9ecabaSChristoph Lameter } 1799d7a5752cSMel Gorman 1800d7a5752cSMel Gorman #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 1801d7a5752cSMel Gorman 1802d7a5752cSMel Gorman /* 1803d7a5752cSMel Gorman * Return an index indicating how much of the available free memory is 1804d7a5752cSMel Gorman * unusable for an allocation of the requested size. 1805d7a5752cSMel Gorman */ 1806d7a5752cSMel Gorman static int unusable_free_index(unsigned int order, 1807d7a5752cSMel Gorman struct contig_page_info *info) 1808d7a5752cSMel Gorman { 1809d7a5752cSMel Gorman /* No free memory is interpreted as all free memory is unusable */ 1810d7a5752cSMel Gorman if (info->free_pages == 0) 1811d7a5752cSMel Gorman return 1000; 1812d7a5752cSMel Gorman 1813d7a5752cSMel Gorman /* 1814d7a5752cSMel Gorman * Index should be a value between 0 and 1. Return a value to 3 1815d7a5752cSMel Gorman * decimal places. 1816d7a5752cSMel Gorman * 1817d7a5752cSMel Gorman * 0 => no fragmentation 1818d7a5752cSMel Gorman * 1 => high fragmentation 1819d7a5752cSMel Gorman */ 1820d7a5752cSMel Gorman return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 1821d7a5752cSMel Gorman 1822d7a5752cSMel Gorman } 1823d7a5752cSMel Gorman 1824d7a5752cSMel Gorman static void unusable_show_print(struct seq_file *m, 1825d7a5752cSMel Gorman pg_data_t *pgdat, struct zone *zone) 1826d7a5752cSMel Gorman { 1827d7a5752cSMel Gorman unsigned int order; 1828d7a5752cSMel Gorman int index; 1829d7a5752cSMel Gorman struct contig_page_info info; 1830d7a5752cSMel Gorman 1831d7a5752cSMel Gorman seq_printf(m, "Node %d, zone %8s ", 1832d7a5752cSMel Gorman pgdat->node_id, 1833d7a5752cSMel Gorman zone->name); 1834d7a5752cSMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 1835d7a5752cSMel Gorman fill_contig_page_info(zone, order, &info); 1836d7a5752cSMel Gorman index = unusable_free_index(order, &info); 1837d7a5752cSMel Gorman seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 1838d7a5752cSMel Gorman } 1839d7a5752cSMel Gorman 1840d7a5752cSMel Gorman seq_putc(m, '\n'); 1841d7a5752cSMel Gorman } 1842d7a5752cSMel Gorman 1843d7a5752cSMel Gorman /* 1844d7a5752cSMel Gorman * Display unusable free space index 1845d7a5752cSMel Gorman * 1846d7a5752cSMel Gorman * The unusable free space index measures how much of the available free 1847d7a5752cSMel Gorman * memory cannot be used to satisfy an allocation of a given size and is a 1848d7a5752cSMel Gorman * value between 0 and 1. The higher the value, the more of free memory is 1849d7a5752cSMel Gorman * unusable and by implication, the worse the external fragmentation is. This 1850d7a5752cSMel Gorman * can be expressed as a percentage by multiplying by 100. 1851d7a5752cSMel Gorman */ 1852d7a5752cSMel Gorman static int unusable_show(struct seq_file *m, void *arg) 1853d7a5752cSMel Gorman { 1854d7a5752cSMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1855d7a5752cSMel Gorman 1856d7a5752cSMel Gorman /* check memoryless node */ 1857a47b53c5SLai Jiangshan if (!node_state(pgdat->node_id, N_MEMORY)) 1858d7a5752cSMel Gorman return 0; 1859d7a5752cSMel Gorman 1860d7a5752cSMel Gorman walk_zones_in_node(m, pgdat, unusable_show_print); 1861d7a5752cSMel Gorman 1862d7a5752cSMel Gorman return 0; 1863d7a5752cSMel Gorman } 1864d7a5752cSMel Gorman 1865d7a5752cSMel Gorman static const struct seq_operations unusable_op = { 1866d7a5752cSMel Gorman .start = frag_start, 1867d7a5752cSMel Gorman .next = frag_next, 1868d7a5752cSMel Gorman .stop = frag_stop, 1869d7a5752cSMel Gorman .show = unusable_show, 1870d7a5752cSMel Gorman }; 1871d7a5752cSMel Gorman 1872d7a5752cSMel Gorman static int unusable_open(struct inode *inode, struct file *file) 1873d7a5752cSMel Gorman { 1874d7a5752cSMel Gorman return seq_open(file, &unusable_op); 1875d7a5752cSMel Gorman } 1876d7a5752cSMel Gorman 1877d7a5752cSMel Gorman static const struct file_operations unusable_file_ops = { 1878d7a5752cSMel Gorman .open = unusable_open, 1879d7a5752cSMel Gorman .read = seq_read, 1880d7a5752cSMel Gorman .llseek = seq_lseek, 1881d7a5752cSMel Gorman .release = seq_release, 1882d7a5752cSMel Gorman }; 1883d7a5752cSMel Gorman 1884f1a5ab12SMel Gorman static void extfrag_show_print(struct seq_file *m, 1885f1a5ab12SMel Gorman pg_data_t *pgdat, struct zone *zone) 1886f1a5ab12SMel Gorman { 1887f1a5ab12SMel Gorman unsigned int order; 1888f1a5ab12SMel Gorman int index; 1889f1a5ab12SMel Gorman 1890f1a5ab12SMel Gorman /* Alloc on stack as interrupts are disabled for zone walk */ 1891f1a5ab12SMel Gorman struct contig_page_info info; 1892f1a5ab12SMel Gorman 1893f1a5ab12SMel Gorman seq_printf(m, "Node %d, zone %8s ", 1894f1a5ab12SMel Gorman pgdat->node_id, 1895f1a5ab12SMel Gorman zone->name); 1896f1a5ab12SMel Gorman for (order = 0; order < MAX_ORDER; ++order) { 1897f1a5ab12SMel Gorman fill_contig_page_info(zone, order, &info); 189856de7263SMel Gorman index = __fragmentation_index(order, &info); 1899f1a5ab12SMel Gorman seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 1900f1a5ab12SMel Gorman } 1901f1a5ab12SMel Gorman 1902f1a5ab12SMel Gorman seq_putc(m, '\n'); 1903f1a5ab12SMel Gorman } 1904f1a5ab12SMel Gorman 1905f1a5ab12SMel Gorman /* 1906f1a5ab12SMel Gorman * Display fragmentation index for orders that allocations would fail for 1907f1a5ab12SMel Gorman */ 1908f1a5ab12SMel Gorman static int extfrag_show(struct seq_file *m, void *arg) 1909f1a5ab12SMel Gorman { 1910f1a5ab12SMel Gorman pg_data_t *pgdat = (pg_data_t *)arg; 1911f1a5ab12SMel Gorman 1912f1a5ab12SMel Gorman walk_zones_in_node(m, pgdat, extfrag_show_print); 1913f1a5ab12SMel Gorman 1914f1a5ab12SMel Gorman return 0; 1915f1a5ab12SMel Gorman } 1916f1a5ab12SMel Gorman 1917f1a5ab12SMel Gorman static const struct seq_operations extfrag_op = { 1918f1a5ab12SMel Gorman .start = frag_start, 1919f1a5ab12SMel Gorman .next = frag_next, 1920f1a5ab12SMel Gorman .stop = frag_stop, 1921f1a5ab12SMel Gorman .show = extfrag_show, 1922f1a5ab12SMel Gorman }; 1923f1a5ab12SMel Gorman 1924f1a5ab12SMel Gorman static int extfrag_open(struct inode *inode, struct file *file) 1925f1a5ab12SMel Gorman { 1926f1a5ab12SMel Gorman return seq_open(file, &extfrag_op); 1927f1a5ab12SMel Gorman } 1928f1a5ab12SMel Gorman 1929f1a5ab12SMel Gorman static const struct file_operations extfrag_file_ops = { 1930f1a5ab12SMel Gorman .open = extfrag_open, 1931f1a5ab12SMel Gorman .read = seq_read, 1932f1a5ab12SMel Gorman .llseek = seq_lseek, 1933f1a5ab12SMel Gorman .release = seq_release, 1934f1a5ab12SMel Gorman }; 1935f1a5ab12SMel Gorman 1936d7a5752cSMel Gorman static int __init extfrag_debug_init(void) 1937d7a5752cSMel Gorman { 1938bde8bd8aSSasikantha babu struct dentry *extfrag_debug_root; 1939bde8bd8aSSasikantha babu 1940d7a5752cSMel Gorman extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 1941d7a5752cSMel Gorman if (!extfrag_debug_root) 1942d7a5752cSMel Gorman return -ENOMEM; 1943d7a5752cSMel Gorman 1944d7a5752cSMel Gorman if (!debugfs_create_file("unusable_index", 0444, 1945d7a5752cSMel Gorman extfrag_debug_root, NULL, &unusable_file_ops)) 1946bde8bd8aSSasikantha babu goto fail; 1947d7a5752cSMel Gorman 1948f1a5ab12SMel Gorman if (!debugfs_create_file("extfrag_index", 0444, 1949f1a5ab12SMel Gorman extfrag_debug_root, NULL, &extfrag_file_ops)) 1950bde8bd8aSSasikantha babu goto fail; 1951f1a5ab12SMel Gorman 1952d7a5752cSMel Gorman return 0; 1953bde8bd8aSSasikantha babu fail: 1954bde8bd8aSSasikantha babu debugfs_remove_recursive(extfrag_debug_root); 1955bde8bd8aSSasikantha babu return -ENOMEM; 1956d7a5752cSMel Gorman } 1957d7a5752cSMel Gorman 1958d7a5752cSMel Gorman module_init(extfrag_debug_init); 1959d7a5752cSMel Gorman #endif 1960