39 /* zero numa counters within a zone */
40 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
45 		atomic_long_set(&zone->vm_numa_event[item], 0);  in zero_zone_numa_counters()
47 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]  in zero_zone_numa_counters()
56 	struct zone *zone;  in zero_zones_numa_counters()  local
58 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
59 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
161  * Manage combined zone based / global counters
172 static void fold_vm_zone_numa_events(struct zone *zone)  in fold_vm_zone_numa_events()  argument
181 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in fold_vm_zone_numa_events()
187 		zone_numa_event_add(zone_numa_events[item], zone, item);  in fold_vm_zone_numa_events()
192 	struct zone *zone;  in fold_vm_numa_events()  local
194 	for_each_populated_zone(zone)  in fold_vm_numa_events()
195 		fold_vm_zone_numa_events(zone);  in fold_vm_numa_events()
201 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
214 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
225 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
232 	 * of memory per zone. More memory means that we can defer updates for  in calculate_normal_threshold()
260 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
273  * Refresh the thresholds for each zone.
278 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
289 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
290 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
293 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
298 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
301 			/* Base nodestat threshold on the largest populated zone. */  in refresh_zone_stat_thresholds()
312 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
315 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
321 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
323 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
329 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
330 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
333 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
335 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
345 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
348 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __mod_zone_page_state()
367 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
416  * argument instead of struct zone *. This allows the inclusion of the code
429  * However, the code must first determine the differential location in a zone
435 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
437 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __inc_zone_state()
449 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
491 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
493 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __dec_zone_state()
505 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
552  * mod_state() modifies the zone counter state through atomic per cpu
560 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
563 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in mod_zone_state()
570 		z = 0;  /* overflow to zone counters */  in mod_zone_state()
580 		 * for all cpus in a zone.  in mod_zone_state()
589 			/* Overflow must be added to zone counters */  in mod_zone_state()
596 		zone_page_state_add(z, zone, item);  in mod_zone_state()
599 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
602 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
695 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
701 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
709 	struct zone *zone;  in inc_zone_page_state()  local
711 	zone = page_zone(page);  in inc_zone_page_state()
713 	__inc_zone_state(zone, item);  in inc_zone_page_state()
796  * Update the zone counters for the current cpu.
805  * statistics in the remote zone struct as well as the global cachelines
814 	struct zone *zone;  in refresh_cpu_vm_stats()  local
820 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
821 		struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;  in refresh_cpu_vm_stats()
822 		struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;  in refresh_cpu_vm_stats()
830 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
842 			changes += decay_pcp_high(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
858 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
869 				drain_zone_pages(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
902 	struct zone *zone;  in cpu_vm_stats_fold()  local
907 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
910 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in cpu_vm_stats_fold()
918 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
929 				zone_numa_event_add(v, zone, i);  in cpu_vm_stats_fold()
955  * this is only called if !populated_zone(zone), which implies no other users of
958 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)  in drain_zonestat()  argument
967 			zone_page_state_add(v, zone, i);  in drain_zonestat()
976 			zone_numa_event_add(v, zone, i);  in drain_zonestat()
992 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
1006 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_event_state()
1066  * Calculate the number of free pages in a zone, how many contiguous
1073 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1092 		blocks = data_race(zone->free_area[order].nr_free);  in fill_contig_page_info()
1136  * Calculates external fragmentation within a zone wrt the given order.
1140 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1144 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1154 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1158 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1517 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1519 	struct zone *zone;  in walk_zones_in_node()  local
1520 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1523 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1524 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1528 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1529 		print(m, pgdat, zone);  in walk_zones_in_node()
1531 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1538 						struct zone *zone)  in frag_show_print()  argument
1542 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1548 		seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));  in frag_show_print()
1553  * This walks the free areas for each zone.
1563 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1568 		seq_printf(m, "Node %4d, zone %8s, type %12s ",  in pagetypeinfo_showfree_print()
1570 					zone->name,  in pagetypeinfo_showfree_print()
1578 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1596 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1598 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1620 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1624 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1625 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1635 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1645 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1727 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1732 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1735 			return zone == compare;  in is_zone_first_populated()
1742 							struct zone *zone)  in zoneinfo_show_print()  argument
1745 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1746 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1768 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1769 		   zone->watermark_boost,  in zoneinfo_show_print()
1770 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1771 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1772 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1773 		   promo_wmark_pages(zone),  in zoneinfo_show_print()
1774 		   zone->spanned_pages,  in zoneinfo_show_print()
1775 		   zone->present_pages,  in zoneinfo_show_print()
1776 		   zone_managed_pages(zone),  in zoneinfo_show_print()
1777 		   zone_cma_pages(zone));  in zoneinfo_show_print()
1781 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1782 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1783 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1787 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1794 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1797 	fold_vm_zone_numa_events(zone);  in zoneinfo_show_print()
1800 			   zone_numa_event_state(zone, i));  in zoneinfo_show_print()
1808 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);  in zoneinfo_show_print()
1823 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);  in zoneinfo_show_print()
1832 		   zone->zone_start_pfn);  in zoneinfo_show_print()
2041 	struct zone *zone;  in need_update()  local
2043 	for_each_populated_zone(zone) {  in need_update()
2044 		struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in need_update()
2053 		if (last_pgdat == zone->zone_pgdat)  in need_update()
2055 		last_pgdat = zone->zone_pgdat;  in need_update()
2056 		n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);  in need_update()
2302 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2308 	seq_printf(m, "Node %d, zone %8s ",  in unusable_show_print()
2310 				zone->name);  in unusable_show_print()
2312 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2352 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2357 	/* Alloc on stack as interrupts are disabled for zone walk */  in extfrag_show_print()
2360 	seq_printf(m, "Node %d, zone %8s ",  in extfrag_show_print()
2362 				zone->name);  in extfrag_show_print()
2364 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()