40 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
45 		atomic_long_set(&zone->vm_numa_event[item], 0);  in zero_zone_numa_counters()
47 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]  in zero_zone_numa_counters()
56 	struct zone *zone;  in zero_zones_numa_counters()  local
58 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
59 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
172 static void fold_vm_zone_numa_events(struct zone *zone)  in fold_vm_zone_numa_events()  argument
181 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in fold_vm_zone_numa_events()
187 		zone_numa_event_add(zone_numa_events[item], zone, item);  in fold_vm_zone_numa_events()
192 	struct zone *zone;  in fold_vm_numa_events()  local
194 	for_each_populated_zone(zone)  in fold_vm_numa_events()
195 		fold_vm_zone_numa_events(zone);  in fold_vm_numa_events()
201 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
214 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
225 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
260 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
278 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
289 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
290 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
293 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
298 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
312 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
315 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
321 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
323 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
329 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
330 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
333 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
335 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
345 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
348 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __mod_zone_page_state()
367 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
435 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
437 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __inc_zone_state()
449 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
491 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
493 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __dec_zone_state()
505 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
560 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
563 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in mod_zone_state()
596 		zone_page_state_add(z, zone, item);  in mod_zone_state()
599 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
602 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
695 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
701 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
709 	struct zone *zone;  in inc_zone_page_state()  local
711 	zone = page_zone(page);  in inc_zone_page_state()
713 	__inc_zone_state(zone, item);  in inc_zone_page_state()
817 	struct zone *zone;  in refresh_cpu_vm_stats()  local
823 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
824 		struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;  in refresh_cpu_vm_stats()
825 		struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;  in refresh_cpu_vm_stats()
833 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
845 			if (decay_pcp_high(zone, this_cpu_ptr(pcp)))  in refresh_cpu_vm_stats()
862 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
873 				drain_zone_pages(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
907 	struct zone *zone;  in cpu_vm_stats_fold()  local
912 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
915 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in cpu_vm_stats_fold()
923 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
934 				zone_numa_event_add(v, zone, i);  in cpu_vm_stats_fold()
963 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)  in drain_zonestat()  argument
972 			zone_page_state_add(v, zone, i);  in drain_zonestat()
981 			zone_numa_event_add(v, zone, i);  in drain_zonestat()
997 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
1011 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_event_state()
1078 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1097 		blocks = data_race(zone->free_area[order].nr_free);  in fill_contig_page_info()
1145 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1149 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1159 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1163 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1542 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1544 	struct zone *zone;  in walk_zones_in_node()  local
1545 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1548 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1549 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1553 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1554 		print(m, pgdat, zone);  in walk_zones_in_node()
1556 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1563 						struct zone *zone)  in frag_show_print()  argument
1567 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1573 		seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));  in frag_show_print()
1588 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1595 					zone->name,  in pagetypeinfo_showfree_print()
1603 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1621 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1623 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1645 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1649 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1650 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1660 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1670 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1752 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1757 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1760 			return zone == compare;  in is_zone_first_populated()
1767 							struct zone *zone)  in zoneinfo_show_print()  argument
1770 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1771 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1793 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1794 		   zone->watermark_boost,  in zoneinfo_show_print()
1795 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1796 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1797 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1798 		   promo_wmark_pages(zone),  in zoneinfo_show_print()
1799 		   zone->spanned_pages,  in zoneinfo_show_print()
1800 		   zone->present_pages,  in zoneinfo_show_print()
1801 		   zone_managed_pages(zone),  in zoneinfo_show_print()
1802 		   zone_cma_pages(zone));  in zoneinfo_show_print()
1806 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1807 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1808 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1812 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1819 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1822 	fold_vm_zone_numa_events(zone);  in zoneinfo_show_print()
1825 			   zone_numa_event_state(zone, i));  in zoneinfo_show_print()
1833 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);  in zoneinfo_show_print()
1848 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);  in zoneinfo_show_print()
1859 		   zone->zone_start_pfn,  in zoneinfo_show_print()
1860 		   zone->nr_reserved_highatomic,  in zoneinfo_show_print()
1861 		   zone->nr_free_highatomic);  in zoneinfo_show_print()
2070 	struct zone *zone;  in need_update()  local
2072 	for_each_populated_zone(zone) {  in need_update()
2073 		struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in need_update()
2082 		if (last_pgdat == zone->zone_pgdat)  in need_update()
2084 		last_pgdat = zone->zone_pgdat;  in need_update()
2085 		n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);  in need_update()
2331 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2339 				zone->name);  in unusable_show_print()
2341 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2381 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2391 				zone->name);  in extfrag_show_print()
2393 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()