Home
last modified time | relevance | path

Searched refs:pgdat (Results 1 – 25 of 28) sorted by relevance

12

/linux/mm/
H A Dshow_mem.c96 pg_data_t *pgdat = NODE_DATA(nid); in si_meminfo_node() local
99 struct zone *zone = &pgdat->node_zones[zone_type]; in si_meminfo_node()
108 val->sharedram = node_page_state(pgdat, NR_SHMEM); in si_meminfo_node()
164 static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) in node_has_managed_zones() argument
168 if (zone_managed_pages(pgdat->node_zones + zone_idx)) in node_has_managed_zones()
188 pg_data_t *pgdat; in show_free_areas() local
229 for_each_online_pgdat(pgdat) { in show_free_areas()
230 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) in show_free_areas()
232 if (!node_has_managed_zones(pgdat, max_zone_idx)) in show_free_areas()
263 pgdat->node_id, in show_free_areas()
[all …]
H A Dmm_init.c83 pg_data_t *pgdat = NODE_DATA(nid); in mminit_verify_zonelist() local
94 zonelist = &pgdat->node_zonelists[listid]; in mminit_verify_zonelist()
95 zone = &pgdat->node_zones[zoneid]; in mminit_verify_zonelist()
697 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) in pgdat_set_deferred_range() argument
699 pgdat->first_deferred_pfn = ULONG_MAX; in pgdat_set_deferred_range()
754 pg_data_t *pgdat = NODE_DATA(nid); in __init_deferred_page() local
761 struct zone *zone = &pgdat->node_zones[zid]; in __init_deferred_page()
775 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} in pgdat_set_deferred_range() argument
1080 struct pglist_data *pgdat = zone->zone_pgdat; in memmap_init_zone_device() local
1085 int nid = pgdat->node_id; in memmap_init_zone_device()
[all …]
H A Dvmscan.c328 struct pglist_data *pgdat = NODE_DATA(nid); in can_demote() local
331 if (!pgdat || !numa_demotion_enabled) in can_demote()
336 node_get_allowed_targets(pgdat, &allowed_mask); in can_demote()
483 static bool skip_throttle_noprogress(pg_data_t *pgdat) in skip_throttle_noprogress() argument
492 if (kswapd_test_hopeless(pgdat)) in skip_throttle_noprogress()
500 for_each_managed_zone_pgdat(zone, pgdat, i, MAX_NR_ZONES - 1) { in skip_throttle_noprogress()
511 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) in reclaim_throttle() argument
513 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; in reclaim_throttle()
542 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { in reclaim_throttle()
543 WRITE_ONCE(pgdat->nr_reclaim_start, in reclaim_throttle()
[all …]
H A Dvmstat.c278 struct pglist_data *pgdat; in refresh_zone_stat_thresholds() local
284 for_each_online_pgdat(pgdat) { in refresh_zone_stat_thresholds()
286 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; in refresh_zone_stat_thresholds()
291 struct pglist_data *pgdat = zone->zone_pgdat; in refresh_zone_stat_thresholds() local
303 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; in refresh_zone_stat_thresholds()
304 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold in refresh_zone_stat_thresholds()
321 void set_pgdat_percpu_threshold(pg_data_t *pgdat, in set_pgdat_percpu_threshold() argument
329 for (i = 0; i < pgdat->nr_zones; i++) { in set_pgdat_percpu_threshold()
330 zone = &pgdat->node_zones[i]; in set_pgdat_percpu_threshold()
377 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, in __mod_node_page_state() argument
[all …]
H A Dmmzone.c18 struct pglist_data *next_online_pgdat(struct pglist_data *pgdat) in next_online_pgdat() argument
20 int nid = next_online_node(pgdat->node_id); in next_online_pgdat()
32 pg_data_t *pgdat = zone->zone_pgdat; in next_zone() local
34 if (zone < pgdat->node_zones + MAX_NR_ZONES - 1) in next_zone()
37 pgdat = next_online_pgdat(pgdat); in next_zone()
38 if (pgdat) in next_zone()
39 zone = pgdat->node_zones; in next_zone()
H A Dworkingset.c202 static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, in pack_shadow() argument
207 eviction = (eviction << NODES_SHIFT) | pgdat->node_id; in pack_shadow()
213 static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, in unpack_shadow() argument
228 *pgdat = NODE_DATA(nid); in unpack_shadow()
248 struct pglist_data *pgdat = folio_pgdat(folio); in lru_gen_eviction() local
256 lruvec = mem_cgroup_lruvec(memcg, pgdat); in lru_gen_eviction()
266 return pack_shadow(memcg_id, pgdat, token, workingset, type); in lru_gen_eviction()
279 struct pglist_data *pgdat; in lru_gen_test_recent() local
281 unpack_shadow(shadow, &memcg_id, &pgdat, token, workingset); in lru_gen_test_recent()
284 *lruvec = mem_cgroup_lruvec(memcg, pgdat); in lru_gen_test_recent()
[all …]
H A Dshuffle.h13 extern void __shuffle_free_memory(pg_data_t *pgdat);
15 static inline void __meminit shuffle_free_memory(pg_data_t *pgdat) in shuffle_free_memory() argument
19 __shuffle_free_memory(pgdat); in shuffle_free_memory()
42 static inline void shuffle_free_memory(pg_data_t *pgdat) in shuffle_free_memory() argument
H A Dnuma.c37 pg_data_t *pgdat; in alloc_offline_node_data() local
38 node_data[nid] = memblock_alloc_or_panic(sizeof(*pgdat), SMP_CACHE_BYTES); in alloc_offline_node_data()
H A Dpage_alloc.c2929 struct pglist_data *pgdat = zone->zone_pgdat; in free_frozen_page_commit() local
2939 if (kswapd_test_hopeless(pgdat) && in free_frozen_page_commit()
2940 next_memory_node(pgdat->node_id) < MAX_NUMNODES) in free_frozen_page_commit()
2941 kswapd_clear_hopeless(pgdat, KSWAPD_CLEAR_HOPELESS_PCP); in free_frozen_page_commit()
5749 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) in build_zonerefs_node() argument
5757 zone = pgdat->node_zones + zone_type; in build_zonerefs_node()
5867 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, in build_zonelists_in_node_order() argument
5873 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; in build_zonelists_in_node_order()
5890 static void build_thisnode_zonelists(pg_data_t *pgdat) in build_thisnode_zonelists() argument
5895 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; in build_thisnode_zonelists()
[all …]
H A Dinternal.h90 pg_data_t *pgdat,
546 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
550 pg_data_t *pgdat = folio_pgdat(folio); in acct_reclaim_writeback() local
551 int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled); in acct_reclaim_writeback()
554 __acct_reclaim_writeback(pgdat, folio, nr_throttled); in acct_reclaim_writeback()
557 static inline void wake_throttle_isolated(pg_data_t *pgdat) in wake_throttle_isolated() argument
561 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED]; in wake_throttle_isolated()
699 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);
701 struct mem_cgroup *memcg, pg_data_t *pgdat);
1147 extern unsigned long node_reclaim(struct pglist_data *pgdat,
[all …]
H A Dpage-writeback.c243 static unsigned long node_dirtyable_memory(struct pglist_data *pgdat) in node_dirtyable_memory() argument
249 struct zone *zone = pgdat->node_zones + z; in node_dirtyable_memory()
262 nr_pages -= min(nr_pages, pgdat->totalreserve_pages); in node_dirtyable_memory()
264 nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE); in node_dirtyable_memory()
265 nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE); in node_dirtyable_memory()
437 static unsigned long node_dirty_limit(struct pglist_data *pgdat) in node_dirty_limit() argument
439 unsigned long node_memory = node_dirtyable_memory(pgdat); in node_dirty_limit()
466 bool node_dirty_ok(struct pglist_data *pgdat) in node_dirty_ok() argument
468 unsigned long limit = node_dirty_limit(pgdat); in node_dirty_ok()
471 nr_pages += node_page_state(pgdat, NR_FILE_DIRTY); in node_dirty_ok()
[all …]
H A Dmemcontrol-v1.c273 pg_data_t *pgdat, in mem_cgroup_soft_reclaim() argument
283 .pgdat = pgdat, in mem_cgroup_soft_reclaim()
313 pgdat, &nr_scanned); in mem_cgroup_soft_reclaim()
322 unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, in memcg1_soft_limit_reclaim() argument
339 mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; in memcg1_soft_limit_reclaim()
362 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, in memcg1_soft_limit_reclaim()
2292 pg_data_t *pgdat; in memcg1_stat_format() local
2297 for_each_online_pgdat(pgdat) { in memcg1_stat_format()
2298 mz = memcg->nodeinfo[pgdat->node_id]; in memcg1_stat_format()
H A Dmigrate.c2647 static bool migrate_balanced_pgdat(struct pglist_data *pgdat, in migrate_balanced_pgdat() argument
2652 for (z = pgdat->nr_zones - 1; z >= 0; z--) { in migrate_balanced_pgdat()
2653 struct zone *zone = pgdat->node_zones + z; in migrate_balanced_pgdat()
2694 pg_data_t *pgdat = NODE_DATA(node); in migrate_misplaced_folio_prepare() local
2718 if (!migrate_balanced_pgdat(pgdat, nr_pages)) { in migrate_misplaced_folio_prepare()
2723 for (z = pgdat->nr_zones - 1; z >= 0; z--) { in migrate_misplaced_folio_prepare()
2724 if (managed_zone(pgdat->node_zones + z)) in migrate_misplaced_folio_prepare()
2735 wakeup_kswapd(pgdat->node_zones + z, 0, in migrate_misplaced_folio_prepare()
2757 pg_data_t *pgdat = NODE_DATA(node); in migrate_misplaced_folio() local
2762 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); in migrate_misplaced_folio()
H A Dmemcontrol.c986 struct pglist_data *pgdat = lruvec_pgdat(lruvec); in mod_memcg_lruvec_state() local
993 pn = memcg->nodeinfo[pgdat->node_id]; in mod_memcg_lruvec_state()
1025 pg_data_t *pgdat = folio_pgdat(folio); in lruvec_stat_mod_folio() local
1033 mod_node_page_state(pgdat, idx, val); in lruvec_stat_mod_folio()
1037 lruvec = mem_cgroup_lruvec(memcg, pgdat); in lruvec_stat_mod_folio()
1045 pg_data_t *pgdat = page_pgdat(virt_to_page(p)); in mod_lruvec_kmem_state() local
1059 mod_node_page_state(pgdat, idx, val); in mod_lruvec_kmem_state()
1061 lruvec = mem_cgroup_lruvec(memcg, pgdat); in mod_lruvec_kmem_state()
1279 int nid = reclaim->pgdat->node_id; in mem_cgroup_iter()
2878 struct pglist_data *pgdat, in account_slab_nmi_safe() argument
[all …]
H A Dmemblock.c2470 static void __init reset_node_managed_pages(pg_data_t *pgdat) in reset_node_managed_pages() argument
2474 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) in reset_node_managed_pages()
2480 struct pglist_data *pgdat; in reset_all_zones_managed_pages() local
2485 for_each_online_pgdat(pgdat) in reset_all_zones_managed_pages()
2486 reset_node_managed_pages(pgdat); in reset_all_zones_managed_pages()
/linux/tools/testing/vma/linux/
H A Dmmzone.h9 struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
11 #define for_each_online_pgdat(pgdat) \ argument
12 for (pgdat = first_online_pgdat(); \
13 pgdat; \
14 pgdat = next_online_pgdat(pgdat))
/linux/tools/testing/memblock/linux/
H A Dmmzone.h9 struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
11 #define for_each_online_pgdat(pgdat) \ argument
12 for (pgdat = first_online_pgdat(); \
13 pgdat; \
14 pgdat = next_online_pgdat(pgdat))
/linux/include/linux/
H A Dvmstat.h168 static inline void node_page_state_add(long x, struct pglist_data *pgdat, in node_page_state_add() argument
171 atomic_long_add(x, &pgdat->vm_stat[item]); in node_page_state_add()
270 extern unsigned long node_page_state(struct pglist_data *pgdat,
272 extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
274 extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
317 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
332 static inline void __mod_node_page_state(struct pglist_data *pgdat, in __mod_node_page_state() argument
346 node_page_state_add(delta, pgdat, item); in __mod_node_page_state()
355 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) in __inc_node_state() argument
357 atomic_long_inc(&pgdat->vm_stat[item]); in __inc_node_state()
[all …]
H A Dmemcontrol.h61 pg_data_t *pgdat; member
706 struct pglist_data *pgdat) in mem_cgroup_lruvec() argument
712 lruvec = &pgdat->__lruvec; in mem_cgroup_lruvec()
719 mz = memcg->nodeinfo[pgdat->node_id]; in mem_cgroup_lruvec()
727 if (unlikely(lruvec->pgdat != pgdat)) in mem_cgroup_lruvec()
728 lruvec->pgdat = pgdat; in mem_cgroup_lruvec()
1188 struct pglist_data *pgdat) in mem_cgroup_lruvec() argument
1190 return &pgdat->__lruvec; in mem_cgroup_lruvec()
1195 struct pglist_data *pgdat = folio_pgdat(folio); in folio_lruvec() local
1196 return &pgdat->__lruvec; in folio_lruvec()
[all …]
H A Dmmzone.h690 void lru_gen_init_pgdat(struct pglist_data *pgdat);
706 static inline void lru_gen_init_pgdat(struct pglist_data *pgdat) in lru_gen_init_pgdat() argument
795 struct pglist_data *pgdat; member
1622 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) in pgdat_end_pfn() argument
1624 return pgdat->node_start_pfn + pgdat->node_spanned_pages; in pgdat_end_pfn()
1629 void build_all_zonelists(pg_data_t *pgdat);
1646 void kswapd_try_clear_hopeless(struct pglist_data *pgdat,
1648 void kswapd_clear_hopeless(pg_data_t *pgdat, enum kswapd_clear_hopeless_reason reason);
1649 bool kswapd_test_hopeless(pg_data_t *pgdat);
1668 return lruvec->pgdat; in lruvec_pgdat()
[all …]
H A Dswap.h546 #define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ argument
547 for ((idx) = 0, (zone) = (pgdat)->node_zones; \
/linux/mm/damon/
H A Dops-common.c309 struct list_head *migrate_folios, struct pglist_data *pgdat, in __damon_migrate_folio_list() argument
324 if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE) in __damon_migrate_folio_list()
339 struct pglist_data *pgdat, in damon_migrate_folio_list() argument
367 &migrate_folios, pgdat, target_nid); in damon_migrate_folio_list()
/linux/tools/perf/util/bpf_skel/
H A Dlock_contention.bpf.c956 struct pglist_data *pgdat = NULL;
959 err = bpf_core_read(&pgdat, sizeof(pgdat), &node_data[i]);
960 if (err < 0 || pgdat == NULL)
963 nr_zones = BPF_CORE_READ(pgdat, nr_zones); in slab_cache_iter()
970 zone_addr = (__u64)(void *)pgdat + (sizeof_zone * k) + zone_off; in slab_cache_iter()
893 struct pglist_data *pgdat = NULL; collect_zone_lock() local
/linux/tools/testing/memblock/
H A Dmmzone.c9 struct pglist_data *next_online_pgdat(struct pglist_data *pgdat) in next_online_pgdat() argument
/linux/kernel/sched/
H A Dfair.c2825 static bool pgdat_free_space_enough(struct pglist_data *pgdat) in should_numa_migrate_memory()
2831 pgdat->node_present_pages >> 4); in should_numa_migrate_memory()
2832 for (z = pgdat->nr_zones - 1; z >= 0; z--) { in should_numa_migrate_memory()
2833 struct zone *zone = pgdat->node_zones + z; in should_numa_migrate_memory()
2873 static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
2880 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
2881 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
2882 start = pgdat->nbp_rl_start;
2884 cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
2885 pgdat
2634 pgdat_free_space_enough(struct pglist_data * pgdat) pgdat_free_space_enough() argument
2682 numa_promotion_rate_limit(struct pglist_data * pgdat,unsigned long rate_limit,int nr) numa_promotion_rate_limit() argument
2702 numa_promotion_adjust_threshold(struct pglist_data * pgdat,unsigned long rate_limit,unsigned int ref_th) numa_promotion_adjust_threshold() argument
2747 struct pglist_data *pgdat; should_numa_migrate_memory() local
[all...]

12