1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common Code for Data Access Monitoring
4 */
5
6 #include <linux/migrate.h>
7 #include <linux/mmu_notifier.h>
8 #include <linux/page_idle.h>
9 #include <linux/pagemap.h>
10 #include <linux/rmap.h>
11 #include <linux/swap.h>
12 #include <linux/leafops.h>
13
14 #include "../internal.h"
15 #include "ops-common.h"
16
17 /*
18 * Get an online page for a pfn if it's in the LRU list. Otherwise, returns
19 * NULL.
20 *
21 * The body of this function is stolen from the 'page_idle_get_folio()'. We
22 * steal rather than reuse it because the code is quite simple.
23 */
damon_get_folio(unsigned long pfn)24 struct folio *damon_get_folio(unsigned long pfn)
25 {
26 struct page *page = pfn_to_online_page(pfn);
27 struct folio *folio;
28
29 if (!page)
30 return NULL;
31
32 folio = page_folio(page);
33 if (!folio_try_get(folio))
34 return NULL;
35 if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) {
36 folio_put(folio);
37 folio = NULL;
38 }
39 return folio;
40 }
41
damon_ptep_mkold(pte_t * pte,struct vm_area_struct * vma,unsigned long addr)42 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
43 {
44 pte_t pteval = ptep_get(pte);
45 struct folio *folio;
46 bool young = false;
47 unsigned long pfn;
48
49 if (likely(pte_present(pteval)))
50 pfn = pte_pfn(pteval);
51 else
52 pfn = softleaf_to_pfn(softleaf_from_pte(pteval));
53
54 folio = damon_get_folio(pfn);
55 if (!folio)
56 return;
57
58 /*
59 * PFN swap PTEs, such as device-exclusive ones, that actually map pages
60 * are "old" from a CPU perspective. The MMU notifier takes care of any
61 * device aspects.
62 */
63 if (likely(pte_present(pteval)))
64 /*
65 * Arch implementation of ptep_test_and_clear_young() may
66 * require aligned @addr
67 */
68 young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr),
69 pte);
70 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE);
71 if (young)
72 folio_set_young(folio);
73
74 folio_set_idle(folio);
75 folio_put(folio);
76 }
77
damon_pmdp_mkold(pmd_t * pmd,struct vm_area_struct * vma,unsigned long addr)78 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)
79 {
80 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
81 pmd_t pmdval = pmdp_get(pmd);
82 struct folio *folio;
83 bool young = false;
84 unsigned long pfn;
85
86 if (likely(pmd_present(pmdval)))
87 pfn = pmd_pfn(pmdval);
88 else
89 pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval));
90
91 folio = damon_get_folio(pfn);
92 if (!folio)
93 return;
94
95 if (likely(pmd_present(pmdval)))
96 young |= pmdp_test_and_clear_young(vma, addr, pmd);
97 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE);
98 if (young)
99 folio_set_young(folio);
100
101 folio_set_idle(folio);
102 folio_put(folio);
103 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
104 }
105
106 #define DAMON_MAX_SUBSCORE (100)
107 #define DAMON_MAX_AGE_IN_LOG (32)
108
damon_hot_score(struct damon_ctx * c,struct damon_region * r,struct damos * s)109 int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
110 struct damos *s)
111 {
112 int freq_subscore;
113 unsigned int age_in_sec;
114 int age_in_log, age_subscore;
115 unsigned int freq_weight = s->quota.weight_nr_accesses;
116 unsigned int age_weight = s->quota.weight_age;
117 int hotness;
118
119 freq_subscore = mult_frac(damon_nr_accesses_mvsum(r, c),
120 DAMON_MAX_SUBSCORE,
121 damon_nr_samples_per_aggr(&c->attrs));
122
123 age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
124 if (age_in_sec)
125 age_in_log = min_t(int, ilog2(age_in_sec) + 1,
126 DAMON_MAX_AGE_IN_LOG);
127 else
128 age_in_log = 0;
129
130
131 /* If frequency is 0, higher age means it's colder */
132 if (freq_subscore == 0)
133 age_in_log *= -1;
134
135 /*
136 * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
137 * Scale it to be in [0, 100] and set it as age subscore.
138 */
139 age_in_log += DAMON_MAX_AGE_IN_LOG;
140 age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
141 DAMON_MAX_AGE_IN_LOG / 2;
142
143 hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
144 if (freq_weight + age_weight)
145 hotness /= freq_weight + age_weight;
146 /*
147 * Transform it to fit in [0, DAMOS_MAX_SCORE]
148 */
149 hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
150 hotness = max(min(hotness, DAMOS_MAX_SCORE), 0);
151
152 return hotness;
153 }
154
damon_cold_score(struct damon_ctx * c,struct damon_region * r,struct damos * s)155 int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
156 struct damos *s)
157 {
158 int hotness = damon_hot_score(c, r, s);
159
160 /* Return coldness of the region */
161 return DAMOS_MAX_SCORE - hotness;
162 }
163
damon_folio_mkold_one(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)164 static bool damon_folio_mkold_one(struct folio *folio,
165 struct vm_area_struct *vma, unsigned long addr, void *arg)
166 {
167 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
168
169 while (page_vma_mapped_walk(&pvmw)) {
170 addr = pvmw.address;
171 if (pvmw.pte)
172 damon_ptep_mkold(pvmw.pte, vma, addr);
173 else
174 damon_pmdp_mkold(pvmw.pmd, vma, addr);
175 }
176 return true;
177 }
178
damon_folio_mkold(struct folio * folio)179 void damon_folio_mkold(struct folio *folio)
180 {
181 struct rmap_walk_control rwc = {
182 .rmap_one = damon_folio_mkold_one,
183 .anon_lock = folio_lock_anon_vma_read,
184 };
185
186 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
187 folio_set_idle(folio);
188 return;
189 }
190
191 if (!folio_trylock(folio))
192 return;
193
194 rmap_walk(folio, &rwc);
195 folio_unlock(folio);
196
197 }
198
damon_folio_young_one(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)199 static bool damon_folio_young_one(struct folio *folio,
200 struct vm_area_struct *vma, unsigned long addr, void *arg)
201 {
202 bool *accessed = arg;
203 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
204 pte_t pte;
205
206 *accessed = false;
207 while (page_vma_mapped_walk(&pvmw)) {
208 addr = pvmw.address;
209 if (pvmw.pte) {
210 pte = ptep_get(pvmw.pte);
211
212 /*
213 * PFN swap PTEs, such as device-exclusive ones, that
214 * actually map pages are "old" from a CPU perspective.
215 * The MMU notifier takes care of any device aspects.
216 */
217 *accessed = (pte_present(pte) && pte_young(pte)) ||
218 !folio_test_idle(folio) ||
219 mmu_notifier_test_young(vma->vm_mm, addr);
220 } else {
221 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
222 pmd_t pmd = pmdp_get(pvmw.pmd);
223
224 *accessed = (pmd_present(pmd) && pmd_young(pmd)) ||
225 !folio_test_idle(folio) ||
226 mmu_notifier_test_young(vma->vm_mm, addr);
227 #else
228 WARN_ON_ONCE(1);
229 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
230 }
231 if (*accessed) {
232 page_vma_mapped_walk_done(&pvmw);
233 break;
234 }
235 }
236
237 /* If accessed, stop walking */
238 return *accessed == false;
239 }
240
damon_folio_young(struct folio * folio)241 bool damon_folio_young(struct folio *folio)
242 {
243 bool accessed = false;
244 struct rmap_walk_control rwc = {
245 .arg = &accessed,
246 .rmap_one = damon_folio_young_one,
247 .anon_lock = folio_lock_anon_vma_read,
248 };
249
250 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
251 if (folio_test_idle(folio))
252 return false;
253 else
254 return true;
255 }
256
257 if (!folio_trylock(folio))
258 return false;
259
260 rmap_walk(folio, &rwc);
261 folio_unlock(folio);
262
263 return accessed;
264 }
265
damos_folio_filter_match(struct damos_filter * filter,struct folio * folio)266 bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio)
267 {
268 bool matched = false;
269 struct mem_cgroup *memcg;
270 size_t folio_sz;
271
272 switch (filter->type) {
273 case DAMOS_FILTER_TYPE_ANON:
274 matched = folio_test_anon(folio);
275 break;
276 case DAMOS_FILTER_TYPE_ACTIVE:
277 matched = folio_test_active(folio);
278 break;
279 case DAMOS_FILTER_TYPE_MEMCG:
280 rcu_read_lock();
281 memcg = folio_memcg_check(folio);
282 if (!memcg)
283 matched = false;
284 else
285 matched = filter->memcg_id == mem_cgroup_id(memcg);
286 rcu_read_unlock();
287 break;
288 case DAMOS_FILTER_TYPE_YOUNG:
289 matched = damon_folio_young(folio);
290 if (matched)
291 damon_folio_mkold(folio);
292 break;
293 case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:
294 folio_sz = folio_size(folio);
295 matched = filter->sz_range.min <= folio_sz &&
296 folio_sz <= filter->sz_range.max;
297 break;
298 case DAMOS_FILTER_TYPE_UNMAPPED:
299 matched = !folio_mapped(folio) || !folio_raw_mapping(folio);
300 break;
301 default:
302 break;
303 }
304
305 return matched == filter->matching;
306 }
307
__damon_migrate_folio_list(struct list_head * migrate_folios,struct pglist_data * pgdat,int target_nid)308 static unsigned int __damon_migrate_folio_list(
309 struct list_head *migrate_folios, struct pglist_data *pgdat,
310 int target_nid)
311 {
312 unsigned int nr_succeeded = 0;
313 struct migration_target_control mtc = {
314 /*
315 * Allocate from 'node', or fail quickly and quietly.
316 * When this happens, 'page' will likely just be discarded
317 * instead of migrated.
318 */
319 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
320 __GFP_NOMEMALLOC | GFP_NOWAIT | __GFP_THISNODE,
321 .nid = target_nid,
322 };
323
324 if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE)
325 return 0;
326
327 if (list_empty(migrate_folios))
328 return 0;
329
330 /* Migration ignores all cpuset and mempolicy settings */
331 migrate_pages(migrate_folios, alloc_migration_target, NULL,
332 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON,
333 &nr_succeeded);
334
335 return nr_succeeded;
336 }
337
damon_migrate_folio_list(struct list_head * folio_list,struct pglist_data * pgdat,int target_nid)338 static unsigned int damon_migrate_folio_list(struct list_head *folio_list,
339 struct pglist_data *pgdat,
340 int target_nid)
341 {
342 unsigned int nr_migrated = 0;
343 struct folio *folio;
344 LIST_HEAD(ret_folios);
345 LIST_HEAD(migrate_folios);
346
347 while (!list_empty(folio_list)) {
348 cond_resched();
349
350 folio = lru_to_folio(folio_list);
351 list_del(&folio->lru);
352
353 if (!folio_trylock(folio))
354 goto keep;
355
356 /* Relocate its contents to another node. */
357 list_add(&folio->lru, &migrate_folios);
358 folio_unlock(folio);
359 continue;
360 keep:
361 list_add(&folio->lru, &ret_folios);
362 }
363 /* 'folio_list' is always empty here */
364
365 /* Migrate folios selected for migration */
366 nr_migrated += __damon_migrate_folio_list(
367 &migrate_folios, pgdat, target_nid);
368 /*
369 * Folios that could not be migrated are still in @migrate_folios. Add
370 * those back on @folio_list
371 */
372 if (!list_empty(&migrate_folios))
373 list_splice_init(&migrate_folios, folio_list);
374
375 try_to_unmap_flush();
376
377 list_splice(&ret_folios, folio_list);
378
379 while (!list_empty(folio_list)) {
380 folio = lru_to_folio(folio_list);
381 list_del(&folio->lru);
382 node_stat_sub_folio(folio, NR_ISOLATED_ANON +
383 folio_is_file_lru(folio));
384 folio_putback_lru(folio);
385 }
386
387 return nr_migrated;
388 }
389
damon_migrate_pages(struct list_head * folio_list,int target_nid)390 unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid)
391 {
392 int nid;
393 unsigned long nr_migrated = 0;
394 LIST_HEAD(node_folio_list);
395 unsigned int noreclaim_flag;
396
397 if (list_empty(folio_list))
398 return nr_migrated;
399
400 if (target_nid < 0 || target_nid >= MAX_NUMNODES ||
401 !node_state(target_nid, N_MEMORY)) {
402 while (!list_empty(folio_list)) {
403 struct folio *folio = lru_to_folio(folio_list);
404
405 list_del(&folio->lru);
406 node_stat_sub_folio(folio, NR_ISOLATED_ANON +
407 folio_is_file_lru(folio));
408 folio_putback_lru(folio);
409 }
410 return nr_migrated;
411 }
412
413 noreclaim_flag = memalloc_noreclaim_save();
414
415 nid = folio_nid(lru_to_folio(folio_list));
416 do {
417 struct folio *folio = lru_to_folio(folio_list);
418
419 if (nid == folio_nid(folio)) {
420 list_move(&folio->lru, &node_folio_list);
421 continue;
422 }
423
424 nr_migrated += damon_migrate_folio_list(&node_folio_list,
425 NODE_DATA(nid),
426 target_nid);
427 nid = folio_nid(lru_to_folio(folio_list));
428 } while (!list_empty(folio_list));
429
430 nr_migrated += damon_migrate_folio_list(&node_folio_list,
431 NODE_DATA(nid),
432 target_nid);
433
434 memalloc_noreclaim_restore(noreclaim_flag);
435
436 return nr_migrated;
437 }
438
damos_ops_has_filter(struct damos * s)439 bool damos_ops_has_filter(struct damos *s)
440 {
441 struct damos_filter *f;
442
443 damos_for_each_ops_filter(f, s)
444 return true;
445 return false;
446 }
447