1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common Code for Data Access Monitoring 4 */ 5 6 #include <linux/migrate.h> 7 #include <linux/mmu_notifier.h> 8 #include <linux/page_idle.h> 9 #include <linux/pagemap.h> 10 #include <linux/rmap.h> 11 #include <linux/swap.h> 12 #include <linux/leafops.h> 13 14 #include "../internal.h" 15 #include "ops-common.h" 16 17 /* 18 * Get an online page for a pfn if it's in the LRU list. Otherwise, returns 19 * NULL. 20 * 21 * The body of this function is stolen from the 'page_idle_get_folio()'. We 22 * steal rather than reuse it because the code is quite simple. 23 */ 24 struct folio *damon_get_folio(unsigned long pfn) 25 { 26 struct page *page = pfn_to_online_page(pfn); 27 struct folio *folio; 28 29 if (!page) 30 return NULL; 31 32 folio = page_folio(page); 33 if (!folio_try_get(folio)) 34 return NULL; 35 if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { 36 folio_put(folio); 37 folio = NULL; 38 } 39 return folio; 40 } 41 42 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr) 43 { 44 pte_t pteval = ptep_get(pte); 45 struct folio *folio; 46 bool young = false; 47 unsigned long pfn; 48 49 if (likely(pte_present(pteval))) 50 pfn = pte_pfn(pteval); 51 else 52 pfn = softleaf_to_pfn(softleaf_from_pte(pteval)); 53 54 folio = damon_get_folio(pfn); 55 if (!folio) 56 return; 57 58 /* 59 * PFN swap PTEs, such as device-exclusive ones, that actually map pages 60 * are "old" from a CPU perspective. The MMU notifier takes care of any 61 * device aspects. 62 */ 63 if (likely(pte_present(pteval))) 64 /* 65 * Arch implementation of ptep_test_and_clear_young() may 66 * require aligned @addr 67 */ 68 young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr), 69 pte); 70 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); 71 if (young) 72 folio_set_young(folio); 73 74 folio_set_idle(folio); 75 folio_put(folio); 76 } 77 78 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr) 79 { 80 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 81 pmd_t pmdval = pmdp_get(pmd); 82 struct folio *folio; 83 bool young = false; 84 unsigned long pfn; 85 86 if (likely(pmd_present(pmdval))) 87 pfn = pmd_pfn(pmdval); 88 else 89 pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval)); 90 91 folio = damon_get_folio(pfn); 92 if (!folio) 93 return; 94 95 if (likely(pmd_present(pmdval))) 96 young |= pmdp_test_and_clear_young(vma, addr, pmd); 97 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE); 98 if (young) 99 folio_set_young(folio); 100 101 folio_set_idle(folio); 102 folio_put(folio); 103 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 104 } 105 106 #define DAMON_MAX_SUBSCORE (100) 107 #define DAMON_MAX_AGE_IN_LOG (32) 108 109 int damon_hot_score(struct damon_ctx *c, struct damon_region *r, 110 struct damos *s) 111 { 112 int freq_subscore; 113 unsigned int age_in_sec; 114 int age_in_log, age_subscore; 115 unsigned int freq_weight = s->quota.weight_nr_accesses; 116 unsigned int age_weight = s->quota.weight_age; 117 int hotness; 118 119 freq_subscore = mult_frac(damon_nr_accesses_mvsum(r, c), 120 DAMON_MAX_SUBSCORE, 121 damon_nr_samples_per_aggr(&c->attrs)); 122 123 age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; 124 if (age_in_sec) 125 age_in_log = min_t(int, ilog2(age_in_sec) + 1, 126 DAMON_MAX_AGE_IN_LOG); 127 else 128 age_in_log = 0; 129 130 131 /* If frequency is 0, higher age means it's colder */ 132 if (freq_subscore == 0) 133 age_in_log *= -1; 134 135 /* 136 * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. 137 * Scale it to be in [0, 100] and set it as age subscore. 138 */ 139 age_in_log += DAMON_MAX_AGE_IN_LOG; 140 age_subscore = age_in_log * DAMON_MAX_SUBSCORE / 141 DAMON_MAX_AGE_IN_LOG / 2; 142 143 hotness = (freq_weight * freq_subscore + age_weight * age_subscore); 144 if (freq_weight + age_weight) 145 hotness /= freq_weight + age_weight; 146 /* 147 * Transform it to fit in [0, DAMOS_MAX_SCORE] 148 */ 149 hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; 150 hotness = max(min(hotness, DAMOS_MAX_SCORE), 0); 151 152 return hotness; 153 } 154 155 int damon_cold_score(struct damon_ctx *c, struct damon_region *r, 156 struct damos *s) 157 { 158 int hotness = damon_hot_score(c, r, s); 159 160 /* Return coldness of the region */ 161 return DAMOS_MAX_SCORE - hotness; 162 } 163 164 static bool damon_folio_mkold_one(struct folio *folio, 165 struct vm_area_struct *vma, unsigned long addr, void *arg) 166 { 167 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 168 169 while (page_vma_mapped_walk(&pvmw)) { 170 addr = pvmw.address; 171 if (pvmw.pte) 172 damon_ptep_mkold(pvmw.pte, vma, addr); 173 else 174 damon_pmdp_mkold(pvmw.pmd, vma, addr); 175 } 176 return true; 177 } 178 179 void damon_folio_mkold(struct folio *folio) 180 { 181 struct rmap_walk_control rwc = { 182 .rmap_one = damon_folio_mkold_one, 183 .anon_lock = folio_lock_anon_vma_read, 184 }; 185 186 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 187 folio_set_idle(folio); 188 return; 189 } 190 191 if (!folio_trylock(folio)) 192 return; 193 194 rmap_walk(folio, &rwc); 195 folio_unlock(folio); 196 197 } 198 199 static bool damon_folio_young_one(struct folio *folio, 200 struct vm_area_struct *vma, unsigned long addr, void *arg) 201 { 202 bool *accessed = arg; 203 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 204 pte_t pte; 205 206 *accessed = false; 207 while (page_vma_mapped_walk(&pvmw)) { 208 addr = pvmw.address; 209 if (pvmw.pte) { 210 pte = ptep_get(pvmw.pte); 211 212 /* 213 * PFN swap PTEs, such as device-exclusive ones, that 214 * actually map pages are "old" from a CPU perspective. 215 * The MMU notifier takes care of any device aspects. 216 */ 217 *accessed = (pte_present(pte) && pte_young(pte)) || 218 !folio_test_idle(folio) || 219 mmu_notifier_test_young(vma->vm_mm, addr); 220 } else { 221 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 222 pmd_t pmd = pmdp_get(pvmw.pmd); 223 224 *accessed = (pmd_present(pmd) && pmd_young(pmd)) || 225 !folio_test_idle(folio) || 226 mmu_notifier_test_young(vma->vm_mm, addr); 227 #else 228 WARN_ON_ONCE(1); 229 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 230 } 231 if (*accessed) { 232 page_vma_mapped_walk_done(&pvmw); 233 break; 234 } 235 } 236 237 /* If accessed, stop walking */ 238 return *accessed == false; 239 } 240 241 bool damon_folio_young(struct folio *folio) 242 { 243 bool accessed = false; 244 struct rmap_walk_control rwc = { 245 .arg = &accessed, 246 .rmap_one = damon_folio_young_one, 247 .anon_lock = folio_lock_anon_vma_read, 248 }; 249 250 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 251 if (folio_test_idle(folio)) 252 return false; 253 else 254 return true; 255 } 256 257 if (!folio_trylock(folio)) 258 return false; 259 260 rmap_walk(folio, &rwc); 261 folio_unlock(folio); 262 263 return accessed; 264 } 265 266 bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio) 267 { 268 bool matched = false; 269 struct mem_cgroup *memcg; 270 size_t folio_sz; 271 272 switch (filter->type) { 273 case DAMOS_FILTER_TYPE_ANON: 274 matched = folio_test_anon(folio); 275 break; 276 case DAMOS_FILTER_TYPE_ACTIVE: 277 matched = folio_test_active(folio); 278 break; 279 case DAMOS_FILTER_TYPE_MEMCG: 280 rcu_read_lock(); 281 memcg = folio_memcg_check(folio); 282 if (!memcg) 283 matched = false; 284 else 285 matched = filter->memcg_id == mem_cgroup_id(memcg); 286 rcu_read_unlock(); 287 break; 288 case DAMOS_FILTER_TYPE_YOUNG: 289 matched = damon_folio_young(folio); 290 if (matched) 291 damon_folio_mkold(folio); 292 break; 293 case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: 294 folio_sz = folio_size(folio); 295 matched = filter->sz_range.min <= folio_sz && 296 folio_sz <= filter->sz_range.max; 297 break; 298 case DAMOS_FILTER_TYPE_UNMAPPED: 299 matched = !folio_mapped(folio) || !folio_raw_mapping(folio); 300 break; 301 default: 302 break; 303 } 304 305 return matched == filter->matching; 306 } 307 308 static unsigned int __damon_migrate_folio_list( 309 struct list_head *migrate_folios, struct pglist_data *pgdat, 310 int target_nid) 311 { 312 unsigned int nr_succeeded = 0; 313 struct migration_target_control mtc = { 314 /* 315 * Allocate from 'node', or fail quickly and quietly. 316 * When this happens, 'page' will likely just be discarded 317 * instead of migrated. 318 */ 319 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | 320 __GFP_NOMEMALLOC | GFP_NOWAIT | __GFP_THISNODE, 321 .nid = target_nid, 322 }; 323 324 if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE) 325 return 0; 326 327 if (list_empty(migrate_folios)) 328 return 0; 329 330 /* Migration ignores all cpuset and mempolicy settings */ 331 migrate_pages(migrate_folios, alloc_migration_target, NULL, 332 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON, 333 &nr_succeeded); 334 335 return nr_succeeded; 336 } 337 338 static unsigned int damon_migrate_folio_list(struct list_head *folio_list, 339 struct pglist_data *pgdat, 340 int target_nid) 341 { 342 unsigned int nr_migrated = 0; 343 struct folio *folio; 344 LIST_HEAD(ret_folios); 345 LIST_HEAD(migrate_folios); 346 347 while (!list_empty(folio_list)) { 348 cond_resched(); 349 350 folio = lru_to_folio(folio_list); 351 list_del(&folio->lru); 352 353 if (!folio_trylock(folio)) 354 goto keep; 355 356 /* Relocate its contents to another node. */ 357 list_add(&folio->lru, &migrate_folios); 358 folio_unlock(folio); 359 continue; 360 keep: 361 list_add(&folio->lru, &ret_folios); 362 } 363 /* 'folio_list' is always empty here */ 364 365 /* Migrate folios selected for migration */ 366 nr_migrated += __damon_migrate_folio_list( 367 &migrate_folios, pgdat, target_nid); 368 /* 369 * Folios that could not be migrated are still in @migrate_folios. Add 370 * those back on @folio_list 371 */ 372 if (!list_empty(&migrate_folios)) 373 list_splice_init(&migrate_folios, folio_list); 374 375 try_to_unmap_flush(); 376 377 list_splice(&ret_folios, folio_list); 378 379 while (!list_empty(folio_list)) { 380 folio = lru_to_folio(folio_list); 381 list_del(&folio->lru); 382 node_stat_sub_folio(folio, NR_ISOLATED_ANON + 383 folio_is_file_lru(folio)); 384 folio_putback_lru(folio); 385 } 386 387 return nr_migrated; 388 } 389 390 unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) 391 { 392 int nid; 393 unsigned long nr_migrated = 0; 394 LIST_HEAD(node_folio_list); 395 unsigned int noreclaim_flag; 396 397 if (list_empty(folio_list)) 398 return nr_migrated; 399 400 if (target_nid < 0 || target_nid >= MAX_NUMNODES || 401 !node_state(target_nid, N_MEMORY)) { 402 while (!list_empty(folio_list)) { 403 struct folio *folio = lru_to_folio(folio_list); 404 405 list_del(&folio->lru); 406 node_stat_sub_folio(folio, NR_ISOLATED_ANON + 407 folio_is_file_lru(folio)); 408 folio_putback_lru(folio); 409 } 410 return nr_migrated; 411 } 412 413 noreclaim_flag = memalloc_noreclaim_save(); 414 415 nid = folio_nid(lru_to_folio(folio_list)); 416 do { 417 struct folio *folio = lru_to_folio(folio_list); 418 419 if (nid == folio_nid(folio)) { 420 list_move(&folio->lru, &node_folio_list); 421 continue; 422 } 423 424 nr_migrated += damon_migrate_folio_list(&node_folio_list, 425 NODE_DATA(nid), 426 target_nid); 427 nid = folio_nid(lru_to_folio(folio_list)); 428 } while (!list_empty(folio_list)); 429 430 nr_migrated += damon_migrate_folio_list(&node_folio_list, 431 NODE_DATA(nid), 432 target_nid); 433 434 memalloc_noreclaim_restore(noreclaim_flag); 435 436 return nr_migrated; 437 } 438 439 bool damos_ops_has_filter(struct damos *s) 440 { 441 struct damos_filter *f; 442 443 damos_for_each_ops_filter(f, s) 444 return true; 445 return false; 446 } 447