1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common Code for Data Access Monitoring 4 */ 5 6 #include <linux/migrate.h> 7 #include <linux/mmu_notifier.h> 8 #include <linux/page_idle.h> 9 #include <linux/pagemap.h> 10 #include <linux/rmap.h> 11 #include <linux/swap.h> 12 #include <linux/leafops.h> 13 14 #include "../internal.h" 15 #include "ops-common.h" 16 17 /* 18 * Get an online page for a pfn if it's in the LRU list. Otherwise, returns 19 * NULL. 20 * 21 * The body of this function is stolen from the 'page_idle_get_folio()'. We 22 * steal rather than reuse it because the code is quite simple. 23 */ 24 struct folio *damon_get_folio(unsigned long pfn) 25 { 26 struct page *page = pfn_to_online_page(pfn); 27 struct folio *folio; 28 29 if (!page) 30 return NULL; 31 32 folio = page_folio(page); 33 if (!folio_try_get(folio)) 34 return NULL; 35 if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { 36 folio_put(folio); 37 folio = NULL; 38 } 39 return folio; 40 } 41 42 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr) 43 { 44 pte_t pteval = ptep_get(pte); 45 struct folio *folio; 46 bool young = false; 47 unsigned long pfn; 48 49 if (likely(pte_present(pteval))) 50 pfn = pte_pfn(pteval); 51 else 52 pfn = softleaf_to_pfn(softleaf_from_pte(pteval)); 53 54 folio = damon_get_folio(pfn); 55 if (!folio) 56 return; 57 58 /* 59 * PFN swap PTEs, such as device-exclusive ones, that actually map pages 60 * are "old" from a CPU perspective. The MMU notifier takes care of any 61 * device aspects. 62 */ 63 if (likely(pte_present(pteval))) 64 young |= ptep_test_and_clear_young(vma, addr, pte); 65 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); 66 if (young) 67 folio_set_young(folio); 68 69 folio_set_idle(folio); 70 folio_put(folio); 71 } 72 73 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr) 74 { 75 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 76 pmd_t pmdval = pmdp_get(pmd); 77 struct folio *folio; 78 bool young = false; 79 unsigned long pfn; 80 81 if (likely(pmd_present(pmdval))) 82 pfn = pmd_pfn(pmdval); 83 else 84 pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval)); 85 86 folio = damon_get_folio(pfn); 87 if (!folio) 88 return; 89 90 if (likely(pmd_present(pmdval))) 91 young |= pmdp_test_and_clear_young(vma, addr, pmd); 92 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE); 93 if (young) 94 folio_set_young(folio); 95 96 folio_set_idle(folio); 97 folio_put(folio); 98 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 99 } 100 101 #define DAMON_MAX_SUBSCORE (100) 102 #define DAMON_MAX_AGE_IN_LOG (32) 103 104 int damon_hot_score(struct damon_ctx *c, struct damon_region *r, 105 struct damos *s) 106 { 107 int freq_subscore; 108 unsigned int age_in_sec; 109 int age_in_log, age_subscore; 110 unsigned int freq_weight = s->quota.weight_nr_accesses; 111 unsigned int age_weight = s->quota.weight_age; 112 int hotness; 113 114 freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / 115 damon_max_nr_accesses(&c->attrs); 116 117 age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; 118 if (age_in_sec) 119 age_in_log = min_t(int, ilog2(age_in_sec) + 1, 120 DAMON_MAX_AGE_IN_LOG); 121 else 122 age_in_log = 0; 123 124 125 /* If frequency is 0, higher age means it's colder */ 126 if (freq_subscore == 0) 127 age_in_log *= -1; 128 129 /* 130 * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. 131 * Scale it to be in [0, 100] and set it as age subscore. 132 */ 133 age_in_log += DAMON_MAX_AGE_IN_LOG; 134 age_subscore = age_in_log * DAMON_MAX_SUBSCORE / 135 DAMON_MAX_AGE_IN_LOG / 2; 136 137 hotness = (freq_weight * freq_subscore + age_weight * age_subscore); 138 if (freq_weight + age_weight) 139 hotness /= freq_weight + age_weight; 140 /* 141 * Transform it to fit in [0, DAMOS_MAX_SCORE] 142 */ 143 hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; 144 hotness = max(min(hotness, DAMOS_MAX_SCORE), 0); 145 146 return hotness; 147 } 148 149 int damon_cold_score(struct damon_ctx *c, struct damon_region *r, 150 struct damos *s) 151 { 152 int hotness = damon_hot_score(c, r, s); 153 154 /* Return coldness of the region */ 155 return DAMOS_MAX_SCORE - hotness; 156 } 157 158 static bool damon_folio_mkold_one(struct folio *folio, 159 struct vm_area_struct *vma, unsigned long addr, void *arg) 160 { 161 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 162 163 while (page_vma_mapped_walk(&pvmw)) { 164 addr = pvmw.address; 165 if (pvmw.pte) 166 damon_ptep_mkold(pvmw.pte, vma, addr); 167 else 168 damon_pmdp_mkold(pvmw.pmd, vma, addr); 169 } 170 return true; 171 } 172 173 void damon_folio_mkold(struct folio *folio) 174 { 175 struct rmap_walk_control rwc = { 176 .rmap_one = damon_folio_mkold_one, 177 .anon_lock = folio_lock_anon_vma_read, 178 }; 179 180 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 181 folio_set_idle(folio); 182 return; 183 } 184 185 if (!folio_trylock(folio)) 186 return; 187 188 rmap_walk(folio, &rwc); 189 folio_unlock(folio); 190 191 } 192 193 static bool damon_folio_young_one(struct folio *folio, 194 struct vm_area_struct *vma, unsigned long addr, void *arg) 195 { 196 bool *accessed = arg; 197 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 198 pte_t pte; 199 200 *accessed = false; 201 while (page_vma_mapped_walk(&pvmw)) { 202 addr = pvmw.address; 203 if (pvmw.pte) { 204 pte = ptep_get(pvmw.pte); 205 206 /* 207 * PFN swap PTEs, such as device-exclusive ones, that 208 * actually map pages are "old" from a CPU perspective. 209 * The MMU notifier takes care of any device aspects. 210 */ 211 *accessed = (pte_present(pte) && pte_young(pte)) || 212 !folio_test_idle(folio) || 213 mmu_notifier_test_young(vma->vm_mm, addr); 214 } else { 215 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 216 pmd_t pmd = pmdp_get(pvmw.pmd); 217 218 *accessed = (pmd_present(pmd) && pmd_young(pmd)) || 219 !folio_test_idle(folio) || 220 mmu_notifier_test_young(vma->vm_mm, addr); 221 #else 222 WARN_ON_ONCE(1); 223 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 224 } 225 if (*accessed) { 226 page_vma_mapped_walk_done(&pvmw); 227 break; 228 } 229 } 230 231 /* If accessed, stop walking */ 232 return *accessed == false; 233 } 234 235 bool damon_folio_young(struct folio *folio) 236 { 237 bool accessed = false; 238 struct rmap_walk_control rwc = { 239 .arg = &accessed, 240 .rmap_one = damon_folio_young_one, 241 .anon_lock = folio_lock_anon_vma_read, 242 }; 243 244 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 245 if (folio_test_idle(folio)) 246 return false; 247 else 248 return true; 249 } 250 251 if (!folio_trylock(folio)) 252 return false; 253 254 rmap_walk(folio, &rwc); 255 folio_unlock(folio); 256 257 return accessed; 258 } 259 260 bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio) 261 { 262 bool matched = false; 263 struct mem_cgroup *memcg; 264 size_t folio_sz; 265 266 switch (filter->type) { 267 case DAMOS_FILTER_TYPE_ANON: 268 matched = folio_test_anon(folio); 269 break; 270 case DAMOS_FILTER_TYPE_ACTIVE: 271 matched = folio_test_active(folio); 272 break; 273 case DAMOS_FILTER_TYPE_MEMCG: 274 rcu_read_lock(); 275 memcg = folio_memcg_check(folio); 276 if (!memcg) 277 matched = false; 278 else 279 matched = filter->memcg_id == mem_cgroup_id(memcg); 280 rcu_read_unlock(); 281 break; 282 case DAMOS_FILTER_TYPE_YOUNG: 283 matched = damon_folio_young(folio); 284 if (matched) 285 damon_folio_mkold(folio); 286 break; 287 case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: 288 folio_sz = folio_size(folio); 289 matched = filter->sz_range.min <= folio_sz && 290 folio_sz <= filter->sz_range.max; 291 break; 292 case DAMOS_FILTER_TYPE_UNMAPPED: 293 matched = !folio_mapped(folio) || !folio_raw_mapping(folio); 294 break; 295 default: 296 break; 297 } 298 299 return matched == filter->matching; 300 } 301 302 static unsigned int __damon_migrate_folio_list( 303 struct list_head *migrate_folios, struct pglist_data *pgdat, 304 int target_nid) 305 { 306 unsigned int nr_succeeded = 0; 307 struct migration_target_control mtc = { 308 /* 309 * Allocate from 'node', or fail quickly and quietly. 310 * When this happens, 'page' will likely just be discarded 311 * instead of migrated. 312 */ 313 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | 314 __GFP_NOMEMALLOC | GFP_NOWAIT, 315 .nid = target_nid, 316 }; 317 318 if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE) 319 return 0; 320 321 if (list_empty(migrate_folios)) 322 return 0; 323 324 /* Migration ignores all cpuset and mempolicy settings */ 325 migrate_pages(migrate_folios, alloc_migration_target, NULL, 326 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON, 327 &nr_succeeded); 328 329 return nr_succeeded; 330 } 331 332 static unsigned int damon_migrate_folio_list(struct list_head *folio_list, 333 struct pglist_data *pgdat, 334 int target_nid) 335 { 336 unsigned int nr_migrated = 0; 337 struct folio *folio; 338 LIST_HEAD(ret_folios); 339 LIST_HEAD(migrate_folios); 340 341 while (!list_empty(folio_list)) { 342 struct folio *folio; 343 344 cond_resched(); 345 346 folio = lru_to_folio(folio_list); 347 list_del(&folio->lru); 348 349 if (!folio_trylock(folio)) 350 goto keep; 351 352 /* Relocate its contents to another node. */ 353 list_add(&folio->lru, &migrate_folios); 354 folio_unlock(folio); 355 continue; 356 keep: 357 list_add(&folio->lru, &ret_folios); 358 } 359 /* 'folio_list' is always empty here */ 360 361 /* Migrate folios selected for migration */ 362 nr_migrated += __damon_migrate_folio_list( 363 &migrate_folios, pgdat, target_nid); 364 /* 365 * Folios that could not be migrated are still in @migrate_folios. Add 366 * those back on @folio_list 367 */ 368 if (!list_empty(&migrate_folios)) 369 list_splice_init(&migrate_folios, folio_list); 370 371 try_to_unmap_flush(); 372 373 list_splice(&ret_folios, folio_list); 374 375 while (!list_empty(folio_list)) { 376 folio = lru_to_folio(folio_list); 377 list_del(&folio->lru); 378 node_stat_sub_folio(folio, NR_ISOLATED_ANON + 379 folio_is_file_lru(folio)); 380 folio_putback_lru(folio); 381 } 382 383 return nr_migrated; 384 } 385 386 unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) 387 { 388 int nid; 389 unsigned long nr_migrated = 0; 390 LIST_HEAD(node_folio_list); 391 unsigned int noreclaim_flag; 392 393 if (list_empty(folio_list)) 394 return nr_migrated; 395 396 if (target_nid < 0 || target_nid >= MAX_NUMNODES || 397 !node_state(target_nid, N_MEMORY)) { 398 while (!list_empty(folio_list)) { 399 struct folio *folio = lru_to_folio(folio_list); 400 401 list_del(&folio->lru); 402 node_stat_sub_folio(folio, NR_ISOLATED_ANON + 403 folio_is_file_lru(folio)); 404 folio_putback_lru(folio); 405 } 406 return nr_migrated; 407 } 408 409 noreclaim_flag = memalloc_noreclaim_save(); 410 411 nid = folio_nid(lru_to_folio(folio_list)); 412 do { 413 struct folio *folio = lru_to_folio(folio_list); 414 415 if (nid == folio_nid(folio)) { 416 list_move(&folio->lru, &node_folio_list); 417 continue; 418 } 419 420 nr_migrated += damon_migrate_folio_list(&node_folio_list, 421 NODE_DATA(nid), 422 target_nid); 423 nid = folio_nid(lru_to_folio(folio_list)); 424 } while (!list_empty(folio_list)); 425 426 nr_migrated += damon_migrate_folio_list(&node_folio_list, 427 NODE_DATA(nid), 428 target_nid); 429 430 memalloc_noreclaim_restore(noreclaim_flag); 431 432 return nr_migrated; 433 } 434 435 bool damos_ops_has_filter(struct damos *s) 436 { 437 struct damos_filter *f; 438 439 damos_for_each_ops_filter(f, s) 440 return true; 441 return false; 442 } 443