1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common Code for Data Access Monitoring 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #include <linux/migrate.h> 9 #include <linux/mmu_notifier.h> 10 #include <linux/page_idle.h> 11 #include <linux/pagemap.h> 12 #include <linux/rmap.h> 13 #include <linux/swap.h> 14 #include <linux/leafops.h> 15 16 #include "../internal.h" 17 #include "ops-common.h" 18 19 /* 20 * Get an online page for a pfn if it's in the LRU list. Otherwise, returns 21 * NULL. 22 * 23 * The body of this function is stolen from the 'page_idle_get_folio()'. We 24 * steal rather than reuse it because the code is quite simple. 25 */ 26 struct folio *damon_get_folio(unsigned long pfn) 27 { 28 struct page *page = pfn_to_online_page(pfn); 29 struct folio *folio; 30 31 if (!page) 32 return NULL; 33 34 folio = page_folio(page); 35 if (!folio_try_get(folio)) 36 return NULL; 37 if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { 38 folio_put(folio); 39 folio = NULL; 40 } 41 return folio; 42 } 43 44 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr) 45 { 46 pte_t pteval = ptep_get(pte); 47 struct folio *folio; 48 bool young = false; 49 unsigned long pfn; 50 51 if (likely(pte_present(pteval))) 52 pfn = pte_pfn(pteval); 53 else 54 pfn = softleaf_to_pfn(softleaf_from_pte(pteval)); 55 56 folio = damon_get_folio(pfn); 57 if (!folio) 58 return; 59 60 /* 61 * PFN swap PTEs, such as device-exclusive ones, that actually map pages 62 * are "old" from a CPU perspective. The MMU notifier takes care of any 63 * device aspects. 64 */ 65 if (likely(pte_present(pteval))) 66 young |= ptep_test_and_clear_young(vma, addr, pte); 67 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); 68 if (young) 69 folio_set_young(folio); 70 71 folio_set_idle(folio); 72 folio_put(folio); 73 } 74 75 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr) 76 { 77 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 78 pmd_t pmdval = pmdp_get(pmd); 79 struct folio *folio; 80 bool young = false; 81 unsigned long pfn; 82 83 if (likely(pmd_present(pmdval))) 84 pfn = pmd_pfn(pmdval); 85 else 86 pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval)); 87 88 folio = damon_get_folio(pfn); 89 if (!folio) 90 return; 91 92 if (likely(pmd_present(pmdval))) 93 young |= pmdp_test_and_clear_young(vma, addr, pmd); 94 young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE); 95 if (young) 96 folio_set_young(folio); 97 98 folio_set_idle(folio); 99 folio_put(folio); 100 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 101 } 102 103 #define DAMON_MAX_SUBSCORE (100) 104 #define DAMON_MAX_AGE_IN_LOG (32) 105 106 int damon_hot_score(struct damon_ctx *c, struct damon_region *r, 107 struct damos *s) 108 { 109 int freq_subscore; 110 unsigned int age_in_sec; 111 int age_in_log, age_subscore; 112 unsigned int freq_weight = s->quota.weight_nr_accesses; 113 unsigned int age_weight = s->quota.weight_age; 114 int hotness; 115 116 freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / 117 damon_max_nr_accesses(&c->attrs); 118 119 age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; 120 if (age_in_sec) 121 age_in_log = min_t(int, ilog2(age_in_sec) + 1, 122 DAMON_MAX_AGE_IN_LOG); 123 else 124 age_in_log = 0; 125 126 127 /* If frequency is 0, higher age means it's colder */ 128 if (freq_subscore == 0) 129 age_in_log *= -1; 130 131 /* 132 * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. 133 * Scale it to be in [0, 100] and set it as age subscore. 134 */ 135 age_in_log += DAMON_MAX_AGE_IN_LOG; 136 age_subscore = age_in_log * DAMON_MAX_SUBSCORE / 137 DAMON_MAX_AGE_IN_LOG / 2; 138 139 hotness = (freq_weight * freq_subscore + age_weight * age_subscore); 140 if (freq_weight + age_weight) 141 hotness /= freq_weight + age_weight; 142 /* 143 * Transform it to fit in [0, DAMOS_MAX_SCORE] 144 */ 145 hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; 146 hotness = max(min(hotness, DAMOS_MAX_SCORE), 0); 147 148 return hotness; 149 } 150 151 int damon_cold_score(struct damon_ctx *c, struct damon_region *r, 152 struct damos *s) 153 { 154 int hotness = damon_hot_score(c, r, s); 155 156 /* Return coldness of the region */ 157 return DAMOS_MAX_SCORE - hotness; 158 } 159 160 static bool damon_folio_mkold_one(struct folio *folio, 161 struct vm_area_struct *vma, unsigned long addr, void *arg) 162 { 163 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 164 165 while (page_vma_mapped_walk(&pvmw)) { 166 addr = pvmw.address; 167 if (pvmw.pte) 168 damon_ptep_mkold(pvmw.pte, vma, addr); 169 else 170 damon_pmdp_mkold(pvmw.pmd, vma, addr); 171 } 172 return true; 173 } 174 175 void damon_folio_mkold(struct folio *folio) 176 { 177 struct rmap_walk_control rwc = { 178 .rmap_one = damon_folio_mkold_one, 179 .anon_lock = folio_lock_anon_vma_read, 180 }; 181 182 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 183 folio_set_idle(folio); 184 return; 185 } 186 187 if (!folio_trylock(folio)) 188 return; 189 190 rmap_walk(folio, &rwc); 191 folio_unlock(folio); 192 193 } 194 195 static bool damon_folio_young_one(struct folio *folio, 196 struct vm_area_struct *vma, unsigned long addr, void *arg) 197 { 198 bool *accessed = arg; 199 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); 200 pte_t pte; 201 202 *accessed = false; 203 while (page_vma_mapped_walk(&pvmw)) { 204 addr = pvmw.address; 205 if (pvmw.pte) { 206 pte = ptep_get(pvmw.pte); 207 208 /* 209 * PFN swap PTEs, such as device-exclusive ones, that 210 * actually map pages are "old" from a CPU perspective. 211 * The MMU notifier takes care of any device aspects. 212 */ 213 *accessed = (pte_present(pte) && pte_young(pte)) || 214 !folio_test_idle(folio) || 215 mmu_notifier_test_young(vma->vm_mm, addr); 216 } else { 217 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 218 pmd_t pmd = pmdp_get(pvmw.pmd); 219 220 *accessed = (pmd_present(pmd) && pmd_young(pmd)) || 221 !folio_test_idle(folio) || 222 mmu_notifier_test_young(vma->vm_mm, addr); 223 #else 224 WARN_ON_ONCE(1); 225 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 226 } 227 if (*accessed) { 228 page_vma_mapped_walk_done(&pvmw); 229 break; 230 } 231 } 232 233 /* If accessed, stop walking */ 234 return *accessed == false; 235 } 236 237 bool damon_folio_young(struct folio *folio) 238 { 239 bool accessed = false; 240 struct rmap_walk_control rwc = { 241 .arg = &accessed, 242 .rmap_one = damon_folio_young_one, 243 .anon_lock = folio_lock_anon_vma_read, 244 }; 245 246 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { 247 if (folio_test_idle(folio)) 248 return false; 249 else 250 return true; 251 } 252 253 if (!folio_trylock(folio)) 254 return false; 255 256 rmap_walk(folio, &rwc); 257 folio_unlock(folio); 258 259 return accessed; 260 } 261 262 bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio) 263 { 264 bool matched = false; 265 struct mem_cgroup *memcg; 266 size_t folio_sz; 267 268 switch (filter->type) { 269 case DAMOS_FILTER_TYPE_ANON: 270 matched = folio_test_anon(folio); 271 break; 272 case DAMOS_FILTER_TYPE_ACTIVE: 273 matched = folio_test_active(folio); 274 break; 275 case DAMOS_FILTER_TYPE_MEMCG: 276 rcu_read_lock(); 277 memcg = folio_memcg_check(folio); 278 if (!memcg) 279 matched = false; 280 else 281 matched = filter->memcg_id == mem_cgroup_id(memcg); 282 rcu_read_unlock(); 283 break; 284 case DAMOS_FILTER_TYPE_YOUNG: 285 matched = damon_folio_young(folio); 286 if (matched) 287 damon_folio_mkold(folio); 288 break; 289 case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE: 290 folio_sz = folio_size(folio); 291 matched = filter->sz_range.min <= folio_sz && 292 folio_sz <= filter->sz_range.max; 293 break; 294 case DAMOS_FILTER_TYPE_UNMAPPED: 295 matched = !folio_mapped(folio) || !folio_raw_mapping(folio); 296 break; 297 default: 298 break; 299 } 300 301 return matched == filter->matching; 302 } 303 304 static unsigned int __damon_migrate_folio_list( 305 struct list_head *migrate_folios, struct pglist_data *pgdat, 306 int target_nid) 307 { 308 unsigned int nr_succeeded = 0; 309 struct migration_target_control mtc = { 310 /* 311 * Allocate from 'node', or fail quickly and quietly. 312 * When this happens, 'page' will likely just be discarded 313 * instead of migrated. 314 */ 315 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | 316 __GFP_NOMEMALLOC | GFP_NOWAIT, 317 .nid = target_nid, 318 }; 319 320 if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE) 321 return 0; 322 323 if (list_empty(migrate_folios)) 324 return 0; 325 326 /* Migration ignores all cpuset and mempolicy settings */ 327 migrate_pages(migrate_folios, alloc_migration_target, NULL, 328 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON, 329 &nr_succeeded); 330 331 return nr_succeeded; 332 } 333 334 static unsigned int damon_migrate_folio_list(struct list_head *folio_list, 335 struct pglist_data *pgdat, 336 int target_nid) 337 { 338 unsigned int nr_migrated = 0; 339 struct folio *folio; 340 LIST_HEAD(ret_folios); 341 LIST_HEAD(migrate_folios); 342 343 while (!list_empty(folio_list)) { 344 struct folio *folio; 345 346 cond_resched(); 347 348 folio = lru_to_folio(folio_list); 349 list_del(&folio->lru); 350 351 if (!folio_trylock(folio)) 352 goto keep; 353 354 /* Relocate its contents to another node. */ 355 list_add(&folio->lru, &migrate_folios); 356 folio_unlock(folio); 357 continue; 358 keep: 359 list_add(&folio->lru, &ret_folios); 360 } 361 /* 'folio_list' is always empty here */ 362 363 /* Migrate folios selected for migration */ 364 nr_migrated += __damon_migrate_folio_list( 365 &migrate_folios, pgdat, target_nid); 366 /* 367 * Folios that could not be migrated are still in @migrate_folios. Add 368 * those back on @folio_list 369 */ 370 if (!list_empty(&migrate_folios)) 371 list_splice_init(&migrate_folios, folio_list); 372 373 try_to_unmap_flush(); 374 375 list_splice(&ret_folios, folio_list); 376 377 while (!list_empty(folio_list)) { 378 folio = lru_to_folio(folio_list); 379 list_del(&folio->lru); 380 folio_putback_lru(folio); 381 } 382 383 return nr_migrated; 384 } 385 386 unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) 387 { 388 int nid; 389 unsigned long nr_migrated = 0; 390 LIST_HEAD(node_folio_list); 391 unsigned int noreclaim_flag; 392 393 if (list_empty(folio_list)) 394 return nr_migrated; 395 396 if (target_nid < 0 || target_nid >= MAX_NUMNODES || 397 !node_state(target_nid, N_MEMORY)) 398 return nr_migrated; 399 400 noreclaim_flag = memalloc_noreclaim_save(); 401 402 nid = folio_nid(lru_to_folio(folio_list)); 403 do { 404 struct folio *folio = lru_to_folio(folio_list); 405 406 if (nid == folio_nid(folio)) { 407 list_move(&folio->lru, &node_folio_list); 408 continue; 409 } 410 411 nr_migrated += damon_migrate_folio_list(&node_folio_list, 412 NODE_DATA(nid), 413 target_nid); 414 nid = folio_nid(lru_to_folio(folio_list)); 415 } while (!list_empty(folio_list)); 416 417 nr_migrated += damon_migrate_folio_list(&node_folio_list, 418 NODE_DATA(nid), 419 target_nid); 420 421 memalloc_noreclaim_restore(noreclaim_flag); 422 423 return nr_migrated; 424 } 425 426 bool damos_ops_has_filter(struct damos *s) 427 { 428 struct damos_filter *f; 429 430 damos_for_each_ops_filter(f, s) 431 return true; 432 return false; 433 } 434