1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON Code for The Physical Address Space 4 */ 5 6 #define pr_fmt(fmt) "damon-pa: " fmt 7 8 #include <linux/mmu_notifier.h> 9 #include <linux/page_idle.h> 10 #include <linux/pagemap.h> 11 #include <linux/rmap.h> 12 #include <linux/swap.h> 13 #include <linux/memory-tiers.h> 14 #include <linux/mm_inline.h> 15 16 #include "../internal.h" 17 #include "ops-common.h" 18 19 static phys_addr_t damon_pa_phys_addr( 20 unsigned long addr, unsigned long addr_unit) 21 { 22 return (phys_addr_t)addr * addr_unit; 23 } 24 25 static unsigned long damon_pa_core_addr( 26 phys_addr_t pa, unsigned long addr_unit) 27 { 28 /* 29 * Use div_u64() for avoiding linking errors related with __udivdi3, 30 * __aeabi_uldivmod, or similar problems. This should also improve the 31 * performance optimization (read div_u64() comment for the detail). 32 */ 33 if (sizeof(pa) == 8 && sizeof(addr_unit) == 4) 34 return div_u64(pa, addr_unit); 35 return pa / addr_unit; 36 } 37 38 static void damon_pa_mkold(phys_addr_t paddr) 39 { 40 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 41 42 if (!folio) 43 return; 44 45 damon_folio_mkold(folio); 46 folio_put(folio); 47 } 48 49 static void __damon_pa_prepare_access_check(struct damon_region *r, 50 struct damon_ctx *ctx) 51 { 52 r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end); 53 54 damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit)); 55 } 56 57 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) 58 { 59 struct damon_target *t; 60 struct damon_region *r; 61 62 damon_for_each_target(t, ctx) { 63 damon_for_each_region(r, t) 64 __damon_pa_prepare_access_check(r, ctx); 65 } 66 } 67 68 static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) 69 { 70 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 71 bool accessed; 72 73 if (!folio) 74 return false; 75 76 accessed = damon_folio_young(folio); 77 *folio_sz = folio_size(folio); 78 folio_put(folio); 79 return accessed; 80 } 81 82 static void __damon_pa_check_access(struct damon_region *r, 83 struct damon_attrs *attrs, unsigned long addr_unit) 84 { 85 static phys_addr_t last_addr; 86 static unsigned long last_folio_sz = PAGE_SIZE; 87 static bool last_accessed; 88 phys_addr_t sampling_addr = damon_pa_phys_addr( 89 r->sampling_addr, addr_unit); 90 91 /* If the region is in the last checked page, reuse the result */ 92 if (ALIGN_DOWN(last_addr, last_folio_sz) == 93 ALIGN_DOWN(sampling_addr, last_folio_sz)) { 94 damon_update_region_access_rate(r, last_accessed, attrs); 95 return; 96 } 97 98 last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); 99 damon_update_region_access_rate(r, last_accessed, attrs); 100 101 last_addr = sampling_addr; 102 } 103 104 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) 105 { 106 struct damon_target *t; 107 struct damon_region *r; 108 unsigned int max_nr_accesses = 0; 109 110 damon_for_each_target(t, ctx) { 111 damon_for_each_region(r, t) { 112 __damon_pa_check_access( 113 r, &ctx->attrs, ctx->addr_unit); 114 max_nr_accesses = max(r->nr_accesses, max_nr_accesses); 115 } 116 } 117 118 return max_nr_accesses; 119 } 120 121 static bool damon_pa_filter_match(struct damon_filter *filter, 122 struct folio *folio) 123 { 124 bool matched = false; 125 struct mem_cgroup *memcg; 126 127 switch (filter->type) { 128 case DAMON_FILTER_TYPE_ANON: 129 if (!folio) { 130 matched = false; 131 break; 132 } 133 matched = folio_test_anon(folio); 134 break; 135 case DAMON_FILTER_TYPE_MEMCG: 136 if (!folio) { 137 matched = false; 138 break; 139 } 140 rcu_read_lock(); 141 memcg = folio_memcg_check(folio); 142 if (!memcg) 143 matched = false; 144 else 145 matched = filter->memcg_id == mem_cgroup_id(memcg); 146 rcu_read_unlock(); 147 break; 148 default: 149 break; 150 } 151 return matched == filter->matching; 152 } 153 154 static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, 155 struct damon_probe *p) 156 { 157 struct damon_filter *f; 158 bool pass = true; 159 160 damon_for_each_filter(f, p) { 161 if (damon_pa_filter_match(f, folio)) { 162 pass = f->allow; 163 break; 164 } 165 pass = !f->allow; 166 } 167 return pass; 168 } 169 170 static void damon_pa_apply_probes(struct damon_ctx *ctx) 171 { 172 struct damon_target *t; 173 struct damon_region *r; 174 struct damon_probe *p; 175 176 damon_for_each_target(t, ctx) { 177 damon_for_each_region(r, t) { 178 int i = 0; 179 phys_addr_t pa; 180 struct folio *folio; 181 182 pa = damon_pa_phys_addr(r->sampling_addr, 183 ctx->addr_unit); 184 folio = damon_get_folio(PHYS_PFN(pa)); 185 damon_for_each_probe(p, ctx) { 186 if (damon_pa_filter_pass(pa, folio, p)) 187 r->probe_hits[i]++; 188 i++; 189 } 190 if (folio) 191 folio_put(folio); 192 } 193 } 194 } 195 196 /* 197 * damos_pa_filter_out - Return true if the page should be filtered out. 198 */ 199 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio) 200 { 201 struct damos_filter *filter; 202 203 if (scheme->core_filters_allowed) 204 return false; 205 206 damos_for_each_ops_filter(filter, scheme) { 207 if (damos_folio_filter_match(filter, folio)) 208 return !filter->allow; 209 } 210 return scheme->ops_filters_default_reject; 211 } 212 213 static bool damon_pa_invalid_damos_folio(struct folio *folio, struct damos *s) 214 { 215 if (!folio) 216 return true; 217 if (folio == s->last_applied) { 218 folio_put(folio); 219 return true; 220 } 221 return false; 222 } 223 224 static unsigned long damon_pa_pageout(struct damon_region *r, 225 unsigned long addr_unit, struct damos *s, 226 unsigned long *sz_filter_passed) 227 { 228 phys_addr_t addr, applied; 229 LIST_HEAD(folio_list); 230 bool install_young_filter = true; 231 struct damos_filter *filter; 232 struct folio *folio = NULL; 233 234 /* check access in page level again by default */ 235 damos_for_each_ops_filter(filter, s) { 236 if (filter->type == DAMOS_FILTER_TYPE_YOUNG) { 237 install_young_filter = false; 238 break; 239 } 240 } 241 if (install_young_filter) { 242 filter = damos_new_filter( 243 DAMOS_FILTER_TYPE_YOUNG, true, false); 244 if (!filter) 245 return 0; 246 damos_add_filter(s, filter); 247 } 248 249 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 250 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 251 folio = damon_get_folio(PHYS_PFN(addr)); 252 if (damon_pa_invalid_damos_folio(folio, s)) { 253 addr += PAGE_SIZE; 254 continue; 255 } 256 257 if (damos_pa_filter_out(s, folio)) 258 goto put_folio; 259 else 260 *sz_filter_passed += folio_size(folio) / addr_unit; 261 262 folio_clear_referenced(folio); 263 folio_test_clear_young(folio); 264 if (!folio_isolate_lru(folio)) 265 goto put_folio; 266 if (folio_test_unevictable(folio)) 267 folio_putback_lru(folio); 268 else 269 list_add(&folio->lru, &folio_list); 270 put_folio: 271 addr += folio_size(folio); 272 folio_put(folio); 273 } 274 if (install_young_filter) 275 damos_destroy_filter(filter); 276 applied = reclaim_pages(&folio_list); 277 cond_resched(); 278 s->last_applied = folio; 279 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 280 } 281 282 static inline unsigned long damon_pa_de_activate( 283 struct damon_region *r, unsigned long addr_unit, 284 struct damos *s, bool activate, 285 unsigned long *sz_filter_passed) 286 { 287 phys_addr_t addr, applied = 0; 288 struct folio *folio = NULL; 289 290 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 291 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 292 folio = damon_get_folio(PHYS_PFN(addr)); 293 if (damon_pa_invalid_damos_folio(folio, s)) { 294 addr += PAGE_SIZE; 295 continue; 296 } 297 298 if (damos_pa_filter_out(s, folio)) 299 goto put_folio; 300 else 301 *sz_filter_passed += folio_size(folio) / addr_unit; 302 303 if (activate) 304 folio_activate(folio); 305 else 306 folio_deactivate(folio); 307 applied += folio_nr_pages(folio); 308 put_folio: 309 addr += folio_size(folio); 310 folio_put(folio); 311 } 312 s->last_applied = folio; 313 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 314 } 315 316 static unsigned long damon_pa_activate_pages(struct damon_region *r, 317 unsigned long addr_unit, struct damos *s, 318 unsigned long *sz_filter_passed) 319 { 320 return damon_pa_de_activate(r, addr_unit, s, true, sz_filter_passed); 321 } 322 323 static unsigned long damon_pa_deactivate_pages(struct damon_region *r, 324 unsigned long addr_unit, struct damos *s, 325 unsigned long *sz_filter_passed) 326 { 327 return damon_pa_de_activate(r, addr_unit, s, false, sz_filter_passed); 328 } 329 330 static unsigned long damon_pa_migrate(struct damon_region *r, 331 unsigned long addr_unit, struct damos *s, 332 unsigned long *sz_filter_passed) 333 { 334 phys_addr_t addr, applied; 335 LIST_HEAD(folio_list); 336 struct folio *folio = NULL; 337 338 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 339 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 340 folio = damon_get_folio(PHYS_PFN(addr)); 341 if (damon_pa_invalid_damos_folio(folio, s)) { 342 addr += PAGE_SIZE; 343 continue; 344 } 345 346 if (damos_pa_filter_out(s, folio)) 347 goto put_folio; 348 else 349 *sz_filter_passed += folio_size(folio) / addr_unit; 350 351 if (!folio_isolate_lru(folio)) 352 goto put_folio; 353 node_stat_add_folio(folio, NR_ISOLATED_ANON + 354 folio_is_file_lru(folio)); 355 list_add(&folio->lru, &folio_list); 356 put_folio: 357 addr += folio_size(folio); 358 folio_put(folio); 359 } 360 applied = damon_migrate_pages(&folio_list, s->target_nid); 361 cond_resched(); 362 s->last_applied = folio; 363 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 364 } 365 366 static unsigned long damon_pa_stat(struct damon_region *r, 367 unsigned long addr_unit, struct damos *s, 368 unsigned long *sz_filter_passed) 369 { 370 phys_addr_t addr; 371 struct folio *folio = NULL; 372 373 if (!damos_ops_has_filter(s)) 374 return 0; 375 376 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 377 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 378 folio = damon_get_folio(PHYS_PFN(addr)); 379 if (damon_pa_invalid_damos_folio(folio, s)) { 380 addr += PAGE_SIZE; 381 continue; 382 } 383 384 if (!damos_pa_filter_out(s, folio)) 385 *sz_filter_passed += folio_size(folio) / addr_unit; 386 addr += folio_size(folio); 387 folio_put(folio); 388 } 389 s->last_applied = folio; 390 return 0; 391 } 392 393 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, 394 struct damon_target *t, struct damon_region *r, 395 struct damos *scheme, unsigned long *sz_filter_passed) 396 { 397 unsigned long aunit = ctx->addr_unit; 398 399 switch (scheme->action) { 400 case DAMOS_PAGEOUT: 401 return damon_pa_pageout(r, aunit, scheme, sz_filter_passed); 402 case DAMOS_LRU_PRIO: 403 return damon_pa_activate_pages(r, aunit, scheme, 404 sz_filter_passed); 405 case DAMOS_LRU_DEPRIO: 406 return damon_pa_deactivate_pages(r, aunit, scheme, 407 sz_filter_passed); 408 case DAMOS_MIGRATE_HOT: 409 case DAMOS_MIGRATE_COLD: 410 return damon_pa_migrate(r, aunit, scheme, sz_filter_passed); 411 case DAMOS_STAT: 412 return damon_pa_stat(r, aunit, scheme, sz_filter_passed); 413 default: 414 /* DAMOS actions that not yet supported by 'paddr'. */ 415 break; 416 } 417 return 0; 418 } 419 420 static int damon_pa_scheme_score(struct damon_ctx *context, 421 struct damon_region *r, struct damos *scheme) 422 { 423 switch (scheme->action) { 424 case DAMOS_PAGEOUT: 425 return damon_cold_score(context, r, scheme); 426 case DAMOS_LRU_PRIO: 427 return damon_hot_score(context, r, scheme); 428 case DAMOS_LRU_DEPRIO: 429 return damon_cold_score(context, r, scheme); 430 case DAMOS_MIGRATE_HOT: 431 return damon_hot_score(context, r, scheme); 432 case DAMOS_MIGRATE_COLD: 433 return damon_cold_score(context, r, scheme); 434 default: 435 break; 436 } 437 438 return DAMOS_MAX_SCORE; 439 } 440 441 static int __init damon_pa_initcall(void) 442 { 443 struct damon_operations ops = { 444 .id = DAMON_OPS_PADDR, 445 .init = NULL, 446 .update = NULL, 447 .prepare_access_checks = damon_pa_prepare_access_checks, 448 .check_accesses = damon_pa_check_accesses, 449 .apply_probes = damon_pa_apply_probes, 450 .target_valid = NULL, 451 .apply_scheme = damon_pa_apply_scheme, 452 .get_scheme_score = damon_pa_scheme_score, 453 }; 454 455 return damon_register_ops(&ops); 456 }; 457 458 subsys_initcall(damon_pa_initcall); 459