1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON Code for The Physical Address Space 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-pa: " fmt 9 10 #include <linux/mmu_notifier.h> 11 #include <linux/page_idle.h> 12 #include <linux/pagemap.h> 13 #include <linux/rmap.h> 14 #include <linux/swap.h> 15 #include <linux/memory-tiers.h> 16 #include <linux/mm_inline.h> 17 18 #include "../internal.h" 19 #include "ops-common.h" 20 21 static phys_addr_t damon_pa_phys_addr( 22 unsigned long addr, unsigned long addr_unit) 23 { 24 return (phys_addr_t)addr * addr_unit; 25 } 26 27 static unsigned long damon_pa_core_addr( 28 phys_addr_t pa, unsigned long addr_unit) 29 { 30 /* 31 * Use div_u64() for avoiding linking errors related with __udivdi3, 32 * __aeabi_uldivmod, or similar problems. This should also improve the 33 * performance optimization (read div_u64() comment for the detail). 34 */ 35 if (sizeof(pa) == 8 && sizeof(addr_unit) == 4) 36 return div_u64(pa, addr_unit); 37 return pa / addr_unit; 38 } 39 40 static void damon_pa_mkold(phys_addr_t paddr) 41 { 42 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 43 44 if (!folio) 45 return; 46 47 damon_folio_mkold(folio); 48 folio_put(folio); 49 } 50 51 static void __damon_pa_prepare_access_check(struct damon_region *r, 52 struct damon_ctx *ctx) 53 { 54 r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end); 55 56 damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit)); 57 } 58 59 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) 60 { 61 struct damon_target *t; 62 struct damon_region *r; 63 64 damon_for_each_target(t, ctx) { 65 damon_for_each_region(r, t) 66 __damon_pa_prepare_access_check(r, ctx); 67 } 68 } 69 70 static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) 71 { 72 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 73 bool accessed; 74 75 if (!folio) 76 return false; 77 78 accessed = damon_folio_young(folio); 79 *folio_sz = folio_size(folio); 80 folio_put(folio); 81 return accessed; 82 } 83 84 static void __damon_pa_check_access(struct damon_region *r, 85 struct damon_attrs *attrs, unsigned long addr_unit) 86 { 87 static phys_addr_t last_addr; 88 static unsigned long last_folio_sz = PAGE_SIZE; 89 static bool last_accessed; 90 phys_addr_t sampling_addr = damon_pa_phys_addr( 91 r->sampling_addr, addr_unit); 92 93 /* If the region is in the last checked page, reuse the result */ 94 if (ALIGN_DOWN(last_addr, last_folio_sz) == 95 ALIGN_DOWN(sampling_addr, last_folio_sz)) { 96 damon_update_region_access_rate(r, last_accessed, attrs); 97 return; 98 } 99 100 last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); 101 damon_update_region_access_rate(r, last_accessed, attrs); 102 103 last_addr = sampling_addr; 104 } 105 106 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) 107 { 108 struct damon_target *t; 109 struct damon_region *r; 110 unsigned int max_nr_accesses = 0; 111 112 damon_for_each_target(t, ctx) { 113 damon_for_each_region(r, t) { 114 __damon_pa_check_access( 115 r, &ctx->attrs, ctx->addr_unit); 116 max_nr_accesses = max(r->nr_accesses, max_nr_accesses); 117 } 118 } 119 120 return max_nr_accesses; 121 } 122 123 static bool damon_pa_filter_match(struct damon_filter *filter, 124 struct folio *folio) 125 { 126 bool matched = false; 127 struct mem_cgroup *memcg; 128 129 switch (filter->type) { 130 case DAMON_FILTER_TYPE_ANON: 131 if (!folio) { 132 matched = false; 133 break; 134 } 135 matched = folio_test_anon(folio); 136 break; 137 case DAMON_FILTER_TYPE_MEMCG: 138 if (!folio) { 139 matched = false; 140 break; 141 } 142 rcu_read_lock(); 143 memcg = folio_memcg_check(folio); 144 if (!memcg) 145 matched = false; 146 else 147 matched = filter->memcg_id == mem_cgroup_id(memcg); 148 rcu_read_unlock(); 149 break; 150 default: 151 break; 152 } 153 return matched == filter->matching; 154 } 155 156 static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, 157 struct damon_probe *p) 158 { 159 struct damon_filter *f; 160 bool pass = true; 161 162 damon_for_each_filter(f, p) { 163 if (damon_pa_filter_match(f, folio)) { 164 pass = f->allow; 165 break; 166 } 167 pass = !f->allow; 168 } 169 return pass; 170 } 171 172 static void damon_pa_apply_probes(struct damon_ctx *ctx) 173 { 174 struct damon_target *t; 175 struct damon_region *r; 176 struct damon_probe *p; 177 178 damon_for_each_target(t, ctx) { 179 damon_for_each_region(r, t) { 180 int i = 0; 181 phys_addr_t pa; 182 struct folio *folio; 183 184 pa = damon_pa_phys_addr(r->sampling_addr, 185 ctx->addr_unit); 186 folio = damon_get_folio(PHYS_PFN(pa)); 187 damon_for_each_probe(p, ctx) { 188 if (damon_pa_filter_pass(pa, folio, p)) 189 r->probe_hits[i]++; 190 i++; 191 } 192 if (folio) 193 folio_put(folio); 194 } 195 } 196 } 197 198 /* 199 * damos_pa_filter_out - Return true if the page should be filtered out. 200 */ 201 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio) 202 { 203 struct damos_filter *filter; 204 205 if (scheme->core_filters_allowed) 206 return false; 207 208 damos_for_each_ops_filter(filter, scheme) { 209 if (damos_folio_filter_match(filter, folio)) 210 return !filter->allow; 211 } 212 return scheme->ops_filters_default_reject; 213 } 214 215 static bool damon_pa_invalid_damos_folio(struct folio *folio, struct damos *s) 216 { 217 if (!folio) 218 return true; 219 if (folio == s->last_applied) { 220 folio_put(folio); 221 return true; 222 } 223 return false; 224 } 225 226 static unsigned long damon_pa_pageout(struct damon_region *r, 227 unsigned long addr_unit, struct damos *s, 228 unsigned long *sz_filter_passed) 229 { 230 phys_addr_t addr, applied; 231 LIST_HEAD(folio_list); 232 bool install_young_filter = true; 233 struct damos_filter *filter; 234 struct folio *folio = NULL; 235 236 /* check access in page level again by default */ 237 damos_for_each_ops_filter(filter, s) { 238 if (filter->type == DAMOS_FILTER_TYPE_YOUNG) { 239 install_young_filter = false; 240 break; 241 } 242 } 243 if (install_young_filter) { 244 filter = damos_new_filter( 245 DAMOS_FILTER_TYPE_YOUNG, true, false); 246 if (!filter) 247 return 0; 248 damos_add_filter(s, filter); 249 } 250 251 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 252 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 253 folio = damon_get_folio(PHYS_PFN(addr)); 254 if (damon_pa_invalid_damos_folio(folio, s)) { 255 addr += PAGE_SIZE; 256 continue; 257 } 258 259 if (damos_pa_filter_out(s, folio)) 260 goto put_folio; 261 else 262 *sz_filter_passed += folio_size(folio) / addr_unit; 263 264 folio_clear_referenced(folio); 265 folio_test_clear_young(folio); 266 if (!folio_isolate_lru(folio)) 267 goto put_folio; 268 if (folio_test_unevictable(folio)) 269 folio_putback_lru(folio); 270 else 271 list_add(&folio->lru, &folio_list); 272 put_folio: 273 addr += folio_size(folio); 274 folio_put(folio); 275 } 276 if (install_young_filter) 277 damos_destroy_filter(filter); 278 applied = reclaim_pages(&folio_list); 279 cond_resched(); 280 s->last_applied = folio; 281 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 282 } 283 284 static inline unsigned long damon_pa_de_activate( 285 struct damon_region *r, unsigned long addr_unit, 286 struct damos *s, bool activate, 287 unsigned long *sz_filter_passed) 288 { 289 phys_addr_t addr, applied = 0; 290 struct folio *folio = NULL; 291 292 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 293 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 294 folio = damon_get_folio(PHYS_PFN(addr)); 295 if (damon_pa_invalid_damos_folio(folio, s)) { 296 addr += PAGE_SIZE; 297 continue; 298 } 299 300 if (damos_pa_filter_out(s, folio)) 301 goto put_folio; 302 else 303 *sz_filter_passed += folio_size(folio) / addr_unit; 304 305 if (activate) 306 folio_activate(folio); 307 else 308 folio_deactivate(folio); 309 applied += folio_nr_pages(folio); 310 put_folio: 311 addr += folio_size(folio); 312 folio_put(folio); 313 } 314 s->last_applied = folio; 315 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 316 } 317 318 static unsigned long damon_pa_activate_pages(struct damon_region *r, 319 unsigned long addr_unit, struct damos *s, 320 unsigned long *sz_filter_passed) 321 { 322 return damon_pa_de_activate(r, addr_unit, s, true, sz_filter_passed); 323 } 324 325 static unsigned long damon_pa_deactivate_pages(struct damon_region *r, 326 unsigned long addr_unit, struct damos *s, 327 unsigned long *sz_filter_passed) 328 { 329 return damon_pa_de_activate(r, addr_unit, s, false, sz_filter_passed); 330 } 331 332 static unsigned long damon_pa_migrate(struct damon_region *r, 333 unsigned long addr_unit, struct damos *s, 334 unsigned long *sz_filter_passed) 335 { 336 phys_addr_t addr, applied; 337 LIST_HEAD(folio_list); 338 struct folio *folio = NULL; 339 340 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 341 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 342 folio = damon_get_folio(PHYS_PFN(addr)); 343 if (damon_pa_invalid_damos_folio(folio, s)) { 344 addr += PAGE_SIZE; 345 continue; 346 } 347 348 if (damos_pa_filter_out(s, folio)) 349 goto put_folio; 350 else 351 *sz_filter_passed += folio_size(folio) / addr_unit; 352 353 if (!folio_isolate_lru(folio)) 354 goto put_folio; 355 list_add(&folio->lru, &folio_list); 356 put_folio: 357 addr += folio_size(folio); 358 folio_put(folio); 359 } 360 applied = damon_migrate_pages(&folio_list, s->target_nid); 361 cond_resched(); 362 s->last_applied = folio; 363 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 364 } 365 366 static unsigned long damon_pa_stat(struct damon_region *r, 367 unsigned long addr_unit, struct damos *s, 368 unsigned long *sz_filter_passed) 369 { 370 phys_addr_t addr; 371 struct folio *folio = NULL; 372 373 if (!damos_ops_has_filter(s)) 374 return 0; 375 376 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 377 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 378 folio = damon_get_folio(PHYS_PFN(addr)); 379 if (damon_pa_invalid_damos_folio(folio, s)) { 380 addr += PAGE_SIZE; 381 continue; 382 } 383 384 if (!damos_pa_filter_out(s, folio)) 385 *sz_filter_passed += folio_size(folio) / addr_unit; 386 addr += folio_size(folio); 387 folio_put(folio); 388 } 389 s->last_applied = folio; 390 return 0; 391 } 392 393 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, 394 struct damon_target *t, struct damon_region *r, 395 struct damos *scheme, unsigned long *sz_filter_passed) 396 { 397 unsigned long aunit = ctx->addr_unit; 398 399 switch (scheme->action) { 400 case DAMOS_PAGEOUT: 401 return damon_pa_pageout(r, aunit, scheme, sz_filter_passed); 402 case DAMOS_LRU_PRIO: 403 return damon_pa_activate_pages(r, aunit, scheme, 404 sz_filter_passed); 405 case DAMOS_LRU_DEPRIO: 406 return damon_pa_deactivate_pages(r, aunit, scheme, 407 sz_filter_passed); 408 case DAMOS_MIGRATE_HOT: 409 case DAMOS_MIGRATE_COLD: 410 return damon_pa_migrate(r, aunit, scheme, sz_filter_passed); 411 case DAMOS_STAT: 412 return damon_pa_stat(r, aunit, scheme, sz_filter_passed); 413 default: 414 /* DAMOS actions that not yet supported by 'paddr'. */ 415 break; 416 } 417 return 0; 418 } 419 420 static int damon_pa_scheme_score(struct damon_ctx *context, 421 struct damon_region *r, struct damos *scheme) 422 { 423 switch (scheme->action) { 424 case DAMOS_PAGEOUT: 425 return damon_cold_score(context, r, scheme); 426 case DAMOS_LRU_PRIO: 427 return damon_hot_score(context, r, scheme); 428 case DAMOS_LRU_DEPRIO: 429 return damon_cold_score(context, r, scheme); 430 case DAMOS_MIGRATE_HOT: 431 return damon_hot_score(context, r, scheme); 432 case DAMOS_MIGRATE_COLD: 433 return damon_cold_score(context, r, scheme); 434 default: 435 break; 436 } 437 438 return DAMOS_MAX_SCORE; 439 } 440 441 static int __init damon_pa_initcall(void) 442 { 443 struct damon_operations ops = { 444 .id = DAMON_OPS_PADDR, 445 .init = NULL, 446 .update = NULL, 447 .prepare_access_checks = damon_pa_prepare_access_checks, 448 .check_accesses = damon_pa_check_accesses, 449 .apply_probes = damon_pa_apply_probes, 450 .target_valid = NULL, 451 .apply_scheme = damon_pa_apply_scheme, 452 .get_scheme_score = damon_pa_scheme_score, 453 }; 454 455 return damon_register_ops(&ops); 456 }; 457 458 subsys_initcall(damon_pa_initcall); 459