1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON Code for The Physical Address Space 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-pa: " fmt 9 10 #include <linux/mmu_notifier.h> 11 #include <linux/page_idle.h> 12 #include <linux/pagemap.h> 13 #include <linux/rmap.h> 14 #include <linux/swap.h> 15 #include <linux/memory-tiers.h> 16 #include <linux/mm_inline.h> 17 18 #include "../internal.h" 19 #include "ops-common.h" 20 21 static phys_addr_t damon_pa_phys_addr( 22 unsigned long addr, unsigned long addr_unit) 23 { 24 return (phys_addr_t)addr * addr_unit; 25 } 26 27 static unsigned long damon_pa_core_addr( 28 phys_addr_t pa, unsigned long addr_unit) 29 { 30 /* 31 * Use div_u64() for avoiding linking errors related with __udivdi3, 32 * __aeabi_uldivmod, or similar problems. This should also improve the 33 * performance optimization (read div_u64() comment for the detail). 34 */ 35 if (sizeof(pa) == 8 && sizeof(addr_unit) == 4) 36 return div_u64(pa, addr_unit); 37 return pa / addr_unit; 38 } 39 40 static void damon_pa_mkold(phys_addr_t paddr) 41 { 42 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 43 44 if (!folio) 45 return; 46 47 damon_folio_mkold(folio); 48 folio_put(folio); 49 } 50 51 static void __damon_pa_prepare_access_check(struct damon_region *r, 52 struct damon_ctx *ctx) 53 { 54 r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end); 55 56 damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit)); 57 } 58 59 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) 60 { 61 struct damon_target *t; 62 struct damon_region *r; 63 64 damon_for_each_target(t, ctx) { 65 damon_for_each_region(r, t) 66 __damon_pa_prepare_access_check(r, ctx); 67 } 68 } 69 70 static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) 71 { 72 struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); 73 bool accessed; 74 75 if (!folio) 76 return false; 77 78 accessed = damon_folio_young(folio); 79 *folio_sz = folio_size(folio); 80 folio_put(folio); 81 return accessed; 82 } 83 84 static void __damon_pa_check_access(struct damon_region *r, 85 struct damon_attrs *attrs, unsigned long addr_unit) 86 { 87 static phys_addr_t last_addr; 88 static unsigned long last_folio_sz = PAGE_SIZE; 89 static bool last_accessed; 90 phys_addr_t sampling_addr = damon_pa_phys_addr( 91 r->sampling_addr, addr_unit); 92 93 /* If the region is in the last checked page, reuse the result */ 94 if (ALIGN_DOWN(last_addr, last_folio_sz) == 95 ALIGN_DOWN(sampling_addr, last_folio_sz)) { 96 damon_update_region_access_rate(r, last_accessed, attrs); 97 return; 98 } 99 100 last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); 101 damon_update_region_access_rate(r, last_accessed, attrs); 102 103 last_addr = sampling_addr; 104 } 105 106 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) 107 { 108 struct damon_target *t; 109 struct damon_region *r; 110 unsigned int max_nr_accesses = 0; 111 112 damon_for_each_target(t, ctx) { 113 damon_for_each_region(r, t) { 114 __damon_pa_check_access( 115 r, &ctx->attrs, ctx->addr_unit); 116 max_nr_accesses = max(r->nr_accesses, max_nr_accesses); 117 } 118 } 119 120 return max_nr_accesses; 121 } 122 123 static bool damon_pa_filter_match(struct damon_filter *filter, 124 struct folio *folio) 125 { 126 bool matched = false; 127 128 switch (filter->type) { 129 case DAMON_FILTER_TYPE_ANON: 130 if (!folio) { 131 matched = false; 132 break; 133 } 134 matched = folio_test_anon(folio); 135 break; 136 default: 137 break; 138 } 139 return matched == filter->matching; 140 } 141 142 static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio, 143 struct damon_probe *p) 144 { 145 struct damon_filter *f; 146 bool pass = true; 147 148 damon_for_each_filter(f, p) { 149 if (damon_pa_filter_match(f, folio)) { 150 pass = f->allow; 151 break; 152 } 153 pass = !f->allow; 154 } 155 return pass; 156 } 157 158 static void damon_pa_apply_probes(struct damon_ctx *ctx) 159 { 160 struct damon_target *t; 161 struct damon_region *r; 162 struct damon_probe *p; 163 164 damon_for_each_target(t, ctx) { 165 damon_for_each_region(r, t) { 166 int i = 0; 167 phys_addr_t pa; 168 struct folio *folio; 169 170 pa = damon_pa_phys_addr(r->sampling_addr, 171 ctx->addr_unit); 172 folio = damon_get_folio(PHYS_PFN(pa)); 173 damon_for_each_probe(p, ctx) { 174 if (damon_pa_filter_pass(pa, folio, p)) 175 r->probe_hits[i]++; 176 i++; 177 } 178 if (folio) 179 folio_put(folio); 180 } 181 } 182 } 183 184 /* 185 * damos_pa_filter_out - Return true if the page should be filtered out. 186 */ 187 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio) 188 { 189 struct damos_filter *filter; 190 191 if (scheme->core_filters_allowed) 192 return false; 193 194 damos_for_each_ops_filter(filter, scheme) { 195 if (damos_folio_filter_match(filter, folio)) 196 return !filter->allow; 197 } 198 return scheme->ops_filters_default_reject; 199 } 200 201 static bool damon_pa_invalid_damos_folio(struct folio *folio, struct damos *s) 202 { 203 if (!folio) 204 return true; 205 if (folio == s->last_applied) { 206 folio_put(folio); 207 return true; 208 } 209 return false; 210 } 211 212 static unsigned long damon_pa_pageout(struct damon_region *r, 213 unsigned long addr_unit, struct damos *s, 214 unsigned long *sz_filter_passed) 215 { 216 phys_addr_t addr, applied; 217 LIST_HEAD(folio_list); 218 bool install_young_filter = true; 219 struct damos_filter *filter; 220 struct folio *folio = NULL; 221 222 /* check access in page level again by default */ 223 damos_for_each_ops_filter(filter, s) { 224 if (filter->type == DAMOS_FILTER_TYPE_YOUNG) { 225 install_young_filter = false; 226 break; 227 } 228 } 229 if (install_young_filter) { 230 filter = damos_new_filter( 231 DAMOS_FILTER_TYPE_YOUNG, true, false); 232 if (!filter) 233 return 0; 234 damos_add_filter(s, filter); 235 } 236 237 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 238 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 239 folio = damon_get_folio(PHYS_PFN(addr)); 240 if (damon_pa_invalid_damos_folio(folio, s)) { 241 addr += PAGE_SIZE; 242 continue; 243 } 244 245 if (damos_pa_filter_out(s, folio)) 246 goto put_folio; 247 else 248 *sz_filter_passed += folio_size(folio) / addr_unit; 249 250 folio_clear_referenced(folio); 251 folio_test_clear_young(folio); 252 if (!folio_isolate_lru(folio)) 253 goto put_folio; 254 if (folio_test_unevictable(folio)) 255 folio_putback_lru(folio); 256 else 257 list_add(&folio->lru, &folio_list); 258 put_folio: 259 addr += folio_size(folio); 260 folio_put(folio); 261 } 262 if (install_young_filter) 263 damos_destroy_filter(filter); 264 applied = reclaim_pages(&folio_list); 265 cond_resched(); 266 s->last_applied = folio; 267 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 268 } 269 270 static inline unsigned long damon_pa_de_activate( 271 struct damon_region *r, unsigned long addr_unit, 272 struct damos *s, bool activate, 273 unsigned long *sz_filter_passed) 274 { 275 phys_addr_t addr, applied = 0; 276 struct folio *folio = NULL; 277 278 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 279 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 280 folio = damon_get_folio(PHYS_PFN(addr)); 281 if (damon_pa_invalid_damos_folio(folio, s)) { 282 addr += PAGE_SIZE; 283 continue; 284 } 285 286 if (damos_pa_filter_out(s, folio)) 287 goto put_folio; 288 else 289 *sz_filter_passed += folio_size(folio) / addr_unit; 290 291 if (activate) 292 folio_activate(folio); 293 else 294 folio_deactivate(folio); 295 applied += folio_nr_pages(folio); 296 put_folio: 297 addr += folio_size(folio); 298 folio_put(folio); 299 } 300 s->last_applied = folio; 301 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 302 } 303 304 static unsigned long damon_pa_activate_pages(struct damon_region *r, 305 unsigned long addr_unit, struct damos *s, 306 unsigned long *sz_filter_passed) 307 { 308 return damon_pa_de_activate(r, addr_unit, s, true, sz_filter_passed); 309 } 310 311 static unsigned long damon_pa_deactivate_pages(struct damon_region *r, 312 unsigned long addr_unit, struct damos *s, 313 unsigned long *sz_filter_passed) 314 { 315 return damon_pa_de_activate(r, addr_unit, s, false, sz_filter_passed); 316 } 317 318 static unsigned long damon_pa_migrate(struct damon_region *r, 319 unsigned long addr_unit, struct damos *s, 320 unsigned long *sz_filter_passed) 321 { 322 phys_addr_t addr, applied; 323 LIST_HEAD(folio_list); 324 struct folio *folio = NULL; 325 326 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 327 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 328 folio = damon_get_folio(PHYS_PFN(addr)); 329 if (damon_pa_invalid_damos_folio(folio, s)) { 330 addr += PAGE_SIZE; 331 continue; 332 } 333 334 if (damos_pa_filter_out(s, folio)) 335 goto put_folio; 336 else 337 *sz_filter_passed += folio_size(folio) / addr_unit; 338 339 if (!folio_isolate_lru(folio)) 340 goto put_folio; 341 list_add(&folio->lru, &folio_list); 342 put_folio: 343 addr += folio_size(folio); 344 folio_put(folio); 345 } 346 applied = damon_migrate_pages(&folio_list, s->target_nid); 347 cond_resched(); 348 s->last_applied = folio; 349 return damon_pa_core_addr(applied * PAGE_SIZE, addr_unit); 350 } 351 352 static unsigned long damon_pa_stat(struct damon_region *r, 353 unsigned long addr_unit, struct damos *s, 354 unsigned long *sz_filter_passed) 355 { 356 phys_addr_t addr; 357 struct folio *folio = NULL; 358 359 if (!damos_ops_has_filter(s)) 360 return 0; 361 362 addr = damon_pa_phys_addr(r->ar.start, addr_unit); 363 while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { 364 folio = damon_get_folio(PHYS_PFN(addr)); 365 if (damon_pa_invalid_damos_folio(folio, s)) { 366 addr += PAGE_SIZE; 367 continue; 368 } 369 370 if (!damos_pa_filter_out(s, folio)) 371 *sz_filter_passed += folio_size(folio) / addr_unit; 372 addr += folio_size(folio); 373 folio_put(folio); 374 } 375 s->last_applied = folio; 376 return 0; 377 } 378 379 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, 380 struct damon_target *t, struct damon_region *r, 381 struct damos *scheme, unsigned long *sz_filter_passed) 382 { 383 unsigned long aunit = ctx->addr_unit; 384 385 switch (scheme->action) { 386 case DAMOS_PAGEOUT: 387 return damon_pa_pageout(r, aunit, scheme, sz_filter_passed); 388 case DAMOS_LRU_PRIO: 389 return damon_pa_activate_pages(r, aunit, scheme, 390 sz_filter_passed); 391 case DAMOS_LRU_DEPRIO: 392 return damon_pa_deactivate_pages(r, aunit, scheme, 393 sz_filter_passed); 394 case DAMOS_MIGRATE_HOT: 395 case DAMOS_MIGRATE_COLD: 396 return damon_pa_migrate(r, aunit, scheme, sz_filter_passed); 397 case DAMOS_STAT: 398 return damon_pa_stat(r, aunit, scheme, sz_filter_passed); 399 default: 400 /* DAMOS actions that not yet supported by 'paddr'. */ 401 break; 402 } 403 return 0; 404 } 405 406 static int damon_pa_scheme_score(struct damon_ctx *context, 407 struct damon_region *r, struct damos *scheme) 408 { 409 switch (scheme->action) { 410 case DAMOS_PAGEOUT: 411 return damon_cold_score(context, r, scheme); 412 case DAMOS_LRU_PRIO: 413 return damon_hot_score(context, r, scheme); 414 case DAMOS_LRU_DEPRIO: 415 return damon_cold_score(context, r, scheme); 416 case DAMOS_MIGRATE_HOT: 417 return damon_hot_score(context, r, scheme); 418 case DAMOS_MIGRATE_COLD: 419 return damon_cold_score(context, r, scheme); 420 default: 421 break; 422 } 423 424 return DAMOS_MAX_SCORE; 425 } 426 427 static int __init damon_pa_initcall(void) 428 { 429 struct damon_operations ops = { 430 .id = DAMON_OPS_PADDR, 431 .init = NULL, 432 .update = NULL, 433 .prepare_access_checks = damon_pa_prepare_access_checks, 434 .check_accesses = damon_pa_check_accesses, 435 .apply_probes = damon_pa_apply_probes, 436 .target_valid = NULL, 437 .apply_scheme = damon_pa_apply_scheme, 438 .get_scheme_score = damon_pa_scheme_score, 439 }; 440 441 return damon_register_ops(&ops); 442 }; 443 444 subsys_initcall(damon_pa_initcall); 445