1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based LRU-lists Sorting 4 */ 5 6 #define pr_fmt(fmt) "damon-lru-sort: " fmt 7 8 #include <linux/damon.h> 9 #include <linux/kstrtox.h> 10 #include <linux/module.h> 11 12 #include "modules-common.h" 13 14 #ifdef MODULE_PARAM_PREFIX 15 #undef MODULE_PARAM_PREFIX 16 #endif 17 #define MODULE_PARAM_PREFIX "damon_lru_sort." 18 19 /* 20 * Enable or disable DAMON_LRU_SORT. 21 * 22 * You can enable DAMON_LRU_SORT by setting the value of this parameter as 23 * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that 24 * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the 25 * watermarks-based activation condition. Refer to below descriptions for the 26 * watermarks parameter for this. 27 */ 28 static bool enabled __read_mostly; 29 30 /* 31 * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. 32 * 33 * Input parameters that updated while DAMON_LRU_SORT is running are not 34 * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT 35 * reads values of parameters except ``enabled`` again. Once the re-reading is 36 * done, this parameter is set as ``N``. If invalid parameters are found while 37 * the re-reading, DAMON_LRU_SORT will be disabled. 38 */ 39 static bool commit_inputs __read_mostly; 40 41 /* 42 * Desired active to [in]active memory ratio in bp (1/10,000). 43 * 44 * While keeping the caps that set by other quotas, DAMON_LRU_SORT 45 * automatically increases and decreases the effective level of the quota 46 * aiming the LRU [de]prioritizations of the hot and cold memory resulting in 47 * this active to [in]active memory ratio. Value zero means disabling this 48 * auto-tuning feature. 49 * 50 * Disabled by default. 51 */ 52 static unsigned long active_mem_bp __read_mostly; 53 module_param(active_mem_bp, ulong, 0600); 54 55 /* 56 * Auto-tune monitoring intervals. 57 * 58 * If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes 59 * DAMON's sampling and aggregation intervals. The auto-tuning aims to capture 60 * meaningful amount of access events in each DAMON-snapshot, while keeping the 61 * sampling interval 5 milliseconds in minimum, and 10 seconds in maximum. 62 * Setting this as ``N`` disables the auto-tuning. 63 * 64 * Disabled by default. 65 */ 66 static bool autotune_monitoring_intervals __read_mostly; 67 module_param(autotune_monitoring_intervals, bool, 0600); 68 69 /* 70 * Filter [non-]young pages accordingly for LRU [de]prioritizations. 71 * 72 * If this is set, check page level access (youngness) once again before each 73 * LRU [de]prioritization operation. LRU prioritization operation is skipped 74 * if the page has not accessed since the last check (not young). LRU 75 * deprioritization operation is skipped if the page has accessed since the 76 * last check (young). The feature is enabled or disabled if this parameter is 77 * set as ``Y`` or ``N``, respectively. 78 * 79 * Disabled by default. 80 */ 81 static bool filter_young_pages __read_mostly; 82 module_param(filter_young_pages, bool, 0600); 83 84 /* 85 * Access frequency threshold for hot memory regions identification in permil. 86 * 87 * If a memory region is accessed in frequency of this or higher, 88 * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the 89 * LRU list, so that it could not be reclaimed under memory pressure. 50% by 90 * default. 91 */ 92 static unsigned long hot_thres_access_freq = 500; 93 module_param(hot_thres_access_freq, ulong, 0600); 94 95 /* 96 * Time threshold for cold memory regions identification in microseconds. 97 * 98 * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT 99 * identifies the region as cold, and mark it as unaccessed on the LRU list, so 100 * that it could be reclaimed first under memory pressure. 120 seconds by 101 * default. 102 */ 103 static unsigned long cold_min_age __read_mostly = 120000000; 104 module_param(cold_min_age, ulong, 0600); 105 106 static struct damos_quota damon_lru_sort_quota = { 107 /* Use up to 10 ms per 1 sec, by default */ 108 .ms = 10, 109 .sz = 0, 110 .reset_interval = 1000, 111 /* Within the quota, mark hotter regions accessed first. */ 112 .weight_sz = 0, 113 .weight_nr_accesses = 1, 114 .weight_age = 1, 115 }; 116 DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); 117 118 static struct damos_watermarks damon_lru_sort_wmarks = { 119 .metric = DAMOS_WMARK_FREE_MEM_RATE, 120 .interval = 5000000, /* 5 seconds */ 121 .high = 200, /* 20 percent */ 122 .mid = 150, /* 15 percent */ 123 .low = 50, /* 5 percent */ 124 }; 125 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); 126 127 static struct damon_attrs damon_lru_sort_mon_attrs = { 128 .sample_interval = 5000, /* 5 ms */ 129 .aggr_interval = 100000, /* 100 ms */ 130 .ops_update_interval = 0, 131 .min_nr_regions = 10, 132 .max_nr_regions = 1000, 133 }; 134 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); 135 136 /* 137 * Start of the target memory region in physical address. 138 * 139 * The start physical address of memory region that DAMON_LRU_SORT will do work 140 * against. By default, the system's entire physical memory is used as the 141 * region. 142 */ 143 static unsigned long monitor_region_start __read_mostly; 144 module_param(monitor_region_start, ulong, 0600); 145 146 /* 147 * End of the target memory region in physical address. 148 * 149 * The end physical address of memory region that DAMON_LRU_SORT will do work 150 * against. By default, the system's entire physical memory is used as the 151 * region. 152 */ 153 static unsigned long monitor_region_end __read_mostly; 154 module_param(monitor_region_end, ulong, 0600); 155 156 /* 157 * Scale factor for DAMON_LRU_SORT to ops address conversion. 158 * 159 * This parameter must not be set to 0. 160 */ 161 static unsigned long addr_unit __read_mostly = 1; 162 163 static struct damos_stat damon_lru_sort_hot_stat; 164 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, 165 lru_sort_tried_hot_regions, lru_sorted_hot_regions, 166 hot_quota_exceeds); 167 168 static struct damos_stat damon_lru_sort_cold_stat; 169 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, 170 lru_sort_tried_cold_regions, lru_sorted_cold_regions, 171 cold_quota_exceeds); 172 173 static struct damos_access_pattern damon_lru_sort_stub_pattern = { 174 /* Find regions having PAGE_SIZE or larger size */ 175 .min_sz_region = PAGE_SIZE, 176 .max_sz_region = ULONG_MAX, 177 /* no matter its access frequency */ 178 .min_nr_accesses = 0, 179 .max_nr_accesses = UINT_MAX, 180 /* no matter its age */ 181 .min_age_region = 0, 182 .max_age_region = UINT_MAX, 183 }; 184 185 static struct damon_ctx *ctx; 186 static struct damon_target *target; 187 188 static struct damos *damon_lru_sort_new_scheme( 189 struct damos_access_pattern *pattern, enum damos_action action) 190 { 191 struct damos_quota quota = damon_lru_sort_quota; 192 193 /* Use half of total quota for hot/cold pages sorting */ 194 quota.ms = quota.ms / 2; 195 196 return damon_new_scheme( 197 /* find the pattern, and */ 198 pattern, 199 /* (de)prioritize on LRU-lists */ 200 action, 201 /* for each aggregation interval */ 202 0, 203 /* under the quota. */ 204 "a, 205 /* (De)activate this according to the watermarks. */ 206 &damon_lru_sort_wmarks, 207 NUMA_NO_NODE); 208 } 209 210 /* Create a DAMON-based operation scheme for hot memory regions */ 211 static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) 212 { 213 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 214 215 pattern.min_nr_accesses = hot_thres; 216 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO); 217 } 218 219 /* Create a DAMON-based operation scheme for cold memory regions */ 220 static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) 221 { 222 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 223 224 pattern.max_nr_accesses = 0; 225 pattern.min_age_region = cold_thres; 226 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO); 227 } 228 229 static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, 230 struct damos *cold_scheme) 231 { 232 struct damos_quota_goal *goal; 233 234 if (!active_mem_bp) 235 return 0; 236 if (10000 < active_mem_bp) 237 return -EINVAL; 238 goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); 239 if (!goal) 240 return -ENOMEM; 241 damos_add_quota_goal(&hot_scheme->quota, goal); 242 /* aim 0.2 % goal conflict, to keep little ping pong */ 243 goal = damos_new_quota_goal(DAMOS_QUOTA_INACTIVE_MEM_BP, 244 10000 - active_mem_bp + 2); 245 if (!goal) 246 return -ENOMEM; 247 damos_add_quota_goal(&cold_scheme->quota, goal); 248 return 0; 249 } 250 251 static int damon_lru_sort_add_filters(struct damos *hot_scheme, 252 struct damos *cold_scheme) 253 { 254 struct damos_filter *filter; 255 256 if (!filter_young_pages) 257 return 0; 258 259 /* disallow prioritizing not-young pages */ 260 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, false, false); 261 if (!filter) 262 return -ENOMEM; 263 damos_add_filter(hot_scheme, filter); 264 265 /* disabllow de-prioritizing young pages */ 266 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); 267 if (!filter) 268 return -ENOMEM; 269 damos_add_filter(cold_scheme, filter); 270 return 0; 271 } 272 273 static int damon_lru_sort_apply_parameters(void) 274 { 275 struct damon_ctx *param_ctx; 276 struct damon_target *param_target; 277 struct damon_attrs attrs; 278 struct damos *hot_scheme, *cold_scheme; 279 unsigned int hot_thres, cold_thres; 280 int err; 281 282 err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); 283 if (err) 284 return err; 285 286 param_ctx->addr_unit = addr_unit; 287 param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); 288 289 if (!is_power_of_2(param_ctx->min_region_sz)) { 290 err = -EINVAL; 291 goto out; 292 } 293 294 if (!damon_lru_sort_mon_attrs.sample_interval) { 295 err = -EINVAL; 296 goto out; 297 } 298 299 attrs = damon_lru_sort_mon_attrs; 300 if (autotune_monitoring_intervals) { 301 attrs.sample_interval = 5000; 302 attrs.aggr_interval = 100000; 303 attrs.intervals_goal.access_bp = 40; 304 attrs.intervals_goal.aggrs = 3; 305 attrs.intervals_goal.min_sample_us = 5000; 306 attrs.intervals_goal.max_sample_us = 10 * 1000 * 1000; 307 } 308 err = damon_set_attrs(param_ctx, &attrs); 309 if (err) 310 goto out; 311 312 err = -ENOMEM; 313 hot_thres = damon_max_nr_accesses(&attrs) * 314 hot_thres_access_freq / 1000; 315 hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); 316 if (!hot_scheme) 317 goto out; 318 319 cold_thres = cold_min_age / attrs.aggr_interval; 320 cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres); 321 if (!cold_scheme) { 322 damon_destroy_scheme(hot_scheme); 323 goto out; 324 } 325 326 damon_set_schemes(param_ctx, &hot_scheme, 1); 327 damon_add_scheme(param_ctx, cold_scheme); 328 329 err = damon_lru_sort_add_quota_goals(hot_scheme, cold_scheme); 330 if (err) 331 goto out; 332 err = damon_lru_sort_add_filters(hot_scheme, cold_scheme); 333 if (err) 334 goto out; 335 336 err = damon_set_region_system_rams_default(param_target, 337 &monitor_region_start, 338 &monitor_region_end, 339 param_ctx->addr_unit, 340 param_ctx->min_region_sz); 341 if (err) 342 goto out; 343 err = damon_commit_ctx(ctx, param_ctx); 344 out: 345 damon_destroy_ctx(param_ctx); 346 return err; 347 } 348 349 static int damon_lru_sort_commit_inputs_fn(void *arg) 350 { 351 return damon_lru_sort_apply_parameters(); 352 } 353 354 static bool damon_lru_sort_damon_has_started; 355 356 static int damon_lru_sort_commit_inputs_store(const char *val, 357 const struct kernel_param *kp) 358 { 359 bool commit_inputs_request; 360 int err; 361 struct damon_call_control control = { 362 .fn = damon_lru_sort_commit_inputs_fn, 363 }; 364 365 if (!val) { 366 commit_inputs_request = true; 367 } else { 368 err = kstrtobool(val, &commit_inputs_request); 369 if (err) 370 return err; 371 } 372 373 if (!commit_inputs_request) 374 return 0; 375 376 /* Skip damon_call() if ctx has not successfully started. */ 377 if (!damon_lru_sort_damon_has_started) 378 return -EINVAL; 379 380 err = damon_call(ctx, &control); 381 382 return err ? err : control.return_code; 383 } 384 385 static const struct kernel_param_ops commit_inputs_param_ops = { 386 .flags = KERNEL_PARAM_OPS_FL_NOARG, 387 .set = damon_lru_sort_commit_inputs_store, 388 .get = param_get_bool, 389 }; 390 391 module_param_cb(commit_inputs, &commit_inputs_param_ops, &commit_inputs, 0600); 392 393 static int damon_lru_sort_damon_call_fn(void *arg) 394 { 395 struct damon_ctx *c = arg; 396 struct damos *s; 397 398 /* update the stats parameter */ 399 damon_for_each_scheme(s, c) { 400 if (s->action == DAMOS_LRU_PRIO) 401 damon_lru_sort_hot_stat = s->stat; 402 else if (s->action == DAMOS_LRU_DEPRIO) 403 damon_lru_sort_cold_stat = s->stat; 404 } 405 406 return 0; 407 } 408 409 static struct damon_call_control call_control = { 410 .fn = damon_lru_sort_damon_call_fn, 411 .repeat = true, 412 }; 413 414 static int damon_lru_sort_turn(bool on) 415 { 416 int err; 417 418 if (!on) 419 return damon_stop(&ctx, 1); 420 421 err = damon_lru_sort_apply_parameters(); 422 if (err) 423 return err; 424 425 err = damon_start(&ctx, 1, true); 426 if (err) 427 return err; 428 if (!damon_lru_sort_damon_has_started) 429 damon_lru_sort_damon_has_started = true; 430 return damon_call(ctx, &call_control); 431 } 432 433 static int damon_lru_sort_addr_unit_store(const char *val, 434 const struct kernel_param *kp) 435 { 436 unsigned long input_addr_unit; 437 int err = kstrtoul(val, 0, &input_addr_unit); 438 439 if (err) 440 return err; 441 if (!input_addr_unit) 442 return -EINVAL; 443 444 addr_unit = input_addr_unit; 445 return 0; 446 } 447 448 static const struct kernel_param_ops addr_unit_param_ops = { 449 .set = damon_lru_sort_addr_unit_store, 450 .get = param_get_ulong, 451 }; 452 453 module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); 454 MODULE_PARM_DESC(addr_unit, 455 "Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)"); 456 457 static bool damon_lru_sort_enabled(void) 458 { 459 if (!ctx) 460 return false; 461 return damon_is_running(ctx); 462 } 463 464 static int damon_lru_sort_enabled_store(const char *val, 465 const struct kernel_param *kp) 466 { 467 int err; 468 469 err = kstrtobool(val, &enabled); 470 if (err) 471 return err; 472 473 if (damon_lru_sort_enabled() == enabled) 474 return 0; 475 476 /* Called before init function. The function will handle this. */ 477 if (!damon_initialized()) 478 return 0; 479 480 /* damon_modules_new_paddr_ctx_target() in the init function failed. */ 481 if (!ctx) 482 return -ENOMEM; 483 484 return damon_lru_sort_turn(enabled); 485 } 486 487 static int damon_lru_sort_enabled_load(char *buffer, 488 const struct kernel_param *kp) 489 { 490 return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N'); 491 } 492 493 static const struct kernel_param_ops enabled_param_ops = { 494 .set = damon_lru_sort_enabled_store, 495 .get = damon_lru_sort_enabled_load, 496 }; 497 498 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 499 MODULE_PARM_DESC(enabled, 500 "Enable or disable DAMON_LRU_SORT (default: disabled)"); 501 502 static int damon_lru_sort_kdamond_pid_store(const char *val, 503 const struct kernel_param *kp) 504 { 505 /* 506 * kdamond_pid is read-only, but kernel command line could write it. 507 * Do nothing here. 508 */ 509 return 0; 510 } 511 512 static int damon_lru_sort_kdamond_pid_load(char *buffer, 513 const struct kernel_param *kp) 514 { 515 int kdamond_pid = -1; 516 517 if (ctx) { 518 kdamond_pid = damon_kdamond_pid(ctx); 519 if (kdamond_pid < 0) 520 kdamond_pid = -1; 521 } 522 return sprintf(buffer, "%d\n", kdamond_pid); 523 } 524 525 static const struct kernel_param_ops kdamond_pid_param_ops = { 526 .set = damon_lru_sort_kdamond_pid_store, 527 .get = damon_lru_sort_kdamond_pid_load, 528 }; 529 530 /* 531 * PID of the DAMON thread 532 * 533 * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. 534 * Else, -1. 535 */ 536 module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); 537 538 static int __init damon_lru_sort_init(void) 539 { 540 int err; 541 542 if (!damon_initialized()) { 543 err = -ENOMEM; 544 goto out; 545 } 546 err = damon_modules_new_paddr_ctx_target(&ctx, &target); 547 if (err) 548 goto out; 549 550 call_control.data = ctx; 551 552 /* 'enabled' has set before this function, probably via command line */ 553 if (enabled) 554 err = damon_lru_sort_turn(true); 555 556 out: 557 if (err && enabled) 558 enabled = false; 559 return err; 560 } 561 562 module_init(damon_lru_sort_init); 563