1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based LRU-lists Sorting 4 */ 5 6 #define pr_fmt(fmt) "damon-lru-sort: " fmt 7 8 #include <linux/damon.h> 9 #include <linux/kstrtox.h> 10 #include <linux/module.h> 11 12 #include "modules-common.h" 13 14 #ifdef MODULE_PARAM_PREFIX 15 #undef MODULE_PARAM_PREFIX 16 #endif 17 #define MODULE_PARAM_PREFIX "damon_lru_sort." 18 19 /* 20 * Enable or disable DAMON_LRU_SORT. 21 * 22 * You can enable DAMON_LRU_SORT by setting the value of this parameter as 23 * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that 24 * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the 25 * watermarks-based activation condition. Refer to below descriptions for the 26 * watermarks parameter for this. 27 */ 28 static bool enabled __read_mostly; 29 30 /* 31 * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. 32 * 33 * Input parameters that updated while DAMON_LRU_SORT is running are not 34 * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT 35 * reads values of parameters except ``enabled`` again. Once the re-reading is 36 * done, this parameter is set as ``N``. If invalid parameters are found while 37 * the re-reading, DAMON_LRU_SORT will be disabled. 38 */ 39 static bool commit_inputs __read_mostly; 40 41 /* 42 * Desired active to [in]active memory ratio in bp (1/10,000). 43 * 44 * While keeping the caps that set by other quotas, DAMON_LRU_SORT 45 * automatically increases and decreases the effective level of the quota 46 * aiming the LRU [de]prioritizations of the hot and cold memory resulting in 47 * this active to [in]active memory ratio. Value zero means disabling this 48 * auto-tuning feature. 49 * 50 * Disabled by default. 51 */ 52 static unsigned long active_mem_bp __read_mostly; 53 module_param(active_mem_bp, ulong, 0600); 54 55 /* 56 * Auto-tune monitoring intervals. 57 * 58 * If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes 59 * DAMON's sampling and aggregation intervals. The auto-tuning aims to capture 60 * meaningful amount of access events in each DAMON-snapshot, while keeping the 61 * sampling interval 5 milliseconds in minimum, and 10 seconds in maximum. 62 * Setting this as ``N`` disables the auto-tuning. 63 * 64 * Disabled by default. 65 */ 66 static bool autotune_monitoring_intervals __read_mostly; 67 module_param(autotune_monitoring_intervals, bool, 0600); 68 69 /* 70 * Filter [non-]young pages accordingly for LRU [de]prioritizations. 71 * 72 * If this is set, check page level access (youngness) once again before each 73 * LRU [de]prioritization operation. LRU prioritization operation is skipped 74 * if the page has not accessed since the last check (not young). LRU 75 * deprioritization operation is skipped if the page has accessed since the 76 * last check (young). The feature is enabled or disabled if this parameter is 77 * set as ``Y`` or ``N``, respectively. 78 * 79 * Disabled by default. 80 */ 81 static bool filter_young_pages __read_mostly; 82 module_param(filter_young_pages, bool, 0600); 83 84 /* 85 * Access frequency threshold for hot memory regions identification in permil. 86 * 87 * If a memory region is accessed in frequency of this or higher, 88 * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the 89 * LRU list, so that it could not be reclaimed under memory pressure. 50% by 90 * default. 91 */ 92 static unsigned long hot_thres_access_freq = 500; 93 module_param(hot_thres_access_freq, ulong, 0600); 94 95 /* 96 * Time threshold for cold memory regions identification in microseconds. 97 * 98 * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT 99 * identifies the region as cold, and mark it as unaccessed on the LRU list, so 100 * that it could be reclaimed first under memory pressure. 120 seconds by 101 * default. 102 */ 103 static unsigned long cold_min_age __read_mostly = 120000000; 104 module_param(cold_min_age, ulong, 0600); 105 106 static struct damos_quota damon_lru_sort_quota = { 107 /* Use up to 10 ms per 1 sec, by default */ 108 .ms = 10, 109 .sz = 0, 110 .reset_interval = 1000, 111 /* Within the quota, mark hotter regions accessed first. */ 112 .weight_sz = 0, 113 .weight_nr_accesses = 1, 114 .weight_age = 1, 115 }; 116 DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); 117 118 static struct damos_watermarks damon_lru_sort_wmarks = { 119 .metric = DAMOS_WMARK_FREE_MEM_RATE, 120 .interval = 5000000, /* 5 seconds */ 121 .high = 200, /* 20 percent */ 122 .mid = 150, /* 15 percent */ 123 .low = 50, /* 5 percent */ 124 }; 125 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); 126 127 static struct damon_attrs damon_lru_sort_mon_attrs = { 128 .sample_interval = 5000, /* 5 ms */ 129 .aggr_interval = 100000, /* 100 ms */ 130 .ops_update_interval = 0, 131 .min_nr_regions = 10, 132 .max_nr_regions = 1000, 133 }; 134 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); 135 136 /* 137 * Start of the target memory region in physical address. 138 * 139 * The start physical address of memory region that DAMON_LRU_SORT will do work 140 * against. By default, the system's entire physical memory is used as the 141 * region. 142 */ 143 static unsigned long monitor_region_start __read_mostly; 144 module_param(monitor_region_start, ulong, 0600); 145 146 /* 147 * End of the target memory region in physical address. 148 * 149 * The end physical address of memory region that DAMON_LRU_SORT will do work 150 * against. By default, the system's entire physical memory is used as the 151 * region. 152 */ 153 static unsigned long monitor_region_end __read_mostly; 154 module_param(monitor_region_end, ulong, 0600); 155 156 /* 157 * Scale factor for DAMON_LRU_SORT to ops address conversion. 158 * 159 * This parameter must not be set to 0. 160 */ 161 static unsigned long addr_unit __read_mostly = 1; 162 163 static struct damos_stat damon_lru_sort_hot_stat; 164 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, 165 lru_sort_tried_hot_regions, lru_sorted_hot_regions, 166 hot_quota_exceeds); 167 168 static struct damos_stat damon_lru_sort_cold_stat; 169 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, 170 lru_sort_tried_cold_regions, lru_sorted_cold_regions, 171 cold_quota_exceeds); 172 173 static struct damos_access_pattern damon_lru_sort_stub_pattern = { 174 /* Find regions having PAGE_SIZE or larger size */ 175 .min_sz_region = PAGE_SIZE, 176 .max_sz_region = ULONG_MAX, 177 /* no matter its access frequency */ 178 .min_nr_accesses = 0, 179 .max_nr_accesses = UINT_MAX, 180 /* no matter its age */ 181 .min_age_region = 0, 182 .max_age_region = UINT_MAX, 183 }; 184 185 static struct damon_ctx *ctx; 186 static struct damon_target *target; 187 188 static struct damos *damon_lru_sort_new_scheme( 189 struct damos_access_pattern *pattern, enum damos_action action) 190 { 191 struct damos_quota quota = damon_lru_sort_quota; 192 193 /* Use half of total quota for hot/cold pages sorting */ 194 quota.ms = quota.ms / 2; 195 196 return damon_new_scheme( 197 /* find the pattern, and */ 198 pattern, 199 /* (de)prioritize on LRU-lists */ 200 action, 201 /* for each aggregation interval */ 202 0, 203 /* under the quota. */ 204 "a, 205 /* (De)activate this according to the watermarks. */ 206 &damon_lru_sort_wmarks, 207 NUMA_NO_NODE); 208 } 209 210 /* Create a DAMON-based operation scheme for hot memory regions */ 211 static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) 212 { 213 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 214 215 pattern.min_nr_accesses = hot_thres; 216 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO); 217 } 218 219 /* Create a DAMON-based operation scheme for cold memory regions */ 220 static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) 221 { 222 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 223 224 pattern.max_nr_accesses = 0; 225 pattern.min_age_region = cold_thres; 226 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO); 227 } 228 229 static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, 230 struct damos *cold_scheme) 231 { 232 struct damos_quota_goal *goal; 233 234 if (!active_mem_bp) 235 return 0; 236 goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); 237 if (!goal) 238 return -ENOMEM; 239 damos_add_quota_goal(&hot_scheme->quota, goal); 240 /* aim 0.2 % goal conflict, to keep little ping pong */ 241 goal = damos_new_quota_goal(DAMOS_QUOTA_INACTIVE_MEM_BP, 242 10000 - active_mem_bp + 2); 243 if (!goal) 244 return -ENOMEM; 245 damos_add_quota_goal(&cold_scheme->quota, goal); 246 return 0; 247 } 248 249 static int damon_lru_sort_add_filters(struct damos *hot_scheme, 250 struct damos *cold_scheme) 251 { 252 struct damos_filter *filter; 253 254 if (!filter_young_pages) 255 return 0; 256 257 /* disallow prioritizing not-young pages */ 258 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, false, false); 259 if (!filter) 260 return -ENOMEM; 261 damos_add_filter(hot_scheme, filter); 262 263 /* disabllow de-prioritizing young pages */ 264 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); 265 if (!filter) 266 return -ENOMEM; 267 damos_add_filter(cold_scheme, filter); 268 return 0; 269 } 270 271 static int damon_lru_sort_apply_parameters(void) 272 { 273 struct damon_ctx *param_ctx; 274 struct damon_target *param_target; 275 struct damon_attrs attrs; 276 struct damos *hot_scheme, *cold_scheme; 277 unsigned int hot_thres, cold_thres; 278 int err; 279 280 err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); 281 if (err) 282 return err; 283 284 param_ctx->addr_unit = addr_unit; 285 param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); 286 287 if (!is_power_of_2(param_ctx->min_region_sz)) { 288 err = -EINVAL; 289 goto out; 290 } 291 292 if (!damon_lru_sort_mon_attrs.sample_interval) { 293 err = -EINVAL; 294 goto out; 295 } 296 297 attrs = damon_lru_sort_mon_attrs; 298 if (autotune_monitoring_intervals) { 299 attrs.sample_interval = 5000; 300 attrs.aggr_interval = 100000; 301 attrs.intervals_goal.access_bp = 40; 302 attrs.intervals_goal.aggrs = 3; 303 attrs.intervals_goal.min_sample_us = 5000; 304 attrs.intervals_goal.max_sample_us = 10 * 1000 * 1000; 305 } 306 err = damon_set_attrs(param_ctx, &attrs); 307 if (err) 308 goto out; 309 310 err = -ENOMEM; 311 hot_thres = damon_max_nr_accesses(&attrs) * 312 hot_thres_access_freq / 1000; 313 hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); 314 if (!hot_scheme) 315 goto out; 316 317 cold_thres = cold_min_age / attrs.aggr_interval; 318 cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres); 319 if (!cold_scheme) { 320 damon_destroy_scheme(hot_scheme); 321 goto out; 322 } 323 324 damon_set_schemes(param_ctx, &hot_scheme, 1); 325 damon_add_scheme(param_ctx, cold_scheme); 326 327 err = damon_lru_sort_add_quota_goals(hot_scheme, cold_scheme); 328 if (err) 329 goto out; 330 err = damon_lru_sort_add_filters(hot_scheme, cold_scheme); 331 if (err) 332 goto out; 333 334 err = damon_set_region_system_rams_default(param_target, 335 &monitor_region_start, 336 &monitor_region_end, 337 param_ctx->addr_unit, 338 param_ctx->min_region_sz); 339 if (err) 340 goto out; 341 err = damon_commit_ctx(ctx, param_ctx); 342 out: 343 damon_destroy_ctx(param_ctx); 344 return err; 345 } 346 347 static int damon_lru_sort_commit_inputs_fn(void *arg) 348 { 349 return damon_lru_sort_apply_parameters(); 350 } 351 352 static int damon_lru_sort_commit_inputs_store(const char *val, 353 const struct kernel_param *kp) 354 { 355 bool commit_inputs_request; 356 int err; 357 struct damon_call_control control = { 358 .fn = damon_lru_sort_commit_inputs_fn, 359 }; 360 361 if (!val) { 362 commit_inputs_request = true; 363 } else { 364 err = kstrtobool(val, &commit_inputs_request); 365 if (err) 366 return err; 367 } 368 369 if (!commit_inputs_request) 370 return 0; 371 372 /* 373 * Skip damon_call() if ctx is not initialized to avoid 374 * NULL pointer dereference. 375 */ 376 if (!ctx) 377 return -EINVAL; 378 379 err = damon_call(ctx, &control); 380 381 return err ? err : control.return_code; 382 } 383 384 static const struct kernel_param_ops commit_inputs_param_ops = { 385 .flags = KERNEL_PARAM_OPS_FL_NOARG, 386 .set = damon_lru_sort_commit_inputs_store, 387 .get = param_get_bool, 388 }; 389 390 module_param_cb(commit_inputs, &commit_inputs_param_ops, &commit_inputs, 0600); 391 392 static int damon_lru_sort_damon_call_fn(void *arg) 393 { 394 struct damon_ctx *c = arg; 395 struct damos *s; 396 397 /* update the stats parameter */ 398 damon_for_each_scheme(s, c) { 399 if (s->action == DAMOS_LRU_PRIO) 400 damon_lru_sort_hot_stat = s->stat; 401 else if (s->action == DAMOS_LRU_DEPRIO) 402 damon_lru_sort_cold_stat = s->stat; 403 } 404 405 return 0; 406 } 407 408 static struct damon_call_control call_control = { 409 .fn = damon_lru_sort_damon_call_fn, 410 .repeat = true, 411 }; 412 413 static int damon_lru_sort_turn(bool on) 414 { 415 int err; 416 417 if (!on) 418 return damon_stop(&ctx, 1); 419 420 err = damon_lru_sort_apply_parameters(); 421 if (err) 422 return err; 423 424 err = damon_start(&ctx, 1, true); 425 if (err) 426 return err; 427 return damon_call(ctx, &call_control); 428 } 429 430 static int damon_lru_sort_addr_unit_store(const char *val, 431 const struct kernel_param *kp) 432 { 433 unsigned long input_addr_unit; 434 int err = kstrtoul(val, 0, &input_addr_unit); 435 436 if (err) 437 return err; 438 if (!input_addr_unit) 439 return -EINVAL; 440 441 addr_unit = input_addr_unit; 442 return 0; 443 } 444 445 static const struct kernel_param_ops addr_unit_param_ops = { 446 .set = damon_lru_sort_addr_unit_store, 447 .get = param_get_ulong, 448 }; 449 450 module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); 451 MODULE_PARM_DESC(addr_unit, 452 "Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)"); 453 454 static bool damon_lru_sort_enabled(void) 455 { 456 if (!ctx) 457 return false; 458 return damon_is_running(ctx); 459 } 460 461 static int damon_lru_sort_enabled_store(const char *val, 462 const struct kernel_param *kp) 463 { 464 int err; 465 466 err = kstrtobool(val, &enabled); 467 if (err) 468 return err; 469 470 if (damon_lru_sort_enabled() == enabled) 471 return 0; 472 473 /* Called before init function. The function will handle this. */ 474 if (!damon_initialized()) 475 return 0; 476 477 /* damon_modules_new_paddr_ctx_target() in the init function failed. */ 478 if (!ctx) 479 return -ENOMEM; 480 481 return damon_lru_sort_turn(enabled); 482 } 483 484 static int damon_lru_sort_enabled_load(char *buffer, 485 const struct kernel_param *kp) 486 { 487 return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N'); 488 } 489 490 static const struct kernel_param_ops enabled_param_ops = { 491 .set = damon_lru_sort_enabled_store, 492 .get = damon_lru_sort_enabled_load, 493 }; 494 495 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 496 MODULE_PARM_DESC(enabled, 497 "Enable or disable DAMON_LRU_SORT (default: disabled)"); 498 499 static int damon_lru_sort_kdamond_pid_store(const char *val, 500 const struct kernel_param *kp) 501 { 502 /* 503 * kdamond_pid is read-only, but kernel command line could write it. 504 * Do nothing here. 505 */ 506 return 0; 507 } 508 509 static int damon_lru_sort_kdamond_pid_load(char *buffer, 510 const struct kernel_param *kp) 511 { 512 int kdamond_pid = -1; 513 514 if (ctx) { 515 kdamond_pid = damon_kdamond_pid(ctx); 516 if (kdamond_pid < 0) 517 kdamond_pid = -1; 518 } 519 return sprintf(buffer, "%d\n", kdamond_pid); 520 } 521 522 static const struct kernel_param_ops kdamond_pid_param_ops = { 523 .set = damon_lru_sort_kdamond_pid_store, 524 .get = damon_lru_sort_kdamond_pid_load, 525 }; 526 527 /* 528 * PID of the DAMON thread 529 * 530 * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. 531 * Else, -1. 532 */ 533 module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); 534 535 static int __init damon_lru_sort_init(void) 536 { 537 int err; 538 539 if (!damon_initialized()) { 540 err = -ENOMEM; 541 goto out; 542 } 543 err = damon_modules_new_paddr_ctx_target(&ctx, &target); 544 if (err) 545 goto out; 546 547 call_control.data = ctx; 548 549 /* 'enabled' has set before this function, probably via command line */ 550 if (enabled) 551 err = damon_lru_sort_turn(true); 552 553 out: 554 if (err && enabled) 555 enabled = false; 556 return err; 557 } 558 559 module_init(damon_lru_sort_init); 560