1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based LRU-lists Sorting 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-lru-sort: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/kstrtox.h> 12 #include <linux/module.h> 13 14 #include "modules-common.h" 15 16 #ifdef MODULE_PARAM_PREFIX 17 #undef MODULE_PARAM_PREFIX 18 #endif 19 #define MODULE_PARAM_PREFIX "damon_lru_sort." 20 21 /* 22 * Enable or disable DAMON_LRU_SORT. 23 * 24 * You can enable DAMON_LRU_SORT by setting the value of this parameter as 25 * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that 26 * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the 27 * watermarks-based activation condition. Refer to below descriptions for the 28 * watermarks parameter for this. 29 */ 30 static bool enabled __read_mostly; 31 32 /* 33 * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. 34 * 35 * Input parameters that updated while DAMON_LRU_SORT is running are not 36 * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT 37 * reads values of parameters except ``enabled`` again. Once the re-reading is 38 * done, this parameter is set as ``N``. If invalid parameters are found while 39 * the re-reading, DAMON_LRU_SORT will be disabled. 40 */ 41 static bool commit_inputs __read_mostly; 42 43 /* 44 * Desired active to [in]active memory ratio in bp (1/10,000). 45 * 46 * While keeping the caps that set by other quotas, DAMON_LRU_SORT 47 * automatically increases and decreases the effective level of the quota 48 * aiming the LRU [de]prioritizations of the hot and cold memory resulting in 49 * this active to [in]active memory ratio. Value zero means disabling this 50 * auto-tuning feature. 51 * 52 * Disabled by default. 53 */ 54 static unsigned long active_mem_bp __read_mostly; 55 module_param(active_mem_bp, ulong, 0600); 56 57 /* 58 * Auto-tune monitoring intervals. 59 * 60 * If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes 61 * DAMON's sampling and aggregation intervals. The auto-tuning aims to capture 62 * meaningful amount of access events in each DAMON-snapshot, while keeping the 63 * sampling interval 5 milliseconds in minimum, and 10 seconds in maximum. 64 * Setting this as ``N`` disables the auto-tuning. 65 * 66 * Disabled by default. 67 */ 68 static bool autotune_monitoring_intervals __read_mostly; 69 module_param(autotune_monitoring_intervals, bool, 0600); 70 71 /* 72 * Filter [non-]young pages accordingly for LRU [de]prioritizations. 73 * 74 * If this is set, check page level access (youngness) once again before each 75 * LRU [de]prioritization operation. LRU prioritization operation is skipped 76 * if the page has not accessed since the last check (not young). LRU 77 * deprioritization operation is skipped if the page has accessed since the 78 * last check (young). The feature is enabled or disabled if this parameter is 79 * set as ``Y`` or ``N``, respectively. 80 * 81 * Disabled by default. 82 */ 83 static bool filter_young_pages __read_mostly; 84 module_param(filter_young_pages, bool, 0600); 85 86 /* 87 * Access frequency threshold for hot memory regions identification in permil. 88 * 89 * If a memory region is accessed in frequency of this or higher, 90 * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the 91 * LRU list, so that it could not be reclaimed under memory pressure. 50% by 92 * default. 93 */ 94 static unsigned long hot_thres_access_freq = 500; 95 module_param(hot_thres_access_freq, ulong, 0600); 96 97 /* 98 * Time threshold for cold memory regions identification in microseconds. 99 * 100 * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT 101 * identifies the region as cold, and mark it as unaccessed on the LRU list, so 102 * that it could be reclaimed first under memory pressure. 120 seconds by 103 * default. 104 */ 105 static unsigned long cold_min_age __read_mostly = 120000000; 106 module_param(cold_min_age, ulong, 0600); 107 108 static struct damos_quota damon_lru_sort_quota = { 109 /* Use up to 10 ms per 1 sec, by default */ 110 .ms = 10, 111 .sz = 0, 112 .reset_interval = 1000, 113 /* Within the quota, mark hotter regions accessed first. */ 114 .weight_sz = 0, 115 .weight_nr_accesses = 1, 116 .weight_age = 1, 117 }; 118 DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); 119 120 static struct damos_watermarks damon_lru_sort_wmarks = { 121 .metric = DAMOS_WMARK_FREE_MEM_RATE, 122 .interval = 5000000, /* 5 seconds */ 123 .high = 200, /* 20 percent */ 124 .mid = 150, /* 15 percent */ 125 .low = 50, /* 5 percent */ 126 }; 127 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); 128 129 static struct damon_attrs damon_lru_sort_mon_attrs = { 130 .sample_interval = 5000, /* 5 ms */ 131 .aggr_interval = 100000, /* 100 ms */ 132 .ops_update_interval = 0, 133 .min_nr_regions = 10, 134 .max_nr_regions = 1000, 135 }; 136 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); 137 138 /* 139 * Start of the target memory region in physical address. 140 * 141 * The start physical address of memory region that DAMON_LRU_SORT will do work 142 * against. By default, the system's entire physical memory is used as the 143 * region. 144 */ 145 static unsigned long monitor_region_start __read_mostly; 146 module_param(monitor_region_start, ulong, 0600); 147 148 /* 149 * End of the target memory region in physical address. 150 * 151 * The end physical address of memory region that DAMON_LRU_SORT will do work 152 * against. By default, the system's entire physical memory is used as the 153 * region. 154 */ 155 static unsigned long monitor_region_end __read_mostly; 156 module_param(monitor_region_end, ulong, 0600); 157 158 /* 159 * Scale factor for DAMON_LRU_SORT to ops address conversion. 160 * 161 * This parameter must not be set to 0. 162 */ 163 static unsigned long addr_unit __read_mostly = 1; 164 165 static struct damos_stat damon_lru_sort_hot_stat; 166 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, 167 lru_sort_tried_hot_regions, lru_sorted_hot_regions, 168 hot_quota_exceeds); 169 170 static struct damos_stat damon_lru_sort_cold_stat; 171 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, 172 lru_sort_tried_cold_regions, lru_sorted_cold_regions, 173 cold_quota_exceeds); 174 175 static struct damos_access_pattern damon_lru_sort_stub_pattern = { 176 /* Find regions having PAGE_SIZE or larger size */ 177 .min_sz_region = PAGE_SIZE, 178 .max_sz_region = ULONG_MAX, 179 /* no matter its access frequency */ 180 .min_nr_accesses = 0, 181 .max_nr_accesses = UINT_MAX, 182 /* no matter its age */ 183 .min_age_region = 0, 184 .max_age_region = UINT_MAX, 185 }; 186 187 static struct damon_ctx *ctx; 188 static struct damon_target *target; 189 190 static struct damos *damon_lru_sort_new_scheme( 191 struct damos_access_pattern *pattern, enum damos_action action) 192 { 193 struct damos_quota quota = damon_lru_sort_quota; 194 195 /* Use half of total quota for hot/cold pages sorting */ 196 quota.ms = quota.ms / 2; 197 198 return damon_new_scheme( 199 /* find the pattern, and */ 200 pattern, 201 /* (de)prioritize on LRU-lists */ 202 action, 203 /* for each aggregation interval */ 204 0, 205 /* under the quota. */ 206 "a, 207 /* (De)activate this according to the watermarks. */ 208 &damon_lru_sort_wmarks, 209 NUMA_NO_NODE); 210 } 211 212 /* Create a DAMON-based operation scheme for hot memory regions */ 213 static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) 214 { 215 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 216 217 pattern.min_nr_accesses = hot_thres; 218 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO); 219 } 220 221 /* Create a DAMON-based operation scheme for cold memory regions */ 222 static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) 223 { 224 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 225 226 pattern.max_nr_accesses = 0; 227 pattern.min_age_region = cold_thres; 228 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO); 229 } 230 231 static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, 232 struct damos *cold_scheme) 233 { 234 struct damos_quota_goal *goal; 235 236 if (!active_mem_bp) 237 return 0; 238 goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); 239 if (!goal) 240 return -ENOMEM; 241 damos_add_quota_goal(&hot_scheme->quota, goal); 242 /* aim 0.2 % goal conflict, to keep little ping pong */ 243 goal = damos_new_quota_goal(DAMOS_QUOTA_INACTIVE_MEM_BP, 244 10000 - active_mem_bp + 2); 245 if (!goal) 246 return -ENOMEM; 247 damos_add_quota_goal(&cold_scheme->quota, goal); 248 return 0; 249 } 250 251 static int damon_lru_sort_add_filters(struct damos *hot_scheme, 252 struct damos *cold_scheme) 253 { 254 struct damos_filter *filter; 255 256 if (!filter_young_pages) 257 return 0; 258 259 /* disallow prioritizing not-young pages */ 260 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, false, false); 261 if (!filter) 262 return -ENOMEM; 263 damos_add_filter(hot_scheme, filter); 264 265 /* disabllow de-prioritizing young pages */ 266 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); 267 if (!filter) 268 return -ENOMEM; 269 damos_add_filter(cold_scheme, filter); 270 return 0; 271 } 272 273 static int damon_lru_sort_apply_parameters(void) 274 { 275 struct damon_ctx *param_ctx; 276 struct damon_target *param_target; 277 struct damon_attrs attrs; 278 struct damos *hot_scheme, *cold_scheme; 279 unsigned int hot_thres, cold_thres; 280 int err; 281 282 err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); 283 if (err) 284 return err; 285 286 param_ctx->addr_unit = addr_unit; 287 param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); 288 289 if (!is_power_of_2(param_ctx->min_region_sz)) { 290 err = -EINVAL; 291 goto out; 292 } 293 294 if (!damon_lru_sort_mon_attrs.sample_interval) { 295 err = -EINVAL; 296 goto out; 297 } 298 299 attrs = damon_lru_sort_mon_attrs; 300 if (autotune_monitoring_intervals) { 301 attrs.sample_interval = 5000; 302 attrs.aggr_interval = 100000; 303 attrs.intervals_goal.access_bp = 40; 304 attrs.intervals_goal.aggrs = 3; 305 attrs.intervals_goal.min_sample_us = 5000; 306 attrs.intervals_goal.max_sample_us = 10 * 1000 * 1000; 307 } 308 err = damon_set_attrs(param_ctx, &attrs); 309 if (err) 310 goto out; 311 312 err = -ENOMEM; 313 hot_thres = damon_max_nr_accesses(&attrs) * 314 hot_thres_access_freq / 1000; 315 hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); 316 if (!hot_scheme) 317 goto out; 318 319 cold_thres = cold_min_age / attrs.aggr_interval; 320 cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres); 321 if (!cold_scheme) { 322 damon_destroy_scheme(hot_scheme); 323 goto out; 324 } 325 326 damon_set_schemes(param_ctx, &hot_scheme, 1); 327 damon_add_scheme(param_ctx, cold_scheme); 328 329 err = damon_lru_sort_add_quota_goals(hot_scheme, cold_scheme); 330 if (err) 331 goto out; 332 err = damon_lru_sort_add_filters(hot_scheme, cold_scheme); 333 if (err) 334 goto out; 335 336 err = damon_set_region_system_rams_default(param_target, 337 &monitor_region_start, 338 &monitor_region_end, 339 param_ctx->addr_unit, 340 param_ctx->min_region_sz); 341 if (err) 342 goto out; 343 err = damon_commit_ctx(ctx, param_ctx); 344 out: 345 damon_destroy_ctx(param_ctx); 346 return err; 347 } 348 349 static int damon_lru_sort_commit_inputs_fn(void *arg) 350 { 351 return damon_lru_sort_apply_parameters(); 352 } 353 354 static int damon_lru_sort_commit_inputs_store(const char *val, 355 const struct kernel_param *kp) 356 { 357 bool commit_inputs_request; 358 int err; 359 struct damon_call_control control = { 360 .fn = damon_lru_sort_commit_inputs_fn, 361 }; 362 363 if (!val) { 364 commit_inputs_request = true; 365 } else { 366 err = kstrtobool(val, &commit_inputs_request); 367 if (err) 368 return err; 369 } 370 371 if (!commit_inputs_request) 372 return 0; 373 374 /* 375 * Skip damon_call() if ctx is not initialized to avoid 376 * NULL pointer dereference. 377 */ 378 if (!ctx) 379 return -EINVAL; 380 381 err = damon_call(ctx, &control); 382 383 return err ? err : control.return_code; 384 } 385 386 static const struct kernel_param_ops commit_inputs_param_ops = { 387 .flags = KERNEL_PARAM_OPS_FL_NOARG, 388 .set = damon_lru_sort_commit_inputs_store, 389 .get = param_get_bool, 390 }; 391 392 module_param_cb(commit_inputs, &commit_inputs_param_ops, &commit_inputs, 0600); 393 394 static int damon_lru_sort_damon_call_fn(void *arg) 395 { 396 struct damon_ctx *c = arg; 397 struct damos *s; 398 399 /* update the stats parameter */ 400 damon_for_each_scheme(s, c) { 401 if (s->action == DAMOS_LRU_PRIO) 402 damon_lru_sort_hot_stat = s->stat; 403 else if (s->action == DAMOS_LRU_DEPRIO) 404 damon_lru_sort_cold_stat = s->stat; 405 } 406 407 return 0; 408 } 409 410 static struct damon_call_control call_control = { 411 .fn = damon_lru_sort_damon_call_fn, 412 .repeat = true, 413 }; 414 415 static int damon_lru_sort_turn(bool on) 416 { 417 int err; 418 419 if (!on) 420 return damon_stop(&ctx, 1); 421 422 err = damon_lru_sort_apply_parameters(); 423 if (err) 424 return err; 425 426 err = damon_start(&ctx, 1, true); 427 if (err) 428 return err; 429 return damon_call(ctx, &call_control); 430 } 431 432 static int damon_lru_sort_addr_unit_store(const char *val, 433 const struct kernel_param *kp) 434 { 435 unsigned long input_addr_unit; 436 int err = kstrtoul(val, 0, &input_addr_unit); 437 438 if (err) 439 return err; 440 if (!input_addr_unit) 441 return -EINVAL; 442 443 addr_unit = input_addr_unit; 444 return 0; 445 } 446 447 static const struct kernel_param_ops addr_unit_param_ops = { 448 .set = damon_lru_sort_addr_unit_store, 449 .get = param_get_ulong, 450 }; 451 452 module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); 453 MODULE_PARM_DESC(addr_unit, 454 "Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)"); 455 456 static bool damon_lru_sort_enabled(void) 457 { 458 if (!ctx) 459 return false; 460 return damon_is_running(ctx); 461 } 462 463 static int damon_lru_sort_enabled_store(const char *val, 464 const struct kernel_param *kp) 465 { 466 int err; 467 468 err = kstrtobool(val, &enabled); 469 if (err) 470 return err; 471 472 if (damon_lru_sort_enabled() == enabled) 473 return 0; 474 475 /* Called before init function. The function will handle this. */ 476 if (!damon_initialized()) 477 return 0; 478 479 return damon_lru_sort_turn(enabled); 480 } 481 482 static int damon_lru_sort_enabled_load(char *buffer, 483 const struct kernel_param *kp) 484 { 485 return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N'); 486 } 487 488 static const struct kernel_param_ops enabled_param_ops = { 489 .set = damon_lru_sort_enabled_store, 490 .get = damon_lru_sort_enabled_load, 491 }; 492 493 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 494 MODULE_PARM_DESC(enabled, 495 "Enable or disable DAMON_LRU_SORT (default: disabled)"); 496 497 static int damon_lru_sort_kdamond_pid_store(const char *val, 498 const struct kernel_param *kp) 499 { 500 /* 501 * kdamond_pid is read-only, but kernel command line could write it. 502 * Do nothing here. 503 */ 504 return 0; 505 } 506 507 static int damon_lru_sort_kdamond_pid_load(char *buffer, 508 const struct kernel_param *kp) 509 { 510 int kdamond_pid = -1; 511 512 if (ctx) { 513 kdamond_pid = damon_kdamond_pid(ctx); 514 if (kdamond_pid < 0) 515 kdamond_pid = -1; 516 } 517 return sprintf(buffer, "%d\n", kdamond_pid); 518 } 519 520 static const struct kernel_param_ops kdamond_pid_param_ops = { 521 .set = damon_lru_sort_kdamond_pid_store, 522 .get = damon_lru_sort_kdamond_pid_load, 523 }; 524 525 /* 526 * PID of the DAMON thread 527 * 528 * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. 529 * Else, -1. 530 */ 531 module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); 532 533 static int __init damon_lru_sort_init(void) 534 { 535 int err; 536 537 if (!damon_initialized()) { 538 err = -ENOMEM; 539 goto out; 540 } 541 err = damon_modules_new_paddr_ctx_target(&ctx, &target); 542 if (err) 543 goto out; 544 545 call_control.data = ctx; 546 547 /* 'enabled' has set before this function, probably via command line */ 548 if (enabled) 549 err = damon_lru_sort_turn(true); 550 551 out: 552 if (err && enabled) 553 enabled = false; 554 return err; 555 } 556 557 module_init(damon_lru_sort_init); 558