1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based LRU-lists Sorting 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-lru-sort: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/kstrtox.h> 12 #include <linux/module.h> 13 14 #include "modules-common.h" 15 16 #ifdef MODULE_PARAM_PREFIX 17 #undef MODULE_PARAM_PREFIX 18 #endif 19 #define MODULE_PARAM_PREFIX "damon_lru_sort." 20 21 /* 22 * Enable or disable DAMON_LRU_SORT. 23 * 24 * You can enable DAMON_LRU_SORT by setting the value of this parameter as 25 * ``Y``. Setting it as ``N`` disables DAMON_LRU_SORT. Note that 26 * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the 27 * watermarks-based activation condition. Refer to below descriptions for the 28 * watermarks parameter for this. 29 */ 30 static bool enabled __read_mostly; 31 32 /* 33 * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``. 34 * 35 * Input parameters that updated while DAMON_LRU_SORT is running are not 36 * applied by default. Once this parameter is set as ``Y``, DAMON_LRU_SORT 37 * reads values of parameters except ``enabled`` again. Once the re-reading is 38 * done, this parameter is set as ``N``. If invalid parameters are found while 39 * the re-reading, DAMON_LRU_SORT will be disabled. 40 */ 41 static bool commit_inputs __read_mostly; 42 module_param(commit_inputs, bool, 0600); 43 44 /* 45 * Desired active to [in]active memory ratio in bp (1/10,000). 46 * 47 * While keeping the caps that set by other quotas, DAMON_LRU_SORT 48 * automatically increases and decreases the effective level of the quota 49 * aiming the LRU [de]prioritizations of the hot and cold memory resulting in 50 * this active to [in]active memory ratio. Value zero means disabling this 51 * auto-tuning feature. 52 * 53 * Disabled by default. 54 */ 55 static unsigned long active_mem_bp __read_mostly; 56 module_param(active_mem_bp, ulong, 0600); 57 58 /* 59 * Auto-tune monitoring intervals. 60 * 61 * If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes 62 * DAMON's sampling and aggregation intervals. The auto-tuning aims to capture 63 * meaningful amount of access events in each DAMON-snapshot, while keeping the 64 * sampling interval 5 milliseconds in minimum, and 10 seconds in maximum. 65 * Setting this as ``N`` disables the auto-tuning. 66 * 67 * Disabled by default. 68 */ 69 static bool autotune_monitoring_intervals __read_mostly; 70 module_param(autotune_monitoring_intervals, bool, 0600); 71 72 /* 73 * Filter [non-]young pages accordingly for LRU [de]prioritizations. 74 * 75 * If this is set, check page level access (youngness) once again before each 76 * LRU [de]prioritization operation. LRU prioritization operation is skipped 77 * if the page has not accessed since the last check (not young). LRU 78 * deprioritization operation is skipped if the page has accessed since the 79 * last check (young). The feature is enabled or disabled if this parameter is 80 * set as ``Y`` or ``N``, respectively. 81 * 82 * Disabled by default. 83 */ 84 static bool filter_young_pages __read_mostly; 85 module_param(filter_young_pages, bool, 0600); 86 87 /* 88 * Access frequency threshold for hot memory regions identification in permil. 89 * 90 * If a memory region is accessed in frequency of this or higher, 91 * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the 92 * LRU list, so that it could not be reclaimed under memory pressure. 50% by 93 * default. 94 */ 95 static unsigned long hot_thres_access_freq = 500; 96 module_param(hot_thres_access_freq, ulong, 0600); 97 98 /* 99 * Time threshold for cold memory regions identification in microseconds. 100 * 101 * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT 102 * identifies the region as cold, and mark it as unaccessed on the LRU list, so 103 * that it could be reclaimed first under memory pressure. 120 seconds by 104 * default. 105 */ 106 static unsigned long cold_min_age __read_mostly = 120000000; 107 module_param(cold_min_age, ulong, 0600); 108 109 static struct damos_quota damon_lru_sort_quota = { 110 /* Use up to 10 ms per 1 sec, by default */ 111 .ms = 10, 112 .sz = 0, 113 .reset_interval = 1000, 114 /* Within the quota, mark hotter regions accessed first. */ 115 .weight_sz = 0, 116 .weight_nr_accesses = 1, 117 .weight_age = 1, 118 }; 119 DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); 120 121 static struct damos_watermarks damon_lru_sort_wmarks = { 122 .metric = DAMOS_WMARK_FREE_MEM_RATE, 123 .interval = 5000000, /* 5 seconds */ 124 .high = 200, /* 20 percent */ 125 .mid = 150, /* 15 percent */ 126 .low = 50, /* 5 percent */ 127 }; 128 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); 129 130 static struct damon_attrs damon_lru_sort_mon_attrs = { 131 .sample_interval = 5000, /* 5 ms */ 132 .aggr_interval = 100000, /* 100 ms */ 133 .ops_update_interval = 0, 134 .min_nr_regions = 10, 135 .max_nr_regions = 1000, 136 }; 137 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); 138 139 /* 140 * Start of the target memory region in physical address. 141 * 142 * The start physical address of memory region that DAMON_LRU_SORT will do work 143 * against. By default, biggest System RAM is used as the region. 144 */ 145 static unsigned long monitor_region_start __read_mostly; 146 module_param(monitor_region_start, ulong, 0600); 147 148 /* 149 * End of the target memory region in physical address. 150 * 151 * The end physical address of memory region that DAMON_LRU_SORT will do work 152 * against. By default, biggest System RAM is used as the region. 153 */ 154 static unsigned long monitor_region_end __read_mostly; 155 module_param(monitor_region_end, ulong, 0600); 156 157 /* 158 * Scale factor for DAMON_LRU_SORT to ops address conversion. 159 * 160 * This parameter must not be set to 0. 161 */ 162 static unsigned long addr_unit __read_mostly = 1; 163 164 static struct damos_stat damon_lru_sort_hot_stat; 165 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, 166 lru_sort_tried_hot_regions, lru_sorted_hot_regions, 167 hot_quota_exceeds); 168 169 static struct damos_stat damon_lru_sort_cold_stat; 170 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, 171 lru_sort_tried_cold_regions, lru_sorted_cold_regions, 172 cold_quota_exceeds); 173 174 static struct damos_access_pattern damon_lru_sort_stub_pattern = { 175 /* Find regions having PAGE_SIZE or larger size */ 176 .min_sz_region = PAGE_SIZE, 177 .max_sz_region = ULONG_MAX, 178 /* no matter its access frequency */ 179 .min_nr_accesses = 0, 180 .max_nr_accesses = UINT_MAX, 181 /* no matter its age */ 182 .min_age_region = 0, 183 .max_age_region = UINT_MAX, 184 }; 185 186 static struct damon_ctx *ctx; 187 static struct damon_target *target; 188 189 static struct damos *damon_lru_sort_new_scheme( 190 struct damos_access_pattern *pattern, enum damos_action action) 191 { 192 struct damos_quota quota = damon_lru_sort_quota; 193 194 /* Use half of total quota for hot/cold pages sorting */ 195 quota.ms = quota.ms / 2; 196 197 return damon_new_scheme( 198 /* find the pattern, and */ 199 pattern, 200 /* (de)prioritize on LRU-lists */ 201 action, 202 /* for each aggregation interval */ 203 0, 204 /* under the quota. */ 205 "a, 206 /* (De)activate this according to the watermarks. */ 207 &damon_lru_sort_wmarks, 208 NUMA_NO_NODE); 209 } 210 211 /* Create a DAMON-based operation scheme for hot memory regions */ 212 static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) 213 { 214 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 215 216 pattern.min_nr_accesses = hot_thres; 217 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO); 218 } 219 220 /* Create a DAMON-based operation scheme for cold memory regions */ 221 static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) 222 { 223 struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; 224 225 pattern.max_nr_accesses = 0; 226 pattern.min_age_region = cold_thres; 227 return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO); 228 } 229 230 static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme, 231 struct damos *cold_scheme) 232 { 233 struct damos_quota_goal *goal; 234 235 if (!active_mem_bp) 236 return 0; 237 goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp); 238 if (!goal) 239 return -ENOMEM; 240 damos_add_quota_goal(&hot_scheme->quota, goal); 241 /* aim 0.2 % goal conflict, to keep little ping pong */ 242 goal = damos_new_quota_goal(DAMOS_QUOTA_INACTIVE_MEM_BP, 243 10000 - active_mem_bp + 2); 244 if (!goal) 245 return -ENOMEM; 246 damos_add_quota_goal(&cold_scheme->quota, goal); 247 return 0; 248 } 249 250 static int damon_lru_sort_add_filters(struct damos *hot_scheme, 251 struct damos *cold_scheme) 252 { 253 struct damos_filter *filter; 254 255 if (!filter_young_pages) 256 return 0; 257 258 /* disallow prioritizing not-young pages */ 259 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, false, false); 260 if (!filter) 261 return -ENOMEM; 262 damos_add_filter(hot_scheme, filter); 263 264 /* disabllow de-prioritizing young pages */ 265 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false); 266 if (!filter) 267 return -ENOMEM; 268 damos_add_filter(cold_scheme, filter); 269 return 0; 270 } 271 272 static int damon_lru_sort_apply_parameters(void) 273 { 274 struct damon_ctx *param_ctx; 275 struct damon_target *param_target; 276 struct damon_attrs attrs; 277 struct damos *hot_scheme, *cold_scheme; 278 unsigned int hot_thres, cold_thres; 279 int err; 280 281 err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); 282 if (err) 283 return err; 284 285 param_ctx->addr_unit = addr_unit; 286 param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); 287 288 if (!damon_lru_sort_mon_attrs.sample_interval) { 289 err = -EINVAL; 290 goto out; 291 } 292 293 attrs = damon_lru_sort_mon_attrs; 294 if (autotune_monitoring_intervals) { 295 attrs.sample_interval = 5000; 296 attrs.aggr_interval = 100000; 297 attrs.intervals_goal.access_bp = 40; 298 attrs.intervals_goal.aggrs = 3; 299 attrs.intervals_goal.min_sample_us = 5000; 300 attrs.intervals_goal.max_sample_us = 10 * 1000 * 1000; 301 } 302 err = damon_set_attrs(param_ctx, &attrs); 303 if (err) 304 goto out; 305 306 err = -ENOMEM; 307 hot_thres = damon_max_nr_accesses(&attrs) * 308 hot_thres_access_freq / 1000; 309 hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres); 310 if (!hot_scheme) 311 goto out; 312 313 cold_thres = cold_min_age / attrs.aggr_interval; 314 cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres); 315 if (!cold_scheme) { 316 damon_destroy_scheme(hot_scheme); 317 goto out; 318 } 319 320 damon_set_schemes(param_ctx, &hot_scheme, 1); 321 damon_add_scheme(param_ctx, cold_scheme); 322 323 err = damon_lru_sort_add_quota_goals(hot_scheme, cold_scheme); 324 if (err) 325 goto out; 326 err = damon_lru_sort_add_filters(hot_scheme, cold_scheme); 327 if (err) 328 goto out; 329 330 err = damon_set_region_biggest_system_ram_default(param_target, 331 &monitor_region_start, 332 &monitor_region_end, 333 param_ctx->addr_unit, 334 param_ctx->min_region_sz); 335 if (err) 336 goto out; 337 err = damon_commit_ctx(ctx, param_ctx); 338 out: 339 damon_destroy_ctx(param_ctx); 340 return err; 341 } 342 343 static int damon_lru_sort_handle_commit_inputs(void) 344 { 345 int err; 346 347 if (!commit_inputs) 348 return 0; 349 350 err = damon_lru_sort_apply_parameters(); 351 commit_inputs = false; 352 return err; 353 } 354 355 static int damon_lru_sort_damon_call_fn(void *arg) 356 { 357 struct damon_ctx *c = arg; 358 struct damos *s; 359 360 /* update the stats parameter */ 361 damon_for_each_scheme(s, c) { 362 if (s->action == DAMOS_LRU_PRIO) 363 damon_lru_sort_hot_stat = s->stat; 364 else if (s->action == DAMOS_LRU_DEPRIO) 365 damon_lru_sort_cold_stat = s->stat; 366 } 367 368 return damon_lru_sort_handle_commit_inputs(); 369 } 370 371 static struct damon_call_control call_control = { 372 .fn = damon_lru_sort_damon_call_fn, 373 .repeat = true, 374 }; 375 376 static int damon_lru_sort_turn(bool on) 377 { 378 int err; 379 380 if (!on) 381 return damon_stop(&ctx, 1); 382 383 err = damon_lru_sort_apply_parameters(); 384 if (err) 385 return err; 386 387 err = damon_start(&ctx, 1, true); 388 if (err) 389 return err; 390 return damon_call(ctx, &call_control); 391 } 392 393 static int damon_lru_sort_addr_unit_store(const char *val, 394 const struct kernel_param *kp) 395 { 396 unsigned long input_addr_unit; 397 int err = kstrtoul(val, 0, &input_addr_unit); 398 399 if (err) 400 return err; 401 if (!input_addr_unit) 402 return -EINVAL; 403 404 addr_unit = input_addr_unit; 405 return 0; 406 } 407 408 static const struct kernel_param_ops addr_unit_param_ops = { 409 .set = damon_lru_sort_addr_unit_store, 410 .get = param_get_ulong, 411 }; 412 413 module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); 414 MODULE_PARM_DESC(addr_unit, 415 "Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)"); 416 417 static bool damon_lru_sort_enabled(void) 418 { 419 if (!ctx) 420 return false; 421 return damon_is_running(ctx); 422 } 423 424 static int damon_lru_sort_enabled_store(const char *val, 425 const struct kernel_param *kp) 426 { 427 int err; 428 429 err = kstrtobool(val, &enabled); 430 if (err) 431 return err; 432 433 if (damon_lru_sort_enabled() == enabled) 434 return 0; 435 436 /* Called before init function. The function will handle this. */ 437 if (!damon_initialized()) 438 return 0; 439 440 return damon_lru_sort_turn(enabled); 441 } 442 443 static int damon_lru_sort_enabled_load(char *buffer, 444 const struct kernel_param *kp) 445 { 446 return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N'); 447 } 448 449 static const struct kernel_param_ops enabled_param_ops = { 450 .set = damon_lru_sort_enabled_store, 451 .get = damon_lru_sort_enabled_load, 452 }; 453 454 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 455 MODULE_PARM_DESC(enabled, 456 "Enable or disable DAMON_LRU_SORT (default: disabled)"); 457 458 static int damon_lru_sort_kdamond_pid_store(const char *val, 459 const struct kernel_param *kp) 460 { 461 /* 462 * kdamond_pid is read-only, but kernel command line could write it. 463 * Do nothing here. 464 */ 465 return 0; 466 } 467 468 static int damon_lru_sort_kdamond_pid_load(char *buffer, 469 const struct kernel_param *kp) 470 { 471 int kdamond_pid = -1; 472 473 if (ctx) { 474 kdamond_pid = damon_kdamond_pid(ctx); 475 if (kdamond_pid < 0) 476 kdamond_pid = -1; 477 } 478 return sprintf(buffer, "%d\n", kdamond_pid); 479 } 480 481 static const struct kernel_param_ops kdamond_pid_param_ops = { 482 .set = damon_lru_sort_kdamond_pid_store, 483 .get = damon_lru_sort_kdamond_pid_load, 484 }; 485 486 /* 487 * PID of the DAMON thread 488 * 489 * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. 490 * Else, -1. 491 */ 492 module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); 493 494 static int __init damon_lru_sort_init(void) 495 { 496 int err; 497 498 if (!damon_initialized()) { 499 err = -ENOMEM; 500 goto out; 501 } 502 err = damon_modules_new_paddr_ctx_target(&ctx, &target); 503 if (err) 504 goto out; 505 506 call_control.data = ctx; 507 508 /* 'enabled' has set before this function, probably via command line */ 509 if (enabled) 510 err = damon_lru_sort_turn(true); 511 512 out: 513 if (err && enabled) 514 enabled = false; 515 return err; 516 } 517 518 module_init(damon_lru_sort_init); 519