1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based page reclamation 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-reclaim: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/kstrtox.h> 12 #include <linux/module.h> 13 14 #include "modules-common.h" 15 16 #ifdef MODULE_PARAM_PREFIX 17 #undef MODULE_PARAM_PREFIX 18 #endif 19 #define MODULE_PARAM_PREFIX "damon_reclaim." 20 21 /* 22 * Enable or disable DAMON_RECLAIM. 23 * 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. 25 * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could 26 * do no real monitoring and reclamation due to the watermarks-based activation 27 * condition. Refer to below descriptions for the watermarks parameter for 28 * this. 29 */ 30 static bool enabled __read_mostly; 31 32 /* 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. 34 * 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied 36 * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values 37 * of parameters except ``enabled`` again. Once the re-reading is done, this 38 * parameter is set as ``N``. If invalid parameters are found while the 39 * re-reading, DAMON_RECLAIM will be disabled. 40 */ 41 static bool commit_inputs __read_mostly; 42 module_param(commit_inputs, bool, 0600); 43 44 /* 45 * Time threshold for cold memory regions identification in microseconds. 46 * 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM 48 * identifies the region as cold, and reclaims. 120 seconds by default. 49 */ 50 static unsigned long min_age __read_mostly = 120000000; 51 module_param(min_age, ulong, 0600); 52 53 static struct damos_quota damon_reclaim_quota = { 54 /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ 55 .ms = 10, 56 .sz = 128 * 1024 * 1024, 57 .reset_interval = 1000, 58 /* Within the quota, page out older regions first. */ 59 .weight_sz = 0, 60 .weight_nr_accesses = 0, 61 .weight_age = 1 62 }; 63 DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); 64 65 /* 66 * Desired level of memory pressure-stall time in microseconds. 67 * 68 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically 69 * increases and decreases the effective level of the quota aiming this level of 70 * memory pressure is incurred. System-wide ``some`` memory PSI in microseconds 71 * per quota reset interval (``quota_reset_interval_ms``) is collected and 72 * compared to this value to see if the aim is satisfied. Value zero means 73 * disabling this auto-tuning feature. 74 * 75 * Disabled by default. 76 */ 77 static unsigned long quota_mem_pressure_us __read_mostly; 78 module_param(quota_mem_pressure_us, ulong, 0600); 79 80 /* 81 * User-specifiable feedback for auto-tuning of the effective quota. 82 * 83 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically 84 * increases and decreases the effective level of the quota aiming receiving this 85 * feedback of value ``10,000`` from the user. DAMON_RECLAIM assumes the feedback 86 * value and the quota are positively proportional. Value zero means disabling 87 * this auto-tuning feature. 88 * 89 * Disabled by default. 90 * 91 */ 92 static unsigned long quota_autotune_feedback __read_mostly; 93 module_param(quota_autotune_feedback, ulong, 0600); 94 95 static struct damos_watermarks damon_reclaim_wmarks = { 96 .metric = DAMOS_WMARK_FREE_MEM_RATE, 97 .interval = 5000000, /* 5 seconds */ 98 .high = 500, /* 50 percent */ 99 .mid = 400, /* 40 percent */ 100 .low = 200, /* 20 percent */ 101 }; 102 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); 103 104 static struct damon_attrs damon_reclaim_mon_attrs = { 105 .sample_interval = 5000, /* 5 ms */ 106 .aggr_interval = 100000, /* 100 ms */ 107 .ops_update_interval = 0, 108 .min_nr_regions = 10, 109 .max_nr_regions = 1000, 110 }; 111 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); 112 113 /* 114 * Start of the target memory region in physical address. 115 * 116 * The start physical address of memory region that DAMON_RECLAIM will do work 117 * against. By default, biggest System RAM is used as the region. 118 */ 119 static unsigned long monitor_region_start __read_mostly; 120 module_param(monitor_region_start, ulong, 0600); 121 122 /* 123 * End of the target memory region in physical address. 124 * 125 * The end physical address of memory region that DAMON_RECLAIM will do work 126 * against. By default, biggest System RAM is used as the region. 127 */ 128 static unsigned long monitor_region_end __read_mostly; 129 module_param(monitor_region_end, ulong, 0600); 130 131 /* 132 * Scale factor for DAMON_RECLAIM to ops address conversion. 133 * 134 * This parameter must not be set to 0. 135 */ 136 static unsigned long addr_unit __read_mostly = 1; 137 138 /* 139 * Skip anonymous pages reclamation. 140 * 141 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous 142 * pages. By default, ``N``. 143 */ 144 static bool skip_anon __read_mostly; 145 module_param(skip_anon, bool, 0600); 146 147 static struct damos_stat damon_reclaim_stat; 148 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, 149 reclaim_tried_regions, reclaimed_regions, quota_exceeds); 150 151 static struct damon_ctx *ctx; 152 static struct damon_target *target; 153 154 static struct damos *damon_reclaim_new_scheme(void) 155 { 156 struct damos_access_pattern pattern = { 157 /* Find regions having PAGE_SIZE or larger size */ 158 .min_sz_region = PAGE_SIZE, 159 .max_sz_region = ULONG_MAX, 160 /* and not accessed at all */ 161 .min_nr_accesses = 0, 162 .max_nr_accesses = 0, 163 /* for min_age or more micro-seconds */ 164 .min_age_region = min_age / 165 damon_reclaim_mon_attrs.aggr_interval, 166 .max_age_region = UINT_MAX, 167 }; 168 169 return damon_new_scheme( 170 &pattern, 171 /* page out those, as soon as found */ 172 DAMOS_PAGEOUT, 173 /* for each aggregation interval */ 174 0, 175 /* under the quota. */ 176 &damon_reclaim_quota, 177 /* (De)activate this according to the watermarks. */ 178 &damon_reclaim_wmarks, 179 NUMA_NO_NODE); 180 } 181 182 static int damon_reclaim_apply_parameters(void) 183 { 184 struct damon_ctx *param_ctx; 185 struct damon_target *param_target; 186 struct damos *scheme; 187 struct damos_quota_goal *goal; 188 struct damos_filter *filter; 189 int err; 190 191 err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); 192 if (err) 193 return err; 194 195 param_ctx->addr_unit = addr_unit; 196 param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1); 197 198 if (!damon_reclaim_mon_attrs.aggr_interval) { 199 err = -EINVAL; 200 goto out; 201 } 202 203 err = damon_set_attrs(param_ctx, &damon_reclaim_mon_attrs); 204 if (err) 205 goto out; 206 207 err = -ENOMEM; 208 scheme = damon_reclaim_new_scheme(); 209 if (!scheme) 210 goto out; 211 damon_set_schemes(param_ctx, &scheme, 1); 212 213 if (quota_mem_pressure_us) { 214 goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US, 215 quota_mem_pressure_us); 216 if (!goal) 217 goto out; 218 damos_add_quota_goal(&scheme->quota, goal); 219 } 220 221 if (quota_autotune_feedback) { 222 goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000); 223 if (!goal) 224 goto out; 225 goal->current_value = quota_autotune_feedback; 226 damos_add_quota_goal(&scheme->quota, goal); 227 } 228 229 if (skip_anon) { 230 filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true, false); 231 if (!filter) 232 goto out; 233 damos_add_filter(scheme, filter); 234 } 235 236 err = damon_set_region_biggest_system_ram_default(param_target, 237 &monitor_region_start, 238 &monitor_region_end, 239 param_ctx->addr_unit, 240 param_ctx->min_region_sz); 241 if (err) 242 goto out; 243 err = damon_commit_ctx(ctx, param_ctx); 244 out: 245 damon_destroy_ctx(param_ctx); 246 return err; 247 } 248 249 static int damon_reclaim_handle_commit_inputs(void) 250 { 251 int err; 252 253 if (!commit_inputs) 254 return 0; 255 256 err = damon_reclaim_apply_parameters(); 257 commit_inputs = false; 258 return err; 259 } 260 261 static int damon_reclaim_damon_call_fn(void *arg) 262 { 263 struct damon_ctx *c = arg; 264 struct damos *s; 265 266 /* update the stats parameter */ 267 damon_for_each_scheme(s, c) 268 damon_reclaim_stat = s->stat; 269 270 return damon_reclaim_handle_commit_inputs(); 271 } 272 273 static struct damon_call_control call_control = { 274 .fn = damon_reclaim_damon_call_fn, 275 .repeat = true, 276 }; 277 278 static int damon_reclaim_turn(bool on) 279 { 280 int err; 281 282 if (!on) 283 return damon_stop(&ctx, 1); 284 285 err = damon_reclaim_apply_parameters(); 286 if (err) 287 return err; 288 289 err = damon_start(&ctx, 1, true); 290 if (err) 291 return err; 292 return damon_call(ctx, &call_control); 293 } 294 295 static int damon_reclaim_addr_unit_store(const char *val, 296 const struct kernel_param *kp) 297 { 298 unsigned long input_addr_unit; 299 int err = kstrtoul(val, 0, &input_addr_unit); 300 301 if (err) 302 return err; 303 if (!input_addr_unit) 304 return -EINVAL; 305 306 addr_unit = input_addr_unit; 307 return 0; 308 } 309 310 static const struct kernel_param_ops addr_unit_param_ops = { 311 .set = damon_reclaim_addr_unit_store, 312 .get = param_get_ulong, 313 }; 314 315 module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); 316 MODULE_PARM_DESC(addr_unit, 317 "Scale factor for DAMON_RECLAIM to ops address conversion (default: 1)"); 318 319 static bool damon_reclaim_enabled(void) 320 { 321 if (!ctx) 322 return false; 323 return damon_is_running(ctx); 324 } 325 326 static int damon_reclaim_enabled_store(const char *val, 327 const struct kernel_param *kp) 328 { 329 int err; 330 331 err = kstrtobool(val, &enabled); 332 if (err) 333 return err; 334 335 if (damon_reclaim_enabled() == enabled) 336 return 0; 337 338 /* Called before init function. The function will handle this. */ 339 if (!damon_initialized()) 340 return 0; 341 342 return damon_reclaim_turn(enabled); 343 } 344 345 static int damon_reclaim_enabled_load(char *buffer, 346 const struct kernel_param *kp) 347 { 348 return sprintf(buffer, "%c\n", damon_reclaim_enabled() ? 'Y' : 'N'); 349 } 350 351 static const struct kernel_param_ops enabled_param_ops = { 352 .set = damon_reclaim_enabled_store, 353 .get = damon_reclaim_enabled_load, 354 }; 355 356 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 357 MODULE_PARM_DESC(enabled, 358 "Enable or disable DAMON_RECLAIM (default: disabled)"); 359 360 static int damon_reclaim_kdamond_pid_store(const char *val, 361 const struct kernel_param *kp) 362 { 363 /* 364 * kdamond_pid is read-only, but kernel command line could write it. 365 * Do nothing here. 366 */ 367 return 0; 368 } 369 370 static int damon_reclaim_kdamond_pid_load(char *buffer, 371 const struct kernel_param *kp) 372 { 373 int kdamond_pid = -1; 374 375 if (ctx) { 376 kdamond_pid = damon_kdamond_pid(ctx); 377 if (kdamond_pid < 0) 378 kdamond_pid = -1; 379 } 380 return sprintf(buffer, "%d\n", kdamond_pid); 381 } 382 383 static const struct kernel_param_ops kdamond_pid_param_ops = { 384 .set = damon_reclaim_kdamond_pid_store, 385 .get = damon_reclaim_kdamond_pid_load, 386 }; 387 388 /* 389 * PID of the DAMON thread 390 * 391 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. 392 * Else, -1. 393 */ 394 module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); 395 396 static int __init damon_reclaim_init(void) 397 { 398 int err; 399 400 if (!damon_initialized()) { 401 err = -ENOMEM; 402 goto out; 403 } 404 err = damon_modules_new_paddr_ctx_target(&ctx, &target); 405 if (err) 406 goto out; 407 408 call_control.data = ctx; 409 410 /* 'enabled' has set before this function, probably via command line */ 411 if (enabled) 412 err = damon_reclaim_turn(true); 413 414 out: 415 if (err && enabled) 416 enabled = false; 417 return err; 418 } 419 420 module_init(damon_reclaim_init); 421