1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * memory tiering: migrate cold pages in node 0 and hot pages in node 1 to node 4 * 1 and node 0, respectively. Adjust the hotness/coldness threshold aiming 5 * resulting 99.6 % node 0 utilization ratio. 6 */ 7 8 #define pr_fmt(fmt) "damon_sample_mtier: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 15 #ifdef MODULE_PARAM_PREFIX 16 #undef MODULE_PARAM_PREFIX 17 #endif 18 #define MODULE_PARAM_PREFIX "damon_sample_mtier." 19 20 static unsigned long node0_start_addr __read_mostly; 21 module_param(node0_start_addr, ulong, 0600); 22 23 static unsigned long node0_end_addr __read_mostly; 24 module_param(node0_end_addr, ulong, 0600); 25 26 static unsigned long node1_start_addr __read_mostly; 27 module_param(node1_start_addr, ulong, 0600); 28 29 static unsigned long node1_end_addr __read_mostly; 30 module_param(node1_end_addr, ulong, 0600); 31 32 static unsigned long node0_mem_used_bp __read_mostly = 9970; 33 module_param(node0_mem_used_bp, ulong, 0600); 34 35 static unsigned long node0_mem_free_bp __read_mostly = 50; 36 module_param(node0_mem_free_bp, ulong, 0600); 37 38 static int damon_sample_mtier_enable_store( 39 const char *val, const struct kernel_param *kp); 40 41 static const struct kernel_param_ops enabled_param_ops = { 42 .set = damon_sample_mtier_enable_store, 43 .get = param_get_bool, 44 }; 45 46 static bool enabled __read_mostly; 47 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 48 MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_MTIER"); 49 50 static bool detect_node_addresses __read_mostly; 51 module_param(detect_node_addresses, bool, 0600); 52 53 static struct damon_ctx *ctxs[2]; 54 55 struct region_range { 56 phys_addr_t start; 57 phys_addr_t end; 58 }; 59 60 static int nid_to_phys(int target_node, struct region_range *range) 61 { 62 if (!node_online(target_node)) { 63 pr_err("NUMA node %d is not online\n", target_node); 64 return -EINVAL; 65 } 66 67 range->start = PFN_PHYS(node_start_pfn(target_node)); 68 range->end = PFN_PHYS(node_end_pfn(target_node)); 69 70 return 0; 71 } 72 73 static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) 74 { 75 struct damon_ctx *ctx; 76 struct damon_attrs attrs; 77 struct damon_target *target; 78 struct damos *scheme; 79 struct damos_quota_goal *quota_goal; 80 struct damos_filter *filter; 81 struct region_range addr; 82 struct damon_addr_range range; 83 int ret; 84 85 ctx = damon_new_ctx(); 86 if (!ctx) 87 return NULL; 88 attrs = (struct damon_attrs) { 89 .sample_interval = 5 * USEC_PER_MSEC, 90 .aggr_interval = 100 * USEC_PER_MSEC, 91 .ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC, 92 .min_nr_regions = 10, 93 .max_nr_regions = 1000, 94 }; 95 96 /* 97 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed 98 * accesses ratio, keeping sampling interval in [5ms, 10s] range. 99 */ 100 attrs.intervals_goal = (struct damon_intervals_goal) { 101 .access_bp = 400, .aggrs = 3, 102 .min_sample_us = 5000, .max_sample_us = 10000000, 103 }; 104 if (damon_set_attrs(ctx, &attrs)) 105 goto free_out; 106 if (damon_select_ops(ctx, DAMON_OPS_PADDR)) 107 goto free_out; 108 109 target = damon_new_target(); 110 if (!target) 111 goto free_out; 112 damon_add_target(ctx, target); 113 114 if (detect_node_addresses) { 115 ret = promote ? nid_to_phys(1, &addr) : nid_to_phys(0, &addr); 116 if (ret) 117 goto free_out; 118 } else { 119 addr.start = promote ? node1_start_addr : node0_start_addr; 120 addr.end = promote ? node1_end_addr : node0_end_addr; 121 } 122 123 if (addr.start >= addr.end) 124 goto free_out; 125 126 range.start = addr.start; 127 range.end = addr.end; 128 129 ret = damon_set_regions(target, &range, 1, DAMON_MIN_REGION_SZ); 130 if (ret) 131 goto free_out; 132 133 scheme = damon_new_scheme( 134 /* access pattern */ 135 &(struct damos_access_pattern) { 136 .min_sz_region = PAGE_SIZE, 137 .max_sz_region = ULONG_MAX, 138 .min_nr_accesses = promote ? 1 : 0, 139 .max_nr_accesses = promote ? UINT_MAX : 0, 140 .min_age_region = 0, 141 .max_age_region = UINT_MAX}, 142 /* action */ 143 promote ? DAMOS_MIGRATE_HOT : DAMOS_MIGRATE_COLD, 144 1000000, /* apply interval (1s) */ 145 &(struct damos_quota){ 146 /* 200 MiB per sec by most */ 147 .reset_interval = 1000, 148 .sz = 200 * 1024 * 1024, 149 /* ignore size of region when prioritizing */ 150 .weight_sz = 0, 151 .weight_nr_accesses = 100, 152 .weight_age = 100, 153 }, 154 &(struct damos_watermarks){}, 155 promote ? 0 : 1); /* migrate target node id */ 156 if (!scheme) 157 goto free_out; 158 damon_set_schemes(ctx, &scheme, 1); 159 quota_goal = damos_new_quota_goal( 160 promote ? DAMOS_QUOTA_NODE_MEM_USED_BP : 161 DAMOS_QUOTA_NODE_MEM_FREE_BP, 162 promote ? node0_mem_used_bp : node0_mem_free_bp); 163 if (!quota_goal) 164 goto free_out; 165 quota_goal->nid = 0; 166 damos_add_quota_goal(&scheme->quota, quota_goal); 167 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, promote); 168 if (!filter) 169 goto free_out; 170 damos_add_filter(scheme, filter); 171 return ctx; 172 free_out: 173 damon_destroy_ctx(ctx); 174 return NULL; 175 } 176 177 static int damon_sample_mtier_start(void) 178 { 179 struct damon_ctx *ctx; 180 181 ctx = damon_sample_mtier_build_ctx(true); 182 if (!ctx) 183 return -ENOMEM; 184 ctxs[0] = ctx; 185 ctx = damon_sample_mtier_build_ctx(false); 186 if (!ctx) { 187 damon_destroy_ctx(ctxs[0]); 188 return -ENOMEM; 189 } 190 ctxs[1] = ctx; 191 return damon_start(ctxs, 2, true); 192 } 193 194 static void damon_sample_mtier_stop(void) 195 { 196 damon_stop(ctxs, 2); 197 damon_destroy_ctx(ctxs[0]); 198 damon_destroy_ctx(ctxs[1]); 199 } 200 201 static int damon_sample_mtier_enable_store( 202 const char *val, const struct kernel_param *kp) 203 { 204 bool is_enabled = enabled; 205 int err; 206 207 err = kstrtobool(val, &enabled); 208 if (err) 209 return err; 210 211 if (enabled == is_enabled) 212 return 0; 213 214 if (!damon_initialized()) 215 return 0; 216 217 if (enabled) { 218 err = damon_sample_mtier_start(); 219 if (err) 220 enabled = false; 221 return err; 222 } 223 damon_sample_mtier_stop(); 224 return 0; 225 } 226 227 static int __init damon_sample_mtier_init(void) 228 { 229 int err = 0; 230 231 if (!damon_initialized()) { 232 if (enabled) 233 enabled = false; 234 return -ENOMEM; 235 } 236 237 if (enabled) { 238 err = damon_sample_mtier_start(); 239 if (err) 240 enabled = false; 241 } 242 return 0; 243 } 244 245 module_init(damon_sample_mtier_init); 246