1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shows data access monitoring results in simple metrics. 4 */ 5 6 #define pr_fmt(fmt) "damon-stat: " fmt 7 8 #include <linux/damon.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/sort.h> 13 14 #ifdef MODULE_PARAM_PREFIX 15 #undef MODULE_PARAM_PREFIX 16 #endif 17 #define MODULE_PARAM_PREFIX "damon_stat." 18 19 static int damon_stat_enabled_store( 20 const char *val, const struct kernel_param *kp); 21 22 static const struct kernel_param_ops enabled_param_ops = { 23 .set = damon_stat_enabled_store, 24 .get = param_get_bool, 25 }; 26 27 static bool enabled __read_mostly = IS_ENABLED( 28 CONFIG_DAMON_STAT_ENABLED_DEFAULT); 29 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 30 MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT"); 31 32 static unsigned long estimated_memory_bandwidth __read_mostly; 33 module_param(estimated_memory_bandwidth, ulong, 0400); 34 MODULE_PARM_DESC(estimated_memory_bandwidth, 35 "Estimated memory bandwidth usage in bytes per second"); 36 37 static long memory_idle_ms_percentiles[101] = {0,}; 38 module_param_array(memory_idle_ms_percentiles, long, NULL, 0400); 39 MODULE_PARM_DESC(memory_idle_ms_percentiles, 40 "Memory idle time percentiles in milliseconds"); 41 42 static unsigned long aggr_interval_us; 43 module_param(aggr_interval_us, ulong, 0400); 44 MODULE_PARM_DESC(aggr_interval_us, 45 "Current tuned aggregation interval in microseconds"); 46 47 static struct damon_ctx *damon_stat_context; 48 49 static unsigned long damon_stat_last_refresh_jiffies; 50 51 static void damon_stat_set_estimated_memory_bandwidth(struct damon_ctx *c) 52 { 53 struct damon_target *t; 54 struct damon_region *r; 55 unsigned long access_bytes = 0; 56 57 damon_for_each_target(t, c) { 58 damon_for_each_region(r, t) 59 access_bytes += (r->ar.end - r->ar.start) * 60 r->nr_accesses; 61 } 62 estimated_memory_bandwidth = access_bytes * USEC_PER_MSEC * 63 MSEC_PER_SEC / c->attrs.aggr_interval; 64 } 65 66 static int damon_stat_idletime(const struct damon_region *r) 67 { 68 if (r->nr_accesses) 69 return -1 * (r->age + 1); 70 return r->age + 1; 71 } 72 73 static int damon_stat_cmp_regions(const void *a, const void *b) 74 { 75 const struct damon_region *ra = *(const struct damon_region **)a; 76 const struct damon_region *rb = *(const struct damon_region **)b; 77 78 return damon_stat_idletime(ra) - damon_stat_idletime(rb); 79 } 80 81 static int damon_stat_sort_regions(struct damon_ctx *c, 82 struct damon_region ***sorted_ptr, int *nr_regions_ptr, 83 unsigned long *total_sz_ptr) 84 { 85 struct damon_target *t; 86 struct damon_region *r; 87 struct damon_region **region_pointers; 88 unsigned int nr_regions = 0; 89 unsigned long total_sz = 0; 90 91 damon_for_each_target(t, c) { 92 /* there is only one target */ 93 region_pointers = kmalloc_objs(*region_pointers, 94 damon_nr_regions(t)); 95 if (!region_pointers) 96 return -ENOMEM; 97 damon_for_each_region(r, t) { 98 region_pointers[nr_regions++] = r; 99 total_sz += r->ar.end - r->ar.start; 100 } 101 } 102 sort(region_pointers, nr_regions, sizeof(*region_pointers), 103 damon_stat_cmp_regions, NULL); 104 *sorted_ptr = region_pointers; 105 *nr_regions_ptr = nr_regions; 106 *total_sz_ptr = total_sz; 107 return 0; 108 } 109 110 static void damon_stat_set_idletime_percentiles(struct damon_ctx *c) 111 { 112 struct damon_region **sorted_regions, *region; 113 int nr_regions; 114 unsigned long total_sz, accounted_bytes = 0; 115 int err, i, next_percentile = 0; 116 117 err = damon_stat_sort_regions(c, &sorted_regions, &nr_regions, 118 &total_sz); 119 if (err) 120 return; 121 for (i = 0; i < nr_regions; i++) { 122 region = sorted_regions[i]; 123 accounted_bytes += region->ar.end - region->ar.start; 124 while (next_percentile <= accounted_bytes * 100 / total_sz) 125 memory_idle_ms_percentiles[next_percentile++] = 126 damon_stat_idletime(region) * 127 (long)c->attrs.aggr_interval / USEC_PER_MSEC; 128 } 129 kfree(sorted_regions); 130 } 131 132 static int damon_stat_damon_call_fn(void *data) 133 { 134 struct damon_ctx *c = data; 135 136 /* avoid unnecessarily frequent stat update */ 137 if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + 138 msecs_to_jiffies(5 * MSEC_PER_SEC))) 139 return 0; 140 damon_stat_last_refresh_jiffies = jiffies; 141 142 aggr_interval_us = c->attrs.aggr_interval; 143 damon_stat_set_estimated_memory_bandwidth(c); 144 damon_stat_set_idletime_percentiles(c); 145 return 0; 146 } 147 148 struct damon_stat_system_ram_range_walk_arg { 149 bool walked; 150 struct resource res; 151 }; 152 153 static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg) 154 { 155 struct damon_stat_system_ram_range_walk_arg *a = arg; 156 157 if (!a->walked) { 158 a->walked = true; 159 a->res.start = res->start; 160 } 161 a->res.end = res->end; 162 return 0; 163 } 164 165 static unsigned long damon_stat_res_to_core_addr(resource_size_t ra, 166 unsigned long addr_unit) 167 { 168 /* 169 * Use div_u64() for avoiding linking errors related with __udivdi3, 170 * __aeabi_uldivmod, or similar problems. This should also improve the 171 * performance optimization (read div_u64() comment for the detail). 172 */ 173 if (sizeof(ra) == 8 && sizeof(addr_unit) == 4) 174 return div_u64(ra, addr_unit); 175 return ra / addr_unit; 176 } 177 178 static int damon_stat_set_monitoring_region(struct damon_target *t, 179 unsigned long addr_unit, unsigned long min_region_sz) 180 { 181 struct damon_addr_range addr_range; 182 struct damon_stat_system_ram_range_walk_arg arg = {}; 183 184 walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn); 185 if (!arg.walked) 186 return -EINVAL; 187 addr_range.start = damon_stat_res_to_core_addr( 188 arg.res.start, addr_unit); 189 addr_range.end = damon_stat_res_to_core_addr( 190 arg.res.end + 1, addr_unit); 191 if (addr_range.end <= addr_range.start) 192 return -EINVAL; 193 return damon_set_regions(t, &addr_range, 1, min_region_sz); 194 } 195 196 static struct damon_ctx *damon_stat_build_ctx(void) 197 { 198 struct damon_ctx *ctx; 199 struct damon_attrs attrs; 200 struct damon_target *target; 201 202 ctx = damon_new_ctx(); 203 if (!ctx) 204 return NULL; 205 attrs = (struct damon_attrs) { 206 .sample_interval = 5 * USEC_PER_MSEC, 207 .aggr_interval = 100 * USEC_PER_MSEC, 208 .ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC, 209 .min_nr_regions = 10, 210 .max_nr_regions = 1000, 211 }; 212 /* 213 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed 214 * accesses ratio, keeping sampling interval in [5ms, 10s] range. 215 */ 216 attrs.intervals_goal = (struct damon_intervals_goal) { 217 .access_bp = 400, .aggrs = 3, 218 .min_sample_us = 5000, .max_sample_us = 10000000, 219 }; 220 if (damon_set_attrs(ctx, &attrs)) 221 goto free_out; 222 223 if (damon_select_ops(ctx, DAMON_OPS_PADDR)) 224 goto free_out; 225 226 target = damon_new_target(); 227 if (!target) 228 goto free_out; 229 damon_add_target(ctx, target); 230 if (damon_stat_set_monitoring_region(target, ctx->addr_unit, 231 ctx->min_region_sz)) 232 goto free_out; 233 return ctx; 234 free_out: 235 damon_destroy_ctx(ctx); 236 return NULL; 237 } 238 239 static struct damon_call_control call_control = { 240 .fn = damon_stat_damon_call_fn, 241 .repeat = true, 242 }; 243 244 static int damon_stat_start(void) 245 { 246 int err; 247 248 if (damon_stat_context) { 249 if (damon_is_running(damon_stat_context)) 250 return -EAGAIN; 251 damon_destroy_ctx(damon_stat_context); 252 } 253 254 damon_stat_context = damon_stat_build_ctx(); 255 if (!damon_stat_context) 256 return -ENOMEM; 257 err = damon_start(&damon_stat_context, 1, true); 258 if (err) 259 return err; 260 261 damon_stat_last_refresh_jiffies = jiffies; 262 call_control.data = damon_stat_context; 263 return damon_call(damon_stat_context, &call_control); 264 } 265 266 static void damon_stat_stop(void) 267 { 268 damon_stop(&damon_stat_context, 1); 269 damon_destroy_ctx(damon_stat_context); 270 damon_stat_context = NULL; 271 } 272 273 static int damon_stat_enabled_store( 274 const char *val, const struct kernel_param *kp) 275 { 276 bool is_enabled = enabled; 277 int err; 278 279 err = kstrtobool(val, &enabled); 280 if (err) 281 return err; 282 283 if (is_enabled == enabled) 284 return 0; 285 286 if (!damon_initialized()) 287 /* 288 * probably called from command line parsing (parse_args()). 289 * Cannot call damon_new_ctx(). Let damon_stat_init() handle. 290 */ 291 return 0; 292 293 if (enabled) { 294 err = damon_stat_start(); 295 if (err) 296 enabled = false; 297 return err; 298 } 299 damon_stat_stop(); 300 return 0; 301 } 302 303 static int __init damon_stat_init(void) 304 { 305 int err = 0; 306 307 if (!damon_initialized()) { 308 err = -ENOMEM; 309 goto out; 310 } 311 312 /* probably set via command line */ 313 if (enabled) 314 err = damon_stat_start(); 315 316 out: 317 if (err && enabled) 318 enabled = false; 319 return err; 320 } 321 322 module_init(damon_stat_init); 323