1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shows data access monitoring results in simple metrics. 4 */ 5 6 #define pr_fmt(fmt) "damon-stat: " fmt 7 8 #include <linux/damon.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/sort.h> 13 14 #ifdef MODULE_PARAM_PREFIX 15 #undef MODULE_PARAM_PREFIX 16 #endif 17 #define MODULE_PARAM_PREFIX "damon_stat." 18 19 static int damon_stat_enabled_store( 20 const char *val, const struct kernel_param *kp); 21 22 static int damon_stat_enabled_load(char *buffer, 23 const struct kernel_param *kp); 24 25 static const struct kernel_param_ops enabled_param_ops = { 26 .set = damon_stat_enabled_store, 27 .get = damon_stat_enabled_load, 28 }; 29 30 static bool enabled __read_mostly = IS_ENABLED( 31 CONFIG_DAMON_STAT_ENABLED_DEFAULT); 32 module_param_cb(enabled, &enabled_param_ops, NULL, 0600); 33 MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT"); 34 35 static unsigned long estimated_memory_bandwidth __read_mostly; 36 module_param(estimated_memory_bandwidth, ulong, 0400); 37 MODULE_PARM_DESC(estimated_memory_bandwidth, 38 "Estimated memory bandwidth usage in bytes per second"); 39 40 static long memory_idle_ms_percentiles[101] = {0,}; 41 module_param_array(memory_idle_ms_percentiles, long, NULL, 0400); 42 MODULE_PARM_DESC(memory_idle_ms_percentiles, 43 "Memory idle time percentiles in milliseconds"); 44 45 static unsigned long aggr_interval_us; 46 module_param(aggr_interval_us, ulong, 0400); 47 MODULE_PARM_DESC(aggr_interval_us, 48 "Current tuned aggregation interval in microseconds"); 49 50 static struct damon_ctx *damon_stat_context; 51 52 static unsigned long damon_stat_last_refresh_jiffies; 53 54 static void damon_stat_set_estimated_memory_bandwidth(struct damon_ctx *c) 55 { 56 struct damon_target *t; 57 struct damon_region *r; 58 unsigned long access_bytes = 0; 59 60 damon_for_each_target(t, c) { 61 damon_for_each_region(r, t) 62 access_bytes += (r->ar.end - r->ar.start) * 63 r->nr_accesses; 64 } 65 estimated_memory_bandwidth = access_bytes * USEC_PER_MSEC * 66 MSEC_PER_SEC / c->attrs.aggr_interval; 67 } 68 69 static int damon_stat_idletime(const struct damon_region *r) 70 { 71 if (r->nr_accesses) 72 return -1 * (r->age + 1); 73 return r->age + 1; 74 } 75 76 static int damon_stat_cmp_regions(const void *a, const void *b) 77 { 78 const struct damon_region *ra = *(const struct damon_region **)a; 79 const struct damon_region *rb = *(const struct damon_region **)b; 80 81 return damon_stat_idletime(ra) - damon_stat_idletime(rb); 82 } 83 84 static int damon_stat_sort_regions(struct damon_ctx *c, 85 struct damon_region ***sorted_ptr, int *nr_regions_ptr, 86 unsigned long *total_sz_ptr) 87 { 88 struct damon_target *t; 89 struct damon_region *r; 90 struct damon_region **region_pointers; 91 unsigned int nr_regions = 0; 92 unsigned long total_sz = 0; 93 94 damon_for_each_target(t, c) { 95 /* there is only one target */ 96 region_pointers = kmalloc_objs(*region_pointers, 97 damon_nr_regions(t)); 98 if (!region_pointers) 99 return -ENOMEM; 100 damon_for_each_region(r, t) { 101 region_pointers[nr_regions++] = r; 102 total_sz += r->ar.end - r->ar.start; 103 } 104 } 105 sort(region_pointers, nr_regions, sizeof(*region_pointers), 106 damon_stat_cmp_regions, NULL); 107 *sorted_ptr = region_pointers; 108 *nr_regions_ptr = nr_regions; 109 *total_sz_ptr = total_sz; 110 return 0; 111 } 112 113 static void damon_stat_set_idletime_percentiles(struct damon_ctx *c) 114 { 115 struct damon_region **sorted_regions, *region; 116 int nr_regions; 117 unsigned long total_sz, accounted_bytes = 0; 118 int err, i, next_percentile = 0; 119 120 err = damon_stat_sort_regions(c, &sorted_regions, &nr_regions, 121 &total_sz); 122 if (err) 123 return; 124 for (i = 0; i < nr_regions; i++) { 125 region = sorted_regions[i]; 126 accounted_bytes += region->ar.end - region->ar.start; 127 while (next_percentile <= accounted_bytes * 100 / total_sz) 128 memory_idle_ms_percentiles[next_percentile++] = 129 damon_stat_idletime(region) * 130 (long)c->attrs.aggr_interval / USEC_PER_MSEC; 131 } 132 kfree(sorted_regions); 133 } 134 135 static int damon_stat_damon_call_fn(void *data) 136 { 137 struct damon_ctx *c = data; 138 139 /* avoid unnecessarily frequent stat update */ 140 if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + 141 msecs_to_jiffies(5 * MSEC_PER_SEC))) 142 return 0; 143 damon_stat_last_refresh_jiffies = jiffies; 144 145 aggr_interval_us = c->attrs.aggr_interval; 146 damon_stat_set_estimated_memory_bandwidth(c); 147 damon_stat_set_idletime_percentiles(c); 148 return 0; 149 } 150 151 struct damon_stat_system_ram_range_walk_arg { 152 bool walked; 153 struct resource res; 154 }; 155 156 static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg) 157 { 158 struct damon_stat_system_ram_range_walk_arg *a = arg; 159 160 if (!a->walked) { 161 a->walked = true; 162 a->res.start = res->start; 163 } 164 a->res.end = res->end; 165 return 0; 166 } 167 168 static unsigned long damon_stat_res_to_core_addr(resource_size_t ra, 169 unsigned long addr_unit) 170 { 171 /* 172 * Use div_u64() for avoiding linking errors related with __udivdi3, 173 * __aeabi_uldivmod, or similar problems. This should also improve the 174 * performance optimization (read div_u64() comment for the detail). 175 */ 176 if (sizeof(ra) == 8 && sizeof(addr_unit) == 4) 177 return div_u64(ra, addr_unit); 178 return ra / addr_unit; 179 } 180 181 static int damon_stat_set_monitoring_region(struct damon_target *t, 182 unsigned long addr_unit, unsigned long min_region_sz) 183 { 184 struct damon_addr_range addr_range; 185 struct damon_stat_system_ram_range_walk_arg arg = {}; 186 187 walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn); 188 if (!arg.walked) 189 return -EINVAL; 190 addr_range.start = damon_stat_res_to_core_addr( 191 arg.res.start, addr_unit); 192 addr_range.end = damon_stat_res_to_core_addr( 193 arg.res.end + 1, addr_unit); 194 if (addr_range.end <= addr_range.start) 195 return -EINVAL; 196 return damon_set_regions(t, &addr_range, 1, min_region_sz); 197 } 198 199 static struct damon_ctx *damon_stat_build_ctx(void) 200 { 201 struct damon_ctx *ctx; 202 struct damon_attrs attrs; 203 struct damon_target *target; 204 205 ctx = damon_new_ctx(); 206 if (!ctx) 207 return NULL; 208 attrs = (struct damon_attrs) { 209 .sample_interval = 5 * USEC_PER_MSEC, 210 .aggr_interval = 100 * USEC_PER_MSEC, 211 .ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC, 212 .min_nr_regions = 10, 213 .max_nr_regions = 1000, 214 }; 215 /* 216 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed 217 * accesses ratio, keeping sampling interval in [5ms, 10s] range. 218 */ 219 attrs.intervals_goal = (struct damon_intervals_goal) { 220 .access_bp = 400, .aggrs = 3, 221 .min_sample_us = 5000, .max_sample_us = 10000000, 222 }; 223 if (damon_set_attrs(ctx, &attrs)) 224 goto free_out; 225 226 if (damon_select_ops(ctx, DAMON_OPS_PADDR)) 227 goto free_out; 228 229 target = damon_new_target(); 230 if (!target) 231 goto free_out; 232 damon_add_target(ctx, target); 233 if (damon_stat_set_monitoring_region(target, ctx->addr_unit, 234 ctx->min_region_sz)) 235 goto free_out; 236 return ctx; 237 free_out: 238 damon_destroy_ctx(ctx); 239 return NULL; 240 } 241 242 static struct damon_call_control call_control = { 243 .fn = damon_stat_damon_call_fn, 244 .repeat = true, 245 }; 246 247 static int damon_stat_start(void) 248 { 249 int err; 250 251 if (damon_stat_context) { 252 if (damon_is_running(damon_stat_context)) 253 return -EAGAIN; 254 damon_destroy_ctx(damon_stat_context); 255 } 256 257 damon_stat_context = damon_stat_build_ctx(); 258 if (!damon_stat_context) 259 return -ENOMEM; 260 err = damon_start(&damon_stat_context, 1, true); 261 if (err) { 262 damon_destroy_ctx(damon_stat_context); 263 damon_stat_context = NULL; 264 return err; 265 } 266 267 damon_stat_last_refresh_jiffies = jiffies; 268 call_control.data = damon_stat_context; 269 return damon_call(damon_stat_context, &call_control); 270 } 271 272 static void damon_stat_stop(void) 273 { 274 damon_stop(&damon_stat_context, 1); 275 damon_destroy_ctx(damon_stat_context); 276 damon_stat_context = NULL; 277 } 278 279 static bool damon_stat_enabled(void) 280 { 281 if (!damon_stat_context) 282 return false; 283 return damon_is_running(damon_stat_context); 284 } 285 286 static int damon_stat_enabled_store( 287 const char *val, const struct kernel_param *kp) 288 { 289 int err; 290 291 err = kstrtobool(val, &enabled); 292 if (err) 293 return err; 294 295 if (damon_stat_enabled() == enabled) 296 return 0; 297 298 if (!damon_initialized()) 299 /* 300 * probably called from command line parsing (parse_args()). 301 * Cannot call damon_new_ctx(). Let damon_stat_init() handle. 302 */ 303 return 0; 304 305 if (enabled) 306 return damon_stat_start(); 307 damon_stat_stop(); 308 return 0; 309 } 310 311 static int damon_stat_enabled_load(char *buffer, const struct kernel_param *kp) 312 { 313 return sprintf(buffer, "%c\n", damon_stat_enabled() ? 'Y' : 'N'); 314 } 315 316 static int __init damon_stat_init(void) 317 { 318 int err = 0; 319 320 if (!damon_initialized()) { 321 err = -ENOMEM; 322 goto out; 323 } 324 325 /* probably set via command line */ 326 if (enabled) 327 err = damon_stat_start(); 328 329 out: 330 if (err && enabled) 331 enabled = false; 332 return err; 333 } 334 335 module_init(damon_stat_init); 336