1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shows data access monitoring results in simple metrics.
4 */
5
6 #define pr_fmt(fmt) "damon-stat: " fmt
7
8 #include <linux/damon.h>
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/sort.h>
13
14 #ifdef MODULE_PARAM_PREFIX
15 #undef MODULE_PARAM_PREFIX
16 #endif
17 #define MODULE_PARAM_PREFIX "damon_stat."
18
19 static int damon_stat_enabled_store(
20 const char *val, const struct kernel_param *kp);
21
22 static const struct kernel_param_ops enabled_param_ops = {
23 .set = damon_stat_enabled_store,
24 .get = param_get_bool,
25 };
26
27 static bool enabled __read_mostly = IS_ENABLED(
28 CONFIG_DAMON_STAT_ENABLED_DEFAULT);
29 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
30 MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT");
31
32 static unsigned long estimated_memory_bandwidth __read_mostly;
33 module_param(estimated_memory_bandwidth, ulong, 0400);
34 MODULE_PARM_DESC(estimated_memory_bandwidth,
35 "Estimated memory bandwidth usage in bytes per second");
36
37 static long memory_idle_ms_percentiles[101] = {0,};
38 module_param_array(memory_idle_ms_percentiles, long, NULL, 0400);
39 MODULE_PARM_DESC(memory_idle_ms_percentiles,
40 "Memory idle time percentiles in milliseconds");
41
42 static unsigned long aggr_interval_us;
43 module_param(aggr_interval_us, ulong, 0400);
44 MODULE_PARM_DESC(aggr_interval_us,
45 "Current tuned aggregation interval in microseconds");
46
47 static struct damon_ctx *damon_stat_context;
48
49 static unsigned long damon_stat_last_refresh_jiffies;
50
damon_stat_set_estimated_memory_bandwidth(struct damon_ctx * c)51 static void damon_stat_set_estimated_memory_bandwidth(struct damon_ctx *c)
52 {
53 struct damon_target *t;
54 struct damon_region *r;
55 unsigned long access_bytes = 0;
56
57 damon_for_each_target(t, c) {
58 damon_for_each_region(r, t)
59 access_bytes += (r->ar.end - r->ar.start) *
60 r->nr_accesses;
61 }
62 estimated_memory_bandwidth = access_bytes * USEC_PER_MSEC *
63 MSEC_PER_SEC / c->attrs.aggr_interval;
64 }
65
damon_stat_idletime(const struct damon_region * r)66 static int damon_stat_idletime(const struct damon_region *r)
67 {
68 if (r->nr_accesses)
69 return -1 * (r->age + 1);
70 return r->age + 1;
71 }
72
damon_stat_cmp_regions(const void * a,const void * b)73 static int damon_stat_cmp_regions(const void *a, const void *b)
74 {
75 const struct damon_region *ra = *(const struct damon_region **)a;
76 const struct damon_region *rb = *(const struct damon_region **)b;
77
78 return damon_stat_idletime(ra) - damon_stat_idletime(rb);
79 }
80
damon_stat_sort_regions(struct damon_ctx * c,struct damon_region *** sorted_ptr,int * nr_regions_ptr,unsigned long * total_sz_ptr)81 static int damon_stat_sort_regions(struct damon_ctx *c,
82 struct damon_region ***sorted_ptr, int *nr_regions_ptr,
83 unsigned long *total_sz_ptr)
84 {
85 struct damon_target *t;
86 struct damon_region *r;
87 struct damon_region **region_pointers;
88 unsigned int nr_regions = 0;
89 unsigned long total_sz = 0;
90
91 damon_for_each_target(t, c) {
92 /* there is only one target */
93 region_pointers = kmalloc_objs(*region_pointers,
94 damon_nr_regions(t));
95 if (!region_pointers)
96 return -ENOMEM;
97 damon_for_each_region(r, t) {
98 region_pointers[nr_regions++] = r;
99 total_sz += r->ar.end - r->ar.start;
100 }
101 }
102 sort(region_pointers, nr_regions, sizeof(*region_pointers),
103 damon_stat_cmp_regions, NULL);
104 *sorted_ptr = region_pointers;
105 *nr_regions_ptr = nr_regions;
106 *total_sz_ptr = total_sz;
107 return 0;
108 }
109
damon_stat_set_idletime_percentiles(struct damon_ctx * c)110 static void damon_stat_set_idletime_percentiles(struct damon_ctx *c)
111 {
112 struct damon_region **sorted_regions, *region;
113 int nr_regions;
114 unsigned long total_sz, accounted_bytes = 0;
115 int err, i, next_percentile = 0;
116
117 err = damon_stat_sort_regions(c, &sorted_regions, &nr_regions,
118 &total_sz);
119 if (err)
120 return;
121 for (i = 0; i < nr_regions; i++) {
122 region = sorted_regions[i];
123 accounted_bytes += region->ar.end - region->ar.start;
124 while (next_percentile <= accounted_bytes * 100 / total_sz)
125 memory_idle_ms_percentiles[next_percentile++] =
126 damon_stat_idletime(region) *
127 (long)c->attrs.aggr_interval / USEC_PER_MSEC;
128 }
129 kfree(sorted_regions);
130 }
131
damon_stat_damon_call_fn(void * data)132 static int damon_stat_damon_call_fn(void *data)
133 {
134 struct damon_ctx *c = data;
135
136 /* avoid unnecessarily frequent stat update */
137 if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies +
138 msecs_to_jiffies(5 * MSEC_PER_SEC)))
139 return 0;
140 damon_stat_last_refresh_jiffies = jiffies;
141
142 aggr_interval_us = c->attrs.aggr_interval;
143 damon_stat_set_estimated_memory_bandwidth(c);
144 damon_stat_set_idletime_percentiles(c);
145 return 0;
146 }
147
148 struct damon_stat_system_ram_range_walk_arg {
149 bool walked;
150 struct resource res;
151 };
152
damon_stat_system_ram_walk_fn(struct resource * res,void * arg)153 static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg)
154 {
155 struct damon_stat_system_ram_range_walk_arg *a = arg;
156
157 if (!a->walked) {
158 a->walked = true;
159 a->res.start = res->start;
160 }
161 a->res.end = res->end;
162 return 0;
163 }
164
damon_stat_res_to_core_addr(resource_size_t ra,unsigned long addr_unit)165 static unsigned long damon_stat_res_to_core_addr(resource_size_t ra,
166 unsigned long addr_unit)
167 {
168 /*
169 * Use div_u64() for avoiding linking errors related with __udivdi3,
170 * __aeabi_uldivmod, or similar problems. This should also improve the
171 * performance optimization (read div_u64() comment for the detail).
172 */
173 if (sizeof(ra) == 8 && sizeof(addr_unit) == 4)
174 return div_u64(ra, addr_unit);
175 return ra / addr_unit;
176 }
177
damon_stat_set_monitoring_region(struct damon_target * t,unsigned long addr_unit,unsigned long min_region_sz)178 static int damon_stat_set_monitoring_region(struct damon_target *t,
179 unsigned long addr_unit, unsigned long min_region_sz)
180 {
181 struct damon_addr_range addr_range;
182 struct damon_stat_system_ram_range_walk_arg arg = {};
183
184 walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn);
185 if (!arg.walked)
186 return -EINVAL;
187 addr_range.start = damon_stat_res_to_core_addr(
188 arg.res.start, addr_unit);
189 addr_range.end = damon_stat_res_to_core_addr(
190 arg.res.end + 1, addr_unit);
191 if (addr_range.end <= addr_range.start)
192 return -EINVAL;
193 return damon_set_regions(t, &addr_range, 1, min_region_sz);
194 }
195
damon_stat_build_ctx(void)196 static struct damon_ctx *damon_stat_build_ctx(void)
197 {
198 struct damon_ctx *ctx;
199 struct damon_attrs attrs;
200 struct damon_target *target;
201
202 ctx = damon_new_ctx();
203 if (!ctx)
204 return NULL;
205 attrs = (struct damon_attrs) {
206 .sample_interval = 5 * USEC_PER_MSEC,
207 .aggr_interval = 100 * USEC_PER_MSEC,
208 .ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC,
209 .min_nr_regions = 10,
210 .max_nr_regions = 1000,
211 };
212 /*
213 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed
214 * accesses ratio, keeping sampling interval in [5ms, 10s] range.
215 */
216 attrs.intervals_goal = (struct damon_intervals_goal) {
217 .access_bp = 400, .aggrs = 3,
218 .min_sample_us = 5000, .max_sample_us = 10000000,
219 };
220 if (damon_set_attrs(ctx, &attrs))
221 goto free_out;
222
223 if (damon_select_ops(ctx, DAMON_OPS_PADDR))
224 goto free_out;
225
226 target = damon_new_target();
227 if (!target)
228 goto free_out;
229 damon_add_target(ctx, target);
230 if (damon_stat_set_monitoring_region(target, ctx->addr_unit,
231 ctx->min_region_sz))
232 goto free_out;
233 return ctx;
234 free_out:
235 damon_destroy_ctx(ctx);
236 return NULL;
237 }
238
239 static struct damon_call_control call_control = {
240 .fn = damon_stat_damon_call_fn,
241 .repeat = true,
242 };
243
damon_stat_start(void)244 static int damon_stat_start(void)
245 {
246 int err;
247
248 damon_stat_context = damon_stat_build_ctx();
249 if (!damon_stat_context)
250 return -ENOMEM;
251 err = damon_start(&damon_stat_context, 1, true);
252 if (err)
253 return err;
254
255 damon_stat_last_refresh_jiffies = jiffies;
256 call_control.data = damon_stat_context;
257 return damon_call(damon_stat_context, &call_control);
258 }
259
damon_stat_stop(void)260 static void damon_stat_stop(void)
261 {
262 damon_stop(&damon_stat_context, 1);
263 damon_destroy_ctx(damon_stat_context);
264 }
265
damon_stat_enabled_store(const char * val,const struct kernel_param * kp)266 static int damon_stat_enabled_store(
267 const char *val, const struct kernel_param *kp)
268 {
269 bool is_enabled = enabled;
270 int err;
271
272 err = kstrtobool(val, &enabled);
273 if (err)
274 return err;
275
276 if (is_enabled == enabled)
277 return 0;
278
279 if (!damon_initialized())
280 /*
281 * probably called from command line parsing (parse_args()).
282 * Cannot call damon_new_ctx(). Let damon_stat_init() handle.
283 */
284 return 0;
285
286 if (enabled) {
287 err = damon_stat_start();
288 if (err)
289 enabled = false;
290 return err;
291 }
292 damon_stat_stop();
293 return 0;
294 }
295
damon_stat_init(void)296 static int __init damon_stat_init(void)
297 {
298 int err = 0;
299
300 if (!damon_initialized()) {
301 err = -ENOMEM;
302 goto out;
303 }
304
305 /* probably set via command line */
306 if (enabled)
307 err = damon_stat_start();
308
309 out:
310 if (err && enabled)
311 enabled = false;
312 return err;
313 }
314
315 module_init(damon_stat_init);
316