1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _RESCTRL_H
3 #define _RESCTRL_H
4
5 #include <linux/cacheinfo.h>
6 #include <linux/kernel.h>
7 #include <linux/list.h>
8 #include <linux/pid.h>
9 #include <linux/resctrl_types.h>
10
11 #ifdef CONFIG_ARCH_HAS_CPU_RESCTRL
12 #include <asm/resctrl.h>
13 #endif
14
15 /* CLOSID, RMID value used by the default control group */
16 #define RESCTRL_RESERVED_CLOSID 0
17 #define RESCTRL_RESERVED_RMID 0
18
19 #define RESCTRL_PICK_ANY_CPU -1
20
21 #ifdef CONFIG_PROC_CPU_RESCTRL
22
23 int proc_resctrl_show(struct seq_file *m,
24 struct pid_namespace *ns,
25 struct pid *pid,
26 struct task_struct *tsk);
27
28 #endif
29
30 /* max value for struct rdt_domain's mbps_val */
31 #define MBA_MAX_MBPS U32_MAX
32
33 /* Walk all possible resources, with variants for only controls or monitors. */
34 #define for_each_rdt_resource(_r) \
35 for ((_r) = resctrl_arch_get_resource(0); \
36 (_r) && (_r)->rid < RDT_NUM_RESOURCES; \
37 (_r) = resctrl_arch_get_resource((_r)->rid + 1))
38
39 #define for_each_capable_rdt_resource(r) \
40 for_each_rdt_resource((r)) \
41 if ((r)->alloc_capable || (r)->mon_capable)
42
43 #define for_each_alloc_capable_rdt_resource(r) \
44 for_each_rdt_resource((r)) \
45 if ((r)->alloc_capable)
46
47 #define for_each_mon_capable_rdt_resource(r) \
48 for_each_rdt_resource((r)) \
49 if ((r)->mon_capable)
50
51 enum resctrl_res_level {
52 RDT_RESOURCE_L3,
53 RDT_RESOURCE_L2,
54 RDT_RESOURCE_MBA,
55 RDT_RESOURCE_SMBA,
56
57 /* Must be the last */
58 RDT_NUM_RESOURCES,
59 };
60
61 /**
62 * enum resctrl_conf_type - The type of configuration.
63 * @CDP_NONE: No prioritisation, both code and data are controlled or monitored.
64 * @CDP_CODE: Configuration applies to instruction fetches.
65 * @CDP_DATA: Configuration applies to reads and writes.
66 */
67 enum resctrl_conf_type {
68 CDP_NONE,
69 CDP_CODE,
70 CDP_DATA,
71 };
72
73 #define CDP_NUM_TYPES (CDP_DATA + 1)
74
75 /*
76 * struct pseudo_lock_region - pseudo-lock region information
77 * @s: Resctrl schema for the resource to which this
78 * pseudo-locked region belongs
79 * @closid: The closid that this pseudo-locked region uses
80 * @d: RDT domain to which this pseudo-locked region
81 * belongs
82 * @cbm: bitmask of the pseudo-locked region
83 * @lock_thread_wq: waitqueue used to wait on the pseudo-locking thread
84 * completion
85 * @thread_done: variable used by waitqueue to test if pseudo-locking
86 * thread completed
87 * @cpu: core associated with the cache on which the setup code
88 * will be run
89 * @line_size: size of the cache lines
90 * @size: size of pseudo-locked region in bytes
91 * @kmem: the kernel memory associated with pseudo-locked region
92 * @minor: minor number of character device associated with this
93 * region
94 * @debugfs_dir: pointer to this region's directory in the debugfs
95 * filesystem
96 * @pm_reqs: Power management QoS requests related to this region
97 */
98 struct pseudo_lock_region {
99 struct resctrl_schema *s;
100 u32 closid;
101 struct rdt_ctrl_domain *d;
102 u32 cbm;
103 wait_queue_head_t lock_thread_wq;
104 int thread_done;
105 int cpu;
106 unsigned int line_size;
107 unsigned int size;
108 void *kmem;
109 unsigned int minor;
110 struct dentry *debugfs_dir;
111 struct list_head pm_reqs;
112 };
113
114 /**
115 * struct resctrl_staged_config - parsed configuration to be applied
116 * @new_ctrl: new ctrl value to be loaded
117 * @have_new_ctrl: whether the user provided new_ctrl is valid
118 */
119 struct resctrl_staged_config {
120 u32 new_ctrl;
121 bool have_new_ctrl;
122 };
123
124 enum resctrl_domain_type {
125 RESCTRL_CTRL_DOMAIN,
126 RESCTRL_MON_DOMAIN,
127 };
128
129 /**
130 * struct rdt_domain_hdr - common header for different domain types
131 * @list: all instances of this resource
132 * @id: unique id for this instance
133 * @type: type of this instance
134 * @cpu_mask: which CPUs share this resource
135 */
136 struct rdt_domain_hdr {
137 struct list_head list;
138 int id;
139 enum resctrl_domain_type type;
140 struct cpumask cpu_mask;
141 };
142
143 /**
144 * struct rdt_ctrl_domain - group of CPUs sharing a resctrl control resource
145 * @hdr: common header for different domain types
146 * @plr: pseudo-locked region (if any) associated with domain
147 * @staged_config: parsed configuration to be applied
148 * @mbps_val: When mba_sc is enabled, this holds the array of user
149 * specified control values for mba_sc in MBps, indexed
150 * by closid
151 */
152 struct rdt_ctrl_domain {
153 struct rdt_domain_hdr hdr;
154 struct pseudo_lock_region *plr;
155 struct resctrl_staged_config staged_config[CDP_NUM_TYPES];
156 u32 *mbps_val;
157 };
158
159 /**
160 * struct rdt_mon_domain - group of CPUs sharing a resctrl monitor resource
161 * @hdr: common header for different domain types
162 * @ci_id: cache info id for this domain
163 * @rmid_busy_llc: bitmap of which limbo RMIDs are above threshold
164 * @mbm_total: saved state for MBM total bandwidth
165 * @mbm_local: saved state for MBM local bandwidth
166 * @mbm_over: worker to periodically read MBM h/w counters
167 * @cqm_limbo: worker to periodically read CQM h/w counters
168 * @mbm_work_cpu: worker CPU for MBM h/w counters
169 * @cqm_work_cpu: worker CPU for CQM h/w counters
170 */
171 struct rdt_mon_domain {
172 struct rdt_domain_hdr hdr;
173 unsigned int ci_id;
174 unsigned long *rmid_busy_llc;
175 struct mbm_state *mbm_total;
176 struct mbm_state *mbm_local;
177 struct delayed_work mbm_over;
178 struct delayed_work cqm_limbo;
179 int mbm_work_cpu;
180 int cqm_work_cpu;
181 };
182
183 /**
184 * struct resctrl_cache - Cache allocation related data
185 * @cbm_len: Length of the cache bit mask
186 * @min_cbm_bits: Minimum number of consecutive bits to be set.
187 * The value 0 means the architecture can support
188 * zero CBM.
189 * @shareable_bits: Bitmask of shareable resource with other
190 * executing entities
191 * @arch_has_sparse_bitmasks: True if a bitmask like f00f is valid.
192 * @arch_has_per_cpu_cfg: True if QOS_CFG register for this cache
193 * level has CPU scope.
194 */
195 struct resctrl_cache {
196 unsigned int cbm_len;
197 unsigned int min_cbm_bits;
198 unsigned int shareable_bits;
199 bool arch_has_sparse_bitmasks;
200 bool arch_has_per_cpu_cfg;
201 };
202
203 /**
204 * enum membw_throttle_mode - System's memory bandwidth throttling mode
205 * @THREAD_THROTTLE_UNDEFINED: Not relevant to the system
206 * @THREAD_THROTTLE_MAX: Memory bandwidth is throttled at the core
207 * always using smallest bandwidth percentage
208 * assigned to threads, aka "max throttling"
209 * @THREAD_THROTTLE_PER_THREAD: Memory bandwidth is throttled at the thread
210 */
211 enum membw_throttle_mode {
212 THREAD_THROTTLE_UNDEFINED = 0,
213 THREAD_THROTTLE_MAX,
214 THREAD_THROTTLE_PER_THREAD,
215 };
216
217 /**
218 * struct resctrl_membw - Memory bandwidth allocation related data
219 * @min_bw: Minimum memory bandwidth percentage user can request
220 * @max_bw: Maximum memory bandwidth value, used as the reset value
221 * @bw_gran: Granularity at which the memory bandwidth is allocated
222 * @delay_linear: True if memory B/W delay is in linear scale
223 * @arch_needs_linear: True if we can't configure non-linear resources
224 * @throttle_mode: Bandwidth throttling mode when threads request
225 * different memory bandwidths
226 * @mba_sc: True if MBA software controller(mba_sc) is enabled
227 * @mb_map: Mapping of memory B/W percentage to memory B/W delay
228 */
229 struct resctrl_membw {
230 u32 min_bw;
231 u32 max_bw;
232 u32 bw_gran;
233 u32 delay_linear;
234 bool arch_needs_linear;
235 enum membw_throttle_mode throttle_mode;
236 bool mba_sc;
237 u32 *mb_map;
238 };
239
240 struct resctrl_schema;
241
242 enum resctrl_scope {
243 RESCTRL_L2_CACHE = 2,
244 RESCTRL_L3_CACHE = 3,
245 RESCTRL_L3_NODE,
246 };
247
248 /**
249 * enum resctrl_schema_fmt - The format user-space provides for a schema.
250 * @RESCTRL_SCHEMA_BITMAP: The schema is a bitmap in hex.
251 * @RESCTRL_SCHEMA_RANGE: The schema is a decimal number.
252 */
253 enum resctrl_schema_fmt {
254 RESCTRL_SCHEMA_BITMAP,
255 RESCTRL_SCHEMA_RANGE,
256 };
257
258 /**
259 * struct rdt_resource - attributes of a resctrl resource
260 * @rid: The index of the resource
261 * @alloc_capable: Is allocation available on this machine
262 * @mon_capable: Is monitor feature available on this machine
263 * @num_rmid: Number of RMIDs available
264 * @ctrl_scope: Scope of this resource for control functions
265 * @mon_scope: Scope of this resource for monitor functions
266 * @cache: Cache allocation related data
267 * @membw: If the component has bandwidth controls, their properties.
268 * @ctrl_domains: RCU list of all control domains for this resource
269 * @mon_domains: RCU list of all monitor domains for this resource
270 * @name: Name to use in "schemata" file.
271 * @schema_fmt: Which format string and parser is used for this schema.
272 * @evt_list: List of monitoring events
273 * @mbm_cfg_mask: Bandwidth sources that can be tracked when bandwidth
274 * monitoring events can be configured.
275 * @cdp_capable: Is the CDP feature available on this resource
276 */
277 struct rdt_resource {
278 int rid;
279 bool alloc_capable;
280 bool mon_capable;
281 int num_rmid;
282 enum resctrl_scope ctrl_scope;
283 enum resctrl_scope mon_scope;
284 struct resctrl_cache cache;
285 struct resctrl_membw membw;
286 struct list_head ctrl_domains;
287 struct list_head mon_domains;
288 char *name;
289 enum resctrl_schema_fmt schema_fmt;
290 struct list_head evt_list;
291 unsigned int mbm_cfg_mask;
292 bool cdp_capable;
293 };
294
295 /*
296 * Get the resource that exists at this level. If the level is not supported
297 * a dummy/not-capable resource can be returned. Levels >= RDT_NUM_RESOURCES
298 * will return NULL.
299 */
300 struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l);
301
302 /**
303 * struct resctrl_schema - configuration abilities of a resource presented to
304 * user-space
305 * @list: Member of resctrl_schema_all.
306 * @name: The name to use in the "schemata" file.
307 * @fmt_str: Format string to show domain value.
308 * @conf_type: Whether this schema is specific to code/data.
309 * @res: The resource structure exported by the architecture to describe
310 * the hardware that is configured by this schema.
311 * @num_closid: The number of closid that can be used with this schema. When
312 * features like CDP are enabled, this will be lower than the
313 * hardware supports for the resource.
314 */
315 struct resctrl_schema {
316 struct list_head list;
317 char name[8];
318 const char *fmt_str;
319 enum resctrl_conf_type conf_type;
320 struct rdt_resource *res;
321 u32 num_closid;
322 };
323
324 struct resctrl_cpu_defaults {
325 u32 closid;
326 u32 rmid;
327 };
328
329 struct resctrl_mon_config_info {
330 struct rdt_resource *r;
331 struct rdt_mon_domain *d;
332 u32 evtid;
333 u32 mon_config;
334 };
335
336 /**
337 * resctrl_arch_sync_cpu_closid_rmid() - Refresh this CPU's CLOSID and RMID.
338 * Call via IPI.
339 * @info: If non-NULL, a pointer to a struct resctrl_cpu_defaults
340 * specifying the new CLOSID and RMID for tasks in the default
341 * resctrl ctrl and mon group when running on this CPU. If NULL,
342 * this CPU is not re-assigned to a different default group.
343 *
344 * Propagates reassignment of CPUs and/or tasks to different resctrl groups
345 * when requested by the resctrl core code.
346 *
347 * This function records the per-cpu defaults specified by @info (if any),
348 * and then reconfigures the CPU's hardware CLOSID and RMID for subsequent
349 * execution based on @current, in the same way as during a task switch.
350 */
351 void resctrl_arch_sync_cpu_closid_rmid(void *info);
352
353 /**
354 * resctrl_get_default_ctrl() - Return the default control value for this
355 * resource.
356 * @r: The resource whose default control type is queried.
357 */
resctrl_get_default_ctrl(struct rdt_resource * r)358 static inline u32 resctrl_get_default_ctrl(struct rdt_resource *r)
359 {
360 switch (r->schema_fmt) {
361 case RESCTRL_SCHEMA_BITMAP:
362 return BIT_MASK(r->cache.cbm_len) - 1;
363 case RESCTRL_SCHEMA_RANGE:
364 return r->membw.max_bw;
365 }
366
367 return WARN_ON_ONCE(1);
368 }
369
370 /* The number of closid supported by this resource regardless of CDP */
371 u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
372 u32 resctrl_arch_system_num_rmid_idx(void);
373 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid);
374
375 bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt);
376
377 /**
378 * resctrl_arch_mon_event_config_write() - Write the config for an event.
379 * @config_info: struct resctrl_mon_config_info describing the resource, domain
380 * and event.
381 *
382 * Reads resource, domain and eventid from @config_info and writes the
383 * event config_info->mon_config into hardware.
384 *
385 * Called via IPI to reach a CPU that is a member of the specified domain.
386 */
387 void resctrl_arch_mon_event_config_write(void *config_info);
388
389 /**
390 * resctrl_arch_mon_event_config_read() - Read the config for an event.
391 * @config_info: struct resctrl_mon_config_info describing the resource, domain
392 * and event.
393 *
394 * Reads resource, domain and eventid from @config_info and reads the
395 * hardware config value into config_info->mon_config.
396 *
397 * Called via IPI to reach a CPU that is a member of the specified domain.
398 */
399 void resctrl_arch_mon_event_config_read(void *config_info);
400
401 /* For use by arch code to remap resctrl's smaller CDP CLOSID range */
resctrl_get_config_index(u32 closid,enum resctrl_conf_type type)402 static inline u32 resctrl_get_config_index(u32 closid,
403 enum resctrl_conf_type type)
404 {
405 switch (type) {
406 default:
407 case CDP_NONE:
408 return closid;
409 case CDP_CODE:
410 return closid * 2 + 1;
411 case CDP_DATA:
412 return closid * 2;
413 }
414 }
415
416 bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l);
417 int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable);
418
419 /*
420 * Update the ctrl_val and apply this config right now.
421 * Must be called on one of the domain's CPUs.
422 */
423 int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
424 u32 closid, enum resctrl_conf_type t, u32 cfg_val);
425
426 u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d,
427 u32 closid, enum resctrl_conf_type type);
428 int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d);
429 int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d);
430 void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d);
431 void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d);
432 void resctrl_online_cpu(unsigned int cpu);
433 void resctrl_offline_cpu(unsigned int cpu);
434
435 /**
436 * resctrl_arch_rmid_read() - Read the eventid counter corresponding to rmid
437 * for this resource and domain.
438 * @r: resource that the counter should be read from.
439 * @d: domain that the counter should be read from.
440 * @closid: closid that matches the rmid. Depending on the architecture, the
441 * counter may match traffic of both @closid and @rmid, or @rmid
442 * only.
443 * @rmid: rmid of the counter to read.
444 * @eventid: eventid to read, e.g. L3 occupancy.
445 * @val: result of the counter read in bytes.
446 * @arch_mon_ctx: An architecture specific value from
447 * resctrl_arch_mon_ctx_alloc(), for MPAM this identifies
448 * the hardware monitor allocated for this read request.
449 *
450 * Some architectures need to sleep when first programming some of the counters.
451 * (specifically: arm64's MPAM cache occupancy counters can return 'not ready'
452 * for a short period of time). Call from a non-migrateable process context on
453 * a CPU that belongs to domain @d. e.g. use smp_call_on_cpu() or
454 * schedule_work_on(). This function can be called with interrupts masked,
455 * e.g. using smp_call_function_any(), but may consistently return an error.
456 *
457 * Return:
458 * 0 on success, or -EIO, -EINVAL etc on error.
459 */
460 int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d,
461 u32 closid, u32 rmid, enum resctrl_event_id eventid,
462 u64 *val, void *arch_mon_ctx);
463
464 /**
465 * resctrl_arch_rmid_read_context_check() - warn about invalid contexts
466 *
467 * When built with CONFIG_DEBUG_ATOMIC_SLEEP generate a warning when
468 * resctrl_arch_rmid_read() is called with preemption disabled.
469 *
470 * The contract with resctrl_arch_rmid_read() is that if interrupts
471 * are unmasked, it can sleep. This allows NOHZ_FULL systems to use an
472 * IPI, (and fail if the call needed to sleep), while most of the time
473 * the work is scheduled, allowing the call to sleep.
474 */
resctrl_arch_rmid_read_context_check(void)475 static inline void resctrl_arch_rmid_read_context_check(void)
476 {
477 if (!irqs_disabled())
478 might_sleep();
479 }
480
481 /**
482 * resctrl_find_domain() - Search for a domain id in a resource domain list.
483 * @h: The domain list to search.
484 * @id: The domain id to search for.
485 * @pos: A pointer to position in the list id should be inserted.
486 *
487 * Search the domain list to find the domain id. If the domain id is
488 * found, return the domain. NULL otherwise. If the domain id is not
489 * found (and NULL returned) then the first domain with id bigger than
490 * the input id can be returned to the caller via @pos.
491 */
492 struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id,
493 struct list_head **pos);
494
495 /**
496 * resctrl_arch_reset_rmid() - Reset any private state associated with rmid
497 * and eventid.
498 * @r: The domain's resource.
499 * @d: The rmid's domain.
500 * @closid: closid that matches the rmid. Depending on the architecture, the
501 * counter may match traffic of both @closid and @rmid, or @rmid only.
502 * @rmid: The rmid whose counter values should be reset.
503 * @eventid: The eventid whose counter values should be reset.
504 *
505 * This can be called from any CPU.
506 */
507 void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d,
508 u32 closid, u32 rmid,
509 enum resctrl_event_id eventid);
510
511 /**
512 * resctrl_arch_reset_rmid_all() - Reset all private state associated with
513 * all rmids and eventids.
514 * @r: The resctrl resource.
515 * @d: The domain for which all architectural counter state will
516 * be cleared.
517 *
518 * This can be called from any CPU.
519 */
520 void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d);
521
522 /**
523 * resctrl_arch_reset_all_ctrls() - Reset the control for each CLOSID to its
524 * default.
525 * @r: The resctrl resource to reset.
526 *
527 * This can be called from any CPU.
528 */
529 void resctrl_arch_reset_all_ctrls(struct rdt_resource *r);
530
531 extern unsigned int resctrl_rmid_realloc_threshold;
532 extern unsigned int resctrl_rmid_realloc_limit;
533
534 int resctrl_init(void);
535 void resctrl_exit(void);
536
537 #ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK
538 u64 resctrl_arch_get_prefetch_disable_bits(void);
539 int resctrl_arch_pseudo_lock_fn(void *_plr);
540 int resctrl_arch_measure_cycles_lat_fn(void *_plr);
541 int resctrl_arch_measure_l2_residency(void *_plr);
542 int resctrl_arch_measure_l3_residency(void *_plr);
543 #else
resctrl_arch_get_prefetch_disable_bits(void)544 static inline u64 resctrl_arch_get_prefetch_disable_bits(void) { return 0; }
resctrl_arch_pseudo_lock_fn(void * _plr)545 static inline int resctrl_arch_pseudo_lock_fn(void *_plr) { return 0; }
resctrl_arch_measure_cycles_lat_fn(void * _plr)546 static inline int resctrl_arch_measure_cycles_lat_fn(void *_plr) { return 0; }
resctrl_arch_measure_l2_residency(void * _plr)547 static inline int resctrl_arch_measure_l2_residency(void *_plr) { return 0; }
resctrl_arch_measure_l3_residency(void * _plr)548 static inline int resctrl_arch_measure_l3_residency(void *_plr) { return 0; }
549 #endif /* CONFIG_RESCTRL_FS_PSEUDO_LOCK */
550 #endif /* _RESCTRL_H */
551