1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _FS_RESCTRL_INTERNAL_H 3 #define _FS_RESCTRL_INTERNAL_H 4 5 #include <linux/resctrl.h> 6 #include <linux/kernfs.h> 7 #include <linux/fs_context.h> 8 #include <linux/tick.h> 9 10 #define CQM_LIMBOCHECK_INTERVAL 1000 11 12 /** 13 * cpumask_any_housekeeping() - Choose any CPU in @mask, preferring those that 14 * aren't marked nohz_full 15 * @mask: The mask to pick a CPU from. 16 * @exclude_cpu:The CPU to avoid picking. 17 * 18 * Returns a CPU from @mask, but not @exclude_cpu. If there are housekeeping 19 * CPUs that don't use nohz_full, these are preferred. Pass 20 * RESCTRL_PICK_ANY_CPU to avoid excluding any CPUs. 21 * 22 * When a CPU is excluded, returns >= nr_cpu_ids if no CPUs are available. 23 */ 24 static inline unsigned int 25 cpumask_any_housekeeping(const struct cpumask *mask, int exclude_cpu) 26 { 27 unsigned int cpu; 28 29 /* Try to find a CPU that isn't nohz_full to use in preference */ 30 if (tick_nohz_full_enabled()) { 31 cpu = cpumask_any_andnot_but(mask, tick_nohz_full_mask, exclude_cpu); 32 if (cpu < nr_cpu_ids) 33 return cpu; 34 } 35 36 return cpumask_any_but(mask, exclude_cpu); 37 } 38 39 struct rdt_fs_context { 40 struct kernfs_fs_context kfc; 41 bool enable_cdpl2; 42 bool enable_cdpl3; 43 bool enable_mba_mbps; 44 bool enable_debug; 45 }; 46 47 static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc) 48 { 49 struct kernfs_fs_context *kfc = fc->fs_private; 50 51 return container_of(kfc, struct rdt_fs_context, kfc); 52 } 53 54 /** 55 * struct mon_evt - Properties of a monitor event 56 * @evtid: event id 57 * @rid: resource id for this event 58 * @name: name of the event 59 * @evt_cfg: Event configuration value that represents the 60 * memory transactions (e.g., READS_TO_LOCAL_MEM, 61 * READS_TO_REMOTE_MEM) being tracked by @evtid. 62 * Only valid if @evtid is an MBM event. 63 * @configurable: true if the event is configurable 64 * @any_cpu: true if the event can be read from any CPU 65 * @is_floating_point: event values are displayed in floating point format 66 * @binary_bits: number of fixed-point binary bits from architecture, 67 * only valid if @is_floating_point is true 68 * @enabled: true if the event is enabled 69 */ 70 struct mon_evt { 71 enum resctrl_event_id evtid; 72 enum resctrl_res_level rid; 73 char *name; 74 u32 evt_cfg; 75 bool configurable; 76 bool any_cpu; 77 bool is_floating_point; 78 unsigned int binary_bits; 79 bool enabled; 80 }; 81 82 extern struct mon_evt mon_event_all[QOS_NUM_EVENTS]; 83 84 #define for_each_mon_event(mevt) for (mevt = &mon_event_all[QOS_FIRST_EVENT]; \ 85 mevt < &mon_event_all[QOS_NUM_EVENTS]; mevt++) 86 87 /* Limit for mon_evt::binary_bits */ 88 #define MAX_BINARY_BITS 27 89 90 /** 91 * struct mon_data - Monitoring details for each event file. 92 * @list: Member of the global @mon_data_kn_priv_list list. 93 * @rid: Resource id associated with the event file. 94 * @evt: Event structure associated with the event file. 95 * @sum: Set for RDT_RESOURCE_L3 when event must be summed 96 * across multiple domains. 97 * @domid: When @sum is zero this is the domain to which 98 * the event file belongs. When @sum is one this 99 * is the id of the L3 cache that all domains to be 100 * summed share. 101 * 102 * Pointed to by the kernfs kn->priv field of monitoring event files. 103 * Readers and writers must hold rdtgroup_mutex. 104 */ 105 struct mon_data { 106 struct list_head list; 107 enum resctrl_res_level rid; 108 struct mon_evt *evt; 109 int domid; 110 bool sum; 111 }; 112 113 /** 114 * struct rmid_read - Data passed across smp_call*() to read event count. 115 * @rgrp: Resource group for which the counter is being read. If it is a parent 116 * resource group then its event count is summed with the count from all 117 * its child resource groups. 118 * @r: Resource describing the properties of the event being read. 119 * @hdr: Header of domain that the counter should be read from. If NULL then 120 * sum all domains in @r sharing L3 @ci.id 121 * @evt: Which monitor event to read. 122 * @first: Initialize MBM counter when true. 123 * @ci: Cacheinfo for L3. Only set when @hdr is NULL. Used when summing 124 * domains. 125 * @is_mbm_cntr: true if "mbm_event" counter assignment mode is enabled and it 126 * is an MBM event. 127 * @err: Error encountered when reading counter. 128 * @val: Returned value of event counter. If @rgrp is a parent resource 129 * group, @val includes the sum of event counts from its child 130 * resource groups. If @hdr is NULL, @val includes the sum of all 131 * domains in @r sharing @ci.id, (summed across child resource groups 132 * if @rgrp is a parent resource group). 133 * @arch_mon_ctx: Hardware monitor allocated for this read request (MPAM only). 134 */ 135 struct rmid_read { 136 struct rdtgroup *rgrp; 137 struct rdt_resource *r; 138 struct rdt_domain_hdr *hdr; 139 struct mon_evt *evt; 140 bool first; 141 struct cacheinfo *ci; 142 bool is_mbm_cntr; 143 int err; 144 u64 val; 145 void *arch_mon_ctx; 146 }; 147 148 extern struct list_head resctrl_schema_all; 149 150 extern bool resctrl_mounted; 151 152 enum rdt_group_type { 153 RDTCTRL_GROUP = 0, 154 RDTMON_GROUP, 155 RDT_NUM_GROUP, 156 }; 157 158 /** 159 * enum rdtgrp_mode - Mode of a RDT resource group 160 * @RDT_MODE_SHAREABLE: This resource group allows sharing of its allocations 161 * @RDT_MODE_EXCLUSIVE: No sharing of this resource group's allocations allowed 162 * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking 163 * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations 164 * allowed AND the allocations are Cache Pseudo-Locked 165 * @RDT_NUM_MODES: Total number of modes 166 * 167 * The mode of a resource group enables control over the allowed overlap 168 * between allocations associated with different resource groups (classes 169 * of service). User is able to modify the mode of a resource group by 170 * writing to the "mode" resctrl file associated with the resource group. 171 * 172 * The "shareable", "exclusive", and "pseudo-locksetup" modes are set by 173 * writing the appropriate text to the "mode" file. A resource group enters 174 * "pseudo-locked" mode after the schemata is written while the resource 175 * group is in "pseudo-locksetup" mode. 176 */ 177 enum rdtgrp_mode { 178 RDT_MODE_SHAREABLE = 0, 179 RDT_MODE_EXCLUSIVE, 180 RDT_MODE_PSEUDO_LOCKSETUP, 181 RDT_MODE_PSEUDO_LOCKED, 182 183 /* Must be last */ 184 RDT_NUM_MODES, 185 }; 186 187 /** 188 * struct mongroup - store mon group's data in resctrl fs. 189 * @mon_data_kn: kernfs node for the mon_data directory 190 * @parent: parent rdtgrp 191 * @crdtgrp_list: child rdtgroup node list 192 * @rmid: rmid for this rdtgroup 193 */ 194 struct mongroup { 195 struct kernfs_node *mon_data_kn; 196 struct rdtgroup *parent; 197 struct list_head crdtgrp_list; 198 u32 rmid; 199 }; 200 201 /** 202 * struct rdtgroup - store rdtgroup's data in resctrl file system. 203 * @kn: kernfs node 204 * @rdtgroup_list: linked list for all rdtgroups 205 * @closid: closid for this rdtgroup 206 * @cpu_mask: CPUs assigned to this rdtgroup 207 * @flags: status bits 208 * @waitcount: how many cpus expect to find this 209 * group when they acquire rdtgroup_mutex 210 * @type: indicates type of this rdtgroup - either 211 * monitor only or ctrl_mon group 212 * @mon: mongroup related data 213 * @mode: mode of resource group 214 * @mba_mbps_event: input monitoring event id when mba_sc is enabled 215 * @plr: pseudo-locked region 216 */ 217 struct rdtgroup { 218 struct kernfs_node *kn; 219 struct list_head rdtgroup_list; 220 u32 closid; 221 struct cpumask cpu_mask; 222 int flags; 223 atomic_t waitcount; 224 enum rdt_group_type type; 225 struct mongroup mon; 226 enum rdtgrp_mode mode; 227 enum resctrl_event_id mba_mbps_event; 228 struct pseudo_lock_region *plr; 229 }; 230 231 /* rdtgroup.flags */ 232 #define RDT_DELETED 1 233 234 /* rftype.flags */ 235 #define RFTYPE_FLAGS_CPUS_LIST 1 236 237 /* 238 * Define the file type flags for base and info directories. 239 */ 240 #define RFTYPE_INFO BIT(0) 241 242 #define RFTYPE_BASE BIT(1) 243 244 #define RFTYPE_CTRL BIT(4) 245 246 #define RFTYPE_MON BIT(5) 247 248 #define RFTYPE_TOP BIT(6) 249 250 #define RFTYPE_RES_CACHE BIT(8) 251 252 #define RFTYPE_RES_MB BIT(9) 253 254 #define RFTYPE_DEBUG BIT(10) 255 256 #define RFTYPE_ASSIGN_CONFIG BIT(11) 257 258 #define RFTYPE_RES_PERF_PKG BIT(12) 259 260 #define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) 261 262 #define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) 263 264 #define RFTYPE_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP) 265 266 #define RFTYPE_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL) 267 268 #define RFTYPE_MON_BASE (RFTYPE_BASE | RFTYPE_MON) 269 270 /* List of all resource groups */ 271 extern struct list_head rdt_all_groups; 272 273 extern int max_name_width; 274 275 /** 276 * struct rftype - describe each file in the resctrl file system 277 * @name: File name 278 * @mode: Access mode 279 * @kf_ops: File operations 280 * @flags: File specific RFTYPE_FLAGS_* flags 281 * @fflags: File specific RFTYPE_* flags 282 * @seq_show: Show content of the file 283 * @write: Write to the file 284 */ 285 struct rftype { 286 char *name; 287 umode_t mode; 288 const struct kernfs_ops *kf_ops; 289 unsigned long flags; 290 unsigned long fflags; 291 292 int (*seq_show)(struct kernfs_open_file *of, 293 struct seq_file *sf, void *v); 294 /* 295 * write() is the generic write callback which maps directly to 296 * kernfs write operation and overrides all other operations. 297 * Maximum write size is determined by ->max_write_len. 298 */ 299 ssize_t (*write)(struct kernfs_open_file *of, 300 char *buf, size_t nbytes, loff_t off); 301 }; 302 303 /** 304 * struct mbm_state - status for each MBM counter in each domain 305 * @prev_bw_bytes: Previous bytes value read for bandwidth calculation 306 * @prev_bw: The most recent bandwidth in MBps 307 */ 308 struct mbm_state { 309 u64 prev_bw_bytes; 310 u32 prev_bw; 311 }; 312 313 extern struct mutex rdtgroup_mutex; 314 315 static inline const char *rdt_kn_name(const struct kernfs_node *kn) 316 { 317 return rcu_dereference_check(kn->name, lockdep_is_held(&rdtgroup_mutex)); 318 } 319 320 extern struct rdtgroup rdtgroup_default; 321 322 extern struct dentry *debugfs_resctrl; 323 324 extern enum resctrl_event_id mba_mbps_default_event; 325 326 void rdt_last_cmd_clear(void); 327 328 void rdt_last_cmd_puts(const char *s); 329 330 __printf(1, 2) 331 void rdt_last_cmd_printf(const char *fmt, ...); 332 333 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); 334 335 void rdtgroup_kn_unlock(struct kernfs_node *kn); 336 337 int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name); 338 339 int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, 340 umode_t mask); 341 342 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, 343 char *buf, size_t nbytes, loff_t off); 344 345 int rdtgroup_schemata_show(struct kernfs_open_file *of, 346 struct seq_file *s, void *v); 347 348 ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, 349 char *buf, size_t nbytes, loff_t off); 350 351 int rdtgroup_mba_mbps_event_show(struct kernfs_open_file *of, 352 struct seq_file *s, void *v); 353 354 bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_ctrl_domain *d, 355 unsigned long cbm, int closid, bool exclusive); 356 357 unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, 358 unsigned long cbm); 359 360 enum rdtgrp_mode rdtgroup_mode_by_closid(int closid); 361 362 int rdtgroup_tasks_assigned(struct rdtgroup *r); 363 364 int closids_supported(void); 365 366 void closid_free(int closid); 367 368 int alloc_rmid(u32 closid); 369 370 void free_rmid(u32 closid, u32 rmid); 371 372 int resctrl_l3_mon_resource_init(void); 373 374 void resctrl_l3_mon_resource_exit(void); 375 376 void mon_event_count(void *info); 377 378 int rdtgroup_mondata_show(struct seq_file *m, void *arg); 379 380 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, 381 struct rdt_domain_hdr *hdr, struct rdtgroup *rdtgrp, 382 cpumask_t *cpumask, struct mon_evt *evt, int first); 383 384 void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, 385 unsigned long delay_ms, 386 int exclude_cpu); 387 388 void mbm_handle_overflow(struct work_struct *work); 389 390 bool is_mba_sc(struct rdt_resource *r); 391 392 void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, 393 int exclude_cpu); 394 395 void cqm_handle_limbo(struct work_struct *work); 396 397 bool has_busy_rmid(struct rdt_l3_mon_domain *d); 398 399 void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); 400 401 void resctrl_file_fflags_init(const char *config, unsigned long fflags); 402 403 void rdt_staged_configs_clear(void); 404 405 bool closid_allocated(unsigned int closid); 406 407 bool closid_alloc_fixed(u32 closid); 408 409 int resctrl_find_cleanest_closid(void); 410 411 void *rdt_kn_parent_priv(struct kernfs_node *kn); 412 413 int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, struct seq_file *s, void *v); 414 415 ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, 416 size_t nbytes, loff_t off); 417 418 void resctrl_bmec_files_show(struct rdt_resource *r, struct kernfs_node *l3_mon_kn, 419 bool show); 420 421 int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, void *v); 422 423 int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, 424 void *v); 425 426 void rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp); 427 428 void rdtgroup_unassign_cntrs(struct rdtgroup *rdtgrp); 429 430 int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v); 431 432 ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes, 433 loff_t off); 434 435 int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, 436 struct seq_file *s, void *v); 437 438 ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf, 439 size_t nbytes, loff_t off); 440 441 int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, void *v); 442 443 ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, size_t nbytes, 444 loff_t off); 445 int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v); 446 447 int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid); 448 449 enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type); 450 451 ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, 452 size_t nbytes, loff_t off); 453 454 const char *rdtgroup_name_by_closid(u32 closid); 455 int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, 456 void *v); 457 ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, 458 size_t nbytes, loff_t off); 459 u32 resctrl_io_alloc_closid(struct rdt_resource *r); 460 461 #ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK 462 int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp); 463 464 int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp); 465 466 bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm); 467 468 bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d); 469 470 int rdt_pseudo_lock_init(void); 471 472 void rdt_pseudo_lock_release(void); 473 474 int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp); 475 476 void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp); 477 478 #else 479 static inline int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) 480 { 481 return -EOPNOTSUPP; 482 } 483 484 static inline int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) 485 { 486 return -EOPNOTSUPP; 487 } 488 489 static inline bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) 490 { 491 return false; 492 } 493 494 static inline bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) 495 { 496 return false; 497 } 498 499 static inline int rdt_pseudo_lock_init(void) { return 0; } 500 static inline void rdt_pseudo_lock_release(void) { } 501 static inline int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) 502 { 503 return -EOPNOTSUPP; 504 } 505 506 static inline void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) { } 507 #endif /* CONFIG_RESCTRL_FS_PSEUDO_LOCK */ 508 509 #endif /* _FS_RESCTRL_INTERNAL_H */ 510