1 /* SPDX-License-Identifier: GPL-2.0 */ 2 // Copyright (C) 2025 Arm Ltd. 3 4 #ifndef MPAM_INTERNAL_H 5 #define MPAM_INTERNAL_H 6 7 #include <linux/arm_mpam.h> 8 #include <linux/atomic.h> 9 #include <linux/bitmap.h> 10 #include <linux/cpumask.h> 11 #include <linux/io.h> 12 #include <linux/llist.h> 13 #include <linux/mutex.h> 14 #include <linux/srcu.h> 15 #include <linux/spinlock.h> 16 #include <linux/srcu.h> 17 #include <linux/types.h> 18 19 #define MPAM_MSC_MAX_NUM_RIS 16 20 21 struct platform_device; 22 23 /* 24 * Structures protected by SRCU may not be freed for a surprising amount of 25 * time (especially if perf is running). To ensure the MPAM error interrupt can 26 * tear down all the structures, build a list of objects that can be garbage 27 * collected once synchronize_srcu() has returned. 28 * If pdev is non-NULL, use devm_kfree(). 29 */ 30 struct mpam_garbage { 31 /* member of mpam_garbage */ 32 struct llist_node llist; 33 34 void *to_free; 35 struct platform_device *pdev; 36 }; 37 38 struct mpam_msc { 39 /* member of mpam_all_msc */ 40 struct list_head all_msc_list; 41 42 int id; 43 struct platform_device *pdev; 44 45 /* Not modified after mpam_is_enabled() becomes true */ 46 enum mpam_msc_iface iface; 47 u32 nrdy_usec; 48 cpumask_t accessibility; 49 atomic_t online_refs; 50 51 /* 52 * probe_lock is only taken during discovery. After discovery these 53 * properties become read-only and the lists are protected by SRCU. 54 */ 55 struct mutex probe_lock; 56 bool probed; 57 u16 partid_max; 58 u8 pmg_max; 59 unsigned long ris_idxs; 60 u32 ris_max; 61 62 /* mpam_msc_ris of this component */ 63 struct list_head ris; 64 65 /* 66 * part_sel_lock protects access to the MSC hardware registers that are 67 * affected by MPAMCFG_PART_SEL. (including the ID registers that vary 68 * by RIS). 69 * If needed, take msc->probe_lock first. 70 */ 71 struct mutex part_sel_lock; 72 73 /* 74 * mon_sel_lock protects access to the MSC hardware registers that are 75 * affected by MPAMCFG_MON_SEL, and the mbwu_state. 76 * Access to mon_sel is needed from both process and interrupt contexts, 77 * but is complicated by firmware-backed platforms that can't make any 78 * access unless they can sleep. 79 * Always use the mpam_mon_sel_lock() helpers. 80 * Accesses to mon_sel need to be able to fail if they occur in the wrong 81 * context. 82 * If needed, take msc->probe_lock first. 83 */ 84 raw_spinlock_t _mon_sel_lock; 85 unsigned long _mon_sel_flags; 86 87 void __iomem *mapped_hwpage; 88 size_t mapped_hwpage_sz; 89 90 struct mpam_garbage garbage; 91 }; 92 93 /* Returning false here means accesses to mon_sel must fail and report an error. */ 94 static inline bool __must_check mpam_mon_sel_lock(struct mpam_msc *msc) 95 { 96 /* Locking will require updating to support a firmware backed interface */ 97 if (WARN_ON_ONCE(msc->iface != MPAM_IFACE_MMIO)) 98 return false; 99 100 raw_spin_lock_irqsave(&msc->_mon_sel_lock, msc->_mon_sel_flags); 101 return true; 102 } 103 104 static inline void mpam_mon_sel_unlock(struct mpam_msc *msc) 105 { 106 raw_spin_unlock_irqrestore(&msc->_mon_sel_lock, msc->_mon_sel_flags); 107 } 108 109 static inline void mpam_mon_sel_lock_held(struct mpam_msc *msc) 110 { 111 lockdep_assert_held_once(&msc->_mon_sel_lock); 112 } 113 114 static inline void mpam_mon_sel_lock_init(struct mpam_msc *msc) 115 { 116 raw_spin_lock_init(&msc->_mon_sel_lock); 117 } 118 119 /* Bits for mpam features bitmaps */ 120 enum mpam_device_features { 121 mpam_feat_cpor_part, 122 mpam_feat_mbw_part, 123 mpam_feat_mbw_min, 124 mpam_feat_mbw_max, 125 mpam_feat_msmon, 126 mpam_feat_msmon_csu, 127 mpam_feat_msmon_csu_hw_nrdy, 128 mpam_feat_msmon_mbwu, 129 mpam_feat_msmon_mbwu_hw_nrdy, 130 MPAM_FEATURE_LAST 131 }; 132 133 struct mpam_props { 134 DECLARE_BITMAP(features, MPAM_FEATURE_LAST); 135 136 u16 cpbm_wd; 137 u16 mbw_pbm_bits; 138 u16 bwa_wd; 139 u16 num_csu_mon; 140 u16 num_mbwu_mon; 141 }; 142 143 #define mpam_has_feature(_feat, x) test_bit(_feat, (x)->features) 144 #define mpam_set_feature(_feat, x) set_bit(_feat, (x)->features) 145 #define mpam_clear_feature(_feat, x) clear_bit(_feat, (x)->features) 146 147 struct mpam_class { 148 /* mpam_components in this class */ 149 struct list_head components; 150 151 cpumask_t affinity; 152 153 struct mpam_props props; 154 u32 nrdy_usec; 155 u8 level; 156 enum mpam_class_types type; 157 158 /* member of mpam_classes */ 159 struct list_head classes_list; 160 161 struct mpam_garbage garbage; 162 }; 163 164 struct mpam_component { 165 u32 comp_id; 166 167 /* mpam_vmsc in this component */ 168 struct list_head vmsc; 169 170 cpumask_t affinity; 171 172 /* member of mpam_class:components */ 173 struct list_head class_list; 174 175 /* parent: */ 176 struct mpam_class *class; 177 178 struct mpam_garbage garbage; 179 }; 180 181 struct mpam_vmsc { 182 /* member of mpam_component:vmsc_list */ 183 struct list_head comp_list; 184 185 /* mpam_msc_ris in this vmsc */ 186 struct list_head ris; 187 188 struct mpam_props props; 189 190 /* All RIS in this vMSC are members of this MSC */ 191 struct mpam_msc *msc; 192 193 /* parent: */ 194 struct mpam_component *comp; 195 196 struct mpam_garbage garbage; 197 }; 198 199 struct mpam_msc_ris { 200 u8 ris_idx; 201 u64 idr; 202 struct mpam_props props; 203 bool in_reset_state; 204 205 cpumask_t affinity; 206 207 /* member of mpam_vmsc:ris */ 208 struct list_head vmsc_list; 209 210 /* member of mpam_msc:ris */ 211 struct list_head msc_list; 212 213 /* parent: */ 214 struct mpam_vmsc *vmsc; 215 216 struct mpam_garbage garbage; 217 }; 218 219 /* List of all classes - protected by srcu*/ 220 extern struct srcu_struct mpam_srcu; 221 extern struct list_head mpam_classes; 222 223 /* System wide partid/pmg values */ 224 extern u16 mpam_partid_max; 225 extern u8 mpam_pmg_max; 226 227 /* Scheduled work callback to enable mpam once all MSC have been probed */ 228 void mpam_enable(struct work_struct *work); 229 void mpam_disable(struct work_struct *work); 230 231 int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, 232 cpumask_t *affinity); 233 234 /* 235 * MPAM MSCs have the following register layout. See: 236 * Arm Memory System Resource Partitioning and Monitoring (MPAM) System 237 * Component Specification. 238 * https://developer.arm.com/documentation/ihi0099/aa/ 239 */ 240 #define MPAM_ARCHITECTURE_V1 0x10 241 242 /* Memory mapped control pages */ 243 /* ID Register offsets in the memory mapped page */ 244 #define MPAMF_IDR 0x0000 /* features id register */ 245 #define MPAMF_IIDR 0x0018 /* implementer id register */ 246 #define MPAMF_AIDR 0x0020 /* architectural id register */ 247 #define MPAMF_IMPL_IDR 0x0028 /* imp-def partitioning */ 248 #define MPAMF_CPOR_IDR 0x0030 /* cache-portion partitioning */ 249 #define MPAMF_CCAP_IDR 0x0038 /* cache-capacity partitioning */ 250 #define MPAMF_MBW_IDR 0x0040 /* mem-bw partitioning */ 251 #define MPAMF_PRI_IDR 0x0048 /* priority partitioning */ 252 #define MPAMF_MSMON_IDR 0x0080 /* performance monitoring features */ 253 #define MPAMF_CSUMON_IDR 0x0088 /* cache-usage monitor */ 254 #define MPAMF_MBWUMON_IDR 0x0090 /* mem-bw usage monitor */ 255 #define MPAMF_PARTID_NRW_IDR 0x0050 /* partid-narrowing */ 256 257 /* Configuration and Status Register offsets in the memory mapped page */ 258 #define MPAMCFG_PART_SEL 0x0100 /* partid to configure */ 259 #define MPAMCFG_CPBM 0x1000 /* cache-portion config */ 260 #define MPAMCFG_CMAX 0x0108 /* cache-capacity config */ 261 #define MPAMCFG_CMIN 0x0110 /* cache-capacity config */ 262 #define MPAMCFG_CASSOC 0x0118 /* cache-associativity config */ 263 #define MPAMCFG_MBW_MIN 0x0200 /* min mem-bw config */ 264 #define MPAMCFG_MBW_MAX 0x0208 /* max mem-bw config */ 265 #define MPAMCFG_MBW_WINWD 0x0220 /* mem-bw accounting window config */ 266 #define MPAMCFG_MBW_PBM 0x2000 /* mem-bw portion bitmap config */ 267 #define MPAMCFG_PRI 0x0400 /* priority partitioning config */ 268 #define MPAMCFG_MBW_PROP 0x0500 /* mem-bw stride config */ 269 #define MPAMCFG_INTPARTID 0x0600 /* partid-narrowing config */ 270 271 #define MSMON_CFG_MON_SEL 0x0800 /* monitor selector */ 272 #define MSMON_CFG_CSU_FLT 0x0810 /* cache-usage monitor filter */ 273 #define MSMON_CFG_CSU_CTL 0x0818 /* cache-usage monitor config */ 274 #define MSMON_CFG_MBWU_FLT 0x0820 /* mem-bw monitor filter */ 275 #define MSMON_CFG_MBWU_CTL 0x0828 /* mem-bw monitor config */ 276 #define MSMON_CSU 0x0840 /* current cache-usage */ 277 #define MSMON_CSU_CAPTURE 0x0848 /* last cache-usage value captured */ 278 #define MSMON_MBWU 0x0860 /* current mem-bw usage value */ 279 #define MSMON_MBWU_CAPTURE 0x0868 /* last mem-bw value captured */ 280 #define MSMON_MBWU_L 0x0880 /* current long mem-bw usage value */ 281 #define MSMON_MBWU_L_CAPTURE 0x0890 /* last long mem-bw value captured */ 282 #define MSMON_CAPT_EVNT 0x0808 /* signal a capture event */ 283 #define MPAMF_ESR 0x00F8 /* error status register */ 284 #define MPAMF_ECR 0x00F0 /* error control register */ 285 286 /* MPAMF_IDR - MPAM features ID register */ 287 #define MPAMF_IDR_PARTID_MAX GENMASK(15, 0) 288 #define MPAMF_IDR_PMG_MAX GENMASK(23, 16) 289 #define MPAMF_IDR_HAS_CCAP_PART BIT(24) 290 #define MPAMF_IDR_HAS_CPOR_PART BIT(25) 291 #define MPAMF_IDR_HAS_MBW_PART BIT(26) 292 #define MPAMF_IDR_HAS_PRI_PART BIT(27) 293 #define MPAMF_IDR_EXT BIT(28) 294 #define MPAMF_IDR_HAS_IMPL_IDR BIT(29) 295 #define MPAMF_IDR_HAS_MSMON BIT(30) 296 #define MPAMF_IDR_HAS_PARTID_NRW BIT(31) 297 #define MPAMF_IDR_HAS_RIS BIT(32) 298 #define MPAMF_IDR_HAS_EXTD_ESR BIT(38) 299 #define MPAMF_IDR_HAS_ESR BIT(39) 300 #define MPAMF_IDR_RIS_MAX GENMASK(59, 56) 301 302 /* MPAMF_MSMON_IDR - MPAM performance monitoring ID register */ 303 #define MPAMF_MSMON_IDR_MSMON_CSU BIT(16) 304 #define MPAMF_MSMON_IDR_MSMON_MBWU BIT(17) 305 #define MPAMF_MSMON_IDR_HAS_LOCAL_CAPT_EVNT BIT(31) 306 307 /* MPAMF_CPOR_IDR - MPAM features cache portion partitioning ID register */ 308 #define MPAMF_CPOR_IDR_CPBM_WD GENMASK(15, 0) 309 310 /* MPAMF_CCAP_IDR - MPAM features cache capacity partitioning ID register */ 311 #define MPAMF_CCAP_IDR_CMAX_WD GENMASK(5, 0) 312 #define MPAMF_CCAP_IDR_CASSOC_WD GENMASK(12, 8) 313 #define MPAMF_CCAP_IDR_HAS_CASSOC BIT(28) 314 #define MPAMF_CCAP_IDR_HAS_CMIN BIT(29) 315 #define MPAMF_CCAP_IDR_NO_CMAX BIT(30) 316 #define MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM BIT(31) 317 318 /* MPAMF_MBW_IDR - MPAM features memory bandwidth partitioning ID register */ 319 #define MPAMF_MBW_IDR_BWA_WD GENMASK(5, 0) 320 #define MPAMF_MBW_IDR_HAS_MIN BIT(10) 321 #define MPAMF_MBW_IDR_HAS_MAX BIT(11) 322 #define MPAMF_MBW_IDR_HAS_PBM BIT(12) 323 #define MPAMF_MBW_IDR_HAS_PROP BIT(13) 324 #define MPAMF_MBW_IDR_WINDWR BIT(14) 325 #define MPAMF_MBW_IDR_BWPBM_WD GENMASK(28, 16) 326 327 /* MPAMF_PRI_IDR - MPAM features priority partitioning ID register */ 328 #define MPAMF_PRI_IDR_HAS_INTPRI BIT(0) 329 #define MPAMF_PRI_IDR_INTPRI_0_IS_LOW BIT(1) 330 #define MPAMF_PRI_IDR_INTPRI_WD GENMASK(9, 4) 331 #define MPAMF_PRI_IDR_HAS_DSPRI BIT(16) 332 #define MPAMF_PRI_IDR_DSPRI_0_IS_LOW BIT(17) 333 #define MPAMF_PRI_IDR_DSPRI_WD GENMASK(25, 20) 334 335 /* MPAMF_CSUMON_IDR - MPAM cache storage usage monitor ID register */ 336 #define MPAMF_CSUMON_IDR_NUM_MON GENMASK(15, 0) 337 #define MPAMF_CSUMON_IDR_HAS_OFLOW_CAPT BIT(24) 338 #define MPAMF_CSUMON_IDR_HAS_CEVNT_OFLW BIT(25) 339 #define MPAMF_CSUMON_IDR_HAS_OFSR BIT(26) 340 #define MPAMF_CSUMON_IDR_HAS_OFLOW_LNKG BIT(27) 341 #define MPAMF_CSUMON_IDR_HAS_XCL BIT(29) 342 #define MPAMF_CSUMON_IDR_CSU_RO BIT(30) 343 #define MPAMF_CSUMON_IDR_HAS_CAPTURE BIT(31) 344 345 /* MPAMF_MBWUMON_IDR - MPAM memory bandwidth usage monitor ID register */ 346 #define MPAMF_MBWUMON_IDR_NUM_MON GENMASK(15, 0) 347 #define MPAMF_MBWUMON_IDR_HAS_RWBW BIT(28) 348 #define MPAMF_MBWUMON_IDR_LWD BIT(29) 349 #define MPAMF_MBWUMON_IDR_HAS_LONG BIT(30) 350 #define MPAMF_MBWUMON_IDR_HAS_CAPTURE BIT(31) 351 352 /* MPAMF_PARTID_NRW_IDR - MPAM PARTID narrowing ID register */ 353 #define MPAMF_PARTID_NRW_IDR_INTPARTID_MAX GENMASK(15, 0) 354 355 /* MPAMF_IIDR - MPAM implementation ID register */ 356 #define MPAMF_IIDR_IMPLEMENTER GENMASK(11, 0) 357 #define MPAMF_IIDR_REVISION GENMASK(15, 12) 358 #define MPAMF_IIDR_VARIANT GENMASK(19, 16) 359 #define MPAMF_IIDR_PRODUCTID GENMASK(31, 20) 360 361 /* MPAMF_AIDR - MPAM architecture ID register */ 362 #define MPAMF_AIDR_ARCH_MINOR_REV GENMASK(3, 0) 363 #define MPAMF_AIDR_ARCH_MAJOR_REV GENMASK(7, 4) 364 365 /* MPAMCFG_PART_SEL - MPAM partition configuration selection register */ 366 #define MPAMCFG_PART_SEL_PARTID_SEL GENMASK(15, 0) 367 #define MPAMCFG_PART_SEL_INTERNAL BIT(16) 368 #define MPAMCFG_PART_SEL_RIS GENMASK(27, 24) 369 370 /* MPAMCFG_CASSOC - MPAM cache maximum associativity partition configuration register */ 371 #define MPAMCFG_CASSOC_CASSOC GENMASK(15, 0) 372 373 /* MPAMCFG_CMAX - MPAM cache capacity configuration register */ 374 #define MPAMCFG_CMAX_SOFTLIM BIT(31) 375 #define MPAMCFG_CMAX_CMAX GENMASK(15, 0) 376 377 /* MPAMCFG_CMIN - MPAM cache capacity configuration register */ 378 #define MPAMCFG_CMIN_CMIN GENMASK(15, 0) 379 380 /* 381 * MPAMCFG_MBW_MIN - MPAM memory minimum bandwidth partitioning configuration 382 * register 383 */ 384 #define MPAMCFG_MBW_MIN_MIN GENMASK(15, 0) 385 386 /* 387 * MPAMCFG_MBW_MAX - MPAM memory maximum bandwidth partitioning configuration 388 * register 389 */ 390 #define MPAMCFG_MBW_MAX_MAX GENMASK(15, 0) 391 #define MPAMCFG_MBW_MAX_HARDLIM BIT(31) 392 393 /* 394 * MPAMCFG_MBW_WINWD - MPAM memory bandwidth partitioning window width 395 * register 396 */ 397 #define MPAMCFG_MBW_WINWD_US_FRAC GENMASK(7, 0) 398 #define MPAMCFG_MBW_WINWD_US_INT GENMASK(23, 8) 399 400 /* MPAMCFG_PRI - MPAM priority partitioning configuration register */ 401 #define MPAMCFG_PRI_INTPRI GENMASK(15, 0) 402 #define MPAMCFG_PRI_DSPRI GENMASK(31, 16) 403 404 /* 405 * MPAMCFG_MBW_PROP - Memory bandwidth proportional stride partitioning 406 * configuration register 407 */ 408 #define MPAMCFG_MBW_PROP_STRIDEM1 GENMASK(15, 0) 409 #define MPAMCFG_MBW_PROP_EN BIT(31) 410 411 /* 412 * MPAMCFG_INTPARTID - MPAM internal partition narrowing configuration register 413 */ 414 #define MPAMCFG_INTPARTID_INTPARTID GENMASK(15, 0) 415 #define MPAMCFG_INTPARTID_INTERNAL BIT(16) 416 417 /* MSMON_CFG_MON_SEL - Memory system performance monitor selection register */ 418 #define MSMON_CFG_MON_SEL_MON_SEL GENMASK(15, 0) 419 #define MSMON_CFG_MON_SEL_RIS GENMASK(27, 24) 420 421 /* MPAMF_ESR - MPAM Error Status Register */ 422 #define MPAMF_ESR_PARTID_MON GENMASK(15, 0) 423 #define MPAMF_ESR_PMG GENMASK(23, 16) 424 #define MPAMF_ESR_ERRCODE GENMASK(27, 24) 425 #define MPAMF_ESR_OVRWR BIT(31) 426 #define MPAMF_ESR_RIS GENMASK(35, 32) 427 428 /* MPAMF_ECR - MPAM Error Control Register */ 429 #define MPAMF_ECR_INTEN BIT(0) 430 431 /* Error conditions in accessing memory mapped registers */ 432 #define MPAM_ERRCODE_NONE 0 433 #define MPAM_ERRCODE_PARTID_SEL_RANGE 1 434 #define MPAM_ERRCODE_REQ_PARTID_RANGE 2 435 #define MPAM_ERRCODE_MSMONCFG_ID_RANGE 3 436 #define MPAM_ERRCODE_REQ_PMG_RANGE 4 437 #define MPAM_ERRCODE_MONITOR_RANGE 5 438 #define MPAM_ERRCODE_INTPARTID_RANGE 6 439 #define MPAM_ERRCODE_UNEXPECTED_INTERNAL 7 440 #define MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL 8 441 #define MPAM_ERRCODE_RIS_NO_CONTROL 9 442 #define MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL 10 443 #define MPAM_ERRCODE_RIS_NO_MONITOR 11 444 445 /* 446 * MSMON_CFG_CSU_CTL - Memory system performance monitor configure cache storage 447 * usage monitor control register 448 * MSMON_CFG_MBWU_CTL - Memory system performance monitor configure memory 449 * bandwidth usage monitor control register 450 */ 451 #define MSMON_CFG_x_CTL_TYPE GENMASK(7, 0) 452 #define MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L BIT(15) 453 #define MSMON_CFG_x_CTL_MATCH_PARTID BIT(16) 454 #define MSMON_CFG_x_CTL_MATCH_PMG BIT(17) 455 #define MSMON_CFG_MBWU_CTL_SCLEN BIT(19) 456 #define MSMON_CFG_x_CTL_SUBTYPE GENMASK(22, 20) 457 #define MSMON_CFG_x_CTL_OFLOW_FRZ BIT(24) 458 #define MSMON_CFG_x_CTL_OFLOW_INTR BIT(25) 459 #define MSMON_CFG_x_CTL_OFLOW_STATUS BIT(26) 460 #define MSMON_CFG_x_CTL_CAPT_RESET BIT(27) 461 #define MSMON_CFG_x_CTL_CAPT_EVNT GENMASK(30, 28) 462 #define MSMON_CFG_x_CTL_EN BIT(31) 463 464 #define MSMON_CFG_MBWU_CTL_TYPE_MBWU 0x42 465 #define MSMON_CFG_CSU_CTL_TYPE_CSU 0x43 466 467 /* 468 * MSMON_CFG_CSU_FLT - Memory system performance monitor configure cache storage 469 * usage monitor filter register 470 * MSMON_CFG_MBWU_FLT - Memory system performance monitor configure memory 471 * bandwidth usage monitor filter register 472 */ 473 #define MSMON_CFG_x_FLT_PARTID GENMASK(15, 0) 474 #define MSMON_CFG_x_FLT_PMG GENMASK(23, 16) 475 476 #define MSMON_CFG_MBWU_FLT_RWBW GENMASK(31, 30) 477 #define MSMON_CFG_CSU_FLT_XCL BIT(31) 478 479 /* 480 * MSMON_CSU - Memory system performance monitor cache storage usage monitor 481 * register 482 * MSMON_CSU_CAPTURE - Memory system performance monitor cache storage usage 483 * capture register 484 * MSMON_MBWU - Memory system performance monitor memory bandwidth usage 485 * monitor register 486 * MSMON_MBWU_CAPTURE - Memory system performance monitor memory bandwidth usage 487 * capture register 488 */ 489 #define MSMON___VALUE GENMASK(30, 0) 490 #define MSMON___NRDY BIT(31) 491 #define MSMON___L_NRDY BIT(63) 492 #define MSMON___L_VALUE GENMASK(43, 0) 493 #define MSMON___LWD_VALUE GENMASK(62, 0) 494 495 /* 496 * MSMON_CAPT_EVNT - Memory system performance monitoring capture event 497 * generation register 498 */ 499 #define MSMON_CAPT_EVNT_NOW BIT(0) 500 501 #endif /* MPAM_INTERNAL_H */ 502