1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2016-2020 Arm Limited 3 // CMN-600 Coherent Mesh Network PMU driver 4 5 #include <linux/acpi.h> 6 #include <linux/bitfield.h> 7 #include <linux/bitops.h> 8 #include <linux/debugfs.h> 9 #include <linux/interrupt.h> 10 #include <linux/io.h> 11 #include <linux/io-64-nonatomic-lo-hi.h> 12 #include <linux/kernel.h> 13 #include <linux/list.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/perf_event.h> 17 #include <linux/platform_device.h> 18 #include <linux/slab.h> 19 #include <linux/sort.h> 20 21 /* Common register stuff */ 22 #define CMN_NODE_INFO 0x0000 23 #define CMN_NI_NODE_TYPE GENMASK_ULL(15, 0) 24 #define CMN_NI_NODE_ID GENMASK_ULL(31, 16) 25 #define CMN_NI_LOGICAL_ID GENMASK_ULL(47, 32) 26 27 #define CMN_NODEID_DEVID(reg) ((reg) & 3) 28 #define CMN_NODEID_EXT_DEVID(reg) ((reg) & 1) 29 #define CMN_NODEID_PID(reg) (((reg) >> 2) & 1) 30 #define CMN_NODEID_EXT_PID(reg) (((reg) >> 1) & 3) 31 #define CMN_NODEID_1x1_PID(reg) (((reg) >> 2) & 7) 32 #define CMN_NODEID_X(reg, bits) ((reg) >> (3 + (bits))) 33 #define CMN_NODEID_Y(reg, bits) (((reg) >> 3) & ((1U << (bits)) - 1)) 34 35 #define CMN_CHILD_INFO 0x0080 36 #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0) 37 #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) 38 39 #define CMN_CHILD_NODE_ADDR GENMASK(27, 0) 40 #define CMN_CHILD_NODE_EXTERNAL BIT(31) 41 42 #define CMN_MAX_DIMENSION 8 43 #define CMN_MAX_XPS (CMN_MAX_DIMENSION * CMN_MAX_DIMENSION) 44 #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4) 45 46 /* The CFG node has various info besides the discovery tree */ 47 #define CMN_CFGM_PERIPH_ID_2 0x0010 48 #define CMN_CFGM_PID2_REVISION GENMASK(7, 4) 49 50 #define CMN_CFGM_INFO_GLOBAL 0x900 51 #define CMN_INFO_MULTIPLE_DTM_EN BIT_ULL(63) 52 #define CMN_INFO_RSP_VC_NUM GENMASK_ULL(53, 52) 53 #define CMN_INFO_DAT_VC_NUM GENMASK_ULL(51, 50) 54 55 /* XPs also have some local topology info which has uses too */ 56 #define CMN_MXP__CONNECT_INFO_P0 0x0008 57 #define CMN_MXP__CONNECT_INFO_P1 0x0010 58 #define CMN_MXP__CONNECT_INFO_P2 0x0028 59 #define CMN_MXP__CONNECT_INFO_P3 0x0030 60 #define CMN_MXP__CONNECT_INFO_P4 0x0038 61 #define CMN_MXP__CONNECT_INFO_P5 0x0040 62 63 /* PMU registers occupy the 3rd 4KB page of each node's region */ 64 #define CMN_PMU_OFFSET 0x2000 65 66 /* For most nodes, this is all there is */ 67 #define CMN_PMU_EVENT_SEL 0x000 68 #define CMN_PMU_EVENTn_ID_SHIFT(n) ((n) * 8) 69 70 /* DTMs live in the PMU space of XP registers */ 71 #define CMN_DTM_WPn(n) (0x1A0 + (n) * 0x18) 72 #define CMN_DTM_WPn_CONFIG(n) (CMN_DTM_WPn(n) + 0x00) 73 #define CMN_DTM_WPn_CONFIG_WP_DEV_SEL2 GENMASK_ULL(18,17) 74 #define CMN_DTM_WPn_CONFIG_WP_COMBINE BIT(6) 75 #define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE BIT(5) 76 #define CMN_DTM_WPn_CONFIG_WP_GRP BIT(4) 77 #define CMN_DTM_WPn_CONFIG_WP_CHN_SEL GENMASK_ULL(3, 1) 78 #define CMN_DTM_WPn_CONFIG_WP_DEV_SEL BIT(0) 79 #define CMN_DTM_WPn_VAL(n) (CMN_DTM_WPn(n) + 0x08) 80 #define CMN_DTM_WPn_MASK(n) (CMN_DTM_WPn(n) + 0x10) 81 82 #define CMN_DTM_PMU_CONFIG 0x210 83 #define CMN__PMEVCNT0_INPUT_SEL GENMASK_ULL(37, 32) 84 #define CMN__PMEVCNT0_INPUT_SEL_WP 0x00 85 #define CMN__PMEVCNT0_INPUT_SEL_XP 0x04 86 #define CMN__PMEVCNT0_INPUT_SEL_DEV 0x10 87 #define CMN__PMEVCNT0_GLOBAL_NUM GENMASK_ULL(18, 16) 88 #define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n) ((n) * 4) 89 #define CMN__PMEVCNT_PAIRED(n) BIT(4 + (n)) 90 #define CMN__PMEVCNT23_COMBINED BIT(2) 91 #define CMN__PMEVCNT01_COMBINED BIT(1) 92 #define CMN_DTM_PMU_CONFIG_PMU_EN BIT(0) 93 94 #define CMN_DTM_PMEVCNT 0x220 95 96 #define CMN_DTM_PMEVCNTSR 0x240 97 98 #define CMN_DTM_UNIT_INFO 0x0910 99 100 #define CMN_DTM_NUM_COUNTERS 4 101 /* Want more local counters? Why not replicate the whole DTM! Ugh... */ 102 #define CMN_DTM_OFFSET(n) ((n) * 0x200) 103 104 /* The DTC node is where the magic happens */ 105 #define CMN_DT_DTC_CTL 0x0a00 106 #define CMN_DT_DTC_CTL_DT_EN BIT(0) 107 108 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */ 109 #define _CMN_DT_CNT_REG(n) ((((n) / 2) * 4 + (n) % 2) * 4) 110 #define CMN_DT_PMEVCNT(n) (CMN_PMU_OFFSET + _CMN_DT_CNT_REG(n)) 111 #define CMN_DT_PMCCNTR (CMN_PMU_OFFSET + 0x40) 112 113 #define CMN_DT_PMEVCNTSR(n) (CMN_PMU_OFFSET + 0x50 + _CMN_DT_CNT_REG(n)) 114 #define CMN_DT_PMCCNTRSR (CMN_PMU_OFFSET + 0x90) 115 116 #define CMN_DT_PMCR (CMN_PMU_OFFSET + 0x100) 117 #define CMN_DT_PMCR_PMU_EN BIT(0) 118 #define CMN_DT_PMCR_CNTR_RST BIT(5) 119 #define CMN_DT_PMCR_OVFL_INTR_EN BIT(6) 120 121 #define CMN_DT_PMOVSR (CMN_PMU_OFFSET + 0x118) 122 #define CMN_DT_PMOVSR_CLR (CMN_PMU_OFFSET + 0x120) 123 124 #define CMN_DT_PMSSR (CMN_PMU_OFFSET + 0x128) 125 #define CMN_DT_PMSSR_SS_STATUS(n) BIT(n) 126 127 #define CMN_DT_PMSRR (CMN_PMU_OFFSET + 0x130) 128 #define CMN_DT_PMSRR_SS_REQ BIT(0) 129 130 #define CMN_DT_NUM_COUNTERS 8 131 #define CMN_MAX_DTCS 4 132 133 /* 134 * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles, 135 * so throwing away one bit to make overflow handling easy is no big deal. 136 */ 137 #define CMN_COUNTER_INIT 0x80000000 138 /* Similarly for the 40-bit cycle counter */ 139 #define CMN_CC_INIT 0x8000000000ULL 140 141 142 /* Event attributes */ 143 #define CMN_CONFIG_TYPE GENMASK_ULL(15, 0) 144 #define CMN_CONFIG_EVENTID GENMASK_ULL(23, 16) 145 #define CMN_CONFIG_OCCUPID GENMASK_ULL(27, 24) 146 #define CMN_CONFIG_BYNODEID BIT_ULL(31) 147 #define CMN_CONFIG_NODEID GENMASK_ULL(47, 32) 148 149 #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config) 150 #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config) 151 #define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config) 152 #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config) 153 #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config) 154 155 #define CMN_CONFIG_WP_COMBINE GENMASK_ULL(27, 24) 156 #define CMN_CONFIG_WP_DEV_SEL GENMASK_ULL(50, 48) 157 #define CMN_CONFIG_WP_CHN_SEL GENMASK_ULL(55, 51) 158 #define CMN_CONFIG_WP_GRP BIT_ULL(56) 159 #define CMN_CONFIG_WP_EXCLUSIVE BIT_ULL(57) 160 #define CMN_CONFIG1_WP_VAL GENMASK_ULL(63, 0) 161 #define CMN_CONFIG2_WP_MASK GENMASK_ULL(63, 0) 162 163 #define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config) 164 #define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config) 165 #define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config) 166 #define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config) 167 #define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config) 168 #define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1) 169 #define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2) 170 171 /* Made-up event IDs for watchpoint direction */ 172 #define CMN_WP_UP 0 173 #define CMN_WP_DOWN 2 174 175 176 enum cmn_model { 177 CMN_ANY = -1, 178 CMN600 = 1, 179 CI700 = 2, 180 }; 181 182 /* CMN-600 r0px shouldn't exist in silicon, thankfully */ 183 enum cmn_revision { 184 CMN600_R1P0, 185 CMN600_R1P1, 186 CMN600_R1P2, 187 CMN600_R1P3, 188 CMN600_R2P0, 189 CMN600_R3P0, 190 CMN600_R3P1, 191 CI700_R0P0 = 0, 192 CI700_R1P0, 193 CI700_R2P0, 194 }; 195 196 enum cmn_node_type { 197 CMN_TYPE_INVALID, 198 CMN_TYPE_DVM, 199 CMN_TYPE_CFG, 200 CMN_TYPE_DTC, 201 CMN_TYPE_HNI, 202 CMN_TYPE_HNF, 203 CMN_TYPE_XP, 204 CMN_TYPE_SBSX, 205 CMN_TYPE_MPAM_S, 206 CMN_TYPE_MPAM_NS, 207 CMN_TYPE_RNI, 208 CMN_TYPE_RND = 0xd, 209 CMN_TYPE_RNSAM = 0xf, 210 CMN_TYPE_MTSX, 211 CMN_TYPE_CXRA = 0x100, 212 CMN_TYPE_CXHA = 0x101, 213 CMN_TYPE_CXLA = 0x102, 214 /* Not a real node type */ 215 CMN_TYPE_WP = 0x7770 216 }; 217 218 struct arm_cmn_node { 219 void __iomem *pmu_base; 220 u16 id, logid; 221 enum cmn_node_type type; 222 223 int dtm; 224 union { 225 /* DN/HN-F/CXHA */ 226 struct { 227 u8 occupid_val; 228 u8 occupid_count; 229 }; 230 /* XP */ 231 u8 dtc; 232 }; 233 union { 234 u8 event[4]; 235 __le32 event_sel; 236 }; 237 }; 238 239 struct arm_cmn_dtm { 240 void __iomem *base; 241 u32 pmu_config_low; 242 union { 243 u8 input_sel[4]; 244 __le32 pmu_config_high; 245 }; 246 s8 wp_event[4]; 247 }; 248 249 struct arm_cmn_dtc { 250 void __iomem *base; 251 int irq; 252 int irq_friend; 253 bool cc_active; 254 255 struct perf_event *counters[CMN_DT_NUM_COUNTERS]; 256 struct perf_event *cycles; 257 }; 258 259 #define CMN_STATE_DISABLED BIT(0) 260 #define CMN_STATE_TXN BIT(1) 261 262 struct arm_cmn { 263 struct device *dev; 264 void __iomem *base; 265 unsigned int state; 266 267 enum cmn_revision rev; 268 enum cmn_model model; 269 u8 mesh_x; 270 u8 mesh_y; 271 u16 num_xps; 272 u16 num_dns; 273 bool multi_dtm; 274 u8 ports_used; 275 struct { 276 unsigned int rsp_vc_num : 2; 277 unsigned int dat_vc_num : 2; 278 }; 279 280 struct arm_cmn_node *xps; 281 struct arm_cmn_node *dns; 282 283 struct arm_cmn_dtm *dtms; 284 struct arm_cmn_dtc *dtc; 285 unsigned int num_dtcs; 286 287 int cpu; 288 struct hlist_node cpuhp_node; 289 290 struct pmu pmu; 291 struct dentry *debug; 292 }; 293 294 #define to_cmn(p) container_of(p, struct arm_cmn, pmu) 295 296 static int arm_cmn_hp_state; 297 298 struct arm_cmn_nodeid { 299 u8 x; 300 u8 y; 301 u8 port; 302 u8 dev; 303 }; 304 305 static int arm_cmn_xyidbits(const struct arm_cmn *cmn) 306 { 307 int dim = max(cmn->mesh_x, cmn->mesh_y); 308 309 return dim > 4 ? 3 : 2; 310 } 311 312 static struct arm_cmn_nodeid arm_cmn_nid(const struct arm_cmn *cmn, u16 id) 313 { 314 struct arm_cmn_nodeid nid; 315 316 if (cmn->num_xps == 1) { 317 nid.x = 0; 318 nid.y = 0; 319 nid.port = CMN_NODEID_1x1_PID(id); 320 nid.dev = CMN_NODEID_DEVID(id); 321 } else { 322 int bits = arm_cmn_xyidbits(cmn); 323 324 nid.x = CMN_NODEID_X(id, bits); 325 nid.y = CMN_NODEID_Y(id, bits); 326 if (cmn->ports_used & 0xc) { 327 nid.port = CMN_NODEID_EXT_PID(id); 328 nid.dev = CMN_NODEID_EXT_DEVID(id); 329 } else { 330 nid.port = CMN_NODEID_PID(id); 331 nid.dev = CMN_NODEID_DEVID(id); 332 } 333 } 334 return nid; 335 } 336 337 static struct arm_cmn_node *arm_cmn_node_to_xp(const struct arm_cmn *cmn, 338 const struct arm_cmn_node *dn) 339 { 340 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 341 int xp_idx = cmn->mesh_x * nid.y + nid.x; 342 343 return cmn->xps + xp_idx; 344 } 345 static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn, 346 enum cmn_node_type type) 347 { 348 struct arm_cmn_node *dn; 349 350 for (dn = cmn->dns; dn->type; dn++) 351 if (dn->type == type) 352 return dn; 353 return NULL; 354 } 355 356 struct dentry *arm_cmn_debugfs; 357 358 #ifdef CONFIG_DEBUG_FS 359 static const char *arm_cmn_device_type(u8 type) 360 { 361 switch(type) { 362 case 0x01: return " RN-I |"; 363 case 0x02: return " RN-D |"; 364 case 0x04: return " RN-F_B |"; 365 case 0x05: return "RN-F_B_E|"; 366 case 0x06: return " RN-F_A |"; 367 case 0x07: return "RN-F_A_E|"; 368 case 0x08: return " HN-T |"; 369 case 0x09: return " HN-I |"; 370 case 0x0a: return " HN-D |"; 371 case 0x0c: return " SN-F |"; 372 case 0x0d: return " SBSX |"; 373 case 0x0e: return " HN-F |"; 374 case 0x0f: return " SN-F_E |"; 375 case 0x10: return " SN-F_D |"; 376 case 0x11: return " CXHA |"; 377 case 0x12: return " CXRA |"; 378 case 0x13: return " CXRH |"; 379 case 0x14: return " RN-F_D |"; 380 case 0x15: return "RN-F_D_E|"; 381 case 0x16: return " RN-F_C |"; 382 case 0x17: return "RN-F_C_E|"; 383 case 0x1c: return " MTSX |"; 384 default: return " |"; 385 } 386 } 387 388 static void arm_cmn_show_logid(struct seq_file *s, int x, int y, int p, int d) 389 { 390 struct arm_cmn *cmn = s->private; 391 struct arm_cmn_node *dn; 392 393 for (dn = cmn->dns; dn->type; dn++) { 394 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 395 396 if (dn->type == CMN_TYPE_XP) 397 continue; 398 /* Ignore the extra components that will overlap on some ports */ 399 if (dn->type < CMN_TYPE_HNI) 400 continue; 401 402 if (nid.x != x || nid.y != y || nid.port != p || nid.dev != d) 403 continue; 404 405 seq_printf(s, " #%-2d |", dn->logid); 406 return; 407 } 408 seq_puts(s, " |"); 409 } 410 411 static int arm_cmn_map_show(struct seq_file *s, void *data) 412 { 413 struct arm_cmn *cmn = s->private; 414 int x, y, p, pmax = fls(cmn->ports_used); 415 416 seq_puts(s, " X"); 417 for (x = 0; x < cmn->mesh_x; x++) 418 seq_printf(s, " %d ", x); 419 seq_puts(s, "\nY P D+"); 420 y = cmn->mesh_y; 421 while (y--) { 422 int xp_base = cmn->mesh_x * y; 423 u8 port[6][CMN_MAX_DIMENSION]; 424 425 for (x = 0; x < cmn->mesh_x; x++) 426 seq_puts(s, "--------+"); 427 428 seq_printf(s, "\n%d |", y); 429 for (x = 0; x < cmn->mesh_x; x++) { 430 struct arm_cmn_node *xp = cmn->xps + xp_base + x; 431 void __iomem *base = xp->pmu_base - CMN_PMU_OFFSET; 432 433 port[0][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P0); 434 port[1][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P1); 435 port[2][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P2); 436 port[3][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P3); 437 port[4][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P4); 438 port[5][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P5); 439 seq_printf(s, " XP #%-2d |", xp_base + x); 440 } 441 442 seq_puts(s, "\n |"); 443 for (x = 0; x < cmn->mesh_x; x++) { 444 u8 dtc = cmn->xps[xp_base + x].dtc; 445 446 if (dtc & (dtc - 1)) 447 seq_puts(s, " DTC ?? |"); 448 else 449 seq_printf(s, " DTC %ld |", __ffs(dtc)); 450 } 451 seq_puts(s, "\n |"); 452 for (x = 0; x < cmn->mesh_x; x++) 453 seq_puts(s, "........|"); 454 455 for (p = 0; p < pmax; p++) { 456 seq_printf(s, "\n %d |", p); 457 for (x = 0; x < cmn->mesh_x; x++) 458 seq_puts(s, arm_cmn_device_type(port[p][x])); 459 seq_puts(s, "\n 0|"); 460 for (x = 0; x < cmn->mesh_x; x++) 461 arm_cmn_show_logid(s, x, y, p, 0); 462 seq_puts(s, "\n 1|"); 463 for (x = 0; x < cmn->mesh_x; x++) 464 arm_cmn_show_logid(s, x, y, p, 1); 465 } 466 seq_puts(s, "\n-----+"); 467 } 468 for (x = 0; x < cmn->mesh_x; x++) 469 seq_puts(s, "--------+"); 470 seq_puts(s, "\n"); 471 return 0; 472 } 473 DEFINE_SHOW_ATTRIBUTE(arm_cmn_map); 474 475 static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) 476 { 477 const char *name = "map"; 478 479 if (id > 0) 480 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id); 481 if (!name) 482 return; 483 484 cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops); 485 } 486 #else 487 static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {} 488 #endif 489 490 struct arm_cmn_hw_event { 491 struct arm_cmn_node *dn; 492 u64 dtm_idx[2]; 493 unsigned int dtc_idx; 494 u8 dtcs_used; 495 u8 num_dns; 496 u8 dtm_offset; 497 }; 498 499 #define for_each_hw_dn(hw, dn, i) \ 500 for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++) 501 502 static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event) 503 { 504 BUILD_BUG_ON(sizeof(struct arm_cmn_hw_event) > offsetof(struct hw_perf_event, target)); 505 return (struct arm_cmn_hw_event *)&event->hw; 506 } 507 508 static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val) 509 { 510 x[pos / 32] |= (u64)val << ((pos % 32) * 2); 511 } 512 513 static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos) 514 { 515 return (x[pos / 32] >> ((pos % 32) * 2)) & 3; 516 } 517 518 struct arm_cmn_event_attr { 519 struct device_attribute attr; 520 enum cmn_model model; 521 enum cmn_node_type type; 522 u8 eventid; 523 u8 occupid; 524 }; 525 526 struct arm_cmn_format_attr { 527 struct device_attribute attr; 528 u64 field; 529 int config; 530 }; 531 532 #define CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid) \ 533 (&((struct arm_cmn_event_attr[]) {{ \ 534 .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ 535 .model = _model, \ 536 .type = _type, \ 537 .eventid = _eventid, \ 538 .occupid = _occupid, \ 539 }})[0].attr.attr) 540 541 static bool arm_cmn_is_occup_event(enum cmn_model model, 542 enum cmn_node_type type, unsigned int id) 543 { 544 if (type == CMN_TYPE_DVM) 545 return (model == CMN600 && id == 0x05) || 546 (model == CI700 && id == 0x0c); 547 return type == CMN_TYPE_HNF && id == 0x0f; 548 } 549 550 static ssize_t arm_cmn_event_show(struct device *dev, 551 struct device_attribute *attr, char *buf) 552 { 553 struct arm_cmn_event_attr *eattr; 554 555 eattr = container_of(attr, typeof(*eattr), attr); 556 557 if (eattr->type == CMN_TYPE_DTC) 558 return sysfs_emit(buf, "type=0x%x\n", eattr->type); 559 560 if (eattr->type == CMN_TYPE_WP) 561 return sysfs_emit(buf, 562 "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", 563 eattr->type, eattr->eventid); 564 565 if (arm_cmn_is_occup_event(eattr->model, eattr->type, eattr->eventid)) 566 return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n", 567 eattr->type, eattr->eventid, eattr->occupid); 568 569 return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, 570 eattr->eventid); 571 } 572 573 static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, 574 struct attribute *attr, 575 int unused) 576 { 577 struct device *dev = kobj_to_dev(kobj); 578 struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev)); 579 struct arm_cmn_event_attr *eattr; 580 581 eattr = container_of(attr, typeof(*eattr), attr.attr); 582 583 if (!(eattr->model & cmn->model)) 584 return 0; 585 586 /* Watchpoints aren't nodes, so avoid confusion */ 587 if (eattr->type == CMN_TYPE_WP) 588 return attr->mode; 589 590 /* Hide XP events for unused interfaces/channels */ 591 if (eattr->type == CMN_TYPE_XP) { 592 unsigned int intf = (eattr->eventid >> 2) & 7; 593 unsigned int chan = eattr->eventid >> 5; 594 595 if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3))) 596 return 0; 597 598 if ((chan == 5 && cmn->rsp_vc_num < 2) || 599 (chan == 6 && cmn->dat_vc_num < 2)) 600 return 0; 601 } 602 603 /* Revision-specific differences */ 604 if (cmn->model == CMN600 && cmn->rev < CMN600_R1P2) { 605 if (eattr->type == CMN_TYPE_HNF && eattr->eventid == 0x1b) 606 return 0; 607 } 608 609 if (!arm_cmn_node(cmn, eattr->type)) 610 return 0; 611 612 return attr->mode; 613 } 614 615 #define _CMN_EVENT_DVM(_model, _name, _event, _occup) \ 616 CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup) 617 #define CMN_EVENT_DTC(_name) \ 618 CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0, 0) 619 #define _CMN_EVENT_HNF(_model, _name, _event, _occup) \ 620 CMN_EVENT_ATTR(_model, hnf_##_name, CMN_TYPE_HNF, _event, _occup) 621 #define CMN_EVENT_HNI(_name, _event) \ 622 CMN_EVENT_ATTR(CMN_ANY, hni_##_name, CMN_TYPE_HNI, _event, 0) 623 #define __CMN_EVENT_XP(_name, _event) \ 624 CMN_EVENT_ATTR(CMN_ANY, mxp_##_name, CMN_TYPE_XP, _event, 0) 625 #define CMN_EVENT_SBSX(_model, _name, _event) \ 626 CMN_EVENT_ATTR(_model, sbsx_##_name, CMN_TYPE_SBSX, _event, 0) 627 #define CMN_EVENT_RNID(_model, _name, _event) \ 628 CMN_EVENT_ATTR(_model, rnid_##_name, CMN_TYPE_RNI, _event, 0) 629 #define CMN_EVENT_MTSX(_name, _event) \ 630 CMN_EVENT_ATTR(CMN_ANY, mtsx_##_name, CMN_TYPE_MTSX, _event, 0) 631 632 #define CMN_EVENT_DVM(_model, _name, _event) \ 633 _CMN_EVENT_DVM(_model, _name, _event, 0) 634 #define CMN_EVENT_HNF(_model, _name, _event) \ 635 _CMN_EVENT_HNF(_model, _name, _event, 0) 636 #define _CMN_EVENT_XP(_name, _event) \ 637 __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \ 638 __CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)), \ 639 __CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)), \ 640 __CMN_EVENT_XP(s_##_name, (_event) | (3 << 2)), \ 641 __CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)), \ 642 __CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2)), \ 643 __CMN_EVENT_XP(p2_##_name, (_event) | (6 << 2)), \ 644 __CMN_EVENT_XP(p3_##_name, (_event) | (7 << 2)) 645 646 /* Good thing there are only 3 fundamental XP events... */ 647 #define CMN_EVENT_XP(_name, _event) \ 648 _CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)), \ 649 _CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)), \ 650 _CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)), \ 651 _CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5)), \ 652 _CMN_EVENT_XP(pub_##_name, (_event) | (4 << 5)), \ 653 _CMN_EVENT_XP(rsp2_##_name, (_event) | (5 << 5)), \ 654 _CMN_EVENT_XP(dat2_##_name, (_event) | (6 << 5)) 655 656 657 static struct attribute *arm_cmn_event_attrs[] = { 658 CMN_EVENT_DTC(cycles), 659 660 /* 661 * DVM node events conflict with HN-I events in the equivalent PMU 662 * slot, but our lazy short-cut of using the DTM counter index for 663 * the PMU index as well happens to avoid that by construction. 664 */ 665 CMN_EVENT_DVM(CMN600, rxreq_dvmop, 0x01), 666 CMN_EVENT_DVM(CMN600, rxreq_dvmsync, 0x02), 667 CMN_EVENT_DVM(CMN600, rxreq_dvmop_vmid_filtered, 0x03), 668 CMN_EVENT_DVM(CMN600, rxreq_retried, 0x04), 669 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_all, 0x05, 0), 670 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_dvmop, 0x05, 1), 671 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_dvmsync, 0x05, 2), 672 CMN_EVENT_DVM(CI700, dvmop_tlbi, 0x01), 673 CMN_EVENT_DVM(CI700, dvmop_bpi, 0x02), 674 CMN_EVENT_DVM(CI700, dvmop_pici, 0x03), 675 CMN_EVENT_DVM(CI700, dvmop_vici, 0x04), 676 CMN_EVENT_DVM(CI700, dvmsync, 0x05), 677 CMN_EVENT_DVM(CI700, vmid_filtered, 0x06), 678 CMN_EVENT_DVM(CI700, rndop_filtered, 0x07), 679 CMN_EVENT_DVM(CI700, retry, 0x08), 680 CMN_EVENT_DVM(CI700, txsnp_flitv, 0x09), 681 CMN_EVENT_DVM(CI700, txsnp_stall, 0x0a), 682 CMN_EVENT_DVM(CI700, trkfull, 0x0b), 683 _CMN_EVENT_DVM(CI700, trk_occupancy_all, 0x0c, 0), 684 _CMN_EVENT_DVM(CI700, trk_occupancy_dvmop, 0x0c, 1), 685 _CMN_EVENT_DVM(CI700, trk_occupancy_dvmsync, 0x0c, 2), 686 687 CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), 688 CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), 689 CMN_EVENT_HNF(CMN_ANY, cache_fill, 0x03), 690 CMN_EVENT_HNF(CMN_ANY, pocq_retry, 0x04), 691 CMN_EVENT_HNF(CMN_ANY, pocq_reqs_recvd, 0x05), 692 CMN_EVENT_HNF(CMN_ANY, sf_hit, 0x06), 693 CMN_EVENT_HNF(CMN_ANY, sf_evictions, 0x07), 694 CMN_EVENT_HNF(CMN_ANY, dir_snoops_sent, 0x08), 695 CMN_EVENT_HNF(CMN_ANY, brd_snoops_sent, 0x09), 696 CMN_EVENT_HNF(CMN_ANY, slc_eviction, 0x0a), 697 CMN_EVENT_HNF(CMN_ANY, slc_fill_invalid_way, 0x0b), 698 CMN_EVENT_HNF(CMN_ANY, mc_retries, 0x0c), 699 CMN_EVENT_HNF(CMN_ANY, mc_reqs, 0x0d), 700 CMN_EVENT_HNF(CMN_ANY, qos_hh_retry, 0x0e), 701 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_all, 0x0f, 0), 702 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_read, 0x0f, 1), 703 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_write, 0x0f, 2), 704 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_atomic, 0x0f, 3), 705 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_stash, 0x0f, 4), 706 CMN_EVENT_HNF(CMN_ANY, pocq_addrhaz, 0x10), 707 CMN_EVENT_HNF(CMN_ANY, pocq_atomic_addrhaz, 0x11), 708 CMN_EVENT_HNF(CMN_ANY, ld_st_swp_adq_full, 0x12), 709 CMN_EVENT_HNF(CMN_ANY, cmp_adq_full, 0x13), 710 CMN_EVENT_HNF(CMN_ANY, txdat_stall, 0x14), 711 CMN_EVENT_HNF(CMN_ANY, txrsp_stall, 0x15), 712 CMN_EVENT_HNF(CMN_ANY, seq_full, 0x16), 713 CMN_EVENT_HNF(CMN_ANY, seq_hit, 0x17), 714 CMN_EVENT_HNF(CMN_ANY, snp_sent, 0x18), 715 CMN_EVENT_HNF(CMN_ANY, sfbi_dir_snp_sent, 0x19), 716 CMN_EVENT_HNF(CMN_ANY, sfbi_brd_snp_sent, 0x1a), 717 CMN_EVENT_HNF(CMN_ANY, snp_sent_untrk, 0x1b), 718 CMN_EVENT_HNF(CMN_ANY, intv_dirty, 0x1c), 719 CMN_EVENT_HNF(CMN_ANY, stash_snp_sent, 0x1d), 720 CMN_EVENT_HNF(CMN_ANY, stash_data_pull, 0x1e), 721 CMN_EVENT_HNF(CMN_ANY, snp_fwded, 0x1f), 722 CMN_EVENT_HNF(CI700, atomic_fwd, 0x20), 723 CMN_EVENT_HNF(CI700, mpam_hardlim, 0x21), 724 CMN_EVENT_HNF(CI700, mpam_softlim, 0x22), 725 726 CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl, 0x20), 727 CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl, 0x21), 728 CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl, 0x22), 729 CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl, 0x23), 730 CMN_EVENT_HNI(wdb_occ_cnt_ovfl, 0x24), 731 CMN_EVENT_HNI(rrt_rd_alloc, 0x25), 732 CMN_EVENT_HNI(rrt_wr_alloc, 0x26), 733 CMN_EVENT_HNI(rdt_rd_alloc, 0x27), 734 CMN_EVENT_HNI(rdt_wr_alloc, 0x28), 735 CMN_EVENT_HNI(wdb_alloc, 0x29), 736 CMN_EVENT_HNI(txrsp_retryack, 0x2a), 737 CMN_EVENT_HNI(arvalid_no_arready, 0x2b), 738 CMN_EVENT_HNI(arready_no_arvalid, 0x2c), 739 CMN_EVENT_HNI(awvalid_no_awready, 0x2d), 740 CMN_EVENT_HNI(awready_no_awvalid, 0x2e), 741 CMN_EVENT_HNI(wvalid_no_wready, 0x2f), 742 CMN_EVENT_HNI(txdat_stall, 0x30), 743 CMN_EVENT_HNI(nonpcie_serialization, 0x31), 744 CMN_EVENT_HNI(pcie_serialization, 0x32), 745 746 CMN_EVENT_XP(txflit_valid, 0x01), 747 CMN_EVENT_XP(txflit_stall, 0x02), 748 CMN_EVENT_XP(partial_dat_flit, 0x03), 749 /* We treat watchpoints as a special made-up class of XP events */ 750 CMN_EVENT_ATTR(CMN_ANY, watchpoint_up, CMN_TYPE_WP, CMN_WP_UP, 0), 751 CMN_EVENT_ATTR(CMN_ANY, watchpoint_down, CMN_TYPE_WP, CMN_WP_DOWN, 0), 752 753 CMN_EVENT_SBSX(CMN_ANY, rd_req, 0x01), 754 CMN_EVENT_SBSX(CMN_ANY, wr_req, 0x02), 755 CMN_EVENT_SBSX(CMN_ANY, cmo_req, 0x03), 756 CMN_EVENT_SBSX(CMN_ANY, txrsp_retryack, 0x04), 757 CMN_EVENT_SBSX(CMN_ANY, txdat_flitv, 0x05), 758 CMN_EVENT_SBSX(CMN_ANY, txrsp_flitv, 0x06), 759 CMN_EVENT_SBSX(CMN_ANY, rd_req_trkr_occ_cnt_ovfl, 0x11), 760 CMN_EVENT_SBSX(CMN_ANY, wr_req_trkr_occ_cnt_ovfl, 0x12), 761 CMN_EVENT_SBSX(CMN_ANY, cmo_req_trkr_occ_cnt_ovfl, 0x13), 762 CMN_EVENT_SBSX(CMN_ANY, wdb_occ_cnt_ovfl, 0x14), 763 CMN_EVENT_SBSX(CMN_ANY, rd_axi_trkr_occ_cnt_ovfl, 0x15), 764 CMN_EVENT_SBSX(CMN_ANY, cmo_axi_trkr_occ_cnt_ovfl, 0x16), 765 CMN_EVENT_SBSX(CI700, rdb_occ_cnt_ovfl, 0x17), 766 CMN_EVENT_SBSX(CMN_ANY, arvalid_no_arready, 0x21), 767 CMN_EVENT_SBSX(CMN_ANY, awvalid_no_awready, 0x22), 768 CMN_EVENT_SBSX(CMN_ANY, wvalid_no_wready, 0x23), 769 CMN_EVENT_SBSX(CMN_ANY, txdat_stall, 0x24), 770 CMN_EVENT_SBSX(CMN_ANY, txrsp_stall, 0x25), 771 772 CMN_EVENT_RNID(CMN_ANY, s0_rdata_beats, 0x01), 773 CMN_EVENT_RNID(CMN_ANY, s1_rdata_beats, 0x02), 774 CMN_EVENT_RNID(CMN_ANY, s2_rdata_beats, 0x03), 775 CMN_EVENT_RNID(CMN_ANY, rxdat_flits, 0x04), 776 CMN_EVENT_RNID(CMN_ANY, txdat_flits, 0x05), 777 CMN_EVENT_RNID(CMN_ANY, txreq_flits_total, 0x06), 778 CMN_EVENT_RNID(CMN_ANY, txreq_flits_retried, 0x07), 779 CMN_EVENT_RNID(CMN_ANY, rrt_occ_ovfl, 0x08), 780 CMN_EVENT_RNID(CMN_ANY, wrt_occ_ovfl, 0x09), 781 CMN_EVENT_RNID(CMN_ANY, txreq_flits_replayed, 0x0a), 782 CMN_EVENT_RNID(CMN_ANY, wrcancel_sent, 0x0b), 783 CMN_EVENT_RNID(CMN_ANY, s0_wdata_beats, 0x0c), 784 CMN_EVENT_RNID(CMN_ANY, s1_wdata_beats, 0x0d), 785 CMN_EVENT_RNID(CMN_ANY, s2_wdata_beats, 0x0e), 786 CMN_EVENT_RNID(CMN_ANY, rrt_alloc, 0x0f), 787 CMN_EVENT_RNID(CMN_ANY, wrt_alloc, 0x10), 788 CMN_EVENT_RNID(CMN600, rdb_unord, 0x11), 789 CMN_EVENT_RNID(CMN600, rdb_replay, 0x12), 790 CMN_EVENT_RNID(CMN600, rdb_hybrid, 0x13), 791 CMN_EVENT_RNID(CMN600, rdb_ord, 0x14), 792 CMN_EVENT_RNID(CI700, padb_occ_ovfl, 0x11), 793 CMN_EVENT_RNID(CI700, rpdb_occ_ovfl, 0x12), 794 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice1, 0x13), 795 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice2, 0x14), 796 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice3, 0x15), 797 CMN_EVENT_RNID(CI700, wrt_throttled, 0x16), 798 799 CMN_EVENT_MTSX(tc_lookup, 0x01), 800 CMN_EVENT_MTSX(tc_fill, 0x02), 801 CMN_EVENT_MTSX(tc_miss, 0x03), 802 CMN_EVENT_MTSX(tdb_forward, 0x04), 803 CMN_EVENT_MTSX(tcq_hazard, 0x05), 804 CMN_EVENT_MTSX(tcq_rd_alloc, 0x06), 805 CMN_EVENT_MTSX(tcq_wr_alloc, 0x07), 806 CMN_EVENT_MTSX(tcq_cmo_alloc, 0x08), 807 CMN_EVENT_MTSX(axi_rd_req, 0x09), 808 CMN_EVENT_MTSX(axi_wr_req, 0x0a), 809 CMN_EVENT_MTSX(tcq_occ_cnt_ovfl, 0x0b), 810 CMN_EVENT_MTSX(tdb_occ_cnt_ovfl, 0x0c), 811 812 NULL 813 }; 814 815 static const struct attribute_group arm_cmn_event_attrs_group = { 816 .name = "events", 817 .attrs = arm_cmn_event_attrs, 818 .is_visible = arm_cmn_event_attr_is_visible, 819 }; 820 821 static ssize_t arm_cmn_format_show(struct device *dev, 822 struct device_attribute *attr, char *buf) 823 { 824 struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr); 825 int lo = __ffs(fmt->field), hi = __fls(fmt->field); 826 827 if (lo == hi) 828 return sysfs_emit(buf, "config:%d\n", lo); 829 830 if (!fmt->config) 831 return sysfs_emit(buf, "config:%d-%d\n", lo, hi); 832 833 return sysfs_emit(buf, "config%d:%d-%d\n", fmt->config, lo, hi); 834 } 835 836 #define _CMN_FORMAT_ATTR(_name, _cfg, _fld) \ 837 (&((struct arm_cmn_format_attr[]) {{ \ 838 .attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL), \ 839 .config = _cfg, \ 840 .field = _fld, \ 841 }})[0].attr.attr) 842 #define CMN_FORMAT_ATTR(_name, _fld) _CMN_FORMAT_ATTR(_name, 0, _fld) 843 844 static struct attribute *arm_cmn_format_attrs[] = { 845 CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE), 846 CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID), 847 CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID), 848 CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID), 849 CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID), 850 851 CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL), 852 CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL), 853 CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP), 854 CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE), 855 CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE), 856 857 _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL), 858 _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK), 859 860 NULL 861 }; 862 863 static const struct attribute_group arm_cmn_format_attrs_group = { 864 .name = "format", 865 .attrs = arm_cmn_format_attrs, 866 }; 867 868 static ssize_t arm_cmn_cpumask_show(struct device *dev, 869 struct device_attribute *attr, char *buf) 870 { 871 struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev)); 872 873 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu)); 874 } 875 876 static struct device_attribute arm_cmn_cpumask_attr = 877 __ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL); 878 879 static struct attribute *arm_cmn_cpumask_attrs[] = { 880 &arm_cmn_cpumask_attr.attr, 881 NULL, 882 }; 883 884 static const struct attribute_group arm_cmn_cpumask_attr_group = { 885 .attrs = arm_cmn_cpumask_attrs, 886 }; 887 888 static const struct attribute_group *arm_cmn_attr_groups[] = { 889 &arm_cmn_event_attrs_group, 890 &arm_cmn_format_attrs_group, 891 &arm_cmn_cpumask_attr_group, 892 NULL 893 }; 894 895 static int arm_cmn_wp_idx(struct perf_event *event) 896 { 897 return CMN_EVENT_EVENTID(event) + CMN_EVENT_WP_GRP(event); 898 } 899 900 static u32 arm_cmn_wp_config(struct perf_event *event) 901 { 902 u32 config; 903 u32 dev = CMN_EVENT_WP_DEV_SEL(event); 904 u32 chn = CMN_EVENT_WP_CHN_SEL(event); 905 u32 grp = CMN_EVENT_WP_GRP(event); 906 u32 exc = CMN_EVENT_WP_EXCLUSIVE(event); 907 u32 combine = CMN_EVENT_WP_COMBINE(event); 908 909 config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) | 910 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) | 911 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) | 912 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE, exc) | 913 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1); 914 if (combine && !grp) 915 config |= CMN_DTM_WPn_CONFIG_WP_COMBINE; 916 917 return config; 918 } 919 920 static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state) 921 { 922 if (!cmn->state) 923 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_PMCR); 924 cmn->state |= state; 925 } 926 927 static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state) 928 { 929 cmn->state &= ~state; 930 if (!cmn->state) 931 writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN, 932 cmn->dtc[0].base + CMN_DT_PMCR); 933 } 934 935 static void arm_cmn_pmu_enable(struct pmu *pmu) 936 { 937 arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED); 938 } 939 940 static void arm_cmn_pmu_disable(struct pmu *pmu) 941 { 942 arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED); 943 } 944 945 static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw, 946 bool snapshot) 947 { 948 struct arm_cmn_dtm *dtm = NULL; 949 struct arm_cmn_node *dn; 950 unsigned int i, offset, dtm_idx; 951 u64 reg, count = 0; 952 953 offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT; 954 for_each_hw_dn(hw, dn, i) { 955 if (dtm != &cmn->dtms[dn->dtm]) { 956 dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; 957 reg = readq_relaxed(dtm->base + offset); 958 } 959 dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 960 count += (u16)(reg >> (dtm_idx * 16)); 961 } 962 return count; 963 } 964 965 static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc) 966 { 967 u64 val = readq_relaxed(dtc->base + CMN_DT_PMCCNTR); 968 969 writeq_relaxed(CMN_CC_INIT, dtc->base + CMN_DT_PMCCNTR); 970 return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1); 971 } 972 973 static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx) 974 { 975 u32 val, pmevcnt = CMN_DT_PMEVCNT(idx); 976 977 val = readl_relaxed(dtc->base + pmevcnt); 978 writel_relaxed(CMN_COUNTER_INIT, dtc->base + pmevcnt); 979 return val - CMN_COUNTER_INIT; 980 } 981 982 static void arm_cmn_init_counter(struct perf_event *event) 983 { 984 struct arm_cmn *cmn = to_cmn(event->pmu); 985 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 986 unsigned int i, pmevcnt = CMN_DT_PMEVCNT(hw->dtc_idx); 987 u64 count; 988 989 for (i = 0; hw->dtcs_used & (1U << i); i++) { 990 writel_relaxed(CMN_COUNTER_INIT, cmn->dtc[i].base + pmevcnt); 991 cmn->dtc[i].counters[hw->dtc_idx] = event; 992 } 993 994 count = arm_cmn_read_dtm(cmn, hw, false); 995 local64_set(&event->hw.prev_count, count); 996 } 997 998 static void arm_cmn_event_read(struct perf_event *event) 999 { 1000 struct arm_cmn *cmn = to_cmn(event->pmu); 1001 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1002 u64 delta, new, prev; 1003 unsigned long flags; 1004 unsigned int i; 1005 1006 if (hw->dtc_idx == CMN_DT_NUM_COUNTERS) { 1007 i = __ffs(hw->dtcs_used); 1008 delta = arm_cmn_read_cc(cmn->dtc + i); 1009 local64_add(delta, &event->count); 1010 return; 1011 } 1012 new = arm_cmn_read_dtm(cmn, hw, false); 1013 prev = local64_xchg(&event->hw.prev_count, new); 1014 1015 delta = new - prev; 1016 1017 local_irq_save(flags); 1018 for (i = 0; hw->dtcs_used & (1U << i); i++) { 1019 new = arm_cmn_read_counter(cmn->dtc + i, hw->dtc_idx); 1020 delta += new << 16; 1021 } 1022 local_irq_restore(flags); 1023 local64_add(delta, &event->count); 1024 } 1025 1026 static void arm_cmn_event_start(struct perf_event *event, int flags) 1027 { 1028 struct arm_cmn *cmn = to_cmn(event->pmu); 1029 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1030 struct arm_cmn_node *dn; 1031 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1032 int i; 1033 1034 if (type == CMN_TYPE_DTC) { 1035 i = __ffs(hw->dtcs_used); 1036 writeq_relaxed(CMN_CC_INIT, cmn->dtc[i].base + CMN_DT_PMCCNTR); 1037 cmn->dtc[i].cc_active = true; 1038 } else if (type == CMN_TYPE_WP) { 1039 int wp_idx = arm_cmn_wp_idx(event); 1040 u64 val = CMN_EVENT_WP_VAL(event); 1041 u64 mask = CMN_EVENT_WP_MASK(event); 1042 1043 for_each_hw_dn(hw, dn, i) { 1044 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); 1045 1046 writeq_relaxed(val, base + CMN_DTM_WPn_VAL(wp_idx)); 1047 writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx)); 1048 } 1049 } else for_each_hw_dn(hw, dn, i) { 1050 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1051 1052 dn->event[dtm_idx] = CMN_EVENT_EVENTID(event); 1053 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); 1054 } 1055 } 1056 1057 static void arm_cmn_event_stop(struct perf_event *event, int flags) 1058 { 1059 struct arm_cmn *cmn = to_cmn(event->pmu); 1060 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1061 struct arm_cmn_node *dn; 1062 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1063 int i; 1064 1065 if (type == CMN_TYPE_DTC) { 1066 i = __ffs(hw->dtcs_used); 1067 cmn->dtc[i].cc_active = false; 1068 } else if (type == CMN_TYPE_WP) { 1069 int wp_idx = arm_cmn_wp_idx(event); 1070 1071 for_each_hw_dn(hw, dn, i) { 1072 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); 1073 1074 writeq_relaxed(0, base + CMN_DTM_WPn_MASK(wp_idx)); 1075 writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx)); 1076 } 1077 } else for_each_hw_dn(hw, dn, i) { 1078 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1079 1080 dn->event[dtm_idx] = 0; 1081 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); 1082 } 1083 1084 arm_cmn_event_read(event); 1085 } 1086 1087 struct arm_cmn_val { 1088 u8 dtm_count[CMN_MAX_DTMS]; 1089 u8 occupid[CMN_MAX_DTMS]; 1090 u8 wp[CMN_MAX_DTMS][4]; 1091 int dtc_count; 1092 bool cycles; 1093 }; 1094 1095 static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, 1096 struct perf_event *event) 1097 { 1098 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1099 struct arm_cmn_node *dn; 1100 enum cmn_node_type type; 1101 int i; 1102 u8 occupid; 1103 1104 if (is_software_event(event)) 1105 return; 1106 1107 type = CMN_EVENT_TYPE(event); 1108 if (type == CMN_TYPE_DTC) { 1109 val->cycles = true; 1110 return; 1111 } 1112 1113 val->dtc_count++; 1114 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1115 occupid = CMN_EVENT_OCCUPID(event) + 1; 1116 else 1117 occupid = 0; 1118 1119 for_each_hw_dn(hw, dn, i) { 1120 int wp_idx, dtm = dn->dtm; 1121 1122 val->dtm_count[dtm]++; 1123 val->occupid[dtm] = occupid; 1124 1125 if (type != CMN_TYPE_WP) 1126 continue; 1127 1128 wp_idx = arm_cmn_wp_idx(event); 1129 val->wp[dtm][wp_idx] = CMN_EVENT_WP_COMBINE(event) + 1; 1130 } 1131 } 1132 1133 static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) 1134 { 1135 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1136 struct arm_cmn_node *dn; 1137 struct perf_event *sibling, *leader = event->group_leader; 1138 enum cmn_node_type type; 1139 struct arm_cmn_val *val; 1140 int i, ret = -EINVAL; 1141 u8 occupid; 1142 1143 if (leader == event) 1144 return 0; 1145 1146 if (event->pmu != leader->pmu && !is_software_event(leader)) 1147 return -EINVAL; 1148 1149 val = kzalloc(sizeof(*val), GFP_KERNEL); 1150 if (!val) 1151 return -ENOMEM; 1152 1153 arm_cmn_val_add_event(cmn, val, leader); 1154 for_each_sibling_event(sibling, leader) 1155 arm_cmn_val_add_event(cmn, val, sibling); 1156 1157 type = CMN_EVENT_TYPE(event); 1158 if (type == CMN_TYPE_DTC) { 1159 ret = val->cycles ? -EINVAL : 0; 1160 goto done; 1161 } 1162 1163 if (val->dtc_count == CMN_DT_NUM_COUNTERS) 1164 goto done; 1165 1166 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1167 occupid = CMN_EVENT_OCCUPID(event) + 1; 1168 else 1169 occupid = 0; 1170 1171 for_each_hw_dn(hw, dn, i) { 1172 int wp_idx, wp_cmb, dtm = dn->dtm; 1173 1174 if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) 1175 goto done; 1176 1177 if (occupid && val->occupid[dtm] && occupid != val->occupid[dtm]) 1178 goto done; 1179 1180 if (type != CMN_TYPE_WP) 1181 continue; 1182 1183 wp_idx = arm_cmn_wp_idx(event); 1184 if (val->wp[dtm][wp_idx]) 1185 goto done; 1186 1187 wp_cmb = val->wp[dtm][wp_idx ^ 1]; 1188 if (wp_cmb && wp_cmb != CMN_EVENT_WP_COMBINE(event) + 1) 1189 goto done; 1190 } 1191 1192 ret = 0; 1193 done: 1194 kfree(val); 1195 return ret; 1196 } 1197 1198 static int arm_cmn_event_init(struct perf_event *event) 1199 { 1200 struct arm_cmn *cmn = to_cmn(event->pmu); 1201 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1202 struct arm_cmn_node *dn; 1203 enum cmn_node_type type; 1204 bool bynodeid; 1205 u16 nodeid, eventid; 1206 1207 if (event->attr.type != event->pmu->type) 1208 return -ENOENT; 1209 1210 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1211 return -EINVAL; 1212 1213 event->cpu = cmn->cpu; 1214 if (event->cpu < 0) 1215 return -EINVAL; 1216 1217 type = CMN_EVENT_TYPE(event); 1218 /* DTC events (i.e. cycles) already have everything they need */ 1219 if (type == CMN_TYPE_DTC) 1220 return 0; 1221 1222 /* For watchpoints we need the actual XP node here */ 1223 if (type == CMN_TYPE_WP) { 1224 type = CMN_TYPE_XP; 1225 /* ...and we need a "real" direction */ 1226 eventid = CMN_EVENT_EVENTID(event); 1227 if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN) 1228 return -EINVAL; 1229 /* ...but the DTM may depend on which port we're watching */ 1230 if (cmn->multi_dtm) 1231 hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; 1232 } 1233 1234 bynodeid = CMN_EVENT_BYNODEID(event); 1235 nodeid = CMN_EVENT_NODEID(event); 1236 1237 hw->dn = arm_cmn_node(cmn, type); 1238 if (!hw->dn) 1239 return -EINVAL; 1240 for (dn = hw->dn; dn->type == type; dn++) { 1241 if (bynodeid && dn->id != nodeid) { 1242 hw->dn++; 1243 continue; 1244 } 1245 hw->dtcs_used |= arm_cmn_node_to_xp(cmn, dn)->dtc; 1246 hw->num_dns++; 1247 if (bynodeid) 1248 break; 1249 } 1250 1251 if (!hw->num_dns) { 1252 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, nodeid); 1253 1254 dev_dbg(cmn->dev, "invalid node 0x%x (%d,%d,%d,%d) type 0x%x\n", 1255 nodeid, nid.x, nid.y, nid.port, nid.dev, type); 1256 return -EINVAL; 1257 } 1258 1259 return arm_cmn_validate_group(cmn, event); 1260 } 1261 1262 static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, 1263 int i) 1264 { 1265 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1266 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1267 1268 while (i--) { 1269 struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; 1270 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1271 1272 if (type == CMN_TYPE_WP) 1273 dtm->wp_event[arm_cmn_wp_idx(event)] = -1; 1274 1275 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1276 hw->dn[i].occupid_count--; 1277 1278 dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); 1279 writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); 1280 } 1281 memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); 1282 1283 for (i = 0; hw->dtcs_used & (1U << i); i++) 1284 cmn->dtc[i].counters[hw->dtc_idx] = NULL; 1285 } 1286 1287 static int arm_cmn_event_add(struct perf_event *event, int flags) 1288 { 1289 struct arm_cmn *cmn = to_cmn(event->pmu); 1290 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1291 struct arm_cmn_dtc *dtc = &cmn->dtc[0]; 1292 struct arm_cmn_node *dn; 1293 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1294 unsigned int i, dtc_idx, input_sel; 1295 1296 if (type == CMN_TYPE_DTC) { 1297 i = 0; 1298 while (cmn->dtc[i].cycles) 1299 if (++i == cmn->num_dtcs) 1300 return -ENOSPC; 1301 1302 cmn->dtc[i].cycles = event; 1303 hw->dtc_idx = CMN_DT_NUM_COUNTERS; 1304 hw->dtcs_used = 1U << i; 1305 1306 if (flags & PERF_EF_START) 1307 arm_cmn_event_start(event, 0); 1308 return 0; 1309 } 1310 1311 /* Grab a free global counter first... */ 1312 dtc_idx = 0; 1313 while (dtc->counters[dtc_idx]) 1314 if (++dtc_idx == CMN_DT_NUM_COUNTERS) 1315 return -ENOSPC; 1316 1317 hw->dtc_idx = dtc_idx; 1318 1319 /* ...then the local counters to feed it. */ 1320 for_each_hw_dn(hw, dn, i) { 1321 struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; 1322 unsigned int dtm_idx, shift; 1323 u64 reg; 1324 1325 dtm_idx = 0; 1326 while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx)) 1327 if (++dtm_idx == CMN_DTM_NUM_COUNTERS) 1328 goto free_dtms; 1329 1330 if (type == CMN_TYPE_XP) { 1331 input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx; 1332 } else if (type == CMN_TYPE_WP) { 1333 int tmp, wp_idx = arm_cmn_wp_idx(event); 1334 u32 cfg = arm_cmn_wp_config(event); 1335 1336 if (dtm->wp_event[wp_idx] >= 0) 1337 goto free_dtms; 1338 1339 tmp = dtm->wp_event[wp_idx ^ 1]; 1340 if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) != 1341 CMN_EVENT_WP_COMBINE(dtc->counters[tmp])) 1342 goto free_dtms; 1343 1344 input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx; 1345 dtm->wp_event[wp_idx] = dtc_idx; 1346 writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx)); 1347 } else { 1348 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 1349 1350 if (cmn->multi_dtm) 1351 nid.port %= 2; 1352 1353 input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx + 1354 (nid.port << 4) + (nid.dev << 2); 1355 1356 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) { 1357 u8 occupid = CMN_EVENT_OCCUPID(event); 1358 1359 if (dn->occupid_count == 0) { 1360 dn->occupid_val = occupid; 1361 writel_relaxed(occupid, 1362 dn->pmu_base + CMN_PMU_EVENT_SEL + 4); 1363 } else if (dn->occupid_val != occupid) { 1364 goto free_dtms; 1365 } 1366 dn->occupid_count++; 1367 } 1368 } 1369 1370 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); 1371 1372 dtm->input_sel[dtm_idx] = input_sel; 1373 shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx); 1374 dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift); 1375 dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, dtc_idx) << shift; 1376 dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx); 1377 reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low; 1378 writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG); 1379 } 1380 1381 /* Go go go! */ 1382 arm_cmn_init_counter(event); 1383 1384 if (flags & PERF_EF_START) 1385 arm_cmn_event_start(event, 0); 1386 1387 return 0; 1388 1389 free_dtms: 1390 arm_cmn_event_clear(cmn, event, i); 1391 return -ENOSPC; 1392 } 1393 1394 static void arm_cmn_event_del(struct perf_event *event, int flags) 1395 { 1396 struct arm_cmn *cmn = to_cmn(event->pmu); 1397 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1398 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1399 1400 arm_cmn_event_stop(event, PERF_EF_UPDATE); 1401 1402 if (type == CMN_TYPE_DTC) 1403 cmn->dtc[__ffs(hw->dtcs_used)].cycles = NULL; 1404 else 1405 arm_cmn_event_clear(cmn, event, hw->num_dns); 1406 } 1407 1408 /* 1409 * We stop the PMU for both add and read, to avoid skew across DTM counters. 1410 * In theory we could use snapshots to read without stopping, but then it 1411 * becomes a lot trickier to deal with overlow and racing against interrupts, 1412 * plus it seems they don't work properly on some hardware anyway :( 1413 */ 1414 static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags) 1415 { 1416 arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN); 1417 } 1418 1419 static void arm_cmn_end_txn(struct pmu *pmu) 1420 { 1421 arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN); 1422 } 1423 1424 static int arm_cmn_commit_txn(struct pmu *pmu) 1425 { 1426 arm_cmn_end_txn(pmu); 1427 return 0; 1428 } 1429 1430 static void arm_cmn_migrate(struct arm_cmn *cmn, unsigned int cpu) 1431 { 1432 unsigned int i; 1433 1434 perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu); 1435 for (i = 0; i < cmn->num_dtcs; i++) 1436 irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu)); 1437 cmn->cpu = cpu; 1438 } 1439 1440 static int arm_cmn_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 1441 { 1442 struct arm_cmn *cmn; 1443 int node; 1444 1445 cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node); 1446 node = dev_to_node(cmn->dev); 1447 if (node != NUMA_NO_NODE && cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node) 1448 arm_cmn_migrate(cmn, cpu); 1449 return 0; 1450 } 1451 1452 static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 1453 { 1454 struct arm_cmn *cmn; 1455 unsigned int target; 1456 int node; 1457 cpumask_t mask; 1458 1459 cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node); 1460 if (cpu != cmn->cpu) 1461 return 0; 1462 1463 node = dev_to_node(cmn->dev); 1464 if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) && 1465 cpumask_andnot(&mask, &mask, cpumask_of(cpu))) 1466 target = cpumask_any(&mask); 1467 else 1468 target = cpumask_any_but(cpu_online_mask, cpu); 1469 if (target < nr_cpu_ids) 1470 arm_cmn_migrate(cmn, target); 1471 return 0; 1472 } 1473 1474 static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id) 1475 { 1476 struct arm_cmn_dtc *dtc = dev_id; 1477 irqreturn_t ret = IRQ_NONE; 1478 1479 for (;;) { 1480 u32 status = readl_relaxed(dtc->base + CMN_DT_PMOVSR); 1481 u64 delta; 1482 int i; 1483 1484 for (i = 0; i < CMN_DTM_NUM_COUNTERS; i++) { 1485 if (status & (1U << i)) { 1486 ret = IRQ_HANDLED; 1487 if (WARN_ON(!dtc->counters[i])) 1488 continue; 1489 delta = (u64)arm_cmn_read_counter(dtc, i) << 16; 1490 local64_add(delta, &dtc->counters[i]->count); 1491 } 1492 } 1493 1494 if (status & (1U << CMN_DT_NUM_COUNTERS)) { 1495 ret = IRQ_HANDLED; 1496 if (dtc->cc_active && !WARN_ON(!dtc->cycles)) { 1497 delta = arm_cmn_read_cc(dtc); 1498 local64_add(delta, &dtc->cycles->count); 1499 } 1500 } 1501 1502 writel_relaxed(status, dtc->base + CMN_DT_PMOVSR_CLR); 1503 1504 if (!dtc->irq_friend) 1505 return ret; 1506 dtc += dtc->irq_friend; 1507 } 1508 } 1509 1510 /* We can reasonably accommodate DTCs of the same CMN sharing IRQs */ 1511 static int arm_cmn_init_irqs(struct arm_cmn *cmn) 1512 { 1513 int i, j, irq, err; 1514 1515 for (i = 0; i < cmn->num_dtcs; i++) { 1516 irq = cmn->dtc[i].irq; 1517 for (j = i; j--; ) { 1518 if (cmn->dtc[j].irq == irq) { 1519 cmn->dtc[j].irq_friend = i - j; 1520 goto next; 1521 } 1522 } 1523 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq, 1524 IRQF_NOBALANCING | IRQF_NO_THREAD, 1525 dev_name(cmn->dev), &cmn->dtc[i]); 1526 if (err) 1527 return err; 1528 1529 err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); 1530 if (err) 1531 return err; 1532 next: 1533 ; /* isn't C great? */ 1534 } 1535 return 0; 1536 } 1537 1538 static void arm_cmn_init_dtm(struct arm_cmn_dtm *dtm, struct arm_cmn_node *xp, int idx) 1539 { 1540 int i; 1541 1542 dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx); 1543 dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN; 1544 for (i = 0; i < 4; i++) { 1545 dtm->wp_event[i] = -1; 1546 writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i)); 1547 writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i)); 1548 } 1549 } 1550 1551 static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx) 1552 { 1553 struct arm_cmn_dtc *dtc = cmn->dtc + idx; 1554 1555 dtc->base = dn->pmu_base - CMN_PMU_OFFSET; 1556 dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx); 1557 if (dtc->irq < 0) 1558 return dtc->irq; 1559 1560 writel_relaxed(0, dtc->base + CMN_DT_PMCR); 1561 writel_relaxed(0x1ff, dtc->base + CMN_DT_PMOVSR_CLR); 1562 writel_relaxed(CMN_DT_PMCR_OVFL_INTR_EN, dtc->base + CMN_DT_PMCR); 1563 1564 return 0; 1565 } 1566 1567 static int arm_cmn_node_cmp(const void *a, const void *b) 1568 { 1569 const struct arm_cmn_node *dna = a, *dnb = b; 1570 int cmp; 1571 1572 cmp = dna->type - dnb->type; 1573 if (!cmp) 1574 cmp = dna->logid - dnb->logid; 1575 return cmp; 1576 } 1577 1578 static int arm_cmn_init_dtcs(struct arm_cmn *cmn) 1579 { 1580 struct arm_cmn_node *dn, *xp; 1581 int dtc_idx = 0; 1582 u8 dtcs_present = (1 << cmn->num_dtcs) - 1; 1583 1584 cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL); 1585 if (!cmn->dtc) 1586 return -ENOMEM; 1587 1588 sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL); 1589 1590 cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP); 1591 1592 for (dn = cmn->dns; dn->type; dn++) { 1593 if (dn->type == CMN_TYPE_XP) { 1594 dn->dtc &= dtcs_present; 1595 continue; 1596 } 1597 1598 xp = arm_cmn_node_to_xp(cmn, dn); 1599 dn->dtm = xp->dtm; 1600 if (cmn->multi_dtm) 1601 dn->dtm += arm_cmn_nid(cmn, dn->id).port / 2; 1602 1603 if (dn->type == CMN_TYPE_DTC) { 1604 int err; 1605 /* We do at least know that a DTC's XP must be in that DTC's domain */ 1606 if (xp->dtc == 0xf) 1607 xp->dtc = 1 << dtc_idx; 1608 err = arm_cmn_init_dtc(cmn, dn, dtc_idx++); 1609 if (err) 1610 return err; 1611 } 1612 1613 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */ 1614 if (dn->type == CMN_TYPE_RND) 1615 dn->type = CMN_TYPE_RNI; 1616 } 1617 1618 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, cmn->dtc[0].base + CMN_DT_DTC_CTL); 1619 1620 return 0; 1621 } 1622 1623 static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node) 1624 { 1625 int level; 1626 u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO); 1627 1628 node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg); 1629 node->id = FIELD_GET(CMN_NI_NODE_ID, reg); 1630 node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg); 1631 1632 node->pmu_base = cmn->base + offset + CMN_PMU_OFFSET; 1633 1634 if (node->type == CMN_TYPE_CFG) 1635 level = 0; 1636 else if (node->type == CMN_TYPE_XP) 1637 level = 1; 1638 else 1639 level = 2; 1640 1641 dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n", 1642 (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ', 1643 node->type, node->logid, offset); 1644 } 1645 1646 static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) 1647 { 1648 void __iomem *cfg_region; 1649 struct arm_cmn_node cfg, *dn; 1650 struct arm_cmn_dtm *dtm; 1651 u16 child_count, child_poff; 1652 u32 xp_offset[CMN_MAX_XPS]; 1653 u64 reg; 1654 int i, j; 1655 size_t sz; 1656 1657 arm_cmn_init_node_info(cmn, rgn_offset, &cfg); 1658 if (cfg.type != CMN_TYPE_CFG) 1659 return -ENODEV; 1660 1661 cfg_region = cmn->base + rgn_offset; 1662 reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_2); 1663 cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); 1664 1665 reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL); 1666 cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; 1667 cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); 1668 cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); 1669 1670 reg = readq_relaxed(cfg_region + CMN_CHILD_INFO); 1671 child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1672 child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg); 1673 1674 cmn->num_xps = child_count; 1675 cmn->num_dns = cmn->num_xps; 1676 1677 /* Pass 1: visit the XPs, enumerate their children */ 1678 for (i = 0; i < cmn->num_xps; i++) { 1679 reg = readq_relaxed(cfg_region + child_poff + i * 8); 1680 xp_offset[i] = reg & CMN_CHILD_NODE_ADDR; 1681 1682 reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO); 1683 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1684 } 1685 1686 /* Cheeky +1 to help terminate pointer-based iteration later */ 1687 dn = devm_kcalloc(cmn->dev, cmn->num_dns + 1, sizeof(*dn), GFP_KERNEL); 1688 if (!dn) 1689 return -ENOMEM; 1690 1691 /* Initial safe upper bound on DTMs for any possible mesh layout */ 1692 i = cmn->num_xps; 1693 if (cmn->multi_dtm) 1694 i += cmn->num_xps + 1; 1695 dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL); 1696 if (!dtm) 1697 return -ENOMEM; 1698 1699 /* Pass 2: now we can actually populate the nodes */ 1700 cmn->dns = dn; 1701 cmn->dtms = dtm; 1702 for (i = 0; i < cmn->num_xps; i++) { 1703 void __iomem *xp_region = cmn->base + xp_offset[i]; 1704 struct arm_cmn_node *xp = dn++; 1705 unsigned int xp_ports = 0; 1706 1707 arm_cmn_init_node_info(cmn, xp_offset[i], xp); 1708 /* 1709 * Thanks to the order in which XP logical IDs seem to be 1710 * assigned, we can handily infer the mesh X dimension by 1711 * looking out for the XP at (0,1) without needing to know 1712 * the exact node ID format, which we can later derive. 1713 */ 1714 if (xp->id == (1 << 3)) 1715 cmn->mesh_x = xp->logid; 1716 1717 if (cmn->model == CMN600) 1718 xp->dtc = 0xf; 1719 else 1720 xp->dtc = 1 << readl_relaxed(xp_region + CMN_DTM_UNIT_INFO); 1721 1722 xp->dtm = dtm - cmn->dtms; 1723 arm_cmn_init_dtm(dtm++, xp, 0); 1724 /* 1725 * Keeping track of connected ports will let us filter out 1726 * unnecessary XP events easily. We can also reliably infer the 1727 * "extra device ports" configuration for the node ID format 1728 * from this, since in that case we will see at least one XP 1729 * with port 2 connected, for the HN-D. 1730 */ 1731 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P0)) 1732 xp_ports |= BIT(0); 1733 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P1)) 1734 xp_ports |= BIT(1); 1735 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P2)) 1736 xp_ports |= BIT(2); 1737 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P3)) 1738 xp_ports |= BIT(3); 1739 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P4)) 1740 xp_ports |= BIT(4); 1741 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P5)) 1742 xp_ports |= BIT(5); 1743 1744 if (cmn->multi_dtm && (xp_ports & 0xc)) 1745 arm_cmn_init_dtm(dtm++, xp, 1); 1746 if (cmn->multi_dtm && (xp_ports & 0x30)) 1747 arm_cmn_init_dtm(dtm++, xp, 2); 1748 1749 cmn->ports_used |= xp_ports; 1750 1751 reg = readq_relaxed(xp_region + CMN_CHILD_INFO); 1752 child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1753 child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg); 1754 1755 for (j = 0; j < child_count; j++) { 1756 reg = readq_relaxed(xp_region + child_poff + j * 8); 1757 /* 1758 * Don't even try to touch anything external, since in general 1759 * we haven't a clue how to power up arbitrary CHI requesters. 1760 * As of CMN-600r1 these could only be RN-SAMs or CXLAs, 1761 * neither of which have any PMU events anyway. 1762 * (Actually, CXLAs do seem to have grown some events in r1p2, 1763 * but they don't go to regular XP DTMs, and they depend on 1764 * secure configuration which we can't easily deal with) 1765 */ 1766 if (reg & CMN_CHILD_NODE_EXTERNAL) { 1767 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); 1768 continue; 1769 } 1770 1771 arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn); 1772 1773 switch (dn->type) { 1774 case CMN_TYPE_DTC: 1775 cmn->num_dtcs++; 1776 dn++; 1777 break; 1778 /* These guys have PMU events */ 1779 case CMN_TYPE_DVM: 1780 case CMN_TYPE_HNI: 1781 case CMN_TYPE_HNF: 1782 case CMN_TYPE_SBSX: 1783 case CMN_TYPE_RNI: 1784 case CMN_TYPE_RND: 1785 case CMN_TYPE_MTSX: 1786 case CMN_TYPE_CXRA: 1787 case CMN_TYPE_CXHA: 1788 dn++; 1789 break; 1790 /* Nothing to see here */ 1791 case CMN_TYPE_MPAM_S: 1792 case CMN_TYPE_MPAM_NS: 1793 case CMN_TYPE_RNSAM: 1794 case CMN_TYPE_CXLA: 1795 break; 1796 /* Something has gone horribly wrong */ 1797 default: 1798 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); 1799 return -ENODEV; 1800 } 1801 } 1802 } 1803 1804 /* Correct for any nodes we skipped */ 1805 cmn->num_dns = dn - cmn->dns; 1806 1807 sz = (void *)(dn + 1) - (void *)cmn->dns; 1808 dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL); 1809 if (dn) 1810 cmn->dns = dn; 1811 1812 sz = (void *)dtm - (void *)cmn->dtms; 1813 dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL); 1814 if (dtm) 1815 cmn->dtms = dtm; 1816 1817 /* 1818 * If mesh_x wasn't set during discovery then we never saw 1819 * an XP at (0,1), thus we must have an Nx1 configuration. 1820 */ 1821 if (!cmn->mesh_x) 1822 cmn->mesh_x = cmn->num_xps; 1823 cmn->mesh_y = cmn->num_xps / cmn->mesh_x; 1824 1825 /* 1x1 config plays havoc with XP event encodings */ 1826 if (cmn->num_xps == 1) 1827 dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); 1828 1829 dev_dbg(cmn->dev, "model %d, periph_id_2 revision %d\n", cmn->model, cmn->rev); 1830 reg = cmn->ports_used; 1831 dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n", 1832 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), ®, 1833 cmn->multi_dtm ? ", multi-DTM" : ""); 1834 1835 return 0; 1836 } 1837 1838 static int arm_cmn600_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn) 1839 { 1840 struct resource *cfg, *root; 1841 1842 cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1843 if (!cfg) 1844 return -EINVAL; 1845 1846 root = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1847 if (!root) 1848 return -EINVAL; 1849 1850 if (!resource_contains(cfg, root)) 1851 swap(cfg, root); 1852 /* 1853 * Note that devm_ioremap_resource() is dumb and won't let the platform 1854 * device claim cfg when the ACPI companion device has already claimed 1855 * root within it. But since they *are* already both claimed in the 1856 * appropriate name, we don't really need to do it again here anyway. 1857 */ 1858 cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg)); 1859 if (!cmn->base) 1860 return -ENOMEM; 1861 1862 return root->start - cfg->start; 1863 } 1864 1865 static int arm_cmn600_of_probe(struct device_node *np) 1866 { 1867 u32 rootnode; 1868 1869 return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode; 1870 } 1871 1872 static int arm_cmn_probe(struct platform_device *pdev) 1873 { 1874 struct arm_cmn *cmn; 1875 const char *name; 1876 static atomic_t id; 1877 int err, rootnode, this_id; 1878 1879 cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); 1880 if (!cmn) 1881 return -ENOMEM; 1882 1883 cmn->dev = &pdev->dev; 1884 cmn->model = (unsigned long)device_get_match_data(cmn->dev); 1885 platform_set_drvdata(pdev, cmn); 1886 1887 if (cmn->model == CMN600 && has_acpi_companion(cmn->dev)) { 1888 rootnode = arm_cmn600_acpi_probe(pdev, cmn); 1889 } else { 1890 rootnode = 0; 1891 cmn->base = devm_platform_ioremap_resource(pdev, 0); 1892 if (IS_ERR(cmn->base)) 1893 return PTR_ERR(cmn->base); 1894 if (cmn->model == CMN600) 1895 rootnode = arm_cmn600_of_probe(pdev->dev.of_node); 1896 } 1897 if (rootnode < 0) 1898 return rootnode; 1899 1900 err = arm_cmn_discover(cmn, rootnode); 1901 if (err) 1902 return err; 1903 1904 err = arm_cmn_init_dtcs(cmn); 1905 if (err) 1906 return err; 1907 1908 err = arm_cmn_init_irqs(cmn); 1909 if (err) 1910 return err; 1911 1912 cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); 1913 cmn->pmu = (struct pmu) { 1914 .module = THIS_MODULE, 1915 .attr_groups = arm_cmn_attr_groups, 1916 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1917 .task_ctx_nr = perf_invalid_context, 1918 .pmu_enable = arm_cmn_pmu_enable, 1919 .pmu_disable = arm_cmn_pmu_disable, 1920 .event_init = arm_cmn_event_init, 1921 .add = arm_cmn_event_add, 1922 .del = arm_cmn_event_del, 1923 .start = arm_cmn_event_start, 1924 .stop = arm_cmn_event_stop, 1925 .read = arm_cmn_event_read, 1926 .start_txn = arm_cmn_start_txn, 1927 .commit_txn = arm_cmn_commit_txn, 1928 .cancel_txn = arm_cmn_end_txn, 1929 }; 1930 1931 this_id = atomic_fetch_inc(&id); 1932 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id); 1933 if (!name) 1934 return -ENOMEM; 1935 1936 err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node); 1937 if (err) 1938 return err; 1939 1940 err = perf_pmu_register(&cmn->pmu, name, -1); 1941 if (err) 1942 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); 1943 else 1944 arm_cmn_debugfs_init(cmn, this_id); 1945 1946 return err; 1947 } 1948 1949 static int arm_cmn_remove(struct platform_device *pdev) 1950 { 1951 struct arm_cmn *cmn = platform_get_drvdata(pdev); 1952 1953 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); 1954 1955 perf_pmu_unregister(&cmn->pmu); 1956 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); 1957 debugfs_remove(cmn->debug); 1958 return 0; 1959 } 1960 1961 #ifdef CONFIG_OF 1962 static const struct of_device_id arm_cmn_of_match[] = { 1963 { .compatible = "arm,cmn-600", .data = (void *)CMN600 }, 1964 { .compatible = "arm,ci-700", .data = (void *)CI700 }, 1965 {} 1966 }; 1967 MODULE_DEVICE_TABLE(of, arm_cmn_of_match); 1968 #endif 1969 1970 #ifdef CONFIG_ACPI 1971 static const struct acpi_device_id arm_cmn_acpi_match[] = { 1972 { "ARMHC600", CMN600 }, 1973 {} 1974 }; 1975 MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match); 1976 #endif 1977 1978 static struct platform_driver arm_cmn_driver = { 1979 .driver = { 1980 .name = "arm-cmn", 1981 .of_match_table = of_match_ptr(arm_cmn_of_match), 1982 .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match), 1983 }, 1984 .probe = arm_cmn_probe, 1985 .remove = arm_cmn_remove, 1986 }; 1987 1988 static int __init arm_cmn_init(void) 1989 { 1990 int ret; 1991 1992 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1993 "perf/arm/cmn:online", 1994 arm_cmn_pmu_online_cpu, 1995 arm_cmn_pmu_offline_cpu); 1996 if (ret < 0) 1997 return ret; 1998 1999 arm_cmn_hp_state = ret; 2000 arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL); 2001 2002 ret = platform_driver_register(&arm_cmn_driver); 2003 if (ret) { 2004 cpuhp_remove_multi_state(arm_cmn_hp_state); 2005 debugfs_remove(arm_cmn_debugfs); 2006 } 2007 return ret; 2008 } 2009 2010 static void __exit arm_cmn_exit(void) 2011 { 2012 platform_driver_unregister(&arm_cmn_driver); 2013 cpuhp_remove_multi_state(arm_cmn_hp_state); 2014 debugfs_remove(arm_cmn_debugfs); 2015 } 2016 2017 module_init(arm_cmn_init); 2018 module_exit(arm_cmn_exit); 2019 2020 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 2021 MODULE_DESCRIPTION("Arm CMN-600 PMU driver"); 2022 MODULE_LICENSE("GPL v2"); 2023