1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2016-2020 Arm Limited 3 // CMN-600 Coherent Mesh Network PMU driver 4 5 #include <linux/acpi.h> 6 #include <linux/bitfield.h> 7 #include <linux/bitops.h> 8 #include <linux/debugfs.h> 9 #include <linux/interrupt.h> 10 #include <linux/io.h> 11 #include <linux/io-64-nonatomic-lo-hi.h> 12 #include <linux/kernel.h> 13 #include <linux/list.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/perf_event.h> 17 #include <linux/platform_device.h> 18 #include <linux/slab.h> 19 #include <linux/sort.h> 20 21 /* Common register stuff */ 22 #define CMN_NODE_INFO 0x0000 23 #define CMN_NI_NODE_TYPE GENMASK_ULL(15, 0) 24 #define CMN_NI_NODE_ID GENMASK_ULL(31, 16) 25 #define CMN_NI_LOGICAL_ID GENMASK_ULL(47, 32) 26 27 #define CMN_NODEID_DEVID(reg) ((reg) & 3) 28 #define CMN_NODEID_EXT_DEVID(reg) ((reg) & 1) 29 #define CMN_NODEID_PID(reg) (((reg) >> 2) & 1) 30 #define CMN_NODEID_EXT_PID(reg) (((reg) >> 1) & 3) 31 #define CMN_NODEID_1x1_PID(reg) (((reg) >> 2) & 7) 32 #define CMN_NODEID_X(reg, bits) ((reg) >> (3 + (bits))) 33 #define CMN_NODEID_Y(reg, bits) (((reg) >> 3) & ((1U << (bits)) - 1)) 34 35 #define CMN_CHILD_INFO 0x0080 36 #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0) 37 #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) 38 39 #define CMN_CHILD_NODE_ADDR GENMASK(27, 0) 40 #define CMN_CHILD_NODE_EXTERNAL BIT(31) 41 42 #define CMN_MAX_DIMENSION 8 43 #define CMN_MAX_XPS (CMN_MAX_DIMENSION * CMN_MAX_DIMENSION) 44 #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4) 45 46 /* The CFG node has various info besides the discovery tree */ 47 #define CMN_CFGM_PERIPH_ID_2 0x0010 48 #define CMN_CFGM_PID2_REVISION GENMASK(7, 4) 49 50 #define CMN_CFGM_INFO_GLOBAL 0x900 51 #define CMN_INFO_MULTIPLE_DTM_EN BIT_ULL(63) 52 #define CMN_INFO_RSP_VC_NUM GENMASK_ULL(53, 52) 53 #define CMN_INFO_DAT_VC_NUM GENMASK_ULL(51, 50) 54 55 /* XPs also have some local topology info which has uses too */ 56 #define CMN_MXP__CONNECT_INFO_P0 0x0008 57 #define CMN_MXP__CONNECT_INFO_P1 0x0010 58 #define CMN_MXP__CONNECT_INFO_P2 0x0028 59 #define CMN_MXP__CONNECT_INFO_P3 0x0030 60 #define CMN_MXP__CONNECT_INFO_P4 0x0038 61 #define CMN_MXP__CONNECT_INFO_P5 0x0040 62 63 /* PMU registers occupy the 3rd 4KB page of each node's region */ 64 #define CMN_PMU_OFFSET 0x2000 65 66 /* For most nodes, this is all there is */ 67 #define CMN_PMU_EVENT_SEL 0x000 68 #define CMN_PMU_EVENTn_ID_SHIFT(n) ((n) * 8) 69 70 /* DTMs live in the PMU space of XP registers */ 71 #define CMN_DTM_WPn(n) (0x1A0 + (n) * 0x18) 72 #define CMN_DTM_WPn_CONFIG(n) (CMN_DTM_WPn(n) + 0x00) 73 #define CMN_DTM_WPn_CONFIG_WP_DEV_SEL2 GENMASK_ULL(18,17) 74 #define CMN_DTM_WPn_CONFIG_WP_COMBINE BIT(9) 75 #define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE BIT(8) 76 #define CMN600_WPn_CONFIG_WP_COMBINE BIT(6) 77 #define CMN600_WPn_CONFIG_WP_EXCLUSIVE BIT(5) 78 #define CMN_DTM_WPn_CONFIG_WP_GRP GENMASK_ULL(5, 4) 79 #define CMN_DTM_WPn_CONFIG_WP_CHN_SEL GENMASK_ULL(3, 1) 80 #define CMN_DTM_WPn_CONFIG_WP_DEV_SEL BIT(0) 81 #define CMN_DTM_WPn_VAL(n) (CMN_DTM_WPn(n) + 0x08) 82 #define CMN_DTM_WPn_MASK(n) (CMN_DTM_WPn(n) + 0x10) 83 84 #define CMN_DTM_PMU_CONFIG 0x210 85 #define CMN__PMEVCNT0_INPUT_SEL GENMASK_ULL(37, 32) 86 #define CMN__PMEVCNT0_INPUT_SEL_WP 0x00 87 #define CMN__PMEVCNT0_INPUT_SEL_XP 0x04 88 #define CMN__PMEVCNT0_INPUT_SEL_DEV 0x10 89 #define CMN__PMEVCNT0_GLOBAL_NUM GENMASK_ULL(18, 16) 90 #define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n) ((n) * 4) 91 #define CMN__PMEVCNT_PAIRED(n) BIT(4 + (n)) 92 #define CMN__PMEVCNT23_COMBINED BIT(2) 93 #define CMN__PMEVCNT01_COMBINED BIT(1) 94 #define CMN_DTM_PMU_CONFIG_PMU_EN BIT(0) 95 96 #define CMN_DTM_PMEVCNT 0x220 97 98 #define CMN_DTM_PMEVCNTSR 0x240 99 100 #define CMN_DTM_UNIT_INFO 0x0910 101 102 #define CMN_DTM_NUM_COUNTERS 4 103 /* Want more local counters? Why not replicate the whole DTM! Ugh... */ 104 #define CMN_DTM_OFFSET(n) ((n) * 0x200) 105 106 /* The DTC node is where the magic happens */ 107 #define CMN_DT_DTC_CTL 0x0a00 108 #define CMN_DT_DTC_CTL_DT_EN BIT(0) 109 110 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */ 111 #define _CMN_DT_CNT_REG(n) ((((n) / 2) * 4 + (n) % 2) * 4) 112 #define CMN_DT_PMEVCNT(n) (CMN_PMU_OFFSET + _CMN_DT_CNT_REG(n)) 113 #define CMN_DT_PMCCNTR (CMN_PMU_OFFSET + 0x40) 114 115 #define CMN_DT_PMEVCNTSR(n) (CMN_PMU_OFFSET + 0x50 + _CMN_DT_CNT_REG(n)) 116 #define CMN_DT_PMCCNTRSR (CMN_PMU_OFFSET + 0x90) 117 118 #define CMN_DT_PMCR (CMN_PMU_OFFSET + 0x100) 119 #define CMN_DT_PMCR_PMU_EN BIT(0) 120 #define CMN_DT_PMCR_CNTR_RST BIT(5) 121 #define CMN_DT_PMCR_OVFL_INTR_EN BIT(6) 122 123 #define CMN_DT_PMOVSR (CMN_PMU_OFFSET + 0x118) 124 #define CMN_DT_PMOVSR_CLR (CMN_PMU_OFFSET + 0x120) 125 126 #define CMN_DT_PMSSR (CMN_PMU_OFFSET + 0x128) 127 #define CMN_DT_PMSSR_SS_STATUS(n) BIT(n) 128 129 #define CMN_DT_PMSRR (CMN_PMU_OFFSET + 0x130) 130 #define CMN_DT_PMSRR_SS_REQ BIT(0) 131 132 #define CMN_DT_NUM_COUNTERS 8 133 #define CMN_MAX_DTCS 4 134 135 /* 136 * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles, 137 * so throwing away one bit to make overflow handling easy is no big deal. 138 */ 139 #define CMN_COUNTER_INIT 0x80000000 140 /* Similarly for the 40-bit cycle counter */ 141 #define CMN_CC_INIT 0x8000000000ULL 142 143 144 /* Event attributes */ 145 #define CMN_CONFIG_TYPE GENMASK_ULL(15, 0) 146 #define CMN_CONFIG_EVENTID GENMASK_ULL(23, 16) 147 #define CMN_CONFIG_OCCUPID GENMASK_ULL(27, 24) 148 #define CMN_CONFIG_BYNODEID BIT_ULL(31) 149 #define CMN_CONFIG_NODEID GENMASK_ULL(47, 32) 150 151 #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config) 152 #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config) 153 #define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config) 154 #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config) 155 #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config) 156 157 #define CMN_CONFIG_WP_COMBINE GENMASK_ULL(27, 24) 158 #define CMN_CONFIG_WP_DEV_SEL GENMASK_ULL(50, 48) 159 #define CMN_CONFIG_WP_CHN_SEL GENMASK_ULL(55, 51) 160 /* Note that we don't yet support the tertiary match group on newer IPs */ 161 #define CMN_CONFIG_WP_GRP BIT_ULL(56) 162 #define CMN_CONFIG_WP_EXCLUSIVE BIT_ULL(57) 163 #define CMN_CONFIG1_WP_VAL GENMASK_ULL(63, 0) 164 #define CMN_CONFIG2_WP_MASK GENMASK_ULL(63, 0) 165 166 #define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config) 167 #define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config) 168 #define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config) 169 #define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config) 170 #define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config) 171 #define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1) 172 #define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2) 173 174 /* Made-up event IDs for watchpoint direction */ 175 #define CMN_WP_UP 0 176 #define CMN_WP_DOWN 2 177 178 179 enum cmn_model { 180 CMN_ANY = -1, 181 CMN600 = 1, 182 CI700 = 2, 183 }; 184 185 /* CMN-600 r0px shouldn't exist in silicon, thankfully */ 186 enum cmn_revision { 187 CMN600_R1P0, 188 CMN600_R1P1, 189 CMN600_R1P2, 190 CMN600_R1P3, 191 CMN600_R2P0, 192 CMN600_R3P0, 193 CMN600_R3P1, 194 CI700_R0P0 = 0, 195 CI700_R1P0, 196 CI700_R2P0, 197 }; 198 199 enum cmn_node_type { 200 CMN_TYPE_INVALID, 201 CMN_TYPE_DVM, 202 CMN_TYPE_CFG, 203 CMN_TYPE_DTC, 204 CMN_TYPE_HNI, 205 CMN_TYPE_HNF, 206 CMN_TYPE_XP, 207 CMN_TYPE_SBSX, 208 CMN_TYPE_MPAM_S, 209 CMN_TYPE_MPAM_NS, 210 CMN_TYPE_RNI, 211 CMN_TYPE_RND = 0xd, 212 CMN_TYPE_RNSAM = 0xf, 213 CMN_TYPE_MTSX, 214 CMN_TYPE_CXRA = 0x100, 215 CMN_TYPE_CXHA = 0x101, 216 CMN_TYPE_CXLA = 0x102, 217 /* Not a real node type */ 218 CMN_TYPE_WP = 0x7770 219 }; 220 221 struct arm_cmn_node { 222 void __iomem *pmu_base; 223 u16 id, logid; 224 enum cmn_node_type type; 225 226 int dtm; 227 union { 228 /* DN/HN-F/CXHA */ 229 struct { 230 u8 occupid_val; 231 u8 occupid_count; 232 }; 233 /* XP */ 234 u8 dtc; 235 }; 236 union { 237 u8 event[4]; 238 __le32 event_sel; 239 }; 240 }; 241 242 struct arm_cmn_dtm { 243 void __iomem *base; 244 u32 pmu_config_low; 245 union { 246 u8 input_sel[4]; 247 __le32 pmu_config_high; 248 }; 249 s8 wp_event[4]; 250 }; 251 252 struct arm_cmn_dtc { 253 void __iomem *base; 254 int irq; 255 int irq_friend; 256 bool cc_active; 257 258 struct perf_event *counters[CMN_DT_NUM_COUNTERS]; 259 struct perf_event *cycles; 260 }; 261 262 #define CMN_STATE_DISABLED BIT(0) 263 #define CMN_STATE_TXN BIT(1) 264 265 struct arm_cmn { 266 struct device *dev; 267 void __iomem *base; 268 unsigned int state; 269 270 enum cmn_revision rev; 271 enum cmn_model model; 272 u8 mesh_x; 273 u8 mesh_y; 274 u16 num_xps; 275 u16 num_dns; 276 bool multi_dtm; 277 u8 ports_used; 278 struct { 279 unsigned int rsp_vc_num : 2; 280 unsigned int dat_vc_num : 2; 281 }; 282 283 struct arm_cmn_node *xps; 284 struct arm_cmn_node *dns; 285 286 struct arm_cmn_dtm *dtms; 287 struct arm_cmn_dtc *dtc; 288 unsigned int num_dtcs; 289 290 int cpu; 291 struct hlist_node cpuhp_node; 292 293 struct pmu pmu; 294 struct dentry *debug; 295 }; 296 297 #define to_cmn(p) container_of(p, struct arm_cmn, pmu) 298 299 static int arm_cmn_hp_state; 300 301 struct arm_cmn_nodeid { 302 u8 x; 303 u8 y; 304 u8 port; 305 u8 dev; 306 }; 307 308 static int arm_cmn_xyidbits(const struct arm_cmn *cmn) 309 { 310 int dim = max(cmn->mesh_x, cmn->mesh_y); 311 312 return dim > 4 ? 3 : 2; 313 } 314 315 static struct arm_cmn_nodeid arm_cmn_nid(const struct arm_cmn *cmn, u16 id) 316 { 317 struct arm_cmn_nodeid nid; 318 319 if (cmn->num_xps == 1) { 320 nid.x = 0; 321 nid.y = 0; 322 nid.port = CMN_NODEID_1x1_PID(id); 323 nid.dev = CMN_NODEID_DEVID(id); 324 } else { 325 int bits = arm_cmn_xyidbits(cmn); 326 327 nid.x = CMN_NODEID_X(id, bits); 328 nid.y = CMN_NODEID_Y(id, bits); 329 if (cmn->ports_used & 0xc) { 330 nid.port = CMN_NODEID_EXT_PID(id); 331 nid.dev = CMN_NODEID_EXT_DEVID(id); 332 } else { 333 nid.port = CMN_NODEID_PID(id); 334 nid.dev = CMN_NODEID_DEVID(id); 335 } 336 } 337 return nid; 338 } 339 340 static struct arm_cmn_node *arm_cmn_node_to_xp(const struct arm_cmn *cmn, 341 const struct arm_cmn_node *dn) 342 { 343 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 344 int xp_idx = cmn->mesh_x * nid.y + nid.x; 345 346 return cmn->xps + xp_idx; 347 } 348 static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn, 349 enum cmn_node_type type) 350 { 351 struct arm_cmn_node *dn; 352 353 for (dn = cmn->dns; dn->type; dn++) 354 if (dn->type == type) 355 return dn; 356 return NULL; 357 } 358 359 static struct dentry *arm_cmn_debugfs; 360 361 #ifdef CONFIG_DEBUG_FS 362 static const char *arm_cmn_device_type(u8 type) 363 { 364 switch(type) { 365 case 0x01: return " RN-I |"; 366 case 0x02: return " RN-D |"; 367 case 0x04: return " RN-F_B |"; 368 case 0x05: return "RN-F_B_E|"; 369 case 0x06: return " RN-F_A |"; 370 case 0x07: return "RN-F_A_E|"; 371 case 0x08: return " HN-T |"; 372 case 0x09: return " HN-I |"; 373 case 0x0a: return " HN-D |"; 374 case 0x0c: return " SN-F |"; 375 case 0x0d: return " SBSX |"; 376 case 0x0e: return " HN-F |"; 377 case 0x0f: return " SN-F_E |"; 378 case 0x10: return " SN-F_D |"; 379 case 0x11: return " CXHA |"; 380 case 0x12: return " CXRA |"; 381 case 0x13: return " CXRH |"; 382 case 0x14: return " RN-F_D |"; 383 case 0x15: return "RN-F_D_E|"; 384 case 0x16: return " RN-F_C |"; 385 case 0x17: return "RN-F_C_E|"; 386 case 0x1c: return " MTSX |"; 387 default: return " |"; 388 } 389 } 390 391 static void arm_cmn_show_logid(struct seq_file *s, int x, int y, int p, int d) 392 { 393 struct arm_cmn *cmn = s->private; 394 struct arm_cmn_node *dn; 395 396 for (dn = cmn->dns; dn->type; dn++) { 397 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 398 399 if (dn->type == CMN_TYPE_XP) 400 continue; 401 /* Ignore the extra components that will overlap on some ports */ 402 if (dn->type < CMN_TYPE_HNI) 403 continue; 404 405 if (nid.x != x || nid.y != y || nid.port != p || nid.dev != d) 406 continue; 407 408 seq_printf(s, " #%-2d |", dn->logid); 409 return; 410 } 411 seq_puts(s, " |"); 412 } 413 414 static int arm_cmn_map_show(struct seq_file *s, void *data) 415 { 416 struct arm_cmn *cmn = s->private; 417 int x, y, p, pmax = fls(cmn->ports_used); 418 419 seq_puts(s, " X"); 420 for (x = 0; x < cmn->mesh_x; x++) 421 seq_printf(s, " %d ", x); 422 seq_puts(s, "\nY P D+"); 423 y = cmn->mesh_y; 424 while (y--) { 425 int xp_base = cmn->mesh_x * y; 426 u8 port[6][CMN_MAX_DIMENSION]; 427 428 for (x = 0; x < cmn->mesh_x; x++) 429 seq_puts(s, "--------+"); 430 431 seq_printf(s, "\n%d |", y); 432 for (x = 0; x < cmn->mesh_x; x++) { 433 struct arm_cmn_node *xp = cmn->xps + xp_base + x; 434 void __iomem *base = xp->pmu_base - CMN_PMU_OFFSET; 435 436 port[0][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P0); 437 port[1][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P1); 438 port[2][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P2); 439 port[3][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P3); 440 port[4][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P4); 441 port[5][x] = readl_relaxed(base + CMN_MXP__CONNECT_INFO_P5); 442 seq_printf(s, " XP #%-2d |", xp_base + x); 443 } 444 445 seq_puts(s, "\n |"); 446 for (x = 0; x < cmn->mesh_x; x++) { 447 u8 dtc = cmn->xps[xp_base + x].dtc; 448 449 if (dtc & (dtc - 1)) 450 seq_puts(s, " DTC ?? |"); 451 else 452 seq_printf(s, " DTC %ld |", __ffs(dtc)); 453 } 454 seq_puts(s, "\n |"); 455 for (x = 0; x < cmn->mesh_x; x++) 456 seq_puts(s, "........|"); 457 458 for (p = 0; p < pmax; p++) { 459 seq_printf(s, "\n %d |", p); 460 for (x = 0; x < cmn->mesh_x; x++) 461 seq_puts(s, arm_cmn_device_type(port[p][x])); 462 seq_puts(s, "\n 0|"); 463 for (x = 0; x < cmn->mesh_x; x++) 464 arm_cmn_show_logid(s, x, y, p, 0); 465 seq_puts(s, "\n 1|"); 466 for (x = 0; x < cmn->mesh_x; x++) 467 arm_cmn_show_logid(s, x, y, p, 1); 468 } 469 seq_puts(s, "\n-----+"); 470 } 471 for (x = 0; x < cmn->mesh_x; x++) 472 seq_puts(s, "--------+"); 473 seq_puts(s, "\n"); 474 return 0; 475 } 476 DEFINE_SHOW_ATTRIBUTE(arm_cmn_map); 477 478 static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) 479 { 480 const char *name = "map"; 481 482 if (id > 0) 483 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id); 484 if (!name) 485 return; 486 487 cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops); 488 } 489 #else 490 static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {} 491 #endif 492 493 struct arm_cmn_hw_event { 494 struct arm_cmn_node *dn; 495 u64 dtm_idx[2]; 496 unsigned int dtc_idx; 497 u8 dtcs_used; 498 u8 num_dns; 499 u8 dtm_offset; 500 }; 501 502 #define for_each_hw_dn(hw, dn, i) \ 503 for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++) 504 505 static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event) 506 { 507 BUILD_BUG_ON(sizeof(struct arm_cmn_hw_event) > offsetof(struct hw_perf_event, target)); 508 return (struct arm_cmn_hw_event *)&event->hw; 509 } 510 511 static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val) 512 { 513 x[pos / 32] |= (u64)val << ((pos % 32) * 2); 514 } 515 516 static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos) 517 { 518 return (x[pos / 32] >> ((pos % 32) * 2)) & 3; 519 } 520 521 struct arm_cmn_event_attr { 522 struct device_attribute attr; 523 enum cmn_model model; 524 enum cmn_node_type type; 525 u8 eventid; 526 u8 occupid; 527 }; 528 529 struct arm_cmn_format_attr { 530 struct device_attribute attr; 531 u64 field; 532 int config; 533 }; 534 535 #define CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid) \ 536 (&((struct arm_cmn_event_attr[]) {{ \ 537 .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ 538 .model = _model, \ 539 .type = _type, \ 540 .eventid = _eventid, \ 541 .occupid = _occupid, \ 542 }})[0].attr.attr) 543 544 static bool arm_cmn_is_occup_event(enum cmn_model model, 545 enum cmn_node_type type, unsigned int id) 546 { 547 if (type == CMN_TYPE_DVM) 548 return (model == CMN600 && id == 0x05) || 549 (model == CI700 && id == 0x0c); 550 return type == CMN_TYPE_HNF && id == 0x0f; 551 } 552 553 static ssize_t arm_cmn_event_show(struct device *dev, 554 struct device_attribute *attr, char *buf) 555 { 556 struct arm_cmn_event_attr *eattr; 557 558 eattr = container_of(attr, typeof(*eattr), attr); 559 560 if (eattr->type == CMN_TYPE_DTC) 561 return sysfs_emit(buf, "type=0x%x\n", eattr->type); 562 563 if (eattr->type == CMN_TYPE_WP) 564 return sysfs_emit(buf, 565 "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", 566 eattr->type, eattr->eventid); 567 568 if (arm_cmn_is_occup_event(eattr->model, eattr->type, eattr->eventid)) 569 return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n", 570 eattr->type, eattr->eventid, eattr->occupid); 571 572 return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, 573 eattr->eventid); 574 } 575 576 static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, 577 struct attribute *attr, 578 int unused) 579 { 580 struct device *dev = kobj_to_dev(kobj); 581 struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev)); 582 struct arm_cmn_event_attr *eattr; 583 584 eattr = container_of(attr, typeof(*eattr), attr.attr); 585 586 if (!(eattr->model & cmn->model)) 587 return 0; 588 589 /* Watchpoints aren't nodes, so avoid confusion */ 590 if (eattr->type == CMN_TYPE_WP) 591 return attr->mode; 592 593 /* Hide XP events for unused interfaces/channels */ 594 if (eattr->type == CMN_TYPE_XP) { 595 unsigned int intf = (eattr->eventid >> 2) & 7; 596 unsigned int chan = eattr->eventid >> 5; 597 598 if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3))) 599 return 0; 600 601 if (chan == 4 && cmn->model == CMN600) 602 return 0; 603 604 if ((chan == 5 && cmn->rsp_vc_num < 2) || 605 (chan == 6 && cmn->dat_vc_num < 2)) 606 return 0; 607 } 608 609 /* Revision-specific differences */ 610 if (cmn->model == CMN600 && cmn->rev < CMN600_R1P2) { 611 if (eattr->type == CMN_TYPE_HNF && eattr->eventid == 0x1b) 612 return 0; 613 } 614 615 if (!arm_cmn_node(cmn, eattr->type)) 616 return 0; 617 618 return attr->mode; 619 } 620 621 #define _CMN_EVENT_DVM(_model, _name, _event, _occup) \ 622 CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup) 623 #define CMN_EVENT_DTC(_name) \ 624 CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0, 0) 625 #define _CMN_EVENT_HNF(_model, _name, _event, _occup) \ 626 CMN_EVENT_ATTR(_model, hnf_##_name, CMN_TYPE_HNF, _event, _occup) 627 #define CMN_EVENT_HNI(_name, _event) \ 628 CMN_EVENT_ATTR(CMN_ANY, hni_##_name, CMN_TYPE_HNI, _event, 0) 629 #define __CMN_EVENT_XP(_name, _event) \ 630 CMN_EVENT_ATTR(CMN_ANY, mxp_##_name, CMN_TYPE_XP, _event, 0) 631 #define CMN_EVENT_SBSX(_model, _name, _event) \ 632 CMN_EVENT_ATTR(_model, sbsx_##_name, CMN_TYPE_SBSX, _event, 0) 633 #define CMN_EVENT_RNID(_model, _name, _event) \ 634 CMN_EVENT_ATTR(_model, rnid_##_name, CMN_TYPE_RNI, _event, 0) 635 #define CMN_EVENT_MTSX(_name, _event) \ 636 CMN_EVENT_ATTR(CMN_ANY, mtsx_##_name, CMN_TYPE_MTSX, _event, 0) 637 638 #define CMN_EVENT_DVM(_model, _name, _event) \ 639 _CMN_EVENT_DVM(_model, _name, _event, 0) 640 #define CMN_EVENT_HNF(_model, _name, _event) \ 641 _CMN_EVENT_HNF(_model, _name, _event, 0) 642 #define _CMN_EVENT_XP(_name, _event) \ 643 __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \ 644 __CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)), \ 645 __CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)), \ 646 __CMN_EVENT_XP(s_##_name, (_event) | (3 << 2)), \ 647 __CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)), \ 648 __CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2)), \ 649 __CMN_EVENT_XP(p2_##_name, (_event) | (6 << 2)), \ 650 __CMN_EVENT_XP(p3_##_name, (_event) | (7 << 2)) 651 652 /* Good thing there are only 3 fundamental XP events... */ 653 #define CMN_EVENT_XP(_name, _event) \ 654 _CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)), \ 655 _CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)), \ 656 _CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)), \ 657 _CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5)), \ 658 _CMN_EVENT_XP(pub_##_name, (_event) | (4 << 5)), \ 659 _CMN_EVENT_XP(rsp2_##_name, (_event) | (5 << 5)), \ 660 _CMN_EVENT_XP(dat2_##_name, (_event) | (6 << 5)) 661 662 663 static struct attribute *arm_cmn_event_attrs[] = { 664 CMN_EVENT_DTC(cycles), 665 666 /* 667 * DVM node events conflict with HN-I events in the equivalent PMU 668 * slot, but our lazy short-cut of using the DTM counter index for 669 * the PMU index as well happens to avoid that by construction. 670 */ 671 CMN_EVENT_DVM(CMN600, rxreq_dvmop, 0x01), 672 CMN_EVENT_DVM(CMN600, rxreq_dvmsync, 0x02), 673 CMN_EVENT_DVM(CMN600, rxreq_dvmop_vmid_filtered, 0x03), 674 CMN_EVENT_DVM(CMN600, rxreq_retried, 0x04), 675 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_all, 0x05, 0), 676 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_dvmop, 0x05, 1), 677 _CMN_EVENT_DVM(CMN600, rxreq_trk_occupancy_dvmsync, 0x05, 2), 678 CMN_EVENT_DVM(CI700, dvmop_tlbi, 0x01), 679 CMN_EVENT_DVM(CI700, dvmop_bpi, 0x02), 680 CMN_EVENT_DVM(CI700, dvmop_pici, 0x03), 681 CMN_EVENT_DVM(CI700, dvmop_vici, 0x04), 682 CMN_EVENT_DVM(CI700, dvmsync, 0x05), 683 CMN_EVENT_DVM(CI700, vmid_filtered, 0x06), 684 CMN_EVENT_DVM(CI700, rndop_filtered, 0x07), 685 CMN_EVENT_DVM(CI700, retry, 0x08), 686 CMN_EVENT_DVM(CI700, txsnp_flitv, 0x09), 687 CMN_EVENT_DVM(CI700, txsnp_stall, 0x0a), 688 CMN_EVENT_DVM(CI700, trkfull, 0x0b), 689 _CMN_EVENT_DVM(CI700, trk_occupancy_all, 0x0c, 0), 690 _CMN_EVENT_DVM(CI700, trk_occupancy_dvmop, 0x0c, 1), 691 _CMN_EVENT_DVM(CI700, trk_occupancy_dvmsync, 0x0c, 2), 692 693 CMN_EVENT_HNF(CMN_ANY, cache_miss, 0x01), 694 CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access, 0x02), 695 CMN_EVENT_HNF(CMN_ANY, cache_fill, 0x03), 696 CMN_EVENT_HNF(CMN_ANY, pocq_retry, 0x04), 697 CMN_EVENT_HNF(CMN_ANY, pocq_reqs_recvd, 0x05), 698 CMN_EVENT_HNF(CMN_ANY, sf_hit, 0x06), 699 CMN_EVENT_HNF(CMN_ANY, sf_evictions, 0x07), 700 CMN_EVENT_HNF(CMN_ANY, dir_snoops_sent, 0x08), 701 CMN_EVENT_HNF(CMN_ANY, brd_snoops_sent, 0x09), 702 CMN_EVENT_HNF(CMN_ANY, slc_eviction, 0x0a), 703 CMN_EVENT_HNF(CMN_ANY, slc_fill_invalid_way, 0x0b), 704 CMN_EVENT_HNF(CMN_ANY, mc_retries, 0x0c), 705 CMN_EVENT_HNF(CMN_ANY, mc_reqs, 0x0d), 706 CMN_EVENT_HNF(CMN_ANY, qos_hh_retry, 0x0e), 707 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_all, 0x0f, 0), 708 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_read, 0x0f, 1), 709 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_write, 0x0f, 2), 710 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_atomic, 0x0f, 3), 711 _CMN_EVENT_HNF(CMN_ANY, qos_pocq_occupancy_stash, 0x0f, 4), 712 CMN_EVENT_HNF(CMN_ANY, pocq_addrhaz, 0x10), 713 CMN_EVENT_HNF(CMN_ANY, pocq_atomic_addrhaz, 0x11), 714 CMN_EVENT_HNF(CMN_ANY, ld_st_swp_adq_full, 0x12), 715 CMN_EVENT_HNF(CMN_ANY, cmp_adq_full, 0x13), 716 CMN_EVENT_HNF(CMN_ANY, txdat_stall, 0x14), 717 CMN_EVENT_HNF(CMN_ANY, txrsp_stall, 0x15), 718 CMN_EVENT_HNF(CMN_ANY, seq_full, 0x16), 719 CMN_EVENT_HNF(CMN_ANY, seq_hit, 0x17), 720 CMN_EVENT_HNF(CMN_ANY, snp_sent, 0x18), 721 CMN_EVENT_HNF(CMN_ANY, sfbi_dir_snp_sent, 0x19), 722 CMN_EVENT_HNF(CMN_ANY, sfbi_brd_snp_sent, 0x1a), 723 CMN_EVENT_HNF(CMN_ANY, snp_sent_untrk, 0x1b), 724 CMN_EVENT_HNF(CMN_ANY, intv_dirty, 0x1c), 725 CMN_EVENT_HNF(CMN_ANY, stash_snp_sent, 0x1d), 726 CMN_EVENT_HNF(CMN_ANY, stash_data_pull, 0x1e), 727 CMN_EVENT_HNF(CMN_ANY, snp_fwded, 0x1f), 728 CMN_EVENT_HNF(CI700, atomic_fwd, 0x20), 729 CMN_EVENT_HNF(CI700, mpam_hardlim, 0x21), 730 CMN_EVENT_HNF(CI700, mpam_softlim, 0x22), 731 732 CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl, 0x20), 733 CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl, 0x21), 734 CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl, 0x22), 735 CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl, 0x23), 736 CMN_EVENT_HNI(wdb_occ_cnt_ovfl, 0x24), 737 CMN_EVENT_HNI(rrt_rd_alloc, 0x25), 738 CMN_EVENT_HNI(rrt_wr_alloc, 0x26), 739 CMN_EVENT_HNI(rdt_rd_alloc, 0x27), 740 CMN_EVENT_HNI(rdt_wr_alloc, 0x28), 741 CMN_EVENT_HNI(wdb_alloc, 0x29), 742 CMN_EVENT_HNI(txrsp_retryack, 0x2a), 743 CMN_EVENT_HNI(arvalid_no_arready, 0x2b), 744 CMN_EVENT_HNI(arready_no_arvalid, 0x2c), 745 CMN_EVENT_HNI(awvalid_no_awready, 0x2d), 746 CMN_EVENT_HNI(awready_no_awvalid, 0x2e), 747 CMN_EVENT_HNI(wvalid_no_wready, 0x2f), 748 CMN_EVENT_HNI(txdat_stall, 0x30), 749 CMN_EVENT_HNI(nonpcie_serialization, 0x31), 750 CMN_EVENT_HNI(pcie_serialization, 0x32), 751 752 CMN_EVENT_XP(txflit_valid, 0x01), 753 CMN_EVENT_XP(txflit_stall, 0x02), 754 CMN_EVENT_XP(partial_dat_flit, 0x03), 755 /* We treat watchpoints as a special made-up class of XP events */ 756 CMN_EVENT_ATTR(CMN_ANY, watchpoint_up, CMN_TYPE_WP, CMN_WP_UP, 0), 757 CMN_EVENT_ATTR(CMN_ANY, watchpoint_down, CMN_TYPE_WP, CMN_WP_DOWN, 0), 758 759 CMN_EVENT_SBSX(CMN_ANY, rd_req, 0x01), 760 CMN_EVENT_SBSX(CMN_ANY, wr_req, 0x02), 761 CMN_EVENT_SBSX(CMN_ANY, cmo_req, 0x03), 762 CMN_EVENT_SBSX(CMN_ANY, txrsp_retryack, 0x04), 763 CMN_EVENT_SBSX(CMN_ANY, txdat_flitv, 0x05), 764 CMN_EVENT_SBSX(CMN_ANY, txrsp_flitv, 0x06), 765 CMN_EVENT_SBSX(CMN_ANY, rd_req_trkr_occ_cnt_ovfl, 0x11), 766 CMN_EVENT_SBSX(CMN_ANY, wr_req_trkr_occ_cnt_ovfl, 0x12), 767 CMN_EVENT_SBSX(CMN_ANY, cmo_req_trkr_occ_cnt_ovfl, 0x13), 768 CMN_EVENT_SBSX(CMN_ANY, wdb_occ_cnt_ovfl, 0x14), 769 CMN_EVENT_SBSX(CMN_ANY, rd_axi_trkr_occ_cnt_ovfl, 0x15), 770 CMN_EVENT_SBSX(CMN_ANY, cmo_axi_trkr_occ_cnt_ovfl, 0x16), 771 CMN_EVENT_SBSX(CI700, rdb_occ_cnt_ovfl, 0x17), 772 CMN_EVENT_SBSX(CMN_ANY, arvalid_no_arready, 0x21), 773 CMN_EVENT_SBSX(CMN_ANY, awvalid_no_awready, 0x22), 774 CMN_EVENT_SBSX(CMN_ANY, wvalid_no_wready, 0x23), 775 CMN_EVENT_SBSX(CMN_ANY, txdat_stall, 0x24), 776 CMN_EVENT_SBSX(CMN_ANY, txrsp_stall, 0x25), 777 778 CMN_EVENT_RNID(CMN_ANY, s0_rdata_beats, 0x01), 779 CMN_EVENT_RNID(CMN_ANY, s1_rdata_beats, 0x02), 780 CMN_EVENT_RNID(CMN_ANY, s2_rdata_beats, 0x03), 781 CMN_EVENT_RNID(CMN_ANY, rxdat_flits, 0x04), 782 CMN_EVENT_RNID(CMN_ANY, txdat_flits, 0x05), 783 CMN_EVENT_RNID(CMN_ANY, txreq_flits_total, 0x06), 784 CMN_EVENT_RNID(CMN_ANY, txreq_flits_retried, 0x07), 785 CMN_EVENT_RNID(CMN_ANY, rrt_occ_ovfl, 0x08), 786 CMN_EVENT_RNID(CMN_ANY, wrt_occ_ovfl, 0x09), 787 CMN_EVENT_RNID(CMN_ANY, txreq_flits_replayed, 0x0a), 788 CMN_EVENT_RNID(CMN_ANY, wrcancel_sent, 0x0b), 789 CMN_EVENT_RNID(CMN_ANY, s0_wdata_beats, 0x0c), 790 CMN_EVENT_RNID(CMN_ANY, s1_wdata_beats, 0x0d), 791 CMN_EVENT_RNID(CMN_ANY, s2_wdata_beats, 0x0e), 792 CMN_EVENT_RNID(CMN_ANY, rrt_alloc, 0x0f), 793 CMN_EVENT_RNID(CMN_ANY, wrt_alloc, 0x10), 794 CMN_EVENT_RNID(CMN600, rdb_unord, 0x11), 795 CMN_EVENT_RNID(CMN600, rdb_replay, 0x12), 796 CMN_EVENT_RNID(CMN600, rdb_hybrid, 0x13), 797 CMN_EVENT_RNID(CMN600, rdb_ord, 0x14), 798 CMN_EVENT_RNID(CI700, padb_occ_ovfl, 0x11), 799 CMN_EVENT_RNID(CI700, rpdb_occ_ovfl, 0x12), 800 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice1, 0x13), 801 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice2, 0x14), 802 CMN_EVENT_RNID(CI700, rrt_occup_ovfl_slice3, 0x15), 803 CMN_EVENT_RNID(CI700, wrt_throttled, 0x16), 804 805 CMN_EVENT_MTSX(tc_lookup, 0x01), 806 CMN_EVENT_MTSX(tc_fill, 0x02), 807 CMN_EVENT_MTSX(tc_miss, 0x03), 808 CMN_EVENT_MTSX(tdb_forward, 0x04), 809 CMN_EVENT_MTSX(tcq_hazard, 0x05), 810 CMN_EVENT_MTSX(tcq_rd_alloc, 0x06), 811 CMN_EVENT_MTSX(tcq_wr_alloc, 0x07), 812 CMN_EVENT_MTSX(tcq_cmo_alloc, 0x08), 813 CMN_EVENT_MTSX(axi_rd_req, 0x09), 814 CMN_EVENT_MTSX(axi_wr_req, 0x0a), 815 CMN_EVENT_MTSX(tcq_occ_cnt_ovfl, 0x0b), 816 CMN_EVENT_MTSX(tdb_occ_cnt_ovfl, 0x0c), 817 818 NULL 819 }; 820 821 static const struct attribute_group arm_cmn_event_attrs_group = { 822 .name = "events", 823 .attrs = arm_cmn_event_attrs, 824 .is_visible = arm_cmn_event_attr_is_visible, 825 }; 826 827 static ssize_t arm_cmn_format_show(struct device *dev, 828 struct device_attribute *attr, char *buf) 829 { 830 struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr); 831 int lo = __ffs(fmt->field), hi = __fls(fmt->field); 832 833 if (lo == hi) 834 return sysfs_emit(buf, "config:%d\n", lo); 835 836 if (!fmt->config) 837 return sysfs_emit(buf, "config:%d-%d\n", lo, hi); 838 839 return sysfs_emit(buf, "config%d:%d-%d\n", fmt->config, lo, hi); 840 } 841 842 #define _CMN_FORMAT_ATTR(_name, _cfg, _fld) \ 843 (&((struct arm_cmn_format_attr[]) {{ \ 844 .attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL), \ 845 .config = _cfg, \ 846 .field = _fld, \ 847 }})[0].attr.attr) 848 #define CMN_FORMAT_ATTR(_name, _fld) _CMN_FORMAT_ATTR(_name, 0, _fld) 849 850 static struct attribute *arm_cmn_format_attrs[] = { 851 CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE), 852 CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID), 853 CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID), 854 CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID), 855 CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID), 856 857 CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL), 858 CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL), 859 CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP), 860 CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE), 861 CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE), 862 863 _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL), 864 _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK), 865 866 NULL 867 }; 868 869 static const struct attribute_group arm_cmn_format_attrs_group = { 870 .name = "format", 871 .attrs = arm_cmn_format_attrs, 872 }; 873 874 static ssize_t arm_cmn_cpumask_show(struct device *dev, 875 struct device_attribute *attr, char *buf) 876 { 877 struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev)); 878 879 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu)); 880 } 881 882 static struct device_attribute arm_cmn_cpumask_attr = 883 __ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL); 884 885 static struct attribute *arm_cmn_cpumask_attrs[] = { 886 &arm_cmn_cpumask_attr.attr, 887 NULL, 888 }; 889 890 static const struct attribute_group arm_cmn_cpumask_attr_group = { 891 .attrs = arm_cmn_cpumask_attrs, 892 }; 893 894 static const struct attribute_group *arm_cmn_attr_groups[] = { 895 &arm_cmn_event_attrs_group, 896 &arm_cmn_format_attrs_group, 897 &arm_cmn_cpumask_attr_group, 898 NULL 899 }; 900 901 static int arm_cmn_wp_idx(struct perf_event *event) 902 { 903 return CMN_EVENT_EVENTID(event) + CMN_EVENT_WP_GRP(event); 904 } 905 906 static u32 arm_cmn_wp_config(struct perf_event *event) 907 { 908 u32 config; 909 u32 dev = CMN_EVENT_WP_DEV_SEL(event); 910 u32 chn = CMN_EVENT_WP_CHN_SEL(event); 911 u32 grp = CMN_EVENT_WP_GRP(event); 912 u32 exc = CMN_EVENT_WP_EXCLUSIVE(event); 913 u32 combine = CMN_EVENT_WP_COMBINE(event); 914 bool is_cmn600 = to_cmn(event->pmu)->model == CMN600; 915 916 config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) | 917 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) | 918 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) | 919 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1); 920 if (exc) 921 config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_EXCLUSIVE : 922 CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE; 923 if (combine && !grp) 924 config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_COMBINE : 925 CMN_DTM_WPn_CONFIG_WP_COMBINE; 926 return config; 927 } 928 929 static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state) 930 { 931 if (!cmn->state) 932 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_PMCR); 933 cmn->state |= state; 934 } 935 936 static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state) 937 { 938 cmn->state &= ~state; 939 if (!cmn->state) 940 writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN, 941 cmn->dtc[0].base + CMN_DT_PMCR); 942 } 943 944 static void arm_cmn_pmu_enable(struct pmu *pmu) 945 { 946 arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED); 947 } 948 949 static void arm_cmn_pmu_disable(struct pmu *pmu) 950 { 951 arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED); 952 } 953 954 static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw, 955 bool snapshot) 956 { 957 struct arm_cmn_dtm *dtm = NULL; 958 struct arm_cmn_node *dn; 959 unsigned int i, offset, dtm_idx; 960 u64 reg, count = 0; 961 962 offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT; 963 for_each_hw_dn(hw, dn, i) { 964 if (dtm != &cmn->dtms[dn->dtm]) { 965 dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; 966 reg = readq_relaxed(dtm->base + offset); 967 } 968 dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 969 count += (u16)(reg >> (dtm_idx * 16)); 970 } 971 return count; 972 } 973 974 static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc) 975 { 976 u64 val = readq_relaxed(dtc->base + CMN_DT_PMCCNTR); 977 978 writeq_relaxed(CMN_CC_INIT, dtc->base + CMN_DT_PMCCNTR); 979 return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1); 980 } 981 982 static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx) 983 { 984 u32 val, pmevcnt = CMN_DT_PMEVCNT(idx); 985 986 val = readl_relaxed(dtc->base + pmevcnt); 987 writel_relaxed(CMN_COUNTER_INIT, dtc->base + pmevcnt); 988 return val - CMN_COUNTER_INIT; 989 } 990 991 static void arm_cmn_init_counter(struct perf_event *event) 992 { 993 struct arm_cmn *cmn = to_cmn(event->pmu); 994 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 995 unsigned int i, pmevcnt = CMN_DT_PMEVCNT(hw->dtc_idx); 996 u64 count; 997 998 for (i = 0; hw->dtcs_used & (1U << i); i++) { 999 writel_relaxed(CMN_COUNTER_INIT, cmn->dtc[i].base + pmevcnt); 1000 cmn->dtc[i].counters[hw->dtc_idx] = event; 1001 } 1002 1003 count = arm_cmn_read_dtm(cmn, hw, false); 1004 local64_set(&event->hw.prev_count, count); 1005 } 1006 1007 static void arm_cmn_event_read(struct perf_event *event) 1008 { 1009 struct arm_cmn *cmn = to_cmn(event->pmu); 1010 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1011 u64 delta, new, prev; 1012 unsigned long flags; 1013 unsigned int i; 1014 1015 if (hw->dtc_idx == CMN_DT_NUM_COUNTERS) { 1016 i = __ffs(hw->dtcs_used); 1017 delta = arm_cmn_read_cc(cmn->dtc + i); 1018 local64_add(delta, &event->count); 1019 return; 1020 } 1021 new = arm_cmn_read_dtm(cmn, hw, false); 1022 prev = local64_xchg(&event->hw.prev_count, new); 1023 1024 delta = new - prev; 1025 1026 local_irq_save(flags); 1027 for (i = 0; hw->dtcs_used & (1U << i); i++) { 1028 new = arm_cmn_read_counter(cmn->dtc + i, hw->dtc_idx); 1029 delta += new << 16; 1030 } 1031 local_irq_restore(flags); 1032 local64_add(delta, &event->count); 1033 } 1034 1035 static void arm_cmn_event_start(struct perf_event *event, int flags) 1036 { 1037 struct arm_cmn *cmn = to_cmn(event->pmu); 1038 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1039 struct arm_cmn_node *dn; 1040 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1041 int i; 1042 1043 if (type == CMN_TYPE_DTC) { 1044 i = __ffs(hw->dtcs_used); 1045 writeq_relaxed(CMN_CC_INIT, cmn->dtc[i].base + CMN_DT_PMCCNTR); 1046 cmn->dtc[i].cc_active = true; 1047 } else if (type == CMN_TYPE_WP) { 1048 int wp_idx = arm_cmn_wp_idx(event); 1049 u64 val = CMN_EVENT_WP_VAL(event); 1050 u64 mask = CMN_EVENT_WP_MASK(event); 1051 1052 for_each_hw_dn(hw, dn, i) { 1053 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); 1054 1055 writeq_relaxed(val, base + CMN_DTM_WPn_VAL(wp_idx)); 1056 writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx)); 1057 } 1058 } else for_each_hw_dn(hw, dn, i) { 1059 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1060 1061 dn->event[dtm_idx] = CMN_EVENT_EVENTID(event); 1062 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); 1063 } 1064 } 1065 1066 static void arm_cmn_event_stop(struct perf_event *event, int flags) 1067 { 1068 struct arm_cmn *cmn = to_cmn(event->pmu); 1069 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1070 struct arm_cmn_node *dn; 1071 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1072 int i; 1073 1074 if (type == CMN_TYPE_DTC) { 1075 i = __ffs(hw->dtcs_used); 1076 cmn->dtc[i].cc_active = false; 1077 } else if (type == CMN_TYPE_WP) { 1078 int wp_idx = arm_cmn_wp_idx(event); 1079 1080 for_each_hw_dn(hw, dn, i) { 1081 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); 1082 1083 writeq_relaxed(0, base + CMN_DTM_WPn_MASK(wp_idx)); 1084 writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx)); 1085 } 1086 } else for_each_hw_dn(hw, dn, i) { 1087 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1088 1089 dn->event[dtm_idx] = 0; 1090 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); 1091 } 1092 1093 arm_cmn_event_read(event); 1094 } 1095 1096 struct arm_cmn_val { 1097 u8 dtm_count[CMN_MAX_DTMS]; 1098 u8 occupid[CMN_MAX_DTMS]; 1099 u8 wp[CMN_MAX_DTMS][4]; 1100 int dtc_count; 1101 bool cycles; 1102 }; 1103 1104 static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, 1105 struct perf_event *event) 1106 { 1107 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1108 struct arm_cmn_node *dn; 1109 enum cmn_node_type type; 1110 int i; 1111 u8 occupid; 1112 1113 if (is_software_event(event)) 1114 return; 1115 1116 type = CMN_EVENT_TYPE(event); 1117 if (type == CMN_TYPE_DTC) { 1118 val->cycles = true; 1119 return; 1120 } 1121 1122 val->dtc_count++; 1123 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1124 occupid = CMN_EVENT_OCCUPID(event) + 1; 1125 else 1126 occupid = 0; 1127 1128 for_each_hw_dn(hw, dn, i) { 1129 int wp_idx, dtm = dn->dtm; 1130 1131 val->dtm_count[dtm]++; 1132 val->occupid[dtm] = occupid; 1133 1134 if (type != CMN_TYPE_WP) 1135 continue; 1136 1137 wp_idx = arm_cmn_wp_idx(event); 1138 val->wp[dtm][wp_idx] = CMN_EVENT_WP_COMBINE(event) + 1; 1139 } 1140 } 1141 1142 static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) 1143 { 1144 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1145 struct arm_cmn_node *dn; 1146 struct perf_event *sibling, *leader = event->group_leader; 1147 enum cmn_node_type type; 1148 struct arm_cmn_val *val; 1149 int i, ret = -EINVAL; 1150 u8 occupid; 1151 1152 if (leader == event) 1153 return 0; 1154 1155 if (event->pmu != leader->pmu && !is_software_event(leader)) 1156 return -EINVAL; 1157 1158 val = kzalloc(sizeof(*val), GFP_KERNEL); 1159 if (!val) 1160 return -ENOMEM; 1161 1162 arm_cmn_val_add_event(cmn, val, leader); 1163 for_each_sibling_event(sibling, leader) 1164 arm_cmn_val_add_event(cmn, val, sibling); 1165 1166 type = CMN_EVENT_TYPE(event); 1167 if (type == CMN_TYPE_DTC) { 1168 ret = val->cycles ? -EINVAL : 0; 1169 goto done; 1170 } 1171 1172 if (val->dtc_count == CMN_DT_NUM_COUNTERS) 1173 goto done; 1174 1175 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1176 occupid = CMN_EVENT_OCCUPID(event) + 1; 1177 else 1178 occupid = 0; 1179 1180 for_each_hw_dn(hw, dn, i) { 1181 int wp_idx, wp_cmb, dtm = dn->dtm; 1182 1183 if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) 1184 goto done; 1185 1186 if (occupid && val->occupid[dtm] && occupid != val->occupid[dtm]) 1187 goto done; 1188 1189 if (type != CMN_TYPE_WP) 1190 continue; 1191 1192 wp_idx = arm_cmn_wp_idx(event); 1193 if (val->wp[dtm][wp_idx]) 1194 goto done; 1195 1196 wp_cmb = val->wp[dtm][wp_idx ^ 1]; 1197 if (wp_cmb && wp_cmb != CMN_EVENT_WP_COMBINE(event) + 1) 1198 goto done; 1199 } 1200 1201 ret = 0; 1202 done: 1203 kfree(val); 1204 return ret; 1205 } 1206 1207 static int arm_cmn_event_init(struct perf_event *event) 1208 { 1209 struct arm_cmn *cmn = to_cmn(event->pmu); 1210 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1211 struct arm_cmn_node *dn; 1212 enum cmn_node_type type; 1213 bool bynodeid; 1214 u16 nodeid, eventid; 1215 1216 if (event->attr.type != event->pmu->type) 1217 return -ENOENT; 1218 1219 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1220 return -EINVAL; 1221 1222 event->cpu = cmn->cpu; 1223 if (event->cpu < 0) 1224 return -EINVAL; 1225 1226 type = CMN_EVENT_TYPE(event); 1227 /* DTC events (i.e. cycles) already have everything they need */ 1228 if (type == CMN_TYPE_DTC) 1229 return 0; 1230 1231 /* For watchpoints we need the actual XP node here */ 1232 if (type == CMN_TYPE_WP) { 1233 type = CMN_TYPE_XP; 1234 /* ...and we need a "real" direction */ 1235 eventid = CMN_EVENT_EVENTID(event); 1236 if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN) 1237 return -EINVAL; 1238 /* ...but the DTM may depend on which port we're watching */ 1239 if (cmn->multi_dtm) 1240 hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; 1241 } 1242 1243 bynodeid = CMN_EVENT_BYNODEID(event); 1244 nodeid = CMN_EVENT_NODEID(event); 1245 1246 hw->dn = arm_cmn_node(cmn, type); 1247 if (!hw->dn) 1248 return -EINVAL; 1249 for (dn = hw->dn; dn->type == type; dn++) { 1250 if (bynodeid && dn->id != nodeid) { 1251 hw->dn++; 1252 continue; 1253 } 1254 hw->dtcs_used |= arm_cmn_node_to_xp(cmn, dn)->dtc; 1255 hw->num_dns++; 1256 if (bynodeid) 1257 break; 1258 } 1259 1260 if (!hw->num_dns) { 1261 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, nodeid); 1262 1263 dev_dbg(cmn->dev, "invalid node 0x%x (%d,%d,%d,%d) type 0x%x\n", 1264 nodeid, nid.x, nid.y, nid.port, nid.dev, type); 1265 return -EINVAL; 1266 } 1267 1268 return arm_cmn_validate_group(cmn, event); 1269 } 1270 1271 static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, 1272 int i) 1273 { 1274 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1275 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1276 1277 while (i--) { 1278 struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; 1279 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); 1280 1281 if (type == CMN_TYPE_WP) 1282 dtm->wp_event[arm_cmn_wp_idx(event)] = -1; 1283 1284 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) 1285 hw->dn[i].occupid_count--; 1286 1287 dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); 1288 writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); 1289 } 1290 memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); 1291 1292 for (i = 0; hw->dtcs_used & (1U << i); i++) 1293 cmn->dtc[i].counters[hw->dtc_idx] = NULL; 1294 } 1295 1296 static int arm_cmn_event_add(struct perf_event *event, int flags) 1297 { 1298 struct arm_cmn *cmn = to_cmn(event->pmu); 1299 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1300 struct arm_cmn_dtc *dtc = &cmn->dtc[0]; 1301 struct arm_cmn_node *dn; 1302 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1303 unsigned int i, dtc_idx, input_sel; 1304 1305 if (type == CMN_TYPE_DTC) { 1306 i = 0; 1307 while (cmn->dtc[i].cycles) 1308 if (++i == cmn->num_dtcs) 1309 return -ENOSPC; 1310 1311 cmn->dtc[i].cycles = event; 1312 hw->dtc_idx = CMN_DT_NUM_COUNTERS; 1313 hw->dtcs_used = 1U << i; 1314 1315 if (flags & PERF_EF_START) 1316 arm_cmn_event_start(event, 0); 1317 return 0; 1318 } 1319 1320 /* Grab a free global counter first... */ 1321 dtc_idx = 0; 1322 while (dtc->counters[dtc_idx]) 1323 if (++dtc_idx == CMN_DT_NUM_COUNTERS) 1324 return -ENOSPC; 1325 1326 hw->dtc_idx = dtc_idx; 1327 1328 /* ...then the local counters to feed it. */ 1329 for_each_hw_dn(hw, dn, i) { 1330 struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; 1331 unsigned int dtm_idx, shift; 1332 u64 reg; 1333 1334 dtm_idx = 0; 1335 while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx)) 1336 if (++dtm_idx == CMN_DTM_NUM_COUNTERS) 1337 goto free_dtms; 1338 1339 if (type == CMN_TYPE_XP) { 1340 input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx; 1341 } else if (type == CMN_TYPE_WP) { 1342 int tmp, wp_idx = arm_cmn_wp_idx(event); 1343 u32 cfg = arm_cmn_wp_config(event); 1344 1345 if (dtm->wp_event[wp_idx] >= 0) 1346 goto free_dtms; 1347 1348 tmp = dtm->wp_event[wp_idx ^ 1]; 1349 if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) != 1350 CMN_EVENT_WP_COMBINE(dtc->counters[tmp])) 1351 goto free_dtms; 1352 1353 input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx; 1354 dtm->wp_event[wp_idx] = dtc_idx; 1355 writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx)); 1356 } else { 1357 struct arm_cmn_nodeid nid = arm_cmn_nid(cmn, dn->id); 1358 1359 if (cmn->multi_dtm) 1360 nid.port %= 2; 1361 1362 input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx + 1363 (nid.port << 4) + (nid.dev << 2); 1364 1365 if (arm_cmn_is_occup_event(cmn->model, type, CMN_EVENT_EVENTID(event))) { 1366 u8 occupid = CMN_EVENT_OCCUPID(event); 1367 1368 if (dn->occupid_count == 0) { 1369 dn->occupid_val = occupid; 1370 writel_relaxed(occupid, 1371 dn->pmu_base + CMN_PMU_EVENT_SEL + 4); 1372 } else if (dn->occupid_val != occupid) { 1373 goto free_dtms; 1374 } 1375 dn->occupid_count++; 1376 } 1377 } 1378 1379 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); 1380 1381 dtm->input_sel[dtm_idx] = input_sel; 1382 shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx); 1383 dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift); 1384 dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, dtc_idx) << shift; 1385 dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx); 1386 reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low; 1387 writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG); 1388 } 1389 1390 /* Go go go! */ 1391 arm_cmn_init_counter(event); 1392 1393 if (flags & PERF_EF_START) 1394 arm_cmn_event_start(event, 0); 1395 1396 return 0; 1397 1398 free_dtms: 1399 arm_cmn_event_clear(cmn, event, i); 1400 return -ENOSPC; 1401 } 1402 1403 static void arm_cmn_event_del(struct perf_event *event, int flags) 1404 { 1405 struct arm_cmn *cmn = to_cmn(event->pmu); 1406 struct arm_cmn_hw_event *hw = to_cmn_hw(event); 1407 enum cmn_node_type type = CMN_EVENT_TYPE(event); 1408 1409 arm_cmn_event_stop(event, PERF_EF_UPDATE); 1410 1411 if (type == CMN_TYPE_DTC) 1412 cmn->dtc[__ffs(hw->dtcs_used)].cycles = NULL; 1413 else 1414 arm_cmn_event_clear(cmn, event, hw->num_dns); 1415 } 1416 1417 /* 1418 * We stop the PMU for both add and read, to avoid skew across DTM counters. 1419 * In theory we could use snapshots to read without stopping, but then it 1420 * becomes a lot trickier to deal with overlow and racing against interrupts, 1421 * plus it seems they don't work properly on some hardware anyway :( 1422 */ 1423 static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags) 1424 { 1425 arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN); 1426 } 1427 1428 static void arm_cmn_end_txn(struct pmu *pmu) 1429 { 1430 arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN); 1431 } 1432 1433 static int arm_cmn_commit_txn(struct pmu *pmu) 1434 { 1435 arm_cmn_end_txn(pmu); 1436 return 0; 1437 } 1438 1439 static void arm_cmn_migrate(struct arm_cmn *cmn, unsigned int cpu) 1440 { 1441 unsigned int i; 1442 1443 perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu); 1444 for (i = 0; i < cmn->num_dtcs; i++) 1445 irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu)); 1446 cmn->cpu = cpu; 1447 } 1448 1449 static int arm_cmn_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 1450 { 1451 struct arm_cmn *cmn; 1452 int node; 1453 1454 cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node); 1455 node = dev_to_node(cmn->dev); 1456 if (node != NUMA_NO_NODE && cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node) 1457 arm_cmn_migrate(cmn, cpu); 1458 return 0; 1459 } 1460 1461 static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 1462 { 1463 struct arm_cmn *cmn; 1464 unsigned int target; 1465 int node; 1466 cpumask_t mask; 1467 1468 cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node); 1469 if (cpu != cmn->cpu) 1470 return 0; 1471 1472 node = dev_to_node(cmn->dev); 1473 if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) && 1474 cpumask_andnot(&mask, &mask, cpumask_of(cpu))) 1475 target = cpumask_any(&mask); 1476 else 1477 target = cpumask_any_but(cpu_online_mask, cpu); 1478 if (target < nr_cpu_ids) 1479 arm_cmn_migrate(cmn, target); 1480 return 0; 1481 } 1482 1483 static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id) 1484 { 1485 struct arm_cmn_dtc *dtc = dev_id; 1486 irqreturn_t ret = IRQ_NONE; 1487 1488 for (;;) { 1489 u32 status = readl_relaxed(dtc->base + CMN_DT_PMOVSR); 1490 u64 delta; 1491 int i; 1492 1493 for (i = 0; i < CMN_DTM_NUM_COUNTERS; i++) { 1494 if (status & (1U << i)) { 1495 ret = IRQ_HANDLED; 1496 if (WARN_ON(!dtc->counters[i])) 1497 continue; 1498 delta = (u64)arm_cmn_read_counter(dtc, i) << 16; 1499 local64_add(delta, &dtc->counters[i]->count); 1500 } 1501 } 1502 1503 if (status & (1U << CMN_DT_NUM_COUNTERS)) { 1504 ret = IRQ_HANDLED; 1505 if (dtc->cc_active && !WARN_ON(!dtc->cycles)) { 1506 delta = arm_cmn_read_cc(dtc); 1507 local64_add(delta, &dtc->cycles->count); 1508 } 1509 } 1510 1511 writel_relaxed(status, dtc->base + CMN_DT_PMOVSR_CLR); 1512 1513 if (!dtc->irq_friend) 1514 return ret; 1515 dtc += dtc->irq_friend; 1516 } 1517 } 1518 1519 /* We can reasonably accommodate DTCs of the same CMN sharing IRQs */ 1520 static int arm_cmn_init_irqs(struct arm_cmn *cmn) 1521 { 1522 int i, j, irq, err; 1523 1524 for (i = 0; i < cmn->num_dtcs; i++) { 1525 irq = cmn->dtc[i].irq; 1526 for (j = i; j--; ) { 1527 if (cmn->dtc[j].irq == irq) { 1528 cmn->dtc[j].irq_friend = i - j; 1529 goto next; 1530 } 1531 } 1532 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq, 1533 IRQF_NOBALANCING | IRQF_NO_THREAD, 1534 dev_name(cmn->dev), &cmn->dtc[i]); 1535 if (err) 1536 return err; 1537 1538 err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); 1539 if (err) 1540 return err; 1541 next: 1542 ; /* isn't C great? */ 1543 } 1544 return 0; 1545 } 1546 1547 static void arm_cmn_init_dtm(struct arm_cmn_dtm *dtm, struct arm_cmn_node *xp, int idx) 1548 { 1549 int i; 1550 1551 dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx); 1552 dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN; 1553 for (i = 0; i < 4; i++) { 1554 dtm->wp_event[i] = -1; 1555 writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i)); 1556 writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i)); 1557 } 1558 } 1559 1560 static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx) 1561 { 1562 struct arm_cmn_dtc *dtc = cmn->dtc + idx; 1563 1564 dtc->base = dn->pmu_base - CMN_PMU_OFFSET; 1565 dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx); 1566 if (dtc->irq < 0) 1567 return dtc->irq; 1568 1569 writel_relaxed(0, dtc->base + CMN_DT_PMCR); 1570 writel_relaxed(0x1ff, dtc->base + CMN_DT_PMOVSR_CLR); 1571 writel_relaxed(CMN_DT_PMCR_OVFL_INTR_EN, dtc->base + CMN_DT_PMCR); 1572 1573 return 0; 1574 } 1575 1576 static int arm_cmn_node_cmp(const void *a, const void *b) 1577 { 1578 const struct arm_cmn_node *dna = a, *dnb = b; 1579 int cmp; 1580 1581 cmp = dna->type - dnb->type; 1582 if (!cmp) 1583 cmp = dna->logid - dnb->logid; 1584 return cmp; 1585 } 1586 1587 static int arm_cmn_init_dtcs(struct arm_cmn *cmn) 1588 { 1589 struct arm_cmn_node *dn, *xp; 1590 int dtc_idx = 0; 1591 u8 dtcs_present = (1 << cmn->num_dtcs) - 1; 1592 1593 cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL); 1594 if (!cmn->dtc) 1595 return -ENOMEM; 1596 1597 sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL); 1598 1599 cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP); 1600 1601 for (dn = cmn->dns; dn->type; dn++) { 1602 if (dn->type == CMN_TYPE_XP) { 1603 dn->dtc &= dtcs_present; 1604 continue; 1605 } 1606 1607 xp = arm_cmn_node_to_xp(cmn, dn); 1608 dn->dtm = xp->dtm; 1609 if (cmn->multi_dtm) 1610 dn->dtm += arm_cmn_nid(cmn, dn->id).port / 2; 1611 1612 if (dn->type == CMN_TYPE_DTC) { 1613 int err; 1614 /* We do at least know that a DTC's XP must be in that DTC's domain */ 1615 if (xp->dtc == 0xf) 1616 xp->dtc = 1 << dtc_idx; 1617 err = arm_cmn_init_dtc(cmn, dn, dtc_idx++); 1618 if (err) 1619 return err; 1620 } 1621 1622 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */ 1623 if (dn->type == CMN_TYPE_RND) 1624 dn->type = CMN_TYPE_RNI; 1625 } 1626 1627 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, cmn->dtc[0].base + CMN_DT_DTC_CTL); 1628 1629 return 0; 1630 } 1631 1632 static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node) 1633 { 1634 int level; 1635 u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO); 1636 1637 node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg); 1638 node->id = FIELD_GET(CMN_NI_NODE_ID, reg); 1639 node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg); 1640 1641 node->pmu_base = cmn->base + offset + CMN_PMU_OFFSET; 1642 1643 if (node->type == CMN_TYPE_CFG) 1644 level = 0; 1645 else if (node->type == CMN_TYPE_XP) 1646 level = 1; 1647 else 1648 level = 2; 1649 1650 dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n", 1651 (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ', 1652 node->type, node->logid, offset); 1653 } 1654 1655 static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) 1656 { 1657 void __iomem *cfg_region; 1658 struct arm_cmn_node cfg, *dn; 1659 struct arm_cmn_dtm *dtm; 1660 u16 child_count, child_poff; 1661 u32 xp_offset[CMN_MAX_XPS]; 1662 u64 reg; 1663 int i, j; 1664 size_t sz; 1665 1666 arm_cmn_init_node_info(cmn, rgn_offset, &cfg); 1667 if (cfg.type != CMN_TYPE_CFG) 1668 return -ENODEV; 1669 1670 cfg_region = cmn->base + rgn_offset; 1671 reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_2); 1672 cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); 1673 1674 reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL); 1675 cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; 1676 cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); 1677 cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); 1678 1679 reg = readq_relaxed(cfg_region + CMN_CHILD_INFO); 1680 child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1681 child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg); 1682 1683 cmn->num_xps = child_count; 1684 cmn->num_dns = cmn->num_xps; 1685 1686 /* Pass 1: visit the XPs, enumerate their children */ 1687 for (i = 0; i < cmn->num_xps; i++) { 1688 reg = readq_relaxed(cfg_region + child_poff + i * 8); 1689 xp_offset[i] = reg & CMN_CHILD_NODE_ADDR; 1690 1691 reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO); 1692 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1693 } 1694 1695 /* Cheeky +1 to help terminate pointer-based iteration later */ 1696 dn = devm_kcalloc(cmn->dev, cmn->num_dns + 1, sizeof(*dn), GFP_KERNEL); 1697 if (!dn) 1698 return -ENOMEM; 1699 1700 /* Initial safe upper bound on DTMs for any possible mesh layout */ 1701 i = cmn->num_xps; 1702 if (cmn->multi_dtm) 1703 i += cmn->num_xps + 1; 1704 dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL); 1705 if (!dtm) 1706 return -ENOMEM; 1707 1708 /* Pass 2: now we can actually populate the nodes */ 1709 cmn->dns = dn; 1710 cmn->dtms = dtm; 1711 for (i = 0; i < cmn->num_xps; i++) { 1712 void __iomem *xp_region = cmn->base + xp_offset[i]; 1713 struct arm_cmn_node *xp = dn++; 1714 unsigned int xp_ports = 0; 1715 1716 arm_cmn_init_node_info(cmn, xp_offset[i], xp); 1717 /* 1718 * Thanks to the order in which XP logical IDs seem to be 1719 * assigned, we can handily infer the mesh X dimension by 1720 * looking out for the XP at (0,1) without needing to know 1721 * the exact node ID format, which we can later derive. 1722 */ 1723 if (xp->id == (1 << 3)) 1724 cmn->mesh_x = xp->logid; 1725 1726 if (cmn->model == CMN600) 1727 xp->dtc = 0xf; 1728 else 1729 xp->dtc = 1 << readl_relaxed(xp_region + CMN_DTM_UNIT_INFO); 1730 1731 xp->dtm = dtm - cmn->dtms; 1732 arm_cmn_init_dtm(dtm++, xp, 0); 1733 /* 1734 * Keeping track of connected ports will let us filter out 1735 * unnecessary XP events easily. We can also reliably infer the 1736 * "extra device ports" configuration for the node ID format 1737 * from this, since in that case we will see at least one XP 1738 * with port 2 connected, for the HN-D. 1739 */ 1740 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P0)) 1741 xp_ports |= BIT(0); 1742 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P1)) 1743 xp_ports |= BIT(1); 1744 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P2)) 1745 xp_ports |= BIT(2); 1746 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P3)) 1747 xp_ports |= BIT(3); 1748 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P4)) 1749 xp_ports |= BIT(4); 1750 if (readq_relaxed(xp_region + CMN_MXP__CONNECT_INFO_P5)) 1751 xp_ports |= BIT(5); 1752 1753 if (cmn->multi_dtm && (xp_ports & 0xc)) 1754 arm_cmn_init_dtm(dtm++, xp, 1); 1755 if (cmn->multi_dtm && (xp_ports & 0x30)) 1756 arm_cmn_init_dtm(dtm++, xp, 2); 1757 1758 cmn->ports_used |= xp_ports; 1759 1760 reg = readq_relaxed(xp_region + CMN_CHILD_INFO); 1761 child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); 1762 child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg); 1763 1764 for (j = 0; j < child_count; j++) { 1765 reg = readq_relaxed(xp_region + child_poff + j * 8); 1766 /* 1767 * Don't even try to touch anything external, since in general 1768 * we haven't a clue how to power up arbitrary CHI requesters. 1769 * As of CMN-600r1 these could only be RN-SAMs or CXLAs, 1770 * neither of which have any PMU events anyway. 1771 * (Actually, CXLAs do seem to have grown some events in r1p2, 1772 * but they don't go to regular XP DTMs, and they depend on 1773 * secure configuration which we can't easily deal with) 1774 */ 1775 if (reg & CMN_CHILD_NODE_EXTERNAL) { 1776 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); 1777 continue; 1778 } 1779 1780 arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn); 1781 1782 switch (dn->type) { 1783 case CMN_TYPE_DTC: 1784 cmn->num_dtcs++; 1785 dn++; 1786 break; 1787 /* These guys have PMU events */ 1788 case CMN_TYPE_DVM: 1789 case CMN_TYPE_HNI: 1790 case CMN_TYPE_HNF: 1791 case CMN_TYPE_SBSX: 1792 case CMN_TYPE_RNI: 1793 case CMN_TYPE_RND: 1794 case CMN_TYPE_MTSX: 1795 case CMN_TYPE_CXRA: 1796 case CMN_TYPE_CXHA: 1797 dn++; 1798 break; 1799 /* Nothing to see here */ 1800 case CMN_TYPE_MPAM_S: 1801 case CMN_TYPE_MPAM_NS: 1802 case CMN_TYPE_RNSAM: 1803 case CMN_TYPE_CXLA: 1804 break; 1805 /* Something has gone horribly wrong */ 1806 default: 1807 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); 1808 return -ENODEV; 1809 } 1810 } 1811 } 1812 1813 /* Correct for any nodes we skipped */ 1814 cmn->num_dns = dn - cmn->dns; 1815 1816 sz = (void *)(dn + 1) - (void *)cmn->dns; 1817 dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL); 1818 if (dn) 1819 cmn->dns = dn; 1820 1821 sz = (void *)dtm - (void *)cmn->dtms; 1822 dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL); 1823 if (dtm) 1824 cmn->dtms = dtm; 1825 1826 /* 1827 * If mesh_x wasn't set during discovery then we never saw 1828 * an XP at (0,1), thus we must have an Nx1 configuration. 1829 */ 1830 if (!cmn->mesh_x) 1831 cmn->mesh_x = cmn->num_xps; 1832 cmn->mesh_y = cmn->num_xps / cmn->mesh_x; 1833 1834 /* 1x1 config plays havoc with XP event encodings */ 1835 if (cmn->num_xps == 1) 1836 dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); 1837 1838 dev_dbg(cmn->dev, "model %d, periph_id_2 revision %d\n", cmn->model, cmn->rev); 1839 reg = cmn->ports_used; 1840 dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n", 1841 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), ®, 1842 cmn->multi_dtm ? ", multi-DTM" : ""); 1843 1844 return 0; 1845 } 1846 1847 static int arm_cmn600_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn) 1848 { 1849 struct resource *cfg, *root; 1850 1851 cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1852 if (!cfg) 1853 return -EINVAL; 1854 1855 root = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1856 if (!root) 1857 return -EINVAL; 1858 1859 if (!resource_contains(cfg, root)) 1860 swap(cfg, root); 1861 /* 1862 * Note that devm_ioremap_resource() is dumb and won't let the platform 1863 * device claim cfg when the ACPI companion device has already claimed 1864 * root within it. But since they *are* already both claimed in the 1865 * appropriate name, we don't really need to do it again here anyway. 1866 */ 1867 cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg)); 1868 if (!cmn->base) 1869 return -ENOMEM; 1870 1871 return root->start - cfg->start; 1872 } 1873 1874 static int arm_cmn600_of_probe(struct device_node *np) 1875 { 1876 u32 rootnode; 1877 1878 return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode; 1879 } 1880 1881 static int arm_cmn_probe(struct platform_device *pdev) 1882 { 1883 struct arm_cmn *cmn; 1884 const char *name; 1885 static atomic_t id; 1886 int err, rootnode, this_id; 1887 1888 cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); 1889 if (!cmn) 1890 return -ENOMEM; 1891 1892 cmn->dev = &pdev->dev; 1893 cmn->model = (unsigned long)device_get_match_data(cmn->dev); 1894 platform_set_drvdata(pdev, cmn); 1895 1896 if (cmn->model == CMN600 && has_acpi_companion(cmn->dev)) { 1897 rootnode = arm_cmn600_acpi_probe(pdev, cmn); 1898 } else { 1899 rootnode = 0; 1900 cmn->base = devm_platform_ioremap_resource(pdev, 0); 1901 if (IS_ERR(cmn->base)) 1902 return PTR_ERR(cmn->base); 1903 if (cmn->model == CMN600) 1904 rootnode = arm_cmn600_of_probe(pdev->dev.of_node); 1905 } 1906 if (rootnode < 0) 1907 return rootnode; 1908 1909 err = arm_cmn_discover(cmn, rootnode); 1910 if (err) 1911 return err; 1912 1913 err = arm_cmn_init_dtcs(cmn); 1914 if (err) 1915 return err; 1916 1917 err = arm_cmn_init_irqs(cmn); 1918 if (err) 1919 return err; 1920 1921 cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); 1922 cmn->pmu = (struct pmu) { 1923 .module = THIS_MODULE, 1924 .attr_groups = arm_cmn_attr_groups, 1925 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1926 .task_ctx_nr = perf_invalid_context, 1927 .pmu_enable = arm_cmn_pmu_enable, 1928 .pmu_disable = arm_cmn_pmu_disable, 1929 .event_init = arm_cmn_event_init, 1930 .add = arm_cmn_event_add, 1931 .del = arm_cmn_event_del, 1932 .start = arm_cmn_event_start, 1933 .stop = arm_cmn_event_stop, 1934 .read = arm_cmn_event_read, 1935 .start_txn = arm_cmn_start_txn, 1936 .commit_txn = arm_cmn_commit_txn, 1937 .cancel_txn = arm_cmn_end_txn, 1938 }; 1939 1940 this_id = atomic_fetch_inc(&id); 1941 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id); 1942 if (!name) 1943 return -ENOMEM; 1944 1945 err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node); 1946 if (err) 1947 return err; 1948 1949 err = perf_pmu_register(&cmn->pmu, name, -1); 1950 if (err) 1951 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); 1952 else 1953 arm_cmn_debugfs_init(cmn, this_id); 1954 1955 return err; 1956 } 1957 1958 static int arm_cmn_remove(struct platform_device *pdev) 1959 { 1960 struct arm_cmn *cmn = platform_get_drvdata(pdev); 1961 1962 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); 1963 1964 perf_pmu_unregister(&cmn->pmu); 1965 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); 1966 debugfs_remove(cmn->debug); 1967 return 0; 1968 } 1969 1970 #ifdef CONFIG_OF 1971 static const struct of_device_id arm_cmn_of_match[] = { 1972 { .compatible = "arm,cmn-600", .data = (void *)CMN600 }, 1973 { .compatible = "arm,ci-700", .data = (void *)CI700 }, 1974 {} 1975 }; 1976 MODULE_DEVICE_TABLE(of, arm_cmn_of_match); 1977 #endif 1978 1979 #ifdef CONFIG_ACPI 1980 static const struct acpi_device_id arm_cmn_acpi_match[] = { 1981 { "ARMHC600", CMN600 }, 1982 {} 1983 }; 1984 MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match); 1985 #endif 1986 1987 static struct platform_driver arm_cmn_driver = { 1988 .driver = { 1989 .name = "arm-cmn", 1990 .of_match_table = of_match_ptr(arm_cmn_of_match), 1991 .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match), 1992 }, 1993 .probe = arm_cmn_probe, 1994 .remove = arm_cmn_remove, 1995 }; 1996 1997 static int __init arm_cmn_init(void) 1998 { 1999 int ret; 2000 2001 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 2002 "perf/arm/cmn:online", 2003 arm_cmn_pmu_online_cpu, 2004 arm_cmn_pmu_offline_cpu); 2005 if (ret < 0) 2006 return ret; 2007 2008 arm_cmn_hp_state = ret; 2009 arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL); 2010 2011 ret = platform_driver_register(&arm_cmn_driver); 2012 if (ret) { 2013 cpuhp_remove_multi_state(arm_cmn_hp_state); 2014 debugfs_remove(arm_cmn_debugfs); 2015 } 2016 return ret; 2017 } 2018 2019 static void __exit arm_cmn_exit(void) 2020 { 2021 platform_driver_unregister(&arm_cmn_driver); 2022 cpuhp_remove_multi_state(arm_cmn_hp_state); 2023 debugfs_remove(arm_cmn_debugfs); 2024 } 2025 2026 module_init(arm_cmn_init); 2027 module_exit(arm_cmn_exit); 2028 2029 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 2030 MODULE_DESCRIPTION("Arm CMN-600 PMU driver"); 2031 MODULE_LICENSE("GPL v2"); 2032