Lines Matching +full:ignore +full:- +full:power +full:- +full:on +full:- +full:sel
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2016-2020 Arm Limited
11 #include <linux/io-64-nonatomic-lo-hi.h>
36 #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4)
77 /* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */
128 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
130 #define CMN_DT_PMEVCNT(dtc, n) ((dtc)->pmu_base + _CMN_DT_CNT_REG(n))
131 #define CMN_DT_PMCCNTR(dtc) ((dtc)->pmu_base + 0x40)
133 #define CMN_DT_PMEVCNTSR(dtc, n) ((dtc)->pmu_base + 0x50 + _CMN_DT_CNT_REG(n))
134 #define CMN_DT_PMCCNTRSR(dtc) ((dtc)->pmu_base + 0x90)
136 #define CMN_DT_PMCR(dtc) ((dtc)->pmu_base + 0x100)
141 #define CMN_DT_PMOVSR(dtc) ((dtc)->pmu_base + 0x118)
142 #define CMN_DT_PMOVSR_CLR(dtc) ((dtc)->pmu_base + 0x120)
144 #define CMN_DT_PMSSR(dtc) ((dtc)->pmu_base + 0x128)
147 #define CMN_DT_PMSRR(dtc) ((dtc)->pmu_base + 0x130)
158 /* Similarly for the 40-bit cycle counter */
169 #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
170 #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
171 #define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
172 #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
173 #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
183 #define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
184 #define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
185 #define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
186 #define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
187 #define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
188 #define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
189 #define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
191 /* Made-up event IDs for watchpoint direction */
204 CMN_ANY = -1,
205 NOT_CMN600 = -2,
218 /* CMN-600 r0px shouldn't exist in silicon, thankfully */
276 SEL_NONE = -1,
295 /* DN/HN-F/CXHA */
377 return fls((cmn->mesh_x - 1) | (cmn->mesh_y - 1)); in arm_cmn_xyidbits()
384 nid.dev = dn->id & ((1U << dn->deviceid_bits) - 1); in arm_cmn_nid()
385 nid.port = (dn->id >> dn->deviceid_bits) & ((1U << dn->portid_bits) - 1); in arm_cmn_nid()
392 int id = dn->id >> (dn->portid_bits + dn->deviceid_bits); in arm_cmn_node_to_xp()
395 int y = id & ((1U << bits) - 1); in arm_cmn_node_to_xp()
397 return cmn->xps + cmn->mesh_x * y + x; in arm_cmn_node_to_xp()
404 for (dn = cmn->dns; dn->type; dn++) in arm_cmn_node()
405 if (dn->type == type) in arm_cmn_node()
412 switch (cmn->part) { in arm_cmn_model()
430 if (cmn->part == PART_CMN_S3) { in arm_cmn_pmu_offset()
431 if (cmn->rev >= REV_CMNS3_R1P0 && dn->type == CMN_TYPE_XP) in arm_cmn_pmu_offset()
441 int offset = CMN_MXP__CONNECT_INFO(port) - arm_cmn_pmu_offset(cmn, xp); in arm_cmn_device_connect_info()
444 if (cmn->part == PART_CMN600 || cmn->part == PART_CMN650) in arm_cmn_device_connect_info()
447 * CI-700 may have extra ports, but still has the in arm_cmn_device_connect_info()
450 if (cmn->part == PART_CI700) in arm_cmn_device_connect_info()
454 return readl_relaxed(xp->pmu_base + offset); in arm_cmn_device_connect_info()
464 case 0x01: return " RN-I |"; in arm_cmn_device_type()
465 case 0x02: return " RN-D |"; in arm_cmn_device_type()
466 case 0x04: return " RN-F_B |"; in arm_cmn_device_type()
467 case 0x05: return "RN-F_B_E|"; in arm_cmn_device_type()
468 case 0x06: return " RN-F_A |"; in arm_cmn_device_type()
469 case 0x07: return "RN-F_A_E|"; in arm_cmn_device_type()
470 case 0x08: return " HN-T |"; in arm_cmn_device_type()
471 case 0x09: return " HN-I |"; in arm_cmn_device_type()
472 case 0x0a: return " HN-D |"; in arm_cmn_device_type()
473 case 0x0b: return " HN-P |"; in arm_cmn_device_type()
474 case 0x0c: return " SN-F |"; in arm_cmn_device_type()
476 case 0x0e: return " HN-F |"; in arm_cmn_device_type()
477 case 0x0f: return " SN-F_E |"; in arm_cmn_device_type()
478 case 0x10: return " SN-F_D |"; in arm_cmn_device_type()
482 case 0x14: return " RN-F_D |"; in arm_cmn_device_type()
483 case 0x15: return "RN-F_D_E|"; in arm_cmn_device_type()
484 case 0x16: return " RN-F_C |"; in arm_cmn_device_type()
485 case 0x17: return "RN-F_C_E|"; in arm_cmn_device_type()
486 case 0x18: return " RN-F_E |"; in arm_cmn_device_type()
487 case 0x19: return "RN-F_E_E|"; in arm_cmn_device_type()
488 case 0x1a: return " HN-S |"; in arm_cmn_device_type()
491 case 0x1d: return " HN-V |"; in arm_cmn_device_type()
493 case 0x20: return " RN-F_F |"; in arm_cmn_device_type()
494 case 0x21: return "RN-F_F_E|"; in arm_cmn_device_type()
495 case 0x22: return " SN-F_F |"; in arm_cmn_device_type()
502 struct arm_cmn *cmn = s->private; in arm_cmn_show_logid()
504 u16 id = xp->id | d | (p << xp->deviceid_bits); in arm_cmn_show_logid()
506 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_show_logid()
507 int pad = dn->logid < 10; in arm_cmn_show_logid()
509 if (dn->type == CMN_TYPE_XP) in arm_cmn_show_logid()
511 /* Ignore the extra components that will overlap on some ports */ in arm_cmn_show_logid()
512 if (dn->type < CMN_TYPE_HNI) in arm_cmn_show_logid()
515 if (dn->id != id) in arm_cmn_show_logid()
518 seq_printf(s, " %*c#%-*d |", pad + 1, ' ', 3 - pad, dn->logid); in arm_cmn_show_logid()
526 struct arm_cmn *cmn = s->private; in arm_cmn_map_show()
527 int x, y, p, pmax = fls(cmn->ports_used); in arm_cmn_map_show()
530 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
531 seq_printf(s, " %-2d ", x); in arm_cmn_map_show()
533 y = cmn->mesh_y; in arm_cmn_map_show()
534 while (y--) { in arm_cmn_map_show()
535 int xp_base = cmn->mesh_x * y; in arm_cmn_map_show()
536 struct arm_cmn_node *xp = cmn->xps + xp_base; in arm_cmn_map_show()
539 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
540 seq_puts(s, "--------+"); in arm_cmn_map_show()
542 seq_printf(s, "\n%-2d |", y); in arm_cmn_map_show()
543 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
546 seq_printf(s, " XP #%-3d|", xp_base + x); in arm_cmn_map_show()
550 for (x = 0; x < cmn->mesh_x; x++) { in arm_cmn_map_show()
559 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
564 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
567 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
570 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
573 seq_puts(s, "\n-----+"); in arm_cmn_map_show()
575 for (x = 0; x < cmn->mesh_x; x++) in arm_cmn_map_show()
576 seq_puts(s, "--------+"); in arm_cmn_map_show()
587 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id); in arm_cmn_debugfs_init()
591 cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops); in arm_cmn_debugfs_init()
616 for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
618 /* @i is the DTC number, @idx is the counter index on that DTC */
620 for (int i = 0, idx; i < CMN_MAX_DTCS; i++) if ((idx = hw->dtc_idx[i]) >= 0)
624 return (struct arm_cmn_hw_event *)&event->hw; in to_cmn_hw()
682 if (eattr->type == CMN_TYPE_DTC) in arm_cmn_event_show()
683 return sysfs_emit(buf, "type=0x%x\n", eattr->type); in arm_cmn_event_show()
685 if (eattr->type == CMN_TYPE_WP) in arm_cmn_event_show()
688 eattr->type, eattr->eventid); in arm_cmn_event_show()
690 if (eattr->fsel > SEL_NONE) in arm_cmn_event_show()
692 eattr->type, eattr->eventid, eattr->occupid); in arm_cmn_event_show()
694 return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, in arm_cmn_event_show()
695 eattr->eventid); in arm_cmn_event_show()
710 if (!(eattr->model & arm_cmn_model(cmn))) in arm_cmn_event_attr_is_visible()
713 type = eattr->type; in arm_cmn_event_attr_is_visible()
714 eventid = eattr->eventid; in arm_cmn_event_attr_is_visible()
718 return attr->mode; in arm_cmn_event_attr_is_visible()
725 if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3))) in arm_cmn_event_attr_is_visible()
728 if (chan == 4 && cmn->part == PART_CMN600) in arm_cmn_event_attr_is_visible()
731 if ((chan == 5 && cmn->rsp_vc_num < 2) || in arm_cmn_event_attr_is_visible()
732 (chan == 6 && cmn->dat_vc_num < 2) || in arm_cmn_event_attr_is_visible()
733 (chan == 7 && cmn->req_vc_num < 2) || in arm_cmn_event_attr_is_visible()
734 (chan == 8 && cmn->snp_vc_num < 2)) in arm_cmn_event_attr_is_visible()
738 /* Revision-specific differences */ in arm_cmn_event_attr_is_visible()
739 if (cmn->part == PART_CMN600) { in arm_cmn_event_attr_is_visible()
740 if (cmn->rev < REV_CMN600_R1P3) { in arm_cmn_event_attr_is_visible()
744 if (cmn->rev < REV_CMN600_R1P2) { in arm_cmn_event_attr_is_visible()
750 } else if (cmn->part == PART_CMN650) { in arm_cmn_event_attr_is_visible()
751 if (cmn->rev < REV_CMN650_R2P0 || cmn->rev == REV_CMN650_R1P2) { in arm_cmn_event_attr_is_visible()
759 } else if (cmn->part == PART_CMN700) { in arm_cmn_event_attr_is_visible()
760 if (cmn->rev < REV_CMN700_R2P0) { in arm_cmn_event_attr_is_visible()
768 if (cmn->rev < REV_CMN700_R1P0) { in arm_cmn_event_attr_is_visible()
777 return attr->mode; in arm_cmn_event_attr_is_visible()
900 * DVM node events conflict with HN-I events in the equivalent PMU
901 * slot, but our lazy short-cut of using the DTM counter index for
999 * HN-P events squat on top of the HN-I similarly to DVM events, except
1022 /* We treat watchpoints as a special made-up class of XP events */
1285 if (!fmt->config) in arm_cmn_format_show()
1286 return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field); in arm_cmn_format_show()
1288 return sysfs_emit(buf, "config%d:%*pbl\n", fmt->config, 64, &fmt->field); in arm_cmn_format_show()
1328 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu)); in arm_cmn_cpumask_show()
1339 return sysfs_emit(buf, "%03x%02x\n", cmn->part, cmn->rev); in arm_cmn_identifier_show()
1367 if (dtm->wp_event[wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1368 if (dtm->wp_event[++wp_idx] >= 0) in arm_cmn_find_free_wp_idx()
1369 return -ENOSPC; in arm_cmn_find_free_wp_idx()
1378 return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos); in arm_cmn_get_assigned_wp_idx()
1388 dtm->wp_event[wp_idx] = hw->dtc_idx[dtc]; in arm_cmn_claim_wp_idx()
1389 arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event)); in arm_cmn_claim_wp_idx()
1400 bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600; in arm_cmn_wp_config()
1402 /* CMN-600 supports only primary and secondary matching groups */ in arm_cmn_wp_config()
1414 /* wp_combine is available only on WP0 and WP2 */ in arm_cmn_wp_config()
1423 if (!cmn->state) in arm_cmn_set_state()
1424 writel_relaxed(0, CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_set_state()
1425 cmn->state |= state; in arm_cmn_set_state()
1430 cmn->state &= ~state; in arm_cmn_clear_state()
1431 if (!cmn->state) in arm_cmn_clear_state()
1433 CMN_DT_PMCR(&cmn->dtc[0])); in arm_cmn_clear_state()
1456 if (dtm != &cmn->dtms[dn->dtm]) { in arm_cmn_read_dtm()
1457 dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_read_dtm()
1458 reg = readq_relaxed(dtm->base + offset); in arm_cmn_read_dtm()
1460 dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_read_dtm()
1472 return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1); in arm_cmn_read_cc()
1481 return val - CMN_COUNTER_INIT; in arm_cmn_read_counter()
1486 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_init_counter()
1491 writel_relaxed(CMN_COUNTER_INIT, CMN_DT_PMEVCNT(&cmn->dtc[i], idx)); in arm_cmn_init_counter()
1492 cmn->dtc[i].counters[idx] = event; in arm_cmn_init_counter()
1496 local64_set(&event->hw.prev_count, count); in arm_cmn_init_counter()
1501 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_read()
1507 delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]); in arm_cmn_event_read()
1508 local64_add(delta, &event->count); in arm_cmn_event_read()
1512 prev = local64_xchg(&event->hw.prev_count, new); in arm_cmn_event_read()
1514 delta = new - prev; in arm_cmn_event_read()
1518 new = arm_cmn_read_counter(cmn->dtc + i, idx); in arm_cmn_event_read()
1522 local64_add(delta, &event->count); in arm_cmn_event_read()
1533 if (!dn->occupid[fsel].count) { in arm_cmn_set_event_sel_hi()
1534 dn->occupid[fsel].val = occupid; in arm_cmn_set_event_sel_hi()
1536 dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) | in arm_cmn_set_event_sel_hi()
1538 dn->occupid[SEL_SN_HOME_SEL].val) | in arm_cmn_set_event_sel_hi()
1540 dn->occupid[SEL_HBT_LBT_SEL].val) | in arm_cmn_set_event_sel_hi()
1542 dn->occupid[SEL_CLASS_OCCUP_ID].val) | in arm_cmn_set_event_sel_hi()
1544 dn->occupid[SEL_OCCUP1ID].val); in arm_cmn_set_event_sel_hi()
1545 writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); in arm_cmn_set_event_sel_hi()
1546 } else if (dn->occupid[fsel].val != occupid) { in arm_cmn_set_event_sel_hi()
1547 return -EBUSY; in arm_cmn_set_event_sel_hi()
1549 dn->occupid[fsel].count++; in arm_cmn_set_event_sel_hi()
1557 dn->event_w[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1558 writeq_relaxed(le64_to_cpu(dn->event_sel_w), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1560 dn->event[dtm_idx] = eventid; in arm_cmn_set_event_sel_lo()
1561 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL); in arm_cmn_set_event_sel_lo()
1567 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_start()
1574 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_start()
1577 dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_start()
1579 dtc->cc_active = true; in arm_cmn_event_start()
1585 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_start()
1592 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_start()
1595 hw->wide_sel); in arm_cmn_event_start()
1601 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_stop()
1608 struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; in arm_cmn_event_stop()
1610 dtc->cc_active = false; in arm_cmn_event_stop()
1611 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_event_stop()
1614 void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset); in arm_cmn_event_stop()
1621 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_stop()
1623 arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel); in arm_cmn_event_stop()
1643 if (val->wp[dtm][wp_idx]) in arm_cmn_val_find_free_wp_config()
1644 if (val->wp[dtm][++wp_idx]) in arm_cmn_val_find_free_wp_config()
1645 return -ENOSPC; in arm_cmn_val_find_free_wp_config()
1663 val->cycles = true; in arm_cmn_val_add_event()
1668 val->dtc_count[dtc]++; in arm_cmn_val_add_event()
1671 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_val_add_event() local
1673 val->dtm_count[dtm]++; in arm_cmn_val_add_event()
1675 if (sel > SEL_NONE) in arm_cmn_val_add_event()
1676 val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1; in arm_cmn_val_add_event()
1682 val->wp[dtm][wp_idx] = 1; in arm_cmn_val_add_event()
1683 val->wp_combine[dtm][wp_idx >> 1] += !!CMN_EVENT_WP_COMBINE(event); in arm_cmn_val_add_event()
1691 struct perf_event *sibling, *leader = event->group_leader; in arm_cmn_validate_group()
1694 int i, ret = -EINVAL; in arm_cmn_validate_group()
1699 if (event->pmu != leader->pmu && !is_software_event(leader)) in arm_cmn_validate_group()
1700 return -EINVAL; in arm_cmn_validate_group()
1704 return -ENOMEM; in arm_cmn_validate_group()
1713 ret = val->cycles ? -EINVAL : 0; in arm_cmn_validate_group()
1718 if (val->dtc_count[dtc] == CMN_DT_NUM_COUNTERS) in arm_cmn_validate_group()
1722 int wp_idx, dtm = dn->dtm, sel = hw->filter_sel; in arm_cmn_validate_group() local
1724 if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) in arm_cmn_validate_group()
1727 if (sel > SEL_NONE && val->occupid[dtm][sel] && in arm_cmn_validate_group()
1728 val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1) in arm_cmn_validate_group()
1739 val->wp_combine[dtm][wp_idx >> 1] != !!CMN_EVENT_WP_COMBINE(event)) in arm_cmn_validate_group()
1756 for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { in arm_cmn_filter_sel()
1758 if (e->model & model && e->type == type && e->eventid == eventid) in arm_cmn_filter_sel()
1759 return e->fsel; in arm_cmn_filter_sel()
1767 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_init()
1774 if (event->attr.type != event->pmu->type) in arm_cmn_event_init()
1775 return -ENOENT; in arm_cmn_event_init()
1777 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) in arm_cmn_event_init()
1778 return -EINVAL; in arm_cmn_event_init()
1780 event->cpu = cmn->cpu; in arm_cmn_event_init()
1781 if (event->cpu < 0) in arm_cmn_event_init()
1782 return -EINVAL; in arm_cmn_event_init()
1795 return -EINVAL; in arm_cmn_event_init()
1796 /* ...but the DTM may depend on which port we're watching */ in arm_cmn_event_init()
1797 if (cmn->multi_dtm) in arm_cmn_event_init()
1798 hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; in arm_cmn_event_init()
1800 (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) { in arm_cmn_event_init()
1801 hw->wide_sel = true; in arm_cmn_event_init()
1808 hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid); in arm_cmn_event_init()
1813 hw->dn = arm_cmn_node(cmn, type); in arm_cmn_event_init()
1814 if (!hw->dn) in arm_cmn_event_init()
1815 return -EINVAL; in arm_cmn_event_init()
1817 memset(hw->dtc_idx, -1, sizeof(hw->dtc_idx)); in arm_cmn_event_init()
1818 for (dn = hw->dn; dn->type == type; dn++) { in arm_cmn_event_init()
1819 if (bynodeid && dn->id != nodeid) { in arm_cmn_event_init()
1820 hw->dn++; in arm_cmn_event_init()
1823 hw->num_dns++; in arm_cmn_event_init()
1824 if (dn->dtc < 0) in arm_cmn_event_init()
1825 memset(hw->dtc_idx, 0, cmn->num_dtcs); in arm_cmn_event_init()
1827 hw->dtc_idx[dn->dtc] = 0; in arm_cmn_event_init()
1833 if (!hw->num_dns) { in arm_cmn_event_init()
1834 dev_dbg(cmn->dev, "invalid node 0x%x type 0x%x\n", nodeid, type); in arm_cmn_event_init()
1835 return -EINVAL; in arm_cmn_event_init()
1847 while (i--) { in arm_cmn_event_clear()
1848 struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; in arm_cmn_event_clear()
1849 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); in arm_cmn_event_clear()
1854 dtm->wp_event[wp_idx] = -1; in arm_cmn_event_clear()
1857 if (hw->filter_sel > SEL_NONE) in arm_cmn_event_clear()
1858 hw->dn[i].occupid[hw->filter_sel].count--; in arm_cmn_event_clear()
1860 dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_clear()
1861 writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_clear()
1863 memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); in arm_cmn_event_clear()
1864 memset(hw->wp_idx, 0, sizeof(hw->wp_idx)); in arm_cmn_event_clear()
1867 cmn->dtc[j].counters[idx] = NULL; in arm_cmn_event_clear()
1872 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_add()
1879 while (cmn->dtc[i].cycles) in arm_cmn_event_add()
1880 if (++i == cmn->num_dtcs) in arm_cmn_event_add()
1881 return -ENOSPC; in arm_cmn_event_add()
1883 cmn->dtc[i].cycles = event; in arm_cmn_event_add()
1884 hw->dtc_idx[0] = i; in arm_cmn_event_add()
1893 if (cmn->part == PART_CMN600 && j > 0) { in arm_cmn_event_add()
1894 idx = hw->dtc_idx[0]; in arm_cmn_event_add()
1897 while (cmn->dtc[j].counters[idx]) in arm_cmn_event_add()
1899 return -ENOSPC; in arm_cmn_event_add()
1901 hw->dtc_idx[j] = idx; in arm_cmn_event_add()
1906 struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; in arm_cmn_event_add()
1907 unsigned int dtm_idx, shift, d = max_t(int, dn->dtc, 0); in arm_cmn_event_add()
1911 while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx)) in arm_cmn_event_add()
1927 tmp = dtm->wp_event[wp_idx ^ 1]; in arm_cmn_event_add()
1929 CMN_EVENT_WP_COMBINE(cmn->dtc[d].counters[tmp])) in arm_cmn_event_add()
1935 writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx)); in arm_cmn_event_add()
1939 if (cmn->multi_dtm) in arm_cmn_event_add()
1945 if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event))) in arm_cmn_event_add()
1949 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); in arm_cmn_event_add()
1951 dtm->input_sel[dtm_idx] = input_sel; in arm_cmn_event_add()
1953 dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift); in arm_cmn_event_add()
1954 dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, hw->dtc_idx[d]) << shift; in arm_cmn_event_add()
1955 dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx); in arm_cmn_event_add()
1956 reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low; in arm_cmn_event_add()
1957 writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_event_add()
1970 return -ENOSPC; in arm_cmn_event_add()
1975 struct arm_cmn *cmn = to_cmn(event->pmu); in arm_cmn_event_del()
1982 cmn->dtc[hw->dtc_idx[0]].cycles = NULL; in arm_cmn_event_del()
1984 arm_cmn_event_clear(cmn, event, hw->num_dns); in arm_cmn_event_del()
1991 * plus it seems they don't work properly on some hardware anyway :(
2013 perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu); in arm_cmn_migrate()
2014 for (i = 0; i < cmn->num_dtcs; i++) in arm_cmn_migrate()
2015 irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu)); in arm_cmn_migrate()
2016 cmn->cpu = cpu; in arm_cmn_migrate()
2025 node = dev_to_node(cmn->dev); in arm_cmn_pmu_online_cpu()
2026 if (cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node) in arm_cmn_pmu_online_cpu()
2038 if (cpu != cmn->cpu) in arm_cmn_pmu_offline_cpu()
2041 node = dev_to_node(cmn->dev); in arm_cmn_pmu_offline_cpu()
2066 if (WARN_ON(!dtc->counters[i])) in arm_cmn_handle_irq()
2069 local64_add(delta, &dtc->counters[i]->count); in arm_cmn_handle_irq()
2075 if (dtc->cc_active && !WARN_ON(!dtc->cycles)) { in arm_cmn_handle_irq()
2077 local64_add(delta, &dtc->cycles->count); in arm_cmn_handle_irq()
2083 if (!dtc->irq_friend) in arm_cmn_handle_irq()
2085 dtc += dtc->irq_friend; in arm_cmn_handle_irq()
2094 for (i = 0; i < cmn->num_dtcs; i++) { in arm_cmn_init_irqs()
2095 irq = cmn->dtc[i].irq; in arm_cmn_init_irqs()
2096 for (j = i; j--; ) { in arm_cmn_init_irqs()
2097 if (cmn->dtc[j].irq == irq) { in arm_cmn_init_irqs()
2098 cmn->dtc[j].irq_friend = i - j; in arm_cmn_init_irqs()
2102 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq, in arm_cmn_init_irqs()
2104 dev_name(cmn->dev), &cmn->dtc[i]); in arm_cmn_init_irqs()
2108 err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); in arm_cmn_init_irqs()
2121 dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx); in arm_cmn_init_dtm()
2122 dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN; in arm_cmn_init_dtm()
2123 writeq_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); in arm_cmn_init_dtm()
2125 dtm->wp_event[i] = -1; in arm_cmn_init_dtm()
2126 writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i)); in arm_cmn_init_dtm()
2127 writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i)); in arm_cmn_init_dtm()
2133 struct arm_cmn_dtc *dtc = cmn->dtc + idx; in arm_cmn_init_dtc()
2135 dtc->pmu_base = dn->pmu_base; in arm_cmn_init_dtc()
2136 dtc->base = dtc->pmu_base - arm_cmn_pmu_offset(cmn, dn); in arm_cmn_init_dtc()
2137 dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx); in arm_cmn_init_dtc()
2138 if (dtc->irq < 0) in arm_cmn_init_dtc()
2139 return dtc->irq; in arm_cmn_init_dtc()
2141 writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); in arm_cmn_init_dtc()
2154 cmp = dna->type - dnb->type; in arm_cmn_node_cmp()
2156 cmp = dna->logid - dnb->logid; in arm_cmn_node_cmp()
2165 cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL); in arm_cmn_init_dtcs()
2166 if (!cmn->dtc) in arm_cmn_init_dtcs()
2167 return -ENOMEM; in arm_cmn_init_dtcs()
2169 sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL); in arm_cmn_init_dtcs()
2171 cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP); in arm_cmn_init_dtcs()
2173 for (dn = cmn->dns; dn->type; dn++) { in arm_cmn_init_dtcs()
2174 if (dn->type == CMN_TYPE_XP) in arm_cmn_init_dtcs()
2178 dn->dtc = xp->dtc; in arm_cmn_init_dtcs()
2179 dn->dtm = xp->dtm; in arm_cmn_init_dtcs()
2180 if (cmn->multi_dtm) in arm_cmn_init_dtcs()
2181 dn->dtm += arm_cmn_nid(dn).port / 2; in arm_cmn_init_dtcs()
2183 if (dn->type == CMN_TYPE_DTC) { in arm_cmn_init_dtcs()
2190 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */ in arm_cmn_init_dtcs()
2191 if (dn->type == CMN_TYPE_RND) in arm_cmn_init_dtcs()
2192 dn->type = CMN_TYPE_RNI; in arm_cmn_init_dtcs()
2194 /* We split the RN-I off already, so let the CCLA part match CCLA events */ in arm_cmn_init_dtcs()
2195 if (dn->type == CMN_TYPE_CCLA_RNI) in arm_cmn_init_dtcs()
2196 dn->type = CMN_TYPE_CCLA; in arm_cmn_init_dtcs()
2208 if (cmn->part == PART_CMN650 || cmn->part == PART_CI700) in arm_cmn_dtc_domain()
2217 u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO); in arm_cmn_init_node_info()
2219 node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg); in arm_cmn_init_node_info()
2220 node->id = FIELD_GET(CMN_NI_NODE_ID, reg); in arm_cmn_init_node_info()
2221 node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg); in arm_cmn_init_node_info()
2223 node->pmu_base = cmn->base + offset + arm_cmn_pmu_offset(cmn, node); in arm_cmn_init_node_info()
2225 if (node->type == CMN_TYPE_CFG) in arm_cmn_init_node_info()
2227 else if (node->type == CMN_TYPE_XP) in arm_cmn_init_node_info()
2232 dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n", in arm_cmn_init_node_info()
2233 (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ', in arm_cmn_init_node_info()
2234 node->type, node->logid, offset); in arm_cmn_init_node_info()
2262 return -ENODEV; in arm_cmn_discover()
2264 cfg_region = cmn->base + rgn_offset; in arm_cmn_discover()
2269 if (cmn->part && cmn->part != part) in arm_cmn_discover()
2270 dev_warn(cmn->dev, in arm_cmn_discover()
2272 cmn->part, part); in arm_cmn_discover()
2273 cmn->part = part; in arm_cmn_discover()
2275 dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); in arm_cmn_discover()
2278 cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); in arm_cmn_discover()
2283 * it; however we also have no way to tell from Non-Secure whether any in arm_cmn_discover()
2288 dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n"); in arm_cmn_discover()
2289 return -ENODEV; in arm_cmn_discover()
2291 cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; in arm_cmn_discover()
2292 cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); in arm_cmn_discover()
2293 cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); in arm_cmn_discover()
2296 cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); in arm_cmn_discover()
2297 cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); in arm_cmn_discover()
2303 cmn->num_xps = child_count; in arm_cmn_discover()
2304 cmn->num_dns = cmn->num_xps; in arm_cmn_discover()
2308 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2310 xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR); in arm_cmn_discover()
2313 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); in arm_cmn_discover()
2319 * bound, account for double the number of non-XP nodes. in arm_cmn_discover()
2321 dn = devm_kcalloc(cmn->dev, cmn->num_dns * 2 - cmn->num_xps, in arm_cmn_discover()
2324 return -ENOMEM; in arm_cmn_discover()
2326 /* Initial safe upper bound on DTMs for any possible mesh layout */ in arm_cmn_discover()
2327 i = cmn->num_xps; in arm_cmn_discover()
2328 if (cmn->multi_dtm) in arm_cmn_discover()
2329 i += cmn->num_xps + 1; in arm_cmn_discover()
2330 dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL); in arm_cmn_discover()
2332 return -ENOMEM; in arm_cmn_discover()
2335 cmn->dns = dn; in arm_cmn_discover()
2336 cmn->dtms = dtm; in arm_cmn_discover()
2337 for (i = 0; i < cmn->num_xps; i++) { in arm_cmn_discover()
2342 xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR); in arm_cmn_discover()
2350 if (xp->id == (1 << 3)) in arm_cmn_discover()
2351 cmn->mesh_x = xp->logid; in arm_cmn_discover()
2353 if (cmn->part == PART_CMN600) in arm_cmn_discover()
2354 xp->dtc = -1; in arm_cmn_discover()
2356 xp->dtc = arm_cmn_dtc_domain(cmn, xp_region); in arm_cmn_discover()
2358 xp->dtm = dtm - cmn->dtms; in arm_cmn_discover()
2362 * unnecessary XP events easily, and also infer the per-XP in arm_cmn_discover()
2369 if (cmn->num_xps == 1) { in arm_cmn_discover()
2370 xp->portid_bits = 3; in arm_cmn_discover()
2371 xp->deviceid_bits = 2; in arm_cmn_discover()
2373 xp->portid_bits = 2; in arm_cmn_discover()
2374 xp->deviceid_bits = 1; in arm_cmn_discover()
2376 xp->portid_bits = 1; in arm_cmn_discover()
2377 xp->deviceid_bits = 2; in arm_cmn_discover()
2380 if (cmn->multi_dtm && (xp_ports > 0x3)) in arm_cmn_discover()
2382 if (cmn->multi_dtm && (xp_ports > 0xf)) in arm_cmn_discover()
2385 cmn->ports_used |= xp_ports; in arm_cmn_discover()
2395 * we haven't a clue how to power up arbitrary CHI requesters. in arm_cmn_discover()
2396 * As of CMN-600r1 these could only be RN-SAMs or CXLAs, in arm_cmn_discover()
2399 * but they don't go to regular XP DTMs, and they depend on in arm_cmn_discover()
2403 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); in arm_cmn_discover()
2409 * A child offset of 0 can only occur on CMN-600; otherwise it in arm_cmn_discover()
2413 if (reg == 0 && cmn->part != PART_CMN600) { in arm_cmn_discover()
2414 dev_dbg(cmn->dev, "bogus child pointer?\n"); in arm_cmn_discover()
2419 dn->portid_bits = xp->portid_bits; in arm_cmn_discover()
2420 dn->deviceid_bits = xp->deviceid_bits; in arm_cmn_discover()
2422 switch (dn->type) { in arm_cmn_discover()
2424 cmn->num_dtcs++; in arm_cmn_discover()
2443 dn->pmu_base += CMN_CCLA_PMU_EVENT_SEL; in arm_cmn_discover()
2465 dn[1].type = arm_cmn_subtype(dn->type); in arm_cmn_discover()
2470 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); in arm_cmn_discover()
2471 return -ENODEV; in arm_cmn_discover()
2477 cmn->num_dns = dn - cmn->dns; in arm_cmn_discover()
2479 /* Cheeky +1 to help terminate pointer-based iteration later */ in arm_cmn_discover()
2480 sz = (void *)(dn + 1) - (void *)cmn->dns; in arm_cmn_discover()
2481 dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL); in arm_cmn_discover()
2483 cmn->dns = dn; in arm_cmn_discover()
2485 sz = (void *)dtm - (void *)cmn->dtms; in arm_cmn_discover()
2486 dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL); in arm_cmn_discover()
2488 cmn->dtms = dtm; in arm_cmn_discover()
2494 if (!cmn->mesh_x) in arm_cmn_discover()
2495 cmn->mesh_x = cmn->num_xps; in arm_cmn_discover()
2496 cmn->mesh_y = cmn->num_xps / cmn->mesh_x; in arm_cmn_discover()
2499 if (cmn->num_xps == 1) in arm_cmn_discover()
2500 dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); in arm_cmn_discover()
2502 dev_dbg(cmn->dev, "periph_id part 0x%03x revision %d\n", cmn->part, cmn->rev); in arm_cmn_discover()
2503 reg = cmn->ports_used; in arm_cmn_discover()
2504 dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n", in arm_cmn_discover()
2505 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), ®, in arm_cmn_discover()
2506 cmn->multi_dtm ? ", multi-DTM" : ""); in arm_cmn_discover()
2517 return -EINVAL; in arm_cmn600_acpi_probe()
2521 return -EINVAL; in arm_cmn600_acpi_probe()
2531 cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg)); in arm_cmn600_acpi_probe()
2532 if (!cmn->base) in arm_cmn600_acpi_probe()
2533 return -ENOMEM; in arm_cmn600_acpi_probe()
2535 return root->start - cfg->start; in arm_cmn600_acpi_probe()
2542 return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode; in arm_cmn600_of_probe()
2552 cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL); in arm_cmn_probe()
2554 return -ENOMEM; in arm_cmn_probe()
2556 cmn->dev = &pdev->dev; in arm_cmn_probe()
2557 cmn->part = (unsigned long)device_get_match_data(cmn->dev); in arm_cmn_probe()
2558 cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev)); in arm_cmn_probe()
2561 if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) { in arm_cmn_probe()
2565 cmn->base = devm_platform_ioremap_resource(pdev, 0); in arm_cmn_probe()
2566 if (IS_ERR(cmn->base)) in arm_cmn_probe()
2567 return PTR_ERR(cmn->base); in arm_cmn_probe()
2568 if (cmn->part == PART_CMN600) in arm_cmn_probe()
2569 rootnode = arm_cmn600_of_probe(pdev->dev.of_node); in arm_cmn_probe()
2586 cmn->pmu = (struct pmu) { in arm_cmn_probe()
2588 .parent = cmn->dev, in arm_cmn_probe()
2606 name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id); in arm_cmn_probe()
2608 return -ENOMEM; in arm_cmn_probe()
2610 err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2614 err = perf_pmu_register(&cmn->pmu, name, -1); in arm_cmn_probe()
2616 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_probe()
2627 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); in arm_cmn_remove()
2629 perf_pmu_unregister(&cmn->pmu); in arm_cmn_remove()
2630 cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node); in arm_cmn_remove()
2631 debugfs_remove(cmn->debug); in arm_cmn_remove()
2636 { .compatible = "arm,cmn-600", .data = (void *)PART_CMN600 },
2637 { .compatible = "arm,cmn-650" },
2638 { .compatible = "arm,cmn-700" },
2639 { .compatible = "arm,cmn-s3" },
2640 { .compatible = "arm,ci-700" },
2659 .name = "arm-cmn",
2680 arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL); in arm_cmn_init()