1 // SPDX-License-Identifier: GPL-2.0-only
30 /* ARMv8 Cortex-A53 specific event types. */
43  * be disabled at run-time based on the PMCEID registers.
166 	return sprintf(page, "event=0x%04llx\n", pmu_attr->id);  in armv8pmu_events_sysfs_show()
176 	 * means we don't have a fixed event<->counter relationship regardless.
281 	if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&  in armv8pmu_event_attr_is_visible()
282 	    test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))  in armv8pmu_event_attr_is_visible()
283 		return attr->mode;  in armv8pmu_event_attr_is_visible()
285 	if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) {  in armv8pmu_event_attr_is_visible()
286 		u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE;  in armv8pmu_event_attr_is_visible()
289 		    test_bit(id, cpu_pmu->pmceid_ext_bitmap))  in armv8pmu_event_attr_is_visible()
290 			return attr->mode;  in armv8pmu_event_attr_is_visible()
333 	return ATTR_CFG_GET_FLD(&event->attr, long);  in armv8pmu_event_is_64bit()
338 	return ATTR_CFG_GET_FLD(&event->attr, rdpmc);  in armv8pmu_event_want_user_access()
380 	u32 slots = FIELD_GET(ARMV8_PMU_SLOTS, cpu_pmu->reg_pmmir);  in slots_show()
392 	u32 bus_slots = FIELD_GET(ARMV8_PMU_BUS_SLOTS, cpu_pmu->reg_pmmir);  in bus_slots_show()
404 	u32 bus_width = FIELD_GET(ARMV8_PMU_BUS_WIDTH, cpu_pmu->reg_pmmir);  in bus_width_show()
409 		val = 1 << (bus_width - 1);  in bus_width_show()
419 	 * PMMIR.THWIDTH is readable and non-zero on aarch32, but it would be  in threshold_max()
427 	 * (2 ^ PMMIR.THWIDTH) - 1.  in threshold_max()
429 	return (1 << FIELD_GET(ARMV8_PMU_THWIDTH, cpu_pmu->reg_pmmir)) - 1;  in threshold_max()
470 		return brbe_num_branch_records(cpu_pmu) ? attr->mode : 0;  in caps_is_visible()
472 	return attr->mode;  in caps_is_visible()
482  * We unconditionally enable ARMv8.5-PMU long event counter support
483  * (64-bit events) where supported. Indicate if this arm_pmu has long
491 	return (IS_ENABLED(CONFIG_ARM64) && is_pmuv3p5(cpu_pmu->pmuver));  in armv8pmu_has_long_event()
496 	return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT;  in armv8pmu_event_has_user_read()
506 	int idx = event->hw.idx;  in armv8pmu_event_is_chained()
507 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_is_chained()
547 	int idx = event->hw.idx;  in armv8pmu_read_hw_counter()
551 		val = (val << 32) | armv8pmu_read_evcntr(idx - 1);  in armv8pmu_read_hw_counter()
556  * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
557  * is set the event counters also become 64-bit counters. Unless the
559  * interrupt upon 32-bit overflow - we achieve this by applying a bias.
563 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_event_needs_bias()
564 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_event_needs_bias()
565 	int idx = hwc->idx;  in armv8pmu_event_needs_bias()
595 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_read_counter()
596 	int idx = hwc->idx;  in armv8pmu_read_counter()
617 	int idx = event->hw.idx;  in armv8pmu_write_hw_counter()
621 		armv8pmu_write_evcntr(idx - 1, lower_32_bits(value));  in armv8pmu_write_hw_counter()
629 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_write_counter()
630 	int idx = hwc->idx;  in armv8pmu_write_counter()
658 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_write_event_type()
659 	int idx = hwc->idx;  in armv8pmu_write_event_type()
670 		armv8pmu_write_evtype(idx - 1, hwc->config_base);  in armv8pmu_write_event_type()
674 			write_pmccfiltr(hwc->config_base);  in armv8pmu_write_event_type()
676 			write_pmicfiltr(hwc->config_base);  in armv8pmu_write_event_type()
678 			armv8pmu_write_evtype(idx, hwc->config_base);  in armv8pmu_write_event_type()
684 	int counter = event->hw.idx;  in armv8pmu_event_cnten_mask()
688 		mask |= BIT(counter - 1);  in armv8pmu_event_cnten_mask()
704 	struct perf_event_attr *attr = &event->attr;  in armv8pmu_enable_event_counter()
726 	struct perf_event_attr *attr = &event->attr;  in armv8pmu_disable_event_counter()
743 	armv8pmu_enable_intens(BIT(event->hw.idx));  in armv8pmu_enable_event_irq()
757 	armv8pmu_disable_intens(BIT(event->hw.idx));  in armv8pmu_disable_event_irq()
798 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_enable_user_access()
800 	if (is_pmuv3p9(cpu_pmu->pmuver)) {  in armv8pmu_enable_user_access()
802 		for_each_set_bit(i, cpuc->used_mask, ARMPMU_MAX_HWEVENTS) {  in armv8pmu_enable_user_access()
803 			if (armv8pmu_event_has_user_read(cpuc->events[i]))  in armv8pmu_enable_user_access()
809 		for_each_andnot_bit(i, cpu_pmu->cntr_mask, cpuc->used_mask,  in armv8pmu_enable_user_access()
839 	struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_start()
844 		nr_user = ctx->nr_user;  in armv8pmu_start()
853 	if (hw_events->branch_users)  in armv8pmu_start()
862 	struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_stop()
864 	if (hw_events->branch_users)  in armv8pmu_stop()
875 	struct perf_branch_stack *branch_stack = cpuc->branch_stack;  in read_branch_records()
885 	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);  in armv8pmu_handle_irq()
910 	for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS) {  in armv8pmu_handle_irq()
911 		struct perf_event *event = cpuc->events[idx];  in armv8pmu_handle_irq()
925 		hwc = &event->hw;  in armv8pmu_handle_irq()
927 		perf_sample_data_init(&data, 0, hwc->last_period);  in armv8pmu_handle_irq()
951 	for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS) {  in armv8pmu_get_single_idx()
952 		if (!test_and_set_bit(idx, cpuc->used_mask))  in armv8pmu_get_single_idx()
955 	return -EAGAIN;  in armv8pmu_get_single_idx()
967 	for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV8_PMU_MAX_GENERAL_COUNTERS) {  in armv8pmu_get_chain_idx()
970 		if (!test_and_set_bit(idx, cpuc->used_mask)) {  in armv8pmu_get_chain_idx()
972 			if (!test_and_set_bit(idx - 1, cpuc->used_mask))  in armv8pmu_get_chain_idx()
975 			clear_bit(idx, cpuc->used_mask);  in armv8pmu_get_chain_idx()
978 	return -EAGAIN;  in armv8pmu_get_chain_idx()
984 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_can_use_pmccntr()
985 	unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;  in armv8pmu_can_use_pmccntr()
994 	if (armv8pmu_event_get_threshold(&event->attr))  in armv8pmu_can_use_pmccntr()
1010 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);  in armv8pmu_get_event_idx()
1011 	struct hw_perf_event *hwc = &event->hw;  in armv8pmu_get_event_idx()
1012 	unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;  in armv8pmu_get_event_idx()
1016 		if (!test_and_set_bit(ARMV8_PMU_CYCLE_IDX, cpuc->used_mask))  in armv8pmu_get_event_idx()
1021 				return -EAGAIN;  in armv8pmu_get_event_idx()
1030 	    !armv8pmu_event_get_threshold(&event->attr) &&  in armv8pmu_get_event_idx()
1031 	    test_bit(ARMV8_PMU_INSTR_IDX, cpu_pmu->cntr_mask) &&  in armv8pmu_get_event_idx()
1033 		if (!test_and_set_bit(ARMV8_PMU_INSTR_IDX, cpuc->used_mask))  in armv8pmu_get_event_idx()
1049 	int idx = event->hw.idx;  in armv8pmu_clear_event_idx()
1051 	clear_bit(idx, cpuc->used_mask);  in armv8pmu_clear_event_idx()
1053 		clear_bit(idx - 1, cpuc->used_mask);  in armv8pmu_clear_event_idx()
1061 	return event->hw.idx + 1;  in armv8pmu_user_event_idx()
1068 	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);  in armv8pmu_sched_task()
1070 	if (!hw_events->branch_users)  in armv8pmu_sched_task()
1086 	struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu);  in armv8pmu_set_event_filter()
1089 	if (attr->exclude_idle) {  in armv8pmu_set_event_filter()
1091 		return -EOPNOTSUPP;  in armv8pmu_set_event_filter()
1096 			return -EOPNOTSUPP;  in armv8pmu_set_event_filter()
1098 		perf_event->attach_state |= PERF_ATTACH_SCHED_CB;  in armv8pmu_set_event_filter()
1108 		if (!attr->exclude_kernel && !attr->exclude_host)  in armv8pmu_set_event_filter()
1110 		if (attr->exclude_guest)  in armv8pmu_set_event_filter()
1112 		if (attr->exclude_host)  in armv8pmu_set_event_filter()
1115 		if (!attr->exclude_hv && !attr->exclude_host)  in armv8pmu_set_event_filter()
1122 	if (attr->exclude_kernel)  in armv8pmu_set_event_filter()
1125 	if (attr->exclude_user)  in armv8pmu_set_event_filter()
1135 		return -EINVAL;  in armv8pmu_set_event_filter()
1148 	event->config_base = config_base;  in armv8pmu_set_event_filter()
1158 	bitmap_to_arr64(&mask, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS);  in armv8pmu_reset()
1188 	if (event->attr.type == PERF_TYPE_HARDWARE &&  in __armv8_pmuv3_map_event_id()
1189 	    event->attr.config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) {  in __armv8_pmuv3_map_event_id()
1192 			     armpmu->pmceid_bitmap))  in __armv8_pmuv3_map_event_id()
1196 			     armpmu->pmceid_bitmap))  in __armv8_pmuv3_map_event_id()
1216 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);  in __armv8_pmuv3_map_event()
1226 		return -EINVAL;  in __armv8_pmuv3_map_event()
1229 		event->hw.flags |= ARMPMU_EVT_64BIT;  in __armv8_pmuv3_map_event()
1235 	 * Most 64-bit events require long counter support, but 64-bit  in __armv8_pmuv3_map_event()
1240 		if (!(event->attach_state & PERF_ATTACH_TASK))  in __armv8_pmuv3_map_event()
1241 			return -EINVAL;  in __armv8_pmuv3_map_event()
1245 			return -EOPNOTSUPP;  in __armv8_pmuv3_map_event()
1247 		event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;  in __armv8_pmuv3_map_event()
1252 	    && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {  in __armv8_pmuv3_map_event()
1300 	struct arm_pmu *cpu_pmu = probe->pmu;  in __armv8pmu_probe_pmu()
1309 	cpu_pmu->pmuver = pmuver;  in __armv8pmu_probe_pmu()
1310 	probe->present = true;  in __armv8pmu_probe_pmu()
1313 	bitmap_set(cpu_pmu->cntr_mask,  in __armv8pmu_probe_pmu()
1317 	set_bit(ARMV8_PMU_CYCLE_IDX, cpu_pmu->cntr_mask);  in __armv8pmu_probe_pmu()
1321 		set_bit(ARMV8_PMU_INSTR_IDX, cpu_pmu->cntr_mask);  in __armv8pmu_probe_pmu()
1326 	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,  in __armv8pmu_probe_pmu()
1332 	bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,  in __armv8pmu_probe_pmu()
1337 		cpu_pmu->reg_pmmir = read_pmmir();  in __armv8pmu_probe_pmu()
1339 		cpu_pmu->reg_pmmir = 0;  in __armv8pmu_probe_pmu()
1350 	for_each_cpu(cpu, &armpmu->supported_cpus) {  in branch_records_alloc()
1353 		events_cpu = per_cpu_ptr(armpmu->hw_events, cpu);  in branch_records_alloc()
1354 		events_cpu->branch_stack = kmalloc(size, GFP_KERNEL);  in branch_records_alloc()
1355 		if (!events_cpu->branch_stack)  in branch_records_alloc()
1356 			return -ENOMEM;  in branch_records_alloc()
1369 	ret = smp_call_function_any(&cpu_pmu->supported_cpus,  in armv8pmu_probe_pmu()
1376 		return -ENODEV;  in armv8pmu_probe_pmu()
1429 	cpu_pmu->handle_irq		= armv8pmu_handle_irq;  in armv8_pmu_init()
1430 	cpu_pmu->enable			= armv8pmu_enable_event;  in armv8_pmu_init()
1431 	cpu_pmu->disable		= armv8pmu_disable_event;  in armv8_pmu_init()
1432 	cpu_pmu->read_counter		= armv8pmu_read_counter;  in armv8_pmu_init()
1433 	cpu_pmu->write_counter		= armv8pmu_write_counter;  in armv8_pmu_init()
1434 	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx;  in armv8_pmu_init()
1435 	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx;  in armv8_pmu_init()
1436 	cpu_pmu->start			= armv8pmu_start;  in armv8_pmu_init()
1437 	cpu_pmu->stop			= armv8pmu_stop;  in armv8_pmu_init()
1438 	cpu_pmu->reset			= armv8pmu_reset;  in armv8_pmu_init()
1439 	cpu_pmu->set_event_filter	= armv8pmu_set_event_filter;  in armv8_pmu_init()
1441 	cpu_pmu->pmu.event_idx		= armv8pmu_user_event_idx;  in armv8_pmu_init()
1443 		cpu_pmu->pmu.sched_task		= armv8pmu_sched_task;  in armv8_pmu_init()
1445 	cpu_pmu->name			= name;  in armv8_pmu_init()
1446 	cpu_pmu->map_event		= map_event;  in armv8_pmu_init()
1447 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &armv8_pmuv3_events_attr_group;  in armv8_pmu_init()
1448 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &armv8_pmuv3_format_attr_group;  in armv8_pmu_init()
1449 	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = &armv8_pmuv3_caps_attr_group;  in armv8_pmu_init()
1510 	{.compatible = "arm,armv8-pmuv3",	.data = armv8_pmuv3_pmu_init},
1511 	{.compatible = "arm,cortex-a34-pmu",	.data = armv8_cortex_a34_pmu_init},
1512 	{.compatible = "arm,cortex-a35-pmu",	.data = armv8_cortex_a35_pmu_init},
1513 	{.compatible = "arm,cortex-a53-pmu",	.data = armv8_cortex_a53_pmu_init},
1514 	{.compatible = "arm,cortex-a55-pmu",	.data = armv8_cortex_a55_pmu_init},
1515 	{.compatible = "arm,cortex-a57-pmu",	.data = armv8_cortex_a57_pmu_init},
1516 	{.compatible = "arm,cortex-a65-pmu",	.data = armv8_cortex_a65_pmu_init},
1517 	{.compatible = "arm,cortex-a72-pmu",	.data = armv8_cortex_a72_pmu_init},
1518 	{.compatible = "arm,cortex-a73-pmu",	.data = armv8_cortex_a73_pmu_init},
1519 	{.compatible = "arm,cortex-a75-pmu",	.data = armv8_cortex_a75_pmu_init},
1520 	{.compatible = "arm,cortex-a76-pmu",	.data = armv8_cortex_a76_pmu_init},
1521 	{.compatible = "arm,cortex-a77-pmu",	.data = armv8_cortex_a77_pmu_init},
1522 	{.compatible = "arm,cortex-a78-pmu",	.data = armv8_cortex_a78_pmu_init},
1523 	{.compatible = "arm,cortex-a510-pmu",	.data = armv9_cortex_a510_pmu_init},
1524 	{.compatible = "arm,cortex-a520-pmu",	.data = armv9_cortex_a520_pmu_init},
1525 	{.compatible = "arm,cortex-a710-pmu",	.data = armv9_cortex_a710_pmu_init},
1526 	{.compatible = "arm,cortex-a715-pmu",	.data = armv9_cortex_a715_pmu_init},
1527 	{.compatible = "arm,cortex-a720-pmu",	.data = armv9_cortex_a720_pmu_init},
1528 	{.compatible = "arm,cortex-a725-pmu",	.data = armv9_cortex_a725_pmu_init},
1529 	{.compatible = "arm,cortex-x1-pmu",	.data = armv8_cortex_x1_pmu_init},
1530 	{.compatible = "arm,cortex-x2-pmu",	.data = armv9_cortex_x2_pmu_init},
1531 	{.compatible = "arm,cortex-x3-pmu",	.data = armv9_cortex_x3_pmu_init},
1532 	{.compatible = "arm,cortex-x4-pmu",	.data = armv9_cortex_x4_pmu_init},
1533 	{.compatible = "arm,cortex-x925-pmu",	.data = armv9_cortex_x925_pmu_init},
1534 	{.compatible = "arm,neoverse-e1-pmu",	.data = armv8_neoverse_e1_pmu_init},
1535 	{.compatible = "arm,neoverse-n1-pmu",	.data = armv8_neoverse_n1_pmu_init},
1536 	{.compatible = "arm,neoverse-n2-pmu",	.data = armv9_neoverse_n2_pmu_init},
1537 	{.compatible = "arm,neoverse-n3-pmu",	.data = armv9_neoverse_n3_pmu_init},
1538 	{.compatible = "arm,neoverse-v1-pmu",	.data = armv8_neoverse_v1_pmu_init},
1539 	{.compatible = "arm,neoverse-v2-pmu",	.data = armv8_neoverse_v2_pmu_init},
1540 	{.compatible = "arm,neoverse-v3-pmu",	.data = armv8_neoverse_v3_pmu_init},
1541 	{.compatible = "arm,neoverse-v3ae-pmu",	.data = armv8_neoverse_v3ae_pmu_init},
1542 	{.compatible = "arm,rainier-pmu",	.data = armv8_rainier_pmu_init},
1543 	{.compatible = "cavium,thunder-pmu",	.data = armv8_cavium_thunder_pmu_init},
1544 	{.compatible = "brcm,vulcan-pmu",	.data = armv8_brcm_vulcan_pmu_init},
1545 	{.compatible = "nvidia,carmel-pmu",	.data = armv8_nvidia_carmel_pmu_init},
1546 	{.compatible = "nvidia,denver-pmu",	.data = armv8_nvidia_denver_pmu_init},
1547 	{.compatible = "samsung,mongoose-pmu",	.data = armv8_samsung_mongoose_pmu_init},
1588 	userpg->cap_user_time = 0;  in device_initcall()
1589 	userpg->cap_user_time_zero = 0;  in device_initcall()
1590 	userpg->cap_user_time_short = 0;  in device_initcall()
1591 	userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event);  in device_initcall()
1593 	if (userpg->cap_user_rdpmc) {  in device_initcall()
1594 		if (event->hw.flags & ARMPMU_EVT_64BIT)  in device_initcall()
1595 			userpg->pmc_width = 64;  in device_initcall()
1597 			userpg->pmc_width = 32;  in device_initcall()
1603 		if (rd->read_sched_clock != arch_timer_read_counter)  in device_initcall()
1606 		userpg->time_mult = rd->mult;  in device_initcall()
1607 		userpg->time_shift = rd->shift;  in device_initcall()
1608 		userpg->time_zero = rd->epoch_ns;  in device_initcall()
1609 		userpg->time_cycles = rd->epoch_cyc;  in device_initcall()
1610 		userpg->time_mask = rd->sched_clock_mask;  in device_initcall()
1617 		ns = mul_u64_u32_shr(rd->epoch_cyc, rd->mult, rd->shift);  in device_initcall()
1618 		userpg->time_zero -= ns;  in device_initcall()
1622 	userpg->time_offset = userpg->time_zero - now;  in device_initcall()
1627 	 * 32-bit value (now specifies a 64-bit value) - refer  in device_initcall()
1630 	if (userpg->time_shift == 32) {  in device_initcall()
1631 		userpg->time_shift = 31;  in device_initcall()
1632 		userpg->time_mult >>= 1;  in device_initcall()
1639 	userpg->cap_user_time = 1;  in device_initcall()
1640 	userpg->cap_user_time_zero = 1;  in device_initcall()
1641 	userpg->cap_user_time_short = 1;  in device_initcall()