1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CPU PMU driver for the Apple M1 and derivatives 4 * 5 * Copyright (C) 2021 Google LLC 6 * 7 * Author: Marc Zyngier <maz@kernel.org> 8 * 9 * Most of the information used in this driver was provided by the 10 * Asahi Linux project. The rest was experimentally discovered. 11 */ 12 13 #include <linux/of.h> 14 #include <linux/perf/arm_pmu.h> 15 #include <linux/platform_device.h> 16 17 #include <asm/apple_m1_pmu.h> 18 #include <asm/irq_regs.h> 19 #include <asm/perf_event.h> 20 21 #define M1_PMU_NR_COUNTERS 10 22 23 #define M1_PMU_CFG_EVENT GENMASK(7, 0) 24 25 #define ANY_BUT_0_1 GENMASK(9, 2) 26 #define ONLY_2_TO_7 GENMASK(7, 2) 27 #define ONLY_2_4_6 (BIT(2) | BIT(4) | BIT(6)) 28 #define ONLY_5_6_7 (BIT(5) | BIT(6) | BIT(7)) 29 30 /* 31 * Description of the events we actually know about, as well as those with 32 * a specific counter affinity. Yes, this is a grand total of two known 33 * counters, and the rest is anybody's guess. 34 * 35 * Not all counters can count all events. Counters #0 and #1 are wired to 36 * count cycles and instructions respectively, and some events have 37 * bizarre mappings (every other counter, or even *one* counter). These 38 * restrictions equally apply to both P and E cores. 39 * 40 * It is worth noting that the PMUs attached to P and E cores are likely 41 * to be different because the underlying uarches are different. At the 42 * moment, we don't really need to distinguish between the two because we 43 * know next to nothing about the events themselves, and we already have 44 * per cpu-type PMU abstractions. 45 * 46 * If we eventually find out that the events are different across 47 * implementations, we'll have to introduce per cpu-type tables. 48 */ 49 enum m1_pmu_events { 50 M1_PMU_PERFCTR_RETIRE_UOP = 0x1, 51 M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE = 0x2, 52 M1_PMU_PERFCTR_L1I_TLB_FILL = 0x4, 53 M1_PMU_PERFCTR_L1D_TLB_FILL = 0x5, 54 M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION = 0x7, 55 M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA = 0x8, 56 M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION = 0xa, 57 M1_PMU_PERFCTR_L2_TLB_MISS_DATA = 0xb, 58 M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC = 0xd, 59 M1_PMU_PERFCTR_SCHEDULE_UOP = 0x52, 60 M1_PMU_PERFCTR_INTERRUPT_PENDING = 0x6c, 61 M1_PMU_PERFCTR_MAP_STALL_DISPATCH = 0x70, 62 M1_PMU_PERFCTR_MAP_REWIND = 0x75, 63 M1_PMU_PERFCTR_MAP_STALL = 0x76, 64 M1_PMU_PERFCTR_MAP_INT_UOP = 0x7c, 65 M1_PMU_PERFCTR_MAP_LDST_UOP = 0x7d, 66 M1_PMU_PERFCTR_MAP_SIMD_UOP = 0x7e, 67 M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC = 0x84, 68 M1_PMU_PERFCTR_INST_ALL = 0x8c, 69 M1_PMU_PERFCTR_INST_BRANCH = 0x8d, 70 M1_PMU_PERFCTR_INST_BRANCH_CALL = 0x8e, 71 M1_PMU_PERFCTR_INST_BRANCH_RET = 0x8f, 72 M1_PMU_PERFCTR_INST_BRANCH_TAKEN = 0x90, 73 M1_PMU_PERFCTR_INST_BRANCH_INDIR = 0x93, 74 M1_PMU_PERFCTR_INST_BRANCH_COND = 0x94, 75 M1_PMU_PERFCTR_INST_INT_LD = 0x95, 76 M1_PMU_PERFCTR_INST_INT_ST = 0x96, 77 M1_PMU_PERFCTR_INST_INT_ALU = 0x97, 78 M1_PMU_PERFCTR_INST_SIMD_LD = 0x98, 79 M1_PMU_PERFCTR_INST_SIMD_ST = 0x99, 80 M1_PMU_PERFCTR_INST_SIMD_ALU = 0x9a, 81 M1_PMU_PERFCTR_INST_LDST = 0x9b, 82 M1_PMU_PERFCTR_INST_BARRIER = 0x9c, 83 M1_PMU_PERFCTR_UNKNOWN_9f = 0x9f, 84 M1_PMU_PERFCTR_L1D_TLB_ACCESS = 0xa0, 85 M1_PMU_PERFCTR_L1D_TLB_MISS = 0xa1, 86 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST = 0xa2, 87 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD = 0xa3, 88 M1_PMU_PERFCTR_LD_UNIT_UOP = 0xa6, 89 M1_PMU_PERFCTR_ST_UNIT_UOP = 0xa7, 90 M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK = 0xa8, 91 M1_PMU_PERFCTR_LDST_X64_UOP = 0xb1, 92 M1_PMU_PERFCTR_LDST_XPG_UOP = 0xb2, 93 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC = 0xb3, 94 M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL = 0xb4, 95 M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC = 0xbf, 96 M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC = 0xc0, 97 M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC = 0xc1, 98 M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC = 0xc4, 99 M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC = 0xc5, 100 M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC = 0xc6, 101 M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC = 0xc8, 102 M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC = 0xca, 103 M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC = 0xcb, 104 M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND = 0xd4, 105 M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE = 0xd6, 106 M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND = 0xdb, 107 M1_PMU_PERFCTR_FETCH_RESTART = 0xde, 108 M1_PMU_PERFCTR_ST_NT_UOP = 0xe5, 109 M1_PMU_PERFCTR_LD_NT_UOP = 0xe6, 110 M1_PMU_PERFCTR_UNKNOWN_f5 = 0xf5, 111 M1_PMU_PERFCTR_UNKNOWN_f6 = 0xf6, 112 M1_PMU_PERFCTR_UNKNOWN_f7 = 0xf7, 113 M1_PMU_PERFCTR_UNKNOWN_f8 = 0xf8, 114 M1_PMU_PERFCTR_UNKNOWN_fd = 0xfd, 115 M1_PMU_PERFCTR_LAST = M1_PMU_CFG_EVENT, 116 117 /* 118 * From this point onwards, these are not actual HW events, 119 * but attributes that get stored in hw->config_base. 120 */ 121 M1_PMU_CFG_COUNT_USER = BIT(8), 122 M1_PMU_CFG_COUNT_KERNEL = BIT(9), 123 }; 124 125 /* 126 * Per-event affinity table. Most events can be installed on counter 127 * 2-9, but there are a number of exceptions. Note that this table 128 * has been created experimentally, and I wouldn't be surprised if more 129 * counters had strange affinities. 130 */ 131 static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = { 132 [0 ... M1_PMU_PERFCTR_LAST] = ANY_BUT_0_1, 133 [M1_PMU_PERFCTR_RETIRE_UOP] = BIT(7), 134 [M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE] = ANY_BUT_0_1 | BIT(0), 135 [M1_PMU_PERFCTR_INST_ALL] = BIT(7) | BIT(1), 136 [M1_PMU_PERFCTR_INST_BRANCH] = ONLY_5_6_7, 137 [M1_PMU_PERFCTR_INST_BRANCH_CALL] = ONLY_5_6_7, 138 [M1_PMU_PERFCTR_INST_BRANCH_RET] = ONLY_5_6_7, 139 [M1_PMU_PERFCTR_INST_BRANCH_TAKEN] = ONLY_5_6_7, 140 [M1_PMU_PERFCTR_INST_BRANCH_INDIR] = ONLY_5_6_7, 141 [M1_PMU_PERFCTR_INST_BRANCH_COND] = ONLY_5_6_7, 142 [M1_PMU_PERFCTR_INST_INT_LD] = ONLY_5_6_7, 143 [M1_PMU_PERFCTR_INST_INT_ST] = BIT(7), 144 [M1_PMU_PERFCTR_INST_INT_ALU] = BIT(7), 145 [M1_PMU_PERFCTR_INST_SIMD_LD] = ONLY_5_6_7, 146 [M1_PMU_PERFCTR_INST_SIMD_ST] = ONLY_5_6_7, 147 [M1_PMU_PERFCTR_INST_SIMD_ALU] = BIT(7), 148 [M1_PMU_PERFCTR_INST_LDST] = BIT(7), 149 [M1_PMU_PERFCTR_INST_BARRIER] = ONLY_5_6_7, 150 [M1_PMU_PERFCTR_UNKNOWN_9f] = BIT(7), 151 [M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC] = ONLY_5_6_7, 152 [M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC] = ONLY_5_6_7, 153 [M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC] = ONLY_5_6_7, 154 [M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC] = ONLY_5_6_7, 155 [M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC] = ONLY_5_6_7, 156 [M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 157 [M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 158 [M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC] = ONLY_5_6_7, 159 [M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC] = ONLY_5_6_7, 160 [M1_PMU_PERFCTR_UNKNOWN_f5] = ONLY_2_4_6, 161 [M1_PMU_PERFCTR_UNKNOWN_f6] = ONLY_2_4_6, 162 [M1_PMU_PERFCTR_UNKNOWN_f7] = ONLY_2_4_6, 163 [M1_PMU_PERFCTR_UNKNOWN_f8] = ONLY_2_TO_7, 164 [M1_PMU_PERFCTR_UNKNOWN_fd] = ONLY_2_4_6, 165 }; 166 167 static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = { 168 PERF_MAP_ALL_UNSUPPORTED, 169 [PERF_COUNT_HW_CPU_CYCLES] = M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE, 170 [PERF_COUNT_HW_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_ALL, 171 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = M1_PMU_PERFCTR_INST_BRANCH, 172 [PERF_COUNT_HW_BRANCH_MISSES] = M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC, 173 }; 174 175 /* sysfs definitions */ 176 static ssize_t m1_pmu_events_sysfs_show(struct device *dev, 177 struct device_attribute *attr, 178 char *page) 179 { 180 struct perf_pmu_events_attr *pmu_attr; 181 182 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 183 184 return sprintf(page, "event=0x%04llx\n", pmu_attr->id); 185 } 186 187 #define M1_PMU_EVENT_ATTR(name, config) \ 188 PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config) 189 190 static struct attribute *m1_pmu_event_attrs[] = { 191 M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE), 192 M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL), 193 NULL, 194 }; 195 196 static const struct attribute_group m1_pmu_events_attr_group = { 197 .name = "events", 198 .attrs = m1_pmu_event_attrs, 199 }; 200 201 PMU_FORMAT_ATTR(event, "config:0-7"); 202 203 static struct attribute *m1_pmu_format_attrs[] = { 204 &format_attr_event.attr, 205 NULL, 206 }; 207 208 static const struct attribute_group m1_pmu_format_attr_group = { 209 .name = "format", 210 .attrs = m1_pmu_format_attrs, 211 }; 212 213 /* Low level accessors. No synchronisation. */ 214 #define PMU_READ_COUNTER(_idx) \ 215 case _idx: return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1) 216 217 #define PMU_WRITE_COUNTER(_val, _idx) \ 218 case _idx: \ 219 write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1); \ 220 return 221 222 static u64 m1_pmu_read_hw_counter(unsigned int index) 223 { 224 switch (index) { 225 PMU_READ_COUNTER(0); 226 PMU_READ_COUNTER(1); 227 PMU_READ_COUNTER(2); 228 PMU_READ_COUNTER(3); 229 PMU_READ_COUNTER(4); 230 PMU_READ_COUNTER(5); 231 PMU_READ_COUNTER(6); 232 PMU_READ_COUNTER(7); 233 PMU_READ_COUNTER(8); 234 PMU_READ_COUNTER(9); 235 } 236 237 BUG(); 238 } 239 240 static void m1_pmu_write_hw_counter(u64 val, unsigned int index) 241 { 242 switch (index) { 243 PMU_WRITE_COUNTER(val, 0); 244 PMU_WRITE_COUNTER(val, 1); 245 PMU_WRITE_COUNTER(val, 2); 246 PMU_WRITE_COUNTER(val, 3); 247 PMU_WRITE_COUNTER(val, 4); 248 PMU_WRITE_COUNTER(val, 5); 249 PMU_WRITE_COUNTER(val, 6); 250 PMU_WRITE_COUNTER(val, 7); 251 PMU_WRITE_COUNTER(val, 8); 252 PMU_WRITE_COUNTER(val, 9); 253 } 254 255 BUG(); 256 } 257 258 #define get_bit_offset(index, mask) (__ffs(mask) + (index)) 259 260 static void __m1_pmu_enable_counter(unsigned int index, bool en) 261 { 262 u64 val, bit; 263 264 switch (index) { 265 case 0 ... 7: 266 bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7)); 267 break; 268 case 8 ... 9: 269 bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9)); 270 break; 271 default: 272 BUG(); 273 } 274 275 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 276 277 if (en) 278 val |= bit; 279 else 280 val &= ~bit; 281 282 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 283 } 284 285 static void m1_pmu_enable_counter(unsigned int index) 286 { 287 __m1_pmu_enable_counter(index, true); 288 } 289 290 static void m1_pmu_disable_counter(unsigned int index) 291 { 292 __m1_pmu_enable_counter(index, false); 293 } 294 295 static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en) 296 { 297 u64 val, bit; 298 299 switch (index) { 300 case 0 ... 7: 301 bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7)); 302 break; 303 case 8 ... 9: 304 bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9)); 305 break; 306 default: 307 BUG(); 308 } 309 310 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 311 312 if (en) 313 val |= bit; 314 else 315 val &= ~bit; 316 317 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 318 } 319 320 static void m1_pmu_enable_counter_interrupt(unsigned int index) 321 { 322 __m1_pmu_enable_counter_interrupt(index, true); 323 } 324 325 static void m1_pmu_disable_counter_interrupt(unsigned int index) 326 { 327 __m1_pmu_enable_counter_interrupt(index, false); 328 } 329 330 static void m1_pmu_configure_counter(unsigned int index, u8 event, 331 bool user, bool kernel) 332 { 333 u64 val, user_bit, kernel_bit; 334 int shift; 335 336 switch (index) { 337 case 0 ... 7: 338 user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7)); 339 kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7)); 340 break; 341 case 8 ... 9: 342 user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9)); 343 kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9)); 344 break; 345 default: 346 BUG(); 347 } 348 349 val = read_sysreg_s(SYS_IMP_APL_PMCR1_EL1); 350 351 if (user) 352 val |= user_bit; 353 else 354 val &= ~user_bit; 355 356 if (kernel) 357 val |= kernel_bit; 358 else 359 val &= ~kernel_bit; 360 361 write_sysreg_s(val, SYS_IMP_APL_PMCR1_EL1); 362 363 /* 364 * Counters 0 and 1 have fixed events. For anything else, 365 * place the event at the expected location in the relevant 366 * register (PMESR0 holds the event configuration for counters 367 * 2-5, resp. PMESR1 for counters 6-9). 368 */ 369 switch (index) { 370 case 0 ... 1: 371 break; 372 case 2 ... 5: 373 shift = (index - 2) * 8; 374 val = read_sysreg_s(SYS_IMP_APL_PMESR0_EL1); 375 val &= ~((u64)0xff << shift); 376 val |= (u64)event << shift; 377 write_sysreg_s(val, SYS_IMP_APL_PMESR0_EL1); 378 break; 379 case 6 ... 9: 380 shift = (index - 6) * 8; 381 val = read_sysreg_s(SYS_IMP_APL_PMESR1_EL1); 382 val &= ~((u64)0xff << shift); 383 val |= (u64)event << shift; 384 write_sysreg_s(val, SYS_IMP_APL_PMESR1_EL1); 385 break; 386 } 387 } 388 389 /* arm_pmu backend */ 390 static void m1_pmu_enable_event(struct perf_event *event) 391 { 392 bool user, kernel; 393 u8 evt; 394 395 evt = event->hw.config_base & M1_PMU_CFG_EVENT; 396 user = event->hw.config_base & M1_PMU_CFG_COUNT_USER; 397 kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL; 398 399 m1_pmu_disable_counter_interrupt(event->hw.idx); 400 m1_pmu_disable_counter(event->hw.idx); 401 isb(); 402 403 m1_pmu_configure_counter(event->hw.idx, evt, user, kernel); 404 m1_pmu_enable_counter(event->hw.idx); 405 m1_pmu_enable_counter_interrupt(event->hw.idx); 406 isb(); 407 } 408 409 static void m1_pmu_disable_event(struct perf_event *event) 410 { 411 m1_pmu_disable_counter_interrupt(event->hw.idx); 412 m1_pmu_disable_counter(event->hw.idx); 413 isb(); 414 } 415 416 static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu) 417 { 418 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 419 struct pt_regs *regs; 420 u64 overflow, state; 421 int idx; 422 423 overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1); 424 if (!overflow) { 425 /* Spurious interrupt? */ 426 state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 427 state &= ~PMCR0_IACT; 428 write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1); 429 isb(); 430 return IRQ_NONE; 431 } 432 433 cpu_pmu->stop(cpu_pmu); 434 435 regs = get_irq_regs(); 436 437 for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) { 438 struct perf_event *event = cpuc->events[idx]; 439 struct perf_sample_data data; 440 441 if (!event) 442 continue; 443 444 armpmu_event_update(event); 445 perf_sample_data_init(&data, 0, event->hw.last_period); 446 if (!armpmu_event_set_period(event)) 447 continue; 448 449 if (perf_event_overflow(event, &data, regs)) 450 m1_pmu_disable_event(event); 451 } 452 453 cpu_pmu->start(cpu_pmu); 454 455 return IRQ_HANDLED; 456 } 457 458 static u64 m1_pmu_read_counter(struct perf_event *event) 459 { 460 return m1_pmu_read_hw_counter(event->hw.idx); 461 } 462 463 static void m1_pmu_write_counter(struct perf_event *event, u64 value) 464 { 465 m1_pmu_write_hw_counter(value, event->hw.idx); 466 isb(); 467 } 468 469 static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc, 470 struct perf_event *event) 471 { 472 unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT; 473 unsigned long affinity = m1_pmu_event_affinity[evtype]; 474 int idx; 475 476 /* 477 * Place the event on the first free counter that can count 478 * this event. 479 * 480 * We could do a better job if we had a view of all the events 481 * counting on the PMU at any given time, and by placing the 482 * most constraining events first. 483 */ 484 for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) { 485 if (!test_and_set_bit(idx, cpuc->used_mask)) 486 return idx; 487 } 488 489 return -EAGAIN; 490 } 491 492 static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc, 493 struct perf_event *event) 494 { 495 clear_bit(event->hw.idx, cpuc->used_mask); 496 } 497 498 static void __m1_pmu_set_mode(u8 mode) 499 { 500 u64 val; 501 502 val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1); 503 val &= ~(PMCR0_IMODE | PMCR0_IACT); 504 val |= FIELD_PREP(PMCR0_IMODE, mode); 505 write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1); 506 isb(); 507 } 508 509 static void m1_pmu_start(struct arm_pmu *cpu_pmu) 510 { 511 __m1_pmu_set_mode(PMCR0_IMODE_FIQ); 512 } 513 514 static void m1_pmu_stop(struct arm_pmu *cpu_pmu) 515 { 516 __m1_pmu_set_mode(PMCR0_IMODE_OFF); 517 } 518 519 static int m1_pmu_map_event(struct perf_event *event) 520 { 521 /* 522 * Although the counters are 48bit wide, bit 47 is what 523 * triggers the overflow interrupt. Advertise the counters 524 * being 47bit wide to mimick the behaviour of the ARM PMU. 525 */ 526 event->hw.flags |= ARMPMU_EVT_47BIT; 527 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); 528 } 529 530 static int m2_pmu_map_event(struct perf_event *event) 531 { 532 /* 533 * Same deal as the above, except that M2 has 64bit counters. 534 * Which, as far as we're concerned, actually means 63 bits. 535 * Yes, this is getting awkward. 536 */ 537 event->hw.flags |= ARMPMU_EVT_63BIT; 538 return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); 539 } 540 541 static void m1_pmu_reset(void *info) 542 { 543 int i; 544 545 __m1_pmu_set_mode(PMCR0_IMODE_OFF); 546 547 for (i = 0; i < M1_PMU_NR_COUNTERS; i++) { 548 m1_pmu_disable_counter(i); 549 m1_pmu_disable_counter_interrupt(i); 550 m1_pmu_write_hw_counter(0, i); 551 } 552 553 isb(); 554 } 555 556 static int m1_pmu_set_event_filter(struct hw_perf_event *event, 557 struct perf_event_attr *attr) 558 { 559 unsigned long config_base = 0; 560 561 if (!attr->exclude_guest) { 562 pr_debug("ARM performance counters do not support mode exclusion\n"); 563 return -EOPNOTSUPP; 564 } 565 if (!attr->exclude_kernel) 566 config_base |= M1_PMU_CFG_COUNT_KERNEL; 567 if (!attr->exclude_user) 568 config_base |= M1_PMU_CFG_COUNT_USER; 569 570 event->config_base = config_base; 571 572 return 0; 573 } 574 575 static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags) 576 { 577 cpu_pmu->handle_irq = m1_pmu_handle_irq; 578 cpu_pmu->enable = m1_pmu_enable_event; 579 cpu_pmu->disable = m1_pmu_disable_event; 580 cpu_pmu->read_counter = m1_pmu_read_counter; 581 cpu_pmu->write_counter = m1_pmu_write_counter; 582 cpu_pmu->get_event_idx = m1_pmu_get_event_idx; 583 cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx; 584 cpu_pmu->start = m1_pmu_start; 585 cpu_pmu->stop = m1_pmu_stop; 586 587 if (flags & ARMPMU_EVT_47BIT) 588 cpu_pmu->map_event = m1_pmu_map_event; 589 else if (flags & ARMPMU_EVT_63BIT) 590 cpu_pmu->map_event = m2_pmu_map_event; 591 else 592 return WARN_ON(-EINVAL); 593 594 cpu_pmu->reset = m1_pmu_reset; 595 cpu_pmu->set_event_filter = m1_pmu_set_event_filter; 596 597 bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS); 598 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group; 599 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group; 600 return 0; 601 } 602 603 /* Device driver gunk */ 604 static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu) 605 { 606 cpu_pmu->name = "apple_icestorm_pmu"; 607 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); 608 } 609 610 static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) 611 { 612 cpu_pmu->name = "apple_firestorm_pmu"; 613 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); 614 } 615 616 static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) 617 { 618 cpu_pmu->name = "apple_avalanche_pmu"; 619 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); 620 } 621 622 static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) 623 { 624 cpu_pmu->name = "apple_blizzard_pmu"; 625 return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); 626 } 627 628 static const struct of_device_id m1_pmu_of_device_ids[] = { 629 { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, }, 630 { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, }, 631 { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, }, 632 { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, }, 633 { }, 634 }; 635 MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids); 636 637 static int m1_pmu_device_probe(struct platform_device *pdev) 638 { 639 return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL); 640 } 641 642 static struct platform_driver m1_pmu_driver = { 643 .driver = { 644 .name = "apple-m1-cpu-pmu", 645 .of_match_table = m1_pmu_of_device_ids, 646 .suppress_bind_attrs = true, 647 }, 648 .probe = m1_pmu_device_probe, 649 }; 650 651 module_platform_driver(m1_pmu_driver); 652