1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ARMv6 Performance counter handling code. 4 * 5 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles 6 * 7 * ARMv6 has 2 configurable performance counters and a single cycle counter. 8 * They all share a single reset bit but can be written to zero so we can use 9 * that for a reset. 10 * 11 * The counters can't be individually enabled or disabled so when we remove 12 * one event and replace it with another we could get spurious counts from the 13 * wrong event. However, we can take advantage of the fact that the 14 * performance counters can export events to the event bus, and the event bus 15 * itself can be monitored. This requires that we *don't* export the events to 16 * the event bus. The procedure for disabling a configurable counter is: 17 * - change the counter to count the ETMEXTOUT[0] signal (0x20). This 18 * effectively stops the counter from counting. 19 * - disable the counter's interrupt generation (each counter has it's 20 * own interrupt enable bit). 21 * Once stopped, the counter value can be written as 0 to reset. 22 * 23 * To enable a counter: 24 * - enable the counter's interrupt generation. 25 * - set the new event type. 26 * 27 * Note: the dedicated cycle counter only counts cycles and can't be 28 * enabled/disabled independently of the others. When we want to disable the 29 * cycle counter, we have to just disable the interrupt reporting and start 30 * ignoring that counter. When re-enabling, we have to reset the value and 31 * enable the interrupt. 32 */ 33 34 #include <asm/cputype.h> 35 #include <asm/irq_regs.h> 36 37 #include <linux/of.h> 38 #include <linux/perf/arm_pmu.h> 39 #include <linux/platform_device.h> 40 41 enum armv6_perf_types { 42 ARMV6_PERFCTR_ICACHE_MISS = 0x0, 43 ARMV6_PERFCTR_IBUF_STALL = 0x1, 44 ARMV6_PERFCTR_DDEP_STALL = 0x2, 45 ARMV6_PERFCTR_ITLB_MISS = 0x3, 46 ARMV6_PERFCTR_DTLB_MISS = 0x4, 47 ARMV6_PERFCTR_BR_EXEC = 0x5, 48 ARMV6_PERFCTR_BR_MISPREDICT = 0x6, 49 ARMV6_PERFCTR_INSTR_EXEC = 0x7, 50 ARMV6_PERFCTR_DCACHE_HIT = 0x9, 51 ARMV6_PERFCTR_DCACHE_ACCESS = 0xA, 52 ARMV6_PERFCTR_DCACHE_MISS = 0xB, 53 ARMV6_PERFCTR_DCACHE_WBACK = 0xC, 54 ARMV6_PERFCTR_SW_PC_CHANGE = 0xD, 55 ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF, 56 ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10, 57 ARMV6_PERFCTR_LSU_FULL_STALL = 0x11, 58 ARMV6_PERFCTR_WBUF_DRAINED = 0x12, 59 ARMV6_PERFCTR_CPU_CYCLES = 0xFF, 60 ARMV6_PERFCTR_NOP = 0x20, 61 }; 62 63 enum armv6_counters { 64 ARMV6_CYCLE_COUNTER = 0, 65 ARMV6_COUNTER0, 66 ARMV6_COUNTER1, 67 }; 68 69 /* 70 * The hardware events that we support. We do support cache operations but 71 * we have harvard caches and no way to combine instruction and data 72 * accesses/misses in hardware. 73 */ 74 static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = { 75 PERF_MAP_ALL_UNSUPPORTED, 76 [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES, 77 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC, 78 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC, 79 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT, 80 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6_PERFCTR_IBUF_STALL, 81 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6_PERFCTR_LSU_FULL_STALL, 82 }; 83 84 static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 85 [PERF_COUNT_HW_CACHE_OP_MAX] 86 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 87 PERF_CACHE_MAP_ALL_UNSUPPORTED, 88 89 /* 90 * The performance counters don't differentiate between read and write 91 * accesses/misses so this isn't strictly correct, but it's the best we 92 * can do. Writes and reads get combined. 93 */ 94 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, 95 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, 96 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, 97 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, 98 99 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, 100 101 /* 102 * The ARM performance counters can count micro DTLB misses, micro ITLB 103 * misses and main TLB misses. There isn't an event for TLB misses, so 104 * use the micro misses here and if users want the main TLB misses they 105 * can use a raw counter. 106 */ 107 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, 108 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, 109 110 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, 111 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, 112 }; 113 114 static inline unsigned long 115 armv6_pmcr_read(void) 116 { 117 u32 val; 118 asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val)); 119 return val; 120 } 121 122 static inline void 123 armv6_pmcr_write(unsigned long val) 124 { 125 asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val)); 126 } 127 128 #define ARMV6_PMCR_ENABLE (1 << 0) 129 #define ARMV6_PMCR_CTR01_RESET (1 << 1) 130 #define ARMV6_PMCR_CCOUNT_RESET (1 << 2) 131 #define ARMV6_PMCR_CCOUNT_DIV (1 << 3) 132 #define ARMV6_PMCR_COUNT0_IEN (1 << 4) 133 #define ARMV6_PMCR_COUNT1_IEN (1 << 5) 134 #define ARMV6_PMCR_CCOUNT_IEN (1 << 6) 135 #define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8) 136 #define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9) 137 #define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10) 138 #define ARMV6_PMCR_EVT_COUNT0_SHIFT 20 139 #define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT) 140 #define ARMV6_PMCR_EVT_COUNT1_SHIFT 12 141 #define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT) 142 143 #define ARMV6_PMCR_OVERFLOWED_MASK \ 144 (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \ 145 ARMV6_PMCR_CCOUNT_OVERFLOW) 146 147 static inline int 148 armv6_pmcr_has_overflowed(unsigned long pmcr) 149 { 150 return pmcr & ARMV6_PMCR_OVERFLOWED_MASK; 151 } 152 153 static inline int 154 armv6_pmcr_counter_has_overflowed(unsigned long pmcr, 155 enum armv6_counters counter) 156 { 157 int ret = 0; 158 159 if (ARMV6_CYCLE_COUNTER == counter) 160 ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW; 161 else if (ARMV6_COUNTER0 == counter) 162 ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW; 163 else if (ARMV6_COUNTER1 == counter) 164 ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW; 165 else 166 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 167 168 return ret; 169 } 170 171 static inline u64 armv6pmu_read_counter(struct perf_event *event) 172 { 173 struct hw_perf_event *hwc = &event->hw; 174 int counter = hwc->idx; 175 unsigned long value = 0; 176 177 if (ARMV6_CYCLE_COUNTER == counter) 178 asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value)); 179 else if (ARMV6_COUNTER0 == counter) 180 asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value)); 181 else if (ARMV6_COUNTER1 == counter) 182 asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value)); 183 else 184 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 185 186 return value; 187 } 188 189 static inline void armv6pmu_write_counter(struct perf_event *event, u64 value) 190 { 191 struct hw_perf_event *hwc = &event->hw; 192 int counter = hwc->idx; 193 194 if (ARMV6_CYCLE_COUNTER == counter) 195 asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value)); 196 else if (ARMV6_COUNTER0 == counter) 197 asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value)); 198 else if (ARMV6_COUNTER1 == counter) 199 asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value)); 200 else 201 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 202 } 203 204 static void armv6pmu_enable_event(struct perf_event *event) 205 { 206 unsigned long val, mask, evt; 207 struct hw_perf_event *hwc = &event->hw; 208 int idx = hwc->idx; 209 210 if (ARMV6_CYCLE_COUNTER == idx) { 211 mask = 0; 212 evt = ARMV6_PMCR_CCOUNT_IEN; 213 } else if (ARMV6_COUNTER0 == idx) { 214 mask = ARMV6_PMCR_EVT_COUNT0_MASK; 215 evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) | 216 ARMV6_PMCR_COUNT0_IEN; 217 } else if (ARMV6_COUNTER1 == idx) { 218 mask = ARMV6_PMCR_EVT_COUNT1_MASK; 219 evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) | 220 ARMV6_PMCR_COUNT1_IEN; 221 } else { 222 WARN_ONCE(1, "invalid counter number (%d)\n", idx); 223 return; 224 } 225 226 /* 227 * Mask out the current event and set the counter to count the event 228 * that we're interested in. 229 */ 230 val = armv6_pmcr_read(); 231 val &= ~mask; 232 val |= evt; 233 armv6_pmcr_write(val); 234 } 235 236 static irqreturn_t 237 armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) 238 { 239 unsigned long pmcr = armv6_pmcr_read(); 240 struct perf_sample_data data; 241 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 242 struct pt_regs *regs; 243 int idx; 244 245 if (!armv6_pmcr_has_overflowed(pmcr)) 246 return IRQ_NONE; 247 248 regs = get_irq_regs(); 249 250 /* 251 * The interrupts are cleared by writing the overflow flags back to 252 * the control register. All of the other bits don't have any effect 253 * if they are rewritten, so write the whole value back. 254 */ 255 armv6_pmcr_write(pmcr); 256 257 for (idx = 0; idx < cpu_pmu->num_events; ++idx) { 258 struct perf_event *event = cpuc->events[idx]; 259 struct hw_perf_event *hwc; 260 261 /* Ignore if we don't have an event. */ 262 if (!event) 263 continue; 264 265 /* 266 * We have a single interrupt for all counters. Check that 267 * each counter has overflowed before we process it. 268 */ 269 if (!armv6_pmcr_counter_has_overflowed(pmcr, idx)) 270 continue; 271 272 hwc = &event->hw; 273 armpmu_event_update(event); 274 perf_sample_data_init(&data, 0, hwc->last_period); 275 if (!armpmu_event_set_period(event)) 276 continue; 277 278 if (perf_event_overflow(event, &data, regs)) 279 cpu_pmu->disable(event); 280 } 281 282 /* 283 * Handle the pending perf events. 284 * 285 * Note: this call *must* be run with interrupts disabled. For 286 * platforms that can have the PMU interrupts raised as an NMI, this 287 * will not work. 288 */ 289 irq_work_run(); 290 291 return IRQ_HANDLED; 292 } 293 294 static void armv6pmu_start(struct arm_pmu *cpu_pmu) 295 { 296 unsigned long val; 297 298 val = armv6_pmcr_read(); 299 val |= ARMV6_PMCR_ENABLE; 300 armv6_pmcr_write(val); 301 } 302 303 static void armv6pmu_stop(struct arm_pmu *cpu_pmu) 304 { 305 unsigned long val; 306 307 val = armv6_pmcr_read(); 308 val &= ~ARMV6_PMCR_ENABLE; 309 armv6_pmcr_write(val); 310 } 311 312 static int 313 armv6pmu_get_event_idx(struct pmu_hw_events *cpuc, 314 struct perf_event *event) 315 { 316 struct hw_perf_event *hwc = &event->hw; 317 /* Always place a cycle counter into the cycle counter. */ 318 if (ARMV6_PERFCTR_CPU_CYCLES == hwc->config_base) { 319 if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask)) 320 return -EAGAIN; 321 322 return ARMV6_CYCLE_COUNTER; 323 } else { 324 /* 325 * For anything other than a cycle counter, try and use 326 * counter0 and counter1. 327 */ 328 if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) 329 return ARMV6_COUNTER1; 330 331 if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) 332 return ARMV6_COUNTER0; 333 334 /* The counters are all in use. */ 335 return -EAGAIN; 336 } 337 } 338 339 static void armv6pmu_clear_event_idx(struct pmu_hw_events *cpuc, 340 struct perf_event *event) 341 { 342 clear_bit(event->hw.idx, cpuc->used_mask); 343 } 344 345 static void armv6pmu_disable_event(struct perf_event *event) 346 { 347 unsigned long val, mask, evt; 348 struct hw_perf_event *hwc = &event->hw; 349 int idx = hwc->idx; 350 351 if (ARMV6_CYCLE_COUNTER == idx) { 352 mask = ARMV6_PMCR_CCOUNT_IEN; 353 evt = 0; 354 } else if (ARMV6_COUNTER0 == idx) { 355 mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK; 356 evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT; 357 } else if (ARMV6_COUNTER1 == idx) { 358 mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK; 359 evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT; 360 } else { 361 WARN_ONCE(1, "invalid counter number (%d)\n", idx); 362 return; 363 } 364 365 /* 366 * Mask out the current event and set the counter to count the number 367 * of ETM bus signal assertion cycles. The external reporting should 368 * be disabled and so this should never increment. 369 */ 370 val = armv6_pmcr_read(); 371 val &= ~mask; 372 val |= evt; 373 armv6_pmcr_write(val); 374 } 375 376 static int armv6_map_event(struct perf_event *event) 377 { 378 return armpmu_map_event(event, &armv6_perf_map, 379 &armv6_perf_cache_map, 0xFF); 380 } 381 382 static void armv6pmu_init(struct arm_pmu *cpu_pmu) 383 { 384 cpu_pmu->handle_irq = armv6pmu_handle_irq; 385 cpu_pmu->enable = armv6pmu_enable_event; 386 cpu_pmu->disable = armv6pmu_disable_event; 387 cpu_pmu->read_counter = armv6pmu_read_counter; 388 cpu_pmu->write_counter = armv6pmu_write_counter; 389 cpu_pmu->get_event_idx = armv6pmu_get_event_idx; 390 cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; 391 cpu_pmu->start = armv6pmu_start; 392 cpu_pmu->stop = armv6pmu_stop; 393 cpu_pmu->map_event = armv6_map_event; 394 cpu_pmu->num_events = 3; 395 } 396 397 static int armv6_1136_pmu_init(struct arm_pmu *cpu_pmu) 398 { 399 armv6pmu_init(cpu_pmu); 400 cpu_pmu->name = "armv6_1136"; 401 return 0; 402 } 403 404 static int armv6_1176_pmu_init(struct arm_pmu *cpu_pmu) 405 { 406 armv6pmu_init(cpu_pmu); 407 cpu_pmu->name = "armv6_1176"; 408 return 0; 409 } 410 411 static const struct of_device_id armv6_pmu_of_device_ids[] = { 412 {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, 413 {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, 414 { /* sentinel value */ } 415 }; 416 417 static int armv6_pmu_device_probe(struct platform_device *pdev) 418 { 419 return arm_pmu_device_probe(pdev, armv6_pmu_of_device_ids, NULL); 420 } 421 422 static struct platform_driver armv6_pmu_driver = { 423 .driver = { 424 .name = "armv6-pmu", 425 .of_match_table = armv6_pmu_of_device_ids, 426 }, 427 .probe = armv6_pmu_device_probe, 428 }; 429 430 builtin_platform_driver(armv6_pmu_driver); 431