1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Perf support for the Statistical Profiling Extension, introduced as 4 * part of ARMv8.2. 5 * 6 * Copyright (C) 2016 ARM Limited 7 * 8 * Author: Will Deacon <will.deacon@arm.com> 9 */ 10 11 #define PMUNAME "arm_spe" 12 #define DRVNAME PMUNAME "_pmu" 13 #define pr_fmt(fmt) DRVNAME ": " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/bitops.h> 17 #include <linux/bug.h> 18 #include <linux/capability.h> 19 #include <linux/cpuhotplug.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/errno.h> 23 #include <linux/interrupt.h> 24 #include <linux/irq.h> 25 #include <linux/kernel.h> 26 #include <linux/list.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/perf_event.h> 30 #include <linux/perf/arm_pmu.h> 31 #include <linux/platform_device.h> 32 #include <linux/printk.h> 33 #include <linux/slab.h> 34 #include <linux/smp.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/barrier.h> 38 #include <asm/cpufeature.h> 39 #include <asm/mmu.h> 40 #include <asm/sysreg.h> 41 42 /* 43 * Cache if the event is allowed to trace Context information. 44 * This allows us to perform the check, i.e, perf_allow_kernel(), 45 * in the context of the event owner, once, during the event_init(). 46 */ 47 #define SPE_PMU_HW_FLAGS_CX 0x00001 48 49 static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); 50 51 static void set_spe_event_has_cx(struct perf_event *event) 52 { 53 if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel()) 54 event->hw.flags |= SPE_PMU_HW_FLAGS_CX; 55 } 56 57 static bool get_spe_event_has_cx(struct perf_event *event) 58 { 59 return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); 60 } 61 62 #define ARM_SPE_BUF_PAD_BYTE 0 63 64 struct arm_spe_pmu_buf { 65 int nr_pages; 66 bool snapshot; 67 void *base; 68 }; 69 70 struct arm_spe_pmu { 71 struct pmu pmu; 72 struct platform_device *pdev; 73 cpumask_t supported_cpus; 74 struct hlist_node hotplug_node; 75 76 int irq; /* PPI */ 77 u16 pmsver; 78 u16 min_period; 79 u16 counter_sz; 80 81 #define SPE_PMU_FEAT_FILT_EVT (1UL << 0) 82 #define SPE_PMU_FEAT_FILT_TYP (1UL << 1) 83 #define SPE_PMU_FEAT_FILT_LAT (1UL << 2) 84 #define SPE_PMU_FEAT_ARCH_INST (1UL << 3) 85 #define SPE_PMU_FEAT_LDS (1UL << 4) 86 #define SPE_PMU_FEAT_ERND (1UL << 5) 87 #define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) 88 #define SPE_PMU_FEAT_DISCARD (1UL << 7) 89 #define SPE_PMU_FEAT_EFT (1UL << 8) 90 #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) 91 u64 features; 92 93 u64 pmsevfr_res0; 94 u16 max_record_sz; 95 u16 align; 96 struct perf_output_handle __percpu *handle; 97 }; 98 99 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) 100 101 /* Convert a free-running index from perf into an SPE buffer offset */ 102 #define PERF_IDX2OFF(idx, buf) \ 103 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) 104 105 /* Keep track of our dynamic hotplug state */ 106 static enum cpuhp_state arm_spe_pmu_online; 107 108 enum arm_spe_pmu_buf_fault_action { 109 SPE_PMU_BUF_FAULT_ACT_SPURIOUS, 110 SPE_PMU_BUF_FAULT_ACT_FATAL, 111 SPE_PMU_BUF_FAULT_ACT_OK, 112 }; 113 114 /* This sysfs gunk was really good fun to write. */ 115 enum arm_spe_pmu_capabilities { 116 SPE_PMU_CAP_ARCH_INST = 0, 117 SPE_PMU_CAP_ERND, 118 SPE_PMU_CAP_FEAT_MAX, 119 SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, 120 SPE_PMU_CAP_MIN_IVAL, 121 SPE_PMU_CAP_EVENT_FILTER, 122 }; 123 124 static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { 125 [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, 126 [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, 127 }; 128 129 static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) 130 { 131 if (cap < SPE_PMU_CAP_FEAT_MAX) 132 return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); 133 134 switch (cap) { 135 case SPE_PMU_CAP_CNT_SZ: 136 return spe_pmu->counter_sz; 137 case SPE_PMU_CAP_MIN_IVAL: 138 return spe_pmu->min_period; 139 case SPE_PMU_CAP_EVENT_FILTER: 140 return ~spe_pmu->pmsevfr_res0; 141 default: 142 WARN(1, "unknown cap %d\n", cap); 143 } 144 145 return 0; 146 } 147 148 static ssize_t arm_spe_pmu_cap_show(struct device *dev, 149 struct device_attribute *attr, 150 char *buf) 151 { 152 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 153 struct dev_ext_attribute *ea = 154 container_of(attr, struct dev_ext_attribute, attr); 155 int cap = (long)ea->var; 156 157 return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 158 } 159 160 static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev, 161 struct device_attribute *attr, 162 char *buf) 163 { 164 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 165 struct dev_ext_attribute *ea = 166 container_of(attr, struct dev_ext_attribute, attr); 167 int cap = (long)ea->var; 168 169 return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 170 } 171 172 #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ 173 &((struct dev_ext_attribute[]) { \ 174 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ 175 })[0].attr.attr 176 177 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ 178 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) 179 #define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var) \ 180 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var) 181 182 static struct attribute *arm_spe_pmu_cap_attr[] = { 183 SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), 184 SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), 185 SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), 186 SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), 187 SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER), 188 NULL, 189 }; 190 191 static const struct attribute_group arm_spe_pmu_cap_group = { 192 .name = "caps", 193 .attrs = arm_spe_pmu_cap_attr, 194 }; 195 196 /* User ABI */ 197 #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ 198 #define ATTR_CFG_FLD_ts_enable_LO 0 199 #define ATTR_CFG_FLD_ts_enable_HI 0 200 #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ 201 #define ATTR_CFG_FLD_pa_enable_LO 1 202 #define ATTR_CFG_FLD_pa_enable_HI 1 203 #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ 204 #define ATTR_CFG_FLD_pct_enable_LO 2 205 #define ATTR_CFG_FLD_pct_enable_HI 2 206 #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ 207 #define ATTR_CFG_FLD_jitter_LO 16 208 #define ATTR_CFG_FLD_jitter_HI 16 209 #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ 210 #define ATTR_CFG_FLD_branch_filter_LO 32 211 #define ATTR_CFG_FLD_branch_filter_HI 32 212 #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ 213 #define ATTR_CFG_FLD_load_filter_LO 33 214 #define ATTR_CFG_FLD_load_filter_HI 33 215 #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ 216 #define ATTR_CFG_FLD_store_filter_LO 34 217 #define ATTR_CFG_FLD_store_filter_HI 34 218 #define ATTR_CFG_FLD_discard_CFG config /* PMBLIMITR_EL1.FM = DISCARD */ 219 #define ATTR_CFG_FLD_discard_LO 35 220 #define ATTR_CFG_FLD_discard_HI 35 221 #define ATTR_CFG_FLD_branch_filter_mask_CFG config /* PMSFCR_EL1.Bm */ 222 #define ATTR_CFG_FLD_branch_filter_mask_LO 36 223 #define ATTR_CFG_FLD_branch_filter_mask_HI 36 224 #define ATTR_CFG_FLD_load_filter_mask_CFG config /* PMSFCR_EL1.LDm */ 225 #define ATTR_CFG_FLD_load_filter_mask_LO 37 226 #define ATTR_CFG_FLD_load_filter_mask_HI 37 227 #define ATTR_CFG_FLD_store_filter_mask_CFG config /* PMSFCR_EL1.STm */ 228 #define ATTR_CFG_FLD_store_filter_mask_LO 38 229 #define ATTR_CFG_FLD_store_filter_mask_HI 38 230 #define ATTR_CFG_FLD_simd_filter_CFG config /* PMSFCR_EL1.SIMD */ 231 #define ATTR_CFG_FLD_simd_filter_LO 39 232 #define ATTR_CFG_FLD_simd_filter_HI 39 233 #define ATTR_CFG_FLD_simd_filter_mask_CFG config /* PMSFCR_EL1.SIMDm */ 234 #define ATTR_CFG_FLD_simd_filter_mask_LO 40 235 #define ATTR_CFG_FLD_simd_filter_mask_HI 40 236 #define ATTR_CFG_FLD_float_filter_CFG config /* PMSFCR_EL1.FP */ 237 #define ATTR_CFG_FLD_float_filter_LO 41 238 #define ATTR_CFG_FLD_float_filter_HI 41 239 #define ATTR_CFG_FLD_float_filter_mask_CFG config /* PMSFCR_EL1.FPm */ 240 #define ATTR_CFG_FLD_float_filter_mask_LO 42 241 #define ATTR_CFG_FLD_float_filter_mask_HI 42 242 243 #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ 244 #define ATTR_CFG_FLD_event_filter_LO 0 245 #define ATTR_CFG_FLD_event_filter_HI 63 246 247 #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ 248 #define ATTR_CFG_FLD_min_latency_LO 0 249 #define ATTR_CFG_FLD_min_latency_HI 11 250 251 #define ATTR_CFG_FLD_inv_event_filter_CFG config3 /* PMSNEVFR_EL1 */ 252 #define ATTR_CFG_FLD_inv_event_filter_LO 0 253 #define ATTR_CFG_FLD_inv_event_filter_HI 63 254 255 GEN_PMU_FORMAT_ATTR(ts_enable); 256 GEN_PMU_FORMAT_ATTR(pa_enable); 257 GEN_PMU_FORMAT_ATTR(pct_enable); 258 GEN_PMU_FORMAT_ATTR(jitter); 259 GEN_PMU_FORMAT_ATTR(branch_filter); 260 GEN_PMU_FORMAT_ATTR(branch_filter_mask); 261 GEN_PMU_FORMAT_ATTR(load_filter); 262 GEN_PMU_FORMAT_ATTR(load_filter_mask); 263 GEN_PMU_FORMAT_ATTR(store_filter); 264 GEN_PMU_FORMAT_ATTR(store_filter_mask); 265 GEN_PMU_FORMAT_ATTR(simd_filter); 266 GEN_PMU_FORMAT_ATTR(simd_filter_mask); 267 GEN_PMU_FORMAT_ATTR(float_filter); 268 GEN_PMU_FORMAT_ATTR(float_filter_mask); 269 GEN_PMU_FORMAT_ATTR(event_filter); 270 GEN_PMU_FORMAT_ATTR(inv_event_filter); 271 GEN_PMU_FORMAT_ATTR(min_latency); 272 GEN_PMU_FORMAT_ATTR(discard); 273 274 static struct attribute *arm_spe_pmu_formats_attr[] = { 275 &format_attr_ts_enable.attr, 276 &format_attr_pa_enable.attr, 277 &format_attr_pct_enable.attr, 278 &format_attr_jitter.attr, 279 &format_attr_branch_filter.attr, 280 &format_attr_branch_filter_mask.attr, 281 &format_attr_load_filter.attr, 282 &format_attr_load_filter_mask.attr, 283 &format_attr_store_filter.attr, 284 &format_attr_store_filter_mask.attr, 285 &format_attr_simd_filter.attr, 286 &format_attr_simd_filter_mask.attr, 287 &format_attr_float_filter.attr, 288 &format_attr_float_filter_mask.attr, 289 &format_attr_event_filter.attr, 290 &format_attr_inv_event_filter.attr, 291 &format_attr_min_latency.attr, 292 &format_attr_discard.attr, 293 NULL, 294 }; 295 296 static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, 297 struct attribute *attr, 298 int unused) 299 { 300 struct device *dev = kobj_to_dev(kobj); 301 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 302 303 if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 304 return 0; 305 306 if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 307 return 0; 308 309 if ((attr == &format_attr_branch_filter_mask.attr || 310 attr == &format_attr_load_filter_mask.attr || 311 attr == &format_attr_store_filter_mask.attr || 312 attr == &format_attr_simd_filter.attr || 313 attr == &format_attr_simd_filter_mask.attr || 314 attr == &format_attr_float_filter.attr || 315 attr == &format_attr_float_filter_mask.attr) && 316 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 317 return 0; 318 319 return attr->mode; 320 } 321 322 static const struct attribute_group arm_spe_pmu_format_group = { 323 .name = "format", 324 .is_visible = arm_spe_pmu_format_attr_is_visible, 325 .attrs = arm_spe_pmu_formats_attr, 326 }; 327 328 static ssize_t cpumask_show(struct device *dev, 329 struct device_attribute *attr, char *buf) 330 { 331 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 332 333 return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); 334 } 335 static DEVICE_ATTR_RO(cpumask); 336 337 static struct attribute *arm_spe_pmu_attrs[] = { 338 &dev_attr_cpumask.attr, 339 NULL, 340 }; 341 342 static const struct attribute_group arm_spe_pmu_group = { 343 .attrs = arm_spe_pmu_attrs, 344 }; 345 346 static const struct attribute_group *arm_spe_pmu_attr_groups[] = { 347 &arm_spe_pmu_group, 348 &arm_spe_pmu_cap_group, 349 &arm_spe_pmu_format_group, 350 NULL, 351 }; 352 353 /* Convert between user ABI and register values */ 354 static u64 arm_spe_event_to_pmscr(struct perf_event *event) 355 { 356 struct perf_event_attr *attr = &event->attr; 357 u64 reg = 0; 358 359 reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable)); 360 reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable)); 361 reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable)); 362 363 if (!attr->exclude_user) 364 reg |= PMSCR_EL1_E0SPE; 365 366 if (!attr->exclude_kernel) 367 reg |= PMSCR_EL1_E1SPE; 368 369 if (get_spe_event_has_cx(event)) 370 reg |= PMSCR_EL1_CX; 371 372 return reg; 373 } 374 375 static void arm_spe_event_sanitise_period(struct perf_event *event) 376 { 377 u64 period = event->hw.sample_period; 378 u64 max_period = PMSIRR_EL1_INTERVAL_MASK; 379 380 /* 381 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a 382 * recommendation for the minimum interval, not a hardware limitation. 383 * 384 * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, 385 * Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) 386 * states: "Software must set this to a nonzero value". Use 1 as the 387 * minimum value. 388 */ 389 u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); 390 391 period = clamp_t(u64, period, min_period, max_period) & max_period; 392 event->hw.sample_period = period; 393 } 394 395 static u64 arm_spe_event_to_pmsirr(struct perf_event *event) 396 { 397 struct perf_event_attr *attr = &event->attr; 398 u64 reg = 0; 399 400 arm_spe_event_sanitise_period(event); 401 402 reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter)); 403 reg |= event->hw.sample_period; 404 405 return reg; 406 } 407 408 static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) 409 { 410 struct perf_event_attr *attr = &event->attr; 411 u64 reg = 0; 412 413 reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter)); 414 reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask)); 415 reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter)); 416 reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask)); 417 reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter)); 418 reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask)); 419 reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter)); 420 reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask)); 421 reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter)); 422 reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask)); 423 424 if (reg) 425 reg |= PMSFCR_EL1_FT; 426 427 if (ATTR_CFG_GET_FLD(attr, event_filter)) 428 reg |= PMSFCR_EL1_FE; 429 430 if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) 431 reg |= PMSFCR_EL1_FnE; 432 433 if (ATTR_CFG_GET_FLD(attr, min_latency)) 434 reg |= PMSFCR_EL1_FL; 435 436 return reg; 437 } 438 439 static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) 440 { 441 struct perf_event_attr *attr = &event->attr; 442 return ATTR_CFG_GET_FLD(attr, event_filter); 443 } 444 445 static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event) 446 { 447 struct perf_event_attr *attr = &event->attr; 448 return ATTR_CFG_GET_FLD(attr, inv_event_filter); 449 } 450 451 static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) 452 { 453 struct perf_event_attr *attr = &event->attr; 454 return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); 455 } 456 457 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) 458 { 459 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 460 u64 head = PERF_IDX2OFF(handle->head, buf); 461 462 memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); 463 if (!buf->snapshot) 464 perf_aux_output_skip(handle, len); 465 } 466 467 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) 468 { 469 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 470 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 471 u64 head = PERF_IDX2OFF(handle->head, buf); 472 u64 limit = buf->nr_pages * PAGE_SIZE; 473 474 /* 475 * The trace format isn't parseable in reverse, so clamp 476 * the limit to half of the buffer size in snapshot mode 477 * so that the worst case is half a buffer of records, as 478 * opposed to a single record. 479 */ 480 if (head < limit >> 1) 481 limit >>= 1; 482 483 /* 484 * If we're within max_record_sz of the limit, we must 485 * pad, move the head index and recompute the limit. 486 */ 487 if (limit - head < spe_pmu->max_record_sz) { 488 arm_spe_pmu_pad_buf(handle, limit - head); 489 handle->head = PERF_IDX2OFF(limit, buf); 490 limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; 491 } 492 493 return limit; 494 } 495 496 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) 497 { 498 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 499 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 500 const u64 bufsize = buf->nr_pages * PAGE_SIZE; 501 u64 limit = bufsize; 502 u64 head, tail, wakeup; 503 504 /* 505 * The head can be misaligned for two reasons: 506 * 507 * 1. The hardware left PMBPTR pointing to the first byte after 508 * a record when generating a buffer management event. 509 * 510 * 2. We used perf_aux_output_skip to consume handle->size bytes 511 * and CIRC_SPACE was used to compute the size, which always 512 * leaves one entry free. 513 * 514 * Deal with this by padding to the next alignment boundary and 515 * moving the head index. If we run out of buffer space, we'll 516 * reduce handle->size to zero and end up reporting truncation. 517 */ 518 head = PERF_IDX2OFF(handle->head, buf); 519 if (!IS_ALIGNED(head, spe_pmu->align)) { 520 unsigned long delta = roundup(head, spe_pmu->align) - head; 521 522 delta = min(delta, handle->size); 523 arm_spe_pmu_pad_buf(handle, delta); 524 head = PERF_IDX2OFF(handle->head, buf); 525 } 526 527 /* If we've run out of free space, then nothing more to do */ 528 if (!handle->size) 529 goto no_space; 530 531 /* Compute the tail and wakeup indices now that we've aligned head */ 532 tail = PERF_IDX2OFF(handle->head + handle->size, buf); 533 wakeup = PERF_IDX2OFF(handle->wakeup, buf); 534 535 /* 536 * Avoid clobbering unconsumed data. We know we have space, so 537 * if we see head == tail we know that the buffer is empty. If 538 * head > tail, then there's nothing to clobber prior to 539 * wrapping. 540 */ 541 if (head < tail) 542 limit = round_down(tail, PAGE_SIZE); 543 544 /* 545 * Wakeup may be arbitrarily far into the future. If it's not in 546 * the current generation, either we'll wrap before hitting it, 547 * or it's in the past and has been handled already. 548 * 549 * If there's a wakeup before we wrap, arrange to be woken up by 550 * the page boundary following it. Keep the tail boundary if 551 * that's lower. 552 */ 553 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) 554 limit = min(limit, round_up(wakeup, PAGE_SIZE)); 555 556 if (limit > head) 557 return limit; 558 559 arm_spe_pmu_pad_buf(handle, handle->size); 560 no_space: 561 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 562 perf_aux_output_end(handle, 0); 563 return 0; 564 } 565 566 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) 567 { 568 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 569 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 570 u64 limit = __arm_spe_pmu_next_off(handle); 571 u64 head = PERF_IDX2OFF(handle->head, buf); 572 573 /* 574 * If the head has come too close to the end of the buffer, 575 * then pad to the end and recompute the limit. 576 */ 577 if (limit && (limit - head < spe_pmu->max_record_sz)) { 578 arm_spe_pmu_pad_buf(handle, limit - head); 579 limit = __arm_spe_pmu_next_off(handle); 580 } 581 582 return limit; 583 } 584 585 static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, 586 struct perf_event *event) 587 { 588 u64 base, limit; 589 struct arm_spe_pmu_buf *buf; 590 591 if (ATTR_CFG_GET_FLD(&event->attr, discard)) { 592 limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD); 593 limit |= PMBLIMITR_EL1_E; 594 goto out_write_limit; 595 } 596 597 /* Start a new aux session */ 598 buf = perf_aux_output_begin(handle, event); 599 if (!buf) { 600 event->hw.state |= PERF_HES_STOPPED; 601 /* 602 * We still need to clear the limit pointer, since the 603 * profiler might only be disabled by virtue of a fault. 604 */ 605 limit = 0; 606 goto out_write_limit; 607 } 608 609 limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) 610 : arm_spe_pmu_next_off(handle); 611 if (limit) 612 limit |= PMBLIMITR_EL1_E; 613 614 limit += (u64)buf->base; 615 base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); 616 write_sysreg_s(base, SYS_PMBPTR_EL1); 617 618 out_write_limit: 619 write_sysreg_s(limit, SYS_PMBLIMITR_EL1); 620 } 621 622 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) 623 { 624 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 625 u64 offset, size; 626 627 offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; 628 size = offset - PERF_IDX2OFF(handle->head, buf); 629 630 if (buf->snapshot) 631 handle->head = offset; 632 633 perf_aux_output_end(handle, size); 634 } 635 636 static void arm_spe_pmu_disable_and_drain_local(void) 637 { 638 /* Disable profiling at EL0 and EL1 */ 639 write_sysreg_s(0, SYS_PMSCR_EL1); 640 isb(); 641 642 /* Drain any buffered data */ 643 psb_csync(); 644 dsb(nsh); 645 646 /* Disable the profiling buffer */ 647 write_sysreg_s(0, SYS_PMBLIMITR_EL1); 648 isb(); 649 } 650 651 /* IRQ handling */ 652 static enum arm_spe_pmu_buf_fault_action 653 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) 654 { 655 const char *err_str; 656 u64 pmbsr; 657 enum arm_spe_pmu_buf_fault_action ret; 658 659 /* 660 * Ensure new profiling data is visible to the CPU and any external 661 * aborts have been resolved. 662 */ 663 psb_csync(); 664 dsb(nsh); 665 666 /* Ensure hardware updates to PMBPTR_EL1 are visible */ 667 isb(); 668 669 /* Service required? */ 670 pmbsr = read_sysreg_s(SYS_PMBSR_EL1); 671 if (!FIELD_GET(PMBSR_EL1_S, pmbsr)) 672 return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; 673 674 /* 675 * If we've lost data, disable profiling and also set the PARTIAL 676 * flag to indicate that the last record is corrupted. 677 */ 678 if (FIELD_GET(PMBSR_EL1_DL, pmbsr)) 679 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | 680 PERF_AUX_FLAG_PARTIAL); 681 682 /* Report collisions to userspace so that it can up the period */ 683 if (FIELD_GET(PMBSR_EL1_COLL, pmbsr)) 684 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); 685 686 /* We only expect buffer management events */ 687 switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) { 688 case PMBSR_EL1_EC_BUF: 689 /* Handled below */ 690 break; 691 case PMBSR_EL1_EC_FAULT_S1: 692 case PMBSR_EL1_EC_FAULT_S2: 693 err_str = "Unexpected buffer fault"; 694 goto out_err; 695 default: 696 err_str = "Unknown error code"; 697 goto out_err; 698 } 699 700 /* Buffer management event */ 701 switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) { 702 case PMBSR_EL1_BUF_BSC_FULL: 703 ret = SPE_PMU_BUF_FAULT_ACT_OK; 704 goto out_stop; 705 default: 706 err_str = "Unknown buffer status code"; 707 } 708 709 out_err: 710 pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", 711 err_str, smp_processor_id(), pmbsr, 712 read_sysreg_s(SYS_PMBPTR_EL1), 713 read_sysreg_s(SYS_PMBLIMITR_EL1)); 714 ret = SPE_PMU_BUF_FAULT_ACT_FATAL; 715 716 out_stop: 717 arm_spe_perf_aux_output_end(handle); 718 return ret; 719 } 720 721 static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) 722 { 723 struct perf_output_handle *handle = dev; 724 struct perf_event *event = handle->event; 725 enum arm_spe_pmu_buf_fault_action act; 726 727 if (!perf_get_aux(handle)) 728 return IRQ_NONE; 729 730 act = arm_spe_pmu_buf_get_fault_act(handle); 731 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 732 return IRQ_NONE; 733 734 /* 735 * Ensure perf callbacks have completed, which may disable the 736 * profiling buffer in response to a TRUNCATION flag. 737 */ 738 irq_work_run(); 739 740 switch (act) { 741 case SPE_PMU_BUF_FAULT_ACT_FATAL: 742 /* 743 * If a fatal exception occurred then leaving the profiling 744 * buffer enabled is a recipe waiting to happen. Since 745 * fatal faults don't always imply truncation, make sure 746 * that the profiling buffer is disabled explicitly before 747 * clearing the syndrome register. 748 */ 749 arm_spe_pmu_disable_and_drain_local(); 750 break; 751 case SPE_PMU_BUF_FAULT_ACT_OK: 752 /* 753 * We handled the fault (the buffer was full), so resume 754 * profiling as long as we didn't detect truncation. 755 * PMBPTR might be misaligned, but we'll burn that bridge 756 * when we get to it. 757 */ 758 if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { 759 arm_spe_perf_aux_output_begin(handle, event); 760 isb(); 761 } 762 break; 763 case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: 764 /* We've seen you before, but GCC has the memory of a sieve. */ 765 break; 766 } 767 768 /* The buffer pointers are now sane, so resume profiling. */ 769 write_sysreg_s(0, SYS_PMBSR_EL1); 770 return IRQ_HANDLED; 771 } 772 773 /* Perf callbacks */ 774 static int arm_spe_pmu_event_init(struct perf_event *event) 775 { 776 u64 reg; 777 struct perf_event_attr *attr = &event->attr; 778 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 779 780 /* This is, of course, deeply driver-specific */ 781 if (attr->type != event->pmu->type) 782 return -ENOENT; 783 784 if (event->cpu >= 0 && 785 !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) 786 return -ENOENT; 787 788 if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0) 789 return -EOPNOTSUPP; 790 791 if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) 792 return -EOPNOTSUPP; 793 794 if (attr->exclude_idle) 795 return -EOPNOTSUPP; 796 797 /* 798 * Feedback-directed frequency throttling doesn't work when we 799 * have a buffer of samples. We'd need to manually count the 800 * samples in the buffer when it fills up and adjust the event 801 * count to reflect that. Instead, just force the user to specify 802 * a sample period. 803 */ 804 if (attr->freq) 805 return -EINVAL; 806 807 reg = arm_spe_event_to_pmsfcr(event); 808 if ((FIELD_GET(PMSFCR_EL1_FE, reg)) && 809 !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) 810 return -EOPNOTSUPP; 811 812 if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) && 813 !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 814 return -EOPNOTSUPP; 815 816 if ((FIELD_GET(PMSFCR_EL1_FT, reg)) && 817 !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) 818 return -EOPNOTSUPP; 819 820 if ((FIELD_GET(PMSFCR_EL1_FL, reg)) && 821 !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) 822 return -EOPNOTSUPP; 823 824 if ((FIELD_GET(PMSFCR_EL1_LDm, reg) || 825 FIELD_GET(PMSFCR_EL1_STm, reg) || 826 FIELD_GET(PMSFCR_EL1_Bm, reg) || 827 FIELD_GET(PMSFCR_EL1_SIMD, reg) || 828 FIELD_GET(PMSFCR_EL1_SIMDm, reg) || 829 FIELD_GET(PMSFCR_EL1_FP, reg) || 830 FIELD_GET(PMSFCR_EL1_FPm, reg)) && 831 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 832 return -EOPNOTSUPP; 833 834 if (ATTR_CFG_GET_FLD(&event->attr, discard) && 835 !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 836 return -EOPNOTSUPP; 837 838 set_spe_event_has_cx(event); 839 reg = arm_spe_event_to_pmscr(event); 840 if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT)) 841 return perf_allow_kernel(); 842 843 return 0; 844 } 845 846 static void arm_spe_pmu_start(struct perf_event *event, int flags) 847 { 848 u64 reg; 849 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 850 struct hw_perf_event *hwc = &event->hw; 851 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 852 853 hwc->state = 0; 854 arm_spe_perf_aux_output_begin(handle, event); 855 if (hwc->state) 856 return; 857 858 reg = arm_spe_event_to_pmsfcr(event); 859 write_sysreg_s(reg, SYS_PMSFCR_EL1); 860 861 reg = arm_spe_event_to_pmsevfr(event); 862 write_sysreg_s(reg, SYS_PMSEVFR_EL1); 863 864 if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) { 865 reg = arm_spe_event_to_pmsnevfr(event); 866 write_sysreg_s(reg, SYS_PMSNEVFR_EL1); 867 } 868 869 reg = arm_spe_event_to_pmslatfr(event); 870 write_sysreg_s(reg, SYS_PMSLATFR_EL1); 871 872 if (flags & PERF_EF_RELOAD) { 873 reg = arm_spe_event_to_pmsirr(event); 874 write_sysreg_s(reg, SYS_PMSIRR_EL1); 875 isb(); 876 reg = local64_read(&hwc->period_left); 877 write_sysreg_s(reg, SYS_PMSICR_EL1); 878 } 879 880 reg = arm_spe_event_to_pmscr(event); 881 isb(); 882 write_sysreg_s(reg, SYS_PMSCR_EL1); 883 } 884 885 static void arm_spe_pmu_stop(struct perf_event *event, int flags) 886 { 887 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 888 struct hw_perf_event *hwc = &event->hw; 889 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 890 891 /* If we're already stopped, then nothing to do */ 892 if (hwc->state & PERF_HES_STOPPED) 893 return; 894 895 /* Stop all trace generation */ 896 arm_spe_pmu_disable_and_drain_local(); 897 898 if (flags & PERF_EF_UPDATE) { 899 /* 900 * If there's a fault pending then ensure we contain it 901 * to this buffer, since we might be on the context-switch 902 * path. 903 */ 904 if (perf_get_aux(handle)) { 905 enum arm_spe_pmu_buf_fault_action act; 906 907 act = arm_spe_pmu_buf_get_fault_act(handle); 908 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 909 arm_spe_perf_aux_output_end(handle); 910 else 911 write_sysreg_s(0, SYS_PMBSR_EL1); 912 } 913 914 /* 915 * This may also contain ECOUNT, but nobody else should 916 * be looking at period_left, since we forbid frequency 917 * based sampling. 918 */ 919 local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); 920 hwc->state |= PERF_HES_UPTODATE; 921 } 922 923 hwc->state |= PERF_HES_STOPPED; 924 } 925 926 static int arm_spe_pmu_add(struct perf_event *event, int flags) 927 { 928 int ret = 0; 929 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 930 struct hw_perf_event *hwc = &event->hw; 931 int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; 932 933 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 934 return -ENOENT; 935 936 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 937 938 if (flags & PERF_EF_START) { 939 arm_spe_pmu_start(event, PERF_EF_RELOAD); 940 if (hwc->state & PERF_HES_STOPPED) 941 ret = -EINVAL; 942 } 943 944 return ret; 945 } 946 947 static void arm_spe_pmu_del(struct perf_event *event, int flags) 948 { 949 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 950 } 951 952 static void arm_spe_pmu_read(struct perf_event *event) 953 { 954 } 955 956 static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, 957 int nr_pages, bool snapshot) 958 { 959 int i, cpu = event->cpu; 960 struct page **pglist; 961 struct arm_spe_pmu_buf *buf; 962 963 /* We need at least two pages for this to work. */ 964 if (nr_pages < 2) 965 return NULL; 966 967 /* 968 * We require an even number of pages for snapshot mode, so that 969 * we can effectively treat the buffer as consisting of two equal 970 * parts and give userspace a fighting chance of getting some 971 * useful data out of it. 972 */ 973 if (snapshot && (nr_pages & 1)) 974 return NULL; 975 976 if (cpu == -1) 977 cpu = raw_smp_processor_id(); 978 979 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); 980 if (!buf) 981 return NULL; 982 983 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); 984 if (!pglist) 985 goto out_free_buf; 986 987 for (i = 0; i < nr_pages; ++i) 988 pglist[i] = virt_to_page(pages[i]); 989 990 buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); 991 if (!buf->base) 992 goto out_free_pglist; 993 994 buf->nr_pages = nr_pages; 995 buf->snapshot = snapshot; 996 997 kfree(pglist); 998 return buf; 999 1000 out_free_pglist: 1001 kfree(pglist); 1002 out_free_buf: 1003 kfree(buf); 1004 return NULL; 1005 } 1006 1007 static void arm_spe_pmu_free_aux(void *aux) 1008 { 1009 struct arm_spe_pmu_buf *buf = aux; 1010 1011 vunmap(buf->base); 1012 kfree(buf); 1013 } 1014 1015 /* Initialisation and teardown functions */ 1016 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) 1017 { 1018 static atomic_t pmu_idx = ATOMIC_INIT(-1); 1019 1020 int idx; 1021 char *name; 1022 struct device *dev = &spe_pmu->pdev->dev; 1023 1024 spe_pmu->pmu = (struct pmu) { 1025 .module = THIS_MODULE, 1026 .parent = &spe_pmu->pdev->dev, 1027 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, 1028 .attr_groups = arm_spe_pmu_attr_groups, 1029 /* 1030 * We hitch a ride on the software context here, so that 1031 * we can support per-task profiling (which is not possible 1032 * with the invalid context as it doesn't get sched callbacks). 1033 * This requires that userspace either uses a dummy event for 1034 * perf_event_open, since the aux buffer is not setup until 1035 * a subsequent mmap, or creates the profiling event in a 1036 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it 1037 * once the buffer has been created. 1038 */ 1039 .task_ctx_nr = perf_sw_context, 1040 .event_init = arm_spe_pmu_event_init, 1041 .add = arm_spe_pmu_add, 1042 .del = arm_spe_pmu_del, 1043 .start = arm_spe_pmu_start, 1044 .stop = arm_spe_pmu_stop, 1045 .read = arm_spe_pmu_read, 1046 .setup_aux = arm_spe_pmu_setup_aux, 1047 .free_aux = arm_spe_pmu_free_aux, 1048 }; 1049 1050 idx = atomic_inc_return(&pmu_idx); 1051 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); 1052 if (!name) { 1053 dev_err(dev, "failed to allocate name for pmu %d\n", idx); 1054 return -ENOMEM; 1055 } 1056 1057 return perf_pmu_register(&spe_pmu->pmu, name, -1); 1058 } 1059 1060 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) 1061 { 1062 perf_pmu_unregister(&spe_pmu->pmu); 1063 } 1064 1065 static void __arm_spe_pmu_dev_probe(void *info) 1066 { 1067 int fld; 1068 u64 reg; 1069 struct arm_spe_pmu *spe_pmu = info; 1070 struct device *dev = &spe_pmu->pdev->dev; 1071 1072 fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), 1073 ID_AA64DFR0_EL1_PMSVer_SHIFT); 1074 if (!fld) { 1075 dev_err(dev, 1076 "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", 1077 fld, smp_processor_id()); 1078 return; 1079 } 1080 spe_pmu->pmsver = (u16)fld; 1081 1082 /* Read PMBIDR first to determine whether or not we have access */ 1083 reg = read_sysreg_s(SYS_PMBIDR_EL1); 1084 if (FIELD_GET(PMBIDR_EL1_P, reg)) { 1085 dev_err(dev, 1086 "profiling buffer owned by higher exception level\n"); 1087 return; 1088 } 1089 1090 /* Minimum alignment. If it's out-of-range, then fail the probe */ 1091 fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg); 1092 spe_pmu->align = 1 << fld; 1093 if (spe_pmu->align > SZ_2K) { 1094 dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", 1095 fld, smp_processor_id()); 1096 return; 1097 } 1098 1099 /* It's now safe to read PMSIDR and figure out what we've got */ 1100 reg = read_sysreg_s(SYS_PMSIDR_EL1); 1101 if (FIELD_GET(PMSIDR_EL1_FE, reg)) 1102 spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; 1103 1104 if (FIELD_GET(PMSIDR_EL1_FnE, reg)) 1105 spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT; 1106 1107 if (FIELD_GET(PMSIDR_EL1_FT, reg)) 1108 spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; 1109 1110 if (FIELD_GET(PMSIDR_EL1_FL, reg)) 1111 spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; 1112 1113 if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg)) 1114 spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; 1115 1116 if (FIELD_GET(PMSIDR_EL1_LDS, reg)) 1117 spe_pmu->features |= SPE_PMU_FEAT_LDS; 1118 1119 if (FIELD_GET(PMSIDR_EL1_ERND, reg)) 1120 spe_pmu->features |= SPE_PMU_FEAT_ERND; 1121 1122 if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2) 1123 spe_pmu->features |= SPE_PMU_FEAT_DISCARD; 1124 1125 if (FIELD_GET(PMSIDR_EL1_EFT, reg)) 1126 spe_pmu->features |= SPE_PMU_FEAT_EFT; 1127 1128 /* This field has a spaced out encoding, so just use a look-up */ 1129 fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); 1130 switch (fld) { 1131 case PMSIDR_EL1_INTERVAL_256: 1132 spe_pmu->min_period = 256; 1133 break; 1134 case PMSIDR_EL1_INTERVAL_512: 1135 spe_pmu->min_period = 512; 1136 break; 1137 case PMSIDR_EL1_INTERVAL_768: 1138 spe_pmu->min_period = 768; 1139 break; 1140 case PMSIDR_EL1_INTERVAL_1024: 1141 spe_pmu->min_period = 1024; 1142 break; 1143 case PMSIDR_EL1_INTERVAL_1536: 1144 spe_pmu->min_period = 1536; 1145 break; 1146 case PMSIDR_EL1_INTERVAL_2048: 1147 spe_pmu->min_period = 2048; 1148 break; 1149 case PMSIDR_EL1_INTERVAL_3072: 1150 spe_pmu->min_period = 3072; 1151 break; 1152 default: 1153 dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", 1154 fld); 1155 fallthrough; 1156 case PMSIDR_EL1_INTERVAL_4096: 1157 spe_pmu->min_period = 4096; 1158 } 1159 1160 /* Maximum record size. If it's out-of-range, then fail the probe */ 1161 fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg); 1162 spe_pmu->max_record_sz = 1 << fld; 1163 if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { 1164 dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", 1165 fld, smp_processor_id()); 1166 return; 1167 } 1168 1169 fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg); 1170 switch (fld) { 1171 default: 1172 dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", 1173 fld); 1174 fallthrough; 1175 case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT: 1176 spe_pmu->counter_sz = 12; 1177 break; 1178 case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT: 1179 spe_pmu->counter_sz = 16; 1180 } 1181 1182 /* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */ 1183 write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1); 1184 spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1); 1185 1186 dev_info(dev, 1187 "probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", 1188 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus), 1189 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); 1190 1191 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; 1192 } 1193 1194 static void __arm_spe_pmu_reset_local(void) 1195 { 1196 /* 1197 * This is probably overkill, as we have no idea where we're 1198 * draining any buffered data to... 1199 */ 1200 arm_spe_pmu_disable_and_drain_local(); 1201 1202 /* Reset the buffer base pointer */ 1203 write_sysreg_s(0, SYS_PMBPTR_EL1); 1204 isb(); 1205 1206 /* Clear any pending management interrupts */ 1207 write_sysreg_s(0, SYS_PMBSR_EL1); 1208 isb(); 1209 } 1210 1211 static void __arm_spe_pmu_setup_one(void *info) 1212 { 1213 struct arm_spe_pmu *spe_pmu = info; 1214 1215 __arm_spe_pmu_reset_local(); 1216 enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); 1217 } 1218 1219 static void __arm_spe_pmu_stop_one(void *info) 1220 { 1221 struct arm_spe_pmu *spe_pmu = info; 1222 1223 disable_percpu_irq(spe_pmu->irq); 1224 __arm_spe_pmu_reset_local(); 1225 } 1226 1227 static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) 1228 { 1229 struct arm_spe_pmu *spe_pmu; 1230 1231 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1232 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1233 return 0; 1234 1235 __arm_spe_pmu_setup_one(spe_pmu); 1236 return 0; 1237 } 1238 1239 static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1240 { 1241 struct arm_spe_pmu *spe_pmu; 1242 1243 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1244 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1245 return 0; 1246 1247 __arm_spe_pmu_stop_one(spe_pmu); 1248 return 0; 1249 } 1250 1251 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) 1252 { 1253 int ret; 1254 cpumask_t *mask = &spe_pmu->supported_cpus; 1255 1256 /* Make sure we probe the hardware on a relevant CPU */ 1257 ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); 1258 if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) 1259 return -ENXIO; 1260 1261 /* Request our PPIs (note that the IRQ is still disabled) */ 1262 ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME, 1263 spe_pmu->handle); 1264 if (ret) 1265 return ret; 1266 1267 /* 1268 * Register our hotplug notifier now so we don't miss any events. 1269 * This will enable the IRQ for any supported CPUs that are already 1270 * up. 1271 */ 1272 ret = cpuhp_state_add_instance(arm_spe_pmu_online, 1273 &spe_pmu->hotplug_node); 1274 if (ret) 1275 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1276 1277 return ret; 1278 } 1279 1280 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) 1281 { 1282 cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); 1283 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1284 } 1285 1286 /* Driver and device probing */ 1287 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) 1288 { 1289 struct platform_device *pdev = spe_pmu->pdev; 1290 int irq = platform_get_irq(pdev, 0); 1291 1292 if (irq < 0) 1293 return -ENXIO; 1294 1295 if (!irq_is_percpu(irq)) { 1296 dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); 1297 return -EINVAL; 1298 } 1299 1300 if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) { 1301 dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq); 1302 return -EINVAL; 1303 } 1304 1305 spe_pmu->irq = irq; 1306 return 0; 1307 } 1308 1309 static const struct of_device_id arm_spe_pmu_of_match[] = { 1310 { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, 1311 { /* Sentinel */ }, 1312 }; 1313 MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); 1314 1315 static const struct platform_device_id arm_spe_match[] = { 1316 { ARMV8_SPE_PDEV_NAME, 0}, 1317 { } 1318 }; 1319 MODULE_DEVICE_TABLE(platform, arm_spe_match); 1320 1321 static int arm_spe_pmu_device_probe(struct platform_device *pdev) 1322 { 1323 int ret; 1324 struct arm_spe_pmu *spe_pmu; 1325 struct device *dev = &pdev->dev; 1326 1327 /* 1328 * If kernelspace is unmapped when running at EL0, then the SPE 1329 * buffer will fault and prematurely terminate the AUX session. 1330 */ 1331 if (arm64_kernel_unmapped_at_el0()) { 1332 dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); 1333 return -EPERM; 1334 } 1335 1336 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); 1337 if (!spe_pmu) 1338 return -ENOMEM; 1339 1340 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); 1341 if (!spe_pmu->handle) 1342 return -ENOMEM; 1343 1344 spe_pmu->pdev = pdev; 1345 platform_set_drvdata(pdev, spe_pmu); 1346 1347 ret = arm_spe_pmu_irq_probe(spe_pmu); 1348 if (ret) 1349 goto out_free_handle; 1350 1351 ret = arm_spe_pmu_dev_init(spe_pmu); 1352 if (ret) 1353 goto out_free_handle; 1354 1355 ret = arm_spe_pmu_perf_init(spe_pmu); 1356 if (ret) 1357 goto out_teardown_dev; 1358 1359 return 0; 1360 1361 out_teardown_dev: 1362 arm_spe_pmu_dev_teardown(spe_pmu); 1363 out_free_handle: 1364 free_percpu(spe_pmu->handle); 1365 return ret; 1366 } 1367 1368 static void arm_spe_pmu_device_remove(struct platform_device *pdev) 1369 { 1370 struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); 1371 1372 arm_spe_pmu_perf_destroy(spe_pmu); 1373 arm_spe_pmu_dev_teardown(spe_pmu); 1374 free_percpu(spe_pmu->handle); 1375 } 1376 1377 static struct platform_driver arm_spe_pmu_driver = { 1378 .id_table = arm_spe_match, 1379 .driver = { 1380 .name = DRVNAME, 1381 .of_match_table = of_match_ptr(arm_spe_pmu_of_match), 1382 .suppress_bind_attrs = true, 1383 }, 1384 .probe = arm_spe_pmu_device_probe, 1385 .remove = arm_spe_pmu_device_remove, 1386 }; 1387 1388 static int __init arm_spe_pmu_init(void) 1389 { 1390 int ret; 1391 1392 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, 1393 arm_spe_pmu_cpu_startup, 1394 arm_spe_pmu_cpu_teardown); 1395 if (ret < 0) 1396 return ret; 1397 arm_spe_pmu_online = ret; 1398 1399 ret = platform_driver_register(&arm_spe_pmu_driver); 1400 if (ret) 1401 cpuhp_remove_multi_state(arm_spe_pmu_online); 1402 1403 return ret; 1404 } 1405 1406 static void __exit arm_spe_pmu_exit(void) 1407 { 1408 platform_driver_unregister(&arm_spe_pmu_driver); 1409 cpuhp_remove_multi_state(arm_spe_pmu_online); 1410 } 1411 1412 module_init(arm_spe_pmu_init); 1413 module_exit(arm_spe_pmu_exit); 1414 1415 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); 1416 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1417 MODULE_LICENSE("GPL v2"); 1418