1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Perf support for the Statistical Profiling Extension, introduced as 4 * part of ARMv8.2. 5 * 6 * Copyright (C) 2016 ARM Limited 7 * 8 * Author: Will Deacon <will.deacon@arm.com> 9 */ 10 11 #define PMUNAME "arm_spe" 12 #define DRVNAME PMUNAME "_pmu" 13 #define pr_fmt(fmt) DRVNAME ": " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/bitops.h> 17 #include <linux/bug.h> 18 #include <linux/capability.h> 19 #include <linux/cpuhotplug.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/errno.h> 23 #include <linux/interrupt.h> 24 #include <linux/irq.h> 25 #include <linux/kernel.h> 26 #include <linux/list.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/perf_event.h> 30 #include <linux/perf/arm_pmu.h> 31 #include <linux/platform_device.h> 32 #include <linux/printk.h> 33 #include <linux/slab.h> 34 #include <linux/smp.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/barrier.h> 38 #include <asm/cpufeature.h> 39 #include <asm/mmu.h> 40 #include <asm/sysreg.h> 41 42 /* 43 * Cache if the event is allowed to trace Context information. 44 * This allows us to perform the check, i.e, perf_allow_kernel(), 45 * in the context of the event owner, once, during the event_init(). 46 */ 47 #define SPE_PMU_HW_FLAGS_CX 0x00001 48 49 static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); 50 51 static void set_spe_event_has_cx(struct perf_event *event) 52 { 53 if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel()) 54 event->hw.flags |= SPE_PMU_HW_FLAGS_CX; 55 } 56 57 static bool get_spe_event_has_cx(struct perf_event *event) 58 { 59 return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); 60 } 61 62 #define ARM_SPE_BUF_PAD_BYTE 0 63 64 struct arm_spe_pmu_buf { 65 int nr_pages; 66 bool snapshot; 67 void *base; 68 }; 69 70 struct arm_spe_pmu { 71 struct pmu pmu; 72 struct platform_device *pdev; 73 cpumask_t supported_cpus; 74 struct hlist_node hotplug_node; 75 76 int irq; /* PPI */ 77 u16 pmsver; 78 u16 min_period; 79 u16 counter_sz; 80 81 #define SPE_PMU_FEAT_FILT_EVT (1UL << 0) 82 #define SPE_PMU_FEAT_FILT_TYP (1UL << 1) 83 #define SPE_PMU_FEAT_FILT_LAT (1UL << 2) 84 #define SPE_PMU_FEAT_ARCH_INST (1UL << 3) 85 #define SPE_PMU_FEAT_LDS (1UL << 4) 86 #define SPE_PMU_FEAT_ERND (1UL << 5) 87 #define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) 88 #define SPE_PMU_FEAT_DISCARD (1UL << 7) 89 #define SPE_PMU_FEAT_EFT (1UL << 8) 90 #define SPE_PMU_FEAT_FDS (1UL << 9) 91 #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) 92 u64 features; 93 94 u64 pmsevfr_res0; 95 u16 max_record_sz; 96 u16 align; 97 struct perf_output_handle __percpu *handle; 98 }; 99 100 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) 101 102 /* Convert a free-running index from perf into an SPE buffer offset */ 103 #define PERF_IDX2OFF(idx, buf) \ 104 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) 105 106 /* Keep track of our dynamic hotplug state */ 107 static enum cpuhp_state arm_spe_pmu_online; 108 109 static void arm_spe_pmu_stop(struct perf_event *event, int flags); 110 111 enum arm_spe_pmu_buf_fault_action { 112 SPE_PMU_BUF_FAULT_ACT_SPURIOUS, 113 SPE_PMU_BUF_FAULT_ACT_FATAL, 114 SPE_PMU_BUF_FAULT_ACT_OK, 115 }; 116 117 /* This sysfs gunk was really good fun to write. */ 118 enum arm_spe_pmu_capabilities { 119 SPE_PMU_CAP_ARCH_INST = 0, 120 SPE_PMU_CAP_ERND, 121 SPE_PMU_CAP_FEAT_MAX, 122 SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, 123 SPE_PMU_CAP_MIN_IVAL, 124 SPE_PMU_CAP_EVENT_FILTER, 125 }; 126 127 static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { 128 [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, 129 [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, 130 }; 131 132 static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) 133 { 134 if (cap < SPE_PMU_CAP_FEAT_MAX) 135 return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); 136 137 switch (cap) { 138 case SPE_PMU_CAP_CNT_SZ: 139 return spe_pmu->counter_sz; 140 case SPE_PMU_CAP_MIN_IVAL: 141 return spe_pmu->min_period; 142 case SPE_PMU_CAP_EVENT_FILTER: 143 return ~spe_pmu->pmsevfr_res0; 144 default: 145 WARN(1, "unknown cap %d\n", cap); 146 } 147 148 return 0; 149 } 150 151 static ssize_t arm_spe_pmu_cap_show(struct device *dev, 152 struct device_attribute *attr, 153 char *buf) 154 { 155 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 156 struct dev_ext_attribute *ea = 157 container_of(attr, struct dev_ext_attribute, attr); 158 int cap = (long)ea->var; 159 160 return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 161 } 162 163 static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev, 164 struct device_attribute *attr, 165 char *buf) 166 { 167 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 168 struct dev_ext_attribute *ea = 169 container_of(attr, struct dev_ext_attribute, attr); 170 int cap = (long)ea->var; 171 172 return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 173 } 174 175 #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ 176 &((struct dev_ext_attribute[]) { \ 177 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ 178 })[0].attr.attr 179 180 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ 181 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) 182 #define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var) \ 183 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var) 184 185 static struct attribute *arm_spe_pmu_cap_attr[] = { 186 SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), 187 SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), 188 SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), 189 SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), 190 SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER), 191 NULL, 192 }; 193 194 static const struct attribute_group arm_spe_pmu_cap_group = { 195 .name = "caps", 196 .attrs = arm_spe_pmu_cap_attr, 197 }; 198 199 /* User ABI */ 200 #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ 201 #define ATTR_CFG_FLD_ts_enable_LO 0 202 #define ATTR_CFG_FLD_ts_enable_HI 0 203 #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ 204 #define ATTR_CFG_FLD_pa_enable_LO 1 205 #define ATTR_CFG_FLD_pa_enable_HI 1 206 #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ 207 #define ATTR_CFG_FLD_pct_enable_LO 2 208 #define ATTR_CFG_FLD_pct_enable_HI 2 209 #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ 210 #define ATTR_CFG_FLD_jitter_LO 16 211 #define ATTR_CFG_FLD_jitter_HI 16 212 #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ 213 #define ATTR_CFG_FLD_branch_filter_LO 32 214 #define ATTR_CFG_FLD_branch_filter_HI 32 215 #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ 216 #define ATTR_CFG_FLD_load_filter_LO 33 217 #define ATTR_CFG_FLD_load_filter_HI 33 218 #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ 219 #define ATTR_CFG_FLD_store_filter_LO 34 220 #define ATTR_CFG_FLD_store_filter_HI 34 221 #define ATTR_CFG_FLD_discard_CFG config /* PMBLIMITR_EL1.FM = DISCARD */ 222 #define ATTR_CFG_FLD_discard_LO 35 223 #define ATTR_CFG_FLD_discard_HI 35 224 #define ATTR_CFG_FLD_branch_filter_mask_CFG config /* PMSFCR_EL1.Bm */ 225 #define ATTR_CFG_FLD_branch_filter_mask_LO 36 226 #define ATTR_CFG_FLD_branch_filter_mask_HI 36 227 #define ATTR_CFG_FLD_load_filter_mask_CFG config /* PMSFCR_EL1.LDm */ 228 #define ATTR_CFG_FLD_load_filter_mask_LO 37 229 #define ATTR_CFG_FLD_load_filter_mask_HI 37 230 #define ATTR_CFG_FLD_store_filter_mask_CFG config /* PMSFCR_EL1.STm */ 231 #define ATTR_CFG_FLD_store_filter_mask_LO 38 232 #define ATTR_CFG_FLD_store_filter_mask_HI 38 233 #define ATTR_CFG_FLD_simd_filter_CFG config /* PMSFCR_EL1.SIMD */ 234 #define ATTR_CFG_FLD_simd_filter_LO 39 235 #define ATTR_CFG_FLD_simd_filter_HI 39 236 #define ATTR_CFG_FLD_simd_filter_mask_CFG config /* PMSFCR_EL1.SIMDm */ 237 #define ATTR_CFG_FLD_simd_filter_mask_LO 40 238 #define ATTR_CFG_FLD_simd_filter_mask_HI 40 239 #define ATTR_CFG_FLD_float_filter_CFG config /* PMSFCR_EL1.FP */ 240 #define ATTR_CFG_FLD_float_filter_LO 41 241 #define ATTR_CFG_FLD_float_filter_HI 41 242 #define ATTR_CFG_FLD_float_filter_mask_CFG config /* PMSFCR_EL1.FPm */ 243 #define ATTR_CFG_FLD_float_filter_mask_LO 42 244 #define ATTR_CFG_FLD_float_filter_mask_HI 42 245 246 #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ 247 #define ATTR_CFG_FLD_event_filter_LO 0 248 #define ATTR_CFG_FLD_event_filter_HI 63 249 250 #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ 251 #define ATTR_CFG_FLD_min_latency_LO 0 252 #define ATTR_CFG_FLD_min_latency_HI 11 253 254 #define ATTR_CFG_FLD_inv_event_filter_CFG config3 /* PMSNEVFR_EL1 */ 255 #define ATTR_CFG_FLD_inv_event_filter_LO 0 256 #define ATTR_CFG_FLD_inv_event_filter_HI 63 257 258 #define ATTR_CFG_FLD_inv_data_src_filter_CFG config4 /* inverse of PMSDSFR_EL1 */ 259 #define ATTR_CFG_FLD_inv_data_src_filter_LO 0 260 #define ATTR_CFG_FLD_inv_data_src_filter_HI 63 261 262 GEN_PMU_FORMAT_ATTR(ts_enable); 263 GEN_PMU_FORMAT_ATTR(pa_enable); 264 GEN_PMU_FORMAT_ATTR(pct_enable); 265 GEN_PMU_FORMAT_ATTR(jitter); 266 GEN_PMU_FORMAT_ATTR(branch_filter); 267 GEN_PMU_FORMAT_ATTR(branch_filter_mask); 268 GEN_PMU_FORMAT_ATTR(load_filter); 269 GEN_PMU_FORMAT_ATTR(load_filter_mask); 270 GEN_PMU_FORMAT_ATTR(store_filter); 271 GEN_PMU_FORMAT_ATTR(store_filter_mask); 272 GEN_PMU_FORMAT_ATTR(simd_filter); 273 GEN_PMU_FORMAT_ATTR(simd_filter_mask); 274 GEN_PMU_FORMAT_ATTR(float_filter); 275 GEN_PMU_FORMAT_ATTR(float_filter_mask); 276 GEN_PMU_FORMAT_ATTR(event_filter); 277 GEN_PMU_FORMAT_ATTR(inv_event_filter); 278 GEN_PMU_FORMAT_ATTR(inv_data_src_filter); 279 GEN_PMU_FORMAT_ATTR(min_latency); 280 GEN_PMU_FORMAT_ATTR(discard); 281 282 static struct attribute *arm_spe_pmu_formats_attr[] = { 283 &format_attr_ts_enable.attr, 284 &format_attr_pa_enable.attr, 285 &format_attr_pct_enable.attr, 286 &format_attr_jitter.attr, 287 &format_attr_branch_filter.attr, 288 &format_attr_branch_filter_mask.attr, 289 &format_attr_load_filter.attr, 290 &format_attr_load_filter_mask.attr, 291 &format_attr_store_filter.attr, 292 &format_attr_store_filter_mask.attr, 293 &format_attr_simd_filter.attr, 294 &format_attr_simd_filter_mask.attr, 295 &format_attr_float_filter.attr, 296 &format_attr_float_filter_mask.attr, 297 &format_attr_event_filter.attr, 298 &format_attr_inv_event_filter.attr, 299 &format_attr_inv_data_src_filter.attr, 300 &format_attr_min_latency.attr, 301 &format_attr_discard.attr, 302 NULL, 303 }; 304 305 static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, 306 struct attribute *attr, 307 int unused) 308 { 309 struct device *dev = kobj_to_dev(kobj); 310 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 311 312 if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 313 return 0; 314 315 if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 316 return 0; 317 318 if (attr == &format_attr_inv_data_src_filter.attr && 319 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 320 return 0; 321 322 if ((attr == &format_attr_branch_filter_mask.attr || 323 attr == &format_attr_load_filter_mask.attr || 324 attr == &format_attr_store_filter_mask.attr || 325 attr == &format_attr_simd_filter.attr || 326 attr == &format_attr_simd_filter_mask.attr || 327 attr == &format_attr_float_filter.attr || 328 attr == &format_attr_float_filter_mask.attr) && 329 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 330 return 0; 331 332 return attr->mode; 333 } 334 335 static const struct attribute_group arm_spe_pmu_format_group = { 336 .name = "format", 337 .is_visible = arm_spe_pmu_format_attr_is_visible, 338 .attrs = arm_spe_pmu_formats_attr, 339 }; 340 341 static ssize_t cpumask_show(struct device *dev, 342 struct device_attribute *attr, char *buf) 343 { 344 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 345 346 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&spe_pmu->supported_cpus)); 347 } 348 static DEVICE_ATTR_RO(cpumask); 349 350 static struct attribute *arm_spe_pmu_attrs[] = { 351 &dev_attr_cpumask.attr, 352 NULL, 353 }; 354 355 static const struct attribute_group arm_spe_pmu_group = { 356 .attrs = arm_spe_pmu_attrs, 357 }; 358 359 static const struct attribute_group *arm_spe_pmu_attr_groups[] = { 360 &arm_spe_pmu_group, 361 &arm_spe_pmu_cap_group, 362 &arm_spe_pmu_format_group, 363 NULL, 364 }; 365 366 /* Convert between user ABI and register values */ 367 static u64 arm_spe_event_to_pmscr(struct perf_event *event) 368 { 369 struct perf_event_attr *attr = &event->attr; 370 u64 reg = 0; 371 372 reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable)); 373 reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable)); 374 reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable)); 375 376 if (!attr->exclude_user) 377 reg |= PMSCR_EL1_E0SPE; 378 379 if (!attr->exclude_kernel) 380 reg |= PMSCR_EL1_E1SPE; 381 382 if (get_spe_event_has_cx(event)) 383 reg |= PMSCR_EL1_CX; 384 385 return reg; 386 } 387 388 static void arm_spe_event_sanitise_period(struct perf_event *event) 389 { 390 u64 period = event->hw.sample_period; 391 u64 max_period = PMSIRR_EL1_INTERVAL_MASK; 392 393 /* 394 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a 395 * recommendation for the minimum interval, not a hardware limitation. 396 * 397 * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, 398 * Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) 399 * states: "Software must set this to a nonzero value". Use 1 as the 400 * minimum value. 401 */ 402 u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); 403 404 period = clamp_t(u64, period, min_period, max_period) & max_period; 405 event->hw.sample_period = period; 406 } 407 408 static u64 arm_spe_event_to_pmsirr(struct perf_event *event) 409 { 410 struct perf_event_attr *attr = &event->attr; 411 u64 reg = 0; 412 413 arm_spe_event_sanitise_period(event); 414 415 reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter)); 416 reg |= event->hw.sample_period; 417 418 return reg; 419 } 420 421 static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) 422 { 423 struct perf_event_attr *attr = &event->attr; 424 u64 reg = 0; 425 426 reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter)); 427 reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask)); 428 reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter)); 429 reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask)); 430 reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter)); 431 reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask)); 432 reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter)); 433 reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask)); 434 reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter)); 435 reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask)); 436 437 if (reg) 438 reg |= PMSFCR_EL1_FT; 439 440 if (ATTR_CFG_GET_FLD(attr, event_filter)) 441 reg |= PMSFCR_EL1_FE; 442 443 if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) 444 reg |= PMSFCR_EL1_FnE; 445 446 if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter)) 447 reg |= PMSFCR_EL1_FDS; 448 449 if (ATTR_CFG_GET_FLD(attr, min_latency)) 450 reg |= PMSFCR_EL1_FL; 451 452 return reg; 453 } 454 455 static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) 456 { 457 struct perf_event_attr *attr = &event->attr; 458 return ATTR_CFG_GET_FLD(attr, event_filter); 459 } 460 461 static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event) 462 { 463 struct perf_event_attr *attr = &event->attr; 464 return ATTR_CFG_GET_FLD(attr, inv_event_filter); 465 } 466 467 static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) 468 { 469 struct perf_event_attr *attr = &event->attr; 470 return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); 471 } 472 473 static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event) 474 { 475 struct perf_event_attr *attr = &event->attr; 476 477 /* 478 * Data src filter is inverted so that the default value of 0 is 479 * equivalent to no filtering. 480 */ 481 return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter); 482 } 483 484 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) 485 { 486 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 487 u64 head = PERF_IDX2OFF(handle->head, buf); 488 489 memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); 490 if (!buf->snapshot) 491 perf_aux_output_skip(handle, len); 492 } 493 494 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) 495 { 496 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 497 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 498 u64 head = PERF_IDX2OFF(handle->head, buf); 499 u64 limit = buf->nr_pages * PAGE_SIZE; 500 501 /* 502 * The trace format isn't parseable in reverse, so clamp 503 * the limit to half of the buffer size in snapshot mode 504 * so that the worst case is half a buffer of records, as 505 * opposed to a single record. 506 */ 507 if (head < limit >> 1) 508 limit >>= 1; 509 510 /* 511 * If we're within max_record_sz of the limit, we must 512 * pad, move the head index and recompute the limit. 513 */ 514 if (limit - head < spe_pmu->max_record_sz) { 515 arm_spe_pmu_pad_buf(handle, limit - head); 516 handle->head = PERF_IDX2OFF(limit, buf); 517 limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; 518 } 519 520 return limit; 521 } 522 523 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) 524 { 525 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 526 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 527 const u64 bufsize = buf->nr_pages * PAGE_SIZE; 528 u64 limit = bufsize; 529 u64 head, tail, wakeup; 530 531 /* 532 * The head can be misaligned for two reasons: 533 * 534 * 1. The hardware left PMBPTR pointing to the first byte after 535 * a record when generating a buffer management event. 536 * 537 * 2. We used perf_aux_output_skip to consume handle->size bytes 538 * and CIRC_SPACE was used to compute the size, which always 539 * leaves one entry free. 540 * 541 * Deal with this by padding to the next alignment boundary and 542 * moving the head index. If we run out of buffer space, we'll 543 * reduce handle->size to zero and end up reporting truncation. 544 */ 545 head = PERF_IDX2OFF(handle->head, buf); 546 if (!IS_ALIGNED(head, spe_pmu->align)) { 547 unsigned long delta = roundup(head, spe_pmu->align) - head; 548 549 delta = min(delta, handle->size); 550 arm_spe_pmu_pad_buf(handle, delta); 551 head = PERF_IDX2OFF(handle->head, buf); 552 } 553 554 /* If we've run out of free space, then nothing more to do */ 555 if (!handle->size) 556 goto no_space; 557 558 /* Compute the tail and wakeup indices now that we've aligned head */ 559 tail = PERF_IDX2OFF(handle->head + handle->size, buf); 560 wakeup = PERF_IDX2OFF(handle->wakeup, buf); 561 562 /* 563 * Avoid clobbering unconsumed data. We know we have space, so 564 * if we see head == tail we know that the buffer is empty. If 565 * head > tail, then there's nothing to clobber prior to 566 * wrapping. 567 */ 568 if (head < tail) 569 limit = round_down(tail, PAGE_SIZE); 570 571 /* 572 * Wakeup may be arbitrarily far into the future. If it's not in 573 * the current generation, either we'll wrap before hitting it, 574 * or it's in the past and has been handled already. 575 * 576 * If there's a wakeup before we wrap, arrange to be woken up by 577 * the page boundary following it. Keep the tail boundary if 578 * that's lower. 579 */ 580 if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) 581 limit = min(limit, round_up(wakeup, PAGE_SIZE)); 582 583 if (limit > head) 584 return limit; 585 586 arm_spe_pmu_pad_buf(handle, handle->size); 587 no_space: 588 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 589 perf_aux_output_end(handle, 0); 590 return 0; 591 } 592 593 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) 594 { 595 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 596 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 597 u64 limit = __arm_spe_pmu_next_off(handle); 598 u64 head = PERF_IDX2OFF(handle->head, buf); 599 600 /* 601 * If the head has come too close to the end of the buffer, 602 * then pad to the end and recompute the limit. 603 */ 604 if (limit && (limit - head < spe_pmu->max_record_sz)) { 605 arm_spe_pmu_pad_buf(handle, limit - head); 606 limit = __arm_spe_pmu_next_off(handle); 607 } 608 609 return limit; 610 } 611 612 static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, 613 struct perf_event *event) 614 { 615 u64 base, limit; 616 struct arm_spe_pmu_buf *buf; 617 618 if (ATTR_CFG_GET_FLD(&event->attr, discard)) { 619 limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD); 620 limit |= PMBLIMITR_EL1_E; 621 goto out_write_limit; 622 } 623 624 /* Start a new aux session */ 625 buf = perf_aux_output_begin(handle, event); 626 if (!buf) { 627 /* 628 * We still need to clear the limit pointer, since the 629 * profiler might only be disabled by virtue of a fault. 630 */ 631 limit = 0; 632 goto out_write_limit; 633 } 634 635 limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) 636 : arm_spe_pmu_next_off(handle); 637 if (limit) 638 limit |= PMBLIMITR_EL1_E; 639 640 limit += (u64)buf->base; 641 base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); 642 write_sysreg_s(base, SYS_PMBPTR_EL1); 643 644 out_write_limit: 645 write_sysreg_s(limit, SYS_PMBLIMITR_EL1); 646 return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; 647 } 648 649 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) 650 { 651 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 652 u64 offset, size; 653 654 offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; 655 size = offset - PERF_IDX2OFF(handle->head, buf); 656 657 if (buf->snapshot) 658 handle->head = offset; 659 660 perf_aux_output_end(handle, size); 661 } 662 663 static void arm_spe_pmu_disable_and_drain_local(void) 664 { 665 /* Disable profiling at EL0 and EL1 */ 666 write_sysreg_s(0, SYS_PMSCR_EL1); 667 isb(); 668 669 /* Drain any buffered data */ 670 psb_csync(); 671 dsb(nsh); 672 673 /* Disable the profiling buffer */ 674 write_sysreg_s(0, SYS_PMBLIMITR_EL1); 675 isb(); 676 } 677 678 /* IRQ handling */ 679 static enum arm_spe_pmu_buf_fault_action 680 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) 681 { 682 const char *err_str; 683 u64 pmbsr; 684 enum arm_spe_pmu_buf_fault_action ret; 685 686 /* 687 * Ensure new profiling data is visible to the CPU and any external 688 * aborts have been resolved. 689 */ 690 psb_csync(); 691 dsb(nsh); 692 693 /* Ensure hardware updates to PMBPTR_EL1 are visible */ 694 isb(); 695 696 /* Service required? */ 697 pmbsr = read_sysreg_s(SYS_PMBSR_EL1); 698 if (!FIELD_GET(PMBSR_EL1_S, pmbsr)) 699 return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; 700 701 /* 702 * If we've lost data, disable profiling and also set the PARTIAL 703 * flag to indicate that the last record is corrupted. 704 */ 705 if (FIELD_GET(PMBSR_EL1_DL, pmbsr)) 706 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | 707 PERF_AUX_FLAG_PARTIAL); 708 709 /* Report collisions to userspace so that it can up the period */ 710 if (FIELD_GET(PMBSR_EL1_COLL, pmbsr)) 711 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); 712 713 /* We only expect buffer management events */ 714 switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) { 715 case PMBSR_EL1_EC_BUF: 716 /* Handled below */ 717 break; 718 case PMBSR_EL1_EC_FAULT_S1: 719 case PMBSR_EL1_EC_FAULT_S2: 720 err_str = "Unexpected buffer fault"; 721 goto out_err; 722 default: 723 err_str = "Unknown error code"; 724 goto out_err; 725 } 726 727 /* Buffer management event */ 728 switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) { 729 case PMBSR_EL1_BUF_BSC_FULL: 730 ret = SPE_PMU_BUF_FAULT_ACT_OK; 731 goto out_stop; 732 default: 733 err_str = "Unknown buffer status code"; 734 } 735 736 out_err: 737 pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", 738 err_str, smp_processor_id(), pmbsr, 739 read_sysreg_s(SYS_PMBPTR_EL1), 740 read_sysreg_s(SYS_PMBLIMITR_EL1)); 741 ret = SPE_PMU_BUF_FAULT_ACT_FATAL; 742 743 out_stop: 744 arm_spe_perf_aux_output_end(handle); 745 return ret; 746 } 747 748 static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) 749 { 750 struct perf_output_handle *handle = dev; 751 struct perf_event *event = handle->event; 752 enum arm_spe_pmu_buf_fault_action act; 753 754 if (!perf_get_aux(handle)) 755 return IRQ_NONE; 756 757 act = arm_spe_pmu_buf_get_fault_act(handle); 758 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 759 return IRQ_NONE; 760 761 /* 762 * Ensure perf callbacks have completed, which may disable the 763 * profiling buffer in response to a TRUNCATION flag. 764 */ 765 irq_work_run(); 766 767 switch (act) { 768 case SPE_PMU_BUF_FAULT_ACT_FATAL: 769 /* 770 * If a fatal exception occurred then leaving the profiling 771 * buffer enabled is a recipe waiting to happen. Since 772 * fatal faults don't always imply truncation, make sure 773 * that the profiling buffer is disabled explicitly before 774 * clearing the syndrome register. 775 */ 776 arm_spe_pmu_disable_and_drain_local(); 777 break; 778 case SPE_PMU_BUF_FAULT_ACT_OK: 779 /* 780 * We handled the fault (the buffer was full), so resume 781 * profiling as long as we didn't detect truncation. 782 * PMBPTR might be misaligned, but we'll burn that bridge 783 * when we get to it. 784 */ 785 if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { 786 if (arm_spe_perf_aux_output_begin(handle, event)) { 787 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 788 break; 789 } 790 isb(); 791 } 792 break; 793 case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: 794 /* We've seen you before, but GCC has the memory of a sieve. */ 795 break; 796 } 797 798 /* The buffer pointers are now sane, so resume profiling. */ 799 write_sysreg_s(0, SYS_PMBSR_EL1); 800 return IRQ_HANDLED; 801 } 802 803 /* Perf callbacks */ 804 static int arm_spe_pmu_event_init(struct perf_event *event) 805 { 806 u64 reg; 807 struct perf_event_attr *attr = &event->attr; 808 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 809 810 /* This is, of course, deeply driver-specific */ 811 if (attr->type != event->pmu->type) 812 return -ENOENT; 813 814 if (event->cpu >= 0 && 815 !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) 816 return -ENOENT; 817 818 if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0) 819 return -EOPNOTSUPP; 820 821 if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) 822 return -EOPNOTSUPP; 823 824 if (arm_spe_event_to_pmsdsfr(event) != U64_MAX && 825 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 826 return -EOPNOTSUPP; 827 828 if (attr->exclude_idle) 829 return -EOPNOTSUPP; 830 831 /* 832 * Feedback-directed frequency throttling doesn't work when we 833 * have a buffer of samples. We'd need to manually count the 834 * samples in the buffer when it fills up and adjust the event 835 * count to reflect that. Instead, just force the user to specify 836 * a sample period. 837 */ 838 if (attr->freq) 839 return -EINVAL; 840 841 reg = arm_spe_event_to_pmsfcr(event); 842 if ((FIELD_GET(PMSFCR_EL1_FE, reg)) && 843 !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) 844 return -EOPNOTSUPP; 845 846 if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) && 847 !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 848 return -EOPNOTSUPP; 849 850 if ((FIELD_GET(PMSFCR_EL1_FT, reg)) && 851 !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) 852 return -EOPNOTSUPP; 853 854 if ((FIELD_GET(PMSFCR_EL1_FL, reg)) && 855 !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) 856 return -EOPNOTSUPP; 857 858 if ((FIELD_GET(PMSFCR_EL1_LDm, reg) || 859 FIELD_GET(PMSFCR_EL1_STm, reg) || 860 FIELD_GET(PMSFCR_EL1_Bm, reg) || 861 FIELD_GET(PMSFCR_EL1_SIMD, reg) || 862 FIELD_GET(PMSFCR_EL1_SIMDm, reg) || 863 FIELD_GET(PMSFCR_EL1_FP, reg) || 864 FIELD_GET(PMSFCR_EL1_FPm, reg)) && 865 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 866 return -EOPNOTSUPP; 867 868 if (ATTR_CFG_GET_FLD(&event->attr, discard) && 869 !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 870 return -EOPNOTSUPP; 871 872 set_spe_event_has_cx(event); 873 reg = arm_spe_event_to_pmscr(event); 874 if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT)) 875 return perf_allow_kernel(); 876 877 return 0; 878 } 879 880 static void arm_spe_pmu_start(struct perf_event *event, int flags) 881 { 882 u64 reg; 883 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 884 struct hw_perf_event *hwc = &event->hw; 885 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 886 887 hwc->state = 0; 888 if (arm_spe_perf_aux_output_begin(handle, event)) { 889 arm_spe_pmu_stop(event, 0); 890 return; 891 } 892 893 perf_event_itrace_started(event); 894 895 reg = arm_spe_event_to_pmsfcr(event); 896 write_sysreg_s(reg, SYS_PMSFCR_EL1); 897 898 reg = arm_spe_event_to_pmsevfr(event); 899 write_sysreg_s(reg, SYS_PMSEVFR_EL1); 900 901 if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) { 902 reg = arm_spe_event_to_pmsnevfr(event); 903 write_sysreg_s(reg, SYS_PMSNEVFR_EL1); 904 } 905 906 if (spe_pmu->features & SPE_PMU_FEAT_FDS) { 907 reg = arm_spe_event_to_pmsdsfr(event); 908 write_sysreg_s(reg, SYS_PMSDSFR_EL1); 909 } 910 911 reg = arm_spe_event_to_pmslatfr(event); 912 write_sysreg_s(reg, SYS_PMSLATFR_EL1); 913 914 if (flags & PERF_EF_RELOAD) { 915 reg = arm_spe_event_to_pmsirr(event); 916 write_sysreg_s(reg, SYS_PMSIRR_EL1); 917 isb(); 918 reg = local64_read(&hwc->period_left); 919 write_sysreg_s(reg, SYS_PMSICR_EL1); 920 } 921 922 reg = arm_spe_event_to_pmscr(event); 923 isb(); 924 write_sysreg_s(reg, SYS_PMSCR_EL1); 925 } 926 927 static void arm_spe_pmu_stop(struct perf_event *event, int flags) 928 { 929 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 930 struct hw_perf_event *hwc = &event->hw; 931 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 932 933 /* If we're already stopped, then nothing to do */ 934 if (hwc->state & PERF_HES_STOPPED) 935 return; 936 937 /* Stop all trace generation */ 938 arm_spe_pmu_disable_and_drain_local(); 939 940 if (flags & PERF_EF_UPDATE) { 941 /* 942 * If there's a fault pending then ensure we contain it 943 * to this buffer, since we might be on the context-switch 944 * path. 945 */ 946 if (perf_get_aux(handle)) { 947 enum arm_spe_pmu_buf_fault_action act; 948 949 act = arm_spe_pmu_buf_get_fault_act(handle); 950 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 951 arm_spe_perf_aux_output_end(handle); 952 else 953 write_sysreg_s(0, SYS_PMBSR_EL1); 954 } 955 956 /* 957 * This may also contain ECOUNT, but nobody else should 958 * be looking at period_left, since we forbid frequency 959 * based sampling. 960 */ 961 local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); 962 hwc->state |= PERF_HES_UPTODATE; 963 } 964 965 hwc->state |= PERF_HES_STOPPED; 966 } 967 968 static int arm_spe_pmu_add(struct perf_event *event, int flags) 969 { 970 int ret = 0; 971 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 972 struct hw_perf_event *hwc = &event->hw; 973 int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; 974 975 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 976 return -ENOENT; 977 978 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 979 980 if (flags & PERF_EF_START) { 981 arm_spe_pmu_start(event, PERF_EF_RELOAD); 982 if (hwc->state & PERF_HES_STOPPED) 983 ret = -EINVAL; 984 } 985 986 return ret; 987 } 988 989 static void arm_spe_pmu_del(struct perf_event *event, int flags) 990 { 991 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 992 } 993 994 static void arm_spe_pmu_read(struct perf_event *event) 995 { 996 } 997 998 static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, 999 int nr_pages, bool snapshot) 1000 { 1001 int i, cpu = event->cpu; 1002 struct page **pglist; 1003 struct arm_spe_pmu_buf *buf; 1004 1005 /* We need at least two pages for this to work. */ 1006 if (nr_pages < 2) 1007 return NULL; 1008 1009 /* 1010 * We require an even number of pages for snapshot mode, so that 1011 * we can effectively treat the buffer as consisting of two equal 1012 * parts and give userspace a fighting chance of getting some 1013 * useful data out of it. 1014 */ 1015 if (snapshot && (nr_pages & 1)) 1016 return NULL; 1017 1018 if (cpu == -1) 1019 cpu = raw_smp_processor_id(); 1020 1021 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); 1022 if (!buf) 1023 return NULL; 1024 1025 pglist = kzalloc_objs(*pglist, nr_pages); 1026 if (!pglist) 1027 goto out_free_buf; 1028 1029 for (i = 0; i < nr_pages; ++i) 1030 pglist[i] = virt_to_page(pages[i]); 1031 1032 buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); 1033 if (!buf->base) 1034 goto out_free_pglist; 1035 1036 buf->nr_pages = nr_pages; 1037 buf->snapshot = snapshot; 1038 1039 kfree(pglist); 1040 return buf; 1041 1042 out_free_pglist: 1043 kfree(pglist); 1044 out_free_buf: 1045 kfree(buf); 1046 return NULL; 1047 } 1048 1049 static void arm_spe_pmu_free_aux(void *aux) 1050 { 1051 struct arm_spe_pmu_buf *buf = aux; 1052 1053 vunmap(buf->base); 1054 kfree(buf); 1055 } 1056 1057 /* Initialisation and teardown functions */ 1058 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) 1059 { 1060 static atomic_t pmu_idx = ATOMIC_INIT(-1); 1061 1062 int idx; 1063 char *name; 1064 struct device *dev = &spe_pmu->pdev->dev; 1065 1066 spe_pmu->pmu = (struct pmu) { 1067 .module = THIS_MODULE, 1068 .parent = &spe_pmu->pdev->dev, 1069 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, 1070 .attr_groups = arm_spe_pmu_attr_groups, 1071 /* 1072 * We hitch a ride on the software context here, so that 1073 * we can support per-task profiling (which is not possible 1074 * with the invalid context as it doesn't get sched callbacks). 1075 * This requires that userspace either uses a dummy event for 1076 * perf_event_open, since the aux buffer is not setup until 1077 * a subsequent mmap, or creates the profiling event in a 1078 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it 1079 * once the buffer has been created. 1080 */ 1081 .task_ctx_nr = perf_sw_context, 1082 .event_init = arm_spe_pmu_event_init, 1083 .add = arm_spe_pmu_add, 1084 .del = arm_spe_pmu_del, 1085 .start = arm_spe_pmu_start, 1086 .stop = arm_spe_pmu_stop, 1087 .read = arm_spe_pmu_read, 1088 .setup_aux = arm_spe_pmu_setup_aux, 1089 .free_aux = arm_spe_pmu_free_aux, 1090 }; 1091 1092 idx = atomic_inc_return(&pmu_idx); 1093 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); 1094 if (!name) { 1095 dev_err(dev, "failed to allocate name for pmu %d\n", idx); 1096 return -ENOMEM; 1097 } 1098 1099 return perf_pmu_register(&spe_pmu->pmu, name, -1); 1100 } 1101 1102 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) 1103 { 1104 perf_pmu_unregister(&spe_pmu->pmu); 1105 } 1106 1107 static void __arm_spe_pmu_dev_probe(void *info) 1108 { 1109 int fld; 1110 u64 reg; 1111 struct arm_spe_pmu *spe_pmu = info; 1112 struct device *dev = &spe_pmu->pdev->dev; 1113 1114 fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), 1115 ID_AA64DFR0_EL1_PMSVer_SHIFT); 1116 if (!fld) { 1117 dev_err(dev, 1118 "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", 1119 fld, smp_processor_id()); 1120 return; 1121 } 1122 spe_pmu->pmsver = (u16)fld; 1123 1124 /* Read PMBIDR first to determine whether or not we have access */ 1125 reg = read_sysreg_s(SYS_PMBIDR_EL1); 1126 if (FIELD_GET(PMBIDR_EL1_P, reg)) { 1127 dev_err(dev, 1128 "profiling buffer owned by higher exception level\n"); 1129 return; 1130 } 1131 1132 /* Minimum alignment. If it's out-of-range, then fail the probe */ 1133 fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg); 1134 spe_pmu->align = 1 << fld; 1135 if (spe_pmu->align > SZ_2K) { 1136 dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", 1137 fld, smp_processor_id()); 1138 return; 1139 } 1140 1141 /* It's now safe to read PMSIDR and figure out what we've got */ 1142 reg = read_sysreg_s(SYS_PMSIDR_EL1); 1143 if (FIELD_GET(PMSIDR_EL1_FE, reg)) 1144 spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; 1145 1146 if (FIELD_GET(PMSIDR_EL1_FnE, reg)) 1147 spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT; 1148 1149 if (FIELD_GET(PMSIDR_EL1_FT, reg)) 1150 spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; 1151 1152 if (FIELD_GET(PMSIDR_EL1_FL, reg)) 1153 spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; 1154 1155 if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg)) 1156 spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; 1157 1158 if (FIELD_GET(PMSIDR_EL1_LDS, reg)) 1159 spe_pmu->features |= SPE_PMU_FEAT_LDS; 1160 1161 if (FIELD_GET(PMSIDR_EL1_ERND, reg)) 1162 spe_pmu->features |= SPE_PMU_FEAT_ERND; 1163 1164 if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2) 1165 spe_pmu->features |= SPE_PMU_FEAT_DISCARD; 1166 1167 if (FIELD_GET(PMSIDR_EL1_EFT, reg)) 1168 spe_pmu->features |= SPE_PMU_FEAT_EFT; 1169 1170 if (FIELD_GET(PMSIDR_EL1_FDS, reg)) 1171 spe_pmu->features |= SPE_PMU_FEAT_FDS; 1172 1173 /* This field has a spaced out encoding, so just use a look-up */ 1174 fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); 1175 switch (fld) { 1176 case PMSIDR_EL1_INTERVAL_256: 1177 spe_pmu->min_period = 256; 1178 break; 1179 case PMSIDR_EL1_INTERVAL_512: 1180 spe_pmu->min_period = 512; 1181 break; 1182 case PMSIDR_EL1_INTERVAL_768: 1183 spe_pmu->min_period = 768; 1184 break; 1185 case PMSIDR_EL1_INTERVAL_1024: 1186 spe_pmu->min_period = 1024; 1187 break; 1188 case PMSIDR_EL1_INTERVAL_1536: 1189 spe_pmu->min_period = 1536; 1190 break; 1191 case PMSIDR_EL1_INTERVAL_2048: 1192 spe_pmu->min_period = 2048; 1193 break; 1194 case PMSIDR_EL1_INTERVAL_3072: 1195 spe_pmu->min_period = 3072; 1196 break; 1197 default: 1198 dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", 1199 fld); 1200 fallthrough; 1201 case PMSIDR_EL1_INTERVAL_4096: 1202 spe_pmu->min_period = 4096; 1203 } 1204 1205 /* Maximum record size. If it's out-of-range, then fail the probe */ 1206 fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg); 1207 spe_pmu->max_record_sz = 1 << fld; 1208 if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { 1209 dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", 1210 fld, smp_processor_id()); 1211 return; 1212 } 1213 1214 fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg); 1215 switch (fld) { 1216 default: 1217 dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", 1218 fld); 1219 fallthrough; 1220 case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT: 1221 spe_pmu->counter_sz = 12; 1222 break; 1223 case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT: 1224 spe_pmu->counter_sz = 16; 1225 } 1226 1227 /* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */ 1228 write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1); 1229 spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1); 1230 1231 dev_info(dev, 1232 "probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", 1233 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus), 1234 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); 1235 1236 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; 1237 } 1238 1239 static void __arm_spe_pmu_reset_local(void) 1240 { 1241 /* 1242 * This is probably overkill, as we have no idea where we're 1243 * draining any buffered data to... 1244 */ 1245 arm_spe_pmu_disable_and_drain_local(); 1246 1247 /* Reset the buffer base pointer */ 1248 write_sysreg_s(0, SYS_PMBPTR_EL1); 1249 isb(); 1250 1251 /* Clear any pending management interrupts */ 1252 write_sysreg_s(0, SYS_PMBSR_EL1); 1253 isb(); 1254 } 1255 1256 static void __arm_spe_pmu_setup_one(void *info) 1257 { 1258 struct arm_spe_pmu *spe_pmu = info; 1259 1260 __arm_spe_pmu_reset_local(); 1261 enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); 1262 } 1263 1264 static void __arm_spe_pmu_stop_one(void *info) 1265 { 1266 struct arm_spe_pmu *spe_pmu = info; 1267 1268 disable_percpu_irq(spe_pmu->irq); 1269 __arm_spe_pmu_reset_local(); 1270 } 1271 1272 static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) 1273 { 1274 struct arm_spe_pmu *spe_pmu; 1275 1276 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1277 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1278 return 0; 1279 1280 __arm_spe_pmu_setup_one(spe_pmu); 1281 return 0; 1282 } 1283 1284 static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1285 { 1286 struct arm_spe_pmu *spe_pmu; 1287 1288 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1289 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1290 return 0; 1291 1292 __arm_spe_pmu_stop_one(spe_pmu); 1293 return 0; 1294 } 1295 1296 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) 1297 { 1298 int ret; 1299 cpumask_t *mask = &spe_pmu->supported_cpus; 1300 1301 /* Make sure we probe the hardware on a relevant CPU */ 1302 ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); 1303 if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) 1304 return -ENXIO; 1305 1306 /* Request our PPIs (note that the IRQ is still disabled) */ 1307 ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler, 1308 DRVNAME, mask, spe_pmu->handle); 1309 if (ret) 1310 return ret; 1311 1312 /* 1313 * Register our hotplug notifier now so we don't miss any events. 1314 * This will enable the IRQ for any supported CPUs that are already 1315 * up. 1316 */ 1317 ret = cpuhp_state_add_instance(arm_spe_pmu_online, 1318 &spe_pmu->hotplug_node); 1319 if (ret) 1320 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1321 1322 return ret; 1323 } 1324 1325 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) 1326 { 1327 cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); 1328 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1329 } 1330 1331 /* Driver and device probing */ 1332 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) 1333 { 1334 struct platform_device *pdev = spe_pmu->pdev; 1335 const struct cpumask *affinity; 1336 int irq; 1337 1338 irq = platform_get_irq_affinity(pdev, 0, &affinity); 1339 if (irq < 0) 1340 return -ENXIO; 1341 1342 if (!irq_is_percpu(irq)) { 1343 dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); 1344 return -EINVAL; 1345 } 1346 1347 cpumask_copy(&spe_pmu->supported_cpus, affinity); 1348 1349 spe_pmu->irq = irq; 1350 return 0; 1351 } 1352 1353 static const struct of_device_id arm_spe_pmu_of_match[] = { 1354 { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, 1355 { /* Sentinel */ }, 1356 }; 1357 MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); 1358 1359 static const struct platform_device_id arm_spe_match[] = { 1360 { ARMV8_SPE_PDEV_NAME, 0}, 1361 { } 1362 }; 1363 MODULE_DEVICE_TABLE(platform, arm_spe_match); 1364 1365 static int arm_spe_pmu_device_probe(struct platform_device *pdev) 1366 { 1367 int ret; 1368 struct arm_spe_pmu *spe_pmu; 1369 struct device *dev = &pdev->dev; 1370 1371 /* 1372 * If kernelspace is unmapped when running at EL0, then the SPE 1373 * buffer will fault and prematurely terminate the AUX session. 1374 */ 1375 if (arm64_kernel_unmapped_at_el0()) { 1376 dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); 1377 return -EPERM; 1378 } 1379 1380 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); 1381 if (!spe_pmu) 1382 return -ENOMEM; 1383 1384 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); 1385 if (!spe_pmu->handle) 1386 return -ENOMEM; 1387 1388 spe_pmu->pdev = pdev; 1389 platform_set_drvdata(pdev, spe_pmu); 1390 1391 ret = arm_spe_pmu_irq_probe(spe_pmu); 1392 if (ret) 1393 goto out_free_handle; 1394 1395 ret = arm_spe_pmu_dev_init(spe_pmu); 1396 if (ret) 1397 goto out_free_handle; 1398 1399 ret = arm_spe_pmu_perf_init(spe_pmu); 1400 if (ret) 1401 goto out_teardown_dev; 1402 1403 return 0; 1404 1405 out_teardown_dev: 1406 arm_spe_pmu_dev_teardown(spe_pmu); 1407 out_free_handle: 1408 free_percpu(spe_pmu->handle); 1409 return ret; 1410 } 1411 1412 static void arm_spe_pmu_device_remove(struct platform_device *pdev) 1413 { 1414 struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); 1415 1416 arm_spe_pmu_perf_destroy(spe_pmu); 1417 arm_spe_pmu_dev_teardown(spe_pmu); 1418 free_percpu(spe_pmu->handle); 1419 } 1420 1421 static struct platform_driver arm_spe_pmu_driver = { 1422 .id_table = arm_spe_match, 1423 .driver = { 1424 .name = DRVNAME, 1425 .of_match_table = of_match_ptr(arm_spe_pmu_of_match), 1426 .suppress_bind_attrs = true, 1427 }, 1428 .probe = arm_spe_pmu_device_probe, 1429 .remove = arm_spe_pmu_device_remove, 1430 }; 1431 1432 static int __init arm_spe_pmu_init(void) 1433 { 1434 int ret; 1435 1436 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, 1437 arm_spe_pmu_cpu_startup, 1438 arm_spe_pmu_cpu_teardown); 1439 if (ret < 0) 1440 return ret; 1441 arm_spe_pmu_online = ret; 1442 1443 ret = platform_driver_register(&arm_spe_pmu_driver); 1444 if (ret) 1445 cpuhp_remove_multi_state(arm_spe_pmu_online); 1446 1447 return ret; 1448 } 1449 1450 static void __exit arm_spe_pmu_exit(void) 1451 { 1452 platform_driver_unregister(&arm_spe_pmu_driver); 1453 cpuhp_remove_multi_state(arm_spe_pmu_online); 1454 } 1455 1456 module_init(arm_spe_pmu_init); 1457 module_exit(arm_spe_pmu_exit); 1458 1459 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); 1460 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1461 MODULE_LICENSE("GPL v2"); 1462