1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Perf support for the Statistical Profiling Extension, introduced as 4 * part of ARMv8.2. 5 * 6 * Copyright (C) 2016 ARM Limited 7 * 8 * Author: Will Deacon <will.deacon@arm.com> 9 */ 10 11 #define PMUNAME "arm_spe" 12 #define DRVNAME PMUNAME "_pmu" 13 #define pr_fmt(fmt) DRVNAME ": " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/bitops.h> 17 #include <linux/bug.h> 18 #include <linux/capability.h> 19 #include <linux/cpuhotplug.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/errno.h> 23 #include <linux/interrupt.h> 24 #include <linux/irq.h> 25 #include <linux/kernel.h> 26 #include <linux/list.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/perf_event.h> 30 #include <linux/perf/arm_pmu.h> 31 #include <linux/platform_device.h> 32 #include <linux/printk.h> 33 #include <linux/slab.h> 34 #include <linux/smp.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/barrier.h> 38 #include <asm/cpufeature.h> 39 #include <asm/mmu.h> 40 #include <asm/sysreg.h> 41 42 /* 43 * Cache if the event is allowed to trace Context information. 44 * This allows us to perform the check, i.e, perf_allow_kernel(), 45 * in the context of the event owner, once, during the event_init(). 46 */ 47 #define SPE_PMU_HW_FLAGS_CX 0x00001 48 49 static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); 50 51 static void set_spe_event_has_cx(struct perf_event *event) 52 { 53 if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel()) 54 event->hw.flags |= SPE_PMU_HW_FLAGS_CX; 55 } 56 57 static bool get_spe_event_has_cx(struct perf_event *event) 58 { 59 return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); 60 } 61 62 #define ARM_SPE_BUF_PAD_BYTE 0 63 64 struct arm_spe_pmu_buf { 65 int nr_pages; 66 bool snapshot; 67 void *base; 68 }; 69 70 struct arm_spe_pmu { 71 struct pmu pmu; 72 struct platform_device *pdev; 73 cpumask_t supported_cpus; 74 struct hlist_node hotplug_node; 75 76 int irq; /* PPI */ 77 u16 pmsver; 78 u16 min_period; 79 u16 counter_sz; 80 81 #define SPE_PMU_FEAT_FILT_EVT (1UL << 0) 82 #define SPE_PMU_FEAT_FILT_TYP (1UL << 1) 83 #define SPE_PMU_FEAT_FILT_LAT (1UL << 2) 84 #define SPE_PMU_FEAT_ARCH_INST (1UL << 3) 85 #define SPE_PMU_FEAT_LDS (1UL << 4) 86 #define SPE_PMU_FEAT_ERND (1UL << 5) 87 #define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) 88 #define SPE_PMU_FEAT_DISCARD (1UL << 7) 89 #define SPE_PMU_FEAT_EFT (1UL << 8) 90 #define SPE_PMU_FEAT_FDS (1UL << 9) 91 #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) 92 u64 features; 93 94 u64 pmsevfr_res0; 95 u16 max_record_sz; 96 u16 align; 97 struct perf_output_handle __percpu *handle; 98 }; 99 100 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) 101 102 /* Convert a free-running index from perf into an SPE buffer offset */ 103 #define PERF_IDX2OFF(idx, buf) \ 104 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) 105 106 /* Keep track of our dynamic hotplug state */ 107 static enum cpuhp_state arm_spe_pmu_online; 108 109 static void arm_spe_pmu_stop(struct perf_event *event, int flags); 110 111 enum arm_spe_pmu_buf_fault_action { 112 SPE_PMU_BUF_FAULT_ACT_SPURIOUS, 113 SPE_PMU_BUF_FAULT_ACT_FATAL, 114 SPE_PMU_BUF_FAULT_ACT_OK, 115 }; 116 117 /* This sysfs gunk was really good fun to write. */ 118 enum arm_spe_pmu_capabilities { 119 SPE_PMU_CAP_ARCH_INST = 0, 120 SPE_PMU_CAP_ERND, 121 SPE_PMU_CAP_FEAT_MAX, 122 SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, 123 SPE_PMU_CAP_MIN_IVAL, 124 SPE_PMU_CAP_EVENT_FILTER, 125 }; 126 127 static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { 128 [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, 129 [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, 130 }; 131 132 static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) 133 { 134 if (cap < SPE_PMU_CAP_FEAT_MAX) 135 return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); 136 137 switch (cap) { 138 case SPE_PMU_CAP_CNT_SZ: 139 return spe_pmu->counter_sz; 140 case SPE_PMU_CAP_MIN_IVAL: 141 return spe_pmu->min_period; 142 case SPE_PMU_CAP_EVENT_FILTER: 143 return ~spe_pmu->pmsevfr_res0; 144 default: 145 WARN(1, "unknown cap %d\n", cap); 146 } 147 148 return 0; 149 } 150 151 static ssize_t arm_spe_pmu_cap_show(struct device *dev, 152 struct device_attribute *attr, 153 char *buf) 154 { 155 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 156 struct dev_ext_attribute *ea = 157 container_of(attr, struct dev_ext_attribute, attr); 158 int cap = (long)ea->var; 159 160 return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 161 } 162 163 static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev, 164 struct device_attribute *attr, 165 char *buf) 166 { 167 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 168 struct dev_ext_attribute *ea = 169 container_of(attr, struct dev_ext_attribute, attr); 170 int cap = (long)ea->var; 171 172 return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 173 } 174 175 #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ 176 &((struct dev_ext_attribute[]) { \ 177 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ 178 })[0].attr.attr 179 180 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ 181 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) 182 #define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var) \ 183 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var) 184 185 static struct attribute *arm_spe_pmu_cap_attr[] = { 186 SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), 187 SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), 188 SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), 189 SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), 190 SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER), 191 NULL, 192 }; 193 194 static const struct attribute_group arm_spe_pmu_cap_group = { 195 .name = "caps", 196 .attrs = arm_spe_pmu_cap_attr, 197 }; 198 199 /* User ABI */ 200 #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ 201 #define ATTR_CFG_FLD_ts_enable_LO 0 202 #define ATTR_CFG_FLD_ts_enable_HI 0 203 #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ 204 #define ATTR_CFG_FLD_pa_enable_LO 1 205 #define ATTR_CFG_FLD_pa_enable_HI 1 206 #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ 207 #define ATTR_CFG_FLD_pct_enable_LO 2 208 #define ATTR_CFG_FLD_pct_enable_HI 2 209 #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ 210 #define ATTR_CFG_FLD_jitter_LO 16 211 #define ATTR_CFG_FLD_jitter_HI 16 212 #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ 213 #define ATTR_CFG_FLD_branch_filter_LO 32 214 #define ATTR_CFG_FLD_branch_filter_HI 32 215 #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ 216 #define ATTR_CFG_FLD_load_filter_LO 33 217 #define ATTR_CFG_FLD_load_filter_HI 33 218 #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ 219 #define ATTR_CFG_FLD_store_filter_LO 34 220 #define ATTR_CFG_FLD_store_filter_HI 34 221 #define ATTR_CFG_FLD_discard_CFG config /* PMBLIMITR_EL1.FM = DISCARD */ 222 #define ATTR_CFG_FLD_discard_LO 35 223 #define ATTR_CFG_FLD_discard_HI 35 224 #define ATTR_CFG_FLD_branch_filter_mask_CFG config /* PMSFCR_EL1.Bm */ 225 #define ATTR_CFG_FLD_branch_filter_mask_LO 36 226 #define ATTR_CFG_FLD_branch_filter_mask_HI 36 227 #define ATTR_CFG_FLD_load_filter_mask_CFG config /* PMSFCR_EL1.LDm */ 228 #define ATTR_CFG_FLD_load_filter_mask_LO 37 229 #define ATTR_CFG_FLD_load_filter_mask_HI 37 230 #define ATTR_CFG_FLD_store_filter_mask_CFG config /* PMSFCR_EL1.STm */ 231 #define ATTR_CFG_FLD_store_filter_mask_LO 38 232 #define ATTR_CFG_FLD_store_filter_mask_HI 38 233 #define ATTR_CFG_FLD_simd_filter_CFG config /* PMSFCR_EL1.SIMD */ 234 #define ATTR_CFG_FLD_simd_filter_LO 39 235 #define ATTR_CFG_FLD_simd_filter_HI 39 236 #define ATTR_CFG_FLD_simd_filter_mask_CFG config /* PMSFCR_EL1.SIMDm */ 237 #define ATTR_CFG_FLD_simd_filter_mask_LO 40 238 #define ATTR_CFG_FLD_simd_filter_mask_HI 40 239 #define ATTR_CFG_FLD_float_filter_CFG config /* PMSFCR_EL1.FP */ 240 #define ATTR_CFG_FLD_float_filter_LO 41 241 #define ATTR_CFG_FLD_float_filter_HI 41 242 #define ATTR_CFG_FLD_float_filter_mask_CFG config /* PMSFCR_EL1.FPm */ 243 #define ATTR_CFG_FLD_float_filter_mask_LO 42 244 #define ATTR_CFG_FLD_float_filter_mask_HI 42 245 246 #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ 247 #define ATTR_CFG_FLD_event_filter_LO 0 248 #define ATTR_CFG_FLD_event_filter_HI 63 249 250 #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ 251 #define ATTR_CFG_FLD_min_latency_LO 0 252 #define ATTR_CFG_FLD_min_latency_HI 11 253 254 #define ATTR_CFG_FLD_inv_event_filter_CFG config3 /* PMSNEVFR_EL1 */ 255 #define ATTR_CFG_FLD_inv_event_filter_LO 0 256 #define ATTR_CFG_FLD_inv_event_filter_HI 63 257 258 #define ATTR_CFG_FLD_inv_data_src_filter_CFG config4 /* inverse of PMSDSFR_EL1 */ 259 #define ATTR_CFG_FLD_inv_data_src_filter_LO 0 260 #define ATTR_CFG_FLD_inv_data_src_filter_HI 63 261 262 GEN_PMU_FORMAT_ATTR(ts_enable); 263 GEN_PMU_FORMAT_ATTR(pa_enable); 264 GEN_PMU_FORMAT_ATTR(pct_enable); 265 GEN_PMU_FORMAT_ATTR(jitter); 266 GEN_PMU_FORMAT_ATTR(branch_filter); 267 GEN_PMU_FORMAT_ATTR(branch_filter_mask); 268 GEN_PMU_FORMAT_ATTR(load_filter); 269 GEN_PMU_FORMAT_ATTR(load_filter_mask); 270 GEN_PMU_FORMAT_ATTR(store_filter); 271 GEN_PMU_FORMAT_ATTR(store_filter_mask); 272 GEN_PMU_FORMAT_ATTR(simd_filter); 273 GEN_PMU_FORMAT_ATTR(simd_filter_mask); 274 GEN_PMU_FORMAT_ATTR(float_filter); 275 GEN_PMU_FORMAT_ATTR(float_filter_mask); 276 GEN_PMU_FORMAT_ATTR(event_filter); 277 GEN_PMU_FORMAT_ATTR(inv_event_filter); 278 GEN_PMU_FORMAT_ATTR(inv_data_src_filter); 279 GEN_PMU_FORMAT_ATTR(min_latency); 280 GEN_PMU_FORMAT_ATTR(discard); 281 282 static struct attribute *arm_spe_pmu_formats_attr[] = { 283 &format_attr_ts_enable.attr, 284 &format_attr_pa_enable.attr, 285 &format_attr_pct_enable.attr, 286 &format_attr_jitter.attr, 287 &format_attr_branch_filter.attr, 288 &format_attr_branch_filter_mask.attr, 289 &format_attr_load_filter.attr, 290 &format_attr_load_filter_mask.attr, 291 &format_attr_store_filter.attr, 292 &format_attr_store_filter_mask.attr, 293 &format_attr_simd_filter.attr, 294 &format_attr_simd_filter_mask.attr, 295 &format_attr_float_filter.attr, 296 &format_attr_float_filter_mask.attr, 297 &format_attr_event_filter.attr, 298 &format_attr_inv_event_filter.attr, 299 &format_attr_inv_data_src_filter.attr, 300 &format_attr_min_latency.attr, 301 &format_attr_discard.attr, 302 NULL, 303 }; 304 305 static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, 306 struct attribute *attr, 307 int unused) 308 { 309 struct device *dev = kobj_to_dev(kobj); 310 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 311 312 if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 313 return 0; 314 315 if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 316 return 0; 317 318 if (attr == &format_attr_inv_data_src_filter.attr && 319 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 320 return 0; 321 322 if ((attr == &format_attr_branch_filter_mask.attr || 323 attr == &format_attr_load_filter_mask.attr || 324 attr == &format_attr_store_filter_mask.attr || 325 attr == &format_attr_simd_filter.attr || 326 attr == &format_attr_simd_filter_mask.attr || 327 attr == &format_attr_float_filter.attr || 328 attr == &format_attr_float_filter_mask.attr) && 329 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 330 return 0; 331 332 return attr->mode; 333 } 334 335 static const struct attribute_group arm_spe_pmu_format_group = { 336 .name = "format", 337 .is_visible = arm_spe_pmu_format_attr_is_visible, 338 .attrs = arm_spe_pmu_formats_attr, 339 }; 340 341 static ssize_t cpumask_show(struct device *dev, 342 struct device_attribute *attr, char *buf) 343 { 344 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 345 346 return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); 347 } 348 static DEVICE_ATTR_RO(cpumask); 349 350 static struct attribute *arm_spe_pmu_attrs[] = { 351 &dev_attr_cpumask.attr, 352 NULL, 353 }; 354 355 static const struct attribute_group arm_spe_pmu_group = { 356 .attrs = arm_spe_pmu_attrs, 357 }; 358 359 static const struct attribute_group *arm_spe_pmu_attr_groups[] = { 360 &arm_spe_pmu_group, 361 &arm_spe_pmu_cap_group, 362 &arm_spe_pmu_format_group, 363 NULL, 364 }; 365 366 /* Convert between user ABI and register values */ 367 static u64 arm_spe_event_to_pmscr(struct perf_event *event) 368 { 369 struct perf_event_attr *attr = &event->attr; 370 u64 reg = 0; 371 372 reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable)); 373 reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable)); 374 reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable)); 375 376 if (!attr->exclude_user) 377 reg |= PMSCR_EL1_E0SPE; 378 379 if (!attr->exclude_kernel) 380 reg |= PMSCR_EL1_E1SPE; 381 382 if (get_spe_event_has_cx(event)) 383 reg |= PMSCR_EL1_CX; 384 385 return reg; 386 } 387 388 static void arm_spe_event_sanitise_period(struct perf_event *event) 389 { 390 u64 period = event->hw.sample_period; 391 u64 max_period = PMSIRR_EL1_INTERVAL_MASK; 392 393 /* 394 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a 395 * recommendation for the minimum interval, not a hardware limitation. 396 * 397 * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, 398 * Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) 399 * states: "Software must set this to a nonzero value". Use 1 as the 400 * minimum value. 401 */ 402 u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); 403 404 period = clamp_t(u64, period, min_period, max_period) & max_period; 405 event->hw.sample_period = period; 406 } 407 408 static u64 arm_spe_event_to_pmsirr(struct perf_event *event) 409 { 410 struct perf_event_attr *attr = &event->attr; 411 u64 reg = 0; 412 413 arm_spe_event_sanitise_period(event); 414 415 reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter)); 416 reg |= event->hw.sample_period; 417 418 return reg; 419 } 420 421 static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) 422 { 423 struct perf_event_attr *attr = &event->attr; 424 u64 reg = 0; 425 426 reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter)); 427 reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask)); 428 reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter)); 429 reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask)); 430 reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter)); 431 reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask)); 432 reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter)); 433 reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask)); 434 reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter)); 435 reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask)); 436 437 if (reg) 438 reg |= PMSFCR_EL1_FT; 439 440 if (ATTR_CFG_GET_FLD(attr, event_filter)) 441 reg |= PMSFCR_EL1_FE; 442 443 if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) 444 reg |= PMSFCR_EL1_FnE; 445 446 if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter)) 447 reg |= PMSFCR_EL1_FDS; 448 449 if (ATTR_CFG_GET_FLD(attr, min_latency)) 450 reg |= PMSFCR_EL1_FL; 451 452 return reg; 453 } 454 455 static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) 456 { 457 struct perf_event_attr *attr = &event->attr; 458 return ATTR_CFG_GET_FLD(attr, event_filter); 459 } 460 461 static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event) 462 { 463 struct perf_event_attr *attr = &event->attr; 464 return ATTR_CFG_GET_FLD(attr, inv_event_filter); 465 } 466 467 static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) 468 { 469 struct perf_event_attr *attr = &event->attr; 470 return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); 471 } 472 473 static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event) 474 { 475 struct perf_event_attr *attr = &event->attr; 476 477 /* 478 * Data src filter is inverted so that the default value of 0 is 479 * equivalent to no filtering. 480 */ 481 return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter); 482 } 483 484 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) 485 { 486 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 487 u64 head = PERF_IDX2OFF(handle->head, buf); 488 489 memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); 490 if (!buf->snapshot) 491 perf_aux_output_skip(handle, len); 492 } 493 494 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) 495 { 496 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 497 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 498 u64 head = PERF_IDX2OFF(handle->head, buf); 499 u64 limit = buf->nr_pages * PAGE_SIZE; 500 501 /* 502 * The trace format isn't parseable in reverse, so clamp 503 * the limit to half of the buffer size in snapshot mode 504 * so that the worst case is half a buffer of records, as 505 * opposed to a single record. 506 */ 507 if (head < limit >> 1) 508 limit >>= 1; 509 510 /* 511 * If we're within max_record_sz of the limit, we must 512 * pad, move the head index and recompute the limit. 513 */ 514 if (limit - head < spe_pmu->max_record_sz) { 515 arm_spe_pmu_pad_buf(handle, limit - head); 516 handle->head = PERF_IDX2OFF(limit, buf); 517 limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; 518 } 519 520 return limit; 521 } 522 523 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) 524 { 525 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 526 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 527 const u64 bufsize = buf->nr_pages * PAGE_SIZE; 528 u64 limit = bufsize; 529 u64 head, tail, wakeup; 530 531 /* 532 * The head can be misaligned for two reasons: 533 * 534 * 1. The hardware left PMBPTR pointing to the first byte after 535 * a record when generating a buffer management event. 536 * 537 * 2. We used perf_aux_output_skip to consume handle->size bytes 538 * and CIRC_SPACE was used to compute the size, which always 539 * leaves one entry free. 540 * 541 * Deal with this by padding to the next alignment boundary and 542 * moving the head index. If we run out of buffer space, we'll 543 * reduce handle->size to zero and end up reporting truncation. 544 */ 545 head = PERF_IDX2OFF(handle->head, buf); 546 if (!IS_ALIGNED(head, spe_pmu->align)) { 547 unsigned long delta = roundup(head, spe_pmu->align) - head; 548 549 delta = min(delta, handle->size); 550 arm_spe_pmu_pad_buf(handle, delta); 551 head = PERF_IDX2OFF(handle->head, buf); 552 } 553 554 /* If we've run out of free space, then nothing more to do */ 555 if (!handle->size) 556 goto no_space; 557 558 /* Compute the tail and wakeup indices now that we've aligned head */ 559 tail = PERF_IDX2OFF(handle->head + handle->size, buf); 560 wakeup = PERF_IDX2OFF(handle->wakeup, buf); 561 562 /* 563 * Avoid clobbering unconsumed data. We know we have space, so 564 * if we see head == tail we know that the buffer is empty. If 565 * head > tail, then there's nothing to clobber prior to 566 * wrapping. 567 */ 568 if (head < tail) 569 limit = round_down(tail, PAGE_SIZE); 570 571 /* 572 * Wakeup may be arbitrarily far into the future. If it's not in 573 * the current generation, either we'll wrap before hitting it, 574 * or it's in the past and has been handled already. 575 * 576 * If there's a wakeup before we wrap, arrange to be woken up by 577 * the page boundary following it. Keep the tail boundary if 578 * that's lower. 579 */ 580 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) 581 limit = min(limit, round_up(wakeup, PAGE_SIZE)); 582 583 if (limit > head) 584 return limit; 585 586 arm_spe_pmu_pad_buf(handle, handle->size); 587 no_space: 588 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 589 perf_aux_output_end(handle, 0); 590 return 0; 591 } 592 593 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) 594 { 595 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 596 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 597 u64 limit = __arm_spe_pmu_next_off(handle); 598 u64 head = PERF_IDX2OFF(handle->head, buf); 599 600 /* 601 * If the head has come too close to the end of the buffer, 602 * then pad to the end and recompute the limit. 603 */ 604 if (limit && (limit - head < spe_pmu->max_record_sz)) { 605 arm_spe_pmu_pad_buf(handle, limit - head); 606 limit = __arm_spe_pmu_next_off(handle); 607 } 608 609 return limit; 610 } 611 612 static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, 613 struct perf_event *event) 614 { 615 u64 base, limit; 616 struct arm_spe_pmu_buf *buf; 617 618 if (ATTR_CFG_GET_FLD(&event->attr, discard)) { 619 limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD); 620 limit |= PMBLIMITR_EL1_E; 621 goto out_write_limit; 622 } 623 624 /* Start a new aux session */ 625 buf = perf_aux_output_begin(handle, event); 626 if (!buf) { 627 /* 628 * We still need to clear the limit pointer, since the 629 * profiler might only be disabled by virtue of a fault. 630 */ 631 limit = 0; 632 goto out_write_limit; 633 } 634 635 limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) 636 : arm_spe_pmu_next_off(handle); 637 if (limit) 638 limit |= PMBLIMITR_EL1_E; 639 640 limit += (u64)buf->base; 641 base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); 642 write_sysreg_s(base, SYS_PMBPTR_EL1); 643 644 out_write_limit: 645 write_sysreg_s(limit, SYS_PMBLIMITR_EL1); 646 return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; 647 } 648 649 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) 650 { 651 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 652 u64 offset, size; 653 654 offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; 655 size = offset - PERF_IDX2OFF(handle->head, buf); 656 657 if (buf->snapshot) 658 handle->head = offset; 659 660 perf_aux_output_end(handle, size); 661 } 662 663 static void arm_spe_pmu_disable_and_drain_local(void) 664 { 665 /* Disable profiling at EL0 and EL1 */ 666 write_sysreg_s(0, SYS_PMSCR_EL1); 667 isb(); 668 669 /* Drain any buffered data */ 670 psb_csync(); 671 dsb(nsh); 672 673 /* Disable the profiling buffer */ 674 write_sysreg_s(0, SYS_PMBLIMITR_EL1); 675 isb(); 676 } 677 678 /* IRQ handling */ 679 static enum arm_spe_pmu_buf_fault_action 680 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) 681 { 682 const char *err_str; 683 u64 pmbsr; 684 enum arm_spe_pmu_buf_fault_action ret; 685 686 /* 687 * Ensure new profiling data is visible to the CPU and any external 688 * aborts have been resolved. 689 */ 690 psb_csync(); 691 dsb(nsh); 692 693 /* Ensure hardware updates to PMBPTR_EL1 are visible */ 694 isb(); 695 696 /* Service required? */ 697 pmbsr = read_sysreg_s(SYS_PMBSR_EL1); 698 if (!FIELD_GET(PMBSR_EL1_S, pmbsr)) 699 return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; 700 701 /* 702 * If we've lost data, disable profiling and also set the PARTIAL 703 * flag to indicate that the last record is corrupted. 704 */ 705 if (FIELD_GET(PMBSR_EL1_DL, pmbsr)) 706 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | 707 PERF_AUX_FLAG_PARTIAL); 708 709 /* Report collisions to userspace so that it can up the period */ 710 if (FIELD_GET(PMBSR_EL1_COLL, pmbsr)) 711 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); 712 713 /* We only expect buffer management events */ 714 switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) { 715 case PMBSR_EL1_EC_BUF: 716 /* Handled below */ 717 break; 718 case PMBSR_EL1_EC_FAULT_S1: 719 case PMBSR_EL1_EC_FAULT_S2: 720 err_str = "Unexpected buffer fault"; 721 goto out_err; 722 default: 723 err_str = "Unknown error code"; 724 goto out_err; 725 } 726 727 /* Buffer management event */ 728 switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) { 729 case PMBSR_EL1_BUF_BSC_FULL: 730 ret = SPE_PMU_BUF_FAULT_ACT_OK; 731 goto out_stop; 732 default: 733 err_str = "Unknown buffer status code"; 734 } 735 736 out_err: 737 pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", 738 err_str, smp_processor_id(), pmbsr, 739 read_sysreg_s(SYS_PMBPTR_EL1), 740 read_sysreg_s(SYS_PMBLIMITR_EL1)); 741 ret = SPE_PMU_BUF_FAULT_ACT_FATAL; 742 743 out_stop: 744 arm_spe_perf_aux_output_end(handle); 745 return ret; 746 } 747 748 static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) 749 { 750 struct perf_output_handle *handle = dev; 751 struct perf_event *event = handle->event; 752 enum arm_spe_pmu_buf_fault_action act; 753 754 if (!perf_get_aux(handle)) 755 return IRQ_NONE; 756 757 act = arm_spe_pmu_buf_get_fault_act(handle); 758 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 759 return IRQ_NONE; 760 761 /* 762 * Ensure perf callbacks have completed, which may disable the 763 * profiling buffer in response to a TRUNCATION flag. 764 */ 765 irq_work_run(); 766 767 switch (act) { 768 case SPE_PMU_BUF_FAULT_ACT_FATAL: 769 /* 770 * If a fatal exception occurred then leaving the profiling 771 * buffer enabled is a recipe waiting to happen. Since 772 * fatal faults don't always imply truncation, make sure 773 * that the profiling buffer is disabled explicitly before 774 * clearing the syndrome register. 775 */ 776 arm_spe_pmu_disable_and_drain_local(); 777 break; 778 case SPE_PMU_BUF_FAULT_ACT_OK: 779 /* 780 * We handled the fault (the buffer was full), so resume 781 * profiling as long as we didn't detect truncation. 782 * PMBPTR might be misaligned, but we'll burn that bridge 783 * when we get to it. 784 */ 785 if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { 786 if (arm_spe_perf_aux_output_begin(handle, event)) { 787 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 788 break; 789 } 790 isb(); 791 } 792 break; 793 case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: 794 /* We've seen you before, but GCC has the memory of a sieve. */ 795 break; 796 } 797 798 /* The buffer pointers are now sane, so resume profiling. */ 799 write_sysreg_s(0, SYS_PMBSR_EL1); 800 return IRQ_HANDLED; 801 } 802 803 /* Perf callbacks */ 804 static int arm_spe_pmu_event_init(struct perf_event *event) 805 { 806 u64 reg; 807 struct perf_event_attr *attr = &event->attr; 808 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 809 810 /* This is, of course, deeply driver-specific */ 811 if (attr->type != event->pmu->type) 812 return -ENOENT; 813 814 if (event->cpu >= 0 && 815 !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) 816 return -ENOENT; 817 818 if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0) 819 return -EOPNOTSUPP; 820 821 if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) 822 return -EOPNOTSUPP; 823 824 if (arm_spe_event_to_pmsdsfr(event) != U64_MAX && 825 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 826 return -EOPNOTSUPP; 827 828 if (attr->exclude_idle) 829 return -EOPNOTSUPP; 830 831 /* 832 * Feedback-directed frequency throttling doesn't work when we 833 * have a buffer of samples. We'd need to manually count the 834 * samples in the buffer when it fills up and adjust the event 835 * count to reflect that. Instead, just force the user to specify 836 * a sample period. 837 */ 838 if (attr->freq) 839 return -EINVAL; 840 841 reg = arm_spe_event_to_pmsfcr(event); 842 if ((FIELD_GET(PMSFCR_EL1_FE, reg)) && 843 !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) 844 return -EOPNOTSUPP; 845 846 if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) && 847 !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 848 return -EOPNOTSUPP; 849 850 if ((FIELD_GET(PMSFCR_EL1_FT, reg)) && 851 !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) 852 return -EOPNOTSUPP; 853 854 if ((FIELD_GET(PMSFCR_EL1_FL, reg)) && 855 !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) 856 return -EOPNOTSUPP; 857 858 if ((FIELD_GET(PMSFCR_EL1_LDm, reg) || 859 FIELD_GET(PMSFCR_EL1_STm, reg) || 860 FIELD_GET(PMSFCR_EL1_Bm, reg) || 861 FIELD_GET(PMSFCR_EL1_SIMD, reg) || 862 FIELD_GET(PMSFCR_EL1_SIMDm, reg) || 863 FIELD_GET(PMSFCR_EL1_FP, reg) || 864 FIELD_GET(PMSFCR_EL1_FPm, reg)) && 865 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 866 return -EOPNOTSUPP; 867 868 if (ATTR_CFG_GET_FLD(&event->attr, discard) && 869 !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 870 return -EOPNOTSUPP; 871 872 set_spe_event_has_cx(event); 873 reg = arm_spe_event_to_pmscr(event); 874 if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT)) 875 return perf_allow_kernel(); 876 877 return 0; 878 } 879 880 static void arm_spe_pmu_start(struct perf_event *event, int flags) 881 { 882 u64 reg; 883 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 884 struct hw_perf_event *hwc = &event->hw; 885 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 886 887 hwc->state = 0; 888 if (arm_spe_perf_aux_output_begin(handle, event)) { 889 arm_spe_pmu_stop(event, 0); 890 return; 891 } 892 893 reg = arm_spe_event_to_pmsfcr(event); 894 write_sysreg_s(reg, SYS_PMSFCR_EL1); 895 896 reg = arm_spe_event_to_pmsevfr(event); 897 write_sysreg_s(reg, SYS_PMSEVFR_EL1); 898 899 if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) { 900 reg = arm_spe_event_to_pmsnevfr(event); 901 write_sysreg_s(reg, SYS_PMSNEVFR_EL1); 902 } 903 904 if (spe_pmu->features & SPE_PMU_FEAT_FDS) { 905 reg = arm_spe_event_to_pmsdsfr(event); 906 write_sysreg_s(reg, SYS_PMSDSFR_EL1); 907 } 908 909 reg = arm_spe_event_to_pmslatfr(event); 910 write_sysreg_s(reg, SYS_PMSLATFR_EL1); 911 912 if (flags & PERF_EF_RELOAD) { 913 reg = arm_spe_event_to_pmsirr(event); 914 write_sysreg_s(reg, SYS_PMSIRR_EL1); 915 isb(); 916 reg = local64_read(&hwc->period_left); 917 write_sysreg_s(reg, SYS_PMSICR_EL1); 918 } 919 920 reg = arm_spe_event_to_pmscr(event); 921 isb(); 922 write_sysreg_s(reg, SYS_PMSCR_EL1); 923 } 924 925 static void arm_spe_pmu_stop(struct perf_event *event, int flags) 926 { 927 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 928 struct hw_perf_event *hwc = &event->hw; 929 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 930 931 /* If we're already stopped, then nothing to do */ 932 if (hwc->state & PERF_HES_STOPPED) 933 return; 934 935 /* Stop all trace generation */ 936 arm_spe_pmu_disable_and_drain_local(); 937 938 if (flags & PERF_EF_UPDATE) { 939 /* 940 * If there's a fault pending then ensure we contain it 941 * to this buffer, since we might be on the context-switch 942 * path. 943 */ 944 if (perf_get_aux(handle)) { 945 enum arm_spe_pmu_buf_fault_action act; 946 947 act = arm_spe_pmu_buf_get_fault_act(handle); 948 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 949 arm_spe_perf_aux_output_end(handle); 950 else 951 write_sysreg_s(0, SYS_PMBSR_EL1); 952 } 953 954 /* 955 * This may also contain ECOUNT, but nobody else should 956 * be looking at period_left, since we forbid frequency 957 * based sampling. 958 */ 959 local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); 960 hwc->state |= PERF_HES_UPTODATE; 961 } 962 963 hwc->state |= PERF_HES_STOPPED; 964 } 965 966 static int arm_spe_pmu_add(struct perf_event *event, int flags) 967 { 968 int ret = 0; 969 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 970 struct hw_perf_event *hwc = &event->hw; 971 int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; 972 973 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 974 return -ENOENT; 975 976 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 977 978 if (flags & PERF_EF_START) { 979 arm_spe_pmu_start(event, PERF_EF_RELOAD); 980 if (hwc->state & PERF_HES_STOPPED) 981 ret = -EINVAL; 982 } 983 984 return ret; 985 } 986 987 static void arm_spe_pmu_del(struct perf_event *event, int flags) 988 { 989 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 990 } 991 992 static void arm_spe_pmu_read(struct perf_event *event) 993 { 994 } 995 996 static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, 997 int nr_pages, bool snapshot) 998 { 999 int i, cpu = event->cpu; 1000 struct page **pglist; 1001 struct arm_spe_pmu_buf *buf; 1002 1003 /* We need at least two pages for this to work. */ 1004 if (nr_pages < 2) 1005 return NULL; 1006 1007 /* 1008 * We require an even number of pages for snapshot mode, so that 1009 * we can effectively treat the buffer as consisting of two equal 1010 * parts and give userspace a fighting chance of getting some 1011 * useful data out of it. 1012 */ 1013 if (snapshot && (nr_pages & 1)) 1014 return NULL; 1015 1016 if (cpu == -1) 1017 cpu = raw_smp_processor_id(); 1018 1019 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); 1020 if (!buf) 1021 return NULL; 1022 1023 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); 1024 if (!pglist) 1025 goto out_free_buf; 1026 1027 for (i = 0; i < nr_pages; ++i) 1028 pglist[i] = virt_to_page(pages[i]); 1029 1030 buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); 1031 if (!buf->base) 1032 goto out_free_pglist; 1033 1034 buf->nr_pages = nr_pages; 1035 buf->snapshot = snapshot; 1036 1037 kfree(pglist); 1038 return buf; 1039 1040 out_free_pglist: 1041 kfree(pglist); 1042 out_free_buf: 1043 kfree(buf); 1044 return NULL; 1045 } 1046 1047 static void arm_spe_pmu_free_aux(void *aux) 1048 { 1049 struct arm_spe_pmu_buf *buf = aux; 1050 1051 vunmap(buf->base); 1052 kfree(buf); 1053 } 1054 1055 /* Initialisation and teardown functions */ 1056 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) 1057 { 1058 static atomic_t pmu_idx = ATOMIC_INIT(-1); 1059 1060 int idx; 1061 char *name; 1062 struct device *dev = &spe_pmu->pdev->dev; 1063 1064 spe_pmu->pmu = (struct pmu) { 1065 .module = THIS_MODULE, 1066 .parent = &spe_pmu->pdev->dev, 1067 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, 1068 .attr_groups = arm_spe_pmu_attr_groups, 1069 /* 1070 * We hitch a ride on the software context here, so that 1071 * we can support per-task profiling (which is not possible 1072 * with the invalid context as it doesn't get sched callbacks). 1073 * This requires that userspace either uses a dummy event for 1074 * perf_event_open, since the aux buffer is not setup until 1075 * a subsequent mmap, or creates the profiling event in a 1076 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it 1077 * once the buffer has been created. 1078 */ 1079 .task_ctx_nr = perf_sw_context, 1080 .event_init = arm_spe_pmu_event_init, 1081 .add = arm_spe_pmu_add, 1082 .del = arm_spe_pmu_del, 1083 .start = arm_spe_pmu_start, 1084 .stop = arm_spe_pmu_stop, 1085 .read = arm_spe_pmu_read, 1086 .setup_aux = arm_spe_pmu_setup_aux, 1087 .free_aux = arm_spe_pmu_free_aux, 1088 }; 1089 1090 idx = atomic_inc_return(&pmu_idx); 1091 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); 1092 if (!name) { 1093 dev_err(dev, "failed to allocate name for pmu %d\n", idx); 1094 return -ENOMEM; 1095 } 1096 1097 return perf_pmu_register(&spe_pmu->pmu, name, -1); 1098 } 1099 1100 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) 1101 { 1102 perf_pmu_unregister(&spe_pmu->pmu); 1103 } 1104 1105 static void __arm_spe_pmu_dev_probe(void *info) 1106 { 1107 int fld; 1108 u64 reg; 1109 struct arm_spe_pmu *spe_pmu = info; 1110 struct device *dev = &spe_pmu->pdev->dev; 1111 1112 fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), 1113 ID_AA64DFR0_EL1_PMSVer_SHIFT); 1114 if (!fld) { 1115 dev_err(dev, 1116 "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", 1117 fld, smp_processor_id()); 1118 return; 1119 } 1120 spe_pmu->pmsver = (u16)fld; 1121 1122 /* Read PMBIDR first to determine whether or not we have access */ 1123 reg = read_sysreg_s(SYS_PMBIDR_EL1); 1124 if (FIELD_GET(PMBIDR_EL1_P, reg)) { 1125 dev_err(dev, 1126 "profiling buffer owned by higher exception level\n"); 1127 return; 1128 } 1129 1130 /* Minimum alignment. If it's out-of-range, then fail the probe */ 1131 fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg); 1132 spe_pmu->align = 1 << fld; 1133 if (spe_pmu->align > SZ_2K) { 1134 dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", 1135 fld, smp_processor_id()); 1136 return; 1137 } 1138 1139 /* It's now safe to read PMSIDR and figure out what we've got */ 1140 reg = read_sysreg_s(SYS_PMSIDR_EL1); 1141 if (FIELD_GET(PMSIDR_EL1_FE, reg)) 1142 spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; 1143 1144 if (FIELD_GET(PMSIDR_EL1_FnE, reg)) 1145 spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT; 1146 1147 if (FIELD_GET(PMSIDR_EL1_FT, reg)) 1148 spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; 1149 1150 if (FIELD_GET(PMSIDR_EL1_FL, reg)) 1151 spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; 1152 1153 if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg)) 1154 spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; 1155 1156 if (FIELD_GET(PMSIDR_EL1_LDS, reg)) 1157 spe_pmu->features |= SPE_PMU_FEAT_LDS; 1158 1159 if (FIELD_GET(PMSIDR_EL1_ERND, reg)) 1160 spe_pmu->features |= SPE_PMU_FEAT_ERND; 1161 1162 if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2) 1163 spe_pmu->features |= SPE_PMU_FEAT_DISCARD; 1164 1165 if (FIELD_GET(PMSIDR_EL1_EFT, reg)) 1166 spe_pmu->features |= SPE_PMU_FEAT_EFT; 1167 1168 if (FIELD_GET(PMSIDR_EL1_FDS, reg)) 1169 spe_pmu->features |= SPE_PMU_FEAT_FDS; 1170 1171 /* This field has a spaced out encoding, so just use a look-up */ 1172 fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); 1173 switch (fld) { 1174 case PMSIDR_EL1_INTERVAL_256: 1175 spe_pmu->min_period = 256; 1176 break; 1177 case PMSIDR_EL1_INTERVAL_512: 1178 spe_pmu->min_period = 512; 1179 break; 1180 case PMSIDR_EL1_INTERVAL_768: 1181 spe_pmu->min_period = 768; 1182 break; 1183 case PMSIDR_EL1_INTERVAL_1024: 1184 spe_pmu->min_period = 1024; 1185 break; 1186 case PMSIDR_EL1_INTERVAL_1536: 1187 spe_pmu->min_period = 1536; 1188 break; 1189 case PMSIDR_EL1_INTERVAL_2048: 1190 spe_pmu->min_period = 2048; 1191 break; 1192 case PMSIDR_EL1_INTERVAL_3072: 1193 spe_pmu->min_period = 3072; 1194 break; 1195 default: 1196 dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", 1197 fld); 1198 fallthrough; 1199 case PMSIDR_EL1_INTERVAL_4096: 1200 spe_pmu->min_period = 4096; 1201 } 1202 1203 /* Maximum record size. If it's out-of-range, then fail the probe */ 1204 fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg); 1205 spe_pmu->max_record_sz = 1 << fld; 1206 if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { 1207 dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", 1208 fld, smp_processor_id()); 1209 return; 1210 } 1211 1212 fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg); 1213 switch (fld) { 1214 default: 1215 dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", 1216 fld); 1217 fallthrough; 1218 case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT: 1219 spe_pmu->counter_sz = 12; 1220 break; 1221 case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT: 1222 spe_pmu->counter_sz = 16; 1223 } 1224 1225 /* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */ 1226 write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1); 1227 spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1); 1228 1229 dev_info(dev, 1230 "probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", 1231 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus), 1232 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); 1233 1234 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; 1235 } 1236 1237 static void __arm_spe_pmu_reset_local(void) 1238 { 1239 /* 1240 * This is probably overkill, as we have no idea where we're 1241 * draining any buffered data to... 1242 */ 1243 arm_spe_pmu_disable_and_drain_local(); 1244 1245 /* Reset the buffer base pointer */ 1246 write_sysreg_s(0, SYS_PMBPTR_EL1); 1247 isb(); 1248 1249 /* Clear any pending management interrupts */ 1250 write_sysreg_s(0, SYS_PMBSR_EL1); 1251 isb(); 1252 } 1253 1254 static void __arm_spe_pmu_setup_one(void *info) 1255 { 1256 struct arm_spe_pmu *spe_pmu = info; 1257 1258 __arm_spe_pmu_reset_local(); 1259 enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); 1260 } 1261 1262 static void __arm_spe_pmu_stop_one(void *info) 1263 { 1264 struct arm_spe_pmu *spe_pmu = info; 1265 1266 disable_percpu_irq(spe_pmu->irq); 1267 __arm_spe_pmu_reset_local(); 1268 } 1269 1270 static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) 1271 { 1272 struct arm_spe_pmu *spe_pmu; 1273 1274 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1275 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1276 return 0; 1277 1278 __arm_spe_pmu_setup_one(spe_pmu); 1279 return 0; 1280 } 1281 1282 static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1283 { 1284 struct arm_spe_pmu *spe_pmu; 1285 1286 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1287 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1288 return 0; 1289 1290 __arm_spe_pmu_stop_one(spe_pmu); 1291 return 0; 1292 } 1293 1294 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) 1295 { 1296 int ret; 1297 cpumask_t *mask = &spe_pmu->supported_cpus; 1298 1299 /* Make sure we probe the hardware on a relevant CPU */ 1300 ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); 1301 if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) 1302 return -ENXIO; 1303 1304 /* Request our PPIs (note that the IRQ is still disabled) */ 1305 ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler, 1306 DRVNAME, mask, spe_pmu->handle); 1307 if (ret) 1308 return ret; 1309 1310 /* 1311 * Register our hotplug notifier now so we don't miss any events. 1312 * This will enable the IRQ for any supported CPUs that are already 1313 * up. 1314 */ 1315 ret = cpuhp_state_add_instance(arm_spe_pmu_online, 1316 &spe_pmu->hotplug_node); 1317 if (ret) 1318 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1319 1320 return ret; 1321 } 1322 1323 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) 1324 { 1325 cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); 1326 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1327 } 1328 1329 /* Driver and device probing */ 1330 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) 1331 { 1332 struct platform_device *pdev = spe_pmu->pdev; 1333 const struct cpumask *affinity; 1334 int irq; 1335 1336 irq = platform_get_irq_affinity(pdev, 0, &affinity); 1337 if (irq < 0) 1338 return -ENXIO; 1339 1340 if (!irq_is_percpu(irq)) { 1341 dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); 1342 return -EINVAL; 1343 } 1344 1345 cpumask_copy(&spe_pmu->supported_cpus, affinity); 1346 1347 spe_pmu->irq = irq; 1348 return 0; 1349 } 1350 1351 static const struct of_device_id arm_spe_pmu_of_match[] = { 1352 { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, 1353 { /* Sentinel */ }, 1354 }; 1355 MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); 1356 1357 static const struct platform_device_id arm_spe_match[] = { 1358 { ARMV8_SPE_PDEV_NAME, 0}, 1359 { } 1360 }; 1361 MODULE_DEVICE_TABLE(platform, arm_spe_match); 1362 1363 static int arm_spe_pmu_device_probe(struct platform_device *pdev) 1364 { 1365 int ret; 1366 struct arm_spe_pmu *spe_pmu; 1367 struct device *dev = &pdev->dev; 1368 1369 /* 1370 * If kernelspace is unmapped when running at EL0, then the SPE 1371 * buffer will fault and prematurely terminate the AUX session. 1372 */ 1373 if (arm64_kernel_unmapped_at_el0()) { 1374 dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); 1375 return -EPERM; 1376 } 1377 1378 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); 1379 if (!spe_pmu) 1380 return -ENOMEM; 1381 1382 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); 1383 if (!spe_pmu->handle) 1384 return -ENOMEM; 1385 1386 spe_pmu->pdev = pdev; 1387 platform_set_drvdata(pdev, spe_pmu); 1388 1389 ret = arm_spe_pmu_irq_probe(spe_pmu); 1390 if (ret) 1391 goto out_free_handle; 1392 1393 ret = arm_spe_pmu_dev_init(spe_pmu); 1394 if (ret) 1395 goto out_free_handle; 1396 1397 ret = arm_spe_pmu_perf_init(spe_pmu); 1398 if (ret) 1399 goto out_teardown_dev; 1400 1401 return 0; 1402 1403 out_teardown_dev: 1404 arm_spe_pmu_dev_teardown(spe_pmu); 1405 out_free_handle: 1406 free_percpu(spe_pmu->handle); 1407 return ret; 1408 } 1409 1410 static void arm_spe_pmu_device_remove(struct platform_device *pdev) 1411 { 1412 struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); 1413 1414 arm_spe_pmu_perf_destroy(spe_pmu); 1415 arm_spe_pmu_dev_teardown(spe_pmu); 1416 free_percpu(spe_pmu->handle); 1417 } 1418 1419 static struct platform_driver arm_spe_pmu_driver = { 1420 .id_table = arm_spe_match, 1421 .driver = { 1422 .name = DRVNAME, 1423 .of_match_table = of_match_ptr(arm_spe_pmu_of_match), 1424 .suppress_bind_attrs = true, 1425 }, 1426 .probe = arm_spe_pmu_device_probe, 1427 .remove = arm_spe_pmu_device_remove, 1428 }; 1429 1430 static int __init arm_spe_pmu_init(void) 1431 { 1432 int ret; 1433 1434 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, 1435 arm_spe_pmu_cpu_startup, 1436 arm_spe_pmu_cpu_teardown); 1437 if (ret < 0) 1438 return ret; 1439 arm_spe_pmu_online = ret; 1440 1441 ret = platform_driver_register(&arm_spe_pmu_driver); 1442 if (ret) 1443 cpuhp_remove_multi_state(arm_spe_pmu_online); 1444 1445 return ret; 1446 } 1447 1448 static void __exit arm_spe_pmu_exit(void) 1449 { 1450 platform_driver_unregister(&arm_spe_pmu_driver); 1451 cpuhp_remove_multi_state(arm_spe_pmu_online); 1452 } 1453 1454 module_init(arm_spe_pmu_init); 1455 module_exit(arm_spe_pmu_exit); 1456 1457 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); 1458 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1459 MODULE_LICENSE("GPL v2"); 1460