1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Perf support for the Statistical Profiling Extension, introduced as 4 * part of ARMv8.2. 5 * 6 * Copyright (C) 2016 ARM Limited 7 * 8 * Author: Will Deacon <will.deacon@arm.com> 9 */ 10 11 #define PMUNAME "arm_spe" 12 #define DRVNAME PMUNAME "_pmu" 13 #define pr_fmt(fmt) DRVNAME ": " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/bitops.h> 17 #include <linux/bug.h> 18 #include <linux/capability.h> 19 #include <linux/cpuhotplug.h> 20 #include <linux/cpumask.h> 21 #include <linux/device.h> 22 #include <linux/errno.h> 23 #include <linux/interrupt.h> 24 #include <linux/irq.h> 25 #include <linux/kernel.h> 26 #include <linux/list.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/perf_event.h> 30 #include <linux/perf/arm_pmu.h> 31 #include <linux/platform_device.h> 32 #include <linux/printk.h> 33 #include <linux/slab.h> 34 #include <linux/smp.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/barrier.h> 38 #include <asm/cpufeature.h> 39 #include <asm/mmu.h> 40 #include <asm/sysreg.h> 41 42 /* 43 * Cache if the event is allowed to trace Context information. 44 * This allows us to perform the check, i.e, perf_allow_kernel(), 45 * in the context of the event owner, once, during the event_init(). 46 */ 47 #define SPE_PMU_HW_FLAGS_CX 0x00001 48 49 static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); 50 51 static void set_spe_event_has_cx(struct perf_event *event) 52 { 53 if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel()) 54 event->hw.flags |= SPE_PMU_HW_FLAGS_CX; 55 } 56 57 static bool get_spe_event_has_cx(struct perf_event *event) 58 { 59 return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); 60 } 61 62 #define ARM_SPE_BUF_PAD_BYTE 0 63 64 struct arm_spe_pmu_buf { 65 int nr_pages; 66 bool snapshot; 67 void *base; 68 }; 69 70 struct arm_spe_pmu { 71 struct pmu pmu; 72 struct platform_device *pdev; 73 cpumask_t supported_cpus; 74 struct hlist_node hotplug_node; 75 76 int irq; /* PPI */ 77 u16 pmsver; 78 u16 min_period; 79 u16 counter_sz; 80 81 #define SPE_PMU_FEAT_FILT_EVT (1UL << 0) 82 #define SPE_PMU_FEAT_FILT_TYP (1UL << 1) 83 #define SPE_PMU_FEAT_FILT_LAT (1UL << 2) 84 #define SPE_PMU_FEAT_ARCH_INST (1UL << 3) 85 #define SPE_PMU_FEAT_LDS (1UL << 4) 86 #define SPE_PMU_FEAT_ERND (1UL << 5) 87 #define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) 88 #define SPE_PMU_FEAT_DISCARD (1UL << 7) 89 #define SPE_PMU_FEAT_EFT (1UL << 8) 90 #define SPE_PMU_FEAT_FDS (1UL << 9) 91 #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) 92 u64 features; 93 94 u64 pmsevfr_res0; 95 u16 max_record_sz; 96 u16 align; 97 struct perf_output_handle __percpu *handle; 98 }; 99 100 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) 101 102 /* Convert a free-running index from perf into an SPE buffer offset */ 103 #define PERF_IDX2OFF(idx, buf) \ 104 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) 105 106 /* Keep track of our dynamic hotplug state */ 107 static enum cpuhp_state arm_spe_pmu_online; 108 109 enum arm_spe_pmu_buf_fault_action { 110 SPE_PMU_BUF_FAULT_ACT_SPURIOUS, 111 SPE_PMU_BUF_FAULT_ACT_FATAL, 112 SPE_PMU_BUF_FAULT_ACT_OK, 113 }; 114 115 /* This sysfs gunk was really good fun to write. */ 116 enum arm_spe_pmu_capabilities { 117 SPE_PMU_CAP_ARCH_INST = 0, 118 SPE_PMU_CAP_ERND, 119 SPE_PMU_CAP_FEAT_MAX, 120 SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, 121 SPE_PMU_CAP_MIN_IVAL, 122 SPE_PMU_CAP_EVENT_FILTER, 123 }; 124 125 static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { 126 [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, 127 [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, 128 }; 129 130 static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) 131 { 132 if (cap < SPE_PMU_CAP_FEAT_MAX) 133 return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); 134 135 switch (cap) { 136 case SPE_PMU_CAP_CNT_SZ: 137 return spe_pmu->counter_sz; 138 case SPE_PMU_CAP_MIN_IVAL: 139 return spe_pmu->min_period; 140 case SPE_PMU_CAP_EVENT_FILTER: 141 return ~spe_pmu->pmsevfr_res0; 142 default: 143 WARN(1, "unknown cap %d\n", cap); 144 } 145 146 return 0; 147 } 148 149 static ssize_t arm_spe_pmu_cap_show(struct device *dev, 150 struct device_attribute *attr, 151 char *buf) 152 { 153 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 154 struct dev_ext_attribute *ea = 155 container_of(attr, struct dev_ext_attribute, attr); 156 int cap = (long)ea->var; 157 158 return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 159 } 160 161 static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev, 162 struct device_attribute *attr, 163 char *buf) 164 { 165 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 166 struct dev_ext_attribute *ea = 167 container_of(attr, struct dev_ext_attribute, attr); 168 int cap = (long)ea->var; 169 170 return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 171 } 172 173 #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ 174 &((struct dev_ext_attribute[]) { \ 175 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ 176 })[0].attr.attr 177 178 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ 179 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) 180 #define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var) \ 181 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var) 182 183 static struct attribute *arm_spe_pmu_cap_attr[] = { 184 SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), 185 SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), 186 SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), 187 SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), 188 SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER), 189 NULL, 190 }; 191 192 static const struct attribute_group arm_spe_pmu_cap_group = { 193 .name = "caps", 194 .attrs = arm_spe_pmu_cap_attr, 195 }; 196 197 /* User ABI */ 198 #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ 199 #define ATTR_CFG_FLD_ts_enable_LO 0 200 #define ATTR_CFG_FLD_ts_enable_HI 0 201 #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ 202 #define ATTR_CFG_FLD_pa_enable_LO 1 203 #define ATTR_CFG_FLD_pa_enable_HI 1 204 #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ 205 #define ATTR_CFG_FLD_pct_enable_LO 2 206 #define ATTR_CFG_FLD_pct_enable_HI 2 207 #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ 208 #define ATTR_CFG_FLD_jitter_LO 16 209 #define ATTR_CFG_FLD_jitter_HI 16 210 #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ 211 #define ATTR_CFG_FLD_branch_filter_LO 32 212 #define ATTR_CFG_FLD_branch_filter_HI 32 213 #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ 214 #define ATTR_CFG_FLD_load_filter_LO 33 215 #define ATTR_CFG_FLD_load_filter_HI 33 216 #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ 217 #define ATTR_CFG_FLD_store_filter_LO 34 218 #define ATTR_CFG_FLD_store_filter_HI 34 219 #define ATTR_CFG_FLD_discard_CFG config /* PMBLIMITR_EL1.FM = DISCARD */ 220 #define ATTR_CFG_FLD_discard_LO 35 221 #define ATTR_CFG_FLD_discard_HI 35 222 #define ATTR_CFG_FLD_branch_filter_mask_CFG config /* PMSFCR_EL1.Bm */ 223 #define ATTR_CFG_FLD_branch_filter_mask_LO 36 224 #define ATTR_CFG_FLD_branch_filter_mask_HI 36 225 #define ATTR_CFG_FLD_load_filter_mask_CFG config /* PMSFCR_EL1.LDm */ 226 #define ATTR_CFG_FLD_load_filter_mask_LO 37 227 #define ATTR_CFG_FLD_load_filter_mask_HI 37 228 #define ATTR_CFG_FLD_store_filter_mask_CFG config /* PMSFCR_EL1.STm */ 229 #define ATTR_CFG_FLD_store_filter_mask_LO 38 230 #define ATTR_CFG_FLD_store_filter_mask_HI 38 231 #define ATTR_CFG_FLD_simd_filter_CFG config /* PMSFCR_EL1.SIMD */ 232 #define ATTR_CFG_FLD_simd_filter_LO 39 233 #define ATTR_CFG_FLD_simd_filter_HI 39 234 #define ATTR_CFG_FLD_simd_filter_mask_CFG config /* PMSFCR_EL1.SIMDm */ 235 #define ATTR_CFG_FLD_simd_filter_mask_LO 40 236 #define ATTR_CFG_FLD_simd_filter_mask_HI 40 237 #define ATTR_CFG_FLD_float_filter_CFG config /* PMSFCR_EL1.FP */ 238 #define ATTR_CFG_FLD_float_filter_LO 41 239 #define ATTR_CFG_FLD_float_filter_HI 41 240 #define ATTR_CFG_FLD_float_filter_mask_CFG config /* PMSFCR_EL1.FPm */ 241 #define ATTR_CFG_FLD_float_filter_mask_LO 42 242 #define ATTR_CFG_FLD_float_filter_mask_HI 42 243 244 #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ 245 #define ATTR_CFG_FLD_event_filter_LO 0 246 #define ATTR_CFG_FLD_event_filter_HI 63 247 248 #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ 249 #define ATTR_CFG_FLD_min_latency_LO 0 250 #define ATTR_CFG_FLD_min_latency_HI 11 251 252 #define ATTR_CFG_FLD_inv_event_filter_CFG config3 /* PMSNEVFR_EL1 */ 253 #define ATTR_CFG_FLD_inv_event_filter_LO 0 254 #define ATTR_CFG_FLD_inv_event_filter_HI 63 255 256 #define ATTR_CFG_FLD_inv_data_src_filter_CFG config4 /* inverse of PMSDSFR_EL1 */ 257 #define ATTR_CFG_FLD_inv_data_src_filter_LO 0 258 #define ATTR_CFG_FLD_inv_data_src_filter_HI 63 259 260 GEN_PMU_FORMAT_ATTR(ts_enable); 261 GEN_PMU_FORMAT_ATTR(pa_enable); 262 GEN_PMU_FORMAT_ATTR(pct_enable); 263 GEN_PMU_FORMAT_ATTR(jitter); 264 GEN_PMU_FORMAT_ATTR(branch_filter); 265 GEN_PMU_FORMAT_ATTR(branch_filter_mask); 266 GEN_PMU_FORMAT_ATTR(load_filter); 267 GEN_PMU_FORMAT_ATTR(load_filter_mask); 268 GEN_PMU_FORMAT_ATTR(store_filter); 269 GEN_PMU_FORMAT_ATTR(store_filter_mask); 270 GEN_PMU_FORMAT_ATTR(simd_filter); 271 GEN_PMU_FORMAT_ATTR(simd_filter_mask); 272 GEN_PMU_FORMAT_ATTR(float_filter); 273 GEN_PMU_FORMAT_ATTR(float_filter_mask); 274 GEN_PMU_FORMAT_ATTR(event_filter); 275 GEN_PMU_FORMAT_ATTR(inv_event_filter); 276 GEN_PMU_FORMAT_ATTR(inv_data_src_filter); 277 GEN_PMU_FORMAT_ATTR(min_latency); 278 GEN_PMU_FORMAT_ATTR(discard); 279 280 static struct attribute *arm_spe_pmu_formats_attr[] = { 281 &format_attr_ts_enable.attr, 282 &format_attr_pa_enable.attr, 283 &format_attr_pct_enable.attr, 284 &format_attr_jitter.attr, 285 &format_attr_branch_filter.attr, 286 &format_attr_branch_filter_mask.attr, 287 &format_attr_load_filter.attr, 288 &format_attr_load_filter_mask.attr, 289 &format_attr_store_filter.attr, 290 &format_attr_store_filter_mask.attr, 291 &format_attr_simd_filter.attr, 292 &format_attr_simd_filter_mask.attr, 293 &format_attr_float_filter.attr, 294 &format_attr_float_filter_mask.attr, 295 &format_attr_event_filter.attr, 296 &format_attr_inv_event_filter.attr, 297 &format_attr_inv_data_src_filter.attr, 298 &format_attr_min_latency.attr, 299 &format_attr_discard.attr, 300 NULL, 301 }; 302 303 static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, 304 struct attribute *attr, 305 int unused) 306 { 307 struct device *dev = kobj_to_dev(kobj); 308 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 309 310 if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 311 return 0; 312 313 if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 314 return 0; 315 316 if (attr == &format_attr_inv_data_src_filter.attr && 317 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 318 return 0; 319 320 if ((attr == &format_attr_branch_filter_mask.attr || 321 attr == &format_attr_load_filter_mask.attr || 322 attr == &format_attr_store_filter_mask.attr || 323 attr == &format_attr_simd_filter.attr || 324 attr == &format_attr_simd_filter_mask.attr || 325 attr == &format_attr_float_filter.attr || 326 attr == &format_attr_float_filter_mask.attr) && 327 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 328 return 0; 329 330 return attr->mode; 331 } 332 333 static const struct attribute_group arm_spe_pmu_format_group = { 334 .name = "format", 335 .is_visible = arm_spe_pmu_format_attr_is_visible, 336 .attrs = arm_spe_pmu_formats_attr, 337 }; 338 339 static ssize_t cpumask_show(struct device *dev, 340 struct device_attribute *attr, char *buf) 341 { 342 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 343 344 return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); 345 } 346 static DEVICE_ATTR_RO(cpumask); 347 348 static struct attribute *arm_spe_pmu_attrs[] = { 349 &dev_attr_cpumask.attr, 350 NULL, 351 }; 352 353 static const struct attribute_group arm_spe_pmu_group = { 354 .attrs = arm_spe_pmu_attrs, 355 }; 356 357 static const struct attribute_group *arm_spe_pmu_attr_groups[] = { 358 &arm_spe_pmu_group, 359 &arm_spe_pmu_cap_group, 360 &arm_spe_pmu_format_group, 361 NULL, 362 }; 363 364 /* Convert between user ABI and register values */ 365 static u64 arm_spe_event_to_pmscr(struct perf_event *event) 366 { 367 struct perf_event_attr *attr = &event->attr; 368 u64 reg = 0; 369 370 reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable)); 371 reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable)); 372 reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable)); 373 374 if (!attr->exclude_user) 375 reg |= PMSCR_EL1_E0SPE; 376 377 if (!attr->exclude_kernel) 378 reg |= PMSCR_EL1_E1SPE; 379 380 if (get_spe_event_has_cx(event)) 381 reg |= PMSCR_EL1_CX; 382 383 return reg; 384 } 385 386 static void arm_spe_event_sanitise_period(struct perf_event *event) 387 { 388 u64 period = event->hw.sample_period; 389 u64 max_period = PMSIRR_EL1_INTERVAL_MASK; 390 391 /* 392 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a 393 * recommendation for the minimum interval, not a hardware limitation. 394 * 395 * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, 396 * Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) 397 * states: "Software must set this to a nonzero value". Use 1 as the 398 * minimum value. 399 */ 400 u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); 401 402 period = clamp_t(u64, period, min_period, max_period) & max_period; 403 event->hw.sample_period = period; 404 } 405 406 static u64 arm_spe_event_to_pmsirr(struct perf_event *event) 407 { 408 struct perf_event_attr *attr = &event->attr; 409 u64 reg = 0; 410 411 arm_spe_event_sanitise_period(event); 412 413 reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter)); 414 reg |= event->hw.sample_period; 415 416 return reg; 417 } 418 419 static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) 420 { 421 struct perf_event_attr *attr = &event->attr; 422 u64 reg = 0; 423 424 reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter)); 425 reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask)); 426 reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter)); 427 reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask)); 428 reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter)); 429 reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask)); 430 reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter)); 431 reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask)); 432 reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter)); 433 reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask)); 434 435 if (reg) 436 reg |= PMSFCR_EL1_FT; 437 438 if (ATTR_CFG_GET_FLD(attr, event_filter)) 439 reg |= PMSFCR_EL1_FE; 440 441 if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) 442 reg |= PMSFCR_EL1_FnE; 443 444 if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter)) 445 reg |= PMSFCR_EL1_FDS; 446 447 if (ATTR_CFG_GET_FLD(attr, min_latency)) 448 reg |= PMSFCR_EL1_FL; 449 450 return reg; 451 } 452 453 static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) 454 { 455 struct perf_event_attr *attr = &event->attr; 456 return ATTR_CFG_GET_FLD(attr, event_filter); 457 } 458 459 static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event) 460 { 461 struct perf_event_attr *attr = &event->attr; 462 return ATTR_CFG_GET_FLD(attr, inv_event_filter); 463 } 464 465 static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) 466 { 467 struct perf_event_attr *attr = &event->attr; 468 return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); 469 } 470 471 static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event) 472 { 473 struct perf_event_attr *attr = &event->attr; 474 475 /* 476 * Data src filter is inverted so that the default value of 0 is 477 * equivalent to no filtering. 478 */ 479 return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter); 480 } 481 482 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) 483 { 484 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 485 u64 head = PERF_IDX2OFF(handle->head, buf); 486 487 memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); 488 if (!buf->snapshot) 489 perf_aux_output_skip(handle, len); 490 } 491 492 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) 493 { 494 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 495 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 496 u64 head = PERF_IDX2OFF(handle->head, buf); 497 u64 limit = buf->nr_pages * PAGE_SIZE; 498 499 /* 500 * The trace format isn't parseable in reverse, so clamp 501 * the limit to half of the buffer size in snapshot mode 502 * so that the worst case is half a buffer of records, as 503 * opposed to a single record. 504 */ 505 if (head < limit >> 1) 506 limit >>= 1; 507 508 /* 509 * If we're within max_record_sz of the limit, we must 510 * pad, move the head index and recompute the limit. 511 */ 512 if (limit - head < spe_pmu->max_record_sz) { 513 arm_spe_pmu_pad_buf(handle, limit - head); 514 handle->head = PERF_IDX2OFF(limit, buf); 515 limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; 516 } 517 518 return limit; 519 } 520 521 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) 522 { 523 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 524 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 525 const u64 bufsize = buf->nr_pages * PAGE_SIZE; 526 u64 limit = bufsize; 527 u64 head, tail, wakeup; 528 529 /* 530 * The head can be misaligned for two reasons: 531 * 532 * 1. The hardware left PMBPTR pointing to the first byte after 533 * a record when generating a buffer management event. 534 * 535 * 2. We used perf_aux_output_skip to consume handle->size bytes 536 * and CIRC_SPACE was used to compute the size, which always 537 * leaves one entry free. 538 * 539 * Deal with this by padding to the next alignment boundary and 540 * moving the head index. If we run out of buffer space, we'll 541 * reduce handle->size to zero and end up reporting truncation. 542 */ 543 head = PERF_IDX2OFF(handle->head, buf); 544 if (!IS_ALIGNED(head, spe_pmu->align)) { 545 unsigned long delta = roundup(head, spe_pmu->align) - head; 546 547 delta = min(delta, handle->size); 548 arm_spe_pmu_pad_buf(handle, delta); 549 head = PERF_IDX2OFF(handle->head, buf); 550 } 551 552 /* If we've run out of free space, then nothing more to do */ 553 if (!handle->size) 554 goto no_space; 555 556 /* Compute the tail and wakeup indices now that we've aligned head */ 557 tail = PERF_IDX2OFF(handle->head + handle->size, buf); 558 wakeup = PERF_IDX2OFF(handle->wakeup, buf); 559 560 /* 561 * Avoid clobbering unconsumed data. We know we have space, so 562 * if we see head == tail we know that the buffer is empty. If 563 * head > tail, then there's nothing to clobber prior to 564 * wrapping. 565 */ 566 if (head < tail) 567 limit = round_down(tail, PAGE_SIZE); 568 569 /* 570 * Wakeup may be arbitrarily far into the future. If it's not in 571 * the current generation, either we'll wrap before hitting it, 572 * or it's in the past and has been handled already. 573 * 574 * If there's a wakeup before we wrap, arrange to be woken up by 575 * the page boundary following it. Keep the tail boundary if 576 * that's lower. 577 */ 578 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) 579 limit = min(limit, round_up(wakeup, PAGE_SIZE)); 580 581 if (limit > head) 582 return limit; 583 584 arm_spe_pmu_pad_buf(handle, handle->size); 585 no_space: 586 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 587 perf_aux_output_end(handle, 0); 588 return 0; 589 } 590 591 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) 592 { 593 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 594 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 595 u64 limit = __arm_spe_pmu_next_off(handle); 596 u64 head = PERF_IDX2OFF(handle->head, buf); 597 598 /* 599 * If the head has come too close to the end of the buffer, 600 * then pad to the end and recompute the limit. 601 */ 602 if (limit && (limit - head < spe_pmu->max_record_sz)) { 603 arm_spe_pmu_pad_buf(handle, limit - head); 604 limit = __arm_spe_pmu_next_off(handle); 605 } 606 607 return limit; 608 } 609 610 static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, 611 struct perf_event *event) 612 { 613 u64 base, limit; 614 struct arm_spe_pmu_buf *buf; 615 616 if (ATTR_CFG_GET_FLD(&event->attr, discard)) { 617 limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD); 618 limit |= PMBLIMITR_EL1_E; 619 goto out_write_limit; 620 } 621 622 /* Start a new aux session */ 623 buf = perf_aux_output_begin(handle, event); 624 if (!buf) { 625 event->hw.state |= PERF_HES_STOPPED; 626 /* 627 * We still need to clear the limit pointer, since the 628 * profiler might only be disabled by virtue of a fault. 629 */ 630 limit = 0; 631 goto out_write_limit; 632 } 633 634 limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) 635 : arm_spe_pmu_next_off(handle); 636 if (limit) 637 limit |= PMBLIMITR_EL1_E; 638 639 limit += (u64)buf->base; 640 base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); 641 write_sysreg_s(base, SYS_PMBPTR_EL1); 642 643 out_write_limit: 644 write_sysreg_s(limit, SYS_PMBLIMITR_EL1); 645 } 646 647 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) 648 { 649 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 650 u64 offset, size; 651 652 offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; 653 size = offset - PERF_IDX2OFF(handle->head, buf); 654 655 if (buf->snapshot) 656 handle->head = offset; 657 658 perf_aux_output_end(handle, size); 659 } 660 661 static void arm_spe_pmu_disable_and_drain_local(void) 662 { 663 /* Disable profiling at EL0 and EL1 */ 664 write_sysreg_s(0, SYS_PMSCR_EL1); 665 isb(); 666 667 /* Drain any buffered data */ 668 psb_csync(); 669 dsb(nsh); 670 671 /* Disable the profiling buffer */ 672 write_sysreg_s(0, SYS_PMBLIMITR_EL1); 673 isb(); 674 } 675 676 /* IRQ handling */ 677 static enum arm_spe_pmu_buf_fault_action 678 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) 679 { 680 const char *err_str; 681 u64 pmbsr; 682 enum arm_spe_pmu_buf_fault_action ret; 683 684 /* 685 * Ensure new profiling data is visible to the CPU and any external 686 * aborts have been resolved. 687 */ 688 psb_csync(); 689 dsb(nsh); 690 691 /* Ensure hardware updates to PMBPTR_EL1 are visible */ 692 isb(); 693 694 /* Service required? */ 695 pmbsr = read_sysreg_s(SYS_PMBSR_EL1); 696 if (!FIELD_GET(PMBSR_EL1_S, pmbsr)) 697 return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; 698 699 /* 700 * If we've lost data, disable profiling and also set the PARTIAL 701 * flag to indicate that the last record is corrupted. 702 */ 703 if (FIELD_GET(PMBSR_EL1_DL, pmbsr)) 704 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | 705 PERF_AUX_FLAG_PARTIAL); 706 707 /* Report collisions to userspace so that it can up the period */ 708 if (FIELD_GET(PMBSR_EL1_COLL, pmbsr)) 709 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); 710 711 /* We only expect buffer management events */ 712 switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) { 713 case PMBSR_EL1_EC_BUF: 714 /* Handled below */ 715 break; 716 case PMBSR_EL1_EC_FAULT_S1: 717 case PMBSR_EL1_EC_FAULT_S2: 718 err_str = "Unexpected buffer fault"; 719 goto out_err; 720 default: 721 err_str = "Unknown error code"; 722 goto out_err; 723 } 724 725 /* Buffer management event */ 726 switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) { 727 case PMBSR_EL1_BUF_BSC_FULL: 728 ret = SPE_PMU_BUF_FAULT_ACT_OK; 729 goto out_stop; 730 default: 731 err_str = "Unknown buffer status code"; 732 } 733 734 out_err: 735 pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", 736 err_str, smp_processor_id(), pmbsr, 737 read_sysreg_s(SYS_PMBPTR_EL1), 738 read_sysreg_s(SYS_PMBLIMITR_EL1)); 739 ret = SPE_PMU_BUF_FAULT_ACT_FATAL; 740 741 out_stop: 742 arm_spe_perf_aux_output_end(handle); 743 return ret; 744 } 745 746 static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) 747 { 748 struct perf_output_handle *handle = dev; 749 struct perf_event *event = handle->event; 750 enum arm_spe_pmu_buf_fault_action act; 751 752 if (!perf_get_aux(handle)) 753 return IRQ_NONE; 754 755 act = arm_spe_pmu_buf_get_fault_act(handle); 756 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 757 return IRQ_NONE; 758 759 /* 760 * Ensure perf callbacks have completed, which may disable the 761 * profiling buffer in response to a TRUNCATION flag. 762 */ 763 irq_work_run(); 764 765 switch (act) { 766 case SPE_PMU_BUF_FAULT_ACT_FATAL: 767 /* 768 * If a fatal exception occurred then leaving the profiling 769 * buffer enabled is a recipe waiting to happen. Since 770 * fatal faults don't always imply truncation, make sure 771 * that the profiling buffer is disabled explicitly before 772 * clearing the syndrome register. 773 */ 774 arm_spe_pmu_disable_and_drain_local(); 775 break; 776 case SPE_PMU_BUF_FAULT_ACT_OK: 777 /* 778 * We handled the fault (the buffer was full), so resume 779 * profiling as long as we didn't detect truncation. 780 * PMBPTR might be misaligned, but we'll burn that bridge 781 * when we get to it. 782 */ 783 if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { 784 arm_spe_perf_aux_output_begin(handle, event); 785 isb(); 786 } 787 break; 788 case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: 789 /* We've seen you before, but GCC has the memory of a sieve. */ 790 break; 791 } 792 793 /* The buffer pointers are now sane, so resume profiling. */ 794 write_sysreg_s(0, SYS_PMBSR_EL1); 795 return IRQ_HANDLED; 796 } 797 798 /* Perf callbacks */ 799 static int arm_spe_pmu_event_init(struct perf_event *event) 800 { 801 u64 reg; 802 struct perf_event_attr *attr = &event->attr; 803 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 804 805 /* This is, of course, deeply driver-specific */ 806 if (attr->type != event->pmu->type) 807 return -ENOENT; 808 809 if (event->cpu >= 0 && 810 !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) 811 return -ENOENT; 812 813 if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0) 814 return -EOPNOTSUPP; 815 816 if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) 817 return -EOPNOTSUPP; 818 819 if (arm_spe_event_to_pmsdsfr(event) != U64_MAX && 820 !(spe_pmu->features & SPE_PMU_FEAT_FDS)) 821 return -EOPNOTSUPP; 822 823 if (attr->exclude_idle) 824 return -EOPNOTSUPP; 825 826 /* 827 * Feedback-directed frequency throttling doesn't work when we 828 * have a buffer of samples. We'd need to manually count the 829 * samples in the buffer when it fills up and adjust the event 830 * count to reflect that. Instead, just force the user to specify 831 * a sample period. 832 */ 833 if (attr->freq) 834 return -EINVAL; 835 836 reg = arm_spe_event_to_pmsfcr(event); 837 if ((FIELD_GET(PMSFCR_EL1_FE, reg)) && 838 !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) 839 return -EOPNOTSUPP; 840 841 if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) && 842 !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) 843 return -EOPNOTSUPP; 844 845 if ((FIELD_GET(PMSFCR_EL1_FT, reg)) && 846 !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) 847 return -EOPNOTSUPP; 848 849 if ((FIELD_GET(PMSFCR_EL1_FL, reg)) && 850 !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) 851 return -EOPNOTSUPP; 852 853 if ((FIELD_GET(PMSFCR_EL1_LDm, reg) || 854 FIELD_GET(PMSFCR_EL1_STm, reg) || 855 FIELD_GET(PMSFCR_EL1_Bm, reg) || 856 FIELD_GET(PMSFCR_EL1_SIMD, reg) || 857 FIELD_GET(PMSFCR_EL1_SIMDm, reg) || 858 FIELD_GET(PMSFCR_EL1_FP, reg) || 859 FIELD_GET(PMSFCR_EL1_FPm, reg)) && 860 !(spe_pmu->features & SPE_PMU_FEAT_EFT)) 861 return -EOPNOTSUPP; 862 863 if (ATTR_CFG_GET_FLD(&event->attr, discard) && 864 !(spe_pmu->features & SPE_PMU_FEAT_DISCARD)) 865 return -EOPNOTSUPP; 866 867 set_spe_event_has_cx(event); 868 reg = arm_spe_event_to_pmscr(event); 869 if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT)) 870 return perf_allow_kernel(); 871 872 return 0; 873 } 874 875 static void arm_spe_pmu_start(struct perf_event *event, int flags) 876 { 877 u64 reg; 878 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 879 struct hw_perf_event *hwc = &event->hw; 880 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 881 882 hwc->state = 0; 883 arm_spe_perf_aux_output_begin(handle, event); 884 if (hwc->state) 885 return; 886 887 reg = arm_spe_event_to_pmsfcr(event); 888 write_sysreg_s(reg, SYS_PMSFCR_EL1); 889 890 reg = arm_spe_event_to_pmsevfr(event); 891 write_sysreg_s(reg, SYS_PMSEVFR_EL1); 892 893 if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) { 894 reg = arm_spe_event_to_pmsnevfr(event); 895 write_sysreg_s(reg, SYS_PMSNEVFR_EL1); 896 } 897 898 if (spe_pmu->features & SPE_PMU_FEAT_FDS) { 899 reg = arm_spe_event_to_pmsdsfr(event); 900 write_sysreg_s(reg, SYS_PMSDSFR_EL1); 901 } 902 903 reg = arm_spe_event_to_pmslatfr(event); 904 write_sysreg_s(reg, SYS_PMSLATFR_EL1); 905 906 if (flags & PERF_EF_RELOAD) { 907 reg = arm_spe_event_to_pmsirr(event); 908 write_sysreg_s(reg, SYS_PMSIRR_EL1); 909 isb(); 910 reg = local64_read(&hwc->period_left); 911 write_sysreg_s(reg, SYS_PMSICR_EL1); 912 } 913 914 reg = arm_spe_event_to_pmscr(event); 915 isb(); 916 write_sysreg_s(reg, SYS_PMSCR_EL1); 917 } 918 919 static void arm_spe_pmu_stop(struct perf_event *event, int flags) 920 { 921 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 922 struct hw_perf_event *hwc = &event->hw; 923 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 924 925 /* If we're already stopped, then nothing to do */ 926 if (hwc->state & PERF_HES_STOPPED) 927 return; 928 929 /* Stop all trace generation */ 930 arm_spe_pmu_disable_and_drain_local(); 931 932 if (flags & PERF_EF_UPDATE) { 933 /* 934 * If there's a fault pending then ensure we contain it 935 * to this buffer, since we might be on the context-switch 936 * path. 937 */ 938 if (perf_get_aux(handle)) { 939 enum arm_spe_pmu_buf_fault_action act; 940 941 act = arm_spe_pmu_buf_get_fault_act(handle); 942 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 943 arm_spe_perf_aux_output_end(handle); 944 else 945 write_sysreg_s(0, SYS_PMBSR_EL1); 946 } 947 948 /* 949 * This may also contain ECOUNT, but nobody else should 950 * be looking at period_left, since we forbid frequency 951 * based sampling. 952 */ 953 local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); 954 hwc->state |= PERF_HES_UPTODATE; 955 } 956 957 hwc->state |= PERF_HES_STOPPED; 958 } 959 960 static int arm_spe_pmu_add(struct perf_event *event, int flags) 961 { 962 int ret = 0; 963 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 964 struct hw_perf_event *hwc = &event->hw; 965 int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; 966 967 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 968 return -ENOENT; 969 970 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 971 972 if (flags & PERF_EF_START) { 973 arm_spe_pmu_start(event, PERF_EF_RELOAD); 974 if (hwc->state & PERF_HES_STOPPED) 975 ret = -EINVAL; 976 } 977 978 return ret; 979 } 980 981 static void arm_spe_pmu_del(struct perf_event *event, int flags) 982 { 983 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 984 } 985 986 static void arm_spe_pmu_read(struct perf_event *event) 987 { 988 } 989 990 static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, 991 int nr_pages, bool snapshot) 992 { 993 int i, cpu = event->cpu; 994 struct page **pglist; 995 struct arm_spe_pmu_buf *buf; 996 997 /* We need at least two pages for this to work. */ 998 if (nr_pages < 2) 999 return NULL; 1000 1001 /* 1002 * We require an even number of pages for snapshot mode, so that 1003 * we can effectively treat the buffer as consisting of two equal 1004 * parts and give userspace a fighting chance of getting some 1005 * useful data out of it. 1006 */ 1007 if (snapshot && (nr_pages & 1)) 1008 return NULL; 1009 1010 if (cpu == -1) 1011 cpu = raw_smp_processor_id(); 1012 1013 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); 1014 if (!buf) 1015 return NULL; 1016 1017 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); 1018 if (!pglist) 1019 goto out_free_buf; 1020 1021 for (i = 0; i < nr_pages; ++i) 1022 pglist[i] = virt_to_page(pages[i]); 1023 1024 buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); 1025 if (!buf->base) 1026 goto out_free_pglist; 1027 1028 buf->nr_pages = nr_pages; 1029 buf->snapshot = snapshot; 1030 1031 kfree(pglist); 1032 return buf; 1033 1034 out_free_pglist: 1035 kfree(pglist); 1036 out_free_buf: 1037 kfree(buf); 1038 return NULL; 1039 } 1040 1041 static void arm_spe_pmu_free_aux(void *aux) 1042 { 1043 struct arm_spe_pmu_buf *buf = aux; 1044 1045 vunmap(buf->base); 1046 kfree(buf); 1047 } 1048 1049 /* Initialisation and teardown functions */ 1050 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) 1051 { 1052 static atomic_t pmu_idx = ATOMIC_INIT(-1); 1053 1054 int idx; 1055 char *name; 1056 struct device *dev = &spe_pmu->pdev->dev; 1057 1058 spe_pmu->pmu = (struct pmu) { 1059 .module = THIS_MODULE, 1060 .parent = &spe_pmu->pdev->dev, 1061 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, 1062 .attr_groups = arm_spe_pmu_attr_groups, 1063 /* 1064 * We hitch a ride on the software context here, so that 1065 * we can support per-task profiling (which is not possible 1066 * with the invalid context as it doesn't get sched callbacks). 1067 * This requires that userspace either uses a dummy event for 1068 * perf_event_open, since the aux buffer is not setup until 1069 * a subsequent mmap, or creates the profiling event in a 1070 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it 1071 * once the buffer has been created. 1072 */ 1073 .task_ctx_nr = perf_sw_context, 1074 .event_init = arm_spe_pmu_event_init, 1075 .add = arm_spe_pmu_add, 1076 .del = arm_spe_pmu_del, 1077 .start = arm_spe_pmu_start, 1078 .stop = arm_spe_pmu_stop, 1079 .read = arm_spe_pmu_read, 1080 .setup_aux = arm_spe_pmu_setup_aux, 1081 .free_aux = arm_spe_pmu_free_aux, 1082 }; 1083 1084 idx = atomic_inc_return(&pmu_idx); 1085 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); 1086 if (!name) { 1087 dev_err(dev, "failed to allocate name for pmu %d\n", idx); 1088 return -ENOMEM; 1089 } 1090 1091 return perf_pmu_register(&spe_pmu->pmu, name, -1); 1092 } 1093 1094 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) 1095 { 1096 perf_pmu_unregister(&spe_pmu->pmu); 1097 } 1098 1099 static void __arm_spe_pmu_dev_probe(void *info) 1100 { 1101 int fld; 1102 u64 reg; 1103 struct arm_spe_pmu *spe_pmu = info; 1104 struct device *dev = &spe_pmu->pdev->dev; 1105 1106 fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), 1107 ID_AA64DFR0_EL1_PMSVer_SHIFT); 1108 if (!fld) { 1109 dev_err(dev, 1110 "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", 1111 fld, smp_processor_id()); 1112 return; 1113 } 1114 spe_pmu->pmsver = (u16)fld; 1115 1116 /* Read PMBIDR first to determine whether or not we have access */ 1117 reg = read_sysreg_s(SYS_PMBIDR_EL1); 1118 if (FIELD_GET(PMBIDR_EL1_P, reg)) { 1119 dev_err(dev, 1120 "profiling buffer owned by higher exception level\n"); 1121 return; 1122 } 1123 1124 /* Minimum alignment. If it's out-of-range, then fail the probe */ 1125 fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg); 1126 spe_pmu->align = 1 << fld; 1127 if (spe_pmu->align > SZ_2K) { 1128 dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", 1129 fld, smp_processor_id()); 1130 return; 1131 } 1132 1133 /* It's now safe to read PMSIDR and figure out what we've got */ 1134 reg = read_sysreg_s(SYS_PMSIDR_EL1); 1135 if (FIELD_GET(PMSIDR_EL1_FE, reg)) 1136 spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; 1137 1138 if (FIELD_GET(PMSIDR_EL1_FnE, reg)) 1139 spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT; 1140 1141 if (FIELD_GET(PMSIDR_EL1_FT, reg)) 1142 spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; 1143 1144 if (FIELD_GET(PMSIDR_EL1_FL, reg)) 1145 spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; 1146 1147 if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg)) 1148 spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; 1149 1150 if (FIELD_GET(PMSIDR_EL1_LDS, reg)) 1151 spe_pmu->features |= SPE_PMU_FEAT_LDS; 1152 1153 if (FIELD_GET(PMSIDR_EL1_ERND, reg)) 1154 spe_pmu->features |= SPE_PMU_FEAT_ERND; 1155 1156 if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2) 1157 spe_pmu->features |= SPE_PMU_FEAT_DISCARD; 1158 1159 if (FIELD_GET(PMSIDR_EL1_EFT, reg)) 1160 spe_pmu->features |= SPE_PMU_FEAT_EFT; 1161 1162 if (FIELD_GET(PMSIDR_EL1_FDS, reg)) 1163 spe_pmu->features |= SPE_PMU_FEAT_FDS; 1164 1165 /* This field has a spaced out encoding, so just use a look-up */ 1166 fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); 1167 switch (fld) { 1168 case PMSIDR_EL1_INTERVAL_256: 1169 spe_pmu->min_period = 256; 1170 break; 1171 case PMSIDR_EL1_INTERVAL_512: 1172 spe_pmu->min_period = 512; 1173 break; 1174 case PMSIDR_EL1_INTERVAL_768: 1175 spe_pmu->min_period = 768; 1176 break; 1177 case PMSIDR_EL1_INTERVAL_1024: 1178 spe_pmu->min_period = 1024; 1179 break; 1180 case PMSIDR_EL1_INTERVAL_1536: 1181 spe_pmu->min_period = 1536; 1182 break; 1183 case PMSIDR_EL1_INTERVAL_2048: 1184 spe_pmu->min_period = 2048; 1185 break; 1186 case PMSIDR_EL1_INTERVAL_3072: 1187 spe_pmu->min_period = 3072; 1188 break; 1189 default: 1190 dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", 1191 fld); 1192 fallthrough; 1193 case PMSIDR_EL1_INTERVAL_4096: 1194 spe_pmu->min_period = 4096; 1195 } 1196 1197 /* Maximum record size. If it's out-of-range, then fail the probe */ 1198 fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg); 1199 spe_pmu->max_record_sz = 1 << fld; 1200 if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { 1201 dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", 1202 fld, smp_processor_id()); 1203 return; 1204 } 1205 1206 fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg); 1207 switch (fld) { 1208 default: 1209 dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", 1210 fld); 1211 fallthrough; 1212 case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT: 1213 spe_pmu->counter_sz = 12; 1214 break; 1215 case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT: 1216 spe_pmu->counter_sz = 16; 1217 } 1218 1219 /* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */ 1220 write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1); 1221 spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1); 1222 1223 dev_info(dev, 1224 "probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", 1225 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus), 1226 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); 1227 1228 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; 1229 } 1230 1231 static void __arm_spe_pmu_reset_local(void) 1232 { 1233 /* 1234 * This is probably overkill, as we have no idea where we're 1235 * draining any buffered data to... 1236 */ 1237 arm_spe_pmu_disable_and_drain_local(); 1238 1239 /* Reset the buffer base pointer */ 1240 write_sysreg_s(0, SYS_PMBPTR_EL1); 1241 isb(); 1242 1243 /* Clear any pending management interrupts */ 1244 write_sysreg_s(0, SYS_PMBSR_EL1); 1245 isb(); 1246 } 1247 1248 static void __arm_spe_pmu_setup_one(void *info) 1249 { 1250 struct arm_spe_pmu *spe_pmu = info; 1251 1252 __arm_spe_pmu_reset_local(); 1253 enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); 1254 } 1255 1256 static void __arm_spe_pmu_stop_one(void *info) 1257 { 1258 struct arm_spe_pmu *spe_pmu = info; 1259 1260 disable_percpu_irq(spe_pmu->irq); 1261 __arm_spe_pmu_reset_local(); 1262 } 1263 1264 static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) 1265 { 1266 struct arm_spe_pmu *spe_pmu; 1267 1268 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1269 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1270 return 0; 1271 1272 __arm_spe_pmu_setup_one(spe_pmu); 1273 return 0; 1274 } 1275 1276 static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1277 { 1278 struct arm_spe_pmu *spe_pmu; 1279 1280 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1281 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1282 return 0; 1283 1284 __arm_spe_pmu_stop_one(spe_pmu); 1285 return 0; 1286 } 1287 1288 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) 1289 { 1290 int ret; 1291 cpumask_t *mask = &spe_pmu->supported_cpus; 1292 1293 /* Make sure we probe the hardware on a relevant CPU */ 1294 ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); 1295 if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) 1296 return -ENXIO; 1297 1298 /* Request our PPIs (note that the IRQ is still disabled) */ 1299 ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler, 1300 DRVNAME, mask, spe_pmu->handle); 1301 if (ret) 1302 return ret; 1303 1304 /* 1305 * Register our hotplug notifier now so we don't miss any events. 1306 * This will enable the IRQ for any supported CPUs that are already 1307 * up. 1308 */ 1309 ret = cpuhp_state_add_instance(arm_spe_pmu_online, 1310 &spe_pmu->hotplug_node); 1311 if (ret) 1312 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1313 1314 return ret; 1315 } 1316 1317 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) 1318 { 1319 cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); 1320 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1321 } 1322 1323 /* Driver and device probing */ 1324 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) 1325 { 1326 struct platform_device *pdev = spe_pmu->pdev; 1327 const struct cpumask *affinity; 1328 int irq; 1329 1330 irq = platform_get_irq_affinity(pdev, 0, &affinity); 1331 if (irq < 0) 1332 return -ENXIO; 1333 1334 if (!irq_is_percpu(irq)) { 1335 dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); 1336 return -EINVAL; 1337 } 1338 1339 cpumask_copy(&spe_pmu->supported_cpus, affinity); 1340 1341 spe_pmu->irq = irq; 1342 return 0; 1343 } 1344 1345 static const struct of_device_id arm_spe_pmu_of_match[] = { 1346 { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, 1347 { /* Sentinel */ }, 1348 }; 1349 MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); 1350 1351 static const struct platform_device_id arm_spe_match[] = { 1352 { ARMV8_SPE_PDEV_NAME, 0}, 1353 { } 1354 }; 1355 MODULE_DEVICE_TABLE(platform, arm_spe_match); 1356 1357 static int arm_spe_pmu_device_probe(struct platform_device *pdev) 1358 { 1359 int ret; 1360 struct arm_spe_pmu *spe_pmu; 1361 struct device *dev = &pdev->dev; 1362 1363 /* 1364 * If kernelspace is unmapped when running at EL0, then the SPE 1365 * buffer will fault and prematurely terminate the AUX session. 1366 */ 1367 if (arm64_kernel_unmapped_at_el0()) { 1368 dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); 1369 return -EPERM; 1370 } 1371 1372 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); 1373 if (!spe_pmu) 1374 return -ENOMEM; 1375 1376 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); 1377 if (!spe_pmu->handle) 1378 return -ENOMEM; 1379 1380 spe_pmu->pdev = pdev; 1381 platform_set_drvdata(pdev, spe_pmu); 1382 1383 ret = arm_spe_pmu_irq_probe(spe_pmu); 1384 if (ret) 1385 goto out_free_handle; 1386 1387 ret = arm_spe_pmu_dev_init(spe_pmu); 1388 if (ret) 1389 goto out_free_handle; 1390 1391 ret = arm_spe_pmu_perf_init(spe_pmu); 1392 if (ret) 1393 goto out_teardown_dev; 1394 1395 return 0; 1396 1397 out_teardown_dev: 1398 arm_spe_pmu_dev_teardown(spe_pmu); 1399 out_free_handle: 1400 free_percpu(spe_pmu->handle); 1401 return ret; 1402 } 1403 1404 static void arm_spe_pmu_device_remove(struct platform_device *pdev) 1405 { 1406 struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); 1407 1408 arm_spe_pmu_perf_destroy(spe_pmu); 1409 arm_spe_pmu_dev_teardown(spe_pmu); 1410 free_percpu(spe_pmu->handle); 1411 } 1412 1413 static struct platform_driver arm_spe_pmu_driver = { 1414 .id_table = arm_spe_match, 1415 .driver = { 1416 .name = DRVNAME, 1417 .of_match_table = of_match_ptr(arm_spe_pmu_of_match), 1418 .suppress_bind_attrs = true, 1419 }, 1420 .probe = arm_spe_pmu_device_probe, 1421 .remove = arm_spe_pmu_device_remove, 1422 }; 1423 1424 static int __init arm_spe_pmu_init(void) 1425 { 1426 int ret; 1427 1428 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, 1429 arm_spe_pmu_cpu_startup, 1430 arm_spe_pmu_cpu_teardown); 1431 if (ret < 0) 1432 return ret; 1433 arm_spe_pmu_online = ret; 1434 1435 ret = platform_driver_register(&arm_spe_pmu_driver); 1436 if (ret) 1437 cpuhp_remove_multi_state(arm_spe_pmu_online); 1438 1439 return ret; 1440 } 1441 1442 static void __exit arm_spe_pmu_exit(void) 1443 { 1444 platform_driver_unregister(&arm_spe_pmu_driver); 1445 cpuhp_remove_multi_state(arm_spe_pmu_online); 1446 } 1447 1448 module_init(arm_spe_pmu_init); 1449 module_exit(arm_spe_pmu_exit); 1450 1451 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); 1452 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1453 MODULE_LICENSE("GPL v2"); 1454