1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe PMU driver 4 * 5 * Copyright (C) 2021-2023 Alibaba Inc. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/cpumask.h> 12 #include <linux/device.h> 13 #include <linux/errno.h> 14 #include <linux/hrtimer.h> 15 #include <linux/kernel.h> 16 #include <linux/list.h> 17 #include <linux/pcie-dwc.h> 18 #include <linux/perf_event.h> 19 #include <linux/pci.h> 20 #include <linux/platform_device.h> 21 #include <linux/smp.h> 22 #include <linux/sysfs.h> 23 #include <linux/types.h> 24 25 #define DWC_PCIE_EVENT_CNT_CTL 0x8 26 27 /* 28 * Event Counter Data Select includes two parts: 29 * - 27-24: Group number(4-bit: 0..0x7) 30 * - 23-16: Event number(8-bit: 0..0x13) within the Group 31 * 32 * Put them together as in TRM. 33 */ 34 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) 35 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) 36 #define DWC_PCIE_CNT_STATUS BIT(7) 37 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) 38 #define DWC_PCIE_PER_EVENT_OFF 0x1 39 #define DWC_PCIE_PER_EVENT_ON 0x3 40 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) 41 #define DWC_PCIE_EVENT_PER_CLEAR 0x1 42 43 /* Event Selection Field has two subfields */ 44 #define DWC_PCIE_CNT_EVENT_SEL_GROUP GENMASK(11, 8) 45 #define DWC_PCIE_CNT_EVENT_SEL_EVID GENMASK(7, 0) 46 47 #define DWC_PCIE_EVENT_CNT_DATA 0xC 48 49 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 50 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) 51 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) 52 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 53 #define DWC_PCIE_DURATION_1MS 0x1 54 #define DWC_PCIE_DURATION_10MS 0x2 55 #define DWC_PCIE_DURATION_100MS 0x3 56 #define DWC_PCIE_DURATION_1S 0x4 57 #define DWC_PCIE_DURATION_2S 0x5 58 #define DWC_PCIE_DURATION_4S 0x6 59 #define DWC_PCIE_DURATION_4US 0xFF 60 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) 61 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 62 63 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 64 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 65 66 /* Event attributes */ 67 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) 68 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) 69 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) 70 71 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) 72 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) 73 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) 74 75 enum dwc_pcie_event_type { 76 DWC_PCIE_TIME_BASE_EVENT, 77 DWC_PCIE_LANE_EVENT, 78 DWC_PCIE_EVENT_TYPE_MAX, 79 }; 80 81 #define DWC_PCIE_LANE_GROUP_6 6 82 #define DWC_PCIE_LANE_GROUP_7 7 83 #define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256 84 85 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) 86 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) 87 #define DWC_PCIE_PMU_TIMER_PERIOD_NS (2 * NSEC_PER_SEC) 88 89 struct dwc_pcie_pmu { 90 struct pmu pmu; 91 struct pci_dev *pdev; /* Root Port device */ 92 u16 ras_des_offset; 93 u32 nr_lanes; 94 95 /* Groups #6 and #7 */ 96 DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); 97 struct perf_event *time_based_event; 98 bool timer_enable; 99 struct hrtimer hrtimer; 100 101 struct hlist_node cpuhp_node; 102 int on_cpu; 103 }; 104 105 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) 106 107 static int dwc_pcie_pmu_hp_state; 108 static struct list_head dwc_pcie_dev_info_head = 109 LIST_HEAD_INIT(dwc_pcie_dev_info_head); 110 static bool notify; 111 112 struct dwc_pcie_dev_info { 113 struct platform_device *plat_dev; 114 struct pci_dev *pdev; 115 struct list_head dev_node; 116 }; 117 118 static ssize_t cpumask_show(struct device *dev, 119 struct device_attribute *attr, 120 char *buf) 121 { 122 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); 123 124 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pcie_pmu->on_cpu))); 125 } 126 static DEVICE_ATTR_RO(cpumask); 127 128 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { 129 &dev_attr_cpumask.attr, 130 NULL 131 }; 132 133 static struct attribute_group dwc_pcie_cpumask_attr_group = { 134 .attrs = dwc_pcie_pmu_cpumask_attrs, 135 }; 136 137 struct dwc_pcie_format_attr { 138 struct device_attribute attr; 139 u64 field; 140 int config; 141 }; 142 143 PMU_FORMAT_ATTR(eventid, "config:0-15"); 144 PMU_FORMAT_ATTR(type, "config:16-19"); 145 PMU_FORMAT_ATTR(lane, "config:20-27"); 146 147 static struct attribute *dwc_pcie_format_attrs[] = { 148 &format_attr_type.attr, 149 &format_attr_eventid.attr, 150 &format_attr_lane.attr, 151 NULL, 152 }; 153 154 static struct attribute_group dwc_pcie_format_attrs_group = { 155 .name = "format", 156 .attrs = dwc_pcie_format_attrs, 157 }; 158 159 struct dwc_pcie_event_attr { 160 struct device_attribute attr; 161 enum dwc_pcie_event_type type; 162 u16 eventid; 163 u8 lane; 164 }; 165 166 static ssize_t dwc_pcie_event_show(struct device *dev, 167 struct device_attribute *attr, char *buf) 168 { 169 struct dwc_pcie_event_attr *eattr; 170 171 eattr = container_of(attr, typeof(*eattr), attr); 172 173 if (eattr->type == DWC_PCIE_LANE_EVENT) 174 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", 175 eattr->eventid, eattr->type); 176 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) 177 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", 178 eattr->eventid, eattr->type); 179 180 return 0; 181 } 182 183 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ 184 (&((struct dwc_pcie_event_attr[]) {{ \ 185 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ 186 .type = _type, \ 187 .eventid = _eventid, \ 188 .lane = _lane, \ 189 }})[0].attr.attr) 190 191 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ 192 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) 193 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ 194 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) 195 196 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { 197 /* Group #0 */ 198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), 199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), 200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), 201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), 202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), 203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), 204 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), 205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), 206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08), 207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09), 208 209 /* Group #1 */ 210 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20), 211 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21), 212 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22), 213 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23), 214 215 /* 216 * Leave it to the user to specify the lane ID to avoid generating 217 * a list of hundreds of events. 218 */ 219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), 220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), 221 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), 222 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), 223 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604), 224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605), 225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606), 226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), 227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), 228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), 229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), 230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), 231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), 232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), 233 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), 234 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), 235 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), 236 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), 237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), 238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), 239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), 240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), 241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), 242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), 243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), 244 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), 245 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), 246 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), 247 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), 248 NULL 249 }; 250 251 static const struct attribute_group dwc_pcie_event_attrs_group = { 252 .name = "events", 253 .attrs = dwc_pcie_pmu_time_event_attrs, 254 }; 255 256 static const struct attribute_group *dwc_pcie_attr_groups[] = { 257 &dwc_pcie_event_attrs_group, 258 &dwc_pcie_format_attrs_group, 259 &dwc_pcie_cpumask_attr_group, 260 NULL 261 }; 262 263 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, 264 struct perf_event *event, 265 bool enable) 266 { 267 struct pci_dev *pdev = pcie_pmu->pdev; 268 u16 ras_des_offset = pcie_pmu->ras_des_offset; 269 int event_id = DWC_PCIE_EVENT_ID(event); 270 int lane = DWC_PCIE_EVENT_LANE(event); 271 u32 ctrl; 272 273 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 274 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 275 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 276 277 if (enable) 278 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 279 else 280 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); 281 282 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 283 ctrl); 284 } 285 286 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, 287 bool enable) 288 { 289 struct pci_dev *pdev = pcie_pmu->pdev; 290 u16 ras_des_offset = pcie_pmu->ras_des_offset; 291 292 pci_clear_and_set_config_dword(pdev, 293 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, 294 DWC_PCIE_TIME_BASED_TIMER_START, enable); 295 } 296 297 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 298 { 299 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 300 struct pci_dev *pdev = pcie_pmu->pdev; 301 int event_id = DWC_PCIE_EVENT_ID(event); 302 int lane = DWC_PCIE_EVENT_LANE(event); 303 u16 ras_des_offset = pcie_pmu->ras_des_offset; 304 u32 val, ctrl; 305 306 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 307 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 308 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 309 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 310 ctrl); 311 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); 312 313 ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 314 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 315 ctrl); 316 317 return val; 318 } 319 320 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) 321 { 322 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 323 struct pci_dev *pdev = pcie_pmu->pdev; 324 int event_id = DWC_PCIE_EVENT_ID(event); 325 u16 ras_des_offset = pcie_pmu->ras_des_offset; 326 u32 lo, hi, ss; 327 u64 val; 328 329 /* 330 * The 64-bit value of the data counter is spread across two 331 * registers that are not synchronized. In order to read them 332 * atomically, ensure that the high 32 bits match before and after 333 * reading the low 32 bits. 334 */ 335 pci_read_config_dword(pdev, 336 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); 337 do { 338 /* snapshot the high 32 bits */ 339 ss = hi; 340 341 pci_read_config_dword( 342 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, 343 &lo); 344 pci_read_config_dword( 345 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, 346 &hi); 347 } while (hi != ss); 348 349 val = ((u64)hi << 32) | lo; 350 /* 351 * The Group#1 event measures the amount of data processed in 16-byte 352 * units. Simplify the end-user interface by multiplying the counter 353 * at the point of read. 354 */ 355 if (event_id >= 0x20 && event_id <= 0x23) 356 val *= 16; 357 358 return val; 359 } 360 361 static void dwc_pcie_pmu_reset_time_based_counter(struct perf_event *event) 362 { 363 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 364 struct hw_perf_event *hwc = &event->hw; 365 u64 prev; 366 367 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 368 369 /* 370 * The hardware counter is reset to zero when disabled. Synchronize 371 * prev_count so that the next event_update() computes the correct 372 * delta against the new counter baseline. 373 */ 374 do { 375 prev = local64_read(&hwc->prev_count); 376 } while (local64_cmpxchg(&hwc->prev_count, prev, 0) != prev); 377 378 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 379 } 380 381 static void dwc_pcie_pmu_event_update(struct perf_event *event) 382 { 383 struct hw_perf_event *hwc = &event->hw; 384 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 385 u64 delta, prev, now; 386 387 if (type == DWC_PCIE_LANE_EVENT) { 388 now = dwc_pcie_pmu_read_lane_event_counter(event) & 389 DWC_PCIE_LANE_EVENT_MAX_PERIOD; 390 local64_add(now, &event->count); 391 return; 392 } 393 394 do { 395 prev = local64_read(&hwc->prev_count); 396 now = dwc_pcie_pmu_read_time_based_counter(event); 397 398 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 399 400 delta = (now - prev) & DWC_PCIE_MAX_PERIOD; 401 local64_add(delta, &event->count); 402 } 403 404 static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event, 405 unsigned long val_lane_events[]) 406 { 407 int event_id, event_nr, group; 408 409 event_id = DWC_PCIE_EVENT_ID(event); 410 event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 411 group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id); 412 413 if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7) 414 return -EINVAL; 415 416 group -= DWC_PCIE_LANE_GROUP_6; 417 418 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 419 val_lane_events)) 420 return -EINVAL; 421 422 return 0; 423 } 424 425 static int dwc_pcie_pmu_validate_group(struct perf_event *event) 426 { 427 struct perf_event *sibling, *leader = event->group_leader; 428 DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); 429 bool time_event = false; 430 int type; 431 432 type = DWC_PCIE_EVENT_TYPE(leader); 433 if (type == DWC_PCIE_TIME_BASE_EVENT) 434 time_event = true; 435 else 436 if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events)) 437 return -ENOSPC; 438 439 for_each_sibling_event(sibling, leader) { 440 type = DWC_PCIE_EVENT_TYPE(sibling); 441 if (type == DWC_PCIE_TIME_BASE_EVENT) { 442 if (time_event) 443 return -ENOSPC; 444 445 time_event = true; 446 continue; 447 } 448 449 if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events)) 450 return -ENOSPC; 451 } 452 453 return 0; 454 } 455 456 static enum hrtimer_restart dwc_pcie_pmu_hrtimer_callback(struct hrtimer *hrtimer) 457 { 458 struct dwc_pcie_pmu *pcie_pmu = container_of(hrtimer, struct dwc_pcie_pmu, hrtimer); 459 struct perf_event *event = pcie_pmu->time_based_event; 460 struct hw_perf_event *hwc; 461 462 if (!event) 463 return HRTIMER_NORESTART; 464 465 hwc = &event->hw; 466 if (hwc->state & PERF_HES_STOPPED) 467 return HRTIMER_NORESTART; 468 469 dwc_pcie_pmu_event_update(event); 470 dwc_pcie_pmu_reset_time_based_counter(event); 471 hrtimer_forward_now(hrtimer, ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS)); 472 473 return HRTIMER_RESTART; 474 } 475 476 static int dwc_pcie_pmu_event_init(struct perf_event *event) 477 { 478 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 479 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 480 struct perf_event *sibling; 481 u32 lane; 482 483 if (event->attr.type != event->pmu->type) 484 return -ENOENT; 485 486 /* We don't support sampling */ 487 if (is_sampling_event(event)) 488 return -EINVAL; 489 490 /* We cannot support task bound events */ 491 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 492 return -EINVAL; 493 494 for_each_sibling_event(sibling, event->group_leader) { 495 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 496 return -EINVAL; 497 } 498 499 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) 500 return -EINVAL; 501 502 if (type == DWC_PCIE_LANE_EVENT) { 503 lane = DWC_PCIE_EVENT_LANE(event); 504 if (lane < 0 || lane >= pcie_pmu->nr_lanes) 505 return -EINVAL; 506 } 507 508 if (dwc_pcie_pmu_validate_group(event)) 509 return -ENOSPC; 510 511 event->cpu = pcie_pmu->on_cpu; 512 513 return 0; 514 } 515 516 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 517 { 518 struct hw_perf_event *hwc = &event->hw; 519 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 520 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 521 522 hwc->state = 0; 523 local64_set(&hwc->prev_count, 0); 524 525 if (type == DWC_PCIE_LANE_EVENT) { 526 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true); 527 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 528 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 529 if (pcie_pmu->timer_enable) 530 hrtimer_start(&pcie_pmu->hrtimer, 531 ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS), 532 HRTIMER_MODE_REL_PINNED_HARD); 533 } 534 } 535 536 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 537 { 538 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 539 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 540 struct hw_perf_event *hwc = &event->hw; 541 542 if (event->hw.state & PERF_HES_STOPPED) 543 return; 544 545 dwc_pcie_pmu_event_update(event); 546 547 if (type == DWC_PCIE_LANE_EVENT) { 548 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false); 549 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 550 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 551 552 if (pcie_pmu->timer_enable) 553 hrtimer_cancel(&pcie_pmu->hrtimer); 554 } 555 556 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 557 } 558 559 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 560 { 561 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 562 struct pci_dev *pdev = pcie_pmu->pdev; 563 struct hw_perf_event *hwc = &event->hw; 564 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 565 int event_id = DWC_PCIE_EVENT_ID(event); 566 int lane = DWC_PCIE_EVENT_LANE(event); 567 u16 ras_des_offset = pcie_pmu->ras_des_offset; 568 u32 ctrl; 569 570 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 571 572 if (type == DWC_PCIE_LANE_EVENT) { 573 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 574 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - 575 DWC_PCIE_LANE_GROUP_6; 576 577 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 578 pcie_pmu->lane_events)) 579 return -ENOSPC; 580 581 /* EVENT_COUNTER_DATA_REG needs clear manually */ 582 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 583 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 584 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | 585 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 586 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 587 ctrl); 588 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 589 if (pcie_pmu->time_based_event) 590 return -ENOSPC; 591 592 pcie_pmu->time_based_event = event; 593 594 /* 595 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely 596 * use it with any manually controlled duration. And it is 597 * cleared when next measurement starts. 598 */ 599 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | 600 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, 601 DWC_PCIE_DURATION_MANUAL_CTL) | 602 DWC_PCIE_TIME_BASED_CNT_ENABLE; 603 pci_write_config_dword( 604 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); 605 } 606 607 if (flags & PERF_EF_START) 608 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); 609 610 perf_event_update_userpage(event); 611 612 return 0; 613 } 614 615 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 616 { 617 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 618 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 619 620 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); 621 perf_event_update_userpage(event); 622 623 if (type == DWC_PCIE_TIME_BASE_EVENT) { 624 pcie_pmu->time_based_event = NULL; 625 } else { 626 int event_id = DWC_PCIE_EVENT_ID(event); 627 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 628 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - 629 DWC_PCIE_LANE_GROUP_6; 630 631 clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 632 pcie_pmu->lane_events); 633 } 634 } 635 636 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) 637 { 638 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); 639 } 640 641 /* 642 * Find the binded DES capability device info of a PCI device. 643 * @pdev: The PCI device. 644 */ 645 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) 646 { 647 struct dwc_pcie_dev_info *dev_info; 648 649 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) 650 if (dev_info->pdev == pdev) 651 return dev_info; 652 653 return NULL; 654 } 655 656 static void dwc_pcie_unregister_pmu(void *data) 657 { 658 struct dwc_pcie_pmu *pcie_pmu = data; 659 660 perf_pmu_unregister(&pcie_pmu->pmu); 661 } 662 663 static u16 dwc_pcie_des_cap(struct pci_dev *pdev) 664 { 665 const struct dwc_pcie_vsec_id *vid; 666 u16 vsec; 667 u32 val; 668 669 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) 670 return 0; 671 672 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) { 673 vsec = pci_find_vsec_capability(pdev, vid->vendor_id, 674 vid->vsec_id); 675 if (vsec) { 676 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, 677 &val); 678 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) { 679 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); 680 return vsec; 681 } 682 } 683 } 684 return 0; 685 } 686 687 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) 688 { 689 platform_device_unregister(dev_info->plat_dev); 690 list_del(&dev_info->dev_node); 691 kfree(dev_info); 692 } 693 694 static int dwc_pcie_register_dev(struct pci_dev *pdev) 695 { 696 struct platform_device *plat_dev; 697 struct dwc_pcie_dev_info *dev_info; 698 u32 sbdf; 699 700 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); 701 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0); 702 if (IS_ERR(plat_dev)) 703 return PTR_ERR(plat_dev); 704 705 dev_info = kzalloc_obj(*dev_info); 706 if (!dev_info) { 707 platform_device_unregister(plat_dev); 708 return -ENOMEM; 709 } 710 711 /* Cache platform device to handle pci device hotplug */ 712 dev_info->plat_dev = plat_dev; 713 dev_info->pdev = pdev; 714 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); 715 716 return 0; 717 } 718 719 static int dwc_pcie_pmu_notifier(struct notifier_block *nb, 720 unsigned long action, void *data) 721 { 722 struct device *dev = data; 723 struct pci_dev *pdev = to_pci_dev(dev); 724 struct dwc_pcie_dev_info *dev_info; 725 726 switch (action) { 727 case BUS_NOTIFY_ADD_DEVICE: 728 if (!dwc_pcie_des_cap(pdev)) 729 return NOTIFY_DONE; 730 if (dwc_pcie_register_dev(pdev)) 731 return NOTIFY_BAD; 732 break; 733 case BUS_NOTIFY_DEL_DEVICE: 734 dev_info = dwc_pcie_find_dev_info(pdev); 735 if (!dev_info) 736 return NOTIFY_DONE; 737 dwc_pcie_unregister_dev(dev_info); 738 break; 739 } 740 741 return NOTIFY_OK; 742 } 743 744 static struct notifier_block dwc_pcie_pmu_nb = { 745 .notifier_call = dwc_pcie_pmu_notifier, 746 }; 747 748 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) 749 { 750 struct pci_dev *pdev; 751 struct dwc_pcie_pmu *pcie_pmu; 752 char *name; 753 u32 sbdf; 754 u16 vsec; 755 int ret; 756 757 sbdf = plat_dev->id; 758 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff), 759 sbdf & 0xff); 760 if (!pdev) { 761 pr_err("No pdev found for the sbdf 0x%x\n", sbdf); 762 return -ENODEV; 763 } 764 765 vsec = dwc_pcie_des_cap(pdev); 766 if (!vsec) 767 return -ENODEV; 768 769 pci_dev_put(pdev); 770 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf); 771 if (!name) 772 return -ENOMEM; 773 774 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); 775 if (!pcie_pmu) 776 return -ENOMEM; 777 778 pcie_pmu->pdev = pdev; 779 pcie_pmu->ras_des_offset = vsec; 780 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); 781 pcie_pmu->on_cpu = -1; 782 hrtimer_setup(&pcie_pmu->hrtimer, dwc_pcie_pmu_hrtimer_callback, 783 CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED_HARD); 784 785 /* 786 * Use timer for updating time-based counts on platforms known 787 * to have narrowed counter. 788 */ 789 if (pdev->vendor == PCI_VENDOR_ID_PICOHEART) 790 pcie_pmu->timer_enable = true; 791 792 pcie_pmu->pmu = (struct pmu){ 793 .name = name, 794 .parent = &plat_dev->dev, 795 .module = THIS_MODULE, 796 .attr_groups = dwc_pcie_attr_groups, 797 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 798 .task_ctx_nr = perf_invalid_context, 799 .event_init = dwc_pcie_pmu_event_init, 800 .add = dwc_pcie_pmu_event_add, 801 .del = dwc_pcie_pmu_event_del, 802 .start = dwc_pcie_pmu_event_start, 803 .stop = dwc_pcie_pmu_event_stop, 804 .read = dwc_pcie_pmu_event_update, 805 }; 806 807 /* Add this instance to the list used by the offline callback */ 808 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, 809 &pcie_pmu->cpuhp_node); 810 if (ret) { 811 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf); 812 return ret; 813 } 814 815 /* Unwind when platform driver removes */ 816 ret = devm_add_action_or_reset(&plat_dev->dev, 817 dwc_pcie_pmu_remove_cpuhp_instance, 818 &pcie_pmu->cpuhp_node); 819 if (ret) 820 return ret; 821 822 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); 823 if (ret) { 824 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf); 825 return ret; 826 } 827 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, 828 pcie_pmu); 829 if (ret) 830 return ret; 831 832 return 0; 833 } 834 835 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 836 { 837 struct dwc_pcie_pmu *pcie_pmu; 838 839 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 840 if (pcie_pmu->on_cpu == -1) 841 pcie_pmu->on_cpu = cpumask_local_spread( 842 0, dev_to_node(&pcie_pmu->pdev->dev)); 843 844 return 0; 845 } 846 847 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 848 { 849 struct dwc_pcie_pmu *pcie_pmu; 850 struct pci_dev *pdev; 851 unsigned int target; 852 int node; 853 854 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 855 /* Nothing to do if this CPU doesn't own the PMU */ 856 if (cpu != pcie_pmu->on_cpu) 857 return 0; 858 859 pcie_pmu->on_cpu = -1; 860 pdev = pcie_pmu->pdev; 861 node = dev_to_node(&pdev->dev); 862 863 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 864 if (target >= nr_cpu_ids) 865 target = cpumask_any_but(cpu_online_mask, cpu); 866 867 if (target >= nr_cpu_ids) { 868 pci_err(pdev, "There is no CPU to set\n"); 869 return 0; 870 } 871 872 /* This PMU does NOT support interrupt, just migrate context. */ 873 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); 874 pcie_pmu->on_cpu = target; 875 876 return 0; 877 } 878 879 static struct platform_driver dwc_pcie_pmu_driver = { 880 .probe = dwc_pcie_pmu_probe, 881 .driver = {.name = "dwc_pcie_pmu",}, 882 }; 883 884 static void dwc_pcie_cleanup_devices(void) 885 { 886 struct dwc_pcie_dev_info *dev_info, *tmp; 887 888 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) { 889 dwc_pcie_unregister_dev(dev_info); 890 } 891 } 892 893 static int __init dwc_pcie_pmu_init(void) 894 { 895 struct pci_dev *pdev = NULL; 896 int ret; 897 898 for_each_pci_dev(pdev) { 899 if (!dwc_pcie_des_cap(pdev)) 900 continue; 901 902 ret = dwc_pcie_register_dev(pdev); 903 if (ret) { 904 pci_dev_put(pdev); 905 goto err_cleanup; 906 } 907 } 908 909 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 910 "perf/dwc_pcie_pmu:online", 911 dwc_pcie_pmu_online_cpu, 912 dwc_pcie_pmu_offline_cpu); 913 if (ret < 0) 914 goto err_cleanup; 915 916 dwc_pcie_pmu_hp_state = ret; 917 918 ret = platform_driver_register(&dwc_pcie_pmu_driver); 919 if (ret) 920 goto err_remove_cpuhp; 921 922 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 923 if (ret) 924 goto err_unregister_driver; 925 notify = true; 926 927 return 0; 928 929 err_unregister_driver: 930 platform_driver_unregister(&dwc_pcie_pmu_driver); 931 err_remove_cpuhp: 932 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 933 err_cleanup: 934 dwc_pcie_cleanup_devices(); 935 return ret; 936 } 937 938 static void __exit dwc_pcie_pmu_exit(void) 939 { 940 if (notify) 941 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 942 dwc_pcie_cleanup_devices(); 943 platform_driver_unregister(&dwc_pcie_pmu_driver); 944 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 945 } 946 947 module_init(dwc_pcie_pmu_init); 948 module_exit(dwc_pcie_pmu_exit); 949 950 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); 951 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 952 MODULE_LICENSE("GPL v2"); 953