1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe PMU driver 4 * 5 * Copyright (C) 2021-2023 Alibaba Inc. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/cpumask.h> 12 #include <linux/device.h> 13 #include <linux/errno.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/perf_event.h> 17 #include <linux/pci.h> 18 #include <linux/platform_device.h> 19 #include <linux/smp.h> 20 #include <linux/sysfs.h> 21 #include <linux/types.h> 22 23 #define DWC_PCIE_EVENT_CNT_CTL 0x8 24 25 /* 26 * Event Counter Data Select includes two parts: 27 * - 27-24: Group number(4-bit: 0..0x7) 28 * - 23-16: Event number(8-bit: 0..0x13) within the Group 29 * 30 * Put them together as in TRM. 31 */ 32 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) 33 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) 34 #define DWC_PCIE_CNT_STATUS BIT(7) 35 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) 36 #define DWC_PCIE_PER_EVENT_OFF 0x1 37 #define DWC_PCIE_PER_EVENT_ON 0x3 38 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) 39 #define DWC_PCIE_EVENT_PER_CLEAR 0x1 40 41 #define DWC_PCIE_EVENT_CNT_DATA 0xC 42 43 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 44 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) 45 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) 46 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 47 #define DWC_PCIE_DURATION_1MS 0x1 48 #define DWC_PCIE_DURATION_10MS 0x2 49 #define DWC_PCIE_DURATION_100MS 0x3 50 #define DWC_PCIE_DURATION_1S 0x4 51 #define DWC_PCIE_DURATION_2S 0x5 52 #define DWC_PCIE_DURATION_4S 0x6 53 #define DWC_PCIE_DURATION_4US 0xFF 54 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) 55 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 56 57 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 58 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 59 60 /* Event attributes */ 61 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) 62 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) 63 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) 64 65 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) 66 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) 67 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) 68 69 enum dwc_pcie_event_type { 70 DWC_PCIE_TIME_BASE_EVENT, 71 DWC_PCIE_LANE_EVENT, 72 DWC_PCIE_EVENT_TYPE_MAX, 73 }; 74 75 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) 76 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) 77 78 struct dwc_pcie_pmu { 79 struct pmu pmu; 80 struct pci_dev *pdev; /* Root Port device */ 81 u16 ras_des_offset; 82 u32 nr_lanes; 83 84 struct hlist_node cpuhp_node; 85 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX]; 86 int on_cpu; 87 }; 88 89 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) 90 91 static int dwc_pcie_pmu_hp_state; 92 static struct list_head dwc_pcie_dev_info_head = 93 LIST_HEAD_INIT(dwc_pcie_dev_info_head); 94 static bool notify; 95 96 struct dwc_pcie_dev_info { 97 struct platform_device *plat_dev; 98 struct pci_dev *pdev; 99 struct list_head dev_node; 100 }; 101 102 struct dwc_pcie_pmu_vsec_id { 103 u16 vendor_id; 104 u16 vsec_id; 105 u8 vsec_rev; 106 }; 107 108 /* 109 * VSEC IDs are allocated by the vendor, so a given ID may mean different 110 * things to different vendors. See PCIe r6.0, sec 7.9.5.2. 111 */ 112 static const struct dwc_pcie_pmu_vsec_id dwc_pcie_pmu_vsec_ids[] = { 113 { .vendor_id = PCI_VENDOR_ID_ALIBABA, 114 .vsec_id = 0x02, .vsec_rev = 0x4 }, 115 { .vendor_id = PCI_VENDOR_ID_AMPERE, 116 .vsec_id = 0x02, .vsec_rev = 0x4 }, 117 { .vendor_id = PCI_VENDOR_ID_QCOM, 118 .vsec_id = 0x02, .vsec_rev = 0x4 }, 119 {} /* terminator */ 120 }; 121 122 static ssize_t cpumask_show(struct device *dev, 123 struct device_attribute *attr, 124 char *buf) 125 { 126 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); 127 128 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); 129 } 130 static DEVICE_ATTR_RO(cpumask); 131 132 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { 133 &dev_attr_cpumask.attr, 134 NULL 135 }; 136 137 static struct attribute_group dwc_pcie_cpumask_attr_group = { 138 .attrs = dwc_pcie_pmu_cpumask_attrs, 139 }; 140 141 struct dwc_pcie_format_attr { 142 struct device_attribute attr; 143 u64 field; 144 int config; 145 }; 146 147 PMU_FORMAT_ATTR(eventid, "config:0-15"); 148 PMU_FORMAT_ATTR(type, "config:16-19"); 149 PMU_FORMAT_ATTR(lane, "config:20-27"); 150 151 static struct attribute *dwc_pcie_format_attrs[] = { 152 &format_attr_type.attr, 153 &format_attr_eventid.attr, 154 &format_attr_lane.attr, 155 NULL, 156 }; 157 158 static struct attribute_group dwc_pcie_format_attrs_group = { 159 .name = "format", 160 .attrs = dwc_pcie_format_attrs, 161 }; 162 163 struct dwc_pcie_event_attr { 164 struct device_attribute attr; 165 enum dwc_pcie_event_type type; 166 u16 eventid; 167 u8 lane; 168 }; 169 170 static ssize_t dwc_pcie_event_show(struct device *dev, 171 struct device_attribute *attr, char *buf) 172 { 173 struct dwc_pcie_event_attr *eattr; 174 175 eattr = container_of(attr, typeof(*eattr), attr); 176 177 if (eattr->type == DWC_PCIE_LANE_EVENT) 178 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", 179 eattr->eventid, eattr->type); 180 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) 181 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", 182 eattr->eventid, eattr->type); 183 184 return 0; 185 } 186 187 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ 188 (&((struct dwc_pcie_event_attr[]) {{ \ 189 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ 190 .type = _type, \ 191 .eventid = _eventid, \ 192 .lane = _lane, \ 193 }})[0].attr.attr) 194 195 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ 196 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) 197 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ 198 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) 199 200 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { 201 /* Group #0 */ 202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), 203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), 204 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), 205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), 206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), 207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), 208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), 209 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), 210 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08), 211 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09), 212 213 /* Group #1 */ 214 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20), 215 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21), 216 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22), 217 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23), 218 219 /* 220 * Leave it to the user to specify the lane ID to avoid generating 221 * a list of hundreds of events. 222 */ 223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), 224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), 225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), 226 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), 227 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604), 228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605), 229 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606), 230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), 231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), 232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), 233 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), 234 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), 235 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), 236 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), 237 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), 238 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), 239 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), 240 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), 241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), 242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), 243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), 244 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), 245 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), 246 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), 247 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), 248 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), 249 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), 250 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), 251 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), 252 NULL 253 }; 254 255 static const struct attribute_group dwc_pcie_event_attrs_group = { 256 .name = "events", 257 .attrs = dwc_pcie_pmu_time_event_attrs, 258 }; 259 260 static const struct attribute_group *dwc_pcie_attr_groups[] = { 261 &dwc_pcie_event_attrs_group, 262 &dwc_pcie_format_attrs_group, 263 &dwc_pcie_cpumask_attr_group, 264 NULL 265 }; 266 267 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, 268 bool enable) 269 { 270 struct pci_dev *pdev = pcie_pmu->pdev; 271 u16 ras_des_offset = pcie_pmu->ras_des_offset; 272 273 if (enable) 274 pci_clear_and_set_config_dword(pdev, 275 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 276 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 277 else 278 pci_clear_and_set_config_dword(pdev, 279 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 280 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); 281 } 282 283 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, 284 bool enable) 285 { 286 struct pci_dev *pdev = pcie_pmu->pdev; 287 u16 ras_des_offset = pcie_pmu->ras_des_offset; 288 289 pci_clear_and_set_config_dword(pdev, 290 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, 291 DWC_PCIE_TIME_BASED_TIMER_START, enable); 292 } 293 294 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 295 { 296 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 297 struct pci_dev *pdev = pcie_pmu->pdev; 298 u16 ras_des_offset = pcie_pmu->ras_des_offset; 299 u32 val; 300 301 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); 302 303 return val; 304 } 305 306 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) 307 { 308 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 309 struct pci_dev *pdev = pcie_pmu->pdev; 310 int event_id = DWC_PCIE_EVENT_ID(event); 311 u16 ras_des_offset = pcie_pmu->ras_des_offset; 312 u32 lo, hi, ss; 313 u64 val; 314 315 /* 316 * The 64-bit value of the data counter is spread across two 317 * registers that are not synchronized. In order to read them 318 * atomically, ensure that the high 32 bits match before and after 319 * reading the low 32 bits. 320 */ 321 pci_read_config_dword(pdev, 322 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); 323 do { 324 /* snapshot the high 32 bits */ 325 ss = hi; 326 327 pci_read_config_dword( 328 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, 329 &lo); 330 pci_read_config_dword( 331 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, 332 &hi); 333 } while (hi != ss); 334 335 val = ((u64)hi << 32) | lo; 336 /* 337 * The Group#1 event measures the amount of data processed in 16-byte 338 * units. Simplify the end-user interface by multiplying the counter 339 * at the point of read. 340 */ 341 if (event_id >= 0x20 && event_id <= 0x23) 342 val *= 16; 343 344 return val; 345 } 346 347 static void dwc_pcie_pmu_event_update(struct perf_event *event) 348 { 349 struct hw_perf_event *hwc = &event->hw; 350 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 351 u64 delta, prev, now = 0; 352 353 do { 354 prev = local64_read(&hwc->prev_count); 355 356 if (type == DWC_PCIE_LANE_EVENT) 357 now = dwc_pcie_pmu_read_lane_event_counter(event); 358 else if (type == DWC_PCIE_TIME_BASE_EVENT) 359 now = dwc_pcie_pmu_read_time_based_counter(event); 360 361 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 362 363 delta = (now - prev) & DWC_PCIE_MAX_PERIOD; 364 /* 32-bit counter for Lane Event Counting */ 365 if (type == DWC_PCIE_LANE_EVENT) 366 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD; 367 368 local64_add(delta, &event->count); 369 } 370 371 static int dwc_pcie_pmu_event_init(struct perf_event *event) 372 { 373 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 374 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 375 struct perf_event *sibling; 376 u32 lane; 377 378 if (event->attr.type != event->pmu->type) 379 return -ENOENT; 380 381 /* We don't support sampling */ 382 if (is_sampling_event(event)) 383 return -EINVAL; 384 385 /* We cannot support task bound events */ 386 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 387 return -EINVAL; 388 389 if (event->group_leader != event && 390 !is_software_event(event->group_leader)) 391 return -EINVAL; 392 393 for_each_sibling_event(sibling, event->group_leader) { 394 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 395 return -EINVAL; 396 } 397 398 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) 399 return -EINVAL; 400 401 if (type == DWC_PCIE_LANE_EVENT) { 402 lane = DWC_PCIE_EVENT_LANE(event); 403 if (lane < 0 || lane >= pcie_pmu->nr_lanes) 404 return -EINVAL; 405 } 406 407 event->cpu = pcie_pmu->on_cpu; 408 409 return 0; 410 } 411 412 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 413 { 414 struct hw_perf_event *hwc = &event->hw; 415 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 416 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 417 418 hwc->state = 0; 419 local64_set(&hwc->prev_count, 0); 420 421 if (type == DWC_PCIE_LANE_EVENT) 422 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true); 423 else if (type == DWC_PCIE_TIME_BASE_EVENT) 424 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 425 } 426 427 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 428 { 429 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 430 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 431 struct hw_perf_event *hwc = &event->hw; 432 433 if (event->hw.state & PERF_HES_STOPPED) 434 return; 435 436 if (type == DWC_PCIE_LANE_EVENT) 437 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false); 438 else if (type == DWC_PCIE_TIME_BASE_EVENT) 439 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 440 441 dwc_pcie_pmu_event_update(event); 442 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 443 } 444 445 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 446 { 447 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 448 struct pci_dev *pdev = pcie_pmu->pdev; 449 struct hw_perf_event *hwc = &event->hw; 450 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 451 int event_id = DWC_PCIE_EVENT_ID(event); 452 int lane = DWC_PCIE_EVENT_LANE(event); 453 u16 ras_des_offset = pcie_pmu->ras_des_offset; 454 u32 ctrl; 455 456 /* one counter for each type and it is in use */ 457 if (pcie_pmu->event[type]) 458 return -ENOSPC; 459 460 pcie_pmu->event[type] = event; 461 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 462 463 if (type == DWC_PCIE_LANE_EVENT) { 464 /* EVENT_COUNTER_DATA_REG needs clear manually */ 465 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 466 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 467 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | 468 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 469 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 470 ctrl); 471 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 472 /* 473 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely 474 * use it with any manually controlled duration. And it is 475 * cleared when next measurement starts. 476 */ 477 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | 478 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, 479 DWC_PCIE_DURATION_MANUAL_CTL) | 480 DWC_PCIE_TIME_BASED_CNT_ENABLE; 481 pci_write_config_dword( 482 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); 483 } 484 485 if (flags & PERF_EF_START) 486 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); 487 488 perf_event_update_userpage(event); 489 490 return 0; 491 } 492 493 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 494 { 495 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 496 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 497 498 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); 499 perf_event_update_userpage(event); 500 pcie_pmu->event[type] = NULL; 501 } 502 503 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) 504 { 505 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); 506 } 507 508 /* 509 * Find the binded DES capability device info of a PCI device. 510 * @pdev: The PCI device. 511 */ 512 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) 513 { 514 struct dwc_pcie_dev_info *dev_info; 515 516 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) 517 if (dev_info->pdev == pdev) 518 return dev_info; 519 520 return NULL; 521 } 522 523 static void dwc_pcie_unregister_pmu(void *data) 524 { 525 struct dwc_pcie_pmu *pcie_pmu = data; 526 527 perf_pmu_unregister(&pcie_pmu->pmu); 528 } 529 530 static u16 dwc_pcie_des_cap(struct pci_dev *pdev) 531 { 532 const struct dwc_pcie_pmu_vsec_id *vid; 533 u16 vsec; 534 u32 val; 535 536 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) 537 return 0; 538 539 for (vid = dwc_pcie_pmu_vsec_ids; vid->vendor_id; vid++) { 540 vsec = pci_find_vsec_capability(pdev, vid->vendor_id, 541 vid->vsec_id); 542 if (vsec) { 543 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, 544 &val); 545 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) { 546 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); 547 return vsec; 548 } 549 } 550 } 551 return 0; 552 } 553 554 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) 555 { 556 platform_device_unregister(dev_info->plat_dev); 557 list_del(&dev_info->dev_node); 558 kfree(dev_info); 559 } 560 561 static int dwc_pcie_register_dev(struct pci_dev *pdev) 562 { 563 struct platform_device *plat_dev; 564 struct dwc_pcie_dev_info *dev_info; 565 u32 sbdf; 566 567 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); 568 plat_dev = platform_device_register_data(NULL, "dwc_pcie_pmu", sbdf, 569 pdev, sizeof(*pdev)); 570 571 if (IS_ERR(plat_dev)) 572 return PTR_ERR(plat_dev); 573 574 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL); 575 if (!dev_info) 576 return -ENOMEM; 577 578 /* Cache platform device to handle pci device hotplug */ 579 dev_info->plat_dev = plat_dev; 580 dev_info->pdev = pdev; 581 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); 582 583 return 0; 584 } 585 586 static int dwc_pcie_pmu_notifier(struct notifier_block *nb, 587 unsigned long action, void *data) 588 { 589 struct device *dev = data; 590 struct pci_dev *pdev = to_pci_dev(dev); 591 struct dwc_pcie_dev_info *dev_info; 592 593 switch (action) { 594 case BUS_NOTIFY_ADD_DEVICE: 595 if (!dwc_pcie_des_cap(pdev)) 596 return NOTIFY_DONE; 597 if (dwc_pcie_register_dev(pdev)) 598 return NOTIFY_BAD; 599 break; 600 case BUS_NOTIFY_DEL_DEVICE: 601 dev_info = dwc_pcie_find_dev_info(pdev); 602 if (!dev_info) 603 return NOTIFY_DONE; 604 dwc_pcie_unregister_dev(dev_info); 605 break; 606 } 607 608 return NOTIFY_OK; 609 } 610 611 static struct notifier_block dwc_pcie_pmu_nb = { 612 .notifier_call = dwc_pcie_pmu_notifier, 613 }; 614 615 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) 616 { 617 struct pci_dev *pdev = plat_dev->dev.platform_data; 618 struct dwc_pcie_pmu *pcie_pmu; 619 char *name; 620 u32 sbdf; 621 u16 vsec; 622 int ret; 623 624 vsec = dwc_pcie_des_cap(pdev); 625 if (!vsec) 626 return -ENODEV; 627 628 sbdf = plat_dev->id; 629 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf); 630 if (!name) 631 return -ENOMEM; 632 633 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); 634 if (!pcie_pmu) 635 return -ENOMEM; 636 637 pcie_pmu->pdev = pdev; 638 pcie_pmu->ras_des_offset = vsec; 639 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); 640 pcie_pmu->on_cpu = -1; 641 pcie_pmu->pmu = (struct pmu){ 642 .name = name, 643 .parent = &pdev->dev, 644 .module = THIS_MODULE, 645 .attr_groups = dwc_pcie_attr_groups, 646 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 647 .task_ctx_nr = perf_invalid_context, 648 .event_init = dwc_pcie_pmu_event_init, 649 .add = dwc_pcie_pmu_event_add, 650 .del = dwc_pcie_pmu_event_del, 651 .start = dwc_pcie_pmu_event_start, 652 .stop = dwc_pcie_pmu_event_stop, 653 .read = dwc_pcie_pmu_event_update, 654 }; 655 656 /* Add this instance to the list used by the offline callback */ 657 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, 658 &pcie_pmu->cpuhp_node); 659 if (ret) { 660 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf); 661 return ret; 662 } 663 664 /* Unwind when platform driver removes */ 665 ret = devm_add_action_or_reset(&plat_dev->dev, 666 dwc_pcie_pmu_remove_cpuhp_instance, 667 &pcie_pmu->cpuhp_node); 668 if (ret) 669 return ret; 670 671 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); 672 if (ret) { 673 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf); 674 return ret; 675 } 676 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, 677 pcie_pmu); 678 if (ret) 679 return ret; 680 681 return 0; 682 } 683 684 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 685 { 686 struct dwc_pcie_pmu *pcie_pmu; 687 688 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 689 if (pcie_pmu->on_cpu == -1) 690 pcie_pmu->on_cpu = cpumask_local_spread( 691 0, dev_to_node(&pcie_pmu->pdev->dev)); 692 693 return 0; 694 } 695 696 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 697 { 698 struct dwc_pcie_pmu *pcie_pmu; 699 struct pci_dev *pdev; 700 unsigned int target; 701 int node; 702 703 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 704 /* Nothing to do if this CPU doesn't own the PMU */ 705 if (cpu != pcie_pmu->on_cpu) 706 return 0; 707 708 pcie_pmu->on_cpu = -1; 709 pdev = pcie_pmu->pdev; 710 node = dev_to_node(&pdev->dev); 711 712 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 713 if (target >= nr_cpu_ids) 714 target = cpumask_any_but(cpu_online_mask, cpu); 715 716 if (target >= nr_cpu_ids) { 717 pci_err(pdev, "There is no CPU to set\n"); 718 return 0; 719 } 720 721 /* This PMU does NOT support interrupt, just migrate context. */ 722 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); 723 pcie_pmu->on_cpu = target; 724 725 return 0; 726 } 727 728 static struct platform_driver dwc_pcie_pmu_driver = { 729 .probe = dwc_pcie_pmu_probe, 730 .driver = {.name = "dwc_pcie_pmu",}, 731 }; 732 733 static int __init dwc_pcie_pmu_init(void) 734 { 735 struct pci_dev *pdev = NULL; 736 int ret; 737 738 for_each_pci_dev(pdev) { 739 if (!dwc_pcie_des_cap(pdev)) 740 continue; 741 742 ret = dwc_pcie_register_dev(pdev); 743 if (ret) { 744 pci_dev_put(pdev); 745 return ret; 746 } 747 } 748 749 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 750 "perf/dwc_pcie_pmu:online", 751 dwc_pcie_pmu_online_cpu, 752 dwc_pcie_pmu_offline_cpu); 753 if (ret < 0) 754 return ret; 755 756 dwc_pcie_pmu_hp_state = ret; 757 758 ret = platform_driver_register(&dwc_pcie_pmu_driver); 759 if (ret) 760 goto platform_driver_register_err; 761 762 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 763 if (ret) 764 goto platform_driver_register_err; 765 notify = true; 766 767 return 0; 768 769 platform_driver_register_err: 770 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 771 772 return ret; 773 } 774 775 static void __exit dwc_pcie_pmu_exit(void) 776 { 777 struct dwc_pcie_dev_info *dev_info, *tmp; 778 779 if (notify) 780 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 781 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) 782 dwc_pcie_unregister_dev(dev_info); 783 platform_driver_unregister(&dwc_pcie_pmu_driver); 784 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 785 } 786 787 module_init(dwc_pcie_pmu_init); 788 module_exit(dwc_pcie_pmu_exit); 789 790 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); 791 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 792 MODULE_LICENSE("GPL v2"); 793