1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe PMU driver 4 * 5 * Copyright (C) 2021-2023 Alibaba Inc. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/cpumask.h> 12 #include <linux/device.h> 13 #include <linux/errno.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/perf_event.h> 17 #include <linux/pci.h> 18 #include <linux/platform_device.h> 19 #include <linux/smp.h> 20 #include <linux/sysfs.h> 21 #include <linux/types.h> 22 23 #define DWC_PCIE_VSEC_RAS_DES_ID 0x02 24 #define DWC_PCIE_EVENT_CNT_CTL 0x8 25 26 /* 27 * Event Counter Data Select includes two parts: 28 * - 27-24: Group number(4-bit: 0..0x7) 29 * - 23-16: Event number(8-bit: 0..0x13) within the Group 30 * 31 * Put them together as in TRM. 32 */ 33 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) 34 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) 35 #define DWC_PCIE_CNT_STATUS BIT(7) 36 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) 37 #define DWC_PCIE_PER_EVENT_OFF 0x1 38 #define DWC_PCIE_PER_EVENT_ON 0x3 39 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) 40 #define DWC_PCIE_EVENT_PER_CLEAR 0x1 41 42 #define DWC_PCIE_EVENT_CNT_DATA 0xC 43 44 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 45 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) 46 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) 47 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 48 #define DWC_PCIE_DURATION_1MS 0x1 49 #define DWC_PCIE_DURATION_10MS 0x2 50 #define DWC_PCIE_DURATION_100MS 0x3 51 #define DWC_PCIE_DURATION_1S 0x4 52 #define DWC_PCIE_DURATION_2S 0x5 53 #define DWC_PCIE_DURATION_4S 0x6 54 #define DWC_PCIE_DURATION_4US 0xFF 55 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) 56 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 57 58 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 59 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 60 61 /* Event attributes */ 62 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) 63 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) 64 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) 65 66 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) 67 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) 68 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) 69 70 enum dwc_pcie_event_type { 71 DWC_PCIE_TIME_BASE_EVENT, 72 DWC_PCIE_LANE_EVENT, 73 DWC_PCIE_EVENT_TYPE_MAX, 74 }; 75 76 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) 77 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) 78 79 struct dwc_pcie_pmu { 80 struct pmu pmu; 81 struct pci_dev *pdev; /* Root Port device */ 82 u16 ras_des_offset; 83 u32 nr_lanes; 84 85 struct list_head pmu_node; 86 struct hlist_node cpuhp_node; 87 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX]; 88 int on_cpu; 89 }; 90 91 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) 92 93 static int dwc_pcie_pmu_hp_state; 94 static struct list_head dwc_pcie_dev_info_head = 95 LIST_HEAD_INIT(dwc_pcie_dev_info_head); 96 static bool notify; 97 98 struct dwc_pcie_dev_info { 99 struct platform_device *plat_dev; 100 struct pci_dev *pdev; 101 struct list_head dev_node; 102 }; 103 104 struct dwc_pcie_vendor_id { 105 int vendor_id; 106 }; 107 108 static const struct dwc_pcie_vendor_id dwc_pcie_vendor_ids[] = { 109 {.vendor_id = PCI_VENDOR_ID_ALIBABA }, 110 {} /* terminator */ 111 }; 112 113 static ssize_t cpumask_show(struct device *dev, 114 struct device_attribute *attr, 115 char *buf) 116 { 117 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); 118 119 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); 120 } 121 static DEVICE_ATTR_RO(cpumask); 122 123 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { 124 &dev_attr_cpumask.attr, 125 NULL 126 }; 127 128 static struct attribute_group dwc_pcie_cpumask_attr_group = { 129 .attrs = dwc_pcie_pmu_cpumask_attrs, 130 }; 131 132 struct dwc_pcie_format_attr { 133 struct device_attribute attr; 134 u64 field; 135 int config; 136 }; 137 138 PMU_FORMAT_ATTR(eventid, "config:0-15"); 139 PMU_FORMAT_ATTR(type, "config:16-19"); 140 PMU_FORMAT_ATTR(lane, "config:20-27"); 141 142 static struct attribute *dwc_pcie_format_attrs[] = { 143 &format_attr_type.attr, 144 &format_attr_eventid.attr, 145 &format_attr_lane.attr, 146 NULL, 147 }; 148 149 static struct attribute_group dwc_pcie_format_attrs_group = { 150 .name = "format", 151 .attrs = dwc_pcie_format_attrs, 152 }; 153 154 struct dwc_pcie_event_attr { 155 struct device_attribute attr; 156 enum dwc_pcie_event_type type; 157 u16 eventid; 158 u8 lane; 159 }; 160 161 static ssize_t dwc_pcie_event_show(struct device *dev, 162 struct device_attribute *attr, char *buf) 163 { 164 struct dwc_pcie_event_attr *eattr; 165 166 eattr = container_of(attr, typeof(*eattr), attr); 167 168 if (eattr->type == DWC_PCIE_LANE_EVENT) 169 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", 170 eattr->eventid, eattr->type); 171 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) 172 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", 173 eattr->eventid, eattr->type); 174 175 return 0; 176 } 177 178 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ 179 (&((struct dwc_pcie_event_attr[]) {{ \ 180 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ 181 .type = _type, \ 182 .eventid = _eventid, \ 183 .lane = _lane, \ 184 }})[0].attr.attr) 185 186 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ 187 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) 188 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ 189 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) 190 191 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { 192 /* Group #0 */ 193 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), 194 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), 195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), 196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), 197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), 198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), 199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), 200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), 201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x08), 202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x09), 203 204 /* Group #1 */ 205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_PCIe_TLP_Data_Payload, 0x20), 206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_PCIe_TLP_Data_Payload, 0x21), 207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_CCIX_TLP_Data_Payload, 0x22), 208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_CCIX_TLP_Data_Payload, 0x23), 209 210 /* 211 * Leave it to the user to specify the lane ID to avoid generating 212 * a list of hundreds of events. 213 */ 214 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), 215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), 216 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), 217 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), 218 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nulified_tlp, 0x604), 219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nulified_tlp, 0x605), 220 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tl, 0x606), 221 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), 222 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), 223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), 224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), 225 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), 226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), 227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), 228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), 229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), 230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), 231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), 232 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), 233 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), 234 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), 235 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), 236 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), 237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), 238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), 239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), 240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), 241 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), 242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), 243 NULL 244 }; 245 246 static const struct attribute_group dwc_pcie_event_attrs_group = { 247 .name = "events", 248 .attrs = dwc_pcie_pmu_time_event_attrs, 249 }; 250 251 static const struct attribute_group *dwc_pcie_attr_groups[] = { 252 &dwc_pcie_event_attrs_group, 253 &dwc_pcie_format_attrs_group, 254 &dwc_pcie_cpumask_attr_group, 255 NULL 256 }; 257 258 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, 259 bool enable) 260 { 261 struct pci_dev *pdev = pcie_pmu->pdev; 262 u16 ras_des_offset = pcie_pmu->ras_des_offset; 263 264 if (enable) 265 pci_clear_and_set_config_dword(pdev, 266 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 267 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 268 else 269 pci_clear_and_set_config_dword(pdev, 270 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 271 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); 272 } 273 274 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, 275 bool enable) 276 { 277 struct pci_dev *pdev = pcie_pmu->pdev; 278 u16 ras_des_offset = pcie_pmu->ras_des_offset; 279 280 pci_clear_and_set_config_dword(pdev, 281 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, 282 DWC_PCIE_TIME_BASED_TIMER_START, enable); 283 } 284 285 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 286 { 287 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 288 struct pci_dev *pdev = pcie_pmu->pdev; 289 u16 ras_des_offset = pcie_pmu->ras_des_offset; 290 u32 val; 291 292 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); 293 294 return val; 295 } 296 297 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) 298 { 299 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 300 struct pci_dev *pdev = pcie_pmu->pdev; 301 int event_id = DWC_PCIE_EVENT_ID(event); 302 u16 ras_des_offset = pcie_pmu->ras_des_offset; 303 u32 lo, hi, ss; 304 u64 val; 305 306 /* 307 * The 64-bit value of the data counter is spread across two 308 * registers that are not synchronized. In order to read them 309 * atomically, ensure that the high 32 bits match before and after 310 * reading the low 32 bits. 311 */ 312 pci_read_config_dword(pdev, 313 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); 314 do { 315 /* snapshot the high 32 bits */ 316 ss = hi; 317 318 pci_read_config_dword( 319 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, 320 &lo); 321 pci_read_config_dword( 322 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, 323 &hi); 324 } while (hi != ss); 325 326 val = ((u64)hi << 32) | lo; 327 /* 328 * The Group#1 event measures the amount of data processed in 16-byte 329 * units. Simplify the end-user interface by multiplying the counter 330 * at the point of read. 331 */ 332 if (event_id >= 0x20 && event_id <= 0x23) 333 val *= 16; 334 335 return val; 336 } 337 338 static void dwc_pcie_pmu_event_update(struct perf_event *event) 339 { 340 struct hw_perf_event *hwc = &event->hw; 341 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 342 u64 delta, prev, now = 0; 343 344 do { 345 prev = local64_read(&hwc->prev_count); 346 347 if (type == DWC_PCIE_LANE_EVENT) 348 now = dwc_pcie_pmu_read_lane_event_counter(event); 349 else if (type == DWC_PCIE_TIME_BASE_EVENT) 350 now = dwc_pcie_pmu_read_time_based_counter(event); 351 352 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 353 354 delta = (now - prev) & DWC_PCIE_MAX_PERIOD; 355 /* 32-bit counter for Lane Event Counting */ 356 if (type == DWC_PCIE_LANE_EVENT) 357 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD; 358 359 local64_add(delta, &event->count); 360 } 361 362 static int dwc_pcie_pmu_event_init(struct perf_event *event) 363 { 364 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 365 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 366 struct perf_event *sibling; 367 u32 lane; 368 369 if (event->attr.type != event->pmu->type) 370 return -ENOENT; 371 372 /* We don't support sampling */ 373 if (is_sampling_event(event)) 374 return -EINVAL; 375 376 /* We cannot support task bound events */ 377 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 378 return -EINVAL; 379 380 if (event->group_leader != event && 381 !is_software_event(event->group_leader)) 382 return -EINVAL; 383 384 for_each_sibling_event(sibling, event->group_leader) { 385 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 386 return -EINVAL; 387 } 388 389 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) 390 return -EINVAL; 391 392 if (type == DWC_PCIE_LANE_EVENT) { 393 lane = DWC_PCIE_EVENT_LANE(event); 394 if (lane < 0 || lane >= pcie_pmu->nr_lanes) 395 return -EINVAL; 396 } 397 398 event->cpu = pcie_pmu->on_cpu; 399 400 return 0; 401 } 402 403 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 404 { 405 struct hw_perf_event *hwc = &event->hw; 406 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 407 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 408 409 hwc->state = 0; 410 local64_set(&hwc->prev_count, 0); 411 412 if (type == DWC_PCIE_LANE_EVENT) 413 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true); 414 else if (type == DWC_PCIE_TIME_BASE_EVENT) 415 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 416 } 417 418 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 419 { 420 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 421 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 422 struct hw_perf_event *hwc = &event->hw; 423 424 if (event->hw.state & PERF_HES_STOPPED) 425 return; 426 427 if (type == DWC_PCIE_LANE_EVENT) 428 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false); 429 else if (type == DWC_PCIE_TIME_BASE_EVENT) 430 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 431 432 dwc_pcie_pmu_event_update(event); 433 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 434 } 435 436 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 437 { 438 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 439 struct pci_dev *pdev = pcie_pmu->pdev; 440 struct hw_perf_event *hwc = &event->hw; 441 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 442 int event_id = DWC_PCIE_EVENT_ID(event); 443 int lane = DWC_PCIE_EVENT_LANE(event); 444 u16 ras_des_offset = pcie_pmu->ras_des_offset; 445 u32 ctrl; 446 447 /* one counter for each type and it is in use */ 448 if (pcie_pmu->event[type]) 449 return -ENOSPC; 450 451 pcie_pmu->event[type] = event; 452 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 453 454 if (type == DWC_PCIE_LANE_EVENT) { 455 /* EVENT_COUNTER_DATA_REG needs clear manually */ 456 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 457 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 458 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | 459 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 460 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 461 ctrl); 462 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 463 /* 464 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely 465 * use it with any manually controlled duration. And it is 466 * cleared when next measurement starts. 467 */ 468 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | 469 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, 470 DWC_PCIE_DURATION_MANUAL_CTL) | 471 DWC_PCIE_TIME_BASED_CNT_ENABLE; 472 pci_write_config_dword( 473 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); 474 } 475 476 if (flags & PERF_EF_START) 477 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); 478 479 perf_event_update_userpage(event); 480 481 return 0; 482 } 483 484 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 485 { 486 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 487 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 488 489 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); 490 perf_event_update_userpage(event); 491 pcie_pmu->event[type] = NULL; 492 } 493 494 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) 495 { 496 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); 497 } 498 499 /* 500 * Find the binded DES capability device info of a PCI device. 501 * @pdev: The PCI device. 502 */ 503 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) 504 { 505 struct dwc_pcie_dev_info *dev_info; 506 507 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) 508 if (dev_info->pdev == pdev) 509 return dev_info; 510 511 return NULL; 512 } 513 514 static void dwc_pcie_unregister_pmu(void *data) 515 { 516 struct dwc_pcie_pmu *pcie_pmu = data; 517 518 perf_pmu_unregister(&pcie_pmu->pmu); 519 } 520 521 static bool dwc_pcie_match_des_cap(struct pci_dev *pdev) 522 { 523 const struct dwc_pcie_vendor_id *vid; 524 u16 vsec = 0; 525 u32 val; 526 527 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) 528 return false; 529 530 for (vid = dwc_pcie_vendor_ids; vid->vendor_id; vid++) { 531 vsec = pci_find_vsec_capability(pdev, vid->vendor_id, 532 DWC_PCIE_VSEC_RAS_DES_ID); 533 if (vsec) 534 break; 535 } 536 if (!vsec) 537 return false; 538 539 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val); 540 if (PCI_VNDR_HEADER_REV(val) != 0x04) 541 return false; 542 543 pci_dbg(pdev, 544 "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); 545 return true; 546 } 547 548 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) 549 { 550 platform_device_unregister(dev_info->plat_dev); 551 list_del(&dev_info->dev_node); 552 kfree(dev_info); 553 } 554 555 static int dwc_pcie_register_dev(struct pci_dev *pdev) 556 { 557 struct platform_device *plat_dev; 558 struct dwc_pcie_dev_info *dev_info; 559 u32 bdf; 560 561 bdf = PCI_DEVID(pdev->bus->number, pdev->devfn); 562 plat_dev = platform_device_register_data(NULL, "dwc_pcie_pmu", bdf, 563 pdev, sizeof(*pdev)); 564 565 if (IS_ERR(plat_dev)) 566 return PTR_ERR(plat_dev); 567 568 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL); 569 if (!dev_info) 570 return -ENOMEM; 571 572 /* Cache platform device to handle pci device hotplug */ 573 dev_info->plat_dev = plat_dev; 574 dev_info->pdev = pdev; 575 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); 576 577 return 0; 578 } 579 580 static int dwc_pcie_pmu_notifier(struct notifier_block *nb, 581 unsigned long action, void *data) 582 { 583 struct device *dev = data; 584 struct pci_dev *pdev = to_pci_dev(dev); 585 struct dwc_pcie_dev_info *dev_info; 586 587 switch (action) { 588 case BUS_NOTIFY_ADD_DEVICE: 589 if (!dwc_pcie_match_des_cap(pdev)) 590 return NOTIFY_DONE; 591 if (dwc_pcie_register_dev(pdev)) 592 return NOTIFY_BAD; 593 break; 594 case BUS_NOTIFY_DEL_DEVICE: 595 dev_info = dwc_pcie_find_dev_info(pdev); 596 if (!dev_info) 597 return NOTIFY_DONE; 598 dwc_pcie_unregister_dev(dev_info); 599 break; 600 } 601 602 return NOTIFY_OK; 603 } 604 605 static struct notifier_block dwc_pcie_pmu_nb = { 606 .notifier_call = dwc_pcie_pmu_notifier, 607 }; 608 609 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) 610 { 611 struct pci_dev *pdev = plat_dev->dev.platform_data; 612 struct dwc_pcie_pmu *pcie_pmu; 613 char *name; 614 u32 bdf, val; 615 u16 vsec; 616 int ret; 617 618 vsec = pci_find_vsec_capability(pdev, pdev->vendor, 619 DWC_PCIE_VSEC_RAS_DES_ID); 620 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val); 621 bdf = PCI_DEVID(pdev->bus->number, pdev->devfn); 622 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", bdf); 623 if (!name) 624 return -ENOMEM; 625 626 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); 627 if (!pcie_pmu) 628 return -ENOMEM; 629 630 pcie_pmu->pdev = pdev; 631 pcie_pmu->ras_des_offset = vsec; 632 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); 633 pcie_pmu->on_cpu = -1; 634 pcie_pmu->pmu = (struct pmu){ 635 .name = name, 636 .parent = &pdev->dev, 637 .module = THIS_MODULE, 638 .attr_groups = dwc_pcie_attr_groups, 639 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 640 .task_ctx_nr = perf_invalid_context, 641 .event_init = dwc_pcie_pmu_event_init, 642 .add = dwc_pcie_pmu_event_add, 643 .del = dwc_pcie_pmu_event_del, 644 .start = dwc_pcie_pmu_event_start, 645 .stop = dwc_pcie_pmu_event_stop, 646 .read = dwc_pcie_pmu_event_update, 647 }; 648 649 /* Add this instance to the list used by the offline callback */ 650 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, 651 &pcie_pmu->cpuhp_node); 652 if (ret) { 653 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, bdf); 654 return ret; 655 } 656 657 /* Unwind when platform driver removes */ 658 ret = devm_add_action_or_reset(&plat_dev->dev, 659 dwc_pcie_pmu_remove_cpuhp_instance, 660 &pcie_pmu->cpuhp_node); 661 if (ret) 662 return ret; 663 664 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); 665 if (ret) { 666 pci_err(pdev, "Error %d registering PMU @%x\n", ret, bdf); 667 return ret; 668 } 669 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, 670 pcie_pmu); 671 if (ret) 672 return ret; 673 674 return 0; 675 } 676 677 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 678 { 679 struct dwc_pcie_pmu *pcie_pmu; 680 681 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 682 if (pcie_pmu->on_cpu == -1) 683 pcie_pmu->on_cpu = cpumask_local_spread( 684 0, dev_to_node(&pcie_pmu->pdev->dev)); 685 686 return 0; 687 } 688 689 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 690 { 691 struct dwc_pcie_pmu *pcie_pmu; 692 struct pci_dev *pdev; 693 int node; 694 cpumask_t mask; 695 unsigned int target; 696 697 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 698 /* Nothing to do if this CPU doesn't own the PMU */ 699 if (cpu != pcie_pmu->on_cpu) 700 return 0; 701 702 pcie_pmu->on_cpu = -1; 703 pdev = pcie_pmu->pdev; 704 node = dev_to_node(&pdev->dev); 705 if (cpumask_and(&mask, cpumask_of_node(node), cpu_online_mask) && 706 cpumask_andnot(&mask, &mask, cpumask_of(cpu))) 707 target = cpumask_any(&mask); 708 else 709 target = cpumask_any_but(cpu_online_mask, cpu); 710 711 if (target >= nr_cpu_ids) { 712 pci_err(pdev, "There is no CPU to set\n"); 713 return 0; 714 } 715 716 /* This PMU does NOT support interrupt, just migrate context. */ 717 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); 718 pcie_pmu->on_cpu = target; 719 720 return 0; 721 } 722 723 static struct platform_driver dwc_pcie_pmu_driver = { 724 .probe = dwc_pcie_pmu_probe, 725 .driver = {.name = "dwc_pcie_pmu",}, 726 }; 727 728 static int __init dwc_pcie_pmu_init(void) 729 { 730 struct pci_dev *pdev = NULL; 731 bool found = false; 732 int ret; 733 734 for_each_pci_dev(pdev) { 735 if (!dwc_pcie_match_des_cap(pdev)) 736 continue; 737 738 ret = dwc_pcie_register_dev(pdev); 739 if (ret) { 740 pci_dev_put(pdev); 741 return ret; 742 } 743 744 found = true; 745 } 746 if (!found) 747 return -ENODEV; 748 749 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 750 "perf/dwc_pcie_pmu:online", 751 dwc_pcie_pmu_online_cpu, 752 dwc_pcie_pmu_offline_cpu); 753 if (ret < 0) 754 return ret; 755 756 dwc_pcie_pmu_hp_state = ret; 757 758 ret = platform_driver_register(&dwc_pcie_pmu_driver); 759 if (ret) 760 goto platform_driver_register_err; 761 762 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 763 if (ret) 764 goto platform_driver_register_err; 765 notify = true; 766 767 return 0; 768 769 platform_driver_register_err: 770 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 771 772 return ret; 773 } 774 775 static void __exit dwc_pcie_pmu_exit(void) 776 { 777 struct dwc_pcie_dev_info *dev_info, *tmp; 778 779 if (notify) 780 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 781 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) 782 dwc_pcie_unregister_dev(dev_info); 783 platform_driver_unregister(&dwc_pcie_pmu_driver); 784 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 785 } 786 787 module_init(dwc_pcie_pmu_init); 788 module_exit(dwc_pcie_pmu_exit); 789 790 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); 791 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 792 MODULE_LICENSE("GPL v2"); 793