1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Synopsys DesignWare PCIe PMU driver 4 * 5 * Copyright (C) 2021-2023 Alibaba Inc. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/cpumask.h> 12 #include <linux/device.h> 13 #include <linux/errno.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/pcie-dwc.h> 17 #include <linux/perf_event.h> 18 #include <linux/pci.h> 19 #include <linux/platform_device.h> 20 #include <linux/smp.h> 21 #include <linux/sysfs.h> 22 #include <linux/types.h> 23 24 #define DWC_PCIE_EVENT_CNT_CTL 0x8 25 26 /* 27 * Event Counter Data Select includes two parts: 28 * - 27-24: Group number(4-bit: 0..0x7) 29 * - 23-16: Event number(8-bit: 0..0x13) within the Group 30 * 31 * Put them together as in TRM. 32 */ 33 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) 34 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) 35 #define DWC_PCIE_CNT_STATUS BIT(7) 36 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) 37 #define DWC_PCIE_PER_EVENT_OFF 0x1 38 #define DWC_PCIE_PER_EVENT_ON 0x3 39 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) 40 #define DWC_PCIE_EVENT_PER_CLEAR 0x1 41 42 /* Event Selection Field has two subfields */ 43 #define DWC_PCIE_CNT_EVENT_SEL_GROUP GENMASK(11, 8) 44 #define DWC_PCIE_CNT_EVENT_SEL_EVID GENMASK(7, 0) 45 46 #define DWC_PCIE_EVENT_CNT_DATA 0xC 47 48 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 49 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) 50 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) 51 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 52 #define DWC_PCIE_DURATION_1MS 0x1 53 #define DWC_PCIE_DURATION_10MS 0x2 54 #define DWC_PCIE_DURATION_100MS 0x3 55 #define DWC_PCIE_DURATION_1S 0x4 56 #define DWC_PCIE_DURATION_2S 0x5 57 #define DWC_PCIE_DURATION_4S 0x6 58 #define DWC_PCIE_DURATION_4US 0xFF 59 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) 60 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 61 62 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 63 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 64 65 /* Event attributes */ 66 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) 67 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) 68 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) 69 70 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) 71 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) 72 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) 73 74 enum dwc_pcie_event_type { 75 DWC_PCIE_TIME_BASE_EVENT, 76 DWC_PCIE_LANE_EVENT, 77 DWC_PCIE_EVENT_TYPE_MAX, 78 }; 79 80 #define DWC_PCIE_LANE_GROUP_6 6 81 #define DWC_PCIE_LANE_GROUP_7 7 82 #define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256 83 84 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) 85 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) 86 87 struct dwc_pcie_pmu { 88 struct pmu pmu; 89 struct pci_dev *pdev; /* Root Port device */ 90 u16 ras_des_offset; 91 u32 nr_lanes; 92 93 /* Groups #6 and #7 */ 94 DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); 95 struct perf_event *time_based_event; 96 97 struct hlist_node cpuhp_node; 98 int on_cpu; 99 }; 100 101 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) 102 103 static int dwc_pcie_pmu_hp_state; 104 static struct list_head dwc_pcie_dev_info_head = 105 LIST_HEAD_INIT(dwc_pcie_dev_info_head); 106 static bool notify; 107 108 struct dwc_pcie_dev_info { 109 struct platform_device *plat_dev; 110 struct pci_dev *pdev; 111 struct list_head dev_node; 112 }; 113 114 static ssize_t cpumask_show(struct device *dev, 115 struct device_attribute *attr, 116 char *buf) 117 { 118 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); 119 120 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); 121 } 122 static DEVICE_ATTR_RO(cpumask); 123 124 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { 125 &dev_attr_cpumask.attr, 126 NULL 127 }; 128 129 static struct attribute_group dwc_pcie_cpumask_attr_group = { 130 .attrs = dwc_pcie_pmu_cpumask_attrs, 131 }; 132 133 struct dwc_pcie_format_attr { 134 struct device_attribute attr; 135 u64 field; 136 int config; 137 }; 138 139 PMU_FORMAT_ATTR(eventid, "config:0-15"); 140 PMU_FORMAT_ATTR(type, "config:16-19"); 141 PMU_FORMAT_ATTR(lane, "config:20-27"); 142 143 static struct attribute *dwc_pcie_format_attrs[] = { 144 &format_attr_type.attr, 145 &format_attr_eventid.attr, 146 &format_attr_lane.attr, 147 NULL, 148 }; 149 150 static struct attribute_group dwc_pcie_format_attrs_group = { 151 .name = "format", 152 .attrs = dwc_pcie_format_attrs, 153 }; 154 155 struct dwc_pcie_event_attr { 156 struct device_attribute attr; 157 enum dwc_pcie_event_type type; 158 u16 eventid; 159 u8 lane; 160 }; 161 162 static ssize_t dwc_pcie_event_show(struct device *dev, 163 struct device_attribute *attr, char *buf) 164 { 165 struct dwc_pcie_event_attr *eattr; 166 167 eattr = container_of(attr, typeof(*eattr), attr); 168 169 if (eattr->type == DWC_PCIE_LANE_EVENT) 170 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n", 171 eattr->eventid, eattr->type); 172 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) 173 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n", 174 eattr->eventid, eattr->type); 175 176 return 0; 177 } 178 179 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ 180 (&((struct dwc_pcie_event_attr[]) {{ \ 181 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ 182 .type = _type, \ 183 .eventid = _eventid, \ 184 .lane = _lane, \ 185 }})[0].attr.attr) 186 187 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ 188 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) 189 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ 190 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) 191 192 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { 193 /* Group #0 */ 194 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), 195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), 196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), 197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), 198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), 199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), 200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), 201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), 202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08), 203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09), 204 205 /* Group #1 */ 206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20), 207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21), 208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22), 209 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23), 210 211 /* 212 * Leave it to the user to specify the lane ID to avoid generating 213 * a list of hundreds of events. 214 */ 215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), 216 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), 217 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), 218 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), 219 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604), 220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605), 221 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606), 222 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), 223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), 224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), 225 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), 226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), 227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), 228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), 229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), 230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), 231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), 232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), 233 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), 234 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), 235 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), 236 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), 237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), 238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), 239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), 240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), 241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), 242 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), 243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), 244 NULL 245 }; 246 247 static const struct attribute_group dwc_pcie_event_attrs_group = { 248 .name = "events", 249 .attrs = dwc_pcie_pmu_time_event_attrs, 250 }; 251 252 static const struct attribute_group *dwc_pcie_attr_groups[] = { 253 &dwc_pcie_event_attrs_group, 254 &dwc_pcie_format_attrs_group, 255 &dwc_pcie_cpumask_attr_group, 256 NULL 257 }; 258 259 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, 260 struct perf_event *event, 261 bool enable) 262 { 263 struct pci_dev *pdev = pcie_pmu->pdev; 264 u16 ras_des_offset = pcie_pmu->ras_des_offset; 265 int event_id = DWC_PCIE_EVENT_ID(event); 266 int lane = DWC_PCIE_EVENT_LANE(event); 267 u32 ctrl; 268 269 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 270 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 271 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 272 273 if (enable) 274 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 275 else 276 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); 277 278 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 279 ctrl); 280 } 281 282 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, 283 bool enable) 284 { 285 struct pci_dev *pdev = pcie_pmu->pdev; 286 u16 ras_des_offset = pcie_pmu->ras_des_offset; 287 288 pci_clear_and_set_config_dword(pdev, 289 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, 290 DWC_PCIE_TIME_BASED_TIMER_START, enable); 291 } 292 293 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 294 { 295 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 296 struct pci_dev *pdev = pcie_pmu->pdev; 297 int event_id = DWC_PCIE_EVENT_ID(event); 298 int lane = DWC_PCIE_EVENT_LANE(event); 299 u16 ras_des_offset = pcie_pmu->ras_des_offset; 300 u32 val, ctrl; 301 302 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 303 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 304 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); 305 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 306 ctrl); 307 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val); 308 309 ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 310 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 311 ctrl); 312 313 return val; 314 } 315 316 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) 317 { 318 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 319 struct pci_dev *pdev = pcie_pmu->pdev; 320 int event_id = DWC_PCIE_EVENT_ID(event); 321 u16 ras_des_offset = pcie_pmu->ras_des_offset; 322 u32 lo, hi, ss; 323 u64 val; 324 325 /* 326 * The 64-bit value of the data counter is spread across two 327 * registers that are not synchronized. In order to read them 328 * atomically, ensure that the high 32 bits match before and after 329 * reading the low 32 bits. 330 */ 331 pci_read_config_dword(pdev, 332 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi); 333 do { 334 /* snapshot the high 32 bits */ 335 ss = hi; 336 337 pci_read_config_dword( 338 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, 339 &lo); 340 pci_read_config_dword( 341 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, 342 &hi); 343 } while (hi != ss); 344 345 val = ((u64)hi << 32) | lo; 346 /* 347 * The Group#1 event measures the amount of data processed in 16-byte 348 * units. Simplify the end-user interface by multiplying the counter 349 * at the point of read. 350 */ 351 if (event_id >= 0x20 && event_id <= 0x23) 352 val *= 16; 353 354 return val; 355 } 356 357 static void dwc_pcie_pmu_event_update(struct perf_event *event) 358 { 359 struct hw_perf_event *hwc = &event->hw; 360 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 361 u64 delta, prev, now; 362 363 if (type == DWC_PCIE_LANE_EVENT) { 364 now = dwc_pcie_pmu_read_lane_event_counter(event) & 365 DWC_PCIE_LANE_EVENT_MAX_PERIOD; 366 local64_add(now, &event->count); 367 return; 368 } 369 370 do { 371 prev = local64_read(&hwc->prev_count); 372 now = dwc_pcie_pmu_read_time_based_counter(event); 373 374 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 375 376 delta = (now - prev) & DWC_PCIE_MAX_PERIOD; 377 local64_add(delta, &event->count); 378 } 379 380 static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event, 381 unsigned long val_lane_events[]) 382 { 383 int event_id, event_nr, group; 384 385 event_id = DWC_PCIE_EVENT_ID(event); 386 event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 387 group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id); 388 389 if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7) 390 return -EINVAL; 391 392 group -= DWC_PCIE_LANE_GROUP_6; 393 394 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 395 val_lane_events)) 396 return -EINVAL; 397 398 return 0; 399 } 400 401 static int dwc_pcie_pmu_validate_group(struct perf_event *event) 402 { 403 struct perf_event *sibling, *leader = event->group_leader; 404 DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); 405 bool time_event = false; 406 int type; 407 408 type = DWC_PCIE_EVENT_TYPE(leader); 409 if (type == DWC_PCIE_TIME_BASE_EVENT) 410 time_event = true; 411 else 412 if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events)) 413 return -ENOSPC; 414 415 for_each_sibling_event(sibling, leader) { 416 type = DWC_PCIE_EVENT_TYPE(sibling); 417 if (type == DWC_PCIE_TIME_BASE_EVENT) { 418 if (time_event) 419 return -ENOSPC; 420 421 time_event = true; 422 continue; 423 } 424 425 if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events)) 426 return -ENOSPC; 427 } 428 429 return 0; 430 } 431 432 static int dwc_pcie_pmu_event_init(struct perf_event *event) 433 { 434 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 435 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 436 struct perf_event *sibling; 437 u32 lane; 438 439 if (event->attr.type != event->pmu->type) 440 return -ENOENT; 441 442 /* We don't support sampling */ 443 if (is_sampling_event(event)) 444 return -EINVAL; 445 446 /* We cannot support task bound events */ 447 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) 448 return -EINVAL; 449 450 for_each_sibling_event(sibling, event->group_leader) { 451 if (sibling->pmu != event->pmu && !is_software_event(sibling)) 452 return -EINVAL; 453 } 454 455 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) 456 return -EINVAL; 457 458 if (type == DWC_PCIE_LANE_EVENT) { 459 lane = DWC_PCIE_EVENT_LANE(event); 460 if (lane < 0 || lane >= pcie_pmu->nr_lanes) 461 return -EINVAL; 462 } 463 464 if (dwc_pcie_pmu_validate_group(event)) 465 return -ENOSPC; 466 467 event->cpu = pcie_pmu->on_cpu; 468 469 return 0; 470 } 471 472 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 473 { 474 struct hw_perf_event *hwc = &event->hw; 475 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 476 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 477 478 hwc->state = 0; 479 local64_set(&hwc->prev_count, 0); 480 481 if (type == DWC_PCIE_LANE_EVENT) 482 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true); 483 else if (type == DWC_PCIE_TIME_BASE_EVENT) 484 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); 485 } 486 487 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 488 { 489 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 490 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 491 struct hw_perf_event *hwc = &event->hw; 492 493 if (event->hw.state & PERF_HES_STOPPED) 494 return; 495 496 dwc_pcie_pmu_event_update(event); 497 498 if (type == DWC_PCIE_LANE_EVENT) 499 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false); 500 else if (type == DWC_PCIE_TIME_BASE_EVENT) 501 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); 502 503 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 504 } 505 506 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 507 { 508 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 509 struct pci_dev *pdev = pcie_pmu->pdev; 510 struct hw_perf_event *hwc = &event->hw; 511 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 512 int event_id = DWC_PCIE_EVENT_ID(event); 513 int lane = DWC_PCIE_EVENT_LANE(event); 514 u16 ras_des_offset = pcie_pmu->ras_des_offset; 515 u32 ctrl; 516 517 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 518 519 if (type == DWC_PCIE_LANE_EVENT) { 520 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 521 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - 522 DWC_PCIE_LANE_GROUP_6; 523 524 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 525 pcie_pmu->lane_events)) 526 return -ENOSPC; 527 528 /* EVENT_COUNTER_DATA_REG needs clear manually */ 529 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | 530 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | 531 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | 532 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); 533 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, 534 ctrl); 535 } else if (type == DWC_PCIE_TIME_BASE_EVENT) { 536 if (pcie_pmu->time_based_event) 537 return -ENOSPC; 538 539 pcie_pmu->time_based_event = event; 540 541 /* 542 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely 543 * use it with any manually controlled duration. And it is 544 * cleared when next measurement starts. 545 */ 546 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | 547 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, 548 DWC_PCIE_DURATION_MANUAL_CTL) | 549 DWC_PCIE_TIME_BASED_CNT_ENABLE; 550 pci_write_config_dword( 551 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl); 552 } 553 554 if (flags & PERF_EF_START) 555 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); 556 557 perf_event_update_userpage(event); 558 559 return 0; 560 } 561 562 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 563 { 564 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); 565 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); 566 567 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE); 568 perf_event_update_userpage(event); 569 570 if (type == DWC_PCIE_TIME_BASE_EVENT) { 571 pcie_pmu->time_based_event = NULL; 572 } else { 573 int event_id = DWC_PCIE_EVENT_ID(event); 574 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id); 575 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) - 576 DWC_PCIE_LANE_GROUP_6; 577 578 clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr, 579 pcie_pmu->lane_events); 580 } 581 } 582 583 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) 584 { 585 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node); 586 } 587 588 /* 589 * Find the binded DES capability device info of a PCI device. 590 * @pdev: The PCI device. 591 */ 592 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) 593 { 594 struct dwc_pcie_dev_info *dev_info; 595 596 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) 597 if (dev_info->pdev == pdev) 598 return dev_info; 599 600 return NULL; 601 } 602 603 static void dwc_pcie_unregister_pmu(void *data) 604 { 605 struct dwc_pcie_pmu *pcie_pmu = data; 606 607 perf_pmu_unregister(&pcie_pmu->pmu); 608 } 609 610 static u16 dwc_pcie_des_cap(struct pci_dev *pdev) 611 { 612 const struct dwc_pcie_vsec_id *vid; 613 u16 vsec; 614 u32 val; 615 616 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)) 617 return 0; 618 619 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) { 620 vsec = pci_find_vsec_capability(pdev, vid->vendor_id, 621 vid->vsec_id); 622 if (vsec) { 623 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, 624 &val); 625 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) { 626 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n"); 627 return vsec; 628 } 629 } 630 } 631 return 0; 632 } 633 634 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) 635 { 636 platform_device_unregister(dev_info->plat_dev); 637 list_del(&dev_info->dev_node); 638 kfree(dev_info); 639 } 640 641 static int dwc_pcie_register_dev(struct pci_dev *pdev) 642 { 643 struct platform_device *plat_dev; 644 struct dwc_pcie_dev_info *dev_info; 645 u32 sbdf; 646 647 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); 648 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0); 649 if (IS_ERR(plat_dev)) 650 return PTR_ERR(plat_dev); 651 652 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL); 653 if (!dev_info) { 654 platform_device_unregister(plat_dev); 655 return -ENOMEM; 656 } 657 658 /* Cache platform device to handle pci device hotplug */ 659 dev_info->plat_dev = plat_dev; 660 dev_info->pdev = pdev; 661 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head); 662 663 return 0; 664 } 665 666 static int dwc_pcie_pmu_notifier(struct notifier_block *nb, 667 unsigned long action, void *data) 668 { 669 struct device *dev = data; 670 struct pci_dev *pdev = to_pci_dev(dev); 671 struct dwc_pcie_dev_info *dev_info; 672 673 switch (action) { 674 case BUS_NOTIFY_ADD_DEVICE: 675 if (!dwc_pcie_des_cap(pdev)) 676 return NOTIFY_DONE; 677 if (dwc_pcie_register_dev(pdev)) 678 return NOTIFY_BAD; 679 break; 680 case BUS_NOTIFY_DEL_DEVICE: 681 dev_info = dwc_pcie_find_dev_info(pdev); 682 if (!dev_info) 683 return NOTIFY_DONE; 684 dwc_pcie_unregister_dev(dev_info); 685 break; 686 } 687 688 return NOTIFY_OK; 689 } 690 691 static struct notifier_block dwc_pcie_pmu_nb = { 692 .notifier_call = dwc_pcie_pmu_notifier, 693 }; 694 695 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) 696 { 697 struct pci_dev *pdev; 698 struct dwc_pcie_pmu *pcie_pmu; 699 char *name; 700 u32 sbdf; 701 u16 vsec; 702 int ret; 703 704 sbdf = plat_dev->id; 705 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff), 706 sbdf & 0xff); 707 if (!pdev) { 708 pr_err("No pdev found for the sbdf 0x%x\n", sbdf); 709 return -ENODEV; 710 } 711 712 vsec = dwc_pcie_des_cap(pdev); 713 if (!vsec) 714 return -ENODEV; 715 716 pci_dev_put(pdev); 717 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf); 718 if (!name) 719 return -ENOMEM; 720 721 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL); 722 if (!pcie_pmu) 723 return -ENOMEM; 724 725 pcie_pmu->pdev = pdev; 726 pcie_pmu->ras_des_offset = vsec; 727 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); 728 pcie_pmu->on_cpu = -1; 729 pcie_pmu->pmu = (struct pmu){ 730 .name = name, 731 .parent = &plat_dev->dev, 732 .module = THIS_MODULE, 733 .attr_groups = dwc_pcie_attr_groups, 734 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 735 .task_ctx_nr = perf_invalid_context, 736 .event_init = dwc_pcie_pmu_event_init, 737 .add = dwc_pcie_pmu_event_add, 738 .del = dwc_pcie_pmu_event_del, 739 .start = dwc_pcie_pmu_event_start, 740 .stop = dwc_pcie_pmu_event_stop, 741 .read = dwc_pcie_pmu_event_update, 742 }; 743 744 /* Add this instance to the list used by the offline callback */ 745 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state, 746 &pcie_pmu->cpuhp_node); 747 if (ret) { 748 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf); 749 return ret; 750 } 751 752 /* Unwind when platform driver removes */ 753 ret = devm_add_action_or_reset(&plat_dev->dev, 754 dwc_pcie_pmu_remove_cpuhp_instance, 755 &pcie_pmu->cpuhp_node); 756 if (ret) 757 return ret; 758 759 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1); 760 if (ret) { 761 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf); 762 return ret; 763 } 764 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, 765 pcie_pmu); 766 if (ret) 767 return ret; 768 769 return 0; 770 } 771 772 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 773 { 774 struct dwc_pcie_pmu *pcie_pmu; 775 776 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 777 if (pcie_pmu->on_cpu == -1) 778 pcie_pmu->on_cpu = cpumask_local_spread( 779 0, dev_to_node(&pcie_pmu->pdev->dev)); 780 781 return 0; 782 } 783 784 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 785 { 786 struct dwc_pcie_pmu *pcie_pmu; 787 struct pci_dev *pdev; 788 unsigned int target; 789 int node; 790 791 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); 792 /* Nothing to do if this CPU doesn't own the PMU */ 793 if (cpu != pcie_pmu->on_cpu) 794 return 0; 795 796 pcie_pmu->on_cpu = -1; 797 pdev = pcie_pmu->pdev; 798 node = dev_to_node(&pdev->dev); 799 800 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 801 if (target >= nr_cpu_ids) 802 target = cpumask_any_but(cpu_online_mask, cpu); 803 804 if (target >= nr_cpu_ids) { 805 pci_err(pdev, "There is no CPU to set\n"); 806 return 0; 807 } 808 809 /* This PMU does NOT support interrupt, just migrate context. */ 810 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); 811 pcie_pmu->on_cpu = target; 812 813 return 0; 814 } 815 816 static struct platform_driver dwc_pcie_pmu_driver = { 817 .probe = dwc_pcie_pmu_probe, 818 .driver = {.name = "dwc_pcie_pmu",}, 819 }; 820 821 static void dwc_pcie_cleanup_devices(void) 822 { 823 struct dwc_pcie_dev_info *dev_info, *tmp; 824 825 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) { 826 dwc_pcie_unregister_dev(dev_info); 827 } 828 } 829 830 static int __init dwc_pcie_pmu_init(void) 831 { 832 struct pci_dev *pdev = NULL; 833 int ret; 834 835 for_each_pci_dev(pdev) { 836 if (!dwc_pcie_des_cap(pdev)) 837 continue; 838 839 ret = dwc_pcie_register_dev(pdev); 840 if (ret) { 841 pci_dev_put(pdev); 842 goto err_cleanup; 843 } 844 } 845 846 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 847 "perf/dwc_pcie_pmu:online", 848 dwc_pcie_pmu_online_cpu, 849 dwc_pcie_pmu_offline_cpu); 850 if (ret < 0) 851 goto err_cleanup; 852 853 dwc_pcie_pmu_hp_state = ret; 854 855 ret = platform_driver_register(&dwc_pcie_pmu_driver); 856 if (ret) 857 goto err_remove_cpuhp; 858 859 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 860 if (ret) 861 goto err_unregister_driver; 862 notify = true; 863 864 return 0; 865 866 err_unregister_driver: 867 platform_driver_unregister(&dwc_pcie_pmu_driver); 868 err_remove_cpuhp: 869 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 870 err_cleanup: 871 dwc_pcie_cleanup_devices(); 872 return ret; 873 } 874 875 static void __exit dwc_pcie_pmu_exit(void) 876 { 877 if (notify) 878 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb); 879 dwc_pcie_cleanup_devices(); 880 platform_driver_unregister(&dwc_pcie_pmu_driver); 881 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state); 882 } 883 884 module_init(dwc_pcie_pmu_init); 885 module_exit(dwc_pcie_pmu_exit); 886 887 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller"); 888 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 889 MODULE_LICENSE("GPL v2"); 890