1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/bitops.h>
10 #include <linux/cpuhotplug.h>
11 #include <linux/cpumask.h>
12 #include <linux/device.h>
13 #include <linux/errno.h>
14 #include <linux/kernel.h>
15 #include <linux/list.h>
16 #include <linux/pcie-dwc.h>
17 #include <linux/perf_event.h>
18 #include <linux/pci.h>
19 #include <linux/platform_device.h>
20 #include <linux/smp.h>
21 #include <linux/sysfs.h>
22 #include <linux/types.h>
23
24 #define DWC_PCIE_EVENT_CNT_CTL 0x8
25
26 /*
27 * Event Counter Data Select includes two parts:
28 * - 27-24: Group number(4-bit: 0..0x7)
29 * - 23-16: Event number(8-bit: 0..0x13) within the Group
30 *
31 * Put them together as in TRM.
32 */
33 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
34 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
35 #define DWC_PCIE_CNT_STATUS BIT(7)
36 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
37 #define DWC_PCIE_PER_EVENT_OFF 0x1
38 #define DWC_PCIE_PER_EVENT_ON 0x3
39 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
40 #define DWC_PCIE_EVENT_PER_CLEAR 0x1
41
42 /* Event Selection Field has two subfields */
43 #define DWC_PCIE_CNT_EVENT_SEL_GROUP GENMASK(11, 8)
44 #define DWC_PCIE_CNT_EVENT_SEL_EVID GENMASK(7, 0)
45
46 #define DWC_PCIE_EVENT_CNT_DATA 0xC
47
48 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
49 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
50 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
51 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0
52 #define DWC_PCIE_DURATION_1MS 0x1
53 #define DWC_PCIE_DURATION_10MS 0x2
54 #define DWC_PCIE_DURATION_100MS 0x3
55 #define DWC_PCIE_DURATION_1S 0x4
56 #define DWC_PCIE_DURATION_2S 0x5
57 #define DWC_PCIE_DURATION_4S 0x6
58 #define DWC_PCIE_DURATION_4US 0xFF
59 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
60 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
61
62 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
63 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
64
65 /* Event attributes */
66 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
67 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
68 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
69
70 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
71 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
72 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
73
74 enum dwc_pcie_event_type {
75 DWC_PCIE_TIME_BASE_EVENT,
76 DWC_PCIE_LANE_EVENT,
77 DWC_PCIE_EVENT_TYPE_MAX,
78 };
79
80 #define DWC_PCIE_LANE_GROUP_6 6
81 #define DWC_PCIE_LANE_GROUP_7 7
82 #define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256
83
84 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
85 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
86
87 struct dwc_pcie_pmu {
88 struct pmu pmu;
89 struct pci_dev *pdev; /* Root Port device */
90 u16 ras_des_offset;
91 u32 nr_lanes;
92
93 /* Groups #6 and #7 */
94 DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
95 struct perf_event *time_based_event;
96
97 struct hlist_node cpuhp_node;
98 int on_cpu;
99 };
100
101 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
102
103 static int dwc_pcie_pmu_hp_state;
104 static struct list_head dwc_pcie_dev_info_head =
105 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
106 static bool notify;
107
108 struct dwc_pcie_dev_info {
109 struct platform_device *plat_dev;
110 struct pci_dev *pdev;
111 struct list_head dev_node;
112 };
113
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)114 static ssize_t cpumask_show(struct device *dev,
115 struct device_attribute *attr,
116 char *buf)
117 {
118 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
119
120 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
121 }
122 static DEVICE_ATTR_RO(cpumask);
123
124 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
125 &dev_attr_cpumask.attr,
126 NULL
127 };
128
129 static struct attribute_group dwc_pcie_cpumask_attr_group = {
130 .attrs = dwc_pcie_pmu_cpumask_attrs,
131 };
132
133 struct dwc_pcie_format_attr {
134 struct device_attribute attr;
135 u64 field;
136 int config;
137 };
138
139 PMU_FORMAT_ATTR(eventid, "config:0-15");
140 PMU_FORMAT_ATTR(type, "config:16-19");
141 PMU_FORMAT_ATTR(lane, "config:20-27");
142
143 static struct attribute *dwc_pcie_format_attrs[] = {
144 &format_attr_type.attr,
145 &format_attr_eventid.attr,
146 &format_attr_lane.attr,
147 NULL,
148 };
149
150 static struct attribute_group dwc_pcie_format_attrs_group = {
151 .name = "format",
152 .attrs = dwc_pcie_format_attrs,
153 };
154
155 struct dwc_pcie_event_attr {
156 struct device_attribute attr;
157 enum dwc_pcie_event_type type;
158 u16 eventid;
159 u8 lane;
160 };
161
dwc_pcie_event_show(struct device * dev,struct device_attribute * attr,char * buf)162 static ssize_t dwc_pcie_event_show(struct device *dev,
163 struct device_attribute *attr, char *buf)
164 {
165 struct dwc_pcie_event_attr *eattr;
166
167 eattr = container_of(attr, typeof(*eattr), attr);
168
169 if (eattr->type == DWC_PCIE_LANE_EVENT)
170 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
171 eattr->eventid, eattr->type);
172 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
173 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
174 eattr->eventid, eattr->type);
175
176 return 0;
177 }
178
179 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
180 (&((struct dwc_pcie_event_attr[]) {{ \
181 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
182 .type = _type, \
183 .eventid = _eventid, \
184 .lane = _lane, \
185 }})[0].attr.attr)
186
187 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
188 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
189 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
190 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
191
192 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
193 /* Group #0 */
194 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08),
203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09),
204
205 /* Group #1 */
206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
208 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
209 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
210
211 /*
212 * Leave it to the user to specify the lane ID to avoid generating
213 * a list of hundreds of events.
214 */
215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
216 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
217 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
218 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
219 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
221 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
222 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
233 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
234 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
235 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
236 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
242 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
244 NULL
245 };
246
247 static const struct attribute_group dwc_pcie_event_attrs_group = {
248 .name = "events",
249 .attrs = dwc_pcie_pmu_time_event_attrs,
250 };
251
252 static const struct attribute_group *dwc_pcie_attr_groups[] = {
253 &dwc_pcie_event_attrs_group,
254 &dwc_pcie_format_attrs_group,
255 &dwc_pcie_cpumask_attr_group,
256 NULL
257 };
258
dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu * pcie_pmu,struct perf_event * event,bool enable)259 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
260 struct perf_event *event,
261 bool enable)
262 {
263 struct pci_dev *pdev = pcie_pmu->pdev;
264 u16 ras_des_offset = pcie_pmu->ras_des_offset;
265 int event_id = DWC_PCIE_EVENT_ID(event);
266 int lane = DWC_PCIE_EVENT_LANE(event);
267 u32 ctrl;
268
269 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
270 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
271 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
272
273 if (enable)
274 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
275 else
276 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
277
278 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
279 ctrl);
280 }
281
dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)282 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
283 bool enable)
284 {
285 struct pci_dev *pdev = pcie_pmu->pdev;
286 u16 ras_des_offset = pcie_pmu->ras_des_offset;
287
288 pci_clear_and_set_config_dword(pdev,
289 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
290 DWC_PCIE_TIME_BASED_TIMER_START, enable);
291 }
292
dwc_pcie_pmu_read_lane_event_counter(struct perf_event * event)293 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
294 {
295 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
296 struct pci_dev *pdev = pcie_pmu->pdev;
297 int event_id = DWC_PCIE_EVENT_ID(event);
298 int lane = DWC_PCIE_EVENT_LANE(event);
299 u16 ras_des_offset = pcie_pmu->ras_des_offset;
300 u32 val, ctrl;
301
302 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
303 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
304 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
305 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
306 ctrl);
307 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
308
309 ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
310 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
311 ctrl);
312
313 return val;
314 }
315
dwc_pcie_pmu_read_time_based_counter(struct perf_event * event)316 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
317 {
318 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
319 struct pci_dev *pdev = pcie_pmu->pdev;
320 int event_id = DWC_PCIE_EVENT_ID(event);
321 u16 ras_des_offset = pcie_pmu->ras_des_offset;
322 u32 lo, hi, ss;
323 u64 val;
324
325 /*
326 * The 64-bit value of the data counter is spread across two
327 * registers that are not synchronized. In order to read them
328 * atomically, ensure that the high 32 bits match before and after
329 * reading the low 32 bits.
330 */
331 pci_read_config_dword(pdev,
332 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
333 do {
334 /* snapshot the high 32 bits */
335 ss = hi;
336
337 pci_read_config_dword(
338 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
339 &lo);
340 pci_read_config_dword(
341 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
342 &hi);
343 } while (hi != ss);
344
345 val = ((u64)hi << 32) | lo;
346 /*
347 * The Group#1 event measures the amount of data processed in 16-byte
348 * units. Simplify the end-user interface by multiplying the counter
349 * at the point of read.
350 */
351 if (event_id >= 0x20 && event_id <= 0x23)
352 val *= 16;
353
354 return val;
355 }
356
dwc_pcie_pmu_event_update(struct perf_event * event)357 static void dwc_pcie_pmu_event_update(struct perf_event *event)
358 {
359 struct hw_perf_event *hwc = &event->hw;
360 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
361 u64 delta, prev, now;
362
363 if (type == DWC_PCIE_LANE_EVENT) {
364 now = dwc_pcie_pmu_read_lane_event_counter(event) &
365 DWC_PCIE_LANE_EVENT_MAX_PERIOD;
366 local64_add(now, &event->count);
367 return;
368 }
369
370 do {
371 prev = local64_read(&hwc->prev_count);
372 now = dwc_pcie_pmu_read_time_based_counter(event);
373
374 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
375
376 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
377 local64_add(delta, &event->count);
378 }
379
dwc_pcie_pmu_validate_add_lane_event(struct perf_event * event,unsigned long val_lane_events[])380 static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event,
381 unsigned long val_lane_events[])
382 {
383 int event_id, event_nr, group;
384
385 event_id = DWC_PCIE_EVENT_ID(event);
386 event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
387 group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id);
388
389 if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7)
390 return -EINVAL;
391
392 group -= DWC_PCIE_LANE_GROUP_6;
393
394 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
395 val_lane_events))
396 return -EINVAL;
397
398 return 0;
399 }
400
dwc_pcie_pmu_validate_group(struct perf_event * event)401 static int dwc_pcie_pmu_validate_group(struct perf_event *event)
402 {
403 struct perf_event *sibling, *leader = event->group_leader;
404 DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
405 bool time_event = false;
406 int type;
407
408 type = DWC_PCIE_EVENT_TYPE(leader);
409 if (type == DWC_PCIE_TIME_BASE_EVENT)
410 time_event = true;
411 else
412 if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events))
413 return -ENOSPC;
414
415 for_each_sibling_event(sibling, leader) {
416 type = DWC_PCIE_EVENT_TYPE(sibling);
417 if (type == DWC_PCIE_TIME_BASE_EVENT) {
418 if (time_event)
419 return -ENOSPC;
420
421 time_event = true;
422 continue;
423 }
424
425 if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events))
426 return -ENOSPC;
427 }
428
429 return 0;
430 }
431
dwc_pcie_pmu_event_init(struct perf_event * event)432 static int dwc_pcie_pmu_event_init(struct perf_event *event)
433 {
434 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
435 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
436 struct perf_event *sibling;
437 u32 lane;
438
439 if (event->attr.type != event->pmu->type)
440 return -ENOENT;
441
442 /* We don't support sampling */
443 if (is_sampling_event(event))
444 return -EINVAL;
445
446 /* We cannot support task bound events */
447 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
448 return -EINVAL;
449
450 for_each_sibling_event(sibling, event->group_leader) {
451 if (sibling->pmu != event->pmu && !is_software_event(sibling))
452 return -EINVAL;
453 }
454
455 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
456 return -EINVAL;
457
458 if (type == DWC_PCIE_LANE_EVENT) {
459 lane = DWC_PCIE_EVENT_LANE(event);
460 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
461 return -EINVAL;
462 }
463
464 if (dwc_pcie_pmu_validate_group(event))
465 return -ENOSPC;
466
467 event->cpu = pcie_pmu->on_cpu;
468
469 return 0;
470 }
471
dwc_pcie_pmu_event_start(struct perf_event * event,int flags)472 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
473 {
474 struct hw_perf_event *hwc = &event->hw;
475 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
476 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
477
478 hwc->state = 0;
479 local64_set(&hwc->prev_count, 0);
480
481 if (type == DWC_PCIE_LANE_EVENT)
482 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true);
483 else if (type == DWC_PCIE_TIME_BASE_EVENT)
484 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
485 }
486
dwc_pcie_pmu_event_stop(struct perf_event * event,int flags)487 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
488 {
489 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
490 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
491 struct hw_perf_event *hwc = &event->hw;
492
493 if (event->hw.state & PERF_HES_STOPPED)
494 return;
495
496 dwc_pcie_pmu_event_update(event);
497
498 if (type == DWC_PCIE_LANE_EVENT)
499 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false);
500 else if (type == DWC_PCIE_TIME_BASE_EVENT)
501 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
502
503 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
504 }
505
dwc_pcie_pmu_event_add(struct perf_event * event,int flags)506 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
507 {
508 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
509 struct pci_dev *pdev = pcie_pmu->pdev;
510 struct hw_perf_event *hwc = &event->hw;
511 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
512 int event_id = DWC_PCIE_EVENT_ID(event);
513 int lane = DWC_PCIE_EVENT_LANE(event);
514 u16 ras_des_offset = pcie_pmu->ras_des_offset;
515 u32 ctrl;
516
517 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
518
519 if (type == DWC_PCIE_LANE_EVENT) {
520 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
521 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
522 DWC_PCIE_LANE_GROUP_6;
523
524 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
525 pcie_pmu->lane_events))
526 return -ENOSPC;
527
528 /* EVENT_COUNTER_DATA_REG needs clear manually */
529 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
530 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
531 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
532 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
533 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
534 ctrl);
535 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
536 if (pcie_pmu->time_based_event)
537 return -ENOSPC;
538
539 pcie_pmu->time_based_event = event;
540
541 /*
542 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
543 * use it with any manually controlled duration. And it is
544 * cleared when next measurement starts.
545 */
546 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
547 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
548 DWC_PCIE_DURATION_MANUAL_CTL) |
549 DWC_PCIE_TIME_BASED_CNT_ENABLE;
550 pci_write_config_dword(
551 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
552 }
553
554 if (flags & PERF_EF_START)
555 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
556
557 perf_event_update_userpage(event);
558
559 return 0;
560 }
561
dwc_pcie_pmu_event_del(struct perf_event * event,int flags)562 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
563 {
564 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
565 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
566
567 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
568 perf_event_update_userpage(event);
569
570 if (type == DWC_PCIE_TIME_BASE_EVENT) {
571 pcie_pmu->time_based_event = NULL;
572 } else {
573 int event_id = DWC_PCIE_EVENT_ID(event);
574 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
575 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
576 DWC_PCIE_LANE_GROUP_6;
577
578 clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
579 pcie_pmu->lane_events);
580 }
581 }
582
dwc_pcie_pmu_remove_cpuhp_instance(void * hotplug_node)583 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
584 {
585 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
586 }
587
588 /*
589 * Find the binded DES capability device info of a PCI device.
590 * @pdev: The PCI device.
591 */
dwc_pcie_find_dev_info(struct pci_dev * pdev)592 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
593 {
594 struct dwc_pcie_dev_info *dev_info;
595
596 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
597 if (dev_info->pdev == pdev)
598 return dev_info;
599
600 return NULL;
601 }
602
dwc_pcie_unregister_pmu(void * data)603 static void dwc_pcie_unregister_pmu(void *data)
604 {
605 struct dwc_pcie_pmu *pcie_pmu = data;
606
607 perf_pmu_unregister(&pcie_pmu->pmu);
608 }
609
dwc_pcie_des_cap(struct pci_dev * pdev)610 static u16 dwc_pcie_des_cap(struct pci_dev *pdev)
611 {
612 const struct dwc_pcie_vsec_id *vid;
613 u16 vsec;
614 u32 val;
615
616 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
617 return 0;
618
619 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) {
620 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
621 vid->vsec_id);
622 if (vsec) {
623 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER,
624 &val);
625 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) {
626 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
627 return vsec;
628 }
629 }
630 }
631 return 0;
632 }
633
dwc_pcie_unregister_dev(struct dwc_pcie_dev_info * dev_info)634 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
635 {
636 platform_device_unregister(dev_info->plat_dev);
637 list_del(&dev_info->dev_node);
638 kfree(dev_info);
639 }
640
dwc_pcie_register_dev(struct pci_dev * pdev)641 static int dwc_pcie_register_dev(struct pci_dev *pdev)
642 {
643 struct platform_device *plat_dev;
644 struct dwc_pcie_dev_info *dev_info;
645 u32 sbdf;
646
647 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
648 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0);
649 if (IS_ERR(plat_dev))
650 return PTR_ERR(plat_dev);
651
652 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
653 if (!dev_info) {
654 platform_device_unregister(plat_dev);
655 return -ENOMEM;
656 }
657
658 /* Cache platform device to handle pci device hotplug */
659 dev_info->plat_dev = plat_dev;
660 dev_info->pdev = pdev;
661 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
662
663 return 0;
664 }
665
dwc_pcie_pmu_notifier(struct notifier_block * nb,unsigned long action,void * data)666 static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
667 unsigned long action, void *data)
668 {
669 struct device *dev = data;
670 struct pci_dev *pdev = to_pci_dev(dev);
671 struct dwc_pcie_dev_info *dev_info;
672
673 switch (action) {
674 case BUS_NOTIFY_ADD_DEVICE:
675 if (!dwc_pcie_des_cap(pdev))
676 return NOTIFY_DONE;
677 if (dwc_pcie_register_dev(pdev))
678 return NOTIFY_BAD;
679 break;
680 case BUS_NOTIFY_DEL_DEVICE:
681 dev_info = dwc_pcie_find_dev_info(pdev);
682 if (!dev_info)
683 return NOTIFY_DONE;
684 dwc_pcie_unregister_dev(dev_info);
685 break;
686 }
687
688 return NOTIFY_OK;
689 }
690
691 static struct notifier_block dwc_pcie_pmu_nb = {
692 .notifier_call = dwc_pcie_pmu_notifier,
693 };
694
dwc_pcie_pmu_probe(struct platform_device * plat_dev)695 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
696 {
697 struct pci_dev *pdev;
698 struct dwc_pcie_pmu *pcie_pmu;
699 char *name;
700 u32 sbdf;
701 u16 vsec;
702 int ret;
703
704 sbdf = plat_dev->id;
705 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff),
706 sbdf & 0xff);
707 if (!pdev) {
708 pr_err("No pdev found for the sbdf 0x%x\n", sbdf);
709 return -ENODEV;
710 }
711
712 vsec = dwc_pcie_des_cap(pdev);
713 if (!vsec)
714 return -ENODEV;
715
716 pci_dev_put(pdev);
717 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
718 if (!name)
719 return -ENOMEM;
720
721 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
722 if (!pcie_pmu)
723 return -ENOMEM;
724
725 pcie_pmu->pdev = pdev;
726 pcie_pmu->ras_des_offset = vsec;
727 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
728 pcie_pmu->on_cpu = -1;
729 pcie_pmu->pmu = (struct pmu){
730 .name = name,
731 .parent = &plat_dev->dev,
732 .module = THIS_MODULE,
733 .attr_groups = dwc_pcie_attr_groups,
734 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
735 .task_ctx_nr = perf_invalid_context,
736 .event_init = dwc_pcie_pmu_event_init,
737 .add = dwc_pcie_pmu_event_add,
738 .del = dwc_pcie_pmu_event_del,
739 .start = dwc_pcie_pmu_event_start,
740 .stop = dwc_pcie_pmu_event_stop,
741 .read = dwc_pcie_pmu_event_update,
742 };
743
744 /* Add this instance to the list used by the offline callback */
745 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
746 &pcie_pmu->cpuhp_node);
747 if (ret) {
748 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
749 return ret;
750 }
751
752 /* Unwind when platform driver removes */
753 ret = devm_add_action_or_reset(&plat_dev->dev,
754 dwc_pcie_pmu_remove_cpuhp_instance,
755 &pcie_pmu->cpuhp_node);
756 if (ret)
757 return ret;
758
759 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
760 if (ret) {
761 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
762 return ret;
763 }
764 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
765 pcie_pmu);
766 if (ret)
767 return ret;
768
769 return 0;
770 }
771
dwc_pcie_pmu_online_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)772 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
773 {
774 struct dwc_pcie_pmu *pcie_pmu;
775
776 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
777 if (pcie_pmu->on_cpu == -1)
778 pcie_pmu->on_cpu = cpumask_local_spread(
779 0, dev_to_node(&pcie_pmu->pdev->dev));
780
781 return 0;
782 }
783
dwc_pcie_pmu_offline_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)784 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
785 {
786 struct dwc_pcie_pmu *pcie_pmu;
787 struct pci_dev *pdev;
788 unsigned int target;
789 int node;
790
791 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
792 /* Nothing to do if this CPU doesn't own the PMU */
793 if (cpu != pcie_pmu->on_cpu)
794 return 0;
795
796 pcie_pmu->on_cpu = -1;
797 pdev = pcie_pmu->pdev;
798 node = dev_to_node(&pdev->dev);
799
800 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
801 if (target >= nr_cpu_ids)
802 target = cpumask_any_but(cpu_online_mask, cpu);
803
804 if (target >= nr_cpu_ids) {
805 pci_err(pdev, "There is no CPU to set\n");
806 return 0;
807 }
808
809 /* This PMU does NOT support interrupt, just migrate context. */
810 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
811 pcie_pmu->on_cpu = target;
812
813 return 0;
814 }
815
816 static struct platform_driver dwc_pcie_pmu_driver = {
817 .probe = dwc_pcie_pmu_probe,
818 .driver = {.name = "dwc_pcie_pmu",},
819 };
820
dwc_pcie_cleanup_devices(void)821 static void dwc_pcie_cleanup_devices(void)
822 {
823 struct dwc_pcie_dev_info *dev_info, *tmp;
824
825 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) {
826 dwc_pcie_unregister_dev(dev_info);
827 }
828 }
829
dwc_pcie_pmu_init(void)830 static int __init dwc_pcie_pmu_init(void)
831 {
832 struct pci_dev *pdev = NULL;
833 int ret;
834
835 for_each_pci_dev(pdev) {
836 if (!dwc_pcie_des_cap(pdev))
837 continue;
838
839 ret = dwc_pcie_register_dev(pdev);
840 if (ret) {
841 pci_dev_put(pdev);
842 goto err_cleanup;
843 }
844 }
845
846 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
847 "perf/dwc_pcie_pmu:online",
848 dwc_pcie_pmu_online_cpu,
849 dwc_pcie_pmu_offline_cpu);
850 if (ret < 0)
851 goto err_cleanup;
852
853 dwc_pcie_pmu_hp_state = ret;
854
855 ret = platform_driver_register(&dwc_pcie_pmu_driver);
856 if (ret)
857 goto err_remove_cpuhp;
858
859 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
860 if (ret)
861 goto err_unregister_driver;
862 notify = true;
863
864 return 0;
865
866 err_unregister_driver:
867 platform_driver_unregister(&dwc_pcie_pmu_driver);
868 err_remove_cpuhp:
869 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
870 err_cleanup:
871 dwc_pcie_cleanup_devices();
872 return ret;
873 }
874
dwc_pcie_pmu_exit(void)875 static void __exit dwc_pcie_pmu_exit(void)
876 {
877 if (notify)
878 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
879 dwc_pcie_cleanup_devices();
880 platform_driver_unregister(&dwc_pcie_pmu_driver);
881 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
882 }
883
884 module_init(dwc_pcie_pmu_init);
885 module_exit(dwc_pcie_pmu_exit);
886
887 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
888 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
889 MODULE_LICENSE("GPL v2");
890