1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/bitops.h>
10 #include <linux/cpuhotplug.h>
11 #include <linux/cpumask.h>
12 #include <linux/device.h>
13 #include <linux/errno.h>
14 #include <linux/hrtimer.h>
15 #include <linux/kernel.h>
16 #include <linux/list.h>
17 #include <linux/pcie-dwc.h>
18 #include <linux/perf_event.h>
19 #include <linux/pci.h>
20 #include <linux/platform_device.h>
21 #include <linux/smp.h>
22 #include <linux/sysfs.h>
23 #include <linux/types.h>
24
25 #define DWC_PCIE_EVENT_CNT_CTL 0x8
26
27 /*
28 * Event Counter Data Select includes two parts:
29 * - 27-24: Group number(4-bit: 0..0x7)
30 * - 23-16: Event number(8-bit: 0..0x13) within the Group
31 *
32 * Put them together as in TRM.
33 */
34 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
35 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
36 #define DWC_PCIE_CNT_STATUS BIT(7)
37 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
38 #define DWC_PCIE_PER_EVENT_OFF 0x1
39 #define DWC_PCIE_PER_EVENT_ON 0x3
40 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
41 #define DWC_PCIE_EVENT_PER_CLEAR 0x1
42
43 /* Event Selection Field has two subfields */
44 #define DWC_PCIE_CNT_EVENT_SEL_GROUP GENMASK(11, 8)
45 #define DWC_PCIE_CNT_EVENT_SEL_EVID GENMASK(7, 0)
46
47 #define DWC_PCIE_EVENT_CNT_DATA 0xC
48
49 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
50 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
51 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
52 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0
53 #define DWC_PCIE_DURATION_1MS 0x1
54 #define DWC_PCIE_DURATION_10MS 0x2
55 #define DWC_PCIE_DURATION_100MS 0x3
56 #define DWC_PCIE_DURATION_1S 0x4
57 #define DWC_PCIE_DURATION_2S 0x5
58 #define DWC_PCIE_DURATION_4S 0x6
59 #define DWC_PCIE_DURATION_4US 0xFF
60 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
61 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
62
63 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
64 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
65
66 /* Event attributes */
67 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
68 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
69 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
70
71 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
72 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
73 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
74
75 enum dwc_pcie_event_type {
76 DWC_PCIE_TIME_BASE_EVENT,
77 DWC_PCIE_LANE_EVENT,
78 DWC_PCIE_EVENT_TYPE_MAX,
79 };
80
81 #define DWC_PCIE_LANE_GROUP_6 6
82 #define DWC_PCIE_LANE_GROUP_7 7
83 #define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256
84
85 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
86 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
87 #define DWC_PCIE_PMU_TIMER_PERIOD_NS (2 * NSEC_PER_SEC)
88
89 struct dwc_pcie_pmu {
90 struct pmu pmu;
91 struct pci_dev *pdev; /* Root Port device */
92 u16 ras_des_offset;
93 u32 nr_lanes;
94
95 /* Groups #6 and #7 */
96 DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
97 struct perf_event *time_based_event;
98 bool timer_enable;
99 struct hrtimer hrtimer;
100
101 struct hlist_node cpuhp_node;
102 int on_cpu;
103 };
104
105 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
106
107 static int dwc_pcie_pmu_hp_state;
108 static struct list_head dwc_pcie_dev_info_head =
109 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
110 static bool notify;
111
112 struct dwc_pcie_dev_info {
113 struct platform_device *plat_dev;
114 struct pci_dev *pdev;
115 struct list_head dev_node;
116 };
117
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)118 static ssize_t cpumask_show(struct device *dev,
119 struct device_attribute *attr,
120 char *buf)
121 {
122 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
123
124 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pcie_pmu->on_cpu)));
125 }
126 static DEVICE_ATTR_RO(cpumask);
127
128 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
129 &dev_attr_cpumask.attr,
130 NULL
131 };
132
133 static struct attribute_group dwc_pcie_cpumask_attr_group = {
134 .attrs = dwc_pcie_pmu_cpumask_attrs,
135 };
136
137 struct dwc_pcie_format_attr {
138 struct device_attribute attr;
139 u64 field;
140 int config;
141 };
142
143 PMU_FORMAT_ATTR(eventid, "config:0-15");
144 PMU_FORMAT_ATTR(type, "config:16-19");
145 PMU_FORMAT_ATTR(lane, "config:20-27");
146
147 static struct attribute *dwc_pcie_format_attrs[] = {
148 &format_attr_type.attr,
149 &format_attr_eventid.attr,
150 &format_attr_lane.attr,
151 NULL,
152 };
153
154 static struct attribute_group dwc_pcie_format_attrs_group = {
155 .name = "format",
156 .attrs = dwc_pcie_format_attrs,
157 };
158
159 struct dwc_pcie_event_attr {
160 struct device_attribute attr;
161 enum dwc_pcie_event_type type;
162 u16 eventid;
163 u8 lane;
164 };
165
dwc_pcie_event_show(struct device * dev,struct device_attribute * attr,char * buf)166 static ssize_t dwc_pcie_event_show(struct device *dev,
167 struct device_attribute *attr, char *buf)
168 {
169 struct dwc_pcie_event_attr *eattr;
170
171 eattr = container_of(attr, typeof(*eattr), attr);
172
173 if (eattr->type == DWC_PCIE_LANE_EVENT)
174 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
175 eattr->eventid, eattr->type);
176 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
177 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
178 eattr->eventid, eattr->type);
179
180 return 0;
181 }
182
183 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
184 (&((struct dwc_pcie_event_attr[]) {{ \
185 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
186 .type = _type, \
187 .eventid = _eventid, \
188 .lane = _lane, \
189 }})[0].attr.attr)
190
191 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
192 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
193 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
194 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
195
196 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
197 /* Group #0 */
198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
199 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
200 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
201 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
202 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
203 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
204 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
205 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
206 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08),
207 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09),
208
209 /* Group #1 */
210 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
211 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
212 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
213 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
214
215 /*
216 * Leave it to the user to specify the lane ID to avoid generating
217 * a list of hundreds of events.
218 */
219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
221 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
222 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
233 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
234 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
235 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
236 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
237 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
238 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
239 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
240 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
241 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
242 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
243 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
244 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
245 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
246 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
247 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
248 NULL
249 };
250
251 static const struct attribute_group dwc_pcie_event_attrs_group = {
252 .name = "events",
253 .attrs = dwc_pcie_pmu_time_event_attrs,
254 };
255
256 static const struct attribute_group *dwc_pcie_attr_groups[] = {
257 &dwc_pcie_event_attrs_group,
258 &dwc_pcie_format_attrs_group,
259 &dwc_pcie_cpumask_attr_group,
260 NULL
261 };
262
dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu * pcie_pmu,struct perf_event * event,bool enable)263 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
264 struct perf_event *event,
265 bool enable)
266 {
267 struct pci_dev *pdev = pcie_pmu->pdev;
268 u16 ras_des_offset = pcie_pmu->ras_des_offset;
269 int event_id = DWC_PCIE_EVENT_ID(event);
270 int lane = DWC_PCIE_EVENT_LANE(event);
271 u32 ctrl;
272
273 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
274 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
275 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
276
277 if (enable)
278 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
279 else
280 ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
281
282 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
283 ctrl);
284 }
285
dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)286 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
287 bool enable)
288 {
289 struct pci_dev *pdev = pcie_pmu->pdev;
290 u16 ras_des_offset = pcie_pmu->ras_des_offset;
291
292 pci_clear_and_set_config_dword(pdev,
293 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
294 DWC_PCIE_TIME_BASED_TIMER_START, enable);
295 }
296
dwc_pcie_pmu_read_lane_event_counter(struct perf_event * event)297 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
298 {
299 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
300 struct pci_dev *pdev = pcie_pmu->pdev;
301 int event_id = DWC_PCIE_EVENT_ID(event);
302 int lane = DWC_PCIE_EVENT_LANE(event);
303 u16 ras_des_offset = pcie_pmu->ras_des_offset;
304 u32 val, ctrl;
305
306 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
307 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
308 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
309 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
310 ctrl);
311 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
312
313 ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
314 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
315 ctrl);
316
317 return val;
318 }
319
dwc_pcie_pmu_read_time_based_counter(struct perf_event * event)320 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
321 {
322 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
323 struct pci_dev *pdev = pcie_pmu->pdev;
324 int event_id = DWC_PCIE_EVENT_ID(event);
325 u16 ras_des_offset = pcie_pmu->ras_des_offset;
326 u32 lo, hi, ss;
327 u64 val;
328
329 /*
330 * The 64-bit value of the data counter is spread across two
331 * registers that are not synchronized. In order to read them
332 * atomically, ensure that the high 32 bits match before and after
333 * reading the low 32 bits.
334 */
335 pci_read_config_dword(pdev,
336 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
337 do {
338 /* snapshot the high 32 bits */
339 ss = hi;
340
341 pci_read_config_dword(
342 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
343 &lo);
344 pci_read_config_dword(
345 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
346 &hi);
347 } while (hi != ss);
348
349 val = ((u64)hi << 32) | lo;
350 /*
351 * The Group#1 event measures the amount of data processed in 16-byte
352 * units. Simplify the end-user interface by multiplying the counter
353 * at the point of read.
354 */
355 if (event_id >= 0x20 && event_id <= 0x23)
356 val *= 16;
357
358 return val;
359 }
360
dwc_pcie_pmu_reset_time_based_counter(struct perf_event * event)361 static void dwc_pcie_pmu_reset_time_based_counter(struct perf_event *event)
362 {
363 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
364 struct hw_perf_event *hwc = &event->hw;
365 u64 prev;
366
367 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
368
369 /*
370 * The hardware counter is reset to zero when disabled. Synchronize
371 * prev_count so that the next event_update() computes the correct
372 * delta against the new counter baseline.
373 */
374 do {
375 prev = local64_read(&hwc->prev_count);
376 } while (local64_cmpxchg(&hwc->prev_count, prev, 0) != prev);
377
378 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
379 }
380
dwc_pcie_pmu_event_update(struct perf_event * event)381 static void dwc_pcie_pmu_event_update(struct perf_event *event)
382 {
383 struct hw_perf_event *hwc = &event->hw;
384 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
385 u64 delta, prev, now;
386
387 if (type == DWC_PCIE_LANE_EVENT) {
388 now = dwc_pcie_pmu_read_lane_event_counter(event) &
389 DWC_PCIE_LANE_EVENT_MAX_PERIOD;
390 local64_add(now, &event->count);
391 return;
392 }
393
394 do {
395 prev = local64_read(&hwc->prev_count);
396 now = dwc_pcie_pmu_read_time_based_counter(event);
397
398 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
399
400 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
401 local64_add(delta, &event->count);
402 }
403
dwc_pcie_pmu_validate_add_lane_event(struct perf_event * event,unsigned long val_lane_events[])404 static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event,
405 unsigned long val_lane_events[])
406 {
407 int event_id, event_nr, group;
408
409 event_id = DWC_PCIE_EVENT_ID(event);
410 event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
411 group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id);
412
413 if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7)
414 return -EINVAL;
415
416 group -= DWC_PCIE_LANE_GROUP_6;
417
418 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
419 val_lane_events))
420 return -EINVAL;
421
422 return 0;
423 }
424
dwc_pcie_pmu_validate_group(struct perf_event * event)425 static int dwc_pcie_pmu_validate_group(struct perf_event *event)
426 {
427 struct perf_event *sibling, *leader = event->group_leader;
428 DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
429 bool time_event = false;
430 int type;
431
432 type = DWC_PCIE_EVENT_TYPE(leader);
433 if (type == DWC_PCIE_TIME_BASE_EVENT)
434 time_event = true;
435 else
436 if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events))
437 return -ENOSPC;
438
439 for_each_sibling_event(sibling, leader) {
440 type = DWC_PCIE_EVENT_TYPE(sibling);
441 if (type == DWC_PCIE_TIME_BASE_EVENT) {
442 if (time_event)
443 return -ENOSPC;
444
445 time_event = true;
446 continue;
447 }
448
449 if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events))
450 return -ENOSPC;
451 }
452
453 return 0;
454 }
455
dwc_pcie_pmu_hrtimer_callback(struct hrtimer * hrtimer)456 static enum hrtimer_restart dwc_pcie_pmu_hrtimer_callback(struct hrtimer *hrtimer)
457 {
458 struct dwc_pcie_pmu *pcie_pmu = container_of(hrtimer, struct dwc_pcie_pmu, hrtimer);
459 struct perf_event *event = pcie_pmu->time_based_event;
460 struct hw_perf_event *hwc;
461
462 if (!event)
463 return HRTIMER_NORESTART;
464
465 hwc = &event->hw;
466 if (hwc->state & PERF_HES_STOPPED)
467 return HRTIMER_NORESTART;
468
469 dwc_pcie_pmu_event_update(event);
470 dwc_pcie_pmu_reset_time_based_counter(event);
471 hrtimer_forward_now(hrtimer, ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS));
472
473 return HRTIMER_RESTART;
474 }
475
dwc_pcie_pmu_event_init(struct perf_event * event)476 static int dwc_pcie_pmu_event_init(struct perf_event *event)
477 {
478 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
479 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
480 struct perf_event *sibling;
481 u32 lane;
482
483 if (event->attr.type != event->pmu->type)
484 return -ENOENT;
485
486 /* We don't support sampling */
487 if (is_sampling_event(event))
488 return -EINVAL;
489
490 /* We cannot support task bound events */
491 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
492 return -EINVAL;
493
494 for_each_sibling_event(sibling, event->group_leader) {
495 if (sibling->pmu != event->pmu && !is_software_event(sibling))
496 return -EINVAL;
497 }
498
499 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
500 return -EINVAL;
501
502 if (type == DWC_PCIE_LANE_EVENT) {
503 lane = DWC_PCIE_EVENT_LANE(event);
504 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
505 return -EINVAL;
506 }
507
508 if (dwc_pcie_pmu_validate_group(event))
509 return -ENOSPC;
510
511 event->cpu = pcie_pmu->on_cpu;
512
513 return 0;
514 }
515
dwc_pcie_pmu_event_start(struct perf_event * event,int flags)516 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
517 {
518 struct hw_perf_event *hwc = &event->hw;
519 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
520 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
521
522 hwc->state = 0;
523 local64_set(&hwc->prev_count, 0);
524
525 if (type == DWC_PCIE_LANE_EVENT) {
526 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true);
527 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
528 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
529 if (pcie_pmu->timer_enable)
530 hrtimer_start(&pcie_pmu->hrtimer,
531 ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS),
532 HRTIMER_MODE_REL_PINNED_HARD);
533 }
534 }
535
dwc_pcie_pmu_event_stop(struct perf_event * event,int flags)536 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
537 {
538 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
539 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
540 struct hw_perf_event *hwc = &event->hw;
541
542 if (event->hw.state & PERF_HES_STOPPED)
543 return;
544
545 dwc_pcie_pmu_event_update(event);
546
547 if (type == DWC_PCIE_LANE_EVENT) {
548 dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false);
549 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
550 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
551
552 if (pcie_pmu->timer_enable)
553 hrtimer_cancel(&pcie_pmu->hrtimer);
554 }
555
556 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
557 }
558
dwc_pcie_pmu_event_add(struct perf_event * event,int flags)559 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
560 {
561 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
562 struct pci_dev *pdev = pcie_pmu->pdev;
563 struct hw_perf_event *hwc = &event->hw;
564 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
565 int event_id = DWC_PCIE_EVENT_ID(event);
566 int lane = DWC_PCIE_EVENT_LANE(event);
567 u16 ras_des_offset = pcie_pmu->ras_des_offset;
568 u32 ctrl;
569
570 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
571
572 if (type == DWC_PCIE_LANE_EVENT) {
573 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
574 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
575 DWC_PCIE_LANE_GROUP_6;
576
577 if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
578 pcie_pmu->lane_events))
579 return -ENOSPC;
580
581 /* EVENT_COUNTER_DATA_REG needs clear manually */
582 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
583 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
584 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
585 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
586 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
587 ctrl);
588 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
589 if (pcie_pmu->time_based_event)
590 return -ENOSPC;
591
592 pcie_pmu->time_based_event = event;
593
594 /*
595 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
596 * use it with any manually controlled duration. And it is
597 * cleared when next measurement starts.
598 */
599 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
600 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
601 DWC_PCIE_DURATION_MANUAL_CTL) |
602 DWC_PCIE_TIME_BASED_CNT_ENABLE;
603 pci_write_config_dword(
604 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
605 }
606
607 if (flags & PERF_EF_START)
608 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
609
610 perf_event_update_userpage(event);
611
612 return 0;
613 }
614
dwc_pcie_pmu_event_del(struct perf_event * event,int flags)615 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
616 {
617 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
618 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
619
620 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
621 perf_event_update_userpage(event);
622
623 if (type == DWC_PCIE_TIME_BASE_EVENT) {
624 pcie_pmu->time_based_event = NULL;
625 } else {
626 int event_id = DWC_PCIE_EVENT_ID(event);
627 int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
628 int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
629 DWC_PCIE_LANE_GROUP_6;
630
631 clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
632 pcie_pmu->lane_events);
633 }
634 }
635
dwc_pcie_pmu_remove_cpuhp_instance(void * hotplug_node)636 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
637 {
638 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
639 }
640
641 /*
642 * Find the binded DES capability device info of a PCI device.
643 * @pdev: The PCI device.
644 */
dwc_pcie_find_dev_info(struct pci_dev * pdev)645 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
646 {
647 struct dwc_pcie_dev_info *dev_info;
648
649 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
650 if (dev_info->pdev == pdev)
651 return dev_info;
652
653 return NULL;
654 }
655
dwc_pcie_unregister_pmu(void * data)656 static void dwc_pcie_unregister_pmu(void *data)
657 {
658 struct dwc_pcie_pmu *pcie_pmu = data;
659
660 perf_pmu_unregister(&pcie_pmu->pmu);
661 }
662
dwc_pcie_des_cap(struct pci_dev * pdev)663 static u16 dwc_pcie_des_cap(struct pci_dev *pdev)
664 {
665 const struct dwc_pcie_vsec_id *vid;
666 u16 vsec;
667 u32 val;
668
669 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
670 return 0;
671
672 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) {
673 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
674 vid->vsec_id);
675 if (vsec) {
676 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER,
677 &val);
678 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) {
679 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
680 return vsec;
681 }
682 }
683 }
684 return 0;
685 }
686
dwc_pcie_unregister_dev(struct dwc_pcie_dev_info * dev_info)687 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
688 {
689 platform_device_unregister(dev_info->plat_dev);
690 list_del(&dev_info->dev_node);
691 kfree(dev_info);
692 }
693
dwc_pcie_register_dev(struct pci_dev * pdev)694 static int dwc_pcie_register_dev(struct pci_dev *pdev)
695 {
696 struct platform_device *plat_dev;
697 struct dwc_pcie_dev_info *dev_info;
698 u32 sbdf;
699
700 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
701 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0);
702 if (IS_ERR(plat_dev))
703 return PTR_ERR(plat_dev);
704
705 dev_info = kzalloc_obj(*dev_info);
706 if (!dev_info) {
707 platform_device_unregister(plat_dev);
708 return -ENOMEM;
709 }
710
711 /* Cache platform device to handle pci device hotplug */
712 dev_info->plat_dev = plat_dev;
713 dev_info->pdev = pdev;
714 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
715
716 return 0;
717 }
718
dwc_pcie_pmu_notifier(struct notifier_block * nb,unsigned long action,void * data)719 static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
720 unsigned long action, void *data)
721 {
722 struct device *dev = data;
723 struct pci_dev *pdev = to_pci_dev(dev);
724 struct dwc_pcie_dev_info *dev_info;
725
726 switch (action) {
727 case BUS_NOTIFY_ADD_DEVICE:
728 if (!dwc_pcie_des_cap(pdev))
729 return NOTIFY_DONE;
730 if (dwc_pcie_register_dev(pdev))
731 return NOTIFY_BAD;
732 break;
733 case BUS_NOTIFY_DEL_DEVICE:
734 dev_info = dwc_pcie_find_dev_info(pdev);
735 if (!dev_info)
736 return NOTIFY_DONE;
737 dwc_pcie_unregister_dev(dev_info);
738 break;
739 }
740
741 return NOTIFY_OK;
742 }
743
744 static struct notifier_block dwc_pcie_pmu_nb = {
745 .notifier_call = dwc_pcie_pmu_notifier,
746 };
747
dwc_pcie_pmu_probe(struct platform_device * plat_dev)748 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
749 {
750 struct pci_dev *pdev;
751 struct dwc_pcie_pmu *pcie_pmu;
752 char *name;
753 u32 sbdf;
754 u16 vsec;
755 int ret;
756
757 sbdf = plat_dev->id;
758 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff),
759 sbdf & 0xff);
760 if (!pdev) {
761 pr_err("No pdev found for the sbdf 0x%x\n", sbdf);
762 return -ENODEV;
763 }
764
765 vsec = dwc_pcie_des_cap(pdev);
766 if (!vsec)
767 return -ENODEV;
768
769 pci_dev_put(pdev);
770 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
771 if (!name)
772 return -ENOMEM;
773
774 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
775 if (!pcie_pmu)
776 return -ENOMEM;
777
778 pcie_pmu->pdev = pdev;
779 pcie_pmu->ras_des_offset = vsec;
780 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
781 pcie_pmu->on_cpu = -1;
782 hrtimer_setup(&pcie_pmu->hrtimer, dwc_pcie_pmu_hrtimer_callback,
783 CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED_HARD);
784
785 /*
786 * Use timer for updating time-based counts on platforms known
787 * to have narrowed counter.
788 */
789 if (pdev->vendor == PCI_VENDOR_ID_PICOHEART)
790 pcie_pmu->timer_enable = true;
791
792 pcie_pmu->pmu = (struct pmu){
793 .name = name,
794 .parent = &plat_dev->dev,
795 .module = THIS_MODULE,
796 .attr_groups = dwc_pcie_attr_groups,
797 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
798 .task_ctx_nr = perf_invalid_context,
799 .event_init = dwc_pcie_pmu_event_init,
800 .add = dwc_pcie_pmu_event_add,
801 .del = dwc_pcie_pmu_event_del,
802 .start = dwc_pcie_pmu_event_start,
803 .stop = dwc_pcie_pmu_event_stop,
804 .read = dwc_pcie_pmu_event_update,
805 };
806
807 /* Add this instance to the list used by the offline callback */
808 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
809 &pcie_pmu->cpuhp_node);
810 if (ret) {
811 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
812 return ret;
813 }
814
815 /* Unwind when platform driver removes */
816 ret = devm_add_action_or_reset(&plat_dev->dev,
817 dwc_pcie_pmu_remove_cpuhp_instance,
818 &pcie_pmu->cpuhp_node);
819 if (ret)
820 return ret;
821
822 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
823 if (ret) {
824 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
825 return ret;
826 }
827 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
828 pcie_pmu);
829 if (ret)
830 return ret;
831
832 return 0;
833 }
834
dwc_pcie_pmu_online_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)835 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
836 {
837 struct dwc_pcie_pmu *pcie_pmu;
838
839 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
840 if (pcie_pmu->on_cpu == -1)
841 pcie_pmu->on_cpu = cpumask_local_spread(
842 0, dev_to_node(&pcie_pmu->pdev->dev));
843
844 return 0;
845 }
846
dwc_pcie_pmu_offline_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)847 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
848 {
849 struct dwc_pcie_pmu *pcie_pmu;
850 struct pci_dev *pdev;
851 unsigned int target;
852 int node;
853
854 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
855 /* Nothing to do if this CPU doesn't own the PMU */
856 if (cpu != pcie_pmu->on_cpu)
857 return 0;
858
859 pcie_pmu->on_cpu = -1;
860 pdev = pcie_pmu->pdev;
861 node = dev_to_node(&pdev->dev);
862
863 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
864 if (target >= nr_cpu_ids)
865 target = cpumask_any_but(cpu_online_mask, cpu);
866
867 if (target >= nr_cpu_ids) {
868 pci_err(pdev, "There is no CPU to set\n");
869 return 0;
870 }
871
872 /* This PMU does NOT support interrupt, just migrate context. */
873 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
874 pcie_pmu->on_cpu = target;
875
876 return 0;
877 }
878
879 static struct platform_driver dwc_pcie_pmu_driver = {
880 .probe = dwc_pcie_pmu_probe,
881 .driver = {.name = "dwc_pcie_pmu",},
882 };
883
dwc_pcie_cleanup_devices(void)884 static void dwc_pcie_cleanup_devices(void)
885 {
886 struct dwc_pcie_dev_info *dev_info, *tmp;
887
888 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) {
889 dwc_pcie_unregister_dev(dev_info);
890 }
891 }
892
dwc_pcie_pmu_init(void)893 static int __init dwc_pcie_pmu_init(void)
894 {
895 struct pci_dev *pdev = NULL;
896 int ret;
897
898 for_each_pci_dev(pdev) {
899 if (!dwc_pcie_des_cap(pdev))
900 continue;
901
902 ret = dwc_pcie_register_dev(pdev);
903 if (ret) {
904 pci_dev_put(pdev);
905 goto err_cleanup;
906 }
907 }
908
909 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
910 "perf/dwc_pcie_pmu:online",
911 dwc_pcie_pmu_online_cpu,
912 dwc_pcie_pmu_offline_cpu);
913 if (ret < 0)
914 goto err_cleanup;
915
916 dwc_pcie_pmu_hp_state = ret;
917
918 ret = platform_driver_register(&dwc_pcie_pmu_driver);
919 if (ret)
920 goto err_remove_cpuhp;
921
922 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
923 if (ret)
924 goto err_unregister_driver;
925 notify = true;
926
927 return 0;
928
929 err_unregister_driver:
930 platform_driver_unregister(&dwc_pcie_pmu_driver);
931 err_remove_cpuhp:
932 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
933 err_cleanup:
934 dwc_pcie_cleanup_devices();
935 return ret;
936 }
937
dwc_pcie_pmu_exit(void)938 static void __exit dwc_pcie_pmu_exit(void)
939 {
940 if (notify)
941 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
942 dwc_pcie_cleanup_devices();
943 platform_driver_unregister(&dwc_pcie_pmu_driver);
944 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
945 }
946
947 module_init(dwc_pcie_pmu_init);
948 module_exit(dwc_pcie_pmu_exit);
949
950 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
951 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
952 MODULE_LICENSE("GPL v2");
953