1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
4 *
5 * Copyright (C) 2021-2024 Marvell.
6 */
7
8 #include <linux/init.h>
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/perf_event.h>
13 #include <linux/hrtimer.h>
14 #include <linux/acpi.h>
15 #include <linux/platform_device.h>
16 #include <linux/bits.h>
17
18 /* SoC variant flags for struct ddr_pmu_platform_data (mutually exclusive in pdata) */
19 #define IS_CN10K BIT(0)
20 #define IS_ODY BIT(1)
21 #define IS_CN20K BIT(2)
22
23 /* Performance Counters Operating Mode Control Registers */
24 #define CN10K_DDRC_PERF_CNT_OP_MODE_CTRL 0x8020
25 #define ODY_DDRC_PERF_CNT_OP_MODE_CTRL 0x20020
26 #define CN20K_DDRC_PERF_CNT_OP_MODE_CTRL 0x20000
27 #define OP_MODE_CTRL_VAL_MANUAL 0x1
28
29 /* Performance Counters Start Operation Control Registers */
30 #define CN10K_DDRC_PERF_CNT_START_OP_CTRL 0x8028
31 #define ODY_DDRC_PERF_CNT_START_OP_CTRL 0x200A0
32 #define CN20K_DDRC_PERF_CNT_START_OP_CTRL 0x20080
33 #define START_OP_CTRL_VAL_START 0x1ULL
34 #define START_OP_CTRL_VAL_ACTIVE 0x2
35
36 /* Performance Counters End Operation Control Registers */
37 #define CN10K_DDRC_PERF_CNT_END_OP_CTRL 0x8030
38 #define ODY_DDRC_PERF_CNT_END_OP_CTRL 0x200E0
39 #define CN20K_DDRC_PERF_CNT_END_OP_CTRL 0x200C0
40 #define END_OP_CTRL_VAL_END 0x1ULL
41
42 /* Performance Counters End Status Registers */
43 #define CN10K_DDRC_PERF_CNT_END_STATUS 0x8038
44 #define ODY_DDRC_PERF_CNT_END_STATUS 0x20120
45 #define CN20K_DDRC_PERF_CNT_END_STATUS 0x20100
46 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1
47
48 /* Performance Counters Configuration Registers */
49 #define CN10K_DDRC_PERF_CFG_BASE 0x8040
50 #define ODY_DDRC_PERF_CFG_BASE 0x20160
51 #define CN20K_DDRC_PERF_CFG_BASE 0x20140
52 #define CN20K_DDRC_PERF_CFG1_BASE 0x20180
53
54 /* 8 Generic event counter + 2 fixed event counters */
55 #define DDRC_PERF_NUM_GEN_COUNTERS 8
56 #define DDRC_PERF_NUM_FIX_COUNTERS 2
57 #define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS
58 #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)
59 #define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \
60 DDRC_PERF_NUM_FIX_COUNTERS)
61
62 /* Generic event counter registers */
63 #define DDRC_PERF_CFG(base, n) ((base) + 8 * (n))
64 #define EVENT_ENABLE BIT_ULL(63)
65
66 /* Two dedicated event counters for DDR reads and writes */
67 #define EVENT_DDR_READS 101
68 #define EVENT_DDR_WRITES 100
69
70 #define DDRC_PERF_REG(base, n) ((base) + 8 * (n))
71 /*
72 * programmable events IDs in programmable event counters.
73 * DO NOT change these event-id numbers, they are used to
74 * program event bitmap in h/w.
75 */
76
77 /* CN20K specific events */
78 #define EVENT_PERF_OP_IS_RD16 61
79 #define EVENT_PERF_OP_IS_RD32 60
80 #define EVENT_PERF_OP_IS_WR16 59
81 #define EVENT_PERF_OP_IS_WR32 58
82 #define EVENT_OP_IS_ENTER_DSM 44
83 #define EVENT_OP_IS_RFM 43
84
85
86 #define EVENT_CN20K_OP_IS_ZQLATCH 62
87 #define EVENT_CN20K_OP_IS_ZQSTART 63
88 #define EVENT_CN20K_OP_IS_TCR_MRR 50
89 #define EVENT_CN20K_OP_IS_DQSOSC_MRR 49
90 #define EVENT_CN20K_OP_IS_DQSOSC_MPC 48
91 #define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR 47
92 #define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD 46
93
94 #define EVENT_DFI_CMD_IS_RETRY 61
95 #define EVENT_RD_UC_ECC_ERROR 60
96 #define EVENT_RD_CRC_ERROR 59
97 #define EVENT_CAPAR_ERROR 58
98 #define EVENT_WR_CRC_ERROR 57
99 #define EVENT_DFI_PARITY_POISON 56
100 #define EVENT_RETRY_FIFO_FULL 46
101 #define EVENT_DFI_CYCLES 45
102
103 #define EVENT_OP_IS_ZQLATCH 55
104 #define EVENT_OP_IS_ZQSTART 54
105 #define EVENT_OP_IS_TCR_MRR 53
106 #define EVENT_OP_IS_DQSOSC_MRR 52
107 #define EVENT_OP_IS_DQSOSC_MPC 51
108 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50
109 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49
110 #define EVENT_BSM_STARVATION 48
111 #define EVENT_BSM_ALLOC 47
112 #define EVENT_LPR_REQ_WITH_NOCREDIT 46
113 #define EVENT_HPR_REQ_WITH_NOCREDIT 45
114 #define EVENT_OP_IS_ZQCS 44
115 #define EVENT_OP_IS_ZQCL 43
116 #define EVENT_OP_IS_LOAD_MODE 42
117 #define EVENT_OP_IS_SPEC_REF 41
118 #define EVENT_OP_IS_CRIT_REF 40
119 #define EVENT_OP_IS_REFRESH 39
120 #define EVENT_OP_IS_CAS_WCK_SUS 38
121 #define EVENT_OP_IS_CAS_WS_OFF 37
122 #define EVENT_OP_IS_CAS_WS 36
123 #define EVENT_OP_IS_ENTER_MPSM 35
124 #define EVENT_OP_IS_ENTER_POWERDOWN 31
125 #define EVENT_OP_IS_ENTER_SELFREF 27
126 #define EVENT_WAW_HAZARD 26
127 #define EVENT_RAW_HAZARD 25
128 #define EVENT_WAR_HAZARD 24
129 #define EVENT_WRITE_COMBINE 23
130 #define EVENT_RDWR_TRANSITIONS 22
131 #define EVENT_PRECHARGE_FOR_OTHER 21
132 #define EVENT_PRECHARGE_FOR_RDWR 20
133 #define EVENT_OP_IS_PRECHARGE 19
134 #define EVENT_OP_IS_MWR 18
135 #define EVENT_OP_IS_WR 17
136 #define EVENT_OP_IS_RD 16
137 #define EVENT_OP_IS_RD_ACTIVATE 15
138 #define EVENT_OP_IS_RD_OR_WR 14
139 #define EVENT_OP_IS_ACTIVATE 13
140 #define EVENT_WR_XACT_WHEN_CRITICAL 12
141 #define EVENT_LPR_XACT_WHEN_CRITICAL 11
142 #define EVENT_HPR_XACT_WHEN_CRITICAL 10
143 #define EVENT_DFI_RD_DATA_CYCLES 9
144 #define EVENT_DFI_WR_DATA_CYCLES 8
145 #define EVENT_ACT_BYPASS 7
146 #define EVENT_READ_BYPASS 6
147 #define EVENT_HIF_HI_PRI_RD 5
148 #define EVENT_HIF_RMW 4
149 #define EVENT_HIF_RD 3
150 #define EVENT_HIF_WR 2
151 #define EVENT_HIF_RD_OR_WR 1
152
153 /* Event counter value registers */
154 #define CN10K_DDRC_PERF_CNT_VALUE_BASE 0x8080
155 #define ODY_DDRC_PERF_CNT_VALUE_BASE 0x201C0
156
157 /* Fixed event counter enable/disable register */
158 #define CN10K_DDRC_PERF_CNT_FREERUN_EN 0x80C0
159 #define DDRC_PERF_FREERUN_WRITE_EN 0x1
160 #define DDRC_PERF_FREERUN_READ_EN 0x2
161
162 /* Fixed event counter control register */
163 #define CN10K_DDRC_PERF_CNT_FREERUN_CTRL 0x80C8
164 #define ODY_DDRC_PERF_CNT_FREERUN_CTRL 0x20240
165 #define DDRC_FREERUN_WRITE_CNT_CLR 0x1
166 #define DDRC_FREERUN_READ_CNT_CLR 0x2
167
168 /* Fixed event counter clear register, defined only for Odyssey */
169 #define ODY_DDRC_PERF_CNT_FREERUN_CLR 0x20248
170
171 #define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48)
172 #define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0)
173
174 /* Fixed event counter value register */
175 #define CN10K_DDRC_PERF_CNT_VALUE_WR_OP 0x80D0
176 #define CN10K_DDRC_PERF_CNT_VALUE_RD_OP 0x80D8
177 #define ODY_DDRC_PERF_CNT_VALUE_WR_OP 0x20250
178 #define ODY_DDRC_PERF_CNT_VALUE_RD_OP 0x20258
179
180 struct cn10k_ddr_pmu {
181 struct pmu pmu;
182 void __iomem *base;
183 const struct ddr_pmu_platform_data *p_data;
184 const struct ddr_pmu_ops *ops;
185 unsigned int cpu;
186 struct device *dev;
187 int active_events;
188 struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
189 struct hrtimer hrtimer;
190 struct hlist_node node;
191 };
192
193 struct ddr_pmu_ops {
194 void (*enable_read_freerun_counter)(struct cn10k_ddr_pmu *pmu,
195 bool enable);
196 void (*enable_write_freerun_counter)(struct cn10k_ddr_pmu *pmu,
197 bool enable);
198 void (*clear_read_freerun_counter)(struct cn10k_ddr_pmu *pmu);
199 void (*clear_write_freerun_counter)(struct cn10k_ddr_pmu *pmu);
200 void (*pmu_overflow_handler)(struct cn10k_ddr_pmu *pmu, int evt_idx);
201 };
202
203 #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)
204
205 struct ddr_pmu_platform_data {
206 u64 counter_overflow_val;
207 u64 counter_max_val;
208 u64 cnt_base;
209 u64 cfg_base;
210 u64 cnt_op_mode_ctrl;
211 u64 cnt_start_op_ctrl;
212 u64 cnt_end_op_ctrl;
213 u64 cnt_end_status;
214 u64 cnt_freerun_en;
215 u64 cnt_freerun_ctrl;
216 u64 cnt_freerun_clr;
217 u64 cnt_value_wr_op;
218 u64 cnt_value_rd_op;
219 u64 cfg1_base;
220 unsigned int silicon_flags; /* IS_CN10K, IS_ODY, or IS_CN20K */
221 };
222
cn10k_ddr_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)223 static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
224 struct device_attribute *attr,
225 char *page)
226 {
227 struct perf_pmu_events_attr *pmu_attr;
228
229 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
230 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
231
232 }
233
234 #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \
235 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)
236
237 static struct attribute *cn10k_ddr_perf_events_attrs[] = {
238 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
239 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
240 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
241 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
242 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
243 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
244 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
245 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
246 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
247 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
248 EVENT_HPR_XACT_WHEN_CRITICAL),
249 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
250 EVENT_LPR_XACT_WHEN_CRITICAL),
251 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
252 EVENT_WR_XACT_WHEN_CRITICAL),
253 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
254 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
255 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
256 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
257 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
258 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
259 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
260 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
261 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
262 EVENT_PRECHARGE_FOR_OTHER),
263 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
264 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
265 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
266 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
267 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
268 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
269 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
270 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
271 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
272 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
273 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
274 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
275 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
276 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
277 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
278 EVENT_HPR_REQ_WITH_NOCREDIT),
279 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
280 EVENT_LPR_REQ_WITH_NOCREDIT),
281 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
282 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
283 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
284 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
285 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
286 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
287 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
288 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
289 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
290 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
291 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
292 /* Free run event counters */
293 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
294 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
295 NULL
296 };
297
298 static struct attribute *odyssey_ddr_perf_events_attrs[] = {
299 /* Programmable */
300 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
301 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
302 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
303 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
304 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
305 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
306 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
307 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access,
308 EVENT_DFI_WR_DATA_CYCLES),
309 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access,
310 EVENT_DFI_RD_DATA_CYCLES),
311 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
312 EVENT_HPR_XACT_WHEN_CRITICAL),
313 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
314 EVENT_LPR_XACT_WHEN_CRITICAL),
315 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
316 EVENT_WR_XACT_WHEN_CRITICAL),
317 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
318 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access,
319 EVENT_OP_IS_RD_OR_WR),
320 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access,
321 EVENT_OP_IS_RD_ACTIVATE),
322 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
323 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
324 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
325 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
326 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr,
327 EVENT_PRECHARGE_FOR_RDWR),
328 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
329 EVENT_PRECHARGE_FOR_OTHER),
330 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
331 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
332 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
333 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
334 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
335 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
336 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown,
337 EVENT_OP_IS_ENTER_POWERDOWN),
338 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
339 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
340 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
341 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
342 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
343 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
344 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
345 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES),
346 CN10K_DDR_PMU_EVENT_ATTR(ddr_retry_fifo_full,
347 EVENT_RETRY_FIFO_FULL),
348 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
349 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
350 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
351 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
352 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
353 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
354 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
355 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
356 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
357 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
358 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
359 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_parity_poison,
360 EVENT_DFI_PARITY_POISON),
361 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_crc_error, EVENT_WR_CRC_ERROR),
362 CN10K_DDR_PMU_EVENT_ATTR(ddr_capar_error, EVENT_CAPAR_ERROR),
363 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_crc_error, EVENT_RD_CRC_ERROR),
364 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_uc_ecc_error, EVENT_RD_UC_ECC_ERROR),
365 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cmd_is_retry, EVENT_DFI_CMD_IS_RETRY),
366 /* Free run event counters */
367 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
368 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
369 NULL
370 };
371
372 static struct attribute *cn20k_ddr_perf_events_attrs[] = {
373 /* Programmable */
374 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
375 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
376 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
377 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
378 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
379 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
380 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
381 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access,
382 EVENT_DFI_WR_DATA_CYCLES),
383 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access,
384 EVENT_DFI_RD_DATA_CYCLES),
385 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
386 EVENT_HPR_XACT_WHEN_CRITICAL),
387 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
388 EVENT_LPR_XACT_WHEN_CRITICAL),
389 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
390 EVENT_WR_XACT_WHEN_CRITICAL),
391 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
392 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access,
393 EVENT_OP_IS_RD_OR_WR),
394 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access,
395 EVENT_OP_IS_RD_ACTIVATE),
396 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
397 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
398 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
399 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
400 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr,
401 EVENT_PRECHARGE_FOR_RDWR),
402 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
403 EVENT_PRECHARGE_FOR_OTHER),
404 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
405 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
406 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
407 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
408 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
409 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
410 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown,
411 EVENT_OP_IS_ENTER_POWERDOWN),
412 CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws, EVENT_OP_IS_CAS_WS),
413 CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws_off, EVENT_OP_IS_CAS_WS_OFF),
414 CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_wck_sus, EVENT_OP_IS_CAS_WCK_SUS),
415 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
416 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
417 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
418 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
419 CN10K_DDR_PMU_EVENT_ATTR(ddr_rfm, EVENT_OP_IS_RFM),
420 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_dsm, EVENT_OP_IS_ENTER_DSM),
421 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES),
422 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
423 EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD),
424 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
425 EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR),
426 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_CN20K_OP_IS_DQSOSC_MPC),
427 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_CN20K_OP_IS_DQSOSC_MRR),
428 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_CN20K_OP_IS_TCR_MRR),
429 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_CN20K_OP_IS_ZQSTART),
430 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_CN20K_OP_IS_ZQLATCH),
431 CN10K_DDR_PMU_EVENT_ATTR(ddr_read16, EVENT_PERF_OP_IS_RD16),
432 CN10K_DDR_PMU_EVENT_ATTR(ddr_read32, EVENT_PERF_OP_IS_RD32),
433 CN10K_DDR_PMU_EVENT_ATTR(ddr_write16, EVENT_PERF_OP_IS_WR16),
434 CN10K_DDR_PMU_EVENT_ATTR(ddr_write32, EVENT_PERF_OP_IS_WR32),
435 /* Free run event counters */
436 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
437 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
438 NULL
439 };
440
441 static struct attribute_group cn20k_ddr_perf_events_attr_group = {
442 .name = "events",
443 .attrs = cn20k_ddr_perf_events_attrs,
444 };
445
446 static struct attribute_group odyssey_ddr_perf_events_attr_group = {
447 .name = "events",
448 .attrs = odyssey_ddr_perf_events_attrs,
449 };
450
451 static struct attribute_group cn10k_ddr_perf_events_attr_group = {
452 .name = "events",
453 .attrs = cn10k_ddr_perf_events_attrs,
454 };
455
456 PMU_FORMAT_ATTR(event, "config:0-8");
457
458 static struct attribute *cn10k_ddr_perf_format_attrs[] = {
459 &format_attr_event.attr,
460 NULL,
461 };
462
463 static struct attribute_group cn10k_ddr_perf_format_attr_group = {
464 .name = "format",
465 .attrs = cn10k_ddr_perf_format_attrs,
466 };
467
cn10k_ddr_perf_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)468 static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
469 struct device_attribute *attr,
470 char *buf)
471 {
472 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
473
474 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
475 }
476
477 static struct device_attribute cn10k_ddr_perf_cpumask_attr =
478 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);
479
480 static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
481 &cn10k_ddr_perf_cpumask_attr.attr,
482 NULL,
483 };
484
485 static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
486 .attrs = cn10k_ddr_perf_cpumask_attrs,
487 };
488
489 static const struct attribute_group *cn10k_attr_groups[] = {
490 &cn10k_ddr_perf_events_attr_group,
491 &cn10k_ddr_perf_format_attr_group,
492 &cn10k_ddr_perf_cpumask_attr_group,
493 NULL,
494 };
495
496 static const struct attribute_group *odyssey_attr_groups[] = {
497 &odyssey_ddr_perf_events_attr_group,
498 &cn10k_ddr_perf_format_attr_group,
499 &cn10k_ddr_perf_cpumask_attr_group,
500 NULL
501 };
502
503 static const struct attribute_group *cn20k_attr_groups[] = {
504 &cn20k_ddr_perf_events_attr_group,
505 &cn10k_ddr_perf_format_attr_group,
506 &cn10k_ddr_perf_cpumask_attr_group,
507 NULL
508 };
509
510 /* Default poll timeout is 100 sec, which is very sufficient for
511 * 48 bit counter incremented max at 5.6 GT/s, which may take many
512 * hours to overflow.
513 */
514 static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
515 module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);
516
cn10k_ddr_pmu_timer_period(void)517 static ktime_t cn10k_ddr_pmu_timer_period(void)
518 {
519 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
520 }
521
ddr_perf_get_event_bitmap(int eventid,u64 * event_bitmap,struct cn10k_ddr_pmu * ddr_pmu)522 static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap,
523 struct cn10k_ddr_pmu *ddr_pmu)
524 {
525 int err = 0;
526
527 switch (eventid) {
528 case EVENT_CN20K_OP_IS_ZQLATCH ... EVENT_CN20K_OP_IS_ZQSTART:
529 if (ddr_pmu->p_data->silicon_flags & IS_CN20K) {
530 *event_bitmap = (1ULL << (eventid - 42));
531 break;
532 }
533 err = -EINVAL;
534 break;
535 case EVENT_DFI_PARITY_POISON ...EVENT_DFI_CMD_IS_RETRY:
536 /*
537 * 58..61: CN20K perf width events share numeric IDs with Odyssey
538 * DFI events; same 1ULL << (eventid - 1) bitmap on both paths.
539 */
540 if (eventid >= EVENT_PERF_OP_IS_WR32 &&
541 eventid <= EVENT_PERF_OP_IS_RD16) {
542 if (ddr_pmu->p_data->silicon_flags & IS_CN20K) {
543 *event_bitmap = (1ULL << (eventid - 1));
544 break;
545 }
546 if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) {
547 err = -EINVAL;
548 break;
549 }
550 *event_bitmap = (1ULL << (eventid - 1));
551 break;
552 }
553 if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) {
554 err = -EINVAL;
555 break;
556 }
557 fallthrough;
558 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
559 case EVENT_OP_IS_CAS_WS ... EVENT_OP_IS_ZQLATCH:
560 *event_bitmap = (1ULL << (eventid - 1));
561 break;
562 case EVENT_OP_IS_ENTER_SELFREF:
563 case EVENT_OP_IS_ENTER_POWERDOWN:
564 case EVENT_OP_IS_ENTER_MPSM:
565 *event_bitmap = (0xFULL << (eventid - 1));
566 break;
567 default:
568 err = -EINVAL;
569 }
570
571 if (err)
572 pr_err("%s Invalid eventid %d\n", __func__, eventid);
573 return err;
574 }
575
cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu * pmu,struct perf_event * event)576 static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
577 struct perf_event *event)
578 {
579 u8 config = event->attr.config;
580 int i;
581
582 /* DDR read free-run counter index */
583 if (config == EVENT_DDR_READS) {
584 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
585 return DDRC_PERF_READ_COUNTER_IDX;
586 }
587
588 /* DDR write free-run counter index */
589 if (config == EVENT_DDR_WRITES) {
590 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
591 return DDRC_PERF_WRITE_COUNTER_IDX;
592 }
593
594 /* Allocate DDR generic counters */
595 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
596 if (pmu->events[i] == NULL) {
597 pmu->events[i] = event;
598 return i;
599 }
600 }
601
602 return -ENOENT;
603 }
604
cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu * pmu,int counter)605 static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
606 {
607 pmu->events[counter] = NULL;
608 }
609
cn10k_ddr_perf_event_init(struct perf_event * event)610 static int cn10k_ddr_perf_event_init(struct perf_event *event)
611 {
612 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
613 struct hw_perf_event *hwc = &event->hw;
614
615 if (event->attr.type != event->pmu->type)
616 return -ENOENT;
617
618 if (is_sampling_event(event)) {
619 dev_info(pmu->dev, "Sampling not supported!\n");
620 return -EOPNOTSUPP;
621 }
622
623 if (event->cpu < 0) {
624 dev_warn(pmu->dev, "Can't provide per-task data!\n");
625 return -EOPNOTSUPP;
626 }
627
628 /* We must NOT create groups containing mixed PMUs */
629 if (event->group_leader->pmu != event->pmu &&
630 !is_software_event(event->group_leader))
631 return -EINVAL;
632
633 /* Set ownership of event to one CPU, same event can not be observed
634 * on multiple cpus at same time.
635 */
636 event->cpu = pmu->cpu;
637 hwc->idx = -1;
638 return 0;
639 }
640
cn10k_ddr_perf_counter_start(struct cn10k_ddr_pmu * ddr_pmu,int counter)641 static void cn10k_ddr_perf_counter_start(struct cn10k_ddr_pmu *ddr_pmu,
642 int counter)
643 {
644 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
645 u64 ctrl_reg = p_data->cnt_start_op_ctrl;
646
647 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
648 DDRC_PERF_REG(ctrl_reg, counter));
649 }
650
cn10k_ddr_perf_counter_stop(struct cn10k_ddr_pmu * ddr_pmu,int counter)651 static void cn10k_ddr_perf_counter_stop(struct cn10k_ddr_pmu *ddr_pmu,
652 int counter)
653 {
654 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
655 u64 ctrl_reg = p_data->cnt_end_op_ctrl;
656
657 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
658 DDRC_PERF_REG(ctrl_reg, counter));
659 }
660
cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu * pmu,int counter,bool enable)661 static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
662 int counter, bool enable)
663 {
664 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
665 unsigned int silicon_flags = pmu->p_data->silicon_flags;
666 u64 ctrl_reg = pmu->p_data->cnt_op_mode_ctrl;
667 const struct ddr_pmu_ops *ops = pmu->ops;
668 u32 reg;
669 u64 val;
670
671 if (counter > DDRC_PERF_NUM_COUNTERS) {
672 pr_err("Error: unsupported counter %d\n", counter);
673 return;
674 }
675
676 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
677 reg = DDRC_PERF_CFG(p_data->cfg_base, counter);
678 val = readq_relaxed(pmu->base + reg);
679
680 if (enable)
681 val |= EVENT_ENABLE;
682 else
683 val &= ~EVENT_ENABLE;
684
685 writeq_relaxed(val, pmu->base + reg);
686
687 if ((silicon_flags & IS_ODY) || (silicon_flags & IS_CN20K)) {
688 if (enable) {
689 /*
690 * Setup the PMU counter to work in
691 * manual mode
692 */
693 reg = DDRC_PERF_REG(ctrl_reg, counter);
694 writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL,
695 pmu->base + reg);
696
697 cn10k_ddr_perf_counter_start(pmu, counter);
698 } else {
699 cn10k_ddr_perf_counter_stop(pmu, counter);
700 }
701 }
702 } else {
703 if (counter == DDRC_PERF_READ_COUNTER_IDX)
704 ops->enable_read_freerun_counter(pmu, enable);
705 else
706 ops->enable_write_freerun_counter(pmu, enable);
707 }
708 }
709
cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu * pmu,int counter)710 static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
711 {
712 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
713 u64 val;
714
715 if (counter == DDRC_PERF_READ_COUNTER_IDX)
716 return readq_relaxed(pmu->base +
717 p_data->cnt_value_rd_op);
718
719 if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
720 return readq_relaxed(pmu->base +
721 p_data->cnt_value_wr_op);
722
723 val = readq_relaxed(pmu->base +
724 DDRC_PERF_REG(p_data->cnt_base, counter));
725 return val;
726 }
727
cn10k_ddr_perf_event_update(struct perf_event * event)728 static void cn10k_ddr_perf_event_update(struct perf_event *event)
729 {
730 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
731 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
732 struct hw_perf_event *hwc = &event->hw;
733 u64 prev_count, new_count, mask;
734
735 do {
736 prev_count = local64_read(&hwc->prev_count);
737 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
738 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);
739
740 mask = p_data->counter_max_val;
741
742 local64_add((new_count - prev_count) & mask, &event->count);
743 }
744
cn10k_ddr_perf_event_start(struct perf_event * event,int flags)745 static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
746 {
747 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
748 struct hw_perf_event *hwc = &event->hw;
749 int counter = hwc->idx;
750
751 local64_set(&hwc->prev_count, 0);
752
753 cn10k_ddr_perf_counter_enable(pmu, counter, true);
754
755 hwc->state = 0;
756 }
757
cn10k_ddr_perf_event_add(struct perf_event * event,int flags)758 static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
759 {
760 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
761 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
762 unsigned int silicon_flags = pmu->p_data->silicon_flags;
763 const struct ddr_pmu_ops *ops = pmu->ops;
764 struct hw_perf_event *hwc = &event->hw;
765 u8 config = event->attr.config;
766 int counter, ret;
767 u32 reg_offset;
768 u64 val;
769
770 counter = cn10k_ddr_perf_alloc_counter(pmu, event);
771 if (counter < 0)
772 return -EAGAIN;
773
774 pmu->active_events++;
775 hwc->idx = counter;
776
777 if (pmu->active_events == 1)
778 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
779 HRTIMER_MODE_REL_PINNED);
780
781 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
782 /* Generic counters, configure event id */
783 reg_offset = DDRC_PERF_CFG(p_data->cfg_base, counter);
784
785 ret = ddr_perf_get_event_bitmap(config, &val, pmu);
786 if (ret)
787 goto err_free_counter;
788
789 if (silicon_flags & IS_CN20K) {
790 if (config == EVENT_CN20K_OP_IS_ZQSTART ||
791 config == EVENT_CN20K_OP_IS_ZQLATCH) {
792 /* ZQ lives in CFG1; clear stale event mask in CFG0 */
793 writeq_relaxed(0, pmu->base +
794 DDRC_PERF_CFG(p_data->cfg_base,
795 counter));
796 reg_offset = DDRC_PERF_CFG(p_data->cfg1_base,
797 counter);
798 } else {
799 /* Clear CFG1 so a prior ZQ select cannot linger */
800 writeq_relaxed(0, pmu->base +
801 DDRC_PERF_CFG(p_data->cfg1_base,
802 counter));
803 }
804 }
805 writeq_relaxed(val, pmu->base + reg_offset);
806 } else {
807 /* fixed event counter, clear counter value */
808 if (counter == DDRC_PERF_READ_COUNTER_IDX)
809 ops->clear_read_freerun_counter(pmu);
810 else
811 ops->clear_write_freerun_counter(pmu);
812 }
813
814 hwc->state |= PERF_HES_STOPPED;
815
816 if (flags & PERF_EF_START)
817 cn10k_ddr_perf_event_start(event, flags);
818
819 return 0;
820
821 err_free_counter:
822 if (pmu->active_events == 1)
823 hrtimer_cancel(&pmu->hrtimer);
824 pmu->active_events--;
825 cn10k_ddr_perf_free_counter(pmu, counter);
826 hwc->idx = -1;
827 return ret;
828 }
829
cn10k_ddr_perf_event_stop(struct perf_event * event,int flags)830 static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
831 {
832 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
833 struct hw_perf_event *hwc = &event->hw;
834 int counter = hwc->idx;
835
836 cn10k_ddr_perf_counter_enable(pmu, counter, false);
837
838 if (flags & PERF_EF_UPDATE)
839 cn10k_ddr_perf_event_update(event);
840
841 hwc->state |= PERF_HES_STOPPED;
842 }
843
cn10k_ddr_perf_event_del(struct perf_event * event,int flags)844 static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
845 {
846 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
847 struct hw_perf_event *hwc = &event->hw;
848 int counter = hwc->idx;
849
850 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);
851
852 cn10k_ddr_perf_free_counter(pmu, counter);
853 pmu->active_events--;
854 hwc->idx = -1;
855
856 /* Cancel timer when no events to capture */
857 if (pmu->active_events == 0)
858 hrtimer_cancel(&pmu->hrtimer);
859 }
860
cn10k_ddr_perf_pmu_enable(struct pmu * pmu)861 static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
862 {
863 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
864 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
865
866 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
867 p_data->cnt_start_op_ctrl);
868 }
869
cn10k_ddr_perf_pmu_disable(struct pmu * pmu)870 static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
871 {
872 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
873 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
874
875 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
876 p_data->cnt_end_op_ctrl);
877 }
878
cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu * pmu)879 static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
880 {
881 struct hw_perf_event *hwc;
882 int i;
883
884 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
885 if (pmu->events[i] == NULL)
886 continue;
887
888 cn10k_ddr_perf_event_update(pmu->events[i]);
889 }
890
891 /* Reset previous count as h/w counter are reset */
892 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
893 if (pmu->events[i] == NULL)
894 continue;
895
896 hwc = &pmu->events[i]->hw;
897 local64_set(&hwc->prev_count, 0);
898 }
899 }
900
ddr_pmu_enable_read_freerun(struct cn10k_ddr_pmu * pmu,bool enable)901 static void ddr_pmu_enable_read_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
902 {
903 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
904 u64 val;
905
906 val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
907 if (enable)
908 val |= DDRC_PERF_FREERUN_READ_EN;
909 else
910 val &= ~DDRC_PERF_FREERUN_READ_EN;
911
912 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
913 }
914
ddr_pmu_enable_write_freerun(struct cn10k_ddr_pmu * pmu,bool enable)915 static void ddr_pmu_enable_write_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
916 {
917 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
918 u64 val;
919
920 val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
921 if (enable)
922 val |= DDRC_PERF_FREERUN_WRITE_EN;
923 else
924 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
925
926 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
927 }
928
ddr_pmu_read_clear_freerun(struct cn10k_ddr_pmu * pmu)929 static void ddr_pmu_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
930 {
931 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
932 u64 val;
933
934 val = DDRC_FREERUN_READ_CNT_CLR;
935 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
936 }
937
ddr_pmu_write_clear_freerun(struct cn10k_ddr_pmu * pmu)938 static void ddr_pmu_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
939 {
940 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
941 u64 val;
942
943 val = DDRC_FREERUN_WRITE_CNT_CLR;
944 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
945 }
946
ddr_pmu_overflow_hander(struct cn10k_ddr_pmu * pmu,int evt_idx)947 static void ddr_pmu_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
948 {
949 cn10k_ddr_perf_event_update_all(pmu);
950 cn10k_ddr_perf_pmu_disable(&pmu->pmu);
951 cn10k_ddr_perf_pmu_enable(&pmu->pmu);
952 }
953
ddr_pmu_ody_enable_read_freerun(struct cn10k_ddr_pmu * pmu,bool enable)954 static void ddr_pmu_ody_enable_read_freerun(struct cn10k_ddr_pmu *pmu,
955 bool enable)
956 {
957 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
958 u64 val;
959
960 val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
961 if (enable)
962 val |= DDRC_PERF_FREERUN_READ_EN;
963 else
964 val &= ~DDRC_PERF_FREERUN_READ_EN;
965
966 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
967 }
968
ddr_pmu_ody_enable_write_freerun(struct cn10k_ddr_pmu * pmu,bool enable)969 static void ddr_pmu_ody_enable_write_freerun(struct cn10k_ddr_pmu *pmu,
970 bool enable)
971 {
972 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
973 u64 val;
974
975 val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
976 if (enable)
977 val |= DDRC_PERF_FREERUN_WRITE_EN;
978 else
979 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
980
981 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
982 }
983
ddr_pmu_ody_read_clear_freerun(struct cn10k_ddr_pmu * pmu)984 static void ddr_pmu_ody_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
985 {
986 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
987 u64 val;
988
989 val = DDRC_FREERUN_READ_CNT_CLR;
990 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
991 }
992
ddr_pmu_ody_write_clear_freerun(struct cn10k_ddr_pmu * pmu)993 static void ddr_pmu_ody_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
994 {
995 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
996 u64 val;
997
998 val = DDRC_FREERUN_WRITE_CNT_CLR;
999 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
1000 }
1001
ddr_pmu_ody_overflow_hander(struct cn10k_ddr_pmu * pmu,int evt_idx)1002 static void ddr_pmu_ody_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
1003 {
1004 /*
1005 * On reaching the maximum value of the counter, the counter freezes
1006 * there. The particular event is updated and the respective counter
1007 * is stopped and started again so that it starts counting from zero
1008 */
1009 cn10k_ddr_perf_event_update(pmu->events[evt_idx]);
1010 cn10k_ddr_perf_counter_stop(pmu, evt_idx);
1011 cn10k_ddr_perf_counter_start(pmu, evt_idx);
1012 }
1013
cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu * pmu)1014 static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
1015 {
1016 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
1017 const struct ddr_pmu_ops *ops = pmu->ops;
1018 struct perf_event *event;
1019 struct hw_perf_event *hwc;
1020 u64 prev_count, new_count;
1021 u64 value;
1022 int i;
1023
1024 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
1025 if (event) {
1026 hwc = &event->hw;
1027 prev_count = local64_read(&hwc->prev_count);
1028 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
1029
1030 /* Overflow condition is when new count less than
1031 * previous count
1032 */
1033 if (new_count < prev_count)
1034 cn10k_ddr_perf_event_update(event);
1035 }
1036
1037 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
1038 if (event) {
1039 hwc = &event->hw;
1040 prev_count = local64_read(&hwc->prev_count);
1041 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
1042
1043 /* Overflow condition is when new count less than
1044 * previous count
1045 */
1046 if (new_count < prev_count)
1047 cn10k_ddr_perf_event_update(event);
1048 }
1049
1050 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
1051 if (pmu->events[i] == NULL)
1052 continue;
1053
1054 value = cn10k_ddr_perf_read_counter(pmu, i);
1055 if (value == p_data->counter_max_val) {
1056 pr_info("Counter-(%d) reached max value\n", i);
1057 ops->pmu_overflow_handler(pmu, i);
1058 }
1059 }
1060
1061 return IRQ_HANDLED;
1062 }
1063
cn10k_ddr_pmu_timer_handler(struct hrtimer * hrtimer)1064 static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
1065 {
1066 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
1067 hrtimer);
1068 unsigned long flags;
1069
1070 local_irq_save(flags);
1071 cn10k_ddr_pmu_overflow_handler(pmu);
1072 local_irq_restore(flags);
1073
1074 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
1075 return HRTIMER_RESTART;
1076 }
1077
cn10k_ddr_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)1078 static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1079 {
1080 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
1081 node);
1082 unsigned int target;
1083
1084 if (cpu != pmu->cpu)
1085 return 0;
1086
1087 target = cpumask_any_but(cpu_online_mask, cpu);
1088 if (target >= nr_cpu_ids)
1089 return 0;
1090
1091 perf_pmu_migrate_context(&pmu->pmu, cpu, target);
1092 pmu->cpu = target;
1093 return 0;
1094 }
1095
1096 static const struct ddr_pmu_ops ddr_pmu_ops = {
1097 .enable_read_freerun_counter = ddr_pmu_enable_read_freerun,
1098 .enable_write_freerun_counter = ddr_pmu_enable_write_freerun,
1099 .clear_read_freerun_counter = ddr_pmu_read_clear_freerun,
1100 .clear_write_freerun_counter = ddr_pmu_write_clear_freerun,
1101 .pmu_overflow_handler = ddr_pmu_overflow_hander,
1102 };
1103
1104 #if defined(CONFIG_ACPI) || defined(CONFIG_OF)
1105 static const struct ddr_pmu_platform_data cn10k_ddr_pmu_pdata = {
1106 .counter_overflow_val = BIT_ULL(48),
1107 .counter_max_val = GENMASK_ULL(48, 0),
1108 .cnt_base = CN10K_DDRC_PERF_CNT_VALUE_BASE,
1109 .cfg_base = CN10K_DDRC_PERF_CFG_BASE,
1110 .cnt_op_mode_ctrl = CN10K_DDRC_PERF_CNT_OP_MODE_CTRL,
1111 .cnt_start_op_ctrl = CN10K_DDRC_PERF_CNT_START_OP_CTRL,
1112 .cnt_end_op_ctrl = CN10K_DDRC_PERF_CNT_END_OP_CTRL,
1113 .cnt_end_status = CN10K_DDRC_PERF_CNT_END_STATUS,
1114 .cnt_freerun_en = CN10K_DDRC_PERF_CNT_FREERUN_EN,
1115 .cnt_freerun_ctrl = CN10K_DDRC_PERF_CNT_FREERUN_CTRL,
1116 .cnt_freerun_clr = 0,
1117 .cnt_value_wr_op = CN10K_DDRC_PERF_CNT_VALUE_WR_OP,
1118 .cnt_value_rd_op = CN10K_DDRC_PERF_CNT_VALUE_RD_OP,
1119 .silicon_flags = IS_CN10K,
1120 };
1121
1122 static const struct ddr_pmu_platform_data cn20k_ddr_pmu_pdata = {
1123 .counter_overflow_val = 0,
1124 .counter_max_val = GENMASK_ULL(63, 0),
1125 .cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE,
1126 .cfg_base = CN20K_DDRC_PERF_CFG_BASE,
1127 .cfg1_base = CN20K_DDRC_PERF_CFG1_BASE,
1128 .cnt_op_mode_ctrl = CN20K_DDRC_PERF_CNT_OP_MODE_CTRL,
1129 .cnt_start_op_ctrl = CN20K_DDRC_PERF_CNT_START_OP_CTRL,
1130 .cnt_end_op_ctrl = CN20K_DDRC_PERF_CNT_END_OP_CTRL,
1131 .cnt_end_status = CN20K_DDRC_PERF_CNT_END_STATUS,
1132 .cnt_freerun_en = 0,
1133 .cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL,
1134 .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR,
1135 .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP,
1136 .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP,
1137 .silicon_flags = IS_CN20K,
1138 };
1139 #endif
1140
1141 static const struct ddr_pmu_ops ddr_pmu_ody_ops = {
1142 .enable_read_freerun_counter = ddr_pmu_ody_enable_read_freerun,
1143 .enable_write_freerun_counter = ddr_pmu_ody_enable_write_freerun,
1144 .clear_read_freerun_counter = ddr_pmu_ody_read_clear_freerun,
1145 .clear_write_freerun_counter = ddr_pmu_ody_write_clear_freerun,
1146 .pmu_overflow_handler = ddr_pmu_ody_overflow_hander,
1147 };
1148
1149 #ifdef CONFIG_ACPI
1150 static const struct ddr_pmu_platform_data odyssey_ddr_pmu_pdata = {
1151 .counter_overflow_val = 0,
1152 .counter_max_val = GENMASK_ULL(63, 0),
1153 .cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE,
1154 .cfg_base = ODY_DDRC_PERF_CFG_BASE,
1155 .cnt_op_mode_ctrl = ODY_DDRC_PERF_CNT_OP_MODE_CTRL,
1156 .cnt_start_op_ctrl = ODY_DDRC_PERF_CNT_START_OP_CTRL,
1157 .cnt_end_op_ctrl = ODY_DDRC_PERF_CNT_END_OP_CTRL,
1158 .cnt_end_status = ODY_DDRC_PERF_CNT_END_STATUS,
1159 .cnt_freerun_en = 0,
1160 .cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL,
1161 .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR,
1162 .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP,
1163 .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP,
1164 .silicon_flags = IS_ODY,
1165 };
1166 #endif
1167
cn10k_ddr_perf_probe(struct platform_device * pdev)1168 static int cn10k_ddr_perf_probe(struct platform_device *pdev)
1169 {
1170 const struct ddr_pmu_platform_data *dev_data;
1171 struct cn10k_ddr_pmu *ddr_pmu;
1172 struct resource *res;
1173 void __iomem *base;
1174 unsigned int silicon_flags;
1175 char *name;
1176 int ret;
1177
1178 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
1179 if (!ddr_pmu)
1180 return -ENOMEM;
1181
1182 ddr_pmu->dev = &pdev->dev;
1183 platform_set_drvdata(pdev, ddr_pmu);
1184
1185 dev_data = device_get_match_data(&pdev->dev);
1186 if (!dev_data) {
1187 dev_err(&pdev->dev, "Error: No device match data found\n");
1188 return -ENODEV;
1189 }
1190
1191 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1192 if (IS_ERR(base))
1193 return PTR_ERR(base);
1194
1195 ddr_pmu->base = base;
1196
1197 ddr_pmu->p_data = dev_data;
1198 silicon_flags = ddr_pmu->p_data->silicon_flags;
1199
1200 if (silicon_flags & IS_CN10K) {
1201 ddr_pmu->ops = &ddr_pmu_ops;
1202 /* Setup the PMU counter to work in manual mode */
1203 writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL, ddr_pmu->base +
1204 ddr_pmu->p_data->cnt_op_mode_ctrl);
1205
1206 ddr_pmu->pmu = (struct pmu) {
1207 .module = THIS_MODULE,
1208 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1209 .task_ctx_nr = perf_invalid_context,
1210 .attr_groups = cn10k_attr_groups,
1211 .event_init = cn10k_ddr_perf_event_init,
1212 .add = cn10k_ddr_perf_event_add,
1213 .del = cn10k_ddr_perf_event_del,
1214 .start = cn10k_ddr_perf_event_start,
1215 .stop = cn10k_ddr_perf_event_stop,
1216 .read = cn10k_ddr_perf_event_update,
1217 .pmu_enable = cn10k_ddr_perf_pmu_enable,
1218 .pmu_disable = cn10k_ddr_perf_pmu_disable,
1219 };
1220 }
1221
1222 if (silicon_flags & IS_ODY) {
1223 ddr_pmu->ops = &ddr_pmu_ody_ops;
1224
1225 ddr_pmu->pmu = (struct pmu) {
1226 .module = THIS_MODULE,
1227 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1228 .task_ctx_nr = perf_invalid_context,
1229 .attr_groups = odyssey_attr_groups,
1230 .event_init = cn10k_ddr_perf_event_init,
1231 .add = cn10k_ddr_perf_event_add,
1232 .del = cn10k_ddr_perf_event_del,
1233 .start = cn10k_ddr_perf_event_start,
1234 .stop = cn10k_ddr_perf_event_stop,
1235 .read = cn10k_ddr_perf_event_update,
1236 };
1237 }
1238
1239 if (silicon_flags & IS_CN20K) {
1240 ddr_pmu->ops = &ddr_pmu_ody_ops;
1241
1242 ddr_pmu->pmu = (struct pmu) {
1243 .module = THIS_MODULE,
1244 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1245 .task_ctx_nr = perf_invalid_context,
1246 .attr_groups = cn20k_attr_groups,
1247 .event_init = cn10k_ddr_perf_event_init,
1248 .add = cn10k_ddr_perf_event_add,
1249 .del = cn10k_ddr_perf_event_del,
1250 .start = cn10k_ddr_perf_event_start,
1251 .stop = cn10k_ddr_perf_event_stop,
1252 .read = cn10k_ddr_perf_event_update,
1253 };
1254 }
1255 /* Choose this cpu to collect perf data */
1256 ddr_pmu->cpu = raw_smp_processor_id();
1257
1258 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
1259 res->start);
1260 if (!name)
1261 return -ENOMEM;
1262
1263 hrtimer_setup(&ddr_pmu->hrtimer, cn10k_ddr_pmu_timer_handler, CLOCK_MONOTONIC,
1264 HRTIMER_MODE_REL);
1265
1266 cpuhp_state_add_instance_nocalls(
1267 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1268 &ddr_pmu->node);
1269
1270 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
1271 if (ret)
1272 goto error;
1273
1274 pr_info("DDR PMU Driver for ddrc@%llx\n", res->start);
1275 return 0;
1276 error:
1277 cpuhp_state_remove_instance_nocalls(
1278 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1279 &ddr_pmu->node);
1280 return ret;
1281 }
1282
cn10k_ddr_perf_remove(struct platform_device * pdev)1283 static void cn10k_ddr_perf_remove(struct platform_device *pdev)
1284 {
1285 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);
1286
1287 /*
1288 * Cancel the poll timer before further teardown so the handler
1289 * cannot run after this function returns.
1290 */
1291 hrtimer_cancel(&ddr_pmu->hrtimer);
1292
1293 cpuhp_state_remove_instance_nocalls(
1294 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1295 &ddr_pmu->node);
1296
1297 perf_pmu_unregister(&ddr_pmu->pmu);
1298 }
1299
1300 #ifdef CONFIG_OF
1301 static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
1302 { .compatible = "marvell,cn10k-ddr-pmu", .data = &cn10k_ddr_pmu_pdata },
1303 { .compatible = "marvell,cn20k-ddr-pmu", .data = &cn20k_ddr_pmu_pdata },
1304 { },
1305 };
1306 MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
1307 #endif
1308
1309 #ifdef CONFIG_ACPI
1310 static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {
1311 {"MRVL000A", (kernel_ulong_t)&cn10k_ddr_pmu_pdata },
1312 {"MRVL000C", (kernel_ulong_t)&odyssey_ddr_pmu_pdata},
1313 {"MRVL000B", (kernel_ulong_t)&cn20k_ddr_pmu_pdata},
1314 {},
1315 };
1316 MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);
1317 #endif
1318
1319 static struct platform_driver cn10k_ddr_pmu_driver = {
1320 .driver = {
1321 .name = "cn10k-ddr-pmu",
1322 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
1323 .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match),
1324 .suppress_bind_attrs = true,
1325 },
1326 .probe = cn10k_ddr_perf_probe,
1327 .remove = cn10k_ddr_perf_remove,
1328 };
1329
cn10k_ddr_pmu_init(void)1330 static int __init cn10k_ddr_pmu_init(void)
1331 {
1332 int ret;
1333
1334 ret = cpuhp_setup_state_multi(
1335 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1336 "perf/marvell/cn10k/ddr:online", NULL,
1337 cn10k_ddr_pmu_offline_cpu);
1338 if (ret)
1339 return ret;
1340
1341 ret = platform_driver_register(&cn10k_ddr_pmu_driver);
1342 if (ret)
1343 cpuhp_remove_multi_state(
1344 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
1345 return ret;
1346 }
1347
cn10k_ddr_pmu_exit(void)1348 static void __exit cn10k_ddr_pmu_exit(void)
1349 {
1350 platform_driver_unregister(&cn10k_ddr_pmu_driver);
1351 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
1352 }
1353
1354 module_init(cn10k_ddr_pmu_init);
1355 module_exit(cn10k_ddr_pmu_exit);
1356
1357 MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");
1358 MODULE_DESCRIPTION("Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver");
1359 MODULE_LICENSE("GPL v2");
1360