1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved.
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/clk.h>
8 #include <linux/clk/tegra.h>
9 #include <linux/debugfs.h>
10 #include <linux/delay.h>
11 #include <linux/kernel.h>
12 #include <linux/mod_devicetable.h>
13 #include <linux/module.h>
14 #include <linux/of_reserved_mem.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/thermal.h>
18 #include <soc/tegra/fuse.h>
19 #include <soc/tegra/mc.h>
20
21 #include "tegra210-emc.h"
22 #include "tegra210-mc.h"
23
24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT 29
26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK \
27 (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
28 #define EMC_CLK_SOURCE_PLLM_LJ 0x4
29 #define EMC_CLK_SOURCE_PLLMB_LJ 0x5
30 #define EMC_CLK_FORCE_CC_TRIGGER BIT(27)
31 #define EMC_CLK_MC_EMC_SAME_FREQ BIT(16)
32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT 0
33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK \
34 (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
35
36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT 29
38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK \
39 (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT 10
41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK \
42 (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
43 #define PLLM_VCOA 0
44 #define PLLM_VCOB 1
45 #define EMC_DLL_SWITCH_OUT 2
46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT 0
47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK \
48 (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
49
50 /* MC_EMEM_ARB_MISC0 */
51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ BIT(27)
52
53 /* EMC_DATA_BRLSHFT_X */
54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX 2
55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX 3
56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX 4
57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX 5
58
59 #define TRIM_REG(chan, rank, reg, byte) \
60 (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
61 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \
62 next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \
63 rank ## _ ## reg ## _INDEX]) >> \
64 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
65 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
66 + \
67 (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
68 byte ## _DATA_BRLSHFT_MASK & \
69 next->trim_perch_regs[EMC ## chan ## \
70 _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
71 EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
72 byte ## _DATA_BRLSHFT_SHIFT) * 64))
73
74 #define CALC_TEMP(rank, reg, byte1, byte2, n) \
75 (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
76 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
77 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
78 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
79 | \
80 ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
81 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
82 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
83 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
84
85 #define REFRESH_SPEEDUP(value, speedup) \
86 (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
87
88 #define LPDDR2_MR4_SRR GENMASK(2, 0)
89
90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
91 &tegra210_emc_r21021,
92 };
93
94 static const struct tegra210_emc_table_register_offsets
95 tegra210_emc_table_register_offsets = {
96 .burst = {
97 EMC_RC,
98 EMC_RFC,
99 EMC_RFCPB,
100 EMC_REFCTRL2,
101 EMC_RFC_SLR,
102 EMC_RAS,
103 EMC_RP,
104 EMC_R2W,
105 EMC_W2R,
106 EMC_R2P,
107 EMC_W2P,
108 EMC_R2R,
109 EMC_TPPD,
110 EMC_CCDMW,
111 EMC_RD_RCD,
112 EMC_WR_RCD,
113 EMC_RRD,
114 EMC_REXT,
115 EMC_WEXT,
116 EMC_WDV_CHK,
117 EMC_WDV,
118 EMC_WSV,
119 EMC_WEV,
120 EMC_WDV_MASK,
121 EMC_WS_DURATION,
122 EMC_WE_DURATION,
123 EMC_QUSE,
124 EMC_QUSE_WIDTH,
125 EMC_IBDLY,
126 EMC_OBDLY,
127 EMC_EINPUT,
128 EMC_MRW6,
129 EMC_EINPUT_DURATION,
130 EMC_PUTERM_EXTRA,
131 EMC_PUTERM_WIDTH,
132 EMC_QRST,
133 EMC_QSAFE,
134 EMC_RDV,
135 EMC_RDV_MASK,
136 EMC_RDV_EARLY,
137 EMC_RDV_EARLY_MASK,
138 EMC_REFRESH,
139 EMC_BURST_REFRESH_NUM,
140 EMC_PRE_REFRESH_REQ_CNT,
141 EMC_PDEX2WR,
142 EMC_PDEX2RD,
143 EMC_PCHG2PDEN,
144 EMC_ACT2PDEN,
145 EMC_AR2PDEN,
146 EMC_RW2PDEN,
147 EMC_CKE2PDEN,
148 EMC_PDEX2CKE,
149 EMC_PDEX2MRR,
150 EMC_TXSR,
151 EMC_TXSRDLL,
152 EMC_TCKE,
153 EMC_TCKESR,
154 EMC_TPD,
155 EMC_TFAW,
156 EMC_TRPAB,
157 EMC_TCLKSTABLE,
158 EMC_TCLKSTOP,
159 EMC_MRW7,
160 EMC_TREFBW,
161 EMC_ODT_WRITE,
162 EMC_FBIO_CFG5,
163 EMC_FBIO_CFG7,
164 EMC_CFG_DIG_DLL,
165 EMC_CFG_DIG_DLL_PERIOD,
166 EMC_PMACRO_IB_RXRT,
167 EMC_CFG_PIPE_1,
168 EMC_CFG_PIPE_2,
169 EMC_PMACRO_QUSE_DDLL_RANK0_4,
170 EMC_PMACRO_QUSE_DDLL_RANK0_5,
171 EMC_PMACRO_QUSE_DDLL_RANK1_4,
172 EMC_PMACRO_QUSE_DDLL_RANK1_5,
173 EMC_MRW8,
174 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
175 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
176 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
177 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
178 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
179 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
180 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
181 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
182 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
183 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
184 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
185 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
186 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
187 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
188 EMC_PMACRO_DDLL_LONG_CMD_0,
189 EMC_PMACRO_DDLL_LONG_CMD_1,
190 EMC_PMACRO_DDLL_LONG_CMD_2,
191 EMC_PMACRO_DDLL_LONG_CMD_3,
192 EMC_PMACRO_DDLL_LONG_CMD_4,
193 EMC_PMACRO_DDLL_SHORT_CMD_0,
194 EMC_PMACRO_DDLL_SHORT_CMD_1,
195 EMC_PMACRO_DDLL_SHORT_CMD_2,
196 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
197 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
198 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
199 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
200 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
201 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
202 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
203 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
204 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
205 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
206 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
207 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
208 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
209 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
210 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
211 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
212 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
213 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
214 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
215 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
216 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
217 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
218 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
219 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
220 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
221 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
222 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
223 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
224 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
225 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
226 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
227 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
228 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
229 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
230 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
231 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
232 EMC_TXDSRVTTGEN,
233 EMC_FDPD_CTRL_DQ,
234 EMC_FDPD_CTRL_CMD,
235 EMC_FBIO_SPARE,
236 EMC_ZCAL_INTERVAL,
237 EMC_ZCAL_WAIT_CNT,
238 EMC_MRS_WAIT_CNT,
239 EMC_MRS_WAIT_CNT2,
240 EMC_AUTO_CAL_CHANNEL,
241 EMC_DLL_CFG_0,
242 EMC_DLL_CFG_1,
243 EMC_PMACRO_AUTOCAL_CFG_COMMON,
244 EMC_PMACRO_ZCTRL,
245 EMC_CFG,
246 EMC_CFG_PIPE,
247 EMC_DYN_SELF_REF_CONTROL,
248 EMC_QPOP,
249 EMC_DQS_BRLSHFT_0,
250 EMC_DQS_BRLSHFT_1,
251 EMC_CMD_BRLSHFT_2,
252 EMC_CMD_BRLSHFT_3,
253 EMC_PMACRO_PAD_CFG_CTRL,
254 EMC_PMACRO_DATA_PAD_RX_CTRL,
255 EMC_PMACRO_CMD_PAD_RX_CTRL,
256 EMC_PMACRO_DATA_RX_TERM_MODE,
257 EMC_PMACRO_CMD_RX_TERM_MODE,
258 EMC_PMACRO_CMD_PAD_TX_CTRL,
259 EMC_PMACRO_DATA_PAD_TX_CTRL,
260 EMC_PMACRO_COMMON_PAD_TX_CTRL,
261 EMC_PMACRO_VTTGEN_CTRL_0,
262 EMC_PMACRO_VTTGEN_CTRL_1,
263 EMC_PMACRO_VTTGEN_CTRL_2,
264 EMC_PMACRO_BRICK_CTRL_RFU1,
265 EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
266 EMC_PMACRO_BRICK_CTRL_RFU2,
267 EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
268 EMC_PMACRO_BG_BIAS_CTRL_0,
269 EMC_CFG_3,
270 EMC_PMACRO_TX_PWRD_0,
271 EMC_PMACRO_TX_PWRD_1,
272 EMC_PMACRO_TX_PWRD_2,
273 EMC_PMACRO_TX_PWRD_3,
274 EMC_PMACRO_TX_PWRD_4,
275 EMC_PMACRO_TX_PWRD_5,
276 EMC_CONFIG_SAMPLE_DELAY,
277 EMC_PMACRO_TX_SEL_CLK_SRC_0,
278 EMC_PMACRO_TX_SEL_CLK_SRC_1,
279 EMC_PMACRO_TX_SEL_CLK_SRC_2,
280 EMC_PMACRO_TX_SEL_CLK_SRC_3,
281 EMC_PMACRO_TX_SEL_CLK_SRC_4,
282 EMC_PMACRO_TX_SEL_CLK_SRC_5,
283 EMC_PMACRO_DDLL_BYPASS,
284 EMC_PMACRO_DDLL_PWRD_0,
285 EMC_PMACRO_DDLL_PWRD_1,
286 EMC_PMACRO_DDLL_PWRD_2,
287 EMC_PMACRO_CMD_CTRL_0,
288 EMC_PMACRO_CMD_CTRL_1,
289 EMC_PMACRO_CMD_CTRL_2,
290 EMC_TR_TIMING_0,
291 EMC_TR_DVFS,
292 EMC_TR_CTRL_1,
293 EMC_TR_RDV,
294 EMC_TR_QPOP,
295 EMC_TR_RDV_MASK,
296 EMC_MRW14,
297 EMC_TR_QSAFE,
298 EMC_TR_QRST,
299 EMC_TRAINING_CTRL,
300 EMC_TRAINING_SETTLE,
301 EMC_TRAINING_VREF_SETTLE,
302 EMC_TRAINING_CA_FINE_CTRL,
303 EMC_TRAINING_CA_CTRL_MISC,
304 EMC_TRAINING_CA_CTRL_MISC1,
305 EMC_TRAINING_CA_VREF_CTRL,
306 EMC_TRAINING_QUSE_CORS_CTRL,
307 EMC_TRAINING_QUSE_FINE_CTRL,
308 EMC_TRAINING_QUSE_CTRL_MISC,
309 EMC_TRAINING_QUSE_VREF_CTRL,
310 EMC_TRAINING_READ_FINE_CTRL,
311 EMC_TRAINING_READ_CTRL_MISC,
312 EMC_TRAINING_READ_VREF_CTRL,
313 EMC_TRAINING_WRITE_FINE_CTRL,
314 EMC_TRAINING_WRITE_CTRL_MISC,
315 EMC_TRAINING_WRITE_VREF_CTRL,
316 EMC_TRAINING_MPC,
317 EMC_MRW15,
318 },
319 .trim = {
320 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
321 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
322 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
323 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
324 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
325 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
326 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
327 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
328 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
329 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
330 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
331 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
332 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
333 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
334 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
335 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
336 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
337 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
338 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
339 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
340 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
341 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
342 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
343 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
344 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
345 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
346 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
347 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
348 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
349 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
350 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
351 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
352 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
353 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
354 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
355 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
356 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
357 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
358 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
359 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
360 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
361 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
362 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
363 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
364 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
365 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
366 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
367 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
368 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
369 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
370 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
371 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
372 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
373 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
374 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
375 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
376 EMC_PMACRO_IB_VREF_DQS_0,
377 EMC_PMACRO_IB_VREF_DQS_1,
378 EMC_PMACRO_IB_VREF_DQ_0,
379 EMC_PMACRO_IB_VREF_DQ_1,
380 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
381 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
382 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
383 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
384 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
385 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
386 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
387 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
388 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
389 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
390 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
391 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
392 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
393 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
394 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
395 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
396 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
397 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
398 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
399 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
400 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
401 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
402 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
403 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
404 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
405 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
406 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
407 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
408 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
409 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
410 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
411 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
412 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
413 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
414 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
415 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
416 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
417 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
418 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
419 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
420 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
421 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
422 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
423 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
424 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
425 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
426 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
427 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
428 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
429 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
430 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
431 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
432 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
433 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
434 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
435 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
436 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
437 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
438 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
439 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
440 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
441 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
442 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
443 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
444 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
445 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
446 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
447 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
448 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
449 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
450 EMC_PMACRO_QUSE_DDLL_RANK0_0,
451 EMC_PMACRO_QUSE_DDLL_RANK0_1,
452 EMC_PMACRO_QUSE_DDLL_RANK0_2,
453 EMC_PMACRO_QUSE_DDLL_RANK0_3,
454 EMC_PMACRO_QUSE_DDLL_RANK1_0,
455 EMC_PMACRO_QUSE_DDLL_RANK1_1,
456 EMC_PMACRO_QUSE_DDLL_RANK1_2,
457 EMC_PMACRO_QUSE_DDLL_RANK1_3
458 },
459 .burst_mc = {
460 MC_EMEM_ARB_CFG,
461 MC_EMEM_ARB_OUTSTANDING_REQ,
462 MC_EMEM_ARB_REFPB_HP_CTRL,
463 MC_EMEM_ARB_REFPB_BANK_CTRL,
464 MC_EMEM_ARB_TIMING_RCD,
465 MC_EMEM_ARB_TIMING_RP,
466 MC_EMEM_ARB_TIMING_RC,
467 MC_EMEM_ARB_TIMING_RAS,
468 MC_EMEM_ARB_TIMING_FAW,
469 MC_EMEM_ARB_TIMING_RRD,
470 MC_EMEM_ARB_TIMING_RAP2PRE,
471 MC_EMEM_ARB_TIMING_WAP2PRE,
472 MC_EMEM_ARB_TIMING_R2R,
473 MC_EMEM_ARB_TIMING_W2W,
474 MC_EMEM_ARB_TIMING_R2W,
475 MC_EMEM_ARB_TIMING_CCDMW,
476 MC_EMEM_ARB_TIMING_W2R,
477 MC_EMEM_ARB_TIMING_RFCPB,
478 MC_EMEM_ARB_DA_TURNS,
479 MC_EMEM_ARB_DA_COVERS,
480 MC_EMEM_ARB_MISC0,
481 MC_EMEM_ARB_MISC1,
482 MC_EMEM_ARB_MISC2,
483 MC_EMEM_ARB_RING1_THROTTLE,
484 MC_EMEM_ARB_DHYST_CTRL,
485 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
486 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
487 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
488 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
489 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
490 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
491 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
492 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
493 },
494 .la_scale = {
495 MC_MLL_MPCORER_PTSA_RATE,
496 MC_FTOP_PTSA_RATE,
497 MC_PTSA_GRANT_DECREMENT,
498 MC_LATENCY_ALLOWANCE_XUSB_0,
499 MC_LATENCY_ALLOWANCE_XUSB_1,
500 MC_LATENCY_ALLOWANCE_TSEC_0,
501 MC_LATENCY_ALLOWANCE_SDMMCA_0,
502 MC_LATENCY_ALLOWANCE_SDMMCAA_0,
503 MC_LATENCY_ALLOWANCE_SDMMC_0,
504 MC_LATENCY_ALLOWANCE_SDMMCAB_0,
505 MC_LATENCY_ALLOWANCE_PPCS_0,
506 MC_LATENCY_ALLOWANCE_PPCS_1,
507 MC_LATENCY_ALLOWANCE_MPCORE_0,
508 MC_LATENCY_ALLOWANCE_HC_0,
509 MC_LATENCY_ALLOWANCE_HC_1,
510 MC_LATENCY_ALLOWANCE_AVPC_0,
511 MC_LATENCY_ALLOWANCE_GPU_0,
512 MC_LATENCY_ALLOWANCE_GPU2_0,
513 MC_LATENCY_ALLOWANCE_NVENC_0,
514 MC_LATENCY_ALLOWANCE_NVDEC_0,
515 MC_LATENCY_ALLOWANCE_VIC_0,
516 MC_LATENCY_ALLOWANCE_VI2_0,
517 MC_LATENCY_ALLOWANCE_ISP2_0,
518 MC_LATENCY_ALLOWANCE_ISP2_1,
519 },
520 .burst_per_channel = {
521 { .bank = 0, .offset = EMC_MRW10, },
522 { .bank = 1, .offset = EMC_MRW10, },
523 { .bank = 0, .offset = EMC_MRW11, },
524 { .bank = 1, .offset = EMC_MRW11, },
525 { .bank = 0, .offset = EMC_MRW12, },
526 { .bank = 1, .offset = EMC_MRW12, },
527 { .bank = 0, .offset = EMC_MRW13, },
528 { .bank = 1, .offset = EMC_MRW13, },
529 },
530 .trim_per_channel = {
531 { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
532 { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
533 { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
534 { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
535 { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
536 { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
537 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
538 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
539 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
540 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
541 },
542 .vref_per_channel = {
543 {
544 .bank = 0,
545 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
546 }, {
547 .bank = 1,
548 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
549 }, {
550 .bank = 0,
551 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
552 }, {
553 .bank = 1,
554 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
555 },
556 },
557 };
558
tegra210_emc_train(struct timer_list * timer)559 static void tegra210_emc_train(struct timer_list *timer)
560 {
561 struct tegra210_emc *emc = timer_container_of(emc, timer, training);
562 unsigned long flags;
563
564 if (!emc->last)
565 return;
566
567 spin_lock_irqsave(&emc->lock, flags);
568
569 if (emc->sequence->periodic_compensation)
570 emc->sequence->periodic_compensation(emc);
571
572 spin_unlock_irqrestore(&emc->lock, flags);
573
574 mod_timer(&emc->training,
575 jiffies + msecs_to_jiffies(emc->training_interval));
576 }
577
tegra210_emc_training_start(struct tegra210_emc * emc)578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
579 {
580 mod_timer(&emc->training,
581 jiffies + msecs_to_jiffies(emc->training_interval));
582 }
583
tegra210_emc_training_stop(struct tegra210_emc * emc)584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
585 {
586 timer_delete(&emc->training);
587 }
588
tegra210_emc_get_temperature(struct tegra210_emc * emc)589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
590 {
591 unsigned long flags;
592 u32 value, max = 0;
593 unsigned int i;
594
595 spin_lock_irqsave(&emc->lock, flags);
596
597 for (i = 0; i < emc->num_devices; i++) {
598 value = tegra210_emc_mrr_read(emc, i, 4);
599
600 if (value & BIT(7))
601 dev_dbg(emc->dev,
602 "sensor reading changed for device %u: %08x\n",
603 i, value);
604
605 value = FIELD_GET(LPDDR2_MR4_SRR, value);
606 if (value > max)
607 max = value;
608 }
609
610 spin_unlock_irqrestore(&emc->lock, flags);
611
612 return max;
613 }
614
tegra210_emc_poll_refresh(struct timer_list * timer)615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
616 {
617 struct tegra210_emc *emc = timer_container_of(emc, timer,
618 refresh_timer);
619 unsigned int temperature;
620
621 if (!emc->debugfs.temperature)
622 temperature = tegra210_emc_get_temperature(emc);
623 else
624 temperature = emc->debugfs.temperature;
625
626 if (temperature == emc->temperature)
627 goto reset;
628
629 switch (temperature) {
630 case 0 ... 3:
631 /* temperature is fine, using regular refresh */
632 dev_dbg(emc->dev, "switching to nominal refresh...\n");
633 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
634 break;
635
636 case 4:
637 dev_dbg(emc->dev, "switching to 2x refresh...\n");
638 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
639 break;
640
641 case 5:
642 dev_dbg(emc->dev, "switching to 4x refresh...\n");
643 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
644 break;
645
646 case 6 ... 7:
647 dev_dbg(emc->dev, "switching to throttle refresh...\n");
648 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
649 break;
650
651 default:
652 WARN(1, "invalid DRAM temperature state %u\n", temperature);
653 return;
654 }
655
656 emc->temperature = temperature;
657
658 reset:
659 if (atomic_read(&emc->refresh_poll) > 0) {
660 unsigned int interval = emc->refresh_poll_interval;
661 unsigned int timeout = msecs_to_jiffies(interval);
662
663 mod_timer(&emc->refresh_timer, jiffies + timeout);
664 }
665 }
666
tegra210_emc_poll_refresh_stop(struct tegra210_emc * emc)667 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
668 {
669 atomic_set(&emc->refresh_poll, 0);
670 timer_delete_sync(&emc->refresh_timer);
671 }
672
tegra210_emc_poll_refresh_start(struct tegra210_emc * emc)673 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
674 {
675 atomic_set(&emc->refresh_poll, 1);
676
677 mod_timer(&emc->refresh_timer,
678 jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
679 }
680
tegra210_emc_cd_max_state(struct thermal_cooling_device * cd,unsigned long * state)681 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
682 unsigned long *state)
683 {
684 *state = 1;
685
686 return 0;
687 }
688
tegra210_emc_cd_get_state(struct thermal_cooling_device * cd,unsigned long * state)689 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
690 unsigned long *state)
691 {
692 struct tegra210_emc *emc = cd->devdata;
693
694 *state = atomic_read(&emc->refresh_poll);
695
696 return 0;
697 }
698
tegra210_emc_cd_set_state(struct thermal_cooling_device * cd,unsigned long state)699 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
700 unsigned long state)
701 {
702 struct tegra210_emc *emc = cd->devdata;
703
704 if (state == atomic_read(&emc->refresh_poll))
705 return 0;
706
707 if (state)
708 tegra210_emc_poll_refresh_start(emc);
709 else
710 tegra210_emc_poll_refresh_stop(emc);
711
712 return 0;
713 }
714
715 static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
716 .get_max_state = tegra210_emc_cd_max_state,
717 .get_cur_state = tegra210_emc_cd_get_state,
718 .set_cur_state = tegra210_emc_cd_set_state,
719 };
720
tegra210_emc_set_clock(struct tegra210_emc * emc,u32 clksrc)721 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
722 {
723 emc->sequence->set_clock(emc, clksrc);
724
725 if (emc->next->periodic_training)
726 tegra210_emc_training_start(emc);
727 else
728 tegra210_emc_training_stop(emc);
729 }
730
tegra210_change_dll_src(struct tegra210_emc * emc,u32 clksrc)731 static void tegra210_change_dll_src(struct tegra210_emc *emc,
732 u32 clksrc)
733 {
734 u32 dll_setting = emc->next->dll_clk_src;
735 u32 emc_clk_src;
736 u32 emc_clk_div;
737
738 emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
739 EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
740 emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
741 EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
742
743 dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
744 DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
745 dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
746 dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
747
748 dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
749 if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
750 dll_setting |= (PLLM_VCOB <<
751 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
752 else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
753 dll_setting |= (PLLM_VCOA <<
754 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
755 else
756 dll_setting |= (EMC_DLL_SWITCH_OUT <<
757 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
758
759 tegra210_clk_emc_dll_update_setting(dll_setting);
760
761 if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
762 tegra210_clk_emc_dll_enable(true);
763 else
764 tegra210_clk_emc_dll_enable(false);
765 }
766
tegra210_emc_set_refresh(struct tegra210_emc * emc,enum tegra210_emc_refresh refresh)767 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
768 enum tegra210_emc_refresh refresh)
769 {
770 struct tegra210_emc_timing *timings;
771 unsigned long flags;
772
773 if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
774 emc->dram_type != DRAM_TYPE_LPDDR4) ||
775 !emc->last)
776 return -ENODEV;
777
778 if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
779 return -EINVAL;
780
781 if (refresh == emc->refresh)
782 return 0;
783
784 spin_lock_irqsave(&emc->lock, flags);
785
786 if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
787 timings = emc->derated;
788 else
789 timings = emc->nominal;
790
791 if (timings != emc->timings) {
792 unsigned int index = emc->last - emc->timings;
793 u32 clksrc;
794
795 clksrc = emc->provider.configs[index].value |
796 EMC_CLK_FORCE_CC_TRIGGER;
797
798 emc->next = &timings[index];
799 emc->timings = timings;
800
801 tegra210_emc_set_clock(emc, clksrc);
802 } else {
803 tegra210_emc_adjust_timing(emc, emc->last);
804 tegra210_emc_timing_update(emc);
805
806 if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
807 emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
808 }
809
810 spin_unlock_irqrestore(&emc->lock, flags);
811
812 return 0;
813 }
814
tegra210_emc_mrr_read(struct tegra210_emc * emc,unsigned int chip,unsigned int address)815 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
816 unsigned int address)
817 {
818 u32 value, ret = 0;
819 unsigned int i;
820
821 value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
822 (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
823 emc_writel(emc, value, EMC_MRR);
824
825 for (i = 0; i < emc->num_channels; i++)
826 WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
827 EMC_EMC_STATUS_MRR_DIVLD, 1),
828 "Timed out waiting for MRR %u (ch=%u)\n", address, i);
829
830 for (i = 0; i < emc->num_channels; i++) {
831 value = emc_channel_readl(emc, i, EMC_MRR);
832 value &= EMC_MRR_DATA_MASK;
833
834 ret = (ret << 16) | value;
835 }
836
837 return ret;
838 }
839
tegra210_emc_do_clock_change(struct tegra210_emc * emc,u32 clksrc)840 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
841 {
842 int err;
843
844 mc_readl(emc->mc, MC_EMEM_ADR_CFG);
845 emc_readl(emc, EMC_INTSTATUS);
846
847 tegra210_clk_emc_update_setting(clksrc);
848
849 err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
850 EMC_INTSTATUS_CLKCHANGE_COMPLETE,
851 true);
852 if (err)
853 dev_warn(emc->dev, "clock change completion error: %d\n", err);
854 }
855
tegra210_emc_find_timing(struct tegra210_emc * emc,unsigned long rate)856 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
857 unsigned long rate)
858 {
859 unsigned int i;
860
861 for (i = 0; i < emc->num_timings; i++)
862 if (emc->timings[i].rate * 1000UL == rate)
863 return &emc->timings[i];
864
865 return NULL;
866 }
867
tegra210_emc_wait_for_update(struct tegra210_emc * emc,unsigned int channel,unsigned int offset,u32 bit_mask,bool state)868 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
869 unsigned int offset, u32 bit_mask, bool state)
870 {
871 unsigned int i;
872 u32 value;
873
874 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
875 value = emc_channel_readl(emc, channel, offset);
876 if (!!(value & bit_mask) == state)
877 return 0;
878
879 udelay(1);
880 }
881
882 return -ETIMEDOUT;
883 }
884
tegra210_emc_set_shadow_bypass(struct tegra210_emc * emc,int set)885 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
886 {
887 u32 emc_dbg = emc_readl(emc, EMC_DBG);
888
889 if (set)
890 emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
891 else
892 emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
893 }
894
tegra210_emc_get_dll_state(struct tegra210_emc_timing * next)895 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
896 {
897 if (next->emc_emrs & 0x1)
898 return 0;
899
900 return 1;
901 }
902
tegra210_emc_timing_update(struct tegra210_emc * emc)903 void tegra210_emc_timing_update(struct tegra210_emc *emc)
904 {
905 unsigned int i;
906 int err = 0;
907
908 emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
909
910 for (i = 0; i < emc->num_channels; i++) {
911 err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
912 EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
913 false);
914 }
915
916 if (err)
917 dev_warn(emc->dev, "timing update error: %d\n", err);
918 }
919
tegra210_emc_actual_osc_clocks(u32 in)920 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
921 {
922 if (in < 0x40)
923 return in * 16;
924 else if (in < 0x80)
925 return 2048;
926 else if (in < 0xc0)
927 return 4096;
928 else
929 return 8192;
930 }
931
tegra210_emc_start_periodic_compensation(struct tegra210_emc * emc)932 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
933 {
934 u32 mpc_req = 0x4b;
935
936 emc_writel(emc, mpc_req, EMC_MPC);
937 mpc_req = emc_readl(emc, EMC_MPC);
938 }
939
tegra210_emc_compensate(struct tegra210_emc_timing * next,u32 offset)940 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
941 {
942 u32 temp = 0, rate = next->rate / 1000;
943 s32 delta[4], delta_taps[4];
944 s32 new[] = {
945 TRIM_REG(0, 0, 0, 0),
946 TRIM_REG(0, 0, 0, 1),
947 TRIM_REG(0, 0, 1, 2),
948 TRIM_REG(0, 0, 1, 3),
949
950 TRIM_REG(1, 0, 2, 4),
951 TRIM_REG(1, 0, 2, 5),
952 TRIM_REG(1, 0, 3, 6),
953 TRIM_REG(1, 0, 3, 7),
954
955 TRIM_REG(0, 1, 0, 0),
956 TRIM_REG(0, 1, 0, 1),
957 TRIM_REG(0, 1, 1, 2),
958 TRIM_REG(0, 1, 1, 3),
959
960 TRIM_REG(1, 1, 2, 4),
961 TRIM_REG(1, 1, 2, 5),
962 TRIM_REG(1, 1, 3, 6),
963 TRIM_REG(1, 1, 3, 7)
964 };
965 unsigned i;
966
967 switch (offset) {
968 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
969 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
970 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
971 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
972 case EMC_DATA_BRLSHFT_0:
973 delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
974 next->trained_dram_clktree[C0D0U0]);
975 delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
976 next->trained_dram_clktree[C0D0U1]);
977 delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
978 next->trained_dram_clktree[C1D0U0]);
979 delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
980 next->trained_dram_clktree[C1D0U1]);
981
982 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
983 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
984 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
985 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
986
987 for (i = 0; i < 4; i++) {
988 if ((delta_taps[i] > next->tree_margin) ||
989 (delta_taps[i] < (-1 * next->tree_margin))) {
990 new[i * 2] = new[i * 2] + delta_taps[i];
991 new[i * 2 + 1] = new[i * 2 + 1] +
992 delta_taps[i];
993 }
994 }
995
996 if (offset == EMC_DATA_BRLSHFT_0) {
997 for (i = 0; i < 8; i++)
998 new[i] = new[i] / 64;
999 } else {
1000 for (i = 0; i < 8; i++)
1001 new[i] = new[i] % 64;
1002 }
1003
1004 break;
1005
1006 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1007 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1008 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1009 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1010 case EMC_DATA_BRLSHFT_1:
1011 delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1012 next->trained_dram_clktree[C0D1U0]);
1013 delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1014 next->trained_dram_clktree[C0D1U1]);
1015 delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1016 next->trained_dram_clktree[C1D1U0]);
1017 delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1018 next->trained_dram_clktree[C1D1U1]);
1019
1020 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1021 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1022 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1023 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1024
1025 for (i = 0; i < 4; i++) {
1026 if ((delta_taps[i] > next->tree_margin) ||
1027 (delta_taps[i] < (-1 * next->tree_margin))) {
1028 new[8 + i * 2] = new[8 + i * 2] +
1029 delta_taps[i];
1030 new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1031 delta_taps[i];
1032 }
1033 }
1034
1035 if (offset == EMC_DATA_BRLSHFT_1) {
1036 for (i = 0; i < 8; i++)
1037 new[i + 8] = new[i + 8] / 64;
1038 } else {
1039 for (i = 0; i < 8; i++)
1040 new[i + 8] = new[i + 8] % 64;
1041 }
1042
1043 break;
1044 }
1045
1046 switch (offset) {
1047 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1048 temp = CALC_TEMP(0, 0, 0, 1, 0);
1049 break;
1050
1051 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1052 temp = CALC_TEMP(0, 1, 2, 3, 2);
1053 break;
1054
1055 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1056 temp = CALC_TEMP(0, 2, 4, 5, 4);
1057 break;
1058
1059 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1060 temp = CALC_TEMP(0, 3, 6, 7, 6);
1061 break;
1062
1063 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1064 temp = CALC_TEMP(1, 0, 0, 1, 8);
1065 break;
1066
1067 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1068 temp = CALC_TEMP(1, 1, 2, 3, 10);
1069 break;
1070
1071 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1072 temp = CALC_TEMP(1, 2, 4, 5, 12);
1073 break;
1074
1075 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1076 temp = CALC_TEMP(1, 3, 6, 7, 14);
1077 break;
1078
1079 case EMC_DATA_BRLSHFT_0:
1080 temp = ((new[0] <<
1081 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1082 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1083 ((new[1] <<
1084 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1085 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1086 ((new[2] <<
1087 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1088 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1089 ((new[3] <<
1090 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1091 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1092 ((new[4] <<
1093 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1094 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1095 ((new[5] <<
1096 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1097 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1098 ((new[6] <<
1099 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1100 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1101 ((new[7] <<
1102 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1103 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1104 break;
1105
1106 case EMC_DATA_BRLSHFT_1:
1107 temp = ((new[8] <<
1108 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1109 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1110 ((new[9] <<
1111 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1112 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1113 ((new[10] <<
1114 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1115 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1116 ((new[11] <<
1117 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1118 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1119 ((new[12] <<
1120 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1121 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1122 ((new[13] <<
1123 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1124 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1125 ((new[14] <<
1126 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1127 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1128 ((new[15] <<
1129 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1130 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1131 break;
1132
1133 default:
1134 break;
1135 }
1136
1137 return temp;
1138 }
1139
tegra210_emc_dll_prelock(struct tegra210_emc * emc,u32 clksrc)1140 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1141 {
1142 unsigned int i;
1143 u32 value;
1144
1145 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1146 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1147 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1148 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1149 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1150 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1151 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1152 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1153 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1154 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1155 emc_writel(emc, 1, EMC_TIMING_CONTROL);
1156
1157 for (i = 0; i < emc->num_channels; i++)
1158 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1159 EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1160 0);
1161
1162 for (i = 0; i < emc->num_channels; i++) {
1163 while (true) {
1164 value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1165 if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1166 break;
1167 }
1168 }
1169
1170 value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1171 emc_writel(emc, value, EMC_DLL_CFG_0);
1172
1173 value = emc_readl(emc, EMC_DLL_CFG_1);
1174 value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1175
1176 if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1177 value |= 150;
1178 else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1179 value |= 100;
1180 else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1181 value |= 70;
1182 else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1183 value |= 30;
1184 else
1185 value |= 20;
1186
1187 emc_writel(emc, value, EMC_DLL_CFG_1);
1188
1189 tegra210_change_dll_src(emc, clksrc);
1190
1191 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1192 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1193 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1194
1195 tegra210_emc_timing_update(emc);
1196
1197 for (i = 0; i < emc->num_channels; i++) {
1198 while (true) {
1199 value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1200 if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1201 break;
1202 }
1203 }
1204
1205 while (true) {
1206 value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1207
1208 if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1209 continue;
1210
1211 if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1212 continue;
1213
1214 break;
1215 }
1216
1217 value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1218
1219 return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1220 }
1221
tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc * emc,u32 clk,bool flip_backward)1222 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1223 bool flip_backward)
1224 {
1225 u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1226 const struct tegra210_emc_timing *timing;
1227
1228 if (flip_backward)
1229 timing = emc->last;
1230 else
1231 timing = emc->next;
1232
1233 cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1234 dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1235 rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1236 cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1237 common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1238
1239 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1240
1241 if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1242 ccfifo_writel(emc, common_tx & 0xa,
1243 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1244 ccfifo_writel(emc, common_tx & 0xf,
1245 EMC_PMACRO_COMMON_PAD_TX_CTRL,
1246 (100000 / clk) + 1);
1247 ramp_up_wait += 100000;
1248 } else {
1249 ccfifo_writel(emc, common_tx | 0x8,
1250 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1251 }
1252
1253 if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1254 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1255 cmd_pad |=
1256 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1257 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1258 cmd_pad &=
1259 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1260 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1261 ccfifo_writel(emc, cmd_pad,
1262 EMC_PMACRO_CMD_PAD_TX_CTRL,
1263 (100000 / clk) + 1);
1264 ramp_up_wait += 100000;
1265
1266 dq_pad |=
1267 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1268 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1269 dq_pad &=
1270 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1271 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1272 ccfifo_writel(emc, dq_pad,
1273 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1274 ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1275 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1276 } else {
1277 ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1278 EMC_PMACRO_BRICK_CTRL_RFU1,
1279 (100000 / clk) + 1);
1280 ramp_up_wait += 100000;
1281 }
1282
1283 ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1284 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1285 ramp_up_wait += 100000;
1286
1287 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1288 cmd_pad |=
1289 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1290 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1291 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1292 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1293 ccfifo_writel(emc, cmd_pad,
1294 EMC_PMACRO_CMD_PAD_TX_CTRL,
1295 (100000 / clk) + 1);
1296 ramp_up_wait += 100000;
1297
1298 dq_pad |=
1299 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1300 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1301 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1302 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1303 ccfifo_writel(emc, dq_pad,
1304 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1305 ccfifo_writel(emc, rfu1,
1306 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1307 } else {
1308 ccfifo_writel(emc, rfu1,
1309 EMC_PMACRO_BRICK_CTRL_RFU1,
1310 (100000 / clk) + 1);
1311 ramp_up_wait += 100000;
1312 }
1313
1314 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1315 EMC_FBIO_CFG5, (100000 / clk) + 10);
1316 ramp_up_wait += 100000 + (10 * clk);
1317 } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1318 ccfifo_writel(emc, rfu1 | 0x06000600,
1319 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1320 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1321 EMC_FBIO_CFG5, (100000 / clk) + 10);
1322 ramp_up_wait += 100000 + 10 * clk;
1323 } else {
1324 ccfifo_writel(emc, rfu1 | 0x00000600,
1325 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1326 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1327 EMC_FBIO_CFG5, 12);
1328 ramp_up_wait += 12 * clk;
1329 }
1330
1331 cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1332 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1333
1334 return ramp_up_wait;
1335 }
1336
tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc * emc,u32 clk,bool flip_backward)1337 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1338 bool flip_backward)
1339 {
1340 u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1341 const struct tegra210_emc_timing *entry;
1342 u32 seq_wait;
1343
1344 if (flip_backward)
1345 entry = emc->next;
1346 else
1347 entry = emc->last;
1348
1349 cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1350 dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1351 rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1352 cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1353 common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1354
1355 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1356
1357 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1358 ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1359 EMC_FBIO_CFG5, 12);
1360 ramp_down_wait = 12 * clk;
1361
1362 seq_wait = (100000 / clk) + 1;
1363
1364 if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1365 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1366 cmd_pad &=
1367 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1368 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1369 cmd_pad |=
1370 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1371 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1372 ccfifo_writel(emc, cmd_pad,
1373 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1374 ramp_down_wait += 100000;
1375
1376 dq_pad &=
1377 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1378 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1379 dq_pad |=
1380 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1381 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1382 ccfifo_writel(emc, dq_pad,
1383 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1384 ccfifo_writel(emc, rfu1 & ~0x01120112,
1385 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1386 } else {
1387 ccfifo_writel(emc, rfu1 & ~0x01120112,
1388 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1389 ramp_down_wait += 100000;
1390 }
1391
1392 ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1393 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1394 ramp_down_wait += 100000;
1395
1396 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1397 cmd_pad &=
1398 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1399 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1400 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1401 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1402 ccfifo_writel(emc, cmd_pad,
1403 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1404 ramp_down_wait += 100000;
1405
1406 dq_pad &=
1407 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1408 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1409 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1410 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1411 ccfifo_writel(emc, dq_pad,
1412 EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1413 ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1414 EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1415 } else {
1416 ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1417 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1418 ramp_down_wait += 100000;
1419 }
1420 } else {
1421 ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1422 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1423 ramp_down_wait += 100000 + (20 * clk);
1424 }
1425
1426 if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1427 ramp_down_wait += 100000;
1428 ccfifo_writel(emc, common_tx & ~0x5,
1429 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1430 ramp_down_wait += 100000;
1431 ccfifo_writel(emc, common_tx & ~0xf,
1432 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1433 ramp_down_wait += 100000;
1434 ccfifo_writel(emc, 0, 0, seq_wait);
1435 ramp_down_wait += 100000;
1436 } else {
1437 ccfifo_writel(emc, common_tx & ~0xf,
1438 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1439 }
1440
1441 return ramp_down_wait;
1442 }
1443
tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing * timing)1444 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1445 {
1446 timing->current_dram_clktree[C0D0U0] =
1447 timing->trained_dram_clktree[C0D0U0];
1448 timing->current_dram_clktree[C0D0U1] =
1449 timing->trained_dram_clktree[C0D0U1];
1450 timing->current_dram_clktree[C1D0U0] =
1451 timing->trained_dram_clktree[C1D0U0];
1452 timing->current_dram_clktree[C1D0U1] =
1453 timing->trained_dram_clktree[C1D0U1];
1454 timing->current_dram_clktree[C1D1U0] =
1455 timing->trained_dram_clktree[C1D1U0];
1456 timing->current_dram_clktree[C1D1U1] =
1457 timing->trained_dram_clktree[C1D1U1];
1458 }
1459
update_dll_control(struct tegra210_emc * emc,u32 value,bool state)1460 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1461 {
1462 unsigned int i;
1463
1464 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1465 tegra210_emc_timing_update(emc);
1466
1467 for (i = 0; i < emc->num_channels; i++)
1468 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1469 EMC_CFG_DIG_DLL_CFG_DLL_EN,
1470 state);
1471 }
1472
tegra210_emc_dll_disable(struct tegra210_emc * emc)1473 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1474 {
1475 u32 value;
1476
1477 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1478 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1479
1480 update_dll_control(emc, value, false);
1481 }
1482
tegra210_emc_dll_enable(struct tegra210_emc * emc)1483 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1484 {
1485 u32 value;
1486
1487 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1488 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1489
1490 update_dll_control(emc, value, true);
1491 }
1492
tegra210_emc_adjust_timing(struct tegra210_emc * emc,struct tegra210_emc_timing * timing)1493 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1494 struct tegra210_emc_timing *timing)
1495 {
1496 u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1497 u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1498 u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1499
1500 switch (emc->refresh) {
1501 case TEGRA210_EMC_REFRESH_NOMINAL:
1502 case TEGRA210_EMC_REFRESH_THROTTLE:
1503 break;
1504
1505 case TEGRA210_EMC_REFRESH_2X:
1506 ref = REFRESH_SPEEDUP(ref, 2);
1507 pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1508 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1509 break;
1510
1511 case TEGRA210_EMC_REFRESH_4X:
1512 ref = REFRESH_SPEEDUP(ref, 4);
1513 pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1514 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1515 break;
1516
1517 default:
1518 dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1519 return;
1520 }
1521
1522 emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1523 emc_writel(emc, pre_ref,
1524 emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1525 emc_writel(emc, dsr_cntrl,
1526 emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1527 }
1528
tegra210_emc_set_rate(struct device * dev,const struct tegra210_clk_emc_config * config)1529 static int tegra210_emc_set_rate(struct device *dev,
1530 const struct tegra210_clk_emc_config *config)
1531 {
1532 struct tegra210_emc *emc = dev_get_drvdata(dev);
1533 struct tegra210_emc_timing *timing = NULL;
1534 unsigned long rate = config->rate;
1535 s64 last_change_delay;
1536 unsigned long flags;
1537 unsigned int i;
1538
1539 if (rate == emc->last->rate * 1000UL)
1540 return 0;
1541
1542 for (i = 0; i < emc->num_timings; i++) {
1543 if (emc->timings[i].rate * 1000UL == rate) {
1544 timing = &emc->timings[i];
1545 break;
1546 }
1547 }
1548
1549 if (!timing)
1550 return -EINVAL;
1551
1552 if (rate > 204000000 && !timing->trained)
1553 return -EINVAL;
1554
1555 emc->next = timing;
1556 last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1557
1558 /* XXX use non-busy-looping sleep? */
1559 if ((last_change_delay >= 0) &&
1560 (last_change_delay < emc->clkchange_delay))
1561 udelay(emc->clkchange_delay - (int)last_change_delay);
1562
1563 spin_lock_irqsave(&emc->lock, flags);
1564 tegra210_emc_set_clock(emc, config->value);
1565 emc->clkchange_time = ktime_get();
1566 emc->last = timing;
1567 spin_unlock_irqrestore(&emc->lock, flags);
1568
1569 return 0;
1570 }
1571
1572 /*
1573 * debugfs interface
1574 *
1575 * The memory controller driver exposes some files in debugfs that can be used
1576 * to control the EMC frequency. The top-level directory can be found here:
1577 *
1578 * /sys/kernel/debug/emc
1579 *
1580 * It contains the following files:
1581 *
1582 * - available_rates: This file contains a list of valid, space-separated
1583 * EMC frequencies.
1584 *
1585 * - min_rate: Writing a value to this file sets the given frequency as the
1586 * floor of the permitted range. If this is higher than the currently
1587 * configured EMC frequency, this will cause the frequency to be
1588 * increased so that it stays within the valid range.
1589 *
1590 * - max_rate: Similarily to the min_rate file, writing a value to this file
1591 * sets the given frequency as the ceiling of the permitted range. If
1592 * the value is lower than the currently configured EMC frequency, this
1593 * will cause the frequency to be decreased so that it stays within the
1594 * valid range.
1595 */
1596
tegra210_emc_validate_rate(struct tegra210_emc * emc,unsigned long rate)1597 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1598 unsigned long rate)
1599 {
1600 unsigned int i;
1601
1602 for (i = 0; i < emc->num_timings; i++)
1603 if (rate == emc->timings[i].rate * 1000UL)
1604 return true;
1605
1606 return false;
1607 }
1608
tegra210_emc_debug_available_rates_show(struct seq_file * s,void * data)1609 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1610 void *data)
1611 {
1612 struct tegra210_emc *emc = s->private;
1613 const char *prefix = "";
1614 unsigned int i;
1615
1616 for (i = 0; i < emc->num_timings; i++) {
1617 seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1618 prefix = " ";
1619 }
1620
1621 seq_puts(s, "\n");
1622
1623 return 0;
1624 }
1625 DEFINE_SHOW_ATTRIBUTE(tegra210_emc_debug_available_rates);
1626
tegra210_emc_debug_min_rate_get(void * data,u64 * rate)1627 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1628 {
1629 struct tegra210_emc *emc = data;
1630
1631 *rate = emc->debugfs.min_rate;
1632
1633 return 0;
1634 }
1635
tegra210_emc_debug_min_rate_set(void * data,u64 rate)1636 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1637 {
1638 struct tegra210_emc *emc = data;
1639 int err;
1640
1641 if (!tegra210_emc_validate_rate(emc, rate))
1642 return -EINVAL;
1643
1644 err = clk_set_min_rate(emc->clk, rate);
1645 if (err < 0)
1646 return err;
1647
1648 emc->debugfs.min_rate = rate;
1649
1650 return 0;
1651 }
1652
1653 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1654 tegra210_emc_debug_min_rate_get,
1655 tegra210_emc_debug_min_rate_set, "%llu\n");
1656
tegra210_emc_debug_max_rate_get(void * data,u64 * rate)1657 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1658 {
1659 struct tegra210_emc *emc = data;
1660
1661 *rate = emc->debugfs.max_rate;
1662
1663 return 0;
1664 }
1665
tegra210_emc_debug_max_rate_set(void * data,u64 rate)1666 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1667 {
1668 struct tegra210_emc *emc = data;
1669 int err;
1670
1671 if (!tegra210_emc_validate_rate(emc, rate))
1672 return -EINVAL;
1673
1674 err = clk_set_max_rate(emc->clk, rate);
1675 if (err < 0)
1676 return err;
1677
1678 emc->debugfs.max_rate = rate;
1679
1680 return 0;
1681 }
1682
1683 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1684 tegra210_emc_debug_max_rate_get,
1685 tegra210_emc_debug_max_rate_set, "%llu\n");
1686
tegra210_emc_debug_temperature_get(void * data,u64 * temperature)1687 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1688 {
1689 struct tegra210_emc *emc = data;
1690 unsigned int value;
1691
1692 if (!emc->debugfs.temperature)
1693 value = tegra210_emc_get_temperature(emc);
1694 else
1695 value = emc->debugfs.temperature;
1696
1697 *temperature = value;
1698
1699 return 0;
1700 }
1701
tegra210_emc_debug_temperature_set(void * data,u64 temperature)1702 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1703 {
1704 struct tegra210_emc *emc = data;
1705
1706 if (temperature > 7)
1707 return -EINVAL;
1708
1709 emc->debugfs.temperature = temperature;
1710
1711 return 0;
1712 }
1713
1714 DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1715 tegra210_emc_debug_temperature_get,
1716 tegra210_emc_debug_temperature_set, "%llu\n");
1717
tegra210_emc_debugfs_init(struct tegra210_emc * emc)1718 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1719 {
1720 struct device *dev = emc->dev;
1721 unsigned int i;
1722 int err;
1723
1724 emc->debugfs.min_rate = ULONG_MAX;
1725 emc->debugfs.max_rate = 0;
1726
1727 for (i = 0; i < emc->num_timings; i++) {
1728 if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1729 emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1730
1731 if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1732 emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1733 }
1734
1735 if (!emc->num_timings) {
1736 emc->debugfs.min_rate = clk_get_rate(emc->clk);
1737 emc->debugfs.max_rate = emc->debugfs.min_rate;
1738 }
1739
1740 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1741 emc->debugfs.max_rate);
1742 if (err < 0) {
1743 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1744 emc->debugfs.min_rate, emc->debugfs.max_rate,
1745 emc->clk);
1746 return;
1747 }
1748
1749 emc->debugfs.root = debugfs_create_dir("emc", NULL);
1750
1751 debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1752 &tegra210_emc_debug_available_rates_fops);
1753 debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1754 &tegra210_emc_debug_min_rate_fops);
1755 debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1756 &tegra210_emc_debug_max_rate_fops);
1757 debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1758 &tegra210_emc_debug_temperature_fops);
1759 }
1760
tegra210_emc_detect(struct tegra210_emc * emc)1761 static void tegra210_emc_detect(struct tegra210_emc *emc)
1762 {
1763 u32 value;
1764
1765 /* probe the number of connected DRAM devices */
1766 value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1767
1768 if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1769 emc->num_devices = 2;
1770 else
1771 emc->num_devices = 1;
1772
1773 /* probe the type of DRAM */
1774 value = emc_readl(emc, EMC_FBIO_CFG5);
1775 emc->dram_type = value & 0x3;
1776
1777 /* probe the number of channels */
1778 value = emc_readl(emc, EMC_FBIO_CFG7);
1779
1780 if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1781 (value & EMC_FBIO_CFG7_CH0_ENABLE))
1782 emc->num_channels = 2;
1783 else
1784 emc->num_channels = 1;
1785 }
1786
tegra210_emc_validate_timings(struct tegra210_emc * emc,struct tegra210_emc_timing * timings,unsigned int num_timings)1787 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1788 struct tegra210_emc_timing *timings,
1789 unsigned int num_timings)
1790 {
1791 unsigned int i;
1792
1793 for (i = 0; i < num_timings; i++) {
1794 u32 min_volt = timings[i].min_volt;
1795 u32 rate = timings[i].rate;
1796
1797 if (!rate)
1798 return -EINVAL;
1799
1800 if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1801 (min_volt < timings[i - 1].min_volt)))
1802 return -EINVAL;
1803
1804 if (timings[i].revision != timings[0].revision)
1805 continue;
1806 }
1807
1808 return 0;
1809 }
1810
tegra210_emc_probe(struct platform_device * pdev)1811 static int tegra210_emc_probe(struct platform_device *pdev)
1812 {
1813 struct thermal_cooling_device *cd;
1814 unsigned long current_rate;
1815 struct tegra210_emc *emc;
1816 struct device_node *np;
1817 unsigned int i;
1818 int err;
1819
1820 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1821 if (!emc)
1822 return -ENOMEM;
1823
1824 emc->clk = devm_clk_get(&pdev->dev, "emc");
1825 if (IS_ERR(emc->clk))
1826 return PTR_ERR(emc->clk);
1827
1828 platform_set_drvdata(pdev, emc);
1829 spin_lock_init(&emc->lock);
1830 emc->dev = &pdev->dev;
1831
1832 emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1833 if (IS_ERR(emc->mc))
1834 return PTR_ERR(emc->mc);
1835
1836 emc->regs = devm_platform_ioremap_resource(pdev, 0);
1837 if (IS_ERR(emc->regs))
1838 return PTR_ERR(emc->regs);
1839
1840 for (i = 0; i < 2; i++) {
1841 emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1842 if (IS_ERR(emc->channel[i]))
1843 return PTR_ERR(emc->channel[i]);
1844
1845 }
1846
1847 tegra210_emc_detect(emc);
1848 np = pdev->dev.of_node;
1849
1850 /* attach to the nominal and (optional) derated tables */
1851 err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1852 if (err < 0) {
1853 dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1854 return err;
1855 }
1856
1857 err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1858 if (err < 0 && err != -ENODEV) {
1859 dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1860 goto release;
1861 }
1862
1863 /* validate the tables */
1864 if (emc->nominal) {
1865 err = tegra210_emc_validate_timings(emc, emc->nominal,
1866 emc->num_timings);
1867 if (err < 0)
1868 goto release;
1869 }
1870
1871 if (emc->derated) {
1872 err = tegra210_emc_validate_timings(emc, emc->derated,
1873 emc->num_timings);
1874 if (err < 0)
1875 goto release;
1876 }
1877
1878 /* default to the nominal table */
1879 emc->timings = emc->nominal;
1880
1881 /* pick the current timing based on the current EMC clock rate */
1882 current_rate = clk_get_rate(emc->clk) / 1000;
1883
1884 for (i = 0; i < emc->num_timings; i++) {
1885 if (emc->timings[i].rate == current_rate) {
1886 emc->last = &emc->timings[i];
1887 break;
1888 }
1889 }
1890
1891 if (i == emc->num_timings) {
1892 dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1893 current_rate);
1894 err = -ENOENT;
1895 goto release;
1896 }
1897
1898 /* pick a compatible clock change sequence for the EMC table */
1899 for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1900 const struct tegra210_emc_sequence *sequence =
1901 tegra210_emc_sequences[i];
1902
1903 if (emc->timings[0].revision == sequence->revision) {
1904 emc->sequence = sequence;
1905 break;
1906 }
1907 }
1908
1909 if (!emc->sequence) {
1910 dev_err(&pdev->dev, "sequence %u not supported\n",
1911 emc->timings[0].revision);
1912 err = -ENOTSUPP;
1913 goto release;
1914 }
1915
1916 emc->offsets = &tegra210_emc_table_register_offsets;
1917 emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1918
1919 emc->provider.owner = THIS_MODULE;
1920 emc->provider.dev = &pdev->dev;
1921 emc->provider.set_rate = tegra210_emc_set_rate;
1922
1923 emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1924 sizeof(*emc->provider.configs),
1925 GFP_KERNEL);
1926 if (!emc->provider.configs) {
1927 err = -ENOMEM;
1928 goto release;
1929 }
1930
1931 emc->provider.num_configs = emc->num_timings;
1932
1933 for (i = 0; i < emc->provider.num_configs; i++) {
1934 struct tegra210_emc_timing *timing = &emc->timings[i];
1935 struct tegra210_clk_emc_config *config =
1936 &emc->provider.configs[i];
1937 u32 value;
1938
1939 config->rate = timing->rate * 1000UL;
1940 config->value = timing->clk_src_emc;
1941
1942 value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1943
1944 if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1945 config->same_freq = false;
1946 else
1947 config->same_freq = true;
1948 }
1949
1950 err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1951 if (err < 0) {
1952 dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1953 goto release;
1954 }
1955
1956 emc->clkchange_delay = 100;
1957 emc->training_interval = 100;
1958 dev_set_drvdata(emc->dev, emc);
1959
1960 timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1961 TIMER_DEFERRABLE);
1962 atomic_set(&emc->refresh_poll, 0);
1963 emc->refresh_poll_interval = 1000;
1964
1965 timer_setup(&emc->training, tegra210_emc_train, 0);
1966
1967 tegra210_emc_debugfs_init(emc);
1968
1969 cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1970 &tegra210_emc_cd_ops);
1971 if (IS_ERR(cd)) {
1972 err = PTR_ERR(cd);
1973 dev_err(emc->dev, "failed to register cooling device: %d\n",
1974 err);
1975 goto detach;
1976 }
1977
1978 return 0;
1979
1980 detach:
1981 debugfs_remove_recursive(emc->debugfs.root);
1982 tegra210_clk_emc_detach(emc->clk);
1983 release:
1984 of_reserved_mem_device_release(emc->dev);
1985
1986 return err;
1987 }
1988
tegra210_emc_remove(struct platform_device * pdev)1989 static void tegra210_emc_remove(struct platform_device *pdev)
1990 {
1991 struct tegra210_emc *emc = platform_get_drvdata(pdev);
1992
1993 debugfs_remove_recursive(emc->debugfs.root);
1994 tegra210_clk_emc_detach(emc->clk);
1995 of_reserved_mem_device_release(emc->dev);
1996 }
1997
tegra210_emc_suspend(struct device * dev)1998 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
1999 {
2000 struct tegra210_emc *emc = dev_get_drvdata(dev);
2001 int err;
2002
2003 err = clk_rate_exclusive_get(emc->clk);
2004 if (err < 0) {
2005 dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2006 return err;
2007 }
2008
2009 emc->resume_rate = clk_get_rate(emc->clk);
2010
2011 clk_set_rate(emc->clk, 204000000);
2012 tegra210_clk_emc_detach(emc->clk);
2013
2014 dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2015
2016 return 0;
2017 }
2018
tegra210_emc_resume(struct device * dev)2019 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2020 {
2021 struct tegra210_emc *emc = dev_get_drvdata(dev);
2022 int err;
2023
2024 err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2025 if (err < 0) {
2026 dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2027 return err;
2028 }
2029
2030 clk_set_rate(emc->clk, emc->resume_rate);
2031 clk_rate_exclusive_put(emc->clk);
2032
2033 dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2034
2035 return 0;
2036 }
2037
2038 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2039 SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2040 };
2041
2042 static const struct of_device_id tegra210_emc_of_match[] = {
2043 { .compatible = "nvidia,tegra210-emc", },
2044 { },
2045 };
2046 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2047
2048 static struct platform_driver tegra210_emc_driver = {
2049 .driver = {
2050 .name = "tegra210-emc",
2051 .of_match_table = tegra210_emc_of_match,
2052 .pm = &tegra210_emc_pm_ops,
2053 },
2054 .probe = tegra210_emc_probe,
2055 .remove = tegra210_emc_remove,
2056 };
2057
2058 module_platform_driver(tegra210_emc_driver);
2059
2060 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2061 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2062 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2063 MODULE_LICENSE("GPL v2");
2064