1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/bitfield.h> 7 #include <linux/clk.h> 8 #include <linux/clk/tegra.h> 9 #include <linux/debugfs.h> 10 #include <linux/delay.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/of_address.h> 14 #include <linux/of_platform.h> 15 #include <linux/of_reserved_mem.h> 16 #include <linux/slab.h> 17 #include <linux/thermal.h> 18 #include <soc/tegra/fuse.h> 19 #include <soc/tegra/mc.h> 20 21 #include "tegra210-emc.h" 22 #include "tegra210-mc.h" 23 24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */ 25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT 29 26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK \ 27 (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT) 28 #define EMC_CLK_SOURCE_PLLM_LJ 0x4 29 #define EMC_CLK_SOURCE_PLLMB_LJ 0x5 30 #define EMC_CLK_FORCE_CC_TRIGGER BIT(27) 31 #define EMC_CLK_MC_EMC_SAME_FREQ BIT(16) 32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT 0 33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK \ 34 (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT) 35 36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */ 37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT 29 38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK \ 39 (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT) 40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT 10 41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK \ 42 (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT) 43 #define PLLM_VCOA 0 44 #define PLLM_VCOB 1 45 #define EMC_DLL_SWITCH_OUT 2 46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT 0 47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK \ 48 (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT) 49 50 /* MC_EMEM_ARB_MISC0 */ 51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ BIT(27) 52 53 /* EMC_DATA_BRLSHFT_X */ 54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX 2 55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX 3 56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX 4 57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX 5 58 59 #define TRIM_REG(chan, rank, reg, byte) \ 60 (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ 61 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \ 62 next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \ 63 rank ## _ ## reg ## _INDEX]) >> \ 64 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ 65 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \ 66 + \ 67 (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \ 68 byte ## _DATA_BRLSHFT_MASK & \ 69 next->trim_perch_regs[EMC ## chan ## \ 70 _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \ 71 EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \ 72 byte ## _DATA_BRLSHFT_SHIFT) * 64)) 73 74 #define CALC_TEMP(rank, reg, byte1, byte2, n) \ 75 (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \ 76 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \ 77 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ 78 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \ 79 | \ 80 ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\ 81 reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \ 82 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \ 83 _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK)) 84 85 #define REFRESH_SPEEDUP(value, speedup) \ 86 (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup)) 87 88 #define LPDDR2_MR4_SRR GENMASK(2, 0) 89 90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = { 91 &tegra210_emc_r21021, 92 }; 93 94 static const struct tegra210_emc_table_register_offsets 95 tegra210_emc_table_register_offsets = { 96 .burst = { 97 EMC_RC, 98 EMC_RFC, 99 EMC_RFCPB, 100 EMC_REFCTRL2, 101 EMC_RFC_SLR, 102 EMC_RAS, 103 EMC_RP, 104 EMC_R2W, 105 EMC_W2R, 106 EMC_R2P, 107 EMC_W2P, 108 EMC_R2R, 109 EMC_TPPD, 110 EMC_CCDMW, 111 EMC_RD_RCD, 112 EMC_WR_RCD, 113 EMC_RRD, 114 EMC_REXT, 115 EMC_WEXT, 116 EMC_WDV_CHK, 117 EMC_WDV, 118 EMC_WSV, 119 EMC_WEV, 120 EMC_WDV_MASK, 121 EMC_WS_DURATION, 122 EMC_WE_DURATION, 123 EMC_QUSE, 124 EMC_QUSE_WIDTH, 125 EMC_IBDLY, 126 EMC_OBDLY, 127 EMC_EINPUT, 128 EMC_MRW6, 129 EMC_EINPUT_DURATION, 130 EMC_PUTERM_EXTRA, 131 EMC_PUTERM_WIDTH, 132 EMC_QRST, 133 EMC_QSAFE, 134 EMC_RDV, 135 EMC_RDV_MASK, 136 EMC_RDV_EARLY, 137 EMC_RDV_EARLY_MASK, 138 EMC_REFRESH, 139 EMC_BURST_REFRESH_NUM, 140 EMC_PRE_REFRESH_REQ_CNT, 141 EMC_PDEX2WR, 142 EMC_PDEX2RD, 143 EMC_PCHG2PDEN, 144 EMC_ACT2PDEN, 145 EMC_AR2PDEN, 146 EMC_RW2PDEN, 147 EMC_CKE2PDEN, 148 EMC_PDEX2CKE, 149 EMC_PDEX2MRR, 150 EMC_TXSR, 151 EMC_TXSRDLL, 152 EMC_TCKE, 153 EMC_TCKESR, 154 EMC_TPD, 155 EMC_TFAW, 156 EMC_TRPAB, 157 EMC_TCLKSTABLE, 158 EMC_TCLKSTOP, 159 EMC_MRW7, 160 EMC_TREFBW, 161 EMC_ODT_WRITE, 162 EMC_FBIO_CFG5, 163 EMC_FBIO_CFG7, 164 EMC_CFG_DIG_DLL, 165 EMC_CFG_DIG_DLL_PERIOD, 166 EMC_PMACRO_IB_RXRT, 167 EMC_CFG_PIPE_1, 168 EMC_CFG_PIPE_2, 169 EMC_PMACRO_QUSE_DDLL_RANK0_4, 170 EMC_PMACRO_QUSE_DDLL_RANK0_5, 171 EMC_PMACRO_QUSE_DDLL_RANK1_4, 172 EMC_PMACRO_QUSE_DDLL_RANK1_5, 173 EMC_MRW8, 174 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4, 175 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5, 176 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0, 177 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1, 178 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2, 179 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3, 180 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4, 181 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5, 182 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0, 183 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1, 184 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2, 185 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3, 186 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4, 187 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5, 188 EMC_PMACRO_DDLL_LONG_CMD_0, 189 EMC_PMACRO_DDLL_LONG_CMD_1, 190 EMC_PMACRO_DDLL_LONG_CMD_2, 191 EMC_PMACRO_DDLL_LONG_CMD_3, 192 EMC_PMACRO_DDLL_LONG_CMD_4, 193 EMC_PMACRO_DDLL_SHORT_CMD_0, 194 EMC_PMACRO_DDLL_SHORT_CMD_1, 195 EMC_PMACRO_DDLL_SHORT_CMD_2, 196 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3, 197 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3, 198 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3, 199 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3, 200 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3, 201 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3, 202 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3, 203 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3, 204 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3, 205 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3, 206 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3, 207 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3, 208 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3, 209 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3, 210 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3, 211 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3, 212 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3, 213 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3, 214 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3, 215 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3, 216 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0, 217 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1, 218 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2, 219 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3, 220 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0, 221 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1, 222 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2, 223 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3, 224 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0, 225 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1, 226 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2, 227 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3, 228 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0, 229 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1, 230 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2, 231 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3, 232 EMC_TXDSRVTTGEN, 233 EMC_FDPD_CTRL_DQ, 234 EMC_FDPD_CTRL_CMD, 235 EMC_FBIO_SPARE, 236 EMC_ZCAL_INTERVAL, 237 EMC_ZCAL_WAIT_CNT, 238 EMC_MRS_WAIT_CNT, 239 EMC_MRS_WAIT_CNT2, 240 EMC_AUTO_CAL_CHANNEL, 241 EMC_DLL_CFG_0, 242 EMC_DLL_CFG_1, 243 EMC_PMACRO_AUTOCAL_CFG_COMMON, 244 EMC_PMACRO_ZCTRL, 245 EMC_CFG, 246 EMC_CFG_PIPE, 247 EMC_DYN_SELF_REF_CONTROL, 248 EMC_QPOP, 249 EMC_DQS_BRLSHFT_0, 250 EMC_DQS_BRLSHFT_1, 251 EMC_CMD_BRLSHFT_2, 252 EMC_CMD_BRLSHFT_3, 253 EMC_PMACRO_PAD_CFG_CTRL, 254 EMC_PMACRO_DATA_PAD_RX_CTRL, 255 EMC_PMACRO_CMD_PAD_RX_CTRL, 256 EMC_PMACRO_DATA_RX_TERM_MODE, 257 EMC_PMACRO_CMD_RX_TERM_MODE, 258 EMC_PMACRO_CMD_PAD_TX_CTRL, 259 EMC_PMACRO_DATA_PAD_TX_CTRL, 260 EMC_PMACRO_COMMON_PAD_TX_CTRL, 261 EMC_PMACRO_VTTGEN_CTRL_0, 262 EMC_PMACRO_VTTGEN_CTRL_1, 263 EMC_PMACRO_VTTGEN_CTRL_2, 264 EMC_PMACRO_BRICK_CTRL_RFU1, 265 EMC_PMACRO_CMD_BRICK_CTRL_FDPD, 266 EMC_PMACRO_BRICK_CTRL_RFU2, 267 EMC_PMACRO_DATA_BRICK_CTRL_FDPD, 268 EMC_PMACRO_BG_BIAS_CTRL_0, 269 EMC_CFG_3, 270 EMC_PMACRO_TX_PWRD_0, 271 EMC_PMACRO_TX_PWRD_1, 272 EMC_PMACRO_TX_PWRD_2, 273 EMC_PMACRO_TX_PWRD_3, 274 EMC_PMACRO_TX_PWRD_4, 275 EMC_PMACRO_TX_PWRD_5, 276 EMC_CONFIG_SAMPLE_DELAY, 277 EMC_PMACRO_TX_SEL_CLK_SRC_0, 278 EMC_PMACRO_TX_SEL_CLK_SRC_1, 279 EMC_PMACRO_TX_SEL_CLK_SRC_2, 280 EMC_PMACRO_TX_SEL_CLK_SRC_3, 281 EMC_PMACRO_TX_SEL_CLK_SRC_4, 282 EMC_PMACRO_TX_SEL_CLK_SRC_5, 283 EMC_PMACRO_DDLL_BYPASS, 284 EMC_PMACRO_DDLL_PWRD_0, 285 EMC_PMACRO_DDLL_PWRD_1, 286 EMC_PMACRO_DDLL_PWRD_2, 287 EMC_PMACRO_CMD_CTRL_0, 288 EMC_PMACRO_CMD_CTRL_1, 289 EMC_PMACRO_CMD_CTRL_2, 290 EMC_TR_TIMING_0, 291 EMC_TR_DVFS, 292 EMC_TR_CTRL_1, 293 EMC_TR_RDV, 294 EMC_TR_QPOP, 295 EMC_TR_RDV_MASK, 296 EMC_MRW14, 297 EMC_TR_QSAFE, 298 EMC_TR_QRST, 299 EMC_TRAINING_CTRL, 300 EMC_TRAINING_SETTLE, 301 EMC_TRAINING_VREF_SETTLE, 302 EMC_TRAINING_CA_FINE_CTRL, 303 EMC_TRAINING_CA_CTRL_MISC, 304 EMC_TRAINING_CA_CTRL_MISC1, 305 EMC_TRAINING_CA_VREF_CTRL, 306 EMC_TRAINING_QUSE_CORS_CTRL, 307 EMC_TRAINING_QUSE_FINE_CTRL, 308 EMC_TRAINING_QUSE_CTRL_MISC, 309 EMC_TRAINING_QUSE_VREF_CTRL, 310 EMC_TRAINING_READ_FINE_CTRL, 311 EMC_TRAINING_READ_CTRL_MISC, 312 EMC_TRAINING_READ_VREF_CTRL, 313 EMC_TRAINING_WRITE_FINE_CTRL, 314 EMC_TRAINING_WRITE_CTRL_MISC, 315 EMC_TRAINING_WRITE_VREF_CTRL, 316 EMC_TRAINING_MPC, 317 EMC_MRW15, 318 }, 319 .trim = { 320 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0, 321 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1, 322 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2, 323 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3, 324 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0, 325 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1, 326 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2, 327 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3, 328 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0, 329 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1, 330 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2, 331 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0, 332 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1, 333 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2, 334 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0, 335 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1, 336 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2, 337 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0, 338 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1, 339 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2, 340 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0, 341 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1, 342 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2, 343 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0, 344 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1, 345 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2, 346 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0, 347 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1, 348 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2, 349 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0, 350 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1, 351 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2, 352 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0, 353 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1, 354 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2, 355 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0, 356 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1, 357 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2, 358 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0, 359 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1, 360 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2, 361 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0, 362 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1, 363 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2, 364 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0, 365 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1, 366 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2, 367 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0, 368 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1, 369 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2, 370 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0, 371 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1, 372 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2, 373 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0, 374 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1, 375 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2, 376 EMC_PMACRO_IB_VREF_DQS_0, 377 EMC_PMACRO_IB_VREF_DQS_1, 378 EMC_PMACRO_IB_VREF_DQ_0, 379 EMC_PMACRO_IB_VREF_DQ_1, 380 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, 381 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, 382 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, 383 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, 384 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4, 385 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5, 386 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, 387 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, 388 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, 389 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, 390 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0, 391 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1, 392 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2, 393 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0, 394 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1, 395 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2, 396 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0, 397 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1, 398 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2, 399 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0, 400 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1, 401 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2, 402 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0, 403 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1, 404 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2, 405 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0, 406 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1, 407 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2, 408 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0, 409 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1, 410 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2, 411 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0, 412 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1, 413 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2, 414 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0, 415 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1, 416 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2, 417 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0, 418 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1, 419 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2, 420 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0, 421 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1, 422 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2, 423 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0, 424 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1, 425 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2, 426 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0, 427 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1, 428 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2, 429 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0, 430 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1, 431 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2, 432 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0, 433 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1, 434 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2, 435 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0, 436 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1, 437 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2, 438 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0, 439 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1, 440 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2, 441 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0, 442 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1, 443 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2, 444 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0, 445 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1, 446 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2, 447 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0, 448 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1, 449 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2, 450 EMC_PMACRO_QUSE_DDLL_RANK0_0, 451 EMC_PMACRO_QUSE_DDLL_RANK0_1, 452 EMC_PMACRO_QUSE_DDLL_RANK0_2, 453 EMC_PMACRO_QUSE_DDLL_RANK0_3, 454 EMC_PMACRO_QUSE_DDLL_RANK1_0, 455 EMC_PMACRO_QUSE_DDLL_RANK1_1, 456 EMC_PMACRO_QUSE_DDLL_RANK1_2, 457 EMC_PMACRO_QUSE_DDLL_RANK1_3 458 }, 459 .burst_mc = { 460 MC_EMEM_ARB_CFG, 461 MC_EMEM_ARB_OUTSTANDING_REQ, 462 MC_EMEM_ARB_REFPB_HP_CTRL, 463 MC_EMEM_ARB_REFPB_BANK_CTRL, 464 MC_EMEM_ARB_TIMING_RCD, 465 MC_EMEM_ARB_TIMING_RP, 466 MC_EMEM_ARB_TIMING_RC, 467 MC_EMEM_ARB_TIMING_RAS, 468 MC_EMEM_ARB_TIMING_FAW, 469 MC_EMEM_ARB_TIMING_RRD, 470 MC_EMEM_ARB_TIMING_RAP2PRE, 471 MC_EMEM_ARB_TIMING_WAP2PRE, 472 MC_EMEM_ARB_TIMING_R2R, 473 MC_EMEM_ARB_TIMING_W2W, 474 MC_EMEM_ARB_TIMING_R2W, 475 MC_EMEM_ARB_TIMING_CCDMW, 476 MC_EMEM_ARB_TIMING_W2R, 477 MC_EMEM_ARB_TIMING_RFCPB, 478 MC_EMEM_ARB_DA_TURNS, 479 MC_EMEM_ARB_DA_COVERS, 480 MC_EMEM_ARB_MISC0, 481 MC_EMEM_ARB_MISC1, 482 MC_EMEM_ARB_MISC2, 483 MC_EMEM_ARB_RING1_THROTTLE, 484 MC_EMEM_ARB_DHYST_CTRL, 485 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0, 486 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1, 487 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2, 488 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3, 489 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4, 490 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5, 491 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6, 492 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7, 493 }, 494 .la_scale = { 495 MC_MLL_MPCORER_PTSA_RATE, 496 MC_FTOP_PTSA_RATE, 497 MC_PTSA_GRANT_DECREMENT, 498 MC_LATENCY_ALLOWANCE_XUSB_0, 499 MC_LATENCY_ALLOWANCE_XUSB_1, 500 MC_LATENCY_ALLOWANCE_TSEC_0, 501 MC_LATENCY_ALLOWANCE_SDMMCA_0, 502 MC_LATENCY_ALLOWANCE_SDMMCAA_0, 503 MC_LATENCY_ALLOWANCE_SDMMC_0, 504 MC_LATENCY_ALLOWANCE_SDMMCAB_0, 505 MC_LATENCY_ALLOWANCE_PPCS_0, 506 MC_LATENCY_ALLOWANCE_PPCS_1, 507 MC_LATENCY_ALLOWANCE_MPCORE_0, 508 MC_LATENCY_ALLOWANCE_HC_0, 509 MC_LATENCY_ALLOWANCE_HC_1, 510 MC_LATENCY_ALLOWANCE_AVPC_0, 511 MC_LATENCY_ALLOWANCE_GPU_0, 512 MC_LATENCY_ALLOWANCE_GPU2_0, 513 MC_LATENCY_ALLOWANCE_NVENC_0, 514 MC_LATENCY_ALLOWANCE_NVDEC_0, 515 MC_LATENCY_ALLOWANCE_VIC_0, 516 MC_LATENCY_ALLOWANCE_VI2_0, 517 MC_LATENCY_ALLOWANCE_ISP2_0, 518 MC_LATENCY_ALLOWANCE_ISP2_1, 519 }, 520 .burst_per_channel = { 521 { .bank = 0, .offset = EMC_MRW10, }, 522 { .bank = 1, .offset = EMC_MRW10, }, 523 { .bank = 0, .offset = EMC_MRW11, }, 524 { .bank = 1, .offset = EMC_MRW11, }, 525 { .bank = 0, .offset = EMC_MRW12, }, 526 { .bank = 1, .offset = EMC_MRW12, }, 527 { .bank = 0, .offset = EMC_MRW13, }, 528 { .bank = 1, .offset = EMC_MRW13, }, 529 }, 530 .trim_per_channel = { 531 { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, }, 532 { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, }, 533 { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, }, 534 { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, }, 535 { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, }, 536 { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, }, 537 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, }, 538 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, }, 539 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, }, 540 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, }, 541 }, 542 .vref_per_channel = { 543 { 544 .bank = 0, 545 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0, 546 }, { 547 .bank = 1, 548 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0, 549 }, { 550 .bank = 0, 551 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1, 552 }, { 553 .bank = 1, 554 .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1, 555 }, 556 }, 557 }; 558 559 static void tegra210_emc_train(struct timer_list *timer) 560 { 561 struct tegra210_emc *emc = from_timer(emc, timer, training); 562 unsigned long flags; 563 564 if (!emc->last) 565 return; 566 567 spin_lock_irqsave(&emc->lock, flags); 568 569 if (emc->sequence->periodic_compensation) 570 emc->sequence->periodic_compensation(emc); 571 572 spin_unlock_irqrestore(&emc->lock, flags); 573 574 mod_timer(&emc->training, 575 jiffies + msecs_to_jiffies(emc->training_interval)); 576 } 577 578 static void tegra210_emc_training_start(struct tegra210_emc *emc) 579 { 580 mod_timer(&emc->training, 581 jiffies + msecs_to_jiffies(emc->training_interval)); 582 } 583 584 static void tegra210_emc_training_stop(struct tegra210_emc *emc) 585 { 586 del_timer(&emc->training); 587 } 588 589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc) 590 { 591 unsigned long flags; 592 u32 value, max = 0; 593 unsigned int i; 594 595 spin_lock_irqsave(&emc->lock, flags); 596 597 for (i = 0; i < emc->num_devices; i++) { 598 value = tegra210_emc_mrr_read(emc, i, 4); 599 600 if (value & BIT(7)) 601 dev_dbg(emc->dev, 602 "sensor reading changed for device %u: %08x\n", 603 i, value); 604 605 value = FIELD_GET(LPDDR2_MR4_SRR, value); 606 if (value > max) 607 max = value; 608 } 609 610 spin_unlock_irqrestore(&emc->lock, flags); 611 612 return max; 613 } 614 615 static void tegra210_emc_poll_refresh(struct timer_list *timer) 616 { 617 struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer); 618 unsigned int temperature; 619 620 if (!emc->debugfs.temperature) 621 temperature = tegra210_emc_get_temperature(emc); 622 else 623 temperature = emc->debugfs.temperature; 624 625 if (temperature == emc->temperature) 626 goto reset; 627 628 switch (temperature) { 629 case 0 ... 3: 630 /* temperature is fine, using regular refresh */ 631 dev_dbg(emc->dev, "switching to nominal refresh...\n"); 632 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL); 633 break; 634 635 case 4: 636 dev_dbg(emc->dev, "switching to 2x refresh...\n"); 637 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X); 638 break; 639 640 case 5: 641 dev_dbg(emc->dev, "switching to 4x refresh...\n"); 642 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X); 643 break; 644 645 case 6 ... 7: 646 dev_dbg(emc->dev, "switching to throttle refresh...\n"); 647 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE); 648 break; 649 650 default: 651 WARN(1, "invalid DRAM temperature state %u\n", temperature); 652 return; 653 } 654 655 emc->temperature = temperature; 656 657 reset: 658 if (atomic_read(&emc->refresh_poll) > 0) { 659 unsigned int interval = emc->refresh_poll_interval; 660 unsigned int timeout = msecs_to_jiffies(interval); 661 662 mod_timer(&emc->refresh_timer, jiffies + timeout); 663 } 664 } 665 666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc) 667 { 668 atomic_set(&emc->refresh_poll, 0); 669 del_timer_sync(&emc->refresh_timer); 670 } 671 672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc) 673 { 674 atomic_set(&emc->refresh_poll, 1); 675 676 mod_timer(&emc->refresh_timer, 677 jiffies + msecs_to_jiffies(emc->refresh_poll_interval)); 678 } 679 680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd, 681 unsigned long *state) 682 { 683 *state = 1; 684 685 return 0; 686 } 687 688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd, 689 unsigned long *state) 690 { 691 struct tegra210_emc *emc = cd->devdata; 692 693 *state = atomic_read(&emc->refresh_poll); 694 695 return 0; 696 } 697 698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd, 699 unsigned long state) 700 { 701 struct tegra210_emc *emc = cd->devdata; 702 703 if (state == atomic_read(&emc->refresh_poll)) 704 return 0; 705 706 if (state) 707 tegra210_emc_poll_refresh_start(emc); 708 else 709 tegra210_emc_poll_refresh_stop(emc); 710 711 return 0; 712 } 713 714 static struct thermal_cooling_device_ops tegra210_emc_cd_ops = { 715 .get_max_state = tegra210_emc_cd_max_state, 716 .get_cur_state = tegra210_emc_cd_get_state, 717 .set_cur_state = tegra210_emc_cd_set_state, 718 }; 719 720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc) 721 { 722 emc->sequence->set_clock(emc, clksrc); 723 724 if (emc->next->periodic_training) 725 tegra210_emc_training_start(emc); 726 else 727 tegra210_emc_training_stop(emc); 728 } 729 730 static void tegra210_change_dll_src(struct tegra210_emc *emc, 731 u32 clksrc) 732 { 733 u32 dll_setting = emc->next->dll_clk_src; 734 u32 emc_clk_src; 735 u32 emc_clk_div; 736 737 emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >> 738 EMC_CLK_EMC_2X_CLK_SRC_SHIFT; 739 emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >> 740 EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT; 741 742 dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK | 743 DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK); 744 dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT; 745 dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT; 746 747 dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK; 748 if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ) 749 dll_setting |= (PLLM_VCOB << 750 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); 751 else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ) 752 dll_setting |= (PLLM_VCOA << 753 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); 754 else 755 dll_setting |= (EMC_DLL_SWITCH_OUT << 756 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT); 757 758 tegra210_clk_emc_dll_update_setting(dll_setting); 759 760 if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll) 761 tegra210_clk_emc_dll_enable(true); 762 else 763 tegra210_clk_emc_dll_enable(false); 764 } 765 766 int tegra210_emc_set_refresh(struct tegra210_emc *emc, 767 enum tegra210_emc_refresh refresh) 768 { 769 struct tegra210_emc_timing *timings; 770 unsigned long flags; 771 772 if ((emc->dram_type != DRAM_TYPE_LPDDR2 && 773 emc->dram_type != DRAM_TYPE_LPDDR4) || 774 !emc->last) 775 return -ENODEV; 776 777 if (refresh > TEGRA210_EMC_REFRESH_THROTTLE) 778 return -EINVAL; 779 780 if (refresh == emc->refresh) 781 return 0; 782 783 spin_lock_irqsave(&emc->lock, flags); 784 785 if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated) 786 timings = emc->derated; 787 else 788 timings = emc->nominal; 789 790 if (timings != emc->timings) { 791 unsigned int index = emc->last - emc->timings; 792 u32 clksrc; 793 794 clksrc = emc->provider.configs[index].value | 795 EMC_CLK_FORCE_CC_TRIGGER; 796 797 emc->next = &timings[index]; 798 emc->timings = timings; 799 800 tegra210_emc_set_clock(emc, clksrc); 801 } else { 802 tegra210_emc_adjust_timing(emc, emc->last); 803 tegra210_emc_timing_update(emc); 804 805 if (refresh != TEGRA210_EMC_REFRESH_NOMINAL) 806 emc_writel(emc, EMC_REF_REF_CMD, EMC_REF); 807 } 808 809 spin_unlock_irqrestore(&emc->lock, flags); 810 811 return 0; 812 } 813 814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip, 815 unsigned int address) 816 { 817 u32 value, ret = 0; 818 unsigned int i; 819 820 value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT | 821 (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT; 822 emc_writel(emc, value, EMC_MRR); 823 824 for (i = 0; i < emc->num_channels; i++) 825 WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 826 EMC_EMC_STATUS_MRR_DIVLD, 1), 827 "Timed out waiting for MRR %u (ch=%u)\n", address, i); 828 829 for (i = 0; i < emc->num_channels; i++) { 830 value = emc_channel_readl(emc, i, EMC_MRR); 831 value &= EMC_MRR_DATA_MASK; 832 833 ret = (ret << 16) | value; 834 } 835 836 return ret; 837 } 838 839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc) 840 { 841 int err; 842 843 mc_readl(emc->mc, MC_EMEM_ADR_CFG); 844 emc_readl(emc, EMC_INTSTATUS); 845 846 tegra210_clk_emc_update_setting(clksrc); 847 848 err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS, 849 EMC_INTSTATUS_CLKCHANGE_COMPLETE, 850 true); 851 if (err) 852 dev_warn(emc->dev, "clock change completion error: %d\n", err); 853 } 854 855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc, 856 unsigned long rate) 857 { 858 unsigned int i; 859 860 for (i = 0; i < emc->num_timings; i++) 861 if (emc->timings[i].rate * 1000UL == rate) 862 return &emc->timings[i]; 863 864 return NULL; 865 } 866 867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel, 868 unsigned int offset, u32 bit_mask, bool state) 869 { 870 unsigned int i; 871 u32 value; 872 873 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) { 874 value = emc_channel_readl(emc, channel, offset); 875 if (!!(value & bit_mask) == state) 876 return 0; 877 878 udelay(1); 879 } 880 881 return -ETIMEDOUT; 882 } 883 884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set) 885 { 886 u32 emc_dbg = emc_readl(emc, EMC_DBG); 887 888 if (set) 889 emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG); 890 else 891 emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG); 892 } 893 894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next) 895 { 896 if (next->emc_emrs & 0x1) 897 return 0; 898 899 return 1; 900 } 901 902 void tegra210_emc_timing_update(struct tegra210_emc *emc) 903 { 904 unsigned int i; 905 int err = 0; 906 907 emc_writel(emc, 0x1, EMC_TIMING_CONTROL); 908 909 for (i = 0; i < emc->num_channels; i++) { 910 err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 911 EMC_EMC_STATUS_TIMING_UPDATE_STALLED, 912 false); 913 } 914 915 if (err) 916 dev_warn(emc->dev, "timing update error: %d\n", err); 917 } 918 919 unsigned long tegra210_emc_actual_osc_clocks(u32 in) 920 { 921 if (in < 0x40) 922 return in * 16; 923 else if (in < 0x80) 924 return 2048; 925 else if (in < 0xc0) 926 return 4096; 927 else 928 return 8192; 929 } 930 931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc) 932 { 933 u32 mpc_req = 0x4b; 934 935 emc_writel(emc, mpc_req, EMC_MPC); 936 mpc_req = emc_readl(emc, EMC_MPC); 937 } 938 939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset) 940 { 941 u32 temp = 0, rate = next->rate / 1000; 942 s32 delta[4], delta_taps[4]; 943 s32 new[] = { 944 TRIM_REG(0, 0, 0, 0), 945 TRIM_REG(0, 0, 0, 1), 946 TRIM_REG(0, 0, 1, 2), 947 TRIM_REG(0, 0, 1, 3), 948 949 TRIM_REG(1, 0, 2, 4), 950 TRIM_REG(1, 0, 2, 5), 951 TRIM_REG(1, 0, 3, 6), 952 TRIM_REG(1, 0, 3, 7), 953 954 TRIM_REG(0, 1, 0, 0), 955 TRIM_REG(0, 1, 0, 1), 956 TRIM_REG(0, 1, 1, 2), 957 TRIM_REG(0, 1, 1, 3), 958 959 TRIM_REG(1, 1, 2, 4), 960 TRIM_REG(1, 1, 2, 5), 961 TRIM_REG(1, 1, 3, 6), 962 TRIM_REG(1, 1, 3, 7) 963 }; 964 unsigned i; 965 966 switch (offset) { 967 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0: 968 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1: 969 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2: 970 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3: 971 case EMC_DATA_BRLSHFT_0: 972 delta[0] = 128 * (next->current_dram_clktree[C0D0U0] - 973 next->trained_dram_clktree[C0D0U0]); 974 delta[1] = 128 * (next->current_dram_clktree[C0D0U1] - 975 next->trained_dram_clktree[C0D0U1]); 976 delta[2] = 128 * (next->current_dram_clktree[C1D0U0] - 977 next->trained_dram_clktree[C1D0U0]); 978 delta[3] = 128 * (next->current_dram_clktree[C1D0U1] - 979 next->trained_dram_clktree[C1D0U1]); 980 981 delta_taps[0] = (delta[0] * (s32)rate) / 1000000; 982 delta_taps[1] = (delta[1] * (s32)rate) / 1000000; 983 delta_taps[2] = (delta[2] * (s32)rate) / 1000000; 984 delta_taps[3] = (delta[3] * (s32)rate) / 1000000; 985 986 for (i = 0; i < 4; i++) { 987 if ((delta_taps[i] > next->tree_margin) || 988 (delta_taps[i] < (-1 * next->tree_margin))) { 989 new[i * 2] = new[i * 2] + delta_taps[i]; 990 new[i * 2 + 1] = new[i * 2 + 1] + 991 delta_taps[i]; 992 } 993 } 994 995 if (offset == EMC_DATA_BRLSHFT_0) { 996 for (i = 0; i < 8; i++) 997 new[i] = new[i] / 64; 998 } else { 999 for (i = 0; i < 8; i++) 1000 new[i] = new[i] % 64; 1001 } 1002 1003 break; 1004 1005 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0: 1006 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1: 1007 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2: 1008 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3: 1009 case EMC_DATA_BRLSHFT_1: 1010 delta[0] = 128 * (next->current_dram_clktree[C0D1U0] - 1011 next->trained_dram_clktree[C0D1U0]); 1012 delta[1] = 128 * (next->current_dram_clktree[C0D1U1] - 1013 next->trained_dram_clktree[C0D1U1]); 1014 delta[2] = 128 * (next->current_dram_clktree[C1D1U0] - 1015 next->trained_dram_clktree[C1D1U0]); 1016 delta[3] = 128 * (next->current_dram_clktree[C1D1U1] - 1017 next->trained_dram_clktree[C1D1U1]); 1018 1019 delta_taps[0] = (delta[0] * (s32)rate) / 1000000; 1020 delta_taps[1] = (delta[1] * (s32)rate) / 1000000; 1021 delta_taps[2] = (delta[2] * (s32)rate) / 1000000; 1022 delta_taps[3] = (delta[3] * (s32)rate) / 1000000; 1023 1024 for (i = 0; i < 4; i++) { 1025 if ((delta_taps[i] > next->tree_margin) || 1026 (delta_taps[i] < (-1 * next->tree_margin))) { 1027 new[8 + i * 2] = new[8 + i * 2] + 1028 delta_taps[i]; 1029 new[8 + i * 2 + 1] = new[8 + i * 2 + 1] + 1030 delta_taps[i]; 1031 } 1032 } 1033 1034 if (offset == EMC_DATA_BRLSHFT_1) { 1035 for (i = 0; i < 8; i++) 1036 new[i + 8] = new[i + 8] / 64; 1037 } else { 1038 for (i = 0; i < 8; i++) 1039 new[i + 8] = new[i + 8] % 64; 1040 } 1041 1042 break; 1043 } 1044 1045 switch (offset) { 1046 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0: 1047 temp = CALC_TEMP(0, 0, 0, 1, 0); 1048 break; 1049 1050 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1: 1051 temp = CALC_TEMP(0, 1, 2, 3, 2); 1052 break; 1053 1054 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2: 1055 temp = CALC_TEMP(0, 2, 4, 5, 4); 1056 break; 1057 1058 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3: 1059 temp = CALC_TEMP(0, 3, 6, 7, 6); 1060 break; 1061 1062 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0: 1063 temp = CALC_TEMP(1, 0, 0, 1, 8); 1064 break; 1065 1066 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1: 1067 temp = CALC_TEMP(1, 1, 2, 3, 10); 1068 break; 1069 1070 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2: 1071 temp = CALC_TEMP(1, 2, 4, 5, 12); 1072 break; 1073 1074 case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3: 1075 temp = CALC_TEMP(1, 3, 6, 7, 14); 1076 break; 1077 1078 case EMC_DATA_BRLSHFT_0: 1079 temp = ((new[0] << 1080 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) & 1081 EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) | 1082 ((new[1] << 1083 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) & 1084 EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) | 1085 ((new[2] << 1086 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) & 1087 EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) | 1088 ((new[3] << 1089 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) & 1090 EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) | 1091 ((new[4] << 1092 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) & 1093 EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) | 1094 ((new[5] << 1095 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) & 1096 EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) | 1097 ((new[6] << 1098 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) & 1099 EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) | 1100 ((new[7] << 1101 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) & 1102 EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK); 1103 break; 1104 1105 case EMC_DATA_BRLSHFT_1: 1106 temp = ((new[8] << 1107 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) & 1108 EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) | 1109 ((new[9] << 1110 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) & 1111 EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) | 1112 ((new[10] << 1113 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) & 1114 EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) | 1115 ((new[11] << 1116 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) & 1117 EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) | 1118 ((new[12] << 1119 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) & 1120 EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) | 1121 ((new[13] << 1122 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) & 1123 EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) | 1124 ((new[14] << 1125 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) & 1126 EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) | 1127 ((new[15] << 1128 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) & 1129 EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK); 1130 break; 1131 1132 default: 1133 break; 1134 } 1135 1136 return temp; 1137 } 1138 1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc) 1140 { 1141 unsigned int i; 1142 u32 value; 1143 1144 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1145 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK; 1146 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT); 1147 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 1148 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK; 1149 value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); 1150 value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; 1151 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; 1152 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; 1153 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1154 emc_writel(emc, 1, EMC_TIMING_CONTROL); 1155 1156 for (i = 0; i < emc->num_channels; i++) 1157 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, 1158 EMC_EMC_STATUS_TIMING_UPDATE_STALLED, 1159 0); 1160 1161 for (i = 0; i < emc->num_channels; i++) { 1162 while (true) { 1163 value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL); 1164 if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0) 1165 break; 1166 } 1167 } 1168 1169 value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX]; 1170 emc_writel(emc, value, EMC_DLL_CFG_0); 1171 1172 value = emc_readl(emc, EMC_DLL_CFG_1); 1173 value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK; 1174 1175 if (emc->next->rate >= 400000 && emc->next->rate < 600000) 1176 value |= 150; 1177 else if (emc->next->rate >= 600000 && emc->next->rate < 800000) 1178 value |= 100; 1179 else if (emc->next->rate >= 800000 && emc->next->rate < 1000000) 1180 value |= 70; 1181 else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000) 1182 value |= 30; 1183 else 1184 value |= 20; 1185 1186 emc_writel(emc, value, EMC_DLL_CFG_1); 1187 1188 tegra210_change_dll_src(emc, clksrc); 1189 1190 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1191 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; 1192 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1193 1194 tegra210_emc_timing_update(emc); 1195 1196 for (i = 0; i < emc->num_channels; i++) { 1197 while (true) { 1198 value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL); 1199 if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN) 1200 break; 1201 } 1202 } 1203 1204 while (true) { 1205 value = emc_readl(emc, EMC_DIG_DLL_STATUS); 1206 1207 if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0) 1208 continue; 1209 1210 if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0) 1211 continue; 1212 1213 break; 1214 } 1215 1216 value = emc_readl(emc, EMC_DIG_DLL_STATUS); 1217 1218 return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK; 1219 } 1220 1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk, 1222 bool flip_backward) 1223 { 1224 u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0; 1225 const struct tegra210_emc_timing *timing; 1226 1227 if (flip_backward) 1228 timing = emc->last; 1229 else 1230 timing = emc->next; 1231 1232 cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX]; 1233 dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 1234 rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX]; 1235 cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX]; 1236 common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX]; 1237 1238 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; 1239 1240 if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) { 1241 ccfifo_writel(emc, common_tx & 0xa, 1242 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0); 1243 ccfifo_writel(emc, common_tx & 0xf, 1244 EMC_PMACRO_COMMON_PAD_TX_CTRL, 1245 (100000 / clk) + 1); 1246 ramp_up_wait += 100000; 1247 } else { 1248 ccfifo_writel(emc, common_tx | 0x8, 1249 EMC_PMACRO_COMMON_PAD_TX_CTRL, 0); 1250 } 1251 1252 if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) { 1253 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) { 1254 cmd_pad |= 1255 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1256 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC; 1257 cmd_pad &= 1258 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1259 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); 1260 ccfifo_writel(emc, cmd_pad, 1261 EMC_PMACRO_CMD_PAD_TX_CTRL, 1262 (100000 / clk) + 1); 1263 ramp_up_wait += 100000; 1264 1265 dq_pad |= 1266 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1267 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC; 1268 dq_pad &= 1269 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1270 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); 1271 ccfifo_writel(emc, dq_pad, 1272 EMC_PMACRO_DATA_PAD_TX_CTRL, 0); 1273 ccfifo_writel(emc, rfu1 & 0xfe40fe40, 1274 EMC_PMACRO_BRICK_CTRL_RFU1, 0); 1275 } else { 1276 ccfifo_writel(emc, rfu1 & 0xfe40fe40, 1277 EMC_PMACRO_BRICK_CTRL_RFU1, 1278 (100000 / clk) + 1); 1279 ramp_up_wait += 100000; 1280 } 1281 1282 ccfifo_writel(emc, rfu1 & 0xfeedfeed, 1283 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1); 1284 ramp_up_wait += 100000; 1285 1286 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) { 1287 cmd_pad |= 1288 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1289 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | 1290 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1291 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC; 1292 ccfifo_writel(emc, cmd_pad, 1293 EMC_PMACRO_CMD_PAD_TX_CTRL, 1294 (100000 / clk) + 1); 1295 ramp_up_wait += 100000; 1296 1297 dq_pad |= 1298 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1299 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | 1300 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1301 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC; 1302 ccfifo_writel(emc, dq_pad, 1303 EMC_PMACRO_DATA_PAD_TX_CTRL, 0); 1304 ccfifo_writel(emc, rfu1, 1305 EMC_PMACRO_BRICK_CTRL_RFU1, 0); 1306 } else { 1307 ccfifo_writel(emc, rfu1, 1308 EMC_PMACRO_BRICK_CTRL_RFU1, 1309 (100000 / clk) + 1); 1310 ramp_up_wait += 100000; 1311 } 1312 1313 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, 1314 EMC_FBIO_CFG5, (100000 / clk) + 10); 1315 ramp_up_wait += 100000 + (10 * clk); 1316 } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) { 1317 ccfifo_writel(emc, rfu1 | 0x06000600, 1318 EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1); 1319 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, 1320 EMC_FBIO_CFG5, (100000 / clk) + 10); 1321 ramp_up_wait += 100000 + 10 * clk; 1322 } else { 1323 ccfifo_writel(emc, rfu1 | 0x00000600, 1324 EMC_PMACRO_BRICK_CTRL_RFU1, 0); 1325 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, 1326 EMC_FBIO_CFG5, 12); 1327 ramp_up_wait += 12 * clk; 1328 } 1329 1330 cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; 1331 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5); 1332 1333 return ramp_up_wait; 1334 } 1335 1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk, 1337 bool flip_backward) 1338 { 1339 u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx; 1340 const struct tegra210_emc_timing *entry; 1341 u32 seq_wait; 1342 1343 if (flip_backward) 1344 entry = emc->next; 1345 else 1346 entry = emc->last; 1347 1348 cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX]; 1349 dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; 1350 rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX]; 1351 cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX]; 1352 common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX]; 1353 1354 cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; 1355 1356 ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0); 1357 ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS, 1358 EMC_FBIO_CFG5, 12); 1359 ramp_down_wait = 12 * clk; 1360 1361 seq_wait = (100000 / clk) + 1; 1362 1363 if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) { 1364 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) { 1365 cmd_pad &= 1366 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1367 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); 1368 cmd_pad |= 1369 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1370 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC; 1371 ccfifo_writel(emc, cmd_pad, 1372 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait); 1373 ramp_down_wait += 100000; 1374 1375 dq_pad &= 1376 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1377 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); 1378 dq_pad |= 1379 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1380 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC; 1381 ccfifo_writel(emc, dq_pad, 1382 EMC_PMACRO_DATA_PAD_TX_CTRL, 0); 1383 ccfifo_writel(emc, rfu1 & ~0x01120112, 1384 EMC_PMACRO_BRICK_CTRL_RFU1, 0); 1385 } else { 1386 ccfifo_writel(emc, rfu1 & ~0x01120112, 1387 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); 1388 ramp_down_wait += 100000; 1389 } 1390 1391 ccfifo_writel(emc, rfu1 & ~0x01bf01bf, 1392 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); 1393 ramp_down_wait += 100000; 1394 1395 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) { 1396 cmd_pad &= 1397 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | 1398 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC | 1399 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | 1400 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC); 1401 ccfifo_writel(emc, cmd_pad, 1402 EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait); 1403 ramp_down_wait += 100000; 1404 1405 dq_pad &= 1406 ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | 1407 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC | 1408 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | 1409 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC); 1410 ccfifo_writel(emc, dq_pad, 1411 EMC_PMACRO_DATA_PAD_TX_CTRL, 0); 1412 ccfifo_writel(emc, rfu1 & ~0x07ff07ff, 1413 EMC_PMACRO_BRICK_CTRL_RFU1, 0); 1414 } else { 1415 ccfifo_writel(emc, rfu1 & ~0x07ff07ff, 1416 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait); 1417 ramp_down_wait += 100000; 1418 } 1419 } else { 1420 ccfifo_writel(emc, rfu1 & ~0xffff07ff, 1421 EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19); 1422 ramp_down_wait += 100000 + (20 * clk); 1423 } 1424 1425 if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) { 1426 ramp_down_wait += 100000; 1427 ccfifo_writel(emc, common_tx & ~0x5, 1428 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); 1429 ramp_down_wait += 100000; 1430 ccfifo_writel(emc, common_tx & ~0xf, 1431 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); 1432 ramp_down_wait += 100000; 1433 ccfifo_writel(emc, 0, 0, seq_wait); 1434 ramp_down_wait += 100000; 1435 } else { 1436 ccfifo_writel(emc, common_tx & ~0xf, 1437 EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait); 1438 } 1439 1440 return ramp_down_wait; 1441 } 1442 1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing) 1444 { 1445 timing->current_dram_clktree[C0D0U0] = 1446 timing->trained_dram_clktree[C0D0U0]; 1447 timing->current_dram_clktree[C0D0U1] = 1448 timing->trained_dram_clktree[C0D0U1]; 1449 timing->current_dram_clktree[C1D0U0] = 1450 timing->trained_dram_clktree[C1D0U0]; 1451 timing->current_dram_clktree[C1D0U1] = 1452 timing->trained_dram_clktree[C1D0U1]; 1453 timing->current_dram_clktree[C1D1U0] = 1454 timing->trained_dram_clktree[C1D1U0]; 1455 timing->current_dram_clktree[C1D1U1] = 1456 timing->trained_dram_clktree[C1D1U1]; 1457 } 1458 1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state) 1460 { 1461 unsigned int i; 1462 1463 emc_writel(emc, value, EMC_CFG_DIG_DLL); 1464 tegra210_emc_timing_update(emc); 1465 1466 for (i = 0; i < emc->num_channels; i++) 1467 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, 1468 EMC_CFG_DIG_DLL_CFG_DLL_EN, 1469 state); 1470 } 1471 1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc) 1473 { 1474 u32 value; 1475 1476 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1477 value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; 1478 1479 update_dll_control(emc, value, false); 1480 } 1481 1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc) 1483 { 1484 u32 value; 1485 1486 value = emc_readl(emc, EMC_CFG_DIG_DLL); 1487 value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; 1488 1489 update_dll_control(emc, value, true); 1490 } 1491 1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc, 1493 struct tegra210_emc_timing *timing) 1494 { 1495 u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX]; 1496 u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX]; 1497 u32 ref = timing->burst_regs[EMC_REFRESH_INDEX]; 1498 1499 switch (emc->refresh) { 1500 case TEGRA210_EMC_REFRESH_NOMINAL: 1501 case TEGRA210_EMC_REFRESH_THROTTLE: 1502 break; 1503 1504 case TEGRA210_EMC_REFRESH_2X: 1505 ref = REFRESH_SPEEDUP(ref, 2); 1506 pre_ref = REFRESH_SPEEDUP(pre_ref, 2); 1507 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2); 1508 break; 1509 1510 case TEGRA210_EMC_REFRESH_4X: 1511 ref = REFRESH_SPEEDUP(ref, 4); 1512 pre_ref = REFRESH_SPEEDUP(pre_ref, 4); 1513 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4); 1514 break; 1515 1516 default: 1517 dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh); 1518 return; 1519 } 1520 1521 emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]); 1522 emc_writel(emc, pre_ref, 1523 emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]); 1524 emc_writel(emc, dsr_cntrl, 1525 emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]); 1526 } 1527 1528 static int tegra210_emc_set_rate(struct device *dev, 1529 const struct tegra210_clk_emc_config *config) 1530 { 1531 struct tegra210_emc *emc = dev_get_drvdata(dev); 1532 struct tegra210_emc_timing *timing = NULL; 1533 unsigned long rate = config->rate; 1534 s64 last_change_delay; 1535 unsigned long flags; 1536 unsigned int i; 1537 1538 if (rate == emc->last->rate * 1000UL) 1539 return 0; 1540 1541 for (i = 0; i < emc->num_timings; i++) { 1542 if (emc->timings[i].rate * 1000UL == rate) { 1543 timing = &emc->timings[i]; 1544 break; 1545 } 1546 } 1547 1548 if (!timing) 1549 return -EINVAL; 1550 1551 if (rate > 204000000 && !timing->trained) 1552 return -EINVAL; 1553 1554 emc->next = timing; 1555 last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time); 1556 1557 /* XXX use non-busy-looping sleep? */ 1558 if ((last_change_delay >= 0) && 1559 (last_change_delay < emc->clkchange_delay)) 1560 udelay(emc->clkchange_delay - (int)last_change_delay); 1561 1562 spin_lock_irqsave(&emc->lock, flags); 1563 tegra210_emc_set_clock(emc, config->value); 1564 emc->clkchange_time = ktime_get(); 1565 emc->last = timing; 1566 spin_unlock_irqrestore(&emc->lock, flags); 1567 1568 return 0; 1569 } 1570 1571 /* 1572 * debugfs interface 1573 * 1574 * The memory controller driver exposes some files in debugfs that can be used 1575 * to control the EMC frequency. The top-level directory can be found here: 1576 * 1577 * /sys/kernel/debug/emc 1578 * 1579 * It contains the following files: 1580 * 1581 * - available_rates: This file contains a list of valid, space-separated 1582 * EMC frequencies. 1583 * 1584 * - min_rate: Writing a value to this file sets the given frequency as the 1585 * floor of the permitted range. If this is higher than the currently 1586 * configured EMC frequency, this will cause the frequency to be 1587 * increased so that it stays within the valid range. 1588 * 1589 * - max_rate: Similarily to the min_rate file, writing a value to this file 1590 * sets the given frequency as the ceiling of the permitted range. If 1591 * the value is lower than the currently configured EMC frequency, this 1592 * will cause the frequency to be decreased so that it stays within the 1593 * valid range. 1594 */ 1595 1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc, 1597 unsigned long rate) 1598 { 1599 unsigned int i; 1600 1601 for (i = 0; i < emc->num_timings; i++) 1602 if (rate == emc->timings[i].rate * 1000UL) 1603 return true; 1604 1605 return false; 1606 } 1607 1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s, 1609 void *data) 1610 { 1611 struct tegra210_emc *emc = s->private; 1612 const char *prefix = ""; 1613 unsigned int i; 1614 1615 for (i = 0; i < emc->num_timings; i++) { 1616 seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000); 1617 prefix = " "; 1618 } 1619 1620 seq_puts(s, "\n"); 1621 1622 return 0; 1623 } 1624 1625 static int tegra210_emc_debug_available_rates_open(struct inode *inode, 1626 struct file *file) 1627 { 1628 return single_open(file, tegra210_emc_debug_available_rates_show, 1629 inode->i_private); 1630 } 1631 1632 static const struct file_operations tegra210_emc_debug_available_rates_fops = { 1633 .open = tegra210_emc_debug_available_rates_open, 1634 .read = seq_read, 1635 .llseek = seq_lseek, 1636 .release = single_release, 1637 }; 1638 1639 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate) 1640 { 1641 struct tegra210_emc *emc = data; 1642 1643 *rate = emc->debugfs.min_rate; 1644 1645 return 0; 1646 } 1647 1648 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate) 1649 { 1650 struct tegra210_emc *emc = data; 1651 int err; 1652 1653 if (!tegra210_emc_validate_rate(emc, rate)) 1654 return -EINVAL; 1655 1656 err = clk_set_min_rate(emc->clk, rate); 1657 if (err < 0) 1658 return err; 1659 1660 emc->debugfs.min_rate = rate; 1661 1662 return 0; 1663 } 1664 1665 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_min_rate_fops, 1666 tegra210_emc_debug_min_rate_get, 1667 tegra210_emc_debug_min_rate_set, "%llu\n"); 1668 1669 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate) 1670 { 1671 struct tegra210_emc *emc = data; 1672 1673 *rate = emc->debugfs.max_rate; 1674 1675 return 0; 1676 } 1677 1678 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate) 1679 { 1680 struct tegra210_emc *emc = data; 1681 int err; 1682 1683 if (!tegra210_emc_validate_rate(emc, rate)) 1684 return -EINVAL; 1685 1686 err = clk_set_max_rate(emc->clk, rate); 1687 if (err < 0) 1688 return err; 1689 1690 emc->debugfs.max_rate = rate; 1691 1692 return 0; 1693 } 1694 1695 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_max_rate_fops, 1696 tegra210_emc_debug_max_rate_get, 1697 tegra210_emc_debug_max_rate_set, "%llu\n"); 1698 1699 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature) 1700 { 1701 struct tegra210_emc *emc = data; 1702 unsigned int value; 1703 1704 if (!emc->debugfs.temperature) 1705 value = tegra210_emc_get_temperature(emc); 1706 else 1707 value = emc->debugfs.temperature; 1708 1709 *temperature = value; 1710 1711 return 0; 1712 } 1713 1714 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature) 1715 { 1716 struct tegra210_emc *emc = data; 1717 1718 if (temperature > 7) 1719 return -EINVAL; 1720 1721 emc->debugfs.temperature = temperature; 1722 1723 return 0; 1724 } 1725 1726 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_temperature_fops, 1727 tegra210_emc_debug_temperature_get, 1728 tegra210_emc_debug_temperature_set, "%llu\n"); 1729 1730 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc) 1731 { 1732 struct device *dev = emc->dev; 1733 unsigned int i; 1734 int err; 1735 1736 emc->debugfs.min_rate = ULONG_MAX; 1737 emc->debugfs.max_rate = 0; 1738 1739 for (i = 0; i < emc->num_timings; i++) { 1740 if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate) 1741 emc->debugfs.min_rate = emc->timings[i].rate * 1000UL; 1742 1743 if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate) 1744 emc->debugfs.max_rate = emc->timings[i].rate * 1000UL; 1745 } 1746 1747 if (!emc->num_timings) { 1748 emc->debugfs.min_rate = clk_get_rate(emc->clk); 1749 emc->debugfs.max_rate = emc->debugfs.min_rate; 1750 } 1751 1752 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, 1753 emc->debugfs.max_rate); 1754 if (err < 0) { 1755 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n", 1756 emc->debugfs.min_rate, emc->debugfs.max_rate, 1757 emc->clk); 1758 return; 1759 } 1760 1761 emc->debugfs.root = debugfs_create_dir("emc", NULL); 1762 if (!emc->debugfs.root) { 1763 dev_err(dev, "failed to create debugfs directory\n"); 1764 return; 1765 } 1766 1767 debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc, 1768 &tegra210_emc_debug_available_rates_fops); 1769 debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc, 1770 &tegra210_emc_debug_min_rate_fops); 1771 debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc, 1772 &tegra210_emc_debug_max_rate_fops); 1773 debugfs_create_file("temperature", 0644, emc->debugfs.root, emc, 1774 &tegra210_emc_debug_temperature_fops); 1775 } 1776 1777 static void tegra210_emc_detect(struct tegra210_emc *emc) 1778 { 1779 u32 value; 1780 1781 /* probe the number of connected DRAM devices */ 1782 value = mc_readl(emc->mc, MC_EMEM_ADR_CFG); 1783 1784 if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV) 1785 emc->num_devices = 2; 1786 else 1787 emc->num_devices = 1; 1788 1789 /* probe the type of DRAM */ 1790 value = emc_readl(emc, EMC_FBIO_CFG5); 1791 emc->dram_type = value & 0x3; 1792 1793 /* probe the number of channels */ 1794 value = emc_readl(emc, EMC_FBIO_CFG7); 1795 1796 if ((value & EMC_FBIO_CFG7_CH1_ENABLE) && 1797 (value & EMC_FBIO_CFG7_CH0_ENABLE)) 1798 emc->num_channels = 2; 1799 else 1800 emc->num_channels = 1; 1801 } 1802 1803 static int tegra210_emc_validate_timings(struct tegra210_emc *emc, 1804 struct tegra210_emc_timing *timings, 1805 unsigned int num_timings) 1806 { 1807 unsigned int i; 1808 1809 for (i = 0; i < num_timings; i++) { 1810 u32 min_volt = timings[i].min_volt; 1811 u32 rate = timings[i].rate; 1812 1813 if (!rate) 1814 return -EINVAL; 1815 1816 if ((i > 0) && ((rate <= timings[i - 1].rate) || 1817 (min_volt < timings[i - 1].min_volt))) 1818 return -EINVAL; 1819 1820 if (timings[i].revision != timings[0].revision) 1821 continue; 1822 } 1823 1824 return 0; 1825 } 1826 1827 static int tegra210_emc_probe(struct platform_device *pdev) 1828 { 1829 struct thermal_cooling_device *cd; 1830 unsigned long current_rate; 1831 struct tegra210_emc *emc; 1832 struct device_node *np; 1833 unsigned int i; 1834 int err; 1835 1836 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 1837 if (!emc) 1838 return -ENOMEM; 1839 1840 emc->clk = devm_clk_get(&pdev->dev, "emc"); 1841 if (IS_ERR(emc->clk)) 1842 return PTR_ERR(emc->clk); 1843 1844 platform_set_drvdata(pdev, emc); 1845 spin_lock_init(&emc->lock); 1846 emc->dev = &pdev->dev; 1847 1848 emc->mc = devm_tegra_memory_controller_get(&pdev->dev); 1849 if (IS_ERR(emc->mc)) 1850 return PTR_ERR(emc->mc); 1851 1852 emc->regs = devm_platform_ioremap_resource(pdev, 0); 1853 if (IS_ERR(emc->regs)) 1854 return PTR_ERR(emc->regs); 1855 1856 for (i = 0; i < 2; i++) { 1857 emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i); 1858 if (IS_ERR(emc->channel[i])) 1859 return PTR_ERR(emc->channel[i]); 1860 1861 } 1862 1863 tegra210_emc_detect(emc); 1864 np = pdev->dev.of_node; 1865 1866 /* attach to the nominal and (optional) derated tables */ 1867 err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal"); 1868 if (err < 0) { 1869 dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err); 1870 return err; 1871 } 1872 1873 err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated"); 1874 if (err < 0 && err != -ENODEV) { 1875 dev_err(emc->dev, "failed to get derated EMC table: %d\n", err); 1876 goto release; 1877 } 1878 1879 /* validate the tables */ 1880 if (emc->nominal) { 1881 err = tegra210_emc_validate_timings(emc, emc->nominal, 1882 emc->num_timings); 1883 if (err < 0) 1884 goto release; 1885 } 1886 1887 if (emc->derated) { 1888 err = tegra210_emc_validate_timings(emc, emc->derated, 1889 emc->num_timings); 1890 if (err < 0) 1891 goto release; 1892 } 1893 1894 /* default to the nominal table */ 1895 emc->timings = emc->nominal; 1896 1897 /* pick the current timing based on the current EMC clock rate */ 1898 current_rate = clk_get_rate(emc->clk) / 1000; 1899 1900 for (i = 0; i < emc->num_timings; i++) { 1901 if (emc->timings[i].rate == current_rate) { 1902 emc->last = &emc->timings[i]; 1903 break; 1904 } 1905 } 1906 1907 if (i == emc->num_timings) { 1908 dev_err(emc->dev, "no EMC table entry found for %lu kHz\n", 1909 current_rate); 1910 err = -ENOENT; 1911 goto release; 1912 } 1913 1914 /* pick a compatible clock change sequence for the EMC table */ 1915 for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) { 1916 const struct tegra210_emc_sequence *sequence = 1917 tegra210_emc_sequences[i]; 1918 1919 if (emc->timings[0].revision == sequence->revision) { 1920 emc->sequence = sequence; 1921 break; 1922 } 1923 } 1924 1925 if (!emc->sequence) { 1926 dev_err(&pdev->dev, "sequence %u not supported\n", 1927 emc->timings[0].revision); 1928 err = -ENOTSUPP; 1929 goto release; 1930 } 1931 1932 emc->offsets = &tegra210_emc_table_register_offsets; 1933 emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL; 1934 1935 emc->provider.owner = THIS_MODULE; 1936 emc->provider.dev = &pdev->dev; 1937 emc->provider.set_rate = tegra210_emc_set_rate; 1938 1939 emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings, 1940 sizeof(*emc->provider.configs), 1941 GFP_KERNEL); 1942 if (!emc->provider.configs) { 1943 err = -ENOMEM; 1944 goto release; 1945 } 1946 1947 emc->provider.num_configs = emc->num_timings; 1948 1949 for (i = 0; i < emc->provider.num_configs; i++) { 1950 struct tegra210_emc_timing *timing = &emc->timings[i]; 1951 struct tegra210_clk_emc_config *config = 1952 &emc->provider.configs[i]; 1953 u32 value; 1954 1955 config->rate = timing->rate * 1000UL; 1956 config->value = timing->clk_src_emc; 1957 1958 value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX]; 1959 1960 if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0) 1961 config->same_freq = false; 1962 else 1963 config->same_freq = true; 1964 } 1965 1966 err = tegra210_clk_emc_attach(emc->clk, &emc->provider); 1967 if (err < 0) { 1968 dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err); 1969 goto release; 1970 } 1971 1972 emc->clkchange_delay = 100; 1973 emc->training_interval = 100; 1974 dev_set_drvdata(emc->dev, emc); 1975 1976 timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh, 1977 TIMER_DEFERRABLE); 1978 atomic_set(&emc->refresh_poll, 0); 1979 emc->refresh_poll_interval = 1000; 1980 1981 timer_setup(&emc->training, tegra210_emc_train, 0); 1982 1983 tegra210_emc_debugfs_init(emc); 1984 1985 cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc, 1986 &tegra210_emc_cd_ops); 1987 if (IS_ERR(cd)) { 1988 err = PTR_ERR(cd); 1989 dev_err(emc->dev, "failed to register cooling device: %d\n", 1990 err); 1991 goto detach; 1992 } 1993 1994 return 0; 1995 1996 detach: 1997 debugfs_remove_recursive(emc->debugfs.root); 1998 tegra210_clk_emc_detach(emc->clk); 1999 release: 2000 of_reserved_mem_device_release(emc->dev); 2001 2002 return err; 2003 } 2004 2005 static int tegra210_emc_remove(struct platform_device *pdev) 2006 { 2007 struct tegra210_emc *emc = platform_get_drvdata(pdev); 2008 2009 debugfs_remove_recursive(emc->debugfs.root); 2010 tegra210_clk_emc_detach(emc->clk); 2011 of_reserved_mem_device_release(emc->dev); 2012 2013 return 0; 2014 } 2015 2016 static int __maybe_unused tegra210_emc_suspend(struct device *dev) 2017 { 2018 struct tegra210_emc *emc = dev_get_drvdata(dev); 2019 int err; 2020 2021 err = clk_rate_exclusive_get(emc->clk); 2022 if (err < 0) { 2023 dev_err(emc->dev, "failed to acquire clock: %d\n", err); 2024 return err; 2025 } 2026 2027 emc->resume_rate = clk_get_rate(emc->clk); 2028 2029 clk_set_rate(emc->clk, 204000000); 2030 tegra210_clk_emc_detach(emc->clk); 2031 2032 dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk)); 2033 2034 return 0; 2035 } 2036 2037 static int __maybe_unused tegra210_emc_resume(struct device *dev) 2038 { 2039 struct tegra210_emc *emc = dev_get_drvdata(dev); 2040 int err; 2041 2042 err = tegra210_clk_emc_attach(emc->clk, &emc->provider); 2043 if (err < 0) { 2044 dev_err(dev, "failed to attach to EMC clock: %d\n", err); 2045 return err; 2046 } 2047 2048 clk_set_rate(emc->clk, emc->resume_rate); 2049 clk_rate_exclusive_put(emc->clk); 2050 2051 dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk)); 2052 2053 return 0; 2054 } 2055 2056 static const struct dev_pm_ops tegra210_emc_pm_ops = { 2057 SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume) 2058 }; 2059 2060 static const struct of_device_id tegra210_emc_of_match[] = { 2061 { .compatible = "nvidia,tegra210-emc", }, 2062 { }, 2063 }; 2064 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match); 2065 2066 static struct platform_driver tegra210_emc_driver = { 2067 .driver = { 2068 .name = "tegra210-emc", 2069 .of_match_table = tegra210_emc_of_match, 2070 .pm = &tegra210_emc_pm_ops, 2071 }, 2072 .probe = tegra210_emc_probe, 2073 .remove = tegra210_emc_remove, 2074 }; 2075 2076 module_platform_driver(tegra210_emc_driver); 2077 2078 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); 2079 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>"); 2080 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver"); 2081 MODULE_LICENSE("GPL v2"); 2082