1e34212c7SDmitry Osipenko // SPDX-License-Identifier: GPL-2.0+ 2e34212c7SDmitry Osipenko /* 3e34212c7SDmitry Osipenko * Tegra30 External Memory Controller driver 4e34212c7SDmitry Osipenko * 5e34212c7SDmitry Osipenko * Based on downstream driver from NVIDIA and tegra124-emc.c 6e34212c7SDmitry Osipenko * Copyright (C) 2011-2014 NVIDIA Corporation 7e34212c7SDmitry Osipenko * 8e34212c7SDmitry Osipenko * Author: Dmitry Osipenko <digetx@gmail.com> 9e34212c7SDmitry Osipenko * Copyright (C) 2019 GRATE-DRIVER project 10e34212c7SDmitry Osipenko */ 11e34212c7SDmitry Osipenko 12e34212c7SDmitry Osipenko #include <linux/clk.h> 13e34212c7SDmitry Osipenko #include <linux/clk/tegra.h> 14e34212c7SDmitry Osipenko #include <linux/completion.h> 15*8cee32b4SThierry Reding #include <linux/debugfs.h> 16e34212c7SDmitry Osipenko #include <linux/delay.h> 17e34212c7SDmitry Osipenko #include <linux/err.h> 18e34212c7SDmitry Osipenko #include <linux/interrupt.h> 19e34212c7SDmitry Osipenko #include <linux/io.h> 20e34212c7SDmitry Osipenko #include <linux/iopoll.h> 21e34212c7SDmitry Osipenko #include <linux/kernel.h> 22e34212c7SDmitry Osipenko #include <linux/module.h> 23e34212c7SDmitry Osipenko #include <linux/of_platform.h> 24e34212c7SDmitry Osipenko #include <linux/platform_device.h> 25e34212c7SDmitry Osipenko #include <linux/sort.h> 26e34212c7SDmitry Osipenko #include <linux/types.h> 27e34212c7SDmitry Osipenko 28e34212c7SDmitry Osipenko #include <soc/tegra/fuse.h> 29e34212c7SDmitry Osipenko 30e34212c7SDmitry Osipenko #include "mc.h" 31e34212c7SDmitry Osipenko 32e34212c7SDmitry Osipenko #define EMC_INTSTATUS 0x000 33e34212c7SDmitry Osipenko #define EMC_INTMASK 0x004 34e34212c7SDmitry Osipenko #define EMC_DBG 0x008 35e34212c7SDmitry Osipenko #define EMC_CFG 0x00c 36e34212c7SDmitry Osipenko #define EMC_REFCTRL 0x020 37e34212c7SDmitry Osipenko #define EMC_TIMING_CONTROL 0x028 38e34212c7SDmitry Osipenko #define EMC_RC 0x02c 39e34212c7SDmitry Osipenko #define EMC_RFC 0x030 40e34212c7SDmitry Osipenko #define EMC_RAS 0x034 41e34212c7SDmitry Osipenko #define EMC_RP 0x038 42e34212c7SDmitry Osipenko #define EMC_R2W 0x03c 43e34212c7SDmitry Osipenko #define EMC_W2R 0x040 44e34212c7SDmitry Osipenko #define EMC_R2P 0x044 45e34212c7SDmitry Osipenko #define EMC_W2P 0x048 46e34212c7SDmitry Osipenko #define EMC_RD_RCD 0x04c 47e34212c7SDmitry Osipenko #define EMC_WR_RCD 0x050 48e34212c7SDmitry Osipenko #define EMC_RRD 0x054 49e34212c7SDmitry Osipenko #define EMC_REXT 0x058 50e34212c7SDmitry Osipenko #define EMC_WDV 0x05c 51e34212c7SDmitry Osipenko #define EMC_QUSE 0x060 52e34212c7SDmitry Osipenko #define EMC_QRST 0x064 53e34212c7SDmitry Osipenko #define EMC_QSAFE 0x068 54e34212c7SDmitry Osipenko #define EMC_RDV 0x06c 55e34212c7SDmitry Osipenko #define EMC_REFRESH 0x070 56e34212c7SDmitry Osipenko #define EMC_BURST_REFRESH_NUM 0x074 57e34212c7SDmitry Osipenko #define EMC_PDEX2WR 0x078 58e34212c7SDmitry Osipenko #define EMC_PDEX2RD 0x07c 59e34212c7SDmitry Osipenko #define EMC_PCHG2PDEN 0x080 60e34212c7SDmitry Osipenko #define EMC_ACT2PDEN 0x084 61e34212c7SDmitry Osipenko #define EMC_AR2PDEN 0x088 62e34212c7SDmitry Osipenko #define EMC_RW2PDEN 0x08c 63e34212c7SDmitry Osipenko #define EMC_TXSR 0x090 64e34212c7SDmitry Osipenko #define EMC_TCKE 0x094 65e34212c7SDmitry Osipenko #define EMC_TFAW 0x098 66e34212c7SDmitry Osipenko #define EMC_TRPAB 0x09c 67e34212c7SDmitry Osipenko #define EMC_TCLKSTABLE 0x0a0 68e34212c7SDmitry Osipenko #define EMC_TCLKSTOP 0x0a4 69e34212c7SDmitry Osipenko #define EMC_TREFBW 0x0a8 70e34212c7SDmitry Osipenko #define EMC_QUSE_EXTRA 0x0ac 71e34212c7SDmitry Osipenko #define EMC_ODT_WRITE 0x0b0 72e34212c7SDmitry Osipenko #define EMC_ODT_READ 0x0b4 73e34212c7SDmitry Osipenko #define EMC_WEXT 0x0b8 74e34212c7SDmitry Osipenko #define EMC_CTT 0x0bc 75e34212c7SDmitry Osipenko #define EMC_MRS_WAIT_CNT 0x0c8 76e34212c7SDmitry Osipenko #define EMC_MRS 0x0cc 77e34212c7SDmitry Osipenko #define EMC_EMRS 0x0d0 78e34212c7SDmitry Osipenko #define EMC_SELF_REF 0x0e0 79e34212c7SDmitry Osipenko #define EMC_MRW 0x0e8 80e34212c7SDmitry Osipenko #define EMC_XM2DQSPADCTRL3 0x0f8 81e34212c7SDmitry Osipenko #define EMC_FBIO_SPARE 0x100 82e34212c7SDmitry Osipenko #define EMC_FBIO_CFG5 0x104 83e34212c7SDmitry Osipenko #define EMC_FBIO_CFG6 0x114 84e34212c7SDmitry Osipenko #define EMC_CFG_RSV 0x120 85e34212c7SDmitry Osipenko #define EMC_AUTO_CAL_CONFIG 0x2a4 86e34212c7SDmitry Osipenko #define EMC_AUTO_CAL_INTERVAL 0x2a8 87e34212c7SDmitry Osipenko #define EMC_AUTO_CAL_STATUS 0x2ac 88e34212c7SDmitry Osipenko #define EMC_STATUS 0x2b4 89e34212c7SDmitry Osipenko #define EMC_CFG_2 0x2b8 90e34212c7SDmitry Osipenko #define EMC_CFG_DIG_DLL 0x2bc 91e34212c7SDmitry Osipenko #define EMC_CFG_DIG_DLL_PERIOD 0x2c0 92e34212c7SDmitry Osipenko #define EMC_CTT_DURATION 0x2d8 93e34212c7SDmitry Osipenko #define EMC_CTT_TERM_CTRL 0x2dc 94e34212c7SDmitry Osipenko #define EMC_ZCAL_INTERVAL 0x2e0 95e34212c7SDmitry Osipenko #define EMC_ZCAL_WAIT_CNT 0x2e4 96e34212c7SDmitry Osipenko #define EMC_ZQ_CAL 0x2ec 97e34212c7SDmitry Osipenko #define EMC_XM2CMDPADCTRL 0x2f0 98e34212c7SDmitry Osipenko #define EMC_XM2DQSPADCTRL2 0x2fc 99e34212c7SDmitry Osipenko #define EMC_XM2DQPADCTRL2 0x304 100e34212c7SDmitry Osipenko #define EMC_XM2CLKPADCTRL 0x308 101e34212c7SDmitry Osipenko #define EMC_XM2COMPPADCTRL 0x30c 102e34212c7SDmitry Osipenko #define EMC_XM2VTTGENPADCTRL 0x310 103e34212c7SDmitry Osipenko #define EMC_XM2VTTGENPADCTRL2 0x314 104e34212c7SDmitry Osipenko #define EMC_XM2QUSEPADCTRL 0x318 105e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS0 0x328 106e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS1 0x32c 107e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS2 0x330 108e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS3 0x334 109e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS4 0x338 110e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS5 0x33c 111e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS6 0x340 112e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQS7 0x344 113e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE0 0x348 114e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE1 0x34c 115e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE2 0x350 116e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE3 0x354 117e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE4 0x358 118e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE5 0x35c 119e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE6 0x360 120e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_QUSE7 0x364 121e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQ0 0x368 122e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQ1 0x36c 123e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQ2 0x370 124e34212c7SDmitry Osipenko #define EMC_DLL_XFORM_DQ3 0x374 125e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS0 0x3a8 126e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS1 0x3ac 127e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS2 0x3b0 128e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS3 0x3b4 129e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS4 0x3b8 130e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS5 0x3bc 131e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS6 0x3c0 132e34212c7SDmitry Osipenko #define EMC_DLI_TRIM_TXDQS7 0x3c4 133e34212c7SDmitry Osipenko #define EMC_STALL_THEN_EXE_BEFORE_CLKCHANGE 0x3c8 134e34212c7SDmitry Osipenko #define EMC_STALL_THEN_EXE_AFTER_CLKCHANGE 0x3cc 135e34212c7SDmitry Osipenko #define EMC_UNSTALL_RW_AFTER_CLKCHANGE 0x3d0 136e34212c7SDmitry Osipenko #define EMC_SEL_DPD_CTRL 0x3d8 137e34212c7SDmitry Osipenko #define EMC_PRE_REFRESH_REQ_CNT 0x3dc 138e34212c7SDmitry Osipenko #define EMC_DYN_SELF_REF_CONTROL 0x3e0 139e34212c7SDmitry Osipenko #define EMC_TXSRDLL 0x3e4 140e34212c7SDmitry Osipenko 141e34212c7SDmitry Osipenko #define EMC_STATUS_TIMING_UPDATE_STALLED BIT(23) 142e34212c7SDmitry Osipenko 143e34212c7SDmitry Osipenko #define EMC_MODE_SET_DLL_RESET BIT(8) 144e34212c7SDmitry Osipenko #define EMC_MODE_SET_LONG_CNT BIT(26) 145e34212c7SDmitry Osipenko 146e34212c7SDmitry Osipenko #define EMC_SELF_REF_CMD_ENABLED BIT(0) 147e34212c7SDmitry Osipenko 148e34212c7SDmitry Osipenko #define DRAM_DEV_SEL_ALL (0 << 30) 149e34212c7SDmitry Osipenko #define DRAM_DEV_SEL_0 (2 << 30) 150e34212c7SDmitry Osipenko #define DRAM_DEV_SEL_1 (1 << 30) 151e34212c7SDmitry Osipenko #define DRAM_BROADCAST(num) \ 152e34212c7SDmitry Osipenko ((num) > 1 ? DRAM_DEV_SEL_ALL : DRAM_DEV_SEL_0) 153e34212c7SDmitry Osipenko 154e34212c7SDmitry Osipenko #define EMC_ZQ_CAL_CMD BIT(0) 155e34212c7SDmitry Osipenko #define EMC_ZQ_CAL_LONG BIT(4) 156e34212c7SDmitry Osipenko #define EMC_ZQ_CAL_LONG_CMD_DEV0 \ 157e34212c7SDmitry Osipenko (DRAM_DEV_SEL_0 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD) 158e34212c7SDmitry Osipenko #define EMC_ZQ_CAL_LONG_CMD_DEV1 \ 159e34212c7SDmitry Osipenko (DRAM_DEV_SEL_1 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD) 160e34212c7SDmitry Osipenko 161e34212c7SDmitry Osipenko #define EMC_DBG_READ_MUX_ASSEMBLY BIT(0) 162e34212c7SDmitry Osipenko #define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) 163e34212c7SDmitry Osipenko #define EMC_DBG_FORCE_UPDATE BIT(2) 164e34212c7SDmitry Osipenko #define EMC_DBG_CFG_PRIORITY BIT(24) 165e34212c7SDmitry Osipenko 166e34212c7SDmitry Osipenko #define EMC_CFG5_QUSE_MODE_SHIFT 13 167e34212c7SDmitry Osipenko #define EMC_CFG5_QUSE_MODE_MASK (7 << EMC_CFG5_QUSE_MODE_SHIFT) 168e34212c7SDmitry Osipenko 169e34212c7SDmitry Osipenko #define EMC_CFG5_QUSE_MODE_INTERNAL_LPBK 2 170e34212c7SDmitry Osipenko #define EMC_CFG5_QUSE_MODE_PULSE_INTERN 3 171e34212c7SDmitry Osipenko 172e34212c7SDmitry Osipenko #define EMC_SEL_DPD_CTRL_QUSE_DPD_ENABLE BIT(9) 173e34212c7SDmitry Osipenko 174e34212c7SDmitry Osipenko #define EMC_XM2COMPPADCTRL_VREF_CAL_ENABLE BIT(10) 175e34212c7SDmitry Osipenko 176e34212c7SDmitry Osipenko #define EMC_XM2QUSEPADCTRL_IVREF_ENABLE BIT(4) 177e34212c7SDmitry Osipenko 178e34212c7SDmitry Osipenko #define EMC_XM2DQSPADCTRL2_VREF_ENABLE BIT(5) 179e34212c7SDmitry Osipenko #define EMC_XM2DQSPADCTRL3_VREF_ENABLE BIT(5) 180e34212c7SDmitry Osipenko 181e34212c7SDmitry Osipenko #define EMC_AUTO_CAL_STATUS_ACTIVE BIT(31) 182e34212c7SDmitry Osipenko 183e34212c7SDmitry Osipenko #define EMC_FBIO_CFG5_DRAM_TYPE_MASK 0x3 184e34212c7SDmitry Osipenko 185e34212c7SDmitry Osipenko #define EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK 0x3ff 186e34212c7SDmitry Osipenko #define EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT 16 187e34212c7SDmitry Osipenko #define EMC_MRS_WAIT_CNT_LONG_WAIT_MASK \ 188e34212c7SDmitry Osipenko (0x3ff << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT) 189e34212c7SDmitry Osipenko 190e34212c7SDmitry Osipenko #define EMC_REFCTRL_DEV_SEL_MASK 0x3 191e34212c7SDmitry Osipenko #define EMC_REFCTRL_ENABLE BIT(31) 192e34212c7SDmitry Osipenko #define EMC_REFCTRL_ENABLE_ALL(num) \ 193e34212c7SDmitry Osipenko (((num) > 1 ? 0 : 2) | EMC_REFCTRL_ENABLE) 194e34212c7SDmitry Osipenko #define EMC_REFCTRL_DISABLE_ALL(num) ((num) > 1 ? 0 : 2) 195e34212c7SDmitry Osipenko 196e34212c7SDmitry Osipenko #define EMC_CFG_PERIODIC_QRST BIT(21) 197e34212c7SDmitry Osipenko #define EMC_CFG_DYN_SREF_ENABLE BIT(28) 198e34212c7SDmitry Osipenko 199e34212c7SDmitry Osipenko #define EMC_CLKCHANGE_REQ_ENABLE BIT(0) 200e34212c7SDmitry Osipenko #define EMC_CLKCHANGE_PD_ENABLE BIT(1) 201e34212c7SDmitry Osipenko #define EMC_CLKCHANGE_SR_ENABLE BIT(2) 202e34212c7SDmitry Osipenko 203e34212c7SDmitry Osipenko #define EMC_TIMING_UPDATE BIT(0) 204e34212c7SDmitry Osipenko 205e34212c7SDmitry Osipenko #define EMC_REFRESH_OVERFLOW_INT BIT(3) 206e34212c7SDmitry Osipenko #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) 207e34212c7SDmitry Osipenko 208e34212c7SDmitry Osipenko enum emc_dram_type { 209e34212c7SDmitry Osipenko DRAM_TYPE_DDR3, 210e34212c7SDmitry Osipenko DRAM_TYPE_DDR1, 211e34212c7SDmitry Osipenko DRAM_TYPE_LPDDR2, 212e34212c7SDmitry Osipenko DRAM_TYPE_DDR2, 213e34212c7SDmitry Osipenko }; 214e34212c7SDmitry Osipenko 215e34212c7SDmitry Osipenko enum emc_dll_change { 216e34212c7SDmitry Osipenko DLL_CHANGE_NONE, 217e34212c7SDmitry Osipenko DLL_CHANGE_ON, 218e34212c7SDmitry Osipenko DLL_CHANGE_OFF 219e34212c7SDmitry Osipenko }; 220e34212c7SDmitry Osipenko 221e34212c7SDmitry Osipenko static const u16 emc_timing_registers[] = { 222e34212c7SDmitry Osipenko [0] = EMC_RC, 223e34212c7SDmitry Osipenko [1] = EMC_RFC, 224e34212c7SDmitry Osipenko [2] = EMC_RAS, 225e34212c7SDmitry Osipenko [3] = EMC_RP, 226e34212c7SDmitry Osipenko [4] = EMC_R2W, 227e34212c7SDmitry Osipenko [5] = EMC_W2R, 228e34212c7SDmitry Osipenko [6] = EMC_R2P, 229e34212c7SDmitry Osipenko [7] = EMC_W2P, 230e34212c7SDmitry Osipenko [8] = EMC_RD_RCD, 231e34212c7SDmitry Osipenko [9] = EMC_WR_RCD, 232e34212c7SDmitry Osipenko [10] = EMC_RRD, 233e34212c7SDmitry Osipenko [11] = EMC_REXT, 234e34212c7SDmitry Osipenko [12] = EMC_WEXT, 235e34212c7SDmitry Osipenko [13] = EMC_WDV, 236e34212c7SDmitry Osipenko [14] = EMC_QUSE, 237e34212c7SDmitry Osipenko [15] = EMC_QRST, 238e34212c7SDmitry Osipenko [16] = EMC_QSAFE, 239e34212c7SDmitry Osipenko [17] = EMC_RDV, 240e34212c7SDmitry Osipenko [18] = EMC_REFRESH, 241e34212c7SDmitry Osipenko [19] = EMC_BURST_REFRESH_NUM, 242e34212c7SDmitry Osipenko [20] = EMC_PRE_REFRESH_REQ_CNT, 243e34212c7SDmitry Osipenko [21] = EMC_PDEX2WR, 244e34212c7SDmitry Osipenko [22] = EMC_PDEX2RD, 245e34212c7SDmitry Osipenko [23] = EMC_PCHG2PDEN, 246e34212c7SDmitry Osipenko [24] = EMC_ACT2PDEN, 247e34212c7SDmitry Osipenko [25] = EMC_AR2PDEN, 248e34212c7SDmitry Osipenko [26] = EMC_RW2PDEN, 249e34212c7SDmitry Osipenko [27] = EMC_TXSR, 250e34212c7SDmitry Osipenko [28] = EMC_TXSRDLL, 251e34212c7SDmitry Osipenko [29] = EMC_TCKE, 252e34212c7SDmitry Osipenko [30] = EMC_TFAW, 253e34212c7SDmitry Osipenko [31] = EMC_TRPAB, 254e34212c7SDmitry Osipenko [32] = EMC_TCLKSTABLE, 255e34212c7SDmitry Osipenko [33] = EMC_TCLKSTOP, 256e34212c7SDmitry Osipenko [34] = EMC_TREFBW, 257e34212c7SDmitry Osipenko [35] = EMC_QUSE_EXTRA, 258e34212c7SDmitry Osipenko [36] = EMC_FBIO_CFG6, 259e34212c7SDmitry Osipenko [37] = EMC_ODT_WRITE, 260e34212c7SDmitry Osipenko [38] = EMC_ODT_READ, 261e34212c7SDmitry Osipenko [39] = EMC_FBIO_CFG5, 262e34212c7SDmitry Osipenko [40] = EMC_CFG_DIG_DLL, 263e34212c7SDmitry Osipenko [41] = EMC_CFG_DIG_DLL_PERIOD, 264e34212c7SDmitry Osipenko [42] = EMC_DLL_XFORM_DQS0, 265e34212c7SDmitry Osipenko [43] = EMC_DLL_XFORM_DQS1, 266e34212c7SDmitry Osipenko [44] = EMC_DLL_XFORM_DQS2, 267e34212c7SDmitry Osipenko [45] = EMC_DLL_XFORM_DQS3, 268e34212c7SDmitry Osipenko [46] = EMC_DLL_XFORM_DQS4, 269e34212c7SDmitry Osipenko [47] = EMC_DLL_XFORM_DQS5, 270e34212c7SDmitry Osipenko [48] = EMC_DLL_XFORM_DQS6, 271e34212c7SDmitry Osipenko [49] = EMC_DLL_XFORM_DQS7, 272e34212c7SDmitry Osipenko [50] = EMC_DLL_XFORM_QUSE0, 273e34212c7SDmitry Osipenko [51] = EMC_DLL_XFORM_QUSE1, 274e34212c7SDmitry Osipenko [52] = EMC_DLL_XFORM_QUSE2, 275e34212c7SDmitry Osipenko [53] = EMC_DLL_XFORM_QUSE3, 276e34212c7SDmitry Osipenko [54] = EMC_DLL_XFORM_QUSE4, 277e34212c7SDmitry Osipenko [55] = EMC_DLL_XFORM_QUSE5, 278e34212c7SDmitry Osipenko [56] = EMC_DLL_XFORM_QUSE6, 279e34212c7SDmitry Osipenko [57] = EMC_DLL_XFORM_QUSE7, 280e34212c7SDmitry Osipenko [58] = EMC_DLI_TRIM_TXDQS0, 281e34212c7SDmitry Osipenko [59] = EMC_DLI_TRIM_TXDQS1, 282e34212c7SDmitry Osipenko [60] = EMC_DLI_TRIM_TXDQS2, 283e34212c7SDmitry Osipenko [61] = EMC_DLI_TRIM_TXDQS3, 284e34212c7SDmitry Osipenko [62] = EMC_DLI_TRIM_TXDQS4, 285e34212c7SDmitry Osipenko [63] = EMC_DLI_TRIM_TXDQS5, 286e34212c7SDmitry Osipenko [64] = EMC_DLI_TRIM_TXDQS6, 287e34212c7SDmitry Osipenko [65] = EMC_DLI_TRIM_TXDQS7, 288e34212c7SDmitry Osipenko [66] = EMC_DLL_XFORM_DQ0, 289e34212c7SDmitry Osipenko [67] = EMC_DLL_XFORM_DQ1, 290e34212c7SDmitry Osipenko [68] = EMC_DLL_XFORM_DQ2, 291e34212c7SDmitry Osipenko [69] = EMC_DLL_XFORM_DQ3, 292e34212c7SDmitry Osipenko [70] = EMC_XM2CMDPADCTRL, 293e34212c7SDmitry Osipenko [71] = EMC_XM2DQSPADCTRL2, 294e34212c7SDmitry Osipenko [72] = EMC_XM2DQPADCTRL2, 295e34212c7SDmitry Osipenko [73] = EMC_XM2CLKPADCTRL, 296e34212c7SDmitry Osipenko [74] = EMC_XM2COMPPADCTRL, 297e34212c7SDmitry Osipenko [75] = EMC_XM2VTTGENPADCTRL, 298e34212c7SDmitry Osipenko [76] = EMC_XM2VTTGENPADCTRL2, 299e34212c7SDmitry Osipenko [77] = EMC_XM2QUSEPADCTRL, 300e34212c7SDmitry Osipenko [78] = EMC_XM2DQSPADCTRL3, 301e34212c7SDmitry Osipenko [79] = EMC_CTT_TERM_CTRL, 302e34212c7SDmitry Osipenko [80] = EMC_ZCAL_INTERVAL, 303e34212c7SDmitry Osipenko [81] = EMC_ZCAL_WAIT_CNT, 304e34212c7SDmitry Osipenko [82] = EMC_MRS_WAIT_CNT, 305e34212c7SDmitry Osipenko [83] = EMC_AUTO_CAL_CONFIG, 306e34212c7SDmitry Osipenko [84] = EMC_CTT, 307e34212c7SDmitry Osipenko [85] = EMC_CTT_DURATION, 308e34212c7SDmitry Osipenko [86] = EMC_DYN_SELF_REF_CONTROL, 309e34212c7SDmitry Osipenko [87] = EMC_FBIO_SPARE, 310e34212c7SDmitry Osipenko [88] = EMC_CFG_RSV, 311e34212c7SDmitry Osipenko }; 312e34212c7SDmitry Osipenko 313e34212c7SDmitry Osipenko struct emc_timing { 314e34212c7SDmitry Osipenko unsigned long rate; 315e34212c7SDmitry Osipenko 316e34212c7SDmitry Osipenko u32 data[ARRAY_SIZE(emc_timing_registers)]; 317e34212c7SDmitry Osipenko 318e34212c7SDmitry Osipenko u32 emc_auto_cal_interval; 319e34212c7SDmitry Osipenko u32 emc_mode_1; 320e34212c7SDmitry Osipenko u32 emc_mode_2; 321e34212c7SDmitry Osipenko u32 emc_mode_reset; 322e34212c7SDmitry Osipenko u32 emc_zcal_cnt_long; 323e34212c7SDmitry Osipenko bool emc_cfg_periodic_qrst; 324e34212c7SDmitry Osipenko bool emc_cfg_dyn_self_ref; 325e34212c7SDmitry Osipenko }; 326e34212c7SDmitry Osipenko 327e34212c7SDmitry Osipenko struct tegra_emc { 328e34212c7SDmitry Osipenko struct device *dev; 329e34212c7SDmitry Osipenko struct tegra_mc *mc; 330e34212c7SDmitry Osipenko struct completion clk_handshake_complete; 331e34212c7SDmitry Osipenko struct notifier_block clk_nb; 332e34212c7SDmitry Osipenko struct clk *clk; 333e34212c7SDmitry Osipenko void __iomem *regs; 334e34212c7SDmitry Osipenko unsigned int irq; 335e34212c7SDmitry Osipenko 336e34212c7SDmitry Osipenko struct emc_timing *timings; 337e34212c7SDmitry Osipenko unsigned int num_timings; 338e34212c7SDmitry Osipenko 339e34212c7SDmitry Osipenko u32 mc_override; 340e34212c7SDmitry Osipenko u32 emc_cfg; 341e34212c7SDmitry Osipenko 342e34212c7SDmitry Osipenko u32 emc_mode_1; 343e34212c7SDmitry Osipenko u32 emc_mode_2; 344e34212c7SDmitry Osipenko u32 emc_mode_reset; 345e34212c7SDmitry Osipenko 346e34212c7SDmitry Osipenko bool vref_cal_toggle : 1; 347e34212c7SDmitry Osipenko bool zcal_long : 1; 348e34212c7SDmitry Osipenko bool dll_on : 1; 349e34212c7SDmitry Osipenko bool prepared : 1; 350e34212c7SDmitry Osipenko bool bad_state : 1; 351*8cee32b4SThierry Reding 352*8cee32b4SThierry Reding struct { 353*8cee32b4SThierry Reding struct dentry *root; 354*8cee32b4SThierry Reding unsigned long min_rate; 355*8cee32b4SThierry Reding unsigned long max_rate; 356*8cee32b4SThierry Reding } debugfs; 357e34212c7SDmitry Osipenko }; 358e34212c7SDmitry Osipenko 359e34212c7SDmitry Osipenko static irqreturn_t tegra_emc_isr(int irq, void *data) 360e34212c7SDmitry Osipenko { 361e34212c7SDmitry Osipenko struct tegra_emc *emc = data; 362e34212c7SDmitry Osipenko u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 363e34212c7SDmitry Osipenko u32 status; 364e34212c7SDmitry Osipenko 365e34212c7SDmitry Osipenko status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; 366e34212c7SDmitry Osipenko if (!status) 367e34212c7SDmitry Osipenko return IRQ_NONE; 368e34212c7SDmitry Osipenko 369e34212c7SDmitry Osipenko /* notify about EMC-CAR handshake completion */ 370e34212c7SDmitry Osipenko if (status & EMC_CLKCHANGE_COMPLETE_INT) 371e34212c7SDmitry Osipenko complete(&emc->clk_handshake_complete); 372e34212c7SDmitry Osipenko 373e34212c7SDmitry Osipenko /* notify about HW problem */ 374e34212c7SDmitry Osipenko if (status & EMC_REFRESH_OVERFLOW_INT) 375e34212c7SDmitry Osipenko dev_err_ratelimited(emc->dev, 376e34212c7SDmitry Osipenko "refresh request overflow timeout\n"); 377e34212c7SDmitry Osipenko 378e34212c7SDmitry Osipenko /* clear interrupts */ 379e34212c7SDmitry Osipenko writel_relaxed(status, emc->regs + EMC_INTSTATUS); 380e34212c7SDmitry Osipenko 381e34212c7SDmitry Osipenko return IRQ_HANDLED; 382e34212c7SDmitry Osipenko } 383e34212c7SDmitry Osipenko 384e34212c7SDmitry Osipenko static struct emc_timing *emc_find_timing(struct tegra_emc *emc, 385e34212c7SDmitry Osipenko unsigned long rate) 386e34212c7SDmitry Osipenko { 387e34212c7SDmitry Osipenko struct emc_timing *timing = NULL; 388e34212c7SDmitry Osipenko unsigned int i; 389e34212c7SDmitry Osipenko 390e34212c7SDmitry Osipenko for (i = 0; i < emc->num_timings; i++) { 391e34212c7SDmitry Osipenko if (emc->timings[i].rate >= rate) { 392e34212c7SDmitry Osipenko timing = &emc->timings[i]; 393e34212c7SDmitry Osipenko break; 394e34212c7SDmitry Osipenko } 395e34212c7SDmitry Osipenko } 396e34212c7SDmitry Osipenko 397e34212c7SDmitry Osipenko if (!timing) { 398e34212c7SDmitry Osipenko dev_err(emc->dev, "no timing for rate %lu\n", rate); 399e34212c7SDmitry Osipenko return NULL; 400e34212c7SDmitry Osipenko } 401e34212c7SDmitry Osipenko 402e34212c7SDmitry Osipenko return timing; 403e34212c7SDmitry Osipenko } 404e34212c7SDmitry Osipenko 405e34212c7SDmitry Osipenko static bool emc_dqs_preset(struct tegra_emc *emc, struct emc_timing *timing, 406e34212c7SDmitry Osipenko bool *schmitt_to_vref) 407e34212c7SDmitry Osipenko { 408e34212c7SDmitry Osipenko bool preset = false; 409e34212c7SDmitry Osipenko u32 val; 410e34212c7SDmitry Osipenko 411e34212c7SDmitry Osipenko if (timing->data[71] & EMC_XM2DQSPADCTRL2_VREF_ENABLE) { 412e34212c7SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_XM2DQSPADCTRL2); 413e34212c7SDmitry Osipenko 414e34212c7SDmitry Osipenko if (!(val & EMC_XM2DQSPADCTRL2_VREF_ENABLE)) { 415e34212c7SDmitry Osipenko val |= EMC_XM2DQSPADCTRL2_VREF_ENABLE; 416e34212c7SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_XM2DQSPADCTRL2); 417e34212c7SDmitry Osipenko 418e34212c7SDmitry Osipenko preset = true; 419e34212c7SDmitry Osipenko } 420e34212c7SDmitry Osipenko } 421e34212c7SDmitry Osipenko 422e34212c7SDmitry Osipenko if (timing->data[78] & EMC_XM2DQSPADCTRL3_VREF_ENABLE) { 423e34212c7SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_XM2DQSPADCTRL3); 424e34212c7SDmitry Osipenko 425e34212c7SDmitry Osipenko if (!(val & EMC_XM2DQSPADCTRL3_VREF_ENABLE)) { 426e34212c7SDmitry Osipenko val |= EMC_XM2DQSPADCTRL3_VREF_ENABLE; 427e34212c7SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_XM2DQSPADCTRL3); 428e34212c7SDmitry Osipenko 429e34212c7SDmitry Osipenko preset = true; 430e34212c7SDmitry Osipenko } 431e34212c7SDmitry Osipenko } 432e34212c7SDmitry Osipenko 433e34212c7SDmitry Osipenko if (timing->data[77] & EMC_XM2QUSEPADCTRL_IVREF_ENABLE) { 434e34212c7SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_XM2QUSEPADCTRL); 435e34212c7SDmitry Osipenko 436e34212c7SDmitry Osipenko if (!(val & EMC_XM2QUSEPADCTRL_IVREF_ENABLE)) { 437e34212c7SDmitry Osipenko val |= EMC_XM2QUSEPADCTRL_IVREF_ENABLE; 438e34212c7SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_XM2QUSEPADCTRL); 439e34212c7SDmitry Osipenko 440e34212c7SDmitry Osipenko *schmitt_to_vref = true; 441e34212c7SDmitry Osipenko preset = true; 442e34212c7SDmitry Osipenko } 443e34212c7SDmitry Osipenko } 444e34212c7SDmitry Osipenko 445e34212c7SDmitry Osipenko return preset; 446e34212c7SDmitry Osipenko } 447e34212c7SDmitry Osipenko 448e34212c7SDmitry Osipenko static int emc_seq_update_timing(struct tegra_emc *emc) 449e34212c7SDmitry Osipenko { 450e34212c7SDmitry Osipenko u32 val; 451e34212c7SDmitry Osipenko int err; 452e34212c7SDmitry Osipenko 453e34212c7SDmitry Osipenko writel_relaxed(EMC_TIMING_UPDATE, emc->regs + EMC_TIMING_CONTROL); 454e34212c7SDmitry Osipenko 455e34212c7SDmitry Osipenko err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_STATUS, val, 456e34212c7SDmitry Osipenko !(val & EMC_STATUS_TIMING_UPDATE_STALLED), 457e34212c7SDmitry Osipenko 1, 200); 458e34212c7SDmitry Osipenko if (err) { 459e34212c7SDmitry Osipenko dev_err(emc->dev, "failed to update timing: %d\n", err); 460e34212c7SDmitry Osipenko return err; 461e34212c7SDmitry Osipenko } 462e34212c7SDmitry Osipenko 463e34212c7SDmitry Osipenko return 0; 464e34212c7SDmitry Osipenko } 465e34212c7SDmitry Osipenko 466e34212c7SDmitry Osipenko static int emc_prepare_mc_clk_cfg(struct tegra_emc *emc, unsigned long rate) 467e34212c7SDmitry Osipenko { 468e34212c7SDmitry Osipenko struct tegra_mc *mc = emc->mc; 469e34212c7SDmitry Osipenko unsigned int misc0_index = 16; 470e34212c7SDmitry Osipenko unsigned int i; 471e34212c7SDmitry Osipenko bool same; 472e34212c7SDmitry Osipenko 473e34212c7SDmitry Osipenko for (i = 0; i < mc->num_timings; i++) { 474e34212c7SDmitry Osipenko if (mc->timings[i].rate != rate) 475e34212c7SDmitry Osipenko continue; 476e34212c7SDmitry Osipenko 477e34212c7SDmitry Osipenko if (mc->timings[i].emem_data[misc0_index] & BIT(27)) 478e34212c7SDmitry Osipenko same = true; 479e34212c7SDmitry Osipenko else 480e34212c7SDmitry Osipenko same = false; 481e34212c7SDmitry Osipenko 482e34212c7SDmitry Osipenko return tegra20_clk_prepare_emc_mc_same_freq(emc->clk, same); 483e34212c7SDmitry Osipenko } 484e34212c7SDmitry Osipenko 485e34212c7SDmitry Osipenko return -EINVAL; 486e34212c7SDmitry Osipenko } 487e34212c7SDmitry Osipenko 488e34212c7SDmitry Osipenko static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) 489e34212c7SDmitry Osipenko { 490e34212c7SDmitry Osipenko struct emc_timing *timing = emc_find_timing(emc, rate); 491e34212c7SDmitry Osipenko enum emc_dll_change dll_change; 492e34212c7SDmitry Osipenko enum emc_dram_type dram_type; 493e34212c7SDmitry Osipenko bool schmitt_to_vref = false; 494e34212c7SDmitry Osipenko unsigned int pre_wait = 0; 495e34212c7SDmitry Osipenko bool qrst_used = false; 496e34212c7SDmitry Osipenko unsigned int dram_num; 497e34212c7SDmitry Osipenko unsigned int i; 498e34212c7SDmitry Osipenko u32 fbio_cfg5; 499e34212c7SDmitry Osipenko u32 emc_dbg; 500e34212c7SDmitry Osipenko u32 val; 501e34212c7SDmitry Osipenko int err; 502e34212c7SDmitry Osipenko 503e34212c7SDmitry Osipenko if (!timing || emc->bad_state) 504e34212c7SDmitry Osipenko return -EINVAL; 505e34212c7SDmitry Osipenko 506e34212c7SDmitry Osipenko dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n", 507e34212c7SDmitry Osipenko __func__, timing->rate, rate); 508e34212c7SDmitry Osipenko 509e34212c7SDmitry Osipenko emc->bad_state = true; 510e34212c7SDmitry Osipenko 511e34212c7SDmitry Osipenko err = emc_prepare_mc_clk_cfg(emc, rate); 512e34212c7SDmitry Osipenko if (err) { 513e34212c7SDmitry Osipenko dev_err(emc->dev, "mc clock preparation failed: %d\n", err); 514e34212c7SDmitry Osipenko return err; 515e34212c7SDmitry Osipenko } 516e34212c7SDmitry Osipenko 517e34212c7SDmitry Osipenko emc->vref_cal_toggle = false; 518e34212c7SDmitry Osipenko emc->mc_override = mc_readl(emc->mc, MC_EMEM_ARB_OVERRIDE); 519e34212c7SDmitry Osipenko emc->emc_cfg = readl_relaxed(emc->regs + EMC_CFG); 520e34212c7SDmitry Osipenko emc_dbg = readl_relaxed(emc->regs + EMC_DBG); 521e34212c7SDmitry Osipenko 522e34212c7SDmitry Osipenko if (emc->dll_on == !!(timing->emc_mode_1 & 0x1)) 523e34212c7SDmitry Osipenko dll_change = DLL_CHANGE_NONE; 524e34212c7SDmitry Osipenko else if (timing->emc_mode_1 & 0x1) 525e34212c7SDmitry Osipenko dll_change = DLL_CHANGE_ON; 526e34212c7SDmitry Osipenko else 527e34212c7SDmitry Osipenko dll_change = DLL_CHANGE_OFF; 528e34212c7SDmitry Osipenko 529e34212c7SDmitry Osipenko emc->dll_on = !!(timing->emc_mode_1 & 0x1); 530e34212c7SDmitry Osipenko 531e34212c7SDmitry Osipenko if (timing->data[80] && !readl_relaxed(emc->regs + EMC_ZCAL_INTERVAL)) 532e34212c7SDmitry Osipenko emc->zcal_long = true; 533e34212c7SDmitry Osipenko else 534e34212c7SDmitry Osipenko emc->zcal_long = false; 535e34212c7SDmitry Osipenko 536e34212c7SDmitry Osipenko fbio_cfg5 = readl_relaxed(emc->regs + EMC_FBIO_CFG5); 537e34212c7SDmitry Osipenko dram_type = fbio_cfg5 & EMC_FBIO_CFG5_DRAM_TYPE_MASK; 538e34212c7SDmitry Osipenko 539e34212c7SDmitry Osipenko dram_num = tegra_mc_get_emem_device_count(emc->mc); 540e34212c7SDmitry Osipenko 541e34212c7SDmitry Osipenko /* disable dynamic self-refresh */ 542e34212c7SDmitry Osipenko if (emc->emc_cfg & EMC_CFG_DYN_SREF_ENABLE) { 543e34212c7SDmitry Osipenko emc->emc_cfg &= ~EMC_CFG_DYN_SREF_ENABLE; 544e34212c7SDmitry Osipenko writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG); 545e34212c7SDmitry Osipenko 546e34212c7SDmitry Osipenko pre_wait = 5; 547e34212c7SDmitry Osipenko } 548e34212c7SDmitry Osipenko 549e34212c7SDmitry Osipenko /* update MC arbiter settings */ 550e34212c7SDmitry Osipenko val = mc_readl(emc->mc, MC_EMEM_ARB_OUTSTANDING_REQ); 551e34212c7SDmitry Osipenko if (!(val & MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE) || 552e34212c7SDmitry Osipenko ((val & MC_EMEM_ARB_OUTSTANDING_REQ_MAX_MASK) > 0x50)) { 553e34212c7SDmitry Osipenko 554e34212c7SDmitry Osipenko val = MC_EMEM_ARB_OUTSTANDING_REQ_LIMIT_ENABLE | 555e34212c7SDmitry Osipenko MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE | 0x50; 556e34212c7SDmitry Osipenko mc_writel(emc->mc, val, MC_EMEM_ARB_OUTSTANDING_REQ); 557e34212c7SDmitry Osipenko mc_writel(emc->mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); 558e34212c7SDmitry Osipenko } 559e34212c7SDmitry Osipenko 560e34212c7SDmitry Osipenko if (emc->mc_override & MC_EMEM_ARB_OVERRIDE_EACK_MASK) 561e34212c7SDmitry Osipenko mc_writel(emc->mc, 562e34212c7SDmitry Osipenko emc->mc_override & ~MC_EMEM_ARB_OVERRIDE_EACK_MASK, 563e34212c7SDmitry Osipenko MC_EMEM_ARB_OVERRIDE); 564e34212c7SDmitry Osipenko 565e34212c7SDmitry Osipenko /* check DQ/DQS VREF delay */ 566e34212c7SDmitry Osipenko if (emc_dqs_preset(emc, timing, &schmitt_to_vref)) { 567e34212c7SDmitry Osipenko if (pre_wait < 3) 568e34212c7SDmitry Osipenko pre_wait = 3; 569e34212c7SDmitry Osipenko } 570e34212c7SDmitry Osipenko 571e34212c7SDmitry Osipenko if (pre_wait) { 572e34212c7SDmitry Osipenko err = emc_seq_update_timing(emc); 573e34212c7SDmitry Osipenko if (err) 574e34212c7SDmitry Osipenko return err; 575e34212c7SDmitry Osipenko 576e34212c7SDmitry Osipenko udelay(pre_wait); 577e34212c7SDmitry Osipenko } 578e34212c7SDmitry Osipenko 579e34212c7SDmitry Osipenko /* disable auto-calibration if VREF mode is switching */ 580e34212c7SDmitry Osipenko if (timing->emc_auto_cal_interval) { 581e34212c7SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_XM2COMPPADCTRL); 582e34212c7SDmitry Osipenko val ^= timing->data[74]; 583e34212c7SDmitry Osipenko 584e34212c7SDmitry Osipenko if (val & EMC_XM2COMPPADCTRL_VREF_CAL_ENABLE) { 585e34212c7SDmitry Osipenko writel_relaxed(0, emc->regs + EMC_AUTO_CAL_INTERVAL); 586e34212c7SDmitry Osipenko 587e34212c7SDmitry Osipenko err = readl_relaxed_poll_timeout_atomic( 588e34212c7SDmitry Osipenko emc->regs + EMC_AUTO_CAL_STATUS, val, 589e34212c7SDmitry Osipenko !(val & EMC_AUTO_CAL_STATUS_ACTIVE), 1, 300); 590e34212c7SDmitry Osipenko if (err) { 591e34212c7SDmitry Osipenko dev_err(emc->dev, 592e34212c7SDmitry Osipenko "failed to disable auto-cal: %d\n", 593e34212c7SDmitry Osipenko err); 594e34212c7SDmitry Osipenko return err; 595e34212c7SDmitry Osipenko } 596e34212c7SDmitry Osipenko 597e34212c7SDmitry Osipenko emc->vref_cal_toggle = true; 598e34212c7SDmitry Osipenko } 599e34212c7SDmitry Osipenko } 600e34212c7SDmitry Osipenko 601e34212c7SDmitry Osipenko /* program shadow registers */ 602e34212c7SDmitry Osipenko for (i = 0; i < ARRAY_SIZE(timing->data); i++) { 603e34212c7SDmitry Osipenko /* EMC_XM2CLKPADCTRL should be programmed separately */ 604e34212c7SDmitry Osipenko if (i != 73) 605e34212c7SDmitry Osipenko writel_relaxed(timing->data[i], 606e34212c7SDmitry Osipenko emc->regs + emc_timing_registers[i]); 607e34212c7SDmitry Osipenko } 608e34212c7SDmitry Osipenko 609e34212c7SDmitry Osipenko err = tegra_mc_write_emem_configuration(emc->mc, timing->rate); 610e34212c7SDmitry Osipenko if (err) 611e34212c7SDmitry Osipenko return err; 612e34212c7SDmitry Osipenko 613e34212c7SDmitry Osipenko /* DDR3: predict MRS long wait count */ 614e34212c7SDmitry Osipenko if (dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_ON) { 615e34212c7SDmitry Osipenko u32 cnt = 512; 616e34212c7SDmitry Osipenko 617e34212c7SDmitry Osipenko if (emc->zcal_long) 618e34212c7SDmitry Osipenko cnt -= dram_num * 256; 619e34212c7SDmitry Osipenko 620e34212c7SDmitry Osipenko val = timing->data[82] & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK; 621e34212c7SDmitry Osipenko if (cnt < val) 622e34212c7SDmitry Osipenko cnt = val; 623e34212c7SDmitry Osipenko 624e34212c7SDmitry Osipenko val = timing->data[82] & ~EMC_MRS_WAIT_CNT_LONG_WAIT_MASK; 625e34212c7SDmitry Osipenko val |= (cnt << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT) & 626e34212c7SDmitry Osipenko EMC_MRS_WAIT_CNT_LONG_WAIT_MASK; 627e34212c7SDmitry Osipenko 628e34212c7SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_MRS_WAIT_CNT); 629e34212c7SDmitry Osipenko } 630e34212c7SDmitry Osipenko 631e34212c7SDmitry Osipenko /* disable interrupt since read access is prohibited after stalling */ 632e34212c7SDmitry Osipenko disable_irq(emc->irq); 633e34212c7SDmitry Osipenko 634e34212c7SDmitry Osipenko /* this read also completes the writes */ 635e34212c7SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_SEL_DPD_CTRL); 636e34212c7SDmitry Osipenko 637e34212c7SDmitry Osipenko if (!(val & EMC_SEL_DPD_CTRL_QUSE_DPD_ENABLE) && schmitt_to_vref) { 638e34212c7SDmitry Osipenko u32 cur_mode, new_mode; 639e34212c7SDmitry Osipenko 640e34212c7SDmitry Osipenko cur_mode = fbio_cfg5 & EMC_CFG5_QUSE_MODE_MASK; 641e34212c7SDmitry Osipenko cur_mode >>= EMC_CFG5_QUSE_MODE_SHIFT; 642e34212c7SDmitry Osipenko 643e34212c7SDmitry Osipenko new_mode = timing->data[39] & EMC_CFG5_QUSE_MODE_MASK; 644e34212c7SDmitry Osipenko new_mode >>= EMC_CFG5_QUSE_MODE_SHIFT; 645e34212c7SDmitry Osipenko 646e34212c7SDmitry Osipenko if ((cur_mode != EMC_CFG5_QUSE_MODE_PULSE_INTERN && 647e34212c7SDmitry Osipenko cur_mode != EMC_CFG5_QUSE_MODE_INTERNAL_LPBK) || 648e34212c7SDmitry Osipenko (new_mode != EMC_CFG5_QUSE_MODE_PULSE_INTERN && 649e34212c7SDmitry Osipenko new_mode != EMC_CFG5_QUSE_MODE_INTERNAL_LPBK)) 650e34212c7SDmitry Osipenko qrst_used = true; 651e34212c7SDmitry Osipenko } 652e34212c7SDmitry Osipenko 653e34212c7SDmitry Osipenko /* flow control marker 1 */ 654e34212c7SDmitry Osipenko writel_relaxed(0x1, emc->regs + EMC_STALL_THEN_EXE_BEFORE_CLKCHANGE); 655e34212c7SDmitry Osipenko 656e34212c7SDmitry Osipenko /* enable periodic reset */ 657e34212c7SDmitry Osipenko if (qrst_used) { 658e34212c7SDmitry Osipenko writel_relaxed(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, 659e34212c7SDmitry Osipenko emc->regs + EMC_DBG); 660e34212c7SDmitry Osipenko writel_relaxed(emc->emc_cfg | EMC_CFG_PERIODIC_QRST, 661e34212c7SDmitry Osipenko emc->regs + EMC_CFG); 662e34212c7SDmitry Osipenko writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 663e34212c7SDmitry Osipenko } 664e34212c7SDmitry Osipenko 665e34212c7SDmitry Osipenko /* disable auto-refresh to save time after clock change */ 666e34212c7SDmitry Osipenko writel_relaxed(EMC_REFCTRL_DISABLE_ALL(dram_num), 667e34212c7SDmitry Osipenko emc->regs + EMC_REFCTRL); 668e34212c7SDmitry Osipenko 669e34212c7SDmitry Osipenko /* turn off DLL and enter self-refresh on DDR3 */ 670e34212c7SDmitry Osipenko if (dram_type == DRAM_TYPE_DDR3) { 671e34212c7SDmitry Osipenko if (dll_change == DLL_CHANGE_OFF) 672e34212c7SDmitry Osipenko writel_relaxed(timing->emc_mode_1, 673e34212c7SDmitry Osipenko emc->regs + EMC_EMRS); 674e34212c7SDmitry Osipenko 675e34212c7SDmitry Osipenko writel_relaxed(DRAM_BROADCAST(dram_num) | 676e34212c7SDmitry Osipenko EMC_SELF_REF_CMD_ENABLED, 677e34212c7SDmitry Osipenko emc->regs + EMC_SELF_REF); 678e34212c7SDmitry Osipenko } 679e34212c7SDmitry Osipenko 680e34212c7SDmitry Osipenko /* flow control marker 2 */ 681e34212c7SDmitry Osipenko writel_relaxed(0x1, emc->regs + EMC_STALL_THEN_EXE_AFTER_CLKCHANGE); 682e34212c7SDmitry Osipenko 683e34212c7SDmitry Osipenko /* enable write-active MUX, update unshadowed pad control */ 684e34212c7SDmitry Osipenko writel_relaxed(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, emc->regs + EMC_DBG); 685e34212c7SDmitry Osipenko writel_relaxed(timing->data[73], emc->regs + EMC_XM2CLKPADCTRL); 686e34212c7SDmitry Osipenko 687e34212c7SDmitry Osipenko /* restore periodic QRST and disable write-active MUX */ 688e34212c7SDmitry Osipenko val = !!(emc->emc_cfg & EMC_CFG_PERIODIC_QRST); 689e34212c7SDmitry Osipenko if (qrst_used || timing->emc_cfg_periodic_qrst != val) { 690e34212c7SDmitry Osipenko if (timing->emc_cfg_periodic_qrst) 691e34212c7SDmitry Osipenko emc->emc_cfg |= EMC_CFG_PERIODIC_QRST; 692e34212c7SDmitry Osipenko else 693e34212c7SDmitry Osipenko emc->emc_cfg &= ~EMC_CFG_PERIODIC_QRST; 694e34212c7SDmitry Osipenko 695e34212c7SDmitry Osipenko writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG); 696e34212c7SDmitry Osipenko } 697e34212c7SDmitry Osipenko writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 698e34212c7SDmitry Osipenko 699e34212c7SDmitry Osipenko /* exit self-refresh on DDR3 */ 700e34212c7SDmitry Osipenko if (dram_type == DRAM_TYPE_DDR3) 701e34212c7SDmitry Osipenko writel_relaxed(DRAM_BROADCAST(dram_num), 702e34212c7SDmitry Osipenko emc->regs + EMC_SELF_REF); 703e34212c7SDmitry Osipenko 704e34212c7SDmitry Osipenko /* set DRAM-mode registers */ 705e34212c7SDmitry Osipenko if (dram_type == DRAM_TYPE_DDR3) { 706e34212c7SDmitry Osipenko if (timing->emc_mode_1 != emc->emc_mode_1) 707e34212c7SDmitry Osipenko writel_relaxed(timing->emc_mode_1, 708e34212c7SDmitry Osipenko emc->regs + EMC_EMRS); 709e34212c7SDmitry Osipenko 710e34212c7SDmitry Osipenko if (timing->emc_mode_2 != emc->emc_mode_2) 711e34212c7SDmitry Osipenko writel_relaxed(timing->emc_mode_2, 712e34212c7SDmitry Osipenko emc->regs + EMC_EMRS); 713e34212c7SDmitry Osipenko 714e34212c7SDmitry Osipenko if (timing->emc_mode_reset != emc->emc_mode_reset || 715e34212c7SDmitry Osipenko dll_change == DLL_CHANGE_ON) { 716e34212c7SDmitry Osipenko val = timing->emc_mode_reset; 717e34212c7SDmitry Osipenko if (dll_change == DLL_CHANGE_ON) { 718e34212c7SDmitry Osipenko val |= EMC_MODE_SET_DLL_RESET; 719e34212c7SDmitry Osipenko val |= EMC_MODE_SET_LONG_CNT; 720e34212c7SDmitry Osipenko } else { 721e34212c7SDmitry Osipenko val &= ~EMC_MODE_SET_DLL_RESET; 722e34212c7SDmitry Osipenko } 723e34212c7SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_MRS); 724e34212c7SDmitry Osipenko } 725e34212c7SDmitry Osipenko } else { 726e34212c7SDmitry Osipenko if (timing->emc_mode_2 != emc->emc_mode_2) 727e34212c7SDmitry Osipenko writel_relaxed(timing->emc_mode_2, 728e34212c7SDmitry Osipenko emc->regs + EMC_MRW); 729e34212c7SDmitry Osipenko 730e34212c7SDmitry Osipenko if (timing->emc_mode_1 != emc->emc_mode_1) 731e34212c7SDmitry Osipenko writel_relaxed(timing->emc_mode_1, 732e34212c7SDmitry Osipenko emc->regs + EMC_MRW); 733e34212c7SDmitry Osipenko } 734e34212c7SDmitry Osipenko 735e34212c7SDmitry Osipenko emc->emc_mode_1 = timing->emc_mode_1; 736e34212c7SDmitry Osipenko emc->emc_mode_2 = timing->emc_mode_2; 737e34212c7SDmitry Osipenko emc->emc_mode_reset = timing->emc_mode_reset; 738e34212c7SDmitry Osipenko 739e34212c7SDmitry Osipenko /* issue ZCAL command if turning ZCAL on */ 740e34212c7SDmitry Osipenko if (emc->zcal_long) { 741e34212c7SDmitry Osipenko writel_relaxed(EMC_ZQ_CAL_LONG_CMD_DEV0, 742e34212c7SDmitry Osipenko emc->regs + EMC_ZQ_CAL); 743e34212c7SDmitry Osipenko 744e34212c7SDmitry Osipenko if (dram_num > 1) 745e34212c7SDmitry Osipenko writel_relaxed(EMC_ZQ_CAL_LONG_CMD_DEV1, 746e34212c7SDmitry Osipenko emc->regs + EMC_ZQ_CAL); 747e34212c7SDmitry Osipenko } 748e34212c7SDmitry Osipenko 749e34212c7SDmitry Osipenko /* re-enable auto-refresh */ 750e34212c7SDmitry Osipenko writel_relaxed(EMC_REFCTRL_ENABLE_ALL(dram_num), 751e34212c7SDmitry Osipenko emc->regs + EMC_REFCTRL); 752e34212c7SDmitry Osipenko 753e34212c7SDmitry Osipenko /* flow control marker 3 */ 754e34212c7SDmitry Osipenko writel_relaxed(0x1, emc->regs + EMC_UNSTALL_RW_AFTER_CLKCHANGE); 755e34212c7SDmitry Osipenko 756e34212c7SDmitry Osipenko reinit_completion(&emc->clk_handshake_complete); 757e34212c7SDmitry Osipenko 758e34212c7SDmitry Osipenko /* interrupt can be re-enabled now */ 759e34212c7SDmitry Osipenko enable_irq(emc->irq); 760e34212c7SDmitry Osipenko 761e34212c7SDmitry Osipenko emc->bad_state = false; 762e34212c7SDmitry Osipenko emc->prepared = true; 763e34212c7SDmitry Osipenko 764e34212c7SDmitry Osipenko return 0; 765e34212c7SDmitry Osipenko } 766e34212c7SDmitry Osipenko 767e34212c7SDmitry Osipenko static int emc_complete_timing_change(struct tegra_emc *emc, 768e34212c7SDmitry Osipenko unsigned long rate) 769e34212c7SDmitry Osipenko { 770e34212c7SDmitry Osipenko struct emc_timing *timing = emc_find_timing(emc, rate); 771e34212c7SDmitry Osipenko unsigned long timeout; 772e34212c7SDmitry Osipenko int ret; 773e34212c7SDmitry Osipenko 774e34212c7SDmitry Osipenko timeout = wait_for_completion_timeout(&emc->clk_handshake_complete, 775e34212c7SDmitry Osipenko msecs_to_jiffies(100)); 776e34212c7SDmitry Osipenko if (timeout == 0) { 777e34212c7SDmitry Osipenko dev_err(emc->dev, "emc-car handshake failed\n"); 778e34212c7SDmitry Osipenko emc->bad_state = true; 779e34212c7SDmitry Osipenko return -EIO; 780e34212c7SDmitry Osipenko } 781e34212c7SDmitry Osipenko 782e34212c7SDmitry Osipenko /* restore auto-calibration */ 783e34212c7SDmitry Osipenko if (emc->vref_cal_toggle) 784e34212c7SDmitry Osipenko writel_relaxed(timing->emc_auto_cal_interval, 785e34212c7SDmitry Osipenko emc->regs + EMC_AUTO_CAL_INTERVAL); 786e34212c7SDmitry Osipenko 787e34212c7SDmitry Osipenko /* restore dynamic self-refresh */ 788e34212c7SDmitry Osipenko if (timing->emc_cfg_dyn_self_ref) { 789e34212c7SDmitry Osipenko emc->emc_cfg |= EMC_CFG_DYN_SREF_ENABLE; 790e34212c7SDmitry Osipenko writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG); 791e34212c7SDmitry Osipenko } 792e34212c7SDmitry Osipenko 793e34212c7SDmitry Osipenko /* set number of clocks to wait after each ZQ command */ 794e34212c7SDmitry Osipenko if (emc->zcal_long) 795e34212c7SDmitry Osipenko writel_relaxed(timing->emc_zcal_cnt_long, 796e34212c7SDmitry Osipenko emc->regs + EMC_ZCAL_WAIT_CNT); 797e34212c7SDmitry Osipenko 798e34212c7SDmitry Osipenko udelay(2); 799e34212c7SDmitry Osipenko /* update restored timing */ 800e34212c7SDmitry Osipenko ret = emc_seq_update_timing(emc); 801e34212c7SDmitry Osipenko if (ret) 802e34212c7SDmitry Osipenko emc->bad_state = true; 803e34212c7SDmitry Osipenko 804e34212c7SDmitry Osipenko /* restore early ACK */ 805e34212c7SDmitry Osipenko mc_writel(emc->mc, emc->mc_override, MC_EMEM_ARB_OVERRIDE); 806e34212c7SDmitry Osipenko 807e34212c7SDmitry Osipenko emc->prepared = false; 808e34212c7SDmitry Osipenko 809e34212c7SDmitry Osipenko return ret; 810e34212c7SDmitry Osipenko } 811e34212c7SDmitry Osipenko 812e34212c7SDmitry Osipenko static int emc_unprepare_timing_change(struct tegra_emc *emc, 813e34212c7SDmitry Osipenko unsigned long rate) 814e34212c7SDmitry Osipenko { 815e34212c7SDmitry Osipenko if (emc->prepared && !emc->bad_state) { 816e34212c7SDmitry Osipenko /* shouldn't ever happen in practice */ 817e34212c7SDmitry Osipenko dev_err(emc->dev, "timing configuration can't be reverted\n"); 818e34212c7SDmitry Osipenko emc->bad_state = true; 819e34212c7SDmitry Osipenko } 820e34212c7SDmitry Osipenko 821e34212c7SDmitry Osipenko return 0; 822e34212c7SDmitry Osipenko } 823e34212c7SDmitry Osipenko 824e34212c7SDmitry Osipenko static int emc_clk_change_notify(struct notifier_block *nb, 825e34212c7SDmitry Osipenko unsigned long msg, void *data) 826e34212c7SDmitry Osipenko { 827e34212c7SDmitry Osipenko struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb); 828e34212c7SDmitry Osipenko struct clk_notifier_data *cnd = data; 829e34212c7SDmitry Osipenko int err; 830e34212c7SDmitry Osipenko 831e34212c7SDmitry Osipenko switch (msg) { 832e34212c7SDmitry Osipenko case PRE_RATE_CHANGE: 833e34212c7SDmitry Osipenko err = emc_prepare_timing_change(emc, cnd->new_rate); 834e34212c7SDmitry Osipenko break; 835e34212c7SDmitry Osipenko 836e34212c7SDmitry Osipenko case ABORT_RATE_CHANGE: 837e34212c7SDmitry Osipenko err = emc_unprepare_timing_change(emc, cnd->old_rate); 838e34212c7SDmitry Osipenko break; 839e34212c7SDmitry Osipenko 840e34212c7SDmitry Osipenko case POST_RATE_CHANGE: 841e34212c7SDmitry Osipenko err = emc_complete_timing_change(emc, cnd->new_rate); 842e34212c7SDmitry Osipenko break; 843e34212c7SDmitry Osipenko 844e34212c7SDmitry Osipenko default: 845e34212c7SDmitry Osipenko return NOTIFY_DONE; 846e34212c7SDmitry Osipenko } 847e34212c7SDmitry Osipenko 848e34212c7SDmitry Osipenko return notifier_from_errno(err); 849e34212c7SDmitry Osipenko } 850e34212c7SDmitry Osipenko 851e34212c7SDmitry Osipenko static int load_one_timing_from_dt(struct tegra_emc *emc, 852e34212c7SDmitry Osipenko struct emc_timing *timing, 853e34212c7SDmitry Osipenko struct device_node *node) 854e34212c7SDmitry Osipenko { 855e34212c7SDmitry Osipenko u32 value; 856e34212c7SDmitry Osipenko int err; 857e34212c7SDmitry Osipenko 858e34212c7SDmitry Osipenko err = of_property_read_u32(node, "clock-frequency", &value); 859e34212c7SDmitry Osipenko if (err) { 860e34212c7SDmitry Osipenko dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n", 861e34212c7SDmitry Osipenko node, err); 862e34212c7SDmitry Osipenko return err; 863e34212c7SDmitry Osipenko } 864e34212c7SDmitry Osipenko 865e34212c7SDmitry Osipenko timing->rate = value; 866e34212c7SDmitry Osipenko 867e34212c7SDmitry Osipenko err = of_property_read_u32_array(node, "nvidia,emc-configuration", 868e34212c7SDmitry Osipenko timing->data, 869e34212c7SDmitry Osipenko ARRAY_SIZE(emc_timing_registers)); 870e34212c7SDmitry Osipenko if (err) { 871e34212c7SDmitry Osipenko dev_err(emc->dev, 872e34212c7SDmitry Osipenko "timing %pOF: failed to read emc timing data: %d\n", 873e34212c7SDmitry Osipenko node, err); 874e34212c7SDmitry Osipenko return err; 875e34212c7SDmitry Osipenko } 876e34212c7SDmitry Osipenko 877e34212c7SDmitry Osipenko #define EMC_READ_BOOL(prop, dtprop) \ 878e34212c7SDmitry Osipenko timing->prop = of_property_read_bool(node, dtprop); 879e34212c7SDmitry Osipenko 880e34212c7SDmitry Osipenko #define EMC_READ_U32(prop, dtprop) \ 881e34212c7SDmitry Osipenko err = of_property_read_u32(node, dtprop, &timing->prop); \ 882e34212c7SDmitry Osipenko if (err) { \ 883e34212c7SDmitry Osipenko dev_err(emc->dev, \ 884e34212c7SDmitry Osipenko "timing %pOFn: failed to read " #prop ": %d\n", \ 885e34212c7SDmitry Osipenko node, err); \ 886e34212c7SDmitry Osipenko return err; \ 887e34212c7SDmitry Osipenko } 888e34212c7SDmitry Osipenko 889e34212c7SDmitry Osipenko EMC_READ_U32(emc_auto_cal_interval, "nvidia,emc-auto-cal-interval") 890e34212c7SDmitry Osipenko EMC_READ_U32(emc_mode_1, "nvidia,emc-mode-1") 891e34212c7SDmitry Osipenko EMC_READ_U32(emc_mode_2, "nvidia,emc-mode-2") 892e34212c7SDmitry Osipenko EMC_READ_U32(emc_mode_reset, "nvidia,emc-mode-reset") 893e34212c7SDmitry Osipenko EMC_READ_U32(emc_zcal_cnt_long, "nvidia,emc-zcal-cnt-long") 894e34212c7SDmitry Osipenko EMC_READ_BOOL(emc_cfg_dyn_self_ref, "nvidia,emc-cfg-dyn-self-ref") 895e34212c7SDmitry Osipenko EMC_READ_BOOL(emc_cfg_periodic_qrst, "nvidia,emc-cfg-periodic-qrst") 896e34212c7SDmitry Osipenko 897e34212c7SDmitry Osipenko #undef EMC_READ_U32 898e34212c7SDmitry Osipenko #undef EMC_READ_BOOL 899e34212c7SDmitry Osipenko 900e34212c7SDmitry Osipenko dev_dbg(emc->dev, "%s: %pOF: rate %lu\n", __func__, node, timing->rate); 901e34212c7SDmitry Osipenko 902e34212c7SDmitry Osipenko return 0; 903e34212c7SDmitry Osipenko } 904e34212c7SDmitry Osipenko 905e34212c7SDmitry Osipenko static int cmp_timings(const void *_a, const void *_b) 906e34212c7SDmitry Osipenko { 907e34212c7SDmitry Osipenko const struct emc_timing *a = _a; 908e34212c7SDmitry Osipenko const struct emc_timing *b = _b; 909e34212c7SDmitry Osipenko 910e34212c7SDmitry Osipenko if (a->rate < b->rate) 911e34212c7SDmitry Osipenko return -1; 912e34212c7SDmitry Osipenko 913e34212c7SDmitry Osipenko if (a->rate > b->rate) 914e34212c7SDmitry Osipenko return 1; 915e34212c7SDmitry Osipenko 916e34212c7SDmitry Osipenko return 0; 917e34212c7SDmitry Osipenko } 918e34212c7SDmitry Osipenko 919e34212c7SDmitry Osipenko static int emc_check_mc_timings(struct tegra_emc *emc) 920e34212c7SDmitry Osipenko { 921e34212c7SDmitry Osipenko struct tegra_mc *mc = emc->mc; 922e34212c7SDmitry Osipenko unsigned int i; 923e34212c7SDmitry Osipenko 924e34212c7SDmitry Osipenko if (emc->num_timings != mc->num_timings) { 925e34212c7SDmitry Osipenko dev_err(emc->dev, "emc/mc timings number mismatch: %u %u\n", 926e34212c7SDmitry Osipenko emc->num_timings, mc->num_timings); 927e34212c7SDmitry Osipenko return -EINVAL; 928e34212c7SDmitry Osipenko } 929e34212c7SDmitry Osipenko 930e34212c7SDmitry Osipenko for (i = 0; i < mc->num_timings; i++) { 931e34212c7SDmitry Osipenko if (emc->timings[i].rate != mc->timings[i].rate) { 932e34212c7SDmitry Osipenko dev_err(emc->dev, 933e34212c7SDmitry Osipenko "emc/mc timing rate mismatch: %lu %lu\n", 934e34212c7SDmitry Osipenko emc->timings[i].rate, mc->timings[i].rate); 935e34212c7SDmitry Osipenko return -EINVAL; 936e34212c7SDmitry Osipenko } 937e34212c7SDmitry Osipenko } 938e34212c7SDmitry Osipenko 939e34212c7SDmitry Osipenko return 0; 940e34212c7SDmitry Osipenko } 941e34212c7SDmitry Osipenko 942e34212c7SDmitry Osipenko static int emc_load_timings_from_dt(struct tegra_emc *emc, 943e34212c7SDmitry Osipenko struct device_node *node) 944e34212c7SDmitry Osipenko { 945e34212c7SDmitry Osipenko struct device_node *child; 946e34212c7SDmitry Osipenko struct emc_timing *timing; 947e34212c7SDmitry Osipenko int child_count; 948e34212c7SDmitry Osipenko int err; 949e34212c7SDmitry Osipenko 950e34212c7SDmitry Osipenko child_count = of_get_child_count(node); 951e34212c7SDmitry Osipenko if (!child_count) { 952e34212c7SDmitry Osipenko dev_err(emc->dev, "no memory timings in: %pOF\n", node); 953e34212c7SDmitry Osipenko return -EINVAL; 954e34212c7SDmitry Osipenko } 955e34212c7SDmitry Osipenko 956e34212c7SDmitry Osipenko emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing), 957e34212c7SDmitry Osipenko GFP_KERNEL); 958e34212c7SDmitry Osipenko if (!emc->timings) 959e34212c7SDmitry Osipenko return -ENOMEM; 960e34212c7SDmitry Osipenko 961e34212c7SDmitry Osipenko emc->num_timings = child_count; 962e34212c7SDmitry Osipenko timing = emc->timings; 963e34212c7SDmitry Osipenko 964e34212c7SDmitry Osipenko for_each_child_of_node(node, child) { 965e34212c7SDmitry Osipenko err = load_one_timing_from_dt(emc, timing++, child); 966e34212c7SDmitry Osipenko if (err) { 967e34212c7SDmitry Osipenko of_node_put(child); 968e34212c7SDmitry Osipenko return err; 969e34212c7SDmitry Osipenko } 970e34212c7SDmitry Osipenko } 971e34212c7SDmitry Osipenko 972e34212c7SDmitry Osipenko sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, 973e34212c7SDmitry Osipenko NULL); 974e34212c7SDmitry Osipenko 975e34212c7SDmitry Osipenko err = emc_check_mc_timings(emc); 976e34212c7SDmitry Osipenko if (err) 977e34212c7SDmitry Osipenko return err; 978e34212c7SDmitry Osipenko 979e34212c7SDmitry Osipenko dev_info(emc->dev, 980e34212c7SDmitry Osipenko "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", 981e34212c7SDmitry Osipenko emc->num_timings, 982e34212c7SDmitry Osipenko tegra_read_ram_code(), 983e34212c7SDmitry Osipenko emc->timings[0].rate / 1000000, 984e34212c7SDmitry Osipenko emc->timings[emc->num_timings - 1].rate / 1000000); 985e34212c7SDmitry Osipenko 986e34212c7SDmitry Osipenko return 0; 987e34212c7SDmitry Osipenko } 988e34212c7SDmitry Osipenko 989e34212c7SDmitry Osipenko static struct device_node *emc_find_node_by_ram_code(struct device *dev) 990e34212c7SDmitry Osipenko { 991e34212c7SDmitry Osipenko struct device_node *np; 992e34212c7SDmitry Osipenko u32 value, ram_code; 993e34212c7SDmitry Osipenko int err; 994e34212c7SDmitry Osipenko 995e34212c7SDmitry Osipenko ram_code = tegra_read_ram_code(); 996e34212c7SDmitry Osipenko 997e34212c7SDmitry Osipenko for_each_child_of_node(dev->of_node, np) { 998e34212c7SDmitry Osipenko err = of_property_read_u32(np, "nvidia,ram-code", &value); 999e34212c7SDmitry Osipenko if (err || value != ram_code) 1000e34212c7SDmitry Osipenko continue; 1001e34212c7SDmitry Osipenko 1002e34212c7SDmitry Osipenko return np; 1003e34212c7SDmitry Osipenko } 1004e34212c7SDmitry Osipenko 1005e34212c7SDmitry Osipenko dev_err(dev, "no memory timings for RAM code %u found in device-tree\n", 1006e34212c7SDmitry Osipenko ram_code); 1007e34212c7SDmitry Osipenko 1008e34212c7SDmitry Osipenko return NULL; 1009e34212c7SDmitry Osipenko } 1010e34212c7SDmitry Osipenko 1011e34212c7SDmitry Osipenko static int emc_setup_hw(struct tegra_emc *emc) 1012e34212c7SDmitry Osipenko { 1013e34212c7SDmitry Osipenko u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT; 1014e34212c7SDmitry Osipenko u32 fbio_cfg5, emc_cfg, emc_dbg; 1015e34212c7SDmitry Osipenko enum emc_dram_type dram_type; 1016e34212c7SDmitry Osipenko 1017e34212c7SDmitry Osipenko fbio_cfg5 = readl_relaxed(emc->regs + EMC_FBIO_CFG5); 1018e34212c7SDmitry Osipenko dram_type = fbio_cfg5 & EMC_FBIO_CFG5_DRAM_TYPE_MASK; 1019e34212c7SDmitry Osipenko 1020e34212c7SDmitry Osipenko emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); 1021e34212c7SDmitry Osipenko 1022e34212c7SDmitry Osipenko /* enable EMC and CAR to handshake on PLL divider/source changes */ 1023e34212c7SDmitry Osipenko emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE; 1024e34212c7SDmitry Osipenko 1025e34212c7SDmitry Osipenko /* configure clock change mode accordingly to DRAM type */ 1026e34212c7SDmitry Osipenko switch (dram_type) { 1027e34212c7SDmitry Osipenko case DRAM_TYPE_LPDDR2: 1028e34212c7SDmitry Osipenko emc_cfg |= EMC_CLKCHANGE_PD_ENABLE; 1029e34212c7SDmitry Osipenko emc_cfg &= ~EMC_CLKCHANGE_SR_ENABLE; 1030e34212c7SDmitry Osipenko break; 1031e34212c7SDmitry Osipenko 1032e34212c7SDmitry Osipenko default: 1033e34212c7SDmitry Osipenko emc_cfg &= ~EMC_CLKCHANGE_SR_ENABLE; 1034e34212c7SDmitry Osipenko emc_cfg &= ~EMC_CLKCHANGE_PD_ENABLE; 1035e34212c7SDmitry Osipenko break; 1036e34212c7SDmitry Osipenko } 1037e34212c7SDmitry Osipenko 1038e34212c7SDmitry Osipenko writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2); 1039e34212c7SDmitry Osipenko 1040e34212c7SDmitry Osipenko /* initialize interrupt */ 1041e34212c7SDmitry Osipenko writel_relaxed(intmask, emc->regs + EMC_INTMASK); 1042e34212c7SDmitry Osipenko writel_relaxed(0xffffffff, emc->regs + EMC_INTSTATUS); 1043e34212c7SDmitry Osipenko 1044e34212c7SDmitry Osipenko /* ensure that unwanted debug features are disabled */ 1045e34212c7SDmitry Osipenko emc_dbg = readl_relaxed(emc->regs + EMC_DBG); 1046e34212c7SDmitry Osipenko emc_dbg |= EMC_DBG_CFG_PRIORITY; 1047e34212c7SDmitry Osipenko emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY; 1048e34212c7SDmitry Osipenko emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE; 1049e34212c7SDmitry Osipenko emc_dbg &= ~EMC_DBG_FORCE_UPDATE; 1050e34212c7SDmitry Osipenko writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 1051e34212c7SDmitry Osipenko 1052e34212c7SDmitry Osipenko return 0; 1053e34212c7SDmitry Osipenko } 1054e34212c7SDmitry Osipenko 1055e34212c7SDmitry Osipenko static long emc_round_rate(unsigned long rate, 1056e34212c7SDmitry Osipenko unsigned long min_rate, 1057e34212c7SDmitry Osipenko unsigned long max_rate, 1058e34212c7SDmitry Osipenko void *arg) 1059e34212c7SDmitry Osipenko { 1060e34212c7SDmitry Osipenko struct emc_timing *timing = NULL; 1061e34212c7SDmitry Osipenko struct tegra_emc *emc = arg; 1062e34212c7SDmitry Osipenko unsigned int i; 1063e34212c7SDmitry Osipenko 1064e34212c7SDmitry Osipenko min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); 1065e34212c7SDmitry Osipenko 1066e34212c7SDmitry Osipenko for (i = 0; i < emc->num_timings; i++) { 1067e34212c7SDmitry Osipenko if (emc->timings[i].rate < rate && i != emc->num_timings - 1) 1068e34212c7SDmitry Osipenko continue; 1069e34212c7SDmitry Osipenko 1070e34212c7SDmitry Osipenko if (emc->timings[i].rate > max_rate) { 1071e34212c7SDmitry Osipenko i = max(i, 1u) - 1; 1072e34212c7SDmitry Osipenko 1073e34212c7SDmitry Osipenko if (emc->timings[i].rate < min_rate) 1074e34212c7SDmitry Osipenko break; 1075e34212c7SDmitry Osipenko } 1076e34212c7SDmitry Osipenko 1077e34212c7SDmitry Osipenko if (emc->timings[i].rate < min_rate) 1078e34212c7SDmitry Osipenko continue; 1079e34212c7SDmitry Osipenko 1080e34212c7SDmitry Osipenko timing = &emc->timings[i]; 1081e34212c7SDmitry Osipenko break; 1082e34212c7SDmitry Osipenko } 1083e34212c7SDmitry Osipenko 1084e34212c7SDmitry Osipenko if (!timing) { 1085e34212c7SDmitry Osipenko dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n", 1086e34212c7SDmitry Osipenko rate, min_rate, max_rate); 1087e34212c7SDmitry Osipenko return -EINVAL; 1088e34212c7SDmitry Osipenko } 1089e34212c7SDmitry Osipenko 1090e34212c7SDmitry Osipenko return timing->rate; 1091e34212c7SDmitry Osipenko } 1092e34212c7SDmitry Osipenko 1093*8cee32b4SThierry Reding /* 1094*8cee32b4SThierry Reding * debugfs interface 1095*8cee32b4SThierry Reding * 1096*8cee32b4SThierry Reding * The memory controller driver exposes some files in debugfs that can be used 1097*8cee32b4SThierry Reding * to control the EMC frequency. The top-level directory can be found here: 1098*8cee32b4SThierry Reding * 1099*8cee32b4SThierry Reding * /sys/kernel/debug/emc 1100*8cee32b4SThierry Reding * 1101*8cee32b4SThierry Reding * It contains the following files: 1102*8cee32b4SThierry Reding * 1103*8cee32b4SThierry Reding * - available_rates: This file contains a list of valid, space-separated 1104*8cee32b4SThierry Reding * EMC frequencies. 1105*8cee32b4SThierry Reding * 1106*8cee32b4SThierry Reding * - min_rate: Writing a value to this file sets the given frequency as the 1107*8cee32b4SThierry Reding * floor of the permitted range. If this is higher than the currently 1108*8cee32b4SThierry Reding * configured EMC frequency, this will cause the frequency to be 1109*8cee32b4SThierry Reding * increased so that it stays within the valid range. 1110*8cee32b4SThierry Reding * 1111*8cee32b4SThierry Reding * - max_rate: Similarily to the min_rate file, writing a value to this file 1112*8cee32b4SThierry Reding * sets the given frequency as the ceiling of the permitted range. If 1113*8cee32b4SThierry Reding * the value is lower than the currently configured EMC frequency, this 1114*8cee32b4SThierry Reding * will cause the frequency to be decreased so that it stays within the 1115*8cee32b4SThierry Reding * valid range. 1116*8cee32b4SThierry Reding */ 1117*8cee32b4SThierry Reding 1118*8cee32b4SThierry Reding static bool tegra_emc_validate_rate(struct tegra_emc *emc, unsigned long rate) 1119*8cee32b4SThierry Reding { 1120*8cee32b4SThierry Reding unsigned int i; 1121*8cee32b4SThierry Reding 1122*8cee32b4SThierry Reding for (i = 0; i < emc->num_timings; i++) 1123*8cee32b4SThierry Reding if (rate == emc->timings[i].rate) 1124*8cee32b4SThierry Reding return true; 1125*8cee32b4SThierry Reding 1126*8cee32b4SThierry Reding return false; 1127*8cee32b4SThierry Reding } 1128*8cee32b4SThierry Reding 1129*8cee32b4SThierry Reding static int tegra_emc_debug_available_rates_show(struct seq_file *s, void *data) 1130*8cee32b4SThierry Reding { 1131*8cee32b4SThierry Reding struct tegra_emc *emc = s->private; 1132*8cee32b4SThierry Reding const char *prefix = ""; 1133*8cee32b4SThierry Reding unsigned int i; 1134*8cee32b4SThierry Reding 1135*8cee32b4SThierry Reding for (i = 0; i < emc->num_timings; i++) { 1136*8cee32b4SThierry Reding seq_printf(s, "%s%lu", prefix, emc->timings[i].rate); 1137*8cee32b4SThierry Reding prefix = " "; 1138*8cee32b4SThierry Reding } 1139*8cee32b4SThierry Reding 1140*8cee32b4SThierry Reding seq_puts(s, "\n"); 1141*8cee32b4SThierry Reding 1142*8cee32b4SThierry Reding return 0; 1143*8cee32b4SThierry Reding } 1144*8cee32b4SThierry Reding 1145*8cee32b4SThierry Reding static int tegra_emc_debug_available_rates_open(struct inode *inode, 1146*8cee32b4SThierry Reding struct file *file) 1147*8cee32b4SThierry Reding { 1148*8cee32b4SThierry Reding return single_open(file, tegra_emc_debug_available_rates_show, 1149*8cee32b4SThierry Reding inode->i_private); 1150*8cee32b4SThierry Reding } 1151*8cee32b4SThierry Reding 1152*8cee32b4SThierry Reding static const struct file_operations tegra_emc_debug_available_rates_fops = { 1153*8cee32b4SThierry Reding .open = tegra_emc_debug_available_rates_open, 1154*8cee32b4SThierry Reding .read = seq_read, 1155*8cee32b4SThierry Reding .llseek = seq_lseek, 1156*8cee32b4SThierry Reding .release = single_release, 1157*8cee32b4SThierry Reding }; 1158*8cee32b4SThierry Reding 1159*8cee32b4SThierry Reding static int tegra_emc_debug_min_rate_get(void *data, u64 *rate) 1160*8cee32b4SThierry Reding { 1161*8cee32b4SThierry Reding struct tegra_emc *emc = data; 1162*8cee32b4SThierry Reding 1163*8cee32b4SThierry Reding *rate = emc->debugfs.min_rate; 1164*8cee32b4SThierry Reding 1165*8cee32b4SThierry Reding return 0; 1166*8cee32b4SThierry Reding } 1167*8cee32b4SThierry Reding 1168*8cee32b4SThierry Reding static int tegra_emc_debug_min_rate_set(void *data, u64 rate) 1169*8cee32b4SThierry Reding { 1170*8cee32b4SThierry Reding struct tegra_emc *emc = data; 1171*8cee32b4SThierry Reding int err; 1172*8cee32b4SThierry Reding 1173*8cee32b4SThierry Reding if (!tegra_emc_validate_rate(emc, rate)) 1174*8cee32b4SThierry Reding return -EINVAL; 1175*8cee32b4SThierry Reding 1176*8cee32b4SThierry Reding err = clk_set_min_rate(emc->clk, rate); 1177*8cee32b4SThierry Reding if (err < 0) 1178*8cee32b4SThierry Reding return err; 1179*8cee32b4SThierry Reding 1180*8cee32b4SThierry Reding emc->debugfs.min_rate = rate; 1181*8cee32b4SThierry Reding 1182*8cee32b4SThierry Reding return 0; 1183*8cee32b4SThierry Reding } 1184*8cee32b4SThierry Reding 1185*8cee32b4SThierry Reding DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_min_rate_fops, 1186*8cee32b4SThierry Reding tegra_emc_debug_min_rate_get, 1187*8cee32b4SThierry Reding tegra_emc_debug_min_rate_set, "%llu\n"); 1188*8cee32b4SThierry Reding 1189*8cee32b4SThierry Reding static int tegra_emc_debug_max_rate_get(void *data, u64 *rate) 1190*8cee32b4SThierry Reding { 1191*8cee32b4SThierry Reding struct tegra_emc *emc = data; 1192*8cee32b4SThierry Reding 1193*8cee32b4SThierry Reding *rate = emc->debugfs.max_rate; 1194*8cee32b4SThierry Reding 1195*8cee32b4SThierry Reding return 0; 1196*8cee32b4SThierry Reding } 1197*8cee32b4SThierry Reding 1198*8cee32b4SThierry Reding static int tegra_emc_debug_max_rate_set(void *data, u64 rate) 1199*8cee32b4SThierry Reding { 1200*8cee32b4SThierry Reding struct tegra_emc *emc = data; 1201*8cee32b4SThierry Reding int err; 1202*8cee32b4SThierry Reding 1203*8cee32b4SThierry Reding if (!tegra_emc_validate_rate(emc, rate)) 1204*8cee32b4SThierry Reding return -EINVAL; 1205*8cee32b4SThierry Reding 1206*8cee32b4SThierry Reding err = clk_set_max_rate(emc->clk, rate); 1207*8cee32b4SThierry Reding if (err < 0) 1208*8cee32b4SThierry Reding return err; 1209*8cee32b4SThierry Reding 1210*8cee32b4SThierry Reding emc->debugfs.max_rate = rate; 1211*8cee32b4SThierry Reding 1212*8cee32b4SThierry Reding return 0; 1213*8cee32b4SThierry Reding } 1214*8cee32b4SThierry Reding 1215*8cee32b4SThierry Reding DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_max_rate_fops, 1216*8cee32b4SThierry Reding tegra_emc_debug_max_rate_get, 1217*8cee32b4SThierry Reding tegra_emc_debug_max_rate_set, "%llu\n"); 1218*8cee32b4SThierry Reding 1219*8cee32b4SThierry Reding static void tegra_emc_debugfs_init(struct tegra_emc *emc) 1220*8cee32b4SThierry Reding { 1221*8cee32b4SThierry Reding struct device *dev = emc->dev; 1222*8cee32b4SThierry Reding unsigned int i; 1223*8cee32b4SThierry Reding int err; 1224*8cee32b4SThierry Reding 1225*8cee32b4SThierry Reding emc->debugfs.min_rate = ULONG_MAX; 1226*8cee32b4SThierry Reding emc->debugfs.max_rate = 0; 1227*8cee32b4SThierry Reding 1228*8cee32b4SThierry Reding for (i = 0; i < emc->num_timings; i++) { 1229*8cee32b4SThierry Reding if (emc->timings[i].rate < emc->debugfs.min_rate) 1230*8cee32b4SThierry Reding emc->debugfs.min_rate = emc->timings[i].rate; 1231*8cee32b4SThierry Reding 1232*8cee32b4SThierry Reding if (emc->timings[i].rate > emc->debugfs.max_rate) 1233*8cee32b4SThierry Reding emc->debugfs.max_rate = emc->timings[i].rate; 1234*8cee32b4SThierry Reding } 1235*8cee32b4SThierry Reding 1236*8cee32b4SThierry Reding err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, 1237*8cee32b4SThierry Reding emc->debugfs.max_rate); 1238*8cee32b4SThierry Reding if (err < 0) { 1239*8cee32b4SThierry Reding dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n", 1240*8cee32b4SThierry Reding emc->debugfs.min_rate, emc->debugfs.max_rate, 1241*8cee32b4SThierry Reding emc->clk); 1242*8cee32b4SThierry Reding } 1243*8cee32b4SThierry Reding 1244*8cee32b4SThierry Reding emc->debugfs.root = debugfs_create_dir("emc", NULL); 1245*8cee32b4SThierry Reding if (!emc->debugfs.root) { 1246*8cee32b4SThierry Reding dev_err(emc->dev, "failed to create debugfs directory\n"); 1247*8cee32b4SThierry Reding return; 1248*8cee32b4SThierry Reding } 1249*8cee32b4SThierry Reding 1250*8cee32b4SThierry Reding debugfs_create_file("available_rates", S_IRUGO, emc->debugfs.root, 1251*8cee32b4SThierry Reding emc, &tegra_emc_debug_available_rates_fops); 1252*8cee32b4SThierry Reding debugfs_create_file("min_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, 1253*8cee32b4SThierry Reding emc, &tegra_emc_debug_min_rate_fops); 1254*8cee32b4SThierry Reding debugfs_create_file("max_rate", S_IRUGO | S_IWUSR, emc->debugfs.root, 1255*8cee32b4SThierry Reding emc, &tegra_emc_debug_max_rate_fops); 1256*8cee32b4SThierry Reding } 1257*8cee32b4SThierry Reding 1258e34212c7SDmitry Osipenko static int tegra_emc_probe(struct platform_device *pdev) 1259e34212c7SDmitry Osipenko { 1260e34212c7SDmitry Osipenko struct platform_device *mc; 1261e34212c7SDmitry Osipenko struct device_node *np; 1262e34212c7SDmitry Osipenko struct tegra_emc *emc; 1263e34212c7SDmitry Osipenko int err; 1264e34212c7SDmitry Osipenko 1265e34212c7SDmitry Osipenko if (of_get_child_count(pdev->dev.of_node) == 0) { 1266e34212c7SDmitry Osipenko dev_info(&pdev->dev, 1267e34212c7SDmitry Osipenko "device-tree node doesn't have memory timings\n"); 1268030d2829SDmitry Osipenko return -ENODEV; 1269e34212c7SDmitry Osipenko } 1270e34212c7SDmitry Osipenko 1271e34212c7SDmitry Osipenko np = of_parse_phandle(pdev->dev.of_node, "nvidia,memory-controller", 0); 1272e34212c7SDmitry Osipenko if (!np) { 1273e34212c7SDmitry Osipenko dev_err(&pdev->dev, "could not get memory controller node\n"); 1274e34212c7SDmitry Osipenko return -ENOENT; 1275e34212c7SDmitry Osipenko } 1276e34212c7SDmitry Osipenko 1277e34212c7SDmitry Osipenko mc = of_find_device_by_node(np); 1278e34212c7SDmitry Osipenko of_node_put(np); 1279e34212c7SDmitry Osipenko if (!mc) 1280e34212c7SDmitry Osipenko return -ENOENT; 1281e34212c7SDmitry Osipenko 1282e34212c7SDmitry Osipenko np = emc_find_node_by_ram_code(&pdev->dev); 1283e34212c7SDmitry Osipenko if (!np) 1284e34212c7SDmitry Osipenko return -EINVAL; 1285e34212c7SDmitry Osipenko 1286e34212c7SDmitry Osipenko emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 1287e34212c7SDmitry Osipenko if (!emc) { 1288e34212c7SDmitry Osipenko of_node_put(np); 1289e34212c7SDmitry Osipenko return -ENOMEM; 1290e34212c7SDmitry Osipenko } 1291e34212c7SDmitry Osipenko 1292e34212c7SDmitry Osipenko emc->mc = platform_get_drvdata(mc); 1293e34212c7SDmitry Osipenko if (!emc->mc) 1294e34212c7SDmitry Osipenko return -EPROBE_DEFER; 1295e34212c7SDmitry Osipenko 1296e34212c7SDmitry Osipenko init_completion(&emc->clk_handshake_complete); 1297e34212c7SDmitry Osipenko emc->clk_nb.notifier_call = emc_clk_change_notify; 1298e34212c7SDmitry Osipenko emc->dev = &pdev->dev; 1299e34212c7SDmitry Osipenko 1300e34212c7SDmitry Osipenko err = emc_load_timings_from_dt(emc, np); 1301e34212c7SDmitry Osipenko of_node_put(np); 1302e34212c7SDmitry Osipenko if (err) 1303e34212c7SDmitry Osipenko return err; 1304e34212c7SDmitry Osipenko 1305e34212c7SDmitry Osipenko emc->regs = devm_platform_ioremap_resource(pdev, 0); 1306e34212c7SDmitry Osipenko if (IS_ERR(emc->regs)) 1307e34212c7SDmitry Osipenko return PTR_ERR(emc->regs); 1308e34212c7SDmitry Osipenko 1309e34212c7SDmitry Osipenko err = emc_setup_hw(emc); 1310e34212c7SDmitry Osipenko if (err) 1311e34212c7SDmitry Osipenko return err; 1312e34212c7SDmitry Osipenko 1313e34212c7SDmitry Osipenko err = platform_get_irq(pdev, 0); 1314e34212c7SDmitry Osipenko if (err < 0) { 1315e34212c7SDmitry Osipenko dev_err(&pdev->dev, "interrupt not specified: %d\n", err); 1316e34212c7SDmitry Osipenko return err; 1317e34212c7SDmitry Osipenko } 1318e34212c7SDmitry Osipenko emc->irq = err; 1319e34212c7SDmitry Osipenko 1320e34212c7SDmitry Osipenko err = devm_request_irq(&pdev->dev, emc->irq, tegra_emc_isr, 0, 1321e34212c7SDmitry Osipenko dev_name(&pdev->dev), emc); 1322e34212c7SDmitry Osipenko if (err) { 1323e34212c7SDmitry Osipenko dev_err(&pdev->dev, "failed to request irq: %d\n", err); 1324e34212c7SDmitry Osipenko return err; 1325e34212c7SDmitry Osipenko } 1326e34212c7SDmitry Osipenko 1327e34212c7SDmitry Osipenko tegra20_clk_set_emc_round_callback(emc_round_rate, emc); 1328e34212c7SDmitry Osipenko 1329e34212c7SDmitry Osipenko emc->clk = devm_clk_get(&pdev->dev, "emc"); 1330e34212c7SDmitry Osipenko if (IS_ERR(emc->clk)) { 1331e34212c7SDmitry Osipenko err = PTR_ERR(emc->clk); 1332e34212c7SDmitry Osipenko dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); 1333e34212c7SDmitry Osipenko goto unset_cb; 1334e34212c7SDmitry Osipenko } 1335e34212c7SDmitry Osipenko 1336e34212c7SDmitry Osipenko err = clk_notifier_register(emc->clk, &emc->clk_nb); 1337e34212c7SDmitry Osipenko if (err) { 1338e34212c7SDmitry Osipenko dev_err(&pdev->dev, "failed to register clk notifier: %d\n", 1339e34212c7SDmitry Osipenko err); 1340e34212c7SDmitry Osipenko goto unset_cb; 1341e34212c7SDmitry Osipenko } 1342e34212c7SDmitry Osipenko 1343e34212c7SDmitry Osipenko platform_set_drvdata(pdev, emc); 1344*8cee32b4SThierry Reding tegra_emc_debugfs_init(emc); 1345e34212c7SDmitry Osipenko 1346e34212c7SDmitry Osipenko return 0; 1347e34212c7SDmitry Osipenko 1348e34212c7SDmitry Osipenko unset_cb: 1349e34212c7SDmitry Osipenko tegra20_clk_set_emc_round_callback(NULL, NULL); 1350e34212c7SDmitry Osipenko 1351e34212c7SDmitry Osipenko return err; 1352e34212c7SDmitry Osipenko } 1353e34212c7SDmitry Osipenko 1354e34212c7SDmitry Osipenko static int tegra_emc_suspend(struct device *dev) 1355e34212c7SDmitry Osipenko { 1356e34212c7SDmitry Osipenko struct tegra_emc *emc = dev_get_drvdata(dev); 1357e34212c7SDmitry Osipenko 1358e34212c7SDmitry Osipenko /* 1359e34212c7SDmitry Osipenko * Suspending in a bad state will hang machine. The "prepared" var 1360e34212c7SDmitry Osipenko * shall be always false here unless it's a kernel bug that caused 1361e34212c7SDmitry Osipenko * suspending in a wrong order. 1362e34212c7SDmitry Osipenko */ 1363e34212c7SDmitry Osipenko if (WARN_ON(emc->prepared) || emc->bad_state) 1364e34212c7SDmitry Osipenko return -EINVAL; 1365e34212c7SDmitry Osipenko 1366e34212c7SDmitry Osipenko emc->bad_state = true; 1367e34212c7SDmitry Osipenko 1368e34212c7SDmitry Osipenko return 0; 1369e34212c7SDmitry Osipenko } 1370e34212c7SDmitry Osipenko 1371e34212c7SDmitry Osipenko static int tegra_emc_resume(struct device *dev) 1372e34212c7SDmitry Osipenko { 1373e34212c7SDmitry Osipenko struct tegra_emc *emc = dev_get_drvdata(dev); 1374e34212c7SDmitry Osipenko 1375e34212c7SDmitry Osipenko emc_setup_hw(emc); 1376e34212c7SDmitry Osipenko emc->bad_state = false; 1377e34212c7SDmitry Osipenko 1378e34212c7SDmitry Osipenko return 0; 1379e34212c7SDmitry Osipenko } 1380e34212c7SDmitry Osipenko 1381e34212c7SDmitry Osipenko static const struct dev_pm_ops tegra_emc_pm_ops = { 1382e34212c7SDmitry Osipenko .suspend = tegra_emc_suspend, 1383e34212c7SDmitry Osipenko .resume = tegra_emc_resume, 1384e34212c7SDmitry Osipenko }; 1385e34212c7SDmitry Osipenko 1386e34212c7SDmitry Osipenko static const struct of_device_id tegra_emc_of_match[] = { 1387e34212c7SDmitry Osipenko { .compatible = "nvidia,tegra30-emc", }, 1388e34212c7SDmitry Osipenko {}, 1389e34212c7SDmitry Osipenko }; 1390e34212c7SDmitry Osipenko 1391e34212c7SDmitry Osipenko static struct platform_driver tegra_emc_driver = { 1392e34212c7SDmitry Osipenko .probe = tegra_emc_probe, 1393e34212c7SDmitry Osipenko .driver = { 1394e34212c7SDmitry Osipenko .name = "tegra30-emc", 1395e34212c7SDmitry Osipenko .of_match_table = tegra_emc_of_match, 1396e34212c7SDmitry Osipenko .pm = &tegra_emc_pm_ops, 1397e34212c7SDmitry Osipenko .suppress_bind_attrs = true, 1398e34212c7SDmitry Osipenko }, 1399e34212c7SDmitry Osipenko }; 1400e34212c7SDmitry Osipenko 1401e34212c7SDmitry Osipenko static int __init tegra_emc_init(void) 1402e34212c7SDmitry Osipenko { 1403e34212c7SDmitry Osipenko return platform_driver_register(&tegra_emc_driver); 1404e34212c7SDmitry Osipenko } 1405e34212c7SDmitry Osipenko subsys_initcall(tegra_emc_init); 1406