196e5da7cSDmitry Osipenko // SPDX-License-Identifier: GPL-2.0 296e5da7cSDmitry Osipenko /* 396e5da7cSDmitry Osipenko * Tegra20 External Memory Controller driver 496e5da7cSDmitry Osipenko * 596e5da7cSDmitry Osipenko * Author: Dmitry Osipenko <digetx@gmail.com> 696e5da7cSDmitry Osipenko */ 796e5da7cSDmitry Osipenko 8131dd9a4SDmitry Osipenko #include <linux/bitfield.h> 996e5da7cSDmitry Osipenko #include <linux/clk.h> 1077ab499dSDmitry Osipenko #include <linux/clk/tegra.h> 118209eefaSThierry Reding #include <linux/debugfs.h> 12dedf62d6SDmitry Osipenko #include <linux/devfreq.h> 1396e5da7cSDmitry Osipenko #include <linux/err.h> 14d5ef16baSDmitry Osipenko #include <linux/interconnect-provider.h> 1596e5da7cSDmitry Osipenko #include <linux/interrupt.h> 16d039cf28SDmitry Osipenko #include <linux/io.h> 17adbcec88SDmitry Osipenko #include <linux/iopoll.h> 1896e5da7cSDmitry Osipenko #include <linux/kernel.h> 1996e5da7cSDmitry Osipenko #include <linux/module.h> 20d5ef16baSDmitry Osipenko #include <linux/mutex.h> 2196e5da7cSDmitry Osipenko #include <linux/of.h> 2296e5da7cSDmitry Osipenko #include <linux/platform_device.h> 23d5ef16baSDmitry Osipenko #include <linux/pm_opp.h> 24d5ef16baSDmitry Osipenko #include <linux/slab.h> 2596e5da7cSDmitry Osipenko #include <linux/sort.h> 2696e5da7cSDmitry Osipenko #include <linux/types.h> 2796e5da7cSDmitry Osipenko 28d5ef16baSDmitry Osipenko #include <soc/tegra/common.h> 2996e5da7cSDmitry Osipenko #include <soc/tegra/fuse.h> 3096e5da7cSDmitry Osipenko 31131dd9a4SDmitry Osipenko #include "../jedec_ddr.h" 32131dd9a4SDmitry Osipenko #include "../of_memory.h" 33131dd9a4SDmitry Osipenko 34d5ef16baSDmitry Osipenko #include "mc.h" 35d5ef16baSDmitry Osipenko 3696e5da7cSDmitry Osipenko #define EMC_INTSTATUS 0x000 3796e5da7cSDmitry Osipenko #define EMC_INTMASK 0x004 38c72396f9SDmitry Osipenko #define EMC_DBG 0x008 39131dd9a4SDmitry Osipenko #define EMC_ADR_CFG_0 0x010 4096e5da7cSDmitry Osipenko #define EMC_TIMING_CONTROL 0x028 4196e5da7cSDmitry Osipenko #define EMC_RC 0x02c 4296e5da7cSDmitry Osipenko #define EMC_RFC 0x030 4396e5da7cSDmitry Osipenko #define EMC_RAS 0x034 4496e5da7cSDmitry Osipenko #define EMC_RP 0x038 4596e5da7cSDmitry Osipenko #define EMC_R2W 0x03c 4696e5da7cSDmitry Osipenko #define EMC_W2R 0x040 4796e5da7cSDmitry Osipenko #define EMC_R2P 0x044 4896e5da7cSDmitry Osipenko #define EMC_W2P 0x048 4996e5da7cSDmitry Osipenko #define EMC_RD_RCD 0x04c 5096e5da7cSDmitry Osipenko #define EMC_WR_RCD 0x050 5196e5da7cSDmitry Osipenko #define EMC_RRD 0x054 5296e5da7cSDmitry Osipenko #define EMC_REXT 0x058 5396e5da7cSDmitry Osipenko #define EMC_WDV 0x05c 5496e5da7cSDmitry Osipenko #define EMC_QUSE 0x060 5596e5da7cSDmitry Osipenko #define EMC_QRST 0x064 5696e5da7cSDmitry Osipenko #define EMC_QSAFE 0x068 5796e5da7cSDmitry Osipenko #define EMC_RDV 0x06c 5896e5da7cSDmitry Osipenko #define EMC_REFRESH 0x070 5996e5da7cSDmitry Osipenko #define EMC_BURST_REFRESH_NUM 0x074 6096e5da7cSDmitry Osipenko #define EMC_PDEX2WR 0x078 6196e5da7cSDmitry Osipenko #define EMC_PDEX2RD 0x07c 6296e5da7cSDmitry Osipenko #define EMC_PCHG2PDEN 0x080 6396e5da7cSDmitry Osipenko #define EMC_ACT2PDEN 0x084 6496e5da7cSDmitry Osipenko #define EMC_AR2PDEN 0x088 6596e5da7cSDmitry Osipenko #define EMC_RW2PDEN 0x08c 6696e5da7cSDmitry Osipenko #define EMC_TXSR 0x090 6796e5da7cSDmitry Osipenko #define EMC_TCKE 0x094 6896e5da7cSDmitry Osipenko #define EMC_TFAW 0x098 6996e5da7cSDmitry Osipenko #define EMC_TRPAB 0x09c 7096e5da7cSDmitry Osipenko #define EMC_TCLKSTABLE 0x0a0 7196e5da7cSDmitry Osipenko #define EMC_TCLKSTOP 0x0a4 7296e5da7cSDmitry Osipenko #define EMC_TREFBW 0x0a8 7396e5da7cSDmitry Osipenko #define EMC_QUSE_EXTRA 0x0ac 7496e5da7cSDmitry Osipenko #define EMC_ODT_WRITE 0x0b0 7596e5da7cSDmitry Osipenko #define EMC_ODT_READ 0x0b4 76131dd9a4SDmitry Osipenko #define EMC_MRR 0x0ec 7796e5da7cSDmitry Osipenko #define EMC_FBIO_CFG5 0x104 7896e5da7cSDmitry Osipenko #define EMC_FBIO_CFG6 0x114 79d5ef16baSDmitry Osipenko #define EMC_STAT_CONTROL 0x160 80d5ef16baSDmitry Osipenko #define EMC_STAT_LLMC_CONTROL 0x178 81d5ef16baSDmitry Osipenko #define EMC_STAT_PWR_CLOCK_LIMIT 0x198 82d5ef16baSDmitry Osipenko #define EMC_STAT_PWR_CLOCKS 0x19c 83d5ef16baSDmitry Osipenko #define EMC_STAT_PWR_COUNT 0x1a0 8496e5da7cSDmitry Osipenko #define EMC_AUTO_CAL_INTERVAL 0x2a8 8596e5da7cSDmitry Osipenko #define EMC_CFG_2 0x2b8 8696e5da7cSDmitry Osipenko #define EMC_CFG_DIG_DLL 0x2bc 8796e5da7cSDmitry Osipenko #define EMC_DLL_XFORM_DQS 0x2c0 8896e5da7cSDmitry Osipenko #define EMC_DLL_XFORM_QUSE 0x2c4 8996e5da7cSDmitry Osipenko #define EMC_ZCAL_REF_CNT 0x2e0 9096e5da7cSDmitry Osipenko #define EMC_ZCAL_WAIT_CNT 0x2e4 9196e5da7cSDmitry Osipenko #define EMC_CFG_CLKTRIM_0 0x2d0 9296e5da7cSDmitry Osipenko #define EMC_CFG_CLKTRIM_1 0x2d4 9396e5da7cSDmitry Osipenko #define EMC_CFG_CLKTRIM_2 0x2d8 9496e5da7cSDmitry Osipenko 9596e5da7cSDmitry Osipenko #define EMC_CLKCHANGE_REQ_ENABLE BIT(0) 9696e5da7cSDmitry Osipenko #define EMC_CLKCHANGE_PD_ENABLE BIT(1) 9796e5da7cSDmitry Osipenko #define EMC_CLKCHANGE_SR_ENABLE BIT(2) 9896e5da7cSDmitry Osipenko 9996e5da7cSDmitry Osipenko #define EMC_TIMING_UPDATE BIT(0) 10096e5da7cSDmitry Osipenko 10196e5da7cSDmitry Osipenko #define EMC_REFRESH_OVERFLOW_INT BIT(3) 10296e5da7cSDmitry Osipenko #define EMC_CLKCHANGE_COMPLETE_INT BIT(4) 103131dd9a4SDmitry Osipenko #define EMC_MRR_DIVLD_INT BIT(5) 10496e5da7cSDmitry Osipenko 105c72396f9SDmitry Osipenko #define EMC_DBG_READ_MUX_ASSEMBLY BIT(0) 106c72396f9SDmitry Osipenko #define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) 107c72396f9SDmitry Osipenko #define EMC_DBG_FORCE_UPDATE BIT(2) 108c72396f9SDmitry Osipenko #define EMC_DBG_READ_DQM_CTRL BIT(9) 109c72396f9SDmitry Osipenko #define EMC_DBG_CFG_PRIORITY BIT(24) 110c72396f9SDmitry Osipenko 111d5ef16baSDmitry Osipenko #define EMC_FBIO_CFG5_DRAM_WIDTH_X16 BIT(4) 112131dd9a4SDmitry Osipenko #define EMC_FBIO_CFG5_DRAM_TYPE GENMASK(1, 0) 113131dd9a4SDmitry Osipenko 114131dd9a4SDmitry Osipenko #define EMC_MRR_DEV_SELECTN GENMASK(31, 30) 115131dd9a4SDmitry Osipenko #define EMC_MRR_MRR_MA GENMASK(23, 16) 116131dd9a4SDmitry Osipenko #define EMC_MRR_MRR_DATA GENMASK(15, 0) 117131dd9a4SDmitry Osipenko 118131dd9a4SDmitry Osipenko #define EMC_ADR_CFG_0_EMEM_NUMDEV GENMASK(25, 24) 119d5ef16baSDmitry Osipenko 120dedf62d6SDmitry Osipenko #define EMC_PWR_GATHER_CLEAR (1 << 8) 121dedf62d6SDmitry Osipenko #define EMC_PWR_GATHER_DISABLE (2 << 8) 122dedf62d6SDmitry Osipenko #define EMC_PWR_GATHER_ENABLE (3 << 8) 123dedf62d6SDmitry Osipenko 124131dd9a4SDmitry Osipenko enum emc_dram_type { 125131dd9a4SDmitry Osipenko DRAM_TYPE_RESERVED, 126131dd9a4SDmitry Osipenko DRAM_TYPE_DDR1, 127131dd9a4SDmitry Osipenko DRAM_TYPE_LPDDR2, 128131dd9a4SDmitry Osipenko DRAM_TYPE_DDR2, 129131dd9a4SDmitry Osipenko }; 130131dd9a4SDmitry Osipenko 13196e5da7cSDmitry Osipenko static const u16 emc_timing_registers[] = { 13296e5da7cSDmitry Osipenko EMC_RC, 13396e5da7cSDmitry Osipenko EMC_RFC, 13496e5da7cSDmitry Osipenko EMC_RAS, 13596e5da7cSDmitry Osipenko EMC_RP, 13696e5da7cSDmitry Osipenko EMC_R2W, 13796e5da7cSDmitry Osipenko EMC_W2R, 13896e5da7cSDmitry Osipenko EMC_R2P, 13996e5da7cSDmitry Osipenko EMC_W2P, 14096e5da7cSDmitry Osipenko EMC_RD_RCD, 14196e5da7cSDmitry Osipenko EMC_WR_RCD, 14296e5da7cSDmitry Osipenko EMC_RRD, 14396e5da7cSDmitry Osipenko EMC_REXT, 14496e5da7cSDmitry Osipenko EMC_WDV, 14596e5da7cSDmitry Osipenko EMC_QUSE, 14696e5da7cSDmitry Osipenko EMC_QRST, 14796e5da7cSDmitry Osipenko EMC_QSAFE, 14896e5da7cSDmitry Osipenko EMC_RDV, 14996e5da7cSDmitry Osipenko EMC_REFRESH, 15096e5da7cSDmitry Osipenko EMC_BURST_REFRESH_NUM, 15196e5da7cSDmitry Osipenko EMC_PDEX2WR, 15296e5da7cSDmitry Osipenko EMC_PDEX2RD, 15396e5da7cSDmitry Osipenko EMC_PCHG2PDEN, 15496e5da7cSDmitry Osipenko EMC_ACT2PDEN, 15596e5da7cSDmitry Osipenko EMC_AR2PDEN, 15696e5da7cSDmitry Osipenko EMC_RW2PDEN, 15796e5da7cSDmitry Osipenko EMC_TXSR, 15896e5da7cSDmitry Osipenko EMC_TCKE, 15996e5da7cSDmitry Osipenko EMC_TFAW, 16096e5da7cSDmitry Osipenko EMC_TRPAB, 16196e5da7cSDmitry Osipenko EMC_TCLKSTABLE, 16296e5da7cSDmitry Osipenko EMC_TCLKSTOP, 16396e5da7cSDmitry Osipenko EMC_TREFBW, 16496e5da7cSDmitry Osipenko EMC_QUSE_EXTRA, 16596e5da7cSDmitry Osipenko EMC_FBIO_CFG6, 16696e5da7cSDmitry Osipenko EMC_ODT_WRITE, 16796e5da7cSDmitry Osipenko EMC_ODT_READ, 16896e5da7cSDmitry Osipenko EMC_FBIO_CFG5, 16996e5da7cSDmitry Osipenko EMC_CFG_DIG_DLL, 17096e5da7cSDmitry Osipenko EMC_DLL_XFORM_DQS, 17196e5da7cSDmitry Osipenko EMC_DLL_XFORM_QUSE, 17296e5da7cSDmitry Osipenko EMC_ZCAL_REF_CNT, 17396e5da7cSDmitry Osipenko EMC_ZCAL_WAIT_CNT, 17496e5da7cSDmitry Osipenko EMC_AUTO_CAL_INTERVAL, 17596e5da7cSDmitry Osipenko EMC_CFG_CLKTRIM_0, 17696e5da7cSDmitry Osipenko EMC_CFG_CLKTRIM_1, 17796e5da7cSDmitry Osipenko EMC_CFG_CLKTRIM_2, 17896e5da7cSDmitry Osipenko }; 17996e5da7cSDmitry Osipenko 18096e5da7cSDmitry Osipenko struct emc_timing { 18196e5da7cSDmitry Osipenko unsigned long rate; 18296e5da7cSDmitry Osipenko u32 data[ARRAY_SIZE(emc_timing_registers)]; 18396e5da7cSDmitry Osipenko }; 18496e5da7cSDmitry Osipenko 185d5ef16baSDmitry Osipenko enum emc_rate_request_type { 186dedf62d6SDmitry Osipenko EMC_RATE_DEVFREQ, 187d5ef16baSDmitry Osipenko EMC_RATE_DEBUG, 188d5ef16baSDmitry Osipenko EMC_RATE_ICC, 189d5ef16baSDmitry Osipenko EMC_RATE_TYPE_MAX, 190d5ef16baSDmitry Osipenko }; 191d5ef16baSDmitry Osipenko 192d5ef16baSDmitry Osipenko struct emc_rate_request { 193d5ef16baSDmitry Osipenko unsigned long min_rate; 194d5ef16baSDmitry Osipenko unsigned long max_rate; 195d5ef16baSDmitry Osipenko }; 196d5ef16baSDmitry Osipenko 19796e5da7cSDmitry Osipenko struct tegra_emc { 19896e5da7cSDmitry Osipenko struct device *dev; 199d5ef16baSDmitry Osipenko struct tegra_mc *mc; 200d5ef16baSDmitry Osipenko struct icc_provider provider; 20196e5da7cSDmitry Osipenko struct notifier_block clk_nb; 20296e5da7cSDmitry Osipenko struct clk *clk; 20396e5da7cSDmitry Osipenko void __iomem *regs; 204d5ef16baSDmitry Osipenko unsigned int dram_bus_width; 20596e5da7cSDmitry Osipenko 20696e5da7cSDmitry Osipenko struct emc_timing *timings; 20796e5da7cSDmitry Osipenko unsigned int num_timings; 2088209eefaSThierry Reding 2098209eefaSThierry Reding struct { 2108209eefaSThierry Reding struct dentry *root; 2118209eefaSThierry Reding unsigned long min_rate; 2128209eefaSThierry Reding unsigned long max_rate; 2138209eefaSThierry Reding } debugfs; 214d5ef16baSDmitry Osipenko 215d5ef16baSDmitry Osipenko /* 216d5ef16baSDmitry Osipenko * There are multiple sources in the EMC driver which could request 217d5ef16baSDmitry Osipenko * a min/max clock rate, these rates are contained in this array. 218d5ef16baSDmitry Osipenko */ 219d5ef16baSDmitry Osipenko struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX]; 220d5ef16baSDmitry Osipenko 221d5ef16baSDmitry Osipenko /* protect shared rate-change code path */ 222d5ef16baSDmitry Osipenko struct mutex rate_lock; 223dedf62d6SDmitry Osipenko 224dedf62d6SDmitry Osipenko struct devfreq_simple_ondemand_data ondemand_data; 225131dd9a4SDmitry Osipenko 226131dd9a4SDmitry Osipenko /* memory chip identity information */ 227131dd9a4SDmitry Osipenko union lpddr2_basic_config4 basic_conf4; 228131dd9a4SDmitry Osipenko unsigned int manufacturer_id; 229131dd9a4SDmitry Osipenko unsigned int revision_id1; 230131dd9a4SDmitry Osipenko unsigned int revision_id2; 231131dd9a4SDmitry Osipenko 232131dd9a4SDmitry Osipenko bool mrr_error; 23396e5da7cSDmitry Osipenko }; 23496e5da7cSDmitry Osipenko 23596e5da7cSDmitry Osipenko static irqreturn_t tegra_emc_isr(int irq, void *data) 23696e5da7cSDmitry Osipenko { 23796e5da7cSDmitry Osipenko struct tegra_emc *emc = data; 238adbcec88SDmitry Osipenko u32 intmask = EMC_REFRESH_OVERFLOW_INT; 23996e5da7cSDmitry Osipenko u32 status; 24096e5da7cSDmitry Osipenko 24196e5da7cSDmitry Osipenko status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask; 24296e5da7cSDmitry Osipenko if (!status) 24396e5da7cSDmitry Osipenko return IRQ_NONE; 24496e5da7cSDmitry Osipenko 24596e5da7cSDmitry Osipenko /* notify about HW problem */ 24696e5da7cSDmitry Osipenko if (status & EMC_REFRESH_OVERFLOW_INT) 24796e5da7cSDmitry Osipenko dev_err_ratelimited(emc->dev, 24896e5da7cSDmitry Osipenko "refresh request overflow timeout\n"); 24996e5da7cSDmitry Osipenko 25096e5da7cSDmitry Osipenko /* clear interrupts */ 25196e5da7cSDmitry Osipenko writel_relaxed(status, emc->regs + EMC_INTSTATUS); 25296e5da7cSDmitry Osipenko 25396e5da7cSDmitry Osipenko return IRQ_HANDLED; 25496e5da7cSDmitry Osipenko } 25596e5da7cSDmitry Osipenko 25696e5da7cSDmitry Osipenko static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc, 25796e5da7cSDmitry Osipenko unsigned long rate) 25896e5da7cSDmitry Osipenko { 25996e5da7cSDmitry Osipenko struct emc_timing *timing = NULL; 26096e5da7cSDmitry Osipenko unsigned int i; 26196e5da7cSDmitry Osipenko 26296e5da7cSDmitry Osipenko for (i = 0; i < emc->num_timings; i++) { 26396e5da7cSDmitry Osipenko if (emc->timings[i].rate >= rate) { 26496e5da7cSDmitry Osipenko timing = &emc->timings[i]; 26596e5da7cSDmitry Osipenko break; 26696e5da7cSDmitry Osipenko } 26796e5da7cSDmitry Osipenko } 26896e5da7cSDmitry Osipenko 26996e5da7cSDmitry Osipenko if (!timing) { 27096e5da7cSDmitry Osipenko dev_err(emc->dev, "no timing for rate %lu\n", rate); 27196e5da7cSDmitry Osipenko return NULL; 27296e5da7cSDmitry Osipenko } 27396e5da7cSDmitry Osipenko 27496e5da7cSDmitry Osipenko return timing; 27596e5da7cSDmitry Osipenko } 27696e5da7cSDmitry Osipenko 27796e5da7cSDmitry Osipenko static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) 27896e5da7cSDmitry Osipenko { 27996e5da7cSDmitry Osipenko struct emc_timing *timing = tegra_emc_find_timing(emc, rate); 28096e5da7cSDmitry Osipenko unsigned int i; 28196e5da7cSDmitry Osipenko 28296e5da7cSDmitry Osipenko if (!timing) 28396e5da7cSDmitry Osipenko return -EINVAL; 28496e5da7cSDmitry Osipenko 28596e5da7cSDmitry Osipenko dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n", 28696e5da7cSDmitry Osipenko __func__, timing->rate, rate); 28796e5da7cSDmitry Osipenko 28896e5da7cSDmitry Osipenko /* program shadow registers */ 28996e5da7cSDmitry Osipenko for (i = 0; i < ARRAY_SIZE(timing->data); i++) 29096e5da7cSDmitry Osipenko writel_relaxed(timing->data[i], 29196e5da7cSDmitry Osipenko emc->regs + emc_timing_registers[i]); 29296e5da7cSDmitry Osipenko 29396e5da7cSDmitry Osipenko /* wait until programming has settled */ 29496e5da7cSDmitry Osipenko readl_relaxed(emc->regs + emc_timing_registers[i - 1]); 29596e5da7cSDmitry Osipenko 29696e5da7cSDmitry Osipenko return 0; 29796e5da7cSDmitry Osipenko } 29896e5da7cSDmitry Osipenko 29996e5da7cSDmitry Osipenko static int emc_complete_timing_change(struct tegra_emc *emc, bool flush) 30096e5da7cSDmitry Osipenko { 301adbcec88SDmitry Osipenko int err; 302adbcec88SDmitry Osipenko u32 v; 30396e5da7cSDmitry Osipenko 30496e5da7cSDmitry Osipenko dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush); 30596e5da7cSDmitry Osipenko 30696e5da7cSDmitry Osipenko if (flush) { 30796e5da7cSDmitry Osipenko /* manually initiate memory timing update */ 30896e5da7cSDmitry Osipenko writel_relaxed(EMC_TIMING_UPDATE, 30996e5da7cSDmitry Osipenko emc->regs + EMC_TIMING_CONTROL); 31096e5da7cSDmitry Osipenko return 0; 31196e5da7cSDmitry Osipenko } 31296e5da7cSDmitry Osipenko 313adbcec88SDmitry Osipenko err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, v, 314adbcec88SDmitry Osipenko v & EMC_CLKCHANGE_COMPLETE_INT, 315adbcec88SDmitry Osipenko 1, 100); 316adbcec88SDmitry Osipenko if (err) { 317adbcec88SDmitry Osipenko dev_err(emc->dev, "emc-car handshake timeout: %d\n", err); 318adbcec88SDmitry Osipenko return err; 31996e5da7cSDmitry Osipenko } 32096e5da7cSDmitry Osipenko 32196e5da7cSDmitry Osipenko return 0; 32296e5da7cSDmitry Osipenko } 32396e5da7cSDmitry Osipenko 32496e5da7cSDmitry Osipenko static int tegra_emc_clk_change_notify(struct notifier_block *nb, 32596e5da7cSDmitry Osipenko unsigned long msg, void *data) 32696e5da7cSDmitry Osipenko { 32796e5da7cSDmitry Osipenko struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb); 32896e5da7cSDmitry Osipenko struct clk_notifier_data *cnd = data; 32996e5da7cSDmitry Osipenko int err; 33096e5da7cSDmitry Osipenko 33196e5da7cSDmitry Osipenko switch (msg) { 33296e5da7cSDmitry Osipenko case PRE_RATE_CHANGE: 33396e5da7cSDmitry Osipenko err = emc_prepare_timing_change(emc, cnd->new_rate); 33496e5da7cSDmitry Osipenko break; 33596e5da7cSDmitry Osipenko 33696e5da7cSDmitry Osipenko case ABORT_RATE_CHANGE: 33796e5da7cSDmitry Osipenko err = emc_prepare_timing_change(emc, cnd->old_rate); 33896e5da7cSDmitry Osipenko if (err) 33996e5da7cSDmitry Osipenko break; 34096e5da7cSDmitry Osipenko 34196e5da7cSDmitry Osipenko err = emc_complete_timing_change(emc, true); 34296e5da7cSDmitry Osipenko break; 34396e5da7cSDmitry Osipenko 34496e5da7cSDmitry Osipenko case POST_RATE_CHANGE: 34596e5da7cSDmitry Osipenko err = emc_complete_timing_change(emc, false); 34696e5da7cSDmitry Osipenko break; 34796e5da7cSDmitry Osipenko 34896e5da7cSDmitry Osipenko default: 34996e5da7cSDmitry Osipenko return NOTIFY_DONE; 35096e5da7cSDmitry Osipenko } 35196e5da7cSDmitry Osipenko 35296e5da7cSDmitry Osipenko return notifier_from_errno(err); 35396e5da7cSDmitry Osipenko } 35496e5da7cSDmitry Osipenko 35596e5da7cSDmitry Osipenko static int load_one_timing_from_dt(struct tegra_emc *emc, 35696e5da7cSDmitry Osipenko struct emc_timing *timing, 35796e5da7cSDmitry Osipenko struct device_node *node) 35896e5da7cSDmitry Osipenko { 35996e5da7cSDmitry Osipenko u32 rate; 36096e5da7cSDmitry Osipenko int err; 36196e5da7cSDmitry Osipenko 36296e5da7cSDmitry Osipenko if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) { 36396e5da7cSDmitry Osipenko dev_err(emc->dev, "incompatible DT node: %pOF\n", node); 36496e5da7cSDmitry Osipenko return -EINVAL; 36596e5da7cSDmitry Osipenko } 36696e5da7cSDmitry Osipenko 36796e5da7cSDmitry Osipenko err = of_property_read_u32(node, "clock-frequency", &rate); 36896e5da7cSDmitry Osipenko if (err) { 36996e5da7cSDmitry Osipenko dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n", 37096e5da7cSDmitry Osipenko node, err); 37196e5da7cSDmitry Osipenko return err; 37296e5da7cSDmitry Osipenko } 37396e5da7cSDmitry Osipenko 37496e5da7cSDmitry Osipenko err = of_property_read_u32_array(node, "nvidia,emc-registers", 37596e5da7cSDmitry Osipenko timing->data, 37696e5da7cSDmitry Osipenko ARRAY_SIZE(emc_timing_registers)); 37796e5da7cSDmitry Osipenko if (err) { 37896e5da7cSDmitry Osipenko dev_err(emc->dev, 37996e5da7cSDmitry Osipenko "timing %pOF: failed to read emc timing data: %d\n", 38096e5da7cSDmitry Osipenko node, err); 38196e5da7cSDmitry Osipenko return err; 38296e5da7cSDmitry Osipenko } 38396e5da7cSDmitry Osipenko 38496e5da7cSDmitry Osipenko /* 38596e5da7cSDmitry Osipenko * The EMC clock rate is twice the bus rate, and the bus rate is 38696e5da7cSDmitry Osipenko * measured in kHz. 38796e5da7cSDmitry Osipenko */ 38896e5da7cSDmitry Osipenko timing->rate = rate * 2 * 1000; 38996e5da7cSDmitry Osipenko 39096e5da7cSDmitry Osipenko dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n", 39196e5da7cSDmitry Osipenko __func__, node, timing->rate); 39296e5da7cSDmitry Osipenko 39396e5da7cSDmitry Osipenko return 0; 39496e5da7cSDmitry Osipenko } 39596e5da7cSDmitry Osipenko 39696e5da7cSDmitry Osipenko static int cmp_timings(const void *_a, const void *_b) 39796e5da7cSDmitry Osipenko { 39896e5da7cSDmitry Osipenko const struct emc_timing *a = _a; 39996e5da7cSDmitry Osipenko const struct emc_timing *b = _b; 40096e5da7cSDmitry Osipenko 40196e5da7cSDmitry Osipenko if (a->rate < b->rate) 40296e5da7cSDmitry Osipenko return -1; 40396e5da7cSDmitry Osipenko 40496e5da7cSDmitry Osipenko if (a->rate > b->rate) 40596e5da7cSDmitry Osipenko return 1; 40696e5da7cSDmitry Osipenko 40796e5da7cSDmitry Osipenko return 0; 40896e5da7cSDmitry Osipenko } 40996e5da7cSDmitry Osipenko 41096e5da7cSDmitry Osipenko static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc, 41196e5da7cSDmitry Osipenko struct device_node *node) 41296e5da7cSDmitry Osipenko { 41396e5da7cSDmitry Osipenko struct emc_timing *timing; 41496e5da7cSDmitry Osipenko int child_count; 41596e5da7cSDmitry Osipenko int err; 41696e5da7cSDmitry Osipenko 41796e5da7cSDmitry Osipenko child_count = of_get_child_count(node); 41896e5da7cSDmitry Osipenko if (!child_count) { 41996e5da7cSDmitry Osipenko dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node); 42096e5da7cSDmitry Osipenko return -EINVAL; 42196e5da7cSDmitry Osipenko } 42296e5da7cSDmitry Osipenko 42396e5da7cSDmitry Osipenko emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing), 42496e5da7cSDmitry Osipenko GFP_KERNEL); 42596e5da7cSDmitry Osipenko if (!emc->timings) 42696e5da7cSDmitry Osipenko return -ENOMEM; 42796e5da7cSDmitry Osipenko 42896e5da7cSDmitry Osipenko timing = emc->timings; 42996e5da7cSDmitry Osipenko 430*818902cbSKrzysztof Kozlowski for_each_child_of_node_scoped(node, child) { 431131dd9a4SDmitry Osipenko if (of_node_name_eq(child, "lpddr2")) 432131dd9a4SDmitry Osipenko continue; 433131dd9a4SDmitry Osipenko 43496e5da7cSDmitry Osipenko err = load_one_timing_from_dt(emc, timing++, child); 435*818902cbSKrzysztof Kozlowski if (err) 43696e5da7cSDmitry Osipenko return err; 437131dd9a4SDmitry Osipenko 438131dd9a4SDmitry Osipenko emc->num_timings++; 43996e5da7cSDmitry Osipenko } 44096e5da7cSDmitry Osipenko 44196e5da7cSDmitry Osipenko sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings, 44296e5da7cSDmitry Osipenko NULL); 44396e5da7cSDmitry Osipenko 444f012ade8SDmitry Osipenko dev_info_once(emc->dev, 445f541efaaSDmitry Osipenko "got %u timings for RAM code %u (min %luMHz max %luMHz)\n", 446f541efaaSDmitry Osipenko emc->num_timings, 447f541efaaSDmitry Osipenko tegra_read_ram_code(), 448f541efaaSDmitry Osipenko emc->timings[0].rate / 1000000, 449f541efaaSDmitry Osipenko emc->timings[emc->num_timings - 1].rate / 1000000); 450f541efaaSDmitry Osipenko 45196e5da7cSDmitry Osipenko return 0; 45296e5da7cSDmitry Osipenko } 45396e5da7cSDmitry Osipenko 45496e5da7cSDmitry Osipenko static struct device_node * 455131dd9a4SDmitry Osipenko tegra_emc_find_node_by_ram_code(struct tegra_emc *emc) 45696e5da7cSDmitry Osipenko { 457131dd9a4SDmitry Osipenko struct device *dev = emc->dev; 45896e5da7cSDmitry Osipenko struct device_node *np; 45996e5da7cSDmitry Osipenko u32 value, ram_code; 46096e5da7cSDmitry Osipenko int err; 46196e5da7cSDmitry Osipenko 462131dd9a4SDmitry Osipenko if (emc->mrr_error) { 463131dd9a4SDmitry Osipenko dev_warn(dev, "memory timings skipped due to MRR error\n"); 464131dd9a4SDmitry Osipenko return NULL; 465131dd9a4SDmitry Osipenko } 466131dd9a4SDmitry Osipenko 467fa4794ffSDmitry Osipenko if (of_get_child_count(dev->of_node) == 0) { 468f012ade8SDmitry Osipenko dev_info_once(dev, "device-tree doesn't have memory timings\n"); 469fa4794ffSDmitry Osipenko return NULL; 470fa4794ffSDmitry Osipenko } 471fa4794ffSDmitry Osipenko 47296e5da7cSDmitry Osipenko if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code")) 47396e5da7cSDmitry Osipenko return of_node_get(dev->of_node); 47496e5da7cSDmitry Osipenko 47596e5da7cSDmitry Osipenko ram_code = tegra_read_ram_code(); 47696e5da7cSDmitry Osipenko 47796e5da7cSDmitry Osipenko for (np = of_find_node_by_name(dev->of_node, "emc-tables"); np; 47896e5da7cSDmitry Osipenko np = of_find_node_by_name(np, "emc-tables")) { 47996e5da7cSDmitry Osipenko err = of_property_read_u32(np, "nvidia,ram-code", &value); 48096e5da7cSDmitry Osipenko if (err || value != ram_code) { 481131dd9a4SDmitry Osipenko struct device_node *lpddr2_np; 482131dd9a4SDmitry Osipenko bool cfg_mismatches = false; 483131dd9a4SDmitry Osipenko 484131dd9a4SDmitry Osipenko lpddr2_np = of_find_node_by_name(np, "lpddr2"); 485131dd9a4SDmitry Osipenko if (lpddr2_np) { 486131dd9a4SDmitry Osipenko const struct lpddr2_info *info; 487131dd9a4SDmitry Osipenko 488131dd9a4SDmitry Osipenko info = of_lpddr2_get_info(lpddr2_np, dev); 489131dd9a4SDmitry Osipenko if (info) { 490131dd9a4SDmitry Osipenko if (info->manufacturer_id >= 0 && 491131dd9a4SDmitry Osipenko info->manufacturer_id != emc->manufacturer_id) 492131dd9a4SDmitry Osipenko cfg_mismatches = true; 493131dd9a4SDmitry Osipenko 494131dd9a4SDmitry Osipenko if (info->revision_id1 >= 0 && 495131dd9a4SDmitry Osipenko info->revision_id1 != emc->revision_id1) 496131dd9a4SDmitry Osipenko cfg_mismatches = true; 497131dd9a4SDmitry Osipenko 498131dd9a4SDmitry Osipenko if (info->revision_id2 >= 0 && 499131dd9a4SDmitry Osipenko info->revision_id2 != emc->revision_id2) 500131dd9a4SDmitry Osipenko cfg_mismatches = true; 501131dd9a4SDmitry Osipenko 502131dd9a4SDmitry Osipenko if (info->density != emc->basic_conf4.density) 503131dd9a4SDmitry Osipenko cfg_mismatches = true; 504131dd9a4SDmitry Osipenko 505131dd9a4SDmitry Osipenko if (info->io_width != emc->basic_conf4.io_width) 506131dd9a4SDmitry Osipenko cfg_mismatches = true; 507131dd9a4SDmitry Osipenko 508131dd9a4SDmitry Osipenko if (info->arch_type != emc->basic_conf4.arch_type) 509131dd9a4SDmitry Osipenko cfg_mismatches = true; 510131dd9a4SDmitry Osipenko } else { 511131dd9a4SDmitry Osipenko dev_err(dev, "failed to parse %pOF\n", lpddr2_np); 512131dd9a4SDmitry Osipenko cfg_mismatches = true; 513131dd9a4SDmitry Osipenko } 514131dd9a4SDmitry Osipenko 515131dd9a4SDmitry Osipenko of_node_put(lpddr2_np); 516131dd9a4SDmitry Osipenko } else { 517131dd9a4SDmitry Osipenko cfg_mismatches = true; 518131dd9a4SDmitry Osipenko } 519131dd9a4SDmitry Osipenko 520131dd9a4SDmitry Osipenko if (cfg_mismatches) { 52196e5da7cSDmitry Osipenko of_node_put(np); 52296e5da7cSDmitry Osipenko continue; 52396e5da7cSDmitry Osipenko } 524131dd9a4SDmitry Osipenko } 52596e5da7cSDmitry Osipenko 52696e5da7cSDmitry Osipenko return np; 52796e5da7cSDmitry Osipenko } 52896e5da7cSDmitry Osipenko 52996e5da7cSDmitry Osipenko dev_err(dev, "no memory timings for RAM code %u found in device tree\n", 53096e5da7cSDmitry Osipenko ram_code); 53196e5da7cSDmitry Osipenko 53296e5da7cSDmitry Osipenko return NULL; 53396e5da7cSDmitry Osipenko } 53496e5da7cSDmitry Osipenko 535131dd9a4SDmitry Osipenko static int emc_read_lpddr_mode_register(struct tegra_emc *emc, 536131dd9a4SDmitry Osipenko unsigned int emem_dev, 537131dd9a4SDmitry Osipenko unsigned int register_addr, 538131dd9a4SDmitry Osipenko unsigned int *register_data) 539131dd9a4SDmitry Osipenko { 5409ff68434SDmitry Osipenko u32 memory_dev = emem_dev ? 1 : 2; 541131dd9a4SDmitry Osipenko u32 val, mr_mask = 0xff; 542131dd9a4SDmitry Osipenko int err; 543131dd9a4SDmitry Osipenko 544131dd9a4SDmitry Osipenko /* clear data-valid interrupt status */ 545131dd9a4SDmitry Osipenko writel_relaxed(EMC_MRR_DIVLD_INT, emc->regs + EMC_INTSTATUS); 546131dd9a4SDmitry Osipenko 547131dd9a4SDmitry Osipenko /* issue mode register read request */ 548131dd9a4SDmitry Osipenko val = FIELD_PREP(EMC_MRR_DEV_SELECTN, memory_dev); 549131dd9a4SDmitry Osipenko val |= FIELD_PREP(EMC_MRR_MRR_MA, register_addr); 550131dd9a4SDmitry Osipenko 551131dd9a4SDmitry Osipenko writel_relaxed(val, emc->regs + EMC_MRR); 552131dd9a4SDmitry Osipenko 553131dd9a4SDmitry Osipenko /* wait for the LPDDR2 data-valid interrupt */ 554131dd9a4SDmitry Osipenko err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, val, 555131dd9a4SDmitry Osipenko val & EMC_MRR_DIVLD_INT, 556131dd9a4SDmitry Osipenko 1, 100); 557131dd9a4SDmitry Osipenko if (err) { 558131dd9a4SDmitry Osipenko dev_err(emc->dev, "mode register %u read failed: %d\n", 559131dd9a4SDmitry Osipenko register_addr, err); 560131dd9a4SDmitry Osipenko emc->mrr_error = true; 561131dd9a4SDmitry Osipenko return err; 562131dd9a4SDmitry Osipenko } 563131dd9a4SDmitry Osipenko 564131dd9a4SDmitry Osipenko /* read out mode register data */ 565131dd9a4SDmitry Osipenko val = readl_relaxed(emc->regs + EMC_MRR); 566131dd9a4SDmitry Osipenko *register_data = FIELD_GET(EMC_MRR_MRR_DATA, val) & mr_mask; 567131dd9a4SDmitry Osipenko 568131dd9a4SDmitry Osipenko return 0; 569131dd9a4SDmitry Osipenko } 570131dd9a4SDmitry Osipenko 571131dd9a4SDmitry Osipenko static void emc_read_lpddr_sdram_info(struct tegra_emc *emc, 572131dd9a4SDmitry Osipenko unsigned int emem_dev, 573131dd9a4SDmitry Osipenko bool print_out) 574131dd9a4SDmitry Osipenko { 575131dd9a4SDmitry Osipenko /* these registers are standard for all LPDDR JEDEC memory chips */ 576131dd9a4SDmitry Osipenko emc_read_lpddr_mode_register(emc, emem_dev, 5, &emc->manufacturer_id); 577131dd9a4SDmitry Osipenko emc_read_lpddr_mode_register(emc, emem_dev, 6, &emc->revision_id1); 578131dd9a4SDmitry Osipenko emc_read_lpddr_mode_register(emc, emem_dev, 7, &emc->revision_id2); 579131dd9a4SDmitry Osipenko emc_read_lpddr_mode_register(emc, emem_dev, 8, &emc->basic_conf4.value); 580131dd9a4SDmitry Osipenko 581131dd9a4SDmitry Osipenko if (!print_out) 582131dd9a4SDmitry Osipenko return; 583131dd9a4SDmitry Osipenko 584131dd9a4SDmitry Osipenko dev_info(emc->dev, "SDRAM[dev%u]: manufacturer: 0x%x (%s) rev1: 0x%x rev2: 0x%x prefetch: S%u density: %uMbit iowidth: %ubit\n", 585131dd9a4SDmitry Osipenko emem_dev, emc->manufacturer_id, 586131dd9a4SDmitry Osipenko lpddr2_jedec_manufacturer(emc->manufacturer_id), 587131dd9a4SDmitry Osipenko emc->revision_id1, emc->revision_id2, 588131dd9a4SDmitry Osipenko 4 >> emc->basic_conf4.arch_type, 589131dd9a4SDmitry Osipenko 64 << emc->basic_conf4.density, 590131dd9a4SDmitry Osipenko 32 >> emc->basic_conf4.io_width); 591131dd9a4SDmitry Osipenko } 592131dd9a4SDmitry Osipenko 59396e5da7cSDmitry Osipenko static int emc_setup_hw(struct tegra_emc *emc) 59496e5da7cSDmitry Osipenko { 595131dd9a4SDmitry Osipenko u32 emc_cfg, emc_dbg, emc_fbio, emc_adr_cfg; 596adbcec88SDmitry Osipenko u32 intmask = EMC_REFRESH_OVERFLOW_INT; 597131dd9a4SDmitry Osipenko static bool print_sdram_info_once; 598131dd9a4SDmitry Osipenko enum emc_dram_type dram_type; 599131dd9a4SDmitry Osipenko const char *dram_type_str; 600131dd9a4SDmitry Osipenko unsigned int emem_numdev; 60196e5da7cSDmitry Osipenko 60296e5da7cSDmitry Osipenko emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2); 60396e5da7cSDmitry Osipenko 60496e5da7cSDmitry Osipenko /* 60596e5da7cSDmitry Osipenko * Depending on a memory type, DRAM should enter either self-refresh 60696e5da7cSDmitry Osipenko * or power-down state on EMC clock change. 60796e5da7cSDmitry Osipenko */ 60896e5da7cSDmitry Osipenko if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) && 60996e5da7cSDmitry Osipenko !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) { 61096e5da7cSDmitry Osipenko dev_err(emc->dev, 61196e5da7cSDmitry Osipenko "bootloader didn't specify DRAM auto-suspend mode\n"); 61296e5da7cSDmitry Osipenko return -EINVAL; 61396e5da7cSDmitry Osipenko } 61496e5da7cSDmitry Osipenko 61596e5da7cSDmitry Osipenko /* enable EMC and CAR to handshake on PLL divider/source changes */ 61696e5da7cSDmitry Osipenko emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE; 61796e5da7cSDmitry Osipenko writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2); 61896e5da7cSDmitry Osipenko 61996e5da7cSDmitry Osipenko /* initialize interrupt */ 62096e5da7cSDmitry Osipenko writel_relaxed(intmask, emc->regs + EMC_INTMASK); 62196e5da7cSDmitry Osipenko writel_relaxed(intmask, emc->regs + EMC_INTSTATUS); 62296e5da7cSDmitry Osipenko 623c72396f9SDmitry Osipenko /* ensure that unwanted debug features are disabled */ 624c72396f9SDmitry Osipenko emc_dbg = readl_relaxed(emc->regs + EMC_DBG); 625c72396f9SDmitry Osipenko emc_dbg |= EMC_DBG_CFG_PRIORITY; 626c72396f9SDmitry Osipenko emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY; 627c72396f9SDmitry Osipenko emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE; 628c72396f9SDmitry Osipenko emc_dbg &= ~EMC_DBG_FORCE_UPDATE; 629c72396f9SDmitry Osipenko writel_relaxed(emc_dbg, emc->regs + EMC_DBG); 630c72396f9SDmitry Osipenko 631d5ef16baSDmitry Osipenko emc_fbio = readl_relaxed(emc->regs + EMC_FBIO_CFG5); 632d5ef16baSDmitry Osipenko 633d5ef16baSDmitry Osipenko if (emc_fbio & EMC_FBIO_CFG5_DRAM_WIDTH_X16) 634d5ef16baSDmitry Osipenko emc->dram_bus_width = 16; 635d5ef16baSDmitry Osipenko else 636d5ef16baSDmitry Osipenko emc->dram_bus_width = 32; 637d5ef16baSDmitry Osipenko 638131dd9a4SDmitry Osipenko dram_type = FIELD_GET(EMC_FBIO_CFG5_DRAM_TYPE, emc_fbio); 639131dd9a4SDmitry Osipenko 640131dd9a4SDmitry Osipenko switch (dram_type) { 641131dd9a4SDmitry Osipenko case DRAM_TYPE_RESERVED: 642131dd9a4SDmitry Osipenko dram_type_str = "INVALID"; 643131dd9a4SDmitry Osipenko break; 644131dd9a4SDmitry Osipenko case DRAM_TYPE_DDR1: 645131dd9a4SDmitry Osipenko dram_type_str = "DDR1"; 646131dd9a4SDmitry Osipenko break; 647131dd9a4SDmitry Osipenko case DRAM_TYPE_LPDDR2: 648131dd9a4SDmitry Osipenko dram_type_str = "LPDDR2"; 649131dd9a4SDmitry Osipenko break; 650131dd9a4SDmitry Osipenko case DRAM_TYPE_DDR2: 651131dd9a4SDmitry Osipenko dram_type_str = "DDR2"; 652131dd9a4SDmitry Osipenko break; 653131dd9a4SDmitry Osipenko } 654131dd9a4SDmitry Osipenko 655131dd9a4SDmitry Osipenko emc_adr_cfg = readl_relaxed(emc->regs + EMC_ADR_CFG_0); 656131dd9a4SDmitry Osipenko emem_numdev = FIELD_GET(EMC_ADR_CFG_0_EMEM_NUMDEV, emc_adr_cfg) + 1; 657131dd9a4SDmitry Osipenko 658131dd9a4SDmitry Osipenko dev_info_once(emc->dev, "%ubit DRAM bus, %u %s %s attached\n", 659131dd9a4SDmitry Osipenko emc->dram_bus_width, emem_numdev, dram_type_str, 660131dd9a4SDmitry Osipenko emem_numdev == 2 ? "devices" : "device"); 661131dd9a4SDmitry Osipenko 662131dd9a4SDmitry Osipenko if (dram_type == DRAM_TYPE_LPDDR2) { 663131dd9a4SDmitry Osipenko while (emem_numdev--) 664131dd9a4SDmitry Osipenko emc_read_lpddr_sdram_info(emc, emem_numdev, 665131dd9a4SDmitry Osipenko !print_sdram_info_once); 666131dd9a4SDmitry Osipenko print_sdram_info_once = true; 667131dd9a4SDmitry Osipenko } 668d5ef16baSDmitry Osipenko 66996e5da7cSDmitry Osipenko return 0; 67096e5da7cSDmitry Osipenko } 67196e5da7cSDmitry Osipenko 67277ab499dSDmitry Osipenko static long emc_round_rate(unsigned long rate, 67377ab499dSDmitry Osipenko unsigned long min_rate, 67477ab499dSDmitry Osipenko unsigned long max_rate, 67577ab499dSDmitry Osipenko void *arg) 67677ab499dSDmitry Osipenko { 67777ab499dSDmitry Osipenko struct emc_timing *timing = NULL; 67877ab499dSDmitry Osipenko struct tegra_emc *emc = arg; 67977ab499dSDmitry Osipenko unsigned int i; 68077ab499dSDmitry Osipenko 681fa4794ffSDmitry Osipenko if (!emc->num_timings) 682fa4794ffSDmitry Osipenko return clk_get_rate(emc->clk); 683fa4794ffSDmitry Osipenko 68477ab499dSDmitry Osipenko min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate); 68577ab499dSDmitry Osipenko 68677ab499dSDmitry Osipenko for (i = 0; i < emc->num_timings; i++) { 68777ab499dSDmitry Osipenko if (emc->timings[i].rate < rate && i != emc->num_timings - 1) 68877ab499dSDmitry Osipenko continue; 68977ab499dSDmitry Osipenko 69077ab499dSDmitry Osipenko if (emc->timings[i].rate > max_rate) { 69177ab499dSDmitry Osipenko i = max(i, 1u) - 1; 69277ab499dSDmitry Osipenko 69377ab499dSDmitry Osipenko if (emc->timings[i].rate < min_rate) 69477ab499dSDmitry Osipenko break; 69577ab499dSDmitry Osipenko } 69677ab499dSDmitry Osipenko 69777ab499dSDmitry Osipenko if (emc->timings[i].rate < min_rate) 69877ab499dSDmitry Osipenko continue; 69977ab499dSDmitry Osipenko 70077ab499dSDmitry Osipenko timing = &emc->timings[i]; 70177ab499dSDmitry Osipenko break; 70277ab499dSDmitry Osipenko } 70377ab499dSDmitry Osipenko 70477ab499dSDmitry Osipenko if (!timing) { 70577ab499dSDmitry Osipenko dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n", 70677ab499dSDmitry Osipenko rate, min_rate, max_rate); 70777ab499dSDmitry Osipenko return -EINVAL; 70877ab499dSDmitry Osipenko } 70977ab499dSDmitry Osipenko 71077ab499dSDmitry Osipenko return timing->rate; 71177ab499dSDmitry Osipenko } 71277ab499dSDmitry Osipenko 713d5ef16baSDmitry Osipenko static void tegra_emc_rate_requests_init(struct tegra_emc *emc) 714d5ef16baSDmitry Osipenko { 715d5ef16baSDmitry Osipenko unsigned int i; 716d5ef16baSDmitry Osipenko 717d5ef16baSDmitry Osipenko for (i = 0; i < EMC_RATE_TYPE_MAX; i++) { 718d5ef16baSDmitry Osipenko emc->requested_rate[i].min_rate = 0; 719d5ef16baSDmitry Osipenko emc->requested_rate[i].max_rate = ULONG_MAX; 720d5ef16baSDmitry Osipenko } 721d5ef16baSDmitry Osipenko } 722d5ef16baSDmitry Osipenko 723d5ef16baSDmitry Osipenko static int emc_request_rate(struct tegra_emc *emc, 724d5ef16baSDmitry Osipenko unsigned long new_min_rate, 725d5ef16baSDmitry Osipenko unsigned long new_max_rate, 726d5ef16baSDmitry Osipenko enum emc_rate_request_type type) 727d5ef16baSDmitry Osipenko { 728d5ef16baSDmitry Osipenko struct emc_rate_request *req = emc->requested_rate; 729d5ef16baSDmitry Osipenko unsigned long min_rate = 0, max_rate = ULONG_MAX; 730d5ef16baSDmitry Osipenko unsigned int i; 731d5ef16baSDmitry Osipenko int err; 732d5ef16baSDmitry Osipenko 733d5ef16baSDmitry Osipenko /* select minimum and maximum rates among the requested rates */ 734d5ef16baSDmitry Osipenko for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) { 735d5ef16baSDmitry Osipenko if (i == type) { 736d5ef16baSDmitry Osipenko min_rate = max(new_min_rate, min_rate); 737d5ef16baSDmitry Osipenko max_rate = min(new_max_rate, max_rate); 738d5ef16baSDmitry Osipenko } else { 739d5ef16baSDmitry Osipenko min_rate = max(req->min_rate, min_rate); 740d5ef16baSDmitry Osipenko max_rate = min(req->max_rate, max_rate); 741d5ef16baSDmitry Osipenko } 742d5ef16baSDmitry Osipenko } 743d5ef16baSDmitry Osipenko 744d5ef16baSDmitry Osipenko if (min_rate > max_rate) { 745d5ef16baSDmitry Osipenko dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n", 746d5ef16baSDmitry Osipenko __func__, type, min_rate, max_rate); 747d5ef16baSDmitry Osipenko return -ERANGE; 748d5ef16baSDmitry Osipenko } 749d5ef16baSDmitry Osipenko 750d5ef16baSDmitry Osipenko /* 751d5ef16baSDmitry Osipenko * EMC rate-changes should go via OPP API because it manages voltage 752d5ef16baSDmitry Osipenko * changes. 753d5ef16baSDmitry Osipenko */ 754d5ef16baSDmitry Osipenko err = dev_pm_opp_set_rate(emc->dev, min_rate); 755d5ef16baSDmitry Osipenko if (err) 756d5ef16baSDmitry Osipenko return err; 757d5ef16baSDmitry Osipenko 758d5ef16baSDmitry Osipenko emc->requested_rate[type].min_rate = new_min_rate; 759d5ef16baSDmitry Osipenko emc->requested_rate[type].max_rate = new_max_rate; 760d5ef16baSDmitry Osipenko 761d5ef16baSDmitry Osipenko return 0; 762d5ef16baSDmitry Osipenko } 763d5ef16baSDmitry Osipenko 764d5ef16baSDmitry Osipenko static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate, 765d5ef16baSDmitry Osipenko enum emc_rate_request_type type) 766d5ef16baSDmitry Osipenko { 767d5ef16baSDmitry Osipenko struct emc_rate_request *req = &emc->requested_rate[type]; 768d5ef16baSDmitry Osipenko int ret; 769d5ef16baSDmitry Osipenko 770d5ef16baSDmitry Osipenko mutex_lock(&emc->rate_lock); 771d5ef16baSDmitry Osipenko ret = emc_request_rate(emc, rate, req->max_rate, type); 772d5ef16baSDmitry Osipenko mutex_unlock(&emc->rate_lock); 773d5ef16baSDmitry Osipenko 774d5ef16baSDmitry Osipenko return ret; 775d5ef16baSDmitry Osipenko } 776d5ef16baSDmitry Osipenko 777d5ef16baSDmitry Osipenko static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate, 778d5ef16baSDmitry Osipenko enum emc_rate_request_type type) 779d5ef16baSDmitry Osipenko { 780d5ef16baSDmitry Osipenko struct emc_rate_request *req = &emc->requested_rate[type]; 781d5ef16baSDmitry Osipenko int ret; 782d5ef16baSDmitry Osipenko 783d5ef16baSDmitry Osipenko mutex_lock(&emc->rate_lock); 784d5ef16baSDmitry Osipenko ret = emc_request_rate(emc, req->min_rate, rate, type); 785d5ef16baSDmitry Osipenko mutex_unlock(&emc->rate_lock); 786d5ef16baSDmitry Osipenko 787d5ef16baSDmitry Osipenko return ret; 788d5ef16baSDmitry Osipenko } 789d5ef16baSDmitry Osipenko 7908209eefaSThierry Reding /* 7918209eefaSThierry Reding * debugfs interface 7928209eefaSThierry Reding * 7938209eefaSThierry Reding * The memory controller driver exposes some files in debugfs that can be used 7948209eefaSThierry Reding * to control the EMC frequency. The top-level directory can be found here: 7958209eefaSThierry Reding * 7968209eefaSThierry Reding * /sys/kernel/debug/emc 7978209eefaSThierry Reding * 7988209eefaSThierry Reding * It contains the following files: 7998209eefaSThierry Reding * 8008209eefaSThierry Reding * - available_rates: This file contains a list of valid, space-separated 8018209eefaSThierry Reding * EMC frequencies. 8028209eefaSThierry Reding * 8038209eefaSThierry Reding * - min_rate: Writing a value to this file sets the given frequency as the 8048209eefaSThierry Reding * floor of the permitted range. If this is higher than the currently 8058209eefaSThierry Reding * configured EMC frequency, this will cause the frequency to be 8068209eefaSThierry Reding * increased so that it stays within the valid range. 8078209eefaSThierry Reding * 8088209eefaSThierry Reding * - max_rate: Similarily to the min_rate file, writing a value to this file 8098209eefaSThierry Reding * sets the given frequency as the ceiling of the permitted range. If 8108209eefaSThierry Reding * the value is lower than the currently configured EMC frequency, this 8118209eefaSThierry Reding * will cause the frequency to be decreased so that it stays within the 8128209eefaSThierry Reding * valid range. 8138209eefaSThierry Reding */ 8148209eefaSThierry Reding 8158209eefaSThierry Reding static bool tegra_emc_validate_rate(struct tegra_emc *emc, unsigned long rate) 8168209eefaSThierry Reding { 8178209eefaSThierry Reding unsigned int i; 8188209eefaSThierry Reding 8198209eefaSThierry Reding for (i = 0; i < emc->num_timings; i++) 8208209eefaSThierry Reding if (rate == emc->timings[i].rate) 8218209eefaSThierry Reding return true; 8228209eefaSThierry Reding 8238209eefaSThierry Reding return false; 8248209eefaSThierry Reding } 8258209eefaSThierry Reding 8268209eefaSThierry Reding static int tegra_emc_debug_available_rates_show(struct seq_file *s, void *data) 8278209eefaSThierry Reding { 8288209eefaSThierry Reding struct tegra_emc *emc = s->private; 8298209eefaSThierry Reding const char *prefix = ""; 8308209eefaSThierry Reding unsigned int i; 8318209eefaSThierry Reding 8328209eefaSThierry Reding for (i = 0; i < emc->num_timings; i++) { 8338209eefaSThierry Reding seq_printf(s, "%s%lu", prefix, emc->timings[i].rate); 8348209eefaSThierry Reding prefix = " "; 8358209eefaSThierry Reding } 8368209eefaSThierry Reding 8378209eefaSThierry Reding seq_puts(s, "\n"); 8388209eefaSThierry Reding 8398209eefaSThierry Reding return 0; 8408209eefaSThierry Reding } 841c8385640SLiu Shixin DEFINE_SHOW_ATTRIBUTE(tegra_emc_debug_available_rates); 8428209eefaSThierry Reding 8438209eefaSThierry Reding static int tegra_emc_debug_min_rate_get(void *data, u64 *rate) 8448209eefaSThierry Reding { 8458209eefaSThierry Reding struct tegra_emc *emc = data; 8468209eefaSThierry Reding 8478209eefaSThierry Reding *rate = emc->debugfs.min_rate; 8488209eefaSThierry Reding 8498209eefaSThierry Reding return 0; 8508209eefaSThierry Reding } 8518209eefaSThierry Reding 8528209eefaSThierry Reding static int tegra_emc_debug_min_rate_set(void *data, u64 rate) 8538209eefaSThierry Reding { 8548209eefaSThierry Reding struct tegra_emc *emc = data; 8558209eefaSThierry Reding int err; 8568209eefaSThierry Reding 8578209eefaSThierry Reding if (!tegra_emc_validate_rate(emc, rate)) 8588209eefaSThierry Reding return -EINVAL; 8598209eefaSThierry Reding 860d5ef16baSDmitry Osipenko err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG); 8618209eefaSThierry Reding if (err < 0) 8628209eefaSThierry Reding return err; 8638209eefaSThierry Reding 8648209eefaSThierry Reding emc->debugfs.min_rate = rate; 8658209eefaSThierry Reding 8668209eefaSThierry Reding return 0; 8678209eefaSThierry Reding } 8688209eefaSThierry Reding 8698209eefaSThierry Reding DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_min_rate_fops, 8708209eefaSThierry Reding tegra_emc_debug_min_rate_get, 8718209eefaSThierry Reding tegra_emc_debug_min_rate_set, "%llu\n"); 8728209eefaSThierry Reding 8738209eefaSThierry Reding static int tegra_emc_debug_max_rate_get(void *data, u64 *rate) 8748209eefaSThierry Reding { 8758209eefaSThierry Reding struct tegra_emc *emc = data; 8768209eefaSThierry Reding 8778209eefaSThierry Reding *rate = emc->debugfs.max_rate; 8788209eefaSThierry Reding 8798209eefaSThierry Reding return 0; 8808209eefaSThierry Reding } 8818209eefaSThierry Reding 8828209eefaSThierry Reding static int tegra_emc_debug_max_rate_set(void *data, u64 rate) 8838209eefaSThierry Reding { 8848209eefaSThierry Reding struct tegra_emc *emc = data; 8858209eefaSThierry Reding int err; 8868209eefaSThierry Reding 8878209eefaSThierry Reding if (!tegra_emc_validate_rate(emc, rate)) 8888209eefaSThierry Reding return -EINVAL; 8898209eefaSThierry Reding 890d5ef16baSDmitry Osipenko err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG); 8918209eefaSThierry Reding if (err < 0) 8928209eefaSThierry Reding return err; 8938209eefaSThierry Reding 8948209eefaSThierry Reding emc->debugfs.max_rate = rate; 8958209eefaSThierry Reding 8968209eefaSThierry Reding return 0; 8978209eefaSThierry Reding } 8988209eefaSThierry Reding 8998209eefaSThierry Reding DEFINE_SIMPLE_ATTRIBUTE(tegra_emc_debug_max_rate_fops, 9008209eefaSThierry Reding tegra_emc_debug_max_rate_get, 9018209eefaSThierry Reding tegra_emc_debug_max_rate_set, "%llu\n"); 9028209eefaSThierry Reding 9038209eefaSThierry Reding static void tegra_emc_debugfs_init(struct tegra_emc *emc) 9048209eefaSThierry Reding { 9058209eefaSThierry Reding struct device *dev = emc->dev; 9068209eefaSThierry Reding unsigned int i; 9078209eefaSThierry Reding int err; 9088209eefaSThierry Reding 9098209eefaSThierry Reding emc->debugfs.min_rate = ULONG_MAX; 9108209eefaSThierry Reding emc->debugfs.max_rate = 0; 9118209eefaSThierry Reding 9128209eefaSThierry Reding for (i = 0; i < emc->num_timings; i++) { 9138209eefaSThierry Reding if (emc->timings[i].rate < emc->debugfs.min_rate) 9148209eefaSThierry Reding emc->debugfs.min_rate = emc->timings[i].rate; 9158209eefaSThierry Reding 9168209eefaSThierry Reding if (emc->timings[i].rate > emc->debugfs.max_rate) 9178209eefaSThierry Reding emc->debugfs.max_rate = emc->timings[i].rate; 9188209eefaSThierry Reding } 9198209eefaSThierry Reding 9202243af41SDmitry Osipenko if (!emc->num_timings) { 9212243af41SDmitry Osipenko emc->debugfs.min_rate = clk_get_rate(emc->clk); 9222243af41SDmitry Osipenko emc->debugfs.max_rate = emc->debugfs.min_rate; 9232243af41SDmitry Osipenko } 9242243af41SDmitry Osipenko 9258209eefaSThierry Reding err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate, 9268209eefaSThierry Reding emc->debugfs.max_rate); 9278209eefaSThierry Reding if (err < 0) { 9288209eefaSThierry Reding dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n", 9298209eefaSThierry Reding emc->debugfs.min_rate, emc->debugfs.max_rate, 9308209eefaSThierry Reding emc->clk); 9318209eefaSThierry Reding } 9328209eefaSThierry Reding 9338209eefaSThierry Reding emc->debugfs.root = debugfs_create_dir("emc", NULL); 9348209eefaSThierry Reding 9356cc8823aSDmitry Osipenko debugfs_create_file("available_rates", 0444, emc->debugfs.root, 9368209eefaSThierry Reding emc, &tegra_emc_debug_available_rates_fops); 9376cc8823aSDmitry Osipenko debugfs_create_file("min_rate", 0644, emc->debugfs.root, 9388209eefaSThierry Reding emc, &tegra_emc_debug_min_rate_fops); 9396cc8823aSDmitry Osipenko debugfs_create_file("max_rate", 0644, emc->debugfs.root, 9408209eefaSThierry Reding emc, &tegra_emc_debug_max_rate_fops); 9418209eefaSThierry Reding } 9428209eefaSThierry Reding 943d5ef16baSDmitry Osipenko static inline struct tegra_emc * 944d5ef16baSDmitry Osipenko to_tegra_emc_provider(struct icc_provider *provider) 945d5ef16baSDmitry Osipenko { 946d5ef16baSDmitry Osipenko return container_of(provider, struct tegra_emc, provider); 947d5ef16baSDmitry Osipenko } 948d5ef16baSDmitry Osipenko 949d5ef16baSDmitry Osipenko static struct icc_node_data * 9500dc5b8abSKrzysztof Kozlowski emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data) 951d5ef16baSDmitry Osipenko { 952d5ef16baSDmitry Osipenko struct icc_provider *provider = data; 953d5ef16baSDmitry Osipenko struct icc_node_data *ndata; 954d5ef16baSDmitry Osipenko struct icc_node *node; 955d5ef16baSDmitry Osipenko 956d5ef16baSDmitry Osipenko /* External Memory is the only possible ICC route */ 957d5ef16baSDmitry Osipenko list_for_each_entry(node, &provider->nodes, node_list) { 958d5ef16baSDmitry Osipenko if (node->id != TEGRA_ICC_EMEM) 959d5ef16baSDmitry Osipenko continue; 960d5ef16baSDmitry Osipenko 961d5ef16baSDmitry Osipenko ndata = kzalloc(sizeof(*ndata), GFP_KERNEL); 962d5ef16baSDmitry Osipenko if (!ndata) 963d5ef16baSDmitry Osipenko return ERR_PTR(-ENOMEM); 964d5ef16baSDmitry Osipenko 965d5ef16baSDmitry Osipenko /* 966d5ef16baSDmitry Osipenko * SRC and DST nodes should have matching TAG in order to have 967d5ef16baSDmitry Osipenko * it set by default for a requested path. 968d5ef16baSDmitry Osipenko */ 969d5ef16baSDmitry Osipenko ndata->tag = TEGRA_MC_ICC_TAG_ISO; 970d5ef16baSDmitry Osipenko ndata->node = node; 971d5ef16baSDmitry Osipenko 972d5ef16baSDmitry Osipenko return ndata; 973d5ef16baSDmitry Osipenko } 974d5ef16baSDmitry Osipenko 975d5ef16baSDmitry Osipenko return ERR_PTR(-EPROBE_DEFER); 976d5ef16baSDmitry Osipenko } 977d5ef16baSDmitry Osipenko 978d5ef16baSDmitry Osipenko static int emc_icc_set(struct icc_node *src, struct icc_node *dst) 979d5ef16baSDmitry Osipenko { 980d5ef16baSDmitry Osipenko struct tegra_emc *emc = to_tegra_emc_provider(dst->provider); 981d5ef16baSDmitry Osipenko unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw); 982d5ef16baSDmitry Osipenko unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw); 983d5ef16baSDmitry Osipenko unsigned long long rate = max(avg_bw, peak_bw); 984d5ef16baSDmitry Osipenko unsigned int dram_data_bus_width_bytes; 985d5ef16baSDmitry Osipenko int err; 986d5ef16baSDmitry Osipenko 987d5ef16baSDmitry Osipenko /* 988d5ef16baSDmitry Osipenko * Tegra20 EMC runs on x2 clock rate of SDRAM bus because DDR data 989d5ef16baSDmitry Osipenko * is sampled on both clock edges. This means that EMC clock rate 990d5ef16baSDmitry Osipenko * equals to the peak data-rate. 991d5ef16baSDmitry Osipenko */ 992d5ef16baSDmitry Osipenko dram_data_bus_width_bytes = emc->dram_bus_width / 8; 993d5ef16baSDmitry Osipenko do_div(rate, dram_data_bus_width_bytes); 994d5ef16baSDmitry Osipenko rate = min_t(u64, rate, U32_MAX); 995d5ef16baSDmitry Osipenko 996d5ef16baSDmitry Osipenko err = emc_set_min_rate(emc, rate, EMC_RATE_ICC); 997d5ef16baSDmitry Osipenko if (err) 998d5ef16baSDmitry Osipenko return err; 999d5ef16baSDmitry Osipenko 1000d5ef16baSDmitry Osipenko return 0; 1001d5ef16baSDmitry Osipenko } 1002d5ef16baSDmitry Osipenko 1003d5ef16baSDmitry Osipenko static int tegra_emc_interconnect_init(struct tegra_emc *emc) 1004d5ef16baSDmitry Osipenko { 1005d5ef16baSDmitry Osipenko const struct tegra_mc_soc *soc; 1006d5ef16baSDmitry Osipenko struct icc_node *node; 1007d5ef16baSDmitry Osipenko int err; 1008d5ef16baSDmitry Osipenko 1009d5ef16baSDmitry Osipenko emc->mc = devm_tegra_memory_controller_get(emc->dev); 1010d5ef16baSDmitry Osipenko if (IS_ERR(emc->mc)) 1011d5ef16baSDmitry Osipenko return PTR_ERR(emc->mc); 1012d5ef16baSDmitry Osipenko 1013d5ef16baSDmitry Osipenko soc = emc->mc->soc; 1014d5ef16baSDmitry Osipenko 1015d5ef16baSDmitry Osipenko emc->provider.dev = emc->dev; 1016d5ef16baSDmitry Osipenko emc->provider.set = emc_icc_set; 1017d5ef16baSDmitry Osipenko emc->provider.data = &emc->provider; 1018d5ef16baSDmitry Osipenko emc->provider.aggregate = soc->icc_ops->aggregate; 1019d5ef16baSDmitry Osipenko emc->provider.xlate_extended = emc_of_icc_xlate_extended; 1020d5ef16baSDmitry Osipenko 1021c5587f61SJohan Hovold icc_provider_init(&emc->provider); 1022d5ef16baSDmitry Osipenko 1023d5ef16baSDmitry Osipenko /* create External Memory Controller node */ 1024d5ef16baSDmitry Osipenko node = icc_node_create(TEGRA_ICC_EMC); 1025d5ef16baSDmitry Osipenko if (IS_ERR(node)) { 1026d5ef16baSDmitry Osipenko err = PTR_ERR(node); 1027c5587f61SJohan Hovold goto err_msg; 1028d5ef16baSDmitry Osipenko } 1029d5ef16baSDmitry Osipenko 1030d5ef16baSDmitry Osipenko node->name = "External Memory Controller"; 1031d5ef16baSDmitry Osipenko icc_node_add(node, &emc->provider); 1032d5ef16baSDmitry Osipenko 1033d5ef16baSDmitry Osipenko /* link External Memory Controller to External Memory (DRAM) */ 1034d5ef16baSDmitry Osipenko err = icc_link_create(node, TEGRA_ICC_EMEM); 1035d5ef16baSDmitry Osipenko if (err) 1036d5ef16baSDmitry Osipenko goto remove_nodes; 1037d5ef16baSDmitry Osipenko 1038d5ef16baSDmitry Osipenko /* create External Memory node */ 1039d5ef16baSDmitry Osipenko node = icc_node_create(TEGRA_ICC_EMEM); 1040d5ef16baSDmitry Osipenko if (IS_ERR(node)) { 1041d5ef16baSDmitry Osipenko err = PTR_ERR(node); 1042d5ef16baSDmitry Osipenko goto remove_nodes; 1043d5ef16baSDmitry Osipenko } 1044d5ef16baSDmitry Osipenko 1045d5ef16baSDmitry Osipenko node->name = "External Memory (DRAM)"; 1046d5ef16baSDmitry Osipenko icc_node_add(node, &emc->provider); 1047d5ef16baSDmitry Osipenko 1048c5587f61SJohan Hovold err = icc_provider_register(&emc->provider); 1049c5587f61SJohan Hovold if (err) 1050c5587f61SJohan Hovold goto remove_nodes; 1051c5587f61SJohan Hovold 1052d5ef16baSDmitry Osipenko return 0; 1053d5ef16baSDmitry Osipenko 1054d5ef16baSDmitry Osipenko remove_nodes: 1055d5ef16baSDmitry Osipenko icc_nodes_remove(&emc->provider); 1056d5ef16baSDmitry Osipenko err_msg: 1057d5ef16baSDmitry Osipenko dev_err(emc->dev, "failed to initialize ICC: %d\n", err); 1058d5ef16baSDmitry Osipenko 1059d5ef16baSDmitry Osipenko return err; 1060d5ef16baSDmitry Osipenko } 1061d5ef16baSDmitry Osipenko 1062cba3902bSDmitry Osipenko static void devm_tegra_emc_unset_callback(void *data) 1063cba3902bSDmitry Osipenko { 1064cba3902bSDmitry Osipenko tegra20_clk_set_emc_round_callback(NULL, NULL); 1065cba3902bSDmitry Osipenko } 1066cba3902bSDmitry Osipenko 1067cba3902bSDmitry Osipenko static void devm_tegra_emc_unreg_clk_notifier(void *data) 1068cba3902bSDmitry Osipenko { 1069cba3902bSDmitry Osipenko struct tegra_emc *emc = data; 1070cba3902bSDmitry Osipenko 1071cba3902bSDmitry Osipenko clk_notifier_unregister(emc->clk, &emc->clk_nb); 1072cba3902bSDmitry Osipenko } 1073cba3902bSDmitry Osipenko 1074cba3902bSDmitry Osipenko static int tegra_emc_init_clk(struct tegra_emc *emc) 1075cba3902bSDmitry Osipenko { 1076cba3902bSDmitry Osipenko int err; 1077cba3902bSDmitry Osipenko 1078cba3902bSDmitry Osipenko tegra20_clk_set_emc_round_callback(emc_round_rate, emc); 1079cba3902bSDmitry Osipenko 1080cba3902bSDmitry Osipenko err = devm_add_action_or_reset(emc->dev, devm_tegra_emc_unset_callback, 1081cba3902bSDmitry Osipenko NULL); 1082cba3902bSDmitry Osipenko if (err) 1083cba3902bSDmitry Osipenko return err; 1084cba3902bSDmitry Osipenko 1085cba3902bSDmitry Osipenko emc->clk = devm_clk_get(emc->dev, NULL); 1086cba3902bSDmitry Osipenko if (IS_ERR(emc->clk)) { 1087cba3902bSDmitry Osipenko dev_err(emc->dev, "failed to get EMC clock: %pe\n", emc->clk); 1088cba3902bSDmitry Osipenko return PTR_ERR(emc->clk); 1089cba3902bSDmitry Osipenko } 1090cba3902bSDmitry Osipenko 1091cba3902bSDmitry Osipenko err = clk_notifier_register(emc->clk, &emc->clk_nb); 1092cba3902bSDmitry Osipenko if (err) { 1093cba3902bSDmitry Osipenko dev_err(emc->dev, "failed to register clk notifier: %d\n", err); 1094cba3902bSDmitry Osipenko return err; 1095cba3902bSDmitry Osipenko } 1096cba3902bSDmitry Osipenko 1097cba3902bSDmitry Osipenko err = devm_add_action_or_reset(emc->dev, 1098cba3902bSDmitry Osipenko devm_tegra_emc_unreg_clk_notifier, emc); 1099cba3902bSDmitry Osipenko if (err) 1100cba3902bSDmitry Osipenko return err; 1101cba3902bSDmitry Osipenko 1102cba3902bSDmitry Osipenko return 0; 1103cba3902bSDmitry Osipenko } 1104cba3902bSDmitry Osipenko 1105dedf62d6SDmitry Osipenko static int tegra_emc_devfreq_target(struct device *dev, unsigned long *freq, 1106dedf62d6SDmitry Osipenko u32 flags) 1107dedf62d6SDmitry Osipenko { 1108dedf62d6SDmitry Osipenko struct tegra_emc *emc = dev_get_drvdata(dev); 1109dedf62d6SDmitry Osipenko struct dev_pm_opp *opp; 1110dedf62d6SDmitry Osipenko unsigned long rate; 1111dedf62d6SDmitry Osipenko 1112dedf62d6SDmitry Osipenko opp = devfreq_recommended_opp(dev, freq, flags); 1113dedf62d6SDmitry Osipenko if (IS_ERR(opp)) { 1114dedf62d6SDmitry Osipenko dev_err(dev, "failed to find opp for %lu Hz\n", *freq); 1115dedf62d6SDmitry Osipenko return PTR_ERR(opp); 1116dedf62d6SDmitry Osipenko } 1117dedf62d6SDmitry Osipenko 1118dedf62d6SDmitry Osipenko rate = dev_pm_opp_get_freq(opp); 1119dedf62d6SDmitry Osipenko dev_pm_opp_put(opp); 1120dedf62d6SDmitry Osipenko 1121dedf62d6SDmitry Osipenko return emc_set_min_rate(emc, rate, EMC_RATE_DEVFREQ); 1122dedf62d6SDmitry Osipenko } 1123dedf62d6SDmitry Osipenko 1124dedf62d6SDmitry Osipenko static int tegra_emc_devfreq_get_dev_status(struct device *dev, 1125dedf62d6SDmitry Osipenko struct devfreq_dev_status *stat) 1126dedf62d6SDmitry Osipenko { 1127dedf62d6SDmitry Osipenko struct tegra_emc *emc = dev_get_drvdata(dev); 1128dedf62d6SDmitry Osipenko 1129dedf62d6SDmitry Osipenko /* freeze counters */ 1130dedf62d6SDmitry Osipenko writel_relaxed(EMC_PWR_GATHER_DISABLE, emc->regs + EMC_STAT_CONTROL); 1131dedf62d6SDmitry Osipenko 1132dedf62d6SDmitry Osipenko /* 1133dedf62d6SDmitry Osipenko * busy_time: number of clocks EMC request was accepted 1134dedf62d6SDmitry Osipenko * total_time: number of clocks PWR_GATHER control was set to ENABLE 1135dedf62d6SDmitry Osipenko */ 1136dedf62d6SDmitry Osipenko stat->busy_time = readl_relaxed(emc->regs + EMC_STAT_PWR_COUNT); 1137dedf62d6SDmitry Osipenko stat->total_time = readl_relaxed(emc->regs + EMC_STAT_PWR_CLOCKS); 1138dedf62d6SDmitry Osipenko stat->current_frequency = clk_get_rate(emc->clk); 1139dedf62d6SDmitry Osipenko 1140dedf62d6SDmitry Osipenko /* clear counters and restart */ 1141dedf62d6SDmitry Osipenko writel_relaxed(EMC_PWR_GATHER_CLEAR, emc->regs + EMC_STAT_CONTROL); 1142dedf62d6SDmitry Osipenko writel_relaxed(EMC_PWR_GATHER_ENABLE, emc->regs + EMC_STAT_CONTROL); 1143dedf62d6SDmitry Osipenko 1144dedf62d6SDmitry Osipenko return 0; 1145dedf62d6SDmitry Osipenko } 1146dedf62d6SDmitry Osipenko 1147dedf62d6SDmitry Osipenko static struct devfreq_dev_profile tegra_emc_devfreq_profile = { 1148dedf62d6SDmitry Osipenko .polling_ms = 30, 1149dedf62d6SDmitry Osipenko .target = tegra_emc_devfreq_target, 1150dedf62d6SDmitry Osipenko .get_dev_status = tegra_emc_devfreq_get_dev_status, 1151dedf62d6SDmitry Osipenko }; 1152dedf62d6SDmitry Osipenko 1153dedf62d6SDmitry Osipenko static int tegra_emc_devfreq_init(struct tegra_emc *emc) 1154dedf62d6SDmitry Osipenko { 1155dedf62d6SDmitry Osipenko struct devfreq *devfreq; 1156dedf62d6SDmitry Osipenko 1157dedf62d6SDmitry Osipenko /* 1158dedf62d6SDmitry Osipenko * PWR_COUNT is 1/2 of PWR_CLOCKS at max, and thus, the up-threshold 1159dedf62d6SDmitry Osipenko * should be less than 50. Secondly, multiple active memory clients 1160dedf62d6SDmitry Osipenko * may cause over 20% of lost clock cycles due to stalls caused by 1161dedf62d6SDmitry Osipenko * competing memory accesses. This means that threshold should be 1162dedf62d6SDmitry Osipenko * set to a less than 30 in order to have a properly working governor. 1163dedf62d6SDmitry Osipenko */ 1164dedf62d6SDmitry Osipenko emc->ondemand_data.upthreshold = 20; 1165dedf62d6SDmitry Osipenko 1166dedf62d6SDmitry Osipenko /* 1167dedf62d6SDmitry Osipenko * Reset statistic gathers state, select global bandwidth for the 1168dedf62d6SDmitry Osipenko * statistics collection mode and set clocks counter saturation 1169dedf62d6SDmitry Osipenko * limit to maximum. 1170dedf62d6SDmitry Osipenko */ 1171dedf62d6SDmitry Osipenko writel_relaxed(0x00000000, emc->regs + EMC_STAT_CONTROL); 1172dedf62d6SDmitry Osipenko writel_relaxed(0x00000000, emc->regs + EMC_STAT_LLMC_CONTROL); 1173dedf62d6SDmitry Osipenko writel_relaxed(0xffffffff, emc->regs + EMC_STAT_PWR_CLOCK_LIMIT); 1174dedf62d6SDmitry Osipenko 1175dedf62d6SDmitry Osipenko devfreq = devm_devfreq_add_device(emc->dev, &tegra_emc_devfreq_profile, 1176dedf62d6SDmitry Osipenko DEVFREQ_GOV_SIMPLE_ONDEMAND, 1177dedf62d6SDmitry Osipenko &emc->ondemand_data); 1178dedf62d6SDmitry Osipenko if (IS_ERR(devfreq)) { 1179dedf62d6SDmitry Osipenko dev_err(emc->dev, "failed to initialize devfreq: %pe", devfreq); 1180dedf62d6SDmitry Osipenko return PTR_ERR(devfreq); 1181dedf62d6SDmitry Osipenko } 1182dedf62d6SDmitry Osipenko 1183dedf62d6SDmitry Osipenko return 0; 1184dedf62d6SDmitry Osipenko } 1185dedf62d6SDmitry Osipenko 118696e5da7cSDmitry Osipenko static int tegra_emc_probe(struct platform_device *pdev) 118796e5da7cSDmitry Osipenko { 1188f8c9670fSDmitry Osipenko struct tegra_core_opp_params opp_params = {}; 118996e5da7cSDmitry Osipenko struct device_node *np; 119096e5da7cSDmitry Osipenko struct tegra_emc *emc; 119196e5da7cSDmitry Osipenko int irq, err; 119296e5da7cSDmitry Osipenko 119396e5da7cSDmitry Osipenko irq = platform_get_irq(pdev, 0); 119496e5da7cSDmitry Osipenko if (irq < 0) { 119596e5da7cSDmitry Osipenko dev_err(&pdev->dev, "please update your device tree\n"); 119696e5da7cSDmitry Osipenko return irq; 119796e5da7cSDmitry Osipenko } 119896e5da7cSDmitry Osipenko 119996e5da7cSDmitry Osipenko emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL); 1200fa4794ffSDmitry Osipenko if (!emc) 120196e5da7cSDmitry Osipenko return -ENOMEM; 120296e5da7cSDmitry Osipenko 1203d5ef16baSDmitry Osipenko mutex_init(&emc->rate_lock); 120496e5da7cSDmitry Osipenko emc->clk_nb.notifier_call = tegra_emc_clk_change_notify; 120596e5da7cSDmitry Osipenko emc->dev = &pdev->dev; 120696e5da7cSDmitry Osipenko 12074e84d0a6SDmitry Osipenko emc->regs = devm_platform_ioremap_resource(pdev, 0); 120896e5da7cSDmitry Osipenko if (IS_ERR(emc->regs)) 120996e5da7cSDmitry Osipenko return PTR_ERR(emc->regs); 121096e5da7cSDmitry Osipenko 121196e5da7cSDmitry Osipenko err = emc_setup_hw(emc); 121296e5da7cSDmitry Osipenko if (err) 121396e5da7cSDmitry Osipenko return err; 121496e5da7cSDmitry Osipenko 1215131dd9a4SDmitry Osipenko np = tegra_emc_find_node_by_ram_code(emc); 1216131dd9a4SDmitry Osipenko if (np) { 1217131dd9a4SDmitry Osipenko err = tegra_emc_load_timings_from_dt(emc, np); 1218131dd9a4SDmitry Osipenko of_node_put(np); 1219131dd9a4SDmitry Osipenko if (err) 1220131dd9a4SDmitry Osipenko return err; 1221131dd9a4SDmitry Osipenko } 1222131dd9a4SDmitry Osipenko 122396e5da7cSDmitry Osipenko err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0, 122496e5da7cSDmitry Osipenko dev_name(&pdev->dev), emc); 122596e5da7cSDmitry Osipenko if (err) { 1226e09312feSDmitry Osipenko dev_err(&pdev->dev, "failed to request IRQ: %d\n", err); 122796e5da7cSDmitry Osipenko return err; 122896e5da7cSDmitry Osipenko } 122996e5da7cSDmitry Osipenko 1230cba3902bSDmitry Osipenko err = tegra_emc_init_clk(emc); 1231cba3902bSDmitry Osipenko if (err) 1232cba3902bSDmitry Osipenko return err; 123396e5da7cSDmitry Osipenko 1234f8c9670fSDmitry Osipenko opp_params.init_state = true; 1235f8c9670fSDmitry Osipenko 1236f8c9670fSDmitry Osipenko err = devm_tegra_core_dev_init_opp_table(&pdev->dev, &opp_params); 1237d5ef16baSDmitry Osipenko if (err) 1238cba3902bSDmitry Osipenko return err; 1239d5ef16baSDmitry Osipenko 12408209eefaSThierry Reding platform_set_drvdata(pdev, emc); 1241d5ef16baSDmitry Osipenko tegra_emc_rate_requests_init(emc); 12428209eefaSThierry Reding tegra_emc_debugfs_init(emc); 1243d5ef16baSDmitry Osipenko tegra_emc_interconnect_init(emc); 1244dedf62d6SDmitry Osipenko tegra_emc_devfreq_init(emc); 12458209eefaSThierry Reding 12460260979bSDmitry Osipenko /* 12470260979bSDmitry Osipenko * Don't allow the kernel module to be unloaded. Unloading adds some 12480260979bSDmitry Osipenko * extra complexity which doesn't really worth the effort in a case of 12490260979bSDmitry Osipenko * this driver. 12500260979bSDmitry Osipenko */ 12510260979bSDmitry Osipenko try_module_get(THIS_MODULE); 12520260979bSDmitry Osipenko 125396e5da7cSDmitry Osipenko return 0; 125496e5da7cSDmitry Osipenko } 125596e5da7cSDmitry Osipenko 125696e5da7cSDmitry Osipenko static const struct of_device_id tegra_emc_of_match[] = { 125796e5da7cSDmitry Osipenko { .compatible = "nvidia,tegra20-emc", }, 125896e5da7cSDmitry Osipenko {}, 125996e5da7cSDmitry Osipenko }; 12600260979bSDmitry Osipenko MODULE_DEVICE_TABLE(of, tegra_emc_of_match); 126196e5da7cSDmitry Osipenko 126296e5da7cSDmitry Osipenko static struct platform_driver tegra_emc_driver = { 126396e5da7cSDmitry Osipenko .probe = tegra_emc_probe, 126496e5da7cSDmitry Osipenko .driver = { 126596e5da7cSDmitry Osipenko .name = "tegra20-emc", 126696e5da7cSDmitry Osipenko .of_match_table = tegra_emc_of_match, 126796e5da7cSDmitry Osipenko .suppress_bind_attrs = true, 1268d5ef16baSDmitry Osipenko .sync_state = icc_sync_state, 126996e5da7cSDmitry Osipenko }, 127096e5da7cSDmitry Osipenko }; 12710260979bSDmitry Osipenko module_platform_driver(tegra_emc_driver); 127296e5da7cSDmitry Osipenko 12730260979bSDmitry Osipenko MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>"); 12740260979bSDmitry Osipenko MODULE_DESCRIPTION("NVIDIA Tegra20 EMC driver"); 127514b43c20SDmitry Osipenko MODULE_SOFTDEP("pre: governor_simpleondemand"); 12760260979bSDmitry Osipenko MODULE_LICENSE("GPL v2"); 1277