1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/slab.h> 20 #include <linux/scatterlist.h> 21 #include <linux/dma-mapping.h> 22 23 #include "../cdma.h" 24 #include "../channel.h" 25 #include "../dev.h" 26 #include "../debug.h" 27 28 /* 29 * Put the restart at the end of pushbuffer memory 30 */ 31 static void push_buffer_init(struct push_buffer *pb) 32 { 33 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); 34 } 35 36 /* 37 * Increment timedout buffer's syncpt via CPU. 38 */ 39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 40 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 41 { 42 struct host1x *host1x = cdma_to_host1x(cdma); 43 struct push_buffer *pb = &cdma->push_buffer; 44 unsigned int i; 45 46 for (i = 0; i < syncpt_incrs; i++) 47 host1x_syncpt_incr(cdma->timeout.syncpt); 48 49 /* after CPU incr, ensure shadow is up to date */ 50 host1x_syncpt_load(cdma->timeout.syncpt); 51 52 /* NOP all the PB slots */ 53 while (nr_slots--) { 54 u32 *p = (u32 *)(pb->mapped + getptr); 55 *(p++) = HOST1X_OPCODE_NOP; 56 *(p++) = HOST1X_OPCODE_NOP; 57 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__, 58 &pb->dma, getptr); 59 getptr = (getptr + 8) & (pb->size - 1); 60 } 61 62 wmb(); 63 } 64 65 /* 66 * Start channel DMA 67 */ 68 static void cdma_start(struct host1x_cdma *cdma) 69 { 70 struct host1x_channel *ch = cdma_to_channel(cdma); 71 72 if (cdma->running) 73 return; 74 75 cdma->last_pos = cdma->push_buffer.pos; 76 77 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 78 HOST1X_CHANNEL_DMACTRL); 79 80 /* set base, put and end pointer */ 81 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); 82 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 83 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size + 4, 84 HOST1X_CHANNEL_DMAEND); 85 86 /* reset GET */ 87 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 88 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 89 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 90 HOST1X_CHANNEL_DMACTRL); 91 92 /* start the command DMA */ 93 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 94 95 cdma->running = true; 96 } 97 98 /* 99 * Similar to cdma_start(), but rather than starting from an idle 100 * state (where DMA GET is set to DMA PUT), on a timeout we restore 101 * DMA GET from an explicit value (so DMA may again be pending). 102 */ 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 104 { 105 struct host1x *host1x = cdma_to_host1x(cdma); 106 struct host1x_channel *ch = cdma_to_channel(cdma); 107 108 if (cdma->running) 109 return; 110 111 cdma->last_pos = cdma->push_buffer.pos; 112 113 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 114 HOST1X_CHANNEL_DMACTRL); 115 116 /* set base, end pointer (all of memory) */ 117 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); 118 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size, 119 HOST1X_CHANNEL_DMAEND); 120 121 /* set GET, by loading the value in PUT (then reset GET) */ 122 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 123 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 124 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 125 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 126 HOST1X_CHANNEL_DMACTRL); 127 128 dev_dbg(host1x->dev, 129 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 130 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 131 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 132 cdma->last_pos); 133 134 /* deassert GET reset and set PUT */ 135 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 136 HOST1X_CHANNEL_DMACTRL); 137 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 138 139 /* start the command DMA */ 140 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 141 142 cdma->running = true; 143 } 144 145 /* 146 * Kick channel DMA into action by writing its PUT offset (if it has changed) 147 */ 148 static void cdma_flush(struct host1x_cdma *cdma) 149 { 150 struct host1x_channel *ch = cdma_to_channel(cdma); 151 152 if (cdma->push_buffer.pos != cdma->last_pos) { 153 host1x_ch_writel(ch, cdma->push_buffer.pos, 154 HOST1X_CHANNEL_DMAPUT); 155 cdma->last_pos = cdma->push_buffer.pos; 156 } 157 } 158 159 static void cdma_stop(struct host1x_cdma *cdma) 160 { 161 struct host1x_channel *ch = cdma_to_channel(cdma); 162 163 mutex_lock(&cdma->lock); 164 165 if (cdma->running) { 166 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 167 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 168 HOST1X_CHANNEL_DMACTRL); 169 cdma->running = false; 170 } 171 172 mutex_unlock(&cdma->lock); 173 } 174 175 /* 176 * Stops both channel's command processor and CDMA immediately. 177 * Also, tears down the channel and resets corresponding module. 178 */ 179 static void cdma_freeze(struct host1x_cdma *cdma) 180 { 181 struct host1x *host = cdma_to_host1x(cdma); 182 struct host1x_channel *ch = cdma_to_channel(cdma); 183 u32 cmdproc_stop; 184 185 if (cdma->torndown && !cdma->running) { 186 dev_warn(host->dev, "Already torn down\n"); 187 return; 188 } 189 190 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 191 192 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 193 cmdproc_stop |= BIT(ch->id); 194 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 195 196 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 197 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 198 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 199 cdma->last_pos); 200 201 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 202 HOST1X_CHANNEL_DMACTRL); 203 204 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 205 206 cdma->running = false; 207 cdma->torndown = true; 208 } 209 210 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 211 { 212 struct host1x *host1x = cdma_to_host1x(cdma); 213 struct host1x_channel *ch = cdma_to_channel(cdma); 214 u32 cmdproc_stop; 215 216 dev_dbg(host1x->dev, 217 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 218 ch->id, getptr); 219 220 cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 221 cmdproc_stop &= ~BIT(ch->id); 222 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 223 224 cdma->torndown = false; 225 cdma_timeout_restart(cdma, getptr); 226 } 227 228 /* 229 * If this timeout fires, it indicates the current sync_queue entry has 230 * exceeded its TTL and the userctx should be timed out and remaining 231 * submits already issued cleaned up (future submits return an error). 232 */ 233 static void cdma_timeout_handler(struct work_struct *work) 234 { 235 u32 prev_cmdproc, cmdproc_stop, syncpt_val; 236 struct host1x_cdma *cdma; 237 struct host1x *host1x; 238 struct host1x_channel *ch; 239 240 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 241 timeout.wq); 242 host1x = cdma_to_host1x(cdma); 243 ch = cdma_to_channel(cdma); 244 245 host1x_debug_dump(cdma_to_host1x(cdma)); 246 247 mutex_lock(&cdma->lock); 248 249 if (!cdma->timeout.client) { 250 dev_dbg(host1x->dev, 251 "cdma_timeout: expired, but has no clientid\n"); 252 mutex_unlock(&cdma->lock); 253 return; 254 } 255 256 /* stop processing to get a clean snapshot */ 257 prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 258 cmdproc_stop = prev_cmdproc | BIT(ch->id); 259 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 260 261 dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n", 262 prev_cmdproc, cmdproc_stop); 263 264 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 265 266 /* has buffer actually completed? */ 267 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 268 dev_dbg(host1x->dev, 269 "cdma_timeout: expired, but buffer had completed\n"); 270 /* restore */ 271 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id)); 272 host1x_sync_writel(host1x, cmdproc_stop, 273 HOST1X_SYNC_CMDPROC_STOP); 274 mutex_unlock(&cdma->lock); 275 return; 276 } 277 278 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 279 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 280 syncpt_val, cdma->timeout.syncpt_val); 281 282 /* stop HW, resetting channel/module */ 283 host1x_hw_cdma_freeze(host1x, cdma); 284 285 host1x_cdma_update_sync_queue(cdma, ch->dev); 286 mutex_unlock(&cdma->lock); 287 } 288 289 /* 290 * Init timeout resources 291 */ 292 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt) 293 { 294 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 295 cdma->timeout.initialized = true; 296 297 return 0; 298 } 299 300 /* 301 * Clean up timeout resources 302 */ 303 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 304 { 305 if (cdma->timeout.initialized) 306 cancel_delayed_work(&cdma->timeout.wq); 307 308 cdma->timeout.initialized = false; 309 } 310 311 static const struct host1x_cdma_ops host1x_cdma_ops = { 312 .start = cdma_start, 313 .stop = cdma_stop, 314 .flush = cdma_flush, 315 316 .timeout_init = cdma_timeout_init, 317 .timeout_destroy = cdma_timeout_destroy, 318 .freeze = cdma_freeze, 319 .resume = cdma_resume, 320 .timeout_cpu_incr = cdma_timeout_cpu_incr, 321 }; 322 323 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 324 .init = push_buffer_init, 325 }; 326