1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "priv.h" 25 #include "cgrp.h" 26 #include "chan.h" 27 #include "chid.h" 28 #include "runl.h" 29 30 #include <core/ramht.h> 31 #include <subdev/timer.h> 32 33 #include <nvif/class.h> 34 35 void 36 nv50_eobj_ramht_del(struct nvkm_chan *chan, int hash) 37 { 38 nvkm_ramht_remove(chan->ramht, hash); 39 } 40 41 int 42 nv50_eobj_ramht_add(struct nvkm_engn *engn, struct nvkm_object *eobj, struct nvkm_chan *chan) 43 { 44 return nvkm_ramht_insert(chan->ramht, eobj, 0, 4, eobj->handle, engn->id << 20); 45 } 46 47 void 48 nv50_chan_stop(struct nvkm_chan *chan) 49 { 50 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; 51 52 nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000); 53 } 54 55 void 56 nv50_chan_start(struct nvkm_chan *chan) 57 { 58 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; 59 60 nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x80000000); 61 } 62 63 void 64 nv50_chan_unbind(struct nvkm_chan *chan) 65 { 66 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; 67 68 nvkm_wr32(device, 0x002600 + (chan->id * 4), 0x00000000); 69 } 70 71 static void 72 nv50_chan_bind(struct nvkm_chan *chan) 73 { 74 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; 75 76 nvkm_wr32(device, 0x002600 + (chan->id * 4), chan->ramfc->addr >> 12); 77 } 78 79 static int 80 nv50_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) 81 { 82 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; 83 const u32 limit2 = ilog2(length / 8); 84 int ret; 85 86 ret = nvkm_gpuobj_new(device, 0x0200, 0x1000, true, chan->inst, &chan->ramfc); 87 if (ret) 88 return ret; 89 90 ret = nvkm_gpuobj_new(device, 0x1200, 0, true, chan->inst, &chan->eng); 91 if (ret) 92 return ret; 93 94 ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->inst, &chan->pgd); 95 if (ret) 96 return ret; 97 98 ret = nvkm_ramht_new(device, 0x8000, 16, chan->inst, &chan->ramht); 99 if (ret) 100 return ret; 101 102 nvkm_kmap(chan->ramfc); 103 nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); 104 nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); 105 nvkm_wo32(chan->ramfc, 0x48, chan->push->node->offset >> 4); 106 nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(offset)); 107 nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(offset) | (limit2 << 16)); 108 nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); 109 nvkm_wo32(chan->ramfc, 0x78, 0x00000000); 110 nvkm_wo32(chan->ramfc, 0x7c, 0x30000000 | devm); 111 nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | 112 (4 << 24) /* SEARCH_FULL */ | 113 (chan->ramht->gpuobj->node->offset >> 4)); 114 nvkm_done(chan->ramfc); 115 return 0; 116 } 117 118 static const struct nvkm_chan_func_ramfc 119 nv50_chan_ramfc = { 120 .write = nv50_chan_ramfc_write, 121 .ctxdma = true, 122 .devm = 0xfff, 123 }; 124 125 const struct nvkm_chan_func_userd 126 nv50_chan_userd = { 127 .bar = 0, 128 .base = 0xc00000, 129 .size = 0x002000, 130 }; 131 132 const struct nvkm_chan_func_inst 133 nv50_chan_inst = { 134 .size = 0x10000, 135 .vmm = true, 136 }; 137 138 static const struct nvkm_chan_func 139 nv50_chan = { 140 .inst = &nv50_chan_inst, 141 .userd = &nv50_chan_userd, 142 .ramfc = &nv50_chan_ramfc, 143 .bind = nv50_chan_bind, 144 .unbind = nv50_chan_unbind, 145 .start = nv50_chan_start, 146 .stop = nv50_chan_stop, 147 }; 148 149 static void 150 nv50_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) 151 { 152 struct nvkm_subdev *subdev = &chan->cgrp->runl->fifo->engine.subdev; 153 struct nvkm_device *device = subdev->device; 154 u64 start = 0, limit = 0; 155 u32 flags = 0, ptr0, save; 156 157 switch (engn->engine->subdev.type) { 158 case NVKM_ENGINE_GR : ptr0 = 0x0000; break; 159 case NVKM_ENGINE_MPEG : ptr0 = 0x0060; break; 160 default: 161 WARN_ON(1); 162 return; 163 } 164 165 if (!cctx) { 166 /* HW bug workaround: 167 * 168 * PFIFO will hang forever if the connected engines don't report 169 * that they've processed the context switch request. 170 * 171 * In order for the kickoff to work, we need to ensure all the 172 * connected engines are in a state where they can answer. 173 * 174 * Newer chipsets don't seem to suffer from this issue, and well, 175 * there's also a "ignore these engines" bitmask reg we can use 176 * if we hit the issue there.. 177 */ 178 save = nvkm_mask(device, 0x00b860, 0x00000001, 0x00000001); 179 180 /* Tell engines to save out contexts. */ 181 nvkm_wr32(device, 0x0032fc, chan->inst->addr >> 12); 182 nvkm_msec(device, 2000, 183 if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) 184 break; 185 ); 186 nvkm_wr32(device, 0x00b860, save); 187 } else { 188 flags = 0x00190000; 189 start = cctx->vctx->inst->addr; 190 limit = start + cctx->vctx->inst->size - 1; 191 } 192 193 nvkm_kmap(chan->eng); 194 nvkm_wo32(chan->eng, ptr0 + 0x00, flags); 195 nvkm_wo32(chan->eng, ptr0 + 0x04, lower_32_bits(limit)); 196 nvkm_wo32(chan->eng, ptr0 + 0x08, lower_32_bits(start)); 197 nvkm_wo32(chan->eng, ptr0 + 0x0c, upper_32_bits(limit) << 24 | 198 lower_32_bits(start)); 199 nvkm_wo32(chan->eng, ptr0 + 0x10, 0x00000000); 200 nvkm_wo32(chan->eng, ptr0 + 0x14, 0x00000000); 201 nvkm_done(chan->eng); 202 } 203 204 static const struct nvkm_engn_func 205 nv50_engn = { 206 .bind = nv50_ectx_bind, 207 .ramht_add = nv50_eobj_ramht_add, 208 .ramht_del = nv50_eobj_ramht_del, 209 }; 210 211 const struct nvkm_engn_func 212 nv50_engn_sw = { 213 .ramht_add = nv50_eobj_ramht_add, 214 .ramht_del = nv50_eobj_ramht_del, 215 }; 216 217 static bool 218 nv50_runl_pending(struct nvkm_runl *runl) 219 { 220 return nvkm_rd32(runl->fifo->engine.subdev.device, 0x0032ec) & 0x00000100; 221 } 222 223 int 224 nv50_runl_wait(struct nvkm_runl *runl) 225 { 226 struct nvkm_fifo *fifo = runl->fifo; 227 228 nvkm_msec(fifo->engine.subdev.device, fifo->timeout.chan_msec, 229 if (!nvkm_runl_update_pending(runl)) 230 return 0; 231 usleep_range(1, 2); 232 ); 233 234 return -ETIMEDOUT; 235 } 236 237 static void 238 nv50_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) 239 { 240 struct nvkm_device *device = runl->fifo->engine.subdev.device; 241 u64 addr = nvkm_memory_addr(memory) + start; 242 243 nvkm_wr32(device, 0x0032f4, addr >> 12); 244 nvkm_wr32(device, 0x0032ec, count); 245 } 246 247 static void 248 nv50_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) 249 { 250 nvkm_wo32(memory, offset, chan->id); 251 } 252 253 static struct nvkm_memory * 254 nv50_runl_alloc(struct nvkm_runl *runl, u32 *offset) 255 { 256 const u32 segment = ALIGN((runl->cgrp_nr + runl->chan_nr) * runl->func->size, 0x1000); 257 const u32 maxsize = (runl->cgid ? runl->cgid->nr : 0) + runl->chid->nr; 258 int ret; 259 260 if (unlikely(!runl->mem)) { 261 ret = nvkm_memory_new(runl->fifo->engine.subdev.device, NVKM_MEM_TARGET_INST, 262 maxsize * 2 * runl->func->size, 0, false, &runl->mem); 263 if (ret) { 264 RUNL_ERROR(runl, "alloc %d\n", ret); 265 return ERR_PTR(ret); 266 } 267 } else { 268 if (runl->offset + segment >= nvkm_memory_size(runl->mem)) { 269 ret = runl->func->wait(runl); 270 if (ret) { 271 RUNL_DEBUG(runl, "rewind timeout"); 272 return ERR_PTR(ret); 273 } 274 275 runl->offset = 0; 276 } 277 } 278 279 *offset = runl->offset; 280 runl->offset += segment; 281 return runl->mem; 282 } 283 284 int 285 nv50_runl_update(struct nvkm_runl *runl) 286 { 287 struct nvkm_memory *memory; 288 struct nvkm_cgrp *cgrp; 289 struct nvkm_chan *chan; 290 u32 start, offset, count; 291 292 /*TODO: prio, interleaving. */ 293 294 RUNL_TRACE(runl, "RAMRL: update cgrps:%d chans:%d", runl->cgrp_nr, runl->chan_nr); 295 memory = nv50_runl_alloc(runl, &start); 296 if (IS_ERR(memory)) 297 return PTR_ERR(memory); 298 299 RUNL_TRACE(runl, "RAMRL: update start:%08x", start); 300 offset = start; 301 302 nvkm_kmap(memory); 303 nvkm_runl_foreach_cgrp(cgrp, runl) { 304 if (cgrp->hw) { 305 CGRP_TRACE(cgrp, " RAMRL+%08x: chans:%d", offset, cgrp->chan_nr); 306 runl->func->insert_cgrp(cgrp, memory, offset); 307 offset += runl->func->size; 308 } 309 310 nvkm_cgrp_foreach_chan(chan, cgrp) { 311 CHAN_TRACE(chan, "RAMRL+%08x: [%s]", offset, chan->name); 312 runl->func->insert_chan(chan, memory, offset); 313 offset += runl->func->size; 314 } 315 } 316 nvkm_done(memory); 317 318 /*TODO: look into using features on newer HW to guarantee forward progress. */ 319 list_rotate_left(&runl->cgrps); 320 321 count = (offset - start) / runl->func->size; 322 RUNL_TRACE(runl, "RAMRL: commit start:%08x count:%d", start, count); 323 324 runl->func->commit(runl, memory, start, count); 325 return 0; 326 } 327 328 const struct nvkm_runl_func 329 nv50_runl = { 330 .size = 4, 331 .update = nv50_runl_update, 332 .insert_chan = nv50_runl_insert_chan, 333 .commit = nv50_runl_commit, 334 .wait = nv50_runl_wait, 335 .pending = nv50_runl_pending, 336 }; 337 338 void 339 nv50_fifo_init(struct nvkm_fifo *fifo) 340 { 341 struct nvkm_runl *runl = nvkm_runl_first(fifo); 342 struct nvkm_device *device = fifo->engine.subdev.device; 343 int i; 344 345 nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); 346 nvkm_mask(device, 0x000200, 0x00000100, 0x00000100); 347 nvkm_wr32(device, 0x00250c, 0x6f3cfc34); 348 nvkm_wr32(device, 0x002044, 0x01003fff); 349 350 nvkm_wr32(device, 0x002100, 0xffffffff); 351 nvkm_wr32(device, 0x002140, 0xbfffffff); 352 353 for (i = 0; i < 128; i++) 354 nvkm_wr32(device, 0x002600 + (i * 4), 0x00000000); 355 356 atomic_set(&runl->changed, 1); 357 runl->func->update(runl); 358 359 nvkm_wr32(device, 0x003200, 0x00000001); 360 nvkm_wr32(device, 0x003250, 0x00000001); 361 nvkm_wr32(device, 0x002500, 0x00000001); 362 } 363 364 int 365 nv50_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) 366 { 367 /* CHID 0 is unusable (some kind of PIO channel?), 127 is "channel invalid". */ 368 return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 1, nr - 2, &fifo->chid); 369 } 370 371 int 372 nv50_fifo_chid_nr(struct nvkm_fifo *fifo) 373 { 374 return 128; 375 } 376 377 static const struct nvkm_fifo_func 378 nv50_fifo = { 379 .chid_nr = nv50_fifo_chid_nr, 380 .chid_ctor = nv50_fifo_chid_ctor, 381 .runl_ctor = nv04_fifo_runl_ctor, 382 .init = nv50_fifo_init, 383 .intr = nv04_fifo_intr, 384 .pause = nv04_fifo_pause, 385 .start = nv04_fifo_start, 386 .runl = &nv50_runl, 387 .engn = &nv50_engn, 388 .engn_sw = &nv50_engn_sw, 389 .cgrp = {{ }, &nv04_cgrp }, 390 .chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan }, 391 }; 392 393 int 394 nv50_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 395 struct nvkm_fifo **pfifo) 396 { 397 return nvkm_fifo_new_(&nv50_fifo, device, type, inst, pfifo); 398 } 399