1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "priv.h" 25 #include "chan.h" 26 #include "chid.h" 27 #include "runl.h" 28 #include "runq.h" 29 30 #include <core/gpuobj.h> 31 #include <subdev/bar.h> 32 #include <subdev/mc.h> 33 #include <subdev/mmu.h> 34 35 #include <nvif/cl0080.h> 36 #include <nvif/unpack.h> 37 38 bool 39 nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *engine) 40 { 41 struct nvkm_runl *runl; 42 struct nvkm_engn *engn; 43 44 nvkm_runl_foreach(runl, engine->subdev.device->fifo) { 45 nvkm_runl_foreach_engn(engn, runl) { 46 if (engn->engine == engine) 47 return engn->func->chsw ? engn->func->chsw(engn) : false; 48 } 49 } 50 51 return false; 52 } 53 54 void 55 nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags) 56 { 57 return fifo->func->pause(fifo, flags); 58 } 59 60 void 61 nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags) 62 { 63 return fifo->func->start(fifo, flags); 64 } 65 66 void 67 nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) 68 { 69 return fifo->func->mmu_fault->recover(fifo, info); 70 } 71 72 static int 73 nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, 74 void *argv, u32 argc, struct nvkm_object **pobject) 75 { 76 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); 77 78 if (oclass->engn == &fifo->func->cgrp.user) 79 return nvkm_ucgrp_new(fifo, oclass, argv, argc, pobject); 80 81 if (oclass->engn == &fifo->func->chan.user) 82 return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject); 83 84 WARN_ON(1); 85 return -ENOSYS; 86 } 87 88 static const struct nvkm_device_oclass 89 nvkm_fifo_class = { 90 .ctor = nvkm_fifo_class_new, 91 }; 92 93 static int 94 nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) 95 { 96 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); 97 const struct nvkm_fifo_func_cgrp *cgrp = &fifo->func->cgrp; 98 const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; 99 int c = 0; 100 101 /* *_CHANNEL_GROUP_* */ 102 if (cgrp->user.oclass) { 103 if (c++ == index) { 104 oclass->base = cgrp->user; 105 oclass->engn = &fifo->func->cgrp.user; 106 *class = &nvkm_fifo_class; 107 return 0; 108 } 109 } 110 111 /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */ 112 if (chan->user.oclass) { 113 if (c++ == index) { 114 oclass->base = chan->user; 115 oclass->engn = &fifo->func->chan.user; 116 *class = &nvkm_fifo_class; 117 return 0; 118 } 119 } 120 121 return c; 122 } 123 124 static int 125 nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) 126 { 127 struct nvkm_fifo *fifo = nvkm_fifo(engine); 128 struct nvkm_runl *runl; 129 130 nvkm_inth_block(&fifo->engine.subdev.inth); 131 132 nvkm_runl_foreach(runl, fifo) 133 nvkm_runl_fini(runl); 134 135 return 0; 136 } 137 138 static int 139 nvkm_fifo_init(struct nvkm_engine *engine) 140 { 141 struct nvkm_fifo *fifo = nvkm_fifo(engine); 142 struct nvkm_runq *runq; 143 struct nvkm_runl *runl; 144 u32 mask = 0; 145 146 if (fifo->func->init_pbdmas) { 147 nvkm_runq_foreach(runq, fifo) 148 mask |= BIT(runq->id); 149 150 fifo->func->init_pbdmas(fifo, mask); 151 152 nvkm_runq_foreach(runq, fifo) 153 runq->func->init(runq); 154 } 155 156 nvkm_runl_foreach(runl, fifo) { 157 if (runl->func->init) 158 runl->func->init(runl); 159 } 160 161 if (fifo->func->init) 162 fifo->func->init(fifo); 163 164 nvkm_inth_allow(&fifo->engine.subdev.inth); 165 return 0; 166 } 167 168 static int 169 nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) 170 { 171 struct nvkm_fifo *fifo = nvkm_fifo(engine); 172 struct nvkm_runl *runl; 173 struct nvkm_engn *engn; 174 int ret; 175 176 ret = nvkm_subdev_oneinit(&fifo->engine.subdev); 177 if (ret) 178 return ret; 179 180 switch (mthd) { 181 case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0; 182 case NV_DEVICE_HOST_RUNLISTS: 183 *data = 0; 184 nvkm_runl_foreach(runl, fifo) 185 *data |= BIT(runl->id); 186 return 0; 187 case NV_DEVICE_HOST_RUNLIST_ENGINES: 188 runl = nvkm_runl_get(fifo, *data, 0); 189 if (runl) { 190 *data = 0; 191 nvkm_runl_foreach_engn(engn, runl) { 192 #define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break 193 switch (engn->engine->subdev.type) { 194 case NVKM_ENGINE_DMAOBJ: 195 break; 196 CASE(SW ); 197 CASE(GR ); 198 CASE(MPEG ); 199 CASE(ME ); 200 CASE(CIPHER); 201 CASE(BSP ); 202 CASE(VP ); 203 CASE(CE ); 204 CASE(SEC ); 205 CASE(MSVLD ); 206 CASE(MSPDEC); 207 CASE(MSPPP ); 208 CASE(MSENC ); 209 CASE(VIC ); 210 CASE(SEC2 ); 211 CASE(NVDEC ); 212 CASE(NVENC ); 213 CASE(NVJPG ); 214 CASE(OFA ); 215 default: 216 WARN_ON(1); 217 break; 218 } 219 #undef CASE 220 } 221 return 0; 222 } 223 return -EINVAL; 224 case NV_DEVICE_HOST_RUNLIST_CHANNELS: 225 if (!fifo->chid) { 226 runl = nvkm_runl_get(fifo, *data, 0); 227 if (runl) { 228 *data = runl->chid->nr; 229 return 0; 230 } 231 } 232 return -EINVAL; 233 default: 234 break; 235 } 236 237 return -ENOSYS; 238 } 239 240 static int 241 nvkm_fifo_oneinit(struct nvkm_engine *engine) 242 { 243 struct nvkm_subdev *subdev = &engine->subdev; 244 struct nvkm_device *device = subdev->device; 245 struct nvkm_fifo *fifo = nvkm_fifo(engine); 246 struct nvkm_runl *runl; 247 struct nvkm_engn *engn; 248 int ret, nr, i; 249 250 /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */ 251 if (fifo->func->chid_nr) { 252 ret = fifo->func->chid_ctor(fifo, fifo->func->chid_nr(fifo)); 253 if (ret) 254 return ret; 255 } 256 257 /* Create runqueues for each PBDMA. */ 258 if (fifo->func->runq_nr) { 259 for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) { 260 if (!nvkm_runq_new(fifo, i)) 261 return -ENOMEM; 262 } 263 } 264 265 /* Create runlists. */ 266 ret = fifo->func->runl_ctor(fifo); 267 if (ret) 268 return ret; 269 270 nvkm_runl_foreach(runl, fifo) { 271 RUNL_DEBUG(runl, "chan:%06x", runl->chan); 272 nvkm_runl_foreach_engn(engn, runl) { 273 ENGN_DEBUG(engn, ""); 274 } 275 } 276 277 /* Register interrupt handler. */ 278 if (fifo->func->intr) { 279 ret = nvkm_inth_add(&device->mc->intr, NVKM_INTR_SUBDEV, NVKM_INTR_PRIO_NORMAL, 280 subdev, fifo->func->intr, &subdev->inth); 281 if (ret) { 282 nvkm_error(subdev, "intr %d\n", ret); 283 return ret; 284 } 285 } 286 287 /* Initialise non-stall intr handling. */ 288 if (fifo->func->nonstall) { 289 if (fifo->func->nonstall_ctor) { 290 ret = fifo->func->nonstall_ctor(fifo); 291 if (ret < 0) { 292 nvkm_error(subdev, "nonstall %d\n", ret); 293 return ret; 294 } 295 } else { 296 ret = 1; 297 } 298 299 ret = nvkm_event_init(fifo->func->nonstall, &fifo->engine.subdev, 1, ret, 300 &fifo->nonstall.event); 301 if (ret) 302 return ret; 303 } 304 305 /* Allocate USERD + BAR1 polling area. */ 306 if (fifo->func->chan.func->userd->bar == 1) { 307 struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device); 308 309 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->chid->nr * 310 fifo->func->chan.func->userd->size, 0, true, 311 &fifo->userd.mem); 312 if (ret) 313 return ret; 314 315 ret = nvkm_vmm_get(bar1, 12, nvkm_memory_size(fifo->userd.mem), &fifo->userd.bar1); 316 if (ret) 317 return ret; 318 319 ret = nvkm_memory_map(fifo->userd.mem, 0, bar1, fifo->userd.bar1, NULL, 0); 320 if (ret) 321 return ret; 322 } 323 324 return 0; 325 } 326 327 static void 328 nvkm_fifo_preinit(struct nvkm_engine *engine) 329 { 330 nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0); 331 } 332 333 static void * 334 nvkm_fifo_dtor(struct nvkm_engine *engine) 335 { 336 struct nvkm_fifo *fifo = nvkm_fifo(engine); 337 struct nvkm_runl *runl, *runt; 338 struct nvkm_runq *runq, *rtmp; 339 340 if (fifo->userd.bar1) 341 nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1); 342 nvkm_memory_unref(&fifo->userd.mem); 343 344 list_for_each_entry_safe(runl, runt, &fifo->runls, head) 345 nvkm_runl_del(runl); 346 list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head) 347 nvkm_runq_del(runq); 348 349 nvkm_chid_unref(&fifo->cgid); 350 nvkm_chid_unref(&fifo->chid); 351 352 mutex_destroy(&fifo->userd.mutex); 353 354 nvkm_event_fini(&fifo->nonstall.event); 355 mutex_destroy(&fifo->mutex); 356 357 if (fifo->func->dtor) 358 fifo->func->dtor(fifo); 359 360 return fifo; 361 } 362 363 static const struct nvkm_engine_func 364 nvkm_fifo = { 365 .dtor = nvkm_fifo_dtor, 366 .preinit = nvkm_fifo_preinit, 367 .oneinit = nvkm_fifo_oneinit, 368 .info = nvkm_fifo_info, 369 .init = nvkm_fifo_init, 370 .fini = nvkm_fifo_fini, 371 .base.sclass = nvkm_fifo_class_get, 372 }; 373 374 int 375 nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 376 enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) 377 { 378 struct nvkm_fifo *fifo; 379 380 if (!(fifo = *pfifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 381 return -ENOMEM; 382 383 fifo->func = func; 384 INIT_LIST_HEAD(&fifo->runqs); 385 INIT_LIST_HEAD(&fifo->runls); 386 /*TODO: Needs to be >CTXSW_TIMEOUT, so RC can recover before this is hit. 387 * CTXSW_TIMEOUT HW default seems to differ between GPUs, so just a 388 * large number for now until we support changing it. 389 */ 390 fifo->timeout.chan_msec = 10000; 391 spin_lock_init(&fifo->lock); 392 mutex_init(&fifo->mutex); 393 394 INIT_LIST_HEAD(&fifo->userd.list); 395 mutex_init(&fifo->userd.mutex); 396 397 return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine); 398 } 399