1 /* 2 * Copyright 2021 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "priv.h" 23 #include "cgrp.h" 24 #include "chan.h" 25 #include "chid.h" 26 #include "runl.h" 27 #include "runq.h" 28 29 #include <core/gpuobj.h> 30 #include <subdev/top.h> 31 #include <subdev/vfn.h> 32 33 #include <nvif/class.h> 34 35 /*TODO: allocate? */ 36 #define GA100_FIFO_NONSTALL_VECTOR 0 37 38 static u32 39 ga100_chan_doorbell_handle(struct nvkm_chan *chan) 40 { 41 return (chan->cgrp->runl->doorbell << 16) | chan->id; 42 } 43 44 static void 45 ga100_chan_stop(struct nvkm_chan *chan) 46 { 47 struct nvkm_runl *runl = chan->cgrp->runl; 48 49 nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0x00000003); 50 } 51 52 static void 53 ga100_chan_start(struct nvkm_chan *chan) 54 { 55 struct nvkm_runl *runl = chan->cgrp->runl; 56 struct nvkm_device *device = runl->fifo->engine.subdev.device; 57 const int gfid = 0; 58 59 nvkm_wr32(device, runl->chan + (chan->id * 4), 0x00000002); 60 nvkm_wr32(device, runl->addr + 0x0090, (gfid << 16) | chan->id); /* INTERNAL_DOORBELL. */ 61 } 62 63 static void 64 ga100_chan_unbind(struct nvkm_chan *chan) 65 { 66 struct nvkm_runl *runl = chan->cgrp->runl; 67 68 nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0xffffffff); 69 } 70 71 static int 72 ga100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) 73 { 74 const u32 limit2 = ilog2(length / 8); 75 76 nvkm_kmap(chan->inst); 77 nvkm_wo32(chan->inst, 0x010, 0x0000face); 78 nvkm_wo32(chan->inst, 0x030, 0x7ffff902); 79 nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset)); 80 nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16)); 81 nvkm_wo32(chan->inst, 0x084, 0x20400000); 82 nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm); 83 nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000); 84 nvkm_wo32(chan->inst, 0x0e8, chan->id); 85 nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000)); 86 nvkm_wo32(chan->inst, 0x0f8, 0x80000000 | GA100_FIFO_NONSTALL_VECTOR); 87 nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); 88 nvkm_done(chan->inst); 89 return 0; 90 } 91 92 static const struct nvkm_chan_func_ramfc 93 ga100_chan_ramfc = { 94 .write = ga100_chan_ramfc_write, 95 .devm = 0xfff, 96 .priv = true, 97 }; 98 99 const struct nvkm_chan_func 100 ga100_chan = { 101 .inst = &gf100_chan_inst, 102 .userd = &gv100_chan_userd, 103 .ramfc = &ga100_chan_ramfc, 104 .unbind = ga100_chan_unbind, 105 .start = ga100_chan_start, 106 .stop = ga100_chan_stop, 107 .preempt = gk110_chan_preempt, 108 .doorbell_handle = ga100_chan_doorbell_handle, 109 }; 110 111 static void 112 ga100_cgrp_preempt(struct nvkm_cgrp *cgrp) 113 { 114 struct nvkm_runl *runl = cgrp->runl; 115 116 nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x01000000 | cgrp->id); 117 } 118 119 const struct nvkm_cgrp_func 120 ga100_cgrp = { 121 .preempt = ga100_cgrp_preempt, 122 }; 123 124 static int 125 ga100_engn_cxid(struct nvkm_engn *engn, bool *cgid) 126 { 127 struct nvkm_runl *runl = engn->runl; 128 struct nvkm_device *device = runl->fifo->engine.subdev.device; 129 u32 stat = nvkm_rd32(device, runl->addr + 0x200 + engn->id * 0x40); 130 131 ENGN_DEBUG(engn, "status %08x", stat); 132 *cgid = true; 133 134 switch ((stat & 0x0000e000) >> 13) { 135 case 0 /* INVALID */: return -ENODEV; 136 case 1 /* VALID */: 137 case 5 /* SAVE */: return (stat & 0x00000fff); 138 case 6 /* LOAD */: return (stat & 0x0fff0000) >> 16; 139 case 7 /* SWITCH */: 140 if (nvkm_engine_chsw_load(engn->engine)) 141 return (stat & 0x0fff0000) >> 16; 142 return (stat & 0x00000fff); 143 default: 144 WARN_ON(1); 145 break; 146 } 147 148 return -ENODEV; 149 } 150 151 const struct nvkm_engn_func 152 ga100_engn = { 153 .cxid = ga100_engn_cxid, 154 .ctor = gk104_ectx_ctor, 155 .bind = gv100_ectx_bind, 156 }; 157 158 const struct nvkm_engn_func 159 ga100_engn_ce = { 160 .cxid = ga100_engn_cxid, 161 .ctor = gv100_ectx_ce_ctor, 162 .bind = gv100_ectx_ce_bind, 163 }; 164 165 static bool 166 ga100_runq_idle(struct nvkm_runq *runq) 167 { 168 struct nvkm_device *device = runq->fifo->engine.subdev.device; 169 170 return !(nvkm_rd32(device, 0x04015c + (runq->id * 0x800)) & 0x0000e000); 171 } 172 173 static bool 174 ga100_runq_intr_1(struct nvkm_runq *runq, struct nvkm_runl *runl) 175 { 176 struct nvkm_device *device = runq->fifo->engine.subdev.device; 177 u32 inte = nvkm_rd32(device, 0x040180 + (runq->id * 0x800)); 178 u32 intr = nvkm_rd32(device, 0x040148 + (runq->id * 0x800)); 179 u32 stat = intr & inte; 180 181 if (!stat) { 182 RUNQ_DEBUG(runq, "inte1 %08x %08x", intr, inte); 183 return false; 184 } 185 186 if (stat & 0x80000000) { 187 u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; 188 struct nvkm_chan *chan; 189 unsigned long flags; 190 191 RUNQ_ERROR(runq, "CTXNOTVALID chid:%d", chid); 192 chan = nvkm_runl_chan_get_chid(runl, chid, &flags); 193 if (chan) { 194 nvkm_chan_error(chan, true); 195 nvkm_chan_put(&chan, flags); 196 } 197 198 nvkm_mask(device, 0x0400ac + (runq->id * 0x800), 0x00030000, 0x00030000); 199 stat &= ~0x80000000; 200 } 201 202 if (stat) { 203 RUNQ_ERROR(runq, "intr1 %08x", stat); 204 nvkm_wr32(device, 0x0401a0 + (runq->id * 0x800), stat); 205 } 206 207 nvkm_wr32(device, 0x040148 + (runq->id * 0x800), intr); 208 return true; 209 } 210 211 static bool 212 ga100_runq_intr_0(struct nvkm_runq *runq, struct nvkm_runl *runl) 213 { 214 struct nvkm_device *device = runq->fifo->engine.subdev.device; 215 u32 inte = nvkm_rd32(device, 0x040170 + (runq->id * 0x800)); 216 u32 intr = nvkm_rd32(device, 0x040108 + (runq->id * 0x800)); 217 u32 stat = intr & inte; 218 219 if (!stat) { 220 RUNQ_DEBUG(runq, "inte0 %08x %08x", intr, inte); 221 return false; 222 } 223 224 /*TODO: expand on this when fixing up gf100's version. */ 225 if (stat & 0xc6afe000) { 226 u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; 227 struct nvkm_chan *chan; 228 unsigned long flags; 229 230 RUNQ_ERROR(runq, "intr0 %08x", stat); 231 chan = nvkm_runl_chan_get_chid(runl, chid, &flags); 232 if (chan) { 233 nvkm_chan_error(chan, true); 234 nvkm_chan_put(&chan, flags); 235 } 236 237 stat &= ~0xc6afe000; 238 } 239 240 if (stat) { 241 RUNQ_ERROR(runq, "intr0 %08x", stat); 242 nvkm_wr32(device, 0x040190 + (runq->id * 0x800), stat); 243 } 244 245 nvkm_wr32(device, 0x040108 + (runq->id * 0x800), intr); 246 return true; 247 } 248 249 static bool 250 ga100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *runl) 251 { 252 bool intr0 = ga100_runq_intr_0(runq, runl); 253 bool intr1 = ga100_runq_intr_1(runq, runl); 254 255 return intr0 || intr1; 256 } 257 258 static void 259 ga100_runq_init(struct nvkm_runq *runq) 260 { 261 struct nvkm_device *device = runq->fifo->engine.subdev.device; 262 263 nvkm_wr32(device, 0x040108 + (runq->id * 0x800), 0xffffffff); /* INTR_0 */ 264 nvkm_wr32(device, 0x040148 + (runq->id * 0x800), 0xffffffff); /* INTR_1 */ 265 nvkm_wr32(device, 0x040170 + (runq->id * 0x800), 0xffffffff); /* INTR_0_EN_SET_TREE */ 266 nvkm_wr32(device, 0x040180 + (runq->id * 0x800), 0xffffffff); /* INTR_1_EN_SET_TREE */ 267 } 268 269 const struct nvkm_runq_func 270 ga100_runq = { 271 .init = ga100_runq_init, 272 .intr = ga100_runq_intr, 273 .idle = ga100_runq_idle, 274 }; 275 276 static bool 277 ga100_runl_preempt_pending(struct nvkm_runl *runl) 278 { 279 return nvkm_rd32(runl->fifo->engine.subdev.device, runl->addr + 0x098) & 0x00100000; 280 } 281 282 static void 283 ga100_runl_preempt(struct nvkm_runl *runl) 284 { 285 nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x00000000); 286 } 287 288 static void 289 ga100_runl_allow(struct nvkm_runl *runl, u32 engm) 290 { 291 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000000); 292 } 293 294 static void 295 ga100_runl_block(struct nvkm_runl *runl, u32 engm) 296 { 297 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000001); 298 } 299 300 static bool 301 ga100_runl_pending(struct nvkm_runl *runl) 302 { 303 struct nvkm_device *device = runl->fifo->engine.subdev.device; 304 305 return nvkm_rd32(device, runl->addr + 0x08c) & 0x00008000; 306 } 307 308 static void 309 ga100_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) 310 { 311 struct nvkm_device *device = runl->fifo->engine.subdev.device; 312 u64 addr = nvkm_memory_addr(memory) + start; 313 314 nvkm_wr32(device, runl->addr + 0x080, lower_32_bits(addr)); 315 nvkm_wr32(device, runl->addr + 0x084, upper_32_bits(addr)); 316 nvkm_wr32(device, runl->addr + 0x088, count); 317 } 318 319 static irqreturn_t 320 ga100_runl_intr(struct nvkm_inth *inth) 321 { 322 struct nvkm_runl *runl = container_of(inth, typeof(*runl), inth); 323 struct nvkm_engn *engn; 324 struct nvkm_device *device = runl->fifo->engine.subdev.device; 325 u32 inte = nvkm_rd32(device, runl->addr + 0x120); 326 u32 intr = nvkm_rd32(device, runl->addr + 0x100); 327 u32 stat = intr & inte; 328 u32 info; 329 330 if (!stat) { 331 RUNL_DEBUG(runl, "inte %08x %08x", intr, inte); 332 return IRQ_NONE; 333 } 334 335 if (stat & 0x00000007) { 336 nvkm_runl_foreach_engn_cond(engn, runl, stat & BIT(engn->id)) { 337 info = nvkm_rd32(device, runl->addr + 0x224 + (engn->id * 0x40)); 338 339 tu102_fifo_intr_ctxsw_timeout_info(engn, info); 340 341 nvkm_wr32(device, runl->addr + 0x100, BIT(engn->id)); 342 stat &= ~BIT(engn->id); 343 } 344 } 345 346 if (stat & 0x00000300) { 347 nvkm_wr32(device, runl->addr + 0x100, stat & 0x00000300); 348 stat &= ~0x00000300; 349 } 350 351 if (stat & 0x00010000) { 352 if (runl->runq[0]) { 353 if (runl->runq[0]->func->intr(runl->runq[0], runl)) 354 stat &= ~0x00010000; 355 } 356 } 357 358 if (stat & 0x00020000) { 359 if (runl->runq[1]) { 360 if (runl->runq[1]->func->intr(runl->runq[1], runl)) 361 stat &= ~0x00020000; 362 } 363 } 364 365 if (stat) { 366 RUNL_ERROR(runl, "intr %08x", stat); 367 nvkm_wr32(device, runl->addr + 0x140, stat); 368 } 369 370 nvkm_wr32(device, runl->addr + 0x180, 0x00000001); 371 return IRQ_HANDLED; 372 } 373 374 static void 375 ga100_runl_fini(struct nvkm_runl *runl) 376 { 377 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x300, 0x80000000, 0x00000000); 378 nvkm_inth_block(&runl->inth); 379 } 380 381 static void 382 ga100_runl_init(struct nvkm_runl *runl) 383 { 384 struct nvkm_fifo *fifo = runl->fifo; 385 struct nvkm_runq *runq; 386 struct nvkm_device *device = fifo->engine.subdev.device; 387 int i; 388 389 /* Submit NULL runlist and preempt. */ 390 nvkm_wr32(device, runl->addr + 0x088, 0x00000000); 391 runl->func->preempt(runl); 392 393 /* Enable doorbell. */ 394 nvkm_mask(device, runl->addr + 0x300, 0x80000000, 0x80000000); 395 396 nvkm_wr32(device, runl->addr + 0x100, 0xffffffff); /* INTR_0 */ 397 nvkm_wr32(device, runl->addr + 0x140, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(0) */ 398 nvkm_wr32(device, runl->addr + 0x120, 0x000f1307); /* INTR_0_EN_SET_TREE(0) */ 399 nvkm_wr32(device, runl->addr + 0x148, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(1) */ 400 nvkm_wr32(device, runl->addr + 0x128, 0x00000000); /* INTR_0_EN_SET_TREE(1) */ 401 402 /* Init PBDMA(s). */ 403 for (i = 0; i < runl->runq_nr; i++) { 404 runq = runl->runq[i]; 405 runq->func->init(runq); 406 } 407 408 nvkm_inth_allow(&runl->inth); 409 } 410 411 const struct nvkm_runl_func 412 ga100_runl = { 413 .init = ga100_runl_init, 414 .fini = ga100_runl_fini, 415 .size = 16, 416 .update = nv50_runl_update, 417 .insert_cgrp = gv100_runl_insert_cgrp, 418 .insert_chan = gv100_runl_insert_chan, 419 .commit = ga100_runl_commit, 420 .wait = nv50_runl_wait, 421 .pending = ga100_runl_pending, 422 .block = ga100_runl_block, 423 .allow = ga100_runl_allow, 424 .preempt = ga100_runl_preempt, 425 .preempt_pending = ga100_runl_preempt_pending, 426 }; 427 428 static int 429 ga100_runl_new(struct nvkm_fifo *fifo, int id, u32 addr, struct nvkm_runl **prunl) 430 { 431 struct nvkm_device *device = fifo->engine.subdev.device; 432 struct nvkm_runl *runl; 433 u32 chcfg = nvkm_rd32(device, addr + 0x004); 434 u32 chnum = 1 << (chcfg & 0x0000000f); 435 u32 chaddr = (chcfg & 0xfffffff0); 436 u32 dbcfg = nvkm_rd32(device, addr + 0x008); 437 u32 vector = nvkm_rd32(device, addr + 0x160); 438 int i, ret; 439 440 runl = *prunl = nvkm_runl_new(fifo, id, addr, chnum); 441 if (IS_ERR(runl)) 442 return PTR_ERR(runl); 443 444 for (i = 0; i < 2; i++) { 445 u32 pbcfg = nvkm_rd32(device, addr + 0x010 + (i * 0x04)); 446 if (pbcfg & 0x80000000) { 447 runl->runq[runl->runq_nr] = 448 nvkm_runq_new(fifo, ((pbcfg & 0x03fffc00) - 0x040000) / 0x800); 449 if (!runl->runq[runl->runq_nr]) 450 return -ENOMEM; 451 452 runl->runq_nr++; 453 } 454 } 455 456 ret = nvkm_inth_add(&device->vfn->intr, vector & 0x00000fff, NVKM_INTR_PRIO_NORMAL, 457 &fifo->engine.subdev, ga100_runl_intr, &runl->inth); 458 if (ret) 459 return ret; 460 461 runl->chan = chaddr; 462 runl->doorbell = dbcfg >> 16; 463 return 0; 464 } 465 466 static irqreturn_t 467 ga100_fifo_nonstall_intr(struct nvkm_inth *inth) 468 { 469 struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), nonstall.intr); 470 471 nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); 472 return IRQ_HANDLED; 473 } 474 475 static void 476 ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index) 477 { 478 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); 479 480 nvkm_inth_block(&fifo->nonstall.intr); 481 } 482 483 static void 484 ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) 485 { 486 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); 487 488 nvkm_inth_allow(&fifo->nonstall.intr); 489 } 490 491 const struct nvkm_event_func 492 ga100_fifo_nonstall = { 493 .init = ga100_fifo_nonstall_allow, 494 .fini = ga100_fifo_nonstall_block, 495 }; 496 497 int 498 ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) 499 { 500 return nvkm_inth_add(&fifo->engine.subdev.device->vfn->intr, GA100_FIFO_NONSTALL_VECTOR, 501 NVKM_INTR_PRIO_NORMAL, &fifo->engine.subdev, ga100_fifo_nonstall_intr, 502 &fifo->nonstall.intr); 503 } 504 505 int 506 ga100_fifo_runl_ctor(struct nvkm_fifo *fifo) 507 { 508 struct nvkm_device *device = fifo->engine.subdev.device; 509 struct nvkm_top_device *tdev; 510 struct nvkm_runl *runl; 511 int id = 0, ret; 512 513 nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) { 514 runl = nvkm_runl_get(fifo, -1, tdev->runlist); 515 if (!runl) { 516 ret = ga100_runl_new(fifo, id++, tdev->runlist, &runl); 517 if (ret) 518 return ret; 519 } 520 521 if (tdev->engine < 0) 522 continue; 523 524 nvkm_runl_add(runl, tdev->engine, (tdev->type == NVKM_ENGINE_CE) ? 525 fifo->func->engn_ce : fifo->func->engn, tdev->type, tdev->inst); 526 } 527 528 return 0; 529 } 530 531 static const struct nvkm_fifo_func 532 ga100_fifo = { 533 .runl_ctor = ga100_fifo_runl_ctor, 534 .mmu_fault = &tu102_fifo_mmu_fault, 535 .nonstall_ctor = ga100_fifo_nonstall_ctor, 536 .nonstall = &ga100_fifo_nonstall, 537 .runl = &ga100_runl, 538 .runq = &ga100_runq, 539 .engn = &ga100_engn, 540 .engn_ce = &ga100_engn_ce, 541 .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, 542 .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan }, 543 }; 544 545 int 546 ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 547 struct nvkm_fifo **pfifo) 548 { 549 return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo); 550 } 551