1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 #include <nvif/push206e.h> 39 40 void 41 nouveau_gem_object_del(struct drm_gem_object *gem) 42 { 43 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 44 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 45 struct device *dev = drm->dev->dev; 46 int ret; 47 48 ret = pm_runtime_get_sync(dev); 49 if (WARN_ON(ret < 0 && ret != -EACCES)) { 50 pm_runtime_put_autosuspend(dev); 51 return; 52 } 53 54 if (gem->import_attach) 55 drm_prime_gem_destroy(gem, nvbo->bo.sg); 56 57 ttm_bo_put(&nvbo->bo); 58 59 pm_runtime_mark_last_busy(dev); 60 pm_runtime_put_autosuspend(dev); 61 } 62 63 int 64 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 65 { 66 struct nouveau_cli *cli = nouveau_cli(file_priv); 67 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 68 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 69 struct device *dev = drm->dev->dev; 70 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 71 struct nouveau_vma *vma; 72 int ret; 73 74 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 75 return 0; 76 77 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 78 if (ret) 79 return ret; 80 81 ret = pm_runtime_get_sync(dev); 82 if (ret < 0 && ret != -EACCES) { 83 pm_runtime_put_autosuspend(dev); 84 goto out; 85 } 86 87 ret = nouveau_vma_new(nvbo, vmm, &vma); 88 pm_runtime_mark_last_busy(dev); 89 pm_runtime_put_autosuspend(dev); 90 out: 91 ttm_bo_unreserve(&nvbo->bo); 92 return ret; 93 } 94 95 struct nouveau_gem_object_unmap { 96 struct nouveau_cli_work work; 97 struct nouveau_vma *vma; 98 }; 99 100 static void 101 nouveau_gem_object_delete(struct nouveau_vma *vma) 102 { 103 nouveau_fence_unref(&vma->fence); 104 nouveau_vma_del(&vma); 105 } 106 107 static void 108 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 109 { 110 struct nouveau_gem_object_unmap *work = 111 container_of(w, typeof(*work), work); 112 nouveau_gem_object_delete(work->vma); 113 kfree(work); 114 } 115 116 static void 117 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 118 { 119 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 120 struct nouveau_gem_object_unmap *work; 121 122 list_del_init(&vma->head); 123 124 if (!fence) { 125 nouveau_gem_object_delete(vma); 126 return; 127 } 128 129 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 130 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 131 nouveau_gem_object_delete(vma); 132 return; 133 } 134 135 work->work.func = nouveau_gem_object_delete_work; 136 work->vma = vma; 137 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 138 } 139 140 void 141 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 142 { 143 struct nouveau_cli *cli = nouveau_cli(file_priv); 144 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 145 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 146 struct device *dev = drm->dev->dev; 147 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 148 struct nouveau_vma *vma; 149 int ret; 150 151 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 152 return; 153 154 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 155 if (ret) 156 return; 157 158 vma = nouveau_vma_find(nvbo, vmm); 159 if (vma) { 160 if (--vma->refs == 0) { 161 ret = pm_runtime_get_sync(dev); 162 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 163 nouveau_gem_object_unmap(nvbo, vma); 164 pm_runtime_mark_last_busy(dev); 165 } 166 pm_runtime_put_autosuspend(dev); 167 } 168 } 169 ttm_bo_unreserve(&nvbo->bo); 170 } 171 172 int 173 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 174 uint32_t tile_mode, uint32_t tile_flags, 175 struct nouveau_bo **pnvbo) 176 { 177 struct nouveau_drm *drm = cli->drm; 178 struct nouveau_bo *nvbo; 179 int ret; 180 181 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 182 domain |= NOUVEAU_GEM_DOMAIN_CPU; 183 184 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 185 tile_flags); 186 if (IS_ERR(nvbo)) 187 return PTR_ERR(nvbo); 188 189 /* Initialize the embedded gem-object. We return a single gem-reference 190 * to the caller, instead of a normal nouveau_bo ttm reference. */ 191 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 192 if (ret) { 193 drm_gem_object_release(&nvbo->bo.base); 194 kfree(nvbo); 195 return ret; 196 } 197 198 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL); 199 if (ret) { 200 nouveau_bo_ref(NULL, &nvbo); 201 return ret; 202 } 203 204 /* we restrict allowed domains on nv50+ to only the types 205 * that were requested at creation time. not possibly on 206 * earlier chips without busting the ABI. 207 */ 208 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 209 NOUVEAU_GEM_DOMAIN_GART; 210 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 211 nvbo->valid_domains &= domain; 212 213 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 214 *pnvbo = nvbo; 215 return 0; 216 } 217 218 static int 219 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 220 struct drm_nouveau_gem_info *rep) 221 { 222 struct nouveau_cli *cli = nouveau_cli(file_priv); 223 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 224 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 225 struct nouveau_vma *vma; 226 227 if (is_power_of_2(nvbo->valid_domains)) 228 rep->domain = nvbo->valid_domains; 229 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 230 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 231 else 232 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 233 rep->offset = nvbo->offset; 234 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 235 vma = nouveau_vma_find(nvbo, vmm); 236 if (!vma) 237 return -EINVAL; 238 239 rep->offset = vma->addr; 240 } 241 242 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 243 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 244 rep->tile_mode = nvbo->mode; 245 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 246 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 247 rep->tile_flags |= nvbo->kind << 8; 248 else 249 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 250 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 251 else 252 rep->tile_flags |= nvbo->zeta; 253 return 0; 254 } 255 256 int 257 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 258 struct drm_file *file_priv) 259 { 260 struct nouveau_cli *cli = nouveau_cli(file_priv); 261 struct drm_nouveau_gem_new *req = data; 262 struct nouveau_bo *nvbo = NULL; 263 int ret = 0; 264 265 ret = nouveau_gem_new(cli, req->info.size, req->align, 266 req->info.domain, req->info.tile_mode, 267 req->info.tile_flags, &nvbo); 268 if (ret) 269 return ret; 270 271 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 272 &req->info.handle); 273 if (ret == 0) { 274 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 275 if (ret) 276 drm_gem_handle_delete(file_priv, req->info.handle); 277 } 278 279 /* drop reference from allocate - handle holds it now */ 280 drm_gem_object_put(&nvbo->bo.base); 281 return ret; 282 } 283 284 static int 285 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 286 uint32_t write_domains, uint32_t valid_domains) 287 { 288 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 289 struct ttm_buffer_object *bo = &nvbo->bo; 290 uint32_t domains = valid_domains & nvbo->valid_domains & 291 (write_domains ? write_domains : read_domains); 292 uint32_t pref_domains = 0;; 293 294 if (!domains) 295 return -EINVAL; 296 297 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 298 299 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 300 bo->mem.mem_type == TTM_PL_VRAM) 301 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 302 303 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 304 bo->mem.mem_type == TTM_PL_TT) 305 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 306 307 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 308 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 309 310 else 311 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 312 313 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 314 315 return 0; 316 } 317 318 struct validate_op { 319 struct list_head list; 320 struct ww_acquire_ctx ticket; 321 }; 322 323 static void 324 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 325 struct nouveau_fence *fence, 326 struct drm_nouveau_gem_pushbuf_bo *pbbo) 327 { 328 struct nouveau_bo *nvbo; 329 struct drm_nouveau_gem_pushbuf_bo *b; 330 331 while (!list_empty(&op->list)) { 332 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 333 b = &pbbo[nvbo->pbbo_index]; 334 335 if (likely(fence)) { 336 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 337 338 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 339 struct nouveau_vma *vma = 340 (void *)(unsigned long)b->user_priv; 341 nouveau_fence_unref(&vma->fence); 342 dma_fence_get(&fence->base); 343 vma->fence = fence; 344 } 345 } 346 347 if (unlikely(nvbo->validate_mapped)) { 348 ttm_bo_kunmap(&nvbo->kmap); 349 nvbo->validate_mapped = false; 350 } 351 352 list_del(&nvbo->entry); 353 nvbo->reserved_by = NULL; 354 ttm_bo_unreserve(&nvbo->bo); 355 drm_gem_object_put(&nvbo->bo.base); 356 } 357 } 358 359 static void 360 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 361 struct nouveau_fence *fence, 362 struct drm_nouveau_gem_pushbuf_bo *pbbo) 363 { 364 validate_fini_no_ticket(op, chan, fence, pbbo); 365 ww_acquire_fini(&op->ticket); 366 } 367 368 static int 369 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 370 struct drm_nouveau_gem_pushbuf_bo *pbbo, 371 int nr_buffers, struct validate_op *op) 372 { 373 struct nouveau_cli *cli = nouveau_cli(file_priv); 374 int trycnt = 0; 375 int ret = -EINVAL, i; 376 struct nouveau_bo *res_bo = NULL; 377 LIST_HEAD(gart_list); 378 LIST_HEAD(vram_list); 379 LIST_HEAD(both_list); 380 381 ww_acquire_init(&op->ticket, &reservation_ww_class); 382 retry: 383 if (++trycnt > 100000) { 384 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 385 return -EINVAL; 386 } 387 388 for (i = 0; i < nr_buffers; i++) { 389 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 390 struct drm_gem_object *gem; 391 struct nouveau_bo *nvbo; 392 393 gem = drm_gem_object_lookup(file_priv, b->handle); 394 if (!gem) { 395 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 396 ret = -ENOENT; 397 break; 398 } 399 nvbo = nouveau_gem_object(gem); 400 if (nvbo == res_bo) { 401 res_bo = NULL; 402 drm_gem_object_put(gem); 403 continue; 404 } 405 406 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 407 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 408 "validation list\n", b->handle); 409 drm_gem_object_put(gem); 410 ret = -EINVAL; 411 break; 412 } 413 414 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 415 if (ret) { 416 list_splice_tail_init(&vram_list, &op->list); 417 list_splice_tail_init(&gart_list, &op->list); 418 list_splice_tail_init(&both_list, &op->list); 419 validate_fini_no_ticket(op, chan, NULL, NULL); 420 if (unlikely(ret == -EDEADLK)) { 421 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 422 &op->ticket); 423 if (!ret) 424 res_bo = nvbo; 425 } 426 if (unlikely(ret)) { 427 if (ret != -ERESTARTSYS) 428 NV_PRINTK(err, cli, "fail reserve\n"); 429 break; 430 } 431 } 432 433 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 434 struct nouveau_vmm *vmm = chan->vmm; 435 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 436 if (!vma) { 437 NV_PRINTK(err, cli, "vma not found!\n"); 438 ret = -EINVAL; 439 break; 440 } 441 442 b->user_priv = (uint64_t)(unsigned long)vma; 443 } else { 444 b->user_priv = (uint64_t)(unsigned long)nvbo; 445 } 446 447 nvbo->reserved_by = file_priv; 448 nvbo->pbbo_index = i; 449 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 450 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 451 list_add_tail(&nvbo->entry, &both_list); 452 else 453 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 454 list_add_tail(&nvbo->entry, &vram_list); 455 else 456 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 457 list_add_tail(&nvbo->entry, &gart_list); 458 else { 459 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 460 b->valid_domains); 461 list_add_tail(&nvbo->entry, &both_list); 462 ret = -EINVAL; 463 break; 464 } 465 if (nvbo == res_bo) 466 goto retry; 467 } 468 469 ww_acquire_done(&op->ticket); 470 list_splice_tail(&vram_list, &op->list); 471 list_splice_tail(&gart_list, &op->list); 472 list_splice_tail(&both_list, &op->list); 473 if (ret) 474 validate_fini(op, chan, NULL, NULL); 475 return ret; 476 477 } 478 479 static int 480 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 481 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 482 { 483 struct nouveau_drm *drm = chan->drm; 484 struct nouveau_bo *nvbo; 485 int ret, relocs = 0; 486 487 list_for_each_entry(nvbo, list, entry) { 488 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 489 490 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 491 b->write_domains, 492 b->valid_domains); 493 if (unlikely(ret)) { 494 NV_PRINTK(err, cli, "fail set_domain\n"); 495 return ret; 496 } 497 498 ret = nouveau_bo_validate(nvbo, true, false); 499 if (unlikely(ret)) { 500 if (ret != -ERESTARTSYS) 501 NV_PRINTK(err, cli, "fail ttm_validate\n"); 502 return ret; 503 } 504 505 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 506 if (unlikely(ret)) { 507 if (ret != -ERESTARTSYS) 508 NV_PRINTK(err, cli, "fail post-validate sync\n"); 509 return ret; 510 } 511 512 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 513 if (nvbo->offset == b->presumed.offset && 514 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 515 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 516 (nvbo->bo.mem.mem_type == TTM_PL_TT && 517 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 518 continue; 519 520 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 521 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 522 else 523 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 524 b->presumed.offset = nvbo->offset; 525 b->presumed.valid = 0; 526 relocs++; 527 } 528 } 529 530 return relocs; 531 } 532 533 static int 534 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 535 struct drm_file *file_priv, 536 struct drm_nouveau_gem_pushbuf_bo *pbbo, 537 int nr_buffers, 538 struct validate_op *op, bool *apply_relocs) 539 { 540 struct nouveau_cli *cli = nouveau_cli(file_priv); 541 int ret; 542 543 INIT_LIST_HEAD(&op->list); 544 545 if (nr_buffers == 0) 546 return 0; 547 548 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 549 if (unlikely(ret)) { 550 if (ret != -ERESTARTSYS) 551 NV_PRINTK(err, cli, "validate_init\n"); 552 return ret; 553 } 554 555 ret = validate_list(chan, cli, &op->list, pbbo); 556 if (unlikely(ret < 0)) { 557 if (ret != -ERESTARTSYS) 558 NV_PRINTK(err, cli, "validating bo list\n"); 559 validate_fini(op, chan, NULL, NULL); 560 return ret; 561 } else if (ret > 0) { 562 *apply_relocs = true; 563 } 564 565 return 0; 566 } 567 568 static inline void 569 u_free(void *addr) 570 { 571 kvfree(addr); 572 } 573 574 static inline void * 575 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 576 { 577 void *mem; 578 void __user *userptr = (void __force __user *)(uintptr_t)user; 579 580 size *= nmemb; 581 582 mem = kvmalloc(size, GFP_KERNEL); 583 if (!mem) 584 return ERR_PTR(-ENOMEM); 585 586 if (copy_from_user(mem, userptr, size)) { 587 u_free(mem); 588 return ERR_PTR(-EFAULT); 589 } 590 591 return mem; 592 } 593 594 static int 595 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 596 struct drm_nouveau_gem_pushbuf *req, 597 struct drm_nouveau_gem_pushbuf_reloc *reloc, 598 struct drm_nouveau_gem_pushbuf_bo *bo) 599 { 600 int ret = 0; 601 unsigned i; 602 603 for (i = 0; i < req->nr_relocs; i++) { 604 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 605 struct drm_nouveau_gem_pushbuf_bo *b; 606 struct nouveau_bo *nvbo; 607 uint32_t data; 608 609 if (unlikely(r->bo_index >= req->nr_buffers)) { 610 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 611 ret = -EINVAL; 612 break; 613 } 614 615 b = &bo[r->bo_index]; 616 if (b->presumed.valid) 617 continue; 618 619 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 620 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 621 ret = -EINVAL; 622 break; 623 } 624 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 625 626 if (unlikely(r->reloc_bo_offset + 4 > 627 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 628 NV_PRINTK(err, cli, "reloc outside of bo\n"); 629 ret = -EINVAL; 630 break; 631 } 632 633 if (!nvbo->kmap.virtual) { 634 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 635 &nvbo->kmap); 636 if (ret) { 637 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 638 break; 639 } 640 nvbo->validate_mapped = true; 641 } 642 643 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 644 data = b->presumed.offset + r->data; 645 else 646 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 647 data = (b->presumed.offset + r->data) >> 32; 648 else 649 data = r->data; 650 651 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 652 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 653 data |= r->tor; 654 else 655 data |= r->vor; 656 } 657 658 ret = ttm_bo_wait(&nvbo->bo, false, false); 659 if (ret) { 660 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 661 break; 662 } 663 664 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 665 } 666 667 return ret; 668 } 669 670 int 671 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 672 struct drm_file *file_priv) 673 { 674 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 675 struct nouveau_cli *cli = nouveau_cli(file_priv); 676 struct nouveau_abi16_chan *temp; 677 struct nouveau_drm *drm = nouveau_drm(dev); 678 struct drm_nouveau_gem_pushbuf *req = data; 679 struct drm_nouveau_gem_pushbuf_push *push; 680 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 681 struct drm_nouveau_gem_pushbuf_bo *bo; 682 struct nouveau_channel *chan = NULL; 683 struct validate_op op; 684 struct nouveau_fence *fence = NULL; 685 int i, j, ret = 0; 686 bool do_reloc = false, sync = false; 687 688 if (unlikely(!abi16)) 689 return -ENOMEM; 690 691 list_for_each_entry(temp, &abi16->channels, head) { 692 if (temp->chan->chid == req->channel) { 693 chan = temp->chan; 694 break; 695 } 696 } 697 698 if (!chan) 699 return nouveau_abi16_put(abi16, -ENOENT); 700 if (unlikely(atomic_read(&chan->killed))) 701 return nouveau_abi16_put(abi16, -ENODEV); 702 703 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 704 705 req->vram_available = drm->gem.vram_available; 706 req->gart_available = drm->gem.gart_available; 707 if (unlikely(req->nr_push == 0)) 708 goto out_next; 709 710 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 711 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 712 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 713 return nouveau_abi16_put(abi16, -EINVAL); 714 } 715 716 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 717 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 718 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 719 return nouveau_abi16_put(abi16, -EINVAL); 720 } 721 722 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 723 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 724 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 725 return nouveau_abi16_put(abi16, -EINVAL); 726 } 727 728 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 729 if (IS_ERR(push)) 730 return nouveau_abi16_put(abi16, PTR_ERR(push)); 731 732 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 733 if (IS_ERR(bo)) { 734 u_free(push); 735 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 736 } 737 738 /* Ensure all push buffers are on validate list */ 739 for (i = 0; i < req->nr_push; i++) { 740 if (push[i].bo_index >= req->nr_buffers) { 741 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 742 ret = -EINVAL; 743 goto out_prevalid; 744 } 745 } 746 747 /* Validate buffer list */ 748 revalidate: 749 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 750 req->nr_buffers, &op, &do_reloc); 751 if (ret) { 752 if (ret != -ERESTARTSYS) 753 NV_PRINTK(err, cli, "validate: %d\n", ret); 754 goto out_prevalid; 755 } 756 757 /* Apply any relocations that are required */ 758 if (do_reloc) { 759 if (!reloc) { 760 validate_fini(&op, chan, NULL, bo); 761 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 762 if (IS_ERR(reloc)) { 763 ret = PTR_ERR(reloc); 764 goto out_prevalid; 765 } 766 767 goto revalidate; 768 } 769 770 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 771 if (ret) { 772 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 773 goto out; 774 } 775 } 776 777 if (chan->dma.ib_max) { 778 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 779 if (ret) { 780 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 781 goto out; 782 } 783 784 for (i = 0; i < req->nr_push; i++) { 785 struct nouveau_vma *vma = (void *)(unsigned long) 786 bo[push[i].bo_index].user_priv; 787 788 nv50_dma_push(chan, vma->addr + push[i].offset, 789 push[i].length); 790 } 791 } else 792 if (drm->client.device.info.chipset >= 0x25) { 793 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 794 if (ret) { 795 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 796 goto out; 797 } 798 799 for (i = 0; i < req->nr_push; i++) { 800 struct nouveau_bo *nvbo = (void *)(unsigned long) 801 bo[push[i].bo_index].user_priv; 802 803 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 804 PUSH_DATA(chan->chan.push, 0); 805 } 806 } else { 807 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 808 if (ret) { 809 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 810 goto out; 811 } 812 813 for (i = 0; i < req->nr_push; i++) { 814 struct nouveau_bo *nvbo = (void *)(unsigned long) 815 bo[push[i].bo_index].user_priv; 816 uint32_t cmd; 817 818 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 819 cmd |= 0x20000000; 820 if (unlikely(cmd != req->suffix0)) { 821 if (!nvbo->kmap.virtual) { 822 ret = ttm_bo_kmap(&nvbo->bo, 0, 823 nvbo->bo.mem. 824 num_pages, 825 &nvbo->kmap); 826 if (ret) { 827 WIND_RING(chan); 828 goto out; 829 } 830 nvbo->validate_mapped = true; 831 } 832 833 nouveau_bo_wr32(nvbo, (push[i].offset + 834 push[i].length - 8) / 4, cmd); 835 } 836 837 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 838 PUSH_DATA(chan->chan.push, 0); 839 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 840 PUSH_DATA(chan->chan.push, 0); 841 } 842 } 843 844 ret = nouveau_fence_new(chan, false, &fence); 845 if (ret) { 846 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 847 WIND_RING(chan); 848 goto out; 849 } 850 851 if (sync) { 852 if (!(ret = nouveau_fence_wait(fence, false, false))) { 853 if ((ret = dma_fence_get_status(&fence->base)) == 1) 854 ret = 0; 855 } 856 } 857 858 out: 859 validate_fini(&op, chan, fence, bo); 860 nouveau_fence_unref(&fence); 861 862 if (do_reloc) { 863 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 864 u64_to_user_ptr(req->buffers); 865 866 for (i = 0; i < req->nr_buffers; i++) { 867 if (bo[i].presumed.valid) 868 continue; 869 870 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 871 sizeof(bo[i].presumed))) { 872 ret = -EFAULT; 873 break; 874 } 875 } 876 } 877 out_prevalid: 878 if (!IS_ERR(reloc)) 879 u_free(reloc); 880 u_free(bo); 881 u_free(push); 882 883 out_next: 884 if (chan->dma.ib_max) { 885 req->suffix0 = 0x00000000; 886 req->suffix1 = 0x00000000; 887 } else 888 if (drm->client.device.info.chipset >= 0x25) { 889 req->suffix0 = 0x00020000; 890 req->suffix1 = 0x00000000; 891 } else { 892 req->suffix0 = 0x20000000 | 893 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 894 req->suffix1 = 0x00000000; 895 } 896 897 return nouveau_abi16_put(abi16, ret); 898 } 899 900 int 901 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 902 struct drm_file *file_priv) 903 { 904 struct drm_nouveau_gem_cpu_prep *req = data; 905 struct drm_gem_object *gem; 906 struct nouveau_bo *nvbo; 907 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 908 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 909 long lret; 910 int ret; 911 912 gem = drm_gem_object_lookup(file_priv, req->handle); 913 if (!gem) 914 return -ENOENT; 915 nvbo = nouveau_gem_object(gem); 916 917 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 918 no_wait ? 0 : 30 * HZ); 919 if (!lret) 920 ret = -EBUSY; 921 else if (lret > 0) 922 ret = 0; 923 else 924 ret = lret; 925 926 nouveau_bo_sync_for_cpu(nvbo); 927 drm_gem_object_put(gem); 928 929 return ret; 930 } 931 932 int 933 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 934 struct drm_file *file_priv) 935 { 936 struct drm_nouveau_gem_cpu_fini *req = data; 937 struct drm_gem_object *gem; 938 struct nouveau_bo *nvbo; 939 940 gem = drm_gem_object_lookup(file_priv, req->handle); 941 if (!gem) 942 return -ENOENT; 943 nvbo = nouveau_gem_object(gem); 944 945 nouveau_bo_sync_for_device(nvbo); 946 drm_gem_object_put(gem); 947 return 0; 948 } 949 950 int 951 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 952 struct drm_file *file_priv) 953 { 954 struct drm_nouveau_gem_info *req = data; 955 struct drm_gem_object *gem; 956 int ret; 957 958 gem = drm_gem_object_lookup(file_priv, req->handle); 959 if (!gem) 960 return -ENOENT; 961 962 ret = nouveau_gem_info(file_priv, gem, req); 963 drm_gem_object_put(gem); 964 return ret; 965 } 966 967