1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drm.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 35 void 36 nouveau_gem_object_del(struct drm_gem_object *gem) 37 { 38 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 39 struct ttm_buffer_object *bo = &nvbo->bo; 40 41 if (gem->import_attach) 42 drm_prime_gem_destroy(gem, nvbo->bo.sg); 43 44 drm_gem_object_release(gem); 45 46 /* reset filp so nouveau_bo_del_ttm() can test for it */ 47 gem->filp = NULL; 48 ttm_bo_unref(&bo); 49 } 50 51 int 52 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 53 { 54 struct nouveau_cli *cli = nouveau_cli(file_priv); 55 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 56 struct nouveau_vma *vma; 57 int ret; 58 59 if (!cli->vm) 60 return 0; 61 62 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 63 if (ret) 64 return ret; 65 66 vma = nouveau_bo_vma_find(nvbo, cli->vm); 67 if (!vma) { 68 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 69 if (!vma) { 70 ret = -ENOMEM; 71 goto out; 72 } 73 74 ret = nouveau_bo_vma_add(nvbo, cli->vm, vma); 75 if (ret) { 76 kfree(vma); 77 goto out; 78 } 79 } else { 80 vma->refcount++; 81 } 82 83 out: 84 ttm_bo_unreserve(&nvbo->bo); 85 return ret; 86 } 87 88 static void 89 nouveau_gem_object_delete(void *data) 90 { 91 struct nouveau_vma *vma = data; 92 nouveau_vm_unmap(vma); 93 nouveau_vm_put(vma); 94 kfree(vma); 95 } 96 97 static void 98 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 99 { 100 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; 101 struct nouveau_fence *fence = NULL; 102 103 list_del(&vma->head); 104 105 if (mapped) { 106 spin_lock(&nvbo->bo.bdev->fence_lock); 107 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 108 spin_unlock(&nvbo->bo.bdev->fence_lock); 109 } 110 111 if (fence) { 112 nouveau_fence_work(fence, nouveau_gem_object_delete, vma); 113 } else { 114 if (mapped) 115 nouveau_vm_unmap(vma); 116 nouveau_vm_put(vma); 117 kfree(vma); 118 } 119 nouveau_fence_unref(&fence); 120 } 121 122 void 123 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 124 { 125 struct nouveau_cli *cli = nouveau_cli(file_priv); 126 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 127 struct nouveau_vma *vma; 128 int ret; 129 130 if (!cli->vm) 131 return; 132 133 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL); 134 if (ret) 135 return; 136 137 vma = nouveau_bo_vma_find(nvbo, cli->vm); 138 if (vma) { 139 if (--vma->refcount == 0) 140 nouveau_gem_object_unmap(nvbo, vma); 141 } 142 ttm_bo_unreserve(&nvbo->bo); 143 } 144 145 int 146 nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 147 uint32_t tile_mode, uint32_t tile_flags, 148 struct nouveau_bo **pnvbo) 149 { 150 struct nouveau_drm *drm = nouveau_drm(dev); 151 struct nouveau_bo *nvbo; 152 u32 flags = 0; 153 int ret; 154 155 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 156 flags |= TTM_PL_FLAG_VRAM; 157 if (domain & NOUVEAU_GEM_DOMAIN_GART) 158 flags |= TTM_PL_FLAG_TT; 159 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 160 flags |= TTM_PL_FLAG_SYSTEM; 161 162 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 163 tile_flags, NULL, pnvbo); 164 if (ret) 165 return ret; 166 nvbo = *pnvbo; 167 168 /* we restrict allowed domains on nv50+ to only the types 169 * that were requested at creation time. not possibly on 170 * earlier chips without busting the ABI. 171 */ 172 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 173 NOUVEAU_GEM_DOMAIN_GART; 174 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 175 nvbo->valid_domains &= domain; 176 177 /* Initialize the embedded gem-object. We return a single gem-reference 178 * to the caller, instead of a normal nouveau_bo ttm reference. */ 179 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size); 180 if (ret) { 181 nouveau_bo_ref(NULL, pnvbo); 182 return -ENOMEM; 183 } 184 185 nvbo->bo.persistent_swap_storage = nvbo->gem.filp; 186 return 0; 187 } 188 189 static int 190 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 191 struct drm_nouveau_gem_info *rep) 192 { 193 struct nouveau_cli *cli = nouveau_cli(file_priv); 194 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 195 struct nouveau_vma *vma; 196 197 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 198 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 199 else 200 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 201 202 rep->offset = nvbo->bo.offset; 203 if (cli->vm) { 204 vma = nouveau_bo_vma_find(nvbo, cli->vm); 205 if (!vma) 206 return -EINVAL; 207 208 rep->offset = vma->offset; 209 } 210 211 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 212 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node); 213 rep->tile_mode = nvbo->tile_mode; 214 rep->tile_flags = nvbo->tile_flags; 215 return 0; 216 } 217 218 int 219 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 220 struct drm_file *file_priv) 221 { 222 struct nouveau_drm *drm = nouveau_drm(dev); 223 struct nouveau_cli *cli = nouveau_cli(file_priv); 224 struct nouveau_fb *pfb = nvkm_fb(&drm->device); 225 struct drm_nouveau_gem_new *req = data; 226 struct nouveau_bo *nvbo = NULL; 227 int ret = 0; 228 229 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 230 NV_PRINTK(error, cli, "bad page flags: 0x%08x\n", req->info.tile_flags); 231 return -EINVAL; 232 } 233 234 ret = nouveau_gem_new(dev, req->info.size, req->align, 235 req->info.domain, req->info.tile_mode, 236 req->info.tile_flags, &nvbo); 237 if (ret) 238 return ret; 239 240 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle); 241 if (ret == 0) { 242 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info); 243 if (ret) 244 drm_gem_handle_delete(file_priv, req->info.handle); 245 } 246 247 /* drop reference from allocate - handle holds it now */ 248 drm_gem_object_unreference_unlocked(&nvbo->gem); 249 return ret; 250 } 251 252 static int 253 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 254 uint32_t write_domains, uint32_t valid_domains) 255 { 256 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 257 struct ttm_buffer_object *bo = &nvbo->bo; 258 uint32_t domains = valid_domains & nvbo->valid_domains & 259 (write_domains ? write_domains : read_domains); 260 uint32_t pref_flags = 0, valid_flags = 0; 261 262 if (!domains) 263 return -EINVAL; 264 265 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 266 valid_flags |= TTM_PL_FLAG_VRAM; 267 268 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 269 valid_flags |= TTM_PL_FLAG_TT; 270 271 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 272 bo->mem.mem_type == TTM_PL_VRAM) 273 pref_flags |= TTM_PL_FLAG_VRAM; 274 275 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 276 bo->mem.mem_type == TTM_PL_TT) 277 pref_flags |= TTM_PL_FLAG_TT; 278 279 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 280 pref_flags |= TTM_PL_FLAG_VRAM; 281 282 else 283 pref_flags |= TTM_PL_FLAG_TT; 284 285 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 286 287 return 0; 288 } 289 290 struct validate_op { 291 struct list_head vram_list; 292 struct list_head gart_list; 293 struct list_head both_list; 294 struct ww_acquire_ctx ticket; 295 }; 296 297 static void 298 validate_fini_list(struct list_head *list, struct nouveau_fence *fence, 299 struct ww_acquire_ctx *ticket) 300 { 301 struct list_head *entry, *tmp; 302 struct nouveau_bo *nvbo; 303 304 list_for_each_safe(entry, tmp, list) { 305 nvbo = list_entry(entry, struct nouveau_bo, entry); 306 307 if (likely(fence)) 308 nouveau_bo_fence(nvbo, fence); 309 310 if (unlikely(nvbo->validate_mapped)) { 311 ttm_bo_kunmap(&nvbo->kmap); 312 nvbo->validate_mapped = false; 313 } 314 315 list_del(&nvbo->entry); 316 nvbo->reserved_by = NULL; 317 ttm_bo_unreserve_ticket(&nvbo->bo, ticket); 318 drm_gem_object_unreference_unlocked(&nvbo->gem); 319 } 320 } 321 322 static void 323 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence) 324 { 325 validate_fini_list(&op->vram_list, fence, &op->ticket); 326 validate_fini_list(&op->gart_list, fence, &op->ticket); 327 validate_fini_list(&op->both_list, fence, &op->ticket); 328 } 329 330 static void 331 validate_fini(struct validate_op *op, struct nouveau_fence *fence) 332 { 333 validate_fini_no_ticket(op, fence); 334 ww_acquire_fini(&op->ticket); 335 } 336 337 static int 338 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 339 struct drm_nouveau_gem_pushbuf_bo *pbbo, 340 int nr_buffers, struct validate_op *op) 341 { 342 struct nouveau_cli *cli = nouveau_cli(file_priv); 343 struct drm_device *dev = chan->drm->dev; 344 int trycnt = 0; 345 int ret, i; 346 struct nouveau_bo *res_bo = NULL; 347 348 ww_acquire_init(&op->ticket, &reservation_ww_class); 349 retry: 350 if (++trycnt > 100000) { 351 NV_PRINTK(error, cli, "%s failed and gave up.\n", __func__); 352 return -EINVAL; 353 } 354 355 for (i = 0; i < nr_buffers; i++) { 356 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 357 struct drm_gem_object *gem; 358 struct nouveau_bo *nvbo; 359 360 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 361 if (!gem) { 362 NV_PRINTK(error, cli, "Unknown handle 0x%08x\n", b->handle); 363 ww_acquire_done(&op->ticket); 364 validate_fini(op, NULL); 365 return -ENOENT; 366 } 367 nvbo = nouveau_gem_object(gem); 368 if (nvbo == res_bo) { 369 res_bo = NULL; 370 drm_gem_object_unreference_unlocked(gem); 371 continue; 372 } 373 374 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 375 NV_PRINTK(error, cli, "multiple instances of buffer %d on " 376 "validation list\n", b->handle); 377 drm_gem_object_unreference_unlocked(gem); 378 ww_acquire_done(&op->ticket); 379 validate_fini(op, NULL); 380 return -EINVAL; 381 } 382 383 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket); 384 if (ret) { 385 validate_fini_no_ticket(op, NULL); 386 if (unlikely(ret == -EDEADLK)) { 387 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 388 &op->ticket); 389 if (!ret) 390 res_bo = nvbo; 391 } 392 if (unlikely(ret)) { 393 ww_acquire_done(&op->ticket); 394 ww_acquire_fini(&op->ticket); 395 drm_gem_object_unreference_unlocked(gem); 396 if (ret != -ERESTARTSYS) 397 NV_PRINTK(error, cli, "fail reserve\n"); 398 return ret; 399 } 400 } 401 402 b->user_priv = (uint64_t)(unsigned long)nvbo; 403 nvbo->reserved_by = file_priv; 404 nvbo->pbbo_index = i; 405 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 406 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 407 list_add_tail(&nvbo->entry, &op->both_list); 408 else 409 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 410 list_add_tail(&nvbo->entry, &op->vram_list); 411 else 412 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 413 list_add_tail(&nvbo->entry, &op->gart_list); 414 else { 415 NV_PRINTK(error, cli, "invalid valid domains: 0x%08x\n", 416 b->valid_domains); 417 list_add_tail(&nvbo->entry, &op->both_list); 418 ww_acquire_done(&op->ticket); 419 validate_fini(op, NULL); 420 return -EINVAL; 421 } 422 if (nvbo == res_bo) 423 goto retry; 424 } 425 426 ww_acquire_done(&op->ticket); 427 return 0; 428 } 429 430 static int 431 validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) 432 { 433 struct nouveau_fence *fence = NULL; 434 int ret = 0; 435 436 spin_lock(&nvbo->bo.bdev->fence_lock); 437 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 438 spin_unlock(&nvbo->bo.bdev->fence_lock); 439 440 if (fence) { 441 ret = nouveau_fence_sync(fence, chan); 442 nouveau_fence_unref(&fence); 443 } 444 445 return ret; 446 } 447 448 static int 449 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 450 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, 451 uint64_t user_pbbo_ptr) 452 { 453 struct nouveau_drm *drm = chan->drm; 454 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 455 (void __force __user *)(uintptr_t)user_pbbo_ptr; 456 struct nouveau_bo *nvbo; 457 int ret, relocs = 0; 458 459 list_for_each_entry(nvbo, list, entry) { 460 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 461 462 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains, 463 b->write_domains, 464 b->valid_domains); 465 if (unlikely(ret)) { 466 NV_PRINTK(error, cli, "fail set_domain\n"); 467 return ret; 468 } 469 470 ret = nouveau_bo_validate(nvbo, true, false); 471 if (unlikely(ret)) { 472 if (ret != -ERESTARTSYS) 473 NV_PRINTK(error, cli, "fail ttm_validate\n"); 474 return ret; 475 } 476 477 ret = validate_sync(chan, nvbo); 478 if (unlikely(ret)) { 479 NV_PRINTK(error, cli, "fail post-validate sync\n"); 480 return ret; 481 } 482 483 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { 484 if (nvbo->bo.offset == b->presumed.offset && 485 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 486 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 487 (nvbo->bo.mem.mem_type == TTM_PL_TT && 488 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 489 continue; 490 491 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 492 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 493 else 494 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 495 b->presumed.offset = nvbo->bo.offset; 496 b->presumed.valid = 0; 497 relocs++; 498 499 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed, 500 &b->presumed, sizeof(b->presumed))) 501 return -EFAULT; 502 } 503 } 504 505 return relocs; 506 } 507 508 static int 509 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 510 struct drm_file *file_priv, 511 struct drm_nouveau_gem_pushbuf_bo *pbbo, 512 uint64_t user_buffers, int nr_buffers, 513 struct validate_op *op, int *apply_relocs) 514 { 515 struct nouveau_cli *cli = nouveau_cli(file_priv); 516 int ret, relocs = 0; 517 518 INIT_LIST_HEAD(&op->vram_list); 519 INIT_LIST_HEAD(&op->gart_list); 520 INIT_LIST_HEAD(&op->both_list); 521 522 if (nr_buffers == 0) 523 return 0; 524 525 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 526 if (unlikely(ret)) { 527 if (ret != -ERESTARTSYS) 528 NV_PRINTK(error, cli, "validate_init\n"); 529 return ret; 530 } 531 532 ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers); 533 if (unlikely(ret < 0)) { 534 if (ret != -ERESTARTSYS) 535 NV_PRINTK(error, cli, "validate vram_list\n"); 536 validate_fini(op, NULL); 537 return ret; 538 } 539 relocs += ret; 540 541 ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers); 542 if (unlikely(ret < 0)) { 543 if (ret != -ERESTARTSYS) 544 NV_PRINTK(error, cli, "validate gart_list\n"); 545 validate_fini(op, NULL); 546 return ret; 547 } 548 relocs += ret; 549 550 ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers); 551 if (unlikely(ret < 0)) { 552 if (ret != -ERESTARTSYS) 553 NV_PRINTK(error, cli, "validate both_list\n"); 554 validate_fini(op, NULL); 555 return ret; 556 } 557 relocs += ret; 558 559 *apply_relocs = relocs; 560 return 0; 561 } 562 563 static inline void 564 u_free(void *addr) 565 { 566 if (!is_vmalloc_addr(addr)) 567 kfree(addr); 568 else 569 vfree(addr); 570 } 571 572 static inline void * 573 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 574 { 575 void *mem; 576 void __user *userptr = (void __force __user *)(uintptr_t)user; 577 578 size *= nmemb; 579 580 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); 581 if (!mem) 582 mem = vmalloc(size); 583 if (!mem) 584 return ERR_PTR(-ENOMEM); 585 586 if (copy_from_user(mem, userptr, size)) { 587 u_free(mem); 588 return ERR_PTR(-EFAULT); 589 } 590 591 return mem; 592 } 593 594 static int 595 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 596 struct drm_nouveau_gem_pushbuf *req, 597 struct drm_nouveau_gem_pushbuf_bo *bo) 598 { 599 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 600 int ret = 0; 601 unsigned i; 602 603 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 604 if (IS_ERR(reloc)) 605 return PTR_ERR(reloc); 606 607 for (i = 0; i < req->nr_relocs; i++) { 608 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 609 struct drm_nouveau_gem_pushbuf_bo *b; 610 struct nouveau_bo *nvbo; 611 uint32_t data; 612 613 if (unlikely(r->bo_index > req->nr_buffers)) { 614 NV_PRINTK(error, cli, "reloc bo index invalid\n"); 615 ret = -EINVAL; 616 break; 617 } 618 619 b = &bo[r->bo_index]; 620 if (b->presumed.valid) 621 continue; 622 623 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 624 NV_PRINTK(error, cli, "reloc container bo index invalid\n"); 625 ret = -EINVAL; 626 break; 627 } 628 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 629 630 if (unlikely(r->reloc_bo_offset + 4 > 631 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 632 NV_PRINTK(error, cli, "reloc outside of bo\n"); 633 ret = -EINVAL; 634 break; 635 } 636 637 if (!nvbo->kmap.virtual) { 638 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 639 &nvbo->kmap); 640 if (ret) { 641 NV_PRINTK(error, cli, "failed kmap for reloc\n"); 642 break; 643 } 644 nvbo->validate_mapped = true; 645 } 646 647 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 648 data = b->presumed.offset + r->data; 649 else 650 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 651 data = (b->presumed.offset + r->data) >> 32; 652 else 653 data = r->data; 654 655 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 656 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 657 data |= r->tor; 658 else 659 data |= r->vor; 660 } 661 662 spin_lock(&nvbo->bo.bdev->fence_lock); 663 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 664 spin_unlock(&nvbo->bo.bdev->fence_lock); 665 if (ret) { 666 NV_PRINTK(error, cli, "reloc wait_idle failed: %d\n", ret); 667 break; 668 } 669 670 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 671 } 672 673 u_free(reloc); 674 return ret; 675 } 676 677 int 678 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 679 struct drm_file *file_priv) 680 { 681 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 682 struct nouveau_cli *cli = nouveau_cli(file_priv); 683 struct nouveau_abi16_chan *temp; 684 struct nouveau_drm *drm = nouveau_drm(dev); 685 struct drm_nouveau_gem_pushbuf *req = data; 686 struct drm_nouveau_gem_pushbuf_push *push; 687 struct drm_nouveau_gem_pushbuf_bo *bo; 688 struct nouveau_channel *chan = NULL; 689 struct validate_op op; 690 struct nouveau_fence *fence = NULL; 691 int i, j, ret = 0, do_reloc = 0; 692 693 if (unlikely(!abi16)) 694 return -ENOMEM; 695 696 list_for_each_entry(temp, &abi16->channels, head) { 697 if (temp->chan->object->handle == (NVDRM_CHAN | req->channel)) { 698 chan = temp->chan; 699 break; 700 } 701 } 702 703 if (!chan) 704 return nouveau_abi16_put(abi16, -ENOENT); 705 706 req->vram_available = drm->gem.vram_available; 707 req->gart_available = drm->gem.gart_available; 708 if (unlikely(req->nr_push == 0)) 709 goto out_next; 710 711 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 712 NV_PRINTK(error, cli, "pushbuf push count exceeds limit: %d max %d\n", 713 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 714 return nouveau_abi16_put(abi16, -EINVAL); 715 } 716 717 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 718 NV_PRINTK(error, cli, "pushbuf bo count exceeds limit: %d max %d\n", 719 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 720 return nouveau_abi16_put(abi16, -EINVAL); 721 } 722 723 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 724 NV_PRINTK(error, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 725 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 726 return nouveau_abi16_put(abi16, -EINVAL); 727 } 728 729 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 730 if (IS_ERR(push)) 731 return nouveau_abi16_put(abi16, PTR_ERR(push)); 732 733 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 734 if (IS_ERR(bo)) { 735 u_free(push); 736 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 737 } 738 739 /* Ensure all push buffers are on validate list */ 740 for (i = 0; i < req->nr_push; i++) { 741 if (push[i].bo_index >= req->nr_buffers) { 742 NV_PRINTK(error, cli, "push %d buffer not in list\n", i); 743 ret = -EINVAL; 744 goto out_prevalid; 745 } 746 } 747 748 /* Validate buffer list */ 749 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 750 req->nr_buffers, &op, &do_reloc); 751 if (ret) { 752 if (ret != -ERESTARTSYS) 753 NV_PRINTK(error, cli, "validate: %d\n", ret); 754 goto out_prevalid; 755 } 756 757 /* Apply any relocations that are required */ 758 if (do_reloc) { 759 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo); 760 if (ret) { 761 NV_PRINTK(error, cli, "reloc apply: %d\n", ret); 762 goto out; 763 } 764 } 765 766 if (chan->dma.ib_max) { 767 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 768 if (ret) { 769 NV_PRINTK(error, cli, "nv50cal_space: %d\n", ret); 770 goto out; 771 } 772 773 for (i = 0; i < req->nr_push; i++) { 774 struct nouveau_bo *nvbo = (void *)(unsigned long) 775 bo[push[i].bo_index].user_priv; 776 777 nv50_dma_push(chan, nvbo, push[i].offset, 778 push[i].length); 779 } 780 } else 781 if (drm->device.info.chipset >= 0x25) { 782 ret = RING_SPACE(chan, req->nr_push * 2); 783 if (ret) { 784 NV_PRINTK(error, cli, "cal_space: %d\n", ret); 785 goto out; 786 } 787 788 for (i = 0; i < req->nr_push; i++) { 789 struct nouveau_bo *nvbo = (void *)(unsigned long) 790 bo[push[i].bo_index].user_priv; 791 792 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 793 OUT_RING(chan, 0); 794 } 795 } else { 796 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 797 if (ret) { 798 NV_PRINTK(error, cli, "jmp_space: %d\n", ret); 799 goto out; 800 } 801 802 for (i = 0; i < req->nr_push; i++) { 803 struct nouveau_bo *nvbo = (void *)(unsigned long) 804 bo[push[i].bo_index].user_priv; 805 uint32_t cmd; 806 807 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 808 cmd |= 0x20000000; 809 if (unlikely(cmd != req->suffix0)) { 810 if (!nvbo->kmap.virtual) { 811 ret = ttm_bo_kmap(&nvbo->bo, 0, 812 nvbo->bo.mem. 813 num_pages, 814 &nvbo->kmap); 815 if (ret) { 816 WIND_RING(chan); 817 goto out; 818 } 819 nvbo->validate_mapped = true; 820 } 821 822 nouveau_bo_wr32(nvbo, (push[i].offset + 823 push[i].length - 8) / 4, cmd); 824 } 825 826 OUT_RING(chan, 0x20000000 | 827 (nvbo->bo.offset + push[i].offset)); 828 OUT_RING(chan, 0); 829 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 830 OUT_RING(chan, 0); 831 } 832 } 833 834 ret = nouveau_fence_new(chan, false, &fence); 835 if (ret) { 836 NV_PRINTK(error, cli, "error fencing pushbuf: %d\n", ret); 837 WIND_RING(chan); 838 goto out; 839 } 840 841 out: 842 validate_fini(&op, fence); 843 nouveau_fence_unref(&fence); 844 845 out_prevalid: 846 u_free(bo); 847 u_free(push); 848 849 out_next: 850 if (chan->dma.ib_max) { 851 req->suffix0 = 0x00000000; 852 req->suffix1 = 0x00000000; 853 } else 854 if (drm->device.info.chipset >= 0x25) { 855 req->suffix0 = 0x00020000; 856 req->suffix1 = 0x00000000; 857 } else { 858 req->suffix0 = 0x20000000 | 859 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 860 req->suffix1 = 0x00000000; 861 } 862 863 return nouveau_abi16_put(abi16, ret); 864 } 865 866 static inline uint32_t 867 domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 868 { 869 uint32_t flags = 0; 870 871 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 872 flags |= TTM_PL_FLAG_VRAM; 873 if (domain & NOUVEAU_GEM_DOMAIN_GART) 874 flags |= TTM_PL_FLAG_TT; 875 876 return flags; 877 } 878 879 int 880 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 881 struct drm_file *file_priv) 882 { 883 struct drm_nouveau_gem_cpu_prep *req = data; 884 struct drm_gem_object *gem; 885 struct nouveau_bo *nvbo; 886 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 887 int ret = -EINVAL; 888 889 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 890 if (!gem) 891 return -ENOENT; 892 nvbo = nouveau_gem_object(gem); 893 894 spin_lock(&nvbo->bo.bdev->fence_lock); 895 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); 896 spin_unlock(&nvbo->bo.bdev->fence_lock); 897 drm_gem_object_unreference_unlocked(gem); 898 return ret; 899 } 900 901 int 902 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 903 struct drm_file *file_priv) 904 { 905 return 0; 906 } 907 908 int 909 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 910 struct drm_file *file_priv) 911 { 912 struct drm_nouveau_gem_info *req = data; 913 struct drm_gem_object *gem; 914 int ret; 915 916 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 917 if (!gem) 918 return -ENOENT; 919 920 ret = nouveau_gem_info(file_priv, gem, req); 921 drm_gem_object_unreference_unlocked(gem); 922 return ret; 923 } 924 925