1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <linux/dma-buf.h> 28 29 #include <subdev/fb.h> 30 31 #include "nouveau_drm.h" 32 #include "nouveau_dma.h" 33 #include "nouveau_fence.h" 34 #include "nouveau_abi16.h" 35 36 #include "nouveau_ttm.h" 37 #include "nouveau_gem.h" 38 39 int 40 nouveau_gem_object_new(struct drm_gem_object *gem) 41 { 42 return 0; 43 } 44 45 void 46 nouveau_gem_object_del(struct drm_gem_object *gem) 47 { 48 struct nouveau_bo *nvbo = gem->driver_private; 49 struct ttm_buffer_object *bo = &nvbo->bo; 50 51 if (!nvbo) 52 return; 53 nvbo->gem = NULL; 54 55 if (unlikely(nvbo->pin_refcnt)) { 56 nvbo->pin_refcnt = 1; 57 nouveau_bo_unpin(nvbo); 58 } 59 60 if (gem->import_attach) 61 drm_prime_gem_destroy(gem, nvbo->bo.sg); 62 63 ttm_bo_unref(&bo); 64 65 drm_gem_object_release(gem); 66 kfree(gem); 67 } 68 69 int 70 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 71 { 72 struct nouveau_cli *cli = nouveau_cli(file_priv); 73 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 74 struct nouveau_vma *vma; 75 int ret; 76 77 if (!cli->base.vm) 78 return 0; 79 80 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 81 if (ret) 82 return ret; 83 84 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 85 if (!vma) { 86 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 87 if (!vma) { 88 ret = -ENOMEM; 89 goto out; 90 } 91 92 ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma); 93 if (ret) { 94 kfree(vma); 95 goto out; 96 } 97 } else { 98 vma->refcount++; 99 } 100 101 out: 102 ttm_bo_unreserve(&nvbo->bo); 103 return ret; 104 } 105 106 void 107 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 108 { 109 struct nouveau_cli *cli = nouveau_cli(file_priv); 110 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 111 struct nouveau_vma *vma; 112 int ret; 113 114 if (!cli->base.vm) 115 return; 116 117 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 118 if (ret) 119 return; 120 121 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 122 if (vma) { 123 if (--vma->refcount == 0) { 124 nouveau_bo_vma_del(nvbo, vma); 125 kfree(vma); 126 } 127 } 128 ttm_bo_unreserve(&nvbo->bo); 129 } 130 131 int 132 nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain, 133 uint32_t tile_mode, uint32_t tile_flags, 134 struct nouveau_bo **pnvbo) 135 { 136 struct nouveau_drm *drm = nouveau_drm(dev); 137 struct nouveau_bo *nvbo; 138 u32 flags = 0; 139 int ret; 140 141 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 142 flags |= TTM_PL_FLAG_VRAM; 143 if (domain & NOUVEAU_GEM_DOMAIN_GART) 144 flags |= TTM_PL_FLAG_TT; 145 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 146 flags |= TTM_PL_FLAG_SYSTEM; 147 148 ret = nouveau_bo_new(dev, size, align, flags, tile_mode, 149 tile_flags, NULL, pnvbo); 150 if (ret) 151 return ret; 152 nvbo = *pnvbo; 153 154 /* we restrict allowed domains on nv50+ to only the types 155 * that were requested at creation time. not possibly on 156 * earlier chips without busting the ABI. 157 */ 158 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 159 NOUVEAU_GEM_DOMAIN_GART; 160 if (nv_device(drm->device)->card_type >= NV_50) 161 nvbo->valid_domains &= domain; 162 163 nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size); 164 if (!nvbo->gem) { 165 nouveau_bo_ref(NULL, pnvbo); 166 return -ENOMEM; 167 } 168 169 nvbo->bo.persistent_swap_storage = nvbo->gem->filp; 170 nvbo->gem->driver_private = nvbo; 171 return 0; 172 } 173 174 static int 175 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 176 struct drm_nouveau_gem_info *rep) 177 { 178 struct nouveau_cli *cli = nouveau_cli(file_priv); 179 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 180 struct nouveau_vma *vma; 181 182 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 183 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 184 else 185 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 186 187 rep->offset = nvbo->bo.offset; 188 if (cli->base.vm) { 189 vma = nouveau_bo_vma_find(nvbo, cli->base.vm); 190 if (!vma) 191 return -EINVAL; 192 193 rep->offset = vma->offset; 194 } 195 196 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 197 rep->map_handle = nvbo->bo.addr_space_offset; 198 rep->tile_mode = nvbo->tile_mode; 199 rep->tile_flags = nvbo->tile_flags; 200 return 0; 201 } 202 203 int 204 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 205 struct drm_file *file_priv) 206 { 207 struct nouveau_drm *drm = nouveau_drm(dev); 208 struct nouveau_fb *pfb = nouveau_fb(drm->device); 209 struct drm_nouveau_gem_new *req = data; 210 struct nouveau_bo *nvbo = NULL; 211 int ret = 0; 212 213 drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping; 214 215 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) { 216 NV_ERROR(drm, "bad page flags: 0x%08x\n", req->info.tile_flags); 217 return -EINVAL; 218 } 219 220 ret = nouveau_gem_new(dev, req->info.size, req->align, 221 req->info.domain, req->info.tile_mode, 222 req->info.tile_flags, &nvbo); 223 if (ret) 224 return ret; 225 226 ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle); 227 if (ret == 0) { 228 ret = nouveau_gem_info(file_priv, nvbo->gem, &req->info); 229 if (ret) 230 drm_gem_handle_delete(file_priv, req->info.handle); 231 } 232 233 /* drop reference from allocate - handle holds it now */ 234 drm_gem_object_unreference_unlocked(nvbo->gem); 235 return ret; 236 } 237 238 static int 239 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 240 uint32_t write_domains, uint32_t valid_domains) 241 { 242 struct nouveau_bo *nvbo = gem->driver_private; 243 struct ttm_buffer_object *bo = &nvbo->bo; 244 uint32_t domains = valid_domains & nvbo->valid_domains & 245 (write_domains ? write_domains : read_domains); 246 uint32_t pref_flags = 0, valid_flags = 0; 247 248 if (!domains) 249 return -EINVAL; 250 251 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 252 valid_flags |= TTM_PL_FLAG_VRAM; 253 254 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 255 valid_flags |= TTM_PL_FLAG_TT; 256 257 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 258 bo->mem.mem_type == TTM_PL_VRAM) 259 pref_flags |= TTM_PL_FLAG_VRAM; 260 261 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 262 bo->mem.mem_type == TTM_PL_TT) 263 pref_flags |= TTM_PL_FLAG_TT; 264 265 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 266 pref_flags |= TTM_PL_FLAG_VRAM; 267 268 else 269 pref_flags |= TTM_PL_FLAG_TT; 270 271 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 272 273 return 0; 274 } 275 276 struct validate_op { 277 struct list_head vram_list; 278 struct list_head gart_list; 279 struct list_head both_list; 280 }; 281 282 static void 283 validate_fini_list(struct list_head *list, struct nouveau_fence *fence) 284 { 285 struct list_head *entry, *tmp; 286 struct nouveau_bo *nvbo; 287 288 list_for_each_safe(entry, tmp, list) { 289 nvbo = list_entry(entry, struct nouveau_bo, entry); 290 291 nouveau_bo_fence(nvbo, fence); 292 293 if (unlikely(nvbo->validate_mapped)) { 294 ttm_bo_kunmap(&nvbo->kmap); 295 nvbo->validate_mapped = false; 296 } 297 298 list_del(&nvbo->entry); 299 nvbo->reserved_by = NULL; 300 ttm_bo_unreserve(&nvbo->bo); 301 drm_gem_object_unreference_unlocked(nvbo->gem); 302 } 303 } 304 305 static void 306 validate_fini(struct validate_op *op, struct nouveau_fence* fence) 307 { 308 validate_fini_list(&op->vram_list, fence); 309 validate_fini_list(&op->gart_list, fence); 310 validate_fini_list(&op->both_list, fence); 311 } 312 313 static int 314 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 315 struct drm_nouveau_gem_pushbuf_bo *pbbo, 316 int nr_buffers, struct validate_op *op) 317 { 318 struct drm_device *dev = chan->drm->dev; 319 struct nouveau_drm *drm = nouveau_drm(dev); 320 uint32_t sequence; 321 int trycnt = 0; 322 int ret, i; 323 324 sequence = atomic_add_return(1, &drm->ttm.validate_sequence); 325 retry: 326 if (++trycnt > 100000) { 327 NV_ERROR(drm, "%s failed and gave up.\n", __func__); 328 return -EINVAL; 329 } 330 331 for (i = 0; i < nr_buffers; i++) { 332 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 333 struct drm_gem_object *gem; 334 struct nouveau_bo *nvbo; 335 336 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 337 if (!gem) { 338 NV_ERROR(drm, "Unknown handle 0x%08x\n", b->handle); 339 validate_fini(op, NULL); 340 return -ENOENT; 341 } 342 nvbo = gem->driver_private; 343 344 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 345 NV_ERROR(drm, "multiple instances of buffer %d on " 346 "validation list\n", b->handle); 347 drm_gem_object_unreference_unlocked(gem); 348 validate_fini(op, NULL); 349 return -EINVAL; 350 } 351 352 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence); 353 if (ret) { 354 validate_fini(op, NULL); 355 if (unlikely(ret == -EAGAIN)) 356 ret = ttm_bo_wait_unreserved(&nvbo->bo, true); 357 drm_gem_object_unreference_unlocked(gem); 358 if (unlikely(ret)) { 359 if (ret != -ERESTARTSYS) 360 NV_ERROR(drm, "fail reserve\n"); 361 return ret; 362 } 363 goto retry; 364 } 365 366 b->user_priv = (uint64_t)(unsigned long)nvbo; 367 nvbo->reserved_by = file_priv; 368 nvbo->pbbo_index = i; 369 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 370 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 371 list_add_tail(&nvbo->entry, &op->both_list); 372 else 373 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 374 list_add_tail(&nvbo->entry, &op->vram_list); 375 else 376 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 377 list_add_tail(&nvbo->entry, &op->gart_list); 378 else { 379 NV_ERROR(drm, "invalid valid domains: 0x%08x\n", 380 b->valid_domains); 381 list_add_tail(&nvbo->entry, &op->both_list); 382 validate_fini(op, NULL); 383 return -EINVAL; 384 } 385 } 386 387 return 0; 388 } 389 390 static int 391 validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) 392 { 393 struct nouveau_fence *fence = NULL; 394 int ret = 0; 395 396 spin_lock(&nvbo->bo.bdev->fence_lock); 397 if (nvbo->bo.sync_obj) 398 fence = nouveau_fence_ref(nvbo->bo.sync_obj); 399 spin_unlock(&nvbo->bo.bdev->fence_lock); 400 401 if (fence) { 402 ret = nouveau_fence_sync(fence, chan); 403 nouveau_fence_unref(&fence); 404 } 405 406 return ret; 407 } 408 409 static int 410 validate_list(struct nouveau_channel *chan, struct list_head *list, 411 struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr) 412 { 413 struct nouveau_drm *drm = chan->drm; 414 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 415 (void __force __user *)(uintptr_t)user_pbbo_ptr; 416 struct nouveau_bo *nvbo; 417 int ret, relocs = 0; 418 419 list_for_each_entry(nvbo, list, entry) { 420 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 421 422 ret = validate_sync(chan, nvbo); 423 if (unlikely(ret)) { 424 NV_ERROR(drm, "fail pre-validate sync\n"); 425 return ret; 426 } 427 428 ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains, 429 b->write_domains, 430 b->valid_domains); 431 if (unlikely(ret)) { 432 NV_ERROR(drm, "fail set_domain\n"); 433 return ret; 434 } 435 436 ret = nouveau_bo_validate(nvbo, true, false); 437 if (unlikely(ret)) { 438 if (ret != -ERESTARTSYS) 439 NV_ERROR(drm, "fail ttm_validate\n"); 440 return ret; 441 } 442 443 ret = validate_sync(chan, nvbo); 444 if (unlikely(ret)) { 445 NV_ERROR(drm, "fail post-validate sync\n"); 446 return ret; 447 } 448 449 if (nv_device(drm->device)->card_type < NV_50) { 450 if (nvbo->bo.offset == b->presumed.offset && 451 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 452 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 453 (nvbo->bo.mem.mem_type == TTM_PL_TT && 454 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 455 continue; 456 457 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 458 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 459 else 460 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 461 b->presumed.offset = nvbo->bo.offset; 462 b->presumed.valid = 0; 463 relocs++; 464 465 if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed, 466 &b->presumed, sizeof(b->presumed))) 467 return -EFAULT; 468 } 469 } 470 471 return relocs; 472 } 473 474 static int 475 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 476 struct drm_file *file_priv, 477 struct drm_nouveau_gem_pushbuf_bo *pbbo, 478 uint64_t user_buffers, int nr_buffers, 479 struct validate_op *op, int *apply_relocs) 480 { 481 struct nouveau_drm *drm = chan->drm; 482 int ret, relocs = 0; 483 484 INIT_LIST_HEAD(&op->vram_list); 485 INIT_LIST_HEAD(&op->gart_list); 486 INIT_LIST_HEAD(&op->both_list); 487 488 if (nr_buffers == 0) 489 return 0; 490 491 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 492 if (unlikely(ret)) { 493 if (ret != -ERESTARTSYS) 494 NV_ERROR(drm, "validate_init\n"); 495 return ret; 496 } 497 498 ret = validate_list(chan, &op->vram_list, pbbo, user_buffers); 499 if (unlikely(ret < 0)) { 500 if (ret != -ERESTARTSYS) 501 NV_ERROR(drm, "validate vram_list\n"); 502 validate_fini(op, NULL); 503 return ret; 504 } 505 relocs += ret; 506 507 ret = validate_list(chan, &op->gart_list, pbbo, user_buffers); 508 if (unlikely(ret < 0)) { 509 if (ret != -ERESTARTSYS) 510 NV_ERROR(drm, "validate gart_list\n"); 511 validate_fini(op, NULL); 512 return ret; 513 } 514 relocs += ret; 515 516 ret = validate_list(chan, &op->both_list, pbbo, user_buffers); 517 if (unlikely(ret < 0)) { 518 if (ret != -ERESTARTSYS) 519 NV_ERROR(drm, "validate both_list\n"); 520 validate_fini(op, NULL); 521 return ret; 522 } 523 relocs += ret; 524 525 *apply_relocs = relocs; 526 return 0; 527 } 528 529 static inline void * 530 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 531 { 532 void *mem; 533 void __user *userptr = (void __force __user *)(uintptr_t)user; 534 535 mem = kmalloc(nmemb * size, GFP_KERNEL); 536 if (!mem) 537 return ERR_PTR(-ENOMEM); 538 539 if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) { 540 kfree(mem); 541 return ERR_PTR(-EFAULT); 542 } 543 544 return mem; 545 } 546 547 static int 548 nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev, 549 struct drm_nouveau_gem_pushbuf *req, 550 struct drm_nouveau_gem_pushbuf_bo *bo) 551 { 552 struct nouveau_drm *drm = nouveau_drm(dev); 553 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 554 int ret = 0; 555 unsigned i; 556 557 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 558 if (IS_ERR(reloc)) 559 return PTR_ERR(reloc); 560 561 for (i = 0; i < req->nr_relocs; i++) { 562 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 563 struct drm_nouveau_gem_pushbuf_bo *b; 564 struct nouveau_bo *nvbo; 565 uint32_t data; 566 567 if (unlikely(r->bo_index > req->nr_buffers)) { 568 NV_ERROR(drm, "reloc bo index invalid\n"); 569 ret = -EINVAL; 570 break; 571 } 572 573 b = &bo[r->bo_index]; 574 if (b->presumed.valid) 575 continue; 576 577 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 578 NV_ERROR(drm, "reloc container bo index invalid\n"); 579 ret = -EINVAL; 580 break; 581 } 582 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 583 584 if (unlikely(r->reloc_bo_offset + 4 > 585 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 586 NV_ERROR(drm, "reloc outside of bo\n"); 587 ret = -EINVAL; 588 break; 589 } 590 591 if (!nvbo->kmap.virtual) { 592 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 593 &nvbo->kmap); 594 if (ret) { 595 NV_ERROR(drm, "failed kmap for reloc\n"); 596 break; 597 } 598 nvbo->validate_mapped = true; 599 } 600 601 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 602 data = b->presumed.offset + r->data; 603 else 604 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 605 data = (b->presumed.offset + r->data) >> 32; 606 else 607 data = r->data; 608 609 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 610 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 611 data |= r->tor; 612 else 613 data |= r->vor; 614 } 615 616 spin_lock(&nvbo->bo.bdev->fence_lock); 617 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 618 spin_unlock(&nvbo->bo.bdev->fence_lock); 619 if (ret) { 620 NV_ERROR(drm, "reloc wait_idle failed: %d\n", ret); 621 break; 622 } 623 624 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 625 } 626 627 kfree(reloc); 628 return ret; 629 } 630 631 int 632 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 633 struct drm_file *file_priv) 634 { 635 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev); 636 struct nouveau_abi16_chan *temp; 637 struct nouveau_drm *drm = nouveau_drm(dev); 638 struct drm_nouveau_gem_pushbuf *req = data; 639 struct drm_nouveau_gem_pushbuf_push *push; 640 struct drm_nouveau_gem_pushbuf_bo *bo; 641 struct nouveau_channel *chan = NULL; 642 struct validate_op op; 643 struct nouveau_fence *fence = NULL; 644 int i, j, ret = 0, do_reloc = 0; 645 646 if (unlikely(!abi16)) 647 return -ENOMEM; 648 649 list_for_each_entry(temp, &abi16->channels, head) { 650 if (temp->chan->handle == (NVDRM_CHAN | req->channel)) { 651 chan = temp->chan; 652 break; 653 } 654 } 655 656 if (!chan) 657 return nouveau_abi16_put(abi16, -ENOENT); 658 659 req->vram_available = drm->gem.vram_available; 660 req->gart_available = drm->gem.gart_available; 661 if (unlikely(req->nr_push == 0)) 662 goto out_next; 663 664 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 665 NV_ERROR(drm, "pushbuf push count exceeds limit: %d max %d\n", 666 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 667 return nouveau_abi16_put(abi16, -EINVAL); 668 } 669 670 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 671 NV_ERROR(drm, "pushbuf bo count exceeds limit: %d max %d\n", 672 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 673 return nouveau_abi16_put(abi16, -EINVAL); 674 } 675 676 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 677 NV_ERROR(drm, "pushbuf reloc count exceeds limit: %d max %d\n", 678 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 679 return nouveau_abi16_put(abi16, -EINVAL); 680 } 681 682 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 683 if (IS_ERR(push)) 684 return nouveau_abi16_put(abi16, PTR_ERR(push)); 685 686 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 687 if (IS_ERR(bo)) { 688 kfree(push); 689 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 690 } 691 692 /* Ensure all push buffers are on validate list */ 693 for (i = 0; i < req->nr_push; i++) { 694 if (push[i].bo_index >= req->nr_buffers) { 695 NV_ERROR(drm, "push %d buffer not in list\n", i); 696 ret = -EINVAL; 697 goto out_prevalid; 698 } 699 } 700 701 /* Validate buffer list */ 702 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 703 req->nr_buffers, &op, &do_reloc); 704 if (ret) { 705 if (ret != -ERESTARTSYS) 706 NV_ERROR(drm, "validate: %d\n", ret); 707 goto out_prevalid; 708 } 709 710 /* Apply any relocations that are required */ 711 if (do_reloc) { 712 ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo); 713 if (ret) { 714 NV_ERROR(drm, "reloc apply: %d\n", ret); 715 goto out; 716 } 717 } 718 719 if (chan->dma.ib_max) { 720 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 721 if (ret) { 722 NV_ERROR(drm, "nv50cal_space: %d\n", ret); 723 goto out; 724 } 725 726 for (i = 0; i < req->nr_push; i++) { 727 struct nouveau_bo *nvbo = (void *)(unsigned long) 728 bo[push[i].bo_index].user_priv; 729 730 nv50_dma_push(chan, nvbo, push[i].offset, 731 push[i].length); 732 } 733 } else 734 if (nv_device(drm->device)->chipset >= 0x25) { 735 ret = RING_SPACE(chan, req->nr_push * 2); 736 if (ret) { 737 NV_ERROR(drm, "cal_space: %d\n", ret); 738 goto out; 739 } 740 741 for (i = 0; i < req->nr_push; i++) { 742 struct nouveau_bo *nvbo = (void *)(unsigned long) 743 bo[push[i].bo_index].user_priv; 744 745 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 746 OUT_RING(chan, 0); 747 } 748 } else { 749 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 750 if (ret) { 751 NV_ERROR(drm, "jmp_space: %d\n", ret); 752 goto out; 753 } 754 755 for (i = 0; i < req->nr_push; i++) { 756 struct nouveau_bo *nvbo = (void *)(unsigned long) 757 bo[push[i].bo_index].user_priv; 758 uint32_t cmd; 759 760 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2); 761 cmd |= 0x20000000; 762 if (unlikely(cmd != req->suffix0)) { 763 if (!nvbo->kmap.virtual) { 764 ret = ttm_bo_kmap(&nvbo->bo, 0, 765 nvbo->bo.mem. 766 num_pages, 767 &nvbo->kmap); 768 if (ret) { 769 WIND_RING(chan); 770 goto out; 771 } 772 nvbo->validate_mapped = true; 773 } 774 775 nouveau_bo_wr32(nvbo, (push[i].offset + 776 push[i].length - 8) / 4, cmd); 777 } 778 779 OUT_RING(chan, 0x20000000 | 780 (nvbo->bo.offset + push[i].offset)); 781 OUT_RING(chan, 0); 782 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 783 OUT_RING(chan, 0); 784 } 785 } 786 787 ret = nouveau_fence_new(chan, &fence); 788 if (ret) { 789 NV_ERROR(drm, "error fencing pushbuf: %d\n", ret); 790 WIND_RING(chan); 791 goto out; 792 } 793 794 out: 795 validate_fini(&op, fence); 796 nouveau_fence_unref(&fence); 797 798 out_prevalid: 799 kfree(bo); 800 kfree(push); 801 802 out_next: 803 if (chan->dma.ib_max) { 804 req->suffix0 = 0x00000000; 805 req->suffix1 = 0x00000000; 806 } else 807 if (nv_device(drm->device)->chipset >= 0x25) { 808 req->suffix0 = 0x00020000; 809 req->suffix1 = 0x00000000; 810 } else { 811 req->suffix0 = 0x20000000 | 812 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2)); 813 req->suffix1 = 0x00000000; 814 } 815 816 return nouveau_abi16_put(abi16, ret); 817 } 818 819 static inline uint32_t 820 domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 821 { 822 uint32_t flags = 0; 823 824 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 825 flags |= TTM_PL_FLAG_VRAM; 826 if (domain & NOUVEAU_GEM_DOMAIN_GART) 827 flags |= TTM_PL_FLAG_TT; 828 829 return flags; 830 } 831 832 int 833 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 834 struct drm_file *file_priv) 835 { 836 struct drm_nouveau_gem_cpu_prep *req = data; 837 struct drm_gem_object *gem; 838 struct nouveau_bo *nvbo; 839 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 840 int ret = -EINVAL; 841 842 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 843 if (!gem) 844 return -ENOENT; 845 nvbo = nouveau_gem_object(gem); 846 847 spin_lock(&nvbo->bo.bdev->fence_lock); 848 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); 849 spin_unlock(&nvbo->bo.bdev->fence_lock); 850 drm_gem_object_unreference_unlocked(gem); 851 return ret; 852 } 853 854 int 855 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 856 struct drm_file *file_priv) 857 { 858 return 0; 859 } 860 861 int 862 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 863 struct drm_file *file_priv) 864 { 865 struct drm_nouveau_gem_info *req = data; 866 struct drm_gem_object *gem; 867 int ret; 868 869 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 870 if (!gem) 871 return -ENOENT; 872 873 ret = nouveau_gem_info(file_priv, gem, req); 874 drm_gem_object_unreference_unlocked(gem); 875 return ret; 876 } 877 878