1 /*
2 * Copyright (C) 2008 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
26
27 #include <drm/drm_gem_ttm_helper.h>
28
29 #include "nouveau_drv.h"
30 #include "nouveau_dma.h"
31 #include "nouveau_fence.h"
32 #include "nouveau_abi16.h"
33
34 #include "nouveau_ttm.h"
35 #include "nouveau_gem.h"
36 #include "nouveau_mem.h"
37 #include "nouveau_vmm.h"
38
39 #include <nvif/class.h>
40 #include <nvif/push206e.h>
41
nouveau_ttm_fault(struct vm_fault * vmf)42 static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf)
43 {
44 struct vm_area_struct *vma = vmf->vma;
45 struct ttm_buffer_object *bo = vma->vm_private_data;
46 pgprot_t prot;
47 vm_fault_t ret;
48
49 ret = ttm_bo_vm_reserve(bo, vmf);
50 if (ret)
51 return ret;
52
53 ret = nouveau_ttm_fault_reserve_notify(bo);
54 if (ret)
55 goto error_unlock;
56
57 nouveau_bo_del_io_reserve_lru(bo);
58 prot = vma_get_page_prot(vma);
59 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
60 nouveau_bo_add_io_reserve_lru(bo);
61 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
62 return ret;
63
64 error_unlock:
65 dma_resv_unlock(bo->base.resv);
66 return ret;
67 }
68
69 static const struct vm_operations_struct nouveau_ttm_vm_ops = {
70 .fault = nouveau_ttm_fault,
71 .open = ttm_bo_vm_open,
72 .close = ttm_bo_vm_close,
73 .access = ttm_bo_vm_access
74 };
75
76 void
nouveau_gem_object_del(struct drm_gem_object * gem)77 nouveau_gem_object_del(struct drm_gem_object *gem)
78 {
79 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
80 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
81 struct device *dev = drm->dev->dev;
82 int ret;
83
84 ret = pm_runtime_get_sync(dev);
85 if (WARN_ON(ret < 0 && ret != -EACCES)) {
86 pm_runtime_put_autosuspend(dev);
87 return;
88 }
89
90 ttm_bo_fini(&nvbo->bo);
91
92 pm_runtime_mark_last_busy(dev);
93 pm_runtime_put_autosuspend(dev);
94 }
95
96 int
nouveau_gem_object_open(struct drm_gem_object * gem,struct drm_file * file_priv)97 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
98 {
99 struct nouveau_cli *cli = nouveau_cli(file_priv);
100 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
101 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
102 struct device *dev = drm->dev->dev;
103 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli);
104 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli);
105 struct nouveau_vma *vma;
106 int ret;
107
108 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
109 return 0;
110
111 if (nvbo->no_share && uvmm &&
112 drm_gpuvm_resv(&uvmm->base) != nvbo->bo.base.resv)
113 return -EPERM;
114
115 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
116 if (ret)
117 return ret;
118
119 ret = pm_runtime_get_sync(dev);
120 if (ret < 0 && ret != -EACCES) {
121 pm_runtime_put_autosuspend(dev);
122 goto out;
123 }
124
125 /* only create a VMA on binding */
126 if (!nouveau_cli_uvmm(cli))
127 ret = nouveau_vma_new(nvbo, vmm, &vma);
128 else
129 ret = 0;
130 pm_runtime_mark_last_busy(dev);
131 pm_runtime_put_autosuspend(dev);
132 out:
133 ttm_bo_unreserve(&nvbo->bo);
134 return ret;
135 }
136
137 struct nouveau_gem_object_unmap {
138 struct nouveau_cli_work work;
139 struct nouveau_vma *vma;
140 };
141
142 static void
nouveau_gem_object_delete(struct nouveau_vma * vma)143 nouveau_gem_object_delete(struct nouveau_vma *vma)
144 {
145 nouveau_fence_unref(&vma->fence);
146 nouveau_vma_del(&vma);
147 }
148
149 static void
nouveau_gem_object_delete_work(struct nouveau_cli_work * w)150 nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
151 {
152 struct nouveau_gem_object_unmap *work =
153 container_of(w, typeof(*work), work);
154 nouveau_gem_object_delete(work->vma);
155 kfree(work);
156 }
157
158 static void
nouveau_gem_object_unmap(struct nouveau_bo * nvbo,struct nouveau_vma * vma)159 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
160 {
161 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
162 struct nouveau_gem_object_unmap *work;
163
164 list_del_init(&vma->head);
165
166 if (!fence) {
167 nouveau_gem_object_delete(vma);
168 return;
169 }
170
171 if (!(work = kmalloc_obj(*work))) {
172 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
173 nouveau_gem_object_delete(vma);
174 return;
175 }
176
177 work->work.func = nouveau_gem_object_delete_work;
178 work->vma = vma;
179 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
180 }
181
182 void
nouveau_gem_object_close(struct drm_gem_object * gem,struct drm_file * file_priv)183 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
184 {
185 struct nouveau_cli *cli = nouveau_cli(file_priv);
186 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
187 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
188 struct device *dev = drm->dev->dev;
189 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli);
190 struct nouveau_vma *vma;
191 int ret;
192
193 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
194 return;
195
196 if (nouveau_cli_uvmm(cli))
197 return;
198
199 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
200 if (ret)
201 return;
202
203 vma = nouveau_vma_find(nvbo, vmm);
204 if (vma) {
205 if (--vma->refs == 0) {
206 ret = pm_runtime_get_sync(dev);
207 if (!WARN_ON(ret < 0 && ret != -EACCES)) {
208 nouveau_gem_object_unmap(nvbo, vma);
209 pm_runtime_mark_last_busy(dev);
210 }
211 pm_runtime_put_autosuspend(dev);
212 }
213 }
214 ttm_bo_unreserve(&nvbo->bo);
215 }
216
217 const struct drm_gem_object_funcs nouveau_gem_object_funcs = {
218 .free = nouveau_gem_object_del,
219 .open = nouveau_gem_object_open,
220 .close = nouveau_gem_object_close,
221 .export = nouveau_gem_prime_export,
222 .pin = nouveau_gem_prime_pin,
223 .unpin = nouveau_gem_prime_unpin,
224 .get_sg_table = nouveau_gem_prime_get_sg_table,
225 .vmap = drm_gem_ttm_vmap,
226 .vunmap = drm_gem_ttm_vunmap,
227 .mmap = drm_gem_ttm_mmap,
228 .vm_ops = &nouveau_ttm_vm_ops,
229 };
230
231 int
nouveau_gem_new(struct nouveau_cli * cli,u64 size,int align,uint32_t domain,uint32_t tile_mode,uint32_t tile_flags,struct nouveau_bo ** pnvbo)232 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
233 uint32_t tile_mode, uint32_t tile_flags,
234 struct nouveau_bo **pnvbo)
235 {
236 struct nouveau_drm *drm = cli->drm;
237 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli);
238 struct dma_resv *resv = NULL;
239 struct nouveau_bo *nvbo;
240 int ret;
241
242 if (domain & NOUVEAU_GEM_DOMAIN_NO_SHARE) {
243 if (unlikely(!uvmm))
244 return -EINVAL;
245
246 resv = drm_gpuvm_resv(&uvmm->base);
247 }
248
249 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
250 domain |= NOUVEAU_GEM_DOMAIN_CPU;
251
252 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
253 tile_flags, false);
254 if (IS_ERR(nvbo))
255 return PTR_ERR(nvbo);
256
257 nvbo->bo.base.funcs = &nouveau_gem_object_funcs;
258 nvbo->no_share = domain & NOUVEAU_GEM_DOMAIN_NO_SHARE;
259
260 /* Initialize the embedded gem-object. We return a single gem-reference
261 * to the caller, instead of a normal nouveau_bo ttm reference. */
262 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size);
263 if (ret) {
264 drm_gem_object_release(&nvbo->bo.base);
265 kfree(nvbo);
266 return ret;
267 }
268
269 if (resv)
270 dma_resv_lock(resv, NULL);
271
272 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, resv);
273
274 if (resv)
275 dma_resv_unlock(resv);
276
277 if (ret)
278 return ret;
279
280 /* we restrict allowed domains on nv50+ to only the types
281 * that were requested at creation time. not possibly on
282 * earlier chips without busting the ABI.
283 */
284 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
285 NOUVEAU_GEM_DOMAIN_GART;
286 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
287 nvbo->valid_domains &= domain;
288
289 if (nvbo->no_share) {
290 nvbo->r_obj = drm_gpuvm_resv_obj(&uvmm->base);
291 drm_gem_object_get(nvbo->r_obj);
292 }
293
294 *pnvbo = nvbo;
295 return 0;
296 }
297
298 static int
nouveau_gem_info(struct drm_file * file_priv,struct drm_gem_object * gem,struct drm_nouveau_gem_info * rep)299 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
300 struct drm_nouveau_gem_info *rep)
301 {
302 struct nouveau_cli *cli = nouveau_cli(file_priv);
303 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
304 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli);
305 struct nouveau_vma *vma;
306
307 if (is_power_of_2(nvbo->valid_domains))
308 rep->domain = nvbo->valid_domains;
309 else if (nvbo->bo.resource->mem_type == TTM_PL_TT)
310 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
311 else
312 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
313 rep->offset = nvbo->offset;
314 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 &&
315 !nouveau_cli_uvmm(cli)) {
316 int ret;
317
318 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
319 if (ret)
320 return ret;
321
322 vma = nouveau_vma_find(nvbo, vmm);
323 if (!vma) {
324 ttm_bo_unreserve(&nvbo->bo);
325 return -EINVAL;
326 }
327
328 rep->offset = vma->addr;
329 ttm_bo_unreserve(&nvbo->bo);
330 } else
331 rep->offset = 0;
332
333 rep->size = nvbo->bo.base.size;
334 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node);
335 rep->tile_mode = nvbo->mode;
336 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
337 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
338 rep->tile_flags |= nvbo->kind << 8;
339 else
340 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
341 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
342 else
343 rep->tile_flags |= nvbo->zeta;
344 return 0;
345 }
346
347 int
nouveau_gem_ioctl_new(struct drm_device * dev,void * data,struct drm_file * file_priv)348 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
349 struct drm_file *file_priv)
350 {
351 struct nouveau_cli *cli = nouveau_cli(file_priv);
352 struct drm_nouveau_gem_new *req = data;
353 struct nouveau_bo *nvbo = NULL;
354 int ret = 0;
355
356 /* If uvmm wasn't initialized until now disable it completely to prevent
357 * userspace from mixing up UAPIs.
358 */
359 nouveau_cli_disable_uvmm_noinit(cli);
360
361 ret = nouveau_gem_new(cli, req->info.size, req->align,
362 req->info.domain, req->info.tile_mode,
363 req->info.tile_flags, &nvbo);
364 if (ret)
365 return ret;
366
367 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base,
368 &req->info.handle);
369 if (ret == 0) {
370 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info);
371 if (ret)
372 drm_gem_handle_delete(file_priv, req->info.handle);
373 }
374
375 /* drop reference from allocate - handle holds it now */
376 drm_gem_object_put(&nvbo->bo.base);
377 return ret;
378 }
379
380 static int
nouveau_gem_set_domain(struct drm_gem_object * gem,uint32_t read_domains,uint32_t write_domains,uint32_t valid_domains)381 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
382 uint32_t write_domains, uint32_t valid_domains)
383 {
384 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
385 struct ttm_buffer_object *bo = &nvbo->bo;
386 uint32_t domains = valid_domains & nvbo->valid_domains &
387 (write_domains ? write_domains : read_domains);
388 uint32_t pref_domains = 0;
389
390 if (!domains)
391 return -EINVAL;
392
393 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART);
394
395 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
396 bo->resource->mem_type == TTM_PL_VRAM)
397 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
398
399 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
400 bo->resource->mem_type == TTM_PL_TT)
401 pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
402
403 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
404 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
405
406 else
407 pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
408
409 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains);
410
411 return 0;
412 }
413
414 struct validate_op {
415 struct list_head list;
416 struct ww_acquire_ctx ticket;
417 };
418
419 static void
validate_fini_no_ticket(struct validate_op * op,struct nouveau_channel * chan,struct nouveau_fence * fence,struct drm_nouveau_gem_pushbuf_bo * pbbo)420 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
421 struct nouveau_fence *fence,
422 struct drm_nouveau_gem_pushbuf_bo *pbbo)
423 {
424 struct nouveau_bo *nvbo;
425 struct drm_nouveau_gem_pushbuf_bo *b;
426
427 while (!list_empty(&op->list)) {
428 nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
429 b = &pbbo[nvbo->pbbo_index];
430
431 if (likely(fence)) {
432 nouveau_bo_fence(nvbo, fence, !!b->write_domains);
433
434 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
435 struct nouveau_vma *vma =
436 (void *)(unsigned long)b->user_priv;
437 nouveau_fence_unref(&vma->fence);
438 dma_fence_get(&fence->base);
439 vma->fence = fence;
440 }
441 }
442
443 if (unlikely(nvbo->validate_mapped)) {
444 ttm_bo_kunmap(&nvbo->kmap);
445 nvbo->validate_mapped = false;
446 }
447
448 list_del(&nvbo->entry);
449 nvbo->reserved_by = NULL;
450 ttm_bo_unreserve(&nvbo->bo);
451 drm_gem_object_put(&nvbo->bo.base);
452 }
453 }
454
455 static void
validate_fini(struct validate_op * op,struct nouveau_channel * chan,struct nouveau_fence * fence,struct drm_nouveau_gem_pushbuf_bo * pbbo)456 validate_fini(struct validate_op *op, struct nouveau_channel *chan,
457 struct nouveau_fence *fence,
458 struct drm_nouveau_gem_pushbuf_bo *pbbo)
459 {
460 validate_fini_no_ticket(op, chan, fence, pbbo);
461 ww_acquire_fini(&op->ticket);
462 }
463
464 static int
validate_init(struct nouveau_channel * chan,struct drm_file * file_priv,struct drm_nouveau_gem_pushbuf_bo * pbbo,int nr_buffers,struct validate_op * op)465 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
466 struct drm_nouveau_gem_pushbuf_bo *pbbo,
467 int nr_buffers, struct validate_op *op)
468 {
469 struct nouveau_cli *cli = nouveau_cli(file_priv);
470 int trycnt = 0;
471 int ret = -EINVAL, i;
472 struct nouveau_bo *res_bo = NULL;
473 LIST_HEAD(gart_list);
474 LIST_HEAD(vram_list);
475 LIST_HEAD(both_list);
476
477 ww_acquire_init(&op->ticket, &reservation_ww_class);
478 retry:
479 if (++trycnt > 100000) {
480 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
481 return -EINVAL;
482 }
483
484 for (i = 0; i < nr_buffers; i++) {
485 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
486 struct drm_gem_object *gem;
487 struct nouveau_bo *nvbo;
488
489 gem = drm_gem_object_lookup(file_priv, b->handle);
490 if (!gem) {
491 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
492 ret = -ENOENT;
493 break;
494 }
495 nvbo = nouveau_gem_object(gem);
496 if (nvbo == res_bo) {
497 res_bo = NULL;
498 drm_gem_object_put(gem);
499 continue;
500 }
501
502 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
503 NV_PRINTK(err, cli, "multiple instances of buffer %d on "
504 "validation list\n", b->handle);
505 drm_gem_object_put(gem);
506 ret = -EINVAL;
507 break;
508 }
509
510 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
511 if (ret) {
512 list_splice_tail_init(&vram_list, &op->list);
513 list_splice_tail_init(&gart_list, &op->list);
514 list_splice_tail_init(&both_list, &op->list);
515 validate_fini_no_ticket(op, chan, NULL, NULL);
516 if (unlikely(ret == -EDEADLK)) {
517 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
518 &op->ticket);
519 if (!ret)
520 res_bo = nvbo;
521 }
522 if (unlikely(ret)) {
523 if (ret != -ERESTARTSYS)
524 NV_PRINTK(err, cli, "fail reserve\n");
525 drm_gem_object_put(gem);
526 break;
527 }
528 }
529
530 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
531 struct nouveau_vmm *vmm = chan->vmm;
532 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
533 if (!vma) {
534 NV_PRINTK(err, cli, "vma not found!\n");
535 drm_gem_object_put(gem);
536 ret = -EINVAL;
537 break;
538 }
539
540 b->user_priv = (uint64_t)(unsigned long)vma;
541 } else {
542 b->user_priv = (uint64_t)(unsigned long)nvbo;
543 }
544
545 nvbo->reserved_by = file_priv;
546 nvbo->pbbo_index = i;
547 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
548 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
549 list_add_tail(&nvbo->entry, &both_list);
550 else
551 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
552 list_add_tail(&nvbo->entry, &vram_list);
553 else
554 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
555 list_add_tail(&nvbo->entry, &gart_list);
556 else {
557 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
558 b->valid_domains);
559 list_add_tail(&nvbo->entry, &both_list);
560 ret = -EINVAL;
561 break;
562 }
563 if (nvbo == res_bo)
564 goto retry;
565 }
566
567 ww_acquire_done(&op->ticket);
568 list_splice_tail(&vram_list, &op->list);
569 list_splice_tail(&gart_list, &op->list);
570 list_splice_tail(&both_list, &op->list);
571 if (ret)
572 validate_fini(op, chan, NULL, NULL);
573 return ret;
574
575 }
576
577 static int
validate_list(struct nouveau_channel * chan,struct list_head * list,struct drm_nouveau_gem_pushbuf_bo * pbbo)578 validate_list(struct nouveau_channel *chan,
579 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo)
580 {
581 struct nouveau_cli *cli = chan->cli;
582 struct nouveau_drm *drm = cli->drm;
583 struct nouveau_bo *nvbo;
584 int ret, relocs = 0;
585
586 list_for_each_entry(nvbo, list, entry) {
587 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
588
589 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains,
590 b->write_domains,
591 b->valid_domains);
592 if (unlikely(ret)) {
593 NV_PRINTK(err, cli, "fail set_domain\n");
594 return ret;
595 }
596
597 ret = nouveau_bo_validate(nvbo, true, false);
598 if (unlikely(ret)) {
599 if (ret != -ERESTARTSYS)
600 NV_PRINTK(err, cli, "fail ttm_validate\n");
601 return ret;
602 }
603
604 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
605 if (unlikely(ret)) {
606 if (ret != -ERESTARTSYS)
607 NV_PRINTK(err, cli, "fail post-validate sync\n");
608 return ret;
609 }
610
611 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
612 if (nvbo->offset == b->presumed.offset &&
613 ((nvbo->bo.resource->mem_type == TTM_PL_VRAM &&
614 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
615 (nvbo->bo.resource->mem_type == TTM_PL_TT &&
616 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
617 continue;
618
619 if (nvbo->bo.resource->mem_type == TTM_PL_TT)
620 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
621 else
622 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
623 b->presumed.offset = nvbo->offset;
624 b->presumed.valid = 0;
625 relocs++;
626 }
627 }
628
629 return relocs;
630 }
631
632 static int
nouveau_gem_pushbuf_validate(struct nouveau_channel * chan,struct drm_file * file_priv,struct drm_nouveau_gem_pushbuf_bo * pbbo,int nr_buffers,struct validate_op * op,bool * apply_relocs)633 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
634 struct drm_file *file_priv,
635 struct drm_nouveau_gem_pushbuf_bo *pbbo,
636 int nr_buffers,
637 struct validate_op *op, bool *apply_relocs)
638 {
639 struct nouveau_cli *cli = nouveau_cli(file_priv);
640 int ret;
641
642 INIT_LIST_HEAD(&op->list);
643
644 if (nr_buffers == 0)
645 return 0;
646
647 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
648 if (unlikely(ret)) {
649 if (ret != -ERESTARTSYS)
650 NV_PRINTK(err, cli, "validate_init\n");
651 return ret;
652 }
653
654 ret = validate_list(chan, &op->list, pbbo);
655 if (unlikely(ret < 0)) {
656 if (ret != -ERESTARTSYS)
657 NV_PRINTK(err, cli, "validating bo list\n");
658 validate_fini(op, chan, NULL, NULL);
659 return ret;
660 } else if (ret > 0) {
661 *apply_relocs = true;
662 }
663
664 return 0;
665 }
666
667 static int
nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli * cli,struct drm_nouveau_gem_pushbuf * req,struct drm_nouveau_gem_pushbuf_reloc * reloc,struct drm_nouveau_gem_pushbuf_bo * bo)668 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
669 struct drm_nouveau_gem_pushbuf *req,
670 struct drm_nouveau_gem_pushbuf_reloc *reloc,
671 struct drm_nouveau_gem_pushbuf_bo *bo)
672 {
673 int ret = 0;
674 unsigned i;
675
676 for (i = 0; i < req->nr_relocs; i++) {
677 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
678 struct drm_nouveau_gem_pushbuf_bo *b;
679 struct nouveau_bo *nvbo;
680 uint32_t data;
681 long lret;
682
683 if (unlikely(r->bo_index >= req->nr_buffers)) {
684 NV_PRINTK(err, cli, "reloc bo index invalid\n");
685 ret = -EINVAL;
686 break;
687 }
688
689 b = &bo[r->bo_index];
690 if (b->presumed.valid)
691 continue;
692
693 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
694 NV_PRINTK(err, cli, "reloc container bo index invalid\n");
695 ret = -EINVAL;
696 break;
697 }
698 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
699
700 if (unlikely((u64)r->reloc_bo_offset + 4 >
701 nvbo->bo.base.size)) {
702 NV_PRINTK(err, cli, "reloc outside of bo\n");
703 ret = -EINVAL;
704 break;
705 }
706
707 if (!nvbo->kmap.virtual) {
708 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size),
709 &nvbo->kmap);
710 if (ret) {
711 NV_PRINTK(err, cli, "failed kmap for reloc\n");
712 break;
713 }
714 nvbo->validate_mapped = true;
715 }
716
717 if (r->flags & NOUVEAU_GEM_RELOC_LOW)
718 data = b->presumed.offset + r->data;
719 else
720 if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
721 data = (b->presumed.offset + r->data) >> 32;
722 else
723 data = r->data;
724
725 if (r->flags & NOUVEAU_GEM_RELOC_OR) {
726 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
727 data |= r->tor;
728 else
729 data |= r->vor;
730 }
731
732 lret = dma_resv_wait_timeout(nvbo->bo.base.resv,
733 DMA_RESV_USAGE_BOOKKEEP,
734 false, 15 * HZ);
735 if (!lret)
736 ret = -EBUSY;
737 else if (lret > 0)
738 ret = 0;
739 else
740 ret = lret;
741
742 if (ret) {
743 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n",
744 ret);
745 break;
746 }
747
748 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
749 }
750
751 return ret;
752 }
753
754 int
nouveau_gem_ioctl_pushbuf(struct drm_device * dev,void * data,struct drm_file * file_priv)755 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
756 struct drm_file *file_priv)
757 {
758 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
759 struct nouveau_cli *cli = nouveau_cli(file_priv);
760 struct nouveau_abi16_chan *temp;
761 struct nouveau_drm *drm = nouveau_drm(dev);
762 struct drm_nouveau_gem_pushbuf *req = data;
763 struct drm_nouveau_gem_pushbuf_push *push;
764 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
765 struct drm_nouveau_gem_pushbuf_bo *bo;
766 struct nouveau_channel *chan = NULL;
767 struct validate_op op;
768 struct nouveau_fence *fence = NULL;
769 int i, j, ret = 0;
770 bool do_reloc = false, sync = false;
771
772 if (unlikely(!abi16))
773 return -ENOMEM;
774
775 if (unlikely(nouveau_cli_uvmm(cli)))
776 return nouveau_abi16_put(abi16, -ENOSYS);
777
778 list_for_each_entry(temp, &abi16->channels, head) {
779 if (temp->chan->chid == req->channel) {
780 chan = temp->chan;
781 break;
782 }
783 }
784
785 if (!chan)
786 return nouveau_abi16_put(abi16, -ENOENT);
787 if (unlikely(atomic_read(&chan->killed)))
788 return nouveau_abi16_put(abi16, -ENODEV);
789
790 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC;
791
792 req->vram_available = drm->gem.vram_available;
793 req->gart_available = drm->gem.gart_available;
794 if (unlikely(req->nr_push == 0))
795 goto out_next;
796
797 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
798 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
799 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
800 return nouveau_abi16_put(abi16, -EINVAL);
801 }
802
803 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
804 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
805 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
806 return nouveau_abi16_put(abi16, -EINVAL);
807 }
808
809 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
810 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
811 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
812 return nouveau_abi16_put(abi16, -EINVAL);
813 }
814
815 push = u_memcpya(req->push, req->nr_push, sizeof(*push));
816 if (IS_ERR(push))
817 return nouveau_abi16_put(abi16, PTR_ERR(push));
818
819 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
820 if (IS_ERR(bo)) {
821 u_free(push);
822 return nouveau_abi16_put(abi16, PTR_ERR(bo));
823 }
824
825 /* Ensure all push buffers are on validate list */
826 for (i = 0; i < req->nr_push; i++) {
827 if (push[i].bo_index >= req->nr_buffers) {
828 NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
829 ret = -EINVAL;
830 goto out_prevalid;
831 }
832 }
833
834 /* Validate buffer list */
835 revalidate:
836 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo,
837 req->nr_buffers, &op, &do_reloc);
838 if (ret) {
839 if (ret != -ERESTARTSYS)
840 NV_PRINTK(err, cli, "validate: %d\n", ret);
841 goto out_prevalid;
842 }
843
844 /* Apply any relocations that are required */
845 if (do_reloc) {
846 if (!reloc) {
847 validate_fini(&op, chan, NULL, bo);
848 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
849 if (IS_ERR(reloc)) {
850 ret = PTR_ERR(reloc);
851 goto out_prevalid;
852 }
853
854 goto revalidate;
855 }
856
857 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo);
858 if (ret) {
859 NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
860 goto out;
861 }
862 }
863
864 if (chan->user.oclass >= NV50_CHANNEL_GPFIFO) {
865 ret = nvif_chan_gpfifo_wait(&chan->chan, req->nr_push + 1, 16);
866 if (ret) {
867 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
868 goto out;
869 }
870
871 for (i = 0; i < req->nr_push; i++) {
872 struct nouveau_vma *vma = (void *)(unsigned long)
873 bo[push[i].bo_index].user_priv;
874 u64 addr = vma->addr + push[i].offset;
875 u32 length = push[i].length & ~NOUVEAU_GEM_PUSHBUF_NO_PREFETCH;
876 bool no_prefetch = push[i].length & NOUVEAU_GEM_PUSHBUF_NO_PREFETCH;
877
878 nvif_chan_gpfifo_push(&chan->chan, addr, length, no_prefetch);
879 }
880
881 nvif_chan_gpfifo_post(&chan->chan);
882 } else
883 if (drm->client.device.info.chipset >= 0x25) {
884 ret = PUSH_WAIT(&chan->chan.push, req->nr_push * 2);
885 if (ret) {
886 NV_PRINTK(err, cli, "cal_space: %d\n", ret);
887 goto out;
888 }
889
890 for (i = 0; i < req->nr_push; i++) {
891 struct nouveau_bo *nvbo = (void *)(unsigned long)
892 bo[push[i].bo_index].user_priv;
893
894 PUSH_CALL(&chan->chan.push, nvbo->offset + push[i].offset);
895 PUSH_DATA(&chan->chan.push, 0);
896 }
897 } else {
898 ret = PUSH_WAIT(&chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
899 if (ret) {
900 NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
901 goto out;
902 }
903
904 for (i = 0; i < req->nr_push; i++) {
905 struct nouveau_bo *nvbo = (void *)(unsigned long)
906 bo[push[i].bo_index].user_priv;
907 uint32_t cmd;
908
909 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
910 cmd |= 0x20000000;
911 if (unlikely(cmd != req->suffix0)) {
912 if (!nvbo->kmap.virtual) {
913 ret = ttm_bo_kmap(&nvbo->bo, 0,
914 PFN_UP(nvbo->bo.base.size),
915 &nvbo->kmap);
916 if (ret) {
917 WIND_RING(chan);
918 goto out;
919 }
920 nvbo->validate_mapped = true;
921 }
922
923 nouveau_bo_wr32(nvbo, (push[i].offset +
924 push[i].length - 8) / 4, cmd);
925 }
926
927 PUSH_JUMP(&chan->chan.push, nvbo->offset + push[i].offset);
928 PUSH_DATA(&chan->chan.push, 0);
929 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
930 PUSH_DATA(&chan->chan.push, 0);
931 }
932 }
933
934 ret = nouveau_fence_new(&fence, chan);
935 if (ret) {
936 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
937 WIND_RING(chan);
938 goto out;
939 }
940
941 if (sync) {
942 if (!(ret = nouveau_fence_wait(fence, false, false))) {
943 if ((ret = dma_fence_get_status(&fence->base)) == 1)
944 ret = 0;
945 }
946 }
947
948 out:
949 validate_fini(&op, chan, fence, bo);
950 nouveau_fence_unref(&fence);
951
952 if (do_reloc) {
953 struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
954 u64_to_user_ptr(req->buffers);
955
956 for (i = 0; i < req->nr_buffers; i++) {
957 if (bo[i].presumed.valid)
958 continue;
959
960 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed,
961 sizeof(bo[i].presumed))) {
962 ret = -EFAULT;
963 break;
964 }
965 }
966 }
967 out_prevalid:
968 if (!IS_ERR(reloc))
969 u_free(reloc);
970 u_free(bo);
971 u_free(push);
972
973 out_next:
974 if (chan->user.oclass >= NV50_CHANNEL_GPFIFO) {
975 req->suffix0 = 0x00000000;
976 req->suffix1 = 0x00000000;
977 } else
978 if (drm->client.device.info.chipset >= 0x25) {
979 req->suffix0 = 0x00020000;
980 req->suffix1 = 0x00000000;
981 } else {
982 req->suffix0 = 0x20000000 |
983 (chan->push.addr + ((chan->dma.cur + 2) << 2));
984 req->suffix1 = 0x00000000;
985 }
986
987 return nouveau_abi16_put(abi16, ret);
988 }
989
990 int
nouveau_gem_ioctl_cpu_prep(struct drm_device * dev,void * data,struct drm_file * file_priv)991 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
992 struct drm_file *file_priv)
993 {
994 struct drm_nouveau_gem_cpu_prep *req = data;
995 struct drm_gem_object *gem;
996 struct nouveau_bo *nvbo;
997 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
998 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
999 long lret;
1000 int ret;
1001
1002 gem = drm_gem_object_lookup(file_priv, req->handle);
1003 if (!gem)
1004 return -ENOENT;
1005 nvbo = nouveau_gem_object(gem);
1006
1007 lret = dma_resv_wait_timeout(nvbo->bo.base.resv,
1008 dma_resv_usage_rw(write), true,
1009 no_wait ? 0 : 30 * HZ);
1010 if (!lret)
1011 ret = -EBUSY;
1012 else if (lret > 0)
1013 ret = 0;
1014 else
1015 ret = lret;
1016
1017 nouveau_bo_sync_for_cpu(nvbo);
1018 drm_gem_object_put(gem);
1019
1020 return ret;
1021 }
1022
1023 int
nouveau_gem_ioctl_cpu_fini(struct drm_device * dev,void * data,struct drm_file * file_priv)1024 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
1025 struct drm_file *file_priv)
1026 {
1027 struct drm_nouveau_gem_cpu_fini *req = data;
1028 struct drm_gem_object *gem;
1029 struct nouveau_bo *nvbo;
1030
1031 gem = drm_gem_object_lookup(file_priv, req->handle);
1032 if (!gem)
1033 return -ENOENT;
1034 nvbo = nouveau_gem_object(gem);
1035
1036 nouveau_bo_sync_for_device(nvbo);
1037 drm_gem_object_put(gem);
1038 return 0;
1039 }
1040
1041 int
nouveau_gem_ioctl_info(struct drm_device * dev,void * data,struct drm_file * file_priv)1042 nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
1043 struct drm_file *file_priv)
1044 {
1045 struct drm_nouveau_gem_info *req = data;
1046 struct drm_gem_object *gem;
1047 int ret;
1048
1049 gem = drm_gem_object_lookup(file_priv, req->handle);
1050 if (!gem)
1051 return -ENOENT;
1052
1053 ret = nouveau_gem_info(file_priv, gem, req);
1054 drm_gem_object_put(gem);
1055 return ret;
1056 }
1057
1058