xref: /linux/drivers/gpu/drm/msm/msm_gem_vma.c (revision 44343e8b250abb2f6bfd615493ca07a7f11f3cc2)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include "drm/drm_file.h"
8 #include "drm/msm_drm.h"
9 #include "linux/file.h"
10 #include "linux/sync_file.h"
11 
12 #include "msm_drv.h"
13 #include "msm_gem.h"
14 #include "msm_gpu.h"
15 #include "msm_mmu.h"
16 #include "msm_syncobj.h"
17 
18 #define vm_dbg(fmt, ...) pr_debug("%s:%d: "fmt"\n", __func__, __LINE__, ##__VA_ARGS__)
19 
20 static uint vm_log_shift = 0;
21 MODULE_PARM_DESC(vm_log_shift, "Length of VM op log");
22 module_param_named(vm_log_shift, vm_log_shift, uint, 0600);
23 
24 /**
25  * struct msm_vm_map_op - create new pgtable mapping
26  */
27 struct msm_vm_map_op {
28 	/** @iova: start address for mapping */
29 	uint64_t iova;
30 	/** @range: size of the region to map */
31 	uint64_t range;
32 	/** @offset: offset into @sgt to map */
33 	uint64_t offset;
34 	/** @sgt: pages to map, or NULL for a PRR mapping */
35 	struct sg_table *sgt;
36 	/** @prot: the mapping protection flags */
37 	int prot;
38 
39 	/**
40 	 * @queue_id: The id of the submitqueue the operation is performed
41 	 * on, or zero for (in particular) UNMAP ops triggered outside of
42 	 * a submitqueue (ie. process cleanup)
43 	 */
44 	int queue_id;
45 };
46 
47 /**
48  * struct msm_vm_unmap_op - unmap a range of pages from pgtable
49  */
50 struct msm_vm_unmap_op {
51 	/** @iova: start address for unmap */
52 	uint64_t iova;
53 	/** @range: size of region to unmap */
54 	uint64_t range;
55 
56 	/** @reason: The reason for the unmap */
57 	const char *reason;
58 
59 	/**
60 	 * @queue_id: The id of the submitqueue the operation is performed
61 	 * on, or zero for (in particular) UNMAP ops triggered outside of
62 	 * a submitqueue (ie. process cleanup)
63 	 */
64 	int queue_id;
65 };
66 
67 /**
68  * struct msm_vma_op - A MAP or UNMAP operation
69  */
70 struct msm_vm_op {
71 	/** @op: The operation type */
72 	enum {
73 		MSM_VM_OP_MAP = 1,
74 		MSM_VM_OP_UNMAP,
75 	} op;
76 	union {
77 		/** @map: Parameters used if op == MSM_VMA_OP_MAP */
78 		struct msm_vm_map_op map;
79 		/** @unmap: Parameters used if op == MSM_VMA_OP_UNMAP */
80 		struct msm_vm_unmap_op unmap;
81 	};
82 	/** @node: list head in msm_vm_bind_job::vm_ops */
83 	struct list_head node;
84 
85 	/**
86 	 * @obj: backing object for pages to be mapped/unmapped
87 	 *
88 	 * Async unmap ops, in particular, must hold a reference to the
89 	 * original GEM object backing the mapping that will be unmapped.
90 	 * But the same can be required in the map path, for example if
91 	 * there is not a corresponding unmap op, such as process exit.
92 	 *
93 	 * This ensures that the pages backing the mapping are not freed
94 	 * before the mapping is torn down.
95 	 */
96 	struct drm_gem_object *obj;
97 };
98 
99 /**
100  * struct msm_vm_bind_job - Tracking for a VM_BIND ioctl
101  *
102  * A table of userspace requested VM updates (MSM_VM_BIND_OP_UNMAP/MAP/MAP_NULL)
103  * gets applied to the vm, generating a list of VM ops (MSM_VM_OP_MAP/UNMAP)
104  * which are applied to the pgtables asynchronously.  For example a userspace
105  * requested MSM_VM_BIND_OP_MAP could end up generating both an MSM_VM_OP_UNMAP
106  * to unmap an existing mapping, and a MSM_VM_OP_MAP to apply the new mapping.
107  */
108 struct msm_vm_bind_job {
109 	/** @base: base class for drm_sched jobs */
110 	struct drm_sched_job base;
111 	/** @vm: The VM being operated on */
112 	struct drm_gpuvm *vm;
113 	/** @fence: The fence that is signaled when job completes */
114 	struct dma_fence *fence;
115 	/** @queue: The queue that the job runs on */
116 	struct msm_gpu_submitqueue *queue;
117 	/** @prealloc: Tracking for pre-allocated MMU pgtable pages */
118 	struct msm_mmu_prealloc prealloc;
119 	/** @vm_ops: a list of struct msm_vm_op */
120 	struct list_head vm_ops;
121 	/** @bos_pinned: are the GEM objects being bound pinned? */
122 	bool bos_pinned;
123 	/** @nr_ops: the number of userspace requested ops */
124 	unsigned int nr_ops;
125 	/**
126 	 * @ops: the userspace requested ops
127 	 *
128 	 * The userspace requested ops are copied/parsed and validated
129 	 * before we start applying the updates to try to do as much up-
130 	 * front error checking as possible, to avoid the VM being in an
131 	 * undefined state due to partially executed VM_BIND.
132 	 *
133 	 * This table also serves to hold a reference to the backing GEM
134 	 * objects.
135 	 */
136 	struct msm_vm_bind_op {
137 		uint32_t op;
138 		uint32_t flags;
139 		union {
140 			struct drm_gem_object *obj;
141 			uint32_t handle;
142 		};
143 		uint64_t obj_offset;
144 		uint64_t iova;
145 		uint64_t range;
146 	} ops[];
147 };
148 
149 #define job_foreach_bo(obj, _job) \
150 	for (unsigned i = 0; i < (_job)->nr_ops; i++) \
151 		if ((obj = (_job)->ops[i].obj))
152 
153 static inline struct msm_vm_bind_job *to_msm_vm_bind_job(struct drm_sched_job *job)
154 {
155 	return container_of(job, struct msm_vm_bind_job, base);
156 }
157 
158 static void
159 msm_gem_vm_free(struct drm_gpuvm *gpuvm)
160 {
161 	struct msm_gem_vm *vm = container_of(gpuvm, struct msm_gem_vm, base);
162 
163 	drm_mm_takedown(&vm->mm);
164 	if (vm->mmu)
165 		vm->mmu->funcs->destroy(vm->mmu);
166 	dma_fence_put(vm->last_fence);
167 	put_pid(vm->pid);
168 	kfree(vm->log);
169 	kfree(vm);
170 }
171 
172 /**
173  * msm_gem_vm_unusable() - Mark a VM as unusable
174  * @gpuvm: the VM to mark unusable
175  */
176 void
177 msm_gem_vm_unusable(struct drm_gpuvm *gpuvm)
178 {
179 	struct msm_gem_vm *vm = to_msm_vm(gpuvm);
180 	uint32_t vm_log_len = (1 << vm->log_shift);
181 	uint32_t vm_log_mask = vm_log_len - 1;
182 	uint32_t nr_vm_logs;
183 	int first;
184 
185 	vm->unusable = true;
186 
187 	/* Bail if no log, or empty log: */
188 	if (!vm->log || !vm->log[0].op)
189 		return;
190 
191 	mutex_lock(&vm->mmu_lock);
192 
193 	/*
194 	 * log_idx is the next entry to overwrite, meaning it is the oldest, or
195 	 * first, entry (other than the special case handled below where the
196 	 * log hasn't wrapped around yet)
197 	 */
198 	first = vm->log_idx;
199 
200 	if (!vm->log[first].op) {
201 		/*
202 		 * If the next log entry has not been written yet, then only
203 		 * entries 0 to idx-1 are valid (ie. we haven't wrapped around
204 		 * yet)
205 		 */
206 		nr_vm_logs = MAX(0, first - 1);
207 		first = 0;
208 	} else {
209 		nr_vm_logs = vm_log_len;
210 	}
211 
212 	pr_err("vm-log:\n");
213 	for (int i = 0; i < nr_vm_logs; i++) {
214 		int idx = (i + first) & vm_log_mask;
215 		struct msm_gem_vm_log_entry *e = &vm->log[idx];
216 		pr_err("  - %s:%d: 0x%016llx-0x%016llx\n",
217 		       e->op, e->queue_id, e->iova,
218 		       e->iova + e->range);
219 	}
220 
221 	mutex_unlock(&vm->mmu_lock);
222 }
223 
224 static void
225 vm_log(struct msm_gem_vm *vm, const char *op, uint64_t iova, uint64_t range, int queue_id)
226 {
227 	int idx;
228 
229 	if (!vm->managed)
230 		lockdep_assert_held(&vm->mmu_lock);
231 
232 	vm_dbg("%s:%p:%d: %016llx %016llx", op, vm, queue_id, iova, iova + range);
233 
234 	if (!vm->log)
235 		return;
236 
237 	idx = vm->log_idx;
238 	vm->log[idx].op = op;
239 	vm->log[idx].iova = iova;
240 	vm->log[idx].range = range;
241 	vm->log[idx].queue_id = queue_id;
242 	vm->log_idx = (vm->log_idx + 1) & ((1 << vm->log_shift) - 1);
243 }
244 
245 static void
246 vm_unmap_op(struct msm_gem_vm *vm, const struct msm_vm_unmap_op *op)
247 {
248 	const char *reason = op->reason;
249 
250 	if (!reason)
251 		reason = "unmap";
252 
253 	vm_log(vm, reason, op->iova, op->range, op->queue_id);
254 
255 	vm->mmu->funcs->unmap(vm->mmu, op->iova, op->range);
256 }
257 
258 static int
259 vm_map_op(struct msm_gem_vm *vm, const struct msm_vm_map_op *op)
260 {
261 	vm_log(vm, "map", op->iova, op->range, op->queue_id);
262 
263 	return vm->mmu->funcs->map(vm->mmu, op->iova, op->sgt, op->offset,
264 				   op->range, op->prot);
265 }
266 
267 /* Actually unmap memory for the vma */
268 void msm_gem_vma_unmap(struct drm_gpuva *vma, const char *reason)
269 {
270 	struct msm_gem_vm *vm = to_msm_vm(vma->vm);
271 	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
272 
273 	/* Don't do anything if the memory isn't mapped */
274 	if (!msm_vma->mapped)
275 		return;
276 
277 	/*
278 	 * The mmu_lock is only needed when preallocation is used.  But
279 	 * in that case we don't need to worry about recursion into
280 	 * shrinker
281 	 */
282 	if (!vm->managed)
283 		 mutex_lock(&vm->mmu_lock);
284 
285 	vm_unmap_op(vm, &(struct msm_vm_unmap_op){
286 		.iova = vma->va.addr,
287 		.range = vma->va.range,
288 		.reason = reason,
289 	});
290 
291 	if (!vm->managed)
292 		mutex_unlock(&vm->mmu_lock);
293 
294 	msm_vma->mapped = false;
295 }
296 
297 /* Map and pin vma: */
298 int
299 msm_gem_vma_map(struct drm_gpuva *vma, int prot, struct sg_table *sgt)
300 {
301 	struct msm_gem_vm *vm = to_msm_vm(vma->vm);
302 	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
303 	int ret;
304 
305 	if (GEM_WARN_ON(!vma->va.addr))
306 		return -EINVAL;
307 
308 	if (msm_vma->mapped)
309 		return 0;
310 
311 	msm_vma->mapped = true;
312 
313 	/*
314 	 * The mmu_lock is only needed when preallocation is used.  But
315 	 * in that case we don't need to worry about recursion into
316 	 * shrinker
317 	 */
318 	if (!vm->managed)
319 		mutex_lock(&vm->mmu_lock);
320 
321 	/*
322 	 * NOTE: iommu/io-pgtable can allocate pages, so we cannot hold
323 	 * a lock across map/unmap which is also used in the job_run()
324 	 * path, as this can cause deadlock in job_run() vs shrinker/
325 	 * reclaim.
326 	 *
327 	 * Revisit this if we can come up with a scheme to pre-alloc pages
328 	 * for the pgtable in map/unmap ops.
329 	 */
330 	ret = vm_map_op(vm, &(struct msm_vm_map_op){
331 		.iova = vma->va.addr,
332 		.range = vma->va.range,
333 		.offset = vma->gem.offset,
334 		.sgt = sgt,
335 		.prot = prot,
336 	});
337 
338 	if (!vm->managed)
339 		mutex_unlock(&vm->mmu_lock);
340 
341 	if (ret)
342 		msm_vma->mapped = false;
343 
344 	return ret;
345 }
346 
347 /* Close an iova.  Warn if it is still in use */
348 void msm_gem_vma_close(struct drm_gpuva *vma)
349 {
350 	struct msm_gem_vm *vm = to_msm_vm(vma->vm);
351 	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
352 
353 	GEM_WARN_ON(msm_vma->mapped);
354 
355 	drm_gpuvm_resv_assert_held(&vm->base);
356 
357 	if (vma->gem.obj)
358 		msm_gem_assert_locked(vma->gem.obj);
359 
360 	if (vma->va.addr && vm->managed)
361 		drm_mm_remove_node(&msm_vma->node);
362 
363 	drm_gpuva_remove(vma);
364 	drm_gpuva_unlink(vma);
365 
366 	kfree(vma);
367 }
368 
369 /* Create a new vma and allocate an iova for it */
370 struct drm_gpuva *
371 msm_gem_vma_new(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj,
372 		u64 offset, u64 range_start, u64 range_end)
373 {
374 	struct msm_gem_vm *vm = to_msm_vm(gpuvm);
375 	struct drm_gpuvm_bo *vm_bo;
376 	struct msm_gem_vma *vma;
377 	int ret;
378 
379 	drm_gpuvm_resv_assert_held(&vm->base);
380 
381 	vma = kzalloc(sizeof(*vma), GFP_KERNEL);
382 	if (!vma)
383 		return ERR_PTR(-ENOMEM);
384 
385 	if (vm->managed) {
386 		BUG_ON(offset != 0);
387 		BUG_ON(!obj);  /* NULL mappings not valid for kernel managed VM */
388 		ret = drm_mm_insert_node_in_range(&vm->mm, &vma->node,
389 						obj->size, PAGE_SIZE, 0,
390 						range_start, range_end, 0);
391 
392 		if (ret)
393 			goto err_free_vma;
394 
395 		range_start = vma->node.start;
396 		range_end   = range_start + obj->size;
397 	}
398 
399 	if (obj)
400 		GEM_WARN_ON((range_end - range_start) > obj->size);
401 
402 	struct drm_gpuva_op_map op_map = {
403 		.va.addr = range_start,
404 		.va.range = range_end - range_start,
405 		.gem.obj = obj,
406 		.gem.offset = offset,
407 	};
408 
409 	drm_gpuva_init_from_op(&vma->base, &op_map);
410 	vma->mapped = false;
411 
412 	ret = drm_gpuva_insert(&vm->base, &vma->base);
413 	if (ret)
414 		goto err_free_range;
415 
416 	if (!obj)
417 		return &vma->base;
418 
419 	vm_bo = drm_gpuvm_bo_obtain(&vm->base, obj);
420 	if (IS_ERR(vm_bo)) {
421 		ret = PTR_ERR(vm_bo);
422 		goto err_va_remove;
423 	}
424 
425 	drm_gpuvm_bo_extobj_add(vm_bo);
426 	drm_gpuva_link(&vma->base, vm_bo);
427 	GEM_WARN_ON(drm_gpuvm_bo_put(vm_bo));
428 
429 	return &vma->base;
430 
431 err_va_remove:
432 	drm_gpuva_remove(&vma->base);
433 err_free_range:
434 	if (vm->managed)
435 		drm_mm_remove_node(&vma->node);
436 err_free_vma:
437 	kfree(vma);
438 	return ERR_PTR(ret);
439 }
440 
441 static int
442 msm_gem_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
443 {
444 	struct drm_gem_object *obj = vm_bo->obj;
445 	struct drm_gpuva *vma;
446 	int ret;
447 
448 	vm_dbg("validate: %p", obj);
449 
450 	msm_gem_assert_locked(obj);
451 
452 	drm_gpuvm_bo_for_each_va (vma, vm_bo) {
453 		ret = msm_gem_pin_vma_locked(obj, vma);
454 		if (ret)
455 			return ret;
456 	}
457 
458 	return 0;
459 }
460 
461 struct op_arg {
462 	unsigned flags;
463 	struct msm_vm_bind_job *job;
464 };
465 
466 static void
467 vm_op_enqueue(struct op_arg *arg, struct msm_vm_op _op)
468 {
469 	struct msm_vm_op *op = kmalloc(sizeof(*op), GFP_KERNEL);
470 	*op = _op;
471 	list_add_tail(&op->node, &arg->job->vm_ops);
472 
473 	if (op->obj)
474 		drm_gem_object_get(op->obj);
475 }
476 
477 static struct drm_gpuva *
478 vma_from_op(struct op_arg *arg, struct drm_gpuva_op_map *op)
479 {
480 	return msm_gem_vma_new(arg->job->vm, op->gem.obj, op->gem.offset,
481 			       op->va.addr, op->va.addr + op->va.range);
482 }
483 
484 static int
485 msm_gem_vm_sm_step_map(struct drm_gpuva_op *op, void *arg)
486 {
487 	struct msm_vm_bind_job *job = ((struct op_arg *)arg)->job;
488 	struct drm_gem_object *obj = op->map.gem.obj;
489 	struct drm_gpuva *vma;
490 	struct sg_table *sgt;
491 	unsigned prot;
492 
493 	vma = vma_from_op(arg, &op->map);
494 	if (WARN_ON(IS_ERR(vma)))
495 		return PTR_ERR(vma);
496 
497 	vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
498 	       vma->va.addr, vma->va.range);
499 
500 	vma->flags = ((struct op_arg *)arg)->flags;
501 
502 	if (obj) {
503 		sgt = to_msm_bo(obj)->sgt;
504 		prot = msm_gem_prot(obj);
505 	} else {
506 		sgt = NULL;
507 		prot = IOMMU_READ | IOMMU_WRITE;
508 	}
509 
510 	vm_op_enqueue(arg, (struct msm_vm_op){
511 		.op = MSM_VM_OP_MAP,
512 		.map = {
513 			.sgt = sgt,
514 			.iova = vma->va.addr,
515 			.range = vma->va.range,
516 			.offset = vma->gem.offset,
517 			.prot = prot,
518 			.queue_id = job->queue->id,
519 		},
520 		.obj = vma->gem.obj,
521 	});
522 
523 	to_msm_vma(vma)->mapped = true;
524 
525 	return 0;
526 }
527 
528 static int
529 msm_gem_vm_sm_step_remap(struct drm_gpuva_op *op, void *arg)
530 {
531 	struct msm_vm_bind_job *job = ((struct op_arg *)arg)->job;
532 	struct drm_gpuvm *vm = job->vm;
533 	struct drm_gpuva *orig_vma = op->remap.unmap->va;
534 	struct drm_gpuva *prev_vma = NULL, *next_vma = NULL;
535 	struct drm_gpuvm_bo *vm_bo = orig_vma->vm_bo;
536 	bool mapped = to_msm_vma(orig_vma)->mapped;
537 	unsigned flags;
538 
539 	vm_dbg("orig_vma: %p:%p:%p: %016llx %016llx", vm, orig_vma,
540 	       orig_vma->gem.obj, orig_vma->va.addr, orig_vma->va.range);
541 
542 	if (mapped) {
543 		uint64_t unmap_start, unmap_range;
544 
545 		drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
546 
547 		vm_op_enqueue(arg, (struct msm_vm_op){
548 			.op = MSM_VM_OP_UNMAP,
549 			.unmap = {
550 				.iova = unmap_start,
551 				.range = unmap_range,
552 				.queue_id = job->queue->id,
553 			},
554 			.obj = orig_vma->gem.obj,
555 		});
556 
557 		/*
558 		 * Part of this GEM obj is still mapped, but we're going to kill the
559 		 * existing VMA and replace it with one or two new ones (ie. two if
560 		 * the unmapped range is in the middle of the existing (unmap) VMA).
561 		 * So just set the state to unmapped:
562 		 */
563 		to_msm_vma(orig_vma)->mapped = false;
564 	}
565 
566 	/*
567 	 * Hold a ref to the vm_bo between the msm_gem_vma_close() and the
568 	 * creation of the new prev/next vma's, in case the vm_bo is tracked
569 	 * in the VM's evict list:
570 	 */
571 	if (vm_bo)
572 		drm_gpuvm_bo_get(vm_bo);
573 
574 	/*
575 	 * The prev_vma and/or next_vma are replacing the unmapped vma, and
576 	 * therefore should preserve it's flags:
577 	 */
578 	flags = orig_vma->flags;
579 
580 	msm_gem_vma_close(orig_vma);
581 
582 	if (op->remap.prev) {
583 		prev_vma = vma_from_op(arg, op->remap.prev);
584 		if (WARN_ON(IS_ERR(prev_vma)))
585 			return PTR_ERR(prev_vma);
586 
587 		vm_dbg("prev_vma: %p:%p: %016llx %016llx", vm, prev_vma, prev_vma->va.addr, prev_vma->va.range);
588 		to_msm_vma(prev_vma)->mapped = mapped;
589 		prev_vma->flags = flags;
590 	}
591 
592 	if (op->remap.next) {
593 		next_vma = vma_from_op(arg, op->remap.next);
594 		if (WARN_ON(IS_ERR(next_vma)))
595 			return PTR_ERR(next_vma);
596 
597 		vm_dbg("next_vma: %p:%p: %016llx %016llx", vm, next_vma, next_vma->va.addr, next_vma->va.range);
598 		to_msm_vma(next_vma)->mapped = mapped;
599 		next_vma->flags = flags;
600 	}
601 
602 	if (!mapped)
603 		drm_gpuvm_bo_evict(vm_bo, true);
604 
605 	/* Drop the previous ref: */
606 	drm_gpuvm_bo_put(vm_bo);
607 
608 	return 0;
609 }
610 
611 static int
612 msm_gem_vm_sm_step_unmap(struct drm_gpuva_op *op, void *arg)
613 {
614 	struct msm_vm_bind_job *job = ((struct op_arg *)arg)->job;
615 	struct drm_gpuva *vma = op->unmap.va;
616 	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
617 
618 	vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
619 	       vma->va.addr, vma->va.range);
620 
621 	if (!msm_vma->mapped)
622 		goto out_close;
623 
624 	vm_op_enqueue(arg, (struct msm_vm_op){
625 		.op = MSM_VM_OP_UNMAP,
626 		.unmap = {
627 			.iova = vma->va.addr,
628 			.range = vma->va.range,
629 			.queue_id = job->queue->id,
630 		},
631 		.obj = vma->gem.obj,
632 	});
633 
634 	msm_vma->mapped = false;
635 
636 out_close:
637 	msm_gem_vma_close(vma);
638 
639 	return 0;
640 }
641 
642 static const struct drm_gpuvm_ops msm_gpuvm_ops = {
643 	.vm_free = msm_gem_vm_free,
644 	.vm_bo_validate = msm_gem_vm_bo_validate,
645 	.sm_step_map = msm_gem_vm_sm_step_map,
646 	.sm_step_remap = msm_gem_vm_sm_step_remap,
647 	.sm_step_unmap = msm_gem_vm_sm_step_unmap,
648 };
649 
650 static struct dma_fence *
651 msm_vma_job_run(struct drm_sched_job *_job)
652 {
653 	struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
654 	struct msm_gem_vm *vm = to_msm_vm(job->vm);
655 	struct drm_gem_object *obj;
656 	int ret = vm->unusable ? -EINVAL : 0;
657 
658 	vm_dbg("");
659 
660 	mutex_lock(&vm->mmu_lock);
661 	vm->mmu->prealloc = &job->prealloc;
662 
663 	while (!list_empty(&job->vm_ops)) {
664 		struct msm_vm_op *op =
665 			list_first_entry(&job->vm_ops, struct msm_vm_op, node);
666 
667 		switch (op->op) {
668 		case MSM_VM_OP_MAP:
669 			/*
670 			 * On error, stop trying to map new things.. but we
671 			 * still want to process the unmaps (or in particular,
672 			 * the drm_gem_object_put()s)
673 			 */
674 			if (!ret)
675 				ret = vm_map_op(vm, &op->map);
676 			break;
677 		case MSM_VM_OP_UNMAP:
678 			vm_unmap_op(vm, &op->unmap);
679 			break;
680 		}
681 		drm_gem_object_put(op->obj);
682 		list_del(&op->node);
683 		kfree(op);
684 	}
685 
686 	vm->mmu->prealloc = NULL;
687 	mutex_unlock(&vm->mmu_lock);
688 
689 	/*
690 	 * We failed to perform at least _some_ of the pgtable updates, so
691 	 * now the VM is in an undefined state.  Game over!
692 	 */
693 	if (ret)
694 		msm_gem_vm_unusable(job->vm);
695 
696 	job_foreach_bo (obj, job) {
697 		msm_gem_lock(obj);
698 		msm_gem_unpin_locked(obj);
699 		msm_gem_unlock(obj);
700 	}
701 
702 	/* VM_BIND ops are synchronous, so no fence to wait on: */
703 	return NULL;
704 }
705 
706 static void
707 msm_vma_job_free(struct drm_sched_job *_job)
708 {
709 	struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
710 	struct msm_gem_vm *vm = to_msm_vm(job->vm);
711 	struct drm_gem_object *obj;
712 
713 	vm->mmu->funcs->prealloc_cleanup(vm->mmu, &job->prealloc);
714 
715 	atomic_sub(job->prealloc.count, &vm->prealloc_throttle.in_flight);
716 
717 	drm_sched_job_cleanup(_job);
718 
719 	job_foreach_bo (obj, job)
720 		drm_gem_object_put(obj);
721 
722 	msm_submitqueue_put(job->queue);
723 	dma_fence_put(job->fence);
724 
725 	/* In error paths, we could have unexecuted ops: */
726 	while (!list_empty(&job->vm_ops)) {
727 		struct msm_vm_op *op =
728 			list_first_entry(&job->vm_ops, struct msm_vm_op, node);
729 		list_del(&op->node);
730 		kfree(op);
731 	}
732 
733 	wake_up(&vm->prealloc_throttle.wait);
734 
735 	kfree(job);
736 }
737 
738 static const struct drm_sched_backend_ops msm_vm_bind_ops = {
739 	.run_job = msm_vma_job_run,
740 	.free_job = msm_vma_job_free
741 };
742 
743 /**
744  * msm_gem_vm_create() - Create and initialize a &msm_gem_vm
745  * @drm: the drm device
746  * @mmu: the backing MMU objects handling mapping/unmapping
747  * @name: the name of the VM
748  * @va_start: the start offset of the VA space
749  * @va_size: the size of the VA space
750  * @managed: is it a kernel managed VM?
751  *
752  * In a kernel managed VM, the kernel handles address allocation, and only
753  * synchronous operations are supported.  In a user managed VM, userspace
754  * handles virtual address allocation, and both async and sync operations
755  * are supported.
756  */
757 struct drm_gpuvm *
758 msm_gem_vm_create(struct drm_device *drm, struct msm_mmu *mmu, const char *name,
759 		  u64 va_start, u64 va_size, bool managed)
760 {
761 	/*
762 	 * We mostly want to use DRM_GPUVM_RESV_PROTECTED, except that
763 	 * makes drm_gpuvm_bo_evict() a no-op for extobjs (ie. we loose
764 	 * tracking that an extobj is evicted) :facepalm:
765 	 */
766 	enum drm_gpuvm_flags flags = 0;
767 	struct msm_gem_vm *vm;
768 	struct drm_gem_object *dummy_gem;
769 	int ret = 0;
770 
771 	if (IS_ERR(mmu))
772 		return ERR_CAST(mmu);
773 
774 	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
775 	if (!vm)
776 		return ERR_PTR(-ENOMEM);
777 
778 	dummy_gem = drm_gpuvm_resv_object_alloc(drm);
779 	if (!dummy_gem) {
780 		ret = -ENOMEM;
781 		goto err_free_vm;
782 	}
783 
784 	if (!managed) {
785 		struct drm_sched_init_args args = {
786 			.ops = &msm_vm_bind_ops,
787 			.num_rqs = 1,
788 			.credit_limit = 1,
789 			.timeout = MAX_SCHEDULE_TIMEOUT,
790 			.name = "msm-vm-bind",
791 			.dev = drm->dev,
792 		};
793 
794 		ret = drm_sched_init(&vm->sched, &args);
795 		if (ret)
796 			goto err_free_dummy;
797 
798 		init_waitqueue_head(&vm->prealloc_throttle.wait);
799 	}
800 
801 	drm_gpuvm_init(&vm->base, name, flags, drm, dummy_gem,
802 		       va_start, va_size, 0, 0, &msm_gpuvm_ops);
803 	drm_gem_object_put(dummy_gem);
804 
805 	vm->mmu = mmu;
806 	mutex_init(&vm->mmu_lock);
807 	vm->managed = managed;
808 
809 	drm_mm_init(&vm->mm, va_start, va_size);
810 
811 	/*
812 	 * We don't really need vm log for kernel managed VMs, as the kernel
813 	 * is responsible for ensuring that GEM objs are mapped if they are
814 	 * used by a submit.  Furthermore we piggyback on mmu_lock to serialize
815 	 * access to the log.
816 	 *
817 	 * Limit the max log_shift to 8 to prevent userspace from asking us
818 	 * for an unreasonable log size.
819 	 */
820 	if (!managed)
821 		vm->log_shift = MIN(vm_log_shift, 8);
822 
823 	if (vm->log_shift) {
824 		vm->log = kmalloc_array(1 << vm->log_shift, sizeof(vm->log[0]),
825 					GFP_KERNEL | __GFP_ZERO);
826 	}
827 
828 	return &vm->base;
829 
830 err_free_dummy:
831 	drm_gem_object_put(dummy_gem);
832 
833 err_free_vm:
834 	kfree(vm);
835 	return ERR_PTR(ret);
836 }
837 
838 /**
839  * msm_gem_vm_close() - Close a VM
840  * @gpuvm: The VM to close
841  *
842  * Called when the drm device file is closed, to tear down VM related resources
843  * (which will drop refcounts to GEM objects that were still mapped into the
844  * VM at the time).
845  */
846 void
847 msm_gem_vm_close(struct drm_gpuvm *gpuvm)
848 {
849 	struct msm_gem_vm *vm = to_msm_vm(gpuvm);
850 	struct drm_gpuva *vma, *tmp;
851 	struct drm_exec exec;
852 
853 	/*
854 	 * For kernel managed VMs, the VMAs are torn down when the handle is
855 	 * closed, so nothing more to do.
856 	 */
857 	if (vm->managed)
858 		return;
859 
860 	if (vm->last_fence)
861 		dma_fence_wait(vm->last_fence, false);
862 
863 	/* Kill the scheduler now, so we aren't racing with it for cleanup: */
864 	drm_sched_stop(&vm->sched, NULL);
865 	drm_sched_fini(&vm->sched);
866 
867 	/* Tear down any remaining mappings: */
868 	drm_exec_init(&exec, 0, 2);
869 	drm_exec_until_all_locked (&exec) {
870 		drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(gpuvm));
871 		drm_exec_retry_on_contention(&exec);
872 
873 		drm_gpuvm_for_each_va_safe (vma, tmp, gpuvm) {
874 			struct drm_gem_object *obj = vma->gem.obj;
875 
876 			/*
877 			 * MSM_BO_NO_SHARE objects share the same resv as the
878 			 * VM, in which case the obj is already locked:
879 			 */
880 			if (obj && (obj->resv == drm_gpuvm_resv(gpuvm)))
881 				obj = NULL;
882 
883 			if (obj) {
884 				drm_exec_lock_obj(&exec, obj);
885 				drm_exec_retry_on_contention(&exec);
886 			}
887 
888 			msm_gem_vma_unmap(vma, "close");
889 			msm_gem_vma_close(vma);
890 
891 			if (obj) {
892 				drm_exec_unlock_obj(&exec, obj);
893 			}
894 		}
895 	}
896 	drm_exec_fini(&exec);
897 }
898 
899 
900 static struct msm_vm_bind_job *
901 vm_bind_job_create(struct drm_device *dev, struct drm_file *file,
902 		   struct msm_gpu_submitqueue *queue, uint32_t nr_ops)
903 {
904 	struct msm_vm_bind_job *job;
905 	uint64_t sz;
906 	int ret;
907 
908 	sz = struct_size(job, ops, nr_ops);
909 
910 	if (sz > SIZE_MAX)
911 		return ERR_PTR(-ENOMEM);
912 
913 	job = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN);
914 	if (!job)
915 		return ERR_PTR(-ENOMEM);
916 
917 	ret = drm_sched_job_init(&job->base, queue->entity, 1, queue,
918 				 file->client_id);
919 	if (ret) {
920 		kfree(job);
921 		return ERR_PTR(ret);
922 	}
923 
924 	job->vm = msm_context_vm(dev, queue->ctx);
925 	job->queue = queue;
926 	INIT_LIST_HEAD(&job->vm_ops);
927 
928 	return job;
929 }
930 
931 static bool invalid_alignment(uint64_t addr)
932 {
933 	/*
934 	 * Technically this is about GPU alignment, not CPU alignment.  But
935 	 * I've not seen any qcom SoC where the SMMU does not support the
936 	 * CPU's smallest page size.
937 	 */
938 	return !PAGE_ALIGNED(addr);
939 }
940 
941 static int
942 lookup_op(struct msm_vm_bind_job *job, const struct drm_msm_vm_bind_op *op)
943 {
944 	struct drm_device *dev = job->vm->drm;
945 	int i = job->nr_ops++;
946 	int ret = 0;
947 
948 	job->ops[i].op = op->op;
949 	job->ops[i].handle = op->handle;
950 	job->ops[i].obj_offset = op->obj_offset;
951 	job->ops[i].iova = op->iova;
952 	job->ops[i].range = op->range;
953 	job->ops[i].flags = op->flags;
954 
955 	if (op->flags & ~MSM_VM_BIND_OP_FLAGS)
956 		ret = UERR(EINVAL, dev, "invalid flags: %x\n", op->flags);
957 
958 	if (invalid_alignment(op->iova))
959 		ret = UERR(EINVAL, dev, "invalid address: %016llx\n", op->iova);
960 
961 	if (invalid_alignment(op->obj_offset))
962 		ret = UERR(EINVAL, dev, "invalid bo_offset: %016llx\n", op->obj_offset);
963 
964 	if (invalid_alignment(op->range))
965 		ret = UERR(EINVAL, dev, "invalid range: %016llx\n", op->range);
966 
967 	if (!drm_gpuvm_range_valid(job->vm, op->iova, op->range))
968 		ret = UERR(EINVAL, dev, "invalid range: %016llx, %016llx\n", op->iova, op->range);
969 
970 	/*
971 	 * MAP must specify a valid handle.  But the handle MBZ for
972 	 * UNMAP or MAP_NULL.
973 	 */
974 	if (op->op == MSM_VM_BIND_OP_MAP) {
975 		if (!op->handle)
976 			ret = UERR(EINVAL, dev, "invalid handle\n");
977 	} else if (op->handle) {
978 		ret = UERR(EINVAL, dev, "handle must be zero\n");
979 	}
980 
981 	switch (op->op) {
982 	case MSM_VM_BIND_OP_MAP:
983 	case MSM_VM_BIND_OP_MAP_NULL:
984 	case MSM_VM_BIND_OP_UNMAP:
985 		break;
986 	default:
987 		ret = UERR(EINVAL, dev, "invalid op: %u\n", op->op);
988 		break;
989 	}
990 
991 	return ret;
992 }
993 
994 /*
995  * ioctl parsing, parameter validation, and GEM handle lookup
996  */
997 static int
998 vm_bind_job_lookup_ops(struct msm_vm_bind_job *job, struct drm_msm_vm_bind *args,
999 		       struct drm_file *file, int *nr_bos)
1000 {
1001 	struct drm_device *dev = job->vm->drm;
1002 	int ret = 0;
1003 	int cnt = 0;
1004 
1005 	if (args->nr_ops == 1) {
1006 		/* Single op case, the op is inlined: */
1007 		ret = lookup_op(job, &args->op);
1008 	} else {
1009 		for (unsigned i = 0; i < args->nr_ops; i++) {
1010 			struct drm_msm_vm_bind_op op;
1011 			void __user *userptr =
1012 				u64_to_user_ptr(args->ops + (i * sizeof(op)));
1013 
1014 			/* make sure we don't have garbage flags, in case we hit
1015 			 * error path before flags is initialized:
1016 			 */
1017 			job->ops[i].flags = 0;
1018 
1019 			if (copy_from_user(&op, userptr, sizeof(op))) {
1020 				ret = -EFAULT;
1021 				break;
1022 			}
1023 
1024 			ret = lookup_op(job, &op);
1025 			if (ret)
1026 				break;
1027 		}
1028 	}
1029 
1030 	if (ret) {
1031 		job->nr_ops = 0;
1032 		goto out;
1033 	}
1034 
1035 	spin_lock(&file->table_lock);
1036 
1037 	for (unsigned i = 0; i < args->nr_ops; i++) {
1038 		struct drm_gem_object *obj;
1039 
1040 		if (!job->ops[i].handle) {
1041 			job->ops[i].obj = NULL;
1042 			continue;
1043 		}
1044 
1045 		/*
1046 		 * normally use drm_gem_object_lookup(), but for bulk lookup
1047 		 * all under single table_lock just hit object_idr directly:
1048 		 */
1049 		obj = idr_find(&file->object_idr, job->ops[i].handle);
1050 		if (!obj) {
1051 			ret = UERR(EINVAL, dev, "invalid handle %u at index %u\n", job->ops[i].handle, i);
1052 			goto out_unlock;
1053 		}
1054 
1055 		drm_gem_object_get(obj);
1056 
1057 		job->ops[i].obj = obj;
1058 		cnt++;
1059 	}
1060 
1061 	*nr_bos = cnt;
1062 
1063 out_unlock:
1064 	spin_unlock(&file->table_lock);
1065 
1066 out:
1067 	return ret;
1068 }
1069 
1070 static void
1071 prealloc_count(struct msm_vm_bind_job *job,
1072 	       struct msm_vm_bind_op *first,
1073 	       struct msm_vm_bind_op *last)
1074 {
1075 	struct msm_mmu *mmu = to_msm_vm(job->vm)->mmu;
1076 
1077 	if (!first)
1078 		return;
1079 
1080 	uint64_t start_iova = first->iova;
1081 	uint64_t end_iova = last->iova + last->range;
1082 
1083 	mmu->funcs->prealloc_count(mmu, &job->prealloc, start_iova, end_iova - start_iova);
1084 }
1085 
1086 static bool
1087 ops_are_same_pte(struct msm_vm_bind_op *first, struct msm_vm_bind_op *next)
1088 {
1089 	/*
1090 	 * Last level pte covers 2MB.. so we should merge two ops, from
1091 	 * the PoV of figuring out how much pgtable pages to pre-allocate
1092 	 * if they land in the same 2MB range:
1093 	 */
1094 	uint64_t pte_mask = ~(SZ_2M - 1);
1095 	return ((first->iova + first->range) & pte_mask) == (next->iova & pte_mask);
1096 }
1097 
1098 /*
1099  * Determine the amount of memory to prealloc for pgtables.  For sparse images,
1100  * in particular, userspace plays some tricks with the order of page mappings
1101  * to get the desired swizzle pattern, resulting in a large # of tiny MAP ops.
1102  * So detect when multiple MAP operations are physically contiguous, and count
1103  * them as a single mapping.  Otherwise the prealloc_count() will not realize
1104  * they can share pagetable pages and vastly overcount.
1105  */
1106 static int
1107 vm_bind_prealloc_count(struct msm_vm_bind_job *job)
1108 {
1109 	struct msm_vm_bind_op *first = NULL, *last = NULL;
1110 	struct msm_gem_vm *vm = to_msm_vm(job->vm);
1111 	int ret;
1112 
1113 	for (int i = 0; i < job->nr_ops; i++) {
1114 		struct msm_vm_bind_op *op = &job->ops[i];
1115 
1116 		/* We only care about MAP/MAP_NULL: */
1117 		if (op->op == MSM_VM_BIND_OP_UNMAP)
1118 			continue;
1119 
1120 		/*
1121 		 * If op is contiguous with last in the current range, then
1122 		 * it becomes the new last in the range and we continue
1123 		 * looping:
1124 		 */
1125 		if (last && ops_are_same_pte(last, op)) {
1126 			last = op;
1127 			continue;
1128 		}
1129 
1130 		/*
1131 		 * If op is not contiguous with the current range, flush
1132 		 * the current range and start anew:
1133 		 */
1134 		prealloc_count(job, first, last);
1135 		first = last = op;
1136 	}
1137 
1138 	/* Flush the remaining range: */
1139 	prealloc_count(job, first, last);
1140 
1141 	/*
1142 	 * Now that we know the needed amount to pre-alloc, throttle on pending
1143 	 * VM_BIND jobs if we already have too much pre-alloc memory in flight
1144 	 */
1145 	ret = wait_event_interruptible(
1146 			vm->prealloc_throttle.wait,
1147 			atomic_read(&vm->prealloc_throttle.in_flight) <= 1024);
1148 	if (ret)
1149 		return ret;
1150 
1151 	atomic_add(job->prealloc.count, &vm->prealloc_throttle.in_flight);
1152 
1153 	return 0;
1154 }
1155 
1156 /*
1157  * Lock VM and GEM objects
1158  */
1159 static int
1160 vm_bind_job_lock_objects(struct msm_vm_bind_job *job, struct drm_exec *exec)
1161 {
1162 	int ret;
1163 
1164 	/* Lock VM and objects: */
1165 	drm_exec_until_all_locked (exec) {
1166 		ret = drm_exec_lock_obj(exec, drm_gpuvm_resv_obj(job->vm));
1167 		drm_exec_retry_on_contention(exec);
1168 		if (ret)
1169 			return ret;
1170 
1171 		for (unsigned i = 0; i < job->nr_ops; i++) {
1172 			const struct msm_vm_bind_op *op = &job->ops[i];
1173 
1174 			switch (op->op) {
1175 			case MSM_VM_BIND_OP_UNMAP:
1176 				ret = drm_gpuvm_sm_unmap_exec_lock(job->vm, exec,
1177 							      op->iova,
1178 							      op->obj_offset);
1179 				break;
1180 			case MSM_VM_BIND_OP_MAP:
1181 			case MSM_VM_BIND_OP_MAP_NULL: {
1182 				struct drm_gpuvm_map_req map_req = {
1183 					.map.va.addr = op->iova,
1184 					.map.va.range = op->range,
1185 					.map.gem.obj = op->obj,
1186 					.map.gem.offset = op->obj_offset,
1187 				};
1188 
1189 				ret = drm_gpuvm_sm_map_exec_lock(job->vm, exec, 1, &map_req);
1190 				break;
1191 			}
1192 			default:
1193 				/*
1194 				 * lookup_op() should have already thrown an error for
1195 				 * invalid ops
1196 				 */
1197 				WARN_ON("unreachable");
1198 			}
1199 
1200 			drm_exec_retry_on_contention(exec);
1201 			if (ret)
1202 				return ret;
1203 		}
1204 	}
1205 
1206 	return 0;
1207 }
1208 
1209 /*
1210  * Pin GEM objects, ensuring that we have backing pages.  Pinning will move
1211  * the object to the pinned LRU so that the shrinker knows to first consider
1212  * other objects for evicting.
1213  */
1214 static int
1215 vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
1216 {
1217 	struct drm_gem_object *obj;
1218 
1219 	/*
1220 	 * First loop, before holding the LRU lock, avoids holding the
1221 	 * LRU lock while calling msm_gem_pin_vma_locked (which could
1222 	 * trigger get_pages())
1223 	 */
1224 	job_foreach_bo (obj, job) {
1225 		struct page **pages;
1226 
1227 		pages = msm_gem_get_pages_locked(obj, MSM_MADV_WILLNEED);
1228 		if (IS_ERR(pages))
1229 			return PTR_ERR(pages);
1230 	}
1231 
1232 	struct msm_drm_private *priv = job->vm->drm->dev_private;
1233 
1234 	/*
1235 	 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
1236 	 * the LRU lock for each individual bo, while (b) avoiding holding the
1237 	 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
1238 	 * get_pages() which could trigger reclaim.. and if we held the LRU lock
1239 	 * could trigger deadlock with the shrinker).
1240 	 */
1241 	mutex_lock(&priv->lru.lock);
1242 	job_foreach_bo (obj, job)
1243 		msm_gem_pin_obj_locked(obj);
1244 	mutex_unlock(&priv->lru.lock);
1245 
1246 	job->bos_pinned = true;
1247 
1248 	return 0;
1249 }
1250 
1251 /*
1252  * Unpin GEM objects.  Normally this is done after the bind job is run.
1253  */
1254 static void
1255 vm_bind_job_unpin_objects(struct msm_vm_bind_job *job)
1256 {
1257 	struct drm_gem_object *obj;
1258 
1259 	if (!job->bos_pinned)
1260 		return;
1261 
1262 	job_foreach_bo (obj, job)
1263 		msm_gem_unpin_locked(obj);
1264 
1265 	job->bos_pinned = false;
1266 }
1267 
1268 /*
1269  * Pre-allocate pgtable memory, and translate the VM bind requests into a
1270  * sequence of pgtable updates to be applied asynchronously.
1271  */
1272 static int
1273 vm_bind_job_prepare(struct msm_vm_bind_job *job)
1274 {
1275 	struct msm_gem_vm *vm = to_msm_vm(job->vm);
1276 	struct msm_mmu *mmu = vm->mmu;
1277 	int ret;
1278 
1279 	ret = mmu->funcs->prealloc_allocate(mmu, &job->prealloc);
1280 	if (ret)
1281 		return ret;
1282 
1283 	for (unsigned i = 0; i < job->nr_ops; i++) {
1284 		const struct msm_vm_bind_op *op = &job->ops[i];
1285 		struct op_arg arg = {
1286 			.job = job,
1287 		};
1288 
1289 		switch (op->op) {
1290 		case MSM_VM_BIND_OP_UNMAP:
1291 			ret = drm_gpuvm_sm_unmap(job->vm, &arg, op->iova,
1292 						 op->range);
1293 			break;
1294 		case MSM_VM_BIND_OP_MAP:
1295 			if (op->flags & MSM_VM_BIND_OP_DUMP)
1296 				arg.flags |= MSM_VMA_DUMP;
1297 			fallthrough;
1298 		case MSM_VM_BIND_OP_MAP_NULL: {
1299 			struct drm_gpuvm_map_req map_req = {
1300 				.map.va.addr = op->iova,
1301 				.map.va.range = op->range,
1302 				.map.gem.obj = op->obj,
1303 				.map.gem.offset = op->obj_offset,
1304 			};
1305 
1306 			ret = drm_gpuvm_sm_map(job->vm, &arg, &map_req);
1307 			break;
1308 		}
1309 		default:
1310 			/*
1311 			 * lookup_op() should have already thrown an error for
1312 			 * invalid ops
1313 			 */
1314 			BUG_ON("unreachable");
1315 		}
1316 
1317 		if (ret) {
1318 			/*
1319 			 * If we've already started modifying the vm, we can't
1320 			 * adequetly describe to userspace the intermediate
1321 			 * state the vm is in.  So throw up our hands!
1322 			 */
1323 			if (i > 0)
1324 				msm_gem_vm_unusable(job->vm);
1325 			return ret;
1326 		}
1327 	}
1328 
1329 	return 0;
1330 }
1331 
1332 /*
1333  * Attach fences to the GEM objects being bound.  This will signify to
1334  * the shrinker that they are busy even after dropping the locks (ie.
1335  * drm_exec_fini())
1336  */
1337 static void
1338 vm_bind_job_attach_fences(struct msm_vm_bind_job *job)
1339 {
1340 	for (unsigned i = 0; i < job->nr_ops; i++) {
1341 		struct drm_gem_object *obj = job->ops[i].obj;
1342 
1343 		if (!obj)
1344 			continue;
1345 
1346 		dma_resv_add_fence(obj->resv, job->fence,
1347 				   DMA_RESV_USAGE_KERNEL);
1348 	}
1349 }
1350 
1351 int
1352 msm_ioctl_vm_bind(struct drm_device *dev, void *data, struct drm_file *file)
1353 {
1354 	struct msm_drm_private *priv = dev->dev_private;
1355 	struct drm_msm_vm_bind *args = data;
1356 	struct msm_context *ctx = file->driver_priv;
1357 	struct msm_vm_bind_job *job = NULL;
1358 	struct msm_gpu *gpu = priv->gpu;
1359 	struct msm_gpu_submitqueue *queue;
1360 	struct msm_syncobj_post_dep *post_deps = NULL;
1361 	struct drm_syncobj **syncobjs_to_reset = NULL;
1362 	struct sync_file *sync_file = NULL;
1363 	struct dma_fence *fence;
1364 	int out_fence_fd = -1;
1365 	int ret, nr_bos = 0;
1366 	unsigned i;
1367 
1368 	if (!gpu)
1369 		return -ENXIO;
1370 
1371 	/*
1372 	 * Maybe we could allow just UNMAP ops?  OTOH userspace should just
1373 	 * immediately close the device file and all will be torn down.
1374 	 */
1375 	if (to_msm_vm(ctx->vm)->unusable)
1376 		return UERR(EPIPE, dev, "context is unusable");
1377 
1378 	/*
1379 	 * Technically, you cannot create a VM_BIND submitqueue in the first
1380 	 * place, if you haven't opted in to VM_BIND context.  But it is
1381 	 * cleaner / less confusing, to check this case directly.
1382 	 */
1383 	if (!msm_context_is_vmbind(ctx))
1384 		return UERR(EINVAL, dev, "context does not support vmbind");
1385 
1386 	if (args->flags & ~MSM_VM_BIND_FLAGS)
1387 		return UERR(EINVAL, dev, "invalid flags");
1388 
1389 	queue = msm_submitqueue_get(ctx, args->queue_id);
1390 	if (!queue)
1391 		return -ENOENT;
1392 
1393 	if (!(queue->flags & MSM_SUBMITQUEUE_VM_BIND)) {
1394 		ret = UERR(EINVAL, dev, "Invalid queue type");
1395 		goto out_post_unlock;
1396 	}
1397 
1398 	if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
1399 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
1400 		if (out_fence_fd < 0) {
1401 			ret = out_fence_fd;
1402 			goto out_post_unlock;
1403 		}
1404 	}
1405 
1406 	job = vm_bind_job_create(dev, file, queue, args->nr_ops);
1407 	if (IS_ERR(job)) {
1408 		ret = PTR_ERR(job);
1409 		goto out_post_unlock;
1410 	}
1411 
1412 	ret = mutex_lock_interruptible(&queue->lock);
1413 	if (ret)
1414 		goto out_post_unlock;
1415 
1416 	if (args->flags & MSM_VM_BIND_FENCE_FD_IN) {
1417 		struct dma_fence *in_fence;
1418 
1419 		in_fence = sync_file_get_fence(args->fence_fd);
1420 
1421 		if (!in_fence) {
1422 			ret = UERR(EINVAL, dev, "invalid in-fence");
1423 			goto out_unlock;
1424 		}
1425 
1426 		ret = drm_sched_job_add_dependency(&job->base, in_fence);
1427 		if (ret)
1428 			goto out_unlock;
1429 	}
1430 
1431 	if (args->in_syncobjs > 0) {
1432 		syncobjs_to_reset = msm_syncobj_parse_deps(dev, &job->base,
1433 							   file, args->in_syncobjs,
1434 							   args->nr_in_syncobjs,
1435 							   args->syncobj_stride);
1436 		if (IS_ERR(syncobjs_to_reset)) {
1437 			ret = PTR_ERR(syncobjs_to_reset);
1438 			goto out_unlock;
1439 		}
1440 	}
1441 
1442 	if (args->out_syncobjs > 0) {
1443 		post_deps = msm_syncobj_parse_post_deps(dev, file,
1444 							args->out_syncobjs,
1445 							args->nr_out_syncobjs,
1446 							args->syncobj_stride);
1447 		if (IS_ERR(post_deps)) {
1448 			ret = PTR_ERR(post_deps);
1449 			goto out_unlock;
1450 		}
1451 	}
1452 
1453 	ret = vm_bind_job_lookup_ops(job, args, file, &nr_bos);
1454 	if (ret)
1455 		goto out_unlock;
1456 
1457 	ret = vm_bind_prealloc_count(job);
1458 	if (ret)
1459 		goto out_unlock;
1460 
1461 	struct drm_exec exec;
1462 	unsigned flags = DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT;
1463 	drm_exec_init(&exec, flags, nr_bos + 1);
1464 
1465 	ret = vm_bind_job_lock_objects(job, &exec);
1466 	if (ret)
1467 		goto out;
1468 
1469 	ret = vm_bind_job_pin_objects(job);
1470 	if (ret)
1471 		goto out;
1472 
1473 	ret = vm_bind_job_prepare(job);
1474 	if (ret)
1475 		goto out;
1476 
1477 	drm_sched_job_arm(&job->base);
1478 
1479 	job->fence = dma_fence_get(&job->base.s_fence->finished);
1480 
1481 	if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
1482 		sync_file = sync_file_create(job->fence);
1483 		if (!sync_file) {
1484 			ret = -ENOMEM;
1485 		} else {
1486 			fd_install(out_fence_fd, sync_file->file);
1487 			args->fence_fd = out_fence_fd;
1488 		}
1489 	}
1490 
1491 	if (ret)
1492 		goto out;
1493 
1494 	vm_bind_job_attach_fences(job);
1495 
1496 	/*
1497 	 * The job can be free'd (and fence unref'd) at any point after
1498 	 * drm_sched_entity_push_job(), so we need to hold our own ref
1499 	 */
1500 	fence = dma_fence_get(job->fence);
1501 
1502 	drm_sched_entity_push_job(&job->base);
1503 
1504 	msm_syncobj_reset(syncobjs_to_reset, args->nr_in_syncobjs);
1505 	msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, fence);
1506 
1507 	dma_fence_put(fence);
1508 
1509 out:
1510 	if (ret)
1511 		vm_bind_job_unpin_objects(job);
1512 
1513 	drm_exec_fini(&exec);
1514 out_unlock:
1515 	mutex_unlock(&queue->lock);
1516 out_post_unlock:
1517 	if (ret && (out_fence_fd >= 0)) {
1518 		put_unused_fd(out_fence_fd);
1519 		if (sync_file)
1520 			fput(sync_file->file);
1521 	}
1522 
1523 	if (!IS_ERR_OR_NULL(job)) {
1524 		if (ret)
1525 			msm_vma_job_free(&job->base);
1526 	} else {
1527 		/*
1528 		 * If the submit hasn't yet taken ownership of the queue
1529 		 * then we need to drop the reference ourself:
1530 		 */
1531 		msm_submitqueue_put(queue);
1532 	}
1533 
1534 	if (!IS_ERR_OR_NULL(post_deps)) {
1535 		for (i = 0; i < args->nr_out_syncobjs; ++i) {
1536 			kfree(post_deps[i].chain);
1537 			drm_syncobj_put(post_deps[i].syncobj);
1538 		}
1539 		kfree(post_deps);
1540 	}
1541 
1542 	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
1543 		for (i = 0; i < args->nr_in_syncobjs; ++i) {
1544 			if (syncobjs_to_reset[i])
1545 				drm_syncobj_put(syncobjs_to_reset[i]);
1546 		}
1547 		kfree(syncobjs_to_reset);
1548 	}
1549 
1550 	return ret;
1551 }
1552