xref: /linux/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c (revision d91517839e5d95adc0cf4b28caa7af62a71de526)
1 /**************************************************************************
2  *
3  * Copyright © 2011 VMware, Inc., Palo Alto, CA., USA
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 #include <drm/drmP.h>
29 #include "vmwgfx_drv.h"
30 
31 #define VMW_FENCE_WRAP (1 << 31)
32 
33 struct vmw_fence_manager {
34 	int num_fence_objects;
35 	struct vmw_private *dev_priv;
36 	spinlock_t lock;
37 	struct list_head fence_list;
38 	struct work_struct work;
39 	u32 user_fence_size;
40 	u32 fence_size;
41 	u32 event_fence_action_size;
42 	bool fifo_down;
43 	struct list_head cleanup_list;
44 	uint32_t pending_actions[VMW_ACTION_MAX];
45 	struct mutex goal_irq_mutex;
46 	bool goal_irq_on; /* Protected by @goal_irq_mutex */
47 	bool seqno_valid; /* Protected by @lock, and may not be set to true
48 			     without the @goal_irq_mutex held. */
49 };
50 
51 struct vmw_user_fence {
52 	struct ttm_base_object base;
53 	struct vmw_fence_obj fence;
54 };
55 
56 /**
57  * struct vmw_event_fence_action - fence action that delivers a drm event.
58  *
59  * @e: A struct drm_pending_event that controls the event delivery.
60  * @action: A struct vmw_fence_action to hook up to a fence.
61  * @fence: A referenced pointer to the fence to keep it alive while @action
62  * hangs on it.
63  * @dev: Pointer to a struct drm_device so we can access the event stuff.
64  * @kref: Both @e and @action has destructors, so we need to refcount.
65  * @size: Size accounted for this object.
66  * @tv_sec: If non-null, the variable pointed to will be assigned
67  * current time tv_sec val when the fence signals.
68  * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
69  * be assigned the current time tv_usec val when the fence signals.
70  */
71 struct vmw_event_fence_action {
72 	struct vmw_fence_action action;
73 	struct list_head fpriv_head;
74 
75 	struct drm_pending_event *event;
76 	struct vmw_fence_obj *fence;
77 	struct drm_device *dev;
78 
79 	uint32_t *tv_sec;
80 	uint32_t *tv_usec;
81 };
82 
83 /**
84  * Note on fencing subsystem usage of irqs:
85  * Typically the vmw_fences_update function is called
86  *
87  * a) When a new fence seqno has been submitted by the fifo code.
88  * b) On-demand when we have waiters. Sleeping waiters will switch on the
89  * ANY_FENCE irq and call vmw_fences_update function each time an ANY_FENCE
90  * irq is received. When the last fence waiter is gone, that IRQ is masked
91  * away.
92  *
93  * In situations where there are no waiters and we don't submit any new fences,
94  * fence objects may not be signaled. This is perfectly OK, since there are
95  * no consumers of the signaled data, but that is NOT ok when there are fence
96  * actions attached to a fence. The fencing subsystem then makes use of the
97  * FENCE_GOAL irq and sets the fence goal seqno to that of the next fence
98  * which has an action attached, and each time vmw_fences_update is called,
99  * the subsystem makes sure the fence goal seqno is updated.
100  *
101  * The fence goal seqno irq is on as long as there are unsignaled fence
102  * objects with actions attached to them.
103  */
104 
105 static void vmw_fence_obj_destroy_locked(struct kref *kref)
106 {
107 	struct vmw_fence_obj *fence =
108 		container_of(kref, struct vmw_fence_obj, kref);
109 
110 	struct vmw_fence_manager *fman = fence->fman;
111 	unsigned int num_fences;
112 
113 	list_del_init(&fence->head);
114 	num_fences = --fman->num_fence_objects;
115 	spin_unlock_irq(&fman->lock);
116 	if (fence->destroy)
117 		fence->destroy(fence);
118 	else
119 		kfree(fence);
120 
121 	spin_lock_irq(&fman->lock);
122 }
123 
124 
125 /**
126  * Execute signal actions on fences recently signaled.
127  * This is done from a workqueue so we don't have to execute
128  * signal actions from atomic context.
129  */
130 
131 static void vmw_fence_work_func(struct work_struct *work)
132 {
133 	struct vmw_fence_manager *fman =
134 		container_of(work, struct vmw_fence_manager, work);
135 	struct list_head list;
136 	struct vmw_fence_action *action, *next_action;
137 	bool seqno_valid;
138 
139 	do {
140 		INIT_LIST_HEAD(&list);
141 		mutex_lock(&fman->goal_irq_mutex);
142 
143 		spin_lock_irq(&fman->lock);
144 		list_splice_init(&fman->cleanup_list, &list);
145 		seqno_valid = fman->seqno_valid;
146 		spin_unlock_irq(&fman->lock);
147 
148 		if (!seqno_valid && fman->goal_irq_on) {
149 			fman->goal_irq_on = false;
150 			vmw_goal_waiter_remove(fman->dev_priv);
151 		}
152 		mutex_unlock(&fman->goal_irq_mutex);
153 
154 		if (list_empty(&list))
155 			return;
156 
157 		/*
158 		 * At this point, only we should be able to manipulate the
159 		 * list heads of the actions we have on the private list.
160 		 * hence fman::lock not held.
161 		 */
162 
163 		list_for_each_entry_safe(action, next_action, &list, head) {
164 			list_del_init(&action->head);
165 			if (action->cleanup)
166 				action->cleanup(action);
167 		}
168 	} while (1);
169 }
170 
171 struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
172 {
173 	struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);
174 
175 	if (unlikely(fman == NULL))
176 		return NULL;
177 
178 	fman->dev_priv = dev_priv;
179 	spin_lock_init(&fman->lock);
180 	INIT_LIST_HEAD(&fman->fence_list);
181 	INIT_LIST_HEAD(&fman->cleanup_list);
182 	INIT_WORK(&fman->work, &vmw_fence_work_func);
183 	fman->fifo_down = true;
184 	fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence));
185 	fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj));
186 	fman->event_fence_action_size =
187 		ttm_round_pot(sizeof(struct vmw_event_fence_action));
188 	mutex_init(&fman->goal_irq_mutex);
189 
190 	return fman;
191 }
192 
193 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)
194 {
195 	unsigned long irq_flags;
196 	bool lists_empty;
197 
198 	(void) cancel_work_sync(&fman->work);
199 
200 	spin_lock_irqsave(&fman->lock, irq_flags);
201 	lists_empty = list_empty(&fman->fence_list) &&
202 		list_empty(&fman->cleanup_list);
203 	spin_unlock_irqrestore(&fman->lock, irq_flags);
204 
205 	BUG_ON(!lists_empty);
206 	kfree(fman);
207 }
208 
209 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
210 			      struct vmw_fence_obj *fence,
211 			      u32 seqno,
212 			      uint32_t mask,
213 			      void (*destroy) (struct vmw_fence_obj *fence))
214 {
215 	unsigned long irq_flags;
216 	unsigned int num_fences;
217 	int ret = 0;
218 
219 	fence->seqno = seqno;
220 	INIT_LIST_HEAD(&fence->seq_passed_actions);
221 	fence->fman = fman;
222 	fence->signaled = 0;
223 	fence->signal_mask = mask;
224 	kref_init(&fence->kref);
225 	fence->destroy = destroy;
226 	init_waitqueue_head(&fence->queue);
227 
228 	spin_lock_irqsave(&fman->lock, irq_flags);
229 	if (unlikely(fman->fifo_down)) {
230 		ret = -EBUSY;
231 		goto out_unlock;
232 	}
233 	list_add_tail(&fence->head, &fman->fence_list);
234 	num_fences = ++fman->num_fence_objects;
235 
236 out_unlock:
237 	spin_unlock_irqrestore(&fman->lock, irq_flags);
238 	return ret;
239 
240 }
241 
242 struct vmw_fence_obj *vmw_fence_obj_reference(struct vmw_fence_obj *fence)
243 {
244 	if (unlikely(fence == NULL))
245 		return NULL;
246 
247 	kref_get(&fence->kref);
248 	return fence;
249 }
250 
251 /**
252  * vmw_fence_obj_unreference
253  *
254  * Note that this function may not be entered with disabled irqs since
255  * it may re-enable them in the destroy function.
256  *
257  */
258 void vmw_fence_obj_unreference(struct vmw_fence_obj **fence_p)
259 {
260 	struct vmw_fence_obj *fence = *fence_p;
261 	struct vmw_fence_manager *fman;
262 
263 	if (unlikely(fence == NULL))
264 		return;
265 
266 	fman = fence->fman;
267 	*fence_p = NULL;
268 	spin_lock_irq(&fman->lock);
269 	BUG_ON(atomic_read(&fence->kref.refcount) == 0);
270 	kref_put(&fence->kref, vmw_fence_obj_destroy_locked);
271 	spin_unlock_irq(&fman->lock);
272 }
273 
274 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,
275 				struct list_head *list)
276 {
277 	struct vmw_fence_action *action, *next_action;
278 
279 	list_for_each_entry_safe(action, next_action, list, head) {
280 		list_del_init(&action->head);
281 		fman->pending_actions[action->type]--;
282 		if (action->seq_passed != NULL)
283 			action->seq_passed(action);
284 
285 		/*
286 		 * Add the cleanup action to the cleanup list so that
287 		 * it will be performed by a worker task.
288 		 */
289 
290 		list_add_tail(&action->head, &fman->cleanup_list);
291 	}
292 }
293 
294 /**
295  * vmw_fence_goal_new_locked - Figure out a new device fence goal
296  * seqno if needed.
297  *
298  * @fman: Pointer to a fence manager.
299  * @passed_seqno: The seqno the device currently signals as passed.
300  *
301  * This function should be called with the fence manager lock held.
302  * It is typically called when we have a new passed_seqno, and
303  * we might need to update the fence goal. It checks to see whether
304  * the current fence goal has already passed, and, in that case,
305  * scans through all unsignaled fences to get the next fence object with an
306  * action attached, and sets the seqno of that fence as a new fence goal.
307  *
308  * returns true if the device goal seqno was updated. False otherwise.
309  */
310 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
311 				      u32 passed_seqno)
312 {
313 	u32 goal_seqno;
314 	__le32 __iomem *fifo_mem;
315 	struct vmw_fence_obj *fence;
316 
317 	if (likely(!fman->seqno_valid))
318 		return false;
319 
320 	fifo_mem = fman->dev_priv->mmio_virt;
321 	goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
322 	if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
323 		return false;
324 
325 	fman->seqno_valid = false;
326 	list_for_each_entry(fence, &fman->fence_list, head) {
327 		if (!list_empty(&fence->seq_passed_actions)) {
328 			fman->seqno_valid = true;
329 			iowrite32(fence->seqno,
330 				  fifo_mem + SVGA_FIFO_FENCE_GOAL);
331 			break;
332 		}
333 	}
334 
335 	return true;
336 }
337 
338 
339 /**
340  * vmw_fence_goal_check_locked - Replace the device fence goal seqno if
341  * needed.
342  *
343  * @fence: Pointer to a struct vmw_fence_obj the seqno of which should be
344  * considered as a device fence goal.
345  *
346  * This function should be called with the fence manager lock held.
347  * It is typically called when an action has been attached to a fence to
348  * check whether the seqno of that fence should be used for a fence
349  * goal interrupt. This is typically needed if the current fence goal is
350  * invalid, or has a higher seqno than that of the current fence object.
351  *
352  * returns true if the device goal seqno was updated. False otherwise.
353  */
354 static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
355 {
356 	u32 goal_seqno;
357 	__le32 __iomem *fifo_mem;
358 
359 	if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC)
360 		return false;
361 
362 	fifo_mem = fence->fman->dev_priv->mmio_virt;
363 	goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
364 	if (likely(fence->fman->seqno_valid &&
365 		   goal_seqno - fence->seqno < VMW_FENCE_WRAP))
366 		return false;
367 
368 	iowrite32(fence->seqno, fifo_mem + SVGA_FIFO_FENCE_GOAL);
369 	fence->fman->seqno_valid = true;
370 
371 	return true;
372 }
373 
374 void vmw_fences_update(struct vmw_fence_manager *fman)
375 {
376 	unsigned long flags;
377 	struct vmw_fence_obj *fence, *next_fence;
378 	struct list_head action_list;
379 	bool needs_rerun;
380 	uint32_t seqno, new_seqno;
381 	__le32 __iomem *fifo_mem = fman->dev_priv->mmio_virt;
382 
383 	seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
384 rerun:
385 	spin_lock_irqsave(&fman->lock, flags);
386 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
387 		if (seqno - fence->seqno < VMW_FENCE_WRAP) {
388 			list_del_init(&fence->head);
389 			fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC;
390 			INIT_LIST_HEAD(&action_list);
391 			list_splice_init(&fence->seq_passed_actions,
392 					 &action_list);
393 			vmw_fences_perform_actions(fman, &action_list);
394 			wake_up_all(&fence->queue);
395 		} else
396 			break;
397 	}
398 
399 	needs_rerun = vmw_fence_goal_new_locked(fman, seqno);
400 
401 	if (!list_empty(&fman->cleanup_list))
402 		(void) schedule_work(&fman->work);
403 	spin_unlock_irqrestore(&fman->lock, flags);
404 
405 	/*
406 	 * Rerun if the fence goal seqno was updated, and the
407 	 * hardware might have raced with that update, so that
408 	 * we missed a fence_goal irq.
409 	 */
410 
411 	if (unlikely(needs_rerun)) {
412 		new_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
413 		if (new_seqno != seqno) {
414 			seqno = new_seqno;
415 			goto rerun;
416 		}
417 	}
418 }
419 
420 bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence,
421 			    uint32_t flags)
422 {
423 	struct vmw_fence_manager *fman = fence->fman;
424 	unsigned long irq_flags;
425 	uint32_t signaled;
426 
427 	spin_lock_irqsave(&fman->lock, irq_flags);
428 	signaled = fence->signaled;
429 	spin_unlock_irqrestore(&fman->lock, irq_flags);
430 
431 	flags &= fence->signal_mask;
432 	if ((signaled & flags) == flags)
433 		return 1;
434 
435 	if ((signaled & DRM_VMW_FENCE_FLAG_EXEC) == 0)
436 		vmw_fences_update(fman);
437 
438 	spin_lock_irqsave(&fman->lock, irq_flags);
439 	signaled = fence->signaled;
440 	spin_unlock_irqrestore(&fman->lock, irq_flags);
441 
442 	return ((signaled & flags) == flags);
443 }
444 
445 int vmw_fence_obj_wait(struct vmw_fence_obj *fence,
446 		       uint32_t flags, bool lazy,
447 		       bool interruptible, unsigned long timeout)
448 {
449 	struct vmw_private *dev_priv = fence->fman->dev_priv;
450 	long ret;
451 
452 	if (likely(vmw_fence_obj_signaled(fence, flags)))
453 		return 0;
454 
455 	vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
456 	vmw_seqno_waiter_add(dev_priv);
457 
458 	if (interruptible)
459 		ret = wait_event_interruptible_timeout
460 			(fence->queue,
461 			 vmw_fence_obj_signaled(fence, flags),
462 			 timeout);
463 	else
464 		ret = wait_event_timeout
465 			(fence->queue,
466 			 vmw_fence_obj_signaled(fence, flags),
467 			 timeout);
468 
469 	vmw_seqno_waiter_remove(dev_priv);
470 
471 	if (unlikely(ret == 0))
472 		ret = -EBUSY;
473 	else if (likely(ret > 0))
474 		ret = 0;
475 
476 	return ret;
477 }
478 
479 void vmw_fence_obj_flush(struct vmw_fence_obj *fence)
480 {
481 	struct vmw_private *dev_priv = fence->fman->dev_priv;
482 
483 	vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
484 }
485 
486 static void vmw_fence_destroy(struct vmw_fence_obj *fence)
487 {
488 	struct vmw_fence_manager *fman = fence->fman;
489 
490 	kfree(fence);
491 	/*
492 	 * Free kernel space accounting.
493 	 */
494 	ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),
495 			    fman->fence_size);
496 }
497 
498 int vmw_fence_create(struct vmw_fence_manager *fman,
499 		     uint32_t seqno,
500 		     uint32_t mask,
501 		     struct vmw_fence_obj **p_fence)
502 {
503 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);
504 	struct vmw_fence_obj *fence;
505 	int ret;
506 
507 	ret = ttm_mem_global_alloc(mem_glob, fman->fence_size,
508 				   false, false);
509 	if (unlikely(ret != 0))
510 		return ret;
511 
512 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
513 	if (unlikely(fence == NULL)) {
514 		ret = -ENOMEM;
515 		goto out_no_object;
516 	}
517 
518 	ret = vmw_fence_obj_init(fman, fence, seqno, mask,
519 				 vmw_fence_destroy);
520 	if (unlikely(ret != 0))
521 		goto out_err_init;
522 
523 	*p_fence = fence;
524 	return 0;
525 
526 out_err_init:
527 	kfree(fence);
528 out_no_object:
529 	ttm_mem_global_free(mem_glob, fman->fence_size);
530 	return ret;
531 }
532 
533 
534 static void vmw_user_fence_destroy(struct vmw_fence_obj *fence)
535 {
536 	struct vmw_user_fence *ufence =
537 		container_of(fence, struct vmw_user_fence, fence);
538 	struct vmw_fence_manager *fman = fence->fman;
539 
540 	ttm_base_object_kfree(ufence, base);
541 	/*
542 	 * Free kernel space accounting.
543 	 */
544 	ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),
545 			    fman->user_fence_size);
546 }
547 
548 static void vmw_user_fence_base_release(struct ttm_base_object **p_base)
549 {
550 	struct ttm_base_object *base = *p_base;
551 	struct vmw_user_fence *ufence =
552 		container_of(base, struct vmw_user_fence, base);
553 	struct vmw_fence_obj *fence = &ufence->fence;
554 
555 	*p_base = NULL;
556 	vmw_fence_obj_unreference(&fence);
557 }
558 
559 int vmw_user_fence_create(struct drm_file *file_priv,
560 			  struct vmw_fence_manager *fman,
561 			  uint32_t seqno,
562 			  uint32_t mask,
563 			  struct vmw_fence_obj **p_fence,
564 			  uint32_t *p_handle)
565 {
566 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
567 	struct vmw_user_fence *ufence;
568 	struct vmw_fence_obj *tmp;
569 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);
570 	int ret;
571 
572 	/*
573 	 * Kernel memory space accounting, since this object may
574 	 * be created by a user-space request.
575 	 */
576 
577 	ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,
578 				   false, false);
579 	if (unlikely(ret != 0))
580 		return ret;
581 
582 	ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
583 	if (unlikely(ufence == NULL)) {
584 		ret = -ENOMEM;
585 		goto out_no_object;
586 	}
587 
588 	ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,
589 				 mask, vmw_user_fence_destroy);
590 	if (unlikely(ret != 0)) {
591 		kfree(ufence);
592 		goto out_no_object;
593 	}
594 
595 	/*
596 	 * The base object holds a reference which is freed in
597 	 * vmw_user_fence_base_release.
598 	 */
599 	tmp = vmw_fence_obj_reference(&ufence->fence);
600 	ret = ttm_base_object_init(tfile, &ufence->base, false,
601 				   VMW_RES_FENCE,
602 				   &vmw_user_fence_base_release, NULL);
603 
604 
605 	if (unlikely(ret != 0)) {
606 		/*
607 		 * Free the base object's reference
608 		 */
609 		vmw_fence_obj_unreference(&tmp);
610 		goto out_err;
611 	}
612 
613 	*p_fence = &ufence->fence;
614 	*p_handle = ufence->base.hash.key;
615 
616 	return 0;
617 out_err:
618 	tmp = &ufence->fence;
619 	vmw_fence_obj_unreference(&tmp);
620 out_no_object:
621 	ttm_mem_global_free(mem_glob, fman->user_fence_size);
622 	return ret;
623 }
624 
625 
626 /**
627  * vmw_fence_fifo_down - signal all unsignaled fence objects.
628  */
629 
630 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
631 {
632 	unsigned long irq_flags;
633 	struct list_head action_list;
634 	int ret;
635 
636 	/*
637 	 * The list may be altered while we traverse it, so always
638 	 * restart when we've released the fman->lock.
639 	 */
640 
641 	spin_lock_irqsave(&fman->lock, irq_flags);
642 	fman->fifo_down = true;
643 	while (!list_empty(&fman->fence_list)) {
644 		struct vmw_fence_obj *fence =
645 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,
646 				   head);
647 		kref_get(&fence->kref);
648 		spin_unlock_irq(&fman->lock);
649 
650 		ret = vmw_fence_obj_wait(fence, fence->signal_mask,
651 					 false, false,
652 					 VMW_FENCE_WAIT_TIMEOUT);
653 
654 		if (unlikely(ret != 0)) {
655 			list_del_init(&fence->head);
656 			fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC;
657 			INIT_LIST_HEAD(&action_list);
658 			list_splice_init(&fence->seq_passed_actions,
659 					 &action_list);
660 			vmw_fences_perform_actions(fman, &action_list);
661 			wake_up_all(&fence->queue);
662 		}
663 
664 		spin_lock_irq(&fman->lock);
665 
666 		BUG_ON(!list_empty(&fence->head));
667 		kref_put(&fence->kref, vmw_fence_obj_destroy_locked);
668 	}
669 	spin_unlock_irqrestore(&fman->lock, irq_flags);
670 }
671 
672 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)
673 {
674 	unsigned long irq_flags;
675 
676 	spin_lock_irqsave(&fman->lock, irq_flags);
677 	fman->fifo_down = false;
678 	spin_unlock_irqrestore(&fman->lock, irq_flags);
679 }
680 
681 
682 int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data,
683 			     struct drm_file *file_priv)
684 {
685 	struct drm_vmw_fence_wait_arg *arg =
686 	    (struct drm_vmw_fence_wait_arg *)data;
687 	unsigned long timeout;
688 	struct ttm_base_object *base;
689 	struct vmw_fence_obj *fence;
690 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
691 	int ret;
692 	uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ);
693 
694 	/*
695 	 * 64-bit division not present on 32-bit systems, so do an
696 	 * approximation. (Divide by 1000000).
697 	 */
698 
699 	wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) -
700 	  (wait_timeout >> 26);
701 
702 	if (!arg->cookie_valid) {
703 		arg->cookie_valid = 1;
704 		arg->kernel_cookie = jiffies + wait_timeout;
705 	}
706 
707 	base = ttm_base_object_lookup(tfile, arg->handle);
708 	if (unlikely(base == NULL)) {
709 		printk(KERN_ERR "Wait invalid fence object handle "
710 		       "0x%08lx.\n",
711 		       (unsigned long)arg->handle);
712 		return -EINVAL;
713 	}
714 
715 	fence = &(container_of(base, struct vmw_user_fence, base)->fence);
716 
717 	timeout = jiffies;
718 	if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) {
719 		ret = ((vmw_fence_obj_signaled(fence, arg->flags)) ?
720 		       0 : -EBUSY);
721 		goto out;
722 	}
723 
724 	timeout = (unsigned long)arg->kernel_cookie - timeout;
725 
726 	ret = vmw_fence_obj_wait(fence, arg->flags, arg->lazy, true, timeout);
727 
728 out:
729 	ttm_base_object_unref(&base);
730 
731 	/*
732 	 * Optionally unref the fence object.
733 	 */
734 
735 	if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF))
736 		return ttm_ref_object_base_unref(tfile, arg->handle,
737 						 TTM_REF_USAGE);
738 	return ret;
739 }
740 
741 int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data,
742 				 struct drm_file *file_priv)
743 {
744 	struct drm_vmw_fence_signaled_arg *arg =
745 		(struct drm_vmw_fence_signaled_arg *) data;
746 	struct ttm_base_object *base;
747 	struct vmw_fence_obj *fence;
748 	struct vmw_fence_manager *fman;
749 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
750 	struct vmw_private *dev_priv = vmw_priv(dev);
751 
752 	base = ttm_base_object_lookup(tfile, arg->handle);
753 	if (unlikely(base == NULL)) {
754 		printk(KERN_ERR "Fence signaled invalid fence object handle "
755 		       "0x%08lx.\n",
756 		       (unsigned long)arg->handle);
757 		return -EINVAL;
758 	}
759 
760 	fence = &(container_of(base, struct vmw_user_fence, base)->fence);
761 	fman = fence->fman;
762 
763 	arg->signaled = vmw_fence_obj_signaled(fence, arg->flags);
764 	spin_lock_irq(&fman->lock);
765 
766 	arg->signaled_flags = fence->signaled;
767 	arg->passed_seqno = dev_priv->last_read_seqno;
768 	spin_unlock_irq(&fman->lock);
769 
770 	ttm_base_object_unref(&base);
771 
772 	return 0;
773 }
774 
775 
776 int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data,
777 			      struct drm_file *file_priv)
778 {
779 	struct drm_vmw_fence_arg *arg =
780 		(struct drm_vmw_fence_arg *) data;
781 
782 	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
783 					 arg->handle,
784 					 TTM_REF_USAGE);
785 }
786 
787 /**
788  * vmw_event_fence_fpriv_gone - Remove references to struct drm_file objects
789  *
790  * @fman: Pointer to a struct vmw_fence_manager
791  * @event_list: Pointer to linked list of struct vmw_event_fence_action objects
792  * with pointers to a struct drm_file object about to be closed.
793  *
794  * This function removes all pending fence events with references to a
795  * specific struct drm_file object about to be closed. The caller is required
796  * to pass a list of all struct vmw_event_fence_action objects with such
797  * events attached. This function is typically called before the
798  * struct drm_file object's event management is taken down.
799  */
800 void vmw_event_fence_fpriv_gone(struct vmw_fence_manager *fman,
801 				struct list_head *event_list)
802 {
803 	struct vmw_event_fence_action *eaction;
804 	struct drm_pending_event *event;
805 	unsigned long irq_flags;
806 
807 	while (1) {
808 		spin_lock_irqsave(&fman->lock, irq_flags);
809 		if (list_empty(event_list))
810 			goto out_unlock;
811 		eaction = list_first_entry(event_list,
812 					   struct vmw_event_fence_action,
813 					   fpriv_head);
814 		list_del_init(&eaction->fpriv_head);
815 		event = eaction->event;
816 		eaction->event = NULL;
817 		spin_unlock_irqrestore(&fman->lock, irq_flags);
818 		event->destroy(event);
819 	}
820 out_unlock:
821 	spin_unlock_irqrestore(&fman->lock, irq_flags);
822 }
823 
824 
825 /**
826  * vmw_event_fence_action_seq_passed
827  *
828  * @action: The struct vmw_fence_action embedded in a struct
829  * vmw_event_fence_action.
830  *
831  * This function is called when the seqno of the fence where @action is
832  * attached has passed. It queues the event on the submitter's event list.
833  * This function is always called from atomic context, and may be called
834  * from irq context.
835  */
836 static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
837 {
838 	struct vmw_event_fence_action *eaction =
839 		container_of(action, struct vmw_event_fence_action, action);
840 	struct drm_device *dev = eaction->dev;
841 	struct drm_pending_event *event = eaction->event;
842 	struct drm_file *file_priv;
843 	unsigned long irq_flags;
844 
845 	if (unlikely(event == NULL))
846 		return;
847 
848 	file_priv = event->file_priv;
849 	spin_lock_irqsave(&dev->event_lock, irq_flags);
850 
851 	if (likely(eaction->tv_sec != NULL)) {
852 		struct timeval tv;
853 
854 		do_gettimeofday(&tv);
855 		*eaction->tv_sec = tv.tv_sec;
856 		*eaction->tv_usec = tv.tv_usec;
857 	}
858 
859 	list_del_init(&eaction->fpriv_head);
860 	list_add_tail(&eaction->event->link, &file_priv->event_list);
861 	eaction->event = NULL;
862 	wake_up_all(&file_priv->event_wait);
863 	spin_unlock_irqrestore(&dev->event_lock, irq_flags);
864 }
865 
866 /**
867  * vmw_event_fence_action_cleanup
868  *
869  * @action: The struct vmw_fence_action embedded in a struct
870  * vmw_event_fence_action.
871  *
872  * This function is the struct vmw_fence_action destructor. It's typically
873  * called from a workqueue.
874  */
875 static void vmw_event_fence_action_cleanup(struct vmw_fence_action *action)
876 {
877 	struct vmw_event_fence_action *eaction =
878 		container_of(action, struct vmw_event_fence_action, action);
879 	struct vmw_fence_manager *fman = eaction->fence->fman;
880 	unsigned long irq_flags;
881 
882 	spin_lock_irqsave(&fman->lock, irq_flags);
883 	list_del(&eaction->fpriv_head);
884 	spin_unlock_irqrestore(&fman->lock, irq_flags);
885 
886 	vmw_fence_obj_unreference(&eaction->fence);
887 	kfree(eaction);
888 }
889 
890 
891 /**
892  * vmw_fence_obj_add_action - Add an action to a fence object.
893  *
894  * @fence - The fence object.
895  * @action - The action to add.
896  *
897  * Note that the action callbacks may be executed before this function
898  * returns.
899  */
900 static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
901 			      struct vmw_fence_action *action)
902 {
903 	struct vmw_fence_manager *fman = fence->fman;
904 	unsigned long irq_flags;
905 	bool run_update = false;
906 
907 	mutex_lock(&fman->goal_irq_mutex);
908 	spin_lock_irqsave(&fman->lock, irq_flags);
909 
910 	fman->pending_actions[action->type]++;
911 	if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) {
912 		struct list_head action_list;
913 
914 		INIT_LIST_HEAD(&action_list);
915 		list_add_tail(&action->head, &action_list);
916 		vmw_fences_perform_actions(fman, &action_list);
917 	} else {
918 		list_add_tail(&action->head, &fence->seq_passed_actions);
919 
920 		/*
921 		 * This function may set fman::seqno_valid, so it must
922 		 * be run with the goal_irq_mutex held.
923 		 */
924 		run_update = vmw_fence_goal_check_locked(fence);
925 	}
926 
927 	spin_unlock_irqrestore(&fman->lock, irq_flags);
928 
929 	if (run_update) {
930 		if (!fman->goal_irq_on) {
931 			fman->goal_irq_on = true;
932 			vmw_goal_waiter_add(fman->dev_priv);
933 		}
934 		vmw_fences_update(fman);
935 	}
936 	mutex_unlock(&fman->goal_irq_mutex);
937 
938 }
939 
940 /**
941  * vmw_event_fence_action_create - Post an event for sending when a fence
942  * object seqno has passed.
943  *
944  * @file_priv: The file connection on which the event should be posted.
945  * @fence: The fence object on which to post the event.
946  * @event: Event to be posted. This event should've been alloced
947  * using k[mz]alloc, and should've been completely initialized.
948  * @interruptible: Interruptible waits if possible.
949  *
950  * As a side effect, the object pointed to by @event may have been
951  * freed when this function returns. If this function returns with
952  * an error code, the caller needs to free that object.
953  */
954 
955 int vmw_event_fence_action_queue(struct drm_file *file_priv,
956 				 struct vmw_fence_obj *fence,
957 				 struct drm_pending_event *event,
958 				 uint32_t *tv_sec,
959 				 uint32_t *tv_usec,
960 				 bool interruptible)
961 {
962 	struct vmw_event_fence_action *eaction;
963 	struct vmw_fence_manager *fman = fence->fman;
964 	struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
965 	unsigned long irq_flags;
966 
967 	eaction = kzalloc(sizeof(*eaction), GFP_KERNEL);
968 	if (unlikely(eaction == NULL))
969 		return -ENOMEM;
970 
971 	eaction->event = event;
972 
973 	eaction->action.seq_passed = vmw_event_fence_action_seq_passed;
974 	eaction->action.cleanup = vmw_event_fence_action_cleanup;
975 	eaction->action.type = VMW_ACTION_EVENT;
976 
977 	eaction->fence = vmw_fence_obj_reference(fence);
978 	eaction->dev = fman->dev_priv->dev;
979 	eaction->tv_sec = tv_sec;
980 	eaction->tv_usec = tv_usec;
981 
982 	spin_lock_irqsave(&fman->lock, irq_flags);
983 	list_add_tail(&eaction->fpriv_head, &vmw_fp->fence_events);
984 	spin_unlock_irqrestore(&fman->lock, irq_flags);
985 
986 	vmw_fence_obj_add_action(fence, &eaction->action);
987 
988 	return 0;
989 }
990 
991 struct vmw_event_fence_pending {
992 	struct drm_pending_event base;
993 	struct drm_vmw_event_fence event;
994 };
995 
996 static int vmw_event_fence_action_create(struct drm_file *file_priv,
997 				  struct vmw_fence_obj *fence,
998 				  uint32_t flags,
999 				  uint64_t user_data,
1000 				  bool interruptible)
1001 {
1002 	struct vmw_event_fence_pending *event;
1003 	struct drm_device *dev = fence->fman->dev_priv->dev;
1004 	unsigned long irq_flags;
1005 	int ret;
1006 
1007 	spin_lock_irqsave(&dev->event_lock, irq_flags);
1008 
1009 	ret = (file_priv->event_space < sizeof(event->event)) ? -EBUSY : 0;
1010 	if (likely(ret == 0))
1011 		file_priv->event_space -= sizeof(event->event);
1012 
1013 	spin_unlock_irqrestore(&dev->event_lock, irq_flags);
1014 
1015 	if (unlikely(ret != 0)) {
1016 		DRM_ERROR("Failed to allocate event space for this file.\n");
1017 		goto out_no_space;
1018 	}
1019 
1020 
1021 	event = kzalloc(sizeof(*event), GFP_KERNEL);
1022 	if (unlikely(event == NULL)) {
1023 		DRM_ERROR("Failed to allocate an event.\n");
1024 		ret = -ENOMEM;
1025 		goto out_no_event;
1026 	}
1027 
1028 	event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED;
1029 	event->event.base.length = sizeof(*event);
1030 	event->event.user_data = user_data;
1031 
1032 	event->base.event = &event->event.base;
1033 	event->base.file_priv = file_priv;
1034 	event->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
1035 
1036 
1037 	if (flags & DRM_VMW_FE_FLAG_REQ_TIME)
1038 		ret = vmw_event_fence_action_queue(file_priv, fence,
1039 						   &event->base,
1040 						   &event->event.tv_sec,
1041 						   &event->event.tv_usec,
1042 						   interruptible);
1043 	else
1044 		ret = vmw_event_fence_action_queue(file_priv, fence,
1045 						   &event->base,
1046 						   NULL,
1047 						   NULL,
1048 						   interruptible);
1049 	if (ret != 0)
1050 		goto out_no_queue;
1051 
1052 out_no_queue:
1053 	event->base.destroy(&event->base);
1054 out_no_event:
1055 	spin_lock_irqsave(&dev->event_lock, irq_flags);
1056 	file_priv->event_space += sizeof(*event);
1057 	spin_unlock_irqrestore(&dev->event_lock, irq_flags);
1058 out_no_space:
1059 	return ret;
1060 }
1061 
1062 int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
1063 			  struct drm_file *file_priv)
1064 {
1065 	struct vmw_private *dev_priv = vmw_priv(dev);
1066 	struct drm_vmw_fence_event_arg *arg =
1067 		(struct drm_vmw_fence_event_arg *) data;
1068 	struct vmw_fence_obj *fence = NULL;
1069 	struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
1070 	struct drm_vmw_fence_rep __user *user_fence_rep =
1071 		(struct drm_vmw_fence_rep __user *)(unsigned long)
1072 		arg->fence_rep;
1073 	uint32_t handle;
1074 	int ret;
1075 
1076 	/*
1077 	 * Look up an existing fence object,
1078 	 * and if user-space wants a new reference,
1079 	 * add one.
1080 	 */
1081 	if (arg->handle) {
1082 		struct ttm_base_object *base =
1083 			ttm_base_object_lookup_for_ref(dev_priv->tdev,
1084 						       arg->handle);
1085 
1086 		if (unlikely(base == NULL)) {
1087 			DRM_ERROR("Fence event invalid fence object handle "
1088 				  "0x%08lx.\n",
1089 				  (unsigned long)arg->handle);
1090 			return -EINVAL;
1091 		}
1092 		fence = &(container_of(base, struct vmw_user_fence,
1093 				       base)->fence);
1094 		(void) vmw_fence_obj_reference(fence);
1095 
1096 		if (user_fence_rep != NULL) {
1097 			bool existed;
1098 
1099 			ret = ttm_ref_object_add(vmw_fp->tfile, base,
1100 						 TTM_REF_USAGE, &existed);
1101 			if (unlikely(ret != 0)) {
1102 				DRM_ERROR("Failed to reference a fence "
1103 					  "object.\n");
1104 				goto out_no_ref_obj;
1105 			}
1106 			handle = base->hash.key;
1107 		}
1108 		ttm_base_object_unref(&base);
1109 	}
1110 
1111 	/*
1112 	 * Create a new fence object.
1113 	 */
1114 	if (!fence) {
1115 		ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
1116 						 &fence,
1117 						 (user_fence_rep) ?
1118 						 &handle : NULL);
1119 		if (unlikely(ret != 0)) {
1120 			DRM_ERROR("Fence event failed to create fence.\n");
1121 			return ret;
1122 		}
1123 	}
1124 
1125 	BUG_ON(fence == NULL);
1126 
1127 	if (arg->flags & DRM_VMW_FE_FLAG_REQ_TIME)
1128 		ret = vmw_event_fence_action_create(file_priv, fence,
1129 						    arg->flags,
1130 						    arg->user_data,
1131 						    true);
1132 	else
1133 		ret = vmw_event_fence_action_create(file_priv, fence,
1134 						    arg->flags,
1135 						    arg->user_data,
1136 						    true);
1137 
1138 	if (unlikely(ret != 0)) {
1139 		if (ret != -ERESTARTSYS)
1140 			DRM_ERROR("Failed to attach event to fence.\n");
1141 		goto out_no_create;
1142 	}
1143 
1144 	vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
1145 				    handle);
1146 	vmw_fence_obj_unreference(&fence);
1147 	return 0;
1148 out_no_create:
1149 	if (user_fence_rep != NULL)
1150 		ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
1151 					  handle, TTM_REF_USAGE);
1152 out_no_ref_obj:
1153 	vmw_fence_obj_unreference(&fence);
1154 	return ret;
1155 }
1156