xref: /linux/drivers/gpu/drm/nouveau/nouveau_fence.c (revision ea518afc992032f7570c0a89ac9240b387dc0faf)
1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <linux/ktime.h>
28 #include <linux/hrtimer.h>
29 #include <linux/sched/signal.h>
30 #include <trace/events/dma_fence.h>
31 
32 #include <nvif/if0020.h>
33 
34 #include "nouveau_drv.h"
35 #include "nouveau_dma.h"
36 #include "nouveau_fence.h"
37 
38 static const struct dma_fence_ops nouveau_fence_ops_uevent;
39 static const struct dma_fence_ops nouveau_fence_ops_legacy;
40 
41 static inline struct nouveau_fence *
42 from_fence(struct dma_fence *fence)
43 {
44 	return container_of(fence, struct nouveau_fence, base);
45 }
46 
47 static inline struct nouveau_fence_chan *
48 nouveau_fctx(struct nouveau_fence *fence)
49 {
50 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
51 }
52 
53 static int
54 nouveau_fence_signal(struct nouveau_fence *fence)
55 {
56 	int drop = 0;
57 
58 	dma_fence_signal_locked(&fence->base);
59 	list_del(&fence->head);
60 	rcu_assign_pointer(fence->channel, NULL);
61 
62 	if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
63 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
64 
65 		if (atomic_dec_and_test(&fctx->notify_ref))
66 			drop = 1;
67 	}
68 
69 	dma_fence_put(&fence->base);
70 	return drop;
71 }
72 
73 static struct nouveau_fence *
74 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
75 {
76 	if (fence->ops != &nouveau_fence_ops_legacy &&
77 	    fence->ops != &nouveau_fence_ops_uevent)
78 		return NULL;
79 
80 	return from_fence(fence);
81 }
82 
83 void
84 nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
85 {
86 	struct nouveau_fence *fence;
87 	unsigned long flags;
88 
89 	spin_lock_irqsave(&fctx->lock, flags);
90 	while (!list_empty(&fctx->pending)) {
91 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
92 
93 		if (error)
94 			dma_fence_set_error(&fence->base, error);
95 
96 		if (nouveau_fence_signal(fence))
97 			nvif_event_block(&fctx->event);
98 	}
99 	fctx->killed = 1;
100 	spin_unlock_irqrestore(&fctx->lock, flags);
101 }
102 
103 void
104 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
105 {
106 	cancel_work_sync(&fctx->allow_block_work);
107 	nouveau_fence_context_kill(fctx, 0);
108 	nvif_event_dtor(&fctx->event);
109 	fctx->dead = 1;
110 
111 	/*
112 	 * Ensure that all accesses to fence->channel complete before freeing
113 	 * the channel.
114 	 */
115 	synchronize_rcu();
116 }
117 
118 static void
119 nouveau_fence_context_put(struct kref *fence_ref)
120 {
121 	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
122 }
123 
124 void
125 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
126 {
127 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
128 }
129 
130 static int
131 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
132 {
133 	struct nouveau_fence *fence;
134 	int drop = 0;
135 	u32 seq = fctx->read(chan);
136 
137 	while (!list_empty(&fctx->pending)) {
138 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
139 
140 		if ((int)(seq - fence->base.seqno) < 0)
141 			break;
142 
143 		drop |= nouveau_fence_signal(fence);
144 	}
145 
146 	return drop;
147 }
148 
149 static int
150 nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc)
151 {
152 	struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event);
153 	unsigned long flags;
154 	int ret = NVIF_EVENT_KEEP;
155 
156 	spin_lock_irqsave(&fctx->lock, flags);
157 	if (!list_empty(&fctx->pending)) {
158 		struct nouveau_fence *fence;
159 		struct nouveau_channel *chan;
160 
161 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
162 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
163 		if (nouveau_fence_update(chan, fctx))
164 			ret = NVIF_EVENT_DROP;
165 	}
166 	spin_unlock_irqrestore(&fctx->lock, flags);
167 
168 	return ret;
169 }
170 
171 static void
172 nouveau_fence_work_allow_block(struct work_struct *work)
173 {
174 	struct nouveau_fence_chan *fctx = container_of(work, struct nouveau_fence_chan,
175 						       allow_block_work);
176 
177 	if (atomic_read(&fctx->notify_ref) == 0)
178 		nvif_event_block(&fctx->event);
179 	else
180 		nvif_event_allow(&fctx->event);
181 }
182 
183 void
184 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
185 {
186 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
187 	struct nouveau_cli *cli = (void *)chan->user.client;
188 	struct {
189 		struct nvif_event_v0 base;
190 		struct nvif_chan_event_v0 host;
191 	} args;
192 	int ret;
193 
194 	INIT_WORK(&fctx->allow_block_work, nouveau_fence_work_allow_block);
195 	INIT_LIST_HEAD(&fctx->flip);
196 	INIT_LIST_HEAD(&fctx->pending);
197 	spin_lock_init(&fctx->lock);
198 	fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid;
199 
200 	if (chan == chan->drm->cechan)
201 		strcpy(fctx->name, "copy engine channel");
202 	else if (chan == chan->drm->channel)
203 		strcpy(fctx->name, "generic kernel channel");
204 	else
205 		strcpy(fctx->name, nvxx_client(&cli->base)->name);
206 
207 	kref_init(&fctx->fence_ref);
208 	if (!priv->uevent)
209 		return;
210 
211 	args.host.version = 0;
212 	args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR;
213 
214 	ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid,
215 			      nouveau_fence_wait_uevent_handler, false,
216 			      &args.base, sizeof(args), &fctx->event);
217 
218 	WARN_ON(ret);
219 }
220 
221 int
222 nouveau_fence_emit(struct nouveau_fence *fence)
223 {
224 	struct nouveau_channel *chan = unrcu_pointer(fence->channel);
225 	struct nouveau_fence_chan *fctx = chan->fence;
226 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
227 	int ret;
228 
229 	fence->timeout  = jiffies + (15 * HZ);
230 
231 	if (priv->uevent)
232 		dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
233 			       &fctx->lock, fctx->context, ++fctx->sequence);
234 	else
235 		dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
236 			       &fctx->lock, fctx->context, ++fctx->sequence);
237 	kref_get(&fctx->fence_ref);
238 
239 	ret = fctx->emit(fence);
240 	if (!ret) {
241 		dma_fence_get(&fence->base);
242 		spin_lock_irq(&fctx->lock);
243 
244 		if (unlikely(fctx->killed)) {
245 			spin_unlock_irq(&fctx->lock);
246 			dma_fence_put(&fence->base);
247 			return -ENODEV;
248 		}
249 
250 		if (nouveau_fence_update(chan, fctx))
251 			nvif_event_block(&fctx->event);
252 
253 		list_add_tail(&fence->head, &fctx->pending);
254 		spin_unlock_irq(&fctx->lock);
255 	}
256 
257 	return ret;
258 }
259 
260 bool
261 nouveau_fence_done(struct nouveau_fence *fence)
262 {
263 	if (fence->base.ops == &nouveau_fence_ops_legacy ||
264 	    fence->base.ops == &nouveau_fence_ops_uevent) {
265 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
266 		struct nouveau_channel *chan;
267 		unsigned long flags;
268 
269 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
270 			return true;
271 
272 		spin_lock_irqsave(&fctx->lock, flags);
273 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
274 		if (chan && nouveau_fence_update(chan, fctx))
275 			nvif_event_block(&fctx->event);
276 		spin_unlock_irqrestore(&fctx->lock, flags);
277 	}
278 	return dma_fence_is_signaled(&fence->base);
279 }
280 
281 static long
282 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
283 {
284 	struct nouveau_fence *fence = from_fence(f);
285 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
286 	unsigned long t = jiffies, timeout = t + wait;
287 
288 	while (!nouveau_fence_done(fence)) {
289 		ktime_t kt;
290 
291 		t = jiffies;
292 
293 		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
294 			__set_current_state(TASK_RUNNING);
295 			return 0;
296 		}
297 
298 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
299 					   TASK_UNINTERRUPTIBLE);
300 
301 		kt = sleep_time;
302 		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
303 		sleep_time *= 2;
304 		if (sleep_time > NSEC_PER_MSEC)
305 			sleep_time = NSEC_PER_MSEC;
306 
307 		if (intr && signal_pending(current))
308 			return -ERESTARTSYS;
309 	}
310 
311 	__set_current_state(TASK_RUNNING);
312 
313 	return timeout - t;
314 }
315 
316 static int
317 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
318 {
319 	int ret = 0;
320 
321 	while (!nouveau_fence_done(fence)) {
322 		if (time_after_eq(jiffies, fence->timeout)) {
323 			ret = -EBUSY;
324 			break;
325 		}
326 
327 		__set_current_state(intr ?
328 				    TASK_INTERRUPTIBLE :
329 				    TASK_UNINTERRUPTIBLE);
330 
331 		if (intr && signal_pending(current)) {
332 			ret = -ERESTARTSYS;
333 			break;
334 		}
335 	}
336 
337 	__set_current_state(TASK_RUNNING);
338 	return ret;
339 }
340 
341 int
342 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
343 {
344 	long ret;
345 
346 	if (!lazy)
347 		return nouveau_fence_wait_busy(fence, intr);
348 
349 	ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
350 	if (ret < 0)
351 		return ret;
352 	else if (!ret)
353 		return -EBUSY;
354 	else
355 		return 0;
356 }
357 
358 int
359 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
360 		   bool exclusive, bool intr)
361 {
362 	struct nouveau_fence_chan *fctx = chan->fence;
363 	struct dma_resv *resv = nvbo->bo.base.resv;
364 	int i, ret;
365 
366 	ret = dma_resv_reserve_fences(resv, 1);
367 	if (ret)
368 		return ret;
369 
370 	/* Waiting for the writes first causes performance regressions
371 	 * under some circumstances. So manually wait for the reads first.
372 	 */
373 	for (i = 0; i < 2; ++i) {
374 		struct dma_resv_iter cursor;
375 		struct dma_fence *fence;
376 
377 		dma_resv_for_each_fence(&cursor, resv,
378 					dma_resv_usage_rw(exclusive),
379 					fence) {
380 			enum dma_resv_usage usage;
381 			struct nouveau_fence *f;
382 
383 			usage = dma_resv_iter_usage(&cursor);
384 			if (i == 0 && usage == DMA_RESV_USAGE_WRITE)
385 				continue;
386 
387 			f = nouveau_local_fence(fence, chan->drm);
388 			if (f) {
389 				struct nouveau_channel *prev;
390 				bool must_wait = true;
391 
392 				rcu_read_lock();
393 				prev = rcu_dereference(f->channel);
394 				if (prev && (prev == chan ||
395 					     fctx->sync(f, prev, chan) == 0))
396 					must_wait = false;
397 				rcu_read_unlock();
398 				if (!must_wait)
399 					continue;
400 			}
401 
402 			ret = dma_fence_wait(fence, intr);
403 			if (ret)
404 				return ret;
405 		}
406 	}
407 
408 	return 0;
409 }
410 
411 void
412 nouveau_fence_unref(struct nouveau_fence **pfence)
413 {
414 	if (*pfence)
415 		dma_fence_put(&(*pfence)->base);
416 	*pfence = NULL;
417 }
418 
419 int
420 nouveau_fence_create(struct nouveau_fence **pfence,
421 		     struct nouveau_channel *chan)
422 {
423 	struct nouveau_fence *fence;
424 
425 	if (unlikely(!chan->fence))
426 		return -ENODEV;
427 
428 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
429 	if (!fence)
430 		return -ENOMEM;
431 
432 	fence->channel = chan;
433 
434 	*pfence = fence;
435 	return 0;
436 }
437 
438 int
439 nouveau_fence_new(struct nouveau_fence **pfence,
440 		  struct nouveau_channel *chan)
441 {
442 	int ret = 0;
443 
444 	ret = nouveau_fence_create(pfence, chan);
445 	if (ret)
446 		return ret;
447 
448 	ret = nouveau_fence_emit(*pfence);
449 	if (ret)
450 		nouveau_fence_unref(pfence);
451 
452 	return ret;
453 }
454 
455 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
456 {
457 	return "nouveau";
458 }
459 
460 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
461 {
462 	struct nouveau_fence *fence = from_fence(f);
463 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
464 
465 	return !fctx->dead ? fctx->name : "dead channel";
466 }
467 
468 /*
469  * In an ideal world, read would not assume the channel context is still alive.
470  * This function may be called from another device, running into free memory as a
471  * result. The drm node should still be there, so we can derive the index from
472  * the fence context.
473  */
474 static bool nouveau_fence_is_signaled(struct dma_fence *f)
475 {
476 	struct nouveau_fence *fence = from_fence(f);
477 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
478 	struct nouveau_channel *chan;
479 	bool ret = false;
480 
481 	rcu_read_lock();
482 	chan = rcu_dereference(fence->channel);
483 	if (chan)
484 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
485 	rcu_read_unlock();
486 
487 	return ret;
488 }
489 
490 static bool nouveau_fence_no_signaling(struct dma_fence *f)
491 {
492 	struct nouveau_fence *fence = from_fence(f);
493 
494 	/*
495 	 * caller should have a reference on the fence,
496 	 * else fence could get freed here
497 	 */
498 	WARN_ON(kref_read(&fence->base.refcount) <= 1);
499 
500 	/*
501 	 * This needs uevents to work correctly, but dma_fence_add_callback relies on
502 	 * being able to enable signaling. It will still get signaled eventually,
503 	 * just not right away.
504 	 */
505 	if (nouveau_fence_is_signaled(f)) {
506 		list_del(&fence->head);
507 
508 		dma_fence_put(&fence->base);
509 		return false;
510 	}
511 
512 	return true;
513 }
514 
515 static void nouveau_fence_release(struct dma_fence *f)
516 {
517 	struct nouveau_fence *fence = from_fence(f);
518 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
519 
520 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
521 	dma_fence_free(&fence->base);
522 }
523 
524 static const struct dma_fence_ops nouveau_fence_ops_legacy = {
525 	.get_driver_name = nouveau_fence_get_get_driver_name,
526 	.get_timeline_name = nouveau_fence_get_timeline_name,
527 	.enable_signaling = nouveau_fence_no_signaling,
528 	.signaled = nouveau_fence_is_signaled,
529 	.wait = nouveau_fence_wait_legacy,
530 	.release = nouveau_fence_release
531 };
532 
533 static bool nouveau_fence_enable_signaling(struct dma_fence *f)
534 {
535 	struct nouveau_fence *fence = from_fence(f);
536 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
537 	bool ret;
538 	bool do_work;
539 
540 	if (atomic_inc_return(&fctx->notify_ref) == 0)
541 		do_work = true;
542 
543 	ret = nouveau_fence_no_signaling(f);
544 	if (ret)
545 		set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
546 	else if (atomic_dec_and_test(&fctx->notify_ref))
547 		do_work = true;
548 
549 	if (do_work)
550 		schedule_work(&fctx->allow_block_work);
551 
552 	return ret;
553 }
554 
555 static const struct dma_fence_ops nouveau_fence_ops_uevent = {
556 	.get_driver_name = nouveau_fence_get_get_driver_name,
557 	.get_timeline_name = nouveau_fence_get_timeline_name,
558 	.enable_signaling = nouveau_fence_enable_signaling,
559 	.signaled = nouveau_fence_is_signaled,
560 	.release = nouveau_fence_release
561 };
562