1 /* 2 * (C) Copyright 2016 Intel Corporation 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; version 2 7 * of the License. 8 */ 9 10 #include <linux/slab.h> 11 #include <linux/dma-fence.h> 12 #include <linux/irq_work.h> 13 #include <linux/reservation.h> 14 15 #include "i915_sw_fence.h" 16 #include "i915_selftest.h" 17 18 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ 19 20 static DEFINE_SPINLOCK(i915_sw_fence_lock); 21 22 enum { 23 DEBUG_FENCE_IDLE = 0, 24 DEBUG_FENCE_NOTIFY, 25 }; 26 27 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 28 29 static void *i915_sw_fence_debug_hint(void *addr) 30 { 31 return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); 32 } 33 34 static struct debug_obj_descr i915_sw_fence_debug_descr = { 35 .name = "i915_sw_fence", 36 .debug_hint = i915_sw_fence_debug_hint, 37 }; 38 39 static inline void debug_fence_init(struct i915_sw_fence *fence) 40 { 41 debug_object_init(fence, &i915_sw_fence_debug_descr); 42 } 43 44 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) 45 { 46 debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); 47 } 48 49 static inline void debug_fence_activate(struct i915_sw_fence *fence) 50 { 51 debug_object_activate(fence, &i915_sw_fence_debug_descr); 52 } 53 54 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 55 int old, int new) 56 { 57 debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); 58 } 59 60 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 61 { 62 debug_object_deactivate(fence, &i915_sw_fence_debug_descr); 63 } 64 65 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 66 { 67 debug_object_destroy(fence, &i915_sw_fence_debug_descr); 68 } 69 70 static inline void debug_fence_free(struct i915_sw_fence *fence) 71 { 72 debug_object_free(fence, &i915_sw_fence_debug_descr); 73 smp_wmb(); /* flush the change in state before reallocation */ 74 } 75 76 static inline void debug_fence_assert(struct i915_sw_fence *fence) 77 { 78 debug_object_assert_init(fence, &i915_sw_fence_debug_descr); 79 } 80 81 #else 82 83 static inline void debug_fence_init(struct i915_sw_fence *fence) 84 { 85 } 86 87 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) 88 { 89 } 90 91 static inline void debug_fence_activate(struct i915_sw_fence *fence) 92 { 93 } 94 95 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 96 int old, int new) 97 { 98 } 99 100 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 101 { 102 } 103 104 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 105 { 106 } 107 108 static inline void debug_fence_free(struct i915_sw_fence *fence) 109 { 110 } 111 112 static inline void debug_fence_assert(struct i915_sw_fence *fence) 113 { 114 } 115 116 #endif 117 118 static int __i915_sw_fence_notify(struct i915_sw_fence *fence, 119 enum i915_sw_fence_notify state) 120 { 121 i915_sw_fence_notify_t fn; 122 123 fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); 124 return fn(fence, state); 125 } 126 127 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 128 void i915_sw_fence_fini(struct i915_sw_fence *fence) 129 { 130 debug_fence_free(fence); 131 } 132 #endif 133 134 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, 135 struct list_head *continuation) 136 { 137 wait_queue_head_t *x = &fence->wait; 138 wait_queue_entry_t *pos, *next; 139 unsigned long flags; 140 141 debug_fence_deactivate(fence); 142 atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ 143 144 /* 145 * To prevent unbounded recursion as we traverse the graph of 146 * i915_sw_fences, we move the entry list from this, the next ready 147 * fence, to the tail of the original fence's entry list 148 * (and so added to the list to be woken). 149 */ 150 151 spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); 152 if (continuation) { 153 list_for_each_entry_safe(pos, next, &x->head, entry) { 154 if (pos->func == autoremove_wake_function) 155 pos->func(pos, TASK_NORMAL, 0, continuation); 156 else 157 list_move_tail(&pos->entry, continuation); 158 } 159 } else { 160 LIST_HEAD(extra); 161 162 do { 163 list_for_each_entry_safe(pos, next, &x->head, entry) 164 pos->func(pos, TASK_NORMAL, 0, &extra); 165 166 if (list_empty(&extra)) 167 break; 168 169 list_splice_tail_init(&extra, &x->head); 170 } while (1); 171 } 172 spin_unlock_irqrestore(&x->lock, flags); 173 174 debug_fence_assert(fence); 175 } 176 177 static void __i915_sw_fence_complete(struct i915_sw_fence *fence, 178 struct list_head *continuation) 179 { 180 debug_fence_assert(fence); 181 182 if (!atomic_dec_and_test(&fence->pending)) 183 return; 184 185 debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); 186 187 if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) 188 return; 189 190 debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); 191 192 __i915_sw_fence_wake_up_all(fence, continuation); 193 194 debug_fence_destroy(fence); 195 __i915_sw_fence_notify(fence, FENCE_FREE); 196 } 197 198 static void i915_sw_fence_complete(struct i915_sw_fence *fence) 199 { 200 debug_fence_assert(fence); 201 202 if (WARN_ON(i915_sw_fence_done(fence))) 203 return; 204 205 __i915_sw_fence_complete(fence, NULL); 206 } 207 208 static void i915_sw_fence_await(struct i915_sw_fence *fence) 209 { 210 debug_fence_assert(fence); 211 WARN_ON(atomic_inc_return(&fence->pending) <= 1); 212 } 213 214 void __i915_sw_fence_init(struct i915_sw_fence *fence, 215 i915_sw_fence_notify_t fn, 216 const char *name, 217 struct lock_class_key *key) 218 { 219 BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); 220 221 debug_fence_init(fence); 222 223 __init_waitqueue_head(&fence->wait, name, key); 224 atomic_set(&fence->pending, 1); 225 fence->flags = (unsigned long)fn; 226 } 227 228 void i915_sw_fence_commit(struct i915_sw_fence *fence) 229 { 230 debug_fence_activate(fence); 231 i915_sw_fence_complete(fence); 232 } 233 234 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) 235 { 236 list_del(&wq->entry); 237 __i915_sw_fence_complete(wq->private, key); 238 239 if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) 240 kfree(wq); 241 return 0; 242 } 243 244 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 245 const struct i915_sw_fence * const signaler) 246 { 247 wait_queue_entry_t *wq; 248 249 if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 250 return false; 251 252 if (fence == signaler) 253 return true; 254 255 list_for_each_entry(wq, &fence->wait.head, entry) { 256 if (wq->func != i915_sw_fence_wake) 257 continue; 258 259 if (__i915_sw_fence_check_if_after(wq->private, signaler)) 260 return true; 261 } 262 263 return false; 264 } 265 266 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) 267 { 268 wait_queue_entry_t *wq; 269 270 if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 271 return; 272 273 list_for_each_entry(wq, &fence->wait.head, entry) { 274 if (wq->func != i915_sw_fence_wake) 275 continue; 276 277 __i915_sw_fence_clear_checked_bit(wq->private); 278 } 279 } 280 281 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 282 const struct i915_sw_fence * const signaler) 283 { 284 unsigned long flags; 285 bool err; 286 287 if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) 288 return false; 289 290 spin_lock_irqsave(&i915_sw_fence_lock, flags); 291 err = __i915_sw_fence_check_if_after(fence, signaler); 292 __i915_sw_fence_clear_checked_bit(fence); 293 spin_unlock_irqrestore(&i915_sw_fence_lock, flags); 294 295 return err; 296 } 297 298 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 299 struct i915_sw_fence *signaler, 300 wait_queue_entry_t *wq, gfp_t gfp) 301 { 302 unsigned long flags; 303 int pending; 304 305 debug_fence_assert(fence); 306 might_sleep_if(gfpflags_allow_blocking(gfp)); 307 308 if (i915_sw_fence_done(signaler)) 309 return 0; 310 311 debug_fence_assert(signaler); 312 313 /* The dependency graph must be acyclic. */ 314 if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) 315 return -EINVAL; 316 317 pending = 0; 318 if (!wq) { 319 wq = kmalloc(sizeof(*wq), gfp); 320 if (!wq) { 321 if (!gfpflags_allow_blocking(gfp)) 322 return -ENOMEM; 323 324 i915_sw_fence_wait(signaler); 325 return 0; 326 } 327 328 pending |= I915_SW_FENCE_FLAG_ALLOC; 329 } 330 331 INIT_LIST_HEAD(&wq->entry); 332 wq->flags = pending; 333 wq->func = i915_sw_fence_wake; 334 wq->private = fence; 335 336 i915_sw_fence_await(fence); 337 338 spin_lock_irqsave(&signaler->wait.lock, flags); 339 if (likely(!i915_sw_fence_done(signaler))) { 340 __add_wait_queue_entry_tail(&signaler->wait, wq); 341 pending = 1; 342 } else { 343 i915_sw_fence_wake(wq, 0, 0, NULL); 344 pending = 0; 345 } 346 spin_unlock_irqrestore(&signaler->wait.lock, flags); 347 348 return pending; 349 } 350 351 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 352 struct i915_sw_fence *signaler, 353 wait_queue_entry_t *wq) 354 { 355 return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); 356 } 357 358 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, 359 struct i915_sw_fence *signaler, 360 gfp_t gfp) 361 { 362 return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); 363 } 364 365 struct i915_sw_dma_fence_cb { 366 struct dma_fence_cb base; 367 struct i915_sw_fence *fence; 368 }; 369 370 struct i915_sw_dma_fence_cb_timer { 371 struct i915_sw_dma_fence_cb base; 372 struct dma_fence *dma; 373 struct timer_list timer; 374 struct irq_work work; 375 struct rcu_head rcu; 376 }; 377 378 static void dma_i915_sw_fence_wake(struct dma_fence *dma, 379 struct dma_fence_cb *data) 380 { 381 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); 382 383 i915_sw_fence_complete(cb->fence); 384 kfree(cb); 385 } 386 387 static void timer_i915_sw_fence_wake(struct timer_list *t) 388 { 389 struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); 390 struct i915_sw_fence *fence; 391 392 fence = xchg(&cb->base.fence, NULL); 393 if (!fence) 394 return; 395 396 pr_warn("asynchronous wait on fence %s:%s:%x timed out\n", 397 cb->dma->ops->get_driver_name(cb->dma), 398 cb->dma->ops->get_timeline_name(cb->dma), 399 cb->dma->seqno); 400 401 i915_sw_fence_complete(fence); 402 } 403 404 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, 405 struct dma_fence_cb *data) 406 { 407 struct i915_sw_dma_fence_cb_timer *cb = 408 container_of(data, typeof(*cb), base.base); 409 struct i915_sw_fence *fence; 410 411 fence = xchg(&cb->base.fence, NULL); 412 if (fence) 413 i915_sw_fence_complete(fence); 414 415 irq_work_queue(&cb->work); 416 } 417 418 static void irq_i915_sw_fence_work(struct irq_work *wrk) 419 { 420 struct i915_sw_dma_fence_cb_timer *cb = 421 container_of(wrk, typeof(*cb), work); 422 423 del_timer_sync(&cb->timer); 424 dma_fence_put(cb->dma); 425 426 kfree_rcu(cb, rcu); 427 } 428 429 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, 430 struct dma_fence *dma, 431 unsigned long timeout, 432 gfp_t gfp) 433 { 434 struct i915_sw_dma_fence_cb *cb; 435 dma_fence_func_t func; 436 int ret; 437 438 debug_fence_assert(fence); 439 might_sleep_if(gfpflags_allow_blocking(gfp)); 440 441 if (dma_fence_is_signaled(dma)) 442 return 0; 443 444 cb = kmalloc(timeout ? 445 sizeof(struct i915_sw_dma_fence_cb_timer) : 446 sizeof(struct i915_sw_dma_fence_cb), 447 gfp); 448 if (!cb) { 449 if (!gfpflags_allow_blocking(gfp)) 450 return -ENOMEM; 451 452 return dma_fence_wait(dma, false); 453 } 454 455 cb->fence = fence; 456 i915_sw_fence_await(fence); 457 458 func = dma_i915_sw_fence_wake; 459 if (timeout) { 460 struct i915_sw_dma_fence_cb_timer *timer = 461 container_of(cb, typeof(*timer), base); 462 463 timer->dma = dma_fence_get(dma); 464 init_irq_work(&timer->work, irq_i915_sw_fence_work); 465 466 timer_setup(&timer->timer, 467 timer_i915_sw_fence_wake, TIMER_IRQSAFE); 468 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); 469 470 func = dma_i915_sw_fence_wake_timer; 471 } 472 473 ret = dma_fence_add_callback(dma, &cb->base, func); 474 if (ret == 0) { 475 ret = 1; 476 } else { 477 func(dma, &cb->base); 478 if (ret == -ENOENT) /* fence already signaled */ 479 ret = 0; 480 } 481 482 return ret; 483 } 484 485 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, 486 struct reservation_object *resv, 487 const struct dma_fence_ops *exclude, 488 bool write, 489 unsigned long timeout, 490 gfp_t gfp) 491 { 492 struct dma_fence *excl; 493 int ret = 0, pending; 494 495 debug_fence_assert(fence); 496 might_sleep_if(gfpflags_allow_blocking(gfp)); 497 498 if (write) { 499 struct dma_fence **shared; 500 unsigned int count, i; 501 502 ret = reservation_object_get_fences_rcu(resv, 503 &excl, &count, &shared); 504 if (ret) 505 return ret; 506 507 for (i = 0; i < count; i++) { 508 if (shared[i]->ops == exclude) 509 continue; 510 511 pending = i915_sw_fence_await_dma_fence(fence, 512 shared[i], 513 timeout, 514 gfp); 515 if (pending < 0) { 516 ret = pending; 517 break; 518 } 519 520 ret |= pending; 521 } 522 523 for (i = 0; i < count; i++) 524 dma_fence_put(shared[i]); 525 kfree(shared); 526 } else { 527 excl = reservation_object_get_excl_rcu(resv); 528 } 529 530 if (ret >= 0 && excl && excl->ops != exclude) { 531 pending = i915_sw_fence_await_dma_fence(fence, 532 excl, 533 timeout, 534 gfp); 535 if (pending < 0) 536 ret = pending; 537 else 538 ret |= pending; 539 } 540 541 dma_fence_put(excl); 542 543 return ret; 544 } 545 546 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 547 #include "selftests/lib_sw_fence.c" 548 #include "selftests/i915_sw_fence.c" 549 #endif 550