1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * (C) Copyright 2016 Intel Corporation 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/dma-fence.h> 9 #include <linux/irq_work.h> 10 #include <linux/dma-resv.h> 11 12 #include "i915_sw_fence.h" 13 #include "i915_selftest.h" 14 15 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) 16 #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) 17 #else 18 #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) 19 #endif 20 21 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ 22 23 static DEFINE_SPINLOCK(i915_sw_fence_lock); 24 25 enum { 26 DEBUG_FENCE_IDLE = 0, 27 DEBUG_FENCE_NOTIFY, 28 }; 29 30 static void *i915_sw_fence_debug_hint(void *addr) 31 { 32 return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); 33 } 34 35 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 36 37 static const struct debug_obj_descr i915_sw_fence_debug_descr = { 38 .name = "i915_sw_fence", 39 .debug_hint = i915_sw_fence_debug_hint, 40 }; 41 42 static inline void debug_fence_init(struct i915_sw_fence *fence) 43 { 44 debug_object_init(fence, &i915_sw_fence_debug_descr); 45 } 46 47 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) 48 { 49 debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); 50 } 51 52 static inline void debug_fence_activate(struct i915_sw_fence *fence) 53 { 54 debug_object_activate(fence, &i915_sw_fence_debug_descr); 55 } 56 57 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 58 int old, int new) 59 { 60 debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); 61 } 62 63 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 64 { 65 debug_object_deactivate(fence, &i915_sw_fence_debug_descr); 66 } 67 68 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 69 { 70 debug_object_destroy(fence, &i915_sw_fence_debug_descr); 71 } 72 73 static inline void debug_fence_free(struct i915_sw_fence *fence) 74 { 75 debug_object_free(fence, &i915_sw_fence_debug_descr); 76 smp_wmb(); /* flush the change in state before reallocation */ 77 } 78 79 static inline void debug_fence_assert(struct i915_sw_fence *fence) 80 { 81 debug_object_assert_init(fence, &i915_sw_fence_debug_descr); 82 } 83 84 #else 85 86 static inline void debug_fence_init(struct i915_sw_fence *fence) 87 { 88 } 89 90 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) 91 { 92 } 93 94 static inline void debug_fence_activate(struct i915_sw_fence *fence) 95 { 96 } 97 98 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 99 int old, int new) 100 { 101 } 102 103 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 104 { 105 } 106 107 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 108 { 109 } 110 111 static inline void debug_fence_free(struct i915_sw_fence *fence) 112 { 113 } 114 115 static inline void debug_fence_assert(struct i915_sw_fence *fence) 116 { 117 } 118 119 #endif 120 121 static int __i915_sw_fence_notify(struct i915_sw_fence *fence, 122 enum i915_sw_fence_notify state) 123 { 124 i915_sw_fence_notify_t fn; 125 126 fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); 127 return fn(fence, state); 128 } 129 130 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 131 void i915_sw_fence_fini(struct i915_sw_fence *fence) 132 { 133 debug_fence_free(fence); 134 } 135 #endif 136 137 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, 138 struct list_head *continuation) 139 { 140 wait_queue_head_t *x = &fence->wait; 141 wait_queue_entry_t *pos, *next; 142 unsigned long flags; 143 144 debug_fence_deactivate(fence); 145 atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ 146 147 /* 148 * To prevent unbounded recursion as we traverse the graph of 149 * i915_sw_fences, we move the entry list from this, the next ready 150 * fence, to the tail of the original fence's entry list 151 * (and so added to the list to be woken). 152 */ 153 154 spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); 155 if (continuation) { 156 list_for_each_entry_safe(pos, next, &x->head, entry) { 157 if (pos->func == autoremove_wake_function) 158 pos->func(pos, TASK_NORMAL, 0, continuation); 159 else 160 list_move_tail(&pos->entry, continuation); 161 } 162 } else { 163 LIST_HEAD(extra); 164 165 do { 166 list_for_each_entry_safe(pos, next, &x->head, entry) { 167 int wake_flags; 168 169 wake_flags = fence->error; 170 if (pos->func == autoremove_wake_function) 171 wake_flags = 0; 172 173 pos->func(pos, TASK_NORMAL, wake_flags, &extra); 174 } 175 176 if (list_empty(&extra)) 177 break; 178 179 list_splice_tail_init(&extra, &x->head); 180 } while (1); 181 } 182 spin_unlock_irqrestore(&x->lock, flags); 183 184 debug_fence_assert(fence); 185 } 186 187 static void __i915_sw_fence_complete(struct i915_sw_fence *fence, 188 struct list_head *continuation) 189 { 190 debug_fence_assert(fence); 191 192 if (!atomic_dec_and_test(&fence->pending)) 193 return; 194 195 debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); 196 197 if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) 198 return; 199 200 debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); 201 202 __i915_sw_fence_wake_up_all(fence, continuation); 203 204 debug_fence_destroy(fence); 205 __i915_sw_fence_notify(fence, FENCE_FREE); 206 } 207 208 void i915_sw_fence_complete(struct i915_sw_fence *fence) 209 { 210 debug_fence_assert(fence); 211 212 if (WARN_ON(i915_sw_fence_done(fence))) 213 return; 214 215 __i915_sw_fence_complete(fence, NULL); 216 } 217 218 bool i915_sw_fence_await(struct i915_sw_fence *fence) 219 { 220 int pending; 221 222 /* 223 * It is only safe to add a new await to the fence while it has 224 * not yet been signaled (i.e. there are still existing signalers). 225 */ 226 pending = atomic_read(&fence->pending); 227 do { 228 if (pending < 1) 229 return false; 230 } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); 231 232 return true; 233 } 234 235 void __i915_sw_fence_init(struct i915_sw_fence *fence, 236 i915_sw_fence_notify_t fn, 237 const char *name, 238 struct lock_class_key *key) 239 { 240 BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); 241 242 __init_waitqueue_head(&fence->wait, name, key); 243 fence->flags = (unsigned long)fn; 244 245 i915_sw_fence_reinit(fence); 246 } 247 248 void i915_sw_fence_reinit(struct i915_sw_fence *fence) 249 { 250 debug_fence_init(fence); 251 252 atomic_set(&fence->pending, 1); 253 fence->error = 0; 254 255 I915_SW_FENCE_BUG_ON(!fence->flags); 256 I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); 257 } 258 259 void i915_sw_fence_commit(struct i915_sw_fence *fence) 260 { 261 debug_fence_activate(fence); 262 i915_sw_fence_complete(fence); 263 } 264 265 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) 266 { 267 i915_sw_fence_set_error_once(wq->private, flags); 268 269 list_del(&wq->entry); 270 __i915_sw_fence_complete(wq->private, key); 271 272 if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) 273 kfree(wq); 274 return 0; 275 } 276 277 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 278 const struct i915_sw_fence * const signaler) 279 { 280 wait_queue_entry_t *wq; 281 282 if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 283 return false; 284 285 if (fence == signaler) 286 return true; 287 288 list_for_each_entry(wq, &fence->wait.head, entry) { 289 if (wq->func != i915_sw_fence_wake) 290 continue; 291 292 if (__i915_sw_fence_check_if_after(wq->private, signaler)) 293 return true; 294 } 295 296 return false; 297 } 298 299 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) 300 { 301 wait_queue_entry_t *wq; 302 303 if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 304 return; 305 306 list_for_each_entry(wq, &fence->wait.head, entry) { 307 if (wq->func != i915_sw_fence_wake) 308 continue; 309 310 __i915_sw_fence_clear_checked_bit(wq->private); 311 } 312 } 313 314 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 315 const struct i915_sw_fence * const signaler) 316 { 317 unsigned long flags; 318 bool err; 319 320 if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) 321 return false; 322 323 spin_lock_irqsave(&i915_sw_fence_lock, flags); 324 err = __i915_sw_fence_check_if_after(fence, signaler); 325 __i915_sw_fence_clear_checked_bit(fence); 326 spin_unlock_irqrestore(&i915_sw_fence_lock, flags); 327 328 return err; 329 } 330 331 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 332 struct i915_sw_fence *signaler, 333 wait_queue_entry_t *wq, gfp_t gfp) 334 { 335 unsigned long flags; 336 int pending; 337 338 debug_fence_assert(fence); 339 might_sleep_if(gfpflags_allow_blocking(gfp)); 340 341 if (i915_sw_fence_done(signaler)) { 342 i915_sw_fence_set_error_once(fence, signaler->error); 343 return 0; 344 } 345 346 debug_fence_assert(signaler); 347 348 /* The dependency graph must be acyclic. */ 349 if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) 350 return -EINVAL; 351 352 pending = 0; 353 if (!wq) { 354 wq = kmalloc(sizeof(*wq), gfp); 355 if (!wq) { 356 if (!gfpflags_allow_blocking(gfp)) 357 return -ENOMEM; 358 359 i915_sw_fence_wait(signaler); 360 i915_sw_fence_set_error_once(fence, signaler->error); 361 return 0; 362 } 363 364 pending |= I915_SW_FENCE_FLAG_ALLOC; 365 } 366 367 INIT_LIST_HEAD(&wq->entry); 368 wq->flags = pending; 369 wq->func = i915_sw_fence_wake; 370 wq->private = fence; 371 372 i915_sw_fence_await(fence); 373 374 spin_lock_irqsave(&signaler->wait.lock, flags); 375 if (likely(!i915_sw_fence_done(signaler))) { 376 __add_wait_queue_entry_tail(&signaler->wait, wq); 377 pending = 1; 378 } else { 379 i915_sw_fence_wake(wq, 0, signaler->error, NULL); 380 pending = 0; 381 } 382 spin_unlock_irqrestore(&signaler->wait.lock, flags); 383 384 return pending; 385 } 386 387 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 388 struct i915_sw_fence *signaler, 389 wait_queue_entry_t *wq) 390 { 391 return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); 392 } 393 394 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, 395 struct i915_sw_fence *signaler, 396 gfp_t gfp) 397 { 398 return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); 399 } 400 401 struct i915_sw_dma_fence_cb_timer { 402 struct i915_sw_dma_fence_cb base; 403 struct dma_fence *dma; 404 struct timer_list timer; 405 struct irq_work work; 406 struct rcu_head rcu; 407 }; 408 409 static void dma_i915_sw_fence_wake(struct dma_fence *dma, 410 struct dma_fence_cb *data) 411 { 412 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); 413 414 i915_sw_fence_set_error_once(cb->fence, dma->error); 415 i915_sw_fence_complete(cb->fence); 416 kfree(cb); 417 } 418 419 static void timer_i915_sw_fence_wake(struct timer_list *t) 420 { 421 struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); 422 struct i915_sw_fence *fence; 423 424 fence = xchg(&cb->base.fence, NULL); 425 if (!fence) 426 return; 427 428 pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", 429 cb->dma->ops->get_driver_name(cb->dma), 430 cb->dma->ops->get_timeline_name(cb->dma), 431 cb->dma->seqno, 432 i915_sw_fence_debug_hint(fence)); 433 434 i915_sw_fence_set_error_once(fence, -ETIMEDOUT); 435 i915_sw_fence_complete(fence); 436 } 437 438 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, 439 struct dma_fence_cb *data) 440 { 441 struct i915_sw_dma_fence_cb_timer *cb = 442 container_of(data, typeof(*cb), base.base); 443 struct i915_sw_fence *fence; 444 445 fence = xchg(&cb->base.fence, NULL); 446 if (fence) { 447 i915_sw_fence_set_error_once(fence, dma->error); 448 i915_sw_fence_complete(fence); 449 } 450 451 irq_work_queue(&cb->work); 452 } 453 454 static void irq_i915_sw_fence_work(struct irq_work *wrk) 455 { 456 struct i915_sw_dma_fence_cb_timer *cb = 457 container_of(wrk, typeof(*cb), work); 458 459 del_timer_sync(&cb->timer); 460 dma_fence_put(cb->dma); 461 462 kfree_rcu(cb, rcu); 463 } 464 465 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, 466 struct dma_fence *dma, 467 unsigned long timeout, 468 gfp_t gfp) 469 { 470 struct i915_sw_dma_fence_cb *cb; 471 dma_fence_func_t func; 472 int ret; 473 474 debug_fence_assert(fence); 475 might_sleep_if(gfpflags_allow_blocking(gfp)); 476 477 if (dma_fence_is_signaled(dma)) { 478 i915_sw_fence_set_error_once(fence, dma->error); 479 return 0; 480 } 481 482 cb = kmalloc(timeout ? 483 sizeof(struct i915_sw_dma_fence_cb_timer) : 484 sizeof(struct i915_sw_dma_fence_cb), 485 gfp); 486 if (!cb) { 487 if (!gfpflags_allow_blocking(gfp)) 488 return -ENOMEM; 489 490 ret = dma_fence_wait(dma, false); 491 if (ret) 492 return ret; 493 494 i915_sw_fence_set_error_once(fence, dma->error); 495 return 0; 496 } 497 498 cb->fence = fence; 499 i915_sw_fence_await(fence); 500 501 func = dma_i915_sw_fence_wake; 502 if (timeout) { 503 struct i915_sw_dma_fence_cb_timer *timer = 504 container_of(cb, typeof(*timer), base); 505 506 timer->dma = dma_fence_get(dma); 507 init_irq_work(&timer->work, irq_i915_sw_fence_work); 508 509 timer_setup(&timer->timer, 510 timer_i915_sw_fence_wake, TIMER_IRQSAFE); 511 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); 512 513 func = dma_i915_sw_fence_wake_timer; 514 } 515 516 ret = dma_fence_add_callback(dma, &cb->base, func); 517 if (ret == 0) { 518 ret = 1; 519 } else { 520 func(dma, &cb->base); 521 if (ret == -ENOENT) /* fence already signaled */ 522 ret = 0; 523 } 524 525 return ret; 526 } 527 528 static void __dma_i915_sw_fence_wake(struct dma_fence *dma, 529 struct dma_fence_cb *data) 530 { 531 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); 532 533 i915_sw_fence_set_error_once(cb->fence, dma->error); 534 i915_sw_fence_complete(cb->fence); 535 } 536 537 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, 538 struct dma_fence *dma, 539 struct i915_sw_dma_fence_cb *cb) 540 { 541 int ret; 542 543 debug_fence_assert(fence); 544 545 if (dma_fence_is_signaled(dma)) { 546 i915_sw_fence_set_error_once(fence, dma->error); 547 return 0; 548 } 549 550 cb->fence = fence; 551 i915_sw_fence_await(fence); 552 553 ret = 1; 554 if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { 555 /* fence already signaled */ 556 __dma_i915_sw_fence_wake(dma, &cb->base); 557 ret = 0; 558 } 559 560 return ret; 561 } 562 563 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, 564 struct dma_resv *resv, 565 const struct dma_fence_ops *exclude, 566 bool write, 567 unsigned long timeout, 568 gfp_t gfp) 569 { 570 struct dma_fence *excl; 571 int ret = 0, pending; 572 573 debug_fence_assert(fence); 574 might_sleep_if(gfpflags_allow_blocking(gfp)); 575 576 if (write) { 577 struct dma_fence **shared; 578 unsigned int count, i; 579 580 ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared); 581 if (ret) 582 return ret; 583 584 for (i = 0; i < count; i++) { 585 if (shared[i]->ops == exclude) 586 continue; 587 588 pending = i915_sw_fence_await_dma_fence(fence, 589 shared[i], 590 timeout, 591 gfp); 592 if (pending < 0) { 593 ret = pending; 594 break; 595 } 596 597 ret |= pending; 598 } 599 600 for (i = 0; i < count; i++) 601 dma_fence_put(shared[i]); 602 kfree(shared); 603 } else { 604 excl = dma_resv_get_excl_rcu(resv); 605 } 606 607 if (ret >= 0 && excl && excl->ops != exclude) { 608 pending = i915_sw_fence_await_dma_fence(fence, 609 excl, 610 timeout, 611 gfp); 612 if (pending < 0) 613 ret = pending; 614 else 615 ret |= pending; 616 } 617 618 dma_fence_put(excl); 619 620 return ret; 621 } 622 623 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 624 #include "selftests/lib_sw_fence.c" 625 #include "selftests/i915_sw_fence.c" 626 #endif 627