1 // SPDX-License-Identifier: MIT 2 3 /* 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <kunit/test.h> 8 #include <linux/delay.h> 9 #include <linux/dma-fence.h> 10 #include <linux/dma-fence-chain.h> 11 #include <linux/kernel.h> 12 #include <linux/kthread.h> 13 #include <linux/mm.h> 14 #include <linux/sched/signal.h> 15 #include <linux/slab.h> 16 #include <linux/spinlock.h> 17 #include <linux/random.h> 18 19 #define CHAIN_SZ (4 << 10) 20 21 static struct kmem_cache *slab_fences; 22 23 static inline struct mock_fence { 24 struct dma_fence base; 25 spinlock_t lock; 26 } *to_mock_fence(struct dma_fence *f) { 27 return container_of(f, struct mock_fence, base); 28 } 29 30 static const char *mock_name(struct dma_fence *f) 31 { 32 return "mock"; 33 } 34 35 static void mock_fence_release(struct dma_fence *f) 36 { 37 kmem_cache_free(slab_fences, to_mock_fence(f)); 38 } 39 40 static const struct dma_fence_ops mock_ops = { 41 .get_driver_name = mock_name, 42 .get_timeline_name = mock_name, 43 .release = mock_fence_release, 44 }; 45 46 static struct dma_fence *mock_fence(void) 47 { 48 struct mock_fence *f; 49 50 f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 51 if (!f) 52 return NULL; 53 54 spin_lock_init(&f->lock); 55 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 56 57 return &f->base; 58 } 59 60 static struct dma_fence *mock_chain(struct dma_fence *prev, 61 struct dma_fence *fence, 62 u64 seqno) 63 { 64 struct dma_fence_chain *f; 65 66 f = dma_fence_chain_alloc(); 67 if (!f) 68 return NULL; 69 70 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence), 71 seqno); 72 73 return &f->base; 74 } 75 76 static void test_sanitycheck(struct kunit *test) 77 { 78 struct dma_fence *f, *chain; 79 80 f = mock_fence(); 81 KUNIT_ASSERT_NOT_NULL(test, f); 82 83 chain = mock_chain(NULL, f, 1); 84 if (chain) 85 dma_fence_enable_sw_signaling(chain); 86 else 87 KUNIT_FAIL(test, "Failed to create chain"); 88 89 dma_fence_signal(f); 90 dma_fence_put(f); 91 92 dma_fence_put(chain); 93 } 94 95 struct fence_chains { 96 unsigned int chain_length; 97 struct dma_fence **fences; 98 struct dma_fence **chains; 99 100 struct dma_fence *tail; 101 }; 102 103 static uint64_t seqno_inc(unsigned int i) 104 { 105 return i + 1; 106 } 107 108 static int fence_chains_init(struct fence_chains *fc, unsigned int count, 109 uint64_t (*seqno_fn)(unsigned int)) 110 { 111 unsigned int i; 112 int err = 0; 113 114 fc->chains = kvmalloc_objs(*fc->chains, count, GFP_KERNEL | __GFP_ZERO); 115 if (!fc->chains) 116 return -ENOMEM; 117 118 fc->fences = kvmalloc_objs(*fc->fences, count, GFP_KERNEL | __GFP_ZERO); 119 if (!fc->fences) { 120 err = -ENOMEM; 121 goto err_chains; 122 } 123 124 fc->tail = NULL; 125 for (i = 0; i < count; i++) { 126 fc->fences[i] = mock_fence(); 127 if (!fc->fences[i]) { 128 err = -ENOMEM; 129 goto unwind; 130 } 131 132 fc->chains[i] = mock_chain(fc->tail, 133 fc->fences[i], 134 seqno_fn(i)); 135 if (!fc->chains[i]) { 136 err = -ENOMEM; 137 goto unwind; 138 } 139 140 fc->tail = fc->chains[i]; 141 142 dma_fence_enable_sw_signaling(fc->chains[i]); 143 } 144 145 fc->chain_length = i; 146 return 0; 147 148 unwind: 149 for (i = 0; i < count; i++) { 150 dma_fence_put(fc->fences[i]); 151 dma_fence_put(fc->chains[i]); 152 } 153 kvfree(fc->fences); 154 err_chains: 155 kvfree(fc->chains); 156 return err; 157 } 158 159 static void fence_chains_fini(struct fence_chains *fc) 160 { 161 unsigned int i; 162 163 for (i = 0; i < fc->chain_length; i++) { 164 dma_fence_signal(fc->fences[i]); 165 dma_fence_put(fc->fences[i]); 166 } 167 kvfree(fc->fences); 168 169 for (i = 0; i < fc->chain_length; i++) 170 dma_fence_put(fc->chains[i]); 171 kvfree(fc->chains); 172 } 173 174 static void test_find_seqno(struct kunit *test) 175 { 176 struct fence_chains fc; 177 struct dma_fence *fence; 178 int err; 179 int i; 180 181 err = fence_chains_init(&fc, 64, seqno_inc); 182 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 183 184 fence = dma_fence_get(fc.tail); 185 err = dma_fence_chain_find_seqno(&fence, 0); 186 dma_fence_put(fence); 187 if (err) { 188 KUNIT_FAIL(test, "Reported %d for find_seqno(0)!", err); 189 goto err; 190 } 191 192 for (i = 0; i < fc.chain_length; i++) { 193 fence = dma_fence_get(fc.tail); 194 err = dma_fence_chain_find_seqno(&fence, i + 1); 195 dma_fence_put(fence); 196 if (err) { 197 KUNIT_FAIL(test, "Reported %d for find_seqno(%d:%d)!", 198 err, fc.chain_length + 1, i + 1); 199 goto err; 200 } 201 if (fence != fc.chains[i]) { 202 KUNIT_FAIL(test, "Incorrect fence reported by find_seqno(%d:%d)", 203 fc.chain_length + 1, i + 1); 204 goto err; 205 } 206 207 dma_fence_get(fence); 208 err = dma_fence_chain_find_seqno(&fence, i + 1); 209 dma_fence_put(fence); 210 if (err) { 211 KUNIT_FAIL(test, "Error reported for finding self"); 212 goto err; 213 } 214 if (fence != fc.chains[i]) { 215 KUNIT_FAIL(test, "Incorrect fence reported by find self"); 216 goto err; 217 } 218 219 dma_fence_get(fence); 220 err = dma_fence_chain_find_seqno(&fence, i + 2); 221 dma_fence_put(fence); 222 if (!err) { 223 KUNIT_FAIL(test, "Error not reported for future fence: find_seqno(%d:%d)!", 224 i + 1, i + 2); 225 goto err; 226 } 227 228 dma_fence_get(fence); 229 err = dma_fence_chain_find_seqno(&fence, i); 230 dma_fence_put(fence); 231 if (err) { 232 KUNIT_FAIL(test, "Error reported for previous fence!"); 233 goto err; 234 } 235 if (i > 0 && fence != fc.chains[i - 1]) { 236 KUNIT_FAIL(test, "Incorrect fence reported by find_seqno(%d:%d)", 237 i + 1, i); 238 goto err; 239 } 240 } 241 242 err: 243 fence_chains_fini(&fc); 244 } 245 246 static void test_find_signaled(struct kunit *test) 247 { 248 struct fence_chains fc; 249 struct dma_fence *fence; 250 int err; 251 252 err = fence_chains_init(&fc, 2, seqno_inc); 253 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 254 255 dma_fence_signal(fc.fences[0]); 256 257 fence = dma_fence_get(fc.tail); 258 err = dma_fence_chain_find_seqno(&fence, 1); 259 dma_fence_put(fence); 260 if (err) { 261 KUNIT_FAIL(test, "Reported %d for find_seqno()!", err); 262 goto err; 263 } 264 265 if (fence && fence != fc.chains[0]) { 266 KUNIT_FAIL(test, "Incorrect chain-fence.seqno:%lld reported for completed seqno:1", 267 fence->seqno); 268 269 dma_fence_get(fence); 270 err = dma_fence_chain_find_seqno(&fence, 1); 271 dma_fence_put(fence); 272 if (err) 273 KUNIT_FAIL(test, "Reported %d for finding self!", err); 274 } 275 276 err: 277 fence_chains_fini(&fc); 278 } 279 280 static void test_find_out_of_order(struct kunit *test) 281 { 282 struct fence_chains fc; 283 struct dma_fence *fence; 284 int err; 285 286 err = fence_chains_init(&fc, 3, seqno_inc); 287 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 288 289 dma_fence_signal(fc.fences[1]); 290 291 fence = dma_fence_get(fc.tail); 292 err = dma_fence_chain_find_seqno(&fence, 2); 293 dma_fence_put(fence); 294 if (err) { 295 KUNIT_FAIL(test, "Reported %d for find_seqno()!", err); 296 goto err; 297 } 298 299 /* 300 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior 301 * of the dma-fence-chain is to make us wait for all the fences up to 302 * the point we want. Since fence 1 is still not signaled, this what 303 * we should get as fence to wait upon (fence 2 being garbage 304 * collected during the traversal of the chain). 305 */ 306 if (fence != fc.chains[0]) 307 KUNIT_FAIL(test, "Incorrect chain-fence.seqno:%lld reported for completed seqno:2", 308 fence ? fence->seqno : 0); 309 310 err: 311 fence_chains_fini(&fc); 312 } 313 314 static uint64_t seqno_inc2(unsigned int i) 315 { 316 return 2 * i + 2; 317 } 318 319 static void test_find_gap(struct kunit *test) 320 { 321 struct fence_chains fc; 322 struct dma_fence *fence; 323 int err; 324 int i; 325 326 err = fence_chains_init(&fc, 64, seqno_inc2); 327 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 328 329 for (i = 0; i < fc.chain_length; i++) { 330 fence = dma_fence_get(fc.tail); 331 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 332 dma_fence_put(fence); 333 if (err) { 334 KUNIT_FAIL(test, "Reported %d for find_seqno(%d:%d)!", 335 err, fc.chain_length + 1, 2 * i + 1); 336 goto err; 337 } 338 if (fence != fc.chains[i]) { 339 KUNIT_FAIL(test, "Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)", 340 fence->seqno, 341 fc.chain_length + 1, 342 2 * i + 1); 343 goto err; 344 } 345 346 dma_fence_get(fence); 347 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 348 dma_fence_put(fence); 349 if (err) { 350 KUNIT_FAIL(test, "Error reported for finding self"); 351 goto err; 352 } 353 if (fence != fc.chains[i]) { 354 KUNIT_FAIL(test, "Incorrect fence reported by find self"); 355 goto err; 356 } 357 } 358 359 err: 360 fence_chains_fini(&fc); 361 } 362 363 struct find_race { 364 struct fence_chains fc; 365 atomic_t children; 366 }; 367 368 static int __find_race(void *arg) 369 { 370 struct find_race *data = arg; 371 int err = 0; 372 373 while (!kthread_should_stop()) { 374 struct dma_fence *fence = dma_fence_get(data->fc.tail); 375 int seqno; 376 377 seqno = get_random_u32_inclusive(1, data->fc.chain_length); 378 379 err = dma_fence_chain_find_seqno(&fence, seqno); 380 if (err) { 381 pr_err("Failed to find fence seqno:%d\n", 382 seqno); 383 dma_fence_put(fence); 384 break; 385 } 386 if (!fence) 387 goto signal; 388 389 /* 390 * We can only find ourselves if we are on fence we were 391 * looking for. 392 */ 393 if (fence->seqno == seqno) { 394 err = dma_fence_chain_find_seqno(&fence, seqno); 395 if (err) { 396 pr_err("Reported an invalid fence for find-self:%d\n", 397 seqno); 398 dma_fence_put(fence); 399 break; 400 } 401 } 402 403 dma_fence_put(fence); 404 405 signal: 406 seqno = get_random_u32_below(data->fc.chain_length - 1); 407 dma_fence_signal(data->fc.fences[seqno]); 408 cond_resched(); 409 } 410 411 if (atomic_dec_and_test(&data->children)) 412 wake_up_var(&data->children); 413 return err; 414 } 415 416 static void test_find_race(struct kunit *test) 417 { 418 struct find_race data; 419 int ncpus = num_online_cpus(); 420 struct task_struct **threads; 421 unsigned long count; 422 int err; 423 int i; 424 425 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 426 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 427 428 threads = kmalloc_objs(*threads, ncpus); 429 if (!threads) { 430 KUNIT_FAIL(test, "Failed to allocate threads array"); 431 goto err; 432 } 433 434 atomic_set(&data.children, 0); 435 for (i = 0; i < ncpus; i++) { 436 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 437 if (IS_ERR(threads[i])) { 438 ncpus = i; 439 break; 440 } 441 atomic_inc(&data.children); 442 get_task_struct(threads[i]); 443 } 444 445 wait_var_event_timeout(&data.children, 446 !atomic_read(&data.children), 447 5 * HZ); 448 449 for (i = 0; i < ncpus; i++) { 450 int ret; 451 452 ret = kthread_stop_put(threads[i]); 453 if (ret && !err) 454 err = ret; 455 } 456 kfree(threads); 457 458 count = 0; 459 for (i = 0; i < data.fc.chain_length; i++) 460 if (dma_fence_is_signaled(data.fc.fences[i])) 461 count++; 462 pr_info("Completed %lu cycles\n", count); 463 464 KUNIT_EXPECT_EQ(test, err, 0); 465 466 err: 467 fence_chains_fini(&data.fc); 468 } 469 470 static void test_signal_forward(struct kunit *test) 471 { 472 struct fence_chains fc; 473 int err; 474 int i; 475 476 err = fence_chains_init(&fc, 64, seqno_inc); 477 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 478 479 for (i = 0; i < fc.chain_length; i++) { 480 dma_fence_signal(fc.fences[i]); 481 482 if (!dma_fence_is_signaled(fc.chains[i])) { 483 KUNIT_FAIL(test, "chain[%d] not signaled!", i); 484 goto err; 485 } 486 487 if (i + 1 < fc.chain_length && 488 dma_fence_is_signaled(fc.chains[i + 1])) { 489 KUNIT_FAIL(test, "chain[%d] is signaled!", i); 490 goto err; 491 } 492 } 493 494 err: 495 fence_chains_fini(&fc); 496 } 497 498 static void test_signal_backward(struct kunit *test) 499 { 500 struct fence_chains fc; 501 int err; 502 int i; 503 504 err = fence_chains_init(&fc, 64, seqno_inc); 505 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 506 507 for (i = fc.chain_length; i--; ) { 508 dma_fence_signal(fc.fences[i]); 509 510 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 511 KUNIT_FAIL(test, "chain[%d] is signaled!", i); 512 goto err; 513 } 514 } 515 516 for (i = 0; i < fc.chain_length; i++) { 517 if (!dma_fence_is_signaled(fc.chains[i])) { 518 KUNIT_FAIL(test, "chain[%d] was not signaled!", i); 519 goto err; 520 } 521 } 522 523 err: 524 fence_chains_fini(&fc); 525 } 526 527 static int __wait_fence_chains(void *arg) 528 { 529 struct fence_chains *fc = arg; 530 531 if (dma_fence_wait(fc->tail, false)) 532 return -EIO; 533 534 return 0; 535 } 536 537 static void test_wait_forward(struct kunit *test) 538 { 539 struct fence_chains fc; 540 struct task_struct *tsk; 541 int err; 542 int i; 543 544 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 545 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 546 547 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 548 if (IS_ERR(tsk)) { 549 KUNIT_FAIL(test, "Failed to create kthread"); 550 goto err; 551 } 552 get_task_struct(tsk); 553 yield_to(tsk, true); 554 555 for (i = 0; i < fc.chain_length; i++) 556 dma_fence_signal(fc.fences[i]); 557 558 err = kthread_stop_put(tsk); 559 KUNIT_EXPECT_EQ(test, err, 0); 560 561 err: 562 fence_chains_fini(&fc); 563 } 564 565 static void test_wait_backward(struct kunit *test) 566 { 567 struct fence_chains fc; 568 struct task_struct *tsk; 569 int err; 570 int i; 571 572 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 573 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 574 575 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 576 if (IS_ERR(tsk)) { 577 KUNIT_FAIL(test, "Failed to create kthread"); 578 goto err; 579 } 580 get_task_struct(tsk); 581 yield_to(tsk, true); 582 583 for (i = fc.chain_length; i--; ) 584 dma_fence_signal(fc.fences[i]); 585 586 err = kthread_stop_put(tsk); 587 KUNIT_EXPECT_EQ(test, err, 0); 588 589 err: 590 fence_chains_fini(&fc); 591 } 592 593 static void randomise_fences(struct fence_chains *fc) 594 { 595 unsigned int count = fc->chain_length; 596 597 /* Fisher-Yates shuffle courtesy of Knuth */ 598 while (--count) { 599 unsigned int swp; 600 601 swp = get_random_u32_below(count + 1); 602 if (swp == count) 603 continue; 604 605 swap(fc->fences[count], fc->fences[swp]); 606 } 607 } 608 609 static void test_wait_random(struct kunit *test) 610 { 611 struct fence_chains fc; 612 struct task_struct *tsk; 613 int err; 614 int i; 615 616 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 617 KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains"); 618 619 randomise_fences(&fc); 620 621 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 622 if (IS_ERR(tsk)) { 623 KUNIT_FAIL(test, "Failed to create kthread"); 624 goto err; 625 } 626 get_task_struct(tsk); 627 yield_to(tsk, true); 628 629 for (i = 0; i < fc.chain_length; i++) 630 dma_fence_signal(fc.fences[i]); 631 632 err = kthread_stop_put(tsk); 633 KUNIT_EXPECT_EQ(test, err, 0); 634 635 err: 636 fence_chains_fini(&fc); 637 } 638 639 static int dma_fence_chain_suite_init(struct kunit_suite *suite) 640 { 641 pr_info("sizeof(dma_fence_chain)=%zu\n", 642 sizeof(struct dma_fence_chain)); 643 644 slab_fences = KMEM_CACHE(mock_fence, 645 SLAB_TYPESAFE_BY_RCU | 646 SLAB_HWCACHE_ALIGN); 647 if (!slab_fences) 648 return -ENOMEM; 649 return 0; 650 } 651 652 static void dma_fence_chain_suite_exit(struct kunit_suite *suite) 653 { 654 kmem_cache_destroy(slab_fences); 655 } 656 657 static struct kunit_case dma_fence_chain_cases[] = { 658 KUNIT_CASE(test_sanitycheck), 659 KUNIT_CASE(test_find_seqno), 660 KUNIT_CASE(test_find_signaled), 661 KUNIT_CASE(test_find_out_of_order), 662 KUNIT_CASE(test_find_gap), 663 KUNIT_CASE(test_find_race), 664 KUNIT_CASE(test_signal_forward), 665 KUNIT_CASE(test_signal_backward), 666 KUNIT_CASE(test_wait_forward), 667 KUNIT_CASE(test_wait_backward), 668 KUNIT_CASE(test_wait_random), 669 {} 670 }; 671 672 static struct kunit_suite dma_fence_chain_test_suite = { 673 .name = "dma-buf-fence-chain", 674 .suite_init = dma_fence_chain_suite_init, 675 .suite_exit = dma_fence_chain_suite_exit, 676 .test_cases = dma_fence_chain_cases, 677 }; 678 679 kunit_test_suite(dma_fence_chain_test_suite); 680