1 /* SPDX-License-Identifier: MIT */ 2 3 /* 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/dma-fence.h> 9 #include <linux/kernel.h> 10 #include <linux/kthread.h> 11 #include <linux/sched/signal.h> 12 #include <linux/slab.h> 13 #include <linux/spinlock.h> 14 15 #include "selftest.h" 16 17 static const char *mock_name(struct dma_fence *f) 18 { 19 return "mock"; 20 } 21 22 static const struct dma_fence_ops mock_ops = { 23 .get_driver_name = mock_name, 24 .get_timeline_name = mock_name, 25 }; 26 27 static struct dma_fence *mock_fence(void) 28 { 29 struct dma_fence *f; 30 31 f = kmalloc(sizeof(*f), GFP_KERNEL); 32 if (!f) 33 return NULL; 34 35 dma_fence_init(f, &mock_ops, NULL, 0, 0); 36 return f; 37 } 38 39 static int sanitycheck(void *arg) 40 { 41 struct dma_fence *f; 42 43 f = mock_fence(); 44 if (!f) 45 return -ENOMEM; 46 47 dma_fence_enable_sw_signaling(f); 48 49 dma_fence_signal(f); 50 dma_fence_put(f); 51 52 return 0; 53 } 54 55 static int test_signaling(void *arg) 56 { 57 struct dma_fence *f; 58 int err = -EINVAL; 59 60 f = mock_fence(); 61 if (!f) 62 return -ENOMEM; 63 64 dma_fence_enable_sw_signaling(f); 65 66 if (dma_fence_is_signaled(f)) { 67 pr_err("Fence unexpectedly signaled on creation\n"); 68 goto err_free; 69 } 70 71 if (dma_fence_check_and_signal(f)) { 72 pr_err("Fence reported being already signaled\n"); 73 goto err_free; 74 } 75 76 if (!dma_fence_is_signaled(f)) { 77 pr_err("Fence not reporting signaled\n"); 78 goto err_free; 79 } 80 81 if (!dma_fence_test_signaled_flag(f)) { 82 pr_err("Fence reported not being already signaled\n"); 83 goto err_free; 84 } 85 86 if (rcu_dereference_protected(f->ops, true)) { 87 pr_err("Fence ops not cleared on signal\n"); 88 goto err_free; 89 } 90 91 err = 0; 92 err_free: 93 dma_fence_put(f); 94 return err; 95 } 96 97 struct simple_cb { 98 struct dma_fence_cb cb; 99 bool seen; 100 }; 101 102 static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb) 103 { 104 smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true); 105 } 106 107 static int test_add_callback(void *arg) 108 { 109 struct simple_cb cb = {}; 110 struct dma_fence *f; 111 int err = -EINVAL; 112 113 f = mock_fence(); 114 if (!f) 115 return -ENOMEM; 116 117 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) { 118 pr_err("Failed to add callback, fence already signaled!\n"); 119 goto err_free; 120 } 121 122 dma_fence_signal(f); 123 if (!cb.seen) { 124 pr_err("Callback failed!\n"); 125 goto err_free; 126 } 127 128 err = 0; 129 err_free: 130 dma_fence_put(f); 131 return err; 132 } 133 134 static int test_late_add_callback(void *arg) 135 { 136 struct simple_cb cb = {}; 137 struct dma_fence *f; 138 int err = -EINVAL; 139 140 f = mock_fence(); 141 if (!f) 142 return -ENOMEM; 143 144 dma_fence_enable_sw_signaling(f); 145 146 dma_fence_signal(f); 147 148 if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) { 149 pr_err("Added callback, but fence was already signaled!\n"); 150 goto err_free; 151 } 152 153 dma_fence_signal(f); 154 if (cb.seen) { 155 pr_err("Callback called after failed attachment !\n"); 156 goto err_free; 157 } 158 159 err = 0; 160 err_free: 161 dma_fence_put(f); 162 return err; 163 } 164 165 static int test_rm_callback(void *arg) 166 { 167 struct simple_cb cb = {}; 168 struct dma_fence *f; 169 int err = -EINVAL; 170 171 f = mock_fence(); 172 if (!f) 173 return -ENOMEM; 174 175 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) { 176 pr_err("Failed to add callback, fence already signaled!\n"); 177 goto err_free; 178 } 179 180 if (!dma_fence_remove_callback(f, &cb.cb)) { 181 pr_err("Failed to remove callback!\n"); 182 goto err_free; 183 } 184 185 dma_fence_signal(f); 186 if (cb.seen) { 187 pr_err("Callback still signaled after removal!\n"); 188 goto err_free; 189 } 190 191 err = 0; 192 err_free: 193 dma_fence_put(f); 194 return err; 195 } 196 197 static int test_late_rm_callback(void *arg) 198 { 199 struct simple_cb cb = {}; 200 struct dma_fence *f; 201 int err = -EINVAL; 202 203 f = mock_fence(); 204 if (!f) 205 return -ENOMEM; 206 207 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) { 208 pr_err("Failed to add callback, fence already signaled!\n"); 209 goto err_free; 210 } 211 212 dma_fence_signal(f); 213 if (!cb.seen) { 214 pr_err("Callback failed!\n"); 215 goto err_free; 216 } 217 218 if (dma_fence_remove_callback(f, &cb.cb)) { 219 pr_err("Callback removal succeed after being executed!\n"); 220 goto err_free; 221 } 222 223 err = 0; 224 err_free: 225 dma_fence_put(f); 226 return err; 227 } 228 229 static int test_status(void *arg) 230 { 231 struct dma_fence *f; 232 int err = -EINVAL; 233 234 f = mock_fence(); 235 if (!f) 236 return -ENOMEM; 237 238 dma_fence_enable_sw_signaling(f); 239 240 if (dma_fence_get_status(f)) { 241 pr_err("Fence unexpectedly has signaled status on creation\n"); 242 goto err_free; 243 } 244 245 dma_fence_signal(f); 246 if (!dma_fence_get_status(f)) { 247 pr_err("Fence not reporting signaled status\n"); 248 goto err_free; 249 } 250 251 err = 0; 252 err_free: 253 dma_fence_put(f); 254 return err; 255 } 256 257 static int test_error(void *arg) 258 { 259 struct dma_fence *f; 260 int err = -EINVAL; 261 262 f = mock_fence(); 263 if (!f) 264 return -ENOMEM; 265 266 dma_fence_enable_sw_signaling(f); 267 268 dma_fence_set_error(f, -EIO); 269 270 if (dma_fence_get_status(f)) { 271 pr_err("Fence unexpectedly has error status before signal\n"); 272 goto err_free; 273 } 274 275 dma_fence_signal(f); 276 if (dma_fence_get_status(f) != -EIO) { 277 pr_err("Fence not reporting error status, got %d\n", 278 dma_fence_get_status(f)); 279 goto err_free; 280 } 281 282 err = 0; 283 err_free: 284 dma_fence_put(f); 285 return err; 286 } 287 288 static int test_wait(void *arg) 289 { 290 struct dma_fence *f; 291 int err = -EINVAL; 292 293 f = mock_fence(); 294 if (!f) 295 return -ENOMEM; 296 297 dma_fence_enable_sw_signaling(f); 298 299 if (dma_fence_wait_timeout(f, false, 0) != 0) { 300 pr_err("Wait reported complete before being signaled\n"); 301 goto err_free; 302 } 303 304 dma_fence_signal(f); 305 306 if (dma_fence_wait_timeout(f, false, 0) != 1) { 307 pr_err("Wait reported incomplete after being signaled\n"); 308 goto err_free; 309 } 310 311 err = 0; 312 err_free: 313 dma_fence_signal(f); 314 dma_fence_put(f); 315 return err; 316 } 317 318 struct wait_timer { 319 struct timer_list timer; 320 struct dma_fence *f; 321 }; 322 323 static void wait_timer(struct timer_list *timer) 324 { 325 struct wait_timer *wt = timer_container_of(wt, timer, timer); 326 327 dma_fence_signal(wt->f); 328 } 329 330 static int test_wait_timeout(void *arg) 331 { 332 struct wait_timer wt; 333 int err = -EINVAL; 334 335 timer_setup_on_stack(&wt.timer, wait_timer, 0); 336 337 wt.f = mock_fence(); 338 if (!wt.f) 339 return -ENOMEM; 340 341 dma_fence_enable_sw_signaling(wt.f); 342 343 if (dma_fence_wait_timeout(wt.f, false, 1) != 0) { 344 pr_err("Wait reported complete before being signaled\n"); 345 goto err_free; 346 } 347 348 mod_timer(&wt.timer, jiffies + 1); 349 350 if (dma_fence_wait_timeout(wt.f, false, HZ) == 0) { 351 if (timer_pending(&wt.timer)) { 352 pr_notice("Timer did not fire within one HZ!\n"); 353 err = 0; /* not our fault! */ 354 } else { 355 pr_err("Wait reported incomplete after timeout\n"); 356 } 357 goto err_free; 358 } 359 360 err = 0; 361 err_free: 362 timer_delete_sync(&wt.timer); 363 timer_destroy_on_stack(&wt.timer); 364 dma_fence_signal(wt.f); 365 dma_fence_put(wt.f); 366 return err; 367 } 368 369 static int test_stub(void *arg) 370 { 371 struct dma_fence *f[64]; 372 int err = -EINVAL; 373 int i; 374 375 for (i = 0; i < ARRAY_SIZE(f); i++) { 376 f[i] = dma_fence_get_stub(); 377 if (!dma_fence_is_signaled(f[i])) { 378 pr_err("Obtained unsignaled stub fence!\n"); 379 goto err; 380 } 381 } 382 383 err = 0; 384 err: 385 while (i--) 386 dma_fence_put(f[i]); 387 return err; 388 } 389 390 /* Now off to the races! */ 391 392 struct race_thread { 393 struct dma_fence __rcu **fences; 394 struct task_struct *task; 395 bool before; 396 int id; 397 }; 398 399 static void __wait_for_callbacks(struct dma_fence *f) 400 { 401 unsigned long flags; 402 403 dma_fence_lock_irqsave(f, flags); 404 dma_fence_unlock_irqrestore(f, flags); 405 } 406 407 static int thread_signal_callback(void *arg) 408 { 409 const struct race_thread *t = arg; 410 unsigned long pass = 0; 411 unsigned long miss = 0; 412 int err = 0; 413 414 while (!err && !kthread_should_stop()) { 415 struct dma_fence *f1, *f2; 416 struct simple_cb cb; 417 418 f1 = mock_fence(); 419 if (!f1) { 420 err = -ENOMEM; 421 break; 422 } 423 424 dma_fence_enable_sw_signaling(f1); 425 426 rcu_assign_pointer(t->fences[t->id], f1); 427 smp_wmb(); 428 429 rcu_read_lock(); 430 do { 431 f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]); 432 } while (!f2 && !kthread_should_stop()); 433 rcu_read_unlock(); 434 435 if (t->before) 436 dma_fence_signal(f1); 437 438 smp_store_mb(cb.seen, false); 439 if (!f2 || 440 dma_fence_add_callback(f2, &cb.cb, simple_callback)) { 441 miss++; 442 cb.seen = true; 443 } 444 445 if (!t->before) 446 dma_fence_signal(f1); 447 448 if (!cb.seen) { 449 dma_fence_wait(f2, false); 450 __wait_for_callbacks(f2); 451 } 452 453 if (!READ_ONCE(cb.seen)) { 454 pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n", 455 t->id, pass, miss, 456 t->before ? "before" : "after", 457 dma_fence_is_signaled(f2) ? "yes" : "no"); 458 err = -EINVAL; 459 } 460 461 dma_fence_put(f2); 462 463 rcu_assign_pointer(t->fences[t->id], NULL); 464 smp_wmb(); 465 466 dma_fence_put(f1); 467 468 pass++; 469 } 470 471 pr_info("%s[%d] completed %lu passes, %lu misses\n", 472 __func__, t->id, pass, miss); 473 return err; 474 } 475 476 static int race_signal_callback(void *arg) 477 { 478 struct dma_fence __rcu *f[2] = {}; 479 int ret = 0; 480 int pass; 481 482 for (pass = 0; !ret && pass <= 1; pass++) { 483 struct race_thread t[2]; 484 int i; 485 486 for (i = 0; i < ARRAY_SIZE(t); i++) { 487 t[i].fences = f; 488 t[i].id = i; 489 t[i].before = pass; 490 t[i].task = kthread_run(thread_signal_callback, &t[i], 491 "dma-fence:%d", i); 492 if (IS_ERR(t[i].task)) { 493 ret = PTR_ERR(t[i].task); 494 while (--i >= 0) 495 kthread_stop_put(t[i].task); 496 return ret; 497 } 498 get_task_struct(t[i].task); 499 } 500 501 msleep(50); 502 503 for (i = 0; i < ARRAY_SIZE(t); i++) { 504 int err; 505 506 err = kthread_stop_put(t[i].task); 507 if (err && !ret) 508 ret = err; 509 } 510 } 511 512 return ret; 513 } 514 515 int dma_fence(void) 516 { 517 static const struct subtest tests[] = { 518 SUBTEST(sanitycheck), 519 SUBTEST(test_signaling), 520 SUBTEST(test_add_callback), 521 SUBTEST(test_late_add_callback), 522 SUBTEST(test_rm_callback), 523 SUBTEST(test_late_rm_callback), 524 SUBTEST(test_status), 525 SUBTEST(test_error), 526 SUBTEST(test_wait), 527 SUBTEST(test_wait_timeout), 528 SUBTEST(test_stub), 529 SUBTEST(race_signal_callback), 530 }; 531 532 pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence)); 533 return subtests(tests, NULL); 534 } 535