1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/epoch.h> 36 #include <sys/gtaskqueue.h> 37 #include <sys/kernel.h> 38 #include <sys/limits.h> 39 #include <sys/lock.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/pcpu.h> 43 #include <sys/proc.h> 44 #include <sys/sched.h> 45 #include <sys/sx.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/turnstile.h> 49 #ifdef EPOCH_TRACE 50 #include <machine/stdarg.h> 51 #include <sys/stack.h> 52 #include <sys/tree.h> 53 #endif 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_kern.h> 57 #include <vm/uma.h> 58 59 #include <ck_epoch.h> 60 61 static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); 62 63 #ifdef __amd64__ 64 #define EPOCH_ALIGN CACHE_LINE_SIZE*2 65 #else 66 #define EPOCH_ALIGN CACHE_LINE_SIZE 67 #endif 68 69 TAILQ_HEAD (epoch_tdlist, epoch_tracker); 70 typedef struct epoch_record { 71 ck_epoch_record_t er_record; 72 struct epoch_context er_drain_ctx; 73 struct epoch *er_parent; 74 volatile struct epoch_tdlist er_tdlist; 75 volatile uint32_t er_gen; 76 uint32_t er_cpuid; 77 } __aligned(EPOCH_ALIGN) *epoch_record_t; 78 79 struct epoch { 80 struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); 81 epoch_record_t e_pcpu_record; 82 int e_idx; 83 int e_flags; 84 struct sx e_drain_sx; 85 struct mtx e_drain_mtx; 86 volatile int e_drain_count; 87 const char *e_name; 88 }; 89 90 /* arbitrary --- needs benchmarking */ 91 #define MAX_ADAPTIVE_SPIN 100 92 #define MAX_EPOCHS 64 93 94 CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context)); 95 SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information"); 96 SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats"); 97 98 /* Stats. */ 99 static counter_u64_t block_count; 100 101 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, nblocked, CTLFLAG_RW, 102 &block_count, "# of times a thread was in an epoch when epoch_wait was called"); 103 static counter_u64_t migrate_count; 104 105 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, migrations, CTLFLAG_RW, 106 &migrate_count, "# of times thread was migrated to another CPU in epoch_wait"); 107 static counter_u64_t turnstile_count; 108 109 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, ncontended, CTLFLAG_RW, 110 &turnstile_count, "# of times a thread was blocked on a lock in an epoch during an epoch_wait"); 111 static counter_u64_t switch_count; 112 113 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW, 114 &switch_count, "# of times a thread voluntarily context switched in epoch_wait"); 115 static counter_u64_t epoch_call_count; 116 117 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW, 118 &epoch_call_count, "# of times a callback was deferred"); 119 static counter_u64_t epoch_call_task_count; 120 121 SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW, 122 &epoch_call_task_count, "# of times a callback task was run"); 123 124 TAILQ_HEAD (threadlist, thread); 125 126 CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, 127 ck_epoch_entry_container) 128 129 epoch_t allepochs[MAX_EPOCHS]; 130 131 DPCPU_DEFINE(struct grouptask, epoch_cb_task); 132 DPCPU_DEFINE(int, epoch_cb_count); 133 134 static __read_mostly int inited; 135 static __read_mostly int epoch_count; 136 __read_mostly epoch_t global_epoch; 137 __read_mostly epoch_t global_epoch_preempt; 138 139 static void epoch_call_task(void *context __unused); 140 static uma_zone_t pcpu_zone_record; 141 142 #ifdef EPOCH_TRACE 143 struct stackentry { 144 RB_ENTRY(stackentry) se_node; 145 struct stack se_stack; 146 }; 147 148 static int 149 stackentry_compare(struct stackentry *a, struct stackentry *b) 150 { 151 152 if (a->se_stack.depth > b->se_stack.depth) 153 return (1); 154 if (a->se_stack.depth < b->se_stack.depth) 155 return (-1); 156 for (int i = 0; i < a->se_stack.depth; i++) { 157 if (a->se_stack.pcs[i] > b->se_stack.pcs[i]) 158 return (1); 159 if (a->se_stack.pcs[i] < b->se_stack.pcs[i]) 160 return (-1); 161 } 162 163 return (0); 164 } 165 166 RB_HEAD(stacktree, stackentry) epoch_stacks = RB_INITIALIZER(&epoch_stacks); 167 RB_GENERATE_STATIC(stacktree, stackentry, se_node, stackentry_compare); 168 169 static struct mtx epoch_stacks_lock; 170 MTX_SYSINIT(epochstacks, &epoch_stacks_lock, "epoch_stacks", MTX_DEF); 171 172 static void epoch_trace_report(const char *fmt, ...) __printflike(1, 2); 173 static inline void 174 epoch_trace_report(const char *fmt, ...) 175 { 176 va_list ap; 177 struct stackentry se, *new; 178 179 stack_zero(&se.se_stack); /* XXX: is it really needed? */ 180 stack_save(&se.se_stack); 181 182 /* Tree is never reduced - go lockless. */ 183 if (RB_FIND(stacktree, &epoch_stacks, &se) != NULL) 184 return; 185 186 new = malloc(sizeof(*new), M_STACK, M_NOWAIT); 187 if (new != NULL) { 188 bcopy(&se.se_stack, &new->se_stack, sizeof(struct stack)); 189 190 mtx_lock(&epoch_stacks_lock); 191 new = RB_INSERT(stacktree, &epoch_stacks, new); 192 mtx_unlock(&epoch_stacks_lock); 193 if (new != NULL) 194 free(new, M_STACK); 195 } 196 197 va_start(ap, fmt); 198 (void)vprintf(fmt, ap); 199 va_end(ap); 200 stack_print_ddb(&se.se_stack); 201 } 202 203 static inline void 204 epoch_trace_enter(struct thread *td, epoch_t epoch, epoch_tracker_t et, 205 const char *file, int line) 206 { 207 epoch_tracker_t iet; 208 209 SLIST_FOREACH(iet, &td->td_epochs, et_tlink) 210 if (iet->et_epoch == epoch) 211 epoch_trace_report("Recursively entering epoch %s " 212 "previously entered at %s:%d\n", 213 epoch->e_name, iet->et_file, iet->et_line); 214 et->et_epoch = epoch; 215 et->et_file = file; 216 et->et_line = line; 217 SLIST_INSERT_HEAD(&td->td_epochs, et, et_tlink); 218 } 219 220 static inline void 221 epoch_trace_exit(struct thread *td, epoch_t epoch, epoch_tracker_t et, 222 const char *file, int line) 223 { 224 225 if (SLIST_FIRST(&td->td_epochs) != et) { 226 epoch_trace_report("Exiting epoch %s in a not nested order. " 227 "Most recently entered %s at %s:%d\n", 228 epoch->e_name, 229 SLIST_FIRST(&td->td_epochs)->et_epoch->e_name, 230 SLIST_FIRST(&td->td_epochs)->et_file, 231 SLIST_FIRST(&td->td_epochs)->et_line); 232 /* This will panic if et is not anywhere on td_epochs. */ 233 SLIST_REMOVE(&td->td_epochs, et, epoch_tracker, et_tlink); 234 } else 235 SLIST_REMOVE_HEAD(&td->td_epochs, et_tlink); 236 } 237 238 /* Used by assertions that check thread state before going to sleep. */ 239 void 240 epoch_trace_list(struct thread *td) 241 { 242 epoch_tracker_t iet; 243 244 SLIST_FOREACH(iet, &td->td_epochs, et_tlink) 245 printf("Epoch %s entered at %s:%d\n", iet->et_epoch->e_name, 246 iet->et_file, iet->et_line); 247 } 248 #endif /* EPOCH_TRACE */ 249 250 static void 251 epoch_init(void *arg __unused) 252 { 253 int cpu; 254 255 block_count = counter_u64_alloc(M_WAITOK); 256 migrate_count = counter_u64_alloc(M_WAITOK); 257 turnstile_count = counter_u64_alloc(M_WAITOK); 258 switch_count = counter_u64_alloc(M_WAITOK); 259 epoch_call_count = counter_u64_alloc(M_WAITOK); 260 epoch_call_task_count = counter_u64_alloc(M_WAITOK); 261 262 pcpu_zone_record = uma_zcreate("epoch_record pcpu", 263 sizeof(struct epoch_record), NULL, NULL, NULL, NULL, 264 UMA_ALIGN_PTR, UMA_ZONE_PCPU); 265 CPU_FOREACH(cpu) { 266 GROUPTASK_INIT(DPCPU_ID_PTR(cpu, epoch_cb_task), 0, 267 epoch_call_task, NULL); 268 taskqgroup_attach_cpu(qgroup_softirq, 269 DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, NULL, NULL, 270 "epoch call task"); 271 } 272 #ifdef EPOCH_TRACE 273 SLIST_INIT(&thread0.td_epochs); 274 #endif 275 inited = 1; 276 global_epoch = epoch_alloc("Global", 0); 277 global_epoch_preempt = epoch_alloc("Global preemptible", EPOCH_PREEMPT); 278 } 279 SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL); 280 281 #if !defined(EARLY_AP_STARTUP) 282 static void 283 epoch_init_smp(void *dummy __unused) 284 { 285 inited = 2; 286 } 287 SYSINIT(epoch_smp, SI_SUB_SMP + 1, SI_ORDER_FIRST, epoch_init_smp, NULL); 288 #endif 289 290 static void 291 epoch_ctor(epoch_t epoch) 292 { 293 epoch_record_t er; 294 int cpu; 295 296 epoch->e_pcpu_record = uma_zalloc_pcpu(pcpu_zone_record, M_WAITOK); 297 CPU_FOREACH(cpu) { 298 er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu); 299 bzero(er, sizeof(*er)); 300 ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); 301 TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist); 302 er->er_cpuid = cpu; 303 er->er_parent = epoch; 304 } 305 } 306 307 static void 308 epoch_adjust_prio(struct thread *td, u_char prio) 309 { 310 311 thread_lock(td); 312 sched_prio(td, prio); 313 thread_unlock(td); 314 } 315 316 epoch_t 317 epoch_alloc(const char *name, int flags) 318 { 319 epoch_t epoch; 320 321 if (__predict_false(!inited)) 322 panic("%s called too early in boot", __func__); 323 epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO | M_WAITOK); 324 ck_epoch_init(&epoch->e_epoch); 325 epoch_ctor(epoch); 326 MPASS(epoch_count < MAX_EPOCHS - 2); 327 epoch->e_flags = flags; 328 epoch->e_idx = epoch_count; 329 epoch->e_name = name; 330 sx_init(&epoch->e_drain_sx, "epoch-drain-sx"); 331 mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF); 332 allepochs[epoch_count++] = epoch; 333 return (epoch); 334 } 335 336 void 337 epoch_free(epoch_t epoch) 338 { 339 340 epoch_drain_callbacks(epoch); 341 allepochs[epoch->e_idx] = NULL; 342 epoch_wait(global_epoch); 343 uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record); 344 mtx_destroy(&epoch->e_drain_mtx); 345 sx_destroy(&epoch->e_drain_sx); 346 free(epoch, M_EPOCH); 347 } 348 349 static epoch_record_t 350 epoch_currecord(epoch_t epoch) 351 { 352 353 return (zpcpu_get_cpu(epoch->e_pcpu_record, curcpu)); 354 } 355 356 #define INIT_CHECK(epoch) \ 357 do { \ 358 if (__predict_false((epoch) == NULL)) \ 359 return; \ 360 } while (0) 361 362 void 363 _epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE) 364 { 365 struct epoch_record *er; 366 struct thread *td; 367 368 MPASS(cold || epoch != NULL); 369 MPASS(epoch->e_flags & EPOCH_PREEMPT); 370 td = curthread; 371 MPASS((vm_offset_t)et >= td->td_kstack && 372 (vm_offset_t)et + sizeof(struct epoch_tracker) <= 373 td->td_kstack + td->td_kstack_pages * PAGE_SIZE); 374 375 INIT_CHECK(epoch); 376 #ifdef EPOCH_TRACE 377 epoch_trace_enter(td, epoch, et, file, line); 378 #endif 379 et->et_td = td; 380 td->td_epochnest++; 381 critical_enter(); 382 sched_pin(); 383 td->td_pre_epoch_prio = td->td_priority; 384 er = epoch_currecord(epoch); 385 TAILQ_INSERT_TAIL(&er->er_tdlist, et, et_link); 386 ck_epoch_begin(&er->er_record, &et->et_section); 387 critical_exit(); 388 } 389 390 void 391 epoch_enter(epoch_t epoch) 392 { 393 struct thread *td; 394 epoch_record_t er; 395 396 MPASS(cold || epoch != NULL); 397 INIT_CHECK(epoch); 398 td = curthread; 399 td->td_epochnest++; 400 critical_enter(); 401 er = epoch_currecord(epoch); 402 ck_epoch_begin(&er->er_record, NULL); 403 } 404 405 void 406 _epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE) 407 { 408 struct epoch_record *er; 409 struct thread *td; 410 411 INIT_CHECK(epoch); 412 td = curthread; 413 critical_enter(); 414 sched_unpin(); 415 MPASS(td->td_epochnest); 416 td->td_epochnest--; 417 er = epoch_currecord(epoch); 418 MPASS(epoch->e_flags & EPOCH_PREEMPT); 419 MPASS(et != NULL); 420 MPASS(et->et_td == td); 421 #ifdef INVARIANTS 422 et->et_td = (void*)0xDEADBEEF; 423 #endif 424 ck_epoch_end(&er->er_record, &et->et_section); 425 TAILQ_REMOVE(&er->er_tdlist, et, et_link); 426 er->er_gen++; 427 if (__predict_false(td->td_pre_epoch_prio != td->td_priority)) 428 epoch_adjust_prio(td, td->td_pre_epoch_prio); 429 critical_exit(); 430 #ifdef EPOCH_TRACE 431 epoch_trace_exit(td, epoch, et, file, line); 432 #endif 433 } 434 435 void 436 epoch_exit(epoch_t epoch) 437 { 438 struct thread *td; 439 epoch_record_t er; 440 441 INIT_CHECK(epoch); 442 td = curthread; 443 MPASS(td->td_epochnest); 444 td->td_epochnest--; 445 er = epoch_currecord(epoch); 446 ck_epoch_end(&er->er_record, NULL); 447 critical_exit(); 448 } 449 450 /* 451 * epoch_block_handler_preempt() is a callback from the CK code when another 452 * thread is currently in an epoch section. 453 */ 454 static void 455 epoch_block_handler_preempt(struct ck_epoch *global __unused, 456 ck_epoch_record_t *cr, void *arg __unused) 457 { 458 epoch_record_t record; 459 struct thread *td, *owner, *curwaittd; 460 struct epoch_tracker *tdwait; 461 struct turnstile *ts; 462 struct lock_object *lock; 463 int spincount, gen; 464 int locksheld __unused; 465 466 record = __containerof(cr, struct epoch_record, er_record); 467 td = curthread; 468 locksheld = td->td_locks; 469 spincount = 0; 470 counter_u64_add(block_count, 1); 471 /* 472 * We lost a race and there's no longer any threads 473 * on the CPU in an epoch section. 474 */ 475 if (TAILQ_EMPTY(&record->er_tdlist)) 476 return; 477 478 if (record->er_cpuid != curcpu) { 479 /* 480 * If the head of the list is running, we can wait for it 481 * to remove itself from the list and thus save us the 482 * overhead of a migration 483 */ 484 gen = record->er_gen; 485 thread_unlock(td); 486 /* 487 * We can't actually check if the waiting thread is running 488 * so we simply poll for it to exit before giving up and 489 * migrating. 490 */ 491 do { 492 cpu_spinwait(); 493 } while (!TAILQ_EMPTY(&record->er_tdlist) && 494 gen == record->er_gen && 495 spincount++ < MAX_ADAPTIVE_SPIN); 496 thread_lock(td); 497 /* 498 * If the generation has changed we can poll again 499 * otherwise we need to migrate. 500 */ 501 if (gen != record->er_gen) 502 return; 503 /* 504 * Being on the same CPU as that of the record on which 505 * we need to wait allows us access to the thread 506 * list associated with that CPU. We can then examine the 507 * oldest thread in the queue and wait on its turnstile 508 * until it resumes and so on until a grace period 509 * elapses. 510 * 511 */ 512 counter_u64_add(migrate_count, 1); 513 sched_bind(td, record->er_cpuid); 514 /* 515 * At this point we need to return to the ck code 516 * to scan to see if a grace period has elapsed. 517 * We can't move on to check the thread list, because 518 * in the meantime new threads may have arrived that 519 * in fact belong to a different epoch. 520 */ 521 return; 522 } 523 /* 524 * Try to find a thread in an epoch section on this CPU 525 * waiting on a turnstile. Otherwise find the lowest 526 * priority thread (highest prio value) and drop our priority 527 * to match to allow it to run. 528 */ 529 TAILQ_FOREACH(tdwait, &record->er_tdlist, et_link) { 530 /* 531 * Propagate our priority to any other waiters to prevent us 532 * from starving them. They will have their original priority 533 * restore on exit from epoch_wait(). 534 */ 535 curwaittd = tdwait->et_td; 536 if (!TD_IS_INHIBITED(curwaittd) && curwaittd->td_priority > td->td_priority) { 537 critical_enter(); 538 thread_unlock(td); 539 thread_lock(curwaittd); 540 sched_prio(curwaittd, td->td_priority); 541 thread_unlock(curwaittd); 542 thread_lock(td); 543 critical_exit(); 544 } 545 if (TD_IS_INHIBITED(curwaittd) && TD_ON_LOCK(curwaittd) && 546 ((ts = curwaittd->td_blocked) != NULL)) { 547 /* 548 * We unlock td to allow turnstile_wait to reacquire 549 * the thread lock. Before unlocking it we enter a 550 * critical section to prevent preemption after we 551 * reenable interrupts by dropping the thread lock in 552 * order to prevent curwaittd from getting to run. 553 */ 554 critical_enter(); 555 thread_unlock(td); 556 557 if (turnstile_lock(ts, &lock, &owner)) { 558 if (ts == curwaittd->td_blocked) { 559 MPASS(TD_IS_INHIBITED(curwaittd) && 560 TD_ON_LOCK(curwaittd)); 561 critical_exit(); 562 turnstile_wait(ts, owner, 563 curwaittd->td_tsqueue); 564 counter_u64_add(turnstile_count, 1); 565 thread_lock(td); 566 return; 567 } 568 turnstile_unlock(ts, lock); 569 } 570 thread_lock(td); 571 critical_exit(); 572 KASSERT(td->td_locks == locksheld, 573 ("%d extra locks held", td->td_locks - locksheld)); 574 } 575 } 576 /* 577 * We didn't find any threads actually blocked on a lock 578 * so we have nothing to do except context switch away. 579 */ 580 counter_u64_add(switch_count, 1); 581 mi_switch(SW_VOL | SWT_RELINQUISH, NULL); 582 583 /* 584 * Release the thread lock while yielding to 585 * allow other threads to acquire the lock 586 * pointed to by TDQ_LOCKPTR(td). Else a 587 * deadlock like situation might happen. (HPS) 588 */ 589 thread_unlock(td); 590 thread_lock(td); 591 } 592 593 void 594 epoch_wait_preempt(epoch_t epoch) 595 { 596 struct thread *td; 597 int was_bound; 598 int old_cpu; 599 int old_pinned; 600 u_char old_prio; 601 int locks __unused; 602 603 MPASS(cold || epoch != NULL); 604 INIT_CHECK(epoch); 605 td = curthread; 606 #ifdef INVARIANTS 607 locks = curthread->td_locks; 608 MPASS(epoch->e_flags & EPOCH_PREEMPT); 609 if ((epoch->e_flags & EPOCH_LOCKED) == 0) 610 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 611 "epoch_wait() can be long running"); 612 KASSERT(!in_epoch(epoch), ("epoch_wait_preempt() called in the middle " 613 "of an epoch section of the same epoch")); 614 #endif 615 thread_lock(td); 616 DROP_GIANT(); 617 618 old_cpu = PCPU_GET(cpuid); 619 old_pinned = td->td_pinned; 620 old_prio = td->td_priority; 621 was_bound = sched_is_bound(td); 622 sched_unbind(td); 623 td->td_pinned = 0; 624 sched_bind(td, old_cpu); 625 626 ck_epoch_synchronize_wait(&epoch->e_epoch, epoch_block_handler_preempt, 627 NULL); 628 629 /* restore CPU binding, if any */ 630 if (was_bound != 0) { 631 sched_bind(td, old_cpu); 632 } else { 633 /* get thread back to initial CPU, if any */ 634 if (old_pinned != 0) 635 sched_bind(td, old_cpu); 636 sched_unbind(td); 637 } 638 /* restore pinned after bind */ 639 td->td_pinned = old_pinned; 640 641 /* restore thread priority */ 642 sched_prio(td, old_prio); 643 thread_unlock(td); 644 PICKUP_GIANT(); 645 KASSERT(td->td_locks == locks, 646 ("%d residual locks held", td->td_locks - locks)); 647 } 648 649 static void 650 epoch_block_handler(struct ck_epoch *g __unused, ck_epoch_record_t *c __unused, 651 void *arg __unused) 652 { 653 cpu_spinwait(); 654 } 655 656 void 657 epoch_wait(epoch_t epoch) 658 { 659 660 MPASS(cold || epoch != NULL); 661 INIT_CHECK(epoch); 662 MPASS(epoch->e_flags == 0); 663 critical_enter(); 664 ck_epoch_synchronize_wait(&epoch->e_epoch, epoch_block_handler, NULL); 665 critical_exit(); 666 } 667 668 void 669 epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)) 670 { 671 epoch_record_t er; 672 ck_epoch_entry_t *cb; 673 674 cb = (void *)ctx; 675 676 MPASS(callback); 677 /* too early in boot to have epoch set up */ 678 if (__predict_false(epoch == NULL)) 679 goto boottime; 680 #if !defined(EARLY_AP_STARTUP) 681 if (__predict_false(inited < 2)) 682 goto boottime; 683 #endif 684 685 critical_enter(); 686 *DPCPU_PTR(epoch_cb_count) += 1; 687 er = epoch_currecord(epoch); 688 ck_epoch_call(&er->er_record, cb, (ck_epoch_cb_t *)callback); 689 critical_exit(); 690 return; 691 boottime: 692 callback(ctx); 693 } 694 695 static void 696 epoch_call_task(void *arg __unused) 697 { 698 ck_stack_entry_t *cursor, *head, *next; 699 ck_epoch_record_t *record; 700 epoch_record_t er; 701 epoch_t epoch; 702 ck_stack_t cb_stack; 703 int i, npending, total; 704 705 ck_stack_init(&cb_stack); 706 critical_enter(); 707 epoch_enter(global_epoch); 708 for (total = i = 0; i < epoch_count; i++) { 709 if (__predict_false((epoch = allepochs[i]) == NULL)) 710 continue; 711 er = epoch_currecord(epoch); 712 record = &er->er_record; 713 if ((npending = record->n_pending) == 0) 714 continue; 715 ck_epoch_poll_deferred(record, &cb_stack); 716 total += npending - record->n_pending; 717 } 718 epoch_exit(global_epoch); 719 *DPCPU_PTR(epoch_cb_count) -= total; 720 critical_exit(); 721 722 counter_u64_add(epoch_call_count, total); 723 counter_u64_add(epoch_call_task_count, 1); 724 725 head = ck_stack_batch_pop_npsc(&cb_stack); 726 for (cursor = head; cursor != NULL; cursor = next) { 727 struct ck_epoch_entry *entry = 728 ck_epoch_entry_container(cursor); 729 730 next = CK_STACK_NEXT(cursor); 731 entry->function(entry); 732 } 733 } 734 735 int 736 in_epoch_verbose(epoch_t epoch, int dump_onfail) 737 { 738 struct epoch_tracker *tdwait; 739 struct thread *td; 740 epoch_record_t er; 741 742 td = curthread; 743 if (td->td_epochnest == 0) 744 return (0); 745 if (__predict_false((epoch) == NULL)) 746 return (0); 747 critical_enter(); 748 er = epoch_currecord(epoch); 749 TAILQ_FOREACH(tdwait, &er->er_tdlist, et_link) 750 if (tdwait->et_td == td) { 751 critical_exit(); 752 return (1); 753 } 754 #ifdef INVARIANTS 755 if (dump_onfail) { 756 MPASS(td->td_pinned); 757 printf("cpu: %d id: %d\n", curcpu, td->td_tid); 758 TAILQ_FOREACH(tdwait, &er->er_tdlist, et_link) 759 printf("td_tid: %d ", tdwait->et_td->td_tid); 760 printf("\n"); 761 } 762 #endif 763 critical_exit(); 764 return (0); 765 } 766 767 int 768 in_epoch(epoch_t epoch) 769 { 770 return (in_epoch_verbose(epoch, 0)); 771 } 772 773 static void 774 epoch_drain_cb(struct epoch_context *ctx) 775 { 776 struct epoch *epoch = 777 __containerof(ctx, struct epoch_record, er_drain_ctx)->er_parent; 778 779 if (atomic_fetchadd_int(&epoch->e_drain_count, -1) == 1) { 780 mtx_lock(&epoch->e_drain_mtx); 781 wakeup(epoch); 782 mtx_unlock(&epoch->e_drain_mtx); 783 } 784 } 785 786 void 787 epoch_drain_callbacks(epoch_t epoch) 788 { 789 epoch_record_t er; 790 struct thread *td; 791 int was_bound; 792 int old_pinned; 793 int old_cpu; 794 int cpu; 795 796 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 797 "epoch_drain_callbacks() may sleep!"); 798 799 /* too early in boot to have epoch set up */ 800 if (__predict_false(epoch == NULL)) 801 return; 802 #if !defined(EARLY_AP_STARTUP) 803 if (__predict_false(inited < 2)) 804 return; 805 #endif 806 DROP_GIANT(); 807 808 sx_xlock(&epoch->e_drain_sx); 809 mtx_lock(&epoch->e_drain_mtx); 810 811 td = curthread; 812 thread_lock(td); 813 old_cpu = PCPU_GET(cpuid); 814 old_pinned = td->td_pinned; 815 was_bound = sched_is_bound(td); 816 sched_unbind(td); 817 td->td_pinned = 0; 818 819 CPU_FOREACH(cpu) 820 epoch->e_drain_count++; 821 CPU_FOREACH(cpu) { 822 er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu); 823 sched_bind(td, cpu); 824 epoch_call(epoch, &er->er_drain_ctx, &epoch_drain_cb); 825 } 826 827 /* restore CPU binding, if any */ 828 if (was_bound != 0) { 829 sched_bind(td, old_cpu); 830 } else { 831 /* get thread back to initial CPU, if any */ 832 if (old_pinned != 0) 833 sched_bind(td, old_cpu); 834 sched_unbind(td); 835 } 836 /* restore pinned after bind */ 837 td->td_pinned = old_pinned; 838 839 thread_unlock(td); 840 841 while (epoch->e_drain_count != 0) 842 msleep(epoch, &epoch->e_drain_mtx, PZERO, "EDRAIN", 0); 843 844 mtx_unlock(&epoch->e_drain_mtx); 845 sx_xunlock(&epoch->e_drain_sx); 846 847 PICKUP_GIANT(); 848 } 849