11a01b4e5SHans Petter Selasky /*- 296fc97c8SStephen Hurd * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) 3eae5868cSHans Petter Selasky * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org) 41a01b4e5SHans Petter Selasky * All rights reserved. 51a01b4e5SHans Petter Selasky * 61a01b4e5SHans Petter Selasky * Redistribution and use in source and binary forms, with or without 71a01b4e5SHans Petter Selasky * modification, are permitted provided that the following conditions 81a01b4e5SHans Petter Selasky * are met: 91a01b4e5SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright 101a01b4e5SHans Petter Selasky * notice unmodified, this list of conditions, and the following 111a01b4e5SHans Petter Selasky * disclaimer. 121a01b4e5SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright 131a01b4e5SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the 141a01b4e5SHans Petter Selasky * documentation and/or other materials provided with the distribution. 151a01b4e5SHans Petter Selasky * 161a01b4e5SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 171a01b4e5SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 181a01b4e5SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 191a01b4e5SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 201a01b4e5SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 211a01b4e5SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 221a01b4e5SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 231a01b4e5SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 241a01b4e5SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 251a01b4e5SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 261a01b4e5SHans Petter Selasky */ 271a01b4e5SHans Petter Selasky 281a01b4e5SHans Petter Selasky #include <sys/cdefs.h> 291a01b4e5SHans Petter Selasky __FBSDID("$FreeBSD$"); 301a01b4e5SHans Petter Selasky 311a01b4e5SHans Petter Selasky #include <sys/types.h> 321a01b4e5SHans Petter Selasky #include <sys/systm.h> 331a01b4e5SHans Petter Selasky #include <sys/malloc.h> 341a01b4e5SHans Petter Selasky #include <sys/kernel.h> 351a01b4e5SHans Petter Selasky #include <sys/lock.h> 361a01b4e5SHans Petter Selasky #include <sys/mutex.h> 371a01b4e5SHans Petter Selasky #include <sys/proc.h> 381a01b4e5SHans Petter Selasky #include <sys/sched.h> 391a01b4e5SHans Petter Selasky #include <sys/smp.h> 401a01b4e5SHans Petter Selasky #include <sys/queue.h> 411a01b4e5SHans Petter Selasky #include <sys/taskqueue.h> 42f3de9af6SHans Petter Selasky #include <sys/kdb.h> 431a01b4e5SHans Petter Selasky 441a01b4e5SHans Petter Selasky #include <ck_epoch.h> 451a01b4e5SHans Petter Selasky 461a01b4e5SHans Petter Selasky #include <linux/rcupdate.h> 471a01b4e5SHans Petter Selasky #include <linux/srcu.h> 481a01b4e5SHans Petter Selasky #include <linux/slab.h> 491a01b4e5SHans Petter Selasky #include <linux/kernel.h> 50f3de9af6SHans Petter Selasky #include <linux/compat.h> 511a01b4e5SHans Petter Selasky 52f3de9af6SHans Petter Selasky /* 53f3de9af6SHans Petter Selasky * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will 54f3de9af6SHans Petter Selasky * not be skipped during panic(). 55f3de9af6SHans Petter Selasky */ 56f3de9af6SHans Petter Selasky #ifdef CONFIG_NO_RCU_SKIP 57f3de9af6SHans Petter Selasky #define RCU_SKIP(void) 0 58f3de9af6SHans Petter Selasky #else 59f3de9af6SHans Petter Selasky #define RCU_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) 60f3de9af6SHans Petter Selasky #endif 611f827dabSHans Petter Selasky 621f827dabSHans Petter Selasky struct callback_head { 631f827dabSHans Petter Selasky STAILQ_ENTRY(callback_head) entry; 641f827dabSHans Petter Selasky rcu_callback_t func; 651f827dabSHans Petter Selasky }; 661f827dabSHans Petter Selasky 67f3de9af6SHans Petter Selasky struct linux_epoch_head { 68f3de9af6SHans Petter Selasky STAILQ_HEAD(, callback_head) cb_head; 69f3de9af6SHans Petter Selasky struct mtx lock; 70f3de9af6SHans Petter Selasky struct task task; 71f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 72f3de9af6SHans Petter Selasky 73f3de9af6SHans Petter Selasky struct linux_epoch_record { 741f827dabSHans Petter Selasky ck_epoch_record_t epoch_record; 75f3de9af6SHans Petter Selasky TAILQ_HEAD(, task_struct) ts_head; 76f3de9af6SHans Petter Selasky int cpuid; 77*6ae24079SHans Petter Selasky int type; 78f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 791a01b4e5SHans Petter Selasky 801a01b4e5SHans Petter Selasky /* 811a01b4e5SHans Petter Selasky * Verify that "struct rcu_head" is big enough to hold "struct 821a01b4e5SHans Petter Selasky * callback_head". This has been done to avoid having to add special 831a01b4e5SHans Petter Selasky * compile flags for including ck_epoch.h to all clients of the 841a01b4e5SHans Petter Selasky * LinuxKPI. 851a01b4e5SHans Petter Selasky */ 86f3de9af6SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head)); 871a01b4e5SHans Petter Selasky 881f827dabSHans Petter Selasky /* 891f827dabSHans Petter Selasky * Verify that "epoch_record" is at beginning of "struct 90f3de9af6SHans Petter Selasky * linux_epoch_record": 911f827dabSHans Petter Selasky */ 92f3de9af6SHans Petter Selasky CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); 931f827dabSHans Petter Selasky 94*6ae24079SHans Petter Selasky CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX); 95*6ae24079SHans Petter Selasky 96eae5868cSHans Petter Selasky static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; 97eae5868cSHans Petter Selasky static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; 98eae5868cSHans Petter Selasky DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); 991f827dabSHans Petter Selasky 1001f827dabSHans Petter Selasky static void linux_rcu_cleaner_func(void *, int); 1011a01b4e5SHans Petter Selasky 1021a01b4e5SHans Petter Selasky static void 1031a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused) 1041a01b4e5SHans Petter Selasky { 105f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 1061a01b4e5SHans Petter Selasky int i; 107eae5868cSHans Petter Selasky int j; 1081a01b4e5SHans Petter Selasky 109eae5868cSHans Petter Selasky for (j = 0; j != RCU_TYPE_MAX; j++) { 110eae5868cSHans Petter Selasky ck_epoch_init(&linux_epoch[j]); 1111a01b4e5SHans Petter Selasky 112eae5868cSHans Petter Selasky head = &linux_epoch_head[j]; 113f3de9af6SHans Petter Selasky 114f3de9af6SHans Petter Selasky mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); 115eae5868cSHans Petter Selasky TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); 116f3de9af6SHans Petter Selasky STAILQ_INIT(&head->cb_head); 117f3de9af6SHans Petter Selasky 1181a01b4e5SHans Petter Selasky CPU_FOREACH(i) { 119f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 1201f827dabSHans Petter Selasky 121eae5868cSHans Petter Selasky record = &DPCPU_ID_GET(i, linux_epoch_record[j]); 1221f827dabSHans Petter Selasky 123f3de9af6SHans Petter Selasky record->cpuid = i; 124*6ae24079SHans Petter Selasky record->type = j; 125eae5868cSHans Petter Selasky ck_epoch_register(&linux_epoch[j], 126eae5868cSHans Petter Selasky &record->epoch_record, NULL); 127f3de9af6SHans Petter Selasky TAILQ_INIT(&record->ts_head); 1281a01b4e5SHans Petter Selasky } 1291a01b4e5SHans Petter Selasky } 130eae5868cSHans Petter Selasky } 131d8e073a9SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); 1321a01b4e5SHans Petter Selasky 1331a01b4e5SHans Petter Selasky static void 1341a01b4e5SHans Petter Selasky linux_rcu_runtime_uninit(void *arg __unused) 1351a01b4e5SHans Petter Selasky { 136f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 137eae5868cSHans Petter Selasky int j; 1381a01b4e5SHans Petter Selasky 139eae5868cSHans Petter Selasky for (j = 0; j != RCU_TYPE_MAX; j++) { 140eae5868cSHans Petter Selasky head = &linux_epoch_head[j]; 1411a01b4e5SHans Petter Selasky 142f3de9af6SHans Petter Selasky mtx_destroy(&head->lock); 1431a01b4e5SHans Petter Selasky } 144eae5868cSHans Petter Selasky } 1451a01b4e5SHans Petter Selasky SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); 1461a01b4e5SHans Petter Selasky 1471f827dabSHans Petter Selasky static void 148eae5868cSHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused) 1491f827dabSHans Petter Selasky { 150f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 1511a01b4e5SHans Petter Selasky struct callback_head *rcu; 152f3de9af6SHans Petter Selasky STAILQ_HEAD(, callback_head) tmp_head; 153eae5868cSHans Petter Selasky uintptr_t offset; 1541f827dabSHans Petter Selasky 155f3de9af6SHans Petter Selasky linux_set_current(curthread); 156f3de9af6SHans Petter Selasky 157eae5868cSHans Petter Selasky head = context; 1581f827dabSHans Petter Selasky 1591f827dabSHans Petter Selasky /* move current callbacks into own queue */ 160f3de9af6SHans Petter Selasky mtx_lock(&head->lock); 161f3de9af6SHans Petter Selasky STAILQ_INIT(&tmp_head); 162f3de9af6SHans Petter Selasky STAILQ_CONCAT(&tmp_head, &head->cb_head); 163f3de9af6SHans Petter Selasky mtx_unlock(&head->lock); 1641f827dabSHans Petter Selasky 1651f827dabSHans Petter Selasky /* synchronize */ 166eae5868cSHans Petter Selasky linux_synchronize_rcu(head - linux_epoch_head); 1671f827dabSHans Petter Selasky 1681f827dabSHans Petter Selasky /* dispatch all callbacks, if any */ 169f3de9af6SHans Petter Selasky while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { 1701a01b4e5SHans Petter Selasky 171f3de9af6SHans Petter Selasky STAILQ_REMOVE_HEAD(&tmp_head, entry); 1721a01b4e5SHans Petter Selasky 1731a01b4e5SHans Petter Selasky offset = (uintptr_t)rcu->func; 1741a01b4e5SHans Petter Selasky 1751a01b4e5SHans Petter Selasky if (offset < LINUX_KFREE_RCU_OFFSET_MAX) 1761a01b4e5SHans Petter Selasky kfree((char *)rcu - offset); 1771a01b4e5SHans Petter Selasky else 1781a01b4e5SHans Petter Selasky rcu->func((struct rcu_head *)rcu); 1791a01b4e5SHans Petter Selasky } 1801a01b4e5SHans Petter Selasky } 1811a01b4e5SHans Petter Selasky 1821a01b4e5SHans Petter Selasky void 183eae5868cSHans Petter Selasky linux_rcu_read_lock(unsigned type) 1841a01b4e5SHans Petter Selasky { 185f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 186f3de9af6SHans Petter Selasky struct task_struct *ts; 187f3de9af6SHans Petter Selasky 188eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 189eae5868cSHans Petter Selasky 190f3de9af6SHans Petter Selasky if (RCU_SKIP()) 191f3de9af6SHans Petter Selasky return; 1921a01b4e5SHans Petter Selasky 1931f827dabSHans Petter Selasky /* 1941f827dabSHans Petter Selasky * Pin thread to current CPU so that the unlock code gets the 195f3de9af6SHans Petter Selasky * same per-CPU epoch record: 1961f827dabSHans Petter Selasky */ 1971a01b4e5SHans Petter Selasky sched_pin(); 1981a01b4e5SHans Petter Selasky 199eae5868cSHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 200f3de9af6SHans Petter Selasky ts = current; 2011f827dabSHans Petter Selasky 2021f827dabSHans Petter Selasky /* 2031f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2041f827dabSHans Petter Selasky * ck_epoch_begin(). Else this function supports recursion. 2051f827dabSHans Petter Selasky */ 2061f827dabSHans Petter Selasky critical_enter(); 207f3de9af6SHans Petter Selasky ck_epoch_begin(&record->epoch_record, NULL); 208*6ae24079SHans Petter Selasky ts->rcu_recurse[type]++; 209*6ae24079SHans Petter Selasky if (ts->rcu_recurse[type] == 1) 210*6ae24079SHans Petter Selasky TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]); 2111f827dabSHans Petter Selasky critical_exit(); 2121a01b4e5SHans Petter Selasky } 2131a01b4e5SHans Petter Selasky 2141a01b4e5SHans Petter Selasky void 215eae5868cSHans Petter Selasky linux_rcu_read_unlock(unsigned type) 2161a01b4e5SHans Petter Selasky { 217f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 218f3de9af6SHans Petter Selasky struct task_struct *ts; 2191a01b4e5SHans Petter Selasky 220eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 221eae5868cSHans Petter Selasky 222f3de9af6SHans Petter Selasky if (RCU_SKIP()) 223f3de9af6SHans Petter Selasky return; 224f3de9af6SHans Petter Selasky 225eae5868cSHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 226f3de9af6SHans Petter Selasky ts = current; 2271f827dabSHans Petter Selasky 2281f827dabSHans Petter Selasky /* 2291f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2301f827dabSHans Petter Selasky * ck_epoch_end(). Else this function supports recursion. 2311f827dabSHans Petter Selasky */ 2321f827dabSHans Petter Selasky critical_enter(); 233f3de9af6SHans Petter Selasky ck_epoch_end(&record->epoch_record, NULL); 234*6ae24079SHans Petter Selasky ts->rcu_recurse[type]--; 235*6ae24079SHans Petter Selasky if (ts->rcu_recurse[type] == 0) 236*6ae24079SHans Petter Selasky TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]); 2371f827dabSHans Petter Selasky critical_exit(); 2381f827dabSHans Petter Selasky 2391a01b4e5SHans Petter Selasky sched_unpin(); 2401a01b4e5SHans Petter Selasky } 2411a01b4e5SHans Petter Selasky 242f3de9af6SHans Petter Selasky static void 243f3de9af6SHans Petter Selasky linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused) 244f3de9af6SHans Petter Selasky { 245f3de9af6SHans Petter Selasky struct linux_epoch_record *record = 246f3de9af6SHans Petter Selasky container_of(epoch_record, struct linux_epoch_record, epoch_record); 247f3de9af6SHans Petter Selasky struct thread *td = curthread; 248f3de9af6SHans Petter Selasky struct task_struct *ts; 249f3de9af6SHans Petter Selasky 250f3de9af6SHans Petter Selasky /* check if blocked on the current CPU */ 251f3de9af6SHans Petter Selasky if (record->cpuid == PCPU_GET(cpuid)) { 252f3de9af6SHans Petter Selasky bool is_sleeping = 0; 253f3de9af6SHans Petter Selasky u_char prio = 0; 254f3de9af6SHans Petter Selasky 255f3de9af6SHans Petter Selasky /* 256f3de9af6SHans Petter Selasky * Find the lowest priority or sleeping thread which 257f3de9af6SHans Petter Selasky * is blocking synchronization on this CPU core. All 258f3de9af6SHans Petter Selasky * the threads in the queue are CPU-pinned and cannot 259f3de9af6SHans Petter Selasky * go anywhere while the current thread is locked. 260f3de9af6SHans Petter Selasky */ 261*6ae24079SHans Petter Selasky TAILQ_FOREACH(ts, &record->ts_head, rcu_entry[record->type]) { 262f3de9af6SHans Petter Selasky if (ts->task_thread->td_priority > prio) 263f3de9af6SHans Petter Selasky prio = ts->task_thread->td_priority; 264f3de9af6SHans Petter Selasky is_sleeping |= (ts->task_thread->td_inhibitors != 0); 265f3de9af6SHans Petter Selasky } 266f3de9af6SHans Petter Selasky 267f3de9af6SHans Petter Selasky if (is_sleeping) { 268f3de9af6SHans Petter Selasky thread_unlock(td); 269f3de9af6SHans Petter Selasky pause("W", 1); 270f3de9af6SHans Petter Selasky thread_lock(td); 271f3de9af6SHans Petter Selasky } else { 272f3de9af6SHans Petter Selasky /* set new thread priority */ 273f3de9af6SHans Petter Selasky sched_prio(td, prio); 274f3de9af6SHans Petter Selasky /* task switch */ 275686bcb5cSJeff Roberson mi_switch(SW_VOL | SWT_RELINQUISH); 276cc79ea3aSHans Petter Selasky /* 277cc79ea3aSHans Petter Selasky * It is important the thread lock is dropped 278cc79ea3aSHans Petter Selasky * while yielding to allow other threads to 279cc79ea3aSHans Petter Selasky * acquire the lock pointed to by 280cc79ea3aSHans Petter Selasky * TDQ_LOCKPTR(td). Currently mi_switch() will 281cc79ea3aSHans Petter Selasky * unlock the thread lock before 282cc79ea3aSHans Petter Selasky * returning. Else a deadlock like situation 283cc79ea3aSHans Petter Selasky * might happen. 284cc79ea3aSHans Petter Selasky */ 285714ed5b2SHans Petter Selasky thread_lock(td); 286f3de9af6SHans Petter Selasky } 287f3de9af6SHans Petter Selasky } else { 288f3de9af6SHans Petter Selasky /* 289f3de9af6SHans Petter Selasky * To avoid spinning move execution to the other CPU 290f3de9af6SHans Petter Selasky * which is blocking synchronization. Set highest 291f3de9af6SHans Petter Selasky * thread priority so that code gets run. The thread 292f3de9af6SHans Petter Selasky * priority will be restored later. 293f3de9af6SHans Petter Selasky */ 294f3de9af6SHans Petter Selasky sched_prio(td, 0); 295f3de9af6SHans Petter Selasky sched_bind(td, record->cpuid); 296f3de9af6SHans Petter Selasky } 297f3de9af6SHans Petter Selasky } 298f3de9af6SHans Petter Selasky 2991a01b4e5SHans Petter Selasky void 300eae5868cSHans Petter Selasky linux_synchronize_rcu(unsigned type) 3011a01b4e5SHans Petter Selasky { 302f3de9af6SHans Petter Selasky struct thread *td; 303f3de9af6SHans Petter Selasky int was_bound; 304f3de9af6SHans Petter Selasky int old_cpu; 305f3de9af6SHans Petter Selasky int old_pinned; 3063f743d78SHans Petter Selasky u_char old_prio; 307f3de9af6SHans Petter Selasky 308eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 309eae5868cSHans Petter Selasky 310f3de9af6SHans Petter Selasky if (RCU_SKIP()) 311f3de9af6SHans Petter Selasky return; 312f3de9af6SHans Petter Selasky 313f3de9af6SHans Petter Selasky WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 314f3de9af6SHans Petter Selasky "linux_synchronize_rcu() can sleep"); 315f3de9af6SHans Petter Selasky 316f3de9af6SHans Petter Selasky td = curthread; 317fedab1b4SKonstantin Belousov DROP_GIANT(); 318f3de9af6SHans Petter Selasky 319f3de9af6SHans Petter Selasky /* 320f3de9af6SHans Petter Selasky * Synchronizing RCU might change the CPU core this function 321f3de9af6SHans Petter Selasky * is running on. Save current values: 322f3de9af6SHans Petter Selasky */ 323f3de9af6SHans Petter Selasky thread_lock(td); 324f3de9af6SHans Petter Selasky 325f3de9af6SHans Petter Selasky old_cpu = PCPU_GET(cpuid); 326f3de9af6SHans Petter Selasky old_pinned = td->td_pinned; 3273f743d78SHans Petter Selasky old_prio = td->td_priority; 328f3de9af6SHans Petter Selasky was_bound = sched_is_bound(td); 329ea165254SHans Petter Selasky sched_unbind(td); 330ea165254SHans Petter Selasky td->td_pinned = 0; 331f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 332f3de9af6SHans Petter Selasky 333eae5868cSHans Petter Selasky ck_epoch_synchronize_wait(&linux_epoch[type], 334f3de9af6SHans Petter Selasky &linux_synchronize_rcu_cb, NULL); 335f3de9af6SHans Petter Selasky 336f3de9af6SHans Petter Selasky /* restore CPU binding, if any */ 337f3de9af6SHans Petter Selasky if (was_bound != 0) { 338f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 339f3de9af6SHans Petter Selasky } else { 340f3de9af6SHans Petter Selasky /* get thread back to initial CPU, if any */ 341f3de9af6SHans Petter Selasky if (old_pinned != 0) 342f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 343f3de9af6SHans Petter Selasky sched_unbind(td); 344f3de9af6SHans Petter Selasky } 345f3de9af6SHans Petter Selasky /* restore pinned after bind */ 346f3de9af6SHans Petter Selasky td->td_pinned = old_pinned; 3473f743d78SHans Petter Selasky 3483f743d78SHans Petter Selasky /* restore thread priority */ 3493f743d78SHans Petter Selasky sched_prio(td, old_prio); 350f3de9af6SHans Petter Selasky thread_unlock(td); 351f3de9af6SHans Petter Selasky 352f3de9af6SHans Petter Selasky PICKUP_GIANT(); 3531a01b4e5SHans Petter Selasky } 3541a01b4e5SHans Petter Selasky 3551a01b4e5SHans Petter Selasky void 356eae5868cSHans Petter Selasky linux_rcu_barrier(unsigned type) 3571a01b4e5SHans Petter Selasky { 358f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 3591a01b4e5SHans Petter Selasky 360eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 3611f827dabSHans Petter Selasky 362eae5868cSHans Petter Selasky linux_synchronize_rcu(type); 363eae5868cSHans Petter Selasky 364eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3651f827dabSHans Petter Selasky 3661f827dabSHans Petter Selasky /* wait for callbacks to complete */ 367f3de9af6SHans Petter Selasky taskqueue_drain(taskqueue_fast, &head->task); 3681a01b4e5SHans Petter Selasky } 3691a01b4e5SHans Petter Selasky 3701a01b4e5SHans Petter Selasky void 371eae5868cSHans Petter Selasky linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) 3721a01b4e5SHans Petter Selasky { 373eae5868cSHans Petter Selasky struct callback_head *rcu; 374eae5868cSHans Petter Selasky struct linux_epoch_head *head; 375eae5868cSHans Petter Selasky 376eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 377eae5868cSHans Petter Selasky 378eae5868cSHans Petter Selasky rcu = (struct callback_head *)context; 379eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3801a01b4e5SHans Petter Selasky 381f3de9af6SHans Petter Selasky mtx_lock(&head->lock); 3821f827dabSHans Petter Selasky rcu->func = func; 383f3de9af6SHans Petter Selasky STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry); 384f3de9af6SHans Petter Selasky taskqueue_enqueue(taskqueue_fast, &head->task); 385f3de9af6SHans Petter Selasky mtx_unlock(&head->lock); 3861a01b4e5SHans Petter Selasky } 3871a01b4e5SHans Petter Selasky 3881a01b4e5SHans Petter Selasky int 3891a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu) 3901a01b4e5SHans Petter Selasky { 3911a01b4e5SHans Petter Selasky return (0); 3921a01b4e5SHans Petter Selasky } 3931a01b4e5SHans Petter Selasky 3941a01b4e5SHans Petter Selasky void 3951a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu) 3961a01b4e5SHans Petter Selasky { 3971a01b4e5SHans Petter Selasky } 3981a01b4e5SHans Petter Selasky 3991a01b4e5SHans Petter Selasky int 4001a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu) 4011a01b4e5SHans Petter Selasky { 402eae5868cSHans Petter Selasky linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); 4031a01b4e5SHans Petter Selasky return (0); 4041a01b4e5SHans Petter Selasky } 4051a01b4e5SHans Petter Selasky 4061a01b4e5SHans Petter Selasky void 4071a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused) 4081a01b4e5SHans Petter Selasky { 409eae5868cSHans Petter Selasky linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); 4101a01b4e5SHans Petter Selasky } 4111a01b4e5SHans Petter Selasky 4121a01b4e5SHans Petter Selasky void 4131a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu) 4141a01b4e5SHans Petter Selasky { 415eae5868cSHans Petter Selasky linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); 4161f827dabSHans Petter Selasky } 4171f827dabSHans Petter Selasky 4181f827dabSHans Petter Selasky void 4191f827dabSHans Petter Selasky srcu_barrier(struct srcu_struct *srcu) 4201f827dabSHans Petter Selasky { 421eae5868cSHans Petter Selasky linux_rcu_barrier(RCU_TYPE_SLEEPABLE); 4221a01b4e5SHans Petter Selasky } 423