11a01b4e5SHans Petter Selasky /*- 296fc97c8SStephen Hurd * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) 317777208SHans Petter Selasky * Copyright (c) 2017-2021 Hans Petter Selasky (hselasky@freebsd.org) 41a01b4e5SHans Petter Selasky * All rights reserved. 51a01b4e5SHans Petter Selasky * 61a01b4e5SHans Petter Selasky * Redistribution and use in source and binary forms, with or without 71a01b4e5SHans Petter Selasky * modification, are permitted provided that the following conditions 81a01b4e5SHans Petter Selasky * are met: 91a01b4e5SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright 101a01b4e5SHans Petter Selasky * notice unmodified, this list of conditions, and the following 111a01b4e5SHans Petter Selasky * disclaimer. 121a01b4e5SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright 131a01b4e5SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the 141a01b4e5SHans Petter Selasky * documentation and/or other materials provided with the distribution. 151a01b4e5SHans Petter Selasky * 161a01b4e5SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 171a01b4e5SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 181a01b4e5SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 191a01b4e5SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 201a01b4e5SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 211a01b4e5SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 221a01b4e5SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 231a01b4e5SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 241a01b4e5SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 251a01b4e5SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 261a01b4e5SHans Petter Selasky */ 271a01b4e5SHans Petter Selasky 281a01b4e5SHans Petter Selasky #include <sys/cdefs.h> 291a01b4e5SHans Petter Selasky __FBSDID("$FreeBSD$"); 301a01b4e5SHans Petter Selasky 311a01b4e5SHans Petter Selasky #include <sys/types.h> 321a01b4e5SHans Petter Selasky #include <sys/systm.h> 331a01b4e5SHans Petter Selasky #include <sys/malloc.h> 341a01b4e5SHans Petter Selasky #include <sys/kernel.h> 351a01b4e5SHans Petter Selasky #include <sys/lock.h> 361a01b4e5SHans Petter Selasky #include <sys/mutex.h> 371a01b4e5SHans Petter Selasky #include <sys/proc.h> 381a01b4e5SHans Petter Selasky #include <sys/sched.h> 391a01b4e5SHans Petter Selasky #include <sys/smp.h> 401a01b4e5SHans Petter Selasky #include <sys/queue.h> 411a01b4e5SHans Petter Selasky #include <sys/taskqueue.h> 42f3de9af6SHans Petter Selasky #include <sys/kdb.h> 431a01b4e5SHans Petter Selasky 441a01b4e5SHans Petter Selasky #include <ck_epoch.h> 451a01b4e5SHans Petter Selasky 461a01b4e5SHans Petter Selasky #include <linux/rcupdate.h> 471a01b4e5SHans Petter Selasky #include <linux/srcu.h> 481a01b4e5SHans Petter Selasky #include <linux/slab.h> 491a01b4e5SHans Petter Selasky #include <linux/kernel.h> 50f3de9af6SHans Petter Selasky #include <linux/compat.h> 51a2b83b59SVladimir Kondratyev #include <linux/llist.h> 52a2b83b59SVladimir Kondratyev #include <linux/irq_work.h> 531a01b4e5SHans Petter Selasky 54f3de9af6SHans Petter Selasky /* 55f3de9af6SHans Petter Selasky * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will 56f3de9af6SHans Petter Selasky * not be skipped during panic(). 57f3de9af6SHans Petter Selasky */ 58f3de9af6SHans Petter Selasky #ifdef CONFIG_NO_RCU_SKIP 59f3de9af6SHans Petter Selasky #define RCU_SKIP(void) 0 60f3de9af6SHans Petter Selasky #else 61f3de9af6SHans Petter Selasky #define RCU_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) 62f3de9af6SHans Petter Selasky #endif 631f827dabSHans Petter Selasky 641f827dabSHans Petter Selasky struct callback_head { 65a2b83b59SVladimir Kondratyev union { 661f827dabSHans Petter Selasky STAILQ_ENTRY(callback_head) entry; 67a2b83b59SVladimir Kondratyev struct llist_node node; 68a2b83b59SVladimir Kondratyev }; 691f827dabSHans Petter Selasky rcu_callback_t func; 701f827dabSHans Petter Selasky }; 711f827dabSHans Petter Selasky 72f3de9af6SHans Petter Selasky struct linux_epoch_head { 73a2b83b59SVladimir Kondratyev struct llist_head cb_head; 74f3de9af6SHans Petter Selasky struct task task; 75f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 76f3de9af6SHans Petter Selasky 77f3de9af6SHans Petter Selasky struct linux_epoch_record { 781f827dabSHans Petter Selasky ck_epoch_record_t epoch_record; 79f3de9af6SHans Petter Selasky TAILQ_HEAD(, task_struct) ts_head; 80f3de9af6SHans Petter Selasky int cpuid; 816ae24079SHans Petter Selasky int type; 82f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 831a01b4e5SHans Petter Selasky 841a01b4e5SHans Petter Selasky /* 851a01b4e5SHans Petter Selasky * Verify that "struct rcu_head" is big enough to hold "struct 861a01b4e5SHans Petter Selasky * callback_head". This has been done to avoid having to add special 871a01b4e5SHans Petter Selasky * compile flags for including ck_epoch.h to all clients of the 881a01b4e5SHans Petter Selasky * LinuxKPI. 891a01b4e5SHans Petter Selasky */ 90f3de9af6SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head)); 911a01b4e5SHans Petter Selasky 921f827dabSHans Petter Selasky /* 9317777208SHans Petter Selasky * Verify that "rcu_section[0]" has the same size as 9417777208SHans Petter Selasky * "ck_epoch_section_t". This has been done to avoid having to add 9517777208SHans Petter Selasky * special compile flags for including ck_epoch.h to all clients of 9617777208SHans Petter Selasky * the LinuxKPI. 9717777208SHans Petter Selasky */ 9817777208SHans Petter Selasky CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] == 9917777208SHans Petter Selasky sizeof(ck_epoch_section_t))); 10017777208SHans Petter Selasky 10117777208SHans Petter Selasky /* 1021f827dabSHans Petter Selasky * Verify that "epoch_record" is at beginning of "struct 103f3de9af6SHans Petter Selasky * linux_epoch_record": 1041f827dabSHans Petter Selasky */ 105f3de9af6SHans Petter Selasky CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); 1061f827dabSHans Petter Selasky 1076ae24079SHans Petter Selasky CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX); 1086ae24079SHans Petter Selasky 109eae5868cSHans Petter Selasky static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; 110eae5868cSHans Petter Selasky static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; 111eae5868cSHans Petter Selasky DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); 1121f827dabSHans Petter Selasky 1131f827dabSHans Petter Selasky static void linux_rcu_cleaner_func(void *, int); 1141a01b4e5SHans Petter Selasky 1151a01b4e5SHans Petter Selasky static void 1161a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused) 1171a01b4e5SHans Petter Selasky { 118f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 1191a01b4e5SHans Petter Selasky int i; 120eae5868cSHans Petter Selasky int j; 1211a01b4e5SHans Petter Selasky 122eae5868cSHans Petter Selasky for (j = 0; j != RCU_TYPE_MAX; j++) { 123eae5868cSHans Petter Selasky ck_epoch_init(&linux_epoch[j]); 1241a01b4e5SHans Petter Selasky 125eae5868cSHans Petter Selasky head = &linux_epoch_head[j]; 126f3de9af6SHans Petter Selasky 127eae5868cSHans Petter Selasky TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); 128a2b83b59SVladimir Kondratyev init_llist_head(&head->cb_head); 129f3de9af6SHans Petter Selasky 1301a01b4e5SHans Petter Selasky CPU_FOREACH(i) { 131f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 1321f827dabSHans Petter Selasky 133eae5868cSHans Petter Selasky record = &DPCPU_ID_GET(i, linux_epoch_record[j]); 1341f827dabSHans Petter Selasky 135f3de9af6SHans Petter Selasky record->cpuid = i; 1366ae24079SHans Petter Selasky record->type = j; 137eae5868cSHans Petter Selasky ck_epoch_register(&linux_epoch[j], 138eae5868cSHans Petter Selasky &record->epoch_record, NULL); 139f3de9af6SHans Petter Selasky TAILQ_INIT(&record->ts_head); 1401a01b4e5SHans Petter Selasky } 1411a01b4e5SHans Petter Selasky } 142eae5868cSHans Petter Selasky } 143d8e073a9SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); 1441a01b4e5SHans Petter Selasky 1451a01b4e5SHans Petter Selasky static void 146eae5868cSHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused) 1471f827dabSHans Petter Selasky { 148a2b83b59SVladimir Kondratyev struct linux_epoch_head *head = context; 1491a01b4e5SHans Petter Selasky struct callback_head *rcu; 150f3de9af6SHans Petter Selasky STAILQ_HEAD(, callback_head) tmp_head; 151a2b83b59SVladimir Kondratyev struct llist_node *node, *next; 152eae5868cSHans Petter Selasky uintptr_t offset; 1531f827dabSHans Petter Selasky 1541f827dabSHans Petter Selasky /* move current callbacks into own queue */ 155f3de9af6SHans Petter Selasky STAILQ_INIT(&tmp_head); 156a2b83b59SVladimir Kondratyev llist_for_each_safe(node, next, llist_del_all(&head->cb_head)) { 157a2b83b59SVladimir Kondratyev rcu = container_of(node, struct callback_head, node); 158a2b83b59SVladimir Kondratyev /* re-reverse list to restore chronological order */ 159a2b83b59SVladimir Kondratyev STAILQ_INSERT_HEAD(&tmp_head, rcu, entry); 160a2b83b59SVladimir Kondratyev } 1611f827dabSHans Petter Selasky 1621f827dabSHans Petter Selasky /* synchronize */ 163eae5868cSHans Petter Selasky linux_synchronize_rcu(head - linux_epoch_head); 1641f827dabSHans Petter Selasky 1651f827dabSHans Petter Selasky /* dispatch all callbacks, if any */ 166f3de9af6SHans Petter Selasky while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { 167f3de9af6SHans Petter Selasky STAILQ_REMOVE_HEAD(&tmp_head, entry); 1681a01b4e5SHans Petter Selasky 1691a01b4e5SHans Petter Selasky offset = (uintptr_t)rcu->func; 1701a01b4e5SHans Petter Selasky 1711a01b4e5SHans Petter Selasky if (offset < LINUX_KFREE_RCU_OFFSET_MAX) 1721a01b4e5SHans Petter Selasky kfree((char *)rcu - offset); 1731a01b4e5SHans Petter Selasky else 1741a01b4e5SHans Petter Selasky rcu->func((struct rcu_head *)rcu); 1751a01b4e5SHans Petter Selasky } 1761a01b4e5SHans Petter Selasky } 1771a01b4e5SHans Petter Selasky 1781a01b4e5SHans Petter Selasky void 179eae5868cSHans Petter Selasky linux_rcu_read_lock(unsigned type) 1801a01b4e5SHans Petter Selasky { 181f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 182f3de9af6SHans Petter Selasky struct task_struct *ts; 183f3de9af6SHans Petter Selasky 184eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 185eae5868cSHans Petter Selasky 186f3de9af6SHans Petter Selasky if (RCU_SKIP()) 187f3de9af6SHans Petter Selasky return; 1881a01b4e5SHans Petter Selasky 18917777208SHans Petter Selasky ts = current; 19017777208SHans Petter Selasky 19117777208SHans Petter Selasky /* assert valid refcount */ 19217777208SHans Petter Selasky MPASS(ts->rcu_recurse[type] != INT_MAX); 19317777208SHans Petter Selasky 19417777208SHans Petter Selasky if (++(ts->rcu_recurse[type]) != 1) 19517777208SHans Petter Selasky return; 19617777208SHans Petter Selasky 1971f827dabSHans Petter Selasky /* 1981f827dabSHans Petter Selasky * Pin thread to current CPU so that the unlock code gets the 199f3de9af6SHans Petter Selasky * same per-CPU epoch record: 2001f827dabSHans Petter Selasky */ 2011a01b4e5SHans Petter Selasky sched_pin(); 2021a01b4e5SHans Petter Selasky 203eae5868cSHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 2041f827dabSHans Petter Selasky 2051f827dabSHans Petter Selasky /* 2061f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2071f827dabSHans Petter Selasky * ck_epoch_begin(). Else this function supports recursion. 2081f827dabSHans Petter Selasky */ 2091f827dabSHans Petter Selasky critical_enter(); 21017777208SHans Petter Selasky ck_epoch_begin(&record->epoch_record, 21117777208SHans Petter Selasky (ck_epoch_section_t *)&ts->rcu_section[type]); 2126ae24079SHans Petter Selasky TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]); 2131f827dabSHans Petter Selasky critical_exit(); 2141a01b4e5SHans Petter Selasky } 2151a01b4e5SHans Petter Selasky 2161a01b4e5SHans Petter Selasky void 217eae5868cSHans Petter Selasky linux_rcu_read_unlock(unsigned type) 2181a01b4e5SHans Petter Selasky { 219f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 220f3de9af6SHans Petter Selasky struct task_struct *ts; 2211a01b4e5SHans Petter Selasky 222eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 223eae5868cSHans Petter Selasky 224f3de9af6SHans Petter Selasky if (RCU_SKIP()) 225f3de9af6SHans Petter Selasky return; 226f3de9af6SHans Petter Selasky 227f3de9af6SHans Petter Selasky ts = current; 2281f827dabSHans Petter Selasky 22917777208SHans Petter Selasky /* assert valid refcount */ 23017777208SHans Petter Selasky MPASS(ts->rcu_recurse[type] > 0); 23117777208SHans Petter Selasky 23217777208SHans Petter Selasky if (--(ts->rcu_recurse[type]) != 0) 23317777208SHans Petter Selasky return; 23417777208SHans Petter Selasky 23517777208SHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 23617777208SHans Petter Selasky 2371f827dabSHans Petter Selasky /* 2381f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2391f827dabSHans Petter Selasky * ck_epoch_end(). Else this function supports recursion. 2401f827dabSHans Petter Selasky */ 2411f827dabSHans Petter Selasky critical_enter(); 24217777208SHans Petter Selasky ck_epoch_end(&record->epoch_record, 24317777208SHans Petter Selasky (ck_epoch_section_t *)&ts->rcu_section[type]); 2446ae24079SHans Petter Selasky TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]); 2451f827dabSHans Petter Selasky critical_exit(); 2461f827dabSHans Petter Selasky 2471a01b4e5SHans Petter Selasky sched_unpin(); 2481a01b4e5SHans Petter Selasky } 2491a01b4e5SHans Petter Selasky 250f3de9af6SHans Petter Selasky static void 251f3de9af6SHans Petter Selasky linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused) 252f3de9af6SHans Petter Selasky { 253f3de9af6SHans Petter Selasky struct linux_epoch_record *record = 254f3de9af6SHans Petter Selasky container_of(epoch_record, struct linux_epoch_record, epoch_record); 255f3de9af6SHans Petter Selasky struct thread *td = curthread; 256f3de9af6SHans Petter Selasky struct task_struct *ts; 257f3de9af6SHans Petter Selasky 258f3de9af6SHans Petter Selasky /* check if blocked on the current CPU */ 259f3de9af6SHans Petter Selasky if (record->cpuid == PCPU_GET(cpuid)) { 260f3de9af6SHans Petter Selasky bool is_sleeping = 0; 261f3de9af6SHans Petter Selasky u_char prio = 0; 262f3de9af6SHans Petter Selasky 263f3de9af6SHans Petter Selasky /* 264f3de9af6SHans Petter Selasky * Find the lowest priority or sleeping thread which 265f3de9af6SHans Petter Selasky * is blocking synchronization on this CPU core. All 266f3de9af6SHans Petter Selasky * the threads in the queue are CPU-pinned and cannot 267f3de9af6SHans Petter Selasky * go anywhere while the current thread is locked. 268f3de9af6SHans Petter Selasky */ 2696ae24079SHans Petter Selasky TAILQ_FOREACH(ts, &record->ts_head, rcu_entry[record->type]) { 270f3de9af6SHans Petter Selasky if (ts->task_thread->td_priority > prio) 271f3de9af6SHans Petter Selasky prio = ts->task_thread->td_priority; 272f3de9af6SHans Petter Selasky is_sleeping |= (ts->task_thread->td_inhibitors != 0); 273f3de9af6SHans Petter Selasky } 274f3de9af6SHans Petter Selasky 275f3de9af6SHans Petter Selasky if (is_sleeping) { 276f3de9af6SHans Petter Selasky thread_unlock(td); 277f3de9af6SHans Petter Selasky pause("W", 1); 278f3de9af6SHans Petter Selasky thread_lock(td); 279f3de9af6SHans Petter Selasky } else { 280f3de9af6SHans Petter Selasky /* set new thread priority */ 281f3de9af6SHans Petter Selasky sched_prio(td, prio); 282f3de9af6SHans Petter Selasky /* task switch */ 283686bcb5cSJeff Roberson mi_switch(SW_VOL | SWT_RELINQUISH); 284cc79ea3aSHans Petter Selasky /* 285cc79ea3aSHans Petter Selasky * It is important the thread lock is dropped 286cc79ea3aSHans Petter Selasky * while yielding to allow other threads to 287cc79ea3aSHans Petter Selasky * acquire the lock pointed to by 288cc79ea3aSHans Petter Selasky * TDQ_LOCKPTR(td). Currently mi_switch() will 289cc79ea3aSHans Petter Selasky * unlock the thread lock before 290cc79ea3aSHans Petter Selasky * returning. Else a deadlock like situation 291cc79ea3aSHans Petter Selasky * might happen. 292cc79ea3aSHans Petter Selasky */ 293714ed5b2SHans Petter Selasky thread_lock(td); 294f3de9af6SHans Petter Selasky } 295f3de9af6SHans Petter Selasky } else { 296f3de9af6SHans Petter Selasky /* 297f3de9af6SHans Petter Selasky * To avoid spinning move execution to the other CPU 298f3de9af6SHans Petter Selasky * which is blocking synchronization. Set highest 299f3de9af6SHans Petter Selasky * thread priority so that code gets run. The thread 300f3de9af6SHans Petter Selasky * priority will be restored later. 301f3de9af6SHans Petter Selasky */ 302f3de9af6SHans Petter Selasky sched_prio(td, 0); 303f3de9af6SHans Petter Selasky sched_bind(td, record->cpuid); 304f3de9af6SHans Petter Selasky } 305f3de9af6SHans Petter Selasky } 306f3de9af6SHans Petter Selasky 3071a01b4e5SHans Petter Selasky void 308eae5868cSHans Petter Selasky linux_synchronize_rcu(unsigned type) 3091a01b4e5SHans Petter Selasky { 310f3de9af6SHans Petter Selasky struct thread *td; 311f3de9af6SHans Petter Selasky int was_bound; 312f3de9af6SHans Petter Selasky int old_cpu; 313f3de9af6SHans Petter Selasky int old_pinned; 3143f743d78SHans Petter Selasky u_char old_prio; 315f3de9af6SHans Petter Selasky 316eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 317eae5868cSHans Petter Selasky 318f3de9af6SHans Petter Selasky if (RCU_SKIP()) 319f3de9af6SHans Petter Selasky return; 320f3de9af6SHans Petter Selasky 321f3de9af6SHans Petter Selasky WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 322f3de9af6SHans Petter Selasky "linux_synchronize_rcu() can sleep"); 323f3de9af6SHans Petter Selasky 324f3de9af6SHans Petter Selasky td = curthread; 325fedab1b4SKonstantin Belousov DROP_GIANT(); 326f3de9af6SHans Petter Selasky 327f3de9af6SHans Petter Selasky /* 328f3de9af6SHans Petter Selasky * Synchronizing RCU might change the CPU core this function 329f3de9af6SHans Petter Selasky * is running on. Save current values: 330f3de9af6SHans Petter Selasky */ 331f3de9af6SHans Petter Selasky thread_lock(td); 332f3de9af6SHans Petter Selasky 333f3de9af6SHans Petter Selasky old_cpu = PCPU_GET(cpuid); 334f3de9af6SHans Petter Selasky old_pinned = td->td_pinned; 3353f743d78SHans Petter Selasky old_prio = td->td_priority; 336f3de9af6SHans Petter Selasky was_bound = sched_is_bound(td); 337ea165254SHans Petter Selasky sched_unbind(td); 338ea165254SHans Petter Selasky td->td_pinned = 0; 339f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 340f3de9af6SHans Petter Selasky 341eae5868cSHans Petter Selasky ck_epoch_synchronize_wait(&linux_epoch[type], 342f3de9af6SHans Petter Selasky &linux_synchronize_rcu_cb, NULL); 343f3de9af6SHans Petter Selasky 344f3de9af6SHans Petter Selasky /* restore CPU binding, if any */ 345f3de9af6SHans Petter Selasky if (was_bound != 0) { 346f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 347f3de9af6SHans Petter Selasky } else { 348f3de9af6SHans Petter Selasky /* get thread back to initial CPU, if any */ 349f3de9af6SHans Petter Selasky if (old_pinned != 0) 350f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 351f3de9af6SHans Petter Selasky sched_unbind(td); 352f3de9af6SHans Petter Selasky } 353f3de9af6SHans Petter Selasky /* restore pinned after bind */ 354f3de9af6SHans Petter Selasky td->td_pinned = old_pinned; 3553f743d78SHans Petter Selasky 3563f743d78SHans Petter Selasky /* restore thread priority */ 3573f743d78SHans Petter Selasky sched_prio(td, old_prio); 358f3de9af6SHans Petter Selasky thread_unlock(td); 359f3de9af6SHans Petter Selasky 360f3de9af6SHans Petter Selasky PICKUP_GIANT(); 3611a01b4e5SHans Petter Selasky } 3621a01b4e5SHans Petter Selasky 3631a01b4e5SHans Petter Selasky void 364eae5868cSHans Petter Selasky linux_rcu_barrier(unsigned type) 3651a01b4e5SHans Petter Selasky { 366f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 3671a01b4e5SHans Petter Selasky 368eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 3691f827dabSHans Petter Selasky 370*1ab61a19SVladimir Kondratyev /* 371*1ab61a19SVladimir Kondratyev * This function is not obligated to wait for a grace period. 372*1ab61a19SVladimir Kondratyev * It only waits for RCU callbacks that have already been posted. 373*1ab61a19SVladimir Kondratyev * If there are no RCU callbacks posted, rcu_barrier() can return 374*1ab61a19SVladimir Kondratyev * immediately. 375*1ab61a19SVladimir Kondratyev */ 376eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3771f827dabSHans Petter Selasky 3781f827dabSHans Petter Selasky /* wait for callbacks to complete */ 379a2b83b59SVladimir Kondratyev taskqueue_drain(linux_irq_work_tq, &head->task); 3801a01b4e5SHans Petter Selasky } 3811a01b4e5SHans Petter Selasky 3821a01b4e5SHans Petter Selasky void 383eae5868cSHans Petter Selasky linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) 3841a01b4e5SHans Petter Selasky { 385eae5868cSHans Petter Selasky struct callback_head *rcu; 386eae5868cSHans Petter Selasky struct linux_epoch_head *head; 387eae5868cSHans Petter Selasky 388eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 389eae5868cSHans Petter Selasky 390eae5868cSHans Petter Selasky rcu = (struct callback_head *)context; 391eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3921a01b4e5SHans Petter Selasky 3931f827dabSHans Petter Selasky rcu->func = func; 394a2b83b59SVladimir Kondratyev llist_add(&rcu->node, &head->cb_head); 395a2b83b59SVladimir Kondratyev taskqueue_enqueue(linux_irq_work_tq, &head->task); 3961a01b4e5SHans Petter Selasky } 3971a01b4e5SHans Petter Selasky 3981a01b4e5SHans Petter Selasky int 3991a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu) 4001a01b4e5SHans Petter Selasky { 4011a01b4e5SHans Petter Selasky return (0); 4021a01b4e5SHans Petter Selasky } 4031a01b4e5SHans Petter Selasky 4041a01b4e5SHans Petter Selasky void 4051a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu) 4061a01b4e5SHans Petter Selasky { 4071a01b4e5SHans Petter Selasky } 4081a01b4e5SHans Petter Selasky 4091a01b4e5SHans Petter Selasky int 4101a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu) 4111a01b4e5SHans Petter Selasky { 412eae5868cSHans Petter Selasky linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); 4131a01b4e5SHans Petter Selasky return (0); 4141a01b4e5SHans Petter Selasky } 4151a01b4e5SHans Petter Selasky 4161a01b4e5SHans Petter Selasky void 4171a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused) 4181a01b4e5SHans Petter Selasky { 419eae5868cSHans Petter Selasky linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); 4201a01b4e5SHans Petter Selasky } 4211a01b4e5SHans Petter Selasky 4221a01b4e5SHans Petter Selasky void 4231a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu) 4241a01b4e5SHans Petter Selasky { 425eae5868cSHans Petter Selasky linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); 4261f827dabSHans Petter Selasky } 4271f827dabSHans Petter Selasky 4281f827dabSHans Petter Selasky void 4291f827dabSHans Petter Selasky srcu_barrier(struct srcu_struct *srcu) 4301f827dabSHans Petter Selasky { 431eae5868cSHans Petter Selasky linux_rcu_barrier(RCU_TYPE_SLEEPABLE); 4321a01b4e5SHans Petter Selasky } 433