11a01b4e5SHans Petter Selasky /*- 296fc97c8SStephen Hurd * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) 317777208SHans Petter Selasky * Copyright (c) 2017-2021 Hans Petter Selasky (hselasky@freebsd.org) 41a01b4e5SHans Petter Selasky * All rights reserved. 51a01b4e5SHans Petter Selasky * 61a01b4e5SHans Petter Selasky * Redistribution and use in source and binary forms, with or without 71a01b4e5SHans Petter Selasky * modification, are permitted provided that the following conditions 81a01b4e5SHans Petter Selasky * are met: 91a01b4e5SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright 101a01b4e5SHans Petter Selasky * notice unmodified, this list of conditions, and the following 111a01b4e5SHans Petter Selasky * disclaimer. 121a01b4e5SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright 131a01b4e5SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the 141a01b4e5SHans Petter Selasky * documentation and/or other materials provided with the distribution. 151a01b4e5SHans Petter Selasky * 161a01b4e5SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 171a01b4e5SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 181a01b4e5SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 191a01b4e5SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 201a01b4e5SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 211a01b4e5SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 221a01b4e5SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 231a01b4e5SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 241a01b4e5SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 251a01b4e5SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 261a01b4e5SHans Petter Selasky */ 271a01b4e5SHans Petter Selasky 281a01b4e5SHans Petter Selasky #include <sys/types.h> 291a01b4e5SHans Petter Selasky #include <sys/systm.h> 301a01b4e5SHans Petter Selasky #include <sys/malloc.h> 311a01b4e5SHans Petter Selasky #include <sys/kernel.h> 321a01b4e5SHans Petter Selasky #include <sys/lock.h> 331a01b4e5SHans Petter Selasky #include <sys/mutex.h> 341a01b4e5SHans Petter Selasky #include <sys/proc.h> 351a01b4e5SHans Petter Selasky #include <sys/sched.h> 361a01b4e5SHans Petter Selasky #include <sys/smp.h> 371a01b4e5SHans Petter Selasky #include <sys/queue.h> 381a01b4e5SHans Petter Selasky #include <sys/taskqueue.h> 39f3de9af6SHans Petter Selasky #include <sys/kdb.h> 401a01b4e5SHans Petter Selasky 411a01b4e5SHans Petter Selasky #include <ck_epoch.h> 421a01b4e5SHans Petter Selasky 431a01b4e5SHans Petter Selasky #include <linux/rcupdate.h> 44*256eb8d5SVladimir Kondratyev #include <linux/sched.h> 451a01b4e5SHans Petter Selasky #include <linux/srcu.h> 461a01b4e5SHans Petter Selasky #include <linux/slab.h> 471a01b4e5SHans Petter Selasky #include <linux/kernel.h> 48f3de9af6SHans Petter Selasky #include <linux/compat.h> 49a2b83b59SVladimir Kondratyev #include <linux/llist.h> 50a2b83b59SVladimir Kondratyev #include <linux/irq_work.h> 511a01b4e5SHans Petter Selasky 52f3de9af6SHans Petter Selasky /* 53f3de9af6SHans Petter Selasky * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will 54f3de9af6SHans Petter Selasky * not be skipped during panic(). 55f3de9af6SHans Petter Selasky */ 56f3de9af6SHans Petter Selasky #ifdef CONFIG_NO_RCU_SKIP 57f3de9af6SHans Petter Selasky #define RCU_SKIP(void) 0 58f3de9af6SHans Petter Selasky #else 59f3de9af6SHans Petter Selasky #define RCU_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) 60f3de9af6SHans Petter Selasky #endif 611f827dabSHans Petter Selasky 621f827dabSHans Petter Selasky struct callback_head { 63a2b83b59SVladimir Kondratyev union { 641f827dabSHans Petter Selasky STAILQ_ENTRY(callback_head) entry; 65a2b83b59SVladimir Kondratyev struct llist_node node; 66a2b83b59SVladimir Kondratyev }; 671f827dabSHans Petter Selasky rcu_callback_t func; 681f827dabSHans Petter Selasky }; 691f827dabSHans Petter Selasky 70f3de9af6SHans Petter Selasky struct linux_epoch_head { 71a2b83b59SVladimir Kondratyev struct llist_head cb_head; 72f3de9af6SHans Petter Selasky struct task task; 73f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 74f3de9af6SHans Petter Selasky 75f3de9af6SHans Petter Selasky struct linux_epoch_record { 761f827dabSHans Petter Selasky ck_epoch_record_t epoch_record; 77f3de9af6SHans Petter Selasky TAILQ_HEAD(, task_struct) ts_head; 78f3de9af6SHans Petter Selasky int cpuid; 796ae24079SHans Petter Selasky int type; 80f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE); 811a01b4e5SHans Petter Selasky 821a01b4e5SHans Petter Selasky /* 831a01b4e5SHans Petter Selasky * Verify that "struct rcu_head" is big enough to hold "struct 841a01b4e5SHans Petter Selasky * callback_head". This has been done to avoid having to add special 851a01b4e5SHans Petter Selasky * compile flags for including ck_epoch.h to all clients of the 861a01b4e5SHans Petter Selasky * LinuxKPI. 871a01b4e5SHans Petter Selasky */ 88f3de9af6SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head)); 891a01b4e5SHans Petter Selasky 901f827dabSHans Petter Selasky /* 9117777208SHans Petter Selasky * Verify that "rcu_section[0]" has the same size as 9217777208SHans Petter Selasky * "ck_epoch_section_t". This has been done to avoid having to add 9317777208SHans Petter Selasky * special compile flags for including ck_epoch.h to all clients of 9417777208SHans Petter Selasky * the LinuxKPI. 9517777208SHans Petter Selasky */ 9617777208SHans Petter Selasky CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] == 9717777208SHans Petter Selasky sizeof(ck_epoch_section_t))); 9817777208SHans Petter Selasky 9917777208SHans Petter Selasky /* 1001f827dabSHans Petter Selasky * Verify that "epoch_record" is at beginning of "struct 101f3de9af6SHans Petter Selasky * linux_epoch_record": 1021f827dabSHans Petter Selasky */ 103f3de9af6SHans Petter Selasky CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); 1041f827dabSHans Petter Selasky 1056ae24079SHans Petter Selasky CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX); 1066ae24079SHans Petter Selasky 107eae5868cSHans Petter Selasky static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; 108eae5868cSHans Petter Selasky static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; 109eae5868cSHans Petter Selasky DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); 1101f827dabSHans Petter Selasky 1111f827dabSHans Petter Selasky static void linux_rcu_cleaner_func(void *, int); 1121a01b4e5SHans Petter Selasky 1131a01b4e5SHans Petter Selasky static void 1141a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused) 1151a01b4e5SHans Petter Selasky { 116f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 1171a01b4e5SHans Petter Selasky int i; 118eae5868cSHans Petter Selasky int j; 1191a01b4e5SHans Petter Selasky 120eae5868cSHans Petter Selasky for (j = 0; j != RCU_TYPE_MAX; j++) { 121eae5868cSHans Petter Selasky ck_epoch_init(&linux_epoch[j]); 1221a01b4e5SHans Petter Selasky 123eae5868cSHans Petter Selasky head = &linux_epoch_head[j]; 124f3de9af6SHans Petter Selasky 125eae5868cSHans Petter Selasky TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); 126a2b83b59SVladimir Kondratyev init_llist_head(&head->cb_head); 127f3de9af6SHans Petter Selasky 1281a01b4e5SHans Petter Selasky CPU_FOREACH(i) { 129f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 1301f827dabSHans Petter Selasky 131eae5868cSHans Petter Selasky record = &DPCPU_ID_GET(i, linux_epoch_record[j]); 1321f827dabSHans Petter Selasky 133f3de9af6SHans Petter Selasky record->cpuid = i; 1346ae24079SHans Petter Selasky record->type = j; 135eae5868cSHans Petter Selasky ck_epoch_register(&linux_epoch[j], 136eae5868cSHans Petter Selasky &record->epoch_record, NULL); 137f3de9af6SHans Petter Selasky TAILQ_INIT(&record->ts_head); 1381a01b4e5SHans Petter Selasky } 1391a01b4e5SHans Petter Selasky } 140eae5868cSHans Petter Selasky } 141d8e073a9SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); 1421a01b4e5SHans Petter Selasky 1431a01b4e5SHans Petter Selasky static void 144eae5868cSHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused) 1451f827dabSHans Petter Selasky { 146a2b83b59SVladimir Kondratyev struct linux_epoch_head *head = context; 1471a01b4e5SHans Petter Selasky struct callback_head *rcu; 148f3de9af6SHans Petter Selasky STAILQ_HEAD(, callback_head) tmp_head; 149a2b83b59SVladimir Kondratyev struct llist_node *node, *next; 150eae5868cSHans Petter Selasky uintptr_t offset; 1511f827dabSHans Petter Selasky 1521f827dabSHans Petter Selasky /* move current callbacks into own queue */ 153f3de9af6SHans Petter Selasky STAILQ_INIT(&tmp_head); 154a2b83b59SVladimir Kondratyev llist_for_each_safe(node, next, llist_del_all(&head->cb_head)) { 155a2b83b59SVladimir Kondratyev rcu = container_of(node, struct callback_head, node); 156a2b83b59SVladimir Kondratyev /* re-reverse list to restore chronological order */ 157a2b83b59SVladimir Kondratyev STAILQ_INSERT_HEAD(&tmp_head, rcu, entry); 158a2b83b59SVladimir Kondratyev } 1591f827dabSHans Petter Selasky 1601f827dabSHans Petter Selasky /* synchronize */ 161eae5868cSHans Petter Selasky linux_synchronize_rcu(head - linux_epoch_head); 1621f827dabSHans Petter Selasky 1631f827dabSHans Petter Selasky /* dispatch all callbacks, if any */ 164f3de9af6SHans Petter Selasky while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { 165f3de9af6SHans Petter Selasky STAILQ_REMOVE_HEAD(&tmp_head, entry); 1661a01b4e5SHans Petter Selasky 1671a01b4e5SHans Petter Selasky offset = (uintptr_t)rcu->func; 1681a01b4e5SHans Petter Selasky 1691a01b4e5SHans Petter Selasky if (offset < LINUX_KFREE_RCU_OFFSET_MAX) 1701a01b4e5SHans Petter Selasky kfree((char *)rcu - offset); 1711a01b4e5SHans Petter Selasky else 1721a01b4e5SHans Petter Selasky rcu->func((struct rcu_head *)rcu); 1731a01b4e5SHans Petter Selasky } 1741a01b4e5SHans Petter Selasky } 1751a01b4e5SHans Petter Selasky 1761a01b4e5SHans Petter Selasky void 177eae5868cSHans Petter Selasky linux_rcu_read_lock(unsigned type) 1781a01b4e5SHans Petter Selasky { 179f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 180f3de9af6SHans Petter Selasky struct task_struct *ts; 181f3de9af6SHans Petter Selasky 182eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 183eae5868cSHans Petter Selasky 184f3de9af6SHans Petter Selasky if (RCU_SKIP()) 185f3de9af6SHans Petter Selasky return; 1861a01b4e5SHans Petter Selasky 18717777208SHans Petter Selasky ts = current; 18817777208SHans Petter Selasky 18917777208SHans Petter Selasky /* assert valid refcount */ 19017777208SHans Petter Selasky MPASS(ts->rcu_recurse[type] != INT_MAX); 19117777208SHans Petter Selasky 19217777208SHans Petter Selasky if (++(ts->rcu_recurse[type]) != 1) 19317777208SHans Petter Selasky return; 19417777208SHans Petter Selasky 1951f827dabSHans Petter Selasky /* 1961f827dabSHans Petter Selasky * Pin thread to current CPU so that the unlock code gets the 197f3de9af6SHans Petter Selasky * same per-CPU epoch record: 1981f827dabSHans Petter Selasky */ 1991a01b4e5SHans Petter Selasky sched_pin(); 2001a01b4e5SHans Petter Selasky 201eae5868cSHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 2021f827dabSHans Petter Selasky 2031f827dabSHans Petter Selasky /* 2041f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2051f827dabSHans Petter Selasky * ck_epoch_begin(). Else this function supports recursion. 2061f827dabSHans Petter Selasky */ 2071f827dabSHans Petter Selasky critical_enter(); 20817777208SHans Petter Selasky ck_epoch_begin(&record->epoch_record, 20917777208SHans Petter Selasky (ck_epoch_section_t *)&ts->rcu_section[type]); 2106ae24079SHans Petter Selasky TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]); 2111f827dabSHans Petter Selasky critical_exit(); 2121a01b4e5SHans Petter Selasky } 2131a01b4e5SHans Petter Selasky 2141a01b4e5SHans Petter Selasky void 215eae5868cSHans Petter Selasky linux_rcu_read_unlock(unsigned type) 2161a01b4e5SHans Petter Selasky { 217f3de9af6SHans Petter Selasky struct linux_epoch_record *record; 218f3de9af6SHans Petter Selasky struct task_struct *ts; 2191a01b4e5SHans Petter Selasky 220eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 221eae5868cSHans Petter Selasky 222f3de9af6SHans Petter Selasky if (RCU_SKIP()) 223f3de9af6SHans Petter Selasky return; 224f3de9af6SHans Petter Selasky 225f3de9af6SHans Petter Selasky ts = current; 2261f827dabSHans Petter Selasky 22717777208SHans Petter Selasky /* assert valid refcount */ 22817777208SHans Petter Selasky MPASS(ts->rcu_recurse[type] > 0); 22917777208SHans Petter Selasky 23017777208SHans Petter Selasky if (--(ts->rcu_recurse[type]) != 0) 23117777208SHans Petter Selasky return; 23217777208SHans Petter Selasky 23317777208SHans Petter Selasky record = &DPCPU_GET(linux_epoch_record[type]); 23417777208SHans Petter Selasky 2351f827dabSHans Petter Selasky /* 2361f827dabSHans Petter Selasky * Use a critical section to prevent recursion inside 2371f827dabSHans Petter Selasky * ck_epoch_end(). Else this function supports recursion. 2381f827dabSHans Petter Selasky */ 2391f827dabSHans Petter Selasky critical_enter(); 24017777208SHans Petter Selasky ck_epoch_end(&record->epoch_record, 24117777208SHans Petter Selasky (ck_epoch_section_t *)&ts->rcu_section[type]); 2426ae24079SHans Petter Selasky TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]); 2431f827dabSHans Petter Selasky critical_exit(); 2441f827dabSHans Petter Selasky 2451a01b4e5SHans Petter Selasky sched_unpin(); 2461a01b4e5SHans Petter Selasky } 2471a01b4e5SHans Petter Selasky 248f3de9af6SHans Petter Selasky static void 249f3de9af6SHans Petter Selasky linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused) 250f3de9af6SHans Petter Selasky { 251f3de9af6SHans Petter Selasky struct linux_epoch_record *record = 252f3de9af6SHans Petter Selasky container_of(epoch_record, struct linux_epoch_record, epoch_record); 253f3de9af6SHans Petter Selasky struct thread *td = curthread; 254f3de9af6SHans Petter Selasky struct task_struct *ts; 255f3de9af6SHans Petter Selasky 256f3de9af6SHans Petter Selasky /* check if blocked on the current CPU */ 257f3de9af6SHans Petter Selasky if (record->cpuid == PCPU_GET(cpuid)) { 258f3de9af6SHans Petter Selasky bool is_sleeping = 0; 259f3de9af6SHans Petter Selasky u_char prio = 0; 260f3de9af6SHans Petter Selasky 261f3de9af6SHans Petter Selasky /* 262f3de9af6SHans Petter Selasky * Find the lowest priority or sleeping thread which 263f3de9af6SHans Petter Selasky * is blocking synchronization on this CPU core. All 264f3de9af6SHans Petter Selasky * the threads in the queue are CPU-pinned and cannot 265f3de9af6SHans Petter Selasky * go anywhere while the current thread is locked. 266f3de9af6SHans Petter Selasky */ 2676ae24079SHans Petter Selasky TAILQ_FOREACH(ts, &record->ts_head, rcu_entry[record->type]) { 268f3de9af6SHans Petter Selasky if (ts->task_thread->td_priority > prio) 269f3de9af6SHans Petter Selasky prio = ts->task_thread->td_priority; 270f3de9af6SHans Petter Selasky is_sleeping |= (ts->task_thread->td_inhibitors != 0); 271f3de9af6SHans Petter Selasky } 272f3de9af6SHans Petter Selasky 273f3de9af6SHans Petter Selasky if (is_sleeping) { 274f3de9af6SHans Petter Selasky thread_unlock(td); 275f3de9af6SHans Petter Selasky pause("W", 1); 276f3de9af6SHans Petter Selasky thread_lock(td); 277f3de9af6SHans Petter Selasky } else { 278f3de9af6SHans Petter Selasky /* set new thread priority */ 279f3de9af6SHans Petter Selasky sched_prio(td, prio); 280f3de9af6SHans Petter Selasky /* task switch */ 281686bcb5cSJeff Roberson mi_switch(SW_VOL | SWT_RELINQUISH); 282cc79ea3aSHans Petter Selasky /* 283cc79ea3aSHans Petter Selasky * It is important the thread lock is dropped 284cc79ea3aSHans Petter Selasky * while yielding to allow other threads to 285cc79ea3aSHans Petter Selasky * acquire the lock pointed to by 286cc79ea3aSHans Petter Selasky * TDQ_LOCKPTR(td). Currently mi_switch() will 287cc79ea3aSHans Petter Selasky * unlock the thread lock before 288cc79ea3aSHans Petter Selasky * returning. Else a deadlock like situation 289cc79ea3aSHans Petter Selasky * might happen. 290cc79ea3aSHans Petter Selasky */ 291714ed5b2SHans Petter Selasky thread_lock(td); 292f3de9af6SHans Petter Selasky } 293f3de9af6SHans Petter Selasky } else { 294f3de9af6SHans Petter Selasky /* 295f3de9af6SHans Petter Selasky * To avoid spinning move execution to the other CPU 296f3de9af6SHans Petter Selasky * which is blocking synchronization. Set highest 297f3de9af6SHans Petter Selasky * thread priority so that code gets run. The thread 298f3de9af6SHans Petter Selasky * priority will be restored later. 299f3de9af6SHans Petter Selasky */ 300f3de9af6SHans Petter Selasky sched_prio(td, 0); 301f3de9af6SHans Petter Selasky sched_bind(td, record->cpuid); 302f3de9af6SHans Petter Selasky } 303f3de9af6SHans Petter Selasky } 304f3de9af6SHans Petter Selasky 3051a01b4e5SHans Petter Selasky void 306eae5868cSHans Petter Selasky linux_synchronize_rcu(unsigned type) 3071a01b4e5SHans Petter Selasky { 308f3de9af6SHans Petter Selasky struct thread *td; 309f3de9af6SHans Petter Selasky int was_bound; 310f3de9af6SHans Petter Selasky int old_cpu; 311f3de9af6SHans Petter Selasky int old_pinned; 3123f743d78SHans Petter Selasky u_char old_prio; 313f3de9af6SHans Petter Selasky 314eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 315eae5868cSHans Petter Selasky 316f3de9af6SHans Petter Selasky if (RCU_SKIP()) 317f3de9af6SHans Petter Selasky return; 318f3de9af6SHans Petter Selasky 319f3de9af6SHans Petter Selasky WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 320f3de9af6SHans Petter Selasky "linux_synchronize_rcu() can sleep"); 321f3de9af6SHans Petter Selasky 322f3de9af6SHans Petter Selasky td = curthread; 323fedab1b4SKonstantin Belousov DROP_GIANT(); 324f3de9af6SHans Petter Selasky 325f3de9af6SHans Petter Selasky /* 326f3de9af6SHans Petter Selasky * Synchronizing RCU might change the CPU core this function 327f3de9af6SHans Petter Selasky * is running on. Save current values: 328f3de9af6SHans Petter Selasky */ 329f3de9af6SHans Petter Selasky thread_lock(td); 330f3de9af6SHans Petter Selasky 331f3de9af6SHans Petter Selasky old_cpu = PCPU_GET(cpuid); 332f3de9af6SHans Petter Selasky old_pinned = td->td_pinned; 3333f743d78SHans Petter Selasky old_prio = td->td_priority; 334f3de9af6SHans Petter Selasky was_bound = sched_is_bound(td); 335ea165254SHans Petter Selasky sched_unbind(td); 336ea165254SHans Petter Selasky td->td_pinned = 0; 337f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 338f3de9af6SHans Petter Selasky 339eae5868cSHans Petter Selasky ck_epoch_synchronize_wait(&linux_epoch[type], 340f3de9af6SHans Petter Selasky &linux_synchronize_rcu_cb, NULL); 341f3de9af6SHans Petter Selasky 342f3de9af6SHans Petter Selasky /* restore CPU binding, if any */ 343f3de9af6SHans Petter Selasky if (was_bound != 0) { 344f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 345f3de9af6SHans Petter Selasky } else { 346f3de9af6SHans Petter Selasky /* get thread back to initial CPU, if any */ 347f3de9af6SHans Petter Selasky if (old_pinned != 0) 348f3de9af6SHans Petter Selasky sched_bind(td, old_cpu); 349f3de9af6SHans Petter Selasky sched_unbind(td); 350f3de9af6SHans Petter Selasky } 351f3de9af6SHans Petter Selasky /* restore pinned after bind */ 352f3de9af6SHans Petter Selasky td->td_pinned = old_pinned; 3533f743d78SHans Petter Selasky 3543f743d78SHans Petter Selasky /* restore thread priority */ 3553f743d78SHans Petter Selasky sched_prio(td, old_prio); 356f3de9af6SHans Petter Selasky thread_unlock(td); 357f3de9af6SHans Petter Selasky 358f3de9af6SHans Petter Selasky PICKUP_GIANT(); 3591a01b4e5SHans Petter Selasky } 3601a01b4e5SHans Petter Selasky 3611a01b4e5SHans Petter Selasky void 362eae5868cSHans Petter Selasky linux_rcu_barrier(unsigned type) 3631a01b4e5SHans Petter Selasky { 364f3de9af6SHans Petter Selasky struct linux_epoch_head *head; 3651a01b4e5SHans Petter Selasky 366eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 3671f827dabSHans Petter Selasky 3681ab61a19SVladimir Kondratyev /* 3691ab61a19SVladimir Kondratyev * This function is not obligated to wait for a grace period. 3701ab61a19SVladimir Kondratyev * It only waits for RCU callbacks that have already been posted. 3711ab61a19SVladimir Kondratyev * If there are no RCU callbacks posted, rcu_barrier() can return 3721ab61a19SVladimir Kondratyev * immediately. 3731ab61a19SVladimir Kondratyev */ 374eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3751f827dabSHans Petter Selasky 3761f827dabSHans Petter Selasky /* wait for callbacks to complete */ 377a2b83b59SVladimir Kondratyev taskqueue_drain(linux_irq_work_tq, &head->task); 3781a01b4e5SHans Petter Selasky } 3791a01b4e5SHans Petter Selasky 3801a01b4e5SHans Petter Selasky void 381eae5868cSHans Petter Selasky linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) 3821a01b4e5SHans Petter Selasky { 383eae5868cSHans Petter Selasky struct callback_head *rcu; 384eae5868cSHans Petter Selasky struct linux_epoch_head *head; 385eae5868cSHans Petter Selasky 386eae5868cSHans Petter Selasky MPASS(type < RCU_TYPE_MAX); 387eae5868cSHans Petter Selasky 388eae5868cSHans Petter Selasky rcu = (struct callback_head *)context; 389eae5868cSHans Petter Selasky head = &linux_epoch_head[type]; 3901a01b4e5SHans Petter Selasky 3911f827dabSHans Petter Selasky rcu->func = func; 392a2b83b59SVladimir Kondratyev llist_add(&rcu->node, &head->cb_head); 393a2b83b59SVladimir Kondratyev taskqueue_enqueue(linux_irq_work_tq, &head->task); 3941a01b4e5SHans Petter Selasky } 3951a01b4e5SHans Petter Selasky 3961a01b4e5SHans Petter Selasky int 3971a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu) 3981a01b4e5SHans Petter Selasky { 3991a01b4e5SHans Petter Selasky return (0); 4001a01b4e5SHans Petter Selasky } 4011a01b4e5SHans Petter Selasky 4021a01b4e5SHans Petter Selasky void 4031a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu) 4041a01b4e5SHans Petter Selasky { 4051a01b4e5SHans Petter Selasky } 4061a01b4e5SHans Petter Selasky 4071a01b4e5SHans Petter Selasky int 4081a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu) 4091a01b4e5SHans Petter Selasky { 410eae5868cSHans Petter Selasky linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); 4111a01b4e5SHans Petter Selasky return (0); 4121a01b4e5SHans Petter Selasky } 4131a01b4e5SHans Petter Selasky 4141a01b4e5SHans Petter Selasky void 4151a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused) 4161a01b4e5SHans Petter Selasky { 417eae5868cSHans Petter Selasky linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); 4181a01b4e5SHans Petter Selasky } 4191a01b4e5SHans Petter Selasky 4201a01b4e5SHans Petter Selasky void 4211a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu) 4221a01b4e5SHans Petter Selasky { 423eae5868cSHans Petter Selasky linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); 4241f827dabSHans Petter Selasky } 4251f827dabSHans Petter Selasky 4261f827dabSHans Petter Selasky void 4271f827dabSHans Petter Selasky srcu_barrier(struct srcu_struct *srcu) 4281f827dabSHans Petter Selasky { 429eae5868cSHans Petter Selasky linux_rcu_barrier(RCU_TYPE_SLEEPABLE); 4301a01b4e5SHans Petter Selasky } 431