xref: /freebsd/sys/compat/linuxkpi/common/src/linux_rcu.c (revision 1ab61a193241f832e63f97ca37880b010469cb38)
11a01b4e5SHans Petter Selasky /*-
296fc97c8SStephen Hurd  * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
317777208SHans Petter Selasky  * Copyright (c) 2017-2021 Hans Petter Selasky (hselasky@freebsd.org)
41a01b4e5SHans Petter Selasky  * All rights reserved.
51a01b4e5SHans Petter Selasky  *
61a01b4e5SHans Petter Selasky  * Redistribution and use in source and binary forms, with or without
71a01b4e5SHans Petter Selasky  * modification, are permitted provided that the following conditions
81a01b4e5SHans Petter Selasky  * are met:
91a01b4e5SHans Petter Selasky  * 1. Redistributions of source code must retain the above copyright
101a01b4e5SHans Petter Selasky  *    notice unmodified, this list of conditions, and the following
111a01b4e5SHans Petter Selasky  *    disclaimer.
121a01b4e5SHans Petter Selasky  * 2. Redistributions in binary form must reproduce the above copyright
131a01b4e5SHans Petter Selasky  *    notice, this list of conditions and the following disclaimer in the
141a01b4e5SHans Petter Selasky  *    documentation and/or other materials provided with the distribution.
151a01b4e5SHans Petter Selasky  *
161a01b4e5SHans Petter Selasky  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
171a01b4e5SHans Petter Selasky  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
181a01b4e5SHans Petter Selasky  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
191a01b4e5SHans Petter Selasky  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
201a01b4e5SHans Petter Selasky  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
211a01b4e5SHans Petter Selasky  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
221a01b4e5SHans Petter Selasky  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
231a01b4e5SHans Petter Selasky  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
241a01b4e5SHans Petter Selasky  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
251a01b4e5SHans Petter Selasky  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
261a01b4e5SHans Petter Selasky  */
271a01b4e5SHans Petter Selasky 
281a01b4e5SHans Petter Selasky #include <sys/cdefs.h>
291a01b4e5SHans Petter Selasky __FBSDID("$FreeBSD$");
301a01b4e5SHans Petter Selasky 
311a01b4e5SHans Petter Selasky #include <sys/types.h>
321a01b4e5SHans Petter Selasky #include <sys/systm.h>
331a01b4e5SHans Petter Selasky #include <sys/malloc.h>
341a01b4e5SHans Petter Selasky #include <sys/kernel.h>
351a01b4e5SHans Petter Selasky #include <sys/lock.h>
361a01b4e5SHans Petter Selasky #include <sys/mutex.h>
371a01b4e5SHans Petter Selasky #include <sys/proc.h>
381a01b4e5SHans Petter Selasky #include <sys/sched.h>
391a01b4e5SHans Petter Selasky #include <sys/smp.h>
401a01b4e5SHans Petter Selasky #include <sys/queue.h>
411a01b4e5SHans Petter Selasky #include <sys/taskqueue.h>
42f3de9af6SHans Petter Selasky #include <sys/kdb.h>
431a01b4e5SHans Petter Selasky 
441a01b4e5SHans Petter Selasky #include <ck_epoch.h>
451a01b4e5SHans Petter Selasky 
461a01b4e5SHans Petter Selasky #include <linux/rcupdate.h>
471a01b4e5SHans Petter Selasky #include <linux/srcu.h>
481a01b4e5SHans Petter Selasky #include <linux/slab.h>
491a01b4e5SHans Petter Selasky #include <linux/kernel.h>
50f3de9af6SHans Petter Selasky #include <linux/compat.h>
51a2b83b59SVladimir Kondratyev #include <linux/llist.h>
52a2b83b59SVladimir Kondratyev #include <linux/irq_work.h>
531a01b4e5SHans Petter Selasky 
54f3de9af6SHans Petter Selasky /*
55f3de9af6SHans Petter Selasky  * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will
56f3de9af6SHans Petter Selasky  * not be skipped during panic().
57f3de9af6SHans Petter Selasky  */
58f3de9af6SHans Petter Selasky #ifdef CONFIG_NO_RCU_SKIP
59f3de9af6SHans Petter Selasky #define	RCU_SKIP(void) 0
60f3de9af6SHans Petter Selasky #else
61f3de9af6SHans Petter Selasky #define	RCU_SKIP(void)	unlikely(SCHEDULER_STOPPED() || kdb_active)
62f3de9af6SHans Petter Selasky #endif
631f827dabSHans Petter Selasky 
641f827dabSHans Petter Selasky struct callback_head {
65a2b83b59SVladimir Kondratyev 	union {
661f827dabSHans Petter Selasky 		STAILQ_ENTRY(callback_head) entry;
67a2b83b59SVladimir Kondratyev 		struct llist_node node;
68a2b83b59SVladimir Kondratyev 	};
691f827dabSHans Petter Selasky 	rcu_callback_t func;
701f827dabSHans Petter Selasky };
711f827dabSHans Petter Selasky 
72f3de9af6SHans Petter Selasky struct linux_epoch_head {
73a2b83b59SVladimir Kondratyev 	struct llist_head cb_head;
74f3de9af6SHans Petter Selasky 	struct task task;
75f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE);
76f3de9af6SHans Petter Selasky 
77f3de9af6SHans Petter Selasky struct linux_epoch_record {
781f827dabSHans Petter Selasky 	ck_epoch_record_t epoch_record;
79f3de9af6SHans Petter Selasky 	TAILQ_HEAD(, task_struct) ts_head;
80f3de9af6SHans Petter Selasky 	int cpuid;
816ae24079SHans Petter Selasky 	int type;
82f3de9af6SHans Petter Selasky } __aligned(CACHE_LINE_SIZE);
831a01b4e5SHans Petter Selasky 
841a01b4e5SHans Petter Selasky /*
851a01b4e5SHans Petter Selasky  * Verify that "struct rcu_head" is big enough to hold "struct
861a01b4e5SHans Petter Selasky  * callback_head". This has been done to avoid having to add special
871a01b4e5SHans Petter Selasky  * compile flags for including ck_epoch.h to all clients of the
881a01b4e5SHans Petter Selasky  * LinuxKPI.
891a01b4e5SHans Petter Selasky  */
90f3de9af6SHans Petter Selasky CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head));
911a01b4e5SHans Petter Selasky 
921f827dabSHans Petter Selasky /*
9317777208SHans Petter Selasky  * Verify that "rcu_section[0]" has the same size as
9417777208SHans Petter Selasky  * "ck_epoch_section_t". This has been done to avoid having to add
9517777208SHans Petter Selasky  * special compile flags for including ck_epoch.h to all clients of
9617777208SHans Petter Selasky  * the LinuxKPI.
9717777208SHans Petter Selasky  */
9817777208SHans Petter Selasky CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] ==
9917777208SHans Petter Selasky     sizeof(ck_epoch_section_t)));
10017777208SHans Petter Selasky 
10117777208SHans Petter Selasky /*
1021f827dabSHans Petter Selasky  * Verify that "epoch_record" is at beginning of "struct
103f3de9af6SHans Petter Selasky  * linux_epoch_record":
1041f827dabSHans Petter Selasky  */
105f3de9af6SHans Petter Selasky CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);
1061f827dabSHans Petter Selasky 
1076ae24079SHans Petter Selasky CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX);
1086ae24079SHans Petter Selasky 
109eae5868cSHans Petter Selasky static ck_epoch_t linux_epoch[RCU_TYPE_MAX];
110eae5868cSHans Petter Selasky static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX];
111eae5868cSHans Petter Selasky DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]);
1121f827dabSHans Petter Selasky 
1131f827dabSHans Petter Selasky static void linux_rcu_cleaner_func(void *, int);
1141a01b4e5SHans Petter Selasky 
1151a01b4e5SHans Petter Selasky static void
1161a01b4e5SHans Petter Selasky linux_rcu_runtime_init(void *arg __unused)
1171a01b4e5SHans Petter Selasky {
118f3de9af6SHans Petter Selasky 	struct linux_epoch_head *head;
1191a01b4e5SHans Petter Selasky 	int i;
120eae5868cSHans Petter Selasky 	int j;
1211a01b4e5SHans Petter Selasky 
122eae5868cSHans Petter Selasky 	for (j = 0; j != RCU_TYPE_MAX; j++) {
123eae5868cSHans Petter Selasky 		ck_epoch_init(&linux_epoch[j]);
1241a01b4e5SHans Petter Selasky 
125eae5868cSHans Petter Selasky 		head = &linux_epoch_head[j];
126f3de9af6SHans Petter Selasky 
127eae5868cSHans Petter Selasky 		TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head);
128a2b83b59SVladimir Kondratyev 		init_llist_head(&head->cb_head);
129f3de9af6SHans Petter Selasky 
1301a01b4e5SHans Petter Selasky 		CPU_FOREACH(i) {
131f3de9af6SHans Petter Selasky 			struct linux_epoch_record *record;
1321f827dabSHans Petter Selasky 
133eae5868cSHans Petter Selasky 			record = &DPCPU_ID_GET(i, linux_epoch_record[j]);
1341f827dabSHans Petter Selasky 
135f3de9af6SHans Petter Selasky 			record->cpuid = i;
1366ae24079SHans Petter Selasky 			record->type = j;
137eae5868cSHans Petter Selasky 			ck_epoch_register(&linux_epoch[j],
138eae5868cSHans Petter Selasky 			    &record->epoch_record, NULL);
139f3de9af6SHans Petter Selasky 			TAILQ_INIT(&record->ts_head);
1401a01b4e5SHans Petter Selasky 		}
1411a01b4e5SHans Petter Selasky 	}
142eae5868cSHans Petter Selasky }
143d8e073a9SHans Petter Selasky SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);
1441a01b4e5SHans Petter Selasky 
1451a01b4e5SHans Petter Selasky static void
146eae5868cSHans Petter Selasky linux_rcu_cleaner_func(void *context, int pending __unused)
1471f827dabSHans Petter Selasky {
148a2b83b59SVladimir Kondratyev 	struct linux_epoch_head *head = context;
1491a01b4e5SHans Petter Selasky 	struct callback_head *rcu;
150f3de9af6SHans Petter Selasky 	STAILQ_HEAD(, callback_head) tmp_head;
151a2b83b59SVladimir Kondratyev 	struct llist_node *node, *next;
152eae5868cSHans Petter Selasky 	uintptr_t offset;
1531f827dabSHans Petter Selasky 
1541f827dabSHans Petter Selasky 	/* move current callbacks into own queue */
155f3de9af6SHans Petter Selasky 	STAILQ_INIT(&tmp_head);
156a2b83b59SVladimir Kondratyev 	llist_for_each_safe(node, next, llist_del_all(&head->cb_head)) {
157a2b83b59SVladimir Kondratyev 		rcu = container_of(node, struct callback_head, node);
158a2b83b59SVladimir Kondratyev 		/* re-reverse list to restore chronological order */
159a2b83b59SVladimir Kondratyev 		STAILQ_INSERT_HEAD(&tmp_head, rcu, entry);
160a2b83b59SVladimir Kondratyev 	}
1611f827dabSHans Petter Selasky 
1621f827dabSHans Petter Selasky 	/* synchronize */
163eae5868cSHans Petter Selasky 	linux_synchronize_rcu(head - linux_epoch_head);
1641f827dabSHans Petter Selasky 
1651f827dabSHans Petter Selasky 	/* dispatch all callbacks, if any */
166f3de9af6SHans Petter Selasky 	while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) {
167f3de9af6SHans Petter Selasky 		STAILQ_REMOVE_HEAD(&tmp_head, entry);
1681a01b4e5SHans Petter Selasky 
1691a01b4e5SHans Petter Selasky 		offset = (uintptr_t)rcu->func;
1701a01b4e5SHans Petter Selasky 
1711a01b4e5SHans Petter Selasky 		if (offset < LINUX_KFREE_RCU_OFFSET_MAX)
1721a01b4e5SHans Petter Selasky 			kfree((char *)rcu - offset);
1731a01b4e5SHans Petter Selasky 		else
1741a01b4e5SHans Petter Selasky 			rcu->func((struct rcu_head *)rcu);
1751a01b4e5SHans Petter Selasky 	}
1761a01b4e5SHans Petter Selasky }
1771a01b4e5SHans Petter Selasky 
1781a01b4e5SHans Petter Selasky void
179eae5868cSHans Petter Selasky linux_rcu_read_lock(unsigned type)
1801a01b4e5SHans Petter Selasky {
181f3de9af6SHans Petter Selasky 	struct linux_epoch_record *record;
182f3de9af6SHans Petter Selasky 	struct task_struct *ts;
183f3de9af6SHans Petter Selasky 
184eae5868cSHans Petter Selasky 	MPASS(type < RCU_TYPE_MAX);
185eae5868cSHans Petter Selasky 
186f3de9af6SHans Petter Selasky 	if (RCU_SKIP())
187f3de9af6SHans Petter Selasky 		return;
1881a01b4e5SHans Petter Selasky 
18917777208SHans Petter Selasky 	ts = current;
19017777208SHans Petter Selasky 
19117777208SHans Petter Selasky 	/* assert valid refcount */
19217777208SHans Petter Selasky 	MPASS(ts->rcu_recurse[type] != INT_MAX);
19317777208SHans Petter Selasky 
19417777208SHans Petter Selasky 	if (++(ts->rcu_recurse[type]) != 1)
19517777208SHans Petter Selasky 		return;
19617777208SHans Petter Selasky 
1971f827dabSHans Petter Selasky 	/*
1981f827dabSHans Petter Selasky 	 * Pin thread to current CPU so that the unlock code gets the
199f3de9af6SHans Petter Selasky 	 * same per-CPU epoch record:
2001f827dabSHans Petter Selasky 	 */
2011a01b4e5SHans Petter Selasky 	sched_pin();
2021a01b4e5SHans Petter Selasky 
203eae5868cSHans Petter Selasky 	record = &DPCPU_GET(linux_epoch_record[type]);
2041f827dabSHans Petter Selasky 
2051f827dabSHans Petter Selasky 	/*
2061f827dabSHans Petter Selasky 	 * Use a critical section to prevent recursion inside
2071f827dabSHans Petter Selasky 	 * ck_epoch_begin(). Else this function supports recursion.
2081f827dabSHans Petter Selasky 	 */
2091f827dabSHans Petter Selasky 	critical_enter();
21017777208SHans Petter Selasky 	ck_epoch_begin(&record->epoch_record,
21117777208SHans Petter Selasky 	    (ck_epoch_section_t *)&ts->rcu_section[type]);
2126ae24079SHans Petter Selasky 	TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
2131f827dabSHans Petter Selasky 	critical_exit();
2141a01b4e5SHans Petter Selasky }
2151a01b4e5SHans Petter Selasky 
2161a01b4e5SHans Petter Selasky void
217eae5868cSHans Petter Selasky linux_rcu_read_unlock(unsigned type)
2181a01b4e5SHans Petter Selasky {
219f3de9af6SHans Petter Selasky 	struct linux_epoch_record *record;
220f3de9af6SHans Petter Selasky 	struct task_struct *ts;
2211a01b4e5SHans Petter Selasky 
222eae5868cSHans Petter Selasky 	MPASS(type < RCU_TYPE_MAX);
223eae5868cSHans Petter Selasky 
224f3de9af6SHans Petter Selasky 	if (RCU_SKIP())
225f3de9af6SHans Petter Selasky 		return;
226f3de9af6SHans Petter Selasky 
227f3de9af6SHans Petter Selasky 	ts = current;
2281f827dabSHans Petter Selasky 
22917777208SHans Petter Selasky 	/* assert valid refcount */
23017777208SHans Petter Selasky 	MPASS(ts->rcu_recurse[type] > 0);
23117777208SHans Petter Selasky 
23217777208SHans Petter Selasky 	if (--(ts->rcu_recurse[type]) != 0)
23317777208SHans Petter Selasky 		return;
23417777208SHans Petter Selasky 
23517777208SHans Petter Selasky 	record = &DPCPU_GET(linux_epoch_record[type]);
23617777208SHans Petter Selasky 
2371f827dabSHans Petter Selasky 	/*
2381f827dabSHans Petter Selasky 	 * Use a critical section to prevent recursion inside
2391f827dabSHans Petter Selasky 	 * ck_epoch_end(). Else this function supports recursion.
2401f827dabSHans Petter Selasky 	 */
2411f827dabSHans Petter Selasky 	critical_enter();
24217777208SHans Petter Selasky 	ck_epoch_end(&record->epoch_record,
24317777208SHans Petter Selasky 	    (ck_epoch_section_t *)&ts->rcu_section[type]);
2446ae24079SHans Petter Selasky 	TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
2451f827dabSHans Petter Selasky 	critical_exit();
2461f827dabSHans Petter Selasky 
2471a01b4e5SHans Petter Selasky 	sched_unpin();
2481a01b4e5SHans Petter Selasky }
2491a01b4e5SHans Petter Selasky 
250f3de9af6SHans Petter Selasky static void
251f3de9af6SHans Petter Selasky linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused)
252f3de9af6SHans Petter Selasky {
253f3de9af6SHans Petter Selasky 	struct linux_epoch_record *record =
254f3de9af6SHans Petter Selasky 	    container_of(epoch_record, struct linux_epoch_record, epoch_record);
255f3de9af6SHans Petter Selasky 	struct thread *td = curthread;
256f3de9af6SHans Petter Selasky 	struct task_struct *ts;
257f3de9af6SHans Petter Selasky 
258f3de9af6SHans Petter Selasky 	/* check if blocked on the current CPU */
259f3de9af6SHans Petter Selasky 	if (record->cpuid == PCPU_GET(cpuid)) {
260f3de9af6SHans Petter Selasky 		bool is_sleeping = 0;
261f3de9af6SHans Petter Selasky 		u_char prio = 0;
262f3de9af6SHans Petter Selasky 
263f3de9af6SHans Petter Selasky 		/*
264f3de9af6SHans Petter Selasky 		 * Find the lowest priority or sleeping thread which
265f3de9af6SHans Petter Selasky 		 * is blocking synchronization on this CPU core. All
266f3de9af6SHans Petter Selasky 		 * the threads in the queue are CPU-pinned and cannot
267f3de9af6SHans Petter Selasky 		 * go anywhere while the current thread is locked.
268f3de9af6SHans Petter Selasky 		 */
2696ae24079SHans Petter Selasky 		TAILQ_FOREACH(ts, &record->ts_head, rcu_entry[record->type]) {
270f3de9af6SHans Petter Selasky 			if (ts->task_thread->td_priority > prio)
271f3de9af6SHans Petter Selasky 				prio = ts->task_thread->td_priority;
272f3de9af6SHans Petter Selasky 			is_sleeping |= (ts->task_thread->td_inhibitors != 0);
273f3de9af6SHans Petter Selasky 		}
274f3de9af6SHans Petter Selasky 
275f3de9af6SHans Petter Selasky 		if (is_sleeping) {
276f3de9af6SHans Petter Selasky 			thread_unlock(td);
277f3de9af6SHans Petter Selasky 			pause("W", 1);
278f3de9af6SHans Petter Selasky 			thread_lock(td);
279f3de9af6SHans Petter Selasky 		} else {
280f3de9af6SHans Petter Selasky 			/* set new thread priority */
281f3de9af6SHans Petter Selasky 			sched_prio(td, prio);
282f3de9af6SHans Petter Selasky 			/* task switch */
283686bcb5cSJeff Roberson 			mi_switch(SW_VOL | SWT_RELINQUISH);
284cc79ea3aSHans Petter Selasky 			/*
285cc79ea3aSHans Petter Selasky 			 * It is important the thread lock is dropped
286cc79ea3aSHans Petter Selasky 			 * while yielding to allow other threads to
287cc79ea3aSHans Petter Selasky 			 * acquire the lock pointed to by
288cc79ea3aSHans Petter Selasky 			 * TDQ_LOCKPTR(td). Currently mi_switch() will
289cc79ea3aSHans Petter Selasky 			 * unlock the thread lock before
290cc79ea3aSHans Petter Selasky 			 * returning. Else a deadlock like situation
291cc79ea3aSHans Petter Selasky 			 * might happen.
292cc79ea3aSHans Petter Selasky 			 */
293714ed5b2SHans Petter Selasky 			thread_lock(td);
294f3de9af6SHans Petter Selasky 		}
295f3de9af6SHans Petter Selasky 	} else {
296f3de9af6SHans Petter Selasky 		/*
297f3de9af6SHans Petter Selasky 		 * To avoid spinning move execution to the other CPU
298f3de9af6SHans Petter Selasky 		 * which is blocking synchronization. Set highest
299f3de9af6SHans Petter Selasky 		 * thread priority so that code gets run. The thread
300f3de9af6SHans Petter Selasky 		 * priority will be restored later.
301f3de9af6SHans Petter Selasky 		 */
302f3de9af6SHans Petter Selasky 		sched_prio(td, 0);
303f3de9af6SHans Petter Selasky 		sched_bind(td, record->cpuid);
304f3de9af6SHans Petter Selasky 	}
305f3de9af6SHans Petter Selasky }
306f3de9af6SHans Petter Selasky 
3071a01b4e5SHans Petter Selasky void
308eae5868cSHans Petter Selasky linux_synchronize_rcu(unsigned type)
3091a01b4e5SHans Petter Selasky {
310f3de9af6SHans Petter Selasky 	struct thread *td;
311f3de9af6SHans Petter Selasky 	int was_bound;
312f3de9af6SHans Petter Selasky 	int old_cpu;
313f3de9af6SHans Petter Selasky 	int old_pinned;
3143f743d78SHans Petter Selasky 	u_char old_prio;
315f3de9af6SHans Petter Selasky 
316eae5868cSHans Petter Selasky 	MPASS(type < RCU_TYPE_MAX);
317eae5868cSHans Petter Selasky 
318f3de9af6SHans Petter Selasky 	if (RCU_SKIP())
319f3de9af6SHans Petter Selasky 		return;
320f3de9af6SHans Petter Selasky 
321f3de9af6SHans Petter Selasky 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
322f3de9af6SHans Petter Selasky 	    "linux_synchronize_rcu() can sleep");
323f3de9af6SHans Petter Selasky 
324f3de9af6SHans Petter Selasky 	td = curthread;
325fedab1b4SKonstantin Belousov 	DROP_GIANT();
326f3de9af6SHans Petter Selasky 
327f3de9af6SHans Petter Selasky 	/*
328f3de9af6SHans Petter Selasky 	 * Synchronizing RCU might change the CPU core this function
329f3de9af6SHans Petter Selasky 	 * is running on. Save current values:
330f3de9af6SHans Petter Selasky 	 */
331f3de9af6SHans Petter Selasky 	thread_lock(td);
332f3de9af6SHans Petter Selasky 
333f3de9af6SHans Petter Selasky 	old_cpu = PCPU_GET(cpuid);
334f3de9af6SHans Petter Selasky 	old_pinned = td->td_pinned;
3353f743d78SHans Petter Selasky 	old_prio = td->td_priority;
336f3de9af6SHans Petter Selasky 	was_bound = sched_is_bound(td);
337ea165254SHans Petter Selasky 	sched_unbind(td);
338ea165254SHans Petter Selasky 	td->td_pinned = 0;
339f3de9af6SHans Petter Selasky 	sched_bind(td, old_cpu);
340f3de9af6SHans Petter Selasky 
341eae5868cSHans Petter Selasky 	ck_epoch_synchronize_wait(&linux_epoch[type],
342f3de9af6SHans Petter Selasky 	    &linux_synchronize_rcu_cb, NULL);
343f3de9af6SHans Petter Selasky 
344f3de9af6SHans Petter Selasky 	/* restore CPU binding, if any */
345f3de9af6SHans Petter Selasky 	if (was_bound != 0) {
346f3de9af6SHans Petter Selasky 		sched_bind(td, old_cpu);
347f3de9af6SHans Petter Selasky 	} else {
348f3de9af6SHans Petter Selasky 		/* get thread back to initial CPU, if any */
349f3de9af6SHans Petter Selasky 		if (old_pinned != 0)
350f3de9af6SHans Petter Selasky 			sched_bind(td, old_cpu);
351f3de9af6SHans Petter Selasky 		sched_unbind(td);
352f3de9af6SHans Petter Selasky 	}
353f3de9af6SHans Petter Selasky 	/* restore pinned after bind */
354f3de9af6SHans Petter Selasky 	td->td_pinned = old_pinned;
3553f743d78SHans Petter Selasky 
3563f743d78SHans Petter Selasky 	/* restore thread priority */
3573f743d78SHans Petter Selasky 	sched_prio(td, old_prio);
358f3de9af6SHans Petter Selasky 	thread_unlock(td);
359f3de9af6SHans Petter Selasky 
360f3de9af6SHans Petter Selasky 	PICKUP_GIANT();
3611a01b4e5SHans Petter Selasky }
3621a01b4e5SHans Petter Selasky 
3631a01b4e5SHans Petter Selasky void
364eae5868cSHans Petter Selasky linux_rcu_barrier(unsigned type)
3651a01b4e5SHans Petter Selasky {
366f3de9af6SHans Petter Selasky 	struct linux_epoch_head *head;
3671a01b4e5SHans Petter Selasky 
368eae5868cSHans Petter Selasky 	MPASS(type < RCU_TYPE_MAX);
3691f827dabSHans Petter Selasky 
370*1ab61a19SVladimir Kondratyev 	/*
371*1ab61a19SVladimir Kondratyev 	 * This function is not obligated to wait for a grace period.
372*1ab61a19SVladimir Kondratyev 	 * It only waits for RCU callbacks that have already been posted.
373*1ab61a19SVladimir Kondratyev 	 * If there are no RCU callbacks posted, rcu_barrier() can return
374*1ab61a19SVladimir Kondratyev 	 * immediately.
375*1ab61a19SVladimir Kondratyev 	 */
376eae5868cSHans Petter Selasky 	head = &linux_epoch_head[type];
3771f827dabSHans Petter Selasky 
3781f827dabSHans Petter Selasky 	/* wait for callbacks to complete */
379a2b83b59SVladimir Kondratyev 	taskqueue_drain(linux_irq_work_tq, &head->task);
3801a01b4e5SHans Petter Selasky }
3811a01b4e5SHans Petter Selasky 
3821a01b4e5SHans Petter Selasky void
383eae5868cSHans Petter Selasky linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func)
3841a01b4e5SHans Petter Selasky {
385eae5868cSHans Petter Selasky 	struct callback_head *rcu;
386eae5868cSHans Petter Selasky 	struct linux_epoch_head *head;
387eae5868cSHans Petter Selasky 
388eae5868cSHans Petter Selasky 	MPASS(type < RCU_TYPE_MAX);
389eae5868cSHans Petter Selasky 
390eae5868cSHans Petter Selasky 	rcu = (struct callback_head *)context;
391eae5868cSHans Petter Selasky 	head = &linux_epoch_head[type];
3921a01b4e5SHans Petter Selasky 
3931f827dabSHans Petter Selasky 	rcu->func = func;
394a2b83b59SVladimir Kondratyev 	llist_add(&rcu->node, &head->cb_head);
395a2b83b59SVladimir Kondratyev 	taskqueue_enqueue(linux_irq_work_tq, &head->task);
3961a01b4e5SHans Petter Selasky }
3971a01b4e5SHans Petter Selasky 
3981a01b4e5SHans Petter Selasky int
3991a01b4e5SHans Petter Selasky init_srcu_struct(struct srcu_struct *srcu)
4001a01b4e5SHans Petter Selasky {
4011a01b4e5SHans Petter Selasky 	return (0);
4021a01b4e5SHans Petter Selasky }
4031a01b4e5SHans Petter Selasky 
4041a01b4e5SHans Petter Selasky void
4051a01b4e5SHans Petter Selasky cleanup_srcu_struct(struct srcu_struct *srcu)
4061a01b4e5SHans Petter Selasky {
4071a01b4e5SHans Petter Selasky }
4081a01b4e5SHans Petter Selasky 
4091a01b4e5SHans Petter Selasky int
4101a01b4e5SHans Petter Selasky srcu_read_lock(struct srcu_struct *srcu)
4111a01b4e5SHans Petter Selasky {
412eae5868cSHans Petter Selasky 	linux_rcu_read_lock(RCU_TYPE_SLEEPABLE);
4131a01b4e5SHans Petter Selasky 	return (0);
4141a01b4e5SHans Petter Selasky }
4151a01b4e5SHans Petter Selasky 
4161a01b4e5SHans Petter Selasky void
4171a01b4e5SHans Petter Selasky srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
4181a01b4e5SHans Petter Selasky {
419eae5868cSHans Petter Selasky 	linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE);
4201a01b4e5SHans Petter Selasky }
4211a01b4e5SHans Petter Selasky 
4221a01b4e5SHans Petter Selasky void
4231a01b4e5SHans Petter Selasky synchronize_srcu(struct srcu_struct *srcu)
4241a01b4e5SHans Petter Selasky {
425eae5868cSHans Petter Selasky 	linux_synchronize_rcu(RCU_TYPE_SLEEPABLE);
4261f827dabSHans Petter Selasky }
4271f827dabSHans Petter Selasky 
4281f827dabSHans Petter Selasky void
4291f827dabSHans Petter Selasky srcu_barrier(struct srcu_struct *srcu)
4301f827dabSHans Petter Selasky {
431eae5868cSHans Petter Selasky 	linux_rcu_barrier(RCU_TYPE_SLEEPABLE);
4321a01b4e5SHans Petter Selasky }
433