xref: /freebsd/sys/compat/linuxkpi/common/src/linux_rcu.c (revision 076ad2f836d5f49dc1375f1677335a48fe0d4b82)
1 /*-
2  * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org)
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/malloc.h>
33 #include <sys/kernel.h>
34 #include <sys/lock.h>
35 #include <sys/mutex.h>
36 #include <sys/proc.h>
37 #include <sys/sched.h>
38 #include <sys/smp.h>
39 #include <sys/queue.h>
40 #include <sys/taskqueue.h>
41 
42 #include <ck_epoch.h>
43 
44 #include <linux/rcupdate.h>
45 #include <linux/srcu.h>
46 #include <linux/slab.h>
47 #include <linux/kernel.h>
48 
49 struct callback_head {
50 	ck_epoch_entry_t epoch_entry;
51 	rcu_callback_t func;
52 	ck_epoch_record_t *epoch_record;
53 	struct task task;
54 };
55 
56 /*
57  * Verify that "struct rcu_head" is big enough to hold "struct
58  * callback_head". This has been done to avoid having to add special
59  * compile flags for including ck_epoch.h to all clients of the
60  * LinuxKPI.
61  */
62 CTASSERT(sizeof(struct rcu_head) >= sizeof(struct callback_head));
63 
64 static ck_epoch_t linux_epoch;
65 static	MALLOC_DEFINE(M_LRCU, "lrcu", "Linux RCU");
66 static	DPCPU_DEFINE(ck_epoch_record_t *, epoch_record);
67 
68 static void
69 linux_rcu_runtime_init(void *arg __unused)
70 {
71 	ck_epoch_record_t **pcpu_record;
72 	ck_epoch_record_t *record;
73 	int i;
74 
75 	ck_epoch_init(&linux_epoch);
76 
77 	CPU_FOREACH(i) {
78 		record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
79 		ck_epoch_register(&linux_epoch, record);
80 		pcpu_record = DPCPU_ID_PTR(i, epoch_record);
81 		*pcpu_record = record;
82 	}
83 
84 	/*
85 	 * Populate the epoch with 5 * ncpus # of records
86 	 */
87 	for (i = 0; i < 5 * mp_ncpus; i++) {
88 		record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
89 		ck_epoch_register(&linux_epoch, record);
90 		ck_epoch_unregister(record);
91 	}
92 }
93 SYSINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_init, NULL);
94 
95 static void
96 linux_rcu_runtime_uninit(void *arg __unused)
97 {
98 	ck_epoch_record_t **pcpu_record;
99 	ck_epoch_record_t *record;
100 	int i;
101 
102 	while ((record = ck_epoch_recycle(&linux_epoch)) != NULL)
103 		free(record, M_LRCU);
104 
105 	CPU_FOREACH(i) {
106 		pcpu_record = DPCPU_ID_PTR(i, epoch_record);
107 		record = *pcpu_record;
108 		*pcpu_record = NULL;
109 		free(record, M_LRCU);
110 	}
111 }
112 SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);
113 
114 static ck_epoch_record_t *
115 linux_rcu_get_record(int canblock)
116 {
117 	ck_epoch_record_t *record;
118 
119 	if (__predict_true((record = ck_epoch_recycle(&linux_epoch)) != NULL))
120 		return (record);
121 	if ((record = malloc(sizeof(*record), M_LRCU, M_NOWAIT | M_ZERO)) != NULL) {
122 		ck_epoch_register(&linux_epoch, record);
123 		return (record);
124 	} else if (!canblock)
125 		return (NULL);
126 
127 	record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO);
128 	ck_epoch_register(&linux_epoch, record);
129 	return (record);
130 }
131 
132 static void
133 linux_rcu_destroy_object(ck_epoch_entry_t *e)
134 {
135 	struct callback_head *rcu;
136 	uintptr_t offset;
137 
138 	rcu = container_of(e, struct callback_head, epoch_entry);
139 
140 	offset = (uintptr_t)rcu->func;
141 
142 	MPASS(rcu->task.ta_pending == 0);
143 
144 	if (offset < LINUX_KFREE_RCU_OFFSET_MAX)
145 		kfree((char *)rcu - offset);
146 	else
147 		rcu->func((struct rcu_head *)rcu);
148 }
149 
150 static void
151 linux_rcu_cleaner_func(void *context, int pending __unused)
152 {
153 	struct callback_head *rcu = context;
154 	ck_epoch_record_t *record = rcu->epoch_record;
155 
156 	ck_epoch_barrier(record);
157 	ck_epoch_unregister(record);
158 }
159 
160 void
161 linux_rcu_read_lock(void)
162 {
163 	ck_epoch_record_t *record;
164 
165 	sched_pin();
166 	record = DPCPU_GET(epoch_record);
167 	MPASS(record != NULL);
168 
169 	ck_epoch_begin(record, NULL);
170 }
171 
172 void
173 linux_rcu_read_unlock(void)
174 {
175 	ck_epoch_record_t *record;
176 
177 	record = DPCPU_GET(epoch_record);
178 	ck_epoch_end(record, NULL);
179 	sched_unpin();
180 }
181 
182 void
183 linux_synchronize_rcu(void)
184 {
185 	ck_epoch_record_t *record;
186 
187 	sched_pin();
188 	record = DPCPU_GET(epoch_record);
189 	MPASS(record != NULL);
190 	ck_epoch_synchronize(record);
191 	sched_unpin();
192 }
193 
194 void
195 linux_rcu_barrier(void)
196 {
197 	ck_epoch_record_t *record;
198 
199 	record = linux_rcu_get_record(0);
200 	ck_epoch_barrier(record);
201 	ck_epoch_unregister(record);
202 }
203 
204 void
205 linux_call_rcu(struct rcu_head *context, rcu_callback_t func)
206 {
207 	struct callback_head *ptr = (struct callback_head *)context;
208 	ck_epoch_record_t *record;
209 
210 	record = linux_rcu_get_record(0);
211 
212 	sched_pin();
213 	MPASS(record != NULL);
214 	ptr->func = func;
215 	ptr->epoch_record = record;
216 	ck_epoch_call(record, &ptr->epoch_entry, linux_rcu_destroy_object);
217 	TASK_INIT(&ptr->task, 0, linux_rcu_cleaner_func, ptr);
218 	taskqueue_enqueue(taskqueue_fast, &ptr->task);
219 	sched_unpin();
220 }
221 
222 int
223 init_srcu_struct(struct srcu_struct *srcu)
224 {
225 	ck_epoch_record_t *record;
226 
227 	record = linux_rcu_get_record(0);
228 	srcu->ss_epoch_record = record;
229 	return (0);
230 }
231 
232 void
233 cleanup_srcu_struct(struct srcu_struct *srcu)
234 {
235 	ck_epoch_record_t *record;
236 
237 	record = srcu->ss_epoch_record;
238 	srcu->ss_epoch_record = NULL;
239 	ck_epoch_unregister(record);
240 }
241 
242 int
243 srcu_read_lock(struct srcu_struct *srcu)
244 {
245 	ck_epoch_begin(srcu->ss_epoch_record, NULL);
246 	return (0);
247 }
248 
249 void
250 srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
251 {
252 	ck_epoch_end(srcu->ss_epoch_record, NULL);
253 }
254 
255 void
256 synchronize_srcu(struct srcu_struct *srcu)
257 {
258 	ck_epoch_synchronize(srcu->ss_epoch_record);
259 }
260