1 /*- 2 * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org) 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/types.h> 31 #include <sys/systm.h> 32 #include <sys/malloc.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/mutex.h> 36 #include <sys/proc.h> 37 #include <sys/sched.h> 38 #include <sys/smp.h> 39 #include <sys/queue.h> 40 #include <sys/taskqueue.h> 41 42 #include <ck_epoch.h> 43 44 #include <linux/rcupdate.h> 45 #include <linux/srcu.h> 46 #include <linux/slab.h> 47 #include <linux/kernel.h> 48 49 struct callback_head { 50 ck_epoch_entry_t epoch_entry; 51 rcu_callback_t func; 52 ck_epoch_record_t *epoch_record; 53 struct task task; 54 }; 55 56 /* 57 * Verify that "struct rcu_head" is big enough to hold "struct 58 * callback_head". This has been done to avoid having to add special 59 * compile flags for including ck_epoch.h to all clients of the 60 * LinuxKPI. 61 */ 62 CTASSERT(sizeof(struct rcu_head) >= sizeof(struct callback_head)); 63 64 static ck_epoch_t linux_epoch; 65 static MALLOC_DEFINE(M_LRCU, "lrcu", "Linux RCU"); 66 static DPCPU_DEFINE(ck_epoch_record_t *, epoch_record); 67 68 static void 69 linux_rcu_runtime_init(void *arg __unused) 70 { 71 ck_epoch_record_t **pcpu_record; 72 ck_epoch_record_t *record; 73 int i; 74 75 ck_epoch_init(&linux_epoch); 76 77 CPU_FOREACH(i) { 78 record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 79 ck_epoch_register(&linux_epoch, record); 80 pcpu_record = DPCPU_ID_PTR(i, epoch_record); 81 *pcpu_record = record; 82 } 83 84 /* 85 * Populate the epoch with 5 * ncpus # of records 86 */ 87 for (i = 0; i < 5 * mp_ncpus; i++) { 88 record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 89 ck_epoch_register(&linux_epoch, record); 90 ck_epoch_unregister(record); 91 } 92 } 93 SYSINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_init, NULL); 94 95 static void 96 linux_rcu_runtime_uninit(void *arg __unused) 97 { 98 ck_epoch_record_t **pcpu_record; 99 ck_epoch_record_t *record; 100 int i; 101 102 while ((record = ck_epoch_recycle(&linux_epoch)) != NULL) 103 free(record, M_LRCU); 104 105 CPU_FOREACH(i) { 106 pcpu_record = DPCPU_ID_PTR(i, epoch_record); 107 record = *pcpu_record; 108 *pcpu_record = NULL; 109 free(record, M_LRCU); 110 } 111 } 112 SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); 113 114 static ck_epoch_record_t * 115 linux_rcu_get_record(int canblock) 116 { 117 ck_epoch_record_t *record; 118 119 if (__predict_true((record = ck_epoch_recycle(&linux_epoch)) != NULL)) 120 return (record); 121 if ((record = malloc(sizeof(*record), M_LRCU, M_NOWAIT | M_ZERO)) != NULL) { 122 ck_epoch_register(&linux_epoch, record); 123 return (record); 124 } else if (!canblock) 125 return (NULL); 126 127 record = malloc(sizeof(*record), M_LRCU, M_WAITOK | M_ZERO); 128 ck_epoch_register(&linux_epoch, record); 129 return (record); 130 } 131 132 static void 133 linux_rcu_destroy_object(ck_epoch_entry_t *e) 134 { 135 struct callback_head *rcu; 136 uintptr_t offset; 137 138 rcu = container_of(e, struct callback_head, epoch_entry); 139 140 offset = (uintptr_t)rcu->func; 141 142 MPASS(rcu->task.ta_pending == 0); 143 144 if (offset < LINUX_KFREE_RCU_OFFSET_MAX) 145 kfree((char *)rcu - offset); 146 else 147 rcu->func((struct rcu_head *)rcu); 148 } 149 150 static void 151 linux_rcu_cleaner_func(void *context, int pending __unused) 152 { 153 struct callback_head *rcu = context; 154 ck_epoch_record_t *record = rcu->epoch_record; 155 156 ck_epoch_barrier(record); 157 ck_epoch_unregister(record); 158 } 159 160 void 161 linux_rcu_read_lock(void) 162 { 163 ck_epoch_record_t *record; 164 165 sched_pin(); 166 record = DPCPU_GET(epoch_record); 167 MPASS(record != NULL); 168 169 ck_epoch_begin(record, NULL); 170 } 171 172 void 173 linux_rcu_read_unlock(void) 174 { 175 ck_epoch_record_t *record; 176 177 record = DPCPU_GET(epoch_record); 178 ck_epoch_end(record, NULL); 179 sched_unpin(); 180 } 181 182 void 183 linux_synchronize_rcu(void) 184 { 185 ck_epoch_record_t *record; 186 187 sched_pin(); 188 record = DPCPU_GET(epoch_record); 189 MPASS(record != NULL); 190 ck_epoch_synchronize(record); 191 sched_unpin(); 192 } 193 194 void 195 linux_rcu_barrier(void) 196 { 197 ck_epoch_record_t *record; 198 199 record = linux_rcu_get_record(0); 200 ck_epoch_barrier(record); 201 ck_epoch_unregister(record); 202 } 203 204 void 205 linux_call_rcu(struct rcu_head *context, rcu_callback_t func) 206 { 207 struct callback_head *ptr = (struct callback_head *)context; 208 ck_epoch_record_t *record; 209 210 record = linux_rcu_get_record(0); 211 212 sched_pin(); 213 MPASS(record != NULL); 214 ptr->func = func; 215 ptr->epoch_record = record; 216 ck_epoch_call(record, &ptr->epoch_entry, linux_rcu_destroy_object); 217 TASK_INIT(&ptr->task, 0, linux_rcu_cleaner_func, ptr); 218 taskqueue_enqueue(taskqueue_fast, &ptr->task); 219 sched_unpin(); 220 } 221 222 int 223 init_srcu_struct(struct srcu_struct *srcu) 224 { 225 ck_epoch_record_t *record; 226 227 record = linux_rcu_get_record(0); 228 srcu->ss_epoch_record = record; 229 return (0); 230 } 231 232 void 233 cleanup_srcu_struct(struct srcu_struct *srcu) 234 { 235 ck_epoch_record_t *record; 236 237 record = srcu->ss_epoch_record; 238 srcu->ss_epoch_record = NULL; 239 ck_epoch_unregister(record); 240 } 241 242 int 243 srcu_read_lock(struct srcu_struct *srcu) 244 { 245 ck_epoch_begin(srcu->ss_epoch_record, NULL); 246 return (0); 247 } 248 249 void 250 srcu_read_unlock(struct srcu_struct *srcu, int key __unused) 251 { 252 ck_epoch_end(srcu->ss_epoch_record, NULL); 253 } 254 255 void 256 synchronize_srcu(struct srcu_struct *srcu) 257 { 258 ck_epoch_synchronize(srcu->ss_epoch_record); 259 } 260