1 /*- 2 * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #ifndef _LINUXKPI_LINUX_RCUPDATE_H_ 31 #define _LINUXKPI_LINUX_RCUPDATE_H_ 32 33 #include <sys/cdefs.h> 34 35 #include <linux/compiler.h> 36 #include <linux/types.h> 37 #include <linux/kernel.h> 38 39 #include <machine/atomic.h> 40 41 extern int linuxkpi_rcu_debug; 42 #define RCU_WARN_ONCE(c, ...) do { \ 43 if (unlikely(linuxkpi_rcu_debug > 0)) \ 44 WARN_ONCE((c), ##__VA_ARGS__); \ 45 } while(0) 46 47 #define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ 48 49 /* BSD specific defines */ 50 #define RCU_TYPE_REGULAR 0 51 #define RCU_TYPE_SLEEPABLE 1 52 #define RCU_TYPE_MAX 2 53 54 #define RCU_INITIALIZER(v) \ 55 ((__typeof(*(v)) *)(v)) 56 57 #define RCU_INIT_POINTER(p, v) do { \ 58 (p) = (v); \ 59 } while (0) 60 61 #define call_rcu(ptr, func) do { \ 62 linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \ 63 } while (0) 64 65 #define rcu_barrier(void) do { \ 66 linux_rcu_barrier(RCU_TYPE_REGULAR); \ 67 } while (0) 68 69 #define rcu_read_lock(void) do { \ 70 linux_rcu_read_lock(RCU_TYPE_REGULAR); \ 71 } while (0) 72 73 #define rcu_read_unlock(void) do { \ 74 linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ 75 } while (0) 76 77 #define rcu_read_lock_held(void) \ 78 linux_rcu_read_lock_held(RCU_TYPE_REGULAR) 79 80 #define synchronize_rcu(void) do { \ 81 linux_synchronize_rcu(RCU_TYPE_REGULAR); \ 82 } while (0) 83 84 #define synchronize_rcu_expedited(void) do { \ 85 linux_synchronize_rcu(RCU_TYPE_REGULAR); \ 86 } while (0) 87 88 #define kfree_rcu(ptr, rcu_head) do { \ 89 CTASSERT(offsetof(__typeof(*(ptr)), rcu_head) < \ 90 LINUX_KFREE_RCU_OFFSET_MAX); \ 91 call_rcu(&(ptr)->rcu_head, (rcu_callback_t)(uintptr_t) \ 92 offsetof(__typeof(*(ptr)), rcu_head)); \ 93 } while (0) 94 95 #define rcu_access_pointer(p) \ 96 ((__typeof(*p) *)READ_ONCE(p)) 97 98 #define rcu_dereference(p) \ 99 ((__typeof(*p) *)READ_ONCE(p)) 100 101 #define __rcu_var_name(n, f, l) \ 102 __CONCAT(__CONCAT(__CONCAT(rcu_, n), _), __COUNTER__) 103 104 #define __rcu_dereference_protected(p, c, n) \ 105 ({ \ 106 RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ 107 __func__, __LINE__, __XSTRING(n)); \ 108 rcu_dereference(p); \ 109 }) 110 111 #define rcu_dereference_protected(p, c) \ 112 __rcu_dereference_protected((p), (c), \ 113 __rcu_var_name(protected, __func__, __LINE__)) 114 115 #define __rcu_dereference_check(p, c, n) \ 116 ({ \ 117 __typeof(*p) *n = rcu_dereference(p); \ 118 RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ 119 __func__, __LINE__, __XSTRING(n)); \ 120 n; \ 121 }) 122 123 #define rcu_dereference_check(p, c) \ 124 __rcu_dereference_check((p), (c) || rcu_read_lock_held(), \ 125 __rcu_var_name(check, __func__, __LINE__)) 126 127 #define rcu_dereference_raw(p) \ 128 ((__typeof(*p) *)READ_ONCE(p)) 129 130 #define rcu_pointer_handoff(p) (p) 131 132 #define rcu_assign_pointer(p, v) do { \ 133 atomic_store_rel_ptr((volatile uintptr_t *)&(p), \ 134 (uintptr_t)(v)); \ 135 } while (0) 136 137 #define rcu_replace_pointer(rcu, ptr, c) \ 138 ({ \ 139 typeof(ptr) __tmp = rcu_dereference_protected(rcu, c); \ 140 rcu_assign_pointer(rcu, ptr); \ 141 __tmp; \ 142 }) 143 144 #define rcu_swap_protected(rcu, ptr, c) do { \ 145 typeof(ptr) p = rcu_dereference_protected(rcu, c); \ 146 rcu_assign_pointer(rcu, ptr); \ 147 (ptr) = p; \ 148 } while (0) 149 150 /* prototypes */ 151 152 void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); 153 void linux_rcu_barrier(unsigned type); 154 void linux_rcu_read_lock(unsigned type); 155 void linux_rcu_read_unlock(unsigned type); 156 bool linux_rcu_read_lock_held(unsigned); 157 void linux_synchronize_rcu(unsigned type); 158 159 /* Empty implementation for !DEBUG */ 160 #define init_rcu_head(...) 161 #define destroy_rcu_head(...) 162 #define init_rcu_head_on_stack(...) 163 #define destroy_rcu_head_on_stack(...) 164 165 #endif /* _LINUXKPI_LINUX_RCUPDATE_H_ */ 166