1 /*- 2 * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2024-2025 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #ifndef _LINUXKPI_LINUX_RCUPDATE_H_ 31 #define _LINUXKPI_LINUX_RCUPDATE_H_ 32 33 #include <sys/cdefs.h> 34 35 #include <linux/compiler.h> 36 #include <linux/types.h> 37 #include <linux/kernel.h> 38 #include <linux/cleanup.h> 39 40 #include <machine/atomic.h> 41 42 extern int linuxkpi_rcu_debug; 43 #define RCU_WARN_ONCE(c, ...) do { \ 44 if (unlikely(linuxkpi_rcu_debug > 0)) \ 45 WARN_ONCE((c), ##__VA_ARGS__); \ 46 } while(0) 47 48 #define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ 49 50 /* BSD specific defines */ 51 #define RCU_TYPE_REGULAR 0 52 #define RCU_TYPE_SLEEPABLE 1 53 #define RCU_TYPE_MAX 2 54 55 #define RCU_INITIALIZER(v) \ 56 ((__typeof(*(v)) *)(v)) 57 58 #define RCU_INIT_POINTER(p, v) do { \ 59 (p) = (v); \ 60 } while (0) 61 62 #define call_rcu(ptr, func) do { \ 63 linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \ 64 } while (0) 65 66 #define rcu_barrier(void) do { \ 67 linux_rcu_barrier(RCU_TYPE_REGULAR); \ 68 } while (0) 69 70 #define rcu_read_lock(void) do { \ 71 linux_rcu_read_lock(RCU_TYPE_REGULAR); \ 72 } while (0) 73 74 #define rcu_read_unlock(void) do { \ 75 linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ 76 } while (0) 77 78 #define rcu_read_lock_held(void) \ 79 linux_rcu_read_lock_held(RCU_TYPE_REGULAR) 80 81 #define synchronize_rcu(void) do { \ 82 linux_synchronize_rcu(RCU_TYPE_REGULAR); \ 83 } while (0) 84 85 #define synchronize_rcu_expedited(void) do { \ 86 linux_synchronize_rcu(RCU_TYPE_REGULAR); \ 87 } while (0) 88 89 #define kfree_rcu(ptr, rcu_head) do { \ 90 CTASSERT(offsetof(__typeof(*(ptr)), rcu_head) < \ 91 LINUX_KFREE_RCU_OFFSET_MAX); \ 92 call_rcu(&(ptr)->rcu_head, (rcu_callback_t)(uintptr_t) \ 93 offsetof(__typeof(*(ptr)), rcu_head)); \ 94 } while (0) 95 96 #define rcu_access_pointer(p) \ 97 ((__typeof(*p) *)READ_ONCE(p)) 98 99 #define rcu_dereference(p) \ 100 ((__typeof(*p) *)READ_ONCE(p)) 101 102 #define __rcu_var_name(n, f, l) \ 103 __CONCAT(__CONCAT(__CONCAT(rcu_, n), _), __COUNTER__) 104 105 #define __rcu_dereference_protected(p, c, n) \ 106 ({ \ 107 RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ 108 __func__, __LINE__, __XSTRING(n)); \ 109 rcu_dereference(p); \ 110 }) 111 112 #define rcu_dereference_protected(p, c) \ 113 __rcu_dereference_protected((p), (c), \ 114 __rcu_var_name(protected, __func__, __LINE__)) 115 116 #define __rcu_dereference_check(p, c, n) \ 117 ({ \ 118 __typeof(*p) *n = rcu_dereference(p); \ 119 RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ 120 __func__, __LINE__, __XSTRING(n)); \ 121 n; \ 122 }) 123 124 #define rcu_dereference_check(p, c) \ 125 __rcu_dereference_check((p), (c) || rcu_read_lock_held(), \ 126 __rcu_var_name(check, __func__, __LINE__)) 127 128 #define rcu_dereference_raw(p) \ 129 ((__typeof(*p) *)READ_ONCE(p)) 130 131 #define rcu_pointer_handoff(p) (p) 132 133 #define rcu_assign_pointer(p, v) do { \ 134 atomic_store_rel_ptr((volatile uintptr_t *)&(p), \ 135 (uintptr_t)(v)); \ 136 } while (0) 137 138 #define rcu_replace_pointer(rcu, ptr, c) \ 139 ({ \ 140 typeof(ptr) __tmp = rcu_dereference_protected(rcu, c); \ 141 rcu_assign_pointer(rcu, ptr); \ 142 __tmp; \ 143 }) 144 145 #define rcu_swap_protected(rcu, ptr, c) do { \ 146 typeof(ptr) p = rcu_dereference_protected(rcu, c); \ 147 rcu_assign_pointer(rcu, ptr); \ 148 (ptr) = p; \ 149 } while (0) 150 151 /* prototypes */ 152 153 void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); 154 void linux_rcu_barrier(unsigned type); 155 void linux_rcu_read_lock(unsigned type); 156 void linux_rcu_read_unlock(unsigned type); 157 bool linux_rcu_read_lock_held(unsigned); 158 void linux_synchronize_rcu(unsigned type); 159 160 /* Empty implementation for !DEBUG */ 161 #define init_rcu_head(...) 162 #define destroy_rcu_head(...) 163 #define init_rcu_head_on_stack(...) 164 #define destroy_rcu_head_on_stack(...) 165 166 DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock()) 167 168 #endif /* _LINUXKPI_LINUX_RCUPDATE_H_ */ 169