1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2001 Matthew Dillon. All Rights Reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* Mutex pool routines. These routines are designed to be used as short 29 * term leaf mutexes (e.g. the last mutex you might acquire other then 30 * calling msleep()). They operate using a shared pool. A mutex is chosen 31 * from the pool based on the supplied pointer (which may or may not be 32 * valid). 33 * 34 * Advantages: 35 * - no structural overhead. Mutexes can be associated with structures 36 * without adding bloat to the structures. 37 * - mutexes can be obtained for invalid pointers, useful when uses 38 * mutexes to interlock destructor ops. 39 * - no initialization/destructor overhead. 40 * - can be used with msleep. 41 * 42 * Disadvantages: 43 * - should generally only be used as leaf mutexes. 44 * - pool/pool dependency ordering cannot be depended on. 45 * - possible L1 cache mastersip contention between cpus. 46 */ 47 48 #include <sys/cdefs.h> 49 __FBSDID("$FreeBSD$"); 50 51 #include <sys/param.h> 52 #include <sys/proc.h> 53 #include <sys/kernel.h> 54 #include <sys/ktr.h> 55 #include <sys/lock.h> 56 #include <sys/malloc.h> 57 #include <sys/mutex.h> 58 #include <sys/systm.h> 59 60 static MALLOC_DEFINE(M_MTXPOOL, "mtx_pool", "mutex pool"); 61 62 /* Pool sizes must be a power of two */ 63 #ifndef MTX_POOL_SLEEP_SIZE 64 #define MTX_POOL_SLEEP_SIZE 1024 65 #endif 66 67 struct mtxpool_header { 68 int mtxpool_size; 69 int mtxpool_mask; 70 int mtxpool_shift; 71 int mtxpool_next __aligned(CACHE_LINE_SIZE); 72 }; 73 74 struct mtx_pool { 75 struct mtxpool_header mtx_pool_header; 76 struct mtx mtx_pool_ary[1]; 77 }; 78 79 #define mtx_pool_size mtx_pool_header.mtxpool_size 80 #define mtx_pool_mask mtx_pool_header.mtxpool_mask 81 #define mtx_pool_shift mtx_pool_header.mtxpool_shift 82 #define mtx_pool_next mtx_pool_header.mtxpool_next 83 84 struct mtx_pool __read_mostly *mtxpool_sleep; 85 86 #if UINTPTR_MAX == UINT64_MAX /* 64 bits */ 87 # define POINTER_BITS 64 88 # define HASH_MULTIPLIER 11400714819323198485u /* (2^64)*(sqrt(5)-1)/2 */ 89 #else /* assume 32 bits */ 90 # define POINTER_BITS 32 91 # define HASH_MULTIPLIER 2654435769u /* (2^32)*(sqrt(5)-1)/2 */ 92 #endif 93 94 /* 95 * Return the (shared) pool mutex associated with the specified address. 96 * The returned mutex is a leaf level mutex, meaning that if you obtain it 97 * you cannot obtain any other mutexes until you release it. You can 98 * legally msleep() on the mutex. 99 */ 100 struct mtx * 101 mtx_pool_find(struct mtx_pool *pool, void *ptr) 102 { 103 int p; 104 105 KASSERT(pool != NULL, ("_mtx_pool_find(): null pool")); 106 /* 107 * Fibonacci hash, see Knuth's 108 * _Art of Computer Programming, Volume 3 / Sorting and Searching_ 109 */ 110 p = ((HASH_MULTIPLIER * (uintptr_t)ptr) >> pool->mtx_pool_shift) & 111 pool->mtx_pool_mask; 112 return (&pool->mtx_pool_ary[p]); 113 } 114 115 static void 116 mtx_pool_initialize(struct mtx_pool *pool, const char *mtx_name, int pool_size, 117 int opts) 118 { 119 int i, maskbits; 120 121 pool->mtx_pool_size = pool_size; 122 pool->mtx_pool_mask = pool_size - 1; 123 for (i = 1, maskbits = 0; (i & pool_size) == 0; i = i << 1) 124 maskbits++; 125 pool->mtx_pool_shift = POINTER_BITS - maskbits; 126 pool->mtx_pool_next = 0; 127 for (i = 0; i < pool_size; ++i) 128 mtx_init(&pool->mtx_pool_ary[i], mtx_name, NULL, opts); 129 } 130 131 struct mtx_pool * 132 mtx_pool_create(const char *mtx_name, int pool_size, int opts) 133 { 134 struct mtx_pool *pool; 135 136 if (pool_size <= 0 || !powerof2(pool_size)) { 137 printf("WARNING: %s pool size is not a power of 2.\n", 138 mtx_name); 139 pool_size = 128; 140 } 141 pool = malloc(sizeof (struct mtx_pool) + 142 ((pool_size - 1) * sizeof (struct mtx)), 143 M_MTXPOOL, M_WAITOK | M_ZERO); 144 mtx_pool_initialize(pool, mtx_name, pool_size, opts); 145 return pool; 146 } 147 148 void 149 mtx_pool_destroy(struct mtx_pool **poolp) 150 { 151 int i; 152 struct mtx_pool *pool = *poolp; 153 154 for (i = pool->mtx_pool_size - 1; i >= 0; --i) 155 mtx_destroy(&pool->mtx_pool_ary[i]); 156 free(pool, M_MTXPOOL); 157 *poolp = NULL; 158 } 159 160 static void 161 mtx_pool_setup_dynamic(void *dummy __unused) 162 { 163 mtxpool_sleep = mtx_pool_create("sleep mtxpool", 164 MTX_POOL_SLEEP_SIZE, MTX_DEF); 165 } 166 167 /* 168 * Obtain a (shared) mutex from the pool. The returned mutex is a leaf 169 * level mutex, meaning that if you obtain it you cannot obtain any other 170 * mutexes until you release it. You can legally msleep() on the mutex. 171 */ 172 struct mtx * 173 mtx_pool_alloc(struct mtx_pool *pool) 174 { 175 int i; 176 177 KASSERT(pool != NULL, ("mtx_pool_alloc(): null pool")); 178 /* 179 * mtx_pool_next is unprotected against multiple accesses, 180 * but simultaneous access by two CPUs should not be very 181 * harmful. 182 */ 183 i = pool->mtx_pool_next; 184 pool->mtx_pool_next = (i + 1) & pool->mtx_pool_mask; 185 return (&pool->mtx_pool_ary[i]); 186 } 187 188 SYSINIT(mtxpooli2, SI_SUB_MTX_POOL_DYNAMIC, SI_ORDER_FIRST, 189 mtx_pool_setup_dynamic, NULL); 190