1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2007 Pawel Jakub Dawidek <pjd@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/kernel.h> 31 #include <sys/systm.h> 32 #include <sys/sysctl.h> 33 #include <sys/errno.h> 34 #include <sys/jail.h> 35 #include <sys/malloc.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/rmlock.h> 39 #include <sys/sx.h> 40 #include <sys/queue.h> 41 #include <sys/proc.h> 42 #include <sys/osd.h> 43 44 /* OSD (Object Specific Data) */ 45 46 /* 47 * Lock key: 48 * (m) osd_module_lock 49 * (o) osd_object_lock 50 * (l) osd_list_lock 51 */ 52 struct osd_master { 53 struct sx osd_module_lock; 54 struct rmlock osd_object_lock; 55 struct mtx osd_list_lock; 56 LIST_HEAD(, osd) osd_list; /* (l) */ 57 osd_destructor_t *osd_destructors; /* (o) */ 58 osd_method_t *osd_methods; /* (m) */ 59 u_int osd_ntslots; /* (m) */ 60 const u_int osd_nmethods; 61 }; 62 63 static MALLOC_DEFINE(M_OSD, "osd", "Object Specific Data"); 64 65 static int osd_debug = 0; 66 SYSCTL_INT(_debug, OID_AUTO, osd, CTLFLAG_RWTUN, &osd_debug, 0, "OSD debug level"); 67 68 #define OSD_DEBUG(...) do { \ 69 if (osd_debug) { \ 70 printf("OSD (%s:%u): ", __func__, __LINE__); \ 71 printf(__VA_ARGS__); \ 72 printf("\n"); \ 73 } \ 74 } while (0) 75 76 static void do_osd_del(u_int type, struct osd *osd, u_int slot, 77 int list_locked); 78 79 /* 80 * List of objects with OSD. 81 */ 82 struct osd_master osdm[OSD_LAST + 1] = { 83 [OSD_JAIL] = { .osd_nmethods = PR_MAXMETHOD }, 84 }; 85 86 static void 87 osd_default_destructor(void *value __unused) 88 { 89 /* Do nothing. */ 90 } 91 92 int 93 osd_register(u_int type, osd_destructor_t destructor, osd_method_t *methods) 94 { 95 void *newptr; 96 u_int i, m; 97 98 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 99 100 /* 101 * If no destructor is given, use default one. We need to use some 102 * destructor, because NULL destructor means unused slot. 103 */ 104 if (destructor == NULL) 105 destructor = osd_default_destructor; 106 107 sx_xlock(&osdm[type].osd_module_lock); 108 /* 109 * First, we try to find unused slot. 110 */ 111 for (i = 0; i < osdm[type].osd_ntslots; i++) { 112 if (osdm[type].osd_destructors[i] == NULL) { 113 OSD_DEBUG("Unused slot found (type=%u, slot=%u).", 114 type, i + 1); 115 break; 116 } 117 } 118 /* 119 * If no unused slot was found, allocate one. 120 */ 121 if (i == osdm[type].osd_ntslots) { 122 osdm[type].osd_ntslots++; 123 if (osdm[type].osd_nmethods != 0) 124 osdm[type].osd_methods = realloc(osdm[type].osd_methods, 125 sizeof(osd_method_t) * osdm[type].osd_ntslots * 126 osdm[type].osd_nmethods, M_OSD, M_WAITOK); 127 newptr = malloc(sizeof(osd_destructor_t) * 128 osdm[type].osd_ntslots, M_OSD, M_WAITOK); 129 rm_wlock(&osdm[type].osd_object_lock); 130 bcopy(osdm[type].osd_destructors, newptr, 131 sizeof(osd_destructor_t) * i); 132 free(osdm[type].osd_destructors, M_OSD); 133 osdm[type].osd_destructors = newptr; 134 rm_wunlock(&osdm[type].osd_object_lock); 135 OSD_DEBUG("New slot allocated (type=%u, slot=%u).", 136 type, i + 1); 137 } 138 139 osdm[type].osd_destructors[i] = destructor; 140 if (osdm[type].osd_nmethods != 0) { 141 for (m = 0; m < osdm[type].osd_nmethods; m++) 142 osdm[type].osd_methods[i * osdm[type].osd_nmethods + m] 143 = methods != NULL ? methods[m] : NULL; 144 } 145 sx_xunlock(&osdm[type].osd_module_lock); 146 return (i + 1); 147 } 148 149 void 150 osd_deregister(u_int type, u_int slot) 151 { 152 struct osd *osd, *tosd; 153 154 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 155 KASSERT(slot > 0, ("Invalid slot.")); 156 157 sx_xlock(&osdm[type].osd_module_lock); 158 rm_wlock(&osdm[type].osd_object_lock); 159 KASSERT(osdm[type].osd_destructors[slot - 1] != NULL, ("Unused slot.")); 160 161 /* 162 * Free all OSD for the given slot. 163 */ 164 mtx_lock(&osdm[type].osd_list_lock); 165 LIST_FOREACH_SAFE(osd, &osdm[type].osd_list, osd_next, tosd) 166 do_osd_del(type, osd, slot, 1); 167 mtx_unlock(&osdm[type].osd_list_lock); 168 169 /* 170 * Set destructor to NULL to free the slot. We don't bother actually 171 * freeing any memory here because we'll gracefully reuse any freed 172 * slots, and reallocating the arrays as a smaller chunk of memory isn't 173 * actually guaranteed to succeed. As such, we'll err on the side of 174 * caution and just leave it be since these are generally modestly sized 175 * allocations. 176 */ 177 osdm[type].osd_destructors[slot - 1] = NULL; 178 OSD_DEBUG("Slot deregistration (type=%u, slot=%u).", type, slot); 179 180 rm_wunlock(&osdm[type].osd_object_lock); 181 sx_xunlock(&osdm[type].osd_module_lock); 182 } 183 184 int 185 osd_set(u_int type, struct osd *osd, u_int slot, void *value) 186 { 187 188 return (osd_set_reserved(type, osd, slot, NULL, value)); 189 } 190 191 void ** 192 osd_reserve(u_int slot) 193 { 194 195 KASSERT(slot > 0, ("Invalid slot.")); 196 197 OSD_DEBUG("Reserving slot array (slot=%u).", slot); 198 return (malloc(sizeof(void *) * slot, M_OSD, M_WAITOK | M_ZERO)); 199 } 200 201 int 202 osd_set_reserved(u_int type, struct osd *osd, u_int slot, void **rsv, 203 void *value) 204 { 205 struct rm_priotracker tracker; 206 207 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 208 KASSERT(slot > 0, ("Invalid slot.")); 209 210 rm_rlock(&osdm[type].osd_object_lock, &tracker); 211 KASSERT(osdm[type].osd_destructors[slot - 1] != NULL, ("Unused slot.")); 212 213 if (slot > osd->osd_nslots) { 214 void **newptr; 215 216 if (value == NULL) { 217 OSD_DEBUG( 218 "Not allocating null slot (type=%u, slot=%u).", 219 type, slot); 220 rm_runlock(&osdm[type].osd_object_lock, &tracker); 221 if (rsv) 222 osd_free_reserved(rsv); 223 return (0); 224 } 225 226 /* 227 * Too few slots allocated here, so we need to extend or create 228 * the array. 229 */ 230 if (rsv) { 231 /* 232 * Use the reserve passed in (assumed to be 233 * the right size). 234 */ 235 newptr = rsv; 236 if (osd->osd_nslots != 0) { 237 memcpy(newptr, osd->osd_slots, 238 sizeof(void *) * osd->osd_nslots); 239 free(osd->osd_slots, M_OSD); 240 } 241 } else { 242 newptr = realloc(osd->osd_slots, sizeof(void *) * slot, 243 M_OSD, M_NOWAIT | M_ZERO); 244 if (newptr == NULL) { 245 rm_runlock(&osdm[type].osd_object_lock, 246 &tracker); 247 return (ENOMEM); 248 } 249 } 250 if (osd->osd_nslots == 0) { 251 /* 252 * First OSD for this object, so we need to put it 253 * onto the list. 254 */ 255 mtx_lock(&osdm[type].osd_list_lock); 256 LIST_INSERT_HEAD(&osdm[type].osd_list, osd, osd_next); 257 mtx_unlock(&osdm[type].osd_list_lock); 258 OSD_DEBUG("Setting first slot (type=%u).", type); 259 } else 260 OSD_DEBUG("Growing slots array (type=%u).", type); 261 osd->osd_slots = newptr; 262 osd->osd_nslots = slot; 263 } else if (rsv) 264 osd_free_reserved(rsv); 265 OSD_DEBUG("Setting slot value (type=%u, slot=%u, value=%p).", type, 266 slot, value); 267 osd->osd_slots[slot - 1] = value; 268 rm_runlock(&osdm[type].osd_object_lock, &tracker); 269 return (0); 270 } 271 272 void 273 osd_free_reserved(void **rsv) 274 { 275 276 OSD_DEBUG("Discarding reserved slot array."); 277 free(rsv, M_OSD); 278 } 279 280 void * 281 osd_get_unlocked(u_int type, struct osd *osd, u_int slot) 282 { 283 void *value; 284 285 KASSERT(osdm[type].osd_destructors[slot - 1] != NULL, ("Unused slot.")); 286 287 if (slot > osd->osd_nslots) { 288 value = NULL; 289 OSD_DEBUG("Slot doesn't exist (type=%u, slot=%u).", type, slot); 290 } else { 291 value = atomic_load_ptr(&osd->osd_slots[slot - 1]); 292 OSD_DEBUG("Returning slot value (type=%u, slot=%u, value=%p).", 293 type, slot, value); 294 } 295 return (value); 296 } 297 298 void * 299 osd_get(u_int type, struct osd *osd, u_int slot) 300 { 301 struct rm_priotracker tracker; 302 void *value; 303 304 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 305 KASSERT(slot > 0, ("Invalid slot.")); 306 307 rm_rlock(&osdm[type].osd_object_lock, &tracker); 308 value = osd_get_unlocked(type, osd, slot); 309 rm_runlock(&osdm[type].osd_object_lock, &tracker); 310 return (value); 311 } 312 313 void 314 osd_del(u_int type, struct osd *osd, u_int slot) 315 { 316 struct rm_priotracker tracker; 317 318 rm_rlock(&osdm[type].osd_object_lock, &tracker); 319 do_osd_del(type, osd, slot, 0); 320 rm_runlock(&osdm[type].osd_object_lock, &tracker); 321 } 322 323 static void 324 do_osd_del(u_int type, struct osd *osd, u_int slot, int list_locked) 325 { 326 int i; 327 328 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 329 KASSERT(slot > 0, ("Invalid slot.")); 330 KASSERT(osdm[type].osd_destructors[slot - 1] != NULL, ("Unused slot.")); 331 332 OSD_DEBUG("Deleting slot (type=%u, slot=%u).", type, slot); 333 334 if (slot > osd->osd_nslots) { 335 OSD_DEBUG("Slot doesn't exist (type=%u, slot=%u).", type, slot); 336 return; 337 } 338 if (osd->osd_slots[slot - 1] != NULL) { 339 osdm[type].osd_destructors[slot - 1](osd->osd_slots[slot - 1]); 340 osd->osd_slots[slot - 1] = NULL; 341 } 342 for (i = osd->osd_nslots - 1; i >= 0; i--) { 343 if (osd->osd_slots[i] != NULL) { 344 OSD_DEBUG("Slot still has a value (type=%u, slot=%u).", 345 type, i + 1); 346 break; 347 } 348 } 349 if (i == -1) { 350 /* No values left for this object. */ 351 OSD_DEBUG("No more slots left (type=%u).", type); 352 if (!list_locked) 353 mtx_lock(&osdm[type].osd_list_lock); 354 LIST_REMOVE(osd, osd_next); 355 if (!list_locked) 356 mtx_unlock(&osdm[type].osd_list_lock); 357 free(osd->osd_slots, M_OSD); 358 osd->osd_slots = NULL; 359 osd->osd_nslots = 0; 360 } else if (slot == osd->osd_nslots) { 361 /* This was the last slot. */ 362 osd->osd_slots = realloc(osd->osd_slots, 363 sizeof(void *) * (i + 1), M_OSD, M_NOWAIT | M_ZERO); 364 /* 365 * We always reallocate to smaller size, so we assume it will 366 * always succeed. 367 */ 368 KASSERT(osd->osd_slots != NULL, ("realloc() failed")); 369 osd->osd_nslots = i + 1; 370 OSD_DEBUG("Reducing slots array to %u (type=%u).", 371 osd->osd_nslots, type); 372 } 373 } 374 375 int 376 osd_call(u_int type, u_int method, void *obj, void *data) 377 { 378 osd_method_t methodfun; 379 int error, i; 380 381 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 382 KASSERT(method < osdm[type].osd_nmethods, ("Invalid method.")); 383 384 /* 385 * Call this method for every slot that defines it, stopping if an 386 * error is encountered. 387 */ 388 error = 0; 389 sx_slock(&osdm[type].osd_module_lock); 390 for (i = 0; i < osdm[type].osd_ntslots; i++) { 391 /* Hole in the slot map; avoid dereferencing. */ 392 if (osdm[type].osd_destructors[i] == NULL) 393 continue; 394 methodfun = osdm[type].osd_methods[i * osdm[type].osd_nmethods + 395 method]; 396 if (methodfun != NULL && (error = methodfun(obj, data)) != 0) 397 break; 398 } 399 sx_sunlock(&osdm[type].osd_module_lock); 400 return (error); 401 } 402 403 void 404 osd_exit(u_int type, struct osd *osd) 405 { 406 struct rm_priotracker tracker; 407 u_int i; 408 409 KASSERT(type >= OSD_FIRST && type <= OSD_LAST, ("Invalid type.")); 410 411 if (osd->osd_nslots == 0) { 412 KASSERT(osd->osd_slots == NULL, ("Non-null osd_slots.")); 413 /* No OSD attached, just leave. */ 414 return; 415 } 416 417 rm_rlock(&osdm[type].osd_object_lock, &tracker); 418 for (i = 1; i <= osd->osd_nslots; i++) { 419 if (osdm[type].osd_destructors[i - 1] != NULL) 420 do_osd_del(type, osd, i, 0); 421 else 422 OSD_DEBUG("Unused slot (type=%u, slot=%u).", type, i); 423 } 424 rm_runlock(&osdm[type].osd_object_lock, &tracker); 425 OSD_DEBUG("Object exit (type=%u).", type); 426 } 427 428 static void 429 osd_init(void *arg __unused) 430 { 431 u_int i; 432 433 for (i = OSD_FIRST; i <= OSD_LAST; i++) { 434 sx_init(&osdm[i].osd_module_lock, "osd_module"); 435 rm_init(&osdm[i].osd_object_lock, "osd_object"); 436 mtx_init(&osdm[i].osd_list_lock, "osd_list", NULL, MTX_DEF); 437 LIST_INIT(&osdm[i].osd_list); 438 osdm[i].osd_destructors = NULL; 439 osdm[i].osd_ntslots = 0; 440 osdm[i].osd_methods = NULL; 441 } 442 } 443 SYSINIT(osd, SI_SUB_LOCK, SI_ORDER_ANY, osd_init, NULL); 444