1 /*- 2 * Copyright (c) 2010 Lawrence Stewart <lstewart@freebsd.org> 3 * Copyright (c) 2010 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Lawrence Stewart while studying at the Centre 7 * for Advanced Internet Architectures, Swinburne University of Technology, 8 * made possible in part by grants from the FreeBSD Foundation and Cisco 9 * University Research Program Fund at Community Foundation Silicon Valley. 10 * 11 * Portions of this software were developed at the Centre for Advanced 12 * Internet Architectures, Swinburne University of Technology, Melbourne, 13 * Australia by Lawrence Stewart under sponsorship from the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include <sys/param.h> 41 #include <sys/kernel.h> 42 #include <sys/hhook.h> 43 #include <sys/jail.h> 44 #include <sys/khelp.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/module.h> 48 #include <sys/module_khelp.h> 49 #include <sys/osd.h> 50 #include <sys/queue.h> 51 #include <sys/refcount.h> 52 #include <sys/rwlock.h> 53 #include <sys/systm.h> 54 55 #include <net/vnet.h> 56 57 static struct rwlock khelp_list_lock; 58 RW_SYSINIT(khelplistlock, &khelp_list_lock, "helper list lock"); 59 60 static TAILQ_HEAD(helper_head, helper) helpers = TAILQ_HEAD_INITIALIZER(helpers); 61 62 /* Private function prototypes. */ 63 static inline void khelp_remove_osd(struct helper *h, struct osd *hosd); 64 65 #define KHELP_LIST_WLOCK() rw_wlock(&khelp_list_lock) 66 #define KHELP_LIST_WUNLOCK() rw_wunlock(&khelp_list_lock) 67 #define KHELP_LIST_RLOCK() rw_rlock(&khelp_list_lock) 68 #define KHELP_LIST_RUNLOCK() rw_runlock(&khelp_list_lock) 69 #define KHELP_LIST_LOCK_ASSERT() rw_assert(&khelp_list_lock, RA_LOCKED) 70 71 int 72 khelp_register_helper(struct helper *h) 73 { 74 struct helper *tmph; 75 int error, i, inserted; 76 77 error = 0; 78 inserted = 0; 79 refcount_init(&h->h_refcount, 0); 80 h->h_id = osd_register(OSD_KHELP, NULL, NULL); 81 82 /* It's only safe to add the hooks after osd_register(). */ 83 if (h->h_nhooks > 0) { 84 for (i = 0; i < h->h_nhooks && !error; i++) { 85 /* We don't require the module to assign hook_helper. */ 86 h->h_hooks[i].hook_helper = h; 87 error = khelp_add_hhook(&h->h_hooks[i], HHOOK_NOWAIT); 88 } 89 90 if (error) { 91 for (i--; i >= 0; i--) 92 khelp_remove_hhook(&h->h_hooks[i]); 93 94 osd_deregister(OSD_KHELP, h->h_id); 95 } 96 } 97 98 if (!error) { 99 KHELP_LIST_WLOCK(); 100 /* 101 * Keep list of helpers sorted in descending h_id order. Due to 102 * the way osd_set() works, a sorted list ensures 103 * init_helper_osd() will operate with improved efficiency. 104 */ 105 TAILQ_FOREACH(tmph, &helpers, h_next) { 106 if (tmph->h_id < h->h_id) { 107 TAILQ_INSERT_BEFORE(tmph, h, h_next); 108 inserted = 1; 109 break; 110 } 111 } 112 113 if (!inserted) 114 TAILQ_INSERT_TAIL(&helpers, h, h_next); 115 KHELP_LIST_WUNLOCK(); 116 } 117 118 return (error); 119 } 120 121 int 122 khelp_deregister_helper(struct helper *h) 123 { 124 struct helper *tmph; 125 int error, i; 126 127 error = 0; 128 129 KHELP_LIST_WLOCK(); 130 if (h->h_refcount > 0) 131 error = EBUSY; 132 else { 133 error = ENOENT; 134 TAILQ_FOREACH(tmph, &helpers, h_next) { 135 if (tmph == h) { 136 TAILQ_REMOVE(&helpers, h, h_next); 137 error = 0; 138 break; 139 } 140 } 141 } 142 KHELP_LIST_WUNLOCK(); 143 144 if (!error) { 145 if (h->h_nhooks > 0) { 146 for (i = 0; i < h->h_nhooks; i++) 147 khelp_remove_hhook(&h->h_hooks[i]); 148 } 149 osd_deregister(OSD_KHELP, h->h_id); 150 } 151 152 return (error); 153 } 154 155 int 156 khelp_init_osd(uint32_t classes, struct osd *hosd) 157 { 158 struct helper *h; 159 void *hdata; 160 int error; 161 162 KASSERT(hosd != NULL, ("struct osd not initialised!")); 163 164 error = 0; 165 166 KHELP_LIST_RLOCK(); 167 TAILQ_FOREACH(h, &helpers, h_next) { 168 /* If helper is correct class and needs to store OSD... */ 169 if (h->h_classes & classes && h->h_flags & HELPER_NEEDS_OSD) { 170 hdata = uma_zalloc(h->h_zone, M_NOWAIT); 171 if (hdata == NULL) { 172 error = ENOMEM; 173 break; 174 } 175 osd_set(OSD_KHELP, hosd, h->h_id, hdata); 176 refcount_acquire(&h->h_refcount); 177 } 178 } 179 180 if (error) { 181 /* Delete OSD that was assigned prior to the error. */ 182 TAILQ_FOREACH(h, &helpers, h_next) { 183 if (h->h_classes & classes) 184 khelp_remove_osd(h, hosd); 185 } 186 } 187 KHELP_LIST_RUNLOCK(); 188 189 return (error); 190 } 191 192 int 193 khelp_destroy_osd(struct osd *hosd) 194 { 195 struct helper *h; 196 int error; 197 198 KASSERT(hosd != NULL, ("struct osd not initialised!")); 199 200 error = 0; 201 202 KHELP_LIST_RLOCK(); 203 /* 204 * Clean up all khelp related OSD. 205 * 206 * XXXLAS: Would be nice to use something like osd_exit() here but it 207 * doesn't have the right semantics for this purpose. 208 */ 209 TAILQ_FOREACH(h, &helpers, h_next) 210 khelp_remove_osd(h, hosd); 211 KHELP_LIST_RUNLOCK(); 212 213 return (error); 214 } 215 216 static inline void 217 khelp_remove_osd(struct helper *h, struct osd *hosd) 218 { 219 void *hdata; 220 221 if (h->h_flags & HELPER_NEEDS_OSD) { 222 /* 223 * If the current helper uses OSD and calling osd_get() 224 * on the helper's h_id returns non-NULL, the helper has 225 * OSD attached to 'hosd' which needs to be cleaned up. 226 */ 227 hdata = osd_get(OSD_KHELP, hosd, h->h_id); 228 if (hdata != NULL) { 229 uma_zfree(h->h_zone, hdata); 230 osd_del(OSD_KHELP, hosd, h->h_id); 231 refcount_release(&h->h_refcount); 232 } 233 } 234 } 235 236 void * 237 khelp_get_osd(struct osd *hosd, int32_t id) 238 { 239 240 return (osd_get(OSD_KHELP, hosd, id)); 241 } 242 243 int32_t 244 khelp_get_id(char *hname) 245 { 246 struct helper *h; 247 int32_t id; 248 249 id = -1; 250 251 KHELP_LIST_RLOCK(); 252 TAILQ_FOREACH(h, &helpers, h_next) { 253 if (strncmp(h->h_name, hname, HELPER_NAME_MAXLEN) == 0) { 254 id = h->h_id; 255 break; 256 } 257 } 258 KHELP_LIST_RUNLOCK(); 259 260 return (id); 261 } 262 263 int 264 khelp_add_hhook(struct hookinfo *hki, uint32_t flags) 265 { 266 VNET_ITERATOR_DECL(vnet_iter); 267 int error; 268 269 error = 0; 270 271 /* 272 * XXXLAS: If a helper is dynamically adding a helper hook function at 273 * runtime using this function, we should update the helper's h_hooks 274 * struct member to include the additional hookinfo struct. 275 */ 276 277 VNET_LIST_RLOCK_NOSLEEP(); 278 VNET_FOREACH(vnet_iter) { 279 CURVNET_SET(vnet_iter); 280 error = hhook_add_hook_lookup(hki, flags); 281 CURVNET_RESTORE(); 282 #ifdef VIMAGE 283 if (error) 284 break; 285 #endif 286 } 287 VNET_LIST_RUNLOCK_NOSLEEP(); 288 289 return (error); 290 } 291 292 int 293 khelp_remove_hhook(struct hookinfo *hki) 294 { 295 VNET_ITERATOR_DECL(vnet_iter); 296 int error; 297 298 error = 0; 299 300 /* 301 * XXXLAS: If a helper is dynamically removing a helper hook function at 302 * runtime using this function, we should update the helper's h_hooks 303 * struct member to remove the defunct hookinfo struct. 304 */ 305 306 VNET_LIST_RLOCK_NOSLEEP(); 307 VNET_FOREACH(vnet_iter) { 308 CURVNET_SET(vnet_iter); 309 error = hhook_remove_hook_lookup(hki); 310 CURVNET_RESTORE(); 311 #ifdef VIMAGE 312 if (error) 313 break; 314 #endif 315 } 316 VNET_LIST_RUNLOCK_NOSLEEP(); 317 318 return (error); 319 } 320 321 int 322 khelp_modevent(module_t mod, int event_type, void *data) 323 { 324 struct khelp_modevent_data *kmd; 325 int error; 326 327 kmd = (struct khelp_modevent_data *)data; 328 error = 0; 329 330 switch(event_type) { 331 case MOD_LOAD: 332 if (kmd->helper->h_flags & HELPER_NEEDS_OSD) { 333 if (kmd->uma_zsize <= 0) { 334 printf("Use KHELP_DECLARE_MOD_UMA() instead!\n"); 335 error = EDOOFUS; 336 break; 337 } 338 kmd->helper->h_zone = uma_zcreate(kmd->name, 339 kmd->uma_zsize, kmd->umactor, kmd->umadtor, NULL, 340 NULL, 0, 0); 341 if (kmd->helper->h_zone == NULL) { 342 error = ENOMEM; 343 break; 344 } 345 } 346 strlcpy(kmd->helper->h_name, kmd->name, HELPER_NAME_MAXLEN); 347 kmd->helper->h_hooks = kmd->hooks; 348 kmd->helper->h_nhooks = kmd->nhooks; 349 if (kmd->helper->mod_init != NULL) 350 error = kmd->helper->mod_init(); 351 if (!error) 352 error = khelp_register_helper(kmd->helper); 353 break; 354 355 case MOD_QUIESCE: 356 case MOD_SHUTDOWN: 357 case MOD_UNLOAD: 358 error = khelp_deregister_helper(kmd->helper); 359 if (!error) { 360 if (kmd->helper->h_flags & HELPER_NEEDS_OSD) 361 uma_zdestroy(kmd->helper->h_zone); 362 if (kmd->helper->mod_destroy != NULL) 363 kmd->helper->mod_destroy(); 364 } else if (error == ENOENT) 365 /* Do nothing and allow unload if helper not in list. */ 366 error = 0; 367 else if (error == EBUSY) 368 printf("Khelp module \"%s\" can't unload until its " 369 "refcount drops from %d to 0.\n", kmd->name, 370 kmd->helper->h_refcount); 371 break; 372 373 default: 374 error = EINVAL; 375 break; 376 } 377 378 return (error); 379 } 380 381 /* 382 * This function is called in two separate situations: 383 * 384 * - When the kernel is booting, it is called directly by the SYSINIT framework 385 * to allow Khelp modules which were compiled into the kernel or loaded by the 386 * boot loader to insert their non-virtualised hook functions into the kernel. 387 * 388 * - When the kernel is booting or a vnet is created, this function is also 389 * called indirectly through khelp_vnet_init() by the vnet initialisation code. 390 * In this situation, Khelp modules are able to insert their virtualised hook 391 * functions into the virtualised hook points in the vnet which is being 392 * initialised. In the case where the kernel is not compiled with "options 393 * VIMAGE", this step is still run once at boot, but the hook functions get 394 * transparently inserted into the standard unvirtualised network stack. 395 */ 396 static void 397 khelp_init(const void *vnet) 398 { 399 struct helper *h; 400 int error, i, vinit; 401 int32_t htype, hid; 402 403 error = 0; 404 vinit = vnet != NULL; 405 406 KHELP_LIST_RLOCK(); 407 TAILQ_FOREACH(h, &helpers, h_next) { 408 for (i = 0; i < h->h_nhooks && !error; i++) { 409 htype = h->h_hooks[i].hook_type; 410 hid = h->h_hooks[i].hook_id; 411 412 /* 413 * If we're doing a virtualised init (vinit != 0) and 414 * the hook point is virtualised, or we're doing a plain 415 * sysinit at boot and the hook point is not 416 * virtualised, insert the hook. 417 */ 418 if ((hhook_head_is_virtualised_lookup(htype, hid) == 419 HHOOK_HEADISINVNET && vinit) || 420 (!hhook_head_is_virtualised_lookup(htype, hid) && 421 !vinit)) { 422 error = hhook_add_hook_lookup(&h->h_hooks[i], 423 HHOOK_NOWAIT); 424 } 425 } 426 427 if (error) { 428 /* Remove any helper's hooks we successfully added. */ 429 for (i--; i >= 0; i--) 430 hhook_remove_hook_lookup(&h->h_hooks[i]); 431 432 printf("%s: Failed to add hooks for helper \"%s\" (%p)", 433 __func__, h->h_name, h); 434 if (vinit) 435 printf(" to vnet %p.\n", vnet); 436 else 437 printf(".\n"); 438 439 error = 0; 440 } 441 } 442 KHELP_LIST_RUNLOCK(); 443 } 444 445 /* 446 * Vnet created and being initialised. 447 */ 448 static void 449 khelp_vnet_init(const void *unused __unused) 450 { 451 452 khelp_init(TD_TO_VNET(curthread)); 453 } 454 455 456 /* 457 * As the kernel boots, allow Khelp modules which were compiled into the kernel 458 * or loaded by the boot loader to insert their non-virtualised hook functions 459 * into the kernel. 460 */ 461 SYSINIT(khelp_init, SI_SUB_PROTO_END, SI_ORDER_FIRST, khelp_init, NULL); 462 463 /* 464 * When a vnet is created and being initialised, we need to insert the helper 465 * hook functions for all currently registered Khelp modules into the vnet's 466 * helper hook points. The hhook KPI provides a mechanism for subsystems which 467 * export helper hook points to clean up on vnet shutdown, so we don't need a 468 * VNET_SYSUNINIT for Khelp. 469 */ 470 VNET_SYSINIT(khelp_vnet_init, SI_SUB_PROTO_END, SI_ORDER_FIRST, 471 khelp_vnet_init, NULL); 472