1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2004-2009 University of Zagreb
5  * Copyright (c) 2006-2009 FreeBSD Foundation
44 #include <sys/kernel.h>
67 /*-
70  * - Virtual network stack management functions.
72  * - Virtual network stack memory allocator, which virtualizes global
75  * - Virtualized SYSINIT's/SYSUNINIT's, which allow network stack subsystems
80 FEATURE(vimage, "VIMAGE kernel virtualization");
85  * The virtual network stack list has two read-write locks, one sleepable and
112  * virtualization-agnostic.
114  * The virtual network stack allocator handles variables in the base kernel
117  * vnet linker set.  These "master" copies of global variables serve two
123  *     to zero-filled.
127  *     will be used to calculate the location of a per-virtual instance
128  *     variable at run-time.
132  * referred to by vnet->vnet_data_mem.  Critical to the design is that each
133  * per-instance memory block is laid out identically to the master block so
135  * optimize run-time access, a precalculated 'base' address,
136  * vnet->vnet_data_base, is stored in each vnet, and is the amount that can
137  * be added to the address of a 'master' instance of a variable to get to the
138  * per-vnet instance.
142  * virtualized globals togther, we reserve space in the kernel's linker set
143  * for potential module variables using a per-vnet character array,
146  * linked, allocates portions of the space to specific globals.  The kernel
151  * existing virtual network stack instances so that the soon-to-be executing
157  * size of all kernel virtualized global variables, and the malloc(9) type
160 #define	VNET_BYTES	(VNET_STOP - VNET_START)
167  * array declared as a virtualized global variable in the kernel itself, and
168  * we want the virtualized global variable space to be page-sized, we may
175  * Space to store virtualized global variables from loadable kernel modules,
226  * Run per-vnet sysinits or sysuninits during vnet creation/destruction.
245 	vnet->vnet_magic_n = VNET_MAGIC_N;  in vnet_alloc()
250 	 * initial values from our 'master' copy.  in vnet_alloc()
252 	vnet->vnet_data_mem = malloc(VNET_SIZE, M_VNET_DATA, M_WAITOK);  in vnet_alloc()
253 	memcpy(vnet->vnet_data_mem, (void *)VNET_START, VNET_BYTES);  in vnet_alloc()
256 	 * All use of vnet-specific data will immediately subtract VNET_START  in vnet_alloc()
257 	 * from the base memory pointer, so pre-calculate that now to avoid  in vnet_alloc()
260 	vnet->vnet_data_base = (uintptr_t)vnet->vnet_data_mem - VNET_START;  in vnet_alloc()
283 	KASSERT(vnet->vnet_sockcnt == 0,  in vnet_destroy()
291 	vnet->vnet_shutdown = true;  in vnet_destroy()
302 	free(vnet->vnet_data_mem, M_VNET_DATA);  in vnet_destroy()
303 	vnet->vnet_data_mem = NULL;  in vnet_destroy()
304 	vnet->vnet_data_base = 0;  in vnet_destroy()
305 	vnet->vnet_magic_n = 0xdeadbeef;  in vnet_destroy()
358 	df->vnd_start = (uintptr_t)&VNET_NAME(modspace);  in vnet_data_startup()
359 	df->vnd_len = VNET_MODMIN;  in vnet_data_startup()
379  * should be used only by the kernel linker.
391 		if (df->vnd_len < size)  in vnet_data_alloc()
393 		if (df->vnd_len == size) {  in vnet_data_alloc()
394 			s = (void *)df->vnd_start;  in vnet_data_alloc()
399 		s = (void *)df->vnd_start;  in vnet_data_alloc()
400 		df->vnd_len -= size;  in vnet_data_alloc()
401 		df->vnd_start = df->vnd_start + size;  in vnet_data_alloc()
429 		if (df->vnd_start > end)  in vnet_data_free()
435 		if (df->vnd_start + df->vnd_len == start) {  in vnet_data_free()
436 			df->vnd_len += size;  in vnet_data_free()
438 			if (df->vnd_start + df->vnd_len == dn->vnd_start) {  in vnet_data_free()
439 				df->vnd_len += dn->vnd_len;  in vnet_data_free()
447 		if (df->vnd_start == end) {  in vnet_data_free()
448 			df->vnd_start = start;  in vnet_data_free()
449 			df->vnd_len += size;  in vnet_data_free()
455 	dn->vnd_start = start;  in vnet_data_free()
456 	dn->vnd_len = size;  in vnet_data_free()
466  * initial value to each already-allocated virtual network stack instance.
475 		memcpy((void *)((uintptr_t)vnet->vnet_data_base +  in vnet_data_copy()
489 	memcpy((void *)(vnet_init_var + ((uintptr_t)start - VNET_START)),  in vnet_save_init()
494  * Restore the 'master' copies of virtualized global variables to theirs
504 	    (void *)(vnet_init_var + ((uintptr_t)start - VNET_START)), size);  in vnet_restore_init()
518 	KASSERT(vs->subsystem > SI_SUB_VNET, ("vnet sysinit too early"));  in vnet_register_sysinit()
523 		if (vs2->subsystem > vs->subsystem)  in vnet_register_sysinit()
525 		if (vs2->subsystem == vs->subsystem && vs2->order > vs->order)  in vnet_register_sysinit()
540 		vs->func(vs->arg);  in vnet_register_sysinit()
570 		if (vs2->subsystem > vs->subsystem)  in vnet_register_sysuninit()
572 		if (vs2->subsystem == vs->subsystem && vs2->order > vs->order)  in vnet_register_sysuninit()
598 		vs->func(vs->arg);  in vnet_deregister_sysuninit()
620 		curvnet->vnet_state = vs->subsystem;  in vnet_sysinit()
621 		vs->func(vs->arg);  in vnet_sysinit()
639 		curvnet->vnet_state = vs->subsystem;  in vnet_sysuninit()
640 		vs->func(vs->arg);  in vnet_sysuninit()
665 	 * possible in here so just re-using the variadic version we  in vnet_global_eventhandler_iterator_func()
672 		((vimage_iterator_func_t)v_ee->func)(v_ee->ee_arg);  in vnet_global_eventhandler_iterator_func()
697 	printf("%s() line %d, prev in %s()", vnr->where_fn, vnr->where_line,  in vnet_print_recursion()
698 	    vnr->prev_fn);  in vnet_print_recursion()
703 	printf("%p -> %p\n", vnr->old_vnet, vnr->new_vnet);  in vnet_print_recursion()
713 		if (vnr->prev_fn == old_fn &&  in vnet_log_recursion()
714 		    vnr->where_fn == curthread->td_vnet_lpush &&  in vnet_log_recursion()
715 		    vnr->where_line == line &&  in vnet_log_recursion()
716 		    (vnr->old_vnet == vnr->new_vnet) == (curvnet == old_vnet))  in vnet_log_recursion()
722 	vnr->prev_fn = old_fn;  in vnet_log_recursion()
723 	vnr->where_fn = curthread->td_vnet_lpush;  in vnet_log_recursion()
724 	vnr->where_line = line;  in vnet_log_recursion()
725 	vnr->old_vnet = old_vnet;  in vnet_log_recursion()
726 	vnr->new_vnet = curvnet;  in vnet_log_recursion()
747 	    vnet->vnet_magic_n,  in db_vnet_print()
748 	    (vnet->vnet_magic_n == VNET_MAGIC_N) ?  in db_vnet_print()
750 	db_printf(" vnet_ifcnt     = %u\n", vnet->vnet_ifcnt);  in db_vnet_print()
751 	db_printf(" vnet_sockcnt   = %u\n", vnet->vnet_sockcnt);  in db_vnet_print()
752 	db_printf(" vnet_data_mem  = %p\n", vnet->vnet_data_mem);  in db_vnet_print()
754 	    (uintmax_t)vnet->vnet_data_base);  in db_vnet_print()
755 	db_printf(" vnet_state     = %#08x\n", vnet->vnet_state);  in db_vnet_print()
756 	db_printf(" vnet_shutdown  = %#03x\n", vnet->vnet_shutdown);  in db_vnet_print()
803 	sym = db_search_symbol((vm_offset_t)vs->func, DB_STGY_PROC, &offset);  in db_show_vnet_print_vs()
806 	xprint("  %#08x %#08x\n", vs->subsystem, vs->order);  in db_show_vnet_print_vs()
808 	    vs->func, (funcname != NULL) ? funcname : "", vs->arg);  in db_show_vnet_print_vs()