xref: /illumos-gate/usr/src/uts/common/io/vnic/vnic_dev.c (revision 9b4e3ac25d882519cad3fc11f0c53b07f4e60536)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/sysmacros.h>
28 #include <sys/conf.h>
29 #include <sys/cmn_err.h>
30 #include <sys/list.h>
31 #include <sys/ksynch.h>
32 #include <sys/kmem.h>
33 #include <sys/stream.h>
34 #include <sys/modctl.h>
35 #include <sys/ddi.h>
36 #include <sys/sunddi.h>
37 #include <sys/atomic.h>
38 #include <sys/stat.h>
39 #include <sys/modhash.h>
40 #include <sys/strsubr.h>
41 #include <sys/strsun.h>
42 #include <sys/dlpi.h>
43 #include <sys/mac.h>
44 #include <sys/mac_provider.h>
45 #include <sys/mac_client.h>
46 #include <sys/mac_client_priv.h>
47 #include <sys/mac_ether.h>
48 #include <sys/dls.h>
49 #include <sys/pattr.h>
50 #include <sys/time.h>
51 #include <sys/vlan.h>
52 #include <sys/vnic.h>
53 #include <sys/vnic_impl.h>
54 #include <sys/mac_flow_impl.h>
55 #include <inet/ip_impl.h>
56 
57 /*
58  * Note that for best performance, the VNIC is a passthrough design.
59  * For each VNIC corresponds a MAC client of the underlying MAC (lower MAC).
60  * This MAC client is opened by the VNIC driver at VNIC creation,
61  * and closed when the VNIC is deleted.
62  * When a MAC client of the VNIC itself opens a VNIC, the MAC layer
63  * (upper MAC) detects that the MAC being opened is a VNIC. Instead
64  * of allocating a new MAC client, it asks the VNIC driver to return
65  * the lower MAC client handle associated with the VNIC, and that handle
66  * is returned to the upper MAC client directly. This allows access
67  * by upper MAC clients of the VNIC to have direct access to the lower
68  * MAC client for the control path and data path.
69  *
70  * Due to this passthrough, some of the entry points exported by the
71  * VNIC driver are never directly invoked. These entry points include
72  * vnic_m_start, vnic_m_stop, vnic_m_promisc, vnic_m_multicst, etc.
73  */
74 
75 static int vnic_m_start(void *);
76 static void vnic_m_stop(void *);
77 static int vnic_m_promisc(void *, boolean_t);
78 static int vnic_m_multicst(void *, boolean_t, const uint8_t *);
79 static int vnic_m_unicst(void *, const uint8_t *);
80 static int vnic_m_stat(void *, uint_t, uint64_t *);
81 static void vnic_m_ioctl(void *, queue_t *, mblk_t *);
82 static mblk_t *vnic_m_tx(void *, mblk_t *);
83 static boolean_t vnic_m_capab_get(void *, mac_capab_t, void *);
84 static void vnic_notify_cb(void *, mac_notify_type_t);
85 
86 static kmem_cache_t	*vnic_cache;
87 static krwlock_t	vnic_lock;
88 static uint_t		vnic_count;
89 
90 /* hash of VNICs (vnic_t's), keyed by VNIC id */
91 static mod_hash_t	*vnic_hash;
92 #define	VNIC_HASHSZ	64
93 #define	VNIC_HASH_KEY(vnic_id)	((mod_hash_key_t)(uintptr_t)vnic_id)
94 
95 #define	VNIC_M_CALLBACK_FLAGS	(MC_IOCTL | MC_GETCAPAB)
96 
97 static mac_callbacks_t vnic_m_callbacks = {
98 	VNIC_M_CALLBACK_FLAGS,
99 	vnic_m_stat,
100 	vnic_m_start,
101 	vnic_m_stop,
102 	vnic_m_promisc,
103 	vnic_m_multicst,
104 	vnic_m_unicst,
105 	vnic_m_tx,
106 	vnic_m_ioctl,
107 	vnic_m_capab_get
108 };
109 
110 void
111 vnic_dev_init(void)
112 {
113 	vnic_cache = kmem_cache_create("vnic_cache",
114 	    sizeof (vnic_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
115 
116 	vnic_hash = mod_hash_create_idhash("vnic_hash",
117 	    VNIC_HASHSZ, mod_hash_null_valdtor);
118 
119 	rw_init(&vnic_lock, NULL, RW_DEFAULT, NULL);
120 
121 	vnic_count = 0;
122 }
123 
124 void
125 vnic_dev_fini(void)
126 {
127 	ASSERT(vnic_count == 0);
128 
129 	rw_destroy(&vnic_lock);
130 	mod_hash_destroy_idhash(vnic_hash);
131 	kmem_cache_destroy(vnic_cache);
132 }
133 
134 uint_t
135 vnic_dev_count(void)
136 {
137 	return (vnic_count);
138 }
139 
140 static vnic_ioc_diag_t
141 vnic_mac2vnic_diag(mac_diag_t diag)
142 {
143 	switch (diag) {
144 	case MAC_DIAG_MACADDR_NIC:
145 		return (VNIC_IOC_DIAG_MACADDR_NIC);
146 	case MAC_DIAG_MACADDR_INUSE:
147 		return (VNIC_IOC_DIAG_MACADDR_INUSE);
148 	case MAC_DIAG_MACADDR_INVALID:
149 		return (VNIC_IOC_DIAG_MACADDR_INVALID);
150 	case MAC_DIAG_MACADDRLEN_INVALID:
151 		return (VNIC_IOC_DIAG_MACADDRLEN_INVALID);
152 	case MAC_DIAG_MACFACTORYSLOTINVALID:
153 		return (VNIC_IOC_DIAG_MACFACTORYSLOTINVALID);
154 	case MAC_DIAG_MACFACTORYSLOTUSED:
155 		return (VNIC_IOC_DIAG_MACFACTORYSLOTUSED);
156 	case MAC_DIAG_MACFACTORYSLOTALLUSED:
157 		return (VNIC_IOC_DIAG_MACFACTORYSLOTALLUSED);
158 	case MAC_DIAG_MACFACTORYNOTSUP:
159 		return (VNIC_IOC_DIAG_MACFACTORYNOTSUP);
160 	case MAC_DIAG_MACPREFIX_INVALID:
161 		return (VNIC_IOC_DIAG_MACPREFIX_INVALID);
162 	case MAC_DIAG_MACPREFIXLEN_INVALID:
163 		return (VNIC_IOC_DIAG_MACPREFIXLEN_INVALID);
164 	case MAC_DIAG_MACNO_HWRINGS:
165 		return (VNIC_IOC_DIAG_NO_HWRINGS);
166 	default:
167 		return (VNIC_IOC_DIAG_NONE);
168 	}
169 }
170 
171 static int
172 vnic_unicast_add(vnic_t *vnic, vnic_mac_addr_type_t vnic_addr_type,
173     int *addr_slot, uint_t prefix_len, int *addr_len_ptr_arg,
174     uint8_t *mac_addr_arg, uint16_t flags, vnic_ioc_diag_t *diag,
175     uint16_t vid)
176 {
177 	mac_diag_t mac_diag;
178 	uint16_t mac_flags = 0;
179 	int err;
180 	uint_t addr_len;
181 
182 	if (flags & VNIC_IOC_CREATE_NODUPCHECK)
183 		mac_flags |= MAC_UNICAST_NODUPCHECK;
184 
185 	switch (vnic_addr_type) {
186 	case VNIC_MAC_ADDR_TYPE_FIXED:
187 		/*
188 		 * The MAC address value to assign to the VNIC
189 		 * is already provided in mac_addr_arg. addr_len_ptr_arg
190 		 * already contains the MAC address length.
191 		 */
192 		break;
193 
194 	case VNIC_MAC_ADDR_TYPE_RANDOM:
195 		/*
196 		 * Random MAC address. There are two sub-cases:
197 		 *
198 		 * 1 - If mac_len == 0, a new MAC address is generated.
199 		 *	The length of the MAC address to generated depends
200 		 *	on the type of MAC used. The prefix to use for the MAC
201 		 *	address is stored in the most significant bytes
202 		 *	of the mac_addr argument, and its length is specified
203 		 *	by the mac_prefix_len argument. This prefix can
204 		 *	correspond to a IEEE OUI in the case of Ethernet,
205 		 *	for example.
206 		 *
207 		 * 2 - If mac_len > 0, the address was already picked
208 		 *	randomly, and is now passed back during VNIC
209 		 *	re-creation. The mac_addr argument contains the MAC
210 		 *	address that was generated. We distinguish this
211 		 *	case from the fixed MAC address case, since we
212 		 *	want the user consumers to know, when they query
213 		 *	the list of VNICs, that a VNIC was assigned a
214 		 *	random MAC address vs assigned a fixed address
215 		 *	specified by the user.
216 		 */
217 
218 		/*
219 		 * If it's a pre-generated address, we're done. mac_addr_arg
220 		 * and addr_len_ptr_arg already contain the MAC address
221 		 * value and length.
222 		 */
223 		if (*addr_len_ptr_arg > 0)
224 			break;
225 
226 		/* generate a new random MAC address */
227 		if ((err = mac_addr_random(vnic->vn_mch,
228 		    prefix_len, mac_addr_arg, &mac_diag)) != 0) {
229 			*diag = vnic_mac2vnic_diag(mac_diag);
230 			return (err);
231 		}
232 		*addr_len_ptr_arg = mac_addr_len(vnic->vn_lower_mh);
233 		break;
234 
235 	case VNIC_MAC_ADDR_TYPE_FACTORY:
236 		err = mac_addr_factory_reserve(vnic->vn_mch, addr_slot);
237 		if (err != 0) {
238 			if (err == EINVAL)
239 				*diag = VNIC_IOC_DIAG_MACFACTORYSLOTINVALID;
240 			if (err == EBUSY)
241 				*diag = VNIC_IOC_DIAG_MACFACTORYSLOTUSED;
242 			if (err == ENOSPC)
243 				*diag = VNIC_IOC_DIAG_MACFACTORYSLOTALLUSED;
244 			return (err);
245 		}
246 
247 		mac_addr_factory_value(vnic->vn_lower_mh, *addr_slot,
248 		    mac_addr_arg, &addr_len, NULL, NULL);
249 		*addr_len_ptr_arg = addr_len;
250 		break;
251 
252 	case VNIC_MAC_ADDR_TYPE_AUTO:
253 		/* first try to allocate a factory MAC address */
254 		err = mac_addr_factory_reserve(vnic->vn_mch, addr_slot);
255 		if (err == 0) {
256 			mac_addr_factory_value(vnic->vn_lower_mh, *addr_slot,
257 			    mac_addr_arg, &addr_len, NULL, NULL);
258 			vnic_addr_type = VNIC_MAC_ADDR_TYPE_FACTORY;
259 			*addr_len_ptr_arg = addr_len;
260 			break;
261 		}
262 
263 		/*
264 		 * Allocating a factory MAC address failed, generate a
265 		 * random MAC address instead.
266 		 */
267 		if ((err = mac_addr_random(vnic->vn_mch,
268 		    prefix_len, mac_addr_arg, &mac_diag)) != 0) {
269 			*diag = vnic_mac2vnic_diag(mac_diag);
270 			return (err);
271 		}
272 		*addr_len_ptr_arg = mac_addr_len(vnic->vn_lower_mh);
273 		vnic_addr_type = VNIC_MAC_ADDR_TYPE_RANDOM;
274 		break;
275 	case VNIC_MAC_ADDR_TYPE_PRIMARY:
276 		/*
277 		 * We get the address here since we copy it in the
278 		 * vnic's vn_addr.
279 		 */
280 		mac_unicast_primary_get(vnic->vn_lower_mh, mac_addr_arg);
281 		*addr_len_ptr_arg = mac_addr_len(vnic->vn_lower_mh);
282 		mac_flags |= MAC_UNICAST_VNIC_PRIMARY;
283 		break;
284 	}
285 
286 	vnic->vn_addr_type = vnic_addr_type;
287 
288 	err = mac_unicast_add(vnic->vn_mch, mac_addr_arg, mac_flags,
289 	    &vnic->vn_muh, vid, &mac_diag);
290 	if (err != 0) {
291 		if (vnic_addr_type == VNIC_MAC_ADDR_TYPE_FACTORY) {
292 			/* release factory MAC address */
293 			mac_addr_factory_release(vnic->vn_mch, *addr_slot);
294 		}
295 		*diag = vnic_mac2vnic_diag(mac_diag);
296 	}
297 
298 	return (err);
299 }
300 
301 /*
302  * Create a new VNIC upon request from administrator.
303  * Returns 0 on success, an errno on failure.
304  */
305 /* ARGSUSED */
306 int
307 vnic_dev_create(datalink_id_t vnic_id, datalink_id_t linkid,
308     vnic_mac_addr_type_t *vnic_addr_type, int *mac_len, uchar_t *mac_addr,
309     int *mac_slot, uint_t mac_prefix_len, uint16_t vid,
310     mac_resource_props_t *mrp, uint32_t flags, vnic_ioc_diag_t *diag)
311 {
312 	vnic_t *vnic;
313 	mac_register_t *mac;
314 	int err;
315 	boolean_t is_anchor = ((flags & VNIC_IOC_CREATE_ANCHOR) != 0);
316 	char vnic_name[MAXNAMELEN];
317 	const mac_info_t *minfop;
318 	uint32_t req_hwgrp_flag = ((flags & VNIC_IOC_CREATE_REQ_HWRINGS) != 0) ?
319 	    MAC_OPEN_FLAGS_REQ_HWRINGS : 0;
320 
321 	*diag = VNIC_IOC_DIAG_NONE;
322 
323 	rw_enter(&vnic_lock, RW_WRITER);
324 
325 	/* does a VNIC with the same id already exist? */
326 	err = mod_hash_find(vnic_hash, VNIC_HASH_KEY(vnic_id),
327 	    (mod_hash_val_t *)&vnic);
328 	if (err == 0) {
329 		rw_exit(&vnic_lock);
330 		return (EEXIST);
331 	}
332 
333 	vnic = kmem_cache_alloc(vnic_cache, KM_NOSLEEP);
334 	if (vnic == NULL) {
335 		rw_exit(&vnic_lock);
336 		return (ENOMEM);
337 	}
338 
339 	bzero(vnic, sizeof (*vnic));
340 
341 	vnic->vn_id = vnic_id;
342 	vnic->vn_link_id = linkid;
343 	vnic->vn_started = B_FALSE;
344 
345 	if (!is_anchor) {
346 		if (linkid == DATALINK_INVALID_LINKID) {
347 			err = EINVAL;
348 			goto bail;
349 		}
350 
351 		/*
352 		 * Open the lower MAC and assign its initial bandwidth and
353 		 * MAC address. We do this here during VNIC creation and
354 		 * do not wait until the upper MAC client open so that we
355 		 * can validate the VNIC creation parameters (bandwidth,
356 		 * MAC address, etc) and reserve a factory MAC address if
357 		 * one was requested.
358 		 */
359 		err = mac_open_by_linkid(linkid, &vnic->vn_lower_mh);
360 		if (err != 0)
361 			goto bail;
362 
363 		/*
364 		 * VNIC(vlan) over VNICs(vlans) is not supported.
365 		 */
366 		if (mac_is_vnic(vnic->vn_lower_mh)) {
367 			err = EINVAL;
368 			goto bail;
369 		}
370 
371 		/* only ethernet support for now */
372 		minfop = mac_info(vnic->vn_lower_mh);
373 		if (minfop->mi_nativemedia != DL_ETHER) {
374 			err = ENOTSUP;
375 			goto bail;
376 		}
377 
378 		(void) dls_mgmt_get_linkinfo(vnic_id, vnic_name, NULL, NULL,
379 		    NULL);
380 		err = mac_client_open(vnic->vn_lower_mh, &vnic->vn_mch,
381 		    vnic_name, MAC_OPEN_FLAGS_IS_VNIC | req_hwgrp_flag);
382 		if (err != 0)
383 			goto bail;
384 
385 		if (mrp != NULL) {
386 			err = mac_client_set_resources(vnic->vn_mch, mrp);
387 			if (err != 0)
388 				goto bail;
389 		}
390 		/* assign a MAC address to the VNIC */
391 
392 		err = vnic_unicast_add(vnic, *vnic_addr_type, mac_slot,
393 		    mac_prefix_len, mac_len, mac_addr, flags, diag, vid);
394 		if (err != 0) {
395 			vnic->vn_muh = NULL;
396 			if (diag != NULL && req_hwgrp_flag != 0)
397 				*diag = VNIC_IOC_DIAG_NO_HWRINGS;
398 			goto bail;
399 		}
400 
401 		/* register to receive notification from underlying MAC */
402 		vnic->vn_mnh = mac_notify_add(vnic->vn_lower_mh, vnic_notify_cb,
403 		    vnic);
404 
405 		*vnic_addr_type = vnic->vn_addr_type;
406 		vnic->vn_addr_len = *mac_len;
407 		vnic->vn_vid = vid;
408 
409 		bcopy(mac_addr, vnic->vn_addr, vnic->vn_addr_len);
410 
411 		if (vnic->vn_addr_type == VNIC_MAC_ADDR_TYPE_FACTORY)
412 			vnic->vn_slot_id = *mac_slot;
413 
414 		/* set the initial VNIC capabilities */
415 		if (!mac_capab_get(vnic->vn_lower_mh, MAC_CAPAB_HCKSUM,
416 		    &vnic->vn_hcksum_txflags))
417 			vnic->vn_hcksum_txflags = 0;
418 	}
419 
420 	/* register with the MAC module */
421 	if ((mac = mac_alloc(MAC_VERSION)) == NULL)
422 		goto bail;
423 
424 	mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
425 	mac->m_driver = vnic;
426 	mac->m_dip = vnic_get_dip();
427 	mac->m_instance = (uint_t)-1;
428 	mac->m_src_addr = vnic->vn_addr;
429 	mac->m_callbacks = &vnic_m_callbacks;
430 
431 	if (!is_anchor) {
432 		/*
433 		 * If this is a VNIC based VLAN, then we check for the
434 		 * margin unless it has been created with the force
435 		 * flag. If we are configuring a VLAN over an etherstub,
436 		 * we don't check the margin even if force is not set.
437 		 */
438 		if (vid == 0 || (flags & VNIC_IOC_CREATE_FORCE) != 0) {
439 			if (vid != VLAN_ID_NONE)
440 				vnic->vn_force = B_TRUE;
441 			/*
442 			 * As the current margin size of the underlying mac is
443 			 * used to determine the margin size of the VNIC
444 			 * itself, request the underlying mac not to change
445 			 * to a smaller margin size.
446 			 */
447 			err = mac_margin_add(vnic->vn_lower_mh,
448 			    &vnic->vn_margin, B_TRUE);
449 			ASSERT(err == 0);
450 		} else {
451 			vnic->vn_margin = VLAN_TAGSZ;
452 			err = mac_margin_add(vnic->vn_lower_mh,
453 			    &vnic->vn_margin, B_FALSE);
454 			if (err != 0) {
455 				mac_free(mac);
456 				if (diag != NULL)
457 					*diag = VNIC_IOC_DIAG_MACMARGIN_INVALID;
458 				goto bail;
459 			}
460 		}
461 
462 		mac_sdu_get(vnic->vn_lower_mh, &mac->m_min_sdu,
463 		    &mac->m_max_sdu);
464 	} else {
465 		vnic->vn_margin = VLAN_TAGSZ;
466 		mac->m_min_sdu = 0;
467 		mac->m_max_sdu = 9000;
468 	}
469 
470 	mac->m_margin = vnic->vn_margin;
471 
472 	err = mac_register(mac, &vnic->vn_mh);
473 	mac_free(mac);
474 	if (err != 0) {
475 		VERIFY(is_anchor || mac_margin_remove(vnic->vn_lower_mh,
476 		    vnic->vn_margin) == 0);
477 		goto bail;
478 	}
479 
480 	/* Set the VNIC's MAC in the client */
481 	if (!is_anchor)
482 		mac_set_upper_mac(vnic->vn_mch, vnic->vn_mh);
483 
484 	if ((err = dls_devnet_create(vnic->vn_mh, vnic->vn_id)) != 0) {
485 		VERIFY(is_anchor || mac_margin_remove(vnic->vn_lower_mh,
486 		    vnic->vn_margin) == 0);
487 		(void) mac_unregister(vnic->vn_mh);
488 		goto bail;
489 	}
490 
491 	/* add new VNIC to hash table */
492 	err = mod_hash_insert(vnic_hash, VNIC_HASH_KEY(vnic_id),
493 	    (mod_hash_val_t)vnic);
494 	ASSERT(err == 0);
495 	vnic_count++;
496 
497 	rw_exit(&vnic_lock);
498 
499 	return (0);
500 
501 bail:
502 	rw_exit(&vnic_lock);
503 	if (!is_anchor) {
504 		if (vnic->vn_mnh != NULL)
505 			(void) mac_notify_remove(vnic->vn_mnh, B_TRUE);
506 		if (vnic->vn_muh != NULL)
507 			(void) mac_unicast_remove(vnic->vn_mch, vnic->vn_muh);
508 		if (vnic->vn_mch != NULL)
509 			mac_client_close(vnic->vn_mch, MAC_CLOSE_FLAGS_IS_VNIC);
510 		if (vnic->vn_lower_mh != NULL)
511 			mac_close(vnic->vn_lower_mh);
512 	}
513 
514 	kmem_cache_free(vnic_cache, vnic);
515 	return (err);
516 }
517 
518 /*
519  * Modify the properties of an existing VNIC.
520  */
521 /* ARGSUSED */
522 int
523 vnic_dev_modify(datalink_id_t vnic_id, uint_t modify_mask,
524     vnic_mac_addr_type_t mac_addr_type, uint_t mac_len, uchar_t *mac_addr,
525     uint_t mac_slot, mac_resource_props_t *mrp)
526 {
527 	vnic_t *vnic = NULL;
528 
529 	rw_enter(&vnic_lock, RW_WRITER);
530 
531 	if (mod_hash_find(vnic_hash, VNIC_HASH_KEY(vnic_id),
532 	    (mod_hash_val_t *)&vnic) != 0) {
533 		rw_exit(&vnic_lock);
534 		return (ENOENT);
535 	}
536 
537 	rw_exit(&vnic_lock);
538 
539 	return (0);
540 }
541 
542 /* ARGSUSED */
543 int
544 vnic_dev_delete(datalink_id_t vnic_id, uint32_t flags)
545 {
546 	vnic_t *vnic = NULL;
547 	mod_hash_val_t val;
548 	datalink_id_t tmpid;
549 	int rc;
550 
551 	rw_enter(&vnic_lock, RW_WRITER);
552 
553 	if (mod_hash_find(vnic_hash, VNIC_HASH_KEY(vnic_id),
554 	    (mod_hash_val_t *)&vnic) != 0) {
555 		rw_exit(&vnic_lock);
556 		return (ENOENT);
557 	}
558 
559 	if ((rc = dls_devnet_destroy(vnic->vn_mh, &tmpid, B_TRUE)) != 0) {
560 		rw_exit(&vnic_lock);
561 		return (rc);
562 	}
563 
564 	ASSERT(vnic_id == tmpid);
565 
566 	/*
567 	 * We cannot unregister the MAC yet. Unregistering would
568 	 * free up mac_impl_t which should not happen at this time.
569 	 * So disable mac_impl_t by calling mac_disable(). This will prevent
570 	 * any new claims on mac_impl_t.
571 	 */
572 	if ((rc = mac_disable(vnic->vn_mh)) != 0) {
573 		(void) dls_devnet_create(vnic->vn_mh, vnic_id);
574 		rw_exit(&vnic_lock);
575 		return (rc);
576 	}
577 
578 	(void) mod_hash_remove(vnic_hash, VNIC_HASH_KEY(vnic_id), &val);
579 	ASSERT(vnic == (vnic_t *)val);
580 	vnic_count--;
581 	rw_exit(&vnic_lock);
582 
583 	/*
584 	 * XXX-nicolas shouldn't have a void cast here, if it's
585 	 * expected that the function will never fail, then we should
586 	 * have an ASSERT().
587 	 */
588 	(void) mac_unregister(vnic->vn_mh);
589 
590 	if (vnic->vn_lower_mh != NULL) {
591 		/*
592 		 * Check if MAC address for the vnic was obtained from the
593 		 * factory MAC addresses. If yes, release it.
594 		 */
595 		if (vnic->vn_addr_type == VNIC_MAC_ADDR_TYPE_FACTORY) {
596 			(void) mac_addr_factory_release(vnic->vn_mch,
597 			    vnic->vn_slot_id);
598 		}
599 		(void) mac_margin_remove(vnic->vn_lower_mh, vnic->vn_margin);
600 		(void) mac_notify_remove(vnic->vn_mnh, B_TRUE);
601 		(void) mac_unicast_remove(vnic->vn_mch, vnic->vn_muh);
602 		mac_client_close(vnic->vn_mch, MAC_CLOSE_FLAGS_IS_VNIC);
603 		mac_close(vnic->vn_lower_mh);
604 	}
605 
606 	kmem_cache_free(vnic_cache, vnic);
607 	return (0);
608 }
609 
610 /* ARGSUSED */
611 mblk_t *
612 vnic_m_tx(void *arg, mblk_t *mp_chain)
613 {
614 	/*
615 	 * This function could be invoked for an anchor VNIC when sending
616 	 * broadcast and multicast packets, and unicast packets which did
617 	 * not match any local known destination.
618 	 */
619 	freemsgchain(mp_chain);
620 	return (NULL);
621 }
622 
623 /*ARGSUSED*/
624 static void
625 vnic_m_ioctl(void *arg, queue_t *q, mblk_t *mp)
626 {
627 	miocnak(q, mp, 0, ENOTSUP);
628 }
629 
630 /*
631  * This entry point cannot be passed-through, since it is invoked
632  * for the per-VNIC kstats which must be exported independently
633  * of the existence of VNIC MAC clients.
634  */
635 static int
636 vnic_m_stat(void *arg, uint_t stat, uint64_t *val)
637 {
638 	vnic_t *vnic = arg;
639 	int rval = 0;
640 
641 	if (vnic->vn_lower_mh == NULL) {
642 		/*
643 		 * It's an anchor VNIC, which does not have any
644 		 * statistics in itself.
645 		 */
646 		return (ENOTSUP);
647 	}
648 
649 	/*
650 	 * ENOTSUP must be reported for unsupported stats, the VNIC
651 	 * driver reports a subset of the stats that would
652 	 * be returned by a real piece of hardware.
653 	 */
654 
655 	switch (stat) {
656 	case MAC_STAT_LINK_STATE:
657 	case MAC_STAT_LINK_UP:
658 	case MAC_STAT_PROMISC:
659 	case MAC_STAT_IFSPEED:
660 	case MAC_STAT_MULTIRCV:
661 	case MAC_STAT_MULTIXMT:
662 	case MAC_STAT_BRDCSTRCV:
663 	case MAC_STAT_BRDCSTXMT:
664 	case MAC_STAT_OPACKETS:
665 	case MAC_STAT_OBYTES:
666 	case MAC_STAT_IERRORS:
667 	case MAC_STAT_OERRORS:
668 	case MAC_STAT_RBYTES:
669 	case MAC_STAT_IPACKETS:
670 		*val = mac_client_stat_get(vnic->vn_mch, stat);
671 		break;
672 	default:
673 		rval = ENOTSUP;
674 	}
675 
676 	return (rval);
677 }
678 
679 /*
680  * Invoked by the upper MAC to retrieve the lower MAC client handle
681  * corresponding to a VNIC. A pointer to this function is obtained
682  * by the upper MAC via capability query.
683  *
684  * XXX-nicolas Note: this currently causes all VNIC MAC clients to
685  * receive the same MAC client handle for the same VNIC. This is ok
686  * as long as we have only one VNIC MAC client which sends and
687  * receives data, but we don't currently enforce this at the MAC layer.
688  */
689 static void *
690 vnic_mac_client_handle(void *vnic_arg)
691 {
692 	vnic_t *vnic = vnic_arg;
693 
694 	return (vnic->vn_mch);
695 }
696 
697 
698 /*
699  * Return information about the specified capability.
700  */
701 /* ARGSUSED */
702 static boolean_t
703 vnic_m_capab_get(void *arg, mac_capab_t cap, void *cap_data)
704 {
705 	vnic_t *vnic = arg;
706 
707 	switch (cap) {
708 	case MAC_CAPAB_HCKSUM: {
709 		uint32_t *hcksum_txflags = cap_data;
710 
711 		*hcksum_txflags = vnic->vn_hcksum_txflags &
712 		    (HCKSUM_INET_FULL_V4 | HCKSUM_IPHDRCKSUM |
713 		    HCKSUM_INET_PARTIAL);
714 		break;
715 	}
716 	case MAC_CAPAB_VNIC: {
717 		mac_capab_vnic_t *vnic_capab = cap_data;
718 
719 		if (vnic->vn_lower_mh == NULL) {
720 			/*
721 			 * It's an anchor VNIC, we don't have an underlying
722 			 * NIC and MAC client handle.
723 			 */
724 			return (B_FALSE);
725 		}
726 
727 		if (vnic_capab != NULL) {
728 			vnic_capab->mcv_arg = vnic;
729 			vnic_capab->mcv_mac_client_handle =
730 			    vnic_mac_client_handle;
731 		}
732 		break;
733 	}
734 	case MAC_CAPAB_ANCHOR_VNIC: {
735 		/* since it's an anchor VNIC we don't have lower mac handle */
736 		if (vnic->vn_lower_mh == NULL) {
737 			ASSERT(vnic->vn_link_id == 0);
738 			return (B_TRUE);
739 		}
740 		return (B_FALSE);
741 	}
742 	case MAC_CAPAB_NO_NATIVEVLAN:
743 	case MAC_CAPAB_NO_ZCOPY:
744 		return (B_TRUE);
745 	default:
746 		return (B_FALSE);
747 	}
748 	return (B_TRUE);
749 }
750 
751 /* ARGSUSED */
752 static int
753 vnic_m_start(void *arg)
754 {
755 	return (0);
756 }
757 
758 /* ARGSUSED */
759 static void
760 vnic_m_stop(void *arg)
761 {
762 }
763 
764 /* ARGSUSED */
765 static int
766 vnic_m_promisc(void *arg, boolean_t on)
767 {
768 	return (0);
769 }
770 
771 /* ARGSUSED */
772 static int
773 vnic_m_multicst(void *arg, boolean_t add, const uint8_t *addrp)
774 {
775 	return (0);
776 }
777 
778 static int
779 vnic_m_unicst(void *arg, const uint8_t *macaddr)
780 {
781 	vnic_t *vnic = arg;
782 
783 	return (mac_vnic_unicast_set(vnic->vn_mch, macaddr));
784 }
785 
786 int
787 vnic_info(vnic_info_t *info)
788 {
789 	vnic_t		*vnic;
790 	int		err;
791 
792 	rw_enter(&vnic_lock, RW_WRITER);
793 
794 	err = mod_hash_find(vnic_hash, VNIC_HASH_KEY(info->vn_vnic_id),
795 	    (mod_hash_val_t *)&vnic);
796 	if (err != 0) {
797 		rw_exit(&vnic_lock);
798 		return (ENOENT);
799 	}
800 
801 	info->vn_link_id = vnic->vn_link_id;
802 	info->vn_mac_addr_type = vnic->vn_addr_type;
803 	info->vn_mac_len = vnic->vn_addr_len;
804 	bcopy(vnic->vn_addr, info->vn_mac_addr, MAXMACADDRLEN);
805 	info->vn_mac_slot = vnic->vn_slot_id;
806 	info->vn_mac_prefix_len = 0;
807 	info->vn_vid = vnic->vn_vid;
808 	info->vn_force = vnic->vn_force;
809 
810 	bzero(&info->vn_resource_props, sizeof (mac_resource_props_t));
811 	if (vnic->vn_mch != NULL)
812 		mac_resource_ctl_get(vnic->vn_mch, &info->vn_resource_props);
813 
814 	rw_exit(&vnic_lock);
815 	return (0);
816 }
817 
818 static void
819 vnic_notify_cb(void *arg, mac_notify_type_t type)
820 {
821 	vnic_t *vnic = arg;
822 
823 	/*
824 	 * Only the VLAN VNIC needs to be notified with primary MAC
825 	 * address change.
826 	 */
827 	if (vnic->vn_addr_type != VNIC_MAC_ADDR_TYPE_PRIMARY)
828 		return;
829 
830 	switch (type) {
831 	case MAC_NOTE_UNICST:
832 		/*  the unicast MAC address value */
833 		mac_unicast_primary_get(vnic->vn_lower_mh, vnic->vn_addr);
834 
835 		/* notify its upper layer MAC about MAC address change */
836 		mac_unicst_update(vnic->vn_mh, (const uint8_t *)vnic->vn_addr);
837 		break;
838 	default:
839 		break;
840 	}
841 }
842