1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Xen network backend - mac client edition. 29 * 30 * A driver that sits above an existing GLDv3/Nemo MAC driver and 31 * relays packets to/from that driver from/to a guest domain. 32 */ 33 34 #include "xnb.h" 35 36 #include <sys/sunddi.h> 37 #include <sys/ddi.h> 38 #include <sys/modctl.h> 39 #include <sys/strsubr.h> 40 #include <sys/mac_client.h> 41 #include <sys/mac_provider.h> 42 #include <sys/mac_client_priv.h> 43 #include <sys/mac.h> 44 #include <net/if.h> 45 #include <sys/dlpi.h> 46 #include <sys/pattr.h> 47 #include <xen/sys/xenbus_impl.h> 48 #include <xen/sys/xendev.h> 49 #include <sys/sdt.h> 50 #include <sys/note.h> 51 52 /* Track multicast addresses. */ 53 typedef struct xmca { 54 struct xmca *next; 55 ether_addr_t addr; 56 } xmca_t; 57 58 /* State about this device instance. */ 59 typedef struct xnbo { 60 mac_handle_t o_mh; 61 mac_client_handle_t o_mch; 62 mac_unicast_handle_t o_mah; 63 mac_promisc_handle_t o_mphp; 64 boolean_t o_running; 65 boolean_t o_promiscuous; 66 uint32_t o_hcksum_capab; 67 xmca_t *o_mca; 68 char o_link_name[LIFNAMSIZ]; 69 boolean_t o_need_rx_filter; 70 boolean_t o_need_setphysaddr; 71 boolean_t o_multicast_control; 72 } xnbo_t; 73 74 static void xnbo_close_mac(xnb_t *); 75 static void i_xnbo_close_mac(xnb_t *, boolean_t); 76 77 /* 78 * Packets from the peer come here. We pass them to the mac device. 79 */ 80 static void 81 xnbo_to_mac(xnb_t *xnbp, mblk_t *mp) 82 { 83 xnbo_t *xnbop = xnbp->xnb_flavour_data; 84 85 ASSERT(mp != NULL); 86 87 if (!xnbop->o_running) { 88 xnbp->xnb_stat_tx_too_early++; 89 goto fail; 90 } 91 92 if (mac_tx(xnbop->o_mch, mp, 0, 93 MAC_DROP_ON_NO_DESC, NULL) != NULL) { 94 xnbp->xnb_stat_mac_full++; 95 } 96 97 return; 98 99 fail: 100 freemsgchain(mp); 101 } 102 103 /* 104 * Process the checksum flags `flags' provided by the peer for the 105 * packet `mp'. 106 */ 107 static mblk_t * 108 xnbo_cksum_from_peer(xnb_t *xnbp, mblk_t *mp, uint16_t flags) 109 { 110 xnbo_t *xnbop = xnbp->xnb_flavour_data; 111 112 ASSERT(mp->b_next == NULL); 113 114 if ((flags & NETTXF_csum_blank) != 0) { 115 /* 116 * The checksum in the packet is blank. Determine 117 * whether we can do hardware offload and, if so, 118 * update the flags on the mblk according. If not, 119 * calculate and insert the checksum using software. 120 */ 121 mp = xnb_process_cksum_flags(xnbp, mp, 122 xnbop->o_hcksum_capab); 123 } 124 125 return (mp); 126 } 127 128 /* 129 * Calculate the checksum flags to be relayed to the peer for the 130 * packet `mp'. 131 */ 132 static uint16_t 133 xnbo_cksum_to_peer(xnb_t *xnbp, mblk_t *mp) 134 { 135 _NOTE(ARGUNUSED(xnbp)); 136 uint16_t r = 0; 137 uint32_t pflags, csum; 138 139 /* 140 * We might also check for HCK_PARTIALCKSUM here and, 141 * providing that the partial checksum covers the TCP/UDP 142 * payload, return NETRXF_data_validated. 143 * 144 * It seems that it's probably not worthwhile, as even MAC 145 * devices which advertise HCKSUM_INET_PARTIAL in their 146 * capabilities tend to use HCK_FULLCKSUM on the receive side 147 * - they are actually saying that in the output path the 148 * caller must use HCK_PARTIALCKSUM. 149 * 150 * Then again, if a NIC supports HCK_PARTIALCKSUM in its' 151 * output path, the host IP stack will use it. If such packets 152 * are destined for the peer (i.e. looped around) we would 153 * gain some advantage. 154 */ 155 156 hcksum_retrieve(mp, NULL, NULL, NULL, NULL, 157 NULL, &csum, &pflags); 158 159 /* 160 * If the MAC driver has asserted that the checksum is 161 * good, let the peer know. 162 */ 163 if (((pflags & HCK_FULLCKSUM) != 0) && 164 (((pflags & HCK_FULLCKSUM_OK) != 0) || 165 (csum == 0xffff))) 166 r |= NETRXF_data_validated; 167 168 return (r); 169 } 170 171 /* 172 * Packets from the mac device come here. We pass them to the peer. 173 */ 174 /*ARGSUSED*/ 175 static void 176 xnbo_from_mac(void *arg, mac_resource_handle_t mrh, mblk_t *mp, 177 boolean_t loopback) 178 { 179 xnb_t *xnbp = arg; 180 181 mp = xnb_copy_to_peer(xnbp, mp); 182 183 if (mp != NULL) 184 freemsgchain(mp); 185 } 186 187 /* 188 * Packets from the mac device come here. We pass them to the peer if 189 * the destination mac address matches or it's a multicast/broadcast 190 * address. 191 */ 192 static void 193 xnbo_from_mac_filter(void *arg, mac_resource_handle_t mrh, mblk_t *mp, 194 boolean_t loopback) 195 { 196 _NOTE(ARGUNUSED(loopback)); 197 xnb_t *xnbp = arg; 198 xnbo_t *xnbop = xnbp->xnb_flavour_data; 199 mblk_t *next, *keep, *keep_head, *free, *free_head; 200 201 keep = keep_head = free = free_head = NULL; 202 203 #define ADD(list, bp) \ 204 if (list != NULL) \ 205 list->b_next = bp; \ 206 else \ 207 list##_head = bp; \ 208 list = bp; 209 210 for (; mp != NULL; mp = next) { 211 mac_header_info_t hdr_info; 212 213 next = mp->b_next; 214 mp->b_next = NULL; 215 216 if (mac_header_info(xnbop->o_mh, mp, &hdr_info) != 0) { 217 ADD(free, mp); 218 continue; 219 } 220 221 if ((hdr_info.mhi_dsttype == MAC_ADDRTYPE_BROADCAST) || 222 (hdr_info.mhi_dsttype == MAC_ADDRTYPE_MULTICAST)) { 223 ADD(keep, mp); 224 continue; 225 } 226 227 if (bcmp(hdr_info.mhi_daddr, xnbp->xnb_mac_addr, 228 sizeof (xnbp->xnb_mac_addr)) == 0) { 229 ADD(keep, mp); 230 continue; 231 } 232 233 ADD(free, mp); 234 } 235 #undef ADD 236 237 if (keep_head != NULL) 238 xnbo_from_mac(xnbp, mrh, keep_head, B_FALSE); 239 240 if (free_head != NULL) 241 freemsgchain(free_head); 242 } 243 244 static boolean_t 245 xnbo_open_mac(xnb_t *xnbp, char *mac) 246 { 247 xnbo_t *xnbop = xnbp->xnb_flavour_data; 248 int err; 249 const mac_info_t *mi; 250 void (*rx_fn)(void *, mac_resource_handle_t, mblk_t *, boolean_t); 251 struct ether_addr ea; 252 uint_t max_sdu; 253 mac_diag_t diag; 254 255 if ((err = mac_open_by_linkname(mac, &xnbop->o_mh)) != 0) { 256 cmn_err(CE_WARN, "xnbo_open_mac: " 257 "cannot open mac for link %s (%d)", mac, err); 258 return (B_FALSE); 259 } 260 ASSERT(xnbop->o_mh != NULL); 261 262 mi = mac_info(xnbop->o_mh); 263 ASSERT(mi != NULL); 264 265 if (mi->mi_media != DL_ETHER) { 266 cmn_err(CE_WARN, "xnbo_open_mac: " 267 "device is not DL_ETHER (%d)", mi->mi_media); 268 i_xnbo_close_mac(xnbp, B_TRUE); 269 return (B_FALSE); 270 } 271 if (mi->mi_media != mi->mi_nativemedia) { 272 cmn_err(CE_WARN, "xnbo_open_mac: " 273 "device media and native media mismatch (%d != %d)", 274 mi->mi_media, mi->mi_nativemedia); 275 i_xnbo_close_mac(xnbp, B_TRUE); 276 return (B_FALSE); 277 } 278 279 mac_sdu_get(xnbop->o_mh, NULL, &max_sdu); 280 if (max_sdu > XNBMAXPKT) { 281 cmn_err(CE_WARN, "xnbo_open_mac: mac device SDU too big (%d)", 282 max_sdu); 283 i_xnbo_close_mac(xnbp, B_TRUE); 284 return (B_FALSE); 285 } 286 287 /* 288 * MAC_OPEN_FLAGS_MULTI_PRIMARY is relevant when we are migrating a 289 * guest on the localhost itself. In this case we would have the MAC 290 * client open for the guest being migrated *and* also for the 291 * migrated guest (i.e. the former will be active till the migration 292 * is complete when the latter will be activated). This flag states 293 * that it is OK for mac_unicast_add to add the primary MAC unicast 294 * address multiple times. 295 */ 296 if (mac_client_open(xnbop->o_mh, &xnbop->o_mch, NULL, 297 MAC_OPEN_FLAGS_USE_DATALINK_NAME | 298 MAC_OPEN_FLAGS_MULTI_PRIMARY) != 0) { 299 cmn_err(CE_WARN, "xnbo_open_mac: " 300 "error (%d) opening mac client", err); 301 i_xnbo_close_mac(xnbp, B_TRUE); 302 return (B_FALSE); 303 } 304 305 if (xnbop->o_need_rx_filter) 306 rx_fn = xnbo_from_mac_filter; 307 else 308 rx_fn = xnbo_from_mac; 309 310 err = mac_unicast_add_set_rx(xnbop->o_mch, NULL, MAC_UNICAST_PRIMARY, 311 &xnbop->o_mah, 0, &diag, xnbop->o_multicast_control ? rx_fn : NULL, 312 xnbp); 313 if (err != 0) { 314 cmn_err(CE_WARN, "xnbo_open_mac: failed to get the primary " 315 "MAC address of %s: %d", mac, err); 316 i_xnbo_close_mac(xnbp, B_TRUE); 317 return (B_FALSE); 318 } 319 if (!xnbop->o_multicast_control) { 320 err = mac_promisc_add(xnbop->o_mch, MAC_CLIENT_PROMISC_ALL, 321 rx_fn, xnbp, &xnbop->o_mphp, MAC_PROMISC_FLAGS_NO_TX_LOOP | 322 MAC_PROMISC_FLAGS_VLAN_TAG_STRIP); 323 if (err != 0) { 324 cmn_err(CE_WARN, "xnbo_open_mac: " 325 "cannot enable promiscuous mode of %s: %d", 326 mac, err); 327 i_xnbo_close_mac(xnbp, B_TRUE); 328 return (B_FALSE); 329 } 330 xnbop->o_promiscuous = B_TRUE; 331 } 332 333 if (xnbop->o_need_setphysaddr) { 334 err = mac_unicast_primary_set(xnbop->o_mh, xnbp->xnb_mac_addr); 335 /* Warn, but continue on. */ 336 if (err != 0) { 337 bcopy(xnbp->xnb_mac_addr, ea.ether_addr_octet, 338 ETHERADDRL); 339 cmn_err(CE_WARN, "xnbo_open_mac: " 340 "cannot set MAC address of %s to " 341 "%s: %d", mac, ether_sprintf(&ea), err); 342 } 343 } 344 345 if (!mac_capab_get(xnbop->o_mh, MAC_CAPAB_HCKSUM, 346 &xnbop->o_hcksum_capab)) 347 xnbop->o_hcksum_capab = 0; 348 349 xnbop->o_running = B_TRUE; 350 351 return (B_TRUE); 352 } 353 354 static void 355 xnbo_close_mac(xnb_t *xnbp) 356 { 357 i_xnbo_close_mac(xnbp, B_FALSE); 358 } 359 360 static void 361 i_xnbo_close_mac(xnb_t *xnbp, boolean_t locked) 362 { 363 xnbo_t *xnbop = xnbp->xnb_flavour_data; 364 xmca_t *loop; 365 366 ASSERT(!locked || MUTEX_HELD(&xnbp->xnb_state_lock)); 367 368 if (xnbop->o_mh == NULL) 369 return; 370 371 if (xnbop->o_running) 372 xnbop->o_running = B_FALSE; 373 374 if (!locked) 375 mutex_enter(&xnbp->xnb_state_lock); 376 loop = xnbop->o_mca; 377 xnbop->o_mca = NULL; 378 if (!locked) 379 mutex_exit(&xnbp->xnb_state_lock); 380 381 while (loop != NULL) { 382 xmca_t *next = loop->next; 383 384 DTRACE_PROBE3(mcast_remove, 385 (char *), "close", 386 (void *), xnbp, 387 (etheraddr_t *), loop->addr); 388 (void) mac_multicast_remove(xnbop->o_mch, loop->addr); 389 kmem_free(loop, sizeof (*loop)); 390 loop = next; 391 } 392 393 if (xnbop->o_promiscuous) { 394 if (xnbop->o_mphp != NULL) { 395 mac_promisc_remove(xnbop->o_mphp); 396 xnbop->o_mphp = NULL; 397 } 398 xnbop->o_promiscuous = B_FALSE; 399 } else { 400 if (xnbop->o_mch != NULL) 401 mac_rx_clear(xnbop->o_mch); 402 } 403 404 if (xnbop->o_mah != NULL) { 405 (void) mac_unicast_remove(xnbop->o_mch, xnbop->o_mah); 406 xnbop->o_mah = NULL; 407 } 408 409 if (xnbop->o_mch != NULL) { 410 mac_client_close(xnbop->o_mch, 0); 411 xnbop->o_mch = NULL; 412 } 413 414 mac_close(xnbop->o_mh); 415 xnbop->o_mh = NULL; 416 } 417 418 /* 419 * Hotplug has completed and we are connected to the peer. We have all 420 * the information we need to exchange traffic, so open the MAC device 421 * and configure it appropriately. 422 */ 423 static boolean_t 424 xnbo_start_connect(xnb_t *xnbp) 425 { 426 xnbo_t *xnbop = xnbp->xnb_flavour_data; 427 428 return (xnbo_open_mac(xnbp, xnbop->o_link_name)); 429 } 430 431 /* 432 * The guest has successfully synchronize with this instance. We read 433 * the configuration of the guest from xenstore to check whether the 434 * guest requests multicast control. If not (the default) we make a 435 * note that the MAC device needs to be used in promiscious mode. 436 */ 437 static boolean_t 438 xnbo_peer_connected(xnb_t *xnbp) 439 { 440 char *oename; 441 int request; 442 xnbo_t *xnbop = xnbp->xnb_flavour_data; 443 444 oename = xvdi_get_oename(xnbp->xnb_devinfo); 445 446 if (xenbus_scanf(XBT_NULL, oename, 447 "request-multicast-control", "%d", &request) != 0) 448 request = 0; 449 xnbop->o_multicast_control = (request > 0); 450 451 return (B_TRUE); 452 } 453 454 /* 455 * The guest domain has closed down the inter-domain connection. We 456 * close the underlying MAC device. 457 */ 458 static void 459 xnbo_peer_disconnected(xnb_t *xnbp) 460 { 461 xnbo_close_mac(xnbp); 462 } 463 464 /* 465 * The hotplug script has completed. We read information from xenstore 466 * about our configuration, most notably the name of the MAC device we 467 * should use. 468 */ 469 static boolean_t 470 xnbo_hotplug_connected(xnb_t *xnbp) 471 { 472 char *xsname; 473 xnbo_t *xnbop = xnbp->xnb_flavour_data; 474 int need; 475 476 xsname = xvdi_get_xsname(xnbp->xnb_devinfo); 477 478 if (xenbus_scanf(XBT_NULL, xsname, 479 "nic", "%s", xnbop->o_link_name) != 0) { 480 cmn_err(CE_WARN, "xnbo_connect: " 481 "cannot read nic name from %s", xsname); 482 return (B_FALSE); 483 } 484 485 if (xenbus_scanf(XBT_NULL, xsname, 486 "SUNW-need-rx-filter", "%d", &need) != 0) 487 need = 0; 488 xnbop->o_need_rx_filter = (need > 0); 489 490 if (xenbus_scanf(XBT_NULL, xsname, 491 "SUNW-need-set-physaddr", "%d", &need) != 0) 492 need = 0; 493 xnbop->o_need_setphysaddr = (need > 0); 494 495 return (B_TRUE); 496 } 497 498 /* 499 * Find the multicast address `addr', return B_TRUE if it is one that 500 * we receive. If `remove', remove it from the set received. 501 */ 502 static boolean_t 503 xnbo_mcast_find(xnb_t *xnbp, ether_addr_t *addr, boolean_t remove) 504 { 505 xnbo_t *xnbop = xnbp->xnb_flavour_data; 506 xmca_t *prev, *del, *this; 507 508 ASSERT(MUTEX_HELD(&xnbp->xnb_state_lock)); 509 ASSERT(xnbop->o_promiscuous == B_FALSE); 510 511 prev = del = NULL; 512 513 this = xnbop->o_mca; 514 515 while (this != NULL) { 516 if (bcmp(&this->addr, addr, sizeof (this->addr)) == 0) { 517 del = this; 518 if (remove) { 519 if (prev == NULL) 520 xnbop->o_mca = this->next; 521 else 522 prev->next = this->next; 523 } 524 break; 525 } 526 527 prev = this; 528 this = this->next; 529 } 530 531 if (del == NULL) 532 return (B_FALSE); 533 534 if (remove) { 535 DTRACE_PROBE3(mcast_remove, 536 (char *), "remove", 537 (void *), xnbp, 538 (etheraddr_t *), del->addr); 539 mac_multicast_remove(xnbop->o_mch, del->addr); 540 kmem_free(del, sizeof (*del)); 541 } 542 543 return (B_TRUE); 544 } 545 546 /* 547 * Add the multicast address `addr' to the set received. 548 */ 549 static boolean_t 550 xnbo_mcast_add(xnb_t *xnbp, ether_addr_t *addr) 551 { 552 xnbo_t *xnbop = xnbp->xnb_flavour_data; 553 boolean_t r = B_FALSE; 554 555 ASSERT(xnbop->o_promiscuous == B_FALSE); 556 557 mutex_enter(&xnbp->xnb_state_lock); 558 559 if (xnbo_mcast_find(xnbp, addr, B_FALSE)) { 560 r = B_TRUE; 561 } else if (mac_multicast_add(xnbop->o_mch, 562 (const uint8_t *)addr) == 0) { 563 xmca_t *mca; 564 565 DTRACE_PROBE3(mcast_add, 566 (char *), "add", 567 (void *), xnbp, 568 (etheraddr_t *), addr); 569 570 mca = kmem_alloc(sizeof (*mca), KM_SLEEP); 571 bcopy(addr, &mca->addr, sizeof (mca->addr)); 572 573 mca->next = xnbop->o_mca; 574 xnbop->o_mca = mca; 575 576 r = B_TRUE; 577 } 578 579 mutex_exit(&xnbp->xnb_state_lock); 580 581 return (r); 582 } 583 584 /* 585 * Remove the multicast address `addr' from the set received. 586 */ 587 static boolean_t 588 xnbo_mcast_del(xnb_t *xnbp, ether_addr_t *addr) 589 { 590 boolean_t r; 591 592 mutex_enter(&xnbp->xnb_state_lock); 593 r = xnbo_mcast_find(xnbp, addr, B_TRUE); 594 mutex_exit(&xnbp->xnb_state_lock); 595 596 return (r); 597 } 598 599 static int 600 xnbo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 601 { 602 static xnb_flavour_t flavour = { 603 xnbo_to_mac, xnbo_peer_connected, xnbo_peer_disconnected, 604 xnbo_hotplug_connected, xnbo_start_connect, 605 xnbo_cksum_from_peer, xnbo_cksum_to_peer, 606 xnbo_mcast_add, xnbo_mcast_del, 607 }; 608 xnbo_t *xnbop; 609 610 switch (cmd) { 611 case DDI_ATTACH: 612 break; 613 case DDI_RESUME: 614 return (DDI_SUCCESS); 615 default: 616 return (DDI_FAILURE); 617 } 618 619 xnbop = kmem_zalloc(sizeof (*xnbop), KM_SLEEP); 620 621 if (xnb_attach(dip, &flavour, xnbop) != DDI_SUCCESS) { 622 kmem_free(xnbop, sizeof (*xnbop)); 623 return (DDI_FAILURE); 624 } 625 626 return (DDI_SUCCESS); 627 } 628 629 static int 630 xnbo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 631 { 632 xnb_t *xnbp = ddi_get_driver_private(dip); 633 xnbo_t *xnbop = xnbp->xnb_flavour_data; 634 635 switch (cmd) { 636 case DDI_DETACH: 637 break; 638 case DDI_SUSPEND: 639 return (DDI_SUCCESS); 640 default: 641 return (DDI_FAILURE); 642 } 643 644 mutex_enter(&xnbp->xnb_tx_lock); 645 mutex_enter(&xnbp->xnb_rx_lock); 646 647 if (!xnbp->xnb_detachable || xnbp->xnb_connected || 648 (xnbp->xnb_tx_buf_count > 0)) { 649 mutex_exit(&xnbp->xnb_rx_lock); 650 mutex_exit(&xnbp->xnb_tx_lock); 651 652 return (DDI_FAILURE); 653 } 654 655 mutex_exit(&xnbp->xnb_rx_lock); 656 mutex_exit(&xnbp->xnb_tx_lock); 657 658 xnbo_close_mac(xnbp); 659 kmem_free(xnbop, sizeof (*xnbop)); 660 661 xnb_detach(dip); 662 663 return (DDI_SUCCESS); 664 } 665 666 static struct cb_ops cb_ops = { 667 nulldev, /* open */ 668 nulldev, /* close */ 669 nodev, /* strategy */ 670 nodev, /* print */ 671 nodev, /* dump */ 672 nodev, /* read */ 673 nodev, /* write */ 674 nodev, /* ioctl */ 675 nodev, /* devmap */ 676 nodev, /* mmap */ 677 nodev, /* segmap */ 678 nochpoll, /* poll */ 679 ddi_prop_op, /* cb_prop_op */ 680 0, /* streamtab */ 681 D_NEW | D_MP | D_64BIT /* Driver compatibility flag */ 682 }; 683 684 static struct dev_ops ops = { 685 DEVO_REV, /* devo_rev */ 686 0, /* devo_refcnt */ 687 nulldev, /* devo_getinfo */ 688 nulldev, /* devo_identify */ 689 nulldev, /* devo_probe */ 690 xnbo_attach, /* devo_attach */ 691 xnbo_detach, /* devo_detach */ 692 nodev, /* devo_reset */ 693 &cb_ops, /* devo_cb_ops */ 694 (struct bus_ops *)0, /* devo_bus_ops */ 695 NULL, /* devo_power */ 696 ddi_quiesce_not_needed, /* devo_quiesce */ 697 }; 698 699 static struct modldrv modldrv = { 700 &mod_driverops, "xnbo driver", &ops, 701 }; 702 703 static struct modlinkage modlinkage = { 704 MODREV_1, &modldrv, NULL 705 }; 706 707 int 708 _init(void) 709 { 710 return (mod_install(&modlinkage)); 711 } 712 713 int 714 _info(struct modinfo *modinfop) 715 { 716 return (mod_info(&modlinkage, modinfop)); 717 } 718 719 int 720 _fini(void) 721 { 722 return (mod_remove(&modlinkage)); 723 } 724