1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * Copyright 2019 Joyent, Inc. 26 */ 27 28 /* 29 * This file defines interfaces between fcoe and its clients (FCoEI/FCoET) 30 */ 31 32 #include <sys/ddi.h> 33 #include <sys/sunddi.h> 34 #include <sys/sunndi.h> 35 #include <sys/byteorder.h> 36 #include <sys/atomic.h> 37 #include <sys/sysmacros.h> 38 #include <sys/cmn_err.h> 39 #include <sys/crc32.h> 40 #include <sys/fcntl.h> 41 #include <sys/unistd.h> 42 #include <sys/mac_client.h> 43 #include <sys/strsubr.h> 44 45 /* 46 * FCoE header files 47 */ 48 #include <sys/fcoe/fcoeio.h> 49 #include <sys/fcoe/fcoe_common.h> 50 51 /* 52 * Driver's own header files 53 */ 54 #include <fcoe.h> 55 #include <fcoe_fc.h> 56 #include <fcoe_eth.h> 57 58 static void fcoe_fill_frame_headers(fcoe_frame_t *frm); 59 static void fcoe_fill_frame_tailers(fcoe_frame_t *frm); 60 static void fcoe_deregister_client(fcoe_port_t *eport); 61 static int fcoe_ctl(fcoe_port_t *eport, int cmd, void *arg); 62 static void fcoe_tx_frame(fcoe_frame_t *frm); 63 static void *fcoe_alloc_netb(fcoe_port_t *eport, 64 uint32_t fc_frame_size, uint8_t **ppfc); 65 static void fcoe_free_netb(void *netb); 66 67 /* 68 * Only this function will be called explicitly by clients 69 * Register the specified client port (fcoei/fcoet) 70 */ 71 fcoe_port_t * 72 fcoe_register_client(fcoe_client_t *client) 73 { 74 fcoe_mac_t *mac; 75 fcoe_port_t *eport; 76 77 if (client->ect_fcoe_ver != fcoe_ver_now) { 78 cmn_err(CE_WARN, "FCoE modules version mismatch, " 79 "fail registering client."); 80 return (NULL); 81 } 82 83 /* 84 * We will not come here, when someone is changing ss_mac_list, 85 * so it's safe to go through ss_mac_list. 86 */ 87 for (mac = list_head(&fcoe_global_ss->ss_mac_list); mac; 88 mac = list_next(&fcoe_global_ss->ss_mac_list, mac)) { 89 if (client->ect_channelid == mac->fm_linkid) { 90 break; 91 } 92 } 93 94 if (mac == NULL) { 95 FCOE_LOG(0, "can't find the MAC you want to bind"); 96 return (NULL); 97 } 98 99 if (mac->fm_flags & FCOE_MAC_FLAG_BOUND) { 100 FCOE_LOG(0, "the MAC you want to bind is bound already"); 101 return (NULL); 102 } 103 104 atomic_or_32(&mac->fm_flags, FCOE_MAC_FLAG_BOUND); 105 bcopy(client, &mac->fm_client, sizeof (fcoe_client_t)); 106 107 /* 108 * fcoe_port_t initialization 109 */ 110 eport = &mac->fm_eport; 111 eport->eport_flags = client->ect_eport_flags | EPORT_FLAG_MAC_IN_USE; 112 eport->eport_fcoe_private = mac; 113 eport->eport_client_private = client->ect_client_port_struct; 114 eport->eport_max_fc_frame_size = 2136; 115 eport->eport_tx_frame = fcoe_tx_frame; 116 eport->eport_alloc_frame = fcoe_allocate_frame; 117 eport->eport_release_frame = fcoe_release_frame; 118 eport->eport_alloc_netb = fcoe_alloc_netb; 119 eport->eport_free_netb = fcoe_free_netb; 120 eport->eport_deregister_client = fcoe_deregister_client; 121 eport->eport_ctl = fcoe_ctl; 122 eport->eport_set_mac_address = fcoe_mac_set_address; 123 124 return (eport); 125 } 126 127 /* 128 * The following routines will be called through vectors in fcoe_port_t 129 */ 130 131 /* 132 * Deregister fcoet/fcoei modules, client should make sure the port is in 133 * offline status already 134 */ 135 static void 136 fcoe_deregister_client(fcoe_port_t *eport) 137 { 138 fcoe_mac_t *mac = EPORT2MAC(eport); 139 140 /* 141 * Wait for all the related frame to be freed, this should be fast 142 * because before deregister fcoei/fcoet will make sure its port 143 * is already in offline status so no frame will be received or sent 144 * any more 145 */ 146 while (mac->fm_frm_cnt > 0) { 147 delay(10); 148 } 149 150 atomic_and_32(&EPORT2MAC(eport)->fm_flags, ~FCOE_MAC_FLAG_BOUND); 151 atomic_and_32(&mac->fm_eport.eport_flags, ~EPORT_FLAG_MAC_IN_USE); 152 if (!(EPORT2MAC(eport)->fm_flags & FCOE_MAC_FLAG_USER_DEL)) { 153 (void) fcoe_close_mac(mac); 154 fcoe_destroy_mac(mac); 155 } 156 } 157 158 /* ARGSUSED */ 159 static int 160 fcoe_ctl(fcoe_port_t *eport, int cmd, void *arg) 161 { 162 fcoe_mac_t *mac = EPORT2MAC(eport); 163 164 switch (cmd) { 165 case FCOE_CMD_PORT_ONLINE: 166 /* 167 * client ask us to online, so it's safe to post event 168 * and data up 169 */ 170 if (fcoe_enable_callback(mac) == FCOE_FAILURE) { 171 return (FCOE_FAILURE); 172 } 173 mac->fm_state = FCOE_MAC_STATE_ONLINE; 174 if (mac->fm_link_state == FCOE_MAC_LINK_STATE_UP) 175 (void) ddi_taskq_dispatch( 176 fcoe_global_ss->ss_watchdog_taskq, 177 fcoe_mac_notify_link_up, mac, DDI_SLEEP); 178 break; 179 case FCOE_CMD_PORT_OFFLINE: 180 if (fcoe_disable_callback(mac) == FCOE_FAILURE) { 181 return (FCOE_FAILURE); 182 } 183 mac->fm_state = FCOE_MAC_STATE_OFFLINE; 184 // in case there are threads waiting 185 mutex_enter(&mac->fm_mutex); 186 cv_broadcast(&mac->fm_tx_cv); 187 mutex_exit(&mac->fm_mutex); 188 break; 189 default: 190 FCOE_LOG("fcoe", "fcoe_ctl, unsupported cmd %x", cmd); 191 break; 192 } 193 194 return (FCOE_SUCCESS); 195 } 196 197 /* 198 * Transmit the specified frame to the link 199 */ 200 static void 201 fcoe_tx_frame(fcoe_frame_t *frm) 202 { 203 mblk_t *ret_mblk = NULL; 204 fcoe_mac_t *mac = FRM2MAC(frm); 205 mac_tx_cookie_t ret_cookie; 206 207 fcoe_fill_frame_headers(frm); 208 fcoe_fill_frame_tailers(frm); 209 210 tx_frame: 211 ret_cookie = mac_tx(mac->fm_cli_handle, FRM2MBLK(frm), 0, 212 MAC_TX_NO_ENQUEUE, &ret_mblk); 213 if (ret_cookie != (mac_tx_cookie_t)NULL) { 214 frm->frm_netb = ret_mblk; 215 mutex_enter(&mac->fm_mutex); 216 (void) cv_reltimedwait(&mac->fm_tx_cv, &mac->fm_mutex, 217 drv_usectohz(100000), TR_CLOCK_TICK); 218 mutex_exit(&mac->fm_mutex); 219 220 if (mac->fm_state == FCOE_MAC_STATE_OFFLINE) { 221 /* 222 * we are doing offline, so just tell the upper that 223 * this is finished, the cmd will be aborted soon. 224 */ 225 fcoe_free_netb(ret_mblk); 226 } else { 227 goto tx_frame; 228 } 229 } 230 231 /* 232 * MAC driver will release the mblk of the frame 233 * We need only release the frame itself 234 */ 235 mutex_enter(&FRM2MAC(frm)->fm_ss->ss_watch_mutex); 236 list_insert_tail(&FRM2MAC(frm)->fm_ss->ss_pfrm_list, 237 FRM2FMI(frm)); 238 mac->fm_frm_cnt ++; 239 if (FRM2MAC(frm)->fm_ss->ss_flags & SS_FLAG_DOG_WAITING) { 240 cv_signal(&FRM2MAC(frm)->fm_ss->ss_watch_cv); 241 } 242 mutex_exit(&FRM2MAC(frm)->fm_ss->ss_watch_mutex); 243 } 244 245 /* 246 * Consider cache allocation in the future 247 */ 248 void 249 fcoe_release_frame(fcoe_frame_t *frame) 250 { 251 kmem_free(frame, frame->frm_alloc_size); 252 } 253 254 static void * 255 fcoe_alloc_netb(fcoe_port_t *eport, uint32_t fc_frame_size, uint8_t **ppfc) 256 { 257 mblk_t *mp; 258 259 mp = fcoe_get_mblk(eport->eport_fcoe_private, 260 fc_frame_size + PADDING_SIZE); 261 if (mp != NULL) { 262 *ppfc = mp->b_rptr + PADDING_HEADER_SIZE; 263 } 264 265 return (mp); 266 } 267 268 static void 269 fcoe_free_netb(void *netb) 270 { 271 freemsgchain((mblk_t *)netb); 272 } 273 274 fcoe_frame_t * 275 fcoe_allocate_frame(fcoe_port_t *eport, uint32_t fc_frame_size, void *xmp) 276 { 277 fcoe_frame_t *frm; 278 fcoe_i_frame_t *fmi; 279 mblk_t *mp = xmp; 280 uint32_t alloc_size; 281 uint32_t raw_frame_size; 282 283 if (fc_frame_size > 2136) { 284 FCOE_LOG("fcoe", "fcoe_allocate_frame %d > 2136", 285 fc_frame_size); 286 return (NULL); 287 } 288 289 if (mp == NULL) { 290 /* 291 * We are allocating solicited frame now 292 */ 293 raw_frame_size = PADDING_SIZE + fc_frame_size; 294 mp = fcoe_get_mblk(EPORT2MAC(eport), raw_frame_size); 295 if (mp == NULL) { 296 return (NULL); 297 } 298 } 299 300 alloc_size = sizeof (fcoe_frame_t) + sizeof (fcoe_i_frame_t) + 301 EPORT2MAC(eport)->fm_client.ect_private_frame_struct_size; 302 303 /* 304 * fcoe_frame_t initialization 305 */ 306 frm = (fcoe_frame_t *)kmem_alloc(alloc_size, KM_SLEEP); 307 frm->frm_alloc_size = alloc_size; 308 frm->frm_fc_frame_size = fc_frame_size; 309 frm->frm_payload_size = fc_frame_size - 310 sizeof (fcoe_fc_frame_header_t); 311 frm->frm_fcoe_private = sizeof (fcoe_frame_t) + (uint8_t *)frm; 312 frm->frm_client_private = sizeof (fcoe_i_frame_t) + 313 (uint8_t *)frm->frm_fcoe_private; 314 frm->frm_flags = 0; 315 frm->frm_eport = eport; 316 frm->frm_netb = mp; 317 318 /* 319 * fcoe_i_frame_t initialization 320 */ 321 fmi = FRM2FMI(frm); 322 fmi->fmi_frame = frm; 323 fmi->fmi_mac = EPORT2MAC(eport); 324 fmi->fmi_efh = (void *)mp->b_rptr; 325 326 fmi->fmi_ffh = (fcoe_frame_header_t *) 327 (sizeof (struct ether_header) + (uint8_t *)fmi->fmi_efh); 328 329 fmi->fmi_fc_frame = sizeof (fcoe_frame_header_t) + 330 (uint8_t *)fmi->fmi_ffh; 331 fmi->fmi_fft = (fcoe_frame_tailer_t *) 332 (fc_frame_size + (uint8_t *)fmi->fmi_fc_frame); 333 334 /* 335 * Continue to initialize fcoe_frame_t 336 */ 337 frm->frm_hdr = (fcoe_fc_frame_header_t *)fmi->fmi_fc_frame; 338 frm->frm_ofh1 = NULL; 339 frm->frm_ofh2 = NULL; 340 frm->frm_fc_frame = (uint8_t *)frm->frm_hdr; 341 frm->frm_payload = sizeof (fcoe_fc_frame_header_t) + 342 (uint8_t *)frm->frm_fc_frame; 343 return (frm); 344 } 345 346 /* 347 * Sub routines called by interface functions 348 */ 349 350 /* 351 * According to spec, fill EthernetII frame header, FCoE frame header 352 * VLAN (not included for now) 353 */ 354 static void 355 fcoe_fill_frame_headers(fcoe_frame_t *frm) 356 { 357 fcoe_i_frame_t *fmi = FRM2FMI(frm); 358 359 /* 360 * Initialize ethernet frame header 361 */ 362 bcopy(FRM2MAC(frm)->fm_current_addr, &fmi->fmi_efh->ether_shost, 363 ETHERADDRL); 364 bcopy(frm->frm_eport->eport_efh_dst, 365 &fmi->fmi_efh->ether_dhost, ETHERADDRL); 366 fmi->fmi_efh->ether_type = htons(ETHERTYPE_FCOE); 367 368 /* 369 * Initialize FCoE frame header 370 */ 371 bzero(fmi->fmi_ffh, sizeof (fcoe_frame_header_t)); 372 FCOE_ENCAPS_VER(fmi->fmi_ffh, FCOE_VER); 373 /* set to SOFi3 for the first frame of a sequence */ 374 if (FRM_SEQ_CNT(frm) == 0) { 375 FCOE_V2B_1(0x2E, fmi->fmi_ffh->ffh_sof); 376 } else { 377 FCOE_V2B_1(0x36, fmi->fmi_ffh->ffh_sof); 378 } 379 } 380 381 /* 382 * According to spec, fill FCOE frame tailer including CRC 383 * VLAN (not included for now) 384 */ 385 static void 386 fcoe_fill_frame_tailers(fcoe_frame_t *frm) 387 { 388 uint32_t crc; 389 390 /* 391 * Initialize FCoE frame tailer 392 * CRC is not big endian, can't use macro V2B 393 */ 394 CRC32(crc, frm->frm_fc_frame, frm->frm_fc_frame_size, 395 (uint32_t)~0, crc32_table); 396 FRM2FMI(frm)->fmi_fft->fft_crc[0] = 0xFF & (~crc); 397 FRM2FMI(frm)->fmi_fft->fft_crc[1] = 0xFF & (~crc >> 8); 398 FRM2FMI(frm)->fmi_fft->fft_crc[2] = 0xFF & (~crc >> 16); 399 FRM2FMI(frm)->fmi_fft->fft_crc[3] = 0xFF & (~crc >> 24); 400 if (FRM_F_CTL(frm) & 0x080000) { 401 FCOE_V2B_1(0x42, FRM2FMI(frm)->fmi_fft->fft_eof); 402 } else { 403 FCOE_V2B_1(0x41, FRM2FMI(frm)->fmi_fft->fft_eof); 404 } 405 406 FRM2FMI(frm)->fmi_fft->fft_resvd[0] = 0; 407 FRM2FMI(frm)->fmi_fft->fft_resvd[1] = 0; 408 FRM2FMI(frm)->fmi_fft->fft_resvd[2] = 0; 409 } 410 411 void 412 fcoe_mac_notify_link_up(void *arg) 413 { 414 fcoe_mac_t *mac = (fcoe_mac_t *)arg; 415 416 ASSERT(mac->fm_flags & FCOE_MAC_FLAG_BOUND); 417 418 mac->fm_client.ect_port_event(&mac->fm_eport, 419 FCOE_NOTIFY_EPORT_LINK_UP); 420 } 421 void 422 fcoe_mac_notify_link_down(void *arg) 423 { 424 fcoe_mac_t *mac = (fcoe_mac_t *)arg; 425 426 if (mac->fm_flags & FCOE_MAC_FLAG_BOUND) { 427 mac->fm_client.ect_port_event(&mac->fm_eport, 428 FCOE_NOTIFY_EPORT_LINK_DOWN); 429 } 430 } 431 432 int 433 fcoe_create_port(dev_info_t *parent, fcoe_mac_t *mac, int is_target) 434 { 435 int rval = 0; 436 dev_info_t *child = NULL; 437 char *devname = is_target ? FCOET_DRIVER_NAME : FCOEI_DRIVER_NAME; 438 439 ndi_devi_alloc_sleep(parent, devname, DEVI_PSEUDO_NODEID, &child); 440 if (child == NULL) { 441 FCOE_LOG("fcoe", "fail to create new devinfo"); 442 return (NDI_FAILURE); 443 } 444 445 if (ddi_prop_update_int(DDI_DEV_T_NONE, child, 446 "mac_id", mac->fm_linkid) != DDI_PROP_SUCCESS) { 447 FCOE_LOG("fcoe", 448 "fcoe%d: prop_update port mac id failed for mac %d", 449 ddi_get_instance(parent), mac->fm_linkid); 450 (void) ndi_devi_free(child); 451 return (NDI_FAILURE); 452 } 453 454 rval = ndi_devi_online(child, NDI_ONLINE_ATTACH); 455 if (rval != NDI_SUCCESS) { 456 FCOE_LOG("fcoe", "fcoe%d: online_driver failed for mac %d", 457 ddi_get_instance(parent), mac->fm_linkid); 458 return (NDI_FAILURE); 459 } 460 mac->fm_client_dev = child; 461 462 return (rval); 463 } 464 465 int 466 fcoe_delete_port(dev_info_t *parent, fcoeio_t *fcoeio, datalink_id_t linkid, 467 uint64_t *is_target) 468 { 469 int rval = 0; 470 fcoe_mac_t *mac; 471 472 mac = fcoe_lookup_mac_by_id(linkid); 473 if (mac == NULL) { 474 fcoeio->fcoeio_status = FCOEIOE_MAC_NOT_FOUND; 475 return (EINVAL); 476 } 477 478 *is_target = EPORT_CLT_TYPE(&mac->fm_eport); 479 if ((mac->fm_flags & FCOE_MAC_FLAG_ENABLED) != FCOE_MAC_FLAG_ENABLED) { 480 fcoeio->fcoeio_status = FCOEIOE_ALREADY; 481 return (EALREADY); 482 } 483 484 if (!(mac->fm_flags & FCOE_MAC_FLAG_BOUND)) { 485 /* 486 * It means that deferred detach has finished 487 * of last delete operation 488 */ 489 goto skip_devi_offline; 490 } 491 492 atomic_and_32(&mac->fm_eport.eport_flags, ~EPORT_FLAG_MAC_IN_USE); 493 mac->fm_flags |= FCOE_MAC_FLAG_USER_DEL; 494 rval = ndi_devi_offline(mac->fm_client_dev, NDI_DEVI_REMOVE); 495 if (rval != NDI_SUCCESS) { 496 FCOE_LOG("fcoe", "fcoe%d: offline_driver %s failed", 497 ddi_get_instance(parent), 498 ddi_get_name(mac->fm_client_dev)); 499 atomic_or_32(&mac->fm_eport.eport_flags, 500 EPORT_FLAG_MAC_IN_USE); 501 502 fcoeio->fcoeio_status = FCOEIOE_OFFLINE_FAILURE; 503 return (EBUSY); 504 } 505 506 skip_devi_offline: 507 (void) fcoe_close_mac(mac); 508 fcoe_destroy_mac(mac); 509 return (0); 510 } 511