1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * An implementation of the IPoIB standard based on PSARC 2001/289. 28 */ 29 30 #include <sys/types.h> 31 #include <sys/conf.h> 32 #include <sys/ddi.h> 33 #include <sys/sunddi.h> 34 #include <sys/modctl.h> 35 #include <sys/stropts.h> 36 #include <sys/stream.h> 37 #include <sys/strsun.h> 38 #include <sys/strsubr.h> 39 #include <sys/dlpi.h> 40 #include <sys/mac_provider.h> 41 42 #include <sys/pattr.h> /* for HCK_FULLCKSUM */ 43 #include <sys/sysmacros.h> /* for offsetof */ 44 #include <sys/disp.h> /* for async thread pri */ 45 #include <sys/atomic.h> /* for atomic_add*() */ 46 #include <sys/ethernet.h> /* for ETHERTYPE_IPV6 */ 47 #include <netinet/in.h> /* for netinet/ip.h below */ 48 #include <netinet/ip.h> /* for struct ip */ 49 #include <netinet/udp.h> /* for struct udphdr */ 50 #include <inet/common.h> /* for inet/ip.h below */ 51 #include <inet/ip.h> /* for ipha_t */ 52 #include <inet/ip6.h> /* for ip6_t */ 53 #include <inet/tcp.h> /* for tcph_t */ 54 #include <netinet/icmp6.h> /* for icmp6_t */ 55 #include <sys/callb.h> 56 #include <sys/modhash.h> 57 58 #include <sys/ib/clients/ibd/ibd.h> 59 #include <sys/ib/mgt/sm_attr.h> /* for SM_INIT_TYPE_* */ 60 #include <sys/note.h> 61 #include <sys/multidata.h> 62 63 #include <sys/ib/mgt/ibmf/ibmf.h> /* for ibd_get_portspeed */ 64 65 #include <sys/priv_names.h> 66 #include <sys/dls.h> 67 #include <sys/dld_ioc.h> 68 #include <sys/policy.h> 69 #include <sys/ibpart.h> 70 #include <sys/file.h> 71 72 /* 73 * The write-up below includes details on the following: 74 * 1. The dladm administrative model. 75 * 2. Late HCA initialization feature. 76 * 3. Brussels support and its implications to the current architecture. 77 * 78 * 1. The dladm administrative model. 79 * ------------------------------------------ 80 * With the dladm model, ibnex will create one ibd instance per port. These 81 * instances will be created independent of the port state. 82 * 83 * The ibd driver is two faceted: One side of it working as the port driver and 84 * the other as the partition object driver. 85 * 86 * The port instance is a child of the HCA, and will have an entry in the devfs. 87 * A DDI attach only happens for the port driver, and its attach is 88 * handled in ibd_port_attach(). Similary, a DDI detach for the port driver is 89 * handled in ibd_port_unattach(). 90 * 91 * The partition object is only a registrant to the mac layer via mac_register() 92 * and does not have an entry in the device tree. There is no DDI softstate 93 * managed by the DDI framework for the partition objects. However, the state is 94 * managed inside the ibd driver, and every partition object hangs off the 95 * "ibd_objlist_head". 96 * 97 * The partition object first comes into existence when a user runs the 98 * 'create-part' subcommand of dladm. This is like invoking the attach entry 99 * point of the partition object. The partition object goes away with the 100 * 'delete-part' subcommand of dladm. This is like invoking the detach entry 101 * point of the partition object. 102 * 103 * The create-part and delete-part subcommands result in dld ioctls that end up 104 * calling ibd_create_parition() and ibd_delete_partition respectively. 105 * There ioctls are registered with the dld layer in _init() via a call to 106 * dld_ioc_register(). 107 * 108 * The port instance by itself cannot be plumbed. It is only the partition 109 * objects that can be plumbed and they alone participate in I/O and not the 110 * port driver. 111 * 112 * There are some info ioctls supported in ibd which are used by dladm(1M) to 113 * display useful information. The info entry point for ibd is 114 * ibd_get_partition_info(). 115 * 116 * 2. Late HCA initialization feature. 117 * ------------------------------------ 118 * As mentioned in section 1, the user creates the partition objects via 119 * dladm(1M). It is possible that: 120 * a) The physical port itself is down and the SM cannot be reached. 121 * b) The PKEY specified by the used has not been created in the SM yet. 122 * c) An IPoIB broadcast group for the specified PKEY is not present. 123 * 124 * In all of the above cases, complete initialization of the partition object is 125 * not possible. However, the new model allows the creation of partition 126 * objects even in such cases but will defer the initialization for later. 127 * When such a partition object is plumbed, the link state will be displayed as 128 * "down". 129 * The driver, at this point, is listening to events that herald the 130 * availability of resources - 131 * i) LINK_UP when the link becomes available 132 * ii) PORT_CHANGE when the PKEY has been created 133 * iii) MCG_CREATED when the IPoIB broadcast group for the given pkey has been 134 * created 135 * via ibd_async_handler() for events i) and ii), and via 136 * ibd_snet_notices_handler() for iii. 137 * The driver handles these events (as and when they arrive) and completes the 138 * initialization of the partition object and transitions it to a usable state. 139 * 140 * 3. Brussels support and its implications to the current architecture. 141 * --------------------------------------------------------------------- 142 * The brussels support introduces two new interfaces to the ibd driver - 143 * ibd_m_getprop() and ibd_m_setprop(). 144 * These interfaces allow setting and retrieval of certain properties. 145 * Some of them are public properties while most other are private properties 146 * meant to be used by developers. Tuning the latter kind can cause 147 * performance issues and should not be used without understanding the 148 * implications. All properties are specific to an instance of either the 149 * partition object or the port driver. 150 * 151 * The public properties are : mtu and linkmode. 152 * mtu is a read-only property. 153 * linkmode can take two values - UD and CM. 154 * 155 * Changing the linkmode requires some bookkeeping in the driver. The 156 * capabilities need to be re-reported to the mac layer. This is done by 157 * calling mac_capab_update(). The maxsdu is updated by calling 158 * mac_maxsdu_update2(). 159 * The private properties retain their values across the change of linkmode. 160 * NOTE: 161 * - The port driver does not support any property apart from mtu. 162 * - All other properties are only meant for the partition object. 163 * - The properties cannot be set when an instance is plumbed. The 164 * instance has to be unplumbed to effect any setting. 165 */ 166 167 /* 168 * Driver wide tunables 169 * 170 * ibd_tx_softintr 171 * ibd_rx_softintr 172 * The softintr mechanism allows ibd to avoid event queue overflows if 173 * the receive/completion handlers are to be expensive. These are enabled 174 * by default. 175 * 176 * ibd_log_sz 177 * This specifies the size of the ibd log buffer in bytes. The buffer is 178 * allocated and logging is enabled only when IBD_LOGGING is defined. 179 * 180 */ 181 uint_t ibd_rx_softintr = 1; 182 uint_t ibd_tx_softintr = 1; 183 184 #ifdef IBD_LOGGING 185 uint_t ibd_log_sz = 0x20000; 186 #endif 187 188 #ifdef IBD_LOGGING 189 #define IBD_LOG_SZ ibd_log_sz 190 #endif 191 192 /* Post IBD_RX_POST_CNT receive work requests at a time. */ 193 #define IBD_RX_POST_CNT 8 194 195 /* Hash into 1 << IBD_LOG_RX_POST number of rx post queues */ 196 #define IBD_LOG_RX_POST 4 197 198 /* Minimum number of receive work requests driver needs to always have */ 199 #define IBD_RWQE_MIN ((IBD_RX_POST_CNT << IBD_LOG_RX_POST) * 4) 200 201 /* 202 * LSO parameters 203 */ 204 #define IBD_LSO_MAXLEN 65536 205 #define IBD_LSO_BUFSZ 8192 206 207 /* 208 * Async operation states 209 */ 210 #define IBD_OP_NOTSTARTED 0 211 #define IBD_OP_ONGOING 1 212 #define IBD_OP_COMPLETED 2 213 #define IBD_OP_ERRORED 3 214 #define IBD_OP_ROUTERED 4 215 216 /* 217 * Start/stop in-progress flags; note that restart must always remain 218 * the OR of start and stop flag values. 219 */ 220 #define IBD_DRV_START_IN_PROGRESS 0x10000000 221 #define IBD_DRV_STOP_IN_PROGRESS 0x20000000 222 #define IBD_DRV_RESTART_IN_PROGRESS 0x30000000 223 #define IBD_DRV_DELETE_IN_PROGRESS IBD_DRV_RESTART_IN_PROGRESS 224 225 /* 226 * Miscellaneous constants 227 */ 228 #define IB_MGID_IPV4_LOWGRP_MASK 0xFFFFFFFF 229 #define IBD_DEF_MAX_SDU 2044 230 #define IBD_DEF_MAX_MTU (IBD_DEF_MAX_SDU + IPOIB_HDRSIZE) 231 #define IBD_DEF_RC_MAX_SDU 65520 232 #define IBD_DEF_RC_MAX_MTU (IBD_DEF_RC_MAX_SDU + IPOIB_HDRSIZE) 233 #define IBD_DEFAULT_QKEY 0xB1B 234 #ifdef IBD_LOGGING 235 #define IBD_DMAX_LINE 100 236 #endif 237 238 /* 239 * Enumerations for link states 240 */ 241 typedef enum { 242 IBD_LINK_DOWN, 243 IBD_LINK_UP, 244 IBD_LINK_UP_ABSENT 245 } ibd_link_op_t; 246 247 /* 248 * Driver State Pointer 249 */ 250 void *ibd_list; 251 252 /* 253 * Driver Global Data 254 */ 255 ibd_global_state_t ibd_gstate; 256 257 /* 258 * Partition object list 259 */ 260 ibd_state_t *ibd_objlist_head = NULL; 261 kmutex_t ibd_objlist_lock; 262 263 int ibd_rc_conn_timeout = 60 * 10; /* 10 minutes */ 264 265 /* 266 * Logging 267 */ 268 #ifdef IBD_LOGGING 269 kmutex_t ibd_lbuf_lock; 270 uint8_t *ibd_lbuf; 271 uint32_t ibd_lbuf_ndx; 272 #endif 273 274 /* 275 * Required system entry points 276 */ 277 static int ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 278 static int ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 279 280 /* 281 * Required driver entry points for GLDv3 282 */ 283 static int ibd_m_stat(void *, uint_t, uint64_t *); 284 static int ibd_m_start(void *); 285 static void ibd_m_stop(void *); 286 static int ibd_m_promisc(void *, boolean_t); 287 static int ibd_m_multicst(void *, boolean_t, const uint8_t *); 288 static int ibd_m_unicst(void *, const uint8_t *); 289 static mblk_t *ibd_m_tx(void *, mblk_t *); 290 static boolean_t ibd_m_getcapab(void *, mac_capab_t, void *); 291 292 static int ibd_m_setprop(void *, const char *, mac_prop_id_t, uint_t, 293 const void *); 294 static int ibd_m_getprop(void *, const char *, mac_prop_id_t, uint_t, void *); 295 static void ibd_m_propinfo(void *, const char *, mac_prop_id_t, 296 mac_prop_info_handle_t); 297 static int ibd_set_priv_prop(ibd_state_t *, const char *, uint_t, 298 const void *); 299 static int ibd_get_priv_prop(ibd_state_t *, const char *, uint_t, void *); 300 301 /* 302 * Private driver entry points for GLDv3 303 */ 304 305 /* 306 * Initialization 307 */ 308 static int ibd_state_init(ibd_state_t *, dev_info_t *); 309 static int ibd_init_txlist(ibd_state_t *); 310 static int ibd_init_rxlist(ibd_state_t *); 311 static int ibd_acache_init(ibd_state_t *); 312 #ifdef IBD_LOGGING 313 static void ibd_log_init(void); 314 #endif 315 316 /* 317 * Termination/cleanup 318 */ 319 static void ibd_state_fini(ibd_state_t *); 320 static void ibd_fini_txlist(ibd_state_t *); 321 static void ibd_fini_rxlist(ibd_state_t *); 322 static void ibd_tx_cleanup(ibd_state_t *, ibd_swqe_t *); 323 static void ibd_tx_cleanup_list(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *); 324 static void ibd_acache_fini(ibd_state_t *); 325 #ifdef IBD_LOGGING 326 static void ibd_log_fini(void); 327 #endif 328 329 /* 330 * Allocation/acquire/map routines 331 */ 332 static int ibd_alloc_tx_copybufs(ibd_state_t *); 333 static int ibd_alloc_rx_copybufs(ibd_state_t *); 334 static int ibd_alloc_tx_lsobufs(ibd_state_t *); 335 static ibd_swqe_t *ibd_acquire_swqe(ibd_state_t *); 336 static int ibd_acquire_lsobufs(ibd_state_t *, uint_t, ibt_wr_ds_t *, 337 uint32_t *); 338 339 /* 340 * Free/release/unmap routines 341 */ 342 static void ibd_free_rwqe(ibd_state_t *, ibd_rwqe_t *); 343 static void ibd_free_tx_copybufs(ibd_state_t *); 344 static void ibd_free_rx_copybufs(ibd_state_t *); 345 static void ibd_free_rx_rsrcs(ibd_state_t *); 346 static void ibd_free_tx_lsobufs(ibd_state_t *); 347 static void ibd_release_swqe(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *, int); 348 static void ibd_release_lsobufs(ibd_state_t *, ibt_wr_ds_t *, uint32_t); 349 static void ibd_free_lsohdr(ibd_swqe_t *, mblk_t *); 350 351 /* 352 * Handlers/callback routines 353 */ 354 static uint_t ibd_intr(caddr_t); 355 static uint_t ibd_tx_recycle(caddr_t); 356 static void ibd_rcq_handler(ibt_cq_hdl_t, void *); 357 static void ibd_scq_handler(ibt_cq_hdl_t, void *); 358 static void ibd_poll_rcq(ibd_state_t *, ibt_cq_hdl_t); 359 static void ibd_poll_scq(ibd_state_t *, ibt_cq_hdl_t); 360 static void ibd_drain_rcq(ibd_state_t *, ibt_cq_hdl_t); 361 static void ibd_drain_scq(ibd_state_t *, ibt_cq_hdl_t); 362 static void ibd_freemsg_cb(char *); 363 static void ibd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t, 364 ibt_async_event_t *); 365 static void ibdpd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t, 366 ibt_async_event_t *); 367 static void ibd_snet_notices_handler(void *, ib_gid_t, 368 ibt_subnet_event_code_t, ibt_subnet_event_t *); 369 370 /* 371 * Send/receive routines 372 */ 373 static boolean_t ibd_send(ibd_state_t *, mblk_t *); 374 static void ibd_post_send(ibd_state_t *, ibd_swqe_t *); 375 static void ibd_post_recv(ibd_state_t *, ibd_rwqe_t *); 376 static mblk_t *ibd_process_rx(ibd_state_t *, ibd_rwqe_t *, ibt_wc_t *); 377 378 /* 379 * Threads 380 */ 381 static void ibd_async_work(ibd_state_t *); 382 383 /* 384 * Async tasks 385 */ 386 static void ibd_async_acache(ibd_state_t *, ipoib_mac_t *); 387 static void ibd_async_multicast(ibd_state_t *, ib_gid_t, int); 388 static void ibd_async_setprom(ibd_state_t *); 389 static void ibd_async_unsetprom(ibd_state_t *); 390 static void ibd_async_reap_group(ibd_state_t *, ibd_mce_t *, ib_gid_t, uint8_t); 391 static void ibd_async_trap(ibd_state_t *, ibd_req_t *); 392 static void ibd_async_txsched(ibd_state_t *); 393 static void ibd_async_link(ibd_state_t *, ibd_req_t *); 394 395 /* 396 * Async task helpers 397 */ 398 static ibd_mce_t *ibd_async_mcache(ibd_state_t *, ipoib_mac_t *, boolean_t *); 399 static ibd_mce_t *ibd_join_group(ibd_state_t *, ib_gid_t, uint8_t); 400 static ibd_mce_t *ibd_mcache_find(ib_gid_t, struct list *); 401 static boolean_t ibd_get_allroutergroup(ibd_state_t *, 402 ipoib_mac_t *, ipoib_mac_t *); 403 static void ibd_leave_group(ibd_state_t *, ib_gid_t, uint8_t); 404 static void ibd_reacquire_group(ibd_state_t *, ibd_mce_t *); 405 static ibt_status_t ibd_iba_join(ibd_state_t *, ib_gid_t, ibd_mce_t *); 406 static ibt_status_t ibd_find_bgroup(ibd_state_t *); 407 static void ibd_n2h_gid(ipoib_mac_t *, ib_gid_t *); 408 static void ibd_h2n_mac(ipoib_mac_t *, ib_qpn_t, ib_sn_prefix_t, ib_guid_t); 409 static uint64_t ibd_get_portspeed(ibd_state_t *); 410 static boolean_t ibd_async_safe(ibd_state_t *); 411 static void ibd_async_done(ibd_state_t *); 412 static ibd_ace_t *ibd_acache_lookup(ibd_state_t *, ipoib_mac_t *, int *, int); 413 static ibd_ace_t *ibd_acache_get_unref(ibd_state_t *); 414 static void ibd_link_mod(ibd_state_t *, ibt_async_code_t); 415 static int ibd_locate_pkey(ib_pkey_t *, uint16_t, ib_pkey_t, uint16_t *); 416 417 /* 418 * Helpers for attach/start routines 419 */ 420 static int ibd_register_mac(ibd_state_t *, dev_info_t *); 421 static int ibd_record_capab(ibd_state_t *); 422 static int ibd_get_port_details(ibd_state_t *); 423 static int ibd_alloc_cqs(ibd_state_t *); 424 static int ibd_setup_ud_channel(ibd_state_t *); 425 static int ibd_start(ibd_state_t *); 426 static int ibd_undo_start(ibd_state_t *, link_state_t); 427 static void ibd_set_mac_progress(ibd_state_t *, uint_t); 428 static void ibd_clr_mac_progress(ibd_state_t *, uint_t); 429 static int ibd_part_attach(ibd_state_t *state, dev_info_t *dip); 430 static void ibd_part_unattach(ibd_state_t *state); 431 static int ibd_port_attach(dev_info_t *); 432 static int ibd_port_unattach(ibd_state_t *state, dev_info_t *dip); 433 static int ibd_get_port_state(ibd_state_t *, link_state_t *); 434 static int ibd_part_busy(ibd_state_t *); 435 436 /* 437 * Miscellaneous helpers 438 */ 439 static int ibd_sched_poll(ibd_state_t *, int, int); 440 static void ibd_resume_transmission(ibd_state_t *); 441 static int ibd_setup_lso(ibd_swqe_t *, mblk_t *, uint32_t, ibt_ud_dest_hdl_t); 442 static int ibd_prepare_sgl(ibd_state_t *, mblk_t *, ibd_swqe_t *, uint_t); 443 static void *list_get_head(list_t *); 444 static int ibd_hash_key_cmp(mod_hash_key_t, mod_hash_key_t); 445 static uint_t ibd_hash_by_id(void *, mod_hash_key_t); 446 447 ibt_status_t ibd_get_part_attr(datalink_id_t, ibt_part_attr_t *); 448 ibt_status_t ibd_get_all_part_attr(ibt_part_attr_t **, int *); 449 450 #ifdef IBD_LOGGING 451 static void ibd_log(const char *, ...); 452 #endif 453 454 DDI_DEFINE_STREAM_OPS(ibd_dev_ops, nulldev, nulldev, ibd_attach, ibd_detach, 455 nodev, NULL, D_MP, NULL, ddi_quiesce_not_needed); 456 457 /* Module Driver Info */ 458 static struct modldrv ibd_modldrv = { 459 &mod_driverops, /* This one is a driver */ 460 "InfiniBand GLDv3 Driver", /* short description */ 461 &ibd_dev_ops /* driver specific ops */ 462 }; 463 464 /* Module Linkage */ 465 static struct modlinkage ibd_modlinkage = { 466 MODREV_1, (void *)&ibd_modldrv, NULL 467 }; 468 469 /* 470 * Module (static) info passed to IBTL during ibt_attach 471 */ 472 static struct ibt_clnt_modinfo_s ibd_clnt_modinfo = { 473 IBTI_V_CURR, 474 IBT_NETWORK, 475 ibd_async_handler, 476 NULL, 477 "IBPART" 478 }; 479 480 static struct ibt_clnt_modinfo_s ibdpd_clnt_modinfo = { 481 IBTI_V_CURR, 482 IBT_NETWORK, 483 ibdpd_async_handler, 484 NULL, 485 "IPIB" 486 }; 487 488 /* 489 * GLDv3 entry points 490 */ 491 #define IBD_M_CALLBACK_FLAGS \ 492 (MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 493 494 static mac_callbacks_t ibd_m_callbacks = { 495 IBD_M_CALLBACK_FLAGS, 496 ibd_m_stat, 497 ibd_m_start, 498 ibd_m_stop, 499 ibd_m_promisc, 500 ibd_m_multicst, 501 ibd_m_unicst, 502 ibd_m_tx, 503 NULL, 504 NULL, 505 ibd_m_getcapab, 506 NULL, 507 NULL, 508 ibd_m_setprop, 509 ibd_m_getprop, 510 ibd_m_propinfo 511 }; 512 513 /* Private properties */ 514 char *ibd_priv_props[] = { 515 "_ibd_broadcast_group", 516 "_ibd_coalesce_completions", 517 "_ibd_create_broadcast_group", 518 "_ibd_hash_size", 519 "_ibd_lso_enable", 520 "_ibd_num_ah", 521 "_ibd_num_lso_bufs", 522 "_ibd_rc_enable_srq", 523 "_ibd_rc_num_rwqe", 524 "_ibd_rc_num_srq", 525 "_ibd_rc_num_swqe", 526 "_ibd_rc_rx_comp_count", 527 "_ibd_rc_rx_comp_usec", 528 "_ibd_rc_rx_copy_thresh", 529 "_ibd_rc_rx_rwqe_thresh", 530 "_ibd_rc_tx_comp_count", 531 "_ibd_rc_tx_comp_usec", 532 "_ibd_rc_tx_copy_thresh", 533 "_ibd_ud_num_rwqe", 534 "_ibd_ud_num_swqe", 535 "_ibd_ud_rx_comp_count", 536 "_ibd_ud_rx_comp_usec", 537 "_ibd_ud_tx_comp_count", 538 "_ibd_ud_tx_comp_usec", 539 "_ibd_ud_tx_copy_thresh", 540 NULL 541 }; 542 543 static int ibd_create_partition(void *, intptr_t, int, cred_t *, int *); 544 static int ibd_delete_partition(void *, intptr_t, int, cred_t *, int *); 545 static int ibd_get_partition_info(void *, intptr_t, int, cred_t *, int *); 546 547 static dld_ioc_info_t ibd_dld_ioctl_list[] = { 548 {IBD_CREATE_IBPART, DLDCOPYINOUT, sizeof (ibpart_ioctl_t), 549 ibd_create_partition, secpolicy_dl_config}, 550 {IBD_DELETE_IBPART, DLDCOPYIN, sizeof (ibpart_ioctl_t), 551 ibd_delete_partition, secpolicy_dl_config}, 552 {IBD_INFO_IBPART, DLDCOPYIN, sizeof (ibd_ioctl_t), 553 ibd_get_partition_info, NULL} 554 }; 555 556 /* 557 * Fill/clear <scope> and <p_key> in multicast/broadcast address 558 */ 559 #define IBD_FILL_SCOPE_PKEY(maddr, scope, pkey) \ 560 { \ 561 *(uint32_t *)((char *)(maddr) + 4) |= \ 562 htonl((uint32_t)(scope) << 16); \ 563 *(uint32_t *)((char *)(maddr) + 8) |= \ 564 htonl((uint32_t)(pkey) << 16); \ 565 } 566 567 #define IBD_CLEAR_SCOPE_PKEY(maddr) \ 568 { \ 569 *(uint32_t *)((char *)(maddr) + 4) &= \ 570 htonl(~((uint32_t)0xF << 16)); \ 571 *(uint32_t *)((char *)(maddr) + 8) &= \ 572 htonl(~((uint32_t)0xFFFF << 16)); \ 573 } 574 575 /* 576 * Rudimentary debugging support 577 */ 578 #ifdef DEBUG 579 int ibd_debuglevel = 100; 580 void 581 debug_print(int l, char *fmt, ...) 582 { 583 va_list ap; 584 585 if (l < ibd_debuglevel) 586 return; 587 va_start(ap, fmt); 588 vcmn_err(CE_CONT, fmt, ap); 589 va_end(ap); 590 } 591 #endif 592 593 /* 594 * Common routine to print warning messages; adds in hca guid, port number 595 * and pkey to be able to identify the IBA interface. 596 */ 597 void 598 ibd_print_warn(ibd_state_t *state, char *fmt, ...) 599 { 600 ib_guid_t hca_guid; 601 char ibd_print_buf[MAXNAMELEN + 256]; 602 int len; 603 va_list ap; 604 char part_name[MAXNAMELEN]; 605 datalink_id_t linkid = state->id_plinkid; 606 607 hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip, 608 0, "hca-guid", 0); 609 (void) dls_mgmt_get_linkinfo(linkid, part_name, NULL, NULL, NULL); 610 len = snprintf(ibd_print_buf, sizeof (ibd_print_buf), 611 "%s%d: HCA GUID %016llx port %d PKEY %02x link %s ", 612 ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip), 613 (u_longlong_t)hca_guid, state->id_port, state->id_pkey, 614 part_name); 615 va_start(ap, fmt); 616 (void) vsnprintf(ibd_print_buf + len, sizeof (ibd_print_buf) - len, 617 fmt, ap); 618 cmn_err(CE_NOTE, "!%s", ibd_print_buf); 619 va_end(ap); 620 } 621 622 /* 623 * Warlock directives 624 */ 625 626 /* 627 * id_lso_lock 628 * 629 * state->id_lso->bkt_nfree may be accessed without a lock to 630 * determine the threshold at which we have to ask the nw layer 631 * to resume transmission (see ibd_resume_transmission()). 632 */ 633 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_lso_lock, 634 ibd_state_t::id_lso)) 635 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_lso)) 636 _NOTE(SCHEME_PROTECTS_DATA("init", ibd_state_t::id_lso_policy)) 637 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_lsobkt_t::bkt_nfree)) 638 639 /* 640 * id_scq_poll_lock 641 */ 642 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_scq_poll_lock, 643 ibd_state_t::id_scq_poll_busy)) 644 645 /* 646 * id_txpost_lock 647 */ 648 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock, 649 ibd_state_t::id_tx_head)) 650 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock, 651 ibd_state_t::id_tx_busy)) 652 653 /* 654 * id_acache_req_lock 655 */ 656 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock, 657 ibd_state_t::id_acache_req_cv)) 658 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock, 659 ibd_state_t::id_req_list)) 660 _NOTE(SCHEME_PROTECTS_DATA("atomic", 661 ibd_acache_s::ac_ref)) 662 663 /* 664 * id_ac_mutex 665 * 666 * This mutex is actually supposed to protect id_ah_op as well, 667 * but this path of the code isn't clean (see update of id_ah_op 668 * in ibd_async_acache(), immediately after the call to 669 * ibd_async_mcache()). For now, we'll skip this check by 670 * declaring that id_ah_op is protected by some internal scheme 671 * that warlock isn't aware of. 672 */ 673 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex, 674 ibd_state_t::id_ah_active)) 675 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex, 676 ibd_state_t::id_ah_free)) 677 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex, 678 ibd_state_t::id_ah_addr)) 679 _NOTE(SCHEME_PROTECTS_DATA("ac mutex should protect this", 680 ibd_state_t::id_ah_op)) 681 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex, 682 ibd_state_t::id_ah_error)) 683 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex, 684 ibd_state_t::id_ac_hot_ace)) 685 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_ah_error)) 686 687 /* 688 * id_mc_mutex 689 */ 690 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex, 691 ibd_state_t::id_mc_full)) 692 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex, 693 ibd_state_t::id_mc_non)) 694 695 /* 696 * id_trap_lock 697 */ 698 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock, 699 ibd_state_t::id_trap_cv)) 700 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock, 701 ibd_state_t::id_trap_stop)) 702 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock, 703 ibd_state_t::id_trap_inprog)) 704 705 /* 706 * id_prom_op 707 */ 708 _NOTE(SCHEME_PROTECTS_DATA("only by async thread", 709 ibd_state_t::id_prom_op)) 710 711 /* 712 * id_sched_lock 713 */ 714 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_sched_lock, 715 ibd_state_t::id_sched_needed)) 716 717 /* 718 * id_link_mutex 719 */ 720 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_link_mutex, 721 ibd_state_t::id_link_state)) 722 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_link_state)) 723 _NOTE(SCHEME_PROTECTS_DATA("only async thr and ibd_m_start", 724 ibd_state_t::id_link_speed)) 725 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_sgid)) 726 727 /* 728 * id_tx_list.dl_mutex 729 */ 730 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex, 731 ibd_state_t::id_tx_list.dl_head)) 732 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex, 733 ibd_state_t::id_tx_list.dl_pending_sends)) 734 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex, 735 ibd_state_t::id_tx_list.dl_cnt)) 736 737 /* 738 * id_rx_list.dl_mutex 739 */ 740 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr", 741 ibd_state_t::id_rx_list.dl_bufs_outstanding)) 742 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr", 743 ibd_state_t::id_rx_list.dl_cnt)) 744 745 /* 746 * rc_timeout_lock 747 */ 748 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock, 749 ibd_state_t::rc_timeout_start)) 750 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock, 751 ibd_state_t::rc_timeout)) 752 753 754 /* 755 * Items protected by atomic updates 756 */ 757 _NOTE(SCHEME_PROTECTS_DATA("atomic update only", 758 ibd_state_s::id_brd_rcv 759 ibd_state_s::id_brd_xmt 760 ibd_state_s::id_multi_rcv 761 ibd_state_s::id_multi_xmt 762 ibd_state_s::id_num_intrs 763 ibd_state_s::id_rcv_bytes 764 ibd_state_s::id_rcv_pkt 765 ibd_state_s::id_rx_post_queue_index 766 ibd_state_s::id_tx_short 767 ibd_state_s::id_xmt_bytes 768 ibd_state_s::id_xmt_pkt 769 ibd_state_s::rc_rcv_trans_byte 770 ibd_state_s::rc_rcv_trans_pkt 771 ibd_state_s::rc_rcv_copy_byte 772 ibd_state_s::rc_rcv_copy_pkt 773 ibd_state_s::rc_xmt_bytes 774 ibd_state_s::rc_xmt_small_pkt 775 ibd_state_s::rc_xmt_fragmented_pkt 776 ibd_state_s::rc_xmt_map_fail_pkt 777 ibd_state_s::rc_xmt_map_succ_pkt 778 ibd_rc_chan_s::rcq_invoking)) 779 780 /* 781 * Non-mutex protection schemes for data elements. Almost all of 782 * these are non-shared items. 783 */ 784 _NOTE(SCHEME_PROTECTS_DATA("unshared or single-threaded", 785 callb_cpr 786 ib_gid_s 787 ib_header_info 788 ibd_acache_rq 789 ibd_acache_s::ac_mce 790 ibd_acache_s::ac_chan 791 ibd_mcache::mc_fullreap 792 ibd_mcache::mc_jstate 793 ibd_mcache::mc_req 794 ibd_rwqe_s 795 ibd_swqe_s 796 ibd_wqe_s 797 ibt_wr_ds_s::ds_va 798 ibt_wr_lso_s 799 ipoib_mac::ipoib_qpn 800 mac_capab_lso_s 801 msgb::b_next 802 msgb::b_cont 803 msgb::b_rptr 804 msgb::b_wptr 805 ibd_state_s::id_bgroup_created 806 ibd_state_s::id_mac_state 807 ibd_state_s::id_mtu 808 ibd_state_s::id_ud_num_rwqe 809 ibd_state_s::id_ud_num_swqe 810 ibd_state_s::id_qpnum 811 ibd_state_s::id_rcq_hdl 812 ibd_state_s::id_rx_buf_sz 813 ibd_state_s::id_rx_bufs 814 ibd_state_s::id_rx_mr_hdl 815 ibd_state_s::id_rx_wqes 816 ibd_state_s::id_rxwcs 817 ibd_state_s::id_rxwcs_size 818 ibd_state_s::id_rx_nqueues 819 ibd_state_s::id_rx_queues 820 ibd_state_s::id_scope 821 ibd_state_s::id_scq_hdl 822 ibd_state_s::id_tx_buf_sz 823 ibd_state_s::id_tx_bufs 824 ibd_state_s::id_tx_mr_hdl 825 ibd_state_s::id_tx_rel_list.dl_cnt 826 ibd_state_s::id_tx_wqes 827 ibd_state_s::id_txwcs 828 ibd_state_s::id_txwcs_size 829 ibd_state_s::rc_listen_hdl 830 ibd_state_s::rc_listen_hdl_OFED_interop 831 ibd_state_s::rc_srq_size 832 ibd_state_s::rc_srq_rwqes 833 ibd_state_s::rc_srq_rx_bufs 834 ibd_state_s::rc_srq_rx_mr_hdl 835 ibd_state_s::rc_tx_largebuf_desc_base 836 ibd_state_s::rc_tx_mr_bufs 837 ibd_state_s::rc_tx_mr_hdl 838 ipha_s 839 icmph_s 840 ibt_path_info_s::pi_sid 841 ibd_rc_chan_s::ace 842 ibd_rc_chan_s::chan_hdl 843 ibd_rc_chan_s::state 844 ibd_rc_chan_s::chan_state 845 ibd_rc_chan_s::is_tx_chan 846 ibd_rc_chan_s::rcq_hdl 847 ibd_rc_chan_s::rcq_size 848 ibd_rc_chan_s::scq_hdl 849 ibd_rc_chan_s::scq_size 850 ibd_rc_chan_s::rx_bufs 851 ibd_rc_chan_s::rx_mr_hdl 852 ibd_rc_chan_s::rx_rwqes 853 ibd_rc_chan_s::tx_wqes 854 ibd_rc_chan_s::tx_mr_bufs 855 ibd_rc_chan_s::tx_mr_hdl 856 ibd_rc_chan_s::tx_rel_list.dl_cnt 857 ibd_rc_chan_s::is_used 858 ibd_rc_tx_largebuf_s::lb_buf 859 ibd_rc_msg_hello_s 860 ibt_cm_return_args_s)) 861 862 /* 863 * ibd_rc_chan_s::next is protected by two mutexes: 864 * 1) ibd_state_s::rc_pass_chan_list.chan_list_mutex 865 * 2) ibd_state_s::rc_obs_act_chan_list.chan_list_mutex. 866 */ 867 _NOTE(SCHEME_PROTECTS_DATA("protected by two mutexes", 868 ibd_rc_chan_s::next)) 869 870 /* 871 * ibd_state_s.rc_tx_large_bufs_lock 872 */ 873 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock, 874 ibd_state_s::rc_tx_largebuf_free_head)) 875 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock, 876 ibd_state_s::rc_tx_largebuf_nfree)) 877 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock, 878 ibd_rc_tx_largebuf_s::lb_next)) 879 880 /* 881 * ibd_acache_s.tx_too_big_mutex 882 */ 883 _NOTE(MUTEX_PROTECTS_DATA(ibd_acache_s::tx_too_big_mutex, 884 ibd_acache_s::tx_too_big_ongoing)) 885 886 /* 887 * tx_wqe_list.dl_mutex 888 */ 889 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex, 890 ibd_rc_chan_s::tx_wqe_list.dl_head)) 891 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex, 892 ibd_rc_chan_s::tx_wqe_list.dl_pending_sends)) 893 _NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex, 894 ibd_rc_chan_s::tx_wqe_list.dl_cnt)) 895 896 /* 897 * ibd_state_s.rc_ace_recycle_lock 898 */ 899 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_ace_recycle_lock, 900 ibd_state_s::rc_ace_recycle)) 901 902 /* 903 * rc_srq_rwqe_list.dl_mutex 904 */ 905 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr", 906 ibd_state_t::rc_srq_rwqe_list.dl_bufs_outstanding)) 907 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr", 908 ibd_state_t::rc_srq_rwqe_list.dl_cnt)) 909 910 /* 911 * Non-mutex protection schemes for data elements. They are counters 912 * for problem diagnosis. Don't need be protected. 913 */ 914 _NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis", 915 ibd_state_s::rc_rcv_alloc_fail 916 ibd_state_s::rc_rcq_err 917 ibd_state_s::rc_ace_not_found 918 ibd_state_s::rc_xmt_drop_too_long_pkt 919 ibd_state_s::rc_xmt_icmp_too_long_pkt 920 ibd_state_s::rc_xmt_reenter_too_long_pkt 921 ibd_state_s::rc_swqe_short 922 ibd_state_s::rc_swqe_mac_update 923 ibd_state_s::rc_xmt_buf_short 924 ibd_state_s::rc_xmt_buf_mac_update 925 ibd_state_s::rc_scq_no_swqe 926 ibd_state_s::rc_scq_no_largebuf 927 ibd_state_s::rc_conn_succ 928 ibd_state_s::rc_conn_fail 929 ibd_state_s::rc_null_conn 930 ibd_state_s::rc_no_estab_conn 931 ibd_state_s::rc_act_close 932 ibd_state_s::rc_pas_close 933 ibd_state_s::rc_delay_ace_recycle 934 ibd_state_s::rc_act_close_simultaneous 935 ibd_state_s::rc_act_close_not_clean 936 ibd_state_s::rc_pas_close_rcq_invoking 937 ibd_state_s::rc_reset_cnt 938 ibd_state_s::rc_timeout_act 939 ibd_state_s::rc_timeout_pas 940 ibd_state_s::rc_stop_connect)) 941 942 #ifdef DEBUG 943 /* 944 * Non-mutex protection schemes for data elements. They are counters 945 * for problem diagnosis. Don't need be protected. 946 */ 947 _NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis", 948 ibd_state_s::rc_rwqe_short 949 ibd_rc_stat_s::rc_rcv_trans_byte 950 ibd_rc_stat_s::rc_rcv_trans_pkt 951 ibd_rc_stat_s::rc_rcv_copy_byte 952 ibd_rc_stat_s::rc_rcv_copy_pkt 953 ibd_rc_stat_s::rc_rcv_alloc_fail 954 ibd_rc_stat_s::rc_rcq_err 955 ibd_rc_stat_s::rc_rwqe_short 956 ibd_rc_stat_s::rc_xmt_bytes 957 ibd_rc_stat_s::rc_xmt_small_pkt 958 ibd_rc_stat_s::rc_xmt_fragmented_pkt 959 ibd_rc_stat_s::rc_xmt_map_fail_pkt 960 ibd_rc_stat_s::rc_xmt_map_succ_pkt 961 ibd_rc_stat_s::rc_ace_not_found 962 ibd_rc_stat_s::rc_scq_no_swqe 963 ibd_rc_stat_s::rc_scq_no_largebuf 964 ibd_rc_stat_s::rc_swqe_short 965 ibd_rc_stat_s::rc_swqe_mac_update 966 ibd_rc_stat_s::rc_xmt_buf_short 967 ibd_rc_stat_s::rc_xmt_buf_mac_update 968 ibd_rc_stat_s::rc_conn_succ 969 ibd_rc_stat_s::rc_conn_fail 970 ibd_rc_stat_s::rc_null_conn 971 ibd_rc_stat_s::rc_no_estab_conn 972 ibd_rc_stat_s::rc_act_close 973 ibd_rc_stat_s::rc_pas_close 974 ibd_rc_stat_s::rc_delay_ace_recycle 975 ibd_rc_stat_s::rc_act_close_simultaneous 976 ibd_rc_stat_s::rc_reset_cnt 977 ibd_rc_stat_s::rc_timeout_act 978 ibd_rc_stat_s::rc_timeout_pas)) 979 #endif 980 981 int 982 _init() 983 { 984 int status; 985 986 status = ddi_soft_state_init(&ibd_list, max(sizeof (ibd_state_t), 987 PAGESIZE), 0); 988 if (status != 0) { 989 DPRINT(10, "_init:failed in ddi_soft_state_init()"); 990 return (status); 991 } 992 993 mutex_init(&ibd_objlist_lock, NULL, MUTEX_DRIVER, NULL); 994 995 mac_init_ops(&ibd_dev_ops, "ibp"); 996 status = mod_install(&ibd_modlinkage); 997 if (status != 0) { 998 DPRINT(10, "_init:failed in mod_install()"); 999 ddi_soft_state_fini(&ibd_list); 1000 mac_fini_ops(&ibd_dev_ops); 1001 return (status); 1002 } 1003 1004 mutex_init(&ibd_gstate.ig_mutex, NULL, MUTEX_DRIVER, NULL); 1005 mutex_enter(&ibd_gstate.ig_mutex); 1006 ibd_gstate.ig_ibt_hdl = NULL; 1007 ibd_gstate.ig_ibt_hdl_ref_cnt = 0; 1008 ibd_gstate.ig_service_list = NULL; 1009 mutex_exit(&ibd_gstate.ig_mutex); 1010 1011 if (dld_ioc_register(IBPART_IOC, ibd_dld_ioctl_list, 1012 DLDIOCCNT(ibd_dld_ioctl_list)) != 0) { 1013 return (EIO); 1014 } 1015 1016 ibt_register_part_attr_cb(ibd_get_part_attr, ibd_get_all_part_attr); 1017 1018 #ifdef IBD_LOGGING 1019 ibd_log_init(); 1020 #endif 1021 return (0); 1022 } 1023 1024 int 1025 _info(struct modinfo *modinfop) 1026 { 1027 return (mod_info(&ibd_modlinkage, modinfop)); 1028 } 1029 1030 int 1031 _fini() 1032 { 1033 int status; 1034 1035 status = mod_remove(&ibd_modlinkage); 1036 if (status != 0) 1037 return (status); 1038 1039 ibt_unregister_part_attr_cb(); 1040 1041 mac_fini_ops(&ibd_dev_ops); 1042 mutex_destroy(&ibd_objlist_lock); 1043 ddi_soft_state_fini(&ibd_list); 1044 mutex_destroy(&ibd_gstate.ig_mutex); 1045 #ifdef IBD_LOGGING 1046 ibd_log_fini(); 1047 #endif 1048 return (0); 1049 } 1050 1051 /* 1052 * Convert the GID part of the mac address from network byte order 1053 * to host order. 1054 */ 1055 static void 1056 ibd_n2h_gid(ipoib_mac_t *mac, ib_gid_t *dgid) 1057 { 1058 ib_sn_prefix_t nbopref; 1059 ib_guid_t nboguid; 1060 1061 bcopy(mac->ipoib_gidpref, &nbopref, sizeof (ib_sn_prefix_t)); 1062 bcopy(mac->ipoib_gidsuff, &nboguid, sizeof (ib_guid_t)); 1063 dgid->gid_prefix = b2h64(nbopref); 1064 dgid->gid_guid = b2h64(nboguid); 1065 } 1066 1067 /* 1068 * Create the IPoIB address in network byte order from host order inputs. 1069 */ 1070 static void 1071 ibd_h2n_mac(ipoib_mac_t *mac, ib_qpn_t qpn, ib_sn_prefix_t prefix, 1072 ib_guid_t guid) 1073 { 1074 ib_sn_prefix_t nbopref; 1075 ib_guid_t nboguid; 1076 1077 mac->ipoib_qpn = htonl(qpn); 1078 nbopref = h2b64(prefix); 1079 nboguid = h2b64(guid); 1080 bcopy(&nbopref, mac->ipoib_gidpref, sizeof (ib_sn_prefix_t)); 1081 bcopy(&nboguid, mac->ipoib_gidsuff, sizeof (ib_guid_t)); 1082 } 1083 1084 /* 1085 * Send to the appropriate all-routers group when the IBA multicast group 1086 * does not exist, based on whether the target group is v4 or v6. 1087 */ 1088 static boolean_t 1089 ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac, 1090 ipoib_mac_t *rmac) 1091 { 1092 boolean_t retval = B_TRUE; 1093 uint32_t adjscope = state->id_scope << 16; 1094 uint32_t topword; 1095 1096 /* 1097 * Copy the first 4 bytes in without assuming any alignment of 1098 * input mac address; this will have IPoIB signature, flags and 1099 * scope bits. 1100 */ 1101 bcopy(mcmac->ipoib_gidpref, &topword, sizeof (uint32_t)); 1102 topword = ntohl(topword); 1103 1104 /* 1105 * Generate proper address for IPv4/v6, adding in the Pkey properly. 1106 */ 1107 if ((topword == (IB_MCGID_IPV4_PREFIX | adjscope)) || 1108 (topword == (IB_MCGID_IPV6_PREFIX | adjscope))) 1109 ibd_h2n_mac(rmac, IB_MC_QPN, (((uint64_t)topword << 32) | 1110 ((uint32_t)(state->id_pkey << 16))), 1111 (INADDR_ALLRTRS_GROUP - INADDR_UNSPEC_GROUP)); 1112 else 1113 /* 1114 * Does not have proper bits in the mgid address. 1115 */ 1116 retval = B_FALSE; 1117 1118 return (retval); 1119 } 1120 1121 /* 1122 * Membership states for different mcg's are tracked by two lists: 1123 * the "non" list is used for promiscuous mode, when all mcg traffic 1124 * needs to be inspected. This type of membership is never used for 1125 * transmission, so there can not be an AH in the active list 1126 * corresponding to a member in this list. This list does not need 1127 * any protection, since all operations are performed by the async 1128 * thread. 1129 * 1130 * "Full" and "SendOnly" membership is tracked using a single list, 1131 * the "full" list. This is because this single list can then be 1132 * searched during transmit to a multicast group (if an AH for the 1133 * mcg is not found in the active list), since at least one type 1134 * of membership must be present before initiating the transmit. 1135 * This list is also emptied during driver detach, since sendonly 1136 * membership acquired during transmit is dropped at detach time 1137 * along with ipv4 broadcast full membership. Insert/deletes to 1138 * this list are done only by the async thread, but it is also 1139 * searched in program context (see multicast disable case), thus 1140 * the id_mc_mutex protects the list. The driver detach path also 1141 * deconstructs the "full" list, but it ensures that the async 1142 * thread will not be accessing the list (by blocking out mcg 1143 * trap handling and making sure no more Tx reaping will happen). 1144 * 1145 * Currently, an IBA attach is done in the SendOnly case too, 1146 * although this is not required. 1147 */ 1148 #define IBD_MCACHE_INSERT_FULL(state, mce) \ 1149 list_insert_head(&state->id_mc_full, mce) 1150 #define IBD_MCACHE_INSERT_NON(state, mce) \ 1151 list_insert_head(&state->id_mc_non, mce) 1152 #define IBD_MCACHE_FIND_FULL(state, mgid) \ 1153 ibd_mcache_find(mgid, &state->id_mc_full) 1154 #define IBD_MCACHE_FIND_NON(state, mgid) \ 1155 ibd_mcache_find(mgid, &state->id_mc_non) 1156 #define IBD_MCACHE_PULLOUT_FULL(state, mce) \ 1157 list_remove(&state->id_mc_full, mce) 1158 #define IBD_MCACHE_PULLOUT_NON(state, mce) \ 1159 list_remove(&state->id_mc_non, mce) 1160 1161 static void * 1162 list_get_head(list_t *list) 1163 { 1164 list_node_t *lhead = list_head(list); 1165 1166 if (lhead != NULL) 1167 list_remove(list, lhead); 1168 return (lhead); 1169 } 1170 1171 /* 1172 * This is always guaranteed to be able to queue the work. 1173 */ 1174 void 1175 ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op) 1176 { 1177 /* Initialize request */ 1178 DPRINT(1, "ibd_queue_work_slot : op: %d \n", op); 1179 ptr->rq_op = op; 1180 1181 /* 1182 * Queue provided slot onto request pool. 1183 */ 1184 mutex_enter(&state->id_acache_req_lock); 1185 list_insert_tail(&state->id_req_list, ptr); 1186 1187 /* Go, fetch, async thread */ 1188 cv_signal(&state->id_acache_req_cv); 1189 mutex_exit(&state->id_acache_req_lock); 1190 } 1191 1192 /* 1193 * Main body of the per interface async thread. 1194 */ 1195 static void 1196 ibd_async_work(ibd_state_t *state) 1197 { 1198 ibd_req_t *ptr; 1199 callb_cpr_t cprinfo; 1200 1201 mutex_enter(&state->id_acache_req_lock); 1202 CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock, 1203 callb_generic_cpr, "ibd_async_work"); 1204 1205 for (;;) { 1206 ptr = list_get_head(&state->id_req_list); 1207 if (ptr != NULL) { 1208 mutex_exit(&state->id_acache_req_lock); 1209 1210 /* 1211 * If we are in late hca initialization mode, do not 1212 * process any other async request other than TRAP. TRAP 1213 * is used for indicating creation of a broadcast group; 1214 * in which case, we need to join/create the group. 1215 */ 1216 if ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) && 1217 (ptr->rq_op != IBD_ASYNC_TRAP)) { 1218 goto free_req_and_continue; 1219 } 1220 1221 /* 1222 * Once we have done the operation, there is no 1223 * guarantee the request slot is going to be valid, 1224 * it might be freed up (as in IBD_ASYNC_LEAVE, REAP, 1225 * TRAP). 1226 * 1227 * Perform the request. 1228 */ 1229 switch (ptr->rq_op) { 1230 case IBD_ASYNC_GETAH: 1231 ibd_async_acache(state, &ptr->rq_mac); 1232 break; 1233 case IBD_ASYNC_JOIN: 1234 case IBD_ASYNC_LEAVE: 1235 ibd_async_multicast(state, 1236 ptr->rq_gid, ptr->rq_op); 1237 break; 1238 case IBD_ASYNC_PROMON: 1239 ibd_async_setprom(state); 1240 break; 1241 case IBD_ASYNC_PROMOFF: 1242 ibd_async_unsetprom(state); 1243 break; 1244 case IBD_ASYNC_REAP: 1245 ibd_async_reap_group(state, 1246 ptr->rq_ptr, ptr->rq_gid, 1247 IB_MC_JSTATE_FULL); 1248 /* 1249 * the req buf contains in mce 1250 * structure, so we do not need 1251 * to free it here. 1252 */ 1253 ptr = NULL; 1254 break; 1255 case IBD_ASYNC_TRAP: 1256 ibd_async_trap(state, ptr); 1257 break; 1258 case IBD_ASYNC_SCHED: 1259 ibd_async_txsched(state); 1260 break; 1261 case IBD_ASYNC_LINK: 1262 ibd_async_link(state, ptr); 1263 break; 1264 case IBD_ASYNC_EXIT: 1265 mutex_enter(&state->id_acache_req_lock); 1266 #ifndef __lock_lint 1267 CALLB_CPR_EXIT(&cprinfo); 1268 #else 1269 mutex_exit(&state->id_acache_req_lock); 1270 #endif 1271 return; 1272 case IBD_ASYNC_RC_TOO_BIG: 1273 ibd_async_rc_process_too_big(state, 1274 ptr); 1275 break; 1276 case IBD_ASYNC_RC_CLOSE_ACT_CHAN: 1277 ibd_async_rc_close_act_chan(state, ptr); 1278 break; 1279 case IBD_ASYNC_RC_RECYCLE_ACE: 1280 ibd_async_rc_recycle_ace(state, ptr); 1281 break; 1282 case IBD_ASYNC_RC_CLOSE_PAS_CHAN: 1283 (void) ibd_rc_pas_close(ptr->rq_ptr, 1284 B_TRUE, B_TRUE); 1285 break; 1286 } 1287 free_req_and_continue: 1288 if (ptr != NULL) 1289 kmem_cache_free(state->id_req_kmc, ptr); 1290 1291 mutex_enter(&state->id_acache_req_lock); 1292 } else { 1293 #ifndef __lock_lint 1294 /* 1295 * Nothing to do: wait till new request arrives. 1296 */ 1297 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1298 cv_wait(&state->id_acache_req_cv, 1299 &state->id_acache_req_lock); 1300 CALLB_CPR_SAFE_END(&cprinfo, 1301 &state->id_acache_req_lock); 1302 #endif 1303 } 1304 } 1305 1306 /*NOTREACHED*/ 1307 _NOTE(NOT_REACHED) 1308 } 1309 1310 /* 1311 * Return when it is safe to queue requests to the async daemon; primarily 1312 * for subnet trap and async event handling. Disallow requests before the 1313 * daemon is created, and when interface deinitilization starts. 1314 */ 1315 static boolean_t 1316 ibd_async_safe(ibd_state_t *state) 1317 { 1318 mutex_enter(&state->id_trap_lock); 1319 if (state->id_trap_stop) { 1320 mutex_exit(&state->id_trap_lock); 1321 return (B_FALSE); 1322 } 1323 state->id_trap_inprog++; 1324 mutex_exit(&state->id_trap_lock); 1325 return (B_TRUE); 1326 } 1327 1328 /* 1329 * Wake up ibd_m_stop() if the unplumb code is waiting for pending subnet 1330 * trap or event handling to complete to kill the async thread and deconstruct 1331 * the mcg/ace list. 1332 */ 1333 static void 1334 ibd_async_done(ibd_state_t *state) 1335 { 1336 mutex_enter(&state->id_trap_lock); 1337 if (--state->id_trap_inprog == 0) 1338 cv_signal(&state->id_trap_cv); 1339 mutex_exit(&state->id_trap_lock); 1340 } 1341 1342 /* 1343 * Hash functions: 1344 * ibd_hash_by_id: Returns the qpn as the hash entry into bucket. 1345 * ibd_hash_key_cmp: Compares two keys, return 0 on success or else 1. 1346 * These operate on mac addresses input into ibd_send, but there is no 1347 * guarantee on the alignment of the ipoib_mac_t structure. 1348 */ 1349 /*ARGSUSED*/ 1350 static uint_t 1351 ibd_hash_by_id(void *hash_data, mod_hash_key_t key) 1352 { 1353 ulong_t ptraddr = (ulong_t)key; 1354 uint_t hval; 1355 1356 /* 1357 * If the input address is 4 byte aligned, we can just dereference 1358 * it. This is most common, since IP will send in a 4 byte aligned 1359 * IP header, which implies the 24 byte IPoIB psuedo header will be 1360 * 4 byte aligned too. 1361 */ 1362 if ((ptraddr & 3) == 0) 1363 return ((uint_t)((ipoib_mac_t *)key)->ipoib_qpn); 1364 1365 bcopy(&(((ipoib_mac_t *)key)->ipoib_qpn), &hval, sizeof (uint_t)); 1366 return (hval); 1367 } 1368 1369 static int 1370 ibd_hash_key_cmp(mod_hash_key_t key1, mod_hash_key_t key2) 1371 { 1372 if (bcmp((char *)key1, (char *)key2, IPOIB_ADDRL) == 0) 1373 return (0); 1374 else 1375 return (1); 1376 } 1377 1378 /* 1379 * Initialize all the per interface caches and lists; AH cache, 1380 * MCG list etc. 1381 */ 1382 static int 1383 ibd_acache_init(ibd_state_t *state) 1384 { 1385 ibd_ace_t *ce; 1386 int i; 1387 1388 mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL); 1389 mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL); 1390 mutex_enter(&state->id_ac_mutex); 1391 list_create(&state->id_ah_free, sizeof (ibd_ace_t), 1392 offsetof(ibd_ace_t, ac_list)); 1393 list_create(&state->id_ah_active, sizeof (ibd_ace_t), 1394 offsetof(ibd_ace_t, ac_list)); 1395 state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash", 1396 state->id_hash_size, mod_hash_null_keydtor, mod_hash_null_valdtor, 1397 ibd_hash_by_id, NULL, ibd_hash_key_cmp, KM_SLEEP); 1398 list_create(&state->id_mc_full, sizeof (ibd_mce_t), 1399 offsetof(ibd_mce_t, mc_list)); 1400 list_create(&state->id_mc_non, sizeof (ibd_mce_t), 1401 offsetof(ibd_mce_t, mc_list)); 1402 state->id_ac_hot_ace = NULL; 1403 1404 state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) * 1405 state->id_num_ah, KM_SLEEP); 1406 for (i = 0; i < state->id_num_ah; i++, ce++) { 1407 if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS, 1408 state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) { 1409 mutex_exit(&state->id_ac_mutex); 1410 ibd_acache_fini(state); 1411 return (DDI_FAILURE); 1412 } else { 1413 CLEAR_REFCYCLE(ce); 1414 ce->ac_mce = NULL; 1415 mutex_init(&ce->tx_too_big_mutex, NULL, 1416 MUTEX_DRIVER, NULL); 1417 IBD_ACACHE_INSERT_FREE(state, ce); 1418 } 1419 } 1420 mutex_exit(&state->id_ac_mutex); 1421 return (DDI_SUCCESS); 1422 } 1423 1424 static void 1425 ibd_acache_fini(ibd_state_t *state) 1426 { 1427 ibd_ace_t *ptr; 1428 1429 mutex_enter(&state->id_ac_mutex); 1430 1431 while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) { 1432 ASSERT(GET_REF(ptr) == 0); 1433 mutex_destroy(&ptr->tx_too_big_mutex); 1434 (void) ibt_free_ud_dest(ptr->ac_dest); 1435 } 1436 1437 while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) { 1438 ASSERT(GET_REF(ptr) == 0); 1439 mutex_destroy(&ptr->tx_too_big_mutex); 1440 (void) ibt_free_ud_dest(ptr->ac_dest); 1441 } 1442 1443 list_destroy(&state->id_ah_free); 1444 list_destroy(&state->id_ah_active); 1445 list_destroy(&state->id_mc_full); 1446 list_destroy(&state->id_mc_non); 1447 kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * state->id_num_ah); 1448 mutex_exit(&state->id_ac_mutex); 1449 mutex_destroy(&state->id_ac_mutex); 1450 mutex_destroy(&state->id_mc_mutex); 1451 } 1452 1453 /* 1454 * Search AH active hash list for a cached path to input destination. 1455 * If we are "just looking", hold == F. When we are in the Tx path, 1456 * we set hold == T to grab a reference on the AH so that it can not 1457 * be recycled to a new destination while the Tx request is posted. 1458 */ 1459 ibd_ace_t * 1460 ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num) 1461 { 1462 ibd_ace_t *ptr; 1463 1464 ASSERT(mutex_owned(&state->id_ac_mutex)); 1465 1466 /* 1467 * Do hash search. 1468 */ 1469 if (mod_hash_find(state->id_ah_active_hash, 1470 (mod_hash_key_t)mac, (mod_hash_val_t)&ptr) == 0) { 1471 if (hold) 1472 INC_REF(ptr, num); 1473 return (ptr); 1474 } 1475 return (NULL); 1476 } 1477 1478 /* 1479 * This is called by the tx side; if an initialized AH is found in 1480 * the active list, it is locked down and can be used; if no entry 1481 * is found, an async request is queued to do path resolution. 1482 */ 1483 static ibd_ace_t * 1484 ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe) 1485 { 1486 ibd_ace_t *ptr; 1487 ibd_req_t *req; 1488 1489 /* 1490 * Only attempt to print when we can; in the mdt pattr case, the 1491 * address is not aligned properly. 1492 */ 1493 if (((ulong_t)mac & 3) == 0) { 1494 DPRINT(4, 1495 "ibd_acache_lookup : lookup for %08X:%08X:%08X:%08X:%08X", 1496 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]), 1497 htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]), 1498 htonl(mac->ipoib_gidsuff[1])); 1499 } 1500 1501 mutex_enter(&state->id_ac_mutex); 1502 1503 if (((ptr = state->id_ac_hot_ace) != NULL) && 1504 (memcmp(&ptr->ac_mac, mac, sizeof (*mac)) == 0)) { 1505 INC_REF(ptr, numwqe); 1506 mutex_exit(&state->id_ac_mutex); 1507 return (ptr); 1508 } 1509 if (((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL)) { 1510 state->id_ac_hot_ace = ptr; 1511 mutex_exit(&state->id_ac_mutex); 1512 return (ptr); 1513 } 1514 1515 /* 1516 * Implementation of a single outstanding async request; if 1517 * the operation is not started yet, queue a request and move 1518 * to ongoing state. Remember in id_ah_addr for which address 1519 * we are queueing the request, in case we need to flag an error; 1520 * Any further requests, for the same or different address, until 1521 * the operation completes, is sent back to GLDv3 to be retried. 1522 * The async thread will update id_ah_op with an error indication 1523 * or will set it to indicate the next look up can start; either 1524 * way, it will mac_tx_update() so that all blocked requests come 1525 * back here. 1526 */ 1527 *err = EAGAIN; 1528 if (state->id_ah_op == IBD_OP_NOTSTARTED) { 1529 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); 1530 if (req != NULL) { 1531 /* 1532 * We did not even find the entry; queue a request 1533 * for it. 1534 */ 1535 bcopy(mac, &(req->rq_mac), IPOIB_ADDRL); 1536 state->id_ah_op = IBD_OP_ONGOING; 1537 ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH); 1538 bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL); 1539 } 1540 } else if ((state->id_ah_op != IBD_OP_ONGOING) && 1541 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) { 1542 /* 1543 * Check the status of the pathrecord lookup request 1544 * we had queued before. 1545 */ 1546 if (state->id_ah_op == IBD_OP_ERRORED) { 1547 *err = EFAULT; 1548 state->id_ah_error++; 1549 } else { 1550 /* 1551 * IBD_OP_ROUTERED case: We need to send to the 1552 * all-router MCG. If we can find the AH for 1553 * the mcg, the Tx will be attempted. If we 1554 * do not find the AH, we return NORESOURCES 1555 * to retry. 1556 */ 1557 ipoib_mac_t routermac; 1558 1559 (void) ibd_get_allroutergroup(state, mac, &routermac); 1560 ptr = ibd_acache_find(state, &routermac, B_TRUE, 1561 numwqe); 1562 } 1563 state->id_ah_op = IBD_OP_NOTSTARTED; 1564 } else if ((state->id_ah_op != IBD_OP_ONGOING) && 1565 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) { 1566 /* 1567 * This case can happen when we get a higher band 1568 * packet. The easiest way is to reset the state machine 1569 * to accommodate the higher priority packet. 1570 */ 1571 state->id_ah_op = IBD_OP_NOTSTARTED; 1572 } 1573 mutex_exit(&state->id_ac_mutex); 1574 1575 return (ptr); 1576 } 1577 1578 /* 1579 * Grab a not-currently-in-use AH/PathRecord from the active 1580 * list to recycle to a new destination. Only the async thread 1581 * executes this code. 1582 */ 1583 static ibd_ace_t * 1584 ibd_acache_get_unref(ibd_state_t *state) 1585 { 1586 ibd_ace_t *ptr = list_tail(&state->id_ah_active); 1587 boolean_t try_rc_chan_recycle = B_FALSE; 1588 1589 ASSERT(mutex_owned(&state->id_ac_mutex)); 1590 1591 /* 1592 * Do plain linear search. 1593 */ 1594 while (ptr != NULL) { 1595 /* 1596 * Note that it is possible that the "cycle" bit 1597 * is set on the AH w/o any reference count. The 1598 * mcg must have been deleted, and the tx cleanup 1599 * just decremented the reference count to 0, but 1600 * hasn't gotten around to grabbing the id_ac_mutex 1601 * to move the AH into the free list. 1602 */ 1603 if (GET_REF(ptr) == 0) { 1604 if (ptr->ac_chan != NULL) { 1605 ASSERT(state->id_enable_rc == B_TRUE); 1606 if (!try_rc_chan_recycle) { 1607 try_rc_chan_recycle = B_TRUE; 1608 ibd_rc_signal_ace_recycle(state, ptr); 1609 } 1610 } else { 1611 IBD_ACACHE_PULLOUT_ACTIVE(state, ptr); 1612 break; 1613 } 1614 } 1615 ptr = list_prev(&state->id_ah_active, ptr); 1616 } 1617 return (ptr); 1618 } 1619 1620 /* 1621 * Invoked to clean up AH from active list in case of multicast 1622 * disable and to handle sendonly memberships during mcg traps. 1623 * And for port up processing for multicast and unicast AHs. 1624 * Normally, the AH is taken off the active list, and put into 1625 * the free list to be recycled for a new destination. In case 1626 * Tx requests on the AH have not completed yet, the AH is marked 1627 * for reaping (which will put the AH on the free list) once the Tx's 1628 * complete; in this case, depending on the "force" input, we take 1629 * out the AH from the active list right now, or leave it also for 1630 * the reap operation. Returns TRUE if the AH is taken off the active 1631 * list (and either put into the free list right now, or arranged for 1632 * later), FALSE otherwise. 1633 */ 1634 boolean_t 1635 ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force) 1636 { 1637 ibd_ace_t *acactive; 1638 boolean_t ret = B_TRUE; 1639 1640 ASSERT(mutex_owned(&state->id_ac_mutex)); 1641 1642 if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) { 1643 1644 /* 1645 * Note that the AH might already have the cycle bit set 1646 * on it; this might happen if sequences of multicast 1647 * enables and disables are coming so fast, that posted 1648 * Tx's to the mcg have not completed yet, and the cycle 1649 * bit is set successively by each multicast disable. 1650 */ 1651 if (SET_CYCLE_IF_REF(acactive)) { 1652 if (!force) { 1653 /* 1654 * The ace is kept on the active list, further 1655 * Tx's can still grab a reference on it; the 1656 * ace is reaped when all pending Tx's 1657 * referencing the AH complete. 1658 */ 1659 ret = B_FALSE; 1660 } else { 1661 /* 1662 * In the mcg trap case, we always pull the 1663 * AH from the active list. And also the port 1664 * up multi/unicast case. 1665 */ 1666 ASSERT(acactive->ac_chan == NULL); 1667 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive); 1668 acactive->ac_mce = NULL; 1669 } 1670 } else { 1671 /* 1672 * Determined the ref count is 0, thus reclaim 1673 * immediately after pulling out the ace from 1674 * the active list. 1675 */ 1676 ASSERT(acactive->ac_chan == NULL); 1677 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive); 1678 acactive->ac_mce = NULL; 1679 IBD_ACACHE_INSERT_FREE(state, acactive); 1680 } 1681 1682 } 1683 return (ret); 1684 } 1685 1686 /* 1687 * Helper function for async path record lookup. If we are trying to 1688 * Tx to a MCG, check our membership, possibly trying to join the 1689 * group if required. If that fails, try to send the packet to the 1690 * all router group (indicated by the redirect output), pointing 1691 * the input mac address to the router mcg address. 1692 */ 1693 static ibd_mce_t * 1694 ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect) 1695 { 1696 ib_gid_t mgid; 1697 ibd_mce_t *mce; 1698 ipoib_mac_t routermac; 1699 1700 *redirect = B_FALSE; 1701 ibd_n2h_gid(mac, &mgid); 1702 1703 /* 1704 * Check the FullMember+SendOnlyNonMember list. 1705 * Since we are the only one who manipulates the 1706 * id_mc_full list, no locks are needed. 1707 */ 1708 mce = IBD_MCACHE_FIND_FULL(state, mgid); 1709 if (mce != NULL) { 1710 DPRINT(4, "ibd_async_mcache : already joined to group"); 1711 return (mce); 1712 } 1713 1714 /* 1715 * Not found; try to join(SendOnlyNonMember) and attach. 1716 */ 1717 DPRINT(4, "ibd_async_mcache : not joined to group"); 1718 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) != 1719 NULL) { 1720 DPRINT(4, "ibd_async_mcache : nonmem joined to group"); 1721 return (mce); 1722 } 1723 1724 /* 1725 * MCGroup not present; try to join the all-router group. If 1726 * any of the following steps succeed, we will be redirecting 1727 * to the all router group. 1728 */ 1729 DPRINT(4, "ibd_async_mcache : nonmem join failed"); 1730 if (!ibd_get_allroutergroup(state, mac, &routermac)) 1731 return (NULL); 1732 *redirect = B_TRUE; 1733 ibd_n2h_gid(&routermac, &mgid); 1734 bcopy(&routermac, mac, IPOIB_ADDRL); 1735 DPRINT(4, "ibd_async_mcache : router mgid : %016llx:%016llx\n", 1736 mgid.gid_prefix, mgid.gid_guid); 1737 1738 /* 1739 * Are we already joined to the router group? 1740 */ 1741 if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) { 1742 DPRINT(4, "ibd_async_mcache : using already joined router" 1743 "group\n"); 1744 return (mce); 1745 } 1746 1747 /* 1748 * Can we join(SendOnlyNonMember) the router group? 1749 */ 1750 DPRINT(4, "ibd_async_mcache : attempting join to router grp"); 1751 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) != 1752 NULL) { 1753 DPRINT(4, "ibd_async_mcache : joined to router grp"); 1754 return (mce); 1755 } 1756 1757 return (NULL); 1758 } 1759 1760 /* 1761 * Async path record lookup code. 1762 */ 1763 static void 1764 ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac) 1765 { 1766 ibd_ace_t *ce; 1767 ibd_mce_t *mce = NULL; 1768 ibt_path_attr_t path_attr; 1769 ibt_path_info_t path_info; 1770 ib_gid_t destgid; 1771 char ret = IBD_OP_NOTSTARTED; 1772 1773 DPRINT(4, "ibd_async_acache : %08X:%08X:%08X:%08X:%08X", 1774 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]), 1775 htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]), 1776 htonl(mac->ipoib_gidsuff[1])); 1777 1778 /* 1779 * Check whether we are trying to transmit to a MCG. 1780 * In that case, we need to make sure we are a member of 1781 * the MCG. 1782 */ 1783 if (mac->ipoib_qpn == htonl(IB_MC_QPN)) { 1784 boolean_t redirected; 1785 1786 /* 1787 * If we can not find or join the group or even 1788 * redirect, error out. 1789 */ 1790 if ((mce = ibd_async_mcache(state, mac, &redirected)) == 1791 NULL) { 1792 state->id_ah_op = IBD_OP_ERRORED; 1793 return; 1794 } 1795 1796 /* 1797 * If we got redirected, we need to determine whether 1798 * the AH for the new mcg is in the cache already, and 1799 * not pull it in then; otherwise proceed to get the 1800 * path for the new mcg. There is no guarantee that 1801 * if the AH is currently in the cache, it will still be 1802 * there when we look in ibd_acache_lookup(), but that's 1803 * okay, we will come back here. 1804 */ 1805 if (redirected) { 1806 ret = IBD_OP_ROUTERED; 1807 DPRINT(4, "ibd_async_acache : redirected to " 1808 "%08X:%08X:%08X:%08X:%08X", 1809 htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]), 1810 htonl(mac->ipoib_gidpref[1]), 1811 htonl(mac->ipoib_gidsuff[0]), 1812 htonl(mac->ipoib_gidsuff[1])); 1813 1814 mutex_enter(&state->id_ac_mutex); 1815 if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) { 1816 state->id_ah_op = IBD_OP_ROUTERED; 1817 mutex_exit(&state->id_ac_mutex); 1818 DPRINT(4, "ibd_async_acache : router AH found"); 1819 return; 1820 } 1821 mutex_exit(&state->id_ac_mutex); 1822 } 1823 } 1824 1825 /* 1826 * Get an AH from the free list. 1827 */ 1828 mutex_enter(&state->id_ac_mutex); 1829 if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) { 1830 /* 1831 * No free ones; try to grab an unreferenced active 1832 * one. Maybe we need to make the active list LRU, 1833 * but that will create more work for Tx callbacks. 1834 * Is there a way of not having to pull out the 1835 * entry from the active list, but just indicate it 1836 * is being recycled? Yes, but that creates one more 1837 * check in the fast lookup path. 1838 */ 1839 if ((ce = ibd_acache_get_unref(state)) == NULL) { 1840 /* 1841 * Pretty serious shortage now. 1842 */ 1843 state->id_ah_op = IBD_OP_NOTSTARTED; 1844 mutex_exit(&state->id_ac_mutex); 1845 DPRINT(10, "ibd_async_acache : failed to find AH " 1846 "slot\n"); 1847 return; 1848 } 1849 /* 1850 * We could check whether ac_mce points to a SendOnly 1851 * member and drop that membership now. Or do it lazily 1852 * at detach time. 1853 */ 1854 ce->ac_mce = NULL; 1855 } 1856 mutex_exit(&state->id_ac_mutex); 1857 ASSERT(ce->ac_mce == NULL); 1858 1859 /* 1860 * Update the entry. 1861 */ 1862 bcopy((char *)mac, &ce->ac_mac, IPOIB_ADDRL); 1863 1864 bzero(&path_info, sizeof (path_info)); 1865 bzero(&path_attr, sizeof (ibt_path_attr_t)); 1866 path_attr.pa_sgid = state->id_sgid; 1867 path_attr.pa_num_dgids = 1; 1868 ibd_n2h_gid(&ce->ac_mac, &destgid); 1869 path_attr.pa_dgids = &destgid; 1870 path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl; 1871 path_attr.pa_pkey = state->id_pkey; 1872 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_PKEY, &path_attr, 1, 1873 &path_info, NULL) != IBT_SUCCESS) { 1874 DPRINT(10, "ibd_async_acache : failed in ibt_get_paths"); 1875 goto error; 1876 } 1877 if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey, 1878 ntohl(ce->ac_mac.ipoib_qpn), 1879 &path_info.pi_prim_cep_path.cep_adds_vect) != IBT_SUCCESS) { 1880 DPRINT(10, "ibd_async_acache : failed in ibt_modify_ud_dest"); 1881 goto error; 1882 } 1883 1884 /* 1885 * mce is set whenever an AH is being associated with a 1886 * MCG; this will come in handy when we leave the MCG. The 1887 * lock protects Tx fastpath from scanning the active list. 1888 */ 1889 if (mce != NULL) 1890 ce->ac_mce = mce; 1891 1892 /* 1893 * initiate a RC mode connection for unicast address 1894 */ 1895 if (state->id_enable_rc && (mac->ipoib_qpn != htonl(IB_MC_QPN)) && 1896 (htonl(mac->ipoib_qpn) & IBD_MAC_ADDR_RC)) { 1897 ASSERT(ce->ac_chan == NULL); 1898 DPRINT(10, "ibd_async_acache: call " 1899 "ibd_rc_try_connect(ace=%p)", ce); 1900 ibd_rc_try_connect(state, ce, &path_info); 1901 if (ce->ac_chan == NULL) { 1902 DPRINT(10, "ibd_async_acache: fail to setup RC" 1903 " channel"); 1904 state->rc_conn_fail++; 1905 goto error; 1906 } 1907 } 1908 1909 mutex_enter(&state->id_ac_mutex); 1910 IBD_ACACHE_INSERT_ACTIVE(state, ce); 1911 state->id_ah_op = ret; 1912 mutex_exit(&state->id_ac_mutex); 1913 return; 1914 error: 1915 /* 1916 * We might want to drop SendOnly membership here if we 1917 * joined above. The lock protects Tx callbacks inserting 1918 * into the free list. 1919 */ 1920 mutex_enter(&state->id_ac_mutex); 1921 state->id_ah_op = IBD_OP_ERRORED; 1922 IBD_ACACHE_INSERT_FREE(state, ce); 1923 mutex_exit(&state->id_ac_mutex); 1924 } 1925 1926 /* 1927 * While restoring port's presence on the subnet on a port up, it is possible 1928 * that the port goes down again. 1929 */ 1930 static void 1931 ibd_async_link(ibd_state_t *state, ibd_req_t *req) 1932 { 1933 ibd_link_op_t opcode = (ibd_link_op_t)req->rq_ptr; 1934 link_state_t lstate = (opcode == IBD_LINK_DOWN) ? LINK_STATE_DOWN : 1935 LINK_STATE_UP; 1936 ibd_mce_t *mce, *pmce; 1937 ibd_ace_t *ace, *pace; 1938 1939 DPRINT(10, "ibd_async_link(): %d", opcode); 1940 1941 /* 1942 * On a link up, revalidate the link speed/width. No point doing 1943 * this on a link down, since we will be unable to do SA operations, 1944 * defaulting to the lowest speed. Also notice that we update our 1945 * notion of speed before calling mac_link_update(), which will do 1946 * necessary higher level notifications for speed changes. 1947 */ 1948 if ((opcode == IBD_LINK_UP_ABSENT) || (opcode == IBD_LINK_UP)) { 1949 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state)) 1950 state->id_link_speed = ibd_get_portspeed(state); 1951 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state)) 1952 } 1953 1954 /* 1955 * Do all the work required to establish our presence on 1956 * the subnet. 1957 */ 1958 if (opcode == IBD_LINK_UP_ABSENT) { 1959 /* 1960 * If in promiscuous mode ... 1961 */ 1962 if (state->id_prom_op == IBD_OP_COMPLETED) { 1963 /* 1964 * Drop all nonmembership. 1965 */ 1966 ibd_async_unsetprom(state); 1967 1968 /* 1969 * Then, try to regain nonmembership to all mcg's. 1970 */ 1971 ibd_async_setprom(state); 1972 1973 } 1974 1975 /* 1976 * Drop all sendonly membership (which also gets rid of the 1977 * AHs); try to reacquire all full membership. 1978 */ 1979 mce = list_head(&state->id_mc_full); 1980 while ((pmce = mce) != NULL) { 1981 mce = list_next(&state->id_mc_full, mce); 1982 if (pmce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON) 1983 ibd_leave_group(state, 1984 pmce->mc_info.mc_adds_vect.av_dgid, 1985 IB_MC_JSTATE_SEND_ONLY_NON); 1986 else 1987 ibd_reacquire_group(state, pmce); 1988 } 1989 1990 /* 1991 * Recycle all active AHs to free list (and if there are 1992 * pending posts, make sure they will go into the free list 1993 * once the Tx's complete). Grab the lock to prevent 1994 * concurrent Tx's as well as Tx cleanups. 1995 */ 1996 mutex_enter(&state->id_ac_mutex); 1997 ace = list_head(&state->id_ah_active); 1998 while ((pace = ace) != NULL) { 1999 boolean_t cycled; 2000 2001 ace = list_next(&state->id_ah_active, ace); 2002 mce = pace->ac_mce; 2003 if (pace->ac_chan != NULL) { 2004 ASSERT(mce == NULL); 2005 ASSERT(state->id_enable_rc == B_TRUE); 2006 if (pace->ac_chan->chan_state == 2007 IBD_RC_STATE_ACT_ESTAB) { 2008 INC_REF(pace, 1); 2009 IBD_ACACHE_PULLOUT_ACTIVE(state, pace); 2010 pace->ac_chan->chan_state = 2011 IBD_RC_STATE_ACT_CLOSING; 2012 ibd_rc_signal_act_close(state, pace); 2013 } else { 2014 state->rc_act_close_simultaneous++; 2015 DPRINT(40, "ibd_async_link: other " 2016 "thread is closing it, ace=%p, " 2017 "ac_chan=%p, chan_state=%d", 2018 pace, pace->ac_chan, 2019 pace->ac_chan->chan_state); 2020 } 2021 } else { 2022 cycled = ibd_acache_recycle(state, 2023 &pace->ac_mac, B_TRUE); 2024 } 2025 /* 2026 * If this is for an mcg, it must be for a fullmember, 2027 * since we got rid of send-only members above when 2028 * processing the mce list. 2029 */ 2030 ASSERT(cycled && ((mce == NULL) || (mce->mc_jstate == 2031 IB_MC_JSTATE_FULL))); 2032 2033 /* 2034 * Check if the fullmember mce needs to be torn down, 2035 * ie whether the DLPI disable has already been done. 2036 * If so, do some of the work of tx_cleanup, namely 2037 * causing leave (which will fail), detach and 2038 * mce-freeing. tx_cleanup will put the AH into free 2039 * list. The reason to duplicate some of this 2040 * tx_cleanup work is because we want to delete the 2041 * AH right now instead of waiting for tx_cleanup, to 2042 * force subsequent Tx's to reacquire an AH. 2043 */ 2044 if ((mce != NULL) && (mce->mc_fullreap)) 2045 ibd_async_reap_group(state, mce, 2046 mce->mc_info.mc_adds_vect.av_dgid, 2047 mce->mc_jstate); 2048 } 2049 mutex_exit(&state->id_ac_mutex); 2050 } 2051 2052 /* 2053 * mac handle is guaranteed to exist since driver does ibt_close_hca() 2054 * (which stops further events from being delivered) before 2055 * mac_unregister(). At this point, it is guaranteed that mac_register 2056 * has already been done. 2057 */ 2058 mutex_enter(&state->id_link_mutex); 2059 state->id_link_state = lstate; 2060 mac_link_update(state->id_mh, lstate); 2061 mutex_exit(&state->id_link_mutex); 2062 2063 ibd_async_done(state); 2064 } 2065 2066 /* 2067 * Check the pkey table to see if we can find the pkey we're looking for. 2068 * Set the pkey index in 'pkix' if found. Return 0 on success and -1 on 2069 * failure. 2070 */ 2071 static int 2072 ibd_locate_pkey(ib_pkey_t *pkey_tbl, uint16_t pkey_tbl_sz, ib_pkey_t pkey, 2073 uint16_t *pkix) 2074 { 2075 uint16_t ndx; 2076 2077 ASSERT(pkix != NULL); 2078 2079 for (ndx = 0; ndx < pkey_tbl_sz; ndx++) { 2080 if (pkey_tbl[ndx] == pkey) { 2081 *pkix = ndx; 2082 return (0); 2083 } 2084 } 2085 return (-1); 2086 } 2087 2088 /* 2089 * Late HCA Initialization: 2090 * If plumb had succeeded without the availability of an active port or the 2091 * pkey, and either of their availability is now being indicated via PORT_UP 2092 * or PORT_CHANGE respectively, try a start of the interface. 2093 * 2094 * Normal Operation: 2095 * When the link is notified up, we need to do a few things, based 2096 * on the port's current p_init_type_reply claiming a reinit has been 2097 * done or not. The reinit steps are: 2098 * 1. If in InitTypeReply, NoLoadReply == PreserveContentReply == 0, verify 2099 * the old Pkey and GID0 are correct. 2100 * 2. Register for mcg traps (already done by ibmf). 2101 * 3. If PreservePresenceReply indicates the SM has restored port's presence 2102 * in subnet, nothing more to do. Else go to next steps (on async daemon). 2103 * 4. Give up all sendonly memberships. 2104 * 5. Acquire all full memberships. 2105 * 6. In promiscuous mode, acquire all non memberships. 2106 * 7. Recycle all AHs to free list. 2107 */ 2108 static void 2109 ibd_link_mod(ibd_state_t *state, ibt_async_code_t code) 2110 { 2111 ibt_hca_portinfo_t *port_infop = NULL; 2112 ibt_status_t ibt_status; 2113 uint_t psize, port_infosz; 2114 ibd_link_op_t opcode; 2115 ibd_req_t *req; 2116 link_state_t new_link_state = LINK_STATE_UP; 2117 uint8_t itreply; 2118 uint16_t pkix; 2119 int ret; 2120 2121 /* 2122 * Let's not race with a plumb or an unplumb; if we detect a 2123 * pkey relocation event later on here, we may have to restart. 2124 */ 2125 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); 2126 2127 mutex_enter(&state->id_link_mutex); 2128 2129 /* 2130 * If the link state is unknown, a plumb has not yet been attempted 2131 * on the interface. Nothing to do. 2132 */ 2133 if (state->id_link_state == LINK_STATE_UNKNOWN) { 2134 mutex_exit(&state->id_link_mutex); 2135 goto link_mod_return; 2136 } 2137 2138 /* 2139 * If link state is down because of plumb failure, and we are not in 2140 * late HCA init, and we were not successfully plumbed, nothing to do. 2141 */ 2142 if ((state->id_link_state == LINK_STATE_DOWN) && 2143 ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 0) && 2144 ((state->id_mac_state & IBD_DRV_STARTED) == 0)) { 2145 mutex_exit(&state->id_link_mutex); 2146 goto link_mod_return; 2147 } 2148 2149 /* 2150 * If this routine was called in response to a port down event, 2151 * we just need to see if this should be informed. 2152 */ 2153 if (code == IBT_ERROR_PORT_DOWN) { 2154 new_link_state = LINK_STATE_DOWN; 2155 goto update_link_state; 2156 } 2157 2158 /* 2159 * If it's not a port down event we've received, try to get the port 2160 * attributes first. If we fail here, the port is as good as down. 2161 * Otherwise, if the link went down by the time the handler gets 2162 * here, give up - we cannot even validate the pkey/gid since those 2163 * are not valid and this is as bad as a port down anyway. 2164 */ 2165 ibt_status = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, 2166 &port_infop, &psize, &port_infosz); 2167 if ((ibt_status != IBT_SUCCESS) || (psize != 1) || 2168 (port_infop->p_linkstate != IBT_PORT_ACTIVE)) { 2169 new_link_state = LINK_STATE_DOWN; 2170 goto update_link_state; 2171 } 2172 2173 /* 2174 * If in the previous attempt, the pkey was not found either due to the 2175 * port state being down, or due to it's absence in the pkey table, 2176 * look for it now and try to start the interface. 2177 */ 2178 if (state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) { 2179 mutex_exit(&state->id_link_mutex); 2180 if ((ret = ibd_start(state)) != 0) { 2181 DPRINT(10, "ibd_linkmod: cannot start from late HCA " 2182 "init, ret=%d", ret); 2183 } 2184 ibt_free_portinfo(port_infop, port_infosz); 2185 goto link_mod_return; 2186 } 2187 2188 /* 2189 * Check the SM InitTypeReply flags. If both NoLoadReply and 2190 * PreserveContentReply are 0, we don't know anything about the 2191 * data loaded into the port attributes, so we need to verify 2192 * if gid0 and pkey are still valid. 2193 */ 2194 itreply = port_infop->p_init_type_reply; 2195 if (((itreply & SM_INIT_TYPE_REPLY_NO_LOAD_REPLY) == 0) && 2196 ((itreply & SM_INIT_TYPE_PRESERVE_CONTENT_REPLY) == 0)) { 2197 /* 2198 * Check to see if the subnet part of GID0 has changed. If 2199 * not, check the simple case first to see if the pkey 2200 * index is the same as before; finally check to see if the 2201 * pkey has been relocated to a different index in the table. 2202 */ 2203 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) 2204 if (bcmp(port_infop->p_sgid_tbl, 2205 &state->id_sgid, sizeof (ib_gid_t)) != 0) { 2206 2207 new_link_state = LINK_STATE_DOWN; 2208 2209 } else if (port_infop->p_pkey_tbl[state->id_pkix] == 2210 state->id_pkey) { 2211 2212 new_link_state = LINK_STATE_UP; 2213 2214 } else if (ibd_locate_pkey(port_infop->p_pkey_tbl, 2215 port_infop->p_pkey_tbl_sz, state->id_pkey, &pkix) == 0) { 2216 2217 ibt_free_portinfo(port_infop, port_infosz); 2218 mutex_exit(&state->id_link_mutex); 2219 2220 /* 2221 * Currently a restart is required if our pkey has moved 2222 * in the pkey table. If we get the ibt_recycle_ud() to 2223 * work as documented (expected), we may be able to 2224 * avoid a complete restart. Note that we've already 2225 * marked both the start and stop 'in-progress' flags, 2226 * so it is ok to go ahead and do this restart. 2227 */ 2228 (void) ibd_undo_start(state, LINK_STATE_DOWN); 2229 if ((ret = ibd_start(state)) != 0) { 2230 DPRINT(10, "ibd_restart: cannot restart, " 2231 "ret=%d", ret); 2232 } 2233 2234 goto link_mod_return; 2235 } else { 2236 new_link_state = LINK_STATE_DOWN; 2237 } 2238 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) 2239 } 2240 2241 update_link_state: 2242 if (port_infop) { 2243 ibt_free_portinfo(port_infop, port_infosz); 2244 } 2245 2246 /* 2247 * If we're reporting a link up, check InitTypeReply to see if 2248 * the SM has ensured that the port's presence in mcg, traps, 2249 * etc. is intact. 2250 */ 2251 if (new_link_state == LINK_STATE_DOWN) { 2252 opcode = IBD_LINK_DOWN; 2253 } else { 2254 if ((itreply & SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) == 2255 SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) { 2256 opcode = IBD_LINK_UP; 2257 } else { 2258 opcode = IBD_LINK_UP_ABSENT; 2259 } 2260 } 2261 2262 /* 2263 * If the old state is the same as the new state, and the SM indicated 2264 * no change in the port parameters, nothing to do. 2265 */ 2266 if ((state->id_link_state == new_link_state) && (opcode != 2267 IBD_LINK_UP_ABSENT)) { 2268 mutex_exit(&state->id_link_mutex); 2269 goto link_mod_return; 2270 } 2271 2272 /* 2273 * Ok, so there was a link state change; see if it's safe to ask 2274 * the async thread to do the work 2275 */ 2276 if (!ibd_async_safe(state)) { 2277 state->id_link_state = new_link_state; 2278 mutex_exit(&state->id_link_mutex); 2279 goto link_mod_return; 2280 } 2281 2282 mutex_exit(&state->id_link_mutex); 2283 2284 /* 2285 * Queue up a request for ibd_async_link() to handle this link 2286 * state change event 2287 */ 2288 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP); 2289 req->rq_ptr = (void *)opcode; 2290 ibd_queue_work_slot(state, req, IBD_ASYNC_LINK); 2291 2292 link_mod_return: 2293 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); 2294 } 2295 2296 /* 2297 * For the port up/down events, IBTL guarantees there will not be concurrent 2298 * invocations of the handler. IBTL might coalesce link transition events, 2299 * and not invoke the handler for _each_ up/down transition, but it will 2300 * invoke the handler with last known state 2301 */ 2302 static void 2303 ibd_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 2304 ibt_async_code_t code, ibt_async_event_t *event) 2305 { 2306 ibd_state_t *state = (ibd_state_t *)clnt_private; 2307 2308 switch (code) { 2309 case IBT_ERROR_CATASTROPHIC_CHAN: 2310 ibd_print_warn(state, "catastrophic channel error"); 2311 break; 2312 case IBT_ERROR_CQ: 2313 ibd_print_warn(state, "completion queue error"); 2314 break; 2315 case IBT_PORT_CHANGE_EVENT: 2316 /* 2317 * Events will be delivered to all instances that have 2318 * done ibt_open_hca() but not yet done ibt_close_hca(). 2319 * Only need to do work for our port; IBTF will deliver 2320 * events for other ports on the hca we have ibt_open_hca'ed 2321 * too. Note that id_port is initialized in ibd_attach() 2322 * before we do an ibt_open_hca() in ibd_attach(). 2323 */ 2324 ASSERT(state->id_hca_hdl == hca_hdl); 2325 if (state->id_port != event->ev_port) 2326 break; 2327 2328 if ((event->ev_port_flags & IBT_PORT_CHANGE_PKEY) == 2329 IBT_PORT_CHANGE_PKEY) { 2330 ibd_link_mod(state, code); 2331 } 2332 break; 2333 case IBT_ERROR_PORT_DOWN: 2334 case IBT_CLNT_REREG_EVENT: 2335 case IBT_EVENT_PORT_UP: 2336 /* 2337 * Events will be delivered to all instances that have 2338 * done ibt_open_hca() but not yet done ibt_close_hca(). 2339 * Only need to do work for our port; IBTF will deliver 2340 * events for other ports on the hca we have ibt_open_hca'ed 2341 * too. Note that id_port is initialized in ibd_attach() 2342 * before we do an ibt_open_hca() in ibd_attach(). 2343 */ 2344 ASSERT(state->id_hca_hdl == hca_hdl); 2345 if (state->id_port != event->ev_port) 2346 break; 2347 2348 ibd_link_mod(state, code); 2349 break; 2350 2351 case IBT_HCA_ATTACH_EVENT: 2352 case IBT_HCA_DETACH_EVENT: 2353 /* 2354 * When a new card is plugged to the system, attach_event is 2355 * invoked. Additionally, a cfgadm needs to be run to make the 2356 * card known to the system, and an ifconfig needs to be run to 2357 * plumb up any ibd interfaces on the card. In the case of card 2358 * unplug, a cfgadm is run that will trigger any RCM scripts to 2359 * unplumb the ibd interfaces on the card; when the card is 2360 * actually unplugged, the detach_event is invoked; 2361 * additionally, if any ibd instances are still active on the 2362 * card (eg there were no associated RCM scripts), driver's 2363 * detach routine is invoked. 2364 */ 2365 break; 2366 default: 2367 break; 2368 } 2369 } 2370 2371 static int 2372 ibd_register_mac(ibd_state_t *state, dev_info_t *dip) 2373 { 2374 mac_register_t *macp; 2375 int ret; 2376 2377 if ((macp = mac_alloc(MAC_VERSION)) == NULL) { 2378 DPRINT(10, "ibd_register_mac: mac_alloc() failed"); 2379 return (DDI_FAILURE); 2380 } 2381 2382 /* 2383 * Note that when we register with mac during attach, we don't 2384 * have the id_macaddr yet, so we'll simply be registering a 2385 * zero macaddr that we'll overwrite later during plumb (in 2386 * ibd_m_start()). Similar is the case with id_mtu - we'll 2387 * update the mac layer with the correct mtu during plumb. 2388 */ 2389 macp->m_type_ident = MAC_PLUGIN_IDENT_IB; 2390 macp->m_driver = state; 2391 macp->m_dip = dip; 2392 macp->m_src_addr = (uint8_t *)&state->id_macaddr; 2393 macp->m_callbacks = &ibd_m_callbacks; 2394 macp->m_min_sdu = 0; 2395 macp->m_multicast_sdu = IBD_DEF_MAX_SDU; 2396 if (state->id_type == IBD_PORT_DRIVER) { 2397 macp->m_max_sdu = IBD_DEF_RC_MAX_SDU; 2398 } else if (state->id_enable_rc) { 2399 macp->m_max_sdu = state->rc_mtu - IPOIB_HDRSIZE; 2400 } else { 2401 macp->m_max_sdu = IBD_DEF_MAX_SDU; 2402 } 2403 macp->m_priv_props = ibd_priv_props; 2404 2405 /* 2406 * Register ourselves with the GLDv3 interface 2407 */ 2408 if ((ret = mac_register(macp, &state->id_mh)) != 0) { 2409 mac_free(macp); 2410 DPRINT(10, 2411 "ibd_register_mac: mac_register() failed, ret=%d", ret); 2412 return (DDI_FAILURE); 2413 } 2414 2415 mac_free(macp); 2416 return (DDI_SUCCESS); 2417 } 2418 2419 static int 2420 ibd_record_capab(ibd_state_t *state) 2421 { 2422 ibt_hca_attr_t hca_attrs; 2423 ibt_status_t ibt_status; 2424 2425 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state)) 2426 2427 /* 2428 * Query the HCA and fetch its attributes 2429 */ 2430 ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs); 2431 ASSERT(ibt_status == IBT_SUCCESS); 2432 2433 /* 2434 * 1. Set the Hardware Checksum capability. Currently we only consider 2435 * full checksum offload. 2436 */ 2437 if (state->id_enable_rc) { 2438 state->id_hwcksum_capab = 0; 2439 } else { 2440 if ((hca_attrs.hca_flags & IBT_HCA_CKSUM_FULL) 2441 == IBT_HCA_CKSUM_FULL) { 2442 state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL; 2443 } 2444 } 2445 2446 /* 2447 * 2. Set LSO policy, capability and maximum length 2448 */ 2449 if (state->id_enable_rc) { 2450 state->id_lso_capable = B_FALSE; 2451 state->id_lso_maxlen = 0; 2452 } else { 2453 if (hca_attrs.hca_max_lso_size > 0) { 2454 state->id_lso_capable = B_TRUE; 2455 if (hca_attrs.hca_max_lso_size > IBD_LSO_MAXLEN) 2456 state->id_lso_maxlen = IBD_LSO_MAXLEN; 2457 else 2458 state->id_lso_maxlen = 2459 hca_attrs.hca_max_lso_size; 2460 } else { 2461 state->id_lso_capable = B_FALSE; 2462 state->id_lso_maxlen = 0; 2463 } 2464 } 2465 2466 /* 2467 * 3. Set Reserved L_Key capability 2468 */ 2469 if ((hca_attrs.hca_flags2 & IBT_HCA2_RES_LKEY) == IBT_HCA2_RES_LKEY) { 2470 state->id_hca_res_lkey_capab = 1; 2471 state->id_res_lkey = hca_attrs.hca_reserved_lkey; 2472 state->rc_enable_iov_map = B_TRUE; 2473 } else { 2474 /* If no reserved lkey, we will not use ibt_map_mem_iov */ 2475 state->rc_enable_iov_map = B_FALSE; 2476 } 2477 2478 /* 2479 * 4. Set maximum sqseg value after checking to see if extended sgl 2480 * size information is provided by the hca 2481 */ 2482 if (hca_attrs.hca_flags & IBT_HCA_WQE_SIZE_INFO) { 2483 state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz; 2484 state->rc_tx_max_sqseg = hca_attrs.hca_conn_send_sgl_sz; 2485 } else { 2486 state->id_max_sqseg = hca_attrs.hca_max_sgl; 2487 state->rc_tx_max_sqseg = hca_attrs.hca_max_sgl; 2488 } 2489 if (state->id_max_sqseg > IBD_MAX_SQSEG) { 2490 state->id_max_sqseg = IBD_MAX_SQSEG; 2491 } else if (state->id_max_sqseg < IBD_MAX_SQSEG) { 2492 ibd_print_warn(state, "Set #sgl = %d instead of default %d", 2493 state->id_max_sqseg, IBD_MAX_SQSEG); 2494 } 2495 if (state->rc_tx_max_sqseg > IBD_MAX_SQSEG) { 2496 state->rc_tx_max_sqseg = IBD_MAX_SQSEG; 2497 } else if (state->rc_tx_max_sqseg < IBD_MAX_SQSEG) { 2498 ibd_print_warn(state, "RC mode: Set #sgl = %d instead of " 2499 "default %d", state->rc_tx_max_sqseg, IBD_MAX_SQSEG); 2500 } 2501 2502 /* 2503 * Translating the virtual address regions into physical regions 2504 * for using the Reserved LKey feature results in a wr sgl that 2505 * is a little longer. Since failing ibt_map_mem_iov() is costly, 2506 * we'll fix a high-water mark (65%) for when we should stop. 2507 */ 2508 state->id_max_sqseg_hiwm = (state->id_max_sqseg * 65) / 100; 2509 state->rc_max_sqseg_hiwm = (state->rc_tx_max_sqseg * 65) / 100; 2510 2511 /* 2512 * 5. Set number of recv and send wqes after checking hca maximum 2513 * channel size. Store the max channel size in the state so that it 2514 * can be referred to when the swqe/rwqe change is requested via 2515 * dladm. 2516 */ 2517 2518 state->id_hca_max_chan_sz = hca_attrs.hca_max_chan_sz; 2519 2520 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_rwqe) 2521 state->id_ud_num_rwqe = hca_attrs.hca_max_chan_sz; 2522 2523 state->id_rx_bufs_outstanding_limit = state->id_ud_num_rwqe - 2524 IBD_RWQE_MIN; 2525 2526 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_swqe) 2527 state->id_ud_num_swqe = hca_attrs.hca_max_chan_sz; 2528 2529 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state)) 2530 2531 return (DDI_SUCCESS); 2532 } 2533 2534 static int 2535 ibd_part_busy(ibd_state_t *state) 2536 { 2537 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) != 0) { 2538 DPRINT(10, "ibd_part_busy: failed: rx bufs outstanding\n"); 2539 return (DDI_FAILURE); 2540 } 2541 2542 if (state->rc_srq_rwqe_list.dl_bufs_outstanding != 0) { 2543 DPRINT(10, "ibd_part_busy: failed: srq bufs outstanding\n"); 2544 return (DDI_FAILURE); 2545 } 2546 2547 /* 2548 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB port is 2549 * connecting to a remote IPoIB port. We can't remove this port. 2550 */ 2551 if (state->id_ah_op == IBD_OP_ONGOING) { 2552 DPRINT(10, "ibd_part_busy: failed: connecting\n"); 2553 return (DDI_FAILURE); 2554 } 2555 2556 return (DDI_SUCCESS); 2557 } 2558 2559 2560 static void 2561 ibd_part_unattach(ibd_state_t *state) 2562 { 2563 uint32_t progress = state->id_mac_state; 2564 ibt_status_t ret; 2565 2566 /* make sure rx resources are freed */ 2567 ibd_free_rx_rsrcs(state); 2568 2569 if (progress & IBD_DRV_RC_SRQ_ALLOCD) { 2570 ASSERT(state->id_enable_rc); 2571 ibd_rc_fini_srq_list(state); 2572 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD); 2573 } 2574 2575 if (progress & IBD_DRV_MAC_REGISTERED) { 2576 (void) mac_unregister(state->id_mh); 2577 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED); 2578 } 2579 2580 if (progress & IBD_DRV_ASYNC_THR_CREATED) { 2581 /* 2582 * No new async requests will be posted since the device 2583 * link state has been marked as unknown; completion handlers 2584 * have been turned off, so Tx handler will not cause any 2585 * more IBD_ASYNC_REAP requests. 2586 * 2587 * Queue a request for the async thread to exit, which will 2588 * be serviced after any pending ones. This can take a while, 2589 * specially if the SM is unreachable, since IBMF will slowly 2590 * timeout each SM request issued by the async thread. Reap 2591 * the thread before continuing on, we do not want it to be 2592 * lingering in modunloaded code. 2593 */ 2594 ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT); 2595 thread_join(state->id_async_thrid); 2596 2597 state->id_mac_state &= (~IBD_DRV_ASYNC_THR_CREATED); 2598 } 2599 2600 if (progress & IBD_DRV_REQ_LIST_INITED) { 2601 list_destroy(&state->id_req_list); 2602 mutex_destroy(&state->id_acache_req_lock); 2603 cv_destroy(&state->id_acache_req_cv); 2604 state->id_mac_state &= ~IBD_DRV_REQ_LIST_INITED; 2605 } 2606 2607 if (progress & IBD_DRV_PD_ALLOCD) { 2608 if ((ret = ibt_free_pd(state->id_hca_hdl, 2609 state->id_pd_hdl)) != IBT_SUCCESS) { 2610 ibd_print_warn(state, "failed to free " 2611 "protection domain, ret=%d", ret); 2612 } 2613 state->id_pd_hdl = NULL; 2614 state->id_mac_state &= (~IBD_DRV_PD_ALLOCD); 2615 } 2616 2617 if (progress & IBD_DRV_HCA_OPENED) { 2618 if ((ret = ibt_close_hca(state->id_hca_hdl)) != 2619 IBT_SUCCESS) { 2620 ibd_print_warn(state, "failed to close " 2621 "HCA device, ret=%d", ret); 2622 } 2623 state->id_hca_hdl = NULL; 2624 state->id_mac_state &= (~IBD_DRV_HCA_OPENED); 2625 } 2626 2627 mutex_enter(&ibd_gstate.ig_mutex); 2628 if (progress & IBD_DRV_IBTL_ATTACH_DONE) { 2629 if ((ret = ibt_detach(state->id_ibt_hdl)) != 2630 IBT_SUCCESS) { 2631 ibd_print_warn(state, 2632 "ibt_detach() failed, ret=%d", ret); 2633 } 2634 state->id_ibt_hdl = NULL; 2635 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE); 2636 ibd_gstate.ig_ibt_hdl_ref_cnt--; 2637 } 2638 if ((ibd_gstate.ig_ibt_hdl_ref_cnt == 0) && 2639 (ibd_gstate.ig_ibt_hdl != NULL)) { 2640 if ((ret = ibt_detach(ibd_gstate.ig_ibt_hdl)) != 2641 IBT_SUCCESS) { 2642 ibd_print_warn(state, "ibt_detach(): global " 2643 "failed, ret=%d", ret); 2644 } 2645 ibd_gstate.ig_ibt_hdl = NULL; 2646 } 2647 mutex_exit(&ibd_gstate.ig_mutex); 2648 2649 if (progress & IBD_DRV_TXINTR_ADDED) { 2650 ddi_remove_softintr(state->id_tx); 2651 state->id_tx = NULL; 2652 state->id_mac_state &= (~IBD_DRV_TXINTR_ADDED); 2653 } 2654 2655 if (progress & IBD_DRV_RXINTR_ADDED) { 2656 ddi_remove_softintr(state->id_rx); 2657 state->id_rx = NULL; 2658 state->id_mac_state &= (~IBD_DRV_RXINTR_ADDED); 2659 } 2660 2661 #ifdef DEBUG 2662 if (progress & IBD_DRV_RC_PRIVATE_STATE) { 2663 kstat_delete(state->rc_ksp); 2664 state->id_mac_state &= (~IBD_DRV_RC_PRIVATE_STATE); 2665 } 2666 #endif 2667 2668 if (progress & IBD_DRV_STATE_INITIALIZED) { 2669 ibd_state_fini(state); 2670 state->id_mac_state &= (~IBD_DRV_STATE_INITIALIZED); 2671 } 2672 } 2673 2674 int 2675 ibd_part_attach(ibd_state_t *state, dev_info_t *dip) 2676 { 2677 ibt_status_t ret; 2678 int rv; 2679 kthread_t *kht; 2680 2681 /* 2682 * Initialize mutexes and condition variables 2683 */ 2684 if (ibd_state_init(state, dip) != DDI_SUCCESS) { 2685 DPRINT(10, "ibd_part_attach: failed in ibd_state_init()"); 2686 return (DDI_FAILURE); 2687 } 2688 state->id_mac_state |= IBD_DRV_STATE_INITIALIZED; 2689 2690 /* 2691 * Allocate rx,tx softintr 2692 */ 2693 if (ibd_rx_softintr == 1) { 2694 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx, 2695 NULL, NULL, ibd_intr, (caddr_t)state)) != DDI_SUCCESS) { 2696 DPRINT(10, "ibd_part_attach: failed in " 2697 "ddi_add_softintr(id_rx), ret=%d", rv); 2698 return (DDI_FAILURE); 2699 } 2700 state->id_mac_state |= IBD_DRV_RXINTR_ADDED; 2701 } 2702 if (ibd_tx_softintr == 1) { 2703 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx, 2704 NULL, NULL, ibd_tx_recycle, 2705 (caddr_t)state)) != DDI_SUCCESS) { 2706 DPRINT(10, "ibd_part_attach: failed in " 2707 "ddi_add_softintr(id_tx), ret=%d", rv); 2708 return (DDI_FAILURE); 2709 } 2710 state->id_mac_state |= IBD_DRV_TXINTR_ADDED; 2711 } 2712 2713 /* 2714 * Attach to IBTL 2715 */ 2716 mutex_enter(&ibd_gstate.ig_mutex); 2717 if (ibd_gstate.ig_ibt_hdl == NULL) { 2718 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state, 2719 &ibd_gstate.ig_ibt_hdl)) != IBT_SUCCESS) { 2720 DPRINT(10, "ibd_part_attach: global: failed in " 2721 "ibt_attach(), ret=%d", ret); 2722 mutex_exit(&ibd_gstate.ig_mutex); 2723 return (DDI_FAILURE); 2724 } 2725 } 2726 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state, 2727 &state->id_ibt_hdl)) != IBT_SUCCESS) { 2728 DPRINT(10, "ibd_part_attach: failed in ibt_attach(), ret=%d", 2729 ret); 2730 mutex_exit(&ibd_gstate.ig_mutex); 2731 return (DDI_FAILURE); 2732 } 2733 ibd_gstate.ig_ibt_hdl_ref_cnt++; 2734 mutex_exit(&ibd_gstate.ig_mutex); 2735 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE; 2736 2737 /* 2738 * Open the HCA 2739 */ 2740 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid, 2741 &state->id_hca_hdl)) != IBT_SUCCESS) { 2742 DPRINT(10, "ibd_part_attach: ibt_open_hca() failed, ret=%d", 2743 ret); 2744 return (DDI_FAILURE); 2745 } 2746 state->id_mac_state |= IBD_DRV_HCA_OPENED; 2747 2748 #ifdef DEBUG 2749 /* Initialize Driver Counters for Reliable Connected Mode */ 2750 if (state->id_enable_rc) { 2751 if (ibd_rc_init_stats(state) != DDI_SUCCESS) { 2752 DPRINT(10, "ibd_part_attach: failed in " 2753 "ibd_rc_init_stats"); 2754 return (DDI_FAILURE); 2755 } 2756 state->id_mac_state |= IBD_DRV_RC_PRIVATE_STATE; 2757 } 2758 #endif 2759 2760 /* 2761 * Record capabilities 2762 */ 2763 (void) ibd_record_capab(state); 2764 2765 /* 2766 * Allocate a protection domain on the HCA 2767 */ 2768 if ((ret = ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS, 2769 &state->id_pd_hdl)) != IBT_SUCCESS) { 2770 DPRINT(10, "ibd_part_attach: ibt_alloc_pd() failed, ret=%d", 2771 ret); 2772 return (DDI_FAILURE); 2773 } 2774 state->id_mac_state |= IBD_DRV_PD_ALLOCD; 2775 2776 2777 /* 2778 * We need to initialise the req_list that is required for the 2779 * operation of the async_thread. 2780 */ 2781 mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL); 2782 cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL); 2783 list_create(&state->id_req_list, sizeof (ibd_req_t), 2784 offsetof(ibd_req_t, rq_list)); 2785 state->id_mac_state |= IBD_DRV_REQ_LIST_INITED; 2786 2787 /* 2788 * Create the async thread; thread_create never fails. 2789 */ 2790 kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0, 2791 TS_RUN, minclsyspri); 2792 state->id_async_thrid = kht->t_did; 2793 state->id_mac_state |= IBD_DRV_ASYNC_THR_CREATED; 2794 2795 return (DDI_SUCCESS); 2796 } 2797 2798 /* 2799 * Attach device to the IO framework. 2800 */ 2801 static int 2802 ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 2803 { 2804 int ret; 2805 2806 switch (cmd) { 2807 case DDI_ATTACH: 2808 ret = ibd_port_attach(dip); 2809 break; 2810 default: 2811 ret = DDI_FAILURE; 2812 break; 2813 } 2814 return (ret); 2815 } 2816 2817 /* 2818 * Detach device from the IO framework. 2819 */ 2820 static int 2821 ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2822 { 2823 ibd_state_t *state; 2824 int instance; 2825 2826 /* 2827 * IBD doesn't support suspend/resume 2828 */ 2829 if (cmd != DDI_DETACH) 2830 return (DDI_FAILURE); 2831 2832 /* 2833 * Get the instance softstate 2834 */ 2835 instance = ddi_get_instance(dip); 2836 state = ddi_get_soft_state(ibd_list, instance); 2837 2838 /* 2839 * Release all resources we're holding still. Note that if we'd 2840 * done ibd_attach(), ibd_m_start() and ibd_m_stop() correctly 2841 * so far, we should find all the flags we need in id_mac_state. 2842 */ 2843 return (ibd_port_unattach(state, dip)); 2844 } 2845 2846 /* 2847 * Pre ibt_attach() driver initialization 2848 */ 2849 static int 2850 ibd_state_init(ibd_state_t *state, dev_info_t *dip) 2851 { 2852 char buf[64]; 2853 2854 mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL); 2855 state->id_link_state = LINK_STATE_UNKNOWN; 2856 2857 mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL); 2858 cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL); 2859 state->id_trap_stop = B_TRUE; 2860 state->id_trap_inprog = 0; 2861 2862 mutex_init(&state->id_scq_poll_lock, NULL, MUTEX_DRIVER, NULL); 2863 mutex_init(&state->id_rcq_poll_lock, NULL, MUTEX_DRIVER, NULL); 2864 state->id_dip = dip; 2865 2866 mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL); 2867 2868 mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2869 mutex_init(&state->id_tx_rel_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2870 mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL); 2871 state->id_tx_busy = 0; 2872 mutex_init(&state->id_lso_lock, NULL, MUTEX_DRIVER, NULL); 2873 2874 state->id_rx_list.dl_bufs_outstanding = 0; 2875 state->id_rx_list.dl_cnt = 0; 2876 mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2877 mutex_init(&state->id_rx_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2878 (void) sprintf(buf, "ibd_req%d_%x_%u", ddi_get_instance(dip), 2879 state->id_pkey, state->id_plinkid); 2880 state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t), 2881 0, NULL, NULL, NULL, NULL, NULL, 0); 2882 2883 /* For Reliable Connected Mode */ 2884 mutex_init(&state->rc_rx_lock, NULL, MUTEX_DRIVER, NULL); 2885 mutex_init(&state->rc_tx_large_bufs_lock, NULL, MUTEX_DRIVER, NULL); 2886 mutex_init(&state->rc_srq_rwqe_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2887 mutex_init(&state->rc_srq_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL); 2888 mutex_init(&state->rc_pass_chan_list.chan_list_mutex, NULL, 2889 MUTEX_DRIVER, NULL); 2890 mutex_init(&state->rc_timeout_lock, NULL, MUTEX_DRIVER, NULL); 2891 2892 /* 2893 * Make the default link mode as RC. If this fails during connection 2894 * setup, the link mode is automatically transitioned to UD. 2895 * Also set the RC MTU. 2896 */ 2897 state->id_enable_rc = IBD_DEF_LINK_MODE; 2898 state->rc_mtu = IBD_DEF_RC_MAX_MTU; 2899 state->id_mtu = IBD_DEF_MAX_MTU; 2900 2901 /* Iniatialize all tunables to default */ 2902 state->id_lso_policy = IBD_DEF_LSO_POLICY; 2903 state->id_num_lso_bufs = IBD_DEF_NUM_LSO_BUFS; 2904 state->id_num_ah = IBD_DEF_NUM_AH; 2905 state->id_hash_size = IBD_DEF_HASH_SIZE; 2906 state->id_create_broadcast_group = IBD_DEF_CREATE_BCAST_GROUP; 2907 state->id_allow_coalesce_comp_tuning = IBD_DEF_COALESCE_COMPLETIONS; 2908 state->id_ud_rx_comp_count = IBD_DEF_UD_RX_COMP_COUNT; 2909 state->id_ud_rx_comp_usec = IBD_DEF_UD_RX_COMP_USEC; 2910 state->id_ud_tx_comp_count = IBD_DEF_UD_TX_COMP_COUNT; 2911 state->id_ud_tx_comp_usec = IBD_DEF_UD_TX_COMP_USEC; 2912 state->id_rc_rx_comp_count = IBD_DEF_RC_RX_COMP_COUNT; 2913 state->id_rc_rx_comp_usec = IBD_DEF_RC_RX_COMP_USEC; 2914 state->id_rc_tx_comp_count = IBD_DEF_RC_TX_COMP_COUNT; 2915 state->id_rc_tx_comp_usec = IBD_DEF_RC_TX_COMP_USEC; 2916 state->id_ud_tx_copy_thresh = IBD_DEF_UD_TX_COPY_THRESH; 2917 state->id_rc_rx_copy_thresh = IBD_DEF_RC_RX_COPY_THRESH; 2918 state->id_rc_tx_copy_thresh = IBD_DEF_RC_TX_COPY_THRESH; 2919 state->id_ud_num_rwqe = IBD_DEF_UD_NUM_RWQE; 2920 state->id_ud_num_swqe = IBD_DEF_UD_NUM_SWQE; 2921 state->id_rc_num_rwqe = IBD_DEF_RC_NUM_RWQE; 2922 state->id_rc_num_swqe = IBD_DEF_RC_NUM_SWQE; 2923 state->rc_enable_srq = IBD_DEF_RC_ENABLE_SRQ; 2924 state->id_rc_num_srq = IBD_DEF_RC_NUM_SRQ; 2925 state->id_rc_rx_rwqe_thresh = IBD_DEF_RC_RX_RWQE_THRESH; 2926 2927 return (DDI_SUCCESS); 2928 } 2929 2930 /* 2931 * Post ibt_detach() driver deconstruction 2932 */ 2933 static void 2934 ibd_state_fini(ibd_state_t *state) 2935 { 2936 kmem_cache_destroy(state->id_req_kmc); 2937 2938 mutex_destroy(&state->id_rx_list.dl_mutex); 2939 mutex_destroy(&state->id_rx_free_list.dl_mutex); 2940 2941 mutex_destroy(&state->id_txpost_lock); 2942 mutex_destroy(&state->id_tx_list.dl_mutex); 2943 mutex_destroy(&state->id_tx_rel_list.dl_mutex); 2944 mutex_destroy(&state->id_lso_lock); 2945 2946 mutex_destroy(&state->id_sched_lock); 2947 mutex_destroy(&state->id_scq_poll_lock); 2948 mutex_destroy(&state->id_rcq_poll_lock); 2949 2950 cv_destroy(&state->id_trap_cv); 2951 mutex_destroy(&state->id_trap_lock); 2952 mutex_destroy(&state->id_link_mutex); 2953 2954 /* For Reliable Connected Mode */ 2955 mutex_destroy(&state->rc_timeout_lock); 2956 mutex_destroy(&state->rc_srq_free_list.dl_mutex); 2957 mutex_destroy(&state->rc_srq_rwqe_list.dl_mutex); 2958 mutex_destroy(&state->rc_pass_chan_list.chan_list_mutex); 2959 mutex_destroy(&state->rc_tx_large_bufs_lock); 2960 mutex_destroy(&state->rc_rx_lock); 2961 } 2962 2963 /* 2964 * Fetch link speed from SA for snmp ifspeed reporting. 2965 */ 2966 static uint64_t 2967 ibd_get_portspeed(ibd_state_t *state) 2968 { 2969 int ret; 2970 ibt_path_info_t path; 2971 ibt_path_attr_t path_attr; 2972 uint8_t num_paths; 2973 uint64_t ifspeed; 2974 2975 /* 2976 * Due to serdes 8b10b encoding on the wire, 2.5 Gbps on wire 2977 * translates to 2 Gbps data rate. Thus, 1X single data rate is 2978 * 2000000000. Start with that as default. 2979 */ 2980 ifspeed = 2000000000; 2981 2982 bzero(&path_attr, sizeof (path_attr)); 2983 2984 /* 2985 * Get the port speed from Loopback path information. 2986 */ 2987 path_attr.pa_dgids = &state->id_sgid; 2988 path_attr.pa_num_dgids = 1; 2989 path_attr.pa_sgid = state->id_sgid; 2990 2991 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS, 2992 &path_attr, 1, &path, &num_paths) != IBT_SUCCESS) 2993 goto earlydone; 2994 2995 if (num_paths < 1) 2996 goto earlydone; 2997 2998 /* 2999 * In case SA does not return an expected value, report the default 3000 * speed as 1X. 3001 */ 3002 ret = 1; 3003 switch (path.pi_prim_cep_path.cep_adds_vect.av_srate) { 3004 case IBT_SRATE_2: /* 1X SDR i.e 2.5 Gbps */ 3005 ret = 1; 3006 break; 3007 case IBT_SRATE_10: /* 4X SDR or 1X QDR i.e 10 Gbps */ 3008 ret = 4; 3009 break; 3010 case IBT_SRATE_30: /* 12X SDR i.e 30 Gbps */ 3011 ret = 12; 3012 break; 3013 case IBT_SRATE_5: /* 1X DDR i.e 5 Gbps */ 3014 ret = 2; 3015 break; 3016 case IBT_SRATE_20: /* 4X DDR or 8X SDR i.e 20 Gbps */ 3017 ret = 8; 3018 break; 3019 case IBT_SRATE_40: /* 8X DDR or 4X QDR i.e 40 Gbps */ 3020 ret = 16; 3021 break; 3022 case IBT_SRATE_60: /* 12X DDR i.e 60 Gbps */ 3023 ret = 24; 3024 break; 3025 case IBT_SRATE_80: /* 8X QDR i.e 80 Gbps */ 3026 ret = 32; 3027 break; 3028 case IBT_SRATE_120: /* 12X QDR i.e 120 Gbps */ 3029 ret = 48; 3030 break; 3031 } 3032 3033 ifspeed *= ret; 3034 3035 earlydone: 3036 return (ifspeed); 3037 } 3038 3039 /* 3040 * Search input mcg list (id_mc_full or id_mc_non) for an entry 3041 * representing the input mcg mgid. 3042 */ 3043 static ibd_mce_t * 3044 ibd_mcache_find(ib_gid_t mgid, struct list *mlist) 3045 { 3046 ibd_mce_t *ptr = list_head(mlist); 3047 3048 /* 3049 * Do plain linear search. 3050 */ 3051 while (ptr != NULL) { 3052 if (bcmp(&mgid, &ptr->mc_info.mc_adds_vect.av_dgid, 3053 sizeof (ib_gid_t)) == 0) 3054 return (ptr); 3055 ptr = list_next(mlist, ptr); 3056 } 3057 return (NULL); 3058 } 3059 3060 /* 3061 * Execute IBA JOIN. 3062 */ 3063 static ibt_status_t 3064 ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce) 3065 { 3066 ibt_mcg_attr_t mcg_attr; 3067 3068 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t)); 3069 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey; 3070 mcg_attr.mc_mgid = mgid; 3071 mcg_attr.mc_join_state = mce->mc_jstate; 3072 mcg_attr.mc_scope = state->id_scope; 3073 mcg_attr.mc_pkey = state->id_pkey; 3074 mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow; 3075 mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl; 3076 mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass; 3077 return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info, 3078 NULL, NULL)); 3079 } 3080 3081 /* 3082 * This code JOINs the port in the proper way (depending on the join 3083 * state) so that IBA fabric will forward mcg packets to/from the port. 3084 * It also attaches the QPN to the mcg so it can receive those mcg 3085 * packets. This code makes sure not to attach the mcg to the QP if 3086 * that has been previously done due to the mcg being joined with a 3087 * different join state, even though this is not required by SWG_0216, 3088 * refid 3610. 3089 */ 3090 static ibd_mce_t * 3091 ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate) 3092 { 3093 ibt_status_t ibt_status; 3094 ibd_mce_t *mce, *tmce, *omce = NULL; 3095 boolean_t do_attach = B_TRUE; 3096 3097 DPRINT(2, "ibd_join_group : join_group state %d : %016llx:%016llx\n", 3098 jstate, mgid.gid_prefix, mgid.gid_guid); 3099 3100 /* 3101 * For enable_multicast Full member joins, we need to do some 3102 * extra work. If there is already an mce on the list that 3103 * indicates full membership, that means the membership has 3104 * not yet been dropped (since the disable_multicast was issued) 3105 * because there are pending Tx's to the mcg; in that case, just 3106 * mark the mce not to be reaped when the Tx completion queues 3107 * an async reap operation. 3108 * 3109 * If there is already an mce on the list indicating sendonly 3110 * membership, try to promote to full membership. Be careful 3111 * not to deallocate the old mce, since there might be an AH 3112 * pointing to it; instead, update the old mce with new data 3113 * that tracks the full membership. 3114 */ 3115 if ((jstate == IB_MC_JSTATE_FULL) && ((omce = 3116 IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) { 3117 if (omce->mc_jstate == IB_MC_JSTATE_FULL) { 3118 ASSERT(omce->mc_fullreap); 3119 omce->mc_fullreap = B_FALSE; 3120 return (omce); 3121 } else { 3122 ASSERT(omce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON); 3123 } 3124 } 3125 3126 /* 3127 * Allocate the ibd_mce_t to track this JOIN. 3128 */ 3129 mce = kmem_zalloc(sizeof (ibd_mce_t), KM_SLEEP); 3130 mce->mc_fullreap = B_FALSE; 3131 mce->mc_jstate = jstate; 3132 3133 if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) { 3134 DPRINT(10, "ibd_join_group : failed ibt_join_mcg() %d", 3135 ibt_status); 3136 kmem_free(mce, sizeof (ibd_mce_t)); 3137 return (NULL); 3138 } 3139 3140 /* 3141 * Is an IBA attach required? Not if the interface is already joined 3142 * to the mcg in a different appropriate join state. 3143 */ 3144 if (jstate == IB_MC_JSTATE_NON) { 3145 tmce = IBD_MCACHE_FIND_FULL(state, mgid); 3146 if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL)) 3147 do_attach = B_FALSE; 3148 } else if (jstate == IB_MC_JSTATE_FULL) { 3149 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL) 3150 do_attach = B_FALSE; 3151 } else { /* jstate == IB_MC_JSTATE_SEND_ONLY_NON */ 3152 do_attach = B_FALSE; 3153 } 3154 3155 if (do_attach) { 3156 /* 3157 * Do the IBA attach. 3158 */ 3159 DPRINT(10, "ibd_join_group: ibt_attach_mcg \n"); 3160 if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl, 3161 &mce->mc_info)) != IBT_SUCCESS) { 3162 DPRINT(10, "ibd_join_group : failed qp attachment " 3163 "%d\n", ibt_status); 3164 /* 3165 * NOTE that we should probably preserve the join info 3166 * in the list and later try to leave again at detach 3167 * time. 3168 */ 3169 (void) ibt_leave_mcg(state->id_sgid, mgid, 3170 state->id_sgid, jstate); 3171 kmem_free(mce, sizeof (ibd_mce_t)); 3172 return (NULL); 3173 } 3174 } 3175 3176 /* 3177 * Insert the ibd_mce_t in the proper list. 3178 */ 3179 if (jstate == IB_MC_JSTATE_NON) { 3180 IBD_MCACHE_INSERT_NON(state, mce); 3181 } else { 3182 /* 3183 * Set up the mc_req fields used for reaping the 3184 * mcg in case of delayed tx completion (see 3185 * ibd_tx_cleanup()). Also done for sendonly join in 3186 * case we are promoted to fullmembership later and 3187 * keep using the same mce. 3188 */ 3189 mce->mc_req.rq_gid = mgid; 3190 mce->mc_req.rq_ptr = mce; 3191 /* 3192 * Check whether this is the case of trying to join 3193 * full member, and we were already joined send only. 3194 * We try to drop our SendOnly membership, but it is 3195 * possible that the mcg does not exist anymore (and 3196 * the subnet trap never reached us), so the leave 3197 * operation might fail. 3198 */ 3199 if (omce != NULL) { 3200 (void) ibt_leave_mcg(state->id_sgid, mgid, 3201 state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON); 3202 omce->mc_jstate = IB_MC_JSTATE_FULL; 3203 bcopy(&mce->mc_info, &omce->mc_info, 3204 sizeof (ibt_mcg_info_t)); 3205 kmem_free(mce, sizeof (ibd_mce_t)); 3206 return (omce); 3207 } 3208 mutex_enter(&state->id_mc_mutex); 3209 IBD_MCACHE_INSERT_FULL(state, mce); 3210 mutex_exit(&state->id_mc_mutex); 3211 } 3212 3213 return (mce); 3214 } 3215 3216 /* 3217 * Called during port up event handling to attempt to reacquire full 3218 * membership to an mcg. Stripped down version of ibd_join_group(). 3219 * Note that it is possible that the mcg might have gone away, and 3220 * gets recreated at this point. 3221 */ 3222 static void 3223 ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce) 3224 { 3225 ib_gid_t mgid; 3226 3227 /* 3228 * If the mc_fullreap flag is set, or this join fails, a subsequent 3229 * reap/leave is going to try to leave the group. We could prevent 3230 * that by adding a boolean flag into ibd_mce_t, if required. 3231 */ 3232 if (mce->mc_fullreap) 3233 return; 3234 3235 mgid = mce->mc_info.mc_adds_vect.av_dgid; 3236 3237 DPRINT(2, "ibd_reacquire_group : %016llx:%016llx\n", mgid.gid_prefix, 3238 mgid.gid_guid); 3239 3240 /* While reacquiring, leave and then join the MCG */ 3241 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, 3242 mce->mc_jstate); 3243 if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS) 3244 ibd_print_warn(state, "Failure on port up to rejoin " 3245 "multicast gid %016llx:%016llx", 3246 (u_longlong_t)mgid.gid_prefix, 3247 (u_longlong_t)mgid.gid_guid); 3248 } 3249 3250 /* 3251 * This code handles delayed Tx completion cleanups for mcg's to which 3252 * disable_multicast has been issued, regular mcg related cleanups during 3253 * disable_multicast, disable_promiscuous and mcg traps, as well as 3254 * cleanups during driver detach time. Depending on the join state, 3255 * it deletes the mce from the appropriate list and issues the IBA 3256 * leave/detach; except in the disable_multicast case when the mce 3257 * is left on the active list for a subsequent Tx completion cleanup. 3258 */ 3259 static void 3260 ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid, 3261 uint8_t jstate) 3262 { 3263 ibd_mce_t *tmce; 3264 boolean_t do_detach = B_TRUE; 3265 3266 /* 3267 * Before detaching, we must check whether the other list 3268 * contains the mcg; if we detach blindly, the consumer 3269 * who set up the other list will also stop receiving 3270 * traffic. 3271 */ 3272 if (jstate == IB_MC_JSTATE_FULL) { 3273 /* 3274 * The following check is only relevant while coming 3275 * from the Tx completion path in the reap case. 3276 */ 3277 if (!mce->mc_fullreap) 3278 return; 3279 mutex_enter(&state->id_mc_mutex); 3280 IBD_MCACHE_PULLOUT_FULL(state, mce); 3281 mutex_exit(&state->id_mc_mutex); 3282 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL) 3283 do_detach = B_FALSE; 3284 } else if (jstate == IB_MC_JSTATE_NON) { 3285 IBD_MCACHE_PULLOUT_NON(state, mce); 3286 tmce = IBD_MCACHE_FIND_FULL(state, mgid); 3287 if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL)) 3288 do_detach = B_FALSE; 3289 } else { /* jstate == IB_MC_JSTATE_SEND_ONLY_NON */ 3290 mutex_enter(&state->id_mc_mutex); 3291 IBD_MCACHE_PULLOUT_FULL(state, mce); 3292 mutex_exit(&state->id_mc_mutex); 3293 do_detach = B_FALSE; 3294 } 3295 3296 /* 3297 * If we are reacting to a mcg trap and leaving our sendonly or 3298 * non membership, the mcg is possibly already gone, so attempting 3299 * to leave might fail. On the other hand, we must try to leave 3300 * anyway, since this might be a trap from long ago, and we could 3301 * have potentially sendonly joined to a recent incarnation of 3302 * the mcg and are about to loose track of this information. 3303 */ 3304 if (do_detach) { 3305 DPRINT(2, "ibd_async_reap_group : ibt_detach_mcg : " 3306 "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid); 3307 (void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info); 3308 } 3309 3310 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate); 3311 kmem_free(mce, sizeof (ibd_mce_t)); 3312 } 3313 3314 /* 3315 * Async code executed due to multicast and promiscuous disable requests 3316 * and mcg trap handling; also executed during driver detach. Mostly, a 3317 * leave and detach is done; except for the fullmember case when Tx 3318 * requests are pending, whence arrangements are made for subsequent 3319 * cleanup on Tx completion. 3320 */ 3321 static void 3322 ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate) 3323 { 3324 ipoib_mac_t mcmac; 3325 boolean_t recycled; 3326 ibd_mce_t *mce; 3327 3328 DPRINT(2, "ibd_leave_group : leave_group state %d : %016llx:%016llx\n", 3329 jstate, mgid.gid_prefix, mgid.gid_guid); 3330 3331 if (jstate == IB_MC_JSTATE_NON) { 3332 recycled = B_TRUE; 3333 mce = IBD_MCACHE_FIND_NON(state, mgid); 3334 /* 3335 * In case we are handling a mcg trap, we might not find 3336 * the mcg in the non list. 3337 */ 3338 if (mce == NULL) { 3339 return; 3340 } 3341 } else { 3342 mce = IBD_MCACHE_FIND_FULL(state, mgid); 3343 3344 /* 3345 * In case we are handling a mcg trap, make sure the trap 3346 * is not arriving late; if we have an mce that indicates 3347 * that we are already a fullmember, that would be a clear 3348 * indication that the trap arrived late (ie, is for a 3349 * previous incarnation of the mcg). 3350 */ 3351 if (jstate == IB_MC_JSTATE_SEND_ONLY_NON) { 3352 if ((mce == NULL) || (mce->mc_jstate == 3353 IB_MC_JSTATE_FULL)) { 3354 return; 3355 } 3356 } else { 3357 ASSERT(jstate == IB_MC_JSTATE_FULL); 3358 3359 /* 3360 * If join group failed, mce will be NULL here. 3361 * This is because in GLDv3 driver, set multicast 3362 * will always return success. 3363 */ 3364 if (mce == NULL) { 3365 return; 3366 } 3367 3368 mce->mc_fullreap = B_TRUE; 3369 } 3370 3371 /* 3372 * If no pending Tx's remain that reference the AH 3373 * for the mcg, recycle it from active to free list. 3374 * Else in the IB_MC_JSTATE_FULL case, just mark the AH, 3375 * so the last completing Tx will cause an async reap 3376 * operation to be invoked, at which time we will drop our 3377 * membership to the mcg so that the pending Tx's complete 3378 * successfully. Refer to comments on "AH and MCE active 3379 * list manipulation" at top of this file. The lock protects 3380 * against Tx fast path and Tx cleanup code. 3381 */ 3382 mutex_enter(&state->id_ac_mutex); 3383 ibd_h2n_mac(&mcmac, IB_MC_QPN, mgid.gid_prefix, mgid.gid_guid); 3384 recycled = ibd_acache_recycle(state, &mcmac, (jstate == 3385 IB_MC_JSTATE_SEND_ONLY_NON)); 3386 mutex_exit(&state->id_ac_mutex); 3387 } 3388 3389 if (recycled) { 3390 DPRINT(2, "ibd_leave_group : leave_group reaping : " 3391 "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid); 3392 ibd_async_reap_group(state, mce, mgid, jstate); 3393 } 3394 } 3395 3396 /* 3397 * Find the broadcast address as defined by IPoIB; implicitly 3398 * determines the IBA scope, mtu, tclass etc of the link the 3399 * interface is going to be a member of. 3400 */ 3401 static ibt_status_t 3402 ibd_find_bgroup(ibd_state_t *state) 3403 { 3404 ibt_mcg_attr_t mcg_attr; 3405 uint_t numg; 3406 uchar_t scopes[] = { IB_MC_SCOPE_SUBNET_LOCAL, 3407 IB_MC_SCOPE_SITE_LOCAL, IB_MC_SCOPE_ORG_LOCAL, 3408 IB_MC_SCOPE_GLOBAL }; 3409 int i, mcgmtu; 3410 boolean_t found = B_FALSE; 3411 int ret; 3412 ibt_mcg_info_t mcg_info; 3413 3414 state->id_bgroup_created = B_FALSE; 3415 state->id_bgroup_present = B_FALSE; 3416 3417 query_bcast_grp: 3418 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t)); 3419 mcg_attr.mc_pkey = state->id_pkey; 3420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3421 state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK; 3422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3423 3424 for (i = 0; i < sizeof (scopes)/sizeof (scopes[0]); i++) { 3425 state->id_scope = mcg_attr.mc_scope = scopes[i]; 3426 3427 /* 3428 * Look for the IPoIB broadcast group. 3429 */ 3430 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3431 state->id_mgid.gid_prefix = 3432 (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) | 3433 ((uint64_t)state->id_scope << 48) | 3434 ((uint32_t)(state->id_pkey << 16))); 3435 mcg_attr.mc_mgid = state->id_mgid; 3436 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3437 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1, 3438 &state->id_mcinfo, &numg) == IBT_SUCCESS) { 3439 found = B_TRUE; 3440 break; 3441 } 3442 } 3443 3444 if (!found) { 3445 if (state->id_create_broadcast_group) { 3446 /* 3447 * If we created the broadcast group, but failed to 3448 * find it, we can't do anything except leave the 3449 * one we created and return failure. 3450 */ 3451 if (state->id_bgroup_created) { 3452 ibd_print_warn(state, "IPoIB broadcast group " 3453 "absent. Unable to query after create."); 3454 goto find_bgroup_fail; 3455 } 3456 3457 /* 3458 * Create the ipoib broadcast group if it didn't exist 3459 */ 3460 bzero(&mcg_attr, sizeof (ibt_mcg_attr_t)); 3461 mcg_attr.mc_qkey = IBD_DEFAULT_QKEY; 3462 mcg_attr.mc_join_state = IB_MC_JSTATE_FULL; 3463 mcg_attr.mc_scope = IB_MC_SCOPE_SUBNET_LOCAL; 3464 mcg_attr.mc_pkey = state->id_pkey; 3465 mcg_attr.mc_flow = 0; 3466 mcg_attr.mc_sl = 0; 3467 mcg_attr.mc_tclass = 0; 3468 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3469 state->id_mgid.gid_prefix = 3470 (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) | 3471 ((uint64_t)IB_MC_SCOPE_SUBNET_LOCAL << 48) | 3472 ((uint32_t)(state->id_pkey << 16))); 3473 mcg_attr.mc_mgid = state->id_mgid; 3474 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid)) 3475 3476 if ((ret = ibt_join_mcg(state->id_sgid, &mcg_attr, 3477 &mcg_info, NULL, NULL)) != IBT_SUCCESS) { 3478 ibd_print_warn(state, "IPoIB broadcast group " 3479 "absent, create failed: ret = %d\n", ret); 3480 state->id_bgroup_created = B_FALSE; 3481 return (IBT_FAILURE); 3482 } 3483 state->id_bgroup_created = B_TRUE; 3484 goto query_bcast_grp; 3485 } else { 3486 ibd_print_warn(state, "IPoIB broadcast group absent"); 3487 return (IBT_FAILURE); 3488 } 3489 } 3490 3491 /* 3492 * Assert that the mcg mtu <= id_mtu. Fill in updated id_mtu. 3493 */ 3494 mcgmtu = (128 << state->id_mcinfo->mc_mtu); 3495 if (state->id_mtu < mcgmtu) { 3496 ibd_print_warn(state, "IPoIB broadcast group MTU %d " 3497 "greater than port's maximum MTU %d", mcgmtu, 3498 state->id_mtu); 3499 ibt_free_mcg_info(state->id_mcinfo, 1); 3500 goto find_bgroup_fail; 3501 } 3502 state->id_mtu = mcgmtu; 3503 state->id_bgroup_present = B_TRUE; 3504 3505 return (IBT_SUCCESS); 3506 3507 find_bgroup_fail: 3508 if (state->id_bgroup_created) { 3509 (void) ibt_leave_mcg(state->id_sgid, 3510 mcg_info.mc_adds_vect.av_dgid, state->id_sgid, 3511 IB_MC_JSTATE_FULL); 3512 } 3513 3514 return (IBT_FAILURE); 3515 } 3516 3517 static int 3518 ibd_alloc_tx_copybufs(ibd_state_t *state) 3519 { 3520 ibt_mr_attr_t mem_attr; 3521 3522 /* 3523 * Allocate one big chunk for all regular tx copy bufs 3524 */ 3525 state->id_tx_buf_sz = state->id_mtu; 3526 if (state->id_lso_policy && state->id_lso_capable && 3527 (state->id_ud_tx_copy_thresh > state->id_mtu)) { 3528 state->id_tx_buf_sz = state->id_ud_tx_copy_thresh; 3529 } 3530 3531 state->id_tx_bufs = kmem_zalloc(state->id_ud_num_swqe * 3532 state->id_tx_buf_sz, KM_SLEEP); 3533 3534 state->id_tx_wqes = kmem_zalloc(state->id_ud_num_swqe * 3535 sizeof (ibd_swqe_t), KM_SLEEP); 3536 3537 /* 3538 * Do one memory registration on the entire txbuf area 3539 */ 3540 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs; 3541 mem_attr.mr_len = state->id_ud_num_swqe * state->id_tx_buf_sz; 3542 mem_attr.mr_as = NULL; 3543 mem_attr.mr_flags = IBT_MR_SLEEP; 3544 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, 3545 &state->id_tx_mr_hdl, &state->id_tx_mr_desc) != IBT_SUCCESS) { 3546 DPRINT(10, "ibd_alloc_tx_copybufs: ibt_register_mr failed"); 3547 kmem_free(state->id_tx_wqes, 3548 state->id_ud_num_swqe * sizeof (ibd_swqe_t)); 3549 kmem_free(state->id_tx_bufs, 3550 state->id_ud_num_swqe * state->id_tx_buf_sz); 3551 state->id_tx_bufs = NULL; 3552 return (DDI_FAILURE); 3553 } 3554 3555 return (DDI_SUCCESS); 3556 } 3557 3558 static int 3559 ibd_alloc_tx_lsobufs(ibd_state_t *state) 3560 { 3561 ibt_mr_attr_t mem_attr; 3562 ibd_lsobuf_t *buflist; 3563 ibd_lsobuf_t *lbufp; 3564 ibd_lsobuf_t *tail; 3565 ibd_lsobkt_t *bktp; 3566 uint8_t *membase; 3567 uint8_t *memp; 3568 uint_t memsz; 3569 int i; 3570 3571 /* 3572 * Allocate the lso bucket 3573 */ 3574 bktp = kmem_zalloc(sizeof (ibd_lsobkt_t), KM_SLEEP); 3575 3576 /* 3577 * Allocate the entire lso memory and register it 3578 */ 3579 memsz = state->id_num_lso_bufs * IBD_LSO_BUFSZ; 3580 membase = kmem_zalloc(memsz, KM_SLEEP); 3581 3582 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)membase; 3583 mem_attr.mr_len = memsz; 3584 mem_attr.mr_as = NULL; 3585 mem_attr.mr_flags = IBT_MR_SLEEP; 3586 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, 3587 &mem_attr, &bktp->bkt_mr_hdl, &bktp->bkt_mr_desc) != IBT_SUCCESS) { 3588 DPRINT(10, "ibd_alloc_tx_lsobufs: ibt_register_mr failed"); 3589 kmem_free(membase, memsz); 3590 kmem_free(bktp, sizeof (ibd_lsobkt_t)); 3591 return (DDI_FAILURE); 3592 } 3593 3594 mutex_enter(&state->id_lso_lock); 3595 3596 /* 3597 * Now allocate the buflist. Note that the elements in the buflist and 3598 * the buffers in the lso memory have a permanent 1-1 relation, so we 3599 * can always derive the address of a buflist entry from the address of 3600 * an lso buffer. 3601 */ 3602 buflist = kmem_zalloc(state->id_num_lso_bufs * sizeof (ibd_lsobuf_t), 3603 KM_SLEEP); 3604 3605 /* 3606 * Set up the lso buf chain 3607 */ 3608 memp = membase; 3609 lbufp = buflist; 3610 for (i = 0; i < state->id_num_lso_bufs; i++) { 3611 lbufp->lb_isfree = 1; 3612 lbufp->lb_buf = memp; 3613 lbufp->lb_next = lbufp + 1; 3614 3615 tail = lbufp; 3616 3617 memp += IBD_LSO_BUFSZ; 3618 lbufp++; 3619 } 3620 tail->lb_next = NULL; 3621 3622 /* 3623 * Set up the LSO buffer information in ibd state 3624 */ 3625 bktp->bkt_bufl = buflist; 3626 bktp->bkt_free_head = buflist; 3627 bktp->bkt_mem = membase; 3628 bktp->bkt_nelem = state->id_num_lso_bufs; 3629 bktp->bkt_nfree = bktp->bkt_nelem; 3630 3631 state->id_lso = bktp; 3632 mutex_exit(&state->id_lso_lock); 3633 3634 return (DDI_SUCCESS); 3635 } 3636 3637 /* 3638 * Statically allocate Tx buffer list(s). 3639 */ 3640 static int 3641 ibd_init_txlist(ibd_state_t *state) 3642 { 3643 ibd_swqe_t *swqe; 3644 ibt_lkey_t lkey; 3645 int i; 3646 uint_t len; 3647 uint8_t *bufaddr; 3648 3649 if (ibd_alloc_tx_copybufs(state) != DDI_SUCCESS) 3650 return (DDI_FAILURE); 3651 3652 if (state->id_lso_policy && state->id_lso_capable) { 3653 if (ibd_alloc_tx_lsobufs(state) != DDI_SUCCESS) 3654 state->id_lso_capable = B_FALSE; 3655 } 3656 3657 mutex_enter(&state->id_tx_list.dl_mutex); 3658 state->id_tx_list.dl_head = NULL; 3659 state->id_tx_list.dl_pending_sends = B_FALSE; 3660 state->id_tx_list.dl_cnt = 0; 3661 mutex_exit(&state->id_tx_list.dl_mutex); 3662 mutex_enter(&state->id_tx_rel_list.dl_mutex); 3663 state->id_tx_rel_list.dl_head = NULL; 3664 state->id_tx_rel_list.dl_pending_sends = B_FALSE; 3665 state->id_tx_rel_list.dl_cnt = 0; 3666 mutex_exit(&state->id_tx_rel_list.dl_mutex); 3667 3668 /* 3669 * Allocate and setup the swqe list 3670 */ 3671 lkey = state->id_tx_mr_desc.md_lkey; 3672 bufaddr = state->id_tx_bufs; 3673 len = state->id_tx_buf_sz; 3674 swqe = state->id_tx_wqes; 3675 mutex_enter(&state->id_tx_list.dl_mutex); 3676 for (i = 0; i < state->id_ud_num_swqe; i++, swqe++, bufaddr += len) { 3677 swqe->swqe_next = NULL; 3678 swqe->swqe_im_mblk = NULL; 3679 3680 swqe->swqe_copybuf.ic_sgl.ds_va = (ib_vaddr_t)(uintptr_t) 3681 bufaddr; 3682 swqe->swqe_copybuf.ic_sgl.ds_key = lkey; 3683 swqe->swqe_copybuf.ic_sgl.ds_len = 0; /* set in send */ 3684 3685 swqe->w_swr.wr_id = (ibt_wrid_t)(uintptr_t)swqe; 3686 swqe->w_swr.wr_flags = IBT_WR_NO_FLAGS; 3687 swqe->w_swr.wr_trans = IBT_UD_SRV; 3688 3689 /* These are set in send */ 3690 swqe->w_swr.wr_nds = 0; 3691 swqe->w_swr.wr_sgl = NULL; 3692 swqe->w_swr.wr_opcode = IBT_WRC_SEND; 3693 3694 /* add to list */ 3695 state->id_tx_list.dl_cnt++; 3696 swqe->swqe_next = state->id_tx_list.dl_head; 3697 state->id_tx_list.dl_head = SWQE_TO_WQE(swqe); 3698 } 3699 mutex_exit(&state->id_tx_list.dl_mutex); 3700 3701 return (DDI_SUCCESS); 3702 } 3703 3704 static int 3705 ibd_acquire_lsobufs(ibd_state_t *state, uint_t req_sz, ibt_wr_ds_t *sgl_p, 3706 uint32_t *nds_p) 3707 { 3708 ibd_lsobkt_t *bktp; 3709 ibd_lsobuf_t *lbufp; 3710 ibd_lsobuf_t *nextp; 3711 ibt_lkey_t lso_lkey; 3712 uint_t frag_sz; 3713 uint_t num_needed; 3714 int i; 3715 3716 ASSERT(sgl_p != NULL); 3717 ASSERT(nds_p != NULL); 3718 ASSERT(req_sz != 0); 3719 3720 /* 3721 * Determine how many bufs we'd need for the size requested 3722 */ 3723 num_needed = req_sz / IBD_LSO_BUFSZ; 3724 if ((frag_sz = req_sz % IBD_LSO_BUFSZ) != 0) 3725 num_needed++; 3726 3727 mutex_enter(&state->id_lso_lock); 3728 3729 /* 3730 * If we don't have enough lso bufs, return failure 3731 */ 3732 ASSERT(state->id_lso != NULL); 3733 bktp = state->id_lso; 3734 if (bktp->bkt_nfree < num_needed) { 3735 mutex_exit(&state->id_lso_lock); 3736 return (-1); 3737 } 3738 3739 /* 3740 * Pick the first 'num_needed' bufs from the free list 3741 */ 3742 lso_lkey = bktp->bkt_mr_desc.md_lkey; 3743 lbufp = bktp->bkt_free_head; 3744 for (i = 0; i < num_needed; i++) { 3745 ASSERT(lbufp->lb_isfree != 0); 3746 ASSERT(lbufp->lb_buf != NULL); 3747 3748 nextp = lbufp->lb_next; 3749 3750 sgl_p[i].ds_va = (ib_vaddr_t)(uintptr_t)lbufp->lb_buf; 3751 sgl_p[i].ds_key = lso_lkey; 3752 sgl_p[i].ds_len = IBD_LSO_BUFSZ; 3753 3754 lbufp->lb_isfree = 0; 3755 lbufp->lb_next = NULL; 3756 3757 lbufp = nextp; 3758 } 3759 bktp->bkt_free_head = lbufp; 3760 3761 /* 3762 * If the requested size is not a multiple of IBD_LSO_BUFSZ, we need 3763 * to adjust the last sgl entry's length. Since we know we need atleast 3764 * one, the i-1 use below is ok. 3765 */ 3766 if (frag_sz) { 3767 sgl_p[i-1].ds_len = frag_sz; 3768 } 3769 3770 /* 3771 * Update nfree count and return 3772 */ 3773 bktp->bkt_nfree -= num_needed; 3774 3775 mutex_exit(&state->id_lso_lock); 3776 3777 *nds_p = num_needed; 3778 3779 return (0); 3780 } 3781 3782 static void 3783 ibd_release_lsobufs(ibd_state_t *state, ibt_wr_ds_t *sgl_p, uint32_t nds) 3784 { 3785 ibd_lsobkt_t *bktp; 3786 ibd_lsobuf_t *lbufp; 3787 uint8_t *lso_mem_end; 3788 uint_t ndx; 3789 int i; 3790 3791 mutex_enter(&state->id_lso_lock); 3792 3793 bktp = state->id_lso; 3794 ASSERT(bktp != NULL); 3795 3796 lso_mem_end = bktp->bkt_mem + bktp->bkt_nelem * IBD_LSO_BUFSZ; 3797 for (i = 0; i < nds; i++) { 3798 uint8_t *va; 3799 3800 va = (uint8_t *)(uintptr_t)sgl_p[i].ds_va; 3801 ASSERT(va >= bktp->bkt_mem && va < lso_mem_end); 3802 3803 /* 3804 * Figure out the buflist element this sgl buffer corresponds 3805 * to and put it back at the head 3806 */ 3807 ndx = (va - bktp->bkt_mem) / IBD_LSO_BUFSZ; 3808 lbufp = bktp->bkt_bufl + ndx; 3809 3810 ASSERT(lbufp->lb_isfree == 0); 3811 ASSERT(lbufp->lb_buf == va); 3812 3813 lbufp->lb_isfree = 1; 3814 lbufp->lb_next = bktp->bkt_free_head; 3815 bktp->bkt_free_head = lbufp; 3816 } 3817 bktp->bkt_nfree += nds; 3818 3819 mutex_exit(&state->id_lso_lock); 3820 } 3821 3822 static void 3823 ibd_free_tx_copybufs(ibd_state_t *state) 3824 { 3825 /* 3826 * Unregister txbuf mr 3827 */ 3828 if (ibt_deregister_mr(state->id_hca_hdl, 3829 state->id_tx_mr_hdl) != IBT_SUCCESS) { 3830 DPRINT(10, "ibd_free_tx_copybufs: ibt_deregister_mr failed"); 3831 } 3832 state->id_tx_mr_hdl = NULL; 3833 3834 /* 3835 * Free txbuf memory 3836 */ 3837 kmem_free(state->id_tx_wqes, state->id_ud_num_swqe * 3838 sizeof (ibd_swqe_t)); 3839 kmem_free(state->id_tx_bufs, state->id_ud_num_swqe * 3840 state->id_tx_buf_sz); 3841 state->id_tx_wqes = NULL; 3842 state->id_tx_bufs = NULL; 3843 } 3844 3845 static void 3846 ibd_free_tx_lsobufs(ibd_state_t *state) 3847 { 3848 ibd_lsobkt_t *bktp; 3849 3850 mutex_enter(&state->id_lso_lock); 3851 3852 if ((bktp = state->id_lso) == NULL) { 3853 mutex_exit(&state->id_lso_lock); 3854 return; 3855 } 3856 3857 /* 3858 * First, free the buflist 3859 */ 3860 ASSERT(bktp->bkt_bufl != NULL); 3861 kmem_free(bktp->bkt_bufl, bktp->bkt_nelem * sizeof (ibd_lsobuf_t)); 3862 3863 /* 3864 * Unregister the LSO memory and free it 3865 */ 3866 ASSERT(bktp->bkt_mr_hdl != NULL); 3867 if (ibt_deregister_mr(state->id_hca_hdl, 3868 bktp->bkt_mr_hdl) != IBT_SUCCESS) { 3869 DPRINT(10, 3870 "ibd_free_lsobufs: ibt_deregister_mr failed"); 3871 } 3872 ASSERT(bktp->bkt_mem); 3873 kmem_free(bktp->bkt_mem, bktp->bkt_nelem * IBD_LSO_BUFSZ); 3874 3875 /* 3876 * Finally free the bucket 3877 */ 3878 kmem_free(bktp, sizeof (ibd_lsobkt_t)); 3879 state->id_lso = NULL; 3880 3881 mutex_exit(&state->id_lso_lock); 3882 } 3883 3884 /* 3885 * Free the statically allocated Tx buffer list. 3886 */ 3887 static void 3888 ibd_fini_txlist(ibd_state_t *state) 3889 { 3890 /* 3891 * Free the allocated swqes 3892 */ 3893 mutex_enter(&state->id_tx_list.dl_mutex); 3894 mutex_enter(&state->id_tx_rel_list.dl_mutex); 3895 state->id_tx_list.dl_head = NULL; 3896 state->id_tx_list.dl_pending_sends = B_FALSE; 3897 state->id_tx_list.dl_cnt = 0; 3898 state->id_tx_rel_list.dl_head = NULL; 3899 state->id_tx_rel_list.dl_pending_sends = B_FALSE; 3900 state->id_tx_rel_list.dl_cnt = 0; 3901 mutex_exit(&state->id_tx_rel_list.dl_mutex); 3902 mutex_exit(&state->id_tx_list.dl_mutex); 3903 3904 ibd_free_tx_lsobufs(state); 3905 ibd_free_tx_copybufs(state); 3906 } 3907 3908 /* 3909 * post a list of rwqes, NULL terminated. 3910 */ 3911 static void 3912 ibd_post_recv_list(ibd_state_t *state, ibd_rwqe_t *rwqe) 3913 { 3914 uint_t i; 3915 uint_t num_posted; 3916 ibt_status_t ibt_status; 3917 ibt_recv_wr_t wrs[IBD_RX_POST_CNT]; 3918 3919 while (rwqe) { 3920 /* Post up to IBD_RX_POST_CNT receive work requests */ 3921 for (i = 0; i < IBD_RX_POST_CNT; i++) { 3922 wrs[i] = rwqe->w_rwr; 3923 rwqe = WQE_TO_RWQE(rwqe->rwqe_next); 3924 if (rwqe == NULL) { 3925 i++; 3926 break; 3927 } 3928 } 3929 3930 /* 3931 * If posting fails for some reason, we'll never receive 3932 * completion intimation, so we'll need to cleanup. But 3933 * we need to make sure we don't clean up nodes whose 3934 * wrs have been successfully posted. We assume that the 3935 * hca driver returns on the first failure to post and 3936 * therefore the first 'num_posted' entries don't need 3937 * cleanup here. 3938 */ 3939 atomic_add_32(&state->id_rx_list.dl_cnt, i); 3940 3941 num_posted = 0; 3942 ibt_status = ibt_post_recv(state->id_chnl_hdl, wrs, i, 3943 &num_posted); 3944 if (ibt_status != IBT_SUCCESS) { 3945 /* This cannot happen unless the device has an error. */ 3946 ibd_print_warn(state, "ibd_post_recv: FATAL: " 3947 "posting multiple wrs failed: " 3948 "requested=%d, done=%d, ret=%d", 3949 IBD_RX_POST_CNT, num_posted, ibt_status); 3950 atomic_add_32(&state->id_rx_list.dl_cnt, 3951 num_posted - i); 3952 } 3953 } 3954 } 3955 3956 /* 3957 * Grab a list of rwqes from the array of lists, and post the list. 3958 */ 3959 static void 3960 ibd_post_recv_intr(ibd_state_t *state) 3961 { 3962 ibd_rx_queue_t *rxp; 3963 ibd_rwqe_t *list; 3964 3965 /* rotate through the rx_queue array, expecting an adequate number */ 3966 state->id_rx_post_queue_index = 3967 (state->id_rx_post_queue_index + 1) & 3968 (state->id_rx_nqueues - 1); 3969 3970 rxp = state->id_rx_queues + state->id_rx_post_queue_index; 3971 mutex_enter(&rxp->rx_post_lock); 3972 list = WQE_TO_RWQE(rxp->rx_head); 3973 rxp->rx_head = NULL; 3974 rxp->rx_cnt = 0; 3975 mutex_exit(&rxp->rx_post_lock); 3976 ibd_post_recv_list(state, list); 3977 } 3978 3979 /* macro explained below */ 3980 #define RX_QUEUE_HASH(rwqe) \ 3981 (((uintptr_t)(rwqe) >> 8) & (state->id_rx_nqueues - 1)) 3982 3983 /* 3984 * Add a rwqe to one of the the Rx lists. If the list is large enough 3985 * (exactly IBD_RX_POST_CNT), post the list to the hardware. 3986 * 3987 * Note: one of 2^N lists is chosen via a hash. This is done 3988 * because using one list is contentious. If the first list is busy 3989 * (mutex_tryenter fails), use a second list (just call mutex_enter). 3990 * 3991 * The number 8 in RX_QUEUE_HASH is a random choice that provides 3992 * even distribution of mapping rwqes to the 2^N queues. 3993 */ 3994 static void 3995 ibd_post_recv(ibd_state_t *state, ibd_rwqe_t *rwqe) 3996 { 3997 ibd_rx_queue_t *rxp; 3998 3999 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe); 4000 4001 if (!mutex_tryenter(&rxp->rx_post_lock)) { 4002 /* Failed. Try a different queue ("ptr + 16" ensures that). */ 4003 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe + 16); 4004 mutex_enter(&rxp->rx_post_lock); 4005 } 4006 rwqe->rwqe_next = rxp->rx_head; 4007 if (++rxp->rx_cnt >= IBD_RX_POST_CNT - 2) { 4008 uint_t active = atomic_inc_32_nv(&state->id_rx_post_active); 4009 4010 /* only call ibt_post_recv() every Nth time through here */ 4011 if ((active & (state->id_rx_nqueues - 1)) == 0) { 4012 rxp->rx_head = NULL; 4013 rxp->rx_cnt = 0; 4014 mutex_exit(&rxp->rx_post_lock); 4015 ibd_post_recv_list(state, rwqe); 4016 return; 4017 } 4018 } 4019 rxp->rx_head = RWQE_TO_WQE(rwqe); 4020 mutex_exit(&rxp->rx_post_lock); 4021 } 4022 4023 static int 4024 ibd_alloc_rx_copybufs(ibd_state_t *state) 4025 { 4026 ibt_mr_attr_t mem_attr; 4027 int i; 4028 4029 /* 4030 * Allocate one big chunk for all regular rx copy bufs 4031 */ 4032 state->id_rx_buf_sz = state->id_mtu + IPOIB_GRH_SIZE; 4033 4034 state->id_rx_bufs = kmem_zalloc(state->id_ud_num_rwqe * 4035 state->id_rx_buf_sz, KM_SLEEP); 4036 4037 state->id_rx_wqes = kmem_zalloc(state->id_ud_num_rwqe * 4038 sizeof (ibd_rwqe_t), KM_SLEEP); 4039 4040 state->id_rx_nqueues = 1 << IBD_LOG_RX_POST; 4041 state->id_rx_queues = kmem_zalloc(state->id_rx_nqueues * 4042 sizeof (ibd_rx_queue_t), KM_SLEEP); 4043 for (i = 0; i < state->id_rx_nqueues; i++) { 4044 ibd_rx_queue_t *rxp = state->id_rx_queues + i; 4045 mutex_init(&rxp->rx_post_lock, NULL, MUTEX_DRIVER, NULL); 4046 } 4047 4048 /* 4049 * Do one memory registration on the entire rxbuf area 4050 */ 4051 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_rx_bufs; 4052 mem_attr.mr_len = state->id_ud_num_rwqe * state->id_rx_buf_sz; 4053 mem_attr.mr_as = NULL; 4054 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE; 4055 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr, 4056 &state->id_rx_mr_hdl, &state->id_rx_mr_desc) != IBT_SUCCESS) { 4057 DPRINT(10, "ibd_alloc_rx_copybufs: ibt_register_mr failed"); 4058 kmem_free(state->id_rx_wqes, 4059 state->id_ud_num_rwqe * sizeof (ibd_rwqe_t)); 4060 kmem_free(state->id_rx_bufs, 4061 state->id_ud_num_rwqe * state->id_rx_buf_sz); 4062 state->id_rx_bufs = NULL; 4063 state->id_rx_wqes = NULL; 4064 return (DDI_FAILURE); 4065 } 4066 4067 return (DDI_SUCCESS); 4068 } 4069 4070 /* 4071 * Allocate the statically allocated Rx buffer list. 4072 */ 4073 static int 4074 ibd_init_rxlist(ibd_state_t *state) 4075 { 4076 ibd_rwqe_t *rwqe, *next; 4077 ibd_wqe_t *list; 4078 ibt_lkey_t lkey; 4079 int i; 4080 uint_t len; 4081 uint8_t *bufaddr; 4082 4083 mutex_enter(&state->id_rx_free_list.dl_mutex); 4084 if (state->id_rx_free_list.dl_head != NULL) { 4085 /* rx rsrcs were never freed. Just repost them */ 4086 len = state->id_rx_buf_sz; 4087 list = state->id_rx_free_list.dl_head; 4088 state->id_rx_free_list.dl_head = NULL; 4089 state->id_rx_free_list.dl_cnt = 0; 4090 mutex_exit(&state->id_rx_free_list.dl_mutex); 4091 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL; 4092 rwqe = WQE_TO_RWQE(rwqe->rwqe_next)) { 4093 if ((rwqe->rwqe_im_mblk = desballoc( 4094 rwqe->rwqe_copybuf.ic_bufaddr, len, 0, 4095 &rwqe->w_freemsg_cb)) == NULL) { 4096 /* allow freemsg_cb to free the rwqes */ 4097 if (atomic_dec_32_nv(&state->id_running) != 0) { 4098 cmn_err(CE_WARN, "ibd_init_rxlist: " 4099 "id_running was not 1\n"); 4100 } 4101 DPRINT(10, "ibd_init_rxlist : " 4102 "failed in desballoc()"); 4103 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL; 4104 rwqe = next) { 4105 next = WQE_TO_RWQE(rwqe->rwqe_next); 4106 if (rwqe->rwqe_im_mblk) { 4107 atomic_inc_32(&state-> 4108 id_rx_list. 4109 dl_bufs_outstanding); 4110 freemsg(rwqe->rwqe_im_mblk); 4111 } else 4112 ibd_free_rwqe(state, rwqe); 4113 } 4114 atomic_inc_32(&state->id_running); 4115 return (DDI_FAILURE); 4116 } 4117 } 4118 ibd_post_recv_list(state, WQE_TO_RWQE(list)); 4119 return (DDI_SUCCESS); 4120 } 4121 mutex_exit(&state->id_rx_free_list.dl_mutex); 4122 4123 if (ibd_alloc_rx_copybufs(state) != DDI_SUCCESS) 4124 return (DDI_FAILURE); 4125 4126 /* 4127 * Allocate and setup the rwqe list 4128 */ 4129 len = state->id_rx_buf_sz; 4130 lkey = state->id_rx_mr_desc.md_lkey; 4131 rwqe = state->id_rx_wqes; 4132 bufaddr = state->id_rx_bufs; 4133 list = NULL; 4134 for (i = 0; i < state->id_ud_num_rwqe; i++, rwqe++, bufaddr += len) { 4135 rwqe->w_state = state; 4136 rwqe->w_freemsg_cb.free_func = ibd_freemsg_cb; 4137 rwqe->w_freemsg_cb.free_arg = (char *)rwqe; 4138 4139 rwqe->rwqe_copybuf.ic_bufaddr = bufaddr; 4140 4141 if ((rwqe->rwqe_im_mblk = desballoc(bufaddr, len, 0, 4142 &rwqe->w_freemsg_cb)) == NULL) { 4143 DPRINT(10, "ibd_init_rxlist : failed in desballoc()"); 4144 /* allow freemsg_cb to free the rwqes */ 4145 if (atomic_dec_32_nv(&state->id_running) != 0) { 4146 cmn_err(CE_WARN, "ibd_init_rxlist: " 4147 "id_running was not 1\n"); 4148 } 4149 DPRINT(10, "ibd_init_rxlist : " 4150 "failed in desballoc()"); 4151 for (rwqe = WQE_TO_RWQE(list); rwqe != NULL; 4152 rwqe = next) { 4153 next = WQE_TO_RWQE(rwqe->rwqe_next); 4154 freemsg(rwqe->rwqe_im_mblk); 4155 } 4156 atomic_inc_32(&state->id_running); 4157 4158 /* remove reference to free'd rwqes */ 4159 mutex_enter(&state->id_rx_free_list.dl_mutex); 4160 state->id_rx_free_list.dl_head = NULL; 4161 state->id_rx_free_list.dl_cnt = 0; 4162 mutex_exit(&state->id_rx_free_list.dl_mutex); 4163 4164 ibd_fini_rxlist(state); 4165 return (DDI_FAILURE); 4166 } 4167 4168 rwqe->rwqe_copybuf.ic_sgl.ds_key = lkey; 4169 rwqe->rwqe_copybuf.ic_sgl.ds_va = 4170 (ib_vaddr_t)(uintptr_t)bufaddr; 4171 rwqe->rwqe_copybuf.ic_sgl.ds_len = len; 4172 rwqe->w_rwr.wr_id = (ibt_wrid_t)(uintptr_t)rwqe; 4173 rwqe->w_rwr.wr_nds = 1; 4174 rwqe->w_rwr.wr_sgl = &rwqe->rwqe_copybuf.ic_sgl; 4175 4176 rwqe->rwqe_next = list; 4177 list = RWQE_TO_WQE(rwqe); 4178 } 4179 ibd_post_recv_list(state, WQE_TO_RWQE(list)); 4180 4181 return (DDI_SUCCESS); 4182 } 4183 4184 static void 4185 ibd_free_rx_copybufs(ibd_state_t *state) 4186 { 4187 int i; 4188 4189 /* 4190 * Unregister rxbuf mr 4191 */ 4192 if (ibt_deregister_mr(state->id_hca_hdl, 4193 state->id_rx_mr_hdl) != IBT_SUCCESS) { 4194 DPRINT(10, "ibd_free_rx_copybufs: ibt_deregister_mr failed"); 4195 } 4196 state->id_rx_mr_hdl = NULL; 4197 4198 /* 4199 * Free rxbuf memory 4200 */ 4201 for (i = 0; i < state->id_rx_nqueues; i++) { 4202 ibd_rx_queue_t *rxp = state->id_rx_queues + i; 4203 mutex_destroy(&rxp->rx_post_lock); 4204 } 4205 kmem_free(state->id_rx_queues, state->id_rx_nqueues * 4206 sizeof (ibd_rx_queue_t)); 4207 kmem_free(state->id_rx_wqes, state->id_ud_num_rwqe * 4208 sizeof (ibd_rwqe_t)); 4209 kmem_free(state->id_rx_bufs, state->id_ud_num_rwqe * 4210 state->id_rx_buf_sz); 4211 state->id_rx_queues = NULL; 4212 state->id_rx_wqes = NULL; 4213 state->id_rx_bufs = NULL; 4214 } 4215 4216 static void 4217 ibd_free_rx_rsrcs(ibd_state_t *state) 4218 { 4219 mutex_enter(&state->id_rx_free_list.dl_mutex); 4220 if (state->id_rx_free_list.dl_head == NULL) { 4221 /* already freed */ 4222 mutex_exit(&state->id_rx_free_list.dl_mutex); 4223 return; 4224 } 4225 ASSERT(state->id_rx_free_list.dl_cnt == state->id_ud_num_rwqe); 4226 ibd_free_rx_copybufs(state); 4227 state->id_rx_free_list.dl_cnt = 0; 4228 state->id_rx_free_list.dl_head = NULL; 4229 mutex_exit(&state->id_rx_free_list.dl_mutex); 4230 } 4231 4232 /* 4233 * Free the statically allocated Rx buffer list. 4234 */ 4235 static void 4236 ibd_fini_rxlist(ibd_state_t *state) 4237 { 4238 ibd_rwqe_t *rwqe; 4239 int i; 4240 4241 /* run through the rx_queue's, calling freemsg() */ 4242 for (i = 0; i < state->id_rx_nqueues; i++) { 4243 ibd_rx_queue_t *rxp = state->id_rx_queues + i; 4244 mutex_enter(&rxp->rx_post_lock); 4245 for (rwqe = WQE_TO_RWQE(rxp->rx_head); rwqe; 4246 rwqe = WQE_TO_RWQE(rwqe->rwqe_next)) { 4247 freemsg(rwqe->rwqe_im_mblk); 4248 rxp->rx_cnt--; 4249 } 4250 rxp->rx_head = NULL; 4251 mutex_exit(&rxp->rx_post_lock); 4252 } 4253 4254 /* cannot free rx resources unless gld returned everything */ 4255 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) == 0) 4256 ibd_free_rx_rsrcs(state); 4257 } 4258 4259 /* 4260 * Free an allocated recv wqe. 4261 */ 4262 /* ARGSUSED */ 4263 static void 4264 ibd_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe) 4265 { 4266 /* 4267 * desballoc() failed (no memory). 4268 * 4269 * This rwqe is placed on a free list so that it 4270 * can be reinstated when memory is available. 4271 * 4272 * NOTE: no code currently exists to reinstate 4273 * these "lost" rwqes. 4274 */ 4275 mutex_enter(&state->id_rx_free_list.dl_mutex); 4276 state->id_rx_free_list.dl_cnt++; 4277 rwqe->rwqe_next = state->id_rx_free_list.dl_head; 4278 state->id_rx_free_list.dl_head = RWQE_TO_WQE(rwqe); 4279 mutex_exit(&state->id_rx_free_list.dl_mutex); 4280 } 4281 4282 /* 4283 * IBA Rx completion queue handler. Guaranteed to be single 4284 * threaded and nonreentrant for this CQ. 4285 */ 4286 /* ARGSUSED */ 4287 static void 4288 ibd_rcq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 4289 { 4290 ibd_state_t *state = (ibd_state_t *)arg; 4291 4292 atomic_inc_64(&state->id_num_intrs); 4293 4294 if (ibd_rx_softintr == 1) { 4295 mutex_enter(&state->id_rcq_poll_lock); 4296 if (state->id_rcq_poll_busy & IBD_CQ_POLLING) { 4297 state->id_rcq_poll_busy |= IBD_REDO_CQ_POLLING; 4298 mutex_exit(&state->id_rcq_poll_lock); 4299 return; 4300 } else { 4301 mutex_exit(&state->id_rcq_poll_lock); 4302 ddi_trigger_softintr(state->id_rx); 4303 } 4304 } else 4305 (void) ibd_intr((caddr_t)state); 4306 } 4307 4308 /* 4309 * CQ handler for Tx completions, when the Tx CQ is in 4310 * interrupt driven mode. 4311 */ 4312 /* ARGSUSED */ 4313 static void 4314 ibd_scq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 4315 { 4316 ibd_state_t *state = (ibd_state_t *)arg; 4317 4318 atomic_inc_64(&state->id_num_intrs); 4319 4320 if (ibd_tx_softintr == 1) { 4321 mutex_enter(&state->id_scq_poll_lock); 4322 if (state->id_scq_poll_busy & IBD_CQ_POLLING) { 4323 state->id_scq_poll_busy |= IBD_REDO_CQ_POLLING; 4324 mutex_exit(&state->id_scq_poll_lock); 4325 return; 4326 } else { 4327 mutex_exit(&state->id_scq_poll_lock); 4328 ddi_trigger_softintr(state->id_tx); 4329 } 4330 } else 4331 (void) ibd_tx_recycle((caddr_t)state); 4332 } 4333 4334 /* 4335 * Multicast group create/delete trap handler. These will be delivered 4336 * on a kernel thread (handling can thus block) and can be invoked 4337 * concurrently. The handler can be invoked anytime after it is 4338 * registered and before ibt_detach(). 4339 */ 4340 /* ARGSUSED */ 4341 static void 4342 ibd_snet_notices_handler(void *arg, ib_gid_t gid, ibt_subnet_event_code_t code, 4343 ibt_subnet_event_t *event) 4344 { 4345 ibd_state_t *state = (ibd_state_t *)arg; 4346 ibd_req_t *req; 4347 4348 /* 4349 * The trap handler will get invoked once for every event for 4350 * every port. The input "gid" is the GID0 of the port the 4351 * trap came in on; we just need to act on traps that came 4352 * to our port, meaning the port on which the ipoib interface 4353 * resides. Since ipoib uses GID0 of the port, we just match 4354 * the gids to check whether we need to handle the trap. 4355 */ 4356 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) 4357 if (bcmp(&gid, &state->id_sgid, sizeof (ib_gid_t)) != 0) 4358 return; 4359 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) 4360 4361 DPRINT(10, "ibd_notices_handler : %d\n", code); 4362 4363 switch (code) { 4364 case IBT_SM_EVENT_UNAVAILABLE: 4365 /* 4366 * If we are in promiscuous mode or have 4367 * sendnonmembers, we need to print a warning 4368 * message right now. Else, just store the 4369 * information, print when we enter promiscuous 4370 * mode or attempt nonmember send. We might 4371 * also want to stop caching sendnonmember. 4372 */ 4373 ibd_print_warn(state, "IBA multicast support " 4374 "degraded due to unavailability of multicast " 4375 "traps"); 4376 break; 4377 case IBT_SM_EVENT_AVAILABLE: 4378 /* 4379 * If we printed a warning message above or 4380 * while trying to nonmember send or get into 4381 * promiscuous mode, print an okay message. 4382 */ 4383 ibd_print_warn(state, "IBA multicast support " 4384 "restored due to availability of multicast " 4385 "traps"); 4386 break; 4387 case IBT_SM_EVENT_MCG_CREATED: 4388 case IBT_SM_EVENT_MCG_DELETED: 4389 /* 4390 * If it is a "deleted" event and we are in late hca 4391 * init, nothing to do. 4392 */ 4393 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 4394 IBD_DRV_IN_LATE_HCA_INIT) && (code == 4395 IBT_SM_EVENT_MCG_DELETED)) { 4396 break; 4397 } 4398 /* 4399 * Common processing of creation/deletion traps. 4400 * First check if the instance is being 4401 * [de]initialized; back off then, without doing 4402 * anything more, since we are not sure if the 4403 * async thread is around, or whether we might 4404 * be racing with the detach code in ibd_m_stop() 4405 * that scans the mcg list. 4406 */ 4407 if (!ibd_async_safe(state)) 4408 return; 4409 4410 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP); 4411 req->rq_gid = event->sm_notice_gid; 4412 req->rq_ptr = (void *)code; 4413 ibd_queue_work_slot(state, req, IBD_ASYNC_TRAP); 4414 break; 4415 } 4416 } 4417 4418 static void 4419 ibd_async_trap(ibd_state_t *state, ibd_req_t *req) 4420 { 4421 ib_gid_t mgid = req->rq_gid; 4422 ibt_subnet_event_code_t code = (ibt_subnet_event_code_t)req->rq_ptr; 4423 int ret; 4424 ib_pkey_t pkey = (mgid.gid_prefix >> 16) & 0xffff; 4425 4426 DPRINT(10, "ibd_async_trap : %d\n", code); 4427 4428 /* 4429 * Check if we have already joined the IPoIB broadcast group for our 4430 * PKEY. If joined, perform the rest of the operation. 4431 * Else, the interface is not initialised. Do the initialisation here 4432 * by calling ibd_start() and return. 4433 */ 4434 4435 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 4436 IBD_DRV_IN_LATE_HCA_INIT) && (state->id_bgroup_present == 0) && 4437 (code == IBT_SM_EVENT_MCG_CREATED)) { 4438 /* 4439 * If we are in late HCA init and a notification for the 4440 * creation of a MCG came in, check if it is the IPoIB MCG for 4441 * this pkey. If not, return. 4442 */ 4443 if ((mgid.gid_guid != IB_MGID_IPV4_LOWGRP_MASK) || (pkey != 4444 state->id_pkey)) { 4445 ibd_async_done(state); 4446 return; 4447 } 4448 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); 4449 /* 4450 * Check if there is still a necessity to start the interface. 4451 * It is possible that the user attempted unplumb at just about 4452 * the same time, and if unplumb succeeded, we have nothing to 4453 * do. 4454 */ 4455 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 4456 IBD_DRV_IN_LATE_HCA_INIT) && 4457 ((ret = ibd_start(state)) != 0)) { 4458 DPRINT(10, "ibd_async_trap: cannot start from late HCA " 4459 "init, ret=%d", ret); 4460 } 4461 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS); 4462 ibd_async_done(state); 4463 return; 4464 } 4465 4466 /* 4467 * Atomically search the nonmember and sendonlymember lists and 4468 * delete. 4469 */ 4470 ibd_leave_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON); 4471 4472 if (state->id_prom_op == IBD_OP_COMPLETED) { 4473 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON); 4474 4475 /* 4476 * If in promiscuous mode, try to join/attach to the new 4477 * mcg. Given the unreliable out-of-order mode of trap 4478 * delivery, we can never be sure whether it is a problem 4479 * if the join fails. Thus, we warn the admin of a failure 4480 * if this was a creation trap. Note that the trap might 4481 * actually be reporting a long past event, and the mcg 4482 * might already have been deleted, thus we might be warning 4483 * in vain. 4484 */ 4485 if ((ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == 4486 NULL) && (code == IBT_SM_EVENT_MCG_CREATED)) 4487 ibd_print_warn(state, "IBA promiscuous mode missed " 4488 "new multicast gid %016llx:%016llx", 4489 (u_longlong_t)mgid.gid_prefix, 4490 (u_longlong_t)mgid.gid_guid); 4491 } 4492 4493 /* 4494 * Free the request slot allocated by the subnet event thread. 4495 */ 4496 ibd_async_done(state); 4497 } 4498 4499 /* 4500 * GLDv3 entry point to get capabilities. 4501 */ 4502 static boolean_t 4503 ibd_m_getcapab(void *arg, mac_capab_t cap, void *cap_data) 4504 { 4505 ibd_state_t *state = arg; 4506 4507 if (state->id_type == IBD_PORT_DRIVER) 4508 return (B_FALSE); 4509 4510 switch (cap) { 4511 case MAC_CAPAB_HCKSUM: { 4512 uint32_t *txflags = cap_data; 4513 4514 /* 4515 * We either do full checksum or not do it at all 4516 */ 4517 if (state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) 4518 *txflags = HCK_FULLCKSUM | HCKSUM_INET_FULL_V4; 4519 else 4520 return (B_FALSE); 4521 break; 4522 } 4523 4524 case MAC_CAPAB_LSO: { 4525 mac_capab_lso_t *cap_lso = cap_data; 4526 4527 /* 4528 * In addition to the capability and policy, since LSO 4529 * relies on hw checksum, we'll not enable LSO if we 4530 * don't have hw checksum. Of course, if the HCA doesn't 4531 * provide the reserved lkey capability, enabling LSO will 4532 * actually affect performance adversely, so we'll disable 4533 * LSO even for that case. 4534 */ 4535 if (!state->id_lso_policy || !state->id_lso_capable) 4536 return (B_FALSE); 4537 4538 if ((state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) == 0) 4539 return (B_FALSE); 4540 4541 if (state->id_hca_res_lkey_capab == 0) { 4542 ibd_print_warn(state, "no reserved-lkey capability, " 4543 "disabling LSO"); 4544 return (B_FALSE); 4545 } 4546 4547 cap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4; 4548 cap_lso->lso_basic_tcp_ipv4.lso_max = state->id_lso_maxlen - 1; 4549 break; 4550 } 4551 4552 default: 4553 return (B_FALSE); 4554 } 4555 4556 return (B_TRUE); 4557 } 4558 4559 /* 4560 * callback function for set/get of properties 4561 */ 4562 static int 4563 ibd_m_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, 4564 uint_t pr_valsize, const void *pr_val) 4565 { 4566 ibd_state_t *state = arg; 4567 int err = 0; 4568 uint32_t link_mode; 4569 4570 /* Cannot set properties on a port driver */ 4571 if (state->id_type == IBD_PORT_DRIVER) { 4572 return (ENOTSUP); 4573 } 4574 4575 switch (pr_num) { 4576 case MAC_PROP_IB_LINKMODE: 4577 if (state->id_mac_state & IBD_DRV_STARTED) { 4578 err = EBUSY; 4579 break; 4580 } 4581 if (pr_val == NULL) { 4582 err = EINVAL; 4583 break; 4584 } 4585 bcopy(pr_val, &link_mode, sizeof (link_mode)); 4586 if (link_mode != IBD_LINK_MODE_UD && 4587 link_mode != IBD_LINK_MODE_RC) { 4588 err = EINVAL; 4589 } else { 4590 if (link_mode == IBD_LINK_MODE_RC) { 4591 if (state->id_enable_rc) { 4592 return (0); 4593 } 4594 state->id_enable_rc = 1; 4595 /* inform MAC framework of new MTU */ 4596 err = mac_maxsdu_update2(state->id_mh, 4597 state->rc_mtu - IPOIB_HDRSIZE, 4598 state->id_mtu - IPOIB_HDRSIZE); 4599 } else { 4600 if (!state->id_enable_rc) { 4601 return (0); 4602 } 4603 state->id_enable_rc = 0; 4604 err = mac_maxsdu_update2(state->id_mh, 4605 state->id_mtu - IPOIB_HDRSIZE, 4606 state->id_mtu - IPOIB_HDRSIZE); 4607 } 4608 (void) ibd_record_capab(state); 4609 mac_capab_update(state->id_mh); 4610 } 4611 break; 4612 case MAC_PROP_PRIVATE: 4613 err = ibd_set_priv_prop(state, pr_name, 4614 pr_valsize, pr_val); 4615 break; 4616 default: 4617 err = ENOTSUP; 4618 break; 4619 } 4620 return (err); 4621 } 4622 4623 static int 4624 ibd_m_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num, 4625 uint_t pr_valsize, void *pr_val) 4626 { 4627 ibd_state_t *state = arg; 4628 int err = 0; 4629 4630 switch (pr_num) { 4631 case MAC_PROP_MTU: 4632 break; 4633 default: 4634 if (state->id_type == IBD_PORT_DRIVER) { 4635 return (ENOTSUP); 4636 } 4637 break; 4638 } 4639 4640 switch (pr_num) { 4641 case MAC_PROP_IB_LINKMODE: 4642 *(uint_t *)pr_val = state->id_enable_rc; 4643 break; 4644 case MAC_PROP_PRIVATE: 4645 err = ibd_get_priv_prop(state, pr_name, pr_valsize, 4646 pr_val); 4647 break; 4648 default: 4649 err = ENOTSUP; 4650 break; 4651 } 4652 return (err); 4653 } 4654 4655 static void 4656 ibd_m_propinfo(void *arg, const char *pr_name, mac_prop_id_t pr_num, 4657 mac_prop_info_handle_t prh) 4658 { 4659 ibd_state_t *state = arg; 4660 4661 switch (pr_num) { 4662 case MAC_PROP_IB_LINKMODE: { 4663 mac_prop_info_set_default_uint32(prh, IBD_DEF_LINK_MODE); 4664 break; 4665 } 4666 case MAC_PROP_MTU: { 4667 uint32_t min, max; 4668 if (state->id_type == IBD_PORT_DRIVER) { 4669 min = 1500; 4670 max = IBD_DEF_RC_MAX_SDU; 4671 } else if (state->id_enable_rc) { 4672 min = max = IBD_DEF_RC_MAX_SDU; 4673 } else { 4674 min = max = state->id_mtu - IPOIB_HDRSIZE; 4675 } 4676 mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); 4677 mac_prop_info_set_range_uint32(prh, min, max); 4678 break; 4679 } 4680 case MAC_PROP_PRIVATE: { 4681 char valstr[64]; 4682 int value; 4683 4684 if (strcmp(pr_name, "_ibd_broadcast_group") == 0) { 4685 mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ); 4686 return; 4687 } else if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) { 4688 value = IBD_DEF_COALESCE_COMPLETIONS; 4689 } else if (strcmp(pr_name, 4690 "_ibd_create_broadcast_group") == 0) { 4691 value = IBD_DEF_CREATE_BCAST_GROUP; 4692 } else if (strcmp(pr_name, "_ibd_hash_size") == 0) { 4693 value = IBD_DEF_HASH_SIZE; 4694 } else if (strcmp(pr_name, "_ibd_lso_enable") == 0) { 4695 value = IBD_DEF_LSO_POLICY; 4696 } else if (strcmp(pr_name, "_ibd_num_ah") == 0) { 4697 value = IBD_DEF_NUM_AH; 4698 } else if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) { 4699 value = IBD_DEF_NUM_LSO_BUFS; 4700 } else if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) { 4701 value = IBD_DEF_RC_ENABLE_SRQ; 4702 } else if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) { 4703 value = IBD_DEF_RC_NUM_RWQE; 4704 } else if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) { 4705 value = IBD_DEF_RC_NUM_SRQ; 4706 } else if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) { 4707 value = IBD_DEF_RC_NUM_SWQE; 4708 } else if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) { 4709 value = IBD_DEF_RC_RX_COMP_COUNT; 4710 } else if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) { 4711 value = IBD_DEF_RC_RX_COMP_USEC; 4712 } else if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) { 4713 value = IBD_DEF_RC_RX_COPY_THRESH; 4714 } else if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) { 4715 value = IBD_DEF_RC_RX_RWQE_THRESH; 4716 } else if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) { 4717 value = IBD_DEF_RC_TX_COMP_COUNT; 4718 } else if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) { 4719 value = IBD_DEF_RC_TX_COMP_USEC; 4720 } else if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) { 4721 value = IBD_DEF_RC_TX_COPY_THRESH; 4722 } else if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) { 4723 value = IBD_DEF_UD_NUM_RWQE; 4724 } else if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) { 4725 value = IBD_DEF_UD_NUM_SWQE; 4726 } else if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) { 4727 value = IBD_DEF_UD_RX_COMP_COUNT; 4728 } else if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) { 4729 value = IBD_DEF_UD_RX_COMP_USEC; 4730 } else if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) { 4731 value = IBD_DEF_UD_TX_COMP_COUNT; 4732 } else if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) { 4733 value = IBD_DEF_UD_TX_COMP_USEC; 4734 } else if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) { 4735 value = IBD_DEF_UD_TX_COPY_THRESH; 4736 } else { 4737 return; 4738 } 4739 4740 (void) snprintf(valstr, sizeof (valstr), "%d", value); 4741 mac_prop_info_set_default_str(prh, valstr); 4742 break; 4743 } 4744 } /* switch (pr_num) */ 4745 } 4746 4747 /* ARGSUSED2 */ 4748 static int 4749 ibd_set_priv_prop(ibd_state_t *state, const char *pr_name, 4750 uint_t pr_valsize, const void *pr_val) 4751 { 4752 int err = 0; 4753 long result; 4754 4755 if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) { 4756 if (pr_val == NULL) { 4757 return (EINVAL); 4758 } 4759 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4760 if (result < 0 || result > 1) { 4761 err = EINVAL; 4762 } else { 4763 state->id_allow_coalesce_comp_tuning = (result == 1) ? 4764 B_TRUE: B_FALSE; 4765 } 4766 return (err); 4767 } 4768 if (strcmp(pr_name, "_ibd_create_broadcast_group") == 0) { 4769 if (state->id_mac_state & IBD_DRV_STARTED) { 4770 return (EBUSY); 4771 } 4772 if (pr_val == NULL) { 4773 return (EINVAL); 4774 } 4775 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4776 if (result < 0 || result > 1) { 4777 err = EINVAL; 4778 } else { 4779 state->id_create_broadcast_group = (result == 1) ? 4780 B_TRUE: B_FALSE; 4781 } 4782 return (err); 4783 } 4784 if (strcmp(pr_name, "_ibd_hash_size") == 0) { 4785 if (state->id_mac_state & IBD_DRV_STARTED) { 4786 return (EBUSY); 4787 } 4788 if (pr_val == NULL) { 4789 return (EINVAL); 4790 } 4791 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4792 if (result < IBD_MIN_HASH_SIZE || result > IBD_MAX_HASH_SIZE) { 4793 err = EINVAL; 4794 } else { 4795 state->id_hash_size = (uint32_t)result; 4796 } 4797 return (err); 4798 } 4799 if (strcmp(pr_name, "_ibd_lso_enable") == 0) { 4800 if (state->id_mac_state & IBD_DRV_STARTED) { 4801 return (EBUSY); 4802 } 4803 if (pr_val == NULL) { 4804 return (EINVAL); 4805 } 4806 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4807 if (result < 0 || result > 1) { 4808 err = EINVAL; 4809 } else { 4810 state->id_lso_policy = (result == 1) ? 4811 B_TRUE: B_FALSE; 4812 } 4813 mac_capab_update(state->id_mh); 4814 return (err); 4815 } 4816 if (strcmp(pr_name, "_ibd_num_ah") == 0) { 4817 if (state->id_mac_state & IBD_DRV_STARTED) { 4818 return (EBUSY); 4819 } 4820 if (pr_val == NULL) { 4821 return (EINVAL); 4822 } 4823 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4824 if (result < IBD_MIN_NUM_AH || result > IBD_MAX_NUM_AH) { 4825 err = EINVAL; 4826 } else { 4827 state->id_num_ah = (uint32_t)result; 4828 } 4829 return (err); 4830 } 4831 if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) { 4832 if (state->id_mac_state & IBD_DRV_STARTED) { 4833 return (EBUSY); 4834 } 4835 if (!state->id_lso_policy || !state->id_lso_capable) { 4836 return (EINVAL); 4837 } 4838 if (pr_val == NULL) { 4839 return (EINVAL); 4840 } 4841 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4842 if (result < IBD_MIN_NUM_LSO_BUFS || 4843 result > IBD_MAX_NUM_LSO_BUFS) { 4844 err = EINVAL; 4845 } else { 4846 state->id_num_lso_bufs = (uint32_t)result; 4847 } 4848 return (err); 4849 } 4850 if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) { 4851 if (state->id_mac_state & IBD_DRV_STARTED) { 4852 return (EBUSY); 4853 } 4854 if (pr_val == NULL) { 4855 return (EINVAL); 4856 } 4857 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4858 if (result < 0 || result > 1) { 4859 err = EINVAL; 4860 } else { 4861 state->rc_enable_srq = (result == 1) ? 4862 B_TRUE: B_FALSE; 4863 } 4864 if (!state->rc_enable_srq) { 4865 state->id_rc_num_srq = 0; 4866 } 4867 return (err); 4868 } 4869 if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) { 4870 if (state->id_mac_state & IBD_DRV_STARTED) { 4871 return (EBUSY); 4872 } 4873 if (pr_val == NULL) { 4874 return (EINVAL); 4875 } 4876 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4877 if (result < IBD_MIN_RC_NUM_RWQE || 4878 result > IBD_MAX_RC_NUM_RWQE) { 4879 err = EINVAL; 4880 } else { 4881 state->id_rc_num_rwqe = (uint32_t)result; 4882 if (state->id_allow_coalesce_comp_tuning && 4883 state->id_rc_rx_comp_count > state->id_rc_num_rwqe) 4884 state->id_rc_rx_comp_count = 4885 state->id_rc_num_rwqe; 4886 if (state->id_rc_num_srq > state->id_rc_num_rwqe) 4887 state->id_rc_num_srq = 4888 state->id_rc_num_rwqe - 1; 4889 /* 4890 * If rx_rwqe_threshold is greater than the number of 4891 * rwqes, pull it back to 25% of number of rwqes. 4892 */ 4893 if (state->id_rc_rx_rwqe_thresh > state->id_rc_num_rwqe) 4894 state->id_rc_rx_rwqe_thresh = 4895 (state->id_rc_num_rwqe >> 2); 4896 4897 } 4898 return (err); 4899 } 4900 if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) { 4901 if (state->id_mac_state & IBD_DRV_STARTED) { 4902 return (EBUSY); 4903 } 4904 if (pr_val == NULL) { 4905 return (EINVAL); 4906 } 4907 if (!state->rc_enable_srq) 4908 return (EINVAL); 4909 4910 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4911 if (result < IBD_MIN_RC_NUM_SRQ || 4912 result >= state->id_rc_num_rwqe) { 4913 err = EINVAL; 4914 } else 4915 state->id_rc_num_srq = (uint32_t)result; 4916 return (err); 4917 } 4918 if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) { 4919 if (state->id_mac_state & IBD_DRV_STARTED) { 4920 return (EBUSY); 4921 } 4922 if (pr_val == NULL) { 4923 return (EINVAL); 4924 } 4925 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4926 if (result < IBD_MIN_RC_NUM_SWQE || 4927 result > IBD_MAX_RC_NUM_SWQE) { 4928 err = EINVAL; 4929 } else { 4930 state->id_rc_num_swqe = (uint32_t)result; 4931 if (state->id_allow_coalesce_comp_tuning && 4932 state->id_rc_tx_comp_count > state->id_rc_num_swqe) 4933 state->id_rc_tx_comp_count = 4934 state->id_rc_num_swqe; 4935 } 4936 return (err); 4937 } 4938 if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) { 4939 if (!state->id_allow_coalesce_comp_tuning) { 4940 return (ENOTSUP); 4941 } 4942 if (pr_val == NULL) { 4943 return (EINVAL); 4944 } 4945 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4946 if (result < 1 || result > state->id_rc_num_rwqe) { 4947 err = EINVAL; 4948 } else { 4949 state->id_rc_rx_comp_count = (uint32_t)result; 4950 } 4951 return (err); 4952 } 4953 if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) { 4954 if (!state->id_allow_coalesce_comp_tuning) { 4955 return (ENOTSUP); 4956 } 4957 if (pr_val == NULL) { 4958 return (EINVAL); 4959 } 4960 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4961 if (result < 1) { 4962 err = EINVAL; 4963 } else { 4964 state->id_rc_rx_comp_usec = (uint32_t)result; 4965 } 4966 return (err); 4967 } 4968 if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) { 4969 if (state->id_mac_state & IBD_DRV_STARTED) { 4970 return (EBUSY); 4971 } 4972 if (pr_val == NULL) { 4973 return (EINVAL); 4974 } 4975 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4976 if (result < IBD_MIN_RC_RX_COPY_THRESH || 4977 result > state->rc_mtu) { 4978 err = EINVAL; 4979 } else { 4980 state->id_rc_rx_copy_thresh = (uint32_t)result; 4981 } 4982 return (err); 4983 } 4984 if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) { 4985 if (state->id_mac_state & IBD_DRV_STARTED) { 4986 return (EBUSY); 4987 } 4988 if (pr_val == NULL) { 4989 return (EINVAL); 4990 } 4991 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 4992 if (result < IBD_MIN_RC_RX_RWQE_THRESH || 4993 result >= state->id_rc_num_rwqe) { 4994 err = EINVAL; 4995 } else { 4996 state->id_rc_rx_rwqe_thresh = (uint32_t)result; 4997 } 4998 return (err); 4999 } 5000 if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) { 5001 if (!state->id_allow_coalesce_comp_tuning) { 5002 return (ENOTSUP); 5003 } 5004 if (pr_val == NULL) { 5005 return (EINVAL); 5006 } 5007 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5008 if (result < 1 || result > state->id_rc_num_swqe) { 5009 err = EINVAL; 5010 } else { 5011 state->id_rc_tx_comp_count = (uint32_t)result; 5012 } 5013 return (err); 5014 } 5015 if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) { 5016 if (!state->id_allow_coalesce_comp_tuning) { 5017 return (ENOTSUP); 5018 } 5019 if (pr_val == NULL) { 5020 return (EINVAL); 5021 } 5022 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5023 if (result < 1) 5024 err = EINVAL; 5025 else { 5026 state->id_rc_tx_comp_usec = (uint32_t)result; 5027 } 5028 return (err); 5029 } 5030 if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) { 5031 if (state->id_mac_state & IBD_DRV_STARTED) { 5032 return (EBUSY); 5033 } 5034 if (pr_val == NULL) { 5035 return (EINVAL); 5036 } 5037 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5038 if (result < IBD_MIN_RC_TX_COPY_THRESH || 5039 result > state->rc_mtu) { 5040 err = EINVAL; 5041 } else { 5042 state->id_rc_tx_copy_thresh = (uint32_t)result; 5043 } 5044 return (err); 5045 } 5046 if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) { 5047 if (state->id_mac_state & IBD_DRV_STARTED) { 5048 return (EBUSY); 5049 } 5050 if (pr_val == NULL) { 5051 return (EINVAL); 5052 } 5053 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5054 if (result < IBD_MIN_UD_NUM_RWQE || 5055 result > IBD_MAX_UD_NUM_RWQE) { 5056 err = EINVAL; 5057 } else { 5058 if (result > state->id_hca_max_chan_sz) { 5059 state->id_ud_num_rwqe = 5060 state->id_hca_max_chan_sz; 5061 } else { 5062 state->id_ud_num_rwqe = (uint32_t)result; 5063 } 5064 if (state->id_allow_coalesce_comp_tuning && 5065 state->id_ud_rx_comp_count > state->id_ud_num_rwqe) 5066 state->id_ud_rx_comp_count = 5067 state->id_ud_num_rwqe; 5068 } 5069 return (err); 5070 } 5071 if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) { 5072 if (state->id_mac_state & IBD_DRV_STARTED) { 5073 return (EBUSY); 5074 } 5075 if (pr_val == NULL) { 5076 return (EINVAL); 5077 } 5078 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5079 if (result < IBD_MIN_UD_NUM_SWQE || 5080 result > IBD_MAX_UD_NUM_SWQE) { 5081 err = EINVAL; 5082 } else { 5083 if (result > state->id_hca_max_chan_sz) { 5084 state->id_ud_num_swqe = 5085 state->id_hca_max_chan_sz; 5086 } else { 5087 state->id_ud_num_swqe = (uint32_t)result; 5088 } 5089 if (state->id_allow_coalesce_comp_tuning && 5090 state->id_ud_tx_comp_count > state->id_ud_num_swqe) 5091 state->id_ud_tx_comp_count = 5092 state->id_ud_num_swqe; 5093 } 5094 return (err); 5095 } 5096 if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) { 5097 if (!state->id_allow_coalesce_comp_tuning) { 5098 return (ENOTSUP); 5099 } 5100 if (pr_val == NULL) { 5101 return (EINVAL); 5102 } 5103 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5104 if (result < 1 || result > state->id_ud_num_rwqe) { 5105 err = EINVAL; 5106 } else { 5107 state->id_ud_rx_comp_count = (uint32_t)result; 5108 } 5109 return (err); 5110 } 5111 if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) { 5112 if (!state->id_allow_coalesce_comp_tuning) { 5113 return (ENOTSUP); 5114 } 5115 if (pr_val == NULL) { 5116 return (EINVAL); 5117 } 5118 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5119 if (result < 1) { 5120 err = EINVAL; 5121 } else { 5122 state->id_ud_rx_comp_usec = (uint32_t)result; 5123 } 5124 return (err); 5125 } 5126 if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) { 5127 if (!state->id_allow_coalesce_comp_tuning) { 5128 return (ENOTSUP); 5129 } 5130 if (pr_val == NULL) { 5131 return (EINVAL); 5132 } 5133 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5134 if (result < 1 || result > state->id_ud_num_swqe) { 5135 err = EINVAL; 5136 } else { 5137 state->id_ud_tx_comp_count = (uint32_t)result; 5138 } 5139 return (err); 5140 } 5141 if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) { 5142 if (!state->id_allow_coalesce_comp_tuning) { 5143 return (ENOTSUP); 5144 } 5145 if (pr_val == NULL) { 5146 return (EINVAL); 5147 } 5148 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5149 if (result < 1) { 5150 err = EINVAL; 5151 } else { 5152 state->id_ud_tx_comp_usec = (uint32_t)result; 5153 } 5154 return (err); 5155 } 5156 if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) { 5157 if (state->id_mac_state & IBD_DRV_STARTED) { 5158 return (EBUSY); 5159 } 5160 if (pr_val == NULL) { 5161 return (EINVAL); 5162 } 5163 (void) ddi_strtol(pr_val, (char **)NULL, 0, &result); 5164 if (result < IBD_MIN_UD_TX_COPY_THRESH || 5165 result > IBD_MAX_UD_TX_COPY_THRESH) { 5166 err = EINVAL; 5167 } else { 5168 state->id_ud_tx_copy_thresh = (uint32_t)result; 5169 } 5170 return (err); 5171 } 5172 return (ENOTSUP); 5173 } 5174 5175 static int 5176 ibd_get_priv_prop(ibd_state_t *state, const char *pr_name, uint_t pr_valsize, 5177 void *pr_val) 5178 { 5179 int err = ENOTSUP; 5180 int value; 5181 5182 if (strcmp(pr_name, "_ibd_broadcast_group") == 0) { 5183 value = state->id_bgroup_present; 5184 err = 0; 5185 goto done; 5186 } 5187 if (strcmp(pr_name, "_ibd_coalesce_completions") == 0) { 5188 value = state->id_allow_coalesce_comp_tuning; 5189 err = 0; 5190 goto done; 5191 } 5192 if (strcmp(pr_name, "_ibd_create_broadcast_group") == 0) { 5193 value = state->id_create_broadcast_group; 5194 err = 0; 5195 goto done; 5196 } 5197 if (strcmp(pr_name, "_ibd_hash_size") == 0) { 5198 value = state->id_hash_size; 5199 err = 0; 5200 goto done; 5201 } 5202 if (strcmp(pr_name, "_ibd_lso_enable") == 0) { 5203 value = state->id_lso_policy; 5204 err = 0; 5205 goto done; 5206 } 5207 if (strcmp(pr_name, "_ibd_num_ah") == 0) { 5208 value = state->id_num_ah; 5209 err = 0; 5210 goto done; 5211 } 5212 if (strcmp(pr_name, "_ibd_num_lso_bufs") == 0) { 5213 value = state->id_num_lso_bufs; 5214 err = 0; 5215 goto done; 5216 } 5217 if (strcmp(pr_name, "_ibd_rc_enable_srq") == 0) { 5218 value = state->rc_enable_srq; 5219 err = 0; 5220 goto done; 5221 } 5222 if (strcmp(pr_name, "_ibd_rc_num_rwqe") == 0) { 5223 value = state->id_rc_num_rwqe; 5224 err = 0; 5225 goto done; 5226 } 5227 if (strcmp(pr_name, "_ibd_rc_num_srq") == 0) { 5228 value = state->id_rc_num_srq; 5229 err = 0; 5230 goto done; 5231 } 5232 if (strcmp(pr_name, "_ibd_rc_num_swqe") == 0) { 5233 value = state->id_rc_num_swqe; 5234 err = 0; 5235 goto done; 5236 } 5237 if (strcmp(pr_name, "_ibd_rc_rx_comp_count") == 0) { 5238 value = state->id_rc_rx_comp_count; 5239 err = 0; 5240 goto done; 5241 } 5242 if (strcmp(pr_name, "_ibd_rc_rx_comp_usec") == 0) { 5243 value = state->id_rc_rx_comp_usec; 5244 err = 0; 5245 goto done; 5246 } 5247 if (strcmp(pr_name, "_ibd_rc_rx_copy_thresh") == 0) { 5248 value = state->id_rc_rx_copy_thresh; 5249 err = 0; 5250 goto done; 5251 } 5252 if (strcmp(pr_name, "_ibd_rc_rx_rwqe_thresh") == 0) { 5253 value = state->id_rc_rx_rwqe_thresh; 5254 err = 0; 5255 goto done; 5256 } 5257 if (strcmp(pr_name, "_ibd_rc_tx_comp_count") == 0) { 5258 value = state->id_rc_tx_comp_count; 5259 err = 0; 5260 goto done; 5261 } 5262 if (strcmp(pr_name, "_ibd_rc_tx_comp_usec") == 0) { 5263 value = state->id_rc_tx_comp_usec; 5264 err = 0; 5265 goto done; 5266 } 5267 if (strcmp(pr_name, "_ibd_rc_tx_copy_thresh") == 0) { 5268 value = state->id_rc_tx_copy_thresh; 5269 err = 0; 5270 goto done; 5271 } 5272 if (strcmp(pr_name, "_ibd_ud_num_rwqe") == 0) { 5273 value = state->id_ud_num_rwqe; 5274 err = 0; 5275 goto done; 5276 } 5277 if (strcmp(pr_name, "_ibd_ud_num_swqe") == 0) { 5278 value = state->id_ud_num_swqe; 5279 err = 0; 5280 goto done; 5281 } 5282 if (strcmp(pr_name, "_ibd_ud_rx_comp_count") == 0) { 5283 value = state->id_ud_rx_comp_count; 5284 err = 0; 5285 goto done; 5286 } 5287 if (strcmp(pr_name, "_ibd_ud_rx_comp_usec") == 0) { 5288 value = state->id_ud_rx_comp_usec; 5289 err = 0; 5290 goto done; 5291 } 5292 if (strcmp(pr_name, "_ibd_ud_tx_comp_count") == 0) { 5293 value = state->id_ud_tx_comp_count; 5294 err = 0; 5295 goto done; 5296 } 5297 if (strcmp(pr_name, "_ibd_ud_tx_comp_usec") == 0) { 5298 value = state->id_ud_tx_comp_usec; 5299 err = 0; 5300 goto done; 5301 } 5302 if (strcmp(pr_name, "_ibd_ud_tx_copy_thresh") == 0) { 5303 value = state->id_ud_tx_copy_thresh; 5304 err = 0; 5305 goto done; 5306 } 5307 done: 5308 if (err == 0) { 5309 (void) snprintf(pr_val, pr_valsize, "%d", value); 5310 } 5311 return (err); 5312 } 5313 5314 static int 5315 ibd_get_port_details(ibd_state_t *state) 5316 { 5317 ibt_hca_portinfo_t *port_infop; 5318 ibt_status_t ret; 5319 uint_t psize, port_infosz; 5320 5321 mutex_enter(&state->id_link_mutex); 5322 5323 /* 5324 * Query for port information 5325 */ 5326 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, 5327 &port_infop, &psize, &port_infosz); 5328 if ((ret != IBT_SUCCESS) || (psize != 1)) { 5329 mutex_exit(&state->id_link_mutex); 5330 DPRINT(10, "ibd_get_port_details: ibt_query_hca_ports() " 5331 "failed, ret=%d", ret); 5332 return (ENETDOWN); 5333 } 5334 5335 /* 5336 * If the link is active, verify the pkey 5337 */ 5338 if (port_infop->p_linkstate == IBT_PORT_ACTIVE) { 5339 if ((ret = ibt_pkey2index(state->id_hca_hdl, state->id_port, 5340 state->id_pkey, &state->id_pkix)) != IBT_SUCCESS) { 5341 state->id_link_state = LINK_STATE_DOWN; 5342 } else { 5343 state->id_link_state = LINK_STATE_UP; 5344 } 5345 state->id_mtu = (128 << port_infop->p_mtu); 5346 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid)) 5347 state->id_sgid = *port_infop->p_sgid_tbl; 5348 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid)) 5349 /* 5350 * Now that the port is active, record the port speed 5351 */ 5352 state->id_link_speed = ibd_get_portspeed(state); 5353 } else { 5354 /* Make sure that these are handled in PORT_UP/CHANGE */ 5355 state->id_mtu = 0; 5356 state->id_link_state = LINK_STATE_DOWN; 5357 state->id_link_speed = 0; 5358 } 5359 mutex_exit(&state->id_link_mutex); 5360 ibt_free_portinfo(port_infop, port_infosz); 5361 5362 return (0); 5363 } 5364 5365 static int 5366 ibd_alloc_cqs(ibd_state_t *state) 5367 { 5368 ibt_hca_attr_t hca_attrs; 5369 ibt_cq_attr_t cq_attr; 5370 ibt_status_t ret; 5371 uint32_t real_size; 5372 uint_t num_rwqe_change = 0; 5373 uint_t num_swqe_change = 0; 5374 5375 ret = ibt_query_hca(state->id_hca_hdl, &hca_attrs); 5376 ASSERT(ret == IBT_SUCCESS); 5377 5378 /* 5379 * Allocate Rx/combined CQ: 5380 * Theoretically, there is no point in having more than #rwqe 5381 * plus #swqe cqe's, except that the CQ will be signaled for 5382 * overflow when the last wqe completes, if none of the previous 5383 * cqe's have been polled. Thus, we allocate just a few less wqe's 5384 * to make sure such overflow does not occur. 5385 */ 5386 cq_attr.cq_sched = NULL; 5387 cq_attr.cq_flags = IBT_CQ_NO_FLAGS; 5388 5389 /* 5390 * Allocate Receive CQ. 5391 */ 5392 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_rwqe + 1)) { 5393 cq_attr.cq_size = state->id_ud_num_rwqe + 1; 5394 } else { 5395 cq_attr.cq_size = hca_attrs.hca_max_cq_sz; 5396 num_rwqe_change = state->id_ud_num_rwqe; 5397 state->id_ud_num_rwqe = cq_attr.cq_size - 1; 5398 } 5399 5400 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr, 5401 &state->id_rcq_hdl, &real_size)) != IBT_SUCCESS) { 5402 DPRINT(10, "ibd_alloc_cqs: ibt_alloc_cq(rcq) " 5403 "failed, ret=%d\n", ret); 5404 return (DDI_FAILURE); 5405 } 5406 5407 if ((ret = ibt_modify_cq(state->id_rcq_hdl, state->id_ud_rx_comp_count, 5408 state->id_ud_rx_comp_usec, 0)) != IBT_SUCCESS) { 5409 DPRINT(10, "ibd_alloc_cqs: Receive CQ interrupt " 5410 "moderation failed, ret=%d\n", ret); 5411 } 5412 5413 /* make the #rx wc's the same as max rx chain size */ 5414 state->id_rxwcs_size = IBD_MAX_RX_MP_LEN; 5415 state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) * 5416 state->id_rxwcs_size, KM_SLEEP); 5417 5418 /* 5419 * Allocate Send CQ. 5420 */ 5421 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_swqe + 1)) { 5422 cq_attr.cq_size = state->id_ud_num_swqe + 1; 5423 } else { 5424 cq_attr.cq_size = hca_attrs.hca_max_cq_sz; 5425 num_swqe_change = state->id_ud_num_swqe; 5426 state->id_ud_num_swqe = cq_attr.cq_size - 1; 5427 } 5428 5429 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr, 5430 &state->id_scq_hdl, &real_size)) != IBT_SUCCESS) { 5431 DPRINT(10, "ibd_alloc_cqs: ibt_alloc_cq(scq) " 5432 "failed, ret=%d\n", ret); 5433 kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) * 5434 state->id_rxwcs_size); 5435 (void) ibt_free_cq(state->id_rcq_hdl); 5436 return (DDI_FAILURE); 5437 } 5438 if ((ret = ibt_modify_cq(state->id_scq_hdl, state->id_ud_tx_comp_count, 5439 state->id_ud_tx_comp_usec, 0)) != IBT_SUCCESS) { 5440 DPRINT(10, "ibd_alloc_cqs: Send CQ interrupt " 5441 "moderation failed, ret=%d\n", ret); 5442 } 5443 5444 state->id_txwcs_size = IBD_TX_POLL_THRESH; 5445 state->id_txwcs = kmem_alloc(sizeof (ibt_wc_t) * 5446 state->id_txwcs_size, KM_SLEEP); 5447 5448 /* 5449 * Print message in case we could not allocate as many wqe's 5450 * as was requested. 5451 */ 5452 if (num_rwqe_change) { 5453 ibd_print_warn(state, "Setting #rwqe = %d instead of default " 5454 "%d", state->id_ud_num_rwqe, num_rwqe_change); 5455 } 5456 if (num_swqe_change) { 5457 ibd_print_warn(state, "Setting #swqe = %d instead of default " 5458 "%d", state->id_ud_num_swqe, num_swqe_change); 5459 } 5460 5461 return (DDI_SUCCESS); 5462 } 5463 5464 static int 5465 ibd_setup_ud_channel(ibd_state_t *state) 5466 { 5467 ibt_ud_chan_alloc_args_t ud_alloc_attr; 5468 ibt_ud_chan_query_attr_t ud_chan_attr; 5469 ibt_status_t ret; 5470 5471 ud_alloc_attr.ud_flags = IBT_ALL_SIGNALED; 5472 if (state->id_hca_res_lkey_capab) 5473 ud_alloc_attr.ud_flags |= IBT_FAST_REG_RES_LKEY; 5474 if (state->id_lso_policy && state->id_lso_capable) 5475 ud_alloc_attr.ud_flags |= IBT_USES_LSO; 5476 5477 ud_alloc_attr.ud_hca_port_num = state->id_port; 5478 ud_alloc_attr.ud_sizes.cs_sq_sgl = state->id_max_sqseg; 5479 ud_alloc_attr.ud_sizes.cs_rq_sgl = IBD_MAX_RQSEG; 5480 ud_alloc_attr.ud_sizes.cs_sq = state->id_ud_num_swqe; 5481 ud_alloc_attr.ud_sizes.cs_rq = state->id_ud_num_rwqe; 5482 ud_alloc_attr.ud_qkey = state->id_mcinfo->mc_qkey; 5483 ud_alloc_attr.ud_scq = state->id_scq_hdl; 5484 ud_alloc_attr.ud_rcq = state->id_rcq_hdl; 5485 ud_alloc_attr.ud_pd = state->id_pd_hdl; 5486 ud_alloc_attr.ud_pkey_ix = state->id_pkix; 5487 ud_alloc_attr.ud_clone_chan = NULL; 5488 5489 if ((ret = ibt_alloc_ud_channel(state->id_hca_hdl, IBT_ACHAN_NO_FLAGS, 5490 &ud_alloc_attr, &state->id_chnl_hdl, NULL)) != IBT_SUCCESS) { 5491 DPRINT(10, "ibd_setup_ud_channel: ibt_alloc_ud_channel() " 5492 "failed, ret=%d\n", ret); 5493 return (DDI_FAILURE); 5494 } 5495 5496 if ((ret = ibt_query_ud_channel(state->id_chnl_hdl, 5497 &ud_chan_attr)) != IBT_SUCCESS) { 5498 DPRINT(10, "ibd_setup_ud_channel: ibt_query_ud_channel() " 5499 "failed, ret=%d\n", ret); 5500 (void) ibt_free_channel(state->id_chnl_hdl); 5501 return (DDI_FAILURE); 5502 } 5503 5504 state->id_qpnum = ud_chan_attr.ud_qpn; 5505 5506 return (DDI_SUCCESS); 5507 } 5508 5509 static int 5510 ibd_undo_start(ibd_state_t *state, link_state_t cur_link_state) 5511 { 5512 uint32_t progress = state->id_mac_state; 5513 uint_t attempts; 5514 ibt_status_t ret; 5515 ib_gid_t mgid; 5516 ibd_mce_t *mce; 5517 uint8_t jstate; 5518 timeout_id_t tid; 5519 5520 if (atomic_dec_32_nv(&state->id_running) != 0) 5521 cmn_err(CE_WARN, "ibd_undo_start: id_running was not 1\n"); 5522 5523 /* 5524 * Before we try to stop/undo whatever we did in ibd_start(), 5525 * we need to mark the link state appropriately to prevent the 5526 * ip layer from using this instance for any new transfers. Note 5527 * that if the original state of the link was "up" when we're 5528 * here, we'll set the final link state to "unknown", to behave 5529 * in the same fashion as other ethernet drivers. 5530 */ 5531 mutex_enter(&state->id_link_mutex); 5532 if (cur_link_state == LINK_STATE_DOWN) { 5533 state->id_link_state = cur_link_state; 5534 } else { 5535 state->id_link_state = LINK_STATE_UNKNOWN; 5536 } 5537 mutex_exit(&state->id_link_mutex); 5538 bzero(&state->id_macaddr, sizeof (ipoib_mac_t)); 5539 mac_link_update(state->id_mh, state->id_link_state); 5540 5541 state->id_mac_state &= (~IBD_DRV_PORT_DETAILS_OBTAINED); 5542 if (progress & IBD_DRV_STARTED) { 5543 state->id_mac_state &= (~IBD_DRV_STARTED); 5544 } 5545 5546 if (progress & IBD_DRV_IN_LATE_HCA_INIT) { 5547 state->id_mac_state &= (~IBD_DRV_IN_LATE_HCA_INIT); 5548 } 5549 5550 /* Stop listen under Reliable Connected Mode */ 5551 if (progress & IBD_DRV_RC_LISTEN) { 5552 ASSERT(state->id_enable_rc); 5553 if (state->rc_listen_hdl != NULL) { 5554 ibd_rc_stop_listen(state); 5555 } 5556 state->id_mac_state &= (~IBD_DRV_RC_LISTEN); 5557 } 5558 5559 /* Stop timeout routine */ 5560 if (progress & IBD_DRV_RC_TIMEOUT) { 5561 ASSERT(state->id_enable_rc); 5562 mutex_enter(&state->rc_timeout_lock); 5563 state->rc_timeout_start = B_FALSE; 5564 tid = state->rc_timeout; 5565 state->rc_timeout = 0; 5566 mutex_exit(&state->rc_timeout_lock); 5567 if (tid != 0) 5568 (void) untimeout(tid); 5569 state->id_mac_state &= (~IBD_DRV_RC_TIMEOUT); 5570 } 5571 5572 if ((state->id_enable_rc) && (progress & IBD_DRV_ACACHE_INITIALIZED)) { 5573 attempts = 100; 5574 while (state->id_ah_op == IBD_OP_ONGOING) { 5575 /* 5576 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB 5577 * port is connecting to a remote IPoIB port. Wait for 5578 * the end of this connecting operation. 5579 */ 5580 delay(drv_usectohz(100000)); 5581 if (--attempts == 0) { 5582 state->rc_stop_connect++; 5583 DPRINT(40, "ibd_undo_start: connecting"); 5584 break; 5585 } 5586 } 5587 mutex_enter(&state->id_sched_lock); 5588 state->id_sched_needed = 0; 5589 mutex_exit(&state->id_sched_lock); 5590 (void) ibd_rc_close_all_chan(state); 5591 } 5592 5593 /* 5594 * First, stop receive interrupts; this stops the driver from 5595 * handing up buffers to higher layers. Wait for receive buffers 5596 * to be returned and give up after 1 second. 5597 */ 5598 if (progress & IBD_DRV_RCQ_NOTIFY_ENABLED) { 5599 attempts = 10; 5600 while (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 5601 0) > 0) { 5602 delay(drv_usectohz(100000)); 5603 if (--attempts == 0) { 5604 /* 5605 * There are pending bufs with the network 5606 * layer and we have no choice but to wait 5607 * for them to be done with. Reap all the 5608 * Tx/Rx completions that were posted since 5609 * we turned off the notification and 5610 * return failure. 5611 */ 5612 cmn_err(CE_CONT, "!ibd: bufs outstanding\n"); 5613 DPRINT(2, "ibd_undo_start: " 5614 "reclaiming failed"); 5615 break; 5616 } 5617 } 5618 state->id_mac_state &= (~IBD_DRV_RCQ_NOTIFY_ENABLED); 5619 } 5620 5621 if (progress & IBD_DRV_RC_LARGEBUF_ALLOCD) { 5622 ibd_rc_fini_tx_largebuf_list(state); 5623 state->id_mac_state &= (~IBD_DRV_RC_LARGEBUF_ALLOCD); 5624 } 5625 5626 if (progress & IBD_DRV_RC_SRQ_ALLOCD) { 5627 ASSERT(state->id_enable_rc); 5628 if (state->rc_srq_rwqe_list.dl_bufs_outstanding == 0) { 5629 if (state->id_ah_op == IBD_OP_ONGOING) { 5630 delay(drv_usectohz(10000)); 5631 if (state->id_ah_op == IBD_OP_ONGOING) { 5632 /* 5633 * "state->id_ah_op == IBD_OP_ONGOING" 5634 * means this IPoIB port is connecting 5635 * to a remote IPoIB port. We can't 5636 * delete SRQ here. 5637 */ 5638 state->rc_stop_connect++; 5639 DPRINT(40, "ibd_undo_start: " 5640 "connecting"); 5641 } else { 5642 ibd_rc_fini_srq_list(state); 5643 state->id_mac_state &= 5644 (~IBD_DRV_RC_SRQ_ALLOCD); 5645 } 5646 } else { 5647 ibd_rc_fini_srq_list(state); 5648 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD); 5649 } 5650 } else { 5651 DPRINT(40, "ibd_undo_start: srq bufs outstanding\n"); 5652 } 5653 } 5654 5655 if (progress & IBD_DRV_SM_NOTICES_REGISTERED) { 5656 ibt_register_subnet_notices(state->id_ibt_hdl, NULL, NULL); 5657 5658 mutex_enter(&state->id_trap_lock); 5659 state->id_trap_stop = B_TRUE; 5660 while (state->id_trap_inprog > 0) 5661 cv_wait(&state->id_trap_cv, &state->id_trap_lock); 5662 mutex_exit(&state->id_trap_lock); 5663 5664 state->id_mac_state &= (~IBD_DRV_SM_NOTICES_REGISTERED); 5665 } 5666 5667 if (progress & IBD_DRV_SCQ_NOTIFY_ENABLED) { 5668 /* 5669 * Flushing the channel ensures that all pending WQE's 5670 * are marked with flush_error and handed to the CQ. It 5671 * does not guarantee the invocation of the CQ handler. 5672 * This call is guaranteed to return successfully for 5673 * UD QPNs. 5674 */ 5675 if ((ret = ibt_flush_channel(state->id_chnl_hdl)) != 5676 IBT_SUCCESS) { 5677 DPRINT(10, "ibd_undo_start: flush_channel " 5678 "failed, ret=%d", ret); 5679 } 5680 5681 /* 5682 * Give some time for the TX CQ handler to process the 5683 * completions. 5684 */ 5685 attempts = 10; 5686 mutex_enter(&state->id_tx_list.dl_mutex); 5687 mutex_enter(&state->id_tx_rel_list.dl_mutex); 5688 while (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt 5689 != state->id_ud_num_swqe) { 5690 if (--attempts == 0) 5691 break; 5692 mutex_exit(&state->id_tx_rel_list.dl_mutex); 5693 mutex_exit(&state->id_tx_list.dl_mutex); 5694 delay(drv_usectohz(100000)); 5695 mutex_enter(&state->id_tx_list.dl_mutex); 5696 mutex_enter(&state->id_tx_rel_list.dl_mutex); 5697 } 5698 ibt_set_cq_handler(state->id_scq_hdl, 0, 0); 5699 if (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt != 5700 state->id_ud_num_swqe) { 5701 cmn_err(CE_WARN, "tx resources not freed\n"); 5702 } 5703 mutex_exit(&state->id_tx_rel_list.dl_mutex); 5704 mutex_exit(&state->id_tx_list.dl_mutex); 5705 5706 attempts = 10; 5707 while (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) { 5708 if (--attempts == 0) 5709 break; 5710 delay(drv_usectohz(100000)); 5711 } 5712 ibt_set_cq_handler(state->id_rcq_hdl, 0, 0); 5713 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) { 5714 cmn_err(CE_WARN, "rx resources not freed\n"); 5715 } 5716 5717 state->id_mac_state &= (~IBD_DRV_SCQ_NOTIFY_ENABLED); 5718 } 5719 5720 if (progress & IBD_DRV_BCAST_GROUP_JOINED) { 5721 /* 5722 * Drop all residual full/non membership. This includes full 5723 * membership to the broadcast group, and any nonmembership 5724 * acquired during transmits. We do this after the Tx completion 5725 * handlers are done, since those might result in some late 5726 * leaves; this also eliminates a potential race with that 5727 * path wrt the mc full list insert/delete. Trap handling 5728 * has also been suppressed at this point. Thus, no locks 5729 * are required while traversing the mc full list. 5730 */ 5731 DPRINT(2, "ibd_undo_start: clear full cache entries"); 5732 mce = list_head(&state->id_mc_full); 5733 while (mce != NULL) { 5734 mgid = mce->mc_info.mc_adds_vect.av_dgid; 5735 jstate = mce->mc_jstate; 5736 mce = list_next(&state->id_mc_full, mce); 5737 ibd_leave_group(state, mgid, jstate); 5738 } 5739 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_JOINED); 5740 } 5741 5742 if (progress & IBD_DRV_RXLIST_ALLOCD) { 5743 ibd_fini_rxlist(state); 5744 state->id_mac_state &= (~IBD_DRV_RXLIST_ALLOCD); 5745 } 5746 5747 if (progress & IBD_DRV_TXLIST_ALLOCD) { 5748 ibd_fini_txlist(state); 5749 state->id_mac_state &= (~IBD_DRV_TXLIST_ALLOCD); 5750 } 5751 5752 if (progress & IBD_DRV_UD_CHANNEL_SETUP) { 5753 if ((ret = ibt_free_channel(state->id_chnl_hdl)) != 5754 IBT_SUCCESS) { 5755 DPRINT(10, "ibd_undo_start: free_channel " 5756 "failed, ret=%d", ret); 5757 } 5758 5759 state->id_mac_state &= (~IBD_DRV_UD_CHANNEL_SETUP); 5760 } 5761 5762 if (progress & IBD_DRV_CQS_ALLOCD) { 5763 kmem_free(state->id_txwcs, 5764 sizeof (ibt_wc_t) * state->id_txwcs_size); 5765 if ((ret = ibt_free_cq(state->id_scq_hdl)) != 5766 IBT_SUCCESS) { 5767 DPRINT(10, "ibd_undo_start: free_cq(scq) " 5768 "failed, ret=%d", ret); 5769 } 5770 5771 kmem_free(state->id_rxwcs, 5772 sizeof (ibt_wc_t) * state->id_rxwcs_size); 5773 if ((ret = ibt_free_cq(state->id_rcq_hdl)) != IBT_SUCCESS) { 5774 DPRINT(10, "ibd_undo_start: free_cq(rcq) failed, " 5775 "ret=%d", ret); 5776 } 5777 5778 state->id_txwcs = NULL; 5779 state->id_rxwcs = NULL; 5780 state->id_scq_hdl = NULL; 5781 state->id_rcq_hdl = NULL; 5782 5783 state->id_mac_state &= (~IBD_DRV_CQS_ALLOCD); 5784 } 5785 5786 if (progress & IBD_DRV_ACACHE_INITIALIZED) { 5787 mutex_enter(&state->id_ac_mutex); 5788 mod_hash_destroy_hash(state->id_ah_active_hash); 5789 mutex_exit(&state->id_ac_mutex); 5790 ibd_acache_fini(state); 5791 5792 state->id_mac_state &= (~IBD_DRV_ACACHE_INITIALIZED); 5793 } 5794 5795 if (progress & IBD_DRV_BCAST_GROUP_FOUND) { 5796 /* 5797 * If we'd created the ipoib broadcast group and had 5798 * successfully joined it, leave it now 5799 */ 5800 if (state->id_bgroup_created) { 5801 mgid = state->id_mcinfo->mc_adds_vect.av_dgid; 5802 jstate = IB_MC_JSTATE_FULL; 5803 (void) ibt_leave_mcg(state->id_sgid, mgid, 5804 state->id_sgid, jstate); 5805 } 5806 ibt_free_mcg_info(state->id_mcinfo, 1); 5807 5808 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_FOUND); 5809 } 5810 5811 return (DDI_SUCCESS); 5812 } 5813 5814 /* 5815 * These pair of routines are used to set/clear the condition that 5816 * the caller is likely to do something to change the id_mac_state. 5817 * If there's already someone doing either a start or a stop (possibly 5818 * due to the async handler detecting a pkey relocation event, a plumb 5819 * or dlpi_open, or an unplumb or dlpi_close coming in), we wait until 5820 * that's done. 5821 */ 5822 static void 5823 ibd_set_mac_progress(ibd_state_t *state, uint_t flag) 5824 { 5825 mutex_enter(&state->id_macst_lock); 5826 while (state->id_mac_state & IBD_DRV_RESTART_IN_PROGRESS) 5827 cv_wait(&state->id_macst_cv, &state->id_macst_lock); 5828 5829 state->id_mac_state |= flag; 5830 mutex_exit(&state->id_macst_lock); 5831 } 5832 5833 static void 5834 ibd_clr_mac_progress(ibd_state_t *state, uint_t flag) 5835 { 5836 mutex_enter(&state->id_macst_lock); 5837 state->id_mac_state &= (~flag); 5838 cv_signal(&state->id_macst_cv); 5839 mutex_exit(&state->id_macst_lock); 5840 } 5841 5842 /* 5843 * GLDv3 entry point to start hardware. 5844 */ 5845 /*ARGSUSED*/ 5846 static int 5847 ibd_m_start(void *arg) 5848 { 5849 ibd_state_t *state = arg; 5850 int ret; 5851 5852 if (state->id_type == IBD_PORT_DRIVER) 5853 return (EINVAL); 5854 5855 ibd_set_mac_progress(state, IBD_DRV_START_IN_PROGRESS); 5856 if (state->id_mac_state & IBD_DRV_IN_DELETION) { 5857 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS); 5858 return (EIO); 5859 } 5860 5861 ret = ibd_start(state); 5862 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS); 5863 return (ret); 5864 } 5865 5866 static int 5867 ibd_start(ibd_state_t *state) 5868 { 5869 int err; 5870 ibt_status_t ret; 5871 int late_hca_init = 0; 5872 5873 if (state->id_mac_state & IBD_DRV_STARTED) 5874 return (DDI_SUCCESS); 5875 5876 /* 5877 * We do not increment the running flag when calling ibd_start() as 5878 * a result of some event which moves the state away from late HCA 5879 * initialization viz. MCG_CREATED, PORT_CHANGE or link availability. 5880 */ 5881 if (!(state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) && 5882 (atomic_inc_32_nv(&state->id_running) != 1)) { 5883 DPRINT(10, "ibd_start: id_running is non-zero"); 5884 cmn_err(CE_WARN, "ibd_start: id_running was not 0\n"); 5885 atomic_dec_32(&state->id_running); 5886 return (EINVAL); 5887 } 5888 5889 /* 5890 * Get port details; if we fail here, something bad happened. 5891 * Fail plumb. 5892 */ 5893 if ((err = ibd_get_port_details(state)) != 0) { 5894 DPRINT(10, "ibd_start: ibd_get_port_details() failed"); 5895 goto start_fail; 5896 } 5897 /* 5898 * If state->id_link_state is DOWN, it indicates that either the port 5899 * is down, or the pkey is not available. In both cases, resort to late 5900 * initialization. Register for subnet notices, and return success. 5901 */ 5902 state->id_mac_state |= IBD_DRV_PORT_DETAILS_OBTAINED; 5903 if (state->id_link_state == LINK_STATE_DOWN) { 5904 late_hca_init = 1; 5905 goto late_hca_init_return; 5906 } 5907 5908 /* 5909 * Find the IPoIB broadcast group 5910 */ 5911 if (ibd_find_bgroup(state) != IBT_SUCCESS) { 5912 /* Resort to late initialization */ 5913 late_hca_init = 1; 5914 goto reg_snet_notices; 5915 } 5916 state->id_mac_state |= IBD_DRV_BCAST_GROUP_FOUND; 5917 5918 /* 5919 * Initialize per-interface caches and lists; if we fail here, 5920 * it is most likely due to a lack of resources 5921 */ 5922 if (ibd_acache_init(state) != DDI_SUCCESS) { 5923 DPRINT(10, "ibd_start: ibd_acache_init() failed"); 5924 err = ENOMEM; 5925 goto start_fail; 5926 } 5927 state->id_mac_state |= IBD_DRV_ACACHE_INITIALIZED; 5928 5929 /* 5930 * Allocate send and receive completion queues 5931 */ 5932 if (ibd_alloc_cqs(state) != DDI_SUCCESS) { 5933 DPRINT(10, "ibd_start: ibd_alloc_cqs() failed"); 5934 err = ENOMEM; 5935 goto start_fail; 5936 } 5937 state->id_mac_state |= IBD_DRV_CQS_ALLOCD; 5938 5939 /* 5940 * Setup a UD channel 5941 */ 5942 if (ibd_setup_ud_channel(state) != DDI_SUCCESS) { 5943 err = ENOMEM; 5944 DPRINT(10, "ibd_start: ibd_setup_ud_channel() failed"); 5945 goto start_fail; 5946 } 5947 state->id_mac_state |= IBD_DRV_UD_CHANNEL_SETUP; 5948 5949 /* 5950 * Allocate and initialize the tx buffer list 5951 */ 5952 if (ibd_init_txlist(state) != DDI_SUCCESS) { 5953 DPRINT(10, "ibd_start: ibd_init_txlist() failed"); 5954 err = ENOMEM; 5955 goto start_fail; 5956 } 5957 state->id_mac_state |= IBD_DRV_TXLIST_ALLOCD; 5958 5959 /* 5960 * Create the send cq handler here 5961 */ 5962 ibt_set_cq_handler(state->id_scq_hdl, ibd_scq_handler, state); 5963 if ((ret = ibt_enable_cq_notify(state->id_scq_hdl, 5964 IBT_NEXT_COMPLETION)) != IBT_SUCCESS) { 5965 DPRINT(10, "ibd_start: ibt_enable_cq_notify(scq) " 5966 "failed, ret=%d", ret); 5967 err = EINVAL; 5968 goto start_fail; 5969 } 5970 state->id_mac_state |= IBD_DRV_SCQ_NOTIFY_ENABLED; 5971 5972 /* 5973 * Allocate and initialize the rx buffer list 5974 */ 5975 if (ibd_init_rxlist(state) != DDI_SUCCESS) { 5976 DPRINT(10, "ibd_start: ibd_init_rxlist() failed"); 5977 err = ENOMEM; 5978 goto start_fail; 5979 } 5980 state->id_mac_state |= IBD_DRV_RXLIST_ALLOCD; 5981 5982 /* 5983 * Join IPoIB broadcast group 5984 */ 5985 if (ibd_join_group(state, state->id_mgid, IB_MC_JSTATE_FULL) == NULL) { 5986 DPRINT(10, "ibd_start: ibd_join_group() failed"); 5987 err = ENOTACTIVE; 5988 goto start_fail; 5989 } 5990 state->id_mac_state |= IBD_DRV_BCAST_GROUP_JOINED; 5991 5992 /* 5993 * When we did mac_register() in ibd_attach(), we didn't register 5994 * the real macaddr and we didn't have the true port mtu. Now that 5995 * we're almost ready, set the local mac address and broadcast 5996 * addresses and update gldv3 about the real values of these 5997 * parameters. 5998 */ 5999 if (state->id_enable_rc) { 6000 ibd_h2n_mac(&state->id_macaddr, 6001 IBD_MAC_ADDR_RC + state->id_qpnum, 6002 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); 6003 ibd_h2n_mac(&state->rc_macaddr_loopback, state->id_qpnum, 6004 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); 6005 } else { 6006 ibd_h2n_mac(&state->id_macaddr, state->id_qpnum, 6007 state->id_sgid.gid_prefix, state->id_sgid.gid_guid); 6008 } 6009 ibd_h2n_mac(&state->id_bcaddr, IB_QPN_MASK, 6010 state->id_mgid.gid_prefix, state->id_mgid.gid_guid); 6011 6012 if (!state->id_enable_rc) { 6013 (void) mac_maxsdu_update2(state->id_mh, 6014 state->id_mtu - IPOIB_HDRSIZE, 6015 state->id_mtu - IPOIB_HDRSIZE); 6016 } 6017 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr); 6018 6019 /* 6020 * Setup the receive cq handler 6021 */ 6022 ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state); 6023 if ((ret = ibt_enable_cq_notify(state->id_rcq_hdl, 6024 IBT_NEXT_COMPLETION)) != IBT_SUCCESS) { 6025 DPRINT(10, "ibd_start: ibt_enable_cq_notify(rcq) " 6026 "failed, ret=%d", ret); 6027 err = EINVAL; 6028 goto start_fail; 6029 } 6030 state->id_mac_state |= IBD_DRV_RCQ_NOTIFY_ENABLED; 6031 6032 reg_snet_notices: 6033 /* 6034 * In case of normal initialization sequence, 6035 * Setup the subnet notices handler after we've initialized the acache/ 6036 * mcache and started the async thread, both of which are required for 6037 * the trap handler to function properly. 6038 * 6039 * Now that the async thread has been started (and we've already done 6040 * a mac_register() during attach so mac_tx_update() can be called 6041 * if necessary without any problem), we can enable the trap handler 6042 * to queue requests to the async thread. 6043 * 6044 * In case of late hca initialization, the subnet notices handler will 6045 * only handle MCG created/deleted event. The action performed as part 6046 * of handling these events is to start the interface. So, the 6047 * acache/mcache initialization is not a necessity in such cases for 6048 * registering the subnet notices handler. Also, if we are in 6049 * ibd_start() as a result of, say, some event handling after entering 6050 * late hca initialization phase no need to register again. 6051 */ 6052 if ((state->id_mac_state & IBD_DRV_SM_NOTICES_REGISTERED) == 0) { 6053 ibt_register_subnet_notices(state->id_ibt_hdl, 6054 ibd_snet_notices_handler, state); 6055 mutex_enter(&state->id_trap_lock); 6056 state->id_trap_stop = B_FALSE; 6057 mutex_exit(&state->id_trap_lock); 6058 state->id_mac_state |= IBD_DRV_SM_NOTICES_REGISTERED; 6059 } 6060 6061 late_hca_init_return: 6062 if (late_hca_init == 1) { 6063 state->id_mac_state |= IBD_DRV_IN_LATE_HCA_INIT; 6064 /* 6065 * In case of late initialization, mark the link state as down, 6066 * immaterial of the actual link state as reported in the 6067 * port_info. 6068 */ 6069 state->id_link_state = LINK_STATE_DOWN; 6070 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr); 6071 mac_link_update(state->id_mh, state->id_link_state); 6072 return (DDI_SUCCESS); 6073 } 6074 6075 if (state->id_enable_rc) { 6076 if (state->rc_enable_srq) { 6077 if (state->id_mac_state & IBD_DRV_RC_SRQ_ALLOCD) { 6078 if (ibd_rc_repost_srq_free_list(state) != 6079 IBT_SUCCESS) { 6080 err = ENOMEM; 6081 goto start_fail; 6082 } 6083 } else { 6084 /* Allocate SRQ resource */ 6085 if (ibd_rc_init_srq_list(state) != 6086 IBT_SUCCESS) { 6087 err = ENOMEM; 6088 goto start_fail; 6089 } 6090 state->id_mac_state |= IBD_DRV_RC_SRQ_ALLOCD; 6091 } 6092 } 6093 6094 if (ibd_rc_init_tx_largebuf_list(state) != IBT_SUCCESS) { 6095 DPRINT(10, "ibd_start: ibd_rc_init_tx_largebuf_list() " 6096 "failed"); 6097 err = ENOMEM; 6098 goto start_fail; 6099 } 6100 state->id_mac_state |= IBD_DRV_RC_LARGEBUF_ALLOCD; 6101 6102 /* RC: begin to listen only after everything is available */ 6103 if (ibd_rc_listen(state) != IBT_SUCCESS) { 6104 DPRINT(10, "ibd_start: ibd_rc_listen() failed"); 6105 err = EINVAL; 6106 goto start_fail; 6107 } 6108 state->id_mac_state |= IBD_DRV_RC_LISTEN; 6109 } 6110 6111 /* 6112 * Indicate link status to GLDv3 and higher layers. By default, 6113 * we assume we are in up state (which must have been true at 6114 * least at the time the broadcast mcg's were probed); if there 6115 * were any up/down transitions till the time we come here, the 6116 * async handler will have updated last known state, which we 6117 * use to tell GLDv3. The async handler will not send any 6118 * notifications to GLDv3 till we reach here in the initialization 6119 * sequence. 6120 */ 6121 mac_link_update(state->id_mh, state->id_link_state); 6122 state->id_mac_state &= ~IBD_DRV_IN_LATE_HCA_INIT; 6123 state->id_mac_state |= IBD_DRV_STARTED; 6124 6125 /* Start timer after everything is ready */ 6126 if (state->id_enable_rc) { 6127 mutex_enter(&state->rc_timeout_lock); 6128 state->rc_timeout_start = B_TRUE; 6129 state->rc_timeout = timeout(ibd_rc_conn_timeout_call, state, 6130 SEC_TO_TICK(ibd_rc_conn_timeout)); 6131 mutex_exit(&state->rc_timeout_lock); 6132 state->id_mac_state |= IBD_DRV_RC_TIMEOUT; 6133 } 6134 6135 return (DDI_SUCCESS); 6136 6137 start_fail: 6138 /* 6139 * If we ran into a problem during ibd_start() and ran into 6140 * some other problem during undoing our partial work, we can't 6141 * do anything about it. Ignore any errors we might get from 6142 * ibd_undo_start() and just return the original error we got. 6143 */ 6144 (void) ibd_undo_start(state, LINK_STATE_DOWN); 6145 return (err); 6146 } 6147 6148 /* 6149 * GLDv3 entry point to stop hardware from receiving packets. 6150 */ 6151 /*ARGSUSED*/ 6152 static void 6153 ibd_m_stop(void *arg) 6154 { 6155 ibd_state_t *state = (ibd_state_t *)arg; 6156 6157 if (state->id_type == IBD_PORT_DRIVER) 6158 return; 6159 6160 ibd_set_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS); 6161 6162 (void) ibd_undo_start(state, state->id_link_state); 6163 6164 ibd_clr_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS); 6165 } 6166 6167 /* 6168 * GLDv3 entry point to modify device's mac address. We do not 6169 * allow address modifications. 6170 */ 6171 static int 6172 ibd_m_unicst(void *arg, const uint8_t *macaddr) 6173 { 6174 ibd_state_t *state = arg; 6175 6176 if (state->id_type == IBD_PORT_DRIVER) 6177 return (EINVAL); 6178 6179 /* 6180 * Don't bother even comparing the macaddr if we haven't 6181 * completed ibd_m_start(). 6182 */ 6183 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) 6184 return (0); 6185 6186 if (bcmp(macaddr, &state->id_macaddr, IPOIB_ADDRL) == 0) 6187 return (0); 6188 else 6189 return (EINVAL); 6190 } 6191 6192 /* 6193 * The blocking part of the IBA join/leave operations are done out 6194 * of here on the async thread. 6195 */ 6196 static void 6197 ibd_async_multicast(ibd_state_t *state, ib_gid_t mgid, int op) 6198 { 6199 DPRINT(3, "ibd_async_multicast : async_setmc op %d :" 6200 "%016llx:%016llx\n", op, mgid.gid_prefix, mgid.gid_guid); 6201 6202 if (op == IBD_ASYNC_JOIN) { 6203 if (ibd_join_group(state, mgid, IB_MC_JSTATE_FULL) == NULL) { 6204 ibd_print_warn(state, "Join multicast group failed :" 6205 "%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid); 6206 } 6207 } else { 6208 /* 6209 * Here, we must search for the proper mcg_info and 6210 * use that to leave the group. 6211 */ 6212 ibd_leave_group(state, mgid, IB_MC_JSTATE_FULL); 6213 } 6214 } 6215 6216 /* 6217 * GLDv3 entry point for multicast enable/disable requests. 6218 * This function queues the operation to the async thread and 6219 * return success for a valid multicast address. 6220 */ 6221 static int 6222 ibd_m_multicst(void *arg, boolean_t add, const uint8_t *mcmac) 6223 { 6224 ibd_state_t *state = (ibd_state_t *)arg; 6225 ipoib_mac_t maddr, *mcast; 6226 ib_gid_t mgid; 6227 ibd_req_t *req; 6228 6229 if (state->id_type == IBD_PORT_DRIVER) 6230 return (EINVAL); 6231 6232 /* 6233 * If we haven't completed ibd_m_start(), async thread wouldn't 6234 * have been started and id_bcaddr wouldn't be set, so there's 6235 * no point in continuing. 6236 */ 6237 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) 6238 return (0); 6239 6240 /* 6241 * The incoming multicast address might not be aligned properly 6242 * on a 4 byte boundary to be considered an ipoib_mac_t. We force 6243 * it to look like one though, to get the offsets of the mc gid, 6244 * since we know we are not going to dereference any values with 6245 * the ipoib_mac_t pointer. 6246 */ 6247 bcopy(mcmac, &maddr, sizeof (ipoib_mac_t)); 6248 mcast = &maddr; 6249 6250 /* 6251 * Check validity of MCG address. We could additionally check 6252 * that a enable/disable is not being issued on the "broadcast" 6253 * mcg, but since this operation is only invokable by privileged 6254 * programs anyway, we allow the flexibility to those dlpi apps. 6255 * Note that we do not validate the "scope" of the IBA mcg. 6256 */ 6257 if ((ntohl(mcast->ipoib_qpn) & IB_QPN_MASK) != IB_MC_QPN) 6258 return (EINVAL); 6259 6260 /* 6261 * fill in multicast pkey and scope 6262 */ 6263 IBD_FILL_SCOPE_PKEY(mcast, state->id_scope, state->id_pkey); 6264 6265 /* 6266 * If someone is trying to JOIN/LEAVE the broadcast group, we do 6267 * nothing (i.e. we stay JOINed to the broadcast group done in 6268 * ibd_m_start()), to mimic ethernet behavior. IPv4 specifically 6269 * requires to be joined to broadcast groups at all times. 6270 * ibd_join_group() has an ASSERT(omce->mc_fullreap) that also 6271 * depends on this. 6272 */ 6273 if (bcmp(mcast, &state->id_bcaddr, IPOIB_ADDRL) == 0) 6274 return (0); 6275 6276 ibd_n2h_gid(mcast, &mgid); 6277 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); 6278 if (req == NULL) 6279 return (ENOMEM); 6280 6281 req->rq_gid = mgid; 6282 6283 if (add) { 6284 DPRINT(1, "ibd_m_multicst : %016llx:%016llx\n", 6285 mgid.gid_prefix, mgid.gid_guid); 6286 ibd_queue_work_slot(state, req, IBD_ASYNC_JOIN); 6287 } else { 6288 DPRINT(1, "ibd_m_multicst : unset_multicast : " 6289 "%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid); 6290 ibd_queue_work_slot(state, req, IBD_ASYNC_LEAVE); 6291 } 6292 return (0); 6293 } 6294 6295 /* 6296 * The blocking part of the IBA promiscuous operations are done 6297 * out of here on the async thread. The dlpireq parameter indicates 6298 * whether this invocation is due to a dlpi request or due to 6299 * a port up/down event. 6300 */ 6301 static void 6302 ibd_async_unsetprom(ibd_state_t *state) 6303 { 6304 ibd_mce_t *mce = list_head(&state->id_mc_non); 6305 ib_gid_t mgid; 6306 6307 DPRINT(2, "ibd_async_unsetprom : async_unset_promisc"); 6308 6309 while (mce != NULL) { 6310 mgid = mce->mc_info.mc_adds_vect.av_dgid; 6311 mce = list_next(&state->id_mc_non, mce); 6312 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON); 6313 } 6314 state->id_prom_op = IBD_OP_NOTSTARTED; 6315 } 6316 6317 /* 6318 * The blocking part of the IBA promiscuous operations are done 6319 * out of here on the async thread. The dlpireq parameter indicates 6320 * whether this invocation is due to a dlpi request or due to 6321 * a port up/down event. 6322 */ 6323 static void 6324 ibd_async_setprom(ibd_state_t *state) 6325 { 6326 ibt_mcg_attr_t mcg_attr; 6327 ibt_mcg_info_t *mcg_info; 6328 ib_gid_t mgid; 6329 uint_t numg; 6330 int i; 6331 char ret = IBD_OP_COMPLETED; 6332 6333 DPRINT(2, "ibd_async_setprom : async_set_promisc"); 6334 6335 /* 6336 * Obtain all active MC groups on the IB fabric with 6337 * specified criteria (scope + Pkey + Qkey + mtu). 6338 */ 6339 bzero(&mcg_attr, sizeof (mcg_attr)); 6340 mcg_attr.mc_pkey = state->id_pkey; 6341 mcg_attr.mc_scope = state->id_scope; 6342 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey; 6343 mcg_attr.mc_mtu_req.r_mtu = state->id_mcinfo->mc_mtu; 6344 mcg_attr.mc_mtu_req.r_selector = IBT_EQU; 6345 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 0, &mcg_info, &numg) != 6346 IBT_SUCCESS) { 6347 ibd_print_warn(state, "Could not get list of IBA multicast " 6348 "groups"); 6349 ret = IBD_OP_ERRORED; 6350 goto done; 6351 } 6352 6353 /* 6354 * Iterate over the returned mcg's and join as NonMember 6355 * to the IP mcg's. 6356 */ 6357 for (i = 0; i < numg; i++) { 6358 /* 6359 * Do a NonMember JOIN on the MC group. 6360 */ 6361 mgid = mcg_info[i].mc_adds_vect.av_dgid; 6362 if (ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == NULL) 6363 ibd_print_warn(state, "IBA promiscuous mode missed " 6364 "multicast gid %016llx:%016llx", 6365 (u_longlong_t)mgid.gid_prefix, 6366 (u_longlong_t)mgid.gid_guid); 6367 } 6368 6369 ibt_free_mcg_info(mcg_info, numg); 6370 DPRINT(4, "ibd_async_setprom : async_set_promisc completes"); 6371 done: 6372 state->id_prom_op = ret; 6373 } 6374 6375 /* 6376 * GLDv3 entry point for multicast promiscuous enable/disable requests. 6377 * GLDv3 assumes phys state receives more packets than multi state, 6378 * which is not true for IPoIB. Thus, treat the multi and phys 6379 * promiscuous states the same way to work with GLDv3's assumption. 6380 */ 6381 static int 6382 ibd_m_promisc(void *arg, boolean_t on) 6383 { 6384 ibd_state_t *state = (ibd_state_t *)arg; 6385 ibd_req_t *req; 6386 6387 if (state->id_type == IBD_PORT_DRIVER) 6388 return (EINVAL); 6389 6390 /* 6391 * Async thread wouldn't have been started if we haven't 6392 * passed ibd_m_start() 6393 */ 6394 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) 6395 return (0); 6396 6397 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); 6398 if (req == NULL) 6399 return (ENOMEM); 6400 if (on) { 6401 DPRINT(1, "ibd_m_promisc : set_promisc : %d", on); 6402 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMON); 6403 } else { 6404 DPRINT(1, "ibd_m_promisc : unset_promisc"); 6405 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMOFF); 6406 } 6407 6408 return (0); 6409 } 6410 6411 /* 6412 * GLDv3 entry point for gathering statistics. 6413 */ 6414 static int 6415 ibd_m_stat(void *arg, uint_t stat, uint64_t *val) 6416 { 6417 ibd_state_t *state = (ibd_state_t *)arg; 6418 6419 switch (stat) { 6420 case MAC_STAT_IFSPEED: 6421 *val = state->id_link_speed; 6422 break; 6423 case MAC_STAT_MULTIRCV: 6424 *val = state->id_multi_rcv; 6425 break; 6426 case MAC_STAT_BRDCSTRCV: 6427 *val = state->id_brd_rcv; 6428 break; 6429 case MAC_STAT_MULTIXMT: 6430 *val = state->id_multi_xmt; 6431 break; 6432 case MAC_STAT_BRDCSTXMT: 6433 *val = state->id_brd_xmt; 6434 break; 6435 case MAC_STAT_RBYTES: 6436 *val = state->id_rcv_bytes + state->rc_rcv_trans_byte 6437 + state->rc_rcv_copy_byte; 6438 break; 6439 case MAC_STAT_IPACKETS: 6440 *val = state->id_rcv_pkt + state->rc_rcv_trans_pkt 6441 + state->rc_rcv_copy_pkt; 6442 break; 6443 case MAC_STAT_OBYTES: 6444 *val = state->id_xmt_bytes + state->rc_xmt_bytes; 6445 break; 6446 case MAC_STAT_OPACKETS: 6447 *val = state->id_xmt_pkt + state->rc_xmt_small_pkt + 6448 state->rc_xmt_fragmented_pkt + 6449 state->rc_xmt_map_fail_pkt + state->rc_xmt_map_succ_pkt; 6450 break; 6451 case MAC_STAT_OERRORS: 6452 *val = state->id_ah_error; /* failed AH translation */ 6453 break; 6454 case MAC_STAT_IERRORS: 6455 *val = 0; 6456 break; 6457 case MAC_STAT_NOXMTBUF: 6458 *val = state->id_tx_short + state->rc_swqe_short + 6459 state->rc_xmt_buf_short; 6460 break; 6461 case MAC_STAT_NORCVBUF: 6462 default: 6463 return (ENOTSUP); 6464 } 6465 6466 return (0); 6467 } 6468 6469 static void 6470 ibd_async_txsched(ibd_state_t *state) 6471 { 6472 ibd_resume_transmission(state); 6473 } 6474 6475 static void 6476 ibd_resume_transmission(ibd_state_t *state) 6477 { 6478 int flag; 6479 int met_thresh = 0; 6480 int thresh = 0; 6481 int ret = -1; 6482 6483 mutex_enter(&state->id_sched_lock); 6484 if (state->id_sched_needed & IBD_RSRC_SWQE) { 6485 mutex_enter(&state->id_tx_list.dl_mutex); 6486 mutex_enter(&state->id_tx_rel_list.dl_mutex); 6487 met_thresh = state->id_tx_list.dl_cnt + 6488 state->id_tx_rel_list.dl_cnt; 6489 mutex_exit(&state->id_tx_rel_list.dl_mutex); 6490 mutex_exit(&state->id_tx_list.dl_mutex); 6491 thresh = IBD_FREE_SWQES_THRESH; 6492 flag = IBD_RSRC_SWQE; 6493 } else if (state->id_sched_needed & IBD_RSRC_LSOBUF) { 6494 ASSERT(state->id_lso != NULL); 6495 mutex_enter(&state->id_lso_lock); 6496 met_thresh = state->id_lso->bkt_nfree; 6497 thresh = IBD_FREE_LSOS_THRESH; 6498 mutex_exit(&state->id_lso_lock); 6499 flag = IBD_RSRC_LSOBUF; 6500 if (met_thresh > thresh) 6501 state->id_sched_lso_cnt++; 6502 } 6503 if (met_thresh > thresh) { 6504 state->id_sched_needed &= ~flag; 6505 state->id_sched_cnt++; 6506 ret = 0; 6507 } 6508 mutex_exit(&state->id_sched_lock); 6509 6510 if (ret == 0) 6511 mac_tx_update(state->id_mh); 6512 } 6513 6514 /* 6515 * Release the send wqe back into free list. 6516 */ 6517 static void 6518 ibd_release_swqe(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail, int n) 6519 { 6520 /* 6521 * Add back on Tx list for reuse. 6522 */ 6523 ASSERT(tail->swqe_next == NULL); 6524 mutex_enter(&state->id_tx_rel_list.dl_mutex); 6525 state->id_tx_rel_list.dl_pending_sends = B_FALSE; 6526 tail->swqe_next = state->id_tx_rel_list.dl_head; 6527 state->id_tx_rel_list.dl_head = SWQE_TO_WQE(head); 6528 state->id_tx_rel_list.dl_cnt += n; 6529 mutex_exit(&state->id_tx_rel_list.dl_mutex); 6530 } 6531 6532 /* 6533 * Acquire a send wqe from free list. 6534 * Returns error number and send wqe pointer. 6535 */ 6536 static ibd_swqe_t * 6537 ibd_acquire_swqe(ibd_state_t *state) 6538 { 6539 ibd_swqe_t *wqe; 6540 6541 mutex_enter(&state->id_tx_rel_list.dl_mutex); 6542 if (state->id_tx_rel_list.dl_head != NULL) { 6543 /* transfer id_tx_rel_list to id_tx_list */ 6544 state->id_tx_list.dl_head = 6545 state->id_tx_rel_list.dl_head; 6546 state->id_tx_list.dl_cnt = 6547 state->id_tx_rel_list.dl_cnt; 6548 state->id_tx_list.dl_pending_sends = B_FALSE; 6549 6550 /* clear id_tx_rel_list */ 6551 state->id_tx_rel_list.dl_head = NULL; 6552 state->id_tx_rel_list.dl_cnt = 0; 6553 mutex_exit(&state->id_tx_rel_list.dl_mutex); 6554 6555 wqe = WQE_TO_SWQE(state->id_tx_list.dl_head); 6556 state->id_tx_list.dl_cnt -= 1; 6557 state->id_tx_list.dl_head = wqe->swqe_next; 6558 } else { /* no free swqe */ 6559 mutex_exit(&state->id_tx_rel_list.dl_mutex); 6560 state->id_tx_list.dl_pending_sends = B_TRUE; 6561 DPRINT(5, "ibd_acquire_swqe: out of Tx wqe"); 6562 state->id_tx_short++; 6563 wqe = NULL; 6564 } 6565 return (wqe); 6566 } 6567 6568 static int 6569 ibd_setup_lso(ibd_swqe_t *node, mblk_t *mp, uint32_t mss, 6570 ibt_ud_dest_hdl_t ud_dest) 6571 { 6572 mblk_t *nmp; 6573 int iph_len, tcph_len; 6574 ibt_wr_lso_t *lso; 6575 uintptr_t ip_start, tcp_start; 6576 uint8_t *dst; 6577 uint_t pending, mblen; 6578 6579 /* 6580 * The code in ibd_send would've set 'wr.ud.udwr_dest' by default; 6581 * we need to adjust it here for lso. 6582 */ 6583 lso = &(node->w_swr.wr.ud_lso); 6584 lso->lso_ud_dest = ud_dest; 6585 lso->lso_mss = mss; 6586 6587 /* 6588 * Calculate the LSO header size and set it in the UD LSO structure. 6589 * Note that the only assumption we make is that each of the IPoIB, 6590 * IP and TCP headers will be contained in a single mblk fragment; 6591 * together, the headers may span multiple mblk fragments. 6592 */ 6593 nmp = mp; 6594 ip_start = (uintptr_t)(nmp->b_rptr) + IPOIB_HDRSIZE; 6595 if (ip_start >= (uintptr_t)(nmp->b_wptr)) { 6596 ip_start = (uintptr_t)nmp->b_cont->b_rptr 6597 + (ip_start - (uintptr_t)(nmp->b_wptr)); 6598 nmp = nmp->b_cont; 6599 6600 } 6601 iph_len = IPH_HDR_LENGTH((ipha_t *)ip_start); 6602 6603 tcp_start = ip_start + iph_len; 6604 if (tcp_start >= (uintptr_t)(nmp->b_wptr)) { 6605 tcp_start = (uintptr_t)nmp->b_cont->b_rptr 6606 + (tcp_start - (uintptr_t)(nmp->b_wptr)); 6607 nmp = nmp->b_cont; 6608 } 6609 tcph_len = TCP_HDR_LENGTH((tcph_t *)tcp_start); 6610 lso->lso_hdr_sz = IPOIB_HDRSIZE + iph_len + tcph_len; 6611 6612 /* 6613 * If the lso header fits entirely within a single mblk fragment, 6614 * we'll avoid an additional copy of the lso header here and just 6615 * pass the b_rptr of the mblk directly. 6616 * 6617 * If this isn't true, we'd have to allocate for it explicitly. 6618 */ 6619 if (lso->lso_hdr_sz <= MBLKL(mp)) { 6620 lso->lso_hdr = mp->b_rptr; 6621 } else { 6622 /* On work completion, remember to free this allocated hdr */ 6623 lso->lso_hdr = kmem_zalloc(lso->lso_hdr_sz, KM_NOSLEEP); 6624 if (lso->lso_hdr == NULL) { 6625 DPRINT(10, "ibd_setup_lso: couldn't allocate lso hdr, " 6626 "sz = %d", lso->lso_hdr_sz); 6627 lso->lso_hdr_sz = 0; 6628 lso->lso_mss = 0; 6629 return (-1); 6630 } 6631 } 6632 6633 /* 6634 * Copy in the lso header only if we need to 6635 */ 6636 if (lso->lso_hdr != mp->b_rptr) { 6637 dst = lso->lso_hdr; 6638 pending = lso->lso_hdr_sz; 6639 6640 for (nmp = mp; nmp && pending; nmp = nmp->b_cont) { 6641 mblen = MBLKL(nmp); 6642 if (pending > mblen) { 6643 bcopy(nmp->b_rptr, dst, mblen); 6644 dst += mblen; 6645 pending -= mblen; 6646 } else { 6647 bcopy(nmp->b_rptr, dst, pending); 6648 break; 6649 } 6650 } 6651 } 6652 6653 return (0); 6654 } 6655 6656 static void 6657 ibd_free_lsohdr(ibd_swqe_t *node, mblk_t *mp) 6658 { 6659 ibt_wr_lso_t *lso; 6660 6661 if ((!node) || (!mp)) 6662 return; 6663 6664 /* 6665 * Free any header space that we might've allocated if we 6666 * did an LSO 6667 */ 6668 if (node->w_swr.wr_opcode == IBT_WRC_SEND_LSO) { 6669 lso = &(node->w_swr.wr.ud_lso); 6670 if ((lso->lso_hdr) && (lso->lso_hdr != mp->b_rptr)) { 6671 kmem_free(lso->lso_hdr, lso->lso_hdr_sz); 6672 lso->lso_hdr = NULL; 6673 lso->lso_hdr_sz = 0; 6674 } 6675 } 6676 } 6677 6678 static void 6679 ibd_post_send(ibd_state_t *state, ibd_swqe_t *node) 6680 { 6681 uint_t i; 6682 uint_t num_posted; 6683 uint_t n_wrs; 6684 ibt_status_t ibt_status; 6685 ibt_send_wr_t wrs[IBD_MAX_TX_POST_MULTIPLE]; 6686 ibd_swqe_t *tx_head, *elem; 6687 ibd_swqe_t *nodes[IBD_MAX_TX_POST_MULTIPLE]; 6688 6689 /* post the one request, then check for more */ 6690 ibt_status = ibt_post_send(state->id_chnl_hdl, 6691 &node->w_swr, 1, NULL); 6692 if (ibt_status != IBT_SUCCESS) { 6693 ibd_print_warn(state, "ibd_post_send: " 6694 "posting one wr failed: ret=%d", ibt_status); 6695 ibd_tx_cleanup(state, node); 6696 } 6697 6698 tx_head = NULL; 6699 for (;;) { 6700 if (tx_head == NULL) { 6701 mutex_enter(&state->id_txpost_lock); 6702 tx_head = state->id_tx_head; 6703 if (tx_head == NULL) { 6704 state->id_tx_busy = 0; 6705 mutex_exit(&state->id_txpost_lock); 6706 return; 6707 } 6708 state->id_tx_head = NULL; 6709 mutex_exit(&state->id_txpost_lock); 6710 } 6711 6712 /* 6713 * Collect pending requests, IBD_MAX_TX_POST_MULTIPLE wrs 6714 * at a time if possible, and keep posting them. 6715 */ 6716 for (n_wrs = 0, elem = tx_head; 6717 (elem) && (n_wrs < IBD_MAX_TX_POST_MULTIPLE); 6718 elem = WQE_TO_SWQE(elem->swqe_next), n_wrs++) { 6719 nodes[n_wrs] = elem; 6720 wrs[n_wrs] = elem->w_swr; 6721 } 6722 tx_head = elem; 6723 6724 ASSERT(n_wrs != 0); 6725 6726 /* 6727 * If posting fails for some reason, we'll never receive 6728 * completion intimation, so we'll need to cleanup. But 6729 * we need to make sure we don't clean up nodes whose 6730 * wrs have been successfully posted. We assume that the 6731 * hca driver returns on the first failure to post and 6732 * therefore the first 'num_posted' entries don't need 6733 * cleanup here. 6734 */ 6735 num_posted = 0; 6736 ibt_status = ibt_post_send(state->id_chnl_hdl, 6737 wrs, n_wrs, &num_posted); 6738 if (ibt_status != IBT_SUCCESS) { 6739 ibd_print_warn(state, "ibd_post_send: " 6740 "posting multiple wrs failed: " 6741 "requested=%d, done=%d, ret=%d", 6742 n_wrs, num_posted, ibt_status); 6743 6744 for (i = num_posted; i < n_wrs; i++) 6745 ibd_tx_cleanup(state, nodes[i]); 6746 } 6747 } 6748 } 6749 6750 static int 6751 ibd_prepare_sgl(ibd_state_t *state, mblk_t *mp, ibd_swqe_t *node, 6752 uint_t lsohdr_sz) 6753 { 6754 ibt_wr_ds_t *sgl; 6755 ibt_status_t ibt_status; 6756 mblk_t *nmp; 6757 mblk_t *data_mp; 6758 uchar_t *bufp; 6759 size_t blksize; 6760 size_t skip; 6761 size_t avail; 6762 uint_t pktsize; 6763 uint_t frag_len; 6764 uint_t pending_hdr; 6765 int nmblks; 6766 int i; 6767 6768 /* 6769 * Let's skip ahead to the data if this is LSO 6770 */ 6771 data_mp = mp; 6772 pending_hdr = 0; 6773 if (lsohdr_sz) { 6774 pending_hdr = lsohdr_sz; 6775 for (nmp = mp; nmp; nmp = nmp->b_cont) { 6776 frag_len = nmp->b_wptr - nmp->b_rptr; 6777 if (frag_len > pending_hdr) 6778 break; 6779 pending_hdr -= frag_len; 6780 } 6781 data_mp = nmp; /* start of data past lso header */ 6782 ASSERT(data_mp != NULL); 6783 } 6784 6785 /* 6786 * Calculate the size of message data and number of msg blocks 6787 */ 6788 pktsize = 0; 6789 for (nmblks = 0, nmp = data_mp; nmp != NULL; 6790 nmp = nmp->b_cont, nmblks++) { 6791 pktsize += MBLKL(nmp); 6792 } 6793 pktsize -= pending_hdr; 6794 6795 /* 6796 * We only do ibt_map_mem_iov() if the pktsize is above the 6797 * "copy-threshold", and if the number of mp fragments is less than 6798 * the maximum acceptable. 6799 */ 6800 if ((state->id_hca_res_lkey_capab) && 6801 (pktsize > state->id_ud_tx_copy_thresh) && 6802 (nmblks < state->id_max_sqseg_hiwm)) { 6803 ibt_iov_t iov_arr[IBD_MAX_SQSEG]; 6804 ibt_iov_attr_t iov_attr; 6805 6806 iov_attr.iov_as = NULL; 6807 iov_attr.iov = iov_arr; 6808 iov_attr.iov_buf = NULL; 6809 iov_attr.iov_list_len = nmblks; 6810 iov_attr.iov_wr_nds = state->id_max_sqseg; 6811 iov_attr.iov_lso_hdr_sz = lsohdr_sz; 6812 iov_attr.iov_flags = IBT_IOV_SLEEP; 6813 6814 for (nmp = data_mp, i = 0; i < nmblks; i++, nmp = nmp->b_cont) { 6815 iov_arr[i].iov_addr = (caddr_t)(void *)nmp->b_rptr; 6816 iov_arr[i].iov_len = MBLKL(nmp); 6817 if (i == 0) { 6818 iov_arr[i].iov_addr += pending_hdr; 6819 iov_arr[i].iov_len -= pending_hdr; 6820 } 6821 } 6822 6823 node->w_buftype = IBD_WQE_MAPPED; 6824 node->w_swr.wr_sgl = node->w_sgl; 6825 6826 ibt_status = ibt_map_mem_iov(state->id_hca_hdl, &iov_attr, 6827 (ibt_all_wr_t *)&node->w_swr, &node->w_mi_hdl); 6828 if (ibt_status != IBT_SUCCESS) { 6829 ibd_print_warn(state, "ibd_send: ibt_map_mem_iov " 6830 "failed, nmblks=%d, ret=%d\n", nmblks, ibt_status); 6831 goto ibd_copy_path; 6832 } 6833 6834 return (0); 6835 } 6836 6837 ibd_copy_path: 6838 if (pktsize <= state->id_tx_buf_sz) { 6839 node->swqe_copybuf.ic_sgl.ds_len = pktsize; 6840 node->w_swr.wr_nds = 1; 6841 node->w_swr.wr_sgl = &node->swqe_copybuf.ic_sgl; 6842 node->w_buftype = IBD_WQE_TXBUF; 6843 6844 /* 6845 * Even though this is the copy path for transfers less than 6846 * id_tx_buf_sz, it could still be an LSO packet. If so, it 6847 * is possible the first data mblk fragment (data_mp) still 6848 * contains part of the LSO header that we need to skip. 6849 */ 6850 bufp = (uchar_t *)(uintptr_t)node->w_swr.wr_sgl->ds_va; 6851 for (nmp = data_mp; nmp != NULL; nmp = nmp->b_cont) { 6852 blksize = MBLKL(nmp) - pending_hdr; 6853 bcopy(nmp->b_rptr + pending_hdr, bufp, blksize); 6854 bufp += blksize; 6855 pending_hdr = 0; 6856 } 6857 6858 return (0); 6859 } 6860 6861 /* 6862 * Copy path for transfers greater than id_tx_buf_sz 6863 */ 6864 node->w_swr.wr_sgl = node->w_sgl; 6865 if (ibd_acquire_lsobufs(state, pktsize, 6866 node->w_swr.wr_sgl, &(node->w_swr.wr_nds)) != 0) { 6867 DPRINT(10, "ibd_prepare_sgl: lso bufs acquire failed"); 6868 return (-1); 6869 } 6870 node->w_buftype = IBD_WQE_LSOBUF; 6871 6872 /* 6873 * Copy the larger-than-id_tx_buf_sz packet into a set of 6874 * fixed-sized, pre-mapped LSO buffers. Note that we might 6875 * need to skip part of the LSO header in the first fragment 6876 * as before. 6877 */ 6878 nmp = data_mp; 6879 skip = pending_hdr; 6880 for (i = 0; i < node->w_swr.wr_nds; i++) { 6881 sgl = node->w_swr.wr_sgl + i; 6882 bufp = (uchar_t *)(uintptr_t)sgl->ds_va; 6883 avail = IBD_LSO_BUFSZ; 6884 while (nmp && avail) { 6885 blksize = MBLKL(nmp) - skip; 6886 if (blksize > avail) { 6887 bcopy(nmp->b_rptr + skip, bufp, avail); 6888 skip += avail; 6889 avail = 0; 6890 } else { 6891 bcopy(nmp->b_rptr + skip, bufp, blksize); 6892 skip = 0; 6893 avail -= blksize; 6894 bufp += blksize; 6895 nmp = nmp->b_cont; 6896 } 6897 } 6898 } 6899 6900 return (0); 6901 } 6902 6903 /* 6904 * Schedule a completion queue polling to reap the resource we're 6905 * short on. If we implement the change to reap tx completions 6906 * in a separate thread, we'll need to wake up that thread here. 6907 */ 6908 static int 6909 ibd_sched_poll(ibd_state_t *state, int resource_type, int q_flag) 6910 { 6911 ibd_req_t *req; 6912 6913 mutex_enter(&state->id_sched_lock); 6914 state->id_sched_needed |= resource_type; 6915 mutex_exit(&state->id_sched_lock); 6916 6917 /* 6918 * If we are asked to queue a work entry, we need to do it 6919 */ 6920 if (q_flag) { 6921 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP); 6922 if (req == NULL) 6923 return (-1); 6924 6925 ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED); 6926 } 6927 6928 return (0); 6929 } 6930 6931 /* 6932 * The passed in packet has this format: 6933 * IPOIB_ADDRL b dest addr :: 2b sap :: 2b 0's :: data 6934 */ 6935 static boolean_t 6936 ibd_send(ibd_state_t *state, mblk_t *mp) 6937 { 6938 ibd_ace_t *ace; 6939 ibd_swqe_t *node; 6940 ipoib_mac_t *dest; 6941 ib_header_info_t *ipibp; 6942 ip6_t *ip6h; 6943 uint_t pktsize; 6944 uint32_t mss; 6945 uint32_t hckflags; 6946 uint32_t lsoflags = 0; 6947 uint_t lsohdr_sz = 0; 6948 int ret, len; 6949 boolean_t dofree = B_FALSE; 6950 boolean_t rc; 6951 /* if (rc_chan == NULL) send by UD; else send by RC; */ 6952 ibd_rc_chan_t *rc_chan; 6953 int nmblks; 6954 mblk_t *nmp; 6955 6956 /* 6957 * If we aren't done with the device initialization and start, 6958 * we shouldn't be here. 6959 */ 6960 if ((state->id_mac_state & IBD_DRV_STARTED) == 0) 6961 return (B_FALSE); 6962 6963 /* 6964 * Obtain an address handle for the destination. 6965 */ 6966 ipibp = (ib_header_info_t *)mp->b_rptr; 6967 dest = (ipoib_mac_t *)&ipibp->ib_dst; 6968 if ((ntohl(dest->ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN) 6969 IBD_FILL_SCOPE_PKEY(dest, state->id_scope, state->id_pkey); 6970 6971 rc_chan = NULL; 6972 ace = ibd_acache_lookup(state, dest, &ret, 1); 6973 if (state->id_enable_rc && (ace != NULL) && 6974 (ace->ac_mac.ipoib_qpn != htonl(IB_MC_QPN))) { 6975 if (ace->ac_chan == NULL) { 6976 state->rc_null_conn++; 6977 } else { 6978 if (ace->ac_chan->chan_state == 6979 IBD_RC_STATE_ACT_ESTAB) { 6980 rc_chan = ace->ac_chan; 6981 rc_chan->is_used = B_TRUE; 6982 mutex_enter(&rc_chan->tx_wqe_list.dl_mutex); 6983 node = WQE_TO_SWQE( 6984 rc_chan->tx_wqe_list.dl_head); 6985 if (node != NULL) { 6986 rc_chan->tx_wqe_list.dl_cnt -= 1; 6987 rc_chan->tx_wqe_list.dl_head = 6988 node->swqe_next; 6989 } else { 6990 node = ibd_rc_acquire_swqes(rc_chan); 6991 } 6992 mutex_exit(&rc_chan->tx_wqe_list.dl_mutex); 6993 6994 if (node == NULL) { 6995 state->rc_swqe_short++; 6996 mutex_enter(&state->id_sched_lock); 6997 state->id_sched_needed |= 6998 IBD_RSRC_RC_SWQE; 6999 mutex_exit(&state->id_sched_lock); 7000 ibd_dec_ref_ace(state, ace); 7001 return (B_FALSE); 7002 } 7003 } else { 7004 state->rc_no_estab_conn++; 7005 } 7006 } 7007 } 7008 7009 if (rc_chan == NULL) { 7010 mutex_enter(&state->id_tx_list.dl_mutex); 7011 node = WQE_TO_SWQE(state->id_tx_list.dl_head); 7012 if (node != NULL) { 7013 state->id_tx_list.dl_cnt -= 1; 7014 state->id_tx_list.dl_head = node->swqe_next; 7015 } else { 7016 node = ibd_acquire_swqe(state); 7017 } 7018 mutex_exit(&state->id_tx_list.dl_mutex); 7019 if (node == NULL) { 7020 /* 7021 * If we don't have an swqe available, schedule a 7022 * transmit completion queue cleanup and hold off on 7023 * sending more packets until we have some free swqes 7024 */ 7025 if (ibd_sched_poll(state, IBD_RSRC_SWQE, 0) == 0) { 7026 if (ace != NULL) { 7027 ibd_dec_ref_ace(state, ace); 7028 } 7029 return (B_FALSE); 7030 } 7031 7032 /* 7033 * If a poll cannot be scheduled, we have no choice but 7034 * to drop this packet 7035 */ 7036 ibd_print_warn(state, "ibd_send: no swqe, pkt drop"); 7037 if (ace != NULL) { 7038 ibd_dec_ref_ace(state, ace); 7039 } 7040 return (B_TRUE); 7041 } 7042 } 7043 7044 /* 7045 * Initialize the commonly used fields in swqe to NULL to protect 7046 * against ibd_tx_cleanup accidentally misinterpreting these on a 7047 * failure. 7048 */ 7049 node->swqe_im_mblk = NULL; 7050 node->w_swr.wr_nds = 0; 7051 node->w_swr.wr_sgl = NULL; 7052 node->w_swr.wr_opcode = IBT_WRC_SEND; 7053 7054 /* 7055 * Calculate the size of message data and number of msg blocks 7056 */ 7057 pktsize = 0; 7058 for (nmblks = 0, nmp = mp; nmp != NULL; 7059 nmp = nmp->b_cont, nmblks++) { 7060 pktsize += MBLKL(nmp); 7061 } 7062 7063 if (bcmp(&ipibp->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0) 7064 atomic_inc_64(&state->id_brd_xmt); 7065 else if ((ntohl(ipibp->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN) 7066 atomic_inc_64(&state->id_multi_xmt); 7067 7068 if (ace != NULL) { 7069 node->w_ahandle = ace; 7070 node->w_swr.wr.ud.udwr_dest = ace->ac_dest; 7071 } else { 7072 DPRINT(5, 7073 "ibd_send: acache lookup %s for %08X:%08X:%08X:%08X:%08X", 7074 ((ret == EFAULT) ? "failed" : "queued"), 7075 htonl(dest->ipoib_qpn), htonl(dest->ipoib_gidpref[0]), 7076 htonl(dest->ipoib_gidpref[1]), 7077 htonl(dest->ipoib_gidsuff[0]), 7078 htonl(dest->ipoib_gidsuff[1])); 7079 state->rc_ace_not_found++; 7080 node->w_ahandle = NULL; 7081 7082 /* 7083 * Here if ibd_acache_lookup() returns EFAULT, it means ibd 7084 * can not find a path for the specific dest address. We 7085 * should get rid of this kind of packet. We also should get 7086 * rid of the packet if we cannot schedule a poll via the 7087 * async thread. For the normal case, ibd will return the 7088 * packet to upper layer and wait for AH creating. 7089 * 7090 * Note that we always queue a work slot entry for the async 7091 * thread when we fail AH lookup (even in intr mode); this is 7092 * due to the convoluted way the code currently looks for AH. 7093 */ 7094 if (ret == EFAULT) { 7095 dofree = B_TRUE; 7096 rc = B_TRUE; 7097 } else if (ibd_sched_poll(state, IBD_RSRC_SWQE, 1) != 0) { 7098 dofree = B_TRUE; 7099 rc = B_TRUE; 7100 } else { 7101 dofree = B_FALSE; 7102 rc = B_FALSE; 7103 } 7104 goto ibd_send_fail; 7105 } 7106 7107 /* 7108 * For ND6 packets, padding is at the front of the source lladdr. 7109 * Insert the padding at front. 7110 */ 7111 if (ntohs(ipibp->ipib_rhdr.ipoib_type) == ETHERTYPE_IPV6) { 7112 if (MBLKL(mp) < sizeof (ib_header_info_t) + IPV6_HDR_LEN) { 7113 if (!pullupmsg(mp, IPV6_HDR_LEN + 7114 sizeof (ib_header_info_t))) { 7115 DPRINT(10, "ibd_send: pullupmsg failure "); 7116 dofree = B_TRUE; 7117 rc = B_TRUE; 7118 goto ibd_send_fail; 7119 } 7120 ipibp = (ib_header_info_t *)mp->b_rptr; 7121 } 7122 ip6h = (ip6_t *)((uchar_t *)ipibp + 7123 sizeof (ib_header_info_t)); 7124 len = ntohs(ip6h->ip6_plen); 7125 if (ip6h->ip6_nxt == IPPROTO_ICMPV6) { 7126 mblk_t *pad; 7127 7128 pad = allocb(4, 0); 7129 pad->b_wptr = (uchar_t *)pad->b_rptr + 4; 7130 linkb(mp, pad); 7131 if (MBLKL(mp) < sizeof (ib_header_info_t) + 7132 IPV6_HDR_LEN + len + 4) { 7133 if (!pullupmsg(mp, sizeof (ib_header_info_t) + 7134 IPV6_HDR_LEN + len + 4)) { 7135 DPRINT(10, "ibd_send: pullupmsg " 7136 "failure "); 7137 dofree = B_TRUE; 7138 rc = B_TRUE; 7139 goto ibd_send_fail; 7140 } 7141 ip6h = (ip6_t *)((uchar_t *)mp->b_rptr + 7142 sizeof (ib_header_info_t)); 7143 } 7144 7145 /* LINTED: E_CONSTANT_CONDITION */ 7146 IBD_PAD_NSNA(ip6h, len, IBD_SEND); 7147 } 7148 } 7149 7150 ASSERT(mp->b_wptr - mp->b_rptr >= sizeof (ib_addrs_t)); 7151 mp->b_rptr += sizeof (ib_addrs_t); 7152 pktsize -= sizeof (ib_addrs_t); 7153 7154 if (rc_chan) { /* send in RC mode */ 7155 ibt_iov_t iov_arr[IBD_MAX_SQSEG]; 7156 ibt_iov_attr_t iov_attr; 7157 uint_t i; 7158 size_t blksize; 7159 uchar_t *bufp; 7160 ibd_rc_tx_largebuf_t *lbufp; 7161 7162 atomic_add_64(&state->rc_xmt_bytes, pktsize); 7163 7164 /* 7165 * Upper layer does Tx checksum, we don't need do any 7166 * checksum here. 7167 */ 7168 ASSERT(node->w_swr.wr_trans == IBT_RC_SRV); 7169 7170 /* 7171 * We only do ibt_map_mem_iov() if the pktsize is above 7172 * the "copy-threshold", and if the number of mp 7173 * fragments is less than the maximum acceptable. 7174 */ 7175 if (pktsize <= state->id_rc_tx_copy_thresh) { 7176 atomic_inc_64(&state->rc_xmt_small_pkt); 7177 /* 7178 * Only process unicast packet in Reliable Connected 7179 * mode. 7180 */ 7181 node->swqe_copybuf.ic_sgl.ds_len = pktsize; 7182 node->w_swr.wr_nds = 1; 7183 node->w_swr.wr_sgl = &node->swqe_copybuf.ic_sgl; 7184 node->w_buftype = IBD_WQE_TXBUF; 7185 7186 bufp = (uchar_t *)(uintptr_t)node->w_swr.wr_sgl->ds_va; 7187 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 7188 blksize = MBLKL(nmp); 7189 bcopy(nmp->b_rptr, bufp, blksize); 7190 bufp += blksize; 7191 } 7192 freemsg(mp); 7193 ASSERT(node->swqe_im_mblk == NULL); 7194 } else { 7195 if ((state->rc_enable_iov_map) && 7196 (nmblks < state->rc_max_sqseg_hiwm)) { 7197 7198 /* do ibt_map_mem_iov() */ 7199 iov_attr.iov_as = NULL; 7200 iov_attr.iov = iov_arr; 7201 iov_attr.iov_buf = NULL; 7202 iov_attr.iov_wr_nds = state->rc_tx_max_sqseg; 7203 iov_attr.iov_lso_hdr_sz = 0; 7204 iov_attr.iov_flags = IBT_IOV_SLEEP; 7205 7206 i = 0; 7207 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 7208 iov_arr[i].iov_len = MBLKL(nmp); 7209 if (iov_arr[i].iov_len != 0) { 7210 iov_arr[i].iov_addr = (caddr_t) 7211 (void *)nmp->b_rptr; 7212 i++; 7213 } 7214 } 7215 iov_attr.iov_list_len = i; 7216 node->w_swr.wr_sgl = node->w_sgl; 7217 7218 ret = ibt_map_mem_iov(state->id_hca_hdl, 7219 &iov_attr, (ibt_all_wr_t *)&node->w_swr, 7220 &node->w_mi_hdl); 7221 if (ret != IBT_SUCCESS) { 7222 atomic_inc_64( 7223 &state->rc_xmt_map_fail_pkt); 7224 DPRINT(30, "ibd_send: ibt_map_mem_iov(" 7225 ") failed, nmblks=%d, real_nmblks" 7226 "=%d, ret=0x%x", nmblks, i, ret); 7227 goto ibd_rc_large_copy; 7228 } 7229 7230 atomic_inc_64(&state->rc_xmt_map_succ_pkt); 7231 node->w_buftype = IBD_WQE_MAPPED; 7232 node->swqe_im_mblk = mp; 7233 } else { 7234 atomic_inc_64(&state->rc_xmt_fragmented_pkt); 7235 ibd_rc_large_copy: 7236 mutex_enter(&state->rc_tx_large_bufs_lock); 7237 if (state->rc_tx_largebuf_nfree == 0) { 7238 state->rc_xmt_buf_short++; 7239 mutex_exit 7240 (&state->rc_tx_large_bufs_lock); 7241 mutex_enter(&state->id_sched_lock); 7242 state->id_sched_needed |= 7243 IBD_RSRC_RC_TX_LARGEBUF; 7244 mutex_exit(&state->id_sched_lock); 7245 dofree = B_FALSE; 7246 rc = B_FALSE; 7247 /* 7248 * If we don't have Tx large bufs, 7249 * return failure. node->w_buftype 7250 * should not be IBD_WQE_RC_COPYBUF, 7251 * otherwise it will cause problem 7252 * in ibd_rc_tx_cleanup() 7253 */ 7254 node->w_buftype = IBD_WQE_TXBUF; 7255 goto ibd_send_fail; 7256 } 7257 7258 lbufp = state->rc_tx_largebuf_free_head; 7259 ASSERT(lbufp->lb_buf != NULL); 7260 state->rc_tx_largebuf_free_head = 7261 lbufp->lb_next; 7262 lbufp->lb_next = NULL; 7263 /* Update nfree count */ 7264 state->rc_tx_largebuf_nfree --; 7265 mutex_exit(&state->rc_tx_large_bufs_lock); 7266 bufp = lbufp->lb_buf; 7267 node->w_sgl[0].ds_va = 7268 (ib_vaddr_t)(uintptr_t)bufp; 7269 node->w_sgl[0].ds_key = 7270 state->rc_tx_mr_desc.md_lkey; 7271 node->w_sgl[0].ds_len = pktsize; 7272 node->w_swr.wr_sgl = node->w_sgl; 7273 node->w_swr.wr_nds = 1; 7274 node->w_buftype = IBD_WQE_RC_COPYBUF; 7275 node->w_rc_tx_largebuf = lbufp; 7276 7277 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 7278 blksize = MBLKL(nmp); 7279 if (blksize != 0) { 7280 bcopy(nmp->b_rptr, bufp, 7281 blksize); 7282 bufp += blksize; 7283 } 7284 } 7285 freemsg(mp); 7286 ASSERT(node->swqe_im_mblk == NULL); 7287 } 7288 } 7289 7290 node->swqe_next = NULL; 7291 mutex_enter(&rc_chan->tx_post_lock); 7292 if (rc_chan->tx_busy) { 7293 if (rc_chan->tx_head) { 7294 rc_chan->tx_tail->swqe_next = 7295 SWQE_TO_WQE(node); 7296 } else { 7297 rc_chan->tx_head = node; 7298 } 7299 rc_chan->tx_tail = node; 7300 mutex_exit(&rc_chan->tx_post_lock); 7301 } else { 7302 rc_chan->tx_busy = 1; 7303 mutex_exit(&rc_chan->tx_post_lock); 7304 ibd_rc_post_send(rc_chan, node); 7305 } 7306 7307 return (B_TRUE); 7308 } /* send by RC */ 7309 7310 if ((state->id_enable_rc) && (pktsize > state->id_mtu)) { 7311 /* 7312 * Too long pktsize. The packet size from GLD should <= 7313 * state->id_mtu + sizeof (ib_addrs_t) 7314 */ 7315 if (ace->ac_mac.ipoib_qpn != htonl(IB_MC_QPN)) { 7316 ibd_req_t *req; 7317 7318 mutex_enter(&ace->tx_too_big_mutex); 7319 if (ace->tx_too_big_ongoing) { 7320 mutex_exit(&ace->tx_too_big_mutex); 7321 state->rc_xmt_reenter_too_long_pkt++; 7322 dofree = B_TRUE; 7323 } else { 7324 ace->tx_too_big_ongoing = B_TRUE; 7325 mutex_exit(&ace->tx_too_big_mutex); 7326 state->rc_xmt_icmp_too_long_pkt++; 7327 7328 req = kmem_cache_alloc(state->id_req_kmc, 7329 KM_NOSLEEP); 7330 if (req == NULL) { 7331 ibd_print_warn(state, "ibd_send: alloc " 7332 "ibd_req_t fail"); 7333 /* Drop it. */ 7334 dofree = B_TRUE; 7335 } else { 7336 req->rq_ptr = mp; 7337 req->rq_ptr2 = ace; 7338 ibd_queue_work_slot(state, req, 7339 IBD_ASYNC_RC_TOO_BIG); 7340 dofree = B_FALSE; 7341 } 7342 } 7343 } else { 7344 ibd_print_warn(state, "Reliable Connected mode is on. " 7345 "Multicast packet length %d > %d is too long to " 7346 "send packet (%d > %d), drop it", 7347 pktsize, state->id_mtu); 7348 state->rc_xmt_drop_too_long_pkt++; 7349 /* Drop it. */ 7350 dofree = B_TRUE; 7351 } 7352 rc = B_TRUE; 7353 goto ibd_send_fail; 7354 } 7355 7356 atomic_add_64(&state->id_xmt_bytes, pktsize); 7357 atomic_inc_64(&state->id_xmt_pkt); 7358 7359 /* 7360 * Do LSO and checksum related work here. For LSO send, adjust the 7361 * ud destination, the opcode and the LSO header information to the 7362 * work request. 7363 */ 7364 mac_lso_get(mp, &mss, &lsoflags); 7365 if ((lsoflags & HW_LSO) != HW_LSO) { 7366 node->w_swr.wr_opcode = IBT_WRC_SEND; 7367 lsohdr_sz = 0; 7368 } else { 7369 if (ibd_setup_lso(node, mp, mss, ace->ac_dest) != 0) { 7370 /* 7371 * The routine can only fail if there's no memory; we 7372 * can only drop the packet if this happens 7373 */ 7374 ibd_print_warn(state, 7375 "ibd_send: no memory, lso posting failed"); 7376 dofree = B_TRUE; 7377 rc = B_TRUE; 7378 goto ibd_send_fail; 7379 } 7380 7381 node->w_swr.wr_opcode = IBT_WRC_SEND_LSO; 7382 lsohdr_sz = (node->w_swr.wr.ud_lso).lso_hdr_sz; 7383 } 7384 7385 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &hckflags); 7386 if ((hckflags & HCK_FULLCKSUM) == HCK_FULLCKSUM) 7387 node->w_swr.wr_flags |= IBT_WR_SEND_CKSUM; 7388 else 7389 node->w_swr.wr_flags &= ~IBT_WR_SEND_CKSUM; 7390 7391 /* 7392 * Prepare the sgl for posting; the routine can only fail if there's 7393 * no lso buf available for posting. If this is the case, we should 7394 * probably resched for lso bufs to become available and then try again. 7395 */ 7396 if (ibd_prepare_sgl(state, mp, node, lsohdr_sz) != 0) { 7397 if (ibd_sched_poll(state, IBD_RSRC_LSOBUF, 1) != 0) { 7398 dofree = B_TRUE; 7399 rc = B_TRUE; 7400 } else { 7401 dofree = B_FALSE; 7402 rc = B_FALSE; 7403 } 7404 goto ibd_send_fail; 7405 } 7406 node->swqe_im_mblk = mp; 7407 7408 /* 7409 * Queue the wqe to hardware; since we can now simply queue a 7410 * post instead of doing it serially, we cannot assume anything 7411 * about the 'node' after ibd_post_send() returns. 7412 */ 7413 node->swqe_next = NULL; 7414 7415 mutex_enter(&state->id_txpost_lock); 7416 if (state->id_tx_busy) { 7417 if (state->id_tx_head) { 7418 state->id_tx_tail->swqe_next = 7419 SWQE_TO_WQE(node); 7420 } else { 7421 state->id_tx_head = node; 7422 } 7423 state->id_tx_tail = node; 7424 mutex_exit(&state->id_txpost_lock); 7425 } else { 7426 state->id_tx_busy = 1; 7427 mutex_exit(&state->id_txpost_lock); 7428 ibd_post_send(state, node); 7429 } 7430 7431 return (B_TRUE); 7432 7433 ibd_send_fail: 7434 if (node && mp) 7435 ibd_free_lsohdr(node, mp); 7436 7437 if (dofree) 7438 freemsg(mp); 7439 7440 if (node != NULL) { 7441 if (rc_chan) { 7442 ibd_rc_tx_cleanup(node); 7443 } else { 7444 ibd_tx_cleanup(state, node); 7445 } 7446 } 7447 7448 return (rc); 7449 } 7450 7451 /* 7452 * GLDv3 entry point for transmitting datagram. 7453 */ 7454 static mblk_t * 7455 ibd_m_tx(void *arg, mblk_t *mp) 7456 { 7457 ibd_state_t *state = (ibd_state_t *)arg; 7458 mblk_t *next; 7459 7460 if (state->id_type == IBD_PORT_DRIVER) { 7461 freemsgchain(mp); 7462 return (NULL); 7463 } 7464 7465 if ((state->id_link_state != LINK_STATE_UP) || 7466 !(state->id_mac_state & IBD_DRV_STARTED)) { 7467 freemsgchain(mp); 7468 mp = NULL; 7469 } 7470 7471 while (mp != NULL) { 7472 next = mp->b_next; 7473 mp->b_next = NULL; 7474 if (ibd_send(state, mp) == B_FALSE) { 7475 /* Send fail */ 7476 mp->b_next = next; 7477 break; 7478 } 7479 mp = next; 7480 } 7481 7482 return (mp); 7483 } 7484 7485 /* 7486 * this handles Tx and Rx completions. With separate CQs, this handles 7487 * only Rx completions. 7488 */ 7489 static uint_t 7490 ibd_intr(caddr_t arg) 7491 { 7492 ibd_state_t *state = (ibd_state_t *)arg; 7493 7494 ibd_poll_rcq(state, state->id_rcq_hdl); 7495 7496 return (DDI_INTR_CLAIMED); 7497 } 7498 7499 /* 7500 * Poll and fully drain the send cq 7501 */ 7502 static void 7503 ibd_drain_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) 7504 { 7505 ibt_wc_t *wcs = state->id_txwcs; 7506 uint_t numwcs = state->id_txwcs_size; 7507 ibd_wqe_t *wqe; 7508 ibd_swqe_t *head, *tail; 7509 ibt_wc_t *wc; 7510 uint_t num_polled; 7511 int i; 7512 7513 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &num_polled) == IBT_SUCCESS) { 7514 head = tail = NULL; 7515 for (i = 0, wc = wcs; i < num_polled; i++, wc++) { 7516 wqe = (ibd_wqe_t *)(uintptr_t)wc->wc_id; 7517 if (wc->wc_status != IBT_WC_SUCCESS) { 7518 /* 7519 * Channel being torn down. 7520 */ 7521 if (wc->wc_status == IBT_WC_WR_FLUSHED_ERR) { 7522 DPRINT(5, "ibd_drain_scq: flush error"); 7523 DPRINT(10, "ibd_drain_scq: Bad " 7524 "status %d", wc->wc_status); 7525 } else { 7526 DPRINT(10, "ibd_drain_scq: " 7527 "unexpected wc_status %d", 7528 wc->wc_status); 7529 } 7530 /* 7531 * Fallthrough to invoke the Tx handler to 7532 * release held resources, e.g., AH refcount. 7533 */ 7534 } 7535 /* 7536 * Add this swqe to the list to be cleaned up. 7537 */ 7538 if (head) 7539 tail->swqe_next = wqe; 7540 else 7541 head = WQE_TO_SWQE(wqe); 7542 tail = WQE_TO_SWQE(wqe); 7543 } 7544 tail->swqe_next = NULL; 7545 ibd_tx_cleanup_list(state, head, tail); 7546 7547 /* 7548 * Resume any blocked transmissions if possible 7549 */ 7550 ibd_resume_transmission(state); 7551 } 7552 } 7553 7554 /* 7555 * Poll and fully drain the receive cq 7556 */ 7557 static void 7558 ibd_drain_rcq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) 7559 { 7560 ibt_wc_t *wcs = state->id_rxwcs; 7561 uint_t numwcs = state->id_rxwcs_size; 7562 ibd_rwqe_t *rwqe; 7563 ibt_wc_t *wc; 7564 uint_t num_polled; 7565 int i; 7566 mblk_t *head, *tail, *mp; 7567 7568 while (ibt_poll_cq(cq_hdl, wcs, numwcs, &num_polled) == IBT_SUCCESS) { 7569 head = tail = NULL; 7570 for (i = 0, wc = wcs; i < num_polled; i++, wc++) { 7571 rwqe = (ibd_rwqe_t *)(uintptr_t)wc->wc_id; 7572 if (wc->wc_status != IBT_WC_SUCCESS) { 7573 /* 7574 * Channel being torn down. 7575 */ 7576 if (wc->wc_status == IBT_WC_WR_FLUSHED_ERR) { 7577 DPRINT(5, "ibd_drain_rcq: " 7578 "expected flushed rwqe"); 7579 } else { 7580 DPRINT(5, "ibd_drain_rcq: " 7581 "unexpected wc_status %d", 7582 wc->wc_status); 7583 } 7584 atomic_inc_32( 7585 &state->id_rx_list.dl_bufs_outstanding); 7586 freemsg(rwqe->rwqe_im_mblk); 7587 continue; 7588 } 7589 mp = ibd_process_rx(state, rwqe, wc); 7590 if (mp == NULL) 7591 continue; 7592 7593 /* 7594 * Add this mp to the list to send to the nw layer. 7595 */ 7596 if (head) 7597 tail->b_next = mp; 7598 else 7599 head = mp; 7600 tail = mp; 7601 } 7602 if (head) 7603 mac_rx(state->id_mh, state->id_rh, head); 7604 7605 /* 7606 * Account for #rwqes polled. 7607 * Post more here, if less than one fourth full. 7608 */ 7609 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, -num_polled) < 7610 (state->id_ud_num_rwqe / 4)) 7611 ibd_post_recv_intr(state); 7612 } 7613 } 7614 7615 /* 7616 * Common code for interrupt handling as well as for polling 7617 * for all completed wqe's while detaching. 7618 */ 7619 static void 7620 ibd_poll_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl) 7621 { 7622 int flag, redo_flag; 7623 int redo = 1; 7624 7625 flag = IBD_CQ_POLLING; 7626 redo_flag = IBD_REDO_CQ_POLLING; 7627 7628 mutex_enter(&state->id_scq_poll_lock); 7629 if (state->id_scq_poll_busy & flag) { 7630 ibd_print_warn(state, "ibd_poll_scq: multiple polling threads"); 7631 state->id_scq_poll_busy |= redo_flag; 7632 mutex_exit(&state->id_scq_poll_lock); 7633 return; 7634 } 7635 state->id_scq_poll_busy |= flag; 7636 mutex_exit(&state->id_scq_poll_lock); 7637 7638 /* 7639 * In some cases (eg detaching), this code can be invoked on 7640 * any cpu after disabling cq notification (thus no concurrency 7641 * exists). Apart from that, the following applies normally: 7642 * Transmit completion handling could be from any cpu if 7643 * Tx CQ is poll driven, but always on Tx interrupt cpu if Tx CQ 7644 * is interrupt driven. 7645 */ 7646 7647 /* 7648 * Poll and drain the CQ 7649 */ 7650 ibd_drain_scq(state, cq_hdl); 7651 7652 /* 7653 * Enable CQ notifications and redrain the cq to catch any 7654 * completions we might have missed after the ibd_drain_scq() 7655 * above and before the ibt_enable_cq_notify() that follows. 7656 * Finally, service any new requests to poll the cq that 7657 * could've come in after the ibt_enable_cq_notify(). 7658 */ 7659 do { 7660 if (ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION) != 7661 IBT_SUCCESS) { 7662 DPRINT(10, "ibd_intr: ibt_enable_cq_notify() failed"); 7663 } 7664 7665 ibd_drain_scq(state, cq_hdl); 7666 7667 mutex_enter(&state->id_scq_poll_lock); 7668 if (state->id_scq_poll_busy & redo_flag) 7669 state->id_scq_poll_busy &= ~redo_flag; 7670 else { 7671 state->id_scq_poll_busy &= ~flag; 7672 redo = 0; 7673 } 7674 mutex_exit(&state->id_scq_poll_lock); 7675 7676 } while (redo); 7677 } 7678 7679 /* 7680 * Common code for interrupt handling as well as for polling 7681 * for all completed wqe's while detaching. 7682 */ 7683 static void 7684 ibd_poll_rcq(ibd_state_t *state, ibt_cq_hdl_t rcq) 7685 { 7686 int flag, redo_flag; 7687 int redo = 1; 7688 7689 flag = IBD_CQ_POLLING; 7690 redo_flag = IBD_REDO_CQ_POLLING; 7691 7692 mutex_enter(&state->id_rcq_poll_lock); 7693 if (state->id_rcq_poll_busy & flag) { 7694 ibd_print_warn(state, "ibd_poll_rcq: multiple polling threads"); 7695 state->id_rcq_poll_busy |= redo_flag; 7696 mutex_exit(&state->id_rcq_poll_lock); 7697 return; 7698 } 7699 state->id_rcq_poll_busy |= flag; 7700 mutex_exit(&state->id_rcq_poll_lock); 7701 7702 /* 7703 * Poll and drain the CQ 7704 */ 7705 ibd_drain_rcq(state, rcq); 7706 7707 /* 7708 * Enable CQ notifications and redrain the cq to catch any 7709 * completions we might have missed after the ibd_drain_cq() 7710 * above and before the ibt_enable_cq_notify() that follows. 7711 * Finally, service any new requests to poll the cq that 7712 * could've come in after the ibt_enable_cq_notify(). 7713 */ 7714 do { 7715 if (ibt_enable_cq_notify(rcq, IBT_NEXT_COMPLETION) != 7716 IBT_SUCCESS) { 7717 DPRINT(10, "ibd_intr: ibt_enable_cq_notify() failed"); 7718 } 7719 7720 ibd_drain_rcq(state, rcq); 7721 7722 mutex_enter(&state->id_rcq_poll_lock); 7723 if (state->id_rcq_poll_busy & redo_flag) 7724 state->id_rcq_poll_busy &= ~redo_flag; 7725 else { 7726 state->id_rcq_poll_busy &= ~flag; 7727 redo = 0; 7728 } 7729 mutex_exit(&state->id_rcq_poll_lock); 7730 7731 } while (redo); 7732 } 7733 7734 /* 7735 * Unmap the memory area associated with a given swqe. 7736 */ 7737 void 7738 ibd_unmap_mem(ibd_state_t *state, ibd_swqe_t *swqe) 7739 { 7740 ibt_status_t stat; 7741 7742 DPRINT(20, "ibd_unmap_mem: wqe=%p, seg=%d\n", swqe, swqe->w_swr.wr_nds); 7743 7744 if (swqe->w_mi_hdl) { 7745 if ((stat = ibt_unmap_mem_iov(state->id_hca_hdl, 7746 swqe->w_mi_hdl)) != IBT_SUCCESS) { 7747 DPRINT(10, 7748 "failed in ibt_unmap_mem_iov, ret=%d\n", stat); 7749 } 7750 swqe->w_mi_hdl = NULL; 7751 } 7752 swqe->w_swr.wr_nds = 0; 7753 } 7754 7755 void 7756 ibd_dec_ref_ace(ibd_state_t *state, ibd_ace_t *ace) 7757 { 7758 /* 7759 * The recycling logic can be eliminated from here 7760 * and put into the async thread if we create another 7761 * list to hold ACE's for unjoined mcg's. 7762 */ 7763 if (DEC_REF_DO_CYCLE(ace)) { 7764 ibd_mce_t *mce; 7765 7766 /* 7767 * Check with the lock taken: we decremented 7768 * reference count without the lock, and some 7769 * transmitter might already have bumped the 7770 * reference count (possible in case of multicast 7771 * disable when we leave the AH on the active 7772 * list). If not still 0, get out, leaving the 7773 * recycle bit intact. 7774 * 7775 * Atomically transition the AH from active 7776 * to free list, and queue a work request to 7777 * leave the group and destroy the mce. No 7778 * transmitter can be looking at the AH or 7779 * the MCE in between, since we have the 7780 * ac_mutex lock. In the SendOnly reap case, 7781 * it is not necessary to hold the ac_mutex 7782 * and recheck the ref count (since the AH was 7783 * taken off the active list), we just do it 7784 * to have uniform processing with the Full 7785 * reap case. 7786 */ 7787 mutex_enter(&state->id_ac_mutex); 7788 mce = ace->ac_mce; 7789 if (GET_REF_CYCLE(ace) == 0) { 7790 CLEAR_REFCYCLE(ace); 7791 /* 7792 * Identify the case of fullmember reap as 7793 * opposed to mcg trap reap. Also, port up 7794 * might set ac_mce to NULL to indicate Tx 7795 * cleanup should do no more than put the 7796 * AH in the free list (see ibd_async_link). 7797 */ 7798 if (mce != NULL) { 7799 ace->ac_mce = NULL; 7800 IBD_ACACHE_PULLOUT_ACTIVE(state, ace); 7801 /* 7802 * mc_req was initialized at mce 7803 * creation time. 7804 */ 7805 ibd_queue_work_slot(state, 7806 &mce->mc_req, IBD_ASYNC_REAP); 7807 } 7808 IBD_ACACHE_INSERT_FREE(state, ace); 7809 } 7810 mutex_exit(&state->id_ac_mutex); 7811 } 7812 } 7813 7814 /* 7815 * Common code that deals with clean ups after a successful or 7816 * erroneous transmission attempt. 7817 */ 7818 static void 7819 ibd_tx_cleanup(ibd_state_t *state, ibd_swqe_t *swqe) 7820 { 7821 ibd_ace_t *ace = swqe->w_ahandle; 7822 7823 DPRINT(20, "ibd_tx_cleanup %p\n", swqe); 7824 7825 /* 7826 * If this was a dynamic mapping in ibd_send(), we need to 7827 * unmap here. If this was an lso buffer we'd used for sending, 7828 * we need to release the lso buf to the pool, since the resource 7829 * is scarce. However, if this was simply a normal send using 7830 * the copybuf (present in each swqe), we don't need to release it. 7831 */ 7832 if (swqe->swqe_im_mblk != NULL) { 7833 if (swqe->w_buftype == IBD_WQE_MAPPED) { 7834 ibd_unmap_mem(state, swqe); 7835 } else if (swqe->w_buftype == IBD_WQE_LSOBUF) { 7836 ibd_release_lsobufs(state, 7837 swqe->w_swr.wr_sgl, swqe->w_swr.wr_nds); 7838 } 7839 ibd_free_lsohdr(swqe, swqe->swqe_im_mblk); 7840 freemsg(swqe->swqe_im_mblk); 7841 swqe->swqe_im_mblk = NULL; 7842 } 7843 7844 /* 7845 * Drop the reference count on the AH; it can be reused 7846 * now for a different destination if there are no more 7847 * posted sends that will use it. This can be eliminated 7848 * if we can always associate each Tx buffer with an AH. 7849 * The ace can be null if we are cleaning up from the 7850 * ibd_send() error path. 7851 */ 7852 if (ace != NULL) { 7853 ibd_dec_ref_ace(state, ace); 7854 } 7855 7856 /* 7857 * Release the send wqe for reuse. 7858 */ 7859 swqe->swqe_next = NULL; 7860 ibd_release_swqe(state, swqe, swqe, 1); 7861 } 7862 7863 static void 7864 ibd_tx_cleanup_list(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail) 7865 { 7866 ibd_ace_t *ace; 7867 ibd_swqe_t *swqe; 7868 int n = 0; 7869 7870 DPRINT(20, "ibd_tx_cleanup_list %p %p\n", head, tail); 7871 7872 for (swqe = head; swqe != NULL; swqe = WQE_TO_SWQE(swqe->swqe_next)) { 7873 7874 /* 7875 * If this was a dynamic mapping in ibd_send(), we need to 7876 * unmap here. If this was an lso buffer we'd used for sending, 7877 * we need to release the lso buf to the pool, since the 7878 * resource is scarce. However, if this was simply a normal 7879 * send using the copybuf (present in each swqe), we don't need 7880 * to release it. 7881 */ 7882 if (swqe->swqe_im_mblk != NULL) { 7883 if (swqe->w_buftype == IBD_WQE_MAPPED) { 7884 ibd_unmap_mem(state, swqe); 7885 } else if (swqe->w_buftype == IBD_WQE_LSOBUF) { 7886 ibd_release_lsobufs(state, 7887 swqe->w_swr.wr_sgl, swqe->w_swr.wr_nds); 7888 } 7889 ibd_free_lsohdr(swqe, swqe->swqe_im_mblk); 7890 freemsg(swqe->swqe_im_mblk); 7891 swqe->swqe_im_mblk = NULL; 7892 } 7893 7894 /* 7895 * Drop the reference count on the AH; it can be reused 7896 * now for a different destination if there are no more 7897 * posted sends that will use it. This can be eliminated 7898 * if we can always associate each Tx buffer with an AH. 7899 * The ace can be null if we are cleaning up from the 7900 * ibd_send() error path. 7901 */ 7902 ace = swqe->w_ahandle; 7903 if (ace != NULL) { 7904 ibd_dec_ref_ace(state, ace); 7905 } 7906 n++; 7907 } 7908 7909 /* 7910 * Release the send wqes for reuse. 7911 */ 7912 ibd_release_swqe(state, head, tail, n); 7913 } 7914 7915 /* 7916 * Processing to be done after receipt of a packet; hand off to GLD 7917 * in the format expected by GLD. The received packet has this 7918 * format: 2b sap :: 00 :: data. 7919 */ 7920 static mblk_t * 7921 ibd_process_rx(ibd_state_t *state, ibd_rwqe_t *rwqe, ibt_wc_t *wc) 7922 { 7923 ib_header_info_t *phdr; 7924 mblk_t *mp; 7925 ipoib_hdr_t *ipibp; 7926 ipha_t *iphap; 7927 ip6_t *ip6h; 7928 int len; 7929 ib_msglen_t pkt_len = wc->wc_bytes_xfer; 7930 uint32_t bufs; 7931 7932 /* 7933 * Track number handed to upper layer that need to be returned. 7934 */ 7935 bufs = atomic_inc_32_nv(&state->id_rx_list.dl_bufs_outstanding); 7936 7937 /* Never run out of rwqes, use allocb when running low */ 7938 if (bufs >= state->id_rx_bufs_outstanding_limit) { 7939 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding); 7940 atomic_inc_32(&state->id_rx_allocb); 7941 mp = allocb(pkt_len, BPRI_HI); 7942 if (mp) { 7943 bcopy(rwqe->rwqe_im_mblk->b_rptr, mp->b_rptr, pkt_len); 7944 ibd_post_recv(state, rwqe); 7945 } else { /* no memory */ 7946 atomic_inc_32(&state->id_rx_allocb_failed); 7947 ibd_post_recv(state, rwqe); 7948 return (NULL); 7949 } 7950 } else { 7951 mp = rwqe->rwqe_im_mblk; 7952 } 7953 7954 7955 /* 7956 * Adjust write pointer depending on how much data came in. 7957 */ 7958 mp->b_wptr = mp->b_rptr + pkt_len; 7959 7960 /* 7961 * Make sure this is NULL or we're in trouble. 7962 */ 7963 if (mp->b_next != NULL) { 7964 ibd_print_warn(state, 7965 "ibd_process_rx: got duplicate mp from rcq?"); 7966 mp->b_next = NULL; 7967 } 7968 7969 /* 7970 * the IB link will deliver one of the IB link layer 7971 * headers called, the Global Routing Header (GRH). 7972 * ibd driver uses the information in GRH to build the 7973 * Header_info structure and pass it with the datagram up 7974 * to GLDv3. 7975 * If the GRH is not valid, indicate to GLDv3 by setting 7976 * the VerTcFlow field to 0. 7977 */ 7978 phdr = (ib_header_info_t *)mp->b_rptr; 7979 if (wc->wc_flags & IBT_WC_GRH_PRESENT) { 7980 phdr->ib_grh.ipoib_sqpn = htonl(wc->wc_qpn); 7981 7982 /* if it is loop back packet, just drop it. */ 7983 if (state->id_enable_rc) { 7984 if (bcmp(&phdr->ib_grh.ipoib_sqpn, 7985 &state->rc_macaddr_loopback, 7986 IPOIB_ADDRL) == 0) { 7987 freemsg(mp); 7988 return (NULL); 7989 } 7990 } else { 7991 if (bcmp(&phdr->ib_grh.ipoib_sqpn, &state->id_macaddr, 7992 IPOIB_ADDRL) == 0) { 7993 freemsg(mp); 7994 return (NULL); 7995 } 7996 } 7997 7998 ovbcopy(&phdr->ib_grh.ipoib_sqpn, &phdr->ib_src, 7999 sizeof (ipoib_mac_t)); 8000 if (*(uint8_t *)(phdr->ib_grh.ipoib_dgid_pref) == 0xFF) { 8001 phdr->ib_dst.ipoib_qpn = htonl(IB_MC_QPN); 8002 IBD_CLEAR_SCOPE_PKEY(&phdr->ib_dst); 8003 } else { 8004 phdr->ib_dst.ipoib_qpn = state->id_macaddr.ipoib_qpn; 8005 } 8006 } else { 8007 /* 8008 * It can not be a IBA multicast packet. Must have been 8009 * unicast for us. Just copy the interface address to dst. 8010 */ 8011 phdr->ib_grh.ipoib_vertcflow = 0; 8012 ovbcopy(&state->id_macaddr, &phdr->ib_dst, 8013 sizeof (ipoib_mac_t)); 8014 } 8015 8016 /* 8017 * For ND6 packets, padding is at the front of the source/target 8018 * lladdr. However the inet6 layer is not aware of it, hence remove 8019 * the padding from such packets. 8020 */ 8021 ipibp = (ipoib_hdr_t *)((uchar_t *)mp->b_rptr + sizeof (ipoib_pgrh_t)); 8022 if (ntohs(ipibp->ipoib_type) == ETHERTYPE_IPV6) { 8023 ip6h = (ip6_t *)((uchar_t *)ipibp + sizeof (ipoib_hdr_t)); 8024 len = ntohs(ip6h->ip6_plen); 8025 if (ip6h->ip6_nxt == IPPROTO_ICMPV6) { 8026 /* LINTED: E_CONSTANT_CONDITION */ 8027 IBD_PAD_NSNA(ip6h, len, IBD_RECV); 8028 } 8029 } 8030 8031 /* 8032 * Update statistics 8033 */ 8034 atomic_add_64(&state->id_rcv_bytes, pkt_len); 8035 atomic_inc_64(&state->id_rcv_pkt); 8036 if (bcmp(&phdr->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0) 8037 atomic_inc_64(&state->id_brd_rcv); 8038 else if ((ntohl(phdr->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN) 8039 atomic_inc_64(&state->id_multi_rcv); 8040 8041 iphap = (ipha_t *)((uchar_t *)ipibp + sizeof (ipoib_hdr_t)); 8042 /* 8043 * Set receive checksum status in mp 8044 * Hardware checksumming can be considered valid only if: 8045 * 1. CQE.IP_OK bit is set 8046 * 2. CQE.CKSUM = 0xffff 8047 * 3. IPv6 routing header is not present in the packet 8048 * 4. If there are no IP_OPTIONS in the IP HEADER 8049 */ 8050 8051 if (((wc->wc_flags & IBT_WC_CKSUM_OK) == IBT_WC_CKSUM_OK) && 8052 (wc->wc_cksum == 0xFFFF) && 8053 (iphap->ipha_version_and_hdr_length == IP_SIMPLE_HDR_VERSION)) { 8054 mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM_OK); 8055 } 8056 8057 return (mp); 8058 } 8059 8060 /* 8061 * Callback code invoked from STREAMs when the receive data buffer is 8062 * free for recycling. 8063 */ 8064 static void 8065 ibd_freemsg_cb(char *arg) 8066 { 8067 ibd_rwqe_t *rwqe = (ibd_rwqe_t *)arg; 8068 ibd_state_t *state = rwqe->w_state; 8069 8070 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding); 8071 8072 /* 8073 * If the driver is stopped, just free the rwqe. 8074 */ 8075 if (atomic_add_32_nv(&state->id_running, 0) == 0) { 8076 DPRINT(6, "ibd_freemsg: wqe being freed"); 8077 rwqe->rwqe_im_mblk = NULL; 8078 ibd_free_rwqe(state, rwqe); 8079 return; 8080 } 8081 8082 rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr, 8083 state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb); 8084 if (rwqe->rwqe_im_mblk == NULL) { 8085 ibd_free_rwqe(state, rwqe); 8086 DPRINT(6, "ibd_freemsg: desballoc failed"); 8087 return; 8088 } 8089 8090 ibd_post_recv(state, rwqe); 8091 } 8092 8093 static uint_t 8094 ibd_tx_recycle(caddr_t arg) 8095 { 8096 ibd_state_t *state = (ibd_state_t *)arg; 8097 8098 /* 8099 * Poll for completed entries 8100 */ 8101 ibd_poll_scq(state, state->id_scq_hdl); 8102 8103 return (DDI_INTR_CLAIMED); 8104 } 8105 8106 #ifdef IBD_LOGGING 8107 static void 8108 ibd_log_init(void) 8109 { 8110 ibd_lbuf = kmem_zalloc(IBD_LOG_SZ, KM_SLEEP); 8111 ibd_lbuf_ndx = 0; 8112 8113 mutex_init(&ibd_lbuf_lock, NULL, MUTEX_DRIVER, NULL); 8114 } 8115 8116 static void 8117 ibd_log_fini(void) 8118 { 8119 if (ibd_lbuf) 8120 kmem_free(ibd_lbuf, IBD_LOG_SZ); 8121 ibd_lbuf_ndx = 0; 8122 ibd_lbuf = NULL; 8123 8124 mutex_destroy(&ibd_lbuf_lock); 8125 } 8126 8127 static void 8128 ibd_log(const char *fmt, ...) 8129 { 8130 va_list ap; 8131 uint32_t off; 8132 uint32_t msglen; 8133 char tmpbuf[IBD_DMAX_LINE]; 8134 8135 if (ibd_lbuf == NULL) 8136 return; 8137 8138 va_start(ap, fmt); 8139 msglen = vsnprintf(tmpbuf, IBD_DMAX_LINE, fmt, ap); 8140 va_end(ap); 8141 8142 if (msglen >= IBD_DMAX_LINE) 8143 msglen = IBD_DMAX_LINE - 1; 8144 8145 mutex_enter(&ibd_lbuf_lock); 8146 8147 off = ibd_lbuf_ndx; /* current msg should go here */ 8148 if ((ibd_lbuf_ndx) && (ibd_lbuf[ibd_lbuf_ndx-1] != '\n')) 8149 ibd_lbuf[ibd_lbuf_ndx-1] = '\n'; 8150 8151 ibd_lbuf_ndx += msglen; /* place where next msg should start */ 8152 ibd_lbuf[ibd_lbuf_ndx] = 0; /* current msg should terminate */ 8153 8154 if (ibd_lbuf_ndx >= (IBD_LOG_SZ - 2 * IBD_DMAX_LINE)) 8155 ibd_lbuf_ndx = 0; 8156 8157 mutex_exit(&ibd_lbuf_lock); 8158 8159 bcopy(tmpbuf, ibd_lbuf+off, msglen); /* no lock needed for this */ 8160 } 8161 #endif 8162 8163 /* ARGSUSED */ 8164 static int 8165 ibd_create_partition(void *karg, intptr_t arg, int mode, cred_t *credp, 8166 int *rvalp) 8167 { 8168 ibd_create_ioctl_t *cmd = karg; 8169 ibd_state_t *state, *port_state, *p; 8170 int i, err, rval = 0; 8171 mac_register_t *macp; 8172 ibt_hca_portinfo_t *pinfop = NULL; 8173 ibt_status_t ibt_status; 8174 uint_t psize, pinfosz; 8175 boolean_t force_create = B_FALSE; 8176 8177 cmd->ibdioc.ioc_status = 0; 8178 8179 if (cmd->ibdioc.ioc_port_inst < 0) { 8180 cmd->ibdioc.ioc_status = IBD_INVALID_PORT_INST; 8181 return (EINVAL); 8182 } 8183 port_state = ddi_get_soft_state(ibd_list, cmd->ibdioc.ioc_port_inst); 8184 if (port_state == NULL) { 8185 DPRINT(10, "ibd_create_partition: failed to get state %d", 8186 cmd->ibdioc.ioc_port_inst); 8187 cmd->ibdioc.ioc_status = IBD_INVALID_PORT_INST; 8188 return (EINVAL); 8189 } 8190 8191 /* Limited PKeys not supported */ 8192 if (cmd->ioc_pkey <= IB_PKEY_INVALID_FULL) { 8193 rval = EINVAL; 8194 goto part_create_return; 8195 } 8196 8197 if (cmd->ioc_force_create == 0) { 8198 /* 8199 * Check if the port pkey table contains the pkey for which 8200 * this partition is being created. 8201 */ 8202 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl, 8203 port_state->id_port, &pinfop, &psize, &pinfosz); 8204 8205 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) { 8206 rval = EINVAL; 8207 goto part_create_return; 8208 } 8209 8210 if (pinfop->p_linkstate != IBT_PORT_ACTIVE) { 8211 rval = ENETDOWN; 8212 cmd->ibdioc.ioc_status = IBD_PORT_IS_DOWN; 8213 goto part_create_return; 8214 } 8215 8216 for (i = 0; i < pinfop->p_pkey_tbl_sz; i++) { 8217 if (pinfop->p_pkey_tbl[i] == cmd->ioc_pkey) { 8218 break; 8219 } 8220 } 8221 if (i == pinfop->p_pkey_tbl_sz) { 8222 rval = EINVAL; 8223 cmd->ibdioc.ioc_status = IBD_PKEY_NOT_PRESENT; 8224 goto part_create_return; 8225 } 8226 } else { 8227 force_create = B_TRUE; 8228 } 8229 8230 mutex_enter(&ibd_objlist_lock); 8231 for (p = ibd_objlist_head; p; p = p->id_next) { 8232 if ((p->id_port_inst == cmd->ibdioc.ioc_port_inst) && 8233 (p->id_pkey == cmd->ioc_pkey) && 8234 (p->id_plinkid == cmd->ioc_partid)) { 8235 mutex_exit(&ibd_objlist_lock); 8236 rval = EEXIST; 8237 cmd->ibdioc.ioc_status = IBD_PARTITION_EXISTS; 8238 goto part_create_return; 8239 } 8240 } 8241 mutex_exit(&ibd_objlist_lock); 8242 8243 state = kmem_zalloc(sizeof (ibd_state_t), KM_SLEEP); 8244 8245 state->id_type = IBD_PARTITION_OBJ; 8246 8247 state->id_plinkid = cmd->ioc_partid; 8248 state->id_dlinkid = cmd->ibdioc.ioc_linkid; 8249 state->id_port_inst = cmd->ibdioc.ioc_port_inst; 8250 8251 state->id_dip = port_state->id_dip; 8252 state->id_port = port_state->id_port; 8253 state->id_pkey = cmd->ioc_pkey; 8254 state->id_hca_guid = port_state->id_hca_guid; 8255 state->id_port_guid = port_state->id_port_guid; 8256 state->id_force_create = force_create; 8257 8258 mutex_init(&state->id_macst_lock, NULL, MUTEX_DRIVER, NULL); 8259 cv_init(&state->id_macst_cv, NULL, CV_DEFAULT, NULL); 8260 8261 if (ibd_part_attach(state, state->id_dip) != DDI_SUCCESS) { 8262 rval = EIO; 8263 cmd->ibdioc.ioc_status = IBD_NO_HW_RESOURCE; 8264 goto fail; 8265 } 8266 8267 if ((macp = mac_alloc(MAC_VERSION)) == NULL) { 8268 rval = EAGAIN; 8269 goto fail; 8270 } 8271 8272 macp->m_type_ident = MAC_PLUGIN_IDENT_IB; 8273 macp->m_dip = port_state->id_dip; 8274 macp->m_instance = (uint_t)-1; 8275 macp->m_driver = state; 8276 macp->m_src_addr = (uint8_t *)&state->id_macaddr; 8277 macp->m_callbacks = &ibd_m_callbacks; 8278 macp->m_min_sdu = 0; 8279 macp->m_multicast_sdu = IBD_DEF_MAX_SDU; 8280 if (state->id_enable_rc) { 8281 macp->m_max_sdu = IBD_DEF_RC_MAX_SDU; 8282 } else { 8283 macp->m_max_sdu = IBD_DEF_MAX_SDU; 8284 } 8285 macp->m_priv_props = ibd_priv_props; 8286 8287 err = mac_register(macp, &state->id_mh); 8288 mac_free(macp); 8289 8290 if (err != 0) { 8291 DPRINT(10, "ibd_create_partition: mac_register() failed %d", 8292 err); 8293 rval = err; 8294 goto fail; 8295 } 8296 8297 err = dls_devnet_create(state->id_mh, 8298 cmd->ioc_partid, crgetzoneid(credp)); 8299 if (err != 0) { 8300 DPRINT(10, "ibd_create_partition: dls_devnet_create() failed " 8301 "%d", err); 8302 rval = err; 8303 (void) mac_unregister(state->id_mh); 8304 goto fail; 8305 } 8306 8307 /* 8308 * Add the new partition state structure to the list 8309 */ 8310 mutex_enter(&ibd_objlist_lock); 8311 if (ibd_objlist_head) 8312 state->id_next = ibd_objlist_head; 8313 8314 ibd_objlist_head = state; 8315 mutex_exit(&ibd_objlist_lock); 8316 8317 part_create_return: 8318 if (pinfop) { 8319 ibt_free_portinfo(pinfop, pinfosz); 8320 } 8321 return (rval); 8322 8323 fail: 8324 if (pinfop) { 8325 ibt_free_portinfo(pinfop, pinfosz); 8326 } 8327 ibd_part_unattach(state); 8328 kmem_free(state, sizeof (ibd_state_t)); 8329 return (rval); 8330 } 8331 8332 /* ARGSUSED */ 8333 static int 8334 ibd_delete_partition(void *karg, intptr_t arg, int mode, cred_t *credp, 8335 int *rvalp) 8336 { 8337 int err; 8338 datalink_id_t tmpid; 8339 ibd_state_t *node, *prev; 8340 ibd_delete_ioctl_t *cmd = karg; 8341 8342 prev = NULL; 8343 8344 mutex_enter(&ibd_objlist_lock); 8345 node = ibd_objlist_head; 8346 8347 /* Find the ibd state structure corresponding to the partition */ 8348 while (node != NULL) { 8349 if (node->id_plinkid == cmd->ioc_partid) 8350 break; 8351 prev = node; 8352 node = node->id_next; 8353 } 8354 8355 if (node == NULL) { 8356 mutex_exit(&ibd_objlist_lock); 8357 return (ENOENT); 8358 } 8359 8360 if ((err = dls_devnet_destroy(node->id_mh, &tmpid, B_TRUE)) != 0) { 8361 DPRINT(10, "ibd_delete_partition: dls_devnet_destroy() failed " 8362 "%d", err); 8363 mutex_exit(&ibd_objlist_lock); 8364 return (err); 8365 } 8366 8367 /* 8368 * Call ibd_part_unattach() only after making sure that the instance has 8369 * not been started yet and is also not in late hca init mode. 8370 */ 8371 ibd_set_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS); 8372 8373 err = 0; 8374 if ((node->id_mac_state & IBD_DRV_STARTED) || 8375 (node->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) || 8376 (ibd_part_busy(node) != DDI_SUCCESS) || 8377 ((err = mac_disable(node->id_mh)) != 0)) { 8378 (void) dls_devnet_create(node->id_mh, cmd->ioc_partid, 8379 crgetzoneid(credp)); 8380 ibd_clr_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS); 8381 mutex_exit(&ibd_objlist_lock); 8382 return (err != 0 ? err : EBUSY); 8383 } 8384 8385 node->id_mac_state |= IBD_DRV_IN_DELETION; 8386 8387 ibd_part_unattach(node); 8388 8389 ibd_clr_mac_progress(node, IBD_DRV_DELETE_IN_PROGRESS); 8390 8391 /* Remove the partition state structure from the linked list */ 8392 if (prev == NULL) 8393 ibd_objlist_head = node->id_next; 8394 else 8395 prev->id_next = node->id_next; 8396 mutex_exit(&ibd_objlist_lock); 8397 8398 if ((err = mac_unregister(node->id_mh)) != 0) { 8399 DPRINT(10, "ibd_delete_partition: mac_unregister() failed %d", 8400 err); 8401 } 8402 8403 cv_destroy(&node->id_macst_cv); 8404 mutex_destroy(&node->id_macst_lock); 8405 8406 kmem_free(node, sizeof (ibd_state_t)); 8407 8408 return (0); 8409 } 8410 8411 /* ARGSUSED */ 8412 static int 8413 ibd_get_partition_info(void *karg, intptr_t arg, int mode, cred_t *cred, 8414 int *rvalp) 8415 { 8416 ibd_ioctl_t cmd; 8417 ibpart_ioctl_t partioc; 8418 ibport_ioctl_t portioc; 8419 #ifdef _MULTI_DATAMODEL 8420 ibport_ioctl32_t portioc32; 8421 #endif 8422 ibd_state_t *state, *port_state; 8423 int size; 8424 ibt_hca_portinfo_t *pinfop = NULL; 8425 ibt_status_t ibt_status; 8426 uint_t psize, pinfosz; 8427 int rval = 0; 8428 8429 size = sizeof (ibd_ioctl_t); 8430 if (ddi_copyin((void *)arg, &cmd, size, mode)) { 8431 return (EFAULT); 8432 } 8433 cmd.ioc_status = 0; 8434 switch (cmd.ioc_info_cmd) { 8435 case IBD_INFO_CMD_IBPART: 8436 size = sizeof (ibpart_ioctl_t); 8437 if (ddi_copyin((void *)arg, &partioc, size, mode)) { 8438 return (EFAULT); 8439 } 8440 8441 mutex_enter(&ibd_objlist_lock); 8442 /* Find the ibd state structure corresponding the partition */ 8443 for (state = ibd_objlist_head; state; state = state->id_next) { 8444 if (state->id_plinkid == cmd.ioc_linkid) { 8445 break; 8446 } 8447 } 8448 8449 if (state == NULL) { 8450 mutex_exit(&ibd_objlist_lock); 8451 return (ENOENT); 8452 } 8453 8454 partioc.ibdioc.ioc_linkid = state->id_dlinkid; 8455 partioc.ibdioc.ioc_port_inst = state->id_port_inst; 8456 partioc.ibdioc.ioc_portnum = state->id_port; 8457 partioc.ibdioc.ioc_hcaguid = state->id_hca_guid; 8458 partioc.ibdioc.ioc_portguid = state->id_port_guid; 8459 partioc.ibdioc.ioc_status = 0; 8460 partioc.ioc_partid = state->id_plinkid; 8461 partioc.ioc_pkey = state->id_pkey; 8462 partioc.ioc_force_create = state->id_force_create; 8463 if (ddi_copyout((void *)&partioc, (void *)arg, size, mode)) { 8464 mutex_exit(&ibd_objlist_lock); 8465 return (EFAULT); 8466 } 8467 mutex_exit(&ibd_objlist_lock); 8468 8469 break; 8470 8471 case IBD_INFO_CMD_IBPORT: 8472 if ((cmd.ioc_port_inst < 0) || ((port_state = 8473 ddi_get_soft_state(ibd_list, cmd.ioc_port_inst)) == NULL)) { 8474 DPRINT(10, "ibd_create_partition: failed to get" 8475 " state %d", cmd.ioc_port_inst); 8476 size = sizeof (ibd_ioctl_t); 8477 cmd.ioc_status = IBD_INVALID_PORT_INST; 8478 if (ddi_copyout((void *)&cmd, (void *)arg, size, 8479 mode)) { 8480 return (EFAULT); 8481 } 8482 return (EINVAL); 8483 } 8484 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl, 8485 port_state->id_port, &pinfop, &psize, &pinfosz); 8486 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) { 8487 return (EINVAL); 8488 } 8489 #ifdef _MULTI_DATAMODEL 8490 switch (ddi_model_convert_from(mode & FMODELS)) { 8491 case DDI_MODEL_ILP32: { 8492 size = sizeof (ibport_ioctl32_t); 8493 if (ddi_copyin((void *)arg, &portioc32, size, mode)) { 8494 rval = EFAULT; 8495 goto fail; 8496 } 8497 portioc32.ibdioc.ioc_status = 0; 8498 portioc32.ibdioc.ioc_portnum = port_state->id_port; 8499 portioc32.ibdioc.ioc_hcaguid = 8500 port_state->id_hca_guid; 8501 portioc32.ibdioc.ioc_portguid = 8502 port_state->id_port_guid; 8503 if (portioc32.ioc_pkey_tbl_sz != 8504 pinfop->p_pkey_tbl_sz) { 8505 rval = EINVAL; 8506 size = sizeof (ibd_ioctl_t); 8507 portioc32.ibdioc.ioc_status = 8508 IBD_INVALID_PKEY_TBL_SIZE; 8509 if (ddi_copyout((void *)&portioc32.ibdioc, 8510 (void *)arg, size, mode)) { 8511 rval = EFAULT; 8512 goto fail; 8513 } 8514 goto fail; 8515 } 8516 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t); 8517 if (ddi_copyout((void *)pinfop->p_pkey_tbl, 8518 (void *)(uintptr_t)portioc32.ioc_pkeys, size, 8519 mode)) { 8520 rval = EFAULT; 8521 goto fail; 8522 } 8523 size = sizeof (ibport_ioctl32_t); 8524 if (ddi_copyout((void *)&portioc32, (void *)arg, size, 8525 mode)) { 8526 rval = EFAULT; 8527 goto fail; 8528 } 8529 break; 8530 } 8531 case DDI_MODEL_NONE: 8532 size = sizeof (ibport_ioctl_t); 8533 if (ddi_copyin((void *)arg, &portioc, size, mode)) { 8534 rval = EFAULT; 8535 goto fail; 8536 } 8537 portioc.ibdioc.ioc_status = 0; 8538 portioc.ibdioc.ioc_portnum = port_state->id_port; 8539 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid; 8540 portioc.ibdioc.ioc_portguid = port_state->id_port_guid; 8541 if (portioc.ioc_pkey_tbl_sz != pinfop->p_pkey_tbl_sz) { 8542 rval = EINVAL; 8543 size = sizeof (ibd_ioctl_t); 8544 portioc.ibdioc.ioc_status = 8545 IBD_INVALID_PKEY_TBL_SIZE; 8546 if (ddi_copyout((void *)&portioc.ibdioc, 8547 (void *)arg, size, mode)) { 8548 rval = EFAULT; 8549 goto fail; 8550 } 8551 goto fail; 8552 } 8553 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t); 8554 if (ddi_copyout((void *)pinfop->p_pkey_tbl, 8555 (void *)(portioc.ioc_pkeys), size, mode)) { 8556 rval = EFAULT; 8557 goto fail; 8558 } 8559 size = sizeof (ibport_ioctl_t); 8560 if (ddi_copyout((void *)&portioc, (void *)arg, size, 8561 mode)) { 8562 rval = EFAULT; 8563 goto fail; 8564 } 8565 break; 8566 } 8567 #else /* ! _MULTI_DATAMODEL */ 8568 size = sizeof (ibport_ioctl_t); 8569 if (ddi_copyin((void *)arg, &portioc, size, mode)) { 8570 rval = EFAULT; 8571 goto fail; 8572 } 8573 portioc.ibdioc.ioc_status = 0; 8574 portioc.ibdioc.ioc_portnum = port_state->id_port; 8575 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid; 8576 portioc.ibdioc.ioc_portguid = port_state->id_port_guid; 8577 if (portioc.ioc_pkey_tbl_sz != pinfop->p_pkey_tbl_sz) { 8578 rval = EINVAL; 8579 size = sizeof (ibd_ioctl_t); 8580 portioc.ibdioc.ioc_status = IBD_INVALID_PKEY_TBL_SIZE; 8581 if (ddi_copyout((void *)&portioc.ibdioc, (void *)arg, 8582 size, mode)) { 8583 rval = EFAULT; 8584 goto fail; 8585 } 8586 goto fail; 8587 } 8588 size = pinfop->p_pkey_tbl_sz * sizeof (ib_pkey_t); 8589 if (ddi_copyout((void *)pinfop->p_pkey_tbl, 8590 (void *)(portioc.ioc_pkeys), size, mode)) { 8591 rval = EFAULT; 8592 goto fail; 8593 } 8594 size = sizeof (ibport_ioctl_t); 8595 if (ddi_copyout((void *)&portioc, (void *)arg, size, 8596 mode)) { 8597 rval = EFAULT; 8598 goto fail; 8599 } 8600 #endif /* _MULTI_DATAMODEL */ 8601 8602 break; 8603 8604 case IBD_INFO_CMD_PKEYTBLSZ: 8605 if ((cmd.ioc_port_inst < 0) || ((port_state = 8606 ddi_get_soft_state(ibd_list, cmd.ioc_port_inst)) == NULL)) { 8607 DPRINT(10, "ibd_create_partition: failed to get" 8608 " state %d", cmd.ioc_port_inst); 8609 size = sizeof (ibd_ioctl_t); 8610 cmd.ioc_status = IBD_INVALID_PORT_INST; 8611 if (ddi_copyout((void *)&cmd, (void *)arg, size, 8612 mode)) { 8613 return (EFAULT); 8614 } 8615 return (EINVAL); 8616 } 8617 ibt_status = ibt_query_hca_ports(port_state->id_hca_hdl, 8618 port_state->id_port, &pinfop, &psize, &pinfosz); 8619 if ((ibt_status != IBT_SUCCESS) || (psize != 1)) { 8620 return (EINVAL); 8621 } 8622 #ifdef _MULTI_DATAMODEL 8623 switch (ddi_model_convert_from(mode & FMODELS)) { 8624 case DDI_MODEL_ILP32: { 8625 size = sizeof (ibport_ioctl32_t); 8626 if (ddi_copyin((void *)arg, &portioc32, size, mode)) { 8627 rval = EFAULT; 8628 goto fail; 8629 } 8630 portioc32.ibdioc.ioc_status = 0; 8631 portioc32.ibdioc.ioc_portnum = port_state->id_port; 8632 portioc32.ibdioc.ioc_hcaguid = 8633 port_state->id_hca_guid; 8634 portioc32.ibdioc.ioc_portguid = 8635 port_state->id_port_guid; 8636 portioc32.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz; 8637 if (ddi_copyout((void *)&portioc32, (void *)arg, size, 8638 mode)) { 8639 rval = EFAULT; 8640 goto fail; 8641 } 8642 break; 8643 } 8644 case DDI_MODEL_NONE: 8645 size = sizeof (ibport_ioctl_t); 8646 if (ddi_copyin((void *)arg, &portioc, size, mode)) { 8647 rval = EFAULT; 8648 goto fail; 8649 } 8650 portioc.ibdioc.ioc_status = 0; 8651 portioc.ibdioc.ioc_portnum = port_state->id_port; 8652 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid; 8653 portioc.ibdioc.ioc_portguid = port_state->id_port_guid; 8654 portioc.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz; 8655 if (ddi_copyout((void *)&portioc, (void *)arg, size, 8656 mode)) { 8657 rval = EFAULT; 8658 goto fail; 8659 } 8660 break; 8661 } 8662 #else /* ! _MULTI_DATAMODEL */ 8663 size = sizeof (ibport_ioctl_t); 8664 if (ddi_copyin((void *)arg, &portioc, size, mode)) { 8665 rval = EFAULT; 8666 goto fail; 8667 } 8668 portioc.ibdioc.ioc_status = 0; 8669 portioc.ibdioc.ioc_portnum = port_state->id_port; 8670 portioc.ibdioc.ioc_hcaguid = port_state->id_hca_guid; 8671 portioc.ibdioc.ioc_portguid = port_state->id_port_guid; 8672 portioc.ioc_pkey_tbl_sz = pinfop->p_pkey_tbl_sz; 8673 if (ddi_copyout((void *)&portioc, (void *)arg, size, 8674 mode)) { 8675 rval = EFAULT; 8676 goto fail; 8677 } 8678 #endif /* _MULTI_DATAMODEL */ 8679 break; 8680 8681 default: 8682 return (EINVAL); 8683 8684 } /* switch (cmd.ioc_info_cmd) */ 8685 fail: 8686 if (pinfop) { 8687 ibt_free_portinfo(pinfop, pinfosz); 8688 } 8689 return (rval); 8690 } 8691 8692 /* ARGSUSED */ 8693 static void 8694 ibdpd_async_handler(void *arg, ibt_hca_hdl_t hca_hdl, 8695 ibt_async_code_t code, ibt_async_event_t *event) 8696 { 8697 ibd_state_t *state = (ibd_state_t *)arg; 8698 link_state_t lstate; 8699 8700 switch (code) { 8701 case IBT_EVENT_PORT_UP: 8702 case IBT_ERROR_PORT_DOWN: 8703 if (ibd_get_port_state(state, &lstate) != 0) 8704 break; 8705 8706 if (state->id_link_state != lstate) { 8707 state->id_link_state = lstate; 8708 mac_link_update(state->id_mh, lstate); 8709 } 8710 break; 8711 default: 8712 break; 8713 } 8714 } 8715 8716 static int 8717 ibd_get_port_state(ibd_state_t *state, link_state_t *lstate) 8718 { 8719 ibt_hca_portinfo_t *port_infop; 8720 uint_t psize, port_infosz; 8721 ibt_status_t ret; 8722 8723 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port, 8724 &port_infop, &psize, &port_infosz); 8725 if ((ret != IBT_SUCCESS) || (psize != 1)) 8726 return (-1); 8727 8728 state->id_sgid = *port_infop->p_sgid_tbl; 8729 state->id_link_speed = ibd_get_portspeed(state); 8730 8731 if (port_infop->p_linkstate == IBT_PORT_ACTIVE) 8732 *lstate = LINK_STATE_UP; 8733 else 8734 *lstate = LINK_STATE_DOWN; 8735 8736 ibt_free_portinfo(port_infop, port_infosz); 8737 return (0); 8738 } 8739 8740 static int 8741 ibd_port_attach(dev_info_t *dip) 8742 { 8743 ibd_state_t *state; 8744 link_state_t lstate; 8745 int instance; 8746 ibt_status_t ret; 8747 8748 /* 8749 * Allocate softstate structure 8750 */ 8751 instance = ddi_get_instance(dip); 8752 if (ddi_soft_state_zalloc(ibd_list, instance) == DDI_FAILURE) { 8753 DPRINT(10, "ibd_port_attach: ddi_soft_state_zalloc() failed"); 8754 return (DDI_FAILURE); 8755 } 8756 8757 state = ddi_get_soft_state(ibd_list, instance); 8758 8759 state->id_dip = dip; 8760 state->id_type = IBD_PORT_DRIVER; 8761 8762 if ((state->id_port = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0, 8763 "port-number", 0)) == 0) { 8764 DPRINT(10, "ibd_port_attach: invalid port number (%d)", 8765 state->id_port); 8766 return (DDI_FAILURE); 8767 } 8768 if ((state->id_hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0, 8769 "hca-guid", 0)) == 0) { 8770 DPRINT(10, "ibd_port_attach: hca has invalid guid (0x%llx)", 8771 state->id_hca_guid); 8772 return (DDI_FAILURE); 8773 } 8774 if ((state->id_port_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0, 8775 "port-guid", 0)) == 0) { 8776 DPRINT(10, "ibd_port_attach: port has invalid guid (0x%llx)", 8777 state->id_port_guid); 8778 return (DDI_FAILURE); 8779 } 8780 8781 /* 8782 * Attach to IBTL 8783 */ 8784 if ((ret = ibt_attach(&ibdpd_clnt_modinfo, dip, state, 8785 &state->id_ibt_hdl)) != IBT_SUCCESS) { 8786 DPRINT(10, "ibd_port_attach: failed in ibt_attach(), ret=%d", 8787 ret); 8788 goto done; 8789 } 8790 8791 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE; 8792 8793 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid, 8794 &state->id_hca_hdl)) != IBT_SUCCESS) { 8795 DPRINT(10, "ibd_port_attach: ibt_open_hca() failed, ret=%d", 8796 ret); 8797 goto done; 8798 } 8799 state->id_mac_state |= IBD_DRV_HCA_OPENED; 8800 8801 /* Update link status */ 8802 8803 if (ibd_get_port_state(state, &lstate) != 0) { 8804 DPRINT(10, "ibd_port_attach: ibt_open_hca() failed, ret=%d", 8805 ret); 8806 goto done; 8807 } 8808 state->id_link_state = lstate; 8809 /* 8810 * Register ibd interfaces with the Nemo framework 8811 */ 8812 if (ibd_register_mac(state, dip) != IBT_SUCCESS) { 8813 DPRINT(10, "ibd_port_attach: failed in ibd_register_mac()"); 8814 goto done; 8815 } 8816 state->id_mac_state |= IBD_DRV_MAC_REGISTERED; 8817 8818 mac_link_update(state->id_mh, lstate); 8819 8820 return (DDI_SUCCESS); 8821 done: 8822 (void) ibd_port_unattach(state, dip); 8823 return (DDI_FAILURE); 8824 } 8825 8826 static int 8827 ibd_port_unattach(ibd_state_t *state, dev_info_t *dip) 8828 { 8829 int instance; 8830 uint32_t progress = state->id_mac_state; 8831 ibt_status_t ret; 8832 8833 if (progress & IBD_DRV_MAC_REGISTERED) { 8834 (void) mac_unregister(state->id_mh); 8835 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED); 8836 } 8837 8838 if (progress & IBD_DRV_HCA_OPENED) { 8839 if ((ret = ibt_close_hca(state->id_hca_hdl)) != 8840 IBT_SUCCESS) { 8841 ibd_print_warn(state, "failed to close " 8842 "HCA device, ret=%d", ret); 8843 } 8844 state->id_hca_hdl = NULL; 8845 state->id_mac_state &= (~IBD_DRV_HCA_OPENED); 8846 } 8847 8848 if (progress & IBD_DRV_IBTL_ATTACH_DONE) { 8849 if ((ret = ibt_detach(state->id_ibt_hdl)) != IBT_SUCCESS) { 8850 ibd_print_warn(state, 8851 "ibt_detach() failed, ret=%d", ret); 8852 } 8853 state->id_ibt_hdl = NULL; 8854 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE); 8855 } 8856 instance = ddi_get_instance(dip); 8857 ddi_soft_state_free(ibd_list, instance); 8858 8859 return (DDI_SUCCESS); 8860 } 8861 8862 ibt_status_t 8863 ibd_get_part_attr(datalink_id_t linkid, ibt_part_attr_t *attr) 8864 { 8865 ibd_state_t *state; 8866 8867 mutex_enter(&ibd_objlist_lock); 8868 8869 /* Find the ibd state structure corresponding the partition */ 8870 for (state = ibd_objlist_head; state; state = state->id_next) { 8871 if (state->id_plinkid == linkid) { 8872 break; 8873 } 8874 } 8875 8876 if (state == NULL) { 8877 mutex_exit(&ibd_objlist_lock); 8878 return (IBT_NO_SUCH_OBJECT); 8879 } 8880 8881 attr->pa_dlinkid = state->id_dlinkid; 8882 attr->pa_plinkid = state->id_plinkid; 8883 attr->pa_port = state->id_port; 8884 attr->pa_hca_guid = state->id_hca_guid; 8885 attr->pa_port_guid = state->id_port_guid; 8886 attr->pa_pkey = state->id_pkey; 8887 8888 mutex_exit(&ibd_objlist_lock); 8889 8890 return (IBT_SUCCESS); 8891 } 8892 8893 ibt_status_t 8894 ibd_get_all_part_attr(ibt_part_attr_t **attr_list, int *nparts) 8895 { 8896 ibd_state_t *state; 8897 int n = 0; 8898 ibt_part_attr_t *attr; 8899 8900 mutex_enter(&ibd_objlist_lock); 8901 8902 for (state = ibd_objlist_head; state; state = state->id_next) 8903 n++; 8904 8905 *nparts = n; 8906 if (n == 0) { 8907 *attr_list = NULL; 8908 mutex_exit(&ibd_objlist_lock); 8909 return (IBT_SUCCESS); 8910 } 8911 8912 *attr_list = kmem_alloc(sizeof (ibt_part_attr_t) * n, KM_SLEEP); 8913 attr = *attr_list; 8914 for (state = ibd_objlist_head; state; state = state->id_next) { 8915 #ifdef DEBUG 8916 ASSERT(n > 0); 8917 n--; 8918 #endif 8919 attr->pa_dlinkid = state->id_dlinkid; 8920 attr->pa_plinkid = state->id_plinkid; 8921 attr->pa_port = state->id_port; 8922 attr->pa_hca_guid = state->id_hca_guid; 8923 attr->pa_port_guid = state->id_port_guid; 8924 attr->pa_pkey = state->id_pkey; 8925 attr++; 8926 } 8927 8928 mutex_exit(&ibd_objlist_lock); 8929 return (IBT_SUCCESS); 8930 } 8931