1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved. 14 * Copyright 2019 Joyent, Inc. 15 * Copyright 2017 Tegile Systems, Inc. All rights reserved. 16 * Copyright 2020 RackTop Systems, Inc. 17 * Copyright 2020 Ryan Zezeski 18 * Copyright 2021 Oxide Computer Company 19 */ 20 21 /* 22 * i40e - Intel 10/40 Gb Ethernet driver 23 * 24 * The i40e driver is the main software device driver for the Intel 40 Gb family 25 * of devices. Note that these devices come in many flavors with both 40 GbE 26 * ports and 10 GbE ports. This device is the successor to the 82599 family of 27 * devices (ixgbe). 28 * 29 * Unlike previous generations of Intel 1 GbE and 10 GbE devices, the 40 GbE 30 * devices defined in the XL710 controller (previously known as Fortville) are a 31 * rather different beast and have a small switch embedded inside of them. In 32 * addition, the way that most of the programming is done has been overhauled. 33 * As opposed to just using PCIe memory mapped registers, it also has an 34 * administrative queue which is used to communicate with firmware running on 35 * the chip. 36 * 37 * Each physical function in the hardware shows up as a device that this driver 38 * will bind to. The hardware splits many resources evenly across all of the 39 * physical functions present on the device, while other resources are instead 40 * shared across the entire card and its up to the device driver to 41 * intelligently partition them. 42 * 43 * ------------ 44 * Organization 45 * ------------ 46 * 47 * This driver is made up of several files which have their own theory 48 * statements spread across them. We'll touch on the high level purpose of each 49 * file here, and then we'll get into more discussion on how the device is 50 * generally modelled with respect to the interfaces in illumos. 51 * 52 * i40e_gld.c: This file contains all of the bindings to MAC and the networking 53 * stack. 54 * 55 * i40e_intr.c: This file contains all of the interrupt service routines and 56 * contains logic to enable and disable interrupts on the hardware. 57 * It also contains the logic to map hardware resources such as the 58 * rings to and from interrupts and controls their ability to fire. 59 * 60 * There is a big theory statement on interrupts present there. 61 * 62 * i40e_main.c: The file that you're currently in. It interfaces with the 63 * traditional OS DDI interfaces and is in charge of configuring 64 * the device. 65 * 66 * i40e_osdep.[ch]: These files contain interfaces and definitions needed to 67 * work with Intel's common code for the device. 68 * 69 * i40e_stats.c: This file contains the general work and logic around our 70 * kstats. A theory statement on their organization and use of the 71 * hardware exists there. 72 * 73 * i40e_sw.h: This header file contains all of the primary structure definitions 74 * and constants that are used across the entire driver. 75 * 76 * i40e_transceiver.c: This file contains all of the logic for sending and 77 * receiving data. It contains all of the ring and DMA 78 * allocation logic, as well as, the actual interfaces to 79 * send and receive data. 80 * 81 * A big theory statement on ring management, descriptors, 82 * and how it ties into the OS is present there. 83 * 84 * -------------- 85 * General Design 86 * -------------- 87 * 88 * Before we go too far into the general way we've laid out data structures and 89 * the like, it's worth taking some time to explain how the hardware is 90 * organized. This organization informs a lot of how we do things at this time 91 * in the driver. 92 * 93 * Each physical device consists of a number of one or more ports, which are 94 * considered physical functions in the PCI sense and thus each get enumerated 95 * by the system, resulting in an instance being created and attached to. While 96 * there are many resources that are unique to each physical function eg. 97 * instance of the device, there are many that are shared across all of them. 98 * Several resources have an amount reserved for each Virtual Station Interface 99 * (VSI) and then a static pool of resources, available for all functions on the 100 * card. 101 * 102 * The most important resource in hardware are its transmit and receive queue 103 * pairs (i40e_trqpair_t). These should be thought of as rings in GLDv3 104 * parlance. There are a set number of these on each device; however, they are 105 * statically partitioned among all of the different physical functions. 106 * 107 * 'Fortville' (the code name for this device family) is basically a switch. To 108 * map MAC addresses and other things to queues, we end up having to create 109 * Virtual Station Interfaces (VSIs) and establish forwarding rules that direct 110 * traffic to a queue. A VSI owns a collection of queues and has a series of 111 * forwarding rules that point to it. One way to think of this is to treat it 112 * like MAC does a VNIC. When MAC refers to a group, a collection of rings and 113 * classification resources, that is a VSI in i40e. 114 * 115 * The sets of VSIs is shared across the entire device, though there may be some 116 * amount that are reserved to each PF. Because the GLDv3 does not let us change 117 * the number of groups dynamically, we instead statically divide this amount 118 * evenly between all the functions that exist. In addition, we have the same 119 * problem with the mac address forwarding rules. There are a static number that 120 * exist shared across all the functions. 121 * 122 * To handle both of these resources, what we end up doing is going through and 123 * determining which functions belong to the same device. Nominally one might do 124 * this by having a nexus driver; however, a prime requirement for a nexus 125 * driver is identifying the various children and activating them. While it is 126 * possible to get this information from NVRAM, we would end up duplicating a 127 * lot of the PCI enumeration logic. Really, at the end of the day, the device 128 * doesn't give us the traditional identification properties we want from a 129 * nexus driver. 130 * 131 * Instead, we rely on some properties that are guaranteed to be unique. While 132 * it might be tempting to leverage the PBA or serial number of the device from 133 * NVRAM, there is nothing that says that two devices can't be mis-programmed to 134 * have the same values in NVRAM. Instead, we uniquely identify a group of 135 * functions based on their parent in the /devices tree, their PCI bus and PCI 136 * function identifiers. Using either on their own may not be sufficient. 137 * 138 * For each unique PCI device that we encounter, we'll create a i40e_device_t. 139 * From there, because we don't have a good way to tell the GLDv3 about sharing 140 * resources between everything, we'll end up just dividing the resources 141 * evenly between all of the functions. Longer term, if we don't have to declare 142 * to the GLDv3 that these resources are shared, then we'll maintain a pool and 143 * have each PF allocate from the pool in the device, thus if only two of four 144 * ports are being used, for example, then all of the resources can still be 145 * used. 146 * 147 * ------------------------------------------- 148 * Transmit and Receive Queue Pair Allocations 149 * ------------------------------------------- 150 * 151 * NVRAM ends up assigning each PF its own share of the transmit and receive LAN 152 * queue pairs, we have no way of modifying it, only observing it. From there, 153 * it's up to us to map these queues to VSIs and VFs. Since we don't support any 154 * VFs at this time, we only focus on assignments to VSIs. 155 * 156 * At the moment, we used a static mapping of transmit/receive queue pairs to a 157 * given VSI (eg. rings to a group). Though in the fullness of time, we want to 158 * make this something which is fully dynamic and take advantage of documented, 159 * but not yet available functionality for adding filters based on VXLAN and 160 * other encapsulation technologies. 161 * 162 * ------------------------------------- 163 * Broadcast, Multicast, and Promiscuous 164 * ------------------------------------- 165 * 166 * As part of the GLDv3, we need to make sure that we can handle receiving 167 * broadcast and multicast traffic. As well as enabling promiscuous mode when 168 * requested. GLDv3 requires that all broadcast and multicast traffic be 169 * retrieved by the default group, eg. the first one. This is the same thing as 170 * the default VSI. 171 * 172 * To receieve broadcast traffic, we enable it through the admin queue, rather 173 * than use one of our filters for it. For multicast traffic, we reserve a 174 * certain number of the hash filters and assign them to a given PF. When we 175 * exceed those, we then switch to using promiscuous mode for multicast traffic. 176 * 177 * More specifically, once we exceed the number of filters (indicated because 178 * the i40e_t`i40e_resources.ifr_nmcastfilt == 179 * i40e_t`i40e_resources.ifr_nmcastfilt_used), we then instead need to toggle 180 * promiscuous mode. If promiscuous mode is toggled then we keep track of the 181 * number of MACs added to it by incrementing i40e_t`i40e_mcast_promisc_count. 182 * That will stay enabled until that count reaches zero indicating that we have 183 * only added multicast addresses that we have a corresponding entry for. 184 * 185 * Because MAC itself wants to toggle promiscuous mode, which includes both 186 * unicast and multicast traffic, we go through and keep track of that 187 * ourselves. That is maintained through the use of the i40e_t`i40e_promisc_on 188 * member. 189 * 190 * -------------- 191 * VSI Management 192 * -------------- 193 * 194 * The PFs share 384 VSIs. The firmware creates one VSI per PF by default. 195 * During chip start we retrieve the SEID of this VSI and assign it as the 196 * default VSI for our VEB (one VEB per PF). We then add additional VSIs to 197 * the VEB up to the determined number of rx groups: i40e_t`i40e_num_rx_groups. 198 * We currently cap this number to I40E_GROUP_MAX to a) make sure all PFs can 199 * allocate the same number of VSIs, and b) to keep the interrupt multiplexing 200 * under control. In the future, when we improve the interrupt allocation, we 201 * may want to revisit this cap to make better use of the available VSIs. The 202 * VSI allocation and configuration can be found in i40e_chip_start(). 203 * 204 * ---------------- 205 * Structure Layout 206 * ---------------- 207 * 208 * The following images relates the core data structures together. The primary 209 * structure in the system is the i40e_t. It itself contains multiple rings, 210 * i40e_trqpair_t's which contain the various transmit and receive data. The 211 * receive data is stored outside of the i40e_trqpair_t and instead in the 212 * i40e_rx_data_t. The i40e_t has a corresponding i40e_device_t which keeps 213 * track of per-physical device state. Finally, for every active descriptor, 214 * there is a corresponding control block, which is where the 215 * i40e_rx_control_block_t and the i40e_tx_control_block_t come from. 216 * 217 * +-----------------------+ +-----------------------+ 218 * | Global i40e_t list | | Global Device list | 219 * | | +--| | 220 * | i40e_glist | | | i40e_dlist | 221 * +-----------------------+ | +-----------------------+ 222 * | v 223 * | +------------------------+ +-----------------------+ 224 * | | Device-wide Structure |----->| Device-wide Structure |--> ... 225 * | | i40e_device_t | | i40e_device_t | 226 * | | | +-----------------------+ 227 * | | dev_info_t * ------+--> Parent in devices tree. 228 * | | uint_t ------+--> PCI bus number 229 * | | uint_t ------+--> PCI device number 230 * | | uint_t ------+--> Number of functions 231 * | | i40e_switch_rsrcs_t ---+--> Captured total switch resources 232 * | | list_t ------+-------------+ 233 * | +------------------------+ | 234 * | ^ | 235 * | +--------+ | 236 * | | v 237 * | +---------------------------+ | +-------------------+ 238 * +->| GLDv3 Device, per PF |-----|-->| GLDv3 Device (PF) |--> ... 239 * | i40e_t | | | i40e_t | 240 * | **Primary Structure** | | +-------------------+ 241 * | | | 242 * | i40e_device_t * --+-----+ 243 * | i40e_state_t --+---> Device State 244 * | i40e_hw_t --+---> Intel common code structure 245 * | mac_handle_t --+---> GLDv3 handle to MAC 246 * | ddi_periodic_t --+---> Link activity timer 247 * | i40e_vsi_t * --+---> Array of VSIs 248 * | i40e_func_rsrc_t --+---> Available hardware resources 249 * | i40e_switch_rsrc_t * --+---> Switch resource snapshot 250 * | i40e_sdu --+---> Current MTU 251 * | i40e_frame_max --+---> Current HW frame size 252 * | i40e_uaddr_t * --+---> Array of assigned unicast MACs 253 * | i40e_maddr_t * --+---> Array of assigned multicast MACs 254 * | i40e_mcast_promisccount --+---> Active multicast state 255 * | i40e_promisc_on --+---> Current promiscuous mode state 256 * | uint_t --+---> Number of transmit/receive pairs 257 * | i40e_rx_group_t * --+---> Array of Rx groups 258 * | kstat_t * --+---> PF kstats 259 * | i40e_pf_stats_t --+---> PF kstat backing data 260 * | i40e_trqpair_t * --+---------+ 261 * +---------------------------+ | 262 * | 263 * v 264 * +-------------------------------+ +-----------------------------+ 265 * | Transmit/Receive Queue Pair |-------| Transmit/Receive Queue Pair |->... 266 * | i40e_trqpair_t | | i40e_trqpair_t | 267 * + Ring Data Structure | +-----------------------------+ 268 * | | 269 * | mac_ring_handle_t +--> MAC RX ring handle 270 * | mac_ring_handle_t +--> MAC TX ring handle 271 * | i40e_rxq_stat_t --+--> RX Queue stats 272 * | i40e_txq_stat_t --+--> TX Queue stats 273 * | uint32_t (tx ring size) +--> TX Ring Size 274 * | uint32_t (tx free list size) +--> TX Free List Size 275 * | i40e_dma_buffer_t --------+--> TX Descriptor ring DMA 276 * | i40e_tx_desc_t * --------+--> TX descriptor ring 277 * | volatile unt32_t * +--> TX Write back head 278 * | uint32_t -------+--> TX ring head 279 * | uint32_t -------+--> TX ring tail 280 * | uint32_t -------+--> Num TX desc free 281 * | i40e_tx_control_block_t * --+--> TX control block array ---+ 282 * | i40e_tx_control_block_t ** --+--> TCB work list ----+ 283 * | i40e_tx_control_block_t ** --+--> TCB free list ---+ 284 * | uint32_t -------+--> Free TCB count | 285 * | i40e_rx_data_t * -------+--+ v 286 * +-------------------------------+ | +---------------------------+ 287 * | | Per-TX Frame Metadata | 288 * | | i40e_tx_control_block_t | 289 * +--------------------+ | | 290 * | mblk to transmit <--+--- mblk_t * | 291 * | type of transmit <--+--- i40e_tx_type_t | 292 * | TX DMA handle <--+--- ddi_dma_handle_t | 293 * v TX DMA buffer <--+--- i40e_dma_buffer_t | 294 * +------------------------------+ +---------------------------+ 295 * | Core Receive Data | 296 * | i40e_rx_data_t | 297 * | | 298 * | i40e_dma_buffer_t --+--> RX descriptor DMA Data 299 * | i40e_rx_desc_t --+--> RX descriptor ring 300 * | uint32_t --+--> Next free desc. 301 * | i40e_rx_control_block_t * --+--> RX Control Block Array ---+ 302 * | i40e_rx_control_block_t ** --+--> RCB work list ---+ 303 * | i40e_rx_control_block_t ** --+--> RCB free list ---+ 304 * +------------------------------+ | 305 * ^ | 306 * | +---------------------------+ | 307 * | | Per-RX Frame Metadata |<---------------+ 308 * | | i40e_rx_control_block_t | 309 * | | | 310 * | | mblk_t * ----+--> Received mblk_t data 311 * | | uint32_t ----+--> Reference count 312 * | | i40e_dma_buffer_t ----+--> Receive data DMA info 313 * | | frtn_t ----+--> mblk free function info 314 * +-----+-- i40e_rx_data_t * | 315 * +---------------------------+ 316 * 317 * ------------- 318 * Lock Ordering 319 * ------------- 320 * 321 * In order to ensure that we don't deadlock, the following represents the 322 * lock order being used. When grabbing locks, follow the following order. Lower 323 * numbers are more important. Thus, the i40e_glock which is number 0, must be 324 * taken before any other locks in the driver. On the other hand, the 325 * i40e_t`i40e_stat_lock, has the highest number because it's the least 326 * important lock. Note, that just because one lock is higher than another does 327 * not mean that all intermediary locks are required. 328 * 329 * 0) i40e_glock 330 * 1) i40e_t`i40e_general_lock 331 * 332 * 2) i40e_trqpair_t`itrq_rx_lock 333 * 3) i40e_trqpair_t`itrq_tx_lock 334 * 4) i40e_trqpair_t`itrq_intr_lock 335 * 5) i40e_t`i40e_rx_pending_lock 336 * 6) i40e_trqpair_t`itrq_tcb_lock 337 * 338 * 7) i40e_t`i40e_stat_lock 339 * 340 * Rules and expectations: 341 * 342 * 1) A thread holding locks belong to one PF should not hold locks belonging to 343 * a second. If for some reason this becomes necessary, locks should be grabbed 344 * based on the list order in the i40e_device_t, which implies that the 345 * i40e_glock is held. 346 * 347 * 2) When grabbing locks between multiple transmit and receive queues, the 348 * locks for the lowest number transmit/receive queue should be grabbed first. 349 * 350 * 3) When grabbing both the transmit and receive lock for a given queue, always 351 * grab i40e_trqpair_t`itrq_rx_lock before the i40e_trqpair_t`itrq_tx_lock. 352 * 353 * 4) The following pairs of locks are not expected to be held at the same time: 354 * 355 * o i40e_t`i40e_rx_pending_lock and i40e_trqpair_t`itrq_tcb_lock 356 * o i40e_trqpair_t`itrq_intr_lock is not expected to be held with any 357 * other lock except i40e_t`i40e_general_lock in mc_start(9E) and 358 * mc_stop(9e). 359 * 360 * ----------- 361 * Future Work 362 * ----------- 363 * 364 * At the moment the i40e_t driver is rather bare bones, allowing us to start 365 * getting data flowing and folks using it while we develop additional features. 366 * While bugs have been filed to cover this future work, the following gives an 367 * overview of expected work: 368 * 369 * o DMA binding and breaking up the locking in ring recycling. 370 * o Enhanced detection of device errors 371 * o Participation in IRM 372 * o FMA device reset 373 * o Stall detection, temperature error detection, etc. 374 * o More dynamic resource pools 375 */ 376 377 #include "i40e_sw.h" 378 379 static char i40e_ident[] = "Intel 10/40Gb Ethernet v1.0.3"; 380 381 /* 382 * The i40e_glock primarily protects the lists below and the i40e_device_t 383 * structures. 384 */ 385 static kmutex_t i40e_glock; 386 static list_t i40e_glist; 387 static list_t i40e_dlist; 388 389 /* 390 * Access attributes for register mapping. 391 */ 392 static ddi_device_acc_attr_t i40e_regs_acc_attr = { 393 DDI_DEVICE_ATTR_V1, 394 DDI_STRUCTURE_LE_ACC, 395 DDI_STRICTORDER_ACC, 396 DDI_FLAGERR_ACC 397 }; 398 399 /* 400 * Logging function for this driver. 401 */ 402 static void 403 i40e_dev_err(i40e_t *i40e, int level, boolean_t console, const char *fmt, 404 va_list ap) 405 { 406 char buf[1024]; 407 408 (void) vsnprintf(buf, sizeof (buf), fmt, ap); 409 410 if (i40e == NULL) { 411 cmn_err(level, (console) ? "%s: %s" : "!%s: %s", 412 I40E_MODULE_NAME, buf); 413 } else { 414 dev_err(i40e->i40e_dip, level, (console) ? "%s" : "!%s", 415 buf); 416 } 417 } 418 419 /* 420 * Because there's the stupid trailing-comma problem with the C preprocessor 421 * and variable arguments, I need to instantiate these. Pardon the redundant 422 * code. 423 */ 424 /*PRINTFLIKE2*/ 425 void 426 i40e_error(i40e_t *i40e, const char *fmt, ...) 427 { 428 va_list ap; 429 430 va_start(ap, fmt); 431 i40e_dev_err(i40e, CE_WARN, B_FALSE, fmt, ap); 432 va_end(ap); 433 } 434 435 /*PRINTFLIKE2*/ 436 void 437 i40e_log(i40e_t *i40e, const char *fmt, ...) 438 { 439 va_list ap; 440 441 va_start(ap, fmt); 442 i40e_dev_err(i40e, CE_NOTE, B_FALSE, fmt, ap); 443 va_end(ap); 444 } 445 446 /*PRINTFLIKE2*/ 447 void 448 i40e_notice(i40e_t *i40e, const char *fmt, ...) 449 { 450 va_list ap; 451 452 va_start(ap, fmt); 453 i40e_dev_err(i40e, CE_NOTE, B_TRUE, fmt, ap); 454 va_end(ap); 455 } 456 457 /* 458 * Various parts of the driver need to know if the controller is from the X722 459 * family, which has a few additional capabilities and different programming 460 * means. We don't consider virtual functions as part of this as they are quite 461 * different and will require substantially more work. 462 */ 463 static boolean_t 464 i40e_is_x722(i40e_t *i40e) 465 { 466 return (i40e->i40e_hw_space.mac.type == I40E_MAC_X722); 467 } 468 469 static void 470 i40e_device_rele(i40e_t *i40e) 471 { 472 i40e_device_t *idp = i40e->i40e_device; 473 474 if (idp == NULL) 475 return; 476 477 mutex_enter(&i40e_glock); 478 VERIFY(idp->id_nreg > 0); 479 list_remove(&idp->id_i40e_list, i40e); 480 idp->id_nreg--; 481 if (idp->id_nreg == 0) { 482 list_remove(&i40e_dlist, idp); 483 list_destroy(&idp->id_i40e_list); 484 kmem_free(idp->id_rsrcs, sizeof (i40e_switch_rsrc_t) * 485 idp->id_rsrcs_alloc); 486 kmem_free(idp, sizeof (i40e_device_t)); 487 } 488 i40e->i40e_device = NULL; 489 mutex_exit(&i40e_glock); 490 } 491 492 static i40e_device_t * 493 i40e_device_find(i40e_t *i40e, dev_info_t *parent, uint_t bus, uint_t device) 494 { 495 i40e_device_t *idp; 496 mutex_enter(&i40e_glock); 497 for (idp = list_head(&i40e_dlist); idp != NULL; 498 idp = list_next(&i40e_dlist, idp)) { 499 if (idp->id_parent == parent && idp->id_pci_bus == bus && 500 idp->id_pci_device == device) { 501 break; 502 } 503 } 504 505 if (idp != NULL) { 506 VERIFY(idp->id_nreg < idp->id_nfuncs); 507 idp->id_nreg++; 508 } else { 509 i40e_hw_t *hw = &i40e->i40e_hw_space; 510 ASSERT(hw->num_ports > 0); 511 ASSERT(hw->num_partitions > 0); 512 513 /* 514 * The Intel common code doesn't exactly keep the number of PCI 515 * functions. But it calculates it during discovery of 516 * partitions and ports. So what we do is undo the calculation 517 * that it does originally, as functions are evenly spread 518 * across ports in the rare case of partitions. 519 */ 520 idp = kmem_alloc(sizeof (i40e_device_t), KM_SLEEP); 521 idp->id_parent = parent; 522 idp->id_pci_bus = bus; 523 idp->id_pci_device = device; 524 idp->id_nfuncs = hw->num_ports * hw->num_partitions; 525 idp->id_nreg = 1; 526 idp->id_rsrcs_alloc = i40e->i40e_switch_rsrc_alloc; 527 idp->id_rsrcs_act = i40e->i40e_switch_rsrc_actual; 528 idp->id_rsrcs = kmem_alloc(sizeof (i40e_switch_rsrc_t) * 529 idp->id_rsrcs_alloc, KM_SLEEP); 530 bcopy(i40e->i40e_switch_rsrcs, idp->id_rsrcs, 531 sizeof (i40e_switch_rsrc_t) * idp->id_rsrcs_alloc); 532 list_create(&idp->id_i40e_list, sizeof (i40e_t), 533 offsetof(i40e_t, i40e_dlink)); 534 535 list_insert_tail(&i40e_dlist, idp); 536 } 537 538 list_insert_tail(&idp->id_i40e_list, i40e); 539 mutex_exit(&i40e_glock); 540 541 return (idp); 542 } 543 544 static void 545 i40e_link_state_set(i40e_t *i40e, link_state_t state) 546 { 547 if (i40e->i40e_link_state == state) 548 return; 549 550 i40e->i40e_link_state = state; 551 mac_link_update(i40e->i40e_mac_hdl, i40e->i40e_link_state); 552 } 553 554 /* 555 * This is a basic link check routine. Mostly we're using this just to see 556 * if we can get any accurate information about the state of the link being 557 * up or down, as well as updating the link state, speed, etc. information. 558 */ 559 void 560 i40e_link_check(i40e_t *i40e) 561 { 562 i40e_hw_t *hw = &i40e->i40e_hw_space; 563 boolean_t ls; 564 int ret; 565 566 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 567 568 hw->phy.get_link_info = B_TRUE; 569 if ((ret = i40e_get_link_status(hw, &ls)) != I40E_SUCCESS) { 570 i40e->i40e_s_link_status_errs++; 571 i40e->i40e_s_link_status_lasterr = ret; 572 return; 573 } 574 575 /* 576 * Firmware abstracts all of the mac and phy information for us, so we 577 * can use i40e_get_link_status to determine the current state. 578 */ 579 if (ls == B_TRUE) { 580 enum i40e_aq_link_speed speed; 581 582 speed = i40e_get_link_speed(hw); 583 584 /* 585 * Translate from an i40e value to a value in Mbits/s. 586 */ 587 switch (speed) { 588 case I40E_LINK_SPEED_100MB: 589 i40e->i40e_link_speed = 100; 590 break; 591 case I40E_LINK_SPEED_1GB: 592 i40e->i40e_link_speed = 1000; 593 break; 594 case I40E_LINK_SPEED_2_5GB: 595 i40e->i40e_link_speed = 2500; 596 break; 597 case I40E_LINK_SPEED_5GB: 598 i40e->i40e_link_speed = 5000; 599 break; 600 case I40E_LINK_SPEED_10GB: 601 i40e->i40e_link_speed = 10000; 602 break; 603 case I40E_LINK_SPEED_20GB: 604 i40e->i40e_link_speed = 20000; 605 break; 606 case I40E_LINK_SPEED_40GB: 607 i40e->i40e_link_speed = 40000; 608 break; 609 case I40E_LINK_SPEED_25GB: 610 i40e->i40e_link_speed = 25000; 611 break; 612 default: 613 i40e->i40e_link_speed = 0; 614 break; 615 } 616 617 /* 618 * At this time, hardware does not support half-duplex 619 * operation, hence why we don't ask the hardware about our 620 * current speed. 621 */ 622 i40e->i40e_link_duplex = LINK_DUPLEX_FULL; 623 i40e_link_state_set(i40e, LINK_STATE_UP); 624 } else { 625 i40e->i40e_link_speed = 0; 626 i40e->i40e_link_duplex = 0; 627 i40e_link_state_set(i40e, LINK_STATE_DOWN); 628 } 629 } 630 631 static void 632 i40e_rem_intrs(i40e_t *i40e) 633 { 634 int i, rc; 635 636 for (i = 0; i < i40e->i40e_intr_count; i++) { 637 rc = ddi_intr_free(i40e->i40e_intr_handles[i]); 638 if (rc != DDI_SUCCESS) { 639 i40e_log(i40e, "failed to free interrupt %d: %d", 640 i, rc); 641 } 642 } 643 644 kmem_free(i40e->i40e_intr_handles, i40e->i40e_intr_size); 645 i40e->i40e_intr_handles = NULL; 646 } 647 648 static void 649 i40e_rem_intr_handlers(i40e_t *i40e) 650 { 651 int i, rc; 652 653 for (i = 0; i < i40e->i40e_intr_count; i++) { 654 rc = ddi_intr_remove_handler(i40e->i40e_intr_handles[i]); 655 if (rc != DDI_SUCCESS) { 656 i40e_log(i40e, "failed to remove interrupt %d: %d", 657 i, rc); 658 } 659 } 660 } 661 662 /* 663 * illumos Fault Management Architecture (FMA) support. 664 */ 665 666 int 667 i40e_check_acc_handle(ddi_acc_handle_t handle) 668 { 669 ddi_fm_error_t de; 670 671 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 672 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 673 return (de.fme_status); 674 } 675 676 int 677 i40e_check_dma_handle(ddi_dma_handle_t handle) 678 { 679 ddi_fm_error_t de; 680 681 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 682 return (de.fme_status); 683 } 684 685 /* 686 * Fault service error handling callback function. 687 */ 688 /* ARGSUSED */ 689 static int 690 i40e_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 691 { 692 pci_ereport_post(dip, err, NULL); 693 return (err->fme_status); 694 } 695 696 static void 697 i40e_fm_init(i40e_t *i40e) 698 { 699 ddi_iblock_cookie_t iblk; 700 701 i40e->i40e_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY, 702 i40e->i40e_dip, DDI_PROP_DONTPASS, "fm_capable", 703 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 704 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 705 706 if (i40e->i40e_fm_capabilities < 0) { 707 i40e->i40e_fm_capabilities = 0; 708 } else if (i40e->i40e_fm_capabilities > 0xf) { 709 i40e->i40e_fm_capabilities = DDI_FM_EREPORT_CAPABLE | 710 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | 711 DDI_FM_ERRCB_CAPABLE; 712 } 713 714 /* 715 * Only register with IO Fault Services if we have some capability 716 */ 717 if (i40e->i40e_fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 718 i40e_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 719 } else { 720 i40e_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 721 } 722 723 if (i40e->i40e_fm_capabilities) { 724 ddi_fm_init(i40e->i40e_dip, &i40e->i40e_fm_capabilities, &iblk); 725 726 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 727 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 728 pci_ereport_setup(i40e->i40e_dip); 729 } 730 731 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 732 ddi_fm_handler_register(i40e->i40e_dip, 733 i40e_fm_error_cb, (void*)i40e); 734 } 735 } 736 737 if (i40e->i40e_fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 738 i40e_init_dma_attrs(i40e, B_TRUE); 739 } else { 740 i40e_init_dma_attrs(i40e, B_FALSE); 741 } 742 } 743 744 static void 745 i40e_fm_fini(i40e_t *i40e) 746 { 747 if (i40e->i40e_fm_capabilities) { 748 749 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 750 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 751 pci_ereport_teardown(i40e->i40e_dip); 752 753 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 754 ddi_fm_handler_unregister(i40e->i40e_dip); 755 756 ddi_fm_fini(i40e->i40e_dip); 757 } 758 } 759 760 void 761 i40e_fm_ereport(i40e_t *i40e, char *detail) 762 { 763 uint64_t ena; 764 char buf[FM_MAX_CLASS]; 765 766 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 767 ena = fm_ena_generate(0, FM_ENA_FMT1); 768 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities)) { 769 ddi_fm_ereport_post(i40e->i40e_dip, buf, ena, DDI_NOSLEEP, 770 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 771 } 772 } 773 774 /* 775 * Here we're trying to set the SEID of the default VSI. In general, 776 * when we come through and look at this shortly after attach, we 777 * expect there to only be a single element present, which is the 778 * default VSI. Importantly, each PF seems to not see any other 779 * devices, in part because of the simple switch mode that we're 780 * using. If for some reason, we see more artifacts, we'll need to 781 * revisit what we're doing here. 782 */ 783 static boolean_t 784 i40e_set_def_vsi_seid(i40e_t *i40e) 785 { 786 i40e_hw_t *hw = &i40e->i40e_hw_space; 787 struct i40e_aqc_get_switch_config_resp *sw_config; 788 uint8_t aq_buf[I40E_AQ_LARGE_BUF]; 789 uint16_t next = 0; 790 int rc; 791 792 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 793 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 794 rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next, 795 NULL); 796 if (rc != I40E_SUCCESS) { 797 i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d", 798 rc, hw->aq.asq_last_status); 799 return (B_FALSE); 800 } 801 802 if (LE_16(sw_config->header.num_reported) != 1) { 803 i40e_error(i40e, "encountered multiple (%d) switching units " 804 "during attach, not proceeding", 805 LE_16(sw_config->header.num_reported)); 806 return (B_FALSE); 807 } 808 809 I40E_DEF_VSI_SEID(i40e) = sw_config->element[0].seid; 810 return (B_TRUE); 811 } 812 813 /* 814 * Get the SEID of the uplink MAC. 815 */ 816 static int 817 i40e_get_mac_seid(i40e_t *i40e) 818 { 819 i40e_hw_t *hw = &i40e->i40e_hw_space; 820 struct i40e_aqc_get_switch_config_resp *sw_config; 821 uint8_t aq_buf[I40E_AQ_LARGE_BUF]; 822 uint16_t next = 0; 823 int rc; 824 825 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 826 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 827 rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next, 828 NULL); 829 if (rc != I40E_SUCCESS) { 830 i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d", 831 rc, hw->aq.asq_last_status); 832 return (-1); 833 } 834 835 return (LE_16(sw_config->element[0].uplink_seid)); 836 } 837 838 /* 839 * We need to fill the i40e_hw_t structure with the capabilities of this PF. We 840 * must also provide the memory for it; however, we don't need to keep it around 841 * to the call to the common code. It takes it and parses it into an internal 842 * structure. 843 */ 844 static boolean_t 845 i40e_get_hw_capabilities(i40e_t *i40e, i40e_hw_t *hw) 846 { 847 struct i40e_aqc_list_capabilities_element_resp *buf; 848 int rc; 849 size_t len; 850 uint16_t needed; 851 int nelems = I40E_HW_CAP_DEFAULT; 852 853 len = nelems * sizeof (*buf); 854 855 for (;;) { 856 ASSERT(len > 0); 857 buf = kmem_alloc(len, KM_SLEEP); 858 rc = i40e_aq_discover_capabilities(hw, buf, len, 859 &needed, i40e_aqc_opc_list_func_capabilities, NULL); 860 kmem_free(buf, len); 861 862 if (hw->aq.asq_last_status == I40E_AQ_RC_ENOMEM && 863 nelems == I40E_HW_CAP_DEFAULT) { 864 if (nelems == needed) { 865 i40e_error(i40e, "Capability discovery failed " 866 "due to byzantine common code"); 867 return (B_FALSE); 868 } 869 len = needed; 870 continue; 871 } else if (rc != I40E_SUCCESS || 872 hw->aq.asq_last_status != I40E_AQ_RC_OK) { 873 i40e_error(i40e, "Capability discovery failed: %d", rc); 874 return (B_FALSE); 875 } 876 877 break; 878 } 879 880 return (B_TRUE); 881 } 882 883 /* 884 * Obtain the switch's capabilities as seen by this PF and keep it around for 885 * our later use. 886 */ 887 static boolean_t 888 i40e_get_switch_resources(i40e_t *i40e) 889 { 890 i40e_hw_t *hw = &i40e->i40e_hw_space; 891 uint8_t cnt = 2; 892 uint8_t act; 893 size_t size; 894 i40e_switch_rsrc_t *buf; 895 896 for (;;) { 897 enum i40e_status_code ret; 898 size = cnt * sizeof (i40e_switch_rsrc_t); 899 ASSERT(size > 0); 900 if (size > UINT16_MAX) 901 return (B_FALSE); 902 buf = kmem_alloc(size, KM_SLEEP); 903 904 ret = i40e_aq_get_switch_resource_alloc(hw, &act, buf, 905 cnt, NULL); 906 if (ret == I40E_ERR_ADMIN_QUEUE_ERROR && 907 hw->aq.asq_last_status == I40E_AQ_RC_EINVAL) { 908 kmem_free(buf, size); 909 cnt += I40E_SWITCH_CAP_DEFAULT; 910 continue; 911 } else if (ret != I40E_SUCCESS) { 912 kmem_free(buf, size); 913 i40e_error(i40e, 914 "failed to retrieve switch statistics: %d", ret); 915 return (B_FALSE); 916 } 917 918 break; 919 } 920 921 i40e->i40e_switch_rsrc_alloc = cnt; 922 i40e->i40e_switch_rsrc_actual = act; 923 i40e->i40e_switch_rsrcs = buf; 924 925 return (B_TRUE); 926 } 927 928 static void 929 i40e_cleanup_resources(i40e_t *i40e) 930 { 931 if (i40e->i40e_uaddrs != NULL) { 932 kmem_free(i40e->i40e_uaddrs, sizeof (i40e_uaddr_t) * 933 i40e->i40e_resources.ifr_nmacfilt); 934 i40e->i40e_uaddrs = NULL; 935 } 936 937 if (i40e->i40e_maddrs != NULL) { 938 kmem_free(i40e->i40e_maddrs, sizeof (i40e_maddr_t) * 939 i40e->i40e_resources.ifr_nmcastfilt); 940 i40e->i40e_maddrs = NULL; 941 } 942 943 if (i40e->i40e_switch_rsrcs != NULL) { 944 size_t sz = sizeof (i40e_switch_rsrc_t) * 945 i40e->i40e_switch_rsrc_alloc; 946 ASSERT(sz > 0); 947 kmem_free(i40e->i40e_switch_rsrcs, sz); 948 i40e->i40e_switch_rsrcs = NULL; 949 } 950 951 if (i40e->i40e_device != NULL) 952 i40e_device_rele(i40e); 953 } 954 955 static boolean_t 956 i40e_get_available_resources(i40e_t *i40e) 957 { 958 dev_info_t *parent; 959 uint16_t bus, device, func; 960 uint_t nregs; 961 int *regs, i; 962 i40e_device_t *idp; 963 i40e_hw_t *hw = &i40e->i40e_hw_space; 964 965 parent = ddi_get_parent(i40e->i40e_dip); 966 967 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, i40e->i40e_dip, 0, "reg", 968 ®s, &nregs) != DDI_PROP_SUCCESS) { 969 return (B_FALSE); 970 } 971 972 if (nregs < 1) { 973 ddi_prop_free(regs); 974 return (B_FALSE); 975 } 976 977 bus = PCI_REG_BUS_G(regs[0]); 978 device = PCI_REG_DEV_G(regs[0]); 979 func = PCI_REG_FUNC_G(regs[0]); 980 ddi_prop_free(regs); 981 982 i40e->i40e_hw_space.bus.func = func; 983 i40e->i40e_hw_space.bus.device = device; 984 985 if (i40e_get_switch_resources(i40e) == B_FALSE) { 986 return (B_FALSE); 987 } 988 989 /* 990 * To calculate the total amount of a resource we have available, we 991 * need to add how many our i40e_t thinks it has guaranteed, if any, and 992 * then we need to go through and divide the number of available on the 993 * device, which was snapshotted before anyone should have allocated 994 * anything, and use that to derive how many are available from the 995 * pool. Longer term, we may want to turn this into something that's 996 * more of a pool-like resource that everything can share (though that 997 * may require some more assistance from MAC). 998 * 999 * Though for transmit and receive queue pairs, we just have to ask 1000 * firmware instead. 1001 */ 1002 idp = i40e_device_find(i40e, parent, bus, device); 1003 i40e->i40e_device = idp; 1004 i40e->i40e_resources.ifr_nvsis = 0; 1005 i40e->i40e_resources.ifr_nvsis_used = 0; 1006 i40e->i40e_resources.ifr_nmacfilt = 0; 1007 i40e->i40e_resources.ifr_nmacfilt_used = 0; 1008 i40e->i40e_resources.ifr_nmcastfilt = 0; 1009 i40e->i40e_resources.ifr_nmcastfilt_used = 0; 1010 1011 for (i = 0; i < i40e->i40e_switch_rsrc_actual; i++) { 1012 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 1013 1014 switch (srp->resource_type) { 1015 case I40E_AQ_RESOURCE_TYPE_VSI: 1016 i40e->i40e_resources.ifr_nvsis += 1017 LE_16(srp->guaranteed); 1018 i40e->i40e_resources.ifr_nvsis_used = LE_16(srp->used); 1019 break; 1020 case I40E_AQ_RESOURCE_TYPE_MACADDR: 1021 i40e->i40e_resources.ifr_nmacfilt += 1022 LE_16(srp->guaranteed); 1023 i40e->i40e_resources.ifr_nmacfilt_used = 1024 LE_16(srp->used); 1025 break; 1026 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 1027 i40e->i40e_resources.ifr_nmcastfilt += 1028 LE_16(srp->guaranteed); 1029 i40e->i40e_resources.ifr_nmcastfilt_used = 1030 LE_16(srp->used); 1031 break; 1032 default: 1033 break; 1034 } 1035 } 1036 1037 for (i = 0; i < idp->id_rsrcs_act; i++) { 1038 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 1039 switch (srp->resource_type) { 1040 case I40E_AQ_RESOURCE_TYPE_VSI: 1041 i40e->i40e_resources.ifr_nvsis += 1042 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1043 break; 1044 case I40E_AQ_RESOURCE_TYPE_MACADDR: 1045 i40e->i40e_resources.ifr_nmacfilt += 1046 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1047 break; 1048 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 1049 i40e->i40e_resources.ifr_nmcastfilt += 1050 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1051 default: 1052 break; 1053 } 1054 } 1055 1056 i40e->i40e_resources.ifr_nrx_queue = hw->func_caps.num_rx_qp; 1057 i40e->i40e_resources.ifr_ntx_queue = hw->func_caps.num_tx_qp; 1058 1059 i40e->i40e_uaddrs = kmem_zalloc(sizeof (i40e_uaddr_t) * 1060 i40e->i40e_resources.ifr_nmacfilt, KM_SLEEP); 1061 i40e->i40e_maddrs = kmem_zalloc(sizeof (i40e_maddr_t) * 1062 i40e->i40e_resources.ifr_nmcastfilt, KM_SLEEP); 1063 1064 /* 1065 * Initialize these as multicast addresses to indicate it's invalid for 1066 * sanity purposes. Think of it like 0xdeadbeef. 1067 */ 1068 for (i = 0; i < i40e->i40e_resources.ifr_nmacfilt; i++) 1069 i40e->i40e_uaddrs[i].iua_mac[0] = 0x01; 1070 1071 return (B_TRUE); 1072 } 1073 1074 static boolean_t 1075 i40e_enable_interrupts(i40e_t *i40e) 1076 { 1077 int i, rc; 1078 1079 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1080 rc = ddi_intr_block_enable(i40e->i40e_intr_handles, 1081 i40e->i40e_intr_count); 1082 if (rc != DDI_SUCCESS) { 1083 i40e_error(i40e, "Interrupt block-enable failed: %d", 1084 rc); 1085 return (B_FALSE); 1086 } 1087 } else { 1088 for (i = 0; i < i40e->i40e_intr_count; i++) { 1089 rc = ddi_intr_enable(i40e->i40e_intr_handles[i]); 1090 if (rc != DDI_SUCCESS) { 1091 i40e_error(i40e, 1092 "Failed to enable interrupt %d: %d", i, rc); 1093 while (--i >= 0) { 1094 (void) ddi_intr_disable( 1095 i40e->i40e_intr_handles[i]); 1096 } 1097 return (B_FALSE); 1098 } 1099 } 1100 } 1101 1102 return (B_TRUE); 1103 } 1104 1105 static boolean_t 1106 i40e_disable_interrupts(i40e_t *i40e) 1107 { 1108 int i, rc; 1109 1110 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1111 rc = ddi_intr_block_disable(i40e->i40e_intr_handles, 1112 i40e->i40e_intr_count); 1113 if (rc != DDI_SUCCESS) { 1114 i40e_error(i40e, 1115 "Interrupt block-disabled failed: %d", rc); 1116 return (B_FALSE); 1117 } 1118 } else { 1119 for (i = 0; i < i40e->i40e_intr_count; i++) { 1120 rc = ddi_intr_disable(i40e->i40e_intr_handles[i]); 1121 if (rc != DDI_SUCCESS) { 1122 i40e_error(i40e, 1123 "Failed to disable interrupt %d: %d", 1124 i, rc); 1125 return (B_FALSE); 1126 } 1127 } 1128 } 1129 1130 return (B_TRUE); 1131 } 1132 1133 /* 1134 * Free receive & transmit rings. 1135 */ 1136 static void 1137 i40e_free_trqpairs(i40e_t *i40e) 1138 { 1139 i40e_trqpair_t *itrq; 1140 1141 if (i40e->i40e_rx_groups != NULL) { 1142 kmem_free(i40e->i40e_rx_groups, 1143 sizeof (i40e_rx_group_t) * i40e->i40e_num_rx_groups); 1144 i40e->i40e_rx_groups = NULL; 1145 } 1146 1147 if (i40e->i40e_trqpairs != NULL) { 1148 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1149 itrq = &i40e->i40e_trqpairs[i]; 1150 mutex_destroy(&itrq->itrq_intr_lock); 1151 mutex_destroy(&itrq->itrq_rx_lock); 1152 mutex_destroy(&itrq->itrq_tx_lock); 1153 mutex_destroy(&itrq->itrq_tcb_lock); 1154 cv_destroy(&itrq->itrq_intr_cv); 1155 cv_destroy(&itrq->itrq_tx_cv); 1156 1157 i40e_stats_trqpair_fini(itrq); 1158 } 1159 1160 kmem_free(i40e->i40e_trqpairs, 1161 sizeof (i40e_trqpair_t) * i40e->i40e_num_trqpairs); 1162 i40e->i40e_trqpairs = NULL; 1163 } 1164 1165 cv_destroy(&i40e->i40e_rx_pending_cv); 1166 mutex_destroy(&i40e->i40e_rx_pending_lock); 1167 mutex_destroy(&i40e->i40e_general_lock); 1168 } 1169 1170 /* 1171 * Allocate transmit and receive rings, as well as other data structures that we 1172 * need. 1173 */ 1174 static boolean_t 1175 i40e_alloc_trqpairs(i40e_t *i40e) 1176 { 1177 void *mutexpri = DDI_INTR_PRI(i40e->i40e_intr_pri); 1178 1179 /* 1180 * Now that we have the priority for the interrupts, initialize 1181 * all relevant locks. 1182 */ 1183 mutex_init(&i40e->i40e_general_lock, NULL, MUTEX_DRIVER, mutexpri); 1184 mutex_init(&i40e->i40e_rx_pending_lock, NULL, MUTEX_DRIVER, mutexpri); 1185 cv_init(&i40e->i40e_rx_pending_cv, NULL, CV_DRIVER, NULL); 1186 1187 i40e->i40e_trqpairs = kmem_zalloc(sizeof (i40e_trqpair_t) * 1188 i40e->i40e_num_trqpairs, KM_SLEEP); 1189 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1190 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 1191 1192 itrq->itrq_i40e = i40e; 1193 mutex_init(&itrq->itrq_intr_lock, NULL, MUTEX_DRIVER, mutexpri); 1194 mutex_init(&itrq->itrq_rx_lock, NULL, MUTEX_DRIVER, mutexpri); 1195 mutex_init(&itrq->itrq_tx_lock, NULL, MUTEX_DRIVER, mutexpri); 1196 mutex_init(&itrq->itrq_tcb_lock, NULL, MUTEX_DRIVER, mutexpri); 1197 cv_init(&itrq->itrq_intr_cv, NULL, CV_DRIVER, NULL); 1198 cv_init(&itrq->itrq_tx_cv, NULL, CV_DRIVER, NULL); 1199 itrq->itrq_index = i; 1200 itrq->itrq_intr_quiesce = B_TRUE; 1201 itrq->itrq_tx_quiesce = B_TRUE; 1202 } 1203 1204 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1205 /* 1206 * Keeping this in a separate iteration makes the 1207 * clean up path safe. 1208 */ 1209 if (!i40e_stats_trqpair_init(&i40e->i40e_trqpairs[i])) { 1210 i40e_free_trqpairs(i40e); 1211 return (B_FALSE); 1212 } 1213 } 1214 1215 i40e->i40e_rx_groups = kmem_zalloc(sizeof (i40e_rx_group_t) * 1216 i40e->i40e_num_rx_groups, KM_SLEEP); 1217 1218 for (uint_t i = 0; i < i40e->i40e_num_rx_groups; i++) { 1219 i40e_rx_group_t *rxg = &i40e->i40e_rx_groups[i]; 1220 1221 rxg->irg_index = i; 1222 rxg->irg_i40e = i40e; 1223 } 1224 1225 return (B_TRUE); 1226 } 1227 1228 1229 1230 /* 1231 * Unless a .conf file already overrode i40e_t structure values, they will 1232 * be 0, and need to be set in conjunction with the now-available HW report. 1233 */ 1234 /* ARGSUSED */ 1235 static void 1236 i40e_hw_to_instance(i40e_t *i40e, i40e_hw_t *hw) 1237 { 1238 if (i40e->i40e_num_trqpairs_per_vsi == 0) { 1239 if (i40e_is_x722(i40e)) { 1240 i40e->i40e_num_trqpairs_per_vsi = 1241 I40E_722_MAX_TC_QUEUES; 1242 } else { 1243 i40e->i40e_num_trqpairs_per_vsi = 1244 I40E_710_MAX_TC_QUEUES; 1245 } 1246 } 1247 1248 if (i40e->i40e_num_rx_groups == 0) { 1249 i40e->i40e_num_rx_groups = I40E_DEF_NUM_RX_GROUPS; 1250 } 1251 } 1252 1253 /* 1254 * Free any resources required by, or setup by, the Intel common code. 1255 */ 1256 static void 1257 i40e_common_code_fini(i40e_t *i40e) 1258 { 1259 i40e_hw_t *hw = &i40e->i40e_hw_space; 1260 int rc; 1261 1262 rc = i40e_shutdown_lan_hmc(hw); 1263 if (rc != I40E_SUCCESS) 1264 i40e_error(i40e, "failed to shutdown LAN hmc: %d", rc); 1265 1266 rc = i40e_shutdown_adminq(hw); 1267 if (rc != I40E_SUCCESS) 1268 i40e_error(i40e, "failed to shutdown admin queue: %d", rc); 1269 } 1270 1271 /* 1272 * Initialize and call Intel common-code routines, includes some setup 1273 * the common code expects from the driver. Also prints on failure, so 1274 * the caller doesn't have to. 1275 */ 1276 static boolean_t 1277 i40e_common_code_init(i40e_t *i40e, i40e_hw_t *hw) 1278 { 1279 int rc; 1280 1281 i40e_clear_hw(hw); 1282 rc = i40e_pf_reset(hw); 1283 if (rc != 0) { 1284 i40e_error(i40e, "failed to reset hardware: %d", rc); 1285 i40e_fm_ereport(i40e, DDI_FM_DEVICE_NO_RESPONSE); 1286 return (B_FALSE); 1287 } 1288 1289 rc = i40e_init_shared_code(hw); 1290 if (rc != 0) { 1291 i40e_error(i40e, "failed to initialize i40e core: %d", rc); 1292 return (B_FALSE); 1293 } 1294 1295 hw->aq.num_arq_entries = I40E_DEF_ADMINQ_SIZE; 1296 hw->aq.num_asq_entries = I40E_DEF_ADMINQ_SIZE; 1297 hw->aq.arq_buf_size = I40E_ADMINQ_BUFSZ; 1298 hw->aq.asq_buf_size = I40E_ADMINQ_BUFSZ; 1299 1300 rc = i40e_init_adminq(hw); 1301 if (rc != 0) { 1302 i40e_error(i40e, "failed to initialize firmware admin queue: " 1303 "%d, potential firmware version mismatch", rc); 1304 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 1305 return (B_FALSE); 1306 } 1307 1308 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 1309 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) { 1310 i40e_log(i40e, "The driver for the device detected a newer " 1311 "version of the NVM image (%d.%d) than expected (%d.%d).\n" 1312 "Please install the most recent version of the network " 1313 "driver.\n", hw->aq.api_maj_ver, hw->aq.api_min_ver, 1314 I40E_FW_API_VERSION_MAJOR, I40E_FW_MINOR_VERSION(hw)); 1315 } else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 1316 hw->aq.api_min_ver < (I40E_FW_MINOR_VERSION(hw) - 1)) { 1317 i40e_log(i40e, "The driver for the device detected an older" 1318 " version of the NVM image (%d.%d) than expected (%d.%d)." 1319 "\nPlease update the NVM image.\n", 1320 hw->aq.api_maj_ver, hw->aq.api_min_ver, 1321 I40E_FW_API_VERSION_MAJOR, I40E_FW_MINOR_VERSION(hw) - 1); 1322 } 1323 1324 i40e_clear_pxe_mode(hw); 1325 1326 /* 1327 * We need to call this so that the common code can discover 1328 * capabilities of the hardware, which it uses throughout the rest. 1329 */ 1330 if (!i40e_get_hw_capabilities(i40e, hw)) { 1331 i40e_error(i40e, "failed to obtain hardware capabilities"); 1332 return (B_FALSE); 1333 } 1334 1335 if (i40e_get_available_resources(i40e) == B_FALSE) { 1336 i40e_error(i40e, "failed to obtain hardware resources"); 1337 return (B_FALSE); 1338 } 1339 1340 i40e_hw_to_instance(i40e, hw); 1341 1342 rc = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 1343 hw->func_caps.num_rx_qp, 0, 0); 1344 if (rc != 0) { 1345 i40e_error(i40e, "failed to initialize hardware memory cache: " 1346 "%d", rc); 1347 return (B_FALSE); 1348 } 1349 1350 rc = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 1351 if (rc != 0) { 1352 i40e_error(i40e, "failed to configure hardware memory cache: " 1353 "%d", rc); 1354 return (B_FALSE); 1355 } 1356 1357 (void) i40e_aq_stop_lldp(hw, TRUE, FALSE, NULL); 1358 1359 rc = i40e_get_mac_addr(hw, hw->mac.addr); 1360 if (rc != I40E_SUCCESS) { 1361 i40e_error(i40e, "failed to retrieve hardware mac address: %d", 1362 rc); 1363 return (B_FALSE); 1364 } 1365 1366 rc = i40e_validate_mac_addr(hw->mac.addr); 1367 if (rc != 0) { 1368 i40e_error(i40e, "failed to validate internal mac address: " 1369 "%d", rc); 1370 return (B_FALSE); 1371 } 1372 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 1373 if ((rc = i40e_get_port_mac_addr(hw, hw->mac.port_addr)) != 1374 I40E_SUCCESS) { 1375 i40e_error(i40e, "failed to retrieve port mac address: %d", 1376 rc); 1377 return (B_FALSE); 1378 } 1379 1380 /* 1381 * We need to obtain the Default Virtual Station SEID (VSI) 1382 * before we can perform other operations on the device. 1383 */ 1384 if (!i40e_set_def_vsi_seid(i40e)) { 1385 i40e_error(i40e, "failed to obtain Default VSI SEID"); 1386 return (B_FALSE); 1387 } 1388 1389 return (B_TRUE); 1390 } 1391 1392 static void 1393 i40e_unconfigure(dev_info_t *devinfo, i40e_t *i40e) 1394 { 1395 int rc; 1396 1397 if (i40e->i40e_attach_progress & I40E_ATTACH_ENABLE_INTR) 1398 (void) i40e_disable_interrupts(i40e); 1399 1400 if ((i40e->i40e_attach_progress & I40E_ATTACH_LINK_TIMER) && 1401 i40e->i40e_periodic_id != 0) { 1402 ddi_periodic_delete(i40e->i40e_periodic_id); 1403 i40e->i40e_periodic_id = 0; 1404 } 1405 1406 if (i40e->i40e_attach_progress & I40E_ATTACH_UFM_INIT) 1407 ddi_ufm_fini(i40e->i40e_ufmh); 1408 1409 if (i40e->i40e_attach_progress & I40E_ATTACH_MAC) { 1410 rc = mac_unregister(i40e->i40e_mac_hdl); 1411 if (rc != 0) { 1412 i40e_error(i40e, "failed to unregister from mac: %d", 1413 rc); 1414 } 1415 } 1416 1417 if (i40e->i40e_attach_progress & I40E_ATTACH_STATS) { 1418 i40e_stats_fini(i40e); 1419 } 1420 1421 if (i40e->i40e_attach_progress & I40E_ATTACH_ADD_INTR) 1422 i40e_rem_intr_handlers(i40e); 1423 1424 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_RINGSLOCKS) 1425 i40e_free_trqpairs(i40e); 1426 1427 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_INTR) 1428 i40e_rem_intrs(i40e); 1429 1430 if (i40e->i40e_attach_progress & I40E_ATTACH_COMMON_CODE) 1431 i40e_common_code_fini(i40e); 1432 1433 i40e_cleanup_resources(i40e); 1434 1435 if (i40e->i40e_attach_progress & I40E_ATTACH_PROPS) 1436 (void) ddi_prop_remove_all(devinfo); 1437 1438 if (i40e->i40e_attach_progress & I40E_ATTACH_REGS_MAP && 1439 i40e->i40e_osdep_space.ios_reg_handle != NULL) { 1440 ddi_regs_map_free(&i40e->i40e_osdep_space.ios_reg_handle); 1441 i40e->i40e_osdep_space.ios_reg_handle = NULL; 1442 } 1443 1444 if ((i40e->i40e_attach_progress & I40E_ATTACH_PCI_CONFIG) && 1445 i40e->i40e_osdep_space.ios_cfg_handle != NULL) { 1446 pci_config_teardown(&i40e->i40e_osdep_space.ios_cfg_handle); 1447 i40e->i40e_osdep_space.ios_cfg_handle = NULL; 1448 } 1449 1450 if (i40e->i40e_attach_progress & I40E_ATTACH_FM_INIT) 1451 i40e_fm_fini(i40e); 1452 1453 kmem_free(i40e->i40e_aqbuf, I40E_ADMINQ_BUFSZ); 1454 kmem_free(i40e, sizeof (i40e_t)); 1455 1456 ddi_set_driver_private(devinfo, NULL); 1457 } 1458 1459 static boolean_t 1460 i40e_final_init(i40e_t *i40e) 1461 { 1462 i40e_hw_t *hw = &i40e->i40e_hw_space; 1463 struct i40e_osdep *osdep = OS_DEP(hw); 1464 uint8_t pbanum[I40E_PBANUM_STRLEN]; 1465 enum i40e_status_code irc; 1466 char buf[I40E_DDI_PROP_LEN]; 1467 1468 pbanum[0] = '\0'; 1469 irc = i40e_read_pba_string(hw, pbanum, sizeof (pbanum)); 1470 if (irc != I40E_SUCCESS) { 1471 i40e_log(i40e, "failed to read PBA string: %d", irc); 1472 } else { 1473 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1474 "printed-board-assembly", (char *)pbanum); 1475 } 1476 1477 #ifdef DEBUG 1478 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.fw_maj_ver, 1479 hw->aq.fw_min_ver) < sizeof (buf)); 1480 ASSERT(snprintf(NULL, 0, "%x", hw->aq.fw_build) < sizeof (buf)); 1481 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.api_maj_ver, 1482 hw->aq.api_min_ver) < sizeof (buf)); 1483 #endif 1484 1485 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.fw_maj_ver, 1486 hw->aq.fw_min_ver); 1487 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1488 "firmware-version", buf); 1489 (void) snprintf(buf, sizeof (buf), "%x", hw->aq.fw_build); 1490 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1491 "firmware-build", buf); 1492 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.api_maj_ver, 1493 hw->aq.api_min_ver); 1494 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1495 "api-version", buf); 1496 1497 if (!i40e_set_hw_bus_info(hw)) 1498 return (B_FALSE); 1499 1500 if (i40e_check_acc_handle(osdep->ios_reg_handle) != DDI_FM_OK) { 1501 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 1502 return (B_FALSE); 1503 } 1504 1505 return (B_TRUE); 1506 } 1507 1508 static void 1509 i40e_identify_hardware(i40e_t *i40e) 1510 { 1511 i40e_hw_t *hw = &i40e->i40e_hw_space; 1512 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1513 1514 hw->vendor_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_VENID); 1515 hw->device_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_DEVID); 1516 hw->revision_id = pci_config_get8(osdep->ios_cfg_handle, 1517 PCI_CONF_REVID); 1518 hw->subsystem_device_id = 1519 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBSYSID); 1520 hw->subsystem_vendor_id = 1521 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBVENID); 1522 1523 /* 1524 * Note that we set the hardware's bus information later on, in 1525 * i40e_get_available_resources(). The common code doesn't seem to 1526 * require that it be set in any ways, it seems to be mostly for 1527 * book-keeping. 1528 */ 1529 } 1530 1531 static boolean_t 1532 i40e_regs_map(i40e_t *i40e) 1533 { 1534 dev_info_t *devinfo = i40e->i40e_dip; 1535 i40e_hw_t *hw = &i40e->i40e_hw_space; 1536 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1537 off_t memsize; 1538 int ret; 1539 1540 if (ddi_dev_regsize(devinfo, I40E_ADAPTER_REGSET, &memsize) != 1541 DDI_SUCCESS) { 1542 i40e_error(i40e, "Used invalid register set to map PCIe regs"); 1543 return (B_FALSE); 1544 } 1545 1546 if ((ret = ddi_regs_map_setup(devinfo, I40E_ADAPTER_REGSET, 1547 (caddr_t *)&hw->hw_addr, 0, memsize, &i40e_regs_acc_attr, 1548 &osdep->ios_reg_handle)) != DDI_SUCCESS) { 1549 i40e_error(i40e, "failed to map device registers: %d", ret); 1550 return (B_FALSE); 1551 } 1552 1553 osdep->ios_reg_size = memsize; 1554 return (B_TRUE); 1555 } 1556 1557 /* 1558 * Update parameters required when a new MTU has been configured. Calculate the 1559 * maximum frame size, as well as, size our DMA buffers which we size in 1560 * increments of 1K. 1561 */ 1562 void 1563 i40e_update_mtu(i40e_t *i40e) 1564 { 1565 uint32_t rx, tx; 1566 1567 i40e->i40e_frame_max = i40e->i40e_sdu + 1568 sizeof (struct ether_vlan_header) + ETHERFCSL; 1569 1570 rx = i40e->i40e_frame_max + I40E_BUF_IPHDR_ALIGNMENT; 1571 i40e->i40e_rx_buf_size = ((rx >> 10) + 1572 ((rx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1573 1574 tx = i40e->i40e_frame_max; 1575 i40e->i40e_tx_buf_size = ((tx >> 10) + 1576 ((tx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1577 } 1578 1579 static int 1580 i40e_get_prop(i40e_t *i40e, char *prop, int min, int max, int def) 1581 { 1582 int val; 1583 1584 val = ddi_prop_get_int(DDI_DEV_T_ANY, i40e->i40e_dip, DDI_PROP_DONTPASS, 1585 prop, def); 1586 if (val > max) 1587 val = max; 1588 if (val < min) 1589 val = min; 1590 return (val); 1591 } 1592 1593 static void 1594 i40e_init_properties(i40e_t *i40e) 1595 { 1596 i40e->i40e_sdu = i40e_get_prop(i40e, "default_mtu", 1597 I40E_MIN_MTU, I40E_MAX_MTU, I40E_DEF_MTU); 1598 1599 i40e->i40e_intr_force = i40e_get_prop(i40e, "intr_force", 1600 I40E_INTR_NONE, I40E_INTR_LEGACY, I40E_INTR_NONE); 1601 1602 i40e->i40e_mr_enable = i40e_get_prop(i40e, "mr_enable", 1603 B_FALSE, B_TRUE, B_TRUE); 1604 1605 i40e->i40e_tx_ring_size = i40e_get_prop(i40e, "tx_ring_size", 1606 I40E_MIN_TX_RING_SIZE, I40E_MAX_TX_RING_SIZE, 1607 I40E_DEF_TX_RING_SIZE); 1608 if ((i40e->i40e_tx_ring_size % I40E_DESC_ALIGN) != 0) { 1609 i40e->i40e_tx_ring_size = P2ROUNDUP(i40e->i40e_tx_ring_size, 1610 I40E_DESC_ALIGN); 1611 } 1612 1613 i40e->i40e_tx_block_thresh = i40e_get_prop(i40e, "tx_resched_threshold", 1614 I40E_MIN_TX_BLOCK_THRESH, 1615 i40e->i40e_tx_ring_size - I40E_TX_MAX_COOKIE, 1616 I40E_DEF_TX_BLOCK_THRESH); 1617 1618 i40e->i40e_num_rx_groups = i40e_get_prop(i40e, "rx_num_groups", 1619 I40E_MIN_NUM_RX_GROUPS, I40E_MAX_NUM_RX_GROUPS, 1620 I40E_DEF_NUM_RX_GROUPS); 1621 1622 i40e->i40e_rx_ring_size = i40e_get_prop(i40e, "rx_ring_size", 1623 I40E_MIN_RX_RING_SIZE, I40E_MAX_RX_RING_SIZE, 1624 I40E_DEF_RX_RING_SIZE); 1625 if ((i40e->i40e_rx_ring_size % I40E_DESC_ALIGN) != 0) { 1626 i40e->i40e_rx_ring_size = P2ROUNDUP(i40e->i40e_rx_ring_size, 1627 I40E_DESC_ALIGN); 1628 } 1629 1630 i40e->i40e_rx_limit_per_intr = i40e_get_prop(i40e, "rx_limit_per_intr", 1631 I40E_MIN_RX_LIMIT_PER_INTR, I40E_MAX_RX_LIMIT_PER_INTR, 1632 I40E_DEF_RX_LIMIT_PER_INTR); 1633 1634 i40e->i40e_tx_hcksum_enable = i40e_get_prop(i40e, "tx_hcksum_enable", 1635 B_FALSE, B_TRUE, B_TRUE); 1636 1637 i40e->i40e_tx_lso_enable = i40e_get_prop(i40e, "tx_lso_enable", 1638 B_FALSE, B_TRUE, B_TRUE); 1639 1640 i40e->i40e_rx_hcksum_enable = i40e_get_prop(i40e, "rx_hcksum_enable", 1641 B_FALSE, B_TRUE, B_TRUE); 1642 1643 i40e->i40e_rx_dma_min = i40e_get_prop(i40e, "rx_dma_threshold", 1644 I40E_MIN_RX_DMA_THRESH, I40E_MAX_RX_DMA_THRESH, 1645 I40E_DEF_RX_DMA_THRESH); 1646 1647 i40e->i40e_tx_dma_min = i40e_get_prop(i40e, "tx_dma_threshold", 1648 I40E_MIN_TX_DMA_THRESH, I40E_MAX_TX_DMA_THRESH, 1649 I40E_DEF_TX_DMA_THRESH); 1650 1651 i40e->i40e_tx_itr = i40e_get_prop(i40e, "tx_intr_throttle", 1652 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_TX_ITR); 1653 1654 i40e->i40e_rx_itr = i40e_get_prop(i40e, "rx_intr_throttle", 1655 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_RX_ITR); 1656 1657 i40e->i40e_other_itr = i40e_get_prop(i40e, "other_intr_throttle", 1658 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_OTHER_ITR); 1659 1660 if (!i40e->i40e_mr_enable) { 1661 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1662 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1663 } 1664 1665 i40e_update_mtu(i40e); 1666 } 1667 1668 /* 1669 * There are a few constraints on interrupts that we're currently imposing, some 1670 * of which are restrictions from hardware. For a fuller treatment, see 1671 * i40e_intr.c. 1672 * 1673 * Currently, to use MSI-X we require two interrupts be available though in 1674 * theory we should participate in IRM and happily use more interrupts. 1675 * 1676 * Hardware only supports a single MSI being programmed and therefore if we 1677 * don't have MSI-X interrupts available at this time, then we ratchet down the 1678 * number of rings and groups available. Obviously, we only bother with a single 1679 * fixed interrupt. 1680 */ 1681 static boolean_t 1682 i40e_alloc_intr_handles(i40e_t *i40e, dev_info_t *devinfo, int intr_type) 1683 { 1684 i40e_hw_t *hw = &i40e->i40e_hw_space; 1685 ddi_acc_handle_t rh = i40e->i40e_osdep_space.ios_reg_handle; 1686 int request, count, actual, rc, min; 1687 uint32_t reg; 1688 1689 switch (intr_type) { 1690 case DDI_INTR_TYPE_FIXED: 1691 case DDI_INTR_TYPE_MSI: 1692 request = 1; 1693 min = 1; 1694 break; 1695 case DDI_INTR_TYPE_MSIX: 1696 min = 2; 1697 if (!i40e->i40e_mr_enable) { 1698 request = 2; 1699 break; 1700 } 1701 reg = I40E_READ_REG(hw, I40E_GLPCI_CNF2); 1702 /* 1703 * Should this read fail, we will drop back to using 1704 * MSI or fixed interrupts. 1705 */ 1706 if (i40e_check_acc_handle(rh) != DDI_FM_OK) { 1707 ddi_fm_service_impact(i40e->i40e_dip, 1708 DDI_SERVICE_DEGRADED); 1709 return (B_FALSE); 1710 } 1711 request = (reg & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 1712 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 1713 request++; /* the register value is n - 1 */ 1714 break; 1715 default: 1716 panic("bad interrupt type passed to i40e_alloc_intr_handles: " 1717 "%d", intr_type); 1718 } 1719 1720 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 1721 if (rc != DDI_SUCCESS || count < min) { 1722 i40e_log(i40e, "Get interrupt number failed, " 1723 "returned %d, count %d", rc, count); 1724 return (B_FALSE); 1725 } 1726 1727 rc = ddi_intr_get_navail(devinfo, intr_type, &count); 1728 if (rc != DDI_SUCCESS || count < min) { 1729 i40e_log(i40e, "Get AVAILABLE interrupt number failed, " 1730 "returned %d, count %d", rc, count); 1731 return (B_FALSE); 1732 } 1733 1734 actual = 0; 1735 i40e->i40e_intr_count = 0; 1736 i40e->i40e_intr_count_max = 0; 1737 i40e->i40e_intr_count_min = 0; 1738 1739 i40e->i40e_intr_size = request * sizeof (ddi_intr_handle_t); 1740 ASSERT(i40e->i40e_intr_size != 0); 1741 i40e->i40e_intr_handles = kmem_alloc(i40e->i40e_intr_size, KM_SLEEP); 1742 1743 rc = ddi_intr_alloc(devinfo, i40e->i40e_intr_handles, intr_type, 0, 1744 min(request, count), &actual, DDI_INTR_ALLOC_NORMAL); 1745 if (rc != DDI_SUCCESS) { 1746 i40e_log(i40e, "Interrupt allocation failed with %d.", rc); 1747 goto alloc_handle_fail; 1748 } 1749 1750 i40e->i40e_intr_count = actual; 1751 i40e->i40e_intr_count_max = request; 1752 i40e->i40e_intr_count_min = min; 1753 1754 if (actual < min) { 1755 i40e_log(i40e, "actual (%d) is less than minimum (%d).", 1756 actual, min); 1757 goto alloc_handle_fail; 1758 } 1759 1760 /* 1761 * Record the priority and capabilities for our first vector. Once 1762 * we have it, that's our priority until detach time. Even if we 1763 * eventually participate in IRM, our priority shouldn't change. 1764 */ 1765 rc = ddi_intr_get_pri(i40e->i40e_intr_handles[0], &i40e->i40e_intr_pri); 1766 if (rc != DDI_SUCCESS) { 1767 i40e_log(i40e, 1768 "Getting interrupt priority failed with %d.", rc); 1769 goto alloc_handle_fail; 1770 } 1771 1772 rc = ddi_intr_get_cap(i40e->i40e_intr_handles[0], &i40e->i40e_intr_cap); 1773 if (rc != DDI_SUCCESS) { 1774 i40e_log(i40e, 1775 "Getting interrupt capabilities failed with %d.", rc); 1776 goto alloc_handle_fail; 1777 } 1778 1779 i40e->i40e_intr_type = intr_type; 1780 return (B_TRUE); 1781 1782 alloc_handle_fail: 1783 1784 i40e_rem_intrs(i40e); 1785 return (B_FALSE); 1786 } 1787 1788 static boolean_t 1789 i40e_alloc_intrs(i40e_t *i40e, dev_info_t *devinfo) 1790 { 1791 i40e_hw_t *hw = &i40e->i40e_hw_space; 1792 int intr_types, rc; 1793 uint_t max_trqpairs; 1794 1795 if (i40e_is_x722(i40e)) { 1796 max_trqpairs = I40E_722_MAX_TC_QUEUES; 1797 } else { 1798 max_trqpairs = I40E_710_MAX_TC_QUEUES; 1799 } 1800 1801 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 1802 if (rc != DDI_SUCCESS) { 1803 i40e_error(i40e, "failed to get supported interrupt types: %d", 1804 rc); 1805 return (B_FALSE); 1806 } 1807 1808 i40e->i40e_intr_type = 0; 1809 1810 /* 1811 * We need to determine the number of queue pairs per traffic 1812 * class. We only have one traffic class (TC0), so we'll base 1813 * this off the number of interrupts provided. Furthermore, 1814 * since we only use one traffic class, the number of queues 1815 * per traffic class and per VSI are the same. 1816 */ 1817 if ((intr_types & DDI_INTR_TYPE_MSIX) && 1818 (i40e->i40e_intr_force <= I40E_INTR_MSIX) && 1819 (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSIX))) { 1820 uint32_t n, qp_cap, num_trqpairs; 1821 1822 /* 1823 * While we want the number of queue pairs to match 1824 * the number of interrupts, we must keep stay in 1825 * bounds of the maximum number of queues per traffic 1826 * class. We subtract one from i40e_intr_count to 1827 * account for interrupt zero; which is currently 1828 * restricted to admin queue commands and other 1829 * interrupt causes. 1830 */ 1831 n = MIN(i40e->i40e_intr_count - 1, max_trqpairs); 1832 ASSERT3U(n, >, 0); 1833 1834 /* 1835 * Round up to the nearest power of two to ensure that 1836 * the QBASE aligns with the TC size which must be 1837 * programmed as a power of two. See the queue mapping 1838 * description in section 7.4.9.5.5.1. 1839 * 1840 * If i40e_intr_count - 1 is not a power of two then 1841 * some queue pairs on the same VSI will have to share 1842 * an interrupt. 1843 * 1844 * We may want to revisit this logic in a future where 1845 * we have more interrupts and more VSIs. Otherwise, 1846 * each VSI will use as many interrupts as possible. 1847 * Using more QPs per VSI means better RSS for each 1848 * group, but at the same time may require more 1849 * sharing of interrupts across VSIs. This may be a 1850 * good candidate for a .conf tunable. 1851 */ 1852 n = 0x1 << ddi_fls(n); 1853 i40e->i40e_num_trqpairs_per_vsi = n; 1854 1855 /* 1856 * Make sure the number of tx/rx qpairs does not exceed 1857 * the device's capabilities. 1858 */ 1859 ASSERT3U(i40e->i40e_num_rx_groups, >, 0); 1860 qp_cap = MIN(hw->func_caps.num_rx_qp, hw->func_caps.num_tx_qp); 1861 num_trqpairs = i40e->i40e_num_trqpairs_per_vsi * 1862 i40e->i40e_num_rx_groups; 1863 if (num_trqpairs > qp_cap) { 1864 i40e->i40e_num_rx_groups = MAX(1, qp_cap / 1865 i40e->i40e_num_trqpairs_per_vsi); 1866 num_trqpairs = i40e->i40e_num_trqpairs_per_vsi * 1867 i40e->i40e_num_rx_groups; 1868 i40e_log(i40e, "Rx groups restricted to %u", 1869 i40e->i40e_num_rx_groups); 1870 } 1871 ASSERT3U(num_trqpairs, >, 0); 1872 i40e->i40e_num_trqpairs = num_trqpairs; 1873 return (B_TRUE); 1874 } 1875 1876 /* 1877 * We only use multiple transmit/receive pairs when MSI-X interrupts are 1878 * available due to the fact that the device basically only supports a 1879 * single MSI interrupt. 1880 */ 1881 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1882 i40e->i40e_num_trqpairs_per_vsi = i40e->i40e_num_trqpairs; 1883 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1884 1885 if ((intr_types & DDI_INTR_TYPE_MSI) && 1886 (i40e->i40e_intr_force <= I40E_INTR_MSI)) { 1887 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSI)) 1888 return (B_TRUE); 1889 } 1890 1891 if (intr_types & DDI_INTR_TYPE_FIXED) { 1892 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_FIXED)) 1893 return (B_TRUE); 1894 } 1895 1896 return (B_FALSE); 1897 } 1898 1899 /* 1900 * Map different interrupts to MSI-X vectors. 1901 */ 1902 static boolean_t 1903 i40e_map_intrs_to_vectors(i40e_t *i40e) 1904 { 1905 if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 1906 return (B_TRUE); 1907 } 1908 1909 /* 1910 * Each queue pair is mapped to a single interrupt, so 1911 * transmit and receive interrupts for a given queue share the 1912 * same vector. Vector zero is reserved for the admin queue. 1913 */ 1914 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1915 uint_t vector = i % (i40e->i40e_intr_count - 1); 1916 1917 i40e->i40e_trqpairs[i].itrq_rx_intrvec = vector + 1; 1918 i40e->i40e_trqpairs[i].itrq_tx_intrvec = vector + 1; 1919 } 1920 1921 return (B_TRUE); 1922 } 1923 1924 static boolean_t 1925 i40e_add_intr_handlers(i40e_t *i40e) 1926 { 1927 int rc, vector; 1928 1929 switch (i40e->i40e_intr_type) { 1930 case DDI_INTR_TYPE_MSIX: 1931 for (vector = 0; vector < i40e->i40e_intr_count; vector++) { 1932 rc = ddi_intr_add_handler( 1933 i40e->i40e_intr_handles[vector], 1934 (ddi_intr_handler_t *)i40e_intr_msix, i40e, 1935 (void *)(uintptr_t)vector); 1936 if (rc != DDI_SUCCESS) { 1937 i40e_log(i40e, "Add interrupt handler (MSI-X) " 1938 "failed: return %d, vector %d", rc, vector); 1939 for (vector--; vector >= 0; vector--) { 1940 (void) ddi_intr_remove_handler( 1941 i40e->i40e_intr_handles[vector]); 1942 } 1943 return (B_FALSE); 1944 } 1945 } 1946 break; 1947 case DDI_INTR_TYPE_MSI: 1948 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1949 (ddi_intr_handler_t *)i40e_intr_msi, i40e, NULL); 1950 if (rc != DDI_SUCCESS) { 1951 i40e_log(i40e, "Add interrupt handler (MSI) failed: " 1952 "return %d", rc); 1953 return (B_FALSE); 1954 } 1955 break; 1956 case DDI_INTR_TYPE_FIXED: 1957 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1958 (ddi_intr_handler_t *)i40e_intr_legacy, i40e, NULL); 1959 if (rc != DDI_SUCCESS) { 1960 i40e_log(i40e, "Add interrupt handler (legacy) failed:" 1961 " return %d", rc); 1962 return (B_FALSE); 1963 } 1964 break; 1965 default: 1966 /* Cast to pacify lint */ 1967 panic("i40e_intr_type %p contains an unknown type: %d", 1968 (void *)i40e, i40e->i40e_intr_type); 1969 } 1970 1971 return (B_TRUE); 1972 } 1973 1974 /* 1975 * Perform periodic checks. Longer term, we should be thinking about additional 1976 * things here: 1977 * 1978 * o Stall Detection 1979 * o Temperature sensor detection 1980 * o Device resetting 1981 * o Statistics updating to avoid wraparound 1982 */ 1983 static void 1984 i40e_timer(void *arg) 1985 { 1986 i40e_t *i40e = arg; 1987 1988 mutex_enter(&i40e->i40e_general_lock); 1989 i40e_link_check(i40e); 1990 mutex_exit(&i40e->i40e_general_lock); 1991 } 1992 1993 /* 1994 * Get the hardware state, and scribble away anything that needs scribbling. 1995 */ 1996 static void 1997 i40e_get_hw_state(i40e_t *i40e, i40e_hw_t *hw) 1998 { 1999 int rc; 2000 2001 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 2002 2003 (void) i40e_aq_get_link_info(hw, TRUE, NULL, NULL); 2004 i40e_link_check(i40e); 2005 2006 /* 2007 * Try and determine our PHY. Note that we may have to retry to and 2008 * delay to detect fiber correctly. 2009 */ 2010 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, &i40e->i40e_phy, 2011 NULL); 2012 if (rc == I40E_ERR_UNKNOWN_PHY) { 2013 i40e_msec_delay(200); 2014 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, 2015 &i40e->i40e_phy, NULL); 2016 } 2017 2018 if (rc != I40E_SUCCESS) { 2019 if (rc == I40E_ERR_UNKNOWN_PHY) { 2020 i40e_error(i40e, "encountered unknown PHY type, " 2021 "not attaching."); 2022 } else { 2023 i40e_error(i40e, "error getting physical capabilities: " 2024 "%d, %d", rc, hw->aq.asq_last_status); 2025 } 2026 } 2027 2028 rc = i40e_update_link_info(hw); 2029 if (rc != I40E_SUCCESS) { 2030 i40e_error(i40e, "failed to update link information: %d", rc); 2031 } 2032 2033 /* 2034 * In general, we don't want to mask off (as in stop from being a cause) 2035 * any of the interrupts that the phy might be able to generate. 2036 */ 2037 rc = i40e_aq_set_phy_int_mask(hw, 0, NULL); 2038 if (rc != I40E_SUCCESS) { 2039 i40e_error(i40e, "failed to update phy link mask: %d", rc); 2040 } 2041 } 2042 2043 /* 2044 * Go through and re-initialize any existing filters that we may have set up for 2045 * this device. Note that we would only expect them to exist if hardware had 2046 * already been initialized and we had just reset it. While we're not 2047 * implementing this yet, we're keeping this around for when we add reset 2048 * capabilities, so this isn't forgotten. 2049 */ 2050 /* ARGSUSED */ 2051 static void 2052 i40e_init_macaddrs(i40e_t *i40e, i40e_hw_t *hw) 2053 { 2054 } 2055 2056 /* 2057 * Set the properties which have common values across all the VSIs. 2058 * Consult the "Add VSI" command section (7.4.9.5.5.1) for a 2059 * complete description of these properties. 2060 */ 2061 static void 2062 i40e_set_shared_vsi_props(i40e_t *i40e, 2063 struct i40e_aqc_vsi_properties_data *info, uint_t vsi_idx) 2064 { 2065 uint_t tc_queues; 2066 uint16_t vsi_qp_base; 2067 2068 /* 2069 * It's important that we use bitwise-OR here; callers to this 2070 * function might enable other sections before calling this 2071 * function. 2072 */ 2073 info->valid_sections |= LE_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID | 2074 I40E_AQ_VSI_PROP_VLAN_VALID); 2075 2076 /* 2077 * Calculate the starting QP index for this VSI. This base is 2078 * relative to the PF queue space; so a value of 0 for PF#1 2079 * represents the absolute index PFLAN_QALLOC_FIRSTQ for PF#1. 2080 */ 2081 vsi_qp_base = vsi_idx * i40e->i40e_num_trqpairs_per_vsi; 2082 info->mapping_flags = LE_16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2083 info->queue_mapping[0] = 2084 LE_16((vsi_qp_base << I40E_AQ_VSI_QUEUE_SHIFT) & 2085 I40E_AQ_VSI_QUEUE_MASK); 2086 2087 /* 2088 * tc_queues determines the size of the traffic class, where 2089 * the size is 2^^tc_queues to a maximum of 64 for the X710 2090 * and 128 for the X722. 2091 * 2092 * Some examples: 2093 * i40e_num_trqpairs_per_vsi == 1 => tc_queues = 0, 2^^0 = 1. 2094 * i40e_num_trqpairs_per_vsi == 7 => tc_queues = 3, 2^^3 = 8. 2095 * i40e_num_trqpairs_per_vsi == 8 => tc_queues = 3, 2^^3 = 8. 2096 * i40e_num_trqpairs_per_vsi == 9 => tc_queues = 4, 2^^4 = 16. 2097 * i40e_num_trqpairs_per_vsi == 17 => tc_queues = 5, 2^^5 = 32. 2098 * i40e_num_trqpairs_per_vsi == 64 => tc_queues = 6, 2^^6 = 64. 2099 */ 2100 tc_queues = ddi_fls(i40e->i40e_num_trqpairs_per_vsi - 1); 2101 2102 /* 2103 * The TC queue mapping is in relation to the VSI queue space. 2104 * Since we are only using one traffic class (TC0) we always 2105 * start at queue offset 0. 2106 */ 2107 info->tc_mapping[0] = 2108 LE_16(((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) & 2109 I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | 2110 ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) & 2111 I40E_AQ_VSI_TC_QUE_NUMBER_MASK)); 2112 2113 /* 2114 * I40E_AQ_VSI_PVLAN_MODE_ALL ("VLAN driver insertion mode") 2115 * 2116 * Allow tagged and untagged packets to be sent to this 2117 * VSI from the host. 2118 * 2119 * I40E_AQ_VSI_PVLAN_EMOD_NOTHING ("VLAN and UP expose mode") 2120 * 2121 * Leave the tag on the frame and place no VLAN 2122 * information in the descriptor. We want this mode 2123 * because our MAC layer will take care of the VLAN tag, 2124 * if there is one. 2125 */ 2126 info->port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2127 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2128 } 2129 2130 /* 2131 * Delete the VSI at this index, if one exists. We assume there is no 2132 * action we can take if this command fails but to log the failure. 2133 */ 2134 static void 2135 i40e_delete_vsi(i40e_t *i40e, uint_t idx) 2136 { 2137 i40e_hw_t *hw = &i40e->i40e_hw_space; 2138 uint16_t seid = i40e->i40e_vsis[idx].iv_seid; 2139 2140 if (seid != 0) { 2141 int rc; 2142 2143 rc = i40e_aq_delete_element(hw, seid, NULL); 2144 2145 if (rc != I40E_SUCCESS) { 2146 i40e_error(i40e, "Failed to delete VSI %d: %d", 2147 rc, hw->aq.asq_last_status); 2148 } 2149 2150 i40e->i40e_vsis[idx].iv_seid = 0; 2151 } 2152 } 2153 2154 /* 2155 * Add a new VSI. 2156 */ 2157 static boolean_t 2158 i40e_add_vsi(i40e_t *i40e, i40e_hw_t *hw, uint_t idx) 2159 { 2160 struct i40e_vsi_context ctx; 2161 i40e_rx_group_t *rxg; 2162 int rc; 2163 2164 /* 2165 * The default VSI is created by the controller. This function 2166 * creates new, non-defualt VSIs only. 2167 */ 2168 ASSERT3U(idx, !=, 0); 2169 2170 bzero(&ctx, sizeof (struct i40e_vsi_context)); 2171 ctx.uplink_seid = i40e->i40e_veb_seid; 2172 ctx.pf_num = hw->pf_id; 2173 ctx.flags = I40E_AQ_VSI_TYPE_PF; 2174 ctx.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 2175 i40e_set_shared_vsi_props(i40e, &ctx.info, idx); 2176 2177 rc = i40e_aq_add_vsi(hw, &ctx, NULL); 2178 if (rc != I40E_SUCCESS) { 2179 i40e_error(i40e, "i40e_aq_add_vsi() failed %d: %d", rc, 2180 hw->aq.asq_last_status); 2181 return (B_FALSE); 2182 } 2183 2184 rxg = &i40e->i40e_rx_groups[idx]; 2185 rxg->irg_vsi_seid = ctx.seid; 2186 i40e->i40e_vsis[idx].iv_number = ctx.vsi_number; 2187 i40e->i40e_vsis[idx].iv_seid = ctx.seid; 2188 i40e->i40e_vsis[idx].iv_stats_id = LE_16(ctx.info.stat_counter_idx); 2189 2190 if (i40e_stat_vsi_init(i40e, idx) == B_FALSE) 2191 return (B_FALSE); 2192 2193 return (B_TRUE); 2194 } 2195 2196 /* 2197 * Configure the hardware for the Default Virtual Station Interface (VSI). 2198 */ 2199 static boolean_t 2200 i40e_config_def_vsi(i40e_t *i40e, i40e_hw_t *hw) 2201 { 2202 struct i40e_vsi_context ctx; 2203 i40e_rx_group_t *def_rxg; 2204 int err; 2205 struct i40e_aqc_remove_macvlan_element_data filt; 2206 2207 bzero(&ctx, sizeof (struct i40e_vsi_context)); 2208 ctx.seid = I40E_DEF_VSI_SEID(i40e); 2209 ctx.pf_num = hw->pf_id; 2210 err = i40e_aq_get_vsi_params(hw, &ctx, NULL); 2211 if (err != I40E_SUCCESS) { 2212 i40e_error(i40e, "get VSI params failed with %d", err); 2213 return (B_FALSE); 2214 } 2215 2216 ctx.info.valid_sections = 0; 2217 i40e->i40e_vsis[0].iv_number = ctx.vsi_number; 2218 i40e->i40e_vsis[0].iv_stats_id = LE_16(ctx.info.stat_counter_idx); 2219 if (i40e_stat_vsi_init(i40e, 0) == B_FALSE) 2220 return (B_FALSE); 2221 2222 i40e_set_shared_vsi_props(i40e, &ctx.info, I40E_DEF_VSI_IDX); 2223 2224 err = i40e_aq_update_vsi_params(hw, &ctx, NULL); 2225 if (err != I40E_SUCCESS) { 2226 i40e_error(i40e, "Update VSI params failed with %d", err); 2227 return (B_FALSE); 2228 } 2229 2230 def_rxg = &i40e->i40e_rx_groups[0]; 2231 def_rxg->irg_vsi_seid = I40E_DEF_VSI_SEID(i40e); 2232 2233 /* 2234 * We have seen three different behaviors in regards to the 2235 * Default VSI and its implicit L2 MAC+VLAN filter. 2236 * 2237 * 1. It has an implicit filter for the factory MAC address 2238 * and this filter counts against 'ifr_nmacfilt_used'. 2239 * 2240 * 2. It has an implicit filter for the factory MAC address 2241 * and this filter DOES NOT count against 'ifr_nmacfilt_used'. 2242 * 2243 * 3. It DOES NOT have an implicit filter. 2244 * 2245 * All three of these cases are accounted for below. If we 2246 * fail to remove the L2 filter (ENOENT) then we assume there 2247 * wasn't one. Otherwise, if we successfully remove the 2248 * filter, we make sure to update the 'ifr_nmacfilt_used' 2249 * count accordingly. 2250 * 2251 * We remove this filter to prevent duplicate delivery of 2252 * packets destined for the primary MAC address as DLS will 2253 * create the same filter on a non-default VSI for the primary 2254 * MAC client. 2255 * 2256 * If you change the following code please test it across as 2257 * many X700 series controllers and firmware revisions as you 2258 * can. 2259 */ 2260 bzero(&filt, sizeof (filt)); 2261 bcopy(hw->mac.port_addr, filt.mac_addr, ETHERADDRL); 2262 filt.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2263 filt.vlan_tag = 0; 2264 2265 ASSERT3U(i40e->i40e_resources.ifr_nmacfilt_used, <=, 1); 2266 i40e_log(i40e, "Num L2 filters: %u", 2267 i40e->i40e_resources.ifr_nmacfilt_used); 2268 2269 err = i40e_aq_remove_macvlan(hw, I40E_DEF_VSI_SEID(i40e), &filt, 1, 2270 NULL); 2271 if (err == I40E_SUCCESS) { 2272 i40e_log(i40e, 2273 "Removed L2 filter from Default VSI with SEID %u", 2274 I40E_DEF_VSI_SEID(i40e)); 2275 } else if (hw->aq.asq_last_status == ENOENT) { 2276 i40e_log(i40e, 2277 "No L2 filter for Default VSI with SEID %u", 2278 I40E_DEF_VSI_SEID(i40e)); 2279 } else { 2280 i40e_error(i40e, "Failed to remove L2 filter from" 2281 " Default VSI with SEID %u: %d (%d)", 2282 I40E_DEF_VSI_SEID(i40e), err, hw->aq.asq_last_status); 2283 2284 return (B_FALSE); 2285 } 2286 2287 /* 2288 * As mentioned above, the controller created an implicit L2 2289 * filter for the primary MAC. We want to remove both the 2290 * filter and decrement the filter count. However, not all 2291 * controllers count this implicit filter against the total 2292 * MAC filter count. So here we are making sure it is either 2293 * one or zero. If it is one, then we know it is for the 2294 * implicit filter and we should decrement since we just 2295 * removed the filter above. If it is zero then we know the 2296 * controller that does not count the implicit filter, and it 2297 * was enough to just remove it; we leave the count alone. 2298 * But if it is neither, then we have never seen a controller 2299 * like this before and we should fail to attach. 2300 * 2301 * It is unfortunate that this code must exist but the 2302 * behavior of this implicit L2 filter and its corresponding 2303 * count were dicovered through empirical testing. The 2304 * programming manuals hint at this filter but do not 2305 * explicitly call out the exact behavior. 2306 */ 2307 if (i40e->i40e_resources.ifr_nmacfilt_used == 1) { 2308 i40e->i40e_resources.ifr_nmacfilt_used--; 2309 } else { 2310 if (i40e->i40e_resources.ifr_nmacfilt_used != 0) { 2311 i40e_error(i40e, "Unexpected L2 filter count: %u" 2312 " (expected 0)", 2313 i40e->i40e_resources.ifr_nmacfilt_used); 2314 return (B_FALSE); 2315 } 2316 } 2317 2318 return (B_TRUE); 2319 } 2320 2321 static boolean_t 2322 i40e_config_rss_key_x722(i40e_t *i40e, i40e_hw_t *hw) 2323 { 2324 for (uint_t i = 0; i < i40e->i40e_num_rx_groups; i++) { 2325 uint32_t seed[I40E_PFQF_HKEY_MAX_INDEX + 1]; 2326 struct i40e_aqc_get_set_rss_key_data key; 2327 const char *u8seed; 2328 enum i40e_status_code status; 2329 uint16_t vsi_number = i40e->i40e_vsis[i].iv_number; 2330 2331 (void) random_get_pseudo_bytes((uint8_t *)seed, sizeof (seed)); 2332 u8seed = (char *)seed; 2333 2334 CTASSERT(sizeof (key) >= (sizeof (key.standard_rss_key) + 2335 sizeof (key.extended_hash_key))); 2336 2337 bcopy(u8seed, key.standard_rss_key, 2338 sizeof (key.standard_rss_key)); 2339 bcopy(&u8seed[sizeof (key.standard_rss_key)], 2340 key.extended_hash_key, sizeof (key.extended_hash_key)); 2341 2342 ASSERT3U(vsi_number, !=, 0); 2343 status = i40e_aq_set_rss_key(hw, vsi_number, &key); 2344 2345 if (status != I40E_SUCCESS) { 2346 i40e_error(i40e, "failed to set RSS key for VSI %u: %d", 2347 vsi_number, status); 2348 return (B_FALSE); 2349 } 2350 } 2351 2352 return (B_TRUE); 2353 } 2354 2355 /* 2356 * Configure the RSS key. For the X710 controller family, this is set on a 2357 * per-PF basis via registers. For the X722, this is done on a per-VSI basis 2358 * through the admin queue. 2359 */ 2360 static boolean_t 2361 i40e_config_rss_key(i40e_t *i40e, i40e_hw_t *hw) 2362 { 2363 if (i40e_is_x722(i40e)) { 2364 if (!i40e_config_rss_key_x722(i40e, hw)) 2365 return (B_FALSE); 2366 } else { 2367 uint32_t seed[I40E_PFQF_HKEY_MAX_INDEX + 1]; 2368 2369 (void) random_get_pseudo_bytes((uint8_t *)seed, sizeof (seed)); 2370 for (uint_t i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 2371 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), seed[i]); 2372 } 2373 2374 return (B_TRUE); 2375 } 2376 2377 /* 2378 * Populate the LUT. The size of each entry in the LUT depends on the controller 2379 * family, with the X722 using a known 7-bit width. On the X710 controller, this 2380 * is programmed through its control registers where as on the X722 this is 2381 * configured through the admin queue. Also of note, the X722 allows the LUT to 2382 * be set on a per-PF or VSI basis. At this time we use the PF setting. If we 2383 * decide to use the per-VSI LUT in the future, then we will need to modify the 2384 * i40e_add_vsi() function to set the RSS LUT bits in the queueing section. 2385 * 2386 * We populate the LUT in a round robin fashion with the rx queue indices from 0 2387 * to i40e_num_trqpairs_per_vsi - 1. 2388 */ 2389 static boolean_t 2390 i40e_config_rss_hlut(i40e_t *i40e, i40e_hw_t *hw) 2391 { 2392 uint32_t *hlut; 2393 uint8_t lut_mask; 2394 uint_t i; 2395 boolean_t ret = B_FALSE; 2396 2397 /* 2398 * We always configure the PF with a table size of 512 bytes in 2399 * i40e_chip_start(). 2400 */ 2401 hlut = kmem_alloc(I40E_HLUT_TABLE_SIZE, KM_NOSLEEP); 2402 if (hlut == NULL) { 2403 i40e_error(i40e, "i40e_config_rss() buffer allocation failed"); 2404 return (B_FALSE); 2405 } 2406 2407 /* 2408 * The width of the X722 is apparently defined to be 7 bits, regardless 2409 * of the capability. 2410 */ 2411 if (i40e_is_x722(i40e)) { 2412 lut_mask = (1 << 7) - 1; 2413 } else { 2414 lut_mask = (1 << hw->func_caps.rss_table_entry_width) - 1; 2415 } 2416 2417 for (i = 0; i < I40E_HLUT_TABLE_SIZE; i++) { 2418 ((uint8_t *)hlut)[i] = 2419 (i % i40e->i40e_num_trqpairs_per_vsi) & lut_mask; 2420 } 2421 2422 if (i40e_is_x722(i40e)) { 2423 enum i40e_status_code status; 2424 2425 status = i40e_aq_set_rss_lut(hw, 0, B_TRUE, (uint8_t *)hlut, 2426 I40E_HLUT_TABLE_SIZE); 2427 2428 if (status != I40E_SUCCESS) { 2429 i40e_error(i40e, "failed to set RSS LUT %d: %d", 2430 status, hw->aq.asq_last_status); 2431 goto out; 2432 } 2433 } else { 2434 for (i = 0; i < I40E_HLUT_TABLE_SIZE >> 2; i++) { 2435 I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), hlut[i]); 2436 } 2437 } 2438 ret = B_TRUE; 2439 out: 2440 kmem_free(hlut, I40E_HLUT_TABLE_SIZE); 2441 return (ret); 2442 } 2443 2444 /* 2445 * Set up RSS. 2446 * 1. Seed the hash key. 2447 * 2. Enable PCTYPEs for the hash filter. 2448 * 3. Populate the LUT. 2449 */ 2450 static boolean_t 2451 i40e_config_rss(i40e_t *i40e, i40e_hw_t *hw) 2452 { 2453 uint64_t hena; 2454 2455 /* 2456 * 1. Seed the hash key 2457 */ 2458 if (!i40e_config_rss_key(i40e, hw)) 2459 return (B_FALSE); 2460 2461 /* 2462 * 2. Configure PCTYPES 2463 */ 2464 hena = (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | 2465 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | 2466 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | 2467 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | 2468 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4) | 2469 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | 2470 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | 2471 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | 2472 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | 2473 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6) | 2474 (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD); 2475 2476 /* 2477 * Add additional types supported by the X722 controller. 2478 */ 2479 if (i40e_is_x722(i40e)) { 2480 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | 2481 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | 2482 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | 2483 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | 2484 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | 2485 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); 2486 } 2487 2488 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); 2489 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); 2490 2491 /* 2492 * 3. Populate LUT 2493 */ 2494 return (i40e_config_rss_hlut(i40e, hw)); 2495 } 2496 2497 /* 2498 * Wrapper to kick the chipset on. 2499 */ 2500 static boolean_t 2501 i40e_chip_start(i40e_t *i40e) 2502 { 2503 i40e_hw_t *hw = &i40e->i40e_hw_space; 2504 struct i40e_filter_control_settings filter; 2505 int rc; 2506 uint8_t err; 2507 2508 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || 2509 (hw->aq.fw_maj_ver < 4)) { 2510 i40e_msec_delay(75); 2511 if (i40e_aq_set_link_restart_an(hw, TRUE, NULL) != 2512 I40E_SUCCESS) { 2513 i40e_error(i40e, "failed to restart link: admin queue " 2514 "error: %d", hw->aq.asq_last_status); 2515 return (B_FALSE); 2516 } 2517 } 2518 2519 /* Determine hardware state */ 2520 i40e_get_hw_state(i40e, hw); 2521 2522 /* For now, we always disable Ethernet Flow Control. */ 2523 hw->fc.requested_mode = I40E_FC_NONE; 2524 rc = i40e_set_fc(hw, &err, B_TRUE); 2525 if (rc != I40E_SUCCESS) { 2526 i40e_error(i40e, "Setting flow control failed, returned %d" 2527 " with error: 0x%x", rc, err); 2528 return (B_FALSE); 2529 } 2530 2531 /* Initialize mac addresses. */ 2532 i40e_init_macaddrs(i40e, hw); 2533 2534 /* 2535 * Set up the filter control. If the hash lut size is changed from 2536 * I40E_HASH_LUT_SIZE_512 then I40E_HLUT_TABLE_SIZE and 2537 * i40e_config_rss_hlut() will need to be updated. 2538 */ 2539 bzero(&filter, sizeof (filter)); 2540 filter.enable_ethtype = TRUE; 2541 filter.enable_macvlan = TRUE; 2542 filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; 2543 2544 rc = i40e_set_filter_control(hw, &filter); 2545 if (rc != I40E_SUCCESS) { 2546 i40e_error(i40e, "i40e_set_filter_control() returned %d", rc); 2547 return (B_FALSE); 2548 } 2549 2550 i40e_intr_chip_init(i40e); 2551 2552 rc = i40e_get_mac_seid(i40e); 2553 if (rc == -1) { 2554 i40e_error(i40e, "failed to obtain MAC Uplink SEID"); 2555 return (B_FALSE); 2556 } 2557 i40e->i40e_mac_seid = (uint16_t)rc; 2558 2559 /* 2560 * Create a VEB in order to support multiple VSIs. Each VSI 2561 * functions as a MAC group. This call sets the PF's MAC as 2562 * the uplink port and the PF's default VSI as the default 2563 * downlink port. 2564 */ 2565 rc = i40e_aq_add_veb(hw, i40e->i40e_mac_seid, I40E_DEF_VSI_SEID(i40e), 2566 0x1, B_TRUE, &i40e->i40e_veb_seid, B_FALSE, NULL); 2567 if (rc != I40E_SUCCESS) { 2568 i40e_error(i40e, "i40e_aq_add_veb() failed %d: %d", rc, 2569 hw->aq.asq_last_status); 2570 return (B_FALSE); 2571 } 2572 2573 if (!i40e_config_def_vsi(i40e, hw)) 2574 return (B_FALSE); 2575 2576 for (uint_t i = 1; i < i40e->i40e_num_rx_groups; i++) { 2577 if (!i40e_add_vsi(i40e, hw, i)) 2578 return (B_FALSE); 2579 } 2580 2581 if (!i40e_config_rss(i40e, hw)) 2582 return (B_FALSE); 2583 2584 i40e_flush(hw); 2585 2586 return (B_TRUE); 2587 } 2588 2589 /* 2590 * Take care of tearing down the rx ring. See 8.3.3.1.2 for more information. 2591 */ 2592 static void 2593 i40e_shutdown_rx_ring(i40e_trqpair_t *itrq) 2594 { 2595 i40e_t *i40e = itrq->itrq_i40e; 2596 i40e_hw_t *hw = &i40e->i40e_hw_space; 2597 uint32_t reg; 2598 2599 /* 2600 * Step 1. 8.3.3.1.2 suggests the interrupt is removed from the 2601 * hardware interrupt linked list (see i40e_intr.c) but for 2602 * simplicity we keep this list immutable until the device 2603 * (distinct from an individual ring) is stopped. 2604 */ 2605 2606 /* 2607 * Step 2. Request the queue by clearing QENA_REQ. It may not be 2608 * set due to unwinding from failures and a partially enabled 2609 * ring set. 2610 */ 2611 reg = I40E_READ_REG(hw, I40E_QRX_ENA(itrq->itrq_index)); 2612 if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK)) 2613 return; 2614 VERIFY((reg & I40E_QRX_ENA_QENA_REQ_MASK) == 2615 I40E_QRX_ENA_QENA_REQ_MASK); 2616 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 2617 I40E_WRITE_REG(hw, I40E_QRX_ENA(itrq->itrq_index), reg); 2618 2619 /* 2620 * Step 3. Wait for the disable to take, by having QENA_STAT in the FPM 2621 * be cleared. Note that we could still receive data in the queue during 2622 * this time. We don't actually wait for this now and instead defer this 2623 * to i40e_shutdown_ring_wait(), after we've interleaved disabling the 2624 * TX queue as well. 2625 */ 2626 } 2627 2628 static void 2629 i40e_shutdown_tx_ring(i40e_trqpair_t *itrq) 2630 { 2631 i40e_t *i40e = itrq->itrq_i40e; 2632 i40e_hw_t *hw = &i40e->i40e_hw_space; 2633 uint32_t reg; 2634 2635 /* 2636 * Step 2. Set the SET_QDIS flag for the queue. 2637 */ 2638 i40e_pre_tx_queue_cfg(hw, itrq->itrq_index, B_FALSE); 2639 2640 /* 2641 * Step 3. Wait at least 400 usec. 2642 */ 2643 drv_usecwait(500); 2644 2645 /* 2646 * Step 4. Clear the QENA_REQ flag which tells hardware to 2647 * quiesce. If QENA_REQ is not already set then that means that 2648 * we likely already tried to disable this queue. 2649 */ 2650 reg = I40E_READ_REG(hw, I40E_QTX_ENA(itrq->itrq_index)); 2651 if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) != 0) { 2652 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 2653 I40E_WRITE_REG(hw, I40E_QTX_ENA(itrq->itrq_index), reg); 2654 } 2655 2656 /* 2657 * Step 5. Wait for the drain to finish. This will be done by the 2658 * hardware removing the QENA_STAT flag from the queue. Rather than 2659 * waiting here, we interleave it with the receive shutdown in 2660 * i40e_shutdown_ring_wait(). 2661 */ 2662 } 2663 2664 /* 2665 * Wait for a ring to be shut down. e.g. Steps 2 and 5 from the above 2666 * functions. 2667 */ 2668 static boolean_t 2669 i40e_shutdown_ring_wait(i40e_trqpair_t *itrq) 2670 { 2671 i40e_t *i40e = itrq->itrq_i40e; 2672 i40e_hw_t *hw = &i40e->i40e_hw_space; 2673 uint32_t reg; 2674 int try; 2675 2676 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2677 reg = I40E_READ_REG(hw, I40E_QRX_ENA(itrq->itrq_index)); 2678 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2679 break; 2680 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2681 } 2682 2683 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) != 0) { 2684 i40e_error(i40e, "timed out disabling rx queue %d", 2685 itrq->itrq_index); 2686 return (B_FALSE); 2687 } 2688 2689 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2690 reg = I40E_READ_REG(hw, I40E_QTX_ENA(itrq->itrq_index)); 2691 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2692 break; 2693 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2694 } 2695 2696 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) != 0) { 2697 i40e_error(i40e, "timed out disabling tx queue %d", 2698 itrq->itrq_index); 2699 return (B_FALSE); 2700 } 2701 2702 return (B_TRUE); 2703 } 2704 2705 2706 /* 2707 * Shutdown an individual ring and release any memory. 2708 */ 2709 boolean_t 2710 i40e_shutdown_ring(i40e_trqpair_t *itrq) 2711 { 2712 boolean_t rv = B_TRUE; 2713 2714 /* 2715 * Tell transmit path to quiesce, and wait until done. 2716 */ 2717 if (i40e_ring_tx_quiesce(itrq)) { 2718 /* Already quiesced. */ 2719 return (B_TRUE); 2720 } 2721 2722 i40e_shutdown_rx_ring(itrq); 2723 i40e_shutdown_tx_ring(itrq); 2724 if (!i40e_shutdown_ring_wait(itrq)) 2725 rv = B_FALSE; 2726 2727 /* 2728 * After the ring has stopped, we need to wait 50ms before 2729 * programming it again. Rather than wait here, we'll record 2730 * the time the ring was stopped. When the ring is started, we'll 2731 * check if enough time has expired and then wait if necessary. 2732 */ 2733 itrq->irtq_time_stopped = gethrtime(); 2734 2735 /* 2736 * The rings have been stopped in the hardware, now wait for 2737 * a possibly active interrupt thread. 2738 */ 2739 i40e_intr_quiesce(itrq); 2740 2741 mutex_enter(&itrq->itrq_tx_lock); 2742 i40e_tx_cleanup_ring(itrq); 2743 mutex_exit(&itrq->itrq_tx_lock); 2744 2745 i40e_free_ring_mem(itrq, B_FALSE); 2746 2747 return (rv); 2748 } 2749 2750 /* 2751 * Shutdown all the rings. 2752 * Called from i40e_stop(), and hopefully the mac layer has already 2753 * called ring stop for each ring, which would make this almost a no-op. 2754 */ 2755 static boolean_t 2756 i40e_shutdown_rings(i40e_t *i40e) 2757 { 2758 boolean_t rv = B_TRUE; 2759 int i; 2760 2761 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2762 if (!i40e_shutdown_ring(&i40e->i40e_trqpairs[i])) 2763 rv = B_FALSE; 2764 } 2765 2766 return (rv); 2767 } 2768 2769 static void 2770 i40e_setup_rx_descs(i40e_trqpair_t *itrq) 2771 { 2772 int i; 2773 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2774 2775 for (i = 0; i < rxd->rxd_ring_size; i++) { 2776 i40e_rx_control_block_t *rcb; 2777 i40e_rx_desc_t *rdesc; 2778 2779 rcb = rxd->rxd_work_list[i]; 2780 rdesc = &rxd->rxd_desc_ring[i]; 2781 2782 rdesc->read.pkt_addr = 2783 CPU_TO_LE64((uintptr_t)rcb->rcb_dma.dmab_dma_address); 2784 rdesc->read.hdr_addr = 0; 2785 } 2786 } 2787 2788 static boolean_t 2789 i40e_setup_rx_hmc(i40e_trqpair_t *itrq) 2790 { 2791 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2792 i40e_t *i40e = itrq->itrq_i40e; 2793 i40e_hw_t *hw = &i40e->i40e_hw_space; 2794 2795 struct i40e_hmc_obj_rxq rctx; 2796 int err; 2797 2798 bzero(&rctx, sizeof (struct i40e_hmc_obj_rxq)); 2799 rctx.base = rxd->rxd_desc_area.dmab_dma_address / 2800 I40E_HMC_RX_CTX_UNIT; 2801 rctx.qlen = rxd->rxd_ring_size; 2802 VERIFY(i40e->i40e_rx_buf_size >= I40E_HMC_RX_DBUFF_MIN); 2803 VERIFY(i40e->i40e_rx_buf_size <= I40E_HMC_RX_DBUFF_MAX); 2804 rctx.dbuff = i40e->i40e_rx_buf_size >> I40E_RXQ_CTX_DBUFF_SHIFT; 2805 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 2806 rctx.dtype = I40E_HMC_RX_DTYPE_NOSPLIT; 2807 rctx.dsize = I40E_HMC_RX_DSIZE_32BYTE; 2808 rctx.crcstrip = I40E_HMC_RX_CRCSTRIP_ENABLE; 2809 rctx.fc_ena = I40E_HMC_RX_FC_DISABLE; 2810 rctx.l2tsel = I40E_HMC_RX_L2TAGORDER; 2811 rctx.hsplit_0 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2812 rctx.hsplit_1 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2813 rctx.showiv = I40E_HMC_RX_INVLAN_DONTSTRIP; 2814 rctx.rxmax = i40e->i40e_frame_max; 2815 rctx.tphrdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2816 rctx.tphwdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2817 rctx.tphdata_ena = I40E_HMC_RX_TPH_DISABLE; 2818 rctx.tphhead_ena = I40E_HMC_RX_TPH_DISABLE; 2819 rctx.lrxqthresh = I40E_HMC_RX_LOWRXQ_NOINTR; 2820 2821 /* 2822 * This must be set to 0x1, see Table 8-12 in section 8.3.3.2.2. 2823 */ 2824 rctx.prefena = I40E_HMC_RX_PREFENA; 2825 2826 err = i40e_clear_lan_rx_queue_context(hw, itrq->itrq_index); 2827 if (err != I40E_SUCCESS) { 2828 i40e_error(i40e, "failed to clear rx queue %d context: %d", 2829 itrq->itrq_index, err); 2830 return (B_FALSE); 2831 } 2832 2833 err = i40e_set_lan_rx_queue_context(hw, itrq->itrq_index, &rctx); 2834 if (err != I40E_SUCCESS) { 2835 i40e_error(i40e, "failed to set rx queue %d context: %d", 2836 itrq->itrq_index, err); 2837 return (B_FALSE); 2838 } 2839 2840 return (B_TRUE); 2841 } 2842 2843 /* 2844 * Take care of setting up the descriptor ring and actually programming the 2845 * device. See 8.3.3.1.1 for the full list of steps we need to do to enable the 2846 * rx rings. 2847 */ 2848 static boolean_t 2849 i40e_setup_rx_ring(i40e_trqpair_t *itrq) 2850 { 2851 i40e_t *i40e = itrq->itrq_i40e; 2852 i40e_hw_t *hw = &i40e->i40e_hw_space; 2853 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2854 uint32_t reg; 2855 int i; 2856 2857 /* 2858 * Step 1. Program all receive ring descriptors. 2859 */ 2860 i40e_setup_rx_descs(itrq); 2861 2862 /* 2863 * Step 2. Program the queue's FPM/HMC context. 2864 */ 2865 if (!i40e_setup_rx_hmc(itrq)) 2866 return (B_FALSE); 2867 2868 /* 2869 * Step 3. Clear the queue's tail pointer and set it to the end 2870 * of the space. 2871 */ 2872 I40E_WRITE_REG(hw, I40E_QRX_TAIL(itrq->itrq_index), 0); 2873 I40E_WRITE_REG(hw, I40E_QRX_TAIL(itrq->itrq_index), 2874 rxd->rxd_ring_size - 1); 2875 2876 /* 2877 * Step 4. Enable the queue via the QENA_REQ. 2878 */ 2879 reg = I40E_READ_REG(hw, I40E_QRX_ENA(itrq->itrq_index)); 2880 VERIFY0(reg & (I40E_QRX_ENA_QENA_REQ_MASK | 2881 I40E_QRX_ENA_QENA_STAT_MASK)); 2882 reg |= I40E_QRX_ENA_QENA_REQ_MASK; 2883 I40E_WRITE_REG(hw, I40E_QRX_ENA(itrq->itrq_index), reg); 2884 2885 /* 2886 * Step 5. Verify that QENA_STAT has been set. It's promised 2887 * that this should occur within about 10 us, but like other 2888 * systems, we give the card a bit more time. 2889 */ 2890 for (i = 0; i < I40E_RING_WAIT_NTRIES; i++) { 2891 reg = I40E_READ_REG(hw, I40E_QRX_ENA(itrq->itrq_index)); 2892 2893 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) 2894 break; 2895 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2896 } 2897 2898 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { 2899 i40e_error(i40e, "failed to enable rx queue %d, timed " 2900 "out.", itrq->itrq_index); 2901 return (B_FALSE); 2902 } 2903 2904 return (B_TRUE); 2905 } 2906 2907 static boolean_t 2908 i40e_setup_tx_hmc(i40e_trqpair_t *itrq) 2909 { 2910 i40e_t *i40e = itrq->itrq_i40e; 2911 i40e_hw_t *hw = &i40e->i40e_hw_space; 2912 2913 struct i40e_hmc_obj_txq tctx; 2914 struct i40e_vsi_context context; 2915 int err; 2916 2917 bzero(&tctx, sizeof (struct i40e_hmc_obj_txq)); 2918 tctx.new_context = I40E_HMC_TX_NEW_CONTEXT; 2919 tctx.base = itrq->itrq_desc_area.dmab_dma_address / 2920 I40E_HMC_TX_CTX_UNIT; 2921 tctx.fc_ena = I40E_HMC_TX_FC_DISABLE; 2922 tctx.timesync_ena = I40E_HMC_TX_TS_DISABLE; 2923 tctx.fd_ena = I40E_HMC_TX_FD_DISABLE; 2924 tctx.alt_vlan_ena = I40E_HMC_TX_ALT_VLAN_DISABLE; 2925 tctx.head_wb_ena = I40E_HMC_TX_WB_ENABLE; 2926 tctx.qlen = itrq->itrq_tx_ring_size; 2927 tctx.tphrdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2928 tctx.tphrpacket_ena = I40E_HMC_TX_TPH_DISABLE; 2929 tctx.tphwdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2930 tctx.head_wb_addr = itrq->itrq_desc_area.dmab_dma_address + 2931 sizeof (i40e_tx_desc_t) * itrq->itrq_tx_ring_size; 2932 2933 /* 2934 * This field isn't actually documented, like crc, but it suggests that 2935 * it should be zeroed. We leave both of these here because of that for 2936 * now. We should check with Intel on why these are here even. 2937 */ 2938 tctx.crc = 0; 2939 tctx.rdylist_act = 0; 2940 2941 /* 2942 * We're supposed to assign the rdylist field with the value of the 2943 * traffic class index for the first device. We query the VSI parameters 2944 * again to get what the handle is. Note that every queue is always 2945 * assigned to traffic class zero, because we don't actually use them. 2946 */ 2947 bzero(&context, sizeof (struct i40e_vsi_context)); 2948 context.seid = I40E_DEF_VSI_SEID(i40e); 2949 context.pf_num = hw->pf_id; 2950 err = i40e_aq_get_vsi_params(hw, &context, NULL); 2951 if (err != I40E_SUCCESS) { 2952 i40e_error(i40e, "get VSI params failed with %d", err); 2953 return (B_FALSE); 2954 } 2955 tctx.rdylist = LE_16(context.info.qs_handle[0]); 2956 2957 err = i40e_clear_lan_tx_queue_context(hw, itrq->itrq_index); 2958 if (err != I40E_SUCCESS) { 2959 i40e_error(i40e, "failed to clear tx queue %d context: %d", 2960 itrq->itrq_index, err); 2961 return (B_FALSE); 2962 } 2963 2964 err = i40e_set_lan_tx_queue_context(hw, itrq->itrq_index, &tctx); 2965 if (err != I40E_SUCCESS) { 2966 i40e_error(i40e, "failed to set tx queue %d context: %d", 2967 itrq->itrq_index, err); 2968 return (B_FALSE); 2969 } 2970 2971 return (B_TRUE); 2972 } 2973 2974 /* 2975 * Take care of setting up the descriptor ring and actually programming the 2976 * device. See 8.4.3.1.1 for what we need to do here. 2977 */ 2978 static boolean_t 2979 i40e_setup_tx_ring(i40e_trqpair_t *itrq) 2980 { 2981 i40e_t *i40e = itrq->itrq_i40e; 2982 i40e_hw_t *hw = &i40e->i40e_hw_space; 2983 uint32_t reg; 2984 int i; 2985 2986 /* 2987 * Step 1. Clear the queue disable flag and verify that the 2988 * index is set correctly. 2989 */ 2990 i40e_pre_tx_queue_cfg(hw, itrq->itrq_index, B_TRUE); 2991 2992 /* 2993 * Step 2. Prepare the queue's FPM/HMC context. 2994 */ 2995 if (!i40e_setup_tx_hmc(itrq)) 2996 return (B_FALSE); 2997 2998 /* 2999 * Step 3. Verify that it's clear that this PF owns this queue. 3000 */ 3001 reg = I40E_QTX_CTL_PF_QUEUE; 3002 reg |= (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3003 I40E_QTX_CTL_PF_INDX_MASK; 3004 I40E_WRITE_REG(hw, I40E_QTX_CTL(itrq->itrq_index), reg); 3005 i40e_flush(hw); 3006 3007 /* 3008 * Step 4. Set the QENA_REQ flag. 3009 */ 3010 reg = I40E_READ_REG(hw, I40E_QTX_ENA(itrq->itrq_index)); 3011 VERIFY0(reg & (I40E_QTX_ENA_QENA_REQ_MASK | 3012 I40E_QTX_ENA_QENA_STAT_MASK)); 3013 reg |= I40E_QTX_ENA_QENA_REQ_MASK; 3014 I40E_WRITE_REG(hw, I40E_QTX_ENA(itrq->itrq_index), reg); 3015 3016 /* 3017 * Step 5. Verify that QENA_STAT has been set. It's promised 3018 * that this should occur within about 10 us, but like BSD, 3019 * we'll try for up to 100 ms for this queue. 3020 */ 3021 for (i = 0; i < I40E_RING_WAIT_NTRIES; i++) { 3022 reg = I40E_READ_REG(hw, I40E_QTX_ENA(itrq->itrq_index)); 3023 3024 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) 3025 break; 3026 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 3027 } 3028 3029 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { 3030 i40e_error(i40e, "failed to enable tx queue %d, timed " 3031 "out", itrq->itrq_index); 3032 return (B_FALSE); 3033 } 3034 3035 return (B_TRUE); 3036 } 3037 3038 int 3039 i40e_setup_ring(i40e_trqpair_t *itrq) 3040 { 3041 i40e_t *i40e = itrq->itrq_i40e; 3042 hrtime_t now, gap; 3043 3044 if (!i40e_alloc_ring_mem(itrq)) { 3045 i40e_error(i40e, "Failed to allocate ring memory"); 3046 return (ENOMEM); 3047 } 3048 3049 /* 3050 * 8.3.3.1.1 Receive Queue Enable Flow states software should 3051 * wait at least 50ms between ring disable and enable. See how 3052 * long we need to wait, and wait only if required. 3053 */ 3054 now = gethrtime(); 3055 gap = NSEC2MSEC(now - itrq->irtq_time_stopped); 3056 if (gap < I40E_RING_ENABLE_GAP && gap != 0) 3057 delay(drv_usectohz(gap * 1000)); 3058 3059 mutex_enter(&itrq->itrq_intr_lock); 3060 if (!i40e_setup_rx_ring(itrq)) 3061 goto failed; 3062 3063 if (!i40e_setup_tx_ring(itrq)) 3064 goto failed; 3065 3066 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 3067 DDI_FM_OK) 3068 goto failed; 3069 3070 itrq->itrq_intr_quiesce = B_FALSE; 3071 mutex_exit(&itrq->itrq_intr_lock); 3072 3073 mutex_enter(&itrq->itrq_tx_lock); 3074 itrq->itrq_tx_quiesce = B_FALSE; 3075 mutex_exit(&itrq->itrq_tx_lock); 3076 3077 return (0); 3078 3079 failed: 3080 mutex_exit(&itrq->itrq_intr_lock); 3081 i40e_free_ring_mem(itrq, B_TRUE); 3082 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3083 3084 return (EIO); 3085 } 3086 3087 void 3088 i40e_stop(i40e_t *i40e) 3089 { 3090 uint_t i; 3091 i40e_hw_t *hw = &i40e->i40e_hw_space; 3092 3093 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 3094 3095 /* 3096 * Shutdown and drain the tx and rx pipeline. We do this using the 3097 * following steps. 3098 * 3099 * 1) Shutdown interrupts to all the queues (trying to keep the admin 3100 * queue alive). 3101 * 3102 * 2) Remove all of the interrupt tx and rx causes by setting the 3103 * interrupt linked lists to zero. 3104 * 3105 * 2) Shutdown the tx and rx rings. Because i40e_shutdown_rings() should 3106 * wait for all the queues to be disabled, once we reach that point 3107 * it should be safe to free associated data. 3108 * 3109 * 4) Wait 50ms after all that is done. This ensures that the rings are 3110 * ready for programming again and we don't have to think about this 3111 * in other parts of the driver. 3112 * 3113 * 5) Disable remaining chip interrupts, (admin queue, etc.) 3114 * 3115 * 6) Verify that FM is happy with all the register accesses we 3116 * performed. 3117 */ 3118 i40e_intr_io_disable_all(i40e); 3119 i40e_intr_io_clear_cause(i40e); 3120 3121 if (!i40e_shutdown_rings(i40e)) 3122 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3123 3124 /* 3125 * We don't delete the default VSI because it replaces the VEB 3126 * after VEB deletion (see the "Delete Element" section). 3127 * Furthermore, since the default VSI is provided by the 3128 * firmware, we never attempt to delete it. 3129 */ 3130 for (i = 1; i < i40e->i40e_num_rx_groups; i++) { 3131 i40e_delete_vsi(i40e, i); 3132 } 3133 3134 if (i40e->i40e_veb_seid != 0) { 3135 int rc = i40e_aq_delete_element(hw, i40e->i40e_veb_seid, NULL); 3136 3137 if (rc != I40E_SUCCESS) { 3138 i40e_error(i40e, "Failed to delete VEB %d: %d", rc, 3139 hw->aq.asq_last_status); 3140 } 3141 3142 i40e->i40e_veb_seid = 0; 3143 } 3144 3145 i40e_intr_chip_fini(i40e); 3146 3147 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 3148 DDI_FM_OK) { 3149 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3150 } 3151 3152 for (i = 0; i < i40e->i40e_num_rx_groups; i++) { 3153 i40e_stat_vsi_fini(i40e, i); 3154 } 3155 3156 i40e->i40e_link_speed = 0; 3157 i40e->i40e_link_duplex = 0; 3158 i40e_link_state_set(i40e, LINK_STATE_UNKNOWN); 3159 } 3160 3161 boolean_t 3162 i40e_start(i40e_t *i40e) 3163 { 3164 i40e_hw_t *hw = &i40e->i40e_hw_space; 3165 boolean_t rc = B_TRUE; 3166 int err; 3167 3168 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 3169 3170 if (!i40e_chip_start(i40e)) { 3171 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 3172 rc = B_FALSE; 3173 goto done; 3174 } 3175 3176 /* 3177 * Enable broadcast traffic; however, do not enable multicast traffic. 3178 * That's handle exclusively through MAC's mc_multicst routines. 3179 */ 3180 err = i40e_aq_set_vsi_broadcast(hw, I40E_DEF_VSI_SEID(i40e), B_TRUE, 3181 NULL); 3182 if (err != I40E_SUCCESS) { 3183 i40e_error(i40e, "failed to set default VSI: %d", err); 3184 rc = B_FALSE; 3185 goto done; 3186 } 3187 3188 err = i40e_aq_set_mac_config(hw, i40e->i40e_frame_max, B_TRUE, 0, 3189 B_FALSE, NULL); 3190 if (err != I40E_SUCCESS) { 3191 i40e_error(i40e, "failed to set MAC config: %d", err); 3192 rc = B_FALSE; 3193 goto done; 3194 } 3195 3196 /* 3197 * Finally, make sure that we're happy from an FM perspective. 3198 */ 3199 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 3200 DDI_FM_OK) { 3201 rc = B_FALSE; 3202 goto done; 3203 } 3204 3205 /* Clear state bits prior to final interrupt enabling. */ 3206 atomic_and_32(&i40e->i40e_state, 3207 ~(I40E_ERROR | I40E_STALL | I40E_OVERTEMP)); 3208 3209 i40e_intr_io_enable_all(i40e); 3210 3211 done: 3212 if (rc == B_FALSE) { 3213 i40e_stop(i40e); 3214 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3215 } 3216 3217 return (rc); 3218 } 3219 3220 /* 3221 * We may have loaned up descriptors to the stack. As such, if we still have 3222 * them outstanding, then we will not continue with detach. 3223 */ 3224 static boolean_t 3225 i40e_drain_rx(i40e_t *i40e) 3226 { 3227 mutex_enter(&i40e->i40e_rx_pending_lock); 3228 while (i40e->i40e_rx_pending > 0) { 3229 if (cv_reltimedwait(&i40e->i40e_rx_pending_cv, 3230 &i40e->i40e_rx_pending_lock, 3231 drv_usectohz(I40E_DRAIN_RX_WAIT), TR_CLOCK_TICK) == -1) { 3232 mutex_exit(&i40e->i40e_rx_pending_lock); 3233 return (B_FALSE); 3234 } 3235 } 3236 mutex_exit(&i40e->i40e_rx_pending_lock); 3237 3238 return (B_TRUE); 3239 } 3240 3241 /* 3242 * DDI UFM Callbacks 3243 */ 3244 static int 3245 i40e_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 3246 ddi_ufm_image_t *img) 3247 { 3248 if (imgno != 0) 3249 return (EINVAL); 3250 3251 ddi_ufm_image_set_desc(img, "Firmware"); 3252 ddi_ufm_image_set_nslots(img, 1); 3253 3254 return (0); 3255 } 3256 3257 static int 3258 i40e_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 3259 uint_t slotno, ddi_ufm_slot_t *slot) 3260 { 3261 i40e_t *i40e = (i40e_t *)arg; 3262 char *fw_ver = NULL, *fw_bld = NULL, *api_ver = NULL; 3263 nvlist_t *misc = NULL; 3264 uint_t flags = DDI_PROP_DONTPASS; 3265 int err; 3266 3267 if (imgno != 0 || slotno != 0 || 3268 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3269 "firmware-version", &fw_ver) != DDI_PROP_SUCCESS || 3270 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3271 "firmware-build", &fw_bld) != DDI_PROP_SUCCESS || 3272 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3273 "api-version", &api_ver) != DDI_PROP_SUCCESS) { 3274 err = EINVAL; 3275 goto err; 3276 } 3277 3278 ddi_ufm_slot_set_attrs(slot, DDI_UFM_ATTR_ACTIVE); 3279 ddi_ufm_slot_set_version(slot, fw_ver); 3280 3281 (void) nvlist_alloc(&misc, NV_UNIQUE_NAME, KM_SLEEP); 3282 if ((err = nvlist_add_string(misc, "firmware-build", fw_bld)) != 0 || 3283 (err = nvlist_add_string(misc, "api-version", api_ver)) != 0) { 3284 goto err; 3285 } 3286 ddi_ufm_slot_set_misc(slot, misc); 3287 3288 ddi_prop_free(fw_ver); 3289 ddi_prop_free(fw_bld); 3290 ddi_prop_free(api_ver); 3291 3292 return (0); 3293 err: 3294 nvlist_free(misc); 3295 if (fw_ver != NULL) 3296 ddi_prop_free(fw_ver); 3297 if (fw_bld != NULL) 3298 ddi_prop_free(fw_bld); 3299 if (api_ver != NULL) 3300 ddi_prop_free(api_ver); 3301 3302 return (err); 3303 } 3304 3305 static int 3306 i40e_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps) 3307 { 3308 *caps = DDI_UFM_CAP_REPORT; 3309 3310 return (0); 3311 } 3312 3313 static ddi_ufm_ops_t i40e_ufm_ops = { 3314 NULL, 3315 i40e_ufm_fill_image, 3316 i40e_ufm_fill_slot, 3317 i40e_ufm_getcaps 3318 }; 3319 3320 static int 3321 i40e_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 3322 { 3323 i40e_t *i40e; 3324 struct i40e_osdep *osdep; 3325 i40e_hw_t *hw; 3326 int instance; 3327 3328 if (cmd != DDI_ATTACH) 3329 return (DDI_FAILURE); 3330 3331 instance = ddi_get_instance(devinfo); 3332 i40e = kmem_zalloc(sizeof (i40e_t), KM_SLEEP); 3333 3334 i40e->i40e_aqbuf = kmem_zalloc(I40E_ADMINQ_BUFSZ, KM_SLEEP); 3335 i40e->i40e_instance = instance; 3336 i40e->i40e_dip = devinfo; 3337 3338 hw = &i40e->i40e_hw_space; 3339 osdep = &i40e->i40e_osdep_space; 3340 hw->back = osdep; 3341 osdep->ios_i40e = i40e; 3342 3343 ddi_set_driver_private(devinfo, i40e); 3344 3345 i40e_fm_init(i40e); 3346 i40e->i40e_attach_progress |= I40E_ATTACH_FM_INIT; 3347 3348 if (pci_config_setup(devinfo, &osdep->ios_cfg_handle) != DDI_SUCCESS) { 3349 i40e_error(i40e, "Failed to map PCI configurations."); 3350 goto attach_fail; 3351 } 3352 i40e->i40e_attach_progress |= I40E_ATTACH_PCI_CONFIG; 3353 3354 i40e_identify_hardware(i40e); 3355 3356 if (!i40e_regs_map(i40e)) { 3357 i40e_error(i40e, "Failed to map device registers."); 3358 goto attach_fail; 3359 } 3360 i40e->i40e_attach_progress |= I40E_ATTACH_REGS_MAP; 3361 3362 i40e_init_properties(i40e); 3363 i40e->i40e_attach_progress |= I40E_ATTACH_PROPS; 3364 3365 if (!i40e_common_code_init(i40e, hw)) 3366 goto attach_fail; 3367 i40e->i40e_attach_progress |= I40E_ATTACH_COMMON_CODE; 3368 3369 /* 3370 * When we participate in IRM, we should make sure that we register 3371 * ourselves with it before callbacks. 3372 */ 3373 if (!i40e_alloc_intrs(i40e, devinfo)) { 3374 i40e_error(i40e, "Failed to allocate interrupts."); 3375 goto attach_fail; 3376 } 3377 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_INTR; 3378 3379 if (!i40e_alloc_trqpairs(i40e)) { 3380 i40e_error(i40e, 3381 "Failed to allocate receive & transmit rings."); 3382 goto attach_fail; 3383 } 3384 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_RINGSLOCKS; 3385 3386 if (!i40e_map_intrs_to_vectors(i40e)) { 3387 i40e_error(i40e, "Failed to map interrupts to vectors."); 3388 goto attach_fail; 3389 } 3390 3391 if (!i40e_add_intr_handlers(i40e)) { 3392 i40e_error(i40e, "Failed to add the interrupt handlers."); 3393 goto attach_fail; 3394 } 3395 i40e->i40e_attach_progress |= I40E_ATTACH_ADD_INTR; 3396 3397 if (!i40e_final_init(i40e)) { 3398 i40e_error(i40e, "Final initialization failed."); 3399 goto attach_fail; 3400 } 3401 i40e->i40e_attach_progress |= I40E_ATTACH_INIT; 3402 3403 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 3404 DDI_FM_OK) { 3405 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3406 goto attach_fail; 3407 } 3408 3409 if (!i40e_stats_init(i40e)) { 3410 i40e_error(i40e, "Stats initialization failed."); 3411 goto attach_fail; 3412 } 3413 i40e->i40e_attach_progress |= I40E_ATTACH_STATS; 3414 3415 if (!i40e_register_mac(i40e)) { 3416 i40e_error(i40e, "Failed to register to MAC/GLDv3"); 3417 goto attach_fail; 3418 } 3419 i40e->i40e_attach_progress |= I40E_ATTACH_MAC; 3420 3421 i40e->i40e_periodic_id = ddi_periodic_add(i40e_timer, i40e, 3422 I40E_CYCLIC_PERIOD, DDI_IPL_0); 3423 if (i40e->i40e_periodic_id == 0) { 3424 i40e_error(i40e, "Failed to add the link-check timer"); 3425 goto attach_fail; 3426 } 3427 i40e->i40e_attach_progress |= I40E_ATTACH_LINK_TIMER; 3428 3429 if (!i40e_enable_interrupts(i40e)) { 3430 i40e_error(i40e, "Failed to enable DDI interrupts"); 3431 goto attach_fail; 3432 } 3433 i40e->i40e_attach_progress |= I40E_ATTACH_ENABLE_INTR; 3434 3435 if (i40e->i40e_hw_space.bus.func == 0) { 3436 if (ddi_ufm_init(i40e->i40e_dip, DDI_UFM_CURRENT_VERSION, 3437 &i40e_ufm_ops, &i40e->i40e_ufmh, i40e) != 0) { 3438 i40e_error(i40e, "failed to initialize UFM subsystem"); 3439 goto attach_fail; 3440 } 3441 ddi_ufm_update(i40e->i40e_ufmh); 3442 i40e->i40e_attach_progress |= I40E_ATTACH_UFM_INIT; 3443 } 3444 3445 atomic_or_32(&i40e->i40e_state, I40E_INITIALIZED); 3446 3447 mutex_enter(&i40e_glock); 3448 list_insert_tail(&i40e_glist, i40e); 3449 mutex_exit(&i40e_glock); 3450 3451 return (DDI_SUCCESS); 3452 3453 attach_fail: 3454 i40e_unconfigure(devinfo, i40e); 3455 return (DDI_FAILURE); 3456 } 3457 3458 static int 3459 i40e_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 3460 { 3461 i40e_t *i40e; 3462 3463 if (cmd != DDI_DETACH) 3464 return (DDI_FAILURE); 3465 3466 i40e = (i40e_t *)ddi_get_driver_private(devinfo); 3467 if (i40e == NULL) { 3468 i40e_log(NULL, "i40e_detach() called with no i40e pointer!"); 3469 return (DDI_FAILURE); 3470 } 3471 3472 if (i40e_drain_rx(i40e) == B_FALSE) { 3473 i40e_log(i40e, "timed out draining DMA resources, %d buffers " 3474 "remain", i40e->i40e_rx_pending); 3475 return (DDI_FAILURE); 3476 } 3477 3478 mutex_enter(&i40e_glock); 3479 list_remove(&i40e_glist, i40e); 3480 mutex_exit(&i40e_glock); 3481 3482 i40e_unconfigure(devinfo, i40e); 3483 3484 return (DDI_SUCCESS); 3485 } 3486 3487 static struct cb_ops i40e_cb_ops = { 3488 nulldev, /* cb_open */ 3489 nulldev, /* cb_close */ 3490 nodev, /* cb_strategy */ 3491 nodev, /* cb_print */ 3492 nodev, /* cb_dump */ 3493 nodev, /* cb_read */ 3494 nodev, /* cb_write */ 3495 nodev, /* cb_ioctl */ 3496 nodev, /* cb_devmap */ 3497 nodev, /* cb_mmap */ 3498 nodev, /* cb_segmap */ 3499 nochpoll, /* cb_chpoll */ 3500 ddi_prop_op, /* cb_prop_op */ 3501 NULL, /* cb_stream */ 3502 D_MP | D_HOTPLUG, /* cb_flag */ 3503 CB_REV, /* cb_rev */ 3504 nodev, /* cb_aread */ 3505 nodev /* cb_awrite */ 3506 }; 3507 3508 static struct dev_ops i40e_dev_ops = { 3509 DEVO_REV, /* devo_rev */ 3510 0, /* devo_refcnt */ 3511 NULL, /* devo_getinfo */ 3512 nulldev, /* devo_identify */ 3513 nulldev, /* devo_probe */ 3514 i40e_attach, /* devo_attach */ 3515 i40e_detach, /* devo_detach */ 3516 nodev, /* devo_reset */ 3517 &i40e_cb_ops, /* devo_cb_ops */ 3518 NULL, /* devo_bus_ops */ 3519 nulldev, /* devo_power */ 3520 ddi_quiesce_not_supported /* devo_quiesce */ 3521 }; 3522 3523 static struct modldrv i40e_modldrv = { 3524 &mod_driverops, 3525 i40e_ident, 3526 &i40e_dev_ops 3527 }; 3528 3529 static struct modlinkage i40e_modlinkage = { 3530 MODREV_1, 3531 &i40e_modldrv, 3532 NULL 3533 }; 3534 3535 /* 3536 * Module Initialization Functions. 3537 */ 3538 int 3539 _init(void) 3540 { 3541 int status; 3542 3543 list_create(&i40e_glist, sizeof (i40e_t), offsetof(i40e_t, i40e_glink)); 3544 list_create(&i40e_dlist, sizeof (i40e_device_t), 3545 offsetof(i40e_device_t, id_link)); 3546 mutex_init(&i40e_glock, NULL, MUTEX_DRIVER, NULL); 3547 mac_init_ops(&i40e_dev_ops, I40E_MODULE_NAME); 3548 3549 status = mod_install(&i40e_modlinkage); 3550 if (status != DDI_SUCCESS) { 3551 mac_fini_ops(&i40e_dev_ops); 3552 mutex_destroy(&i40e_glock); 3553 list_destroy(&i40e_dlist); 3554 list_destroy(&i40e_glist); 3555 } 3556 3557 return (status); 3558 } 3559 3560 int 3561 _info(struct modinfo *modinfop) 3562 { 3563 return (mod_info(&i40e_modlinkage, modinfop)); 3564 } 3565 3566 int 3567 _fini(void) 3568 { 3569 int status; 3570 3571 status = mod_remove(&i40e_modlinkage); 3572 if (status == DDI_SUCCESS) { 3573 mac_fini_ops(&i40e_dev_ops); 3574 mutex_destroy(&i40e_glock); 3575 list_destroy(&i40e_dlist); 3576 list_destroy(&i40e_glist); 3577 } 3578 3579 return (status); 3580 } 3581