1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2024 Kevin Bowling <kbowling@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 #include "if_igb_iov.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #define em_mac_min e1000_82571 36 #define igb_mac_min e1000_82575 37 38 /********************************************************************* 39 * Driver version: 40 *********************************************************************/ 41 static const char em_driver_version[] = "7.7.8-fbsd"; 42 static const char igb_driver_version[] = "2.5.28-fbsd"; 43 44 /********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into e1000_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54 static const pci_vendor_info_t em_vendor_info_array[] = 55 { 56 /* Intel(R) - lem-class legacy devices */ 57 PVID(0x8086, E1000_DEV_ID_82540EM, 58 "Intel(R) Legacy PRO/1000 MT 82540EM"), 59 PVID(0x8086, E1000_DEV_ID_82540EM_LOM, 60 "Intel(R) Legacy PRO/1000 MT 82540EM (LOM)"), 61 PVID(0x8086, E1000_DEV_ID_82540EP, 62 "Intel(R) Legacy PRO/1000 MT 82540EP"), 63 PVID(0x8086, E1000_DEV_ID_82540EP_LOM, 64 "Intel(R) Legacy PRO/1000 MT 82540EP (LOM)"), 65 PVID(0x8086, E1000_DEV_ID_82540EP_LP, 66 "Intel(R) Legacy PRO/1000 MT 82540EP (Mobile)"), 67 68 PVID(0x8086, E1000_DEV_ID_82541EI, 69 "Intel(R) Legacy PRO/1000 MT 82541EI (Copper)"), 70 PVID(0x8086, E1000_DEV_ID_82541ER, 71 "Intel(R) Legacy PRO/1000 82541ER"), 72 PVID(0x8086, E1000_DEV_ID_82541ER_LOM, 73 "Intel(R) Legacy PRO/1000 MT 82541ER"), 74 PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, 75 "Intel(R) Legacy PRO/1000 MT 82541EI (Mobile)"), 76 PVID(0x8086, E1000_DEV_ID_82541GI, 77 "Intel(R) Legacy PRO/1000 MT 82541GI"), 78 PVID(0x8086, E1000_DEV_ID_82541GI_LF, 79 "Intel(R) Legacy PRO/1000 GT 82541PI"), 80 PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, 81 "Intel(R) Legacy PRO/1000 MT 82541GI (Mobile)"), 82 83 PVID(0x8086, E1000_DEV_ID_82542, 84 "Intel(R) Legacy PRO/1000 82542 (Fiber)"), 85 86 PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, 87 "Intel(R) Legacy PRO/1000 F 82543GC (Fiber)"), 88 PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, 89 "Intel(R) Legacy PRO/1000 T 82543GC (Copper)"), 90 91 PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, 92 "Intel(R) Legacy PRO/1000 XT 82544EI (Copper)"), 93 PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, 94 "Intel(R) Legacy PRO/1000 XF 82544EI (Fiber)"), 95 PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, 96 "Intel(R) Legacy PRO/1000 T 82544GC (Copper)"), 97 PVID(0x8086, E1000_DEV_ID_82544GC_LOM, 98 "Intel(R) Legacy PRO/1000 XT 82544GC (LOM)"), 99 100 PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, 101 "Intel(R) Legacy PRO/1000 MT 82545EM (Copper)"), 102 PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, 103 "Intel(R) Legacy PRO/1000 MF 82545EM (Fiber)"), 104 PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, 105 "Intel(R) Legacy PRO/1000 MT 82545GM (Copper)"), 106 PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, 107 "Intel(R) Legacy PRO/1000 MF 82545GM (Fiber)"), 108 PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, 109 "Intel(R) Legacy PRO/1000 MB 82545GM (SERDES)"), 110 111 PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, 112 "Intel(R) Legacy PRO/1000 MT 82546EB (Copper)"), 113 PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, 114 "Intel(R) Legacy PRO/1000 MF 82546EB (Fiber)"), 115 PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, 116 "Intel(R) Legacy PRO/1000 MT 82546EB (Quad Copper"), 117 PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, 118 "Intel(R) Legacy PRO/1000 MT 82546GB (Copper)"), 119 PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, 120 "Intel(R) Legacy PRO/1000 MF 82546GB (Fiber)"), 121 PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, 122 "Intel(R) Legacy PRO/1000 MB 82546GB (SERDES)"), 123 PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, 124 "Intel(R) Legacy PRO/1000 P 82546GB (PCIe)"), 125 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, 126 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 127 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 128 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 129 130 PVID(0x8086, E1000_DEV_ID_82547EI, 131 "Intel(R) Legacy PRO/1000 CT 82547EI"), 132 PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, 133 "Intel(R) Legacy PRO/1000 CT 82547EI (Mobile)"), 134 PVID(0x8086, E1000_DEV_ID_82547GI, 135 "Intel(R) Legacy PRO/1000 CT 82547GI"), 136 137 /* Intel(R) - em-class devices */ 138 PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, 139 "Intel(R) PRO/1000 PT 82571EB/82571GB (Copper)"), 140 PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, 141 "Intel(R) PRO/1000 PF 82571EB/82571GB (Fiber)"), 142 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, 143 "Intel(R) PRO/1000 PB 82571EB (SERDES)"), 144 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 145 "Intel(R) PRO/1000 82571EB (Dual Mezzanine)"), 146 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 147 "Intel(R) PRO/1000 82571EB (Quad Mezzanine)"), 148 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 149 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 150 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 151 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 152 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 153 "Intel(R) PRO/1000 PF 82571EB (Quad Fiber)"), 154 PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 155 "Intel(R) PRO/1000 PT 82571PT (Quad Copper)"), 156 PVID(0x8086, E1000_DEV_ID_82572EI, 157 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 158 PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, 159 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 160 PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, 161 "Intel(R) PRO/1000 PF 82572EI (Fiber)"), 162 PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, 163 "Intel(R) PRO/1000 82572EI (SERDES)"), 164 PVID(0x8086, E1000_DEV_ID_82573E, 165 "Intel(R) PRO/1000 82573E (Copper)"), 166 PVID(0x8086, E1000_DEV_ID_82573E_IAMT, 167 "Intel(R) PRO/1000 82573E AMT (Copper)"), 168 PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 82573L"), 169 PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) 82583V"), 170 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 171 "Intel(R) 80003ES2LAN (Copper)"), 172 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 173 "Intel(R) 80003ES2LAN (SERDES)"), 174 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 175 "Intel(R) 80003ES2LAN (Dual Copper)"), 176 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 177 "Intel(R) 80003ES2LAN (Dual SERDES)"), 178 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, 179 "Intel(R) 82566MM ICH8 AMT (Mobile)"), 180 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) 82566DM ICH8 AMT"), 181 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) 82566DC ICH8"), 182 PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) 82562V ICH8"), 183 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) 82562GT ICH8"), 184 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) 82562G ICH8"), 185 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) 82566MC ICH8"), 186 PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) 82567V-3 ICH8"), 187 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, 188 "Intel(R) 82567LM ICH9 AMT"), 189 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, 190 "Intel(R) 82566DM-2 ICH9 AMT"), 191 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) 82566DC-2 ICH9"), 192 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) 82567LF ICH9"), 193 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) 82567V ICH9"), 194 PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) 82562V-2 ICH9"), 195 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) 82562GT-2 ICH9"), 196 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) 82562G-2 ICH9"), 197 PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) 82567LM-4 ICH9"), 198 PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) Gigabit CT 82574L"), 199 PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) 82574L-Apple"), 200 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) 82567LM-2 ICH10"), 201 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) 82567LF-2 ICH10"), 202 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) 82567V-2 ICH10"), 203 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) 82567LM-3 ICH10"), 204 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) 82567LF-3 ICH10"), 205 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) 82567V-4 ICH10"), 206 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) 82577LM"), 207 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) 82577LC"), 208 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) 82578DM"), 209 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) 82578DC"), 210 PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) 82579LM"), 211 PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) 82579V"), 212 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) I217-LM LPT"), 213 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) I217-V LPT"), 214 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, 215 "Intel(R) I218-LM LPTLP"), 216 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) I218-V LPTLP"), 217 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) I218-LM (2)"), 218 PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) I218-V (2)"), 219 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) I218-LM (3)"), 220 PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) I218-V (3)"), 221 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) I219-LM SPT"), 222 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) I219-V SPT"), 223 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, 224 "Intel(R) I219-LM SPT-H(2)"), 225 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, 226 "Intel(R) I219-V SPT-H(2)"), 227 PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, 228 "Intel(R) I219-LM LBG(3)"), 229 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, 230 "Intel(R) I219-LM SPT(4)"), 231 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) I219-V SPT(4)"), 232 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, 233 "Intel(R) I219-LM SPT(5)"), 234 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) I219-V SPT(5)"), 235 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM6, 236 "Intel(R) I219-LM CNP(6)"), 237 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V6, "Intel(R) I219-V CNP(6)"), 238 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM7, 239 "Intel(R) I219-LM CNP(7)"), 240 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V7, "Intel(R) I219-V CNP(7)"), 241 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM8, 242 "Intel(R) I219-LM ICP(8)"), 243 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V8, "Intel(R) I219-V ICP(8)"), 244 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM9, 245 "Intel(R) I219-LM ICP(9)"), 246 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V9, "Intel(R) I219-V ICP(9)"), 247 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM10, 248 "Intel(R) I219-LM CMP(10)"), 249 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V10, 250 "Intel(R) I219-V CMP(10)"), 251 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM11, 252 "Intel(R) I219-LM CMP(11)"), 253 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V11, 254 "Intel(R) I219-V CMP(11)"), 255 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM12, 256 "Intel(R) I219-LM CMP(12)"), 257 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V12, 258 "Intel(R) I219-V CMP(12)"), 259 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM13, 260 "Intel(R) I219-LM TGP(13)"), 261 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V13, 262 "Intel(R) I219-V TGP(13)"), 263 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM14, 264 "Intel(R) I219-LM TGP(14)"), 265 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V14, 266 "Intel(R) I219-V GTP(14)"), 267 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM15, 268 "Intel(R) I219-LM TGP(15)"), 269 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V15, 270 "Intel(R) I219-V TGP(15)"), 271 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM16, 272 "Intel(R) I219-LM ADL(16)"), 273 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V16, 274 "Intel(R) I219-V ADL(16)"), 275 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM17, 276 "Intel(R) I219-LM ADL(17)"), 277 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V17, 278 "Intel(R) I219-V ADL(17)"), 279 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_LM18, 280 "Intel(R) I219-LM MTP(18)"), 281 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_V18, 282 "Intel(R) I219-V MTP(18)"), 283 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM19, 284 "Intel(R) I219-LM ADL(19)"), 285 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V19, 286 "Intel(R) I219-V ADL(19)"), 287 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM20, 288 "Intel(R) I219-LM LNL(20)"), 289 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V20, 290 "Intel(R) I219-V LNL(20)"), 291 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM21, 292 "Intel(R) I219-LM LNL(21)"), 293 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V21, 294 "Intel(R) I219-V LNL(21)"), 295 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM22, 296 "Intel(R) I219-LM RPL(22)"), 297 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V22, 298 "Intel(R) I219-V RPL(22)"), 299 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM23, 300 "Intel(R) I219-LM RPL(23)"), 301 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V23, 302 "Intel(R) I219-V RPL(23)"), 303 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_LM24, 304 "Intel(R) I219-LM ARL(24)"), 305 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_V24, 306 "Intel(R) I219-V ARL(24)"), 307 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM25, 308 "Intel(R) I219-LM PTP(25)"), 309 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V25, 310 "Intel(R) I219-V PTP(25)"), 311 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM26, 312 "Intel(R) I219-LM PTP(26)"), 313 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V26, 314 "Intel(R) I219-V PTP(26)"), 315 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM27, 316 "Intel(R) I219-LM PTP(27)"), 317 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V27, 318 "Intel(R) I219-V PTP(27)"), 319 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_LM29, 320 "Intel(R) I219-LM NVL(29)"), 321 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_V29, 322 "Intel(R) I219-V NVL(29)"), 323 /* required last entry */ 324 PVID_END 325 }; 326 327 static const pci_vendor_info_t igb_vendor_info_array[] = 328 { 329 /* Intel(R) - igb-class devices */ 330 PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, 331 "Intel(R) PRO/1000 82575EB (Copper)"), 332 PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 333 "Intel(R) PRO/1000 82575EB (SERDES)"), 334 PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 335 "Intel(R) PRO/1000 VT 82575GB (Quad Copper)"), 336 PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 82576"), 337 PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 82576NS"), 338 PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, 339 "Intel(R) PRO/1000 82576NS (SERDES)"), 340 PVID(0x8086, E1000_DEV_ID_82576_FIBER, 341 "Intel(R) PRO/1000 EF 82576 (Dual Fiber)"), 342 PVID(0x8086, E1000_DEV_ID_82576_SERDES, 343 "Intel(R) PRO/1000 82576 (Dual SERDES)"), 344 PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 345 "Intel(R) PRO/1000 ET 82576 (Quad SERDES)"), 346 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 347 "Intel(R) PRO/1000 ET 82576 (Quad Copper)"), 348 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 349 "Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"), 350 PVID(0x8086, E1000_DEV_ID_82580_COPPER, 351 "Intel(R) I340 82580 (Copper)"), 352 PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"), 353 PVID(0x8086, E1000_DEV_ID_82580_SERDES, 354 "Intel(R) I340 82580 (SERDES)"), 355 PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) I340 82580 (SGMII)"), 356 PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 357 "Intel(R) I340-T2 82580 (Dual Copper)"), 358 PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, 359 "Intel(R) I340-F4 82580 (Quad Fiber)"), 360 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, 361 "Intel(R) DH89XXCC (SERDES)"), 362 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, 363 "Intel(R) I347-AT4 DH89XXCC"), 364 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) DH89XXCC (SFP)"), 365 PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, 366 "Intel(R) DH89XXCC (Backplane)"), 367 PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) I350 (Copper)"), 368 PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"), 369 PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"), 370 PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"), 371 PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"), 372 PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, 373 "Intel(R) I210 IT (Copper)"), 374 PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) I210 (OEM)"), 375 PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, 376 "Intel(R) I210 Flashless (Copper)"), 377 PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, 378 "Intel(R) I210 Flashless (SERDES)"), 379 PVID(0x8086, E1000_DEV_ID_I210_SGMII_FLASHLESS, 380 "Intel(R) I210 Flashless (SGMII)"), 381 PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) I210 (Fiber)"), 382 PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) I210 (SERDES)"), 383 PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) I210 (SGMII)"), 384 PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) I211 (Copper)"), 385 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 386 "Intel(R) I354 (1.0 GbE Backplane)"), 387 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 388 "Intel(R) I354 (2.5 GbE Backplane)"), 389 PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) I354 (SGMII)"), 390 /* required last entry */ 391 PVID_END 392 }; 393 394 static const pci_vendor_info_t igbv_vendor_info_array[] = { 395 PVID(0x8086, E1000_DEV_ID_82576_VF, 396 "Intel(R) PRO/1000 82576 Virtual Function"), 397 PVID(0x8086, E1000_DEV_ID_82576_VF_HV, 398 "Intel(R) PRO/1000 82576 Virtual Function"), 399 PVID(0x8086, E1000_DEV_ID_I350_VF, 400 "Intel(R) I350 Virtual Function"), 401 PVID(0x8086, E1000_DEV_ID_I350_VF_HV, 402 "Intel(R) I350 Virtual Function"), 403 PVID_END 404 }; 405 406 /********************************************************************* 407 * Function prototypes 408 *********************************************************************/ 409 static void *em_register(device_t); 410 static void *igb_register(device_t); 411 static void *igbv_register(device_t); 412 static int igb_device_attach(device_t); 413 #ifdef PCI_IOV 414 static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *); 415 static void igb_device_iov_uninit(device_t); 416 #endif 417 static int em_if_detach(if_ctx_t); 418 static int em_if_shutdown(if_ctx_t); 419 static int em_if_suspend(if_ctx_t); 420 static int em_if_resume(if_ctx_t); 421 422 static int em_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 423 int); 424 static int em_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 425 int); 426 static void em_if_queues_free(if_ctx_t); 427 428 static uint64_t em_if_get_vf_counter(if_ctx_t, ift_counter); 429 static uint64_t em_if_get_counter(if_ctx_t, ift_counter); 430 static void em_if_init(if_ctx_t); 431 static void em_if_stop(if_ctx_t); 432 static void em_if_media_status(if_ctx_t, struct ifmediareq *); 433 static int em_if_media_change(if_ctx_t); 434 static int em_if_mtu_set(if_ctx_t, uint32_t); 435 static void em_if_timer(if_ctx_t, uint16_t); 436 static void em_if_vlan_register(if_ctx_t, u16); 437 static void em_if_vlan_unregister(if_ctx_t, u16); 438 static bool em_if_needs_restart(if_ctx_t, enum iflib_restart_event); 439 440 static void em_identify_hardware(if_ctx_t); 441 static int em_allocate_pci_resources(if_ctx_t); 442 static void em_free_pci_resources(if_ctx_t); 443 static void em_reset(if_ctx_t); 444 static int em_setup_interface(if_ctx_t); 445 static int em_setup_msix(if_ctx_t); 446 447 static void em_initialize_transmit_unit(if_ctx_t); 448 static void em_initialize_receive_unit(if_ctx_t); 449 450 static void em_if_intr_enable(if_ctx_t); 451 static void em_if_intr_disable(if_ctx_t); 452 static void igb_if_intr_enable(if_ctx_t); 453 static void igb_if_intr_disable(if_ctx_t); 454 static int em_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 455 static int em_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 456 static int igb_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 457 static int igb_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 458 static void em_if_multi_set(if_ctx_t); 459 static void em_if_update_admin_status(if_ctx_t); 460 static void em_if_debug(if_ctx_t); 461 static void em_initialize_vf_stats(struct e1000_softc *); 462 static void em_rebase_vf_stats(struct e1000_softc *); 463 static void em_update_vf_stats_counters(struct e1000_softc *); 464 static void em_add_hw_stats(struct e1000_softc *); 465 static bool em_mac_has_eee(enum e1000_mac_type); 466 static int em_if_set_promisc(if_ctx_t, int); 467 static bool em_if_defer_promisc(struct e1000_softc *); 468 static bool em_if_vlan_filter_capable(if_ctx_t); 469 static bool em_if_vlan_filter_used(if_ctx_t); 470 static void em_if_vlan_filter_enable(struct e1000_softc *); 471 static void em_if_vlan_filter_disable(struct e1000_softc *); 472 static void em_if_vlan_filter_write(struct e1000_softc *, int); 473 static void em_setup_vlan_hw_support(if_ctx_t ctx); 474 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 475 static void em_print_nvm_info(struct e1000_softc *); 476 static void em_fw_version_locked(if_ctx_t); 477 static void em_sbuf_fw_version(struct e1000_fw_version *, struct sbuf *); 478 static void em_print_fw_version(struct e1000_softc *); 479 static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 480 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 481 static int em_get_rs(SYSCTL_HANDLER_ARGS); 482 static void em_print_debug_info(struct e1000_softc *); 483 static void em_newitr(struct e1000_softc *, struct em_rx_queue *, 484 struct rx_ring *); 485 static bool em_automask_tso(if_ctx_t); 486 static int em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 487 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 488 static void em_add_int_delay_sysctl(struct e1000_softc *, const char *, 489 const char *, struct em_int_delay_info *, int, int); 490 /* Management and WOL Support */ 491 static void em_init_manageability(struct e1000_softc *); 492 static void em_release_manageability(struct e1000_softc *); 493 static void em_get_hw_control(struct e1000_softc *); 494 static void em_release_hw_control(struct e1000_softc *); 495 static void em_get_wakeup(if_ctx_t); 496 static void em_enable_wakeup(if_ctx_t); 497 static int em_enable_phy_wakeup(struct e1000_softc *); 498 static void em_disable_aspm(struct e1000_softc *); 499 500 int em_intr(void *); 501 502 /* MSI-X handlers */ 503 static int em_if_msix_intr_assign(if_ctx_t, int); 504 static int em_msix_link(void *); 505 static void em_handle_link(void *); 506 507 static void em_enable_vectors_82574(if_ctx_t); 508 509 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); 510 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); 511 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); 512 static void em_if_led_func(if_ctx_t, int); 513 514 static int em_get_regs(SYSCTL_HANDLER_ARGS); 515 static void lem_smartspeed(struct e1000_softc *); 516 static void igb_configure_queues(struct e1000_softc *); 517 static void igb_initialize_interrupt_rate(struct e1000_softc *); 518 static void em_flush_desc_rings(struct e1000_softc *); 519 520 521 /********************************************************************* 522 * FreeBSD Device Interface Entry Points 523 *********************************************************************/ 524 static device_method_t em_methods[] = { 525 /* Device interface */ 526 DEVMETHOD(device_register, em_register), 527 DEVMETHOD(device_probe, iflib_device_probe), 528 DEVMETHOD(device_attach, iflib_device_attach), 529 DEVMETHOD(device_detach, iflib_device_detach), 530 DEVMETHOD(device_shutdown, iflib_device_shutdown), 531 DEVMETHOD(device_suspend, iflib_device_suspend), 532 DEVMETHOD(device_resume, iflib_device_resume), 533 DEVMETHOD_END 534 }; 535 536 static device_method_t igb_methods[] = { 537 /* Device interface */ 538 DEVMETHOD(device_register, igb_register), 539 DEVMETHOD(device_probe, iflib_device_probe), 540 DEVMETHOD(device_attach, igb_device_attach), 541 DEVMETHOD(device_detach, iflib_device_detach), 542 DEVMETHOD(device_shutdown, iflib_device_shutdown), 543 DEVMETHOD(device_suspend, iflib_device_suspend), 544 DEVMETHOD(device_resume, iflib_device_resume), 545 #ifdef PCI_IOV 546 DEVMETHOD(pci_iov_init, igb_device_iov_init), 547 DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit), 548 DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), 549 #endif 550 DEVMETHOD_END 551 }; 552 553 static device_method_t igbv_methods[] = { 554 /* Device interface */ 555 DEVMETHOD(device_register, igbv_register), 556 DEVMETHOD(device_probe, iflib_device_probe), 557 DEVMETHOD(device_attach, iflib_device_attach), 558 DEVMETHOD(device_detach, iflib_device_detach), 559 DEVMETHOD(device_shutdown, iflib_device_shutdown), 560 DEVMETHOD(device_suspend, iflib_device_suspend), 561 DEVMETHOD(device_resume, iflib_device_resume), 562 DEVMETHOD_END 563 }; 564 565 566 static driver_t em_driver = { 567 "em", em_methods, sizeof(struct e1000_softc), 568 }; 569 570 DRIVER_MODULE(em, pci, em_driver, 0, 0); 571 572 MODULE_DEPEND(em, pci, 1, 1, 1); 573 MODULE_DEPEND(em, ether, 1, 1, 1); 574 MODULE_DEPEND(em, iflib, 1, 1, 1); 575 576 IFLIB_PNP_INFO(pci, em, em_vendor_info_array); 577 578 static driver_t igb_driver = { 579 "igb", igb_methods, sizeof(struct e1000_softc), 580 }; 581 582 DRIVER_MODULE(igb, pci, igb_driver, 0, 0); 583 584 MODULE_DEPEND(igb, pci, 1, 1, 1); 585 MODULE_DEPEND(igb, ether, 1, 1, 1); 586 MODULE_DEPEND(igb, iflib, 1, 1, 1); 587 588 IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array); 589 590 static driver_t igbv_driver = { 591 "igbv", igbv_methods, sizeof(struct e1000_softc), 592 }; 593 594 DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0); 595 596 MODULE_DEPEND(igbv, pci, 1, 1, 1); 597 MODULE_DEPEND(igbv, ether, 1, 1, 1); 598 MODULE_DEPEND(igbv, iflib, 1, 1, 1); 599 600 IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array); 601 602 static device_method_t em_if_methods[] = { 603 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 604 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 605 DEVMETHOD(ifdi_detach, em_if_detach), 606 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 607 DEVMETHOD(ifdi_suspend, em_if_suspend), 608 DEVMETHOD(ifdi_resume, em_if_resume), 609 DEVMETHOD(ifdi_init, em_if_init), 610 DEVMETHOD(ifdi_stop, em_if_stop), 611 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 612 DEVMETHOD(ifdi_intr_enable, em_if_intr_enable), 613 DEVMETHOD(ifdi_intr_disable, em_if_intr_disable), 614 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 615 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 616 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 617 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 618 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 619 DEVMETHOD(ifdi_media_status, em_if_media_status), 620 DEVMETHOD(ifdi_media_change, em_if_media_change), 621 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 622 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 623 DEVMETHOD(ifdi_timer, em_if_timer), 624 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 625 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 626 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 627 DEVMETHOD(ifdi_led_func, em_if_led_func), 628 DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), 629 DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), 630 DEVMETHOD(ifdi_debug, em_if_debug), 631 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 632 DEVMETHOD_END 633 }; 634 635 static driver_t em_if_driver = { 636 "em_if", em_if_methods, sizeof(struct e1000_softc) 637 }; 638 639 static device_method_t igb_if_methods[] = { 640 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 641 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 642 DEVMETHOD(ifdi_detach, em_if_detach), 643 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 644 DEVMETHOD(ifdi_suspend, em_if_suspend), 645 DEVMETHOD(ifdi_resume, em_if_resume), 646 DEVMETHOD(ifdi_init, em_if_init), 647 DEVMETHOD(ifdi_stop, em_if_stop), 648 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 649 DEVMETHOD(ifdi_intr_enable, igb_if_intr_enable), 650 DEVMETHOD(ifdi_intr_disable, igb_if_intr_disable), 651 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 652 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 653 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 654 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 655 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 656 DEVMETHOD(ifdi_media_status, em_if_media_status), 657 DEVMETHOD(ifdi_media_change, em_if_media_change), 658 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 659 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 660 DEVMETHOD(ifdi_timer, em_if_timer), 661 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 662 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 663 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 664 DEVMETHOD(ifdi_led_func, em_if_led_func), 665 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 666 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 667 DEVMETHOD(ifdi_debug, em_if_debug), 668 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 669 #ifdef PCI_IOV 670 DEVMETHOD(ifdi_iov_init, igb_if_iov_init), 671 DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit), 672 DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add), 673 #endif 674 DEVMETHOD_END 675 }; 676 677 static driver_t igb_if_driver = { 678 "igb_if", igb_if_methods, sizeof(struct e1000_softc) 679 }; 680 681 static device_method_t igbv_if_methods[] = { 682 DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre), 683 DEVMETHOD(ifdi_attach_post, igbv_if_attach_post), 684 DEVMETHOD(ifdi_detach, em_if_detach), 685 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 686 DEVMETHOD(ifdi_suspend, em_if_suspend), 687 DEVMETHOD(ifdi_resume, em_if_resume), 688 DEVMETHOD(ifdi_init, em_if_init), 689 DEVMETHOD(ifdi_stop, em_if_stop), 690 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 691 DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable), 692 DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable), 693 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 694 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 695 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 696 DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status), 697 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 698 DEVMETHOD(ifdi_media_status, em_if_media_status), 699 DEVMETHOD(ifdi_media_change, igbv_if_media_change), 700 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 701 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 702 DEVMETHOD(ifdi_timer, em_if_timer), 703 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 704 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 705 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 706 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 707 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 708 DEVMETHOD(ifdi_debug, em_if_debug), 709 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 710 DEVMETHOD_END 711 }; 712 713 static driver_t igbv_if_driver = { 714 "igbv_if", igbv_if_methods, sizeof(struct e1000_softc) 715 }; 716 717 /********************************************************************* 718 * Tunable default values. 719 *********************************************************************/ 720 721 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 722 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 723 724 /* Allow common code without TSO */ 725 #ifndef CSUM_TSO 726 #define CSUM_TSO 0 727 #endif 728 729 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 730 "EM driver parameters"); 731 732 static int em_disable_crc_stripping = 0; 733 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 734 &em_disable_crc_stripping, 0, "Disable CRC Stripping"); 735 736 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 737 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 738 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, 739 &em_tx_int_delay_dflt, 0, "Default transmit interrupt delay in usecs"); 740 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, 741 &em_rx_int_delay_dflt, 0, "Default receive interrupt delay in usecs"); 742 743 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 744 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 745 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN, 746 &em_tx_abs_int_delay_dflt, 0, 747 "Default transmit interrupt delay limit in usecs"); 748 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, 749 &em_rx_abs_int_delay_dflt, 0, 750 "Default receive interrupt delay limit in usecs"); 751 752 static int em_smart_pwr_down = false; 753 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 754 &em_smart_pwr_down, 755 0, "Set to true to leave smart power down enabled on newer adapters"); 756 757 static bool em_unsupported_tso = false; 758 SYSCTL_BOOL(_hw_em, OID_AUTO, unsupported_tso, CTLFLAG_RDTUN, 759 &em_unsupported_tso, 0, "Allow unsupported em(4) TSO configurations"); 760 761 /* Controls whether promiscuous also shows bad packets */ 762 static int em_debug_sbp = false; 763 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, 764 "Show bad packets in promiscuous mode"); 765 766 /* Energy efficient ethernet - default to OFF */ 767 static int eee_setting = 1; 768 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, 769 "Enable Energy Efficient Ethernet"); 770 771 /* 772 * AIM: Adaptive Interrupt Moderation 773 * which means that the interrupt rate is varied over time based on the 774 * traffic for that interrupt vector 775 */ 776 static int em_enable_aim = 1; 777 SYSCTL_INT(_hw_em, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &em_enable_aim, 778 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 779 780 /* 781 ** Tuneable Interrupt rate 782 */ 783 static int em_max_interrupt_rate = EM_INTS_DEFAULT; 784 SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 785 &em_max_interrupt_rate, 0, "Maximum interrupts per second"); 786 787 /* Global used in WOL setup with multiport cards */ 788 static int global_quad_port_a = 0; 789 790 extern struct if_txrx igb_txrx; 791 extern struct if_txrx em_txrx; 792 extern struct if_txrx lem_txrx; 793 794 static struct if_shared_ctx em_sctx_init = { 795 .isc_magic = IFLIB_MAGIC, 796 .isc_q_align = PAGE_SIZE, 797 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 798 .isc_tx_maxsegsize = PAGE_SIZE, 799 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 800 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 801 .isc_rx_maxsize = MJUM9BYTES, 802 .isc_rx_nsegments = 1, 803 .isc_rx_maxsegsize = MJUM9BYTES, 804 .isc_nfl = 1, 805 .isc_nrxqs = 1, 806 .isc_ntxqs = 1, 807 .isc_admin_intrcnt = 1, 808 .isc_vendor_info = em_vendor_info_array, 809 .isc_driver_version = em_driver_version, 810 .isc_driver = &em_if_driver, 811 .isc_flags = 812 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 813 814 .isc_nrxd_min = {EM_MIN_RXD}, 815 .isc_ntxd_min = {EM_MIN_TXD}, 816 .isc_nrxd_max = {EM_MAX_RXD}, 817 .isc_ntxd_max = {EM_MAX_TXD}, 818 .isc_nrxd_default = {EM_DEFAULT_RXD}, 819 .isc_ntxd_default = {EM_DEFAULT_TXD}, 820 }; 821 822 static struct if_shared_ctx igb_sctx_init = { 823 .isc_magic = IFLIB_MAGIC, 824 .isc_q_align = PAGE_SIZE, 825 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 826 .isc_tx_maxsegsize = PAGE_SIZE, 827 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 828 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 829 .isc_rx_maxsize = MJUM9BYTES, 830 .isc_rx_nsegments = 1, 831 .isc_rx_maxsegsize = MJUM9BYTES, 832 .isc_nfl = 1, 833 .isc_nrxqs = 1, 834 .isc_ntxqs = 1, 835 .isc_admin_intrcnt = 1, 836 .isc_vendor_info = igb_vendor_info_array, 837 .isc_driver_version = igb_driver_version, 838 .isc_driver = &igb_if_driver, 839 .isc_flags = 840 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 841 842 .isc_nrxd_min = {EM_MIN_RXD}, 843 .isc_ntxd_min = {EM_MIN_TXD}, 844 .isc_nrxd_max = {IGB_MAX_RXD}, 845 .isc_ntxd_max = {IGB_MAX_TXD}, 846 .isc_nrxd_default = {EM_DEFAULT_RXD}, 847 .isc_ntxd_default = {EM_DEFAULT_TXD}, 848 }; 849 850 /* 851 * igb PFs and igbv VFs share the common datapath implementation. Keep a 852 * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or 853 * interrupt modes. 854 */ 855 static struct if_shared_ctx igbv_sctx_init = { 856 .isc_magic = IFLIB_MAGIC, 857 .isc_q_align = PAGE_SIZE, 858 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 859 .isc_tx_maxsegsize = PAGE_SIZE, 860 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 861 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 862 .isc_rx_maxsize = MJUM9BYTES, 863 .isc_rx_nsegments = 1, 864 .isc_rx_maxsegsize = MJUM9BYTES, 865 .isc_nfl = 1, 866 .isc_nrxqs = 1, 867 .isc_ntxqs = 1, 868 .isc_admin_intrcnt = 1, 869 .isc_vendor_info = igbv_vendor_info_array, 870 .isc_driver_version = igb_driver_version, 871 .isc_driver = &igbv_if_driver, 872 .isc_flags = 873 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM | 874 IFLIB_IS_VF, 875 876 .isc_nrxd_min = {EM_MIN_RXD}, 877 .isc_ntxd_min = {EM_MIN_TXD}, 878 .isc_nrxd_max = {IGB_MAX_RXD}, 879 .isc_ntxd_max = {IGB_MAX_TXD}, 880 .isc_nrxd_default = {EM_DEFAULT_RXD}, 881 .isc_ntxd_default = {EM_DEFAULT_TXD}, 882 }; 883 884 /***************************************************************** 885 * 886 * Dump Registers 887 * 888 ****************************************************************/ 889 #define IGB_REGS_LEN 739 890 891 static int em_get_regs(SYSCTL_HANDLER_ARGS) 892 { 893 struct e1000_softc *sc = (struct e1000_softc *)arg1; 894 struct e1000_hw *hw = &sc->hw; 895 struct sbuf *sb; 896 u32 *regs_buff; 897 int rc; 898 uint32_t rxqid, txqid; 899 900 /* 901 * This sysctl is registered before iflib allocates the queue arrays, 902 * and remains registered while iflib tears them down. 903 */ 904 if (sc->rx_queues == NULL || sc->tx_queues == NULL) 905 return (ENXIO); 906 907 regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK); 908 memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); 909 rxqid = sc->rx_queues[0].rxr.me; 910 txqid = sc->tx_queues[0].txr.me; 911 912 rc = sysctl_wire_old_buffer(req, 0); 913 MPASS(rc == 0); 914 if (rc != 0) { 915 free(regs_buff, M_DEVBUF); 916 return (rc); 917 } 918 919 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 920 MPASS(sb != NULL); 921 if (sb == NULL) { 922 free(regs_buff, M_DEVBUF); 923 return (ENOMEM); 924 } 925 926 /* General Registers */ 927 regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); 928 regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); 929 regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); 930 regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); 931 regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); 932 regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid)); 933 regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid)); 934 regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid)); 935 regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid)); 936 regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid)); 937 regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid)); 938 regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); 939 regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid)); 940 regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid)); 941 regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid)); 942 regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid)); 943 regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid)); 944 regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid)); 945 regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); 946 regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); 947 regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); 948 regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); 949 950 sbuf_printf(sb, "General Registers\n"); 951 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 952 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 953 sbuf_printf(sb, "\tCTRL_EXT\t %08x\n\n", regs_buff[2]); 954 955 sbuf_printf(sb, "Interrupt Registers\n"); 956 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 957 958 sbuf_printf(sb, "RX Registers\n"); 959 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 960 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 961 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 962 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 963 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 964 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 965 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 966 967 sbuf_printf(sb, "TX Registers\n"); 968 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 969 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 970 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 971 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 972 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 973 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 974 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 975 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 976 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 977 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 978 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 979 980 free(regs_buff, M_DEVBUF); 981 982 #ifdef DUMP_DESCS 983 { 984 if_softc_ctx_t scctx = sc->shared; 985 struct rx_ring *rxr = &rx_que->rxr; 986 struct tx_ring *txr = &tx_que->txr; 987 int ntxd = scctx->isc_ntxd[0]; 988 int nrxd = scctx->isc_nrxd[0]; 989 int j; 990 991 for (j = 0; j < nrxd; j++) { 992 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 993 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 994 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 995 PRIx64 " Error:%d Length:%d\n", 996 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 997 } 998 999 for (j = 0; j < min(ntxd, 256); j++) { 1000 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 1001 1002 sbuf_printf(sb, 1003 "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x" 1004 " eop: %d DD=%d\n", 1005 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 1006 buf->eop != -1 ? 1007 txr->tx_base[buf->eop].upper.fields.status & 1008 E1000_TXD_STAT_DD : 0); 1009 1010 } 1011 } 1012 #endif 1013 1014 rc = sbuf_finish(sb); 1015 sbuf_delete(sb); 1016 return(rc); 1017 } 1018 1019 static void * 1020 em_register(device_t dev) 1021 { 1022 return (&em_sctx_init); 1023 } 1024 1025 static void * 1026 igb_register(device_t dev) 1027 { 1028 return (&igb_sctx_init); 1029 } 1030 1031 static void * 1032 igbv_register(device_t dev) 1033 { 1034 return (&igbv_sctx_init); 1035 } 1036 1037 static int 1038 igb_device_attach(device_t dev) 1039 { 1040 struct e1000_softc *sc; 1041 if_ctx_t ctx; 1042 int error; 1043 1044 error = iflib_device_attach(dev); 1045 if (error != 0) 1046 return (error); 1047 1048 ctx = device_get_softc(dev); 1049 sc = iflib_get_softc(ctx); 1050 (void)igb_iov_attach(sc); 1051 return (0); 1052 } 1053 1054 #ifdef PCI_IOV 1055 static int 1056 igb_device_iov_init(device_t dev, uint16_t num_vfs, 1057 const nvlist_t *params) 1058 { 1059 struct e1000_softc *sc; 1060 if_ctx_t ctx; 1061 int error; 1062 1063 ctx = device_get_softc(dev); 1064 sc = iflib_get_softc(ctx); 1065 error = igb_iov_validate(sc, num_vfs); 1066 if (error != 0) 1067 return (error); 1068 return (iflib_device_iov_init_restart(dev, num_vfs, params)); 1069 } 1070 1071 static void 1072 igb_device_iov_uninit(device_t dev) 1073 { 1074 struct e1000_softc *sc; 1075 if_ctx_t ctx; 1076 1077 ctx = device_get_softc(dev); 1078 sc = iflib_get_softc(ctx); 1079 /* 1080 * pci_iov(4) has already detached the VF devices. Tell the stop 1081 * half of iflib's restart transaction not to wait for acknowledgements 1082 * from VFs which can no longer service their mailbox vectors. 1083 */ 1084 atomic_store_rel_32(&sc->iov_teardown, 1); 1085 iflib_device_iov_uninit_restart(dev); 1086 } 1087 1088 #endif 1089 1090 static int 1091 em_set_num_queues(if_ctx_t ctx) 1092 { 1093 struct e1000_softc *sc = iflib_get_softc(ctx); 1094 int maxqueues; 1095 1096 /* Sanity check based on HW */ 1097 switch (sc->hw.mac.type) { 1098 case e1000_82576: 1099 case e1000_82580: 1100 case e1000_i350: 1101 case e1000_i354: 1102 maxqueues = 8; 1103 break; 1104 case e1000_i210: 1105 case e1000_82575: 1106 maxqueues = 4; 1107 break; 1108 case e1000_i211: 1109 case e1000_82574: 1110 maxqueues = 2; 1111 break; 1112 case e1000_vfadapt: 1113 /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */ 1114 case e1000_vfadapt_i350: 1115 maxqueues = 1; 1116 break; 1117 default: 1118 maxqueues = 1; 1119 break; 1120 } 1121 1122 return (maxqueues); 1123 } 1124 1125 #define LEM_CAPS ( \ 1126 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1127 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1128 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6) 1129 1130 #define EM_CAPS ( \ 1131 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1132 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1133 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1134 IFCAP_TSO6) 1135 1136 #define IGB_CAPS ( \ 1137 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1138 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1139 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1140 IFCAP_TSO6) 1141 1142 /* 1143 * VLAN filtering is an effective VF capability, but its policy is owned by 1144 * the PF and cannot be disabled from the VF. vlan(4) registration callbacks 1145 * are independent of this capability bit. 1146 */ 1147 #define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL) 1148 1149 void 1150 em_add_device_sysctls(struct e1000_softc *sc) 1151 { 1152 struct e1000_hw *hw; 1153 struct sysctl_oid_list *child; 1154 struct sysctl_ctx_list *ctx_list; 1155 1156 hw = &sc->hw; 1157 ctx_list = device_get_sysctl_ctx(sc->dev); 1158 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1159 1160 sc->enable_aim = em_enable_aim; 1161 SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", 1162 CTLFLAG_RW, &sc->enable_aim, 0, 1163 "Interrupt Moderation (1=normal, 2=lowlatency)"); 1164 1165 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug", 1166 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1167 em_sysctl_debug_info, "I", "Debug Information"); 1168 1169 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", 1170 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1171 em_get_rs, "I", "Dump RS indexes"); 1172 1173 if (sc->vf_ifp) { 1174 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1175 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1176 igbv_get_regs, "A", "Dump VF registers"); 1177 return; 1178 } 1179 1180 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", 1181 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1182 em_sysctl_nvm_info, "I", "NVM Information"); 1183 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", 1184 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 1185 em_sysctl_print_fw_version, "A", 1186 "Prints FW/NVM Versions"); 1187 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc", 1188 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1189 em_set_flowcntl, "I", "Flow Control"); 1190 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1191 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1192 em_get_regs, "A", "Dump Registers"); 1193 1194 if (hw->mac.type >= e1000_i350) { 1195 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac", 1196 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1197 igb_sysctl_dmac, "I", "DMA Coalesce"); 1198 } 1199 1200 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1201 "tso_tcp_flags_mask_first_segment", 1202 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1203 sc, 0, em_sysctl_tso_tcp_flags_mask, "IU", 1204 "TSO TCP flags mask for first segment"); 1205 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1206 "tso_tcp_flags_mask_middle_segment", 1207 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1208 sc, 1, em_sysctl_tso_tcp_flags_mask, "IU", 1209 "TSO TCP flags mask for middle segment"); 1210 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1211 "tso_tcp_flags_mask_last_segment", 1212 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1213 sc, 2, em_sysctl_tso_tcp_flags_mask, "IU", 1214 "TSO TCP flags mask for last segment"); 1215 } 1216 1217 /********************************************************************* 1218 * Device initialization routine 1219 * 1220 * The attach entry point is called when the driver is being loaded. 1221 * This routine identifies the type of hardware, allocates all resources 1222 * and initializes the hardware. 1223 * 1224 * return 0 on success, positive on failure 1225 *********************************************************************/ 1226 int 1227 em_if_attach_pre(if_ctx_t ctx) 1228 { 1229 struct e1000_softc *sc; 1230 if_softc_ctx_t scctx; 1231 device_t dev; 1232 struct e1000_hw *hw; 1233 struct sysctl_oid_list *child; 1234 struct sysctl_ctx_list *ctx_list; 1235 int error = 0; 1236 1237 INIT_DEBUGOUT("em_if_attach_pre: begin"); 1238 dev = iflib_get_dev(ctx); 1239 sc = iflib_get_softc(ctx); 1240 1241 if (em_max_interrupt_rate <= 0) { 1242 device_printf(dev, 1243 "Invalid max_interrupt_rate %d; using default %d\n", 1244 em_max_interrupt_rate, EM_INTS_DEFAULT); 1245 em_max_interrupt_rate = EM_INTS_DEFAULT; 1246 } 1247 1248 sc->ctx = sc->osdep.ctx = ctx; 1249 sc->dev = sc->osdep.dev = dev; 1250 scctx = sc->shared = iflib_get_softc_ctx(ctx); 1251 sc->media = iflib_get_media(ctx); 1252 hw = &sc->hw; 1253 sc->vf_ifp = 1254 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0; 1255 sc->osdep.vf = sc->vf_ifp; 1256 1257 /* Determine hardware and mac info */ 1258 em_identify_hardware(ctx); 1259 sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt; 1260 1261 /* VF sysctls are deferred until attach-post confirms MSI-X. */ 1262 ctx_list = device_get_sysctl_ctx(dev); 1263 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 1264 if (!sc->vf_ifp) 1265 em_add_device_sysctls(sc); 1266 1267 scctx->isc_tx_nsegments = EM_MAX_SCATTER; 1268 scctx->isc_nrxqsets_max = 1269 scctx->isc_ntxqsets_max = em_set_num_queues(ctx); 1270 if (bootverbose) 1271 device_printf(dev, "attach_pre capping queues at %d\n", 1272 scctx->isc_ntxqsets_max); 1273 1274 if (hw->mac.type >= igb_mac_min) { 1275 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1276 sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); 1277 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1278 sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); 1279 scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); 1280 scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); 1281 scctx->isc_txrx = &igb_txrx; 1282 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1283 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1284 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1285 scctx->isc_capabilities = scctx->isc_capenable = 1286 sc->vf_ifp ? IGBV_CAPS : IGB_CAPS; 1287 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 1288 CSUM_IP6_TCP | CSUM_IP6_UDP; 1289 if (hw->mac.type != e1000_82575) 1290 scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; 1291 /* 1292 ** Some new devices, as with ixgbe, now may 1293 ** use a different BAR, so we need to keep 1294 ** track of which is used. 1295 */ 1296 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1297 } else if (hw->mac.type >= em_mac_min) { 1298 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1299 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1300 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1301 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); 1302 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1303 scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); 1304 scctx->isc_txrx = &em_txrx; 1305 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1306 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1307 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1308 scctx->isc_capabilities = scctx->isc_capenable = EM_CAPS; 1309 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1310 CSUM_IP6_TCP | CSUM_IP6_UDP; 1311 1312 /* Disable TSO on all em(4) until ring stalls are debugged */ 1313 scctx->isc_capenable &= ~IFCAP_TSO; 1314 1315 /* 1316 * Disable TSO on SPT due to errata that downclocks DMA 1317 * performance 1318 * i218-i219 Specification Update 1.5.4.5 1319 */ 1320 if (hw->mac.type == e1000_pch_spt) 1321 scctx->isc_capenable &= ~IFCAP_TSO; 1322 1323 /* 1324 * We support MSI-X with 82574 only, but indicate to iflib(4) 1325 * that it shall give MSI at least a try with other devices. 1326 */ 1327 if (hw->mac.type == e1000_82574) { 1328 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1329 } else { 1330 scctx->isc_msix_bar = -1; 1331 scctx->isc_disable_msix = 1; 1332 } 1333 } else { 1334 scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * 1335 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1336 scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * 1337 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 1338 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1339 scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); 1340 scctx->isc_txrx = &lem_txrx; 1341 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1342 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1343 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1344 scctx->isc_capabilities = scctx->isc_capenable = LEM_CAPS; 1345 if (em_unsupported_tso) 1346 scctx->isc_capabilities |= IFCAP_TSO6; 1347 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1348 CSUM_IP6_TCP | CSUM_IP6_UDP; 1349 1350 /* Disable TSO on all lem(4) until ring stalls debugged */ 1351 scctx->isc_capenable &= ~IFCAP_TSO; 1352 1353 /* 82541ER doesn't do HW tagging */ 1354 if (hw->device_id == E1000_DEV_ID_82541ER || 1355 hw->device_id == E1000_DEV_ID_82541ER_LOM) { 1356 scctx->isc_capabilities &= ~IFCAP_VLAN_HWTAGGING; 1357 scctx->isc_capenable = scctx->isc_capabilities; 1358 } 1359 /* This is the first e1000 chip and it does not do offloads */ 1360 if (hw->mac.type == e1000_82542) { 1361 scctx->isc_capabilities &= ~(IFCAP_HWCSUM | 1362 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6 | 1363 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER | 1364 IFCAP_TSO | IFCAP_VLAN_HWTSO); 1365 scctx->isc_capenable = scctx->isc_capabilities; 1366 } 1367 /* These can't do TSO for various reasons */ 1368 if (hw->mac.type < e1000_82544 || 1369 hw->mac.type == e1000_82547 || 1370 hw->mac.type == e1000_82547_rev_2) { 1371 scctx->isc_capabilities &= 1372 ~(IFCAP_TSO |IFCAP_VLAN_HWTSO); 1373 scctx->isc_capenable = scctx->isc_capabilities; 1374 } 1375 /* XXXKB: No IPv6 before this? */ 1376 if (hw->mac.type < e1000_82545){ 1377 scctx->isc_capabilities &= ~IFCAP_HWCSUM_IPV6; 1378 scctx->isc_capenable = scctx->isc_capabilities; 1379 } 1380 /* 1381 * "PCI/PCI-X SDM 4.0" page 33 (b): 1382 * FDX requirement on these chips 1383 */ 1384 if (hw->mac.type == e1000_82547 || 1385 hw->mac.type == e1000_82547_rev_2) 1386 scctx->isc_capenable &= ~(IFCAP_HWCSUM | 1387 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6); 1388 1389 /* INTx only */ 1390 scctx->isc_msix_bar = 0; 1391 } 1392 1393 /* Setup PCI resources */ 1394 if (em_allocate_pci_resources(ctx)) { 1395 device_printf(dev, "Allocation of PCI resources failed\n"); 1396 error = ENXIO; 1397 goto err_pci; 1398 } 1399 1400 /* 1401 ** For ICH8 and family we need to 1402 ** map the flash memory, and this 1403 ** must happen after the MAC is 1404 ** identified 1405 */ 1406 if ((hw->mac.type == e1000_ich8lan) || 1407 (hw->mac.type == e1000_ich9lan) || 1408 (hw->mac.type == e1000_ich10lan) || 1409 (hw->mac.type == e1000_pchlan) || 1410 (hw->mac.type == e1000_pch2lan) || 1411 (hw->mac.type == e1000_pch_lpt)) { 1412 int rid = EM_BAR_TYPE_FLASH; 1413 sc->flash = bus_alloc_resource_any(dev, 1414 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1415 if (sc->flash == NULL) { 1416 device_printf(dev, "Mapping of Flash failed\n"); 1417 error = ENXIO; 1418 goto err_pci; 1419 } 1420 /* This is used in the shared code */ 1421 hw->flash_address = (u8 *)sc->flash; 1422 sc->osdep.flash_bus_space_tag = 1423 rman_get_bustag(sc->flash); 1424 sc->osdep.flash_bus_space_handle = 1425 rman_get_bushandle(sc->flash); 1426 } 1427 /* 1428 ** In the new SPT device flash is not a 1429 ** separate BAR, rather it is also in BAR0, 1430 ** so use the same tag and an offset handle for the 1431 ** FLASH read/write macros in the shared code. 1432 */ 1433 else if (hw->mac.type >= e1000_pch_spt) { 1434 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag; 1435 sc->osdep.flash_bus_space_handle = 1436 sc->osdep.mem_bus_space_handle + E1000_FLASH_BASE_ADDR; 1437 } 1438 1439 /* Do Shared Code initialization */ 1440 error = e1000_setup_init_funcs(hw, true); 1441 if (error) { 1442 device_printf(dev, "Setup of Shared code failed, error %d\n", 1443 error); 1444 error = ENXIO; 1445 goto err_pci; 1446 } 1447 1448 em_setup_msix(ctx); 1449 e1000_get_bus_info(hw); 1450 1451 /* 1452 * Some conventional PCI systems hang when e1000 devices use 1453 * DMA addresses above 4 GB. Keep PCI-mode DMA below that boundary 1454 * by default; PCI-X and PCIe retain 64-bit DMA. 1455 */ 1456 if (hw->bus.type == e1000_bus_type_pci) { 1457 SYSCTL_ADD_BOOL(ctx_list, child, OID_AUTO, "allow_64bit_dma", 1458 CTLFLAG_RDTUN, &sc->allow_64bit_dma, 0, 1459 "Allow 64-bit DMA in conventional PCI mode"); 1460 if (sc->allow_64bit_dma) 1461 device_printf(dev, "64-bit DMA in conventional PCI mode. " 1462 "Some chipsets are unstable.\n"); 1463 else { 1464 scctx->isc_dma_width = 32; 1465 device_printf(dev, "32-bit DMA in conventional PCI mode. " 1466 "Set dev.%s.%d.allow_64bit_dma=1 at boot to enable " 1467 "64-bit DMA if the chipset is stable with it.\n", 1468 device_get_name(dev), device_get_unit(dev)); 1469 } 1470 } 1471 1472 /* Set up some sysctls for the tunable interrupt delays */ 1473 if (hw->mac.type < igb_mac_min) { 1474 em_add_int_delay_sysctl(sc, "rx_int_delay", 1475 "receive interrupt delay in usecs", &sc->rx_int_delay, 1476 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); 1477 em_add_int_delay_sysctl(sc, "tx_int_delay", 1478 "transmit interrupt delay in usecs", &sc->tx_int_delay, 1479 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt); 1480 } 1481 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) { 1482 em_add_int_delay_sysctl(sc, "rx_abs_int_delay", 1483 "receive interrupt delay limit in usecs", 1484 &sc->rx_abs_int_delay, 1485 E1000_REGISTER(hw, E1000_RADV), em_rx_abs_int_delay_dflt); 1486 em_add_int_delay_sysctl(sc, "tx_abs_int_delay", 1487 "transmit interrupt delay limit in usecs", 1488 &sc->tx_abs_int_delay, 1489 E1000_REGISTER(hw, E1000_TADV), em_tx_abs_int_delay_dflt); 1490 } 1491 1492 hw->mac.autoneg = DO_AUTO_NEG; 1493 hw->phy.autoneg_wait_to_complete = false; 1494 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1495 1496 if (hw->mac.type < em_mac_min) { 1497 e1000_init_script_state_82541(hw, true); 1498 e1000_set_tbi_compatibility_82543(hw, true); 1499 } 1500 /* Copper options */ 1501 if (hw->phy.media_type == e1000_media_type_copper) { 1502 hw->phy.mdix = AUTO_ALL_MODES; 1503 hw->phy.disable_polarity_correction = false; 1504 hw->phy.ms_type = EM_MASTER_SLAVE; 1505 } 1506 1507 /* 1508 * Set the frame limits assuming 1509 * standard ethernet sized frames. 1510 */ 1511 scctx->isc_max_frame_size = hw->mac.max_frame_size = 1512 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 1513 1514 /* 1515 * This controls when hardware reports transmit completion 1516 * status. 1517 */ 1518 hw->mac.report_tx_early = 1; 1519 1520 /* Allocate multicast array memory. */ 1521 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 1522 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 1523 if (sc->mta == NULL) { 1524 device_printf(dev, 1525 "Can not allocate multicast setup array\n"); 1526 error = ENOMEM; 1527 goto err_late; 1528 } 1529 1530 /* Clear the IFCAP_TSO auto mask */ 1531 sc->tso_automasked = 0; 1532 1533 /* Check SOL/IDER usage on physical functions. */ 1534 if (!sc->vf_ifp && e1000_check_reset_block(hw)) 1535 device_printf(dev, 1536 "PHY reset is blocked due to SOL/IDER session.\n"); 1537 1538 /* Sysctl for setting Energy Efficient Ethernet */ 1539 if (!sc->vf_ifp) { 1540 if (hw->mac.type < igb_mac_min) 1541 hw->dev_spec.ich8lan.eee_disable = eee_setting; 1542 else 1543 hw->dev_spec._82575.eee_disable = eee_setting; 1544 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", 1545 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1546 em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); 1547 } 1548 1549 /* 1550 ** Start from a known state, this is 1551 ** important in reading the nvm and 1552 ** mac from that. 1553 */ 1554 error = e1000_reset_hw(hw); 1555 if (sc->vf_ifp) { 1556 atomic_store_rel_32(&sc->vf_mbx_ready, 1557 error == E1000_SUCCESS); 1558 if (error != E1000_SUCCESS) 1559 igbv_log_reset_failure(sc, error, true); 1560 } 1561 1562 /* Make sure a PF has a good EEPROM before we read from it. */ 1563 if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) { 1564 /* 1565 ** Some PCI-E parts fail the first check due to 1566 ** the link being in sleep state, call it again, 1567 ** if it fails a second time its a real issue. 1568 */ 1569 if (e1000_validate_nvm_checksum(hw) < 0) { 1570 device_printf(dev, 1571 "The EEPROM Checksum Is Not Valid\n"); 1572 error = EIO; 1573 goto err_late; 1574 } 1575 } 1576 1577 /* Copy the permanent MAC address out of the EEPROM */ 1578 if (e1000_read_mac_addr(hw) < 0) { 1579 device_printf(dev, 1580 "EEPROM read error while reading MAC address\n"); 1581 error = EIO; 1582 goto err_late; 1583 } 1584 1585 if (!em_is_valid_ether_addr(hw->mac.addr)) { 1586 if (sc->vf_ifp) { 1587 device_printf(dev, 1588 "PF did not assign a MAC address; using a " 1589 "locally generated address\n"); 1590 ether_gen_addr(iflib_get_ifp(ctx), 1591 (struct ether_addr *)hw->mac.addr); 1592 } else { 1593 device_printf(dev, "Invalid MAC address\n"); 1594 error = EIO; 1595 goto err_late; 1596 } 1597 } 1598 1599 if (!sc->vf_ifp) { 1600 /* Save NVM versions while holding the IFLIB context lock. */ 1601 em_fw_version_locked(ctx); 1602 em_print_fw_version(sc); 1603 } 1604 1605 /* 1606 * Get Wake-on-Lan and Management info for later use 1607 */ 1608 if (!sc->vf_ifp) { 1609 em_get_wakeup(ctx); 1610 1611 /* Enable only WOL MAGIC by default. */ 1612 scctx->isc_capenable &= ~IFCAP_WOL; 1613 if (sc->wol != 0) 1614 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 1615 } 1616 1617 iflib_set_mac(ctx, hw->mac.addr); 1618 1619 return (0); 1620 1621 err_late: 1622 em_release_hw_control(sc); 1623 err_pci: 1624 em_free_pci_resources(ctx); 1625 free(sc->mta, M_DEVBUF); 1626 sc->mta = NULL; 1627 1628 return (error); 1629 } 1630 1631 int 1632 em_if_attach_post(if_ctx_t ctx) 1633 { 1634 struct e1000_softc *sc = iflib_get_softc(ctx); 1635 struct e1000_hw *hw = &sc->hw; 1636 int error = 0; 1637 1638 /* Setup OS specific network interface */ 1639 error = em_setup_interface(ctx); 1640 if (error != 0) { 1641 device_printf(sc->dev, "Interface setup failed: %d\n", error); 1642 goto err_late; 1643 } 1644 1645 if (sc->vf_ifp) 1646 (void)igbv_reset(ctx); 1647 else 1648 em_reset(ctx); 1649 1650 /* Initialize statistics */ 1651 if (sc->vf_ifp) 1652 em_initialize_vf_stats(sc); 1653 else 1654 sc->ustats.stats = (struct e1000_hw_stats){}; 1655 1656 em_update_stats_counters(sc); 1657 atomic_readandclear_32(&sc->stats_pending); 1658 hw->mac.get_link_status = 1; 1659 if (sc->vf_ifp) 1660 igbv_if_update_admin_status(ctx); 1661 else 1662 em_if_update_admin_status(ctx); 1663 em_add_hw_stats(sc); 1664 1665 /* Non-AMT based hardware can now take control from firmware */ 1666 if (sc->has_manage && !sc->has_amt) 1667 em_get_hw_control(sc); 1668 1669 INIT_DEBUGOUT("em_if_attach_post: end"); 1670 1671 return (0); 1672 1673 err_late: 1674 /* 1675 * Upon em_if_attach_post() error, iflib calls em_if_detach() to 1676 * free resources 1677 */ 1678 return (error); 1679 } 1680 1681 /********************************************************************* 1682 * Device removal routine 1683 * 1684 * The detach entry point is called when the driver is being removed. 1685 * This routine stops the adapter and deallocates all the resources 1686 * that were allocated for driver operation. 1687 * 1688 * return 0 on success, positive on failure 1689 *********************************************************************/ 1690 static int 1691 em_if_detach(if_ctx_t ctx) 1692 { 1693 struct e1000_softc *sc = iflib_get_softc(ctx); 1694 1695 INIT_DEBUGOUT("em_if_detach: begin"); 1696 1697 igb_iov_detach(sc); 1698 if (sc->vf_ifp) { 1699 igbv_queue_retry_detach(sc); 1700 igbv_mbx_retry_detach(sc); 1701 } else { 1702 e1000_phy_hw_reset(&sc->hw); 1703 } 1704 1705 em_release_manageability(sc); 1706 em_release_hw_control(sc); 1707 em_free_pci_resources(ctx); 1708 free(sc->mta, M_DEVBUF); 1709 sc->mta = NULL; 1710 1711 return (0); 1712 } 1713 1714 /********************************************************************* 1715 * 1716 * Shutdown entry point 1717 * 1718 **********************************************************************/ 1719 1720 static int 1721 em_if_shutdown(if_ctx_t ctx) 1722 { 1723 return em_if_suspend(ctx); 1724 } 1725 1726 /* 1727 * Suspend/resume device methods. 1728 */ 1729 static int 1730 em_if_suspend(if_ctx_t ctx) 1731 { 1732 struct e1000_softc *sc = iflib_get_softc(ctx); 1733 1734 if (sc->vf_ifp) { 1735 igbv_queue_retry_stop(sc); 1736 igbv_mbx_retry_stop(sc); 1737 } 1738 em_release_manageability(sc); 1739 em_release_hw_control(sc); 1740 em_enable_wakeup(ctx); 1741 return (0); 1742 } 1743 1744 static int 1745 em_if_resume(if_ctx_t ctx) 1746 { 1747 struct e1000_softc *sc = iflib_get_softc(ctx); 1748 1749 if (sc->hw.mac.type == e1000_pch2lan) 1750 e1000_resume_workarounds_pchlan(&sc->hw); 1751 1752 return(0); 1753 } 1754 1755 static int 1756 em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1757 { 1758 int max_frame_size; 1759 struct e1000_softc *sc = iflib_get_softc(ctx); 1760 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1761 1762 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1763 1764 switch (sc->hw.mac.type) { 1765 case e1000_82571: 1766 case e1000_82572: 1767 case e1000_ich9lan: 1768 case e1000_ich10lan: 1769 case e1000_pch2lan: 1770 case e1000_pch_lpt: 1771 case e1000_pch_spt: 1772 case e1000_pch_cnp: 1773 case e1000_pch_tgp: 1774 case e1000_pch_adp: 1775 case e1000_pch_mtp: 1776 case e1000_pch_ptp: 1777 case e1000_pch_nvp: 1778 case e1000_82574: 1779 case e1000_82583: 1780 case e1000_80003es2lan: 1781 /* 9K Jumbo Frame size */ 1782 max_frame_size = 9234; 1783 break; 1784 case e1000_pchlan: 1785 max_frame_size = 4096; 1786 break; 1787 case e1000_82542: 1788 case e1000_ich8lan: 1789 /* Adapters that do not support jumbo frames */ 1790 max_frame_size = ETHER_MAX_LEN; 1791 break; 1792 default: 1793 if (sc->hw.mac.type >= igb_mac_min) 1794 max_frame_size = IGB_MAX_FRAME_SIZE; 1795 else /* lem */ 1796 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1797 } 1798 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 1799 return (EINVAL); 1800 } 1801 1802 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 1803 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1804 return (0); 1805 } 1806 1807 /********************************************************************* 1808 * Init entry point 1809 * 1810 * This routine is used in two ways. It is used by the stack as 1811 * init entry point in network interface structure. It is also used 1812 * by the driver as a hw/sw initialization routine to get to a 1813 * consistent state. 1814 * 1815 **********************************************************************/ 1816 static void 1817 em_if_init(if_ctx_t ctx) 1818 { 1819 struct e1000_softc *sc = iflib_get_softc(ctx); 1820 if_softc_ctx_t scctx = sc->shared; 1821 if_t ifp = iflib_get_ifp(ctx); 1822 struct em_tx_queue *tx_que; 1823 int i; 1824 1825 INIT_DEBUGOUT("em_if_init: begin"); 1826 if (sc->vf_ifp) { 1827 igbv_queue_retry_prepare(sc); 1828 igbv_mbx_retry_prepare(sc); 1829 sc->vf_reset_pending = true; 1830 } 1831 1832 /* Get the latest mac address, User can use a LAA */ 1833 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1834 1835 /* 1836 * A VF restores its address only after its reset handshake establishes 1837 * CTS. The PF path programs RAR[0] directly here. 1838 */ 1839 if (!sc->vf_ifp) 1840 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1841 1842 /* 1843 * With the 82571 adapter, RAR[0] may be overwritten 1844 * when the other port is reset, we make a duplicate 1845 * in RAR[14] for that eventuality, this assures 1846 * the interface continues to function. 1847 */ 1848 if (sc->hw.mac.type == e1000_82571) { 1849 e1000_set_laa_state_82571(&sc->hw, true); 1850 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1851 E1000_RAR_ENTRIES - 1); 1852 } 1853 1854 /* Initialize the hardware */ 1855 igb_iov_reset_prepare(sc); 1856 if (sc->vf_ifp) { 1857 (void)igbv_reset(ctx); 1858 em_rebase_vf_stats(sc); 1859 } else { 1860 em_reset(ctx); 1861 } 1862 if (sc->vf_ifp && !sc->vf_queues_sanitized) { 1863 /* 1864 * Do not program or enable rings while retained queue state 1865 * might still contain a previous VF owner's DMA address. A 1866 * bounded callout retries initialization after iflib leaves the 1867 * failed initialization stopped. 1868 */ 1869 igbv_queue_retry_failed(ctx); 1870 return; 1871 } 1872 if (sc->vf_ifp && 1873 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1874 igbv_mbx_retry_failed(ctx); 1875 return; 1876 } 1877 if (sc->vf_ifp) 1878 igbv_reconcile_mac(sc, ifp); 1879 /* Re-arm a link-up transition deferred for this reset. */ 1880 if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING || 1881 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING) 1882 sc->link_state = EM_LINK_STATE_DOWN; 1883 if (sc->vf_ifp) 1884 igbv_if_update_admin_status(ctx); 1885 else 1886 em_if_update_admin_status(ctx); 1887 1888 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 1889 i++, tx_que++) { 1890 struct tx_ring *txr = &tx_que->txr; 1891 1892 txr->tx_rs_cidx = txr->tx_rs_pidx; 1893 1894 /* Initialize the last processed descriptor to be the end of 1895 * the ring, rather than the start, so that we avoid an 1896 * off-by-one error when calculating how many descriptors are 1897 * done in the credits_update function. 1898 */ 1899 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 1900 } 1901 1902 /* The VF VLAN EtherType is fixed and has no VET register. */ 1903 if (!sc->vf_ifp) 1904 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 1905 1906 /* Clear bad data from Rx FIFOs */ 1907 if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp) 1908 e1000_rx_fifo_flush_base(&sc->hw); 1909 1910 /* Configure for OS presence */ 1911 em_init_manageability(sc); 1912 1913 /* Prepare transmit descriptors and buffers */ 1914 if (sc->vf_ifp) 1915 igbv_initialize_transmit_unit(ctx); 1916 else 1917 em_initialize_transmit_unit(ctx); 1918 1919 /* 1920 * A failed VF reset has no CTS channel on which to restore mailbox 1921 * state. The reset detector schedules another complete init, which 1922 * replays these interface-owned lists after the handshake succeeds. 1923 */ 1924 em_if_multi_set(ctx); 1925 1926 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 1927 if (sc->vf_ifp) 1928 igbv_initialize_receive_unit(ctx); 1929 else 1930 em_initialize_receive_unit(ctx); 1931 1932 /* Set up VLAN support and filter. */ 1933 em_setup_vlan_hw_support(ctx); 1934 1935 /* Don't lose promiscuous settings. */ 1936 em_if_set_promisc_impl(ctx, if_getflags(ifp)); 1937 atomic_readandclear_32(&sc->promisc_pending); 1938 1939 /* Restore PF/VF pool configuration after the global reset. */ 1940 igb_iov_initialize(sc); 1941 1942 if (sc->hw.mac.ops.clear_hw_cntrs != NULL) 1943 sc->hw.mac.ops.clear_hw_cntrs(&sc->hw); 1944 1945 /* MSI-X configuration for 82574 */ 1946 if (sc->hw.mac.type == e1000_82574) { 1947 int tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 1948 1949 tmp |= E1000_CTRL_EXT_PBA_CLR; 1950 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 1951 /* Set the IVAR - interrupt vector routing. */ 1952 E1000_WRITE_REG(&sc->hw, E1000_IVAR, sc->ivars); 1953 } else if (sc->intr_type == IFLIB_INTR_MSIX) { 1954 /* Set up queue routing */ 1955 igb_configure_queues(sc); 1956 } 1957 if (sc->hw.mac.type >= igb_mac_min) 1958 igb_initialize_interrupt_rate(sc); 1959 1960 if (!sc->vf_ifp) { 1961 /* Clear pending PF interrupts and request a link check. */ 1962 E1000_READ_REG(&sc->hw, E1000_ICR); 1963 E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); 1964 } 1965 1966 /* AMT based hardware can now take control from firmware */ 1967 if (sc->has_manage && sc->has_amt) 1968 em_get_hw_control(sc); 1969 1970 /* Set Energy Efficient Ethernet */ 1971 if (sc->hw.mac.type >= igb_mac_min && 1972 sc->hw.phy.media_type == e1000_media_type_copper) { 1973 if (sc->hw.mac.type == e1000_i354) 1974 e1000_set_eee_i354(&sc->hw, true, true); 1975 else 1976 e1000_set_eee_i350(&sc->hw, true, true); 1977 } 1978 if (sc->vf_ifp) 1979 sc->vf_reset_pending = false; 1980 } 1981 1982 /* 1983 * RX publishes its byte and packet counters as one snapshot when iflib 1984 * returns descriptors to hardware. This also covers watchdog-driven RX 1985 * processing, which can run while the interrupt vector is unmasked. 1986 */ 1987 static __inline void 1988 em_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 1989 { 1990 uint64_t snapshot; 1991 u32 now_bytes, now_packets; 1992 1993 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 1994 now_bytes = snapshot >> 32; 1995 now_packets = (u32)snapshot; 1996 *bytes = now_bytes - rxr->rx_bytes_last; 1997 *packets = now_packets - rxr->rx_packets_last; 1998 rxr->rx_bytes_last = now_bytes; 1999 rxr->rx_packets_last = now_packets; 2000 } 2001 2002 /* 2003 * TX publishes its byte and packet counters as one snapshot at the doorbell, 2004 * because encapsulation can overlap the interrupt filter. The two halves 2005 * remain independent free running u32 counters, so their deltas are correct 2006 * across wrap. 2007 */ 2008 static __inline void 2009 em_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 2010 { 2011 uint64_t snapshot; 2012 u32 now_bytes, now_packets; 2013 2014 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 2015 now_bytes = snapshot >> 32; 2016 now_packets = (u32)snapshot; 2017 *bytes = now_bytes - txr->tx_bytes_last; 2018 *packets = now_packets - txr->tx_packets_last; 2019 txr->tx_bytes_last = now_bytes; 2020 txr->tx_packets_last = now_packets; 2021 } 2022 2023 /********************************************************************* 2024 * 2025 * Do Adaptive Interrupt Moderation: 2026 * - Calculate based on average size over the last interval 2027 * 2028 * Returns interrupts per second rather than a register value, so that the 2029 * caller's EM_INTS_TO_ITR()/IGB_INTS_TO_EITR() conversion applies, or zero 2030 * if the interval carried no packet to measure. 2031 * 2032 *********************************************************************/ 2033 static u32 2034 em_ring_itr(struct e1000_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 2035 u32 txpackets) 2036 { 2037 u32 newitr = 0; 2038 2039 if (txbytes && txpackets) 2040 newitr = txbytes / txpackets; 2041 if (rxbytes && rxpackets) 2042 newitr = max(newitr, rxbytes / rxpackets); 2043 2044 /* 2045 * No packet was observed, so there is no size to work from. Report no 2046 * observation and let the caller keep the rate it already has. 2047 */ 2048 if (newitr == 0) 2049 return (0); 2050 2051 newitr += 24; /* account for hardware frame, crc */ 2052 /* set an upper boundary */ 2053 newitr = min(newitr, 3000); 2054 /* Be nice to the mid range */ 2055 if ((newitr > 300) && (newitr < 1200)) 2056 newitr = (newitr / 3); 2057 else 2058 newitr = (newitr / 2); 2059 2060 /* The value above was written straight to EITR; make it a rate */ 2061 newitr = EM_AIM_DIVIDEND / newitr; 2062 2063 /* 2064 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 2065 * into the low latency end. The original was unbounded and would ask 2066 * for ~95k ints/s on minimum sized frames. There is deliberately no 2067 * floor, so jumbo traffic settles near 2.7k ints/s. 2068 */ 2069 if (sc->enable_aim == 1) 2070 newitr = min(newitr, EM_INTS_20K); 2071 else 2072 newitr = min(newitr, EM_INTS_70K); 2073 2074 return (newitr); 2075 } 2076 2077 /********************************************************************* 2078 * 2079 * Helper to calculate next (E)ITR value for AIM 2080 * 2081 *********************************************************************/ 2082 static void 2083 em_newitr(struct e1000_softc *sc, struct em_rx_queue *que, 2084 struct rx_ring *rxr) 2085 { 2086 struct e1000_hw *hw = &sc->hw; 2087 struct em_tx_queue *tx_que; 2088 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 2089 u32 newitr; 2090 int i; 2091 2092 em_aim_rx_delta(rxr, &rxbytes, &rxpackets); 2093 2094 /* 2095 * A vector can service more than one TX ring when iflib is configured 2096 * with unequal RX and TX queue counts. Sample every ring routed to 2097 * this vector rather than treating the vector as a TX queue index. 2098 */ 2099 txbytes = txpackets = 0; 2100 for (i = 0; i < sc->tx_num_queues; i++) { 2101 tx_que = &sc->tx_queues[i]; 2102 if (tx_que->msix != que->msix) 2103 continue; 2104 em_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 2105 txbytes += ringbytes; 2106 txpackets += ringpackets; 2107 } 2108 2109 /* Idle, do nothing */ 2110 if (txbytes == 0 && rxbytes == 0) 2111 return; 2112 2113 if (sc->enable_aim == 0) { 2114 newitr = em_max_interrupt_rate; 2115 } else if (sc->link_speed < SPEED_1000) { 2116 /* Use half default (4K) ITR if sub-gig */ 2117 newitr = EM_INTS_4K; 2118 } else if (!sc->vf_ifp && 2119 sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 2120 /* Want at least enough packet buffer for two frames to AIM */ 2121 newitr = em_max_interrupt_rate; 2122 } else { 2123 newitr = em_ring_itr(sc, rxbytes, rxpackets, txbytes, 2124 txpackets); 2125 /* No usable observation; leave the rate where it is */ 2126 if (newitr == 0) 2127 return; 2128 } 2129 2130 if (hw->mac.type >= igb_mac_min) { 2131 newitr = IGB_INTS_TO_EITR(newitr); 2132 2133 if (hw->mac.type == e1000_82575) 2134 newitr |= newitr << 16; 2135 else 2136 newitr |= E1000_EITR_CNT_IGNR; 2137 2138 if (newitr != que->itr_setting) { 2139 que->itr_setting = newitr; 2140 E1000_WRITE_REG(hw, E1000_EITR(que->msix), 2141 que->itr_setting); 2142 } 2143 } else { 2144 newitr = EM_INTS_TO_ITR(newitr); 2145 2146 if (newitr != que->itr_setting) { 2147 que->itr_setting = newitr; 2148 if (hw->mac.type == e1000_82574 && 2149 sc->intr_type == IFLIB_INTR_MSIX) { 2150 E1000_WRITE_REG(hw, 2151 E1000_EITR_82574(que->msix), 2152 que->itr_setting); 2153 } else { 2154 E1000_WRITE_REG(hw, E1000_ITR, 2155 que->itr_setting); 2156 } 2157 } 2158 } 2159 } 2160 2161 /********************************************************************* 2162 * 2163 * Fast Legacy/MSI Combined Interrupt Service routine 2164 * 2165 *********************************************************************/ 2166 int 2167 em_intr(void *arg) 2168 { 2169 struct e1000_softc *sc = arg; 2170 struct e1000_hw *hw = &sc->hw; 2171 struct em_rx_queue *que = &sc->rx_queues[0]; 2172 struct rx_ring *rxr = &que->rxr; 2173 if_ctx_t ctx = sc->ctx; 2174 u32 reg_icr; 2175 2176 reg_icr = E1000_READ_REG(hw, E1000_ICR); 2177 2178 /* Hot eject? */ 2179 if (reg_icr == 0xffffffff) 2180 return FILTER_STRAY; 2181 2182 /* Definitely not our interrupt. */ 2183 if (reg_icr == 0x0) 2184 return FILTER_STRAY; 2185 2186 /* 2187 * Starting with the 82571 chip, bit 31 should be used to 2188 * determine whether the interrupt belongs to us. 2189 */ 2190 if (hw->mac.type >= e1000_82571 && 2191 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 2192 return FILTER_STRAY; 2193 2194 /* 2195 * Only MSI-X interrupts have one-shot behavior by taking advantage 2196 * of the EIAC register. Thus, explicitly disable interrupts. This 2197 * also works around the MSI message reordering errata on certain 2198 * systems. 2199 */ 2200 IFDI_INTR_DISABLE(ctx); 2201 2202 /* Link status change */ 2203 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 2204 em_handle_link(ctx); 2205 2206 if (reg_icr & E1000_ICR_RXO) 2207 sc->rx_overruns++; 2208 2209 if (hw->mac.type >= e1000_82540) 2210 em_newitr(sc, que, rxr); 2211 2212 return (FILTER_SCHEDULE_THREAD); 2213 } 2214 2215 static int 2216 em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 2217 { 2218 struct e1000_softc *sc = iflib_get_softc(ctx); 2219 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 2220 2221 E1000_WRITE_REG(&sc->hw, E1000_IMS, rxq->eims); 2222 return (0); 2223 } 2224 2225 static int 2226 em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 2227 { 2228 struct e1000_softc *sc = iflib_get_softc(ctx); 2229 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 2230 2231 E1000_WRITE_REG(&sc->hw, E1000_IMS, txq->eims); 2232 return (0); 2233 } 2234 2235 static int 2236 igb_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 2237 { 2238 struct e1000_softc *sc = iflib_get_softc(ctx); 2239 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 2240 2241 E1000_WRITE_REG(&sc->hw, E1000_EIMS, rxq->eims); 2242 return (0); 2243 } 2244 2245 static int 2246 igb_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 2247 { 2248 struct e1000_softc *sc = iflib_get_softc(ctx); 2249 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 2250 2251 E1000_WRITE_REG(&sc->hw, E1000_EIMS, txq->eims); 2252 return (0); 2253 } 2254 2255 /********************************************************************* 2256 * 2257 * MSI-X RX Interrupt Service routine 2258 * 2259 **********************************************************************/ 2260 static int 2261 em_msix_que(void *arg) 2262 { 2263 struct em_rx_queue *que = arg; 2264 struct e1000_softc *sc = que->sc; 2265 struct rx_ring *rxr = &que->rxr; 2266 2267 ++que->irqs; 2268 2269 em_newitr(sc, que, rxr); 2270 2271 return (FILTER_SCHEDULE_THREAD); 2272 } 2273 2274 /********************************************************************* 2275 * 2276 * MSI-X Link Fast Interrupt Service routine 2277 * 2278 **********************************************************************/ 2279 static int 2280 em_msix_link(void *arg) 2281 { 2282 struct e1000_softc *sc = arg; 2283 u32 reg_icr; 2284 2285 ++sc->link_irq; 2286 MPASS(sc->hw.back != NULL); 2287 /* 2288 * The VF's admin vector represents mailbox and link activity. It has 2289 * no PF ICR at E1000_ICR, so process every admin-vector interrupt, 2290 * matching the igbvf misc-vector model. 2291 */ 2292 if (sc->vf_ifp) { 2293 sc->hw.mac.get_link_status = true; 2294 iflib_admin_intr_deferred(sc->ctx); 2295 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 2296 return (FILTER_HANDLED); 2297 } 2298 2299 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 2300 2301 /* 2302 * Enabling or disabling SR-IOV can briefly make PF MMIO reads return 2303 * all ones. This is not an interrupt cause; in particular, do not 2304 * turn it into a malicious-driver event. 2305 */ 2306 if (__predict_false(reg_icr == 0xffffffff)) 2307 goto rearm; 2308 2309 if (reg_icr & E1000_ICR_RXO) 2310 sc->rx_overruns++; 2311 2312 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 2313 em_handle_link(sc->ctx); 2314 if (reg_icr & E1000_ICR_MDDET) 2315 igb_iov_mdd_event(sc); 2316 if (reg_icr & E1000_ICR_VMMB) 2317 iflib_admin_intr_deferred(sc->ctx); 2318 2319 rearm: 2320 /* Re-arm unconditionally */ 2321 if (sc->hw.mac.type >= igb_mac_min) { 2322 E1000_WRITE_REG(&sc->hw, E1000_IMS, 2323 E1000_IMS_LSC | igb_iov_intr_mask(sc)); 2324 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 2325 } else if (sc->hw.mac.type == e1000_82574) { 2326 E1000_WRITE_REG(&sc->hw, E1000_IMS, 2327 E1000_IMS_LSC | E1000_IMS_OTHER); 2328 /* 2329 * Because we must read the ICR for this interrupt it may 2330 * clear other causes using autoclear, for this reason we 2331 * simply create a soft interrupt for all these vectors. 2332 */ 2333 if (reg_icr) 2334 E1000_WRITE_REG(&sc->hw, E1000_ICS, sc->ims); 2335 } else 2336 E1000_WRITE_REG(&sc->hw, E1000_IMS, E1000_IMS_LSC); 2337 2338 return (FILTER_HANDLED); 2339 } 2340 2341 static void 2342 em_handle_link(void *context) 2343 { 2344 if_ctx_t ctx = context; 2345 struct e1000_softc *sc = iflib_get_softc(ctx); 2346 2347 sc->hw.mac.get_link_status = 1; 2348 iflib_admin_intr_deferred(ctx); 2349 } 2350 2351 /********************************************************************* 2352 * 2353 * Media Ioctl callback 2354 * 2355 * This routine is called whenever the user queries the status of 2356 * the interface using ifconfig. 2357 * 2358 **********************************************************************/ 2359 static void 2360 em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 2361 { 2362 struct e1000_softc *sc = iflib_get_softc(ctx); 2363 u_char fiber_type = IFM_1000_SX; 2364 2365 INIT_DEBUGOUT("em_if_media_status: begin"); 2366 2367 iflib_admin_intr_deferred(ctx); 2368 2369 ifmr->ifm_status = IFM_AVALID; 2370 ifmr->ifm_active = IFM_ETHER; 2371 2372 if (sc->link_state == EM_LINK_STATE_DOWN || 2373 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING) { 2374 return; 2375 } 2376 2377 ifmr->ifm_status |= IFM_ACTIVE; 2378 2379 if ((sc->hw.phy.media_type == e1000_media_type_fiber) || 2380 (sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2381 if (sc->hw.mac.type == e1000_82545) 2382 fiber_type = IFM_1000_LX; 2383 ifmr->ifm_active |= fiber_type | IFM_FDX; 2384 } else { 2385 switch (sc->link_speed) { 2386 case 10: 2387 ifmr->ifm_active |= IFM_10_T; 2388 break; 2389 case 100: 2390 ifmr->ifm_active |= IFM_100_TX; 2391 break; 2392 case 1000: 2393 ifmr->ifm_active |= IFM_1000_T; 2394 break; 2395 } 2396 if (sc->link_duplex == FULL_DUPLEX) 2397 ifmr->ifm_active |= IFM_FDX; 2398 else 2399 ifmr->ifm_active |= IFM_HDX; 2400 } 2401 } 2402 2403 /********************************************************************* 2404 * 2405 * Media Ioctl callback 2406 * 2407 * This routine is called when the user changes speed/duplex using 2408 * media/mediopt option with ifconfig. 2409 * 2410 **********************************************************************/ 2411 static int 2412 em_if_media_change(if_ctx_t ctx) 2413 { 2414 struct e1000_softc *sc = iflib_get_softc(ctx); 2415 struct ifmedia *ifm = iflib_get_media(ctx); 2416 2417 INIT_DEBUGOUT("em_if_media_change: begin"); 2418 2419 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2420 return (EINVAL); 2421 2422 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2423 case IFM_AUTO: 2424 sc->hw.mac.autoneg = DO_AUTO_NEG; 2425 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2426 break; 2427 case IFM_1000_LX: 2428 case IFM_1000_SX: 2429 case IFM_1000_T: 2430 sc->hw.mac.autoneg = DO_AUTO_NEG; 2431 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2432 break; 2433 case IFM_100_TX: 2434 sc->hw.mac.autoneg = false; 2435 sc->hw.phy.autoneg_advertised = 0; 2436 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2437 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 2438 else 2439 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 2440 break; 2441 case IFM_10_T: 2442 sc->hw.mac.autoneg = false; 2443 sc->hw.phy.autoneg_advertised = 0; 2444 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2445 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 2446 else 2447 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 2448 break; 2449 default: 2450 device_printf(sc->dev, "Unsupported media type\n"); 2451 } 2452 2453 return (0); 2454 } 2455 2456 static int 2457 em_if_set_promisc(if_ctx_t ctx, int flags) 2458 { 2459 struct e1000_softc *sc; 2460 2461 sc = iflib_get_softc(ctx); 2462 if (em_if_defer_promisc(sc)) 2463 return (0); 2464 return (em_if_set_promisc_impl(ctx, flags)); 2465 } 2466 2467 static bool 2468 em_if_defer_promisc(struct e1000_softc *sc) 2469 { 2470 switch (sc->hw.mac.type) { 2471 case e1000_82576: 2472 case e1000_i350: 2473 case e1000_vfadapt: 2474 case e1000_vfadapt_i350: 2475 break; 2476 default: 2477 return (false); 2478 } 2479 2480 /* 2481 * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox 2482 * and IOV register operations later from the locked admin task. 2483 * A deferred VF mailbox rejection cannot be returned to ifconfig; the 2484 * admin task logs it instead. 2485 */ 2486 atomic_set_32(&sc->promisc_pending, 1); 2487 iflib_admin_intr_deferred(sc->ctx); 2488 return (true); 2489 } 2490 2491 int 2492 em_if_set_promisc_impl(if_ctx_t ctx, int flags) 2493 { 2494 struct e1000_softc *sc = iflib_get_softc(ctx); 2495 if_t ifp = iflib_get_ifp(ctx); 2496 enum e1000_promisc_type type; 2497 s32 error; 2498 u32 reg_rctl; 2499 int mcnt = 0; 2500 2501 if (sc->vf_ifp) { 2502 if (flags & IFF_PROMISC) 2503 type = e1000_promisc_enabled; 2504 else if (flags & IFF_ALLMULTI) 2505 type = e1000_promisc_multicast; 2506 else 2507 type = e1000_promisc_disabled; 2508 error = e1000_promisc_set_vf(&sc->hw, type); 2509 if (error != E1000_SUCCESS) { 2510 device_printf(sc->dev, 2511 "VF promiscuous-mode request failed\n"); 2512 return (EPERM); 2513 } 2514 return (0); 2515 } 2516 2517 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2518 reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE); 2519 if (flags & IFF_ALLMULTI) 2520 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 2521 else 2522 mcnt = min(if_llmaddr_count(ifp), 2523 MAX_NUM_MULTICAST_ADDRESSES); 2524 2525 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 2526 reg_rctl &= (~E1000_RCTL_MPE); 2527 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2528 2529 if (flags & IFF_PROMISC) { 2530 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2531 /* Turn this on if you want to see bad packets */ 2532 if (em_debug_sbp) 2533 reg_rctl |= E1000_RCTL_SBP; 2534 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2535 if (igb_iov_enabled(sc)) 2536 em_if_vlan_filter_enable(sc); 2537 else 2538 em_if_vlan_filter_disable(sc); 2539 } else { 2540 if (flags & IFF_ALLMULTI) { 2541 reg_rctl |= E1000_RCTL_MPE; 2542 reg_rctl &= ~E1000_RCTL_UPE; 2543 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2544 } 2545 if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx)) 2546 em_if_vlan_filter_enable(sc); 2547 } 2548 igb_iov_update_pf_vmolr(sc); 2549 igb_iov_rebuild_vlan(sc); 2550 return (0); 2551 } 2552 2553 static u_int 2554 em_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 2555 { 2556 u8 *mta = arg; 2557 2558 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 2559 return (0); 2560 2561 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 2562 2563 return (1); 2564 } 2565 2566 /********************************************************************* 2567 * Multicast Update 2568 * 2569 * This routine is called whenever multicast address list is updated. 2570 * 2571 **********************************************************************/ 2572 static void 2573 em_if_multi_set(if_ctx_t ctx) 2574 { 2575 struct e1000_softc *sc = iflib_get_softc(ctx); 2576 if_t ifp = iflib_get_ifp(ctx); 2577 u8 *mta; /* Multicast array memory */ 2578 u32 reg_rctl = 0; 2579 int mcnt = 0; 2580 2581 IOCTL_DEBUGOUT("em_set_multi: begin"); 2582 2583 mta = sc->mta; 2584 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 2585 2586 if (sc->hw.mac.type == e1000_82542 && 2587 sc->hw.revision_id == E1000_REVISION_2) { 2588 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2589 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2590 e1000_pci_clear_mwi(&sc->hw); 2591 reg_rctl |= E1000_RCTL_RST; 2592 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2593 msec_delay(5); 2594 } 2595 2596 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta); 2597 2598 if (sc->vf_ifp) { 2599 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 2600 igbv_update_uc_addr_list(sc, ifp); 2601 return; 2602 } 2603 2604 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES && 2605 !igb_iov_enabled(sc)) 2606 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 2607 2608 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2609 2610 if (if_getflags(ifp) & IFF_PROMISC) 2611 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2612 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 2613 if_getflags(ifp) & IFF_ALLMULTI) { 2614 reg_rctl |= E1000_RCTL_MPE; 2615 reg_rctl &= ~E1000_RCTL_UPE; 2616 } else 2617 reg_rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 2618 2619 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2620 2621 if (sc->hw.mac.type == e1000_82542 && 2622 sc->hw.revision_id == E1000_REVISION_2) { 2623 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 2624 reg_rctl &= ~E1000_RCTL_RST; 2625 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 2626 msec_delay(5); 2627 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2628 e1000_pci_set_mwi(&sc->hw); 2629 } 2630 igb_iov_rebuild_mta(sc); 2631 igb_iov_update_pf_vmolr(sc); 2632 } 2633 2634 /********************************************************************* 2635 * Timer routine 2636 * 2637 * This routine schedules em_if_update_admin_status() to check for 2638 * link status and to gather statistics as well as to perform some 2639 * controller-specific hardware patting. 2640 * 2641 **********************************************************************/ 2642 static void 2643 em_if_timer(if_ctx_t ctx, uint16_t qid) 2644 { 2645 struct e1000_softc *sc; 2646 2647 if (qid != 0) 2648 return; 2649 2650 sc = iflib_get_softc(ctx); 2651 atomic_set_32(&sc->stats_pending, 1); 2652 iflib_admin_intr_deferred(ctx); 2653 } 2654 2655 static void 2656 em_if_update_admin_status(if_ctx_t ctx) 2657 { 2658 struct e1000_softc *sc = iflib_get_softc(ctx); 2659 struct e1000_hw *hw = &sc->hw; 2660 device_t dev = iflib_get_dev(ctx); 2661 u32 link_check, thstat, ctrl; 2662 bool reset_requested = false; 2663 2664 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 2665 2666 if (atomic_readandclear_32(&sc->promisc_pending) != 0) 2667 (void)em_if_set_promisc_impl(ctx, 2668 if_getflags(iflib_get_ifp(ctx))); 2669 igb_iov_handle_mdd(sc); 2670 igb_iov_handle_mbx(sc); 2671 2672 link_check = thstat = ctrl = 0; 2673 /* Get the cached link value or read phy for real */ 2674 switch (hw->phy.media_type) { 2675 case e1000_media_type_copper: 2676 if (hw->mac.get_link_status) { 2677 if (hw->mac.type == e1000_pch_spt) 2678 msec_delay(50); 2679 /* Do the work to read phy */ 2680 e1000_check_for_link(hw); 2681 link_check = !hw->mac.get_link_status; 2682 if (link_check) /* ESB2 fix */ 2683 e1000_cfg_on_link_up(hw); 2684 } else { 2685 link_check = true; 2686 } 2687 break; 2688 case e1000_media_type_fiber: 2689 e1000_check_for_link(hw); 2690 link_check = 2691 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 2692 break; 2693 case e1000_media_type_internal_serdes: 2694 e1000_check_for_link(hw); 2695 link_check = hw->mac.serdes_has_link; 2696 break; 2697 default: 2698 break; 2699 } 2700 2701 /* Check for thermal downshift or shutdown */ 2702 if (hw->mac.type == e1000_i350) { 2703 thstat = E1000_READ_REG(hw, E1000_THSTAT); 2704 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); 2705 } 2706 2707 /* Now check for a transition */ 2708 if (link_check && 2709 (sc->link_state == EM_LINK_STATE_DOWN || 2710 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { 2711 bool reset_pending; 2712 2713 reset_pending = 2714 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING; 2715 e1000_get_speed_and_duplex(hw, &sc->link_speed, 2716 &sc->link_duplex); 2717 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2718 if ((sc->link_speed != SPEED_1000) && 2719 ((hw->mac.type == e1000_82571) || 2720 (hw->mac.type == e1000_82572))) { 2721 int tarc0; 2722 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2723 tarc0 &= ~TARC_SPEED_MODE_BIT; 2724 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2725 } 2726 if (bootverbose) 2727 device_printf(dev, "Link is up %d Mbps %s\n", 2728 sc->link_speed, 2729 ((sc->link_duplex == FULL_DUPLEX) ? 2730 "Full Duplex" : "Half Duplex")); 2731 sc->link_state = EM_LINK_STATE_UP; 2732 sc->smartspeed = 0; 2733 if (hw->mac.type == e1000_i350 && 2734 (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == 2735 E1000_CTRL_EXT_LINK_MODE_GMII && 2736 (thstat & E1000_THSTAT_LINK_THROTTLE)) 2737 device_printf(dev, "Link: thermal downshift\n"); 2738 /* Delay Link Up for Phy update */ 2739 if (((hw->mac.type == e1000_i210) || 2740 (hw->mac.type == e1000_i211)) && 2741 (hw->phy.id == I210_I_PHY_ID)) 2742 msec_delay(I210_LINK_DELAY); 2743 /* Reset if the media type changed. */ 2744 if (hw->dev_spec._82575.media_changed && 2745 hw->mac.type >= igb_mac_min) { 2746 hw->dev_spec._82575.media_changed = false; 2747 sc->flags |= IGB_MEDIA_RESET; 2748 if (igb_iov_enabled(sc)) { 2749 iflib_request_reset(ctx); 2750 iflib_admin_intr_deferred(ctx); 2751 reset_requested = true; 2752 } else 2753 em_reset(ctx); 2754 } 2755 /* Only do TSO on gigabit for older chips due to errata */ 2756 if (hw->mac.type < igb_mac_min) 2757 reset_requested = em_automask_tso(ctx); 2758 2759 if (reset_pending || reset_requested) { 2760 /* 2761 * The PHY is up, but publish it only after the TSO 2762 * capability-change reset. 2763 */ 2764 sc->link_state = EM_LINK_STATE_UP_RESET_PENDING; 2765 } else { 2766 iflib_link_state_change(ctx, LINK_STATE_UP, 2767 IF_Mbps(sc->link_speed)); 2768 } 2769 igb_iov_ping_all_vfs(sc); 2770 } else if (!link_check && 2771 (sc->link_state == EM_LINK_STATE_UP || 2772 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { 2773 bool link_was_published; 2774 bool reset_pending; 2775 2776 link_was_published = sc->link_state == EM_LINK_STATE_UP; 2777 reset_pending = 2778 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING; 2779 sc->link_speed = 0; 2780 sc->link_duplex = 0; 2781 sc->link_state = reset_pending ? 2782 EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN; 2783 if (link_was_published) 2784 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 2785 igb_iov_ping_all_vfs(sc); 2786 } 2787 /* 2788 * Mailbox, link, and timer events share this admin task. The PF 2789 * statistics sweep performs 66 MMIO reads, so run it only when the 2790 * ordinary iflib timer requests a sample rather than once per mailbox 2791 * message. Exported counters can consequently trail hardware by the 2792 * timer interval (normally 500 ms). 2793 */ 2794 if (atomic_readandclear_32(&sc->stats_pending) != 0) 2795 em_update_stats_counters(sc); 2796 2797 /* Reset LAA into RAR[0] on 82571 */ 2798 if (hw->mac.type == e1000_82571 && e1000_get_laa_state_82571(hw)) 2799 e1000_rar_set(hw, hw->mac.addr, 0); 2800 2801 if (hw->mac.type < em_mac_min) 2802 lem_smartspeed(sc); 2803 } 2804 2805 /********************************************************************* 2806 * 2807 * This routine disables all traffic on the adapter by issuing a 2808 * global reset on the MAC. 2809 * 2810 **********************************************************************/ 2811 static void 2812 em_if_stop(if_ctx_t ctx) 2813 { 2814 struct e1000_softc *sc = iflib_get_softc(ctx); 2815 2816 INIT_DEBUGOUT("em_if_stop: begin"); 2817 2818 if (sc->vf_ifp) { 2819 igbv_queue_retry_stop(sc); 2820 igbv_mbx_retry_stop(sc); 2821 } 2822 2823 /* I219 needs special flushing to avoid hangs */ 2824 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 2825 em_flush_desc_rings(sc); 2826 2827 igb_iov_reset_prepare(sc); 2828 if (!sc->vf_ifp || 2829 (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 && 2830 (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) 2831 e1000_reset_hw(&sc->hw); 2832 if (sc->vf_ifp) 2833 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 2834 if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp) 2835 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 2836 2837 if (!sc->vf_ifp) { 2838 e1000_led_off(&sc->hw); 2839 e1000_cleanup_led(&sc->hw); 2840 } else { 2841 sc->link_speed = 0; 2842 sc->link_duplex = 0; 2843 if (sc->link_state != EM_LINK_STATE_DOWN) { 2844 sc->link_state = EM_LINK_STATE_DOWN; 2845 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 2846 } 2847 } 2848 } 2849 2850 /********************************************************************* 2851 * 2852 * Determine hardware revision. 2853 * 2854 **********************************************************************/ 2855 static void 2856 em_identify_hardware(if_ctx_t ctx) 2857 { 2858 device_t dev = iflib_get_dev(ctx); 2859 struct e1000_softc *sc = iflib_get_softc(ctx); 2860 2861 /* Make sure our PCI config space has the necessary stuff set */ 2862 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2863 2864 /* Save off the information about this board */ 2865 sc->hw.vendor_id = pci_get_vendor(dev); 2866 sc->hw.device_id = pci_get_device(dev); 2867 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2868 sc->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 2869 sc->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 2870 2871 /* Do Shared Code Init and Setup */ 2872 if (e1000_set_mac_type(&sc->hw)) { 2873 device_printf(dev, "Setup init failure\n"); 2874 return; 2875 } 2876 2877 /* 2878 * Function type comes from the selected iflib shared context, not from 2879 * enum ordering. Keep the detected MAC type as an independent check 2880 * that the igb/igbv probe tables selected the right policy. 2881 */ 2882 KASSERT(sc->vf_ifp == 2883 (sc->hw.mac.type == e1000_vfadapt || 2884 sc->hw.mac.type == e1000_vfadapt_i350), 2885 ("%s: iflib function type and MAC type disagree", __func__)); 2886 } 2887 2888 static int 2889 em_allocate_pci_resources(if_ctx_t ctx) 2890 { 2891 struct e1000_softc *sc = iflib_get_softc(ctx); 2892 device_t dev = iflib_get_dev(ctx); 2893 int rid, val; 2894 2895 rid = PCIR_BAR(0); 2896 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 2897 RF_ACTIVE); 2898 if (sc->memory == NULL) { 2899 device_printf(dev, 2900 "Unable to allocate bus resource: memory\n"); 2901 return (ENXIO); 2902 } 2903 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 2904 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 2905 #ifdef INVARIANTS 2906 sc->osdep.mem_bus_space_size = rman_get_size(sc->memory); 2907 #endif 2908 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 2909 2910 /* Only older adapters use IO mapping */ 2911 if (sc->hw.mac.type < em_mac_min && sc->hw.mac.type > e1000_82543) { 2912 /* Figure our where our IO BAR is ? */ 2913 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2914 val = pci_read_config(dev, rid, 4); 2915 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2916 break; 2917 } 2918 rid += 4; 2919 /* check for 64bit BAR */ 2920 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2921 rid += 4; 2922 } 2923 if (rid >= PCIR_CIS) { 2924 device_printf(dev, "Unable to locate IO BAR\n"); 2925 return (ENXIO); 2926 } 2927 sc->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 2928 &rid, RF_ACTIVE); 2929 if (sc->ioport == NULL) { 2930 device_printf(dev, 2931 "Unable to allocate bus resource: ioport\n"); 2932 return (ENXIO); 2933 } 2934 sc->hw.io_base = 0; 2935 sc->osdep.io_bus_space_tag = 2936 rman_get_bustag(sc->ioport); 2937 sc->osdep.io_bus_space_handle = 2938 rman_get_bushandle(sc->ioport); 2939 } 2940 2941 sc->hw.back = &sc->osdep; 2942 2943 return (0); 2944 } 2945 2946 /********************************************************************* 2947 * 2948 * Set up the MSI-X Interrupt handlers 2949 * 2950 **********************************************************************/ 2951 static int 2952 em_if_msix_intr_assign(if_ctx_t ctx, int msix) 2953 { 2954 struct e1000_softc *sc = iflib_get_softc(ctx); 2955 struct em_rx_queue *rx_que = sc->rx_queues; 2956 struct em_tx_queue *tx_que = sc->tx_queues; 2957 int error, rid, i, vector = 0, rx_vectors; 2958 char buf[16]; 2959 2960 /* First set up ring resources */ 2961 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 2962 rid = vector + 1; 2963 snprintf(buf, sizeof(buf), "rxq%d", i); 2964 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 2965 IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); 2966 if (error) { 2967 device_printf(iflib_get_dev(ctx), 2968 "Failed to allocate que int %d err: %d", 2969 i, error); 2970 sc->rx_num_queues = i + 1; 2971 goto fail; 2972 } 2973 2974 rx_que->msix = vector; 2975 2976 /* 2977 * Set the bit to enable interrupt 2978 * in E1000_IMS -- bits 20 and 21 2979 * are for RX0 and RX1, note this has 2980 * NOTHING to do with the MSI-X vector 2981 */ 2982 if (sc->hw.mac.type == e1000_82574) { 2983 rx_que->eims = 1 << (20 + i); 2984 sc->ims |= rx_que->eims; 2985 sc->ivars |= (8 | rx_que->msix) << (i * 4); 2986 } else if (sc->hw.mac.type == e1000_82575) 2987 rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; 2988 else 2989 rx_que->eims = 1 << vector; 2990 } 2991 rx_vectors = vector; 2992 2993 vector = 0; 2994 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 2995 snprintf(buf, sizeof(buf), "txq%d", i); 2996 tx_que = &sc->tx_queues[i]; 2997 iflib_softirq_alloc_generic(ctx, 2998 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 2999 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 3000 3001 tx_que->msix = (vector % sc->rx_num_queues); 3002 3003 /* 3004 * Set the bit to enable interrupt 3005 * in E1000_IMS -- bits 22 and 23 3006 * are for TX0 and TX1, note this has 3007 * NOTHING to do with the MSI-X vector 3008 */ 3009 if (sc->hw.mac.type == e1000_82574) { 3010 tx_que->eims = 1 << (22 + i); 3011 sc->ims |= tx_que->eims; 3012 sc->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); 3013 } else if (sc->hw.mac.type == e1000_82575) { 3014 tx_que->eims = E1000_EICR_TX_QUEUE0 << i; 3015 } else { 3016 tx_que->eims = 1 << i; 3017 } 3018 } 3019 3020 /* Link interrupt */ 3021 rid = rx_vectors + 1; 3022 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 3023 em_msix_link, sc, 0, "aq"); 3024 3025 if (error) { 3026 device_printf(iflib_get_dev(ctx), 3027 "Failed to register admin handler"); 3028 goto fail; 3029 } 3030 sc->linkvec = rx_vectors; 3031 if (sc->hw.mac.type < igb_mac_min) { 3032 sc->ivars |= (8 | rx_vectors) << 16; 3033 sc->ivars |= 0x80000000; 3034 /* Enable the "Other" interrupt type for link status change */ 3035 sc->ims |= E1000_IMS_OTHER; 3036 } 3037 3038 return (0); 3039 fail: 3040 iflib_irq_free(ctx, &sc->irq); 3041 rx_que = sc->rx_queues; 3042 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 3043 iflib_irq_free(ctx, &rx_que->que_irq); 3044 return (error); 3045 } 3046 3047 static void 3048 igb_configure_queues(struct e1000_softc *sc) 3049 { 3050 struct e1000_hw *hw = &sc->hw; 3051 struct em_rx_queue *rx_que; 3052 struct em_tx_queue *tx_que; 3053 u32 tmp, ivar = 0; 3054 3055 /* 3056 * Queue ownership can change when SR-IOV is enabled or disabled. 3057 * Rebuild the interrupt mask for the current layout instead of 3058 * retaining vectors from a previous initialization. 3059 */ 3060 sc->que_mask = 0; 3061 sc->link_mask = 0; 3062 3063 /* GPIE controls the PF interrupt block and is not in the VF BAR. */ 3064 if (!sc->vf_ifp && hw->mac.type != e1000_82575) 3065 E1000_WRITE_REG(hw, E1000_GPIE, 3066 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 3067 E1000_GPIE_PBA | E1000_GPIE_NSICR); 3068 3069 /* Turn on MSI-X */ 3070 switch (hw->mac.type) { 3071 case e1000_82580: 3072 case e1000_i350: 3073 case e1000_i354: 3074 case e1000_i210: 3075 case e1000_i211: 3076 case e1000_vfadapt: 3077 case e1000_vfadapt_i350: 3078 /* RX entries */ 3079 for (int i = 0; i < sc->rx_num_queues; i++) { 3080 uint32_t index, qid; 3081 3082 rx_que = &sc->rx_queues[i]; 3083 qid = rx_que->rxr.me; 3084 index = qid >> 1; 3085 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 3086 if (qid & 1) { 3087 ivar &= 0xFF00FFFF; 3088 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 3089 16; 3090 } else { 3091 ivar &= 0xFFFFFF00; 3092 ivar |= rx_que->msix | E1000_IVAR_VALID; 3093 } 3094 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 3095 sc->que_mask |= rx_que->eims; 3096 } 3097 /* TX entries */ 3098 for (int i = 0; i < sc->tx_num_queues; i++) { 3099 uint32_t index, qid; 3100 3101 tx_que = &sc->tx_queues[i]; 3102 qid = tx_que->txr.me; 3103 index = qid >> 1; 3104 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 3105 if (qid & 1) { 3106 ivar &= 0x00FFFFFF; 3107 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 3108 24; 3109 } else { 3110 ivar &= 0xFFFF00FF; 3111 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 3112 8; 3113 } 3114 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 3115 sc->que_mask |= tx_que->eims; 3116 } 3117 3118 /* And for the link interrupt */ 3119 if (sc->vf_ifp) { 3120 /* 3121 * VTIVAR_MISC maps the VF mailbox in bits 7:0. 3122 * The PF IVAR_MISC maps other causes in bits 15:8. 3123 */ 3124 ivar = sc->linkvec | E1000_IVAR_VALID; 3125 } else 3126 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 3127 sc->link_mask = 1 << sc->linkvec; 3128 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 3129 break; 3130 case e1000_82576: 3131 /* RX entries */ 3132 for (int i = 0; i < sc->rx_num_queues; i++) { 3133 uint32_t index, qid; 3134 3135 rx_que = &sc->rx_queues[i]; 3136 qid = rx_que->rxr.me; 3137 index = qid & 0x7; /* Each IVAR has two entries */ 3138 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 3139 if (qid < 8) { 3140 ivar &= 0xFFFFFF00; 3141 ivar |= rx_que->msix | E1000_IVAR_VALID; 3142 } else { 3143 ivar &= 0xFF00FFFF; 3144 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 3145 16; 3146 } 3147 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 3148 sc->que_mask |= rx_que->eims; 3149 } 3150 /* TX entries */ 3151 for (int i = 0; i < sc->tx_num_queues; i++) { 3152 uint32_t index, qid; 3153 3154 tx_que = &sc->tx_queues[i]; 3155 qid = tx_que->txr.me; 3156 index = qid & 0x7; /* Each IVAR has two entries */ 3157 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 3158 if (qid < 8) { 3159 ivar &= 0xFFFF00FF; 3160 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 3161 8; 3162 } else { 3163 ivar &= 0x00FFFFFF; 3164 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 3165 24; 3166 } 3167 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 3168 sc->que_mask |= tx_que->eims; 3169 } 3170 3171 /* And for the link interrupt */ 3172 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 3173 sc->link_mask = 1 << sc->linkvec; 3174 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 3175 break; 3176 3177 case e1000_82575: 3178 /* enable MSI-X support*/ 3179 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 3180 tmp |= E1000_CTRL_EXT_PBA_CLR; 3181 /* Auto-Mask interrupts upon ICR read. */ 3182 tmp |= E1000_CTRL_EXT_EIAME; 3183 tmp |= E1000_CTRL_EXT_IRCA; 3184 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 3185 3186 /* Queues */ 3187 for (int i = 0; i < sc->rx_num_queues; i++) { 3188 rx_que = &sc->rx_queues[i]; 3189 tmp = E1000_EICR_RX_QUEUE0 << i; 3190 tmp |= E1000_EICR_TX_QUEUE0 << i; 3191 rx_que->eims = tmp; 3192 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), i, 3193 rx_que->eims); 3194 sc->que_mask |= rx_que->eims; 3195 } 3196 3197 /* Link */ 3198 E1000_WRITE_REG(hw, E1000_MSIXBM(sc->linkvec), 3199 E1000_EIMS_OTHER); 3200 sc->link_mask |= E1000_EIMS_OTHER; 3201 default: 3202 break; 3203 } 3204 3205 return; 3206 } 3207 3208 static void 3209 igb_initialize_interrupt_rate(struct e1000_softc *sc) 3210 { 3211 struct e1000_hw *hw = &sc->hw; 3212 struct em_rx_queue *rx_que; 3213 u32 newitr; 3214 3215 newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); 3216 if (hw->mac.type == e1000_82575) 3217 newitr |= newitr << 16; 3218 else 3219 newitr |= E1000_EITR_CNT_IGNR; 3220 3221 for (int i = 0; i < sc->rx_num_queues; i++) { 3222 rx_que = &sc->rx_queues[i]; 3223 rx_que->itr_setting = newitr; 3224 E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), 3225 rx_que->itr_setting); 3226 } 3227 if (sc->intr_type == IFLIB_INTR_MSIX) 3228 E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr); 3229 } 3230 3231 static void 3232 em_free_pci_resources(if_ctx_t ctx) 3233 { 3234 struct e1000_softc *sc = iflib_get_softc(ctx); 3235 struct em_rx_queue *que = sc->rx_queues; 3236 device_t dev = iflib_get_dev(ctx); 3237 3238 /* Release all MSI-X queue resources */ 3239 if (sc->intr_type == IFLIB_INTR_MSIX) 3240 iflib_irq_free(ctx, &sc->irq); 3241 3242 if (que != NULL) { 3243 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 3244 iflib_irq_free(ctx, &que->que_irq); 3245 } 3246 } 3247 3248 if (sc->memory != NULL) { 3249 bus_release_resource(dev, SYS_RES_MEMORY, 3250 rman_get_rid(sc->memory), sc->memory); 3251 sc->memory = NULL; 3252 } 3253 3254 if (sc->flash != NULL) { 3255 bus_release_resource(dev, SYS_RES_MEMORY, 3256 rman_get_rid(sc->flash), sc->flash); 3257 sc->flash = NULL; 3258 } 3259 3260 if (sc->ioport != NULL) { 3261 bus_release_resource(dev, SYS_RES_IOPORT, 3262 rman_get_rid(sc->ioport), sc->ioport); 3263 sc->ioport = NULL; 3264 } 3265 } 3266 3267 /* Set up MSI or MSI-X */ 3268 static int 3269 em_setup_msix(if_ctx_t ctx) 3270 { 3271 struct e1000_softc *sc = iflib_get_softc(ctx); 3272 3273 if (sc->hw.mac.type == e1000_82574) { 3274 em_enable_vectors_82574(ctx); 3275 } 3276 return (0); 3277 } 3278 3279 /********************************************************************* 3280 * 3281 * Workaround for SmartSpeed on 82541 and 82547 controllers 3282 * 3283 **********************************************************************/ 3284 static void 3285 lem_smartspeed(struct e1000_softc *sc) 3286 { 3287 u16 phy_tmp; 3288 3289 if (sc->link_state == EM_LINK_STATE_UP || 3290 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING || 3291 (sc->hw.phy.type != e1000_phy_igp) || 3292 sc->hw.mac.autoneg == 0 || 3293 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3294 return; 3295 3296 if (sc->smartspeed == 0) { 3297 /* If Master/Slave config fault is asserted twice, 3298 * we assume back-to-back */ 3299 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 3300 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3301 return; 3302 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 3303 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3304 e1000_read_phy_reg(&sc->hw, 3305 PHY_1000T_CTRL, &phy_tmp); 3306 if(phy_tmp & CR_1000T_MS_ENABLE) { 3307 phy_tmp &= ~CR_1000T_MS_ENABLE; 3308 e1000_write_phy_reg(&sc->hw, 3309 PHY_1000T_CTRL, phy_tmp); 3310 sc->smartspeed++; 3311 if(sc->hw.mac.autoneg && 3312 !e1000_copper_link_autoneg(&sc->hw) && 3313 !e1000_read_phy_reg(&sc->hw, 3314 PHY_CONTROL, &phy_tmp)) { 3315 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3316 MII_CR_RESTART_AUTO_NEG); 3317 e1000_write_phy_reg(&sc->hw, 3318 PHY_CONTROL, phy_tmp); 3319 } 3320 } 3321 } 3322 return; 3323 } else if(sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3324 /* If still no link, perhaps using 2/3 pair cable */ 3325 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 3326 phy_tmp |= CR_1000T_MS_ENABLE; 3327 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 3328 if(sc->hw.mac.autoneg && 3329 !e1000_copper_link_autoneg(&sc->hw) && 3330 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 3331 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3332 MII_CR_RESTART_AUTO_NEG); 3333 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 3334 } 3335 } 3336 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3337 if(sc->smartspeed++ == EM_SMARTSPEED_MAX) 3338 sc->smartspeed = 0; 3339 } 3340 3341 /********************************************************************* 3342 * 3343 * Initialize the DMA Coalescing feature 3344 * 3345 **********************************************************************/ 3346 static void 3347 igb_init_dmac(struct e1000_softc *sc, u32 pba) 3348 { 3349 device_t dev = sc->dev; 3350 struct e1000_hw *hw = &sc->hw; 3351 u32 dmac, reg = ~E1000_DMACR_DMAC_EN; 3352 u16 hwm; 3353 u16 max_frame_size; 3354 3355 KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF", 3356 __func__)); 3357 3358 if (hw->mac.type == e1000_i211) 3359 return; 3360 3361 /* 3362 * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve 3363 * the configured value so it can be restored after IOV is disabled. 3364 */ 3365 if (igb_iov_enabled(sc)) { 3366 if (hw->mac.type > e1000_82580) 3367 E1000_WRITE_REG(hw, E1000_DMACR, 0); 3368 return; 3369 } 3370 3371 max_frame_size = sc->shared->isc_max_frame_size; 3372 if (hw->mac.type > e1000_82580) { 3373 3374 if (sc->dmac == 0) { /* Disabling it */ 3375 E1000_WRITE_REG(hw, E1000_DMACR, reg); 3376 return; 3377 } else 3378 device_printf(dev, "DMA Coalescing enabled\n"); 3379 3380 /* Set starting threshold */ 3381 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); 3382 3383 hwm = 64 * pba - max_frame_size / 16; 3384 if (hwm < 64 * (pba - 6)) 3385 hwm = 64 * (pba - 6); 3386 reg = E1000_READ_REG(hw, E1000_FCRTC); 3387 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 3388 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 3389 & E1000_FCRTC_RTH_COAL_MASK); 3390 E1000_WRITE_REG(hw, E1000_FCRTC, reg); 3391 3392 3393 dmac = pba - max_frame_size / 512; 3394 if (dmac < pba - 10) 3395 dmac = pba - 10; 3396 reg = E1000_READ_REG(hw, E1000_DMACR); 3397 reg &= ~E1000_DMACR_DMACTHR_MASK; 3398 reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT) 3399 & E1000_DMACR_DMACTHR_MASK); 3400 3401 /* transition to L0x or L1 if available..*/ 3402 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 3403 3404 /* Check if status is 2.5Gb backplane connection 3405 * before configuration of watchdog timer, which is 3406 * in msec values in 12.8usec intervals 3407 * watchdog timer= msec values in 32usec intervals 3408 * for non 2.5Gb connection 3409 */ 3410 if (hw->mac.type == e1000_i354) { 3411 int status = E1000_READ_REG(hw, E1000_STATUS); 3412 if ((status & E1000_STATUS_2P5_SKU) && 3413 (!(status & E1000_STATUS_2P5_SKU_OVER))) 3414 reg |= ((sc->dmac * 5) >> 6); 3415 else 3416 reg |= (sc->dmac >> 5); 3417 } else { 3418 reg |= (sc->dmac >> 5); 3419 } 3420 3421 E1000_WRITE_REG(hw, E1000_DMACR, reg); 3422 3423 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); 3424 3425 /* Set the interval before transition */ 3426 reg = E1000_READ_REG(hw, E1000_DMCTLX); 3427 if (hw->mac.type == e1000_i350) 3428 reg |= IGB_DMCTLX_DCFLUSH_DIS; 3429 /* 3430 ** in 2.5Gb connection, TTLX unit is 0.4 usec 3431 ** which is 0x4*2 = 0xA. But delay is still 4 usec 3432 */ 3433 if (hw->mac.type == e1000_i354) { 3434 int status = E1000_READ_REG(hw, E1000_STATUS); 3435 if ((status & E1000_STATUS_2P5_SKU) && 3436 (!(status & E1000_STATUS_2P5_SKU_OVER))) 3437 reg |= 0xA; 3438 else 3439 reg |= 0x4; 3440 } else { 3441 reg |= 0x4; 3442 } 3443 3444 E1000_WRITE_REG(hw, E1000_DMCTLX, reg); 3445 3446 /* free space in tx packet buffer to wake from DMA coal */ 3447 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - 3448 (2 * max_frame_size)) >> 6); 3449 3450 /* make low power state decision controlled by DMA coal */ 3451 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 3452 reg &= ~E1000_PCIEMISC_LX_DECISION; 3453 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); 3454 3455 } else if (hw->mac.type == e1000_82580) { 3456 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 3457 E1000_WRITE_REG(hw, E1000_PCIEMISC, 3458 reg & ~E1000_PCIEMISC_LX_DECISION); 3459 E1000_WRITE_REG(hw, E1000_DMACR, 0); 3460 } 3461 } 3462 /********************************************************************* 3463 * The 3 following flush routines are used as a workaround in the 3464 * I219 client parts and only for them. 3465 * 3466 * em_flush_tx_ring - remove all descriptors from the tx_ring 3467 * 3468 * We want to clear all pending descriptors from the TX ring. 3469 * zeroing happens when the HW reads the regs. We assign the ring itself as 3470 * the data of the next descriptor. We don't care about the data we are about 3471 * to reset the HW. 3472 **********************************************************************/ 3473 static void 3474 em_flush_tx_ring(struct e1000_softc *sc) 3475 { 3476 struct e1000_hw *hw = &sc->hw; 3477 struct tx_ring *txr = &sc->tx_queues->txr; 3478 struct e1000_tx_desc *txd; 3479 u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; 3480 u16 size = 512; 3481 3482 tctl = E1000_READ_REG(hw, E1000_TCTL); 3483 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); 3484 3485 txd = &txr->tx_base[txr->tx_cidx_processed]; 3486 3487 /* Just use the ring as a dummy buffer addr */ 3488 txd->buffer_addr = txr->tx_paddr; 3489 txd->lower.data = htole32(txd_lower | size); 3490 txd->upper.data = 0; 3491 3492 /* flush descriptors to memory before notifying the HW */ 3493 wmb(); 3494 3495 E1000_WRITE_REG(hw, E1000_TDT(0), txr->tx_cidx_processed); 3496 mb(); 3497 usec_delay(250); 3498 } 3499 3500 /********************************************************************* 3501 * em_flush_rx_ring - remove all descriptors from the rx_ring 3502 * 3503 * Mark all descriptors in the RX ring as consumed and disable the rx ring 3504 **********************************************************************/ 3505 static void 3506 em_flush_rx_ring(struct e1000_softc *sc) 3507 { 3508 struct e1000_hw *hw = &sc->hw; 3509 u32 rctl, rxdctl; 3510 3511 rctl = E1000_READ_REG(hw, E1000_RCTL); 3512 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3513 E1000_WRITE_FLUSH(hw); 3514 usec_delay(150); 3515 3516 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 3517 /* zero the lower 14 bits (prefetch and host thresholds) */ 3518 rxdctl &= 0xffffc000; 3519 /* 3520 * update thresholds: prefetch threshold to 31, host threshold to 1 3521 * and make sure the granularity is "descriptors" and not 3522 * "cache lines" 3523 */ 3524 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); 3525 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 3526 3527 /* momentarily enable the RX ring for the changes to take effect */ 3528 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); 3529 E1000_WRITE_FLUSH(hw); 3530 usec_delay(150); 3531 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3532 } 3533 3534 /********************************************************************* 3535 * em_flush_desc_rings - remove all descriptors from the descriptor rings 3536 * 3537 * In I219, the descriptor rings must be emptied before resetting the HW 3538 * or before changing the device state to D3 during runtime (runtime PM). 3539 * 3540 * Failure to do this will cause the HW to enter a unit hang state which can 3541 * only be released by PCI reset on the device 3542 * 3543 **********************************************************************/ 3544 static void 3545 em_flush_desc_rings(struct e1000_softc *sc) 3546 { 3547 struct e1000_hw *hw = &sc->hw; 3548 device_t dev = sc->dev; 3549 u16 hang_state; 3550 u32 fext_nvm11, tdlen; 3551 3552 /* First, disable MULR fix in FEXTNVM11 */ 3553 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); 3554 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 3555 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); 3556 3557 /* do nothing if we're not in faulty state, or the queue is empty */ 3558 tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); 3559 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 3560 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 3561 return; 3562 em_flush_tx_ring(sc); 3563 3564 /* recheck, maybe the fault is caused by the rx ring */ 3565 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 3566 if (hang_state & FLUSH_DESC_REQUIRED) 3567 em_flush_rx_ring(sc); 3568 } 3569 3570 3571 /********************************************************************* 3572 * 3573 * Initialize the hardware to a configuration as specified by the 3574 * sc structure. 3575 * 3576 **********************************************************************/ 3577 static void 3578 em_reset(if_ctx_t ctx) 3579 { 3580 device_t dev = iflib_get_dev(ctx); 3581 struct e1000_softc *sc = iflib_get_softc(ctx); 3582 if_t ifp = iflib_get_ifp(ctx); 3583 struct e1000_hw *hw = &sc->hw; 3584 u32 rx_buffer_size; 3585 u32 pba; 3586 3587 INIT_DEBUGOUT("em_reset: begin"); 3588 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 3589 3590 /* Let the firmware know the OS is in control */ 3591 em_get_hw_control(sc); 3592 3593 /* Set up smart power down as default off on newer adapters. */ 3594 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 3595 hw->mac.type == e1000_82572)) { 3596 u16 phy_tmp = 0; 3597 3598 /* Speed up time to link by disabling smart power down. */ 3599 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 3600 phy_tmp &= ~IGP02E1000_PM_SPD; 3601 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 3602 } 3603 3604 /* 3605 * Packet Buffer Allocation (PBA) 3606 * Writing PBA sets the receive portion of the buffer 3607 * the remainder is used for the transmit buffer. 3608 */ 3609 switch (hw->mac.type) { 3610 /* 82547: Total Packet Buffer is 40K */ 3611 case e1000_82547: 3612 case e1000_82547_rev_2: 3613 if (hw->mac.max_frame_size > 8192) 3614 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 3615 else 3616 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 3617 break; 3618 /* 82571/82572/80003es2lan: Total Packet Buffer is 48K */ 3619 case e1000_82571: 3620 case e1000_82572: 3621 case e1000_80003es2lan: 3622 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 3623 break; 3624 /* 82573: Total Packet Buffer is 32K */ 3625 case e1000_82573: 3626 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 3627 break; 3628 /* 82574/82583: Total Packet Buffer is 40K */ 3629 case e1000_82574: 3630 case e1000_82583: 3631 if (hw->mac.max_frame_size > 8192) 3632 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 3633 else 3634 pba = E1000_PBA_32K; /* 32K for RX, 8K for Tx */ 3635 break; 3636 case e1000_ich8lan: 3637 pba = E1000_PBA_8K; 3638 break; 3639 case e1000_ich9lan: 3640 case e1000_ich10lan: 3641 /* Boost Receive side for jumbo frames */ 3642 if (hw->mac.max_frame_size > 4096) 3643 pba = E1000_PBA_14K; 3644 else 3645 pba = E1000_PBA_10K; 3646 break; 3647 case e1000_pchlan: 3648 case e1000_pch2lan: 3649 case e1000_pch_lpt: 3650 case e1000_pch_spt: 3651 case e1000_pch_cnp: 3652 case e1000_pch_tgp: 3653 case e1000_pch_adp: 3654 case e1000_pch_mtp: 3655 case e1000_pch_ptp: 3656 case e1000_pch_nvp: 3657 pba = E1000_PBA_26K; 3658 break; 3659 case e1000_82575: 3660 pba = E1000_PBA_32K; 3661 break; 3662 case e1000_82576: 3663 pba = E1000_READ_REG(hw, E1000_RXPBS); 3664 pba &= E1000_RXPBS_SIZE_MASK_82576; 3665 break; 3666 case e1000_82580: 3667 case e1000_i350: 3668 case e1000_i354: 3669 pba = E1000_READ_REG(hw, E1000_RXPBS); 3670 pba = e1000_rxpbs_adjust_82580(pba); 3671 break; 3672 case e1000_i210: 3673 case e1000_i211: 3674 pba = E1000_PBA_34K; 3675 break; 3676 default: 3677 /* Remaining devices assumed to have Packet Buffer of 64K. */ 3678 if (hw->mac.max_frame_size > 8192) 3679 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 3680 else 3681 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 3682 } 3683 3684 /* Special needs in case of Jumbo frames */ 3685 if ((hw->mac.type == e1000_82575) && (if_getmtu(ifp) > ETHERMTU)) { 3686 u32 tx_space, min_tx, min_rx; 3687 pba = E1000_READ_REG(hw, E1000_PBA); 3688 tx_space = pba >> 16; 3689 pba &= 0xffff; 3690 min_tx = (hw->mac.max_frame_size + 3691 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 3692 min_tx = roundup2(min_tx, 1024); 3693 min_tx >>= 10; 3694 min_rx = hw->mac.max_frame_size; 3695 min_rx = roundup2(min_rx, 1024); 3696 min_rx >>= 10; 3697 if (tx_space < min_tx && 3698 ((min_tx - tx_space) < pba)) { 3699 pba = pba - (min_tx - tx_space); 3700 /* 3701 * if short on rx space, rx wins 3702 * and must trump tx adjustment 3703 */ 3704 if (pba < min_rx) 3705 pba = min_rx; 3706 } 3707 E1000_WRITE_REG(hw, E1000_PBA, pba); 3708 } 3709 3710 if (hw->mac.type < igb_mac_min) 3711 E1000_WRITE_REG(hw, E1000_PBA, pba); 3712 3713 INIT_DEBUGOUT1("em_reset: pba=%dK", pba); 3714 3715 /* 3716 * These parameters control the automatic generation (Tx) and 3717 * response (Rx) to Ethernet PAUSE frames. 3718 * - High water mark should allow for at least two frames to be 3719 * received after sending an XOFF. 3720 * - Low water mark works best when it is very near the high water 3721 mark. 3722 * This allows the receiver to restart by sending XON when it has 3723 * drained a bit. Here we use an arbitrary value of 1500 which will 3724 * restart after one full frame is pulled from the buffer. There 3725 * could be several smaller frames in the buffer and if so they will 3726 * not trigger the XON until their total number reduces the buffer 3727 * by 1500. 3728 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 3729 */ 3730 rx_buffer_size = (pba & 0xffff) << 10; 3731 hw->fc.high_water = rx_buffer_size - 3732 roundup2(hw->mac.max_frame_size, 1024); 3733 hw->fc.low_water = hw->fc.high_water - 1500; 3734 3735 if (sc->fc) /* locally set flow control value? */ 3736 hw->fc.requested_mode = sc->fc; 3737 else 3738 hw->fc.requested_mode = e1000_fc_full; 3739 3740 if (hw->mac.type == e1000_80003es2lan) 3741 hw->fc.pause_time = 0xFFFF; 3742 else 3743 hw->fc.pause_time = EM_FC_PAUSE_TIME; 3744 3745 hw->fc.send_xon = true; 3746 3747 /* Device specific overrides/settings */ 3748 switch (hw->mac.type) { 3749 case e1000_pchlan: 3750 /* Workaround: no TX flow ctrl for PCH */ 3751 hw->fc.requested_mode = e1000_fc_rx_pause; 3752 hw->fc.pause_time = 0xFFFF; /* override */ 3753 if (if_getmtu(ifp) > ETHERMTU) { 3754 hw->fc.high_water = 0x3500; 3755 hw->fc.low_water = 0x1500; 3756 } else { 3757 hw->fc.high_water = 0x5000; 3758 hw->fc.low_water = 0x3000; 3759 } 3760 hw->fc.refresh_time = 0x1000; 3761 break; 3762 case e1000_pch2lan: 3763 case e1000_pch_lpt: 3764 case e1000_pch_spt: 3765 case e1000_pch_cnp: 3766 case e1000_pch_tgp: 3767 case e1000_pch_adp: 3768 case e1000_pch_mtp: 3769 case e1000_pch_ptp: 3770 case e1000_pch_nvp: 3771 hw->fc.high_water = 0x5C20; 3772 hw->fc.low_water = 0x5048; 3773 hw->fc.pause_time = 0xFFFF; 3774 hw->fc.refresh_time = 0xFFFF; 3775 /* Jumbos need adjusted PBA */ 3776 if (if_getmtu(ifp) > ETHERMTU) 3777 pba = E1000_PBA_12K; 3778 else 3779 pba = E1000_PBA_26K; 3780 E1000_WRITE_REG(hw, E1000_PBA, pba); 3781 break; 3782 case e1000_82575: 3783 case e1000_82576: 3784 /* 8-byte granularity */ 3785 hw->fc.low_water = hw->fc.high_water - 8; 3786 break; 3787 case e1000_82580: 3788 case e1000_i350: 3789 case e1000_i354: 3790 case e1000_i210: 3791 case e1000_i211: 3792 /* 16-byte granularity */ 3793 hw->fc.low_water = hw->fc.high_water - 16; 3794 break; 3795 case e1000_ich9lan: 3796 case e1000_ich10lan: 3797 if (if_getmtu(ifp) > ETHERMTU) { 3798 hw->fc.high_water = 0x2800; 3799 hw->fc.low_water = hw->fc.high_water - 8; 3800 break; 3801 } 3802 /* FALLTHROUGH */ 3803 default: 3804 if (hw->mac.type == e1000_80003es2lan) 3805 hw->fc.pause_time = 0xFFFF; 3806 break; 3807 } 3808 3809 /* I219 needs some special flushing to avoid hangs */ 3810 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 3811 em_flush_desc_rings(sc); 3812 3813 /* Issue a global reset */ 3814 e1000_reset_hw(hw); 3815 if (hw->mac.type >= igb_mac_min) { 3816 E1000_WRITE_REG(hw, E1000_WUC, 0); 3817 } else { 3818 E1000_WRITE_REG(hw, E1000_WUFC, 0); 3819 em_disable_aspm(sc); 3820 } 3821 if (sc->flags & IGB_MEDIA_RESET) { 3822 e1000_setup_init_funcs(hw, true); 3823 e1000_get_bus_info(hw); 3824 sc->flags &= ~IGB_MEDIA_RESET; 3825 } 3826 /* and a re-init */ 3827 if (e1000_init_hw(hw) < 0) { 3828 device_printf(dev, "Hardware Initialization Failed\n"); 3829 return; 3830 } 3831 if (hw->mac.type >= igb_mac_min) 3832 igb_init_dmac(sc, pba); 3833 3834 /* Save the receive packet-buffer allocation for AIM. */ 3835 sc->pba = pba; 3836 3837 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 3838 e1000_get_phy_info(hw); 3839 e1000_check_for_link(hw); 3840 } 3841 3842 /* 3843 * Initialise the RSS mapping for NICs that support multiple transmit/ 3844 * receive rings. 3845 */ 3846 3847 #define RSSKEYLEN 10 3848 static void 3849 em_initialize_rss_mapping(struct e1000_softc *sc) 3850 { 3851 uint8_t rss_key[4 * RSSKEYLEN]; 3852 uint32_t reta = 0; 3853 struct e1000_hw *hw = &sc->hw; 3854 int i; 3855 3856 /* 3857 * Configure RSS key 3858 */ 3859 arc4rand(rss_key, sizeof(rss_key), 0); 3860 for (i = 0; i < RSSKEYLEN; ++i) { 3861 uint32_t rssrk = 0; 3862 3863 rssrk = EM_RSSRK_VAL(rss_key, i); 3864 E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); 3865 } 3866 3867 /* 3868 * Configure RSS redirect table in following fashion: 3869 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 3870 */ 3871 for (i = 0; i < sizeof(reta); ++i) { 3872 uint32_t q; 3873 3874 q = (i % sc->rx_num_queues) << 7; 3875 reta |= q << (8 * i); 3876 } 3877 3878 for (i = 0; i < 32; ++i) 3879 E1000_WRITE_REG(hw, E1000_RETA(i), reta); 3880 3881 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | 3882 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3883 E1000_MRQC_RSS_FIELD_IPV4 | 3884 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | 3885 E1000_MRQC_RSS_FIELD_IPV6_EX | 3886 E1000_MRQC_RSS_FIELD_IPV6); 3887 } 3888 3889 static void 3890 igb_initialize_rss_mapping(struct e1000_softc *sc) 3891 { 3892 struct e1000_hw *hw = &sc->hw; 3893 int i; 3894 int queue_id; 3895 u32 reta; 3896 u32 rss_key[10], mrqc, shift = 0; 3897 3898 /* XXX? */ 3899 if (hw->mac.type == e1000_82575) 3900 shift = 6; 3901 3902 /* 3903 * The redirection table controls which destination 3904 * queue each bucket redirects traffic to. 3905 * Each DWORD represents four queues, with the LSB 3906 * being the first queue in the DWORD. 3907 * 3908 * This just allocates buckets to queues using round-robin 3909 * allocation. 3910 * 3911 * NOTE: It Just Happens to line up with the default 3912 * RSS allocation method. 3913 */ 3914 3915 /* Warning FM follows */ 3916 reta = 0; 3917 for (i = 0; i < 128; i++) { 3918 #ifdef RSS 3919 queue_id = rss_get_indirection_to_bucket(i); 3920 /* 3921 * If we have more queues than buckets, we'll 3922 * end up mapping buckets to a subset of the 3923 * queues. 3924 * 3925 * If we have more buckets than queues, we'll 3926 * end up instead assigning multiple buckets 3927 * to queues. 3928 * 3929 * Both are suboptimal, but we need to handle 3930 * the case so we don't go out of bounds 3931 * indexing arrays and such. 3932 */ 3933 queue_id = queue_id % sc->rx_num_queues; 3934 #else 3935 queue_id = (i % sc->rx_num_queues); 3936 #endif 3937 /* Adjust if required */ 3938 queue_id = queue_id << shift; 3939 3940 /* 3941 * The low 8 bits are for hash value (n+0); 3942 * The next 8 bits are for hash value (n+1), etc. 3943 */ 3944 reta = reta >> 8; 3945 reta = reta | ( ((uint32_t) queue_id) << 24); 3946 if ((i & 3) == 3) { 3947 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3948 reta = 0; 3949 } 3950 } 3951 3952 /* Now fill in hash table */ 3953 3954 /* 3955 * MRQC: Multiple Receive Queues Command 3956 * Set queuing to RSS control, number depends on the device. 3957 */ 3958 mrqc = E1000_MRQC_ENABLE_RSS_MQ; 3959 3960 /* XXX ew typecasting */ 3961 rss_getkey((uint8_t *) &rss_key); 3962 for (i = 0; i < 10; i++) 3963 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]); 3964 3965 /* 3966 * Configure the RSS fields to hash upon. 3967 */ 3968 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 3969 E1000_MRQC_RSS_FIELD_IPV4_TCP); 3970 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 3971 E1000_MRQC_RSS_FIELD_IPV6_TCP); 3972 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 3973 E1000_MRQC_RSS_FIELD_IPV6_UDP); 3974 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 3975 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3976 3977 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 3978 } 3979 3980 /********************************************************************* 3981 * 3982 * Setup networking device structure and register interface media. 3983 * 3984 **********************************************************************/ 3985 static int 3986 em_setup_interface(if_ctx_t ctx) 3987 { 3988 if_t ifp = iflib_get_ifp(ctx); 3989 struct e1000_softc *sc = iflib_get_softc(ctx); 3990 if_softc_ctx_t scctx = sc->shared; 3991 3992 INIT_DEBUGOUT("em_setup_interface: begin"); 3993 3994 /* Single Queue */ 3995 if (sc->tx_num_queues == 1) { 3996 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 3997 if_setsendqready(ifp); 3998 } 3999 4000 /* 4001 * Specify the media types supported by this adapter and register 4002 * callbacks to update media and link information 4003 */ 4004 if (sc->vf_ifp) { 4005 ifmedia_add(sc->media, 4006 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 4007 ifmedia_set(sc->media, 4008 IFM_ETHER | IFM_1000_T | IFM_FDX); 4009 return (0); 4010 } 4011 4012 if (sc->hw.phy.media_type == e1000_media_type_fiber || 4013 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 4014 u_char fiber_type = IFM_1000_SX; /* default type */ 4015 4016 if (sc->hw.mac.type == e1000_82545) 4017 fiber_type = IFM_1000_LX; 4018 ifmedia_add(sc->media, 4019 IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); 4020 ifmedia_add(sc->media, IFM_ETHER | fiber_type, 0, NULL); 4021 } else { 4022 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 4023 ifmedia_add(sc->media, 4024 IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 4025 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 4026 ifmedia_add(sc->media, 4027 IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 4028 if (sc->hw.phy.type != e1000_phy_ife) { 4029 ifmedia_add(sc->media, 4030 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 4031 ifmedia_add(sc->media, 4032 IFM_ETHER | IFM_1000_T, 0, NULL); 4033 } 4034 } 4035 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 4036 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 4037 return (0); 4038 } 4039 4040 static int 4041 em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 4042 int ntxqs, int ntxqsets) 4043 { 4044 struct e1000_softc *sc = iflib_get_softc(ctx); 4045 if_softc_ctx_t scctx = sc->shared; 4046 int error = E1000_SUCCESS; 4047 struct em_tx_queue *que; 4048 int i, j; 4049 4050 MPASS(sc->tx_num_queues > 0); 4051 MPASS(sc->tx_num_queues == ntxqsets); 4052 4053 /* First allocate the top level queue structs */ 4054 if (!(sc->tx_queues = 4055 (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * 4056 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 4057 device_printf(iflib_get_dev(ctx), 4058 "Unable to allocate queue memory\n"); 4059 return(ENOMEM); 4060 } 4061 4062 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 4063 /* Set up some basics */ 4064 4065 struct tx_ring *txr = &que->txr; 4066 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 4067 ("%s: misaligned TX AIM snapshot %p", __func__, 4068 &txr->tx_aim_snapshot)); 4069 txr->sc = que->sc = sc; 4070 que->me = txr->me = i; 4071 4072 /* Allocate report status array */ 4073 if (!(txr->tx_rsq = 4074 (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], 4075 M_DEVBUF, M_NOWAIT | M_ZERO))) { 4076 device_printf(iflib_get_dev(ctx), 4077 "failed to allocate rs_idxs memory\n"); 4078 error = ENOMEM; 4079 goto fail; 4080 } 4081 for (j = 0; j < scctx->isc_ntxd[0]; j++) 4082 txr->tx_rsq[j] = QIDX_INVALID; 4083 /* get the virtual and physical address of hardware queues */ 4084 txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; 4085 txr->tx_paddr = paddrs[i*ntxqs]; 4086 } 4087 4088 if (bootverbose) 4089 device_printf(iflib_get_dev(ctx), 4090 "allocated for %d tx_queues\n", sc->tx_num_queues); 4091 return (0); 4092 fail: 4093 em_if_queues_free(ctx); 4094 return (error); 4095 } 4096 4097 static int 4098 em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 4099 int nrxqs, int nrxqsets) 4100 { 4101 struct e1000_softc *sc = iflib_get_softc(ctx); 4102 int error = E1000_SUCCESS; 4103 struct em_rx_queue *que; 4104 int i; 4105 4106 MPASS(sc->rx_num_queues > 0); 4107 MPASS(sc->rx_num_queues == nrxqsets); 4108 4109 /* First allocate the top level queue structs */ 4110 if (!(sc->rx_queues = 4111 (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * 4112 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 4113 device_printf(iflib_get_dev(ctx), 4114 "Unable to allocate queue memory\n"); 4115 error = ENOMEM; 4116 goto fail; 4117 } 4118 4119 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 4120 /* Set up some basics */ 4121 struct rx_ring *rxr = &que->rxr; 4122 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 4123 ("%s: misaligned RX AIM snapshot %p", __func__, 4124 &rxr->rx_aim_snapshot)); 4125 rxr->sc = que->sc = sc; 4126 rxr->que = que; 4127 que->me = rxr->me = i; 4128 4129 /* get the virtual and physical address of hardware queues */ 4130 rxr->rx_base = 4131 (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; 4132 rxr->rx_paddr = paddrs[i*nrxqs]; 4133 } 4134 4135 if (bootverbose) 4136 device_printf(iflib_get_dev(ctx), 4137 "allocated for %d rx_queues\n", sc->rx_num_queues); 4138 4139 return (0); 4140 fail: 4141 em_if_queues_free(ctx); 4142 return (error); 4143 } 4144 4145 static void 4146 em_if_queues_free(if_ctx_t ctx) 4147 { 4148 struct e1000_softc *sc = iflib_get_softc(ctx); 4149 struct em_tx_queue *tx_que = sc->tx_queues; 4150 struct em_rx_queue *rx_que = sc->rx_queues; 4151 4152 if (tx_que != NULL) { 4153 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 4154 struct tx_ring *txr = &tx_que->txr; 4155 if (txr->tx_rsq == NULL) 4156 break; 4157 4158 free(txr->tx_rsq, M_DEVBUF); 4159 txr->tx_rsq = NULL; 4160 } 4161 free(sc->tx_queues, M_DEVBUF); 4162 sc->tx_queues = NULL; 4163 } 4164 4165 if (rx_que != NULL) { 4166 free(sc->rx_queues, M_DEVBUF); 4167 sc->rx_queues = NULL; 4168 } 4169 } 4170 4171 static u32 4172 em_legacy_txdctl(struct e1000_hw *hw) 4173 { 4174 u32 txdctl; 4175 4176 /* 4177 * Start with the established full-descriptor writeback policy. 4178 * Several generations have descriptor-queue errata for which it is 4179 * a documented workaround. The unsafe early controllers are 4180 * overridden below. 4181 */ 4182 txdctl = EM_TX_PTHRESH | (EM_TX_HTHRESH << 8) | 4183 (EM_TX_WTHRESH << 16) | E1000_TXDCTL_GRAN; 4184 4185 switch (hw->mac.type) { 4186 case e1000_82571: 4187 case e1000_82572: 4188 case e1000_82573: 4189 case e1000_82574: 4190 case e1000_82583: 4191 case e1000_80003es2lan: 4192 /* Match the Intel shared-code policy for these families. */ 4193 txdctl |= E1000_TXDCTL_COUNT_DESC; 4194 break; 4195 case e1000_ich8lan: 4196 case e1000_ich9lan: 4197 case e1000_ich10lan: 4198 case e1000_pchlan: 4199 case e1000_pch2lan: 4200 case e1000_pch_lpt: 4201 case e1000_pch_spt: 4202 case e1000_pch_cnp: 4203 case e1000_pch_tgp: 4204 case e1000_pch_adp: 4205 case e1000_pch_mtp: 4206 case e1000_pch_ptp: 4207 case e1000_pch_nvp: 4208 /* Preserve the required bit set by the integrated shared code. */ 4209 txdctl |= (1U << 22); 4210 break; 4211 case e1000_82542: 4212 case e1000_82543: 4213 case e1000_82544: 4214 /* 4215 * 82543 erratum 35 and 82544 erratum 20 require 4216 * WTHRESH=0. Leave all descriptor-control thresholds at 4217 * their reset values on these early controllers. 4218 */ 4219 txdctl = 0; 4220 break; 4221 case e1000_82540: 4222 case e1000_82545: 4223 case e1000_82545_rev_3: 4224 case e1000_82546: 4225 case e1000_82546_rev_3: 4226 case e1000_82541: 4227 case e1000_82541_rev_2: 4228 case e1000_82547: 4229 case e1000_82547_rev_2: 4230 break; 4231 default: 4232 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 4233 hw->mac.type)); 4234 break; 4235 } 4236 4237 return (txdctl); 4238 } 4239 4240 static u32 4241 igb_txdctl(struct e1000_hw *hw) 4242 { 4243 u32 pthresh; 4244 4245 switch (hw->mac.type) { 4246 case e1000_i354: 4247 pthresh = I354_TX_PTHRESH; 4248 break; 4249 case e1000_82575: 4250 case e1000_82576: 4251 case e1000_82580: 4252 case e1000_i350: 4253 case e1000_i210: 4254 case e1000_i211: 4255 case e1000_vfadapt: 4256 case e1000_vfadapt_i350: 4257 pthresh = IGB_TX_PTHRESH; 4258 break; 4259 default: 4260 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 4261 hw->mac.type)); 4262 pthresh = IGB_TX_PTHRESH; 4263 break; 4264 } 4265 4266 return (pthresh | (IGB_TX_HTHRESH << 8) | 4267 E1000_TXDCTL_QUEUE_ENABLE); 4268 } 4269 4270 /********************************************************************* 4271 * 4272 * Enable transmit unit. 4273 * 4274 **********************************************************************/ 4275 void 4276 em_initialize_transmit_rings(if_ctx_t ctx) 4277 { 4278 struct e1000_softc *sc = iflib_get_softc(ctx); 4279 if_softc_ctx_t scctx = sc->shared; 4280 struct em_tx_queue *que; 4281 struct tx_ring *txr; 4282 struct e1000_hw *hw = &sc->hw; 4283 u32 txdctl; 4284 4285 for (int i = 0; i < sc->tx_num_queues; i++) { 4286 u64 bus_addr; 4287 caddr_t offp, endp; 4288 uint32_t qid; 4289 4290 que = &sc->tx_queues[i]; 4291 txr = &que->txr; 4292 qid = txr->me; 4293 bus_addr = txr->tx_paddr; 4294 4295 /* Clear checksum offload context. */ 4296 offp = (caddr_t)txr + offsetof(struct tx_ring, csum_flags); 4297 endp = (caddr_t)(txr + 1); 4298 memset(offp, 0, endp - offp); 4299 4300 if (hw->mac.type >= igb_mac_min) { 4301 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid)); 4302 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), 4303 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE); 4304 E1000_WRITE_FLUSH(hw); 4305 } 4306 4307 /* Base and Len of TX Ring */ 4308 E1000_WRITE_REG(hw, E1000_TDLEN(qid), 4309 scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); 4310 E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32)); 4311 E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr); 4312 /* Init the HEAD/TAIL indices */ 4313 E1000_WRITE_REG(hw, E1000_TDT(qid), 0); 4314 E1000_WRITE_REG(hw, E1000_TDH(qid), 0); 4315 4316 HW_DEBUGOUT2("Base = %x, Length = %x\n", 4317 E1000_READ_REG(hw, E1000_TDBAL(qid)), 4318 E1000_READ_REG(hw, E1000_TDLEN(qid))); 4319 4320 if (hw->mac.type < igb_mac_min) 4321 txdctl = em_legacy_txdctl(hw); 4322 else 4323 txdctl = igb_txdctl(hw); 4324 4325 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl); 4326 } 4327 } 4328 4329 static void 4330 em_initialize_transmit_unit(if_ctx_t ctx) 4331 { 4332 struct e1000_softc *sc = iflib_get_softc(ctx); 4333 struct e1000_hw *hw = &sc->hw; 4334 u32 tctl, tarc, tipg = 0; 4335 4336 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 4337 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 4338 4339 em_initialize_transmit_rings(ctx); 4340 4341 /* Set the default values for the Tx Inter Packet Gap timer */ 4342 switch (hw->mac.type) { 4343 case e1000_80003es2lan: 4344 tipg = DEFAULT_82543_TIPG_IPGR1; 4345 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 4346 E1000_TIPG_IPGR2_SHIFT; 4347 break; 4348 case e1000_82542: 4349 tipg = DEFAULT_82542_TIPG_IPGT; 4350 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 4351 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 4352 break; 4353 default: 4354 if (hw->phy.media_type == e1000_media_type_fiber || 4355 hw->phy.media_type == e1000_media_type_internal_serdes) 4356 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 4357 else 4358 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 4359 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 4360 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 4361 } 4362 4363 if (hw->mac.type < igb_mac_min) { 4364 E1000_WRITE_REG(hw, E1000_TIPG, tipg); 4365 E1000_WRITE_REG(hw, E1000_TIDV, sc->tx_int_delay.value); 4366 4367 if (sc->tx_int_delay.value > 0) 4368 sc->txd_cmd |= E1000_TXD_CMD_IDE; 4369 } 4370 4371 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) 4372 E1000_WRITE_REG(hw, E1000_TADV, sc->tx_abs_int_delay.value); 4373 4374 if (hw->mac.type == e1000_82571 || hw->mac.type == e1000_82572) { 4375 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 4376 tarc |= TARC_SPEED_MODE_BIT; 4377 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 4378 } else if (hw->mac.type == e1000_80003es2lan) { 4379 /* errata: program both queues to unweighted RR */ 4380 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 4381 tarc |= 1; 4382 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 4383 tarc = E1000_READ_REG(hw, E1000_TARC(1)); 4384 tarc |= 1; 4385 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 4386 } else if (hw->mac.type == e1000_82574) { 4387 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 4388 tarc |= TARC_ERRATA_BIT; 4389 if ( sc->tx_num_queues > 1) { 4390 tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); 4391 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 4392 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 4393 } else 4394 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 4395 } 4396 4397 /* Program the Transmit Control Register */ 4398 tctl = E1000_READ_REG(hw, E1000_TCTL); 4399 tctl &= ~E1000_TCTL_CT; 4400 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 4401 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 4402 4403 if (hw->mac.type >= e1000_82571 && hw->mac.type < igb_mac_min) 4404 tctl |= E1000_TCTL_MULR; 4405 4406 /* This write will effectively turn on the transmit unit. */ 4407 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 4408 4409 /* SPT and KBL errata workarounds */ 4410 if (hw->mac.type == e1000_pch_spt) { 4411 u32 reg; 4412 reg = E1000_READ_REG(hw, E1000_IOSFPC); 4413 reg |= E1000_RCTL_RDMTS_HEX; 4414 E1000_WRITE_REG(hw, E1000_IOSFPC, reg); 4415 /* i218-i219 Specification Update 1.5.4.5 */ 4416 reg = E1000_READ_REG(hw, E1000_TARC(0)); 4417 reg &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 4418 reg |= E1000_TARC0_CB_MULTIQ_2_REQ; 4419 E1000_WRITE_REG(hw, E1000_TARC(0), reg); 4420 } 4421 } 4422 4423 /********************************************************************* 4424 * 4425 * Enable receive unit. 4426 * 4427 **********************************************************************/ 4428 #define BSIZEPKT_ROUNDUP ((1<<E1000_SRRCTL_BSIZEPKT_SHIFT)-1) 4429 4430 static u32 4431 igb_rxdctl(struct e1000_softc *sc, u32 rxdctl) 4432 { 4433 struct e1000_hw *hw; 4434 u32 mask, pthresh, wthresh; 4435 4436 hw = &sc->hw; 4437 mask = IGB_RXDCTL_THRESH_MASK; 4438 switch (hw->mac.type) { 4439 case e1000_82575: 4440 mask = IGB_82575_RXDCTL_THRESH_MASK; 4441 pthresh = IGB_RX_PTHRESH; 4442 wthresh = IGB_RX_WTHRESH; 4443 break; 4444 case e1000_82576: 4445 pthresh = IGB_RX_PTHRESH; 4446 wthresh = sc->intr_type == IFLIB_INTR_MSIX ? 4447 IGB_82576_RX_WTHRESH : IGB_RX_WTHRESH; 4448 break; 4449 case e1000_vfadapt: 4450 /* 82576 VFs always need the MSI-X writeback workaround. */ 4451 pthresh = IGB_RX_PTHRESH; 4452 wthresh = IGB_82576_RX_WTHRESH; 4453 break; 4454 case e1000_i354: 4455 pthresh = I354_RX_PTHRESH; 4456 wthresh = IGB_RX_WTHRESH; 4457 break; 4458 case e1000_82580: 4459 case e1000_i350: 4460 case e1000_i210: 4461 case e1000_i211: 4462 case e1000_vfadapt_i350: 4463 pthresh = IGB_RX_PTHRESH; 4464 wthresh = IGB_RX_WTHRESH; 4465 break; 4466 default: 4467 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 4468 hw->mac.type)); 4469 pthresh = IGB_RX_PTHRESH; 4470 wthresh = IGB_RX_WTHRESH; 4471 break; 4472 } 4473 4474 rxdctl &= ~mask; 4475 rxdctl |= pthresh | (IGB_RX_HTHRESH << 8) | 4476 (wthresh << 16) | E1000_RXDCTL_QUEUE_ENABLE; 4477 return (rxdctl); 4478 } 4479 4480 void 4481 igb_initialize_receive_rings(if_ctx_t ctx, bool drop) 4482 { 4483 struct e1000_softc *sc = iflib_get_softc(ctx); 4484 if_softc_ctx_t scctx = sc->shared; 4485 struct e1000_hw *hw = &sc->hw; 4486 struct em_rx_queue *que; 4487 u32 srrctl; 4488 4489 srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 4490 E1000_SRRCTL_BSIZEPKT_SHIFT; 4491 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 4492 if (drop) 4493 srrctl |= E1000_SRRCTL_DROP_EN; 4494 4495 for (int i = 0; i < sc->rx_num_queues; i++) { 4496 struct rx_ring *rxr; 4497 u64 bus_addr; 4498 u32 rxdctl; 4499 uint32_t qid; 4500 4501 que = &sc->rx_queues[i]; 4502 rxr = &que->rxr; 4503 bus_addr = rxr->rx_paddr; 4504 qid = rxr->me; 4505 4506 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid)); 4507 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), 4508 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE); 4509 E1000_WRITE_FLUSH(hw); 4510 4511 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 4512 scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); 4513 E1000_WRITE_REG(hw, E1000_RDBAH(qid), 4514 (uint32_t)(bus_addr >> 32)); 4515 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr); 4516 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 4517 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 4518 E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl); 4519 4520 rxdctl = igb_rxdctl(sc, rxdctl); 4521 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl); 4522 } 4523 } 4524 4525 static bool 4526 em_integrated_jumbo_rx(struct e1000_hw *hw) 4527 { 4528 switch (hw->mac.type) { 4529 case e1000_ich9lan: 4530 case e1000_ich10lan: 4531 case e1000_pchlan: 4532 case e1000_pch2lan: 4533 case e1000_pch_lpt: 4534 case e1000_pch_spt: 4535 case e1000_pch_cnp: 4536 case e1000_pch_tgp: 4537 case e1000_pch_adp: 4538 case e1000_pch_mtp: 4539 case e1000_pch_ptp: 4540 case e1000_pch_nvp: 4541 return (true); 4542 default: 4543 return (false); 4544 } 4545 } 4546 4547 static void 4548 em_initialize_receive_unit(if_ctx_t ctx) 4549 { 4550 struct e1000_softc *sc = iflib_get_softc(ctx); 4551 if_softc_ctx_t scctx = sc->shared; 4552 if_t ifp = iflib_get_ifp(ctx); 4553 struct e1000_hw *hw = &sc->hw; 4554 struct em_rx_queue *que; 4555 int i; 4556 uint32_t rctl, rxcsum; 4557 4558 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 4559 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 4560 4561 /* 4562 * Make sure receives are disabled while setting up the descriptor 4563 * ring. 4564 */ 4565 rctl = E1000_READ_REG(hw, E1000_RCTL); 4566 /* Do not disable if ever enabled on this hardware. */ 4567 if (hw->mac.type != e1000_82574 && 4568 hw->mac.type != e1000_82583) 4569 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4570 4571 /* Setup the Receive Control Register. */ 4572 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4573 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 4574 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 4575 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4576 rctl &= ~E1000_RCTL_SBP; 4577 4578 if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU) 4579 rctl |= E1000_RCTL_LPE; 4580 else 4581 rctl &= ~E1000_RCTL_LPE; 4582 if (!em_disable_crc_stripping) 4583 rctl |= E1000_RCTL_SECRC; 4584 4585 /* lem/em default interrupt moderation */ 4586 if (hw->mac.type < igb_mac_min) { 4587 if (hw->mac.type >= e1000_82540) { 4588 E1000_WRITE_REG(hw, E1000_RADV, 4589 sc->rx_abs_int_delay.value); 4590 4591 /* Set the default interrupt throttling rate */ 4592 E1000_WRITE_REG(hw, E1000_ITR, 4593 EM_INTS_TO_ITR(em_max_interrupt_rate)); 4594 4595 /* 4596 * The 82574 MSI-X EITR registers are programmed 4597 * with the same value further below. Either way 4598 * the hardware now holds the default rate, so seed 4599 * the software copy to match; otherwise a stale 4600 * itr_setting left over from AIM makes em_newitr() 4601 * skip the write that would restore it. 4602 */ 4603 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; 4604 i++, que++) 4605 que->itr_setting = 4606 EM_INTS_TO_ITR(em_max_interrupt_rate); 4607 } 4608 4609 /* XXX TEMPORARY WORKAROUND: on some systems with 82573 4610 * long latencies are observed, like Lenovo X60. This 4611 * change eliminates the problem, but since having positive 4612 * values in RDTR is a known source of problems on other 4613 * platforms another solution is being sought. 4614 */ 4615 if (hw->mac.type == e1000_82573) 4616 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 4617 else 4618 E1000_WRITE_REG(hw, E1000_RDTR, 4619 sc->rx_int_delay.value); 4620 } 4621 4622 if (hw->mac.type >= em_mac_min) { 4623 uint32_t rfctl; 4624 /* Use extended rx descriptor formats */ 4625 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 4626 rfctl |= E1000_RFCTL_EXTEN; 4627 4628 /* 4629 * When using MSI-X interrupts we need to throttle 4630 * using the EITR register (82574 only) 4631 */ 4632 if (hw->mac.type == e1000_82574) { 4633 for (int i = 0; i < 4; i++) 4634 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 4635 EM_INTS_TO_ITR(em_max_interrupt_rate)); 4636 /* Disable accelerated acknowledge */ 4637 rfctl |= E1000_RFCTL_ACK_DIS; 4638 } 4639 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 4640 } 4641 4642 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 4643 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 4644 rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; 4645 if (hw->mac.type > e1000_82575) 4646 rxcsum |= E1000_RXCSUM_CRCOFL; 4647 else if (hw->mac.type < em_mac_min && 4648 if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) 4649 rxcsum |= E1000_RXCSUM_IPV6OFL; 4650 } else { 4651 rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4652 if (hw->mac.type > e1000_82575) 4653 rxcsum &= ~E1000_RXCSUM_CRCOFL; 4654 else if (hw->mac.type < em_mac_min) 4655 rxcsum &= ~E1000_RXCSUM_IPV6OFL; 4656 } 4657 4658 if (sc->rx_num_queues > 1) { 4659 /* RSS hash needed in the Rx descriptor */ 4660 rxcsum |= E1000_RXCSUM_PCSD; 4661 4662 if (hw->mac.type >= igb_mac_min) 4663 igb_initialize_rss_mapping(sc); 4664 else 4665 em_initialize_rss_mapping(sc); 4666 } 4667 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4668 4669 for (i = 0, que = sc->rx_queues; 4670 hw->mac.type < igb_mac_min && i < sc->rx_num_queues; 4671 i++, que++) { 4672 struct rx_ring *rxr = &que->rxr; 4673 /* Setup the Base and Length of the Rx Descriptor Ring */ 4674 u64 bus_addr = rxr->rx_paddr; 4675 uint32_t qid = rxr->me; 4676 #if 0 4677 u32 rdt = sc->rx_num_queues -1; /* default */ 4678 #endif 4679 4680 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 4681 scctx->isc_nrxd[0] * 4682 sizeof(union e1000_rx_desc_extended)); 4683 E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32)); 4684 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr); 4685 /* Setup the Head and Tail Descriptor Pointers */ 4686 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 4687 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 4688 } 4689 4690 /* Increase receive-descriptor prefetching for integrated jumbo MACs. */ 4691 if (em_integrated_jumbo_rx(hw) && if_getmtu(ifp) > ETHERMTU) { 4692 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 4693 4694 rxdctl &= ~(EM_RXDCTL_PTHRESH_MASK | 4695 EM_RXDCTL_HTHRESH_MASK); 4696 rxdctl |= EM_JUMBO_RX_PTHRESH | 4697 (EM_JUMBO_RX_HTHRESH << 8); 4698 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 4699 } else if (hw->mac.type == e1000_82574) { 4700 /* RXDCTL(0) writes are mirrored to RXDCTL(1) on 82574. */ 4701 for (int i = 0; i < sc->rx_num_queues; i++) { 4702 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 4703 4704 rxdctl &= ~EM_RXDCTL_THRESH_MASK; 4705 rxdctl |= EM_82574_RX_PTHRESH | 4706 (EM_82574_RX_HTHRESH << 8) | 4707 (EM_82574_RX_WTHRESH << 16) | 4708 E1000_RXDCTL_THRESH_UNIT_DESC; 4709 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 4710 } 4711 } else if (hw->mac.type >= igb_mac_min) { 4712 bool drop; 4713 u32 psize; 4714 4715 if (igb_iov_enabled(sc)) { 4716 E1000_WRITE_REG(hw, E1000_RLPML, 4717 IGB_IOV_MAX_FRAME_SIZE); 4718 } else if (if_getmtu(ifp) > ETHERMTU) { 4719 psize = scctx->isc_max_frame_size; 4720 /* are we on a vlan? */ 4721 if (if_vlantrunkinuse(ifp)) 4722 psize += VLAN_TAG_SIZE; 4723 4724 E1000_WRITE_REG(hw, E1000_RLPML, psize); 4725 } 4726 4727 /* 4728 * If TX flow control is disabled and there's >1 queue 4729 * defined, enable DROP. 4730 * 4731 * This drops frames rather than hanging the RX MAC for all 4732 * queues. 4733 */ 4734 drop = igb_iov_enabled(sc) || 4735 ((sc->rx_num_queues > 1) && 4736 (sc->fc == e1000_fc_none || 4737 sc->fc == e1000_fc_rx_pause)); 4738 igb_initialize_receive_rings(ctx, drop); 4739 } else if (hw->mac.type >= e1000_pch2lan) { 4740 if (if_getmtu(ifp) > ETHERMTU) 4741 e1000_lv_jumbo_workaround_ich8lan(hw, true); 4742 else 4743 e1000_lv_jumbo_workaround_ich8lan(hw, false); 4744 } 4745 4746 /* Make sure VLAN Filters are off */ 4747 rctl &= ~E1000_RCTL_VFE; 4748 4749 /* Set up packet buffer size, overridden by per queue srrctl on igb */ 4750 if (hw->mac.type < igb_mac_min) { 4751 if (sc->rx_mbuf_sz > 2048 && sc->rx_mbuf_sz <= 4096) 4752 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 4753 else if (sc->rx_mbuf_sz > 4096 && sc->rx_mbuf_sz <= 8192) 4754 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 4755 else if (sc->rx_mbuf_sz > 8192) 4756 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX; 4757 else { 4758 rctl |= E1000_RCTL_SZ_2048; 4759 rctl &= ~E1000_RCTL_BSEX; 4760 } 4761 } else 4762 rctl |= E1000_RCTL_SZ_2048; 4763 4764 /* 4765 * rctl bits 11:10 are as follows 4766 * lem: reserved 4767 * em: DTYPE 4768 * igb: reserved 4769 * and should be 00 on all of the above 4770 */ 4771 rctl &= ~0x00000C00; 4772 4773 /* Write out the settings */ 4774 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4775 4776 return; 4777 } 4778 4779 static void 4780 em_if_vlan_register(if_ctx_t ctx, u16 vtag) 4781 { 4782 struct e1000_softc *sc = iflib_get_softc(ctx); 4783 bool present; 4784 u32 index, mask; 4785 4786 index = (vtag >> 5) & 0x7F; 4787 mask = 1U << (vtag & 0x1F); 4788 present = (sc->shadow_vfta[index] & mask) != 0; 4789 /* 4790 * On a VF, record registration intent for replay even if the PF is not 4791 * ready to accept it yet. 4792 */ 4793 sc->shadow_vfta[index] |= mask; 4794 sc->vf_vfta_stale[index] &= ~mask; 4795 if (!present) 4796 ++sc->num_vlans; 4797 if (sc->vf_ifp && 4798 e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { 4799 igbv_vlan_retry_add(sc, vtag); 4800 device_printf(sc->dev, 4801 "VF VLAN %u add request failed\n", vtag); 4802 } else if (sc->vf_ifp) 4803 igbv_vlan_retry_clear(sc, vtag); 4804 if (!sc->vf_ifp) { 4805 if (igb_iov_enabled(sc)) 4806 igb_iov_rebuild_vlan(sc); 4807 else 4808 em_if_vlan_filter_write(sc, index); 4809 } 4810 } 4811 4812 static void 4813 em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 4814 { 4815 struct e1000_softc *sc = iflib_get_softc(ctx); 4816 bool present; 4817 u32 index, mask; 4818 4819 index = (vtag >> 5) & 0x7F; 4820 mask = 1U << (vtag & 0x1F); 4821 present = (sc->shadow_vfta[index] & mask) != 0; 4822 if (sc->vf_ifp) 4823 igbv_vlan_retry_clear(sc, vtag); 4824 if (sc->vf_ifp && 4825 e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { 4826 device_printf(sc->dev, 4827 "VF VLAN %u remove request failed\n", vtag); 4828 /* 4829 * Hardware might still admit this VID. Preserve its receive 4830 * tag until a successful VF reset proves the stale filter gone. 4831 */ 4832 sc->vf_vfta_stale[index] |= mask; 4833 } else { 4834 sc->vf_vfta_stale[index] &= ~mask; 4835 } 4836 sc->shadow_vfta[index] &= ~mask; 4837 if (present) 4838 --sc->num_vlans; 4839 if (!sc->vf_ifp) { 4840 if (igb_iov_enabled(sc)) 4841 igb_iov_rebuild_vlan(sc); 4842 else 4843 em_if_vlan_filter_write(sc, index); 4844 } 4845 } 4846 4847 static bool 4848 em_if_vlan_filter_capable(if_ctx_t ctx) 4849 { 4850 if_t ifp = iflib_get_ifp(ctx); 4851 4852 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) && 4853 !em_disable_crc_stripping) 4854 return (true); 4855 4856 return (false); 4857 } 4858 4859 static bool 4860 em_if_vlan_filter_used(if_ctx_t ctx) 4861 { 4862 struct e1000_softc *sc = iflib_get_softc(ctx); 4863 4864 if (!em_if_vlan_filter_capable(ctx)) 4865 return (false); 4866 4867 for (int i = 0; i < EM_VFTA_SIZE; i++) 4868 if (sc->shadow_vfta[i] != 0) 4869 return (true); 4870 4871 return (false); 4872 } 4873 4874 static void 4875 em_if_vlan_filter_enable(struct e1000_softc *sc) 4876 { 4877 struct e1000_hw *hw = &sc->hw; 4878 u32 reg; 4879 4880 reg = E1000_READ_REG(hw, E1000_RCTL); 4881 reg &= ~E1000_RCTL_CFIEN; 4882 reg |= E1000_RCTL_VFE; 4883 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4884 } 4885 4886 static void 4887 em_if_vlan_filter_disable(struct e1000_softc *sc) 4888 { 4889 struct e1000_hw *hw = &sc->hw; 4890 u32 reg; 4891 4892 reg = E1000_READ_REG(hw, E1000_RCTL); 4893 reg &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 4894 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4895 } 4896 4897 static void 4898 em_if_vlan_filter_write(struct e1000_softc *sc, int changed_index) 4899 { 4900 struct e1000_hw *hw = &sc->hw; 4901 4902 KASSERT(!sc->vf_ifp, ("VLAN filter write on VF\n")); 4903 4904 /* Disable interrupts for lem(4) devices during the filter change */ 4905 if (hw->mac.type < em_mac_min) 4906 em_if_intr_disable(sc->ctx); 4907 4908 /* 4909 * Restore every retained VLAN after reset. Also write the changed 4910 * word when its final VLAN was removed so stale hardware membership 4911 * does not survive a zero shadow value. 4912 */ 4913 for (int i = 0; i < EM_VFTA_SIZE; i++) 4914 if (sc->shadow_vfta[i] != 0 || i == changed_index) 4915 e1000_write_vfta(hw, i, sc->shadow_vfta[i]); 4916 4917 /* Re-enable interrupts for lem-class devices */ 4918 if (hw->mac.type < em_mac_min) 4919 em_if_intr_enable(sc->ctx); 4920 } 4921 4922 static void 4923 em_setup_vlan_hw_support(if_ctx_t ctx) 4924 { 4925 struct e1000_softc *sc = iflib_get_softc(ctx); 4926 struct e1000_hw *hw = &sc->hw; 4927 if_t ifp = iflib_get_ifp(ctx); 4928 s32 error; 4929 u32 max_frame_size, reg; 4930 u16 vid; 4931 int restore_failures; 4932 4933 /* 4934 * Only PFs have control over VLAN HW filtering 4935 * configuration. VFs have to act as if it's always 4936 * enabled. 4937 */ 4938 if (sc->vf_ifp) { 4939 max_frame_size = min(sc->shared->isc_max_frame_size + 4940 VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); 4941 e1000_rlpml_set_vf(hw, max_frame_size); 4942 restore_failures = 0; 4943 for (vid = 0; vid < 4096; vid++) { 4944 if ((sc->shadow_vfta[vid >> 5] & 4945 (1U << (vid & 0x1f))) == 0) 4946 continue; 4947 /* 4948 * Desired state remains in shadow_vfta for the next 4949 * replay if the PF mailbox is absent during reset. 4950 */ 4951 error = e1000_vfta_set_vf(hw, vid, true); 4952 if (error != E1000_SUCCESS) { 4953 igbv_vlan_retry_add(sc, vid); 4954 restore_failures++; 4955 } else 4956 igbv_vlan_retry_clear(sc, vid); 4957 } 4958 if (restore_failures != 0) 4959 device_printf(sc->dev, 4960 "VF VLAN restore failed for %d VIDs; retrying\n", 4961 restore_failures); 4962 return; 4963 } 4964 4965 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 4966 !em_disable_crc_stripping) { 4967 reg = E1000_READ_REG(hw, E1000_CTRL); 4968 reg |= E1000_CTRL_VME; 4969 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4970 } else { 4971 reg = E1000_READ_REG(hw, E1000_CTRL); 4972 reg &= ~E1000_CTRL_VME; 4973 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4974 } 4975 4976 /* 4977 * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN 4978 * filtering is disabled, the IOV VLAN rebuild instead makes the PF 4979 * VLAN-promiscuous without disabling the global filter. 4980 */ 4981 if (!em_if_vlan_filter_capable(ctx)) { 4982 if (igb_iov_enabled(sc)) { 4983 #ifdef PCI_IOV 4984 sc->iov_pf_vlan_promisc = true; 4985 #endif 4986 em_if_vlan_filter_enable(sc); 4987 } else 4988 em_if_vlan_filter_disable(sc); 4989 return; 4990 } 4991 #ifdef PCI_IOV 4992 if (igb_iov_enabled(sc)) 4993 sc->iov_pf_vlan_promisc = false; 4994 #endif 4995 4996 /* 4997 * A soft reset zero's out the VFTA, so 4998 * we need to repopulate it now. 4999 * We also insert VLAN 0 in the filter list, so we pass VLAN 0 tagged 5000 * traffic through. This will write the entire table. 5001 */ 5002 em_if_vlan_register(ctx, 0); 5003 5004 /* Enable the Filter Table */ 5005 em_if_vlan_filter_enable(sc); 5006 } 5007 5008 static void 5009 em_if_intr_enable(if_ctx_t ctx) 5010 { 5011 struct e1000_softc *sc = iflib_get_softc(ctx); 5012 struct e1000_hw *hw = &sc->hw; 5013 u32 ims_mask = IMS_ENABLE_MASK; 5014 5015 if (sc->intr_type == IFLIB_INTR_MSIX) { 5016 E1000_WRITE_REG(hw, EM_EIAC, sc->ims); 5017 ims_mask |= sc->ims; 5018 } 5019 5020 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 5021 E1000_WRITE_FLUSH(hw); 5022 } 5023 5024 static void 5025 em_if_intr_disable(if_ctx_t ctx) 5026 { 5027 struct e1000_softc *sc = iflib_get_softc(ctx); 5028 struct e1000_hw *hw = &sc->hw; 5029 5030 if (sc->intr_type == IFLIB_INTR_MSIX) 5031 E1000_WRITE_REG(hw, EM_EIAC, 0); 5032 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 5033 E1000_WRITE_FLUSH(hw); 5034 } 5035 5036 static void 5037 igb_if_intr_enable(if_ctx_t ctx) 5038 { 5039 struct e1000_softc *sc = iflib_get_softc(ctx); 5040 struct e1000_hw *hw = &sc->hw; 5041 u32 mask, reg; 5042 5043 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 5044 mask = (sc->que_mask | sc->link_mask); 5045 /* 5046 * VF interrupt controls are also mapped into these registers. 5047 * Preserve them and change only the PF vectors we own. 5048 */ 5049 reg = E1000_READ_REG(hw, E1000_EIAC); 5050 E1000_WRITE_REG(hw, E1000_EIAC, reg | mask); 5051 reg = E1000_READ_REG(hw, E1000_EIAM); 5052 E1000_WRITE_REG(hw, E1000_EIAM, reg | mask); 5053 igb_iov_intr_drain_stale(sc); 5054 E1000_WRITE_REG(hw, E1000_EIMS, mask); 5055 E1000_WRITE_REG(hw, E1000_IMS, 5056 E1000_IMS_LSC | igb_iov_intr_mask(sc)); 5057 } else 5058 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 5059 E1000_WRITE_FLUSH(hw); 5060 } 5061 5062 static void 5063 igb_if_intr_disable(if_ctx_t ctx) 5064 { 5065 struct e1000_softc *sc = iflib_get_softc(ctx); 5066 struct e1000_hw *hw = &sc->hw; 5067 u32 mask, reg; 5068 5069 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 5070 /* 5071 * Do not use a blanket EIMC write here. VF interrupt controls 5072 * are mapped into the same PF register space, so clearing bits 5073 * we do not own can leave running VFs with interrupts masked. 5074 * Before initial queue configuration the owned mask is zero 5075 * because this driver has not enabled a vector yet. 5076 */ 5077 mask = (sc->que_mask | sc->link_mask); 5078 reg = E1000_READ_REG(hw, E1000_EIAM); 5079 E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask); 5080 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5081 reg = E1000_READ_REG(hw, E1000_EIAC); 5082 E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask); 5083 } 5084 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 5085 E1000_WRITE_FLUSH(hw); 5086 } 5087 5088 /* 5089 * Bit of a misnomer, what this really means is 5090 * to enable OS management of the system... aka 5091 * to disable special hardware management features 5092 */ 5093 static void 5094 em_init_manageability(struct e1000_softc *sc) 5095 { 5096 /* A shared code workaround */ 5097 #define E1000_82542_MANC2H E1000_MANC2H 5098 if (sc->has_manage) { 5099 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 5100 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 5101 5102 /* disable hardware interception of ARP */ 5103 manc &= ~(E1000_MANC_ARP_EN); 5104 5105 /* enable receiving management packets to the host */ 5106 manc |= E1000_MANC_EN_MNG2HOST; 5107 #define E1000_MNG2HOST_PORT_623 (1 << 5) 5108 #define E1000_MNG2HOST_PORT_664 (1 << 6) 5109 manc2h |= E1000_MNG2HOST_PORT_623; 5110 manc2h |= E1000_MNG2HOST_PORT_664; 5111 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 5112 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 5113 } 5114 } 5115 5116 /* 5117 * Give control back to hardware management 5118 * controller if there is one. 5119 */ 5120 static void 5121 em_release_manageability(struct e1000_softc *sc) 5122 { 5123 if (sc->has_manage) { 5124 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 5125 5126 /* re-enable hardware interception of ARP */ 5127 manc |= E1000_MANC_ARP_EN; 5128 manc &= ~E1000_MANC_EN_MNG2HOST; 5129 5130 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 5131 } 5132 } 5133 5134 /* 5135 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 5136 * For ASF and Pass Through versions of f/w this means 5137 * that the driver is loaded. For AMT version type f/w 5138 * this means that the network i/f is open. 5139 */ 5140 static void 5141 em_get_hw_control(struct e1000_softc *sc) 5142 { 5143 u32 ctrl_ext, swsm; 5144 5145 if (sc->vf_ifp) 5146 return; 5147 5148 if (sc->hw.mac.type == e1000_82573) { 5149 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 5150 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 5151 swsm | E1000_SWSM_DRV_LOAD); 5152 return; 5153 } 5154 /* else */ 5155 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 5156 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 5157 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 5158 } 5159 5160 /* 5161 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 5162 * For ASF and Pass Through versions of f/w this means that 5163 * the driver is no longer loaded. For AMT versions of the 5164 * f/w this means that the network i/f is closed. 5165 */ 5166 static void 5167 em_release_hw_control(struct e1000_softc *sc) 5168 { 5169 u32 ctrl_ext, swsm; 5170 5171 if (!sc->has_manage) 5172 return; 5173 5174 if (sc->hw.mac.type == e1000_82573) { 5175 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 5176 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 5177 swsm & ~E1000_SWSM_DRV_LOAD); 5178 return; 5179 } 5180 /* else */ 5181 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 5182 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 5183 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 5184 return; 5185 } 5186 5187 bool 5188 em_is_valid_ether_addr(const u8 *addr) 5189 { 5190 static const u8 zero_addr[ETHER_ADDR_LEN]; 5191 5192 return (!ETHER_IS_MULTICAST(addr) && 5193 memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0); 5194 } 5195 5196 static bool 5197 em_automask_tso(if_ctx_t ctx) 5198 { 5199 struct e1000_softc *sc = iflib_get_softc(ctx); 5200 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 5201 if_t ifp = iflib_get_ifp(ctx); 5202 bool reset_needed; 5203 int drvflags; 5204 5205 if (!em_unsupported_tso && sc->link_speed && 5206 sc->link_speed != SPEED_1000 && 5207 scctx->isc_capenable & IFCAP_TSO) { 5208 device_printf(sc->dev, 5209 "Disabling TSO for 10/100 Ethernet.\n"); 5210 sc->tso_automasked = scctx->isc_capenable & IFCAP_TSO; 5211 scctx->isc_capenable &= ~IFCAP_TSO; 5212 if_setcapenablebit(ifp, 0, IFCAP_TSO); 5213 } else if (sc->link_speed == SPEED_1000 && sc->tso_automasked) { 5214 device_printf(sc->dev, "Re-enabling TSO for GbE.\n"); 5215 scctx->isc_capenable |= sc->tso_automasked; 5216 if_setcapenablebit(ifp, sc->tso_automasked, 0); 5217 sc->tso_automasked = 0; 5218 } else { 5219 return (false); 5220 } 5221 5222 /* 5223 * Reset a running interface, or one being initialized while 5224 * administratively up. OACTIVE remains set after iflib_stop(), so 5225 * it alone cannot distinguish initialization from an interface that 5226 * is down. In other states, the next initialization will apply the 5227 * updated capabilities. 5228 */ 5229 drvflags = if_getdrvflags(ifp); 5230 reset_needed = (drvflags & IFF_DRV_RUNNING) != 0 || 5231 ((drvflags & IFF_DRV_OACTIVE) != 0 && 5232 (if_getflags(ifp) & IFF_UP) != 0); 5233 if (!reset_needed) 5234 return (false); 5235 5236 /* iflib_init_locked handles ifnet hwassistbits */ 5237 iflib_request_reset(ctx); 5238 return (true); 5239 } 5240 5241 /* 5242 ** Parse the interface capabilities with regard 5243 ** to both system management and wake-on-lan for 5244 ** later use. 5245 */ 5246 static void 5247 em_get_wakeup(if_ctx_t ctx) 5248 { 5249 struct e1000_softc *sc = iflib_get_softc(ctx); 5250 device_t dev = iflib_get_dev(ctx); 5251 u16 eeprom_data = 0, device_id, apme_mask; 5252 5253 sc->has_manage = e1000_enable_mng_pass_thru(&sc->hw); 5254 apme_mask = EM_EEPROM_APME; 5255 5256 switch (sc->hw.mac.type) { 5257 case e1000_82542: 5258 case e1000_82543: 5259 break; 5260 case e1000_82544: 5261 e1000_read_nvm(&sc->hw, 5262 NVM_INIT_CONTROL2_REG, 1, &eeprom_data); 5263 apme_mask = EM_82544_APME; 5264 break; 5265 case e1000_82546: 5266 case e1000_82546_rev_3: 5267 if (sc->hw.bus.func == 1) { 5268 e1000_read_nvm(&sc->hw, 5269 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 5270 break; 5271 } else 5272 e1000_read_nvm(&sc->hw, 5273 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 5274 break; 5275 case e1000_82573: 5276 case e1000_82583: 5277 sc->has_amt = true; 5278 /* FALLTHROUGH */ 5279 case e1000_82571: 5280 case e1000_82572: 5281 case e1000_80003es2lan: 5282 if (sc->hw.bus.func == 1) { 5283 e1000_read_nvm(&sc->hw, 5284 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 5285 break; 5286 } else 5287 e1000_read_nvm(&sc->hw, 5288 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 5289 break; 5290 case e1000_ich8lan: 5291 case e1000_ich9lan: 5292 case e1000_ich10lan: 5293 case e1000_pchlan: 5294 case e1000_pch2lan: 5295 case e1000_pch_lpt: 5296 case e1000_pch_spt: 5297 case e1000_82575: /* listing all igb devices */ 5298 case e1000_82576: 5299 case e1000_82580: 5300 case e1000_i350: 5301 case e1000_i354: 5302 case e1000_i210: 5303 case e1000_i211: 5304 apme_mask = E1000_WUC_APME; 5305 sc->has_amt = true; 5306 eeprom_data = E1000_READ_REG(&sc->hw, E1000_WUC); 5307 break; 5308 default: 5309 e1000_read_nvm(&sc->hw, 5310 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 5311 break; 5312 } 5313 if (eeprom_data & apme_mask) 5314 sc->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 5315 /* 5316 * We have the eeprom settings, now apply the special cases 5317 * where the eeprom may be wrong or the board won't support 5318 * wake on lan on a particular port 5319 */ 5320 device_id = pci_get_device(dev); 5321 switch (device_id) { 5322 case E1000_DEV_ID_82546GB_PCIE: 5323 sc->wol = 0; 5324 break; 5325 case E1000_DEV_ID_82546EB_FIBER: 5326 case E1000_DEV_ID_82546GB_FIBER: 5327 /* Wake events only supported on port A for dual fiber 5328 * regardless of eeprom setting */ 5329 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & 5330 E1000_STATUS_FUNC_1) 5331 sc->wol = 0; 5332 break; 5333 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 5334 /* if quad port adapter, disable WoL on all but port A */ 5335 if (global_quad_port_a != 0) 5336 sc->wol = 0; 5337 /* Reset for multiple quad port adapters */ 5338 if (++global_quad_port_a == 4) 5339 global_quad_port_a = 0; 5340 break; 5341 case E1000_DEV_ID_82571EB_FIBER: 5342 /* Wake events only supported on port A for dual fiber 5343 * regardless of eeprom setting */ 5344 if (E1000_READ_REG(&sc->hw, E1000_STATUS) & 5345 E1000_STATUS_FUNC_1) 5346 sc->wol = 0; 5347 break; 5348 case E1000_DEV_ID_82571EB_QUAD_COPPER: 5349 case E1000_DEV_ID_82571EB_QUAD_FIBER: 5350 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 5351 /* if quad port adapter, disable WoL on all but port A */ 5352 if (global_quad_port_a != 0) 5353 sc->wol = 0; 5354 /* Reset for multiple quad port adapters */ 5355 if (++global_quad_port_a == 4) 5356 global_quad_port_a = 0; 5357 break; 5358 } 5359 } 5360 5361 5362 /* 5363 * Enable PCI Wake On Lan capability 5364 */ 5365 static void 5366 em_enable_wakeup(if_ctx_t ctx) 5367 { 5368 struct e1000_softc *sc = iflib_get_softc(ctx); 5369 device_t dev = iflib_get_dev(ctx); 5370 if_t ifp = iflib_get_ifp(ctx); 5371 int error = 0; 5372 u32 ctrl, ctrl_ext, rctl; 5373 5374 if (sc->vf_ifp) 5375 return; 5376 if (!pci_has_pm(dev)) 5377 return; 5378 5379 /* 5380 * Determine type of Wakeup: note that wol 5381 * is set with all bits on by default. 5382 */ 5383 if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) 5384 sc->wol &= ~E1000_WUFC_MAG; 5385 5386 if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0) 5387 sc->wol &= ~E1000_WUFC_EX; 5388 5389 if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) 5390 sc->wol &= ~E1000_WUFC_MC; 5391 else { 5392 rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 5393 rctl |= E1000_RCTL_MPE; 5394 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 5395 } 5396 5397 if (!(sc->wol & (E1000_WUFC_EX | E1000_WUFC_MAG | E1000_WUFC_MC))) 5398 goto pme; 5399 5400 /* Advertise the wakeup capability */ 5401 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 5402 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 5403 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 5404 5405 /* Keep the laser running on Fiber adapters */ 5406 if (sc->hw.phy.media_type == e1000_media_type_fiber || 5407 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 5408 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 5409 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 5410 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, ctrl_ext); 5411 } 5412 5413 if ((sc->hw.mac.type == e1000_ich8lan) || 5414 (sc->hw.mac.type == e1000_pchlan) || 5415 (sc->hw.mac.type == e1000_ich9lan) || 5416 (sc->hw.mac.type == e1000_ich10lan)) 5417 e1000_suspend_workarounds_ich8lan(&sc->hw); 5418 5419 if ( sc->hw.mac.type >= e1000_pchlan) { 5420 error = em_enable_phy_wakeup(sc); 5421 if (error) 5422 goto pme; 5423 } else { 5424 /* Enable wakeup by the MAC */ 5425 E1000_WRITE_REG(&sc->hw, E1000_WUC, E1000_WUC_PME_EN); 5426 E1000_WRITE_REG(&sc->hw, E1000_WUFC, sc->wol); 5427 } 5428 5429 if (sc->hw.phy.type == e1000_phy_igp_3) 5430 e1000_igp3_phy_powerdown_workaround_ich8lan(&sc->hw); 5431 5432 pme: 5433 if (!error && (if_getcapenable(ifp) & IFCAP_WOL)) 5434 pci_enable_pme(dev); 5435 5436 return; 5437 } 5438 5439 /* 5440 * WOL in the newer chipset interfaces (pchlan) 5441 * require thing to be copied into the phy 5442 */ 5443 static int 5444 em_enable_phy_wakeup(struct e1000_softc *sc) 5445 { 5446 struct e1000_hw *hw = &sc->hw; 5447 u32 mreg, ret = 0; 5448 u16 preg; 5449 5450 /* copy MAC RARs to PHY RARs */ 5451 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 5452 5453 /* copy MAC MTA to PHY MTA */ 5454 for (int i = 0; i < hw->mac.mta_reg_count; i++) { 5455 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 5456 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 5457 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 5458 (u16)((mreg >> 16) & 0xFFFF)); 5459 } 5460 5461 /* configure PHY Rx Control register */ 5462 e1000_read_phy_reg(hw, BM_RCTL, &preg); 5463 mreg = E1000_READ_REG(hw, E1000_RCTL); 5464 if (mreg & E1000_RCTL_UPE) 5465 preg |= BM_RCTL_UPE; 5466 if (mreg & E1000_RCTL_MPE) 5467 preg |= BM_RCTL_MPE; 5468 preg &= ~(BM_RCTL_MO_MASK); 5469 if (mreg & E1000_RCTL_MO_3) 5470 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 5471 << BM_RCTL_MO_SHIFT); 5472 if (mreg & E1000_RCTL_BAM) 5473 preg |= BM_RCTL_BAM; 5474 if (mreg & E1000_RCTL_PMCF) 5475 preg |= BM_RCTL_PMCF; 5476 mreg = E1000_READ_REG(hw, E1000_CTRL); 5477 if (mreg & E1000_CTRL_RFCE) 5478 preg |= BM_RCTL_RFCE; 5479 e1000_write_phy_reg(hw, BM_RCTL, preg); 5480 5481 /* enable PHY wakeup in MAC register */ 5482 E1000_WRITE_REG(hw, E1000_WUC, 5483 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN | E1000_WUC_APME); 5484 E1000_WRITE_REG(hw, E1000_WUFC, sc->wol); 5485 5486 /* configure and enable PHY wakeup in PHY registers */ 5487 e1000_write_phy_reg(hw, BM_WUFC, sc->wol); 5488 e1000_write_phy_reg(hw, BM_WUC, E1000_WUC_PME_EN); 5489 5490 /* activate PHY wakeup */ 5491 ret = hw->phy.ops.acquire(hw); 5492 if (ret) { 5493 printf("Could not acquire PHY\n"); 5494 return ret; 5495 } 5496 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 5497 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 5498 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 5499 if (ret) { 5500 printf("Could not read PHY page 769\n"); 5501 goto out; 5502 } 5503 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 5504 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 5505 if (ret) 5506 printf("Could not set PHY Host Wakeup bit\n"); 5507 out: 5508 hw->phy.ops.release(hw); 5509 5510 return ret; 5511 } 5512 5513 static void 5514 em_if_led_func(if_ctx_t ctx, int onoff) 5515 { 5516 struct e1000_softc *sc = iflib_get_softc(ctx); 5517 5518 if (onoff) { 5519 e1000_setup_led(&sc->hw); 5520 if (sc->hw.phy.media_type == e1000_media_type_internal_serdes) 5521 e1000_blink_led(&sc->hw); 5522 else 5523 e1000_led_on(&sc->hw); 5524 } else { 5525 e1000_led_off(&sc->hw); 5526 e1000_cleanup_led(&sc->hw); 5527 } 5528 } 5529 5530 /* 5531 * Disable the L0S and L1 LINK states 5532 */ 5533 static void 5534 em_disable_aspm(struct e1000_softc *sc) 5535 { 5536 int base, reg; 5537 u16 link_cap,link_ctrl; 5538 device_t dev = sc->dev; 5539 5540 switch (sc->hw.mac.type) { 5541 case e1000_82573: 5542 case e1000_82574: 5543 case e1000_82583: 5544 break; 5545 default: 5546 return; 5547 } 5548 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) 5549 return; 5550 reg = base + PCIER_LINK_CAP; 5551 link_cap = pci_read_config(dev, reg, 2); 5552 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0) 5553 return; 5554 reg = base + PCIER_LINK_CTL; 5555 link_ctrl = pci_read_config(dev, reg, 2); 5556 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC; 5557 pci_write_config(dev, reg, link_ctrl, 2); 5558 return; 5559 } 5560 5561 /********************************************************************** 5562 * 5563 * Update the board statistics counters. 5564 * 5565 **********************************************************************/ 5566 void 5567 em_update_stats_counters(struct e1000_softc *sc) 5568 { 5569 struct e1000_hw_stats *stats; 5570 u64 prev_xoffrxc; 5571 5572 if (sc->vf_ifp) { 5573 em_update_vf_stats_counters(sc); 5574 return; 5575 } 5576 5577 stats = &sc->ustats.stats; 5578 prev_xoffrxc = stats->xoffrxc; 5579 5580 if(sc->hw.phy.media_type == e1000_media_type_copper || 5581 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 5582 stats->symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 5583 stats->sec += E1000_READ_REG(&sc->hw, E1000_SEC); 5584 } 5585 stats->crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 5586 stats->mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 5587 stats->scc += E1000_READ_REG(&sc->hw, E1000_SCC); 5588 stats->ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 5589 5590 stats->mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 5591 stats->latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 5592 stats->colc += E1000_READ_REG(&sc->hw, E1000_COLC); 5593 stats->dc += E1000_READ_REG(&sc->hw, E1000_DC); 5594 stats->rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 5595 stats->xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 5596 stats->xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 5597 stats->xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 5598 /* 5599 ** For watchdog management we need to know if we have been 5600 ** paused during the last interval, so capture that here. 5601 */ 5602 if (stats->xoffrxc != prev_xoffrxc) 5603 sc->shared->isc_pause_frames = 1; 5604 stats->xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 5605 stats->fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 5606 stats->prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 5607 stats->prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 5608 stats->prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 5609 stats->prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 5610 stats->prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 5611 stats->prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 5612 stats->gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 5613 stats->bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 5614 stats->mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 5615 stats->gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 5616 5617 /* For the 64-bit byte counters the low dword must be read first. */ 5618 /* Both registers clear on the read of the high dword */ 5619 5620 stats->gorc += E1000_READ_REG(&sc->hw, E1000_GORCL) + 5621 ((u64)E1000_READ_REG(&sc->hw, E1000_GORCH) << 32); 5622 stats->gotc += E1000_READ_REG(&sc->hw, E1000_GOTCL) + 5623 ((u64)E1000_READ_REG(&sc->hw, E1000_GOTCH) << 32); 5624 5625 stats->rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 5626 stats->ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 5627 stats->rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 5628 stats->roc += E1000_READ_REG(&sc->hw, E1000_ROC); 5629 stats->rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 5630 5631 stats->mgprc += E1000_READ_REG(&sc->hw, E1000_MGTPRC); 5632 stats->mgpdc += E1000_READ_REG(&sc->hw, E1000_MGTPDC); 5633 stats->mgptc += E1000_READ_REG(&sc->hw, E1000_MGTPTC); 5634 5635 stats->tor += E1000_READ_REG(&sc->hw, E1000_TORH); 5636 stats->tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 5637 5638 stats->tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 5639 stats->tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 5640 stats->ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 5641 stats->ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 5642 stats->ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 5643 stats->ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 5644 stats->ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 5645 stats->ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 5646 stats->mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 5647 stats->bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 5648 5649 /* TLPIC and RLPIC are clear-on-read. */ 5650 if (em_mac_has_eee(sc->hw.mac.type)) { 5651 stats->tlpic += E1000_READ_REG(&sc->hw, E1000_TLPIC); 5652 stats->rlpic += E1000_READ_REG(&sc->hw, E1000_RLPIC); 5653 } 5654 5655 /* Interrupt Counts */ 5656 5657 stats->iac += E1000_READ_REG(&sc->hw, E1000_IAC); 5658 stats->icrxptc += E1000_READ_REG(&sc->hw, E1000_ICRXPTC); 5659 stats->icrxatc += E1000_READ_REG(&sc->hw, E1000_ICRXATC); 5660 stats->ictxptc += E1000_READ_REG(&sc->hw, E1000_ICTXPTC); 5661 stats->ictxatc += E1000_READ_REG(&sc->hw, E1000_ICTXATC); 5662 stats->ictxqec += E1000_READ_REG(&sc->hw, E1000_ICTXQEC); 5663 stats->ictxqmtc += E1000_READ_REG(&sc->hw, E1000_ICTXQMTC); 5664 stats->icrxdmtc += E1000_READ_REG(&sc->hw, E1000_ICRXDMTC); 5665 stats->icrxoc += E1000_READ_REG(&sc->hw, E1000_ICRXOC); 5666 5667 if (sc->hw.mac.type >= e1000_82543) { 5668 stats->algnerrc += 5669 E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 5670 stats->rxerrc += 5671 E1000_READ_REG(&sc->hw, E1000_RXERRC); 5672 stats->tncrs += 5673 E1000_READ_REG(&sc->hw, E1000_TNCRS); 5674 stats->cexterr += 5675 E1000_READ_REG(&sc->hw, E1000_CEXTERR); 5676 stats->tsctc += 5677 E1000_READ_REG(&sc->hw, E1000_TSCTC); 5678 stats->tsctfc += 5679 E1000_READ_REG(&sc->hw, E1000_TSCTFC); 5680 } 5681 } 5682 5683 static bool 5684 em_mac_has_eee(enum e1000_mac_type type) 5685 { 5686 5687 return ((type >= e1000_pch2lan && type < e1000_82575) || 5688 (type >= e1000_i350 && type <= e1000_i211)); 5689 } 5690 5691 static void 5692 em_initialize_vf_stats(struct e1000_softc *sc) 5693 { 5694 struct e1000_vf_stats *stats; 5695 5696 stats = &sc->ustats.vf_stats; 5697 *stats = (struct e1000_vf_stats){}; 5698 em_rebase_vf_stats(sc); 5699 } 5700 5701 static void 5702 em_rebase_vf_stats(struct e1000_softc *sc) 5703 { 5704 struct e1000_vf_stats *stats; 5705 5706 /* 5707 * A PF reset starts a new VF counter epoch. Preserve the accumulated 5708 * totals while establishing a new raw baseline so the reset is not 5709 * mistaken for a 32-bit wrap. 5710 */ 5711 stats = &sc->ustats.vf_stats; 5712 #define INIT_VF_REG(reg, name) do { \ 5713 stats->last_##name = E1000_READ_REG(&sc->hw, reg); \ 5714 } while (0) 5715 INIT_VF_REG(E1000_VFGPRC, gprc); 5716 INIT_VF_REG(E1000_VFGORC, gorc); 5717 INIT_VF_REG(E1000_VFGPTC, gptc); 5718 INIT_VF_REG(E1000_VFGOTC, gotc); 5719 /* 5720 * I350 specification update erratum 31 says VFMPRC is not 5721 * accessible from VF memory. The 0xf3c register remains valid on 5722 * 82576 VFs, but must not be read on vfadapt_i350. 5723 */ 5724 if (sc->hw.mac.type == e1000_vfadapt) 5725 INIT_VF_REG(E1000_VFMPRC, mprc); 5726 else 5727 stats->last_mprc = 0; 5728 INIT_VF_REG(E1000_VFGOTLBC, gotlbc); 5729 INIT_VF_REG(E1000_VFGPTLBC, gptlbc); 5730 INIT_VF_REG(E1000_VFGORLBC, gorlbc); 5731 INIT_VF_REG(E1000_VFGPRLBC, gprlbc); 5732 #undef INIT_VF_REG 5733 } 5734 5735 static void 5736 em_update_vf_stats_counters(struct e1000_softc *sc) 5737 { 5738 struct e1000_vf_stats *stats; 5739 5740 stats = &sc->ustats.vf_stats; 5741 5742 /* 5743 * Internal VF loopback traffic can continue without physical link, 5744 * so sample the counters regardless of link state. 5745 */ 5746 UPDATE_VF_REG(E1000_VFGPRC, 5747 stats->last_gprc, stats->gprc); 5748 UPDATE_VF_REG(E1000_VFGORC, 5749 stats->last_gorc, stats->gorc); 5750 UPDATE_VF_REG(E1000_VFGPTC, 5751 stats->last_gptc, stats->gptc); 5752 UPDATE_VF_REG(E1000_VFGOTC, 5753 stats->last_gotc, stats->gotc); 5754 if (sc->hw.mac.type == e1000_vfadapt) 5755 UPDATE_VF_REG(E1000_VFMPRC, 5756 stats->last_mprc, stats->mprc); 5757 UPDATE_VF_REG(E1000_VFGOTLBC, 5758 stats->last_gotlbc, stats->gotlbc); 5759 UPDATE_VF_REG(E1000_VFGPTLBC, 5760 stats->last_gptlbc, stats->gptlbc); 5761 UPDATE_VF_REG(E1000_VFGORLBC, 5762 stats->last_gorlbc, stats->gorlbc); 5763 UPDATE_VF_REG(E1000_VFGPRLBC, 5764 stats->last_gprlbc, stats->gprlbc); 5765 } 5766 5767 static uint64_t 5768 em_if_get_vf_counter(if_ctx_t ctx, ift_counter cnt) 5769 { 5770 struct e1000_softc *sc = iflib_get_softc(ctx); 5771 if_t ifp = iflib_get_ifp(ctx); 5772 5773 switch (cnt) { 5774 case IFCOUNTER_IERRORS: 5775 return sc->dropped_pkts; 5776 default: 5777 return (if_get_counter_default(ifp, cnt)); 5778 } 5779 } 5780 5781 static uint64_t 5782 em_if_get_counter(if_ctx_t ctx, ift_counter cnt) 5783 { 5784 struct e1000_softc *sc = iflib_get_softc(ctx); 5785 struct e1000_hw_stats *stats; 5786 if_t ifp = iflib_get_ifp(ctx); 5787 5788 if (sc->vf_ifp) 5789 return (em_if_get_vf_counter(ctx, cnt)); 5790 5791 stats = &sc->ustats.stats; 5792 5793 switch (cnt) { 5794 case IFCOUNTER_COLLISIONS: 5795 return (stats->colc); 5796 case IFCOUNTER_IERRORS: 5797 return (sc->dropped_pkts + stats->rxerrc + 5798 stats->crcerrs + stats->algnerrc + 5799 stats->ruc + stats->roc + 5800 stats->mpc + stats->cexterr); 5801 case IFCOUNTER_OERRORS: 5802 return (if_get_counter_default(ifp, cnt) + 5803 stats->ecol + stats->latecol); 5804 default: 5805 return (if_get_counter_default(ifp, cnt)); 5806 } 5807 } 5808 5809 /* em_if_needs_restart - Tell iflib when the driver needs to be reinitialized 5810 * @ctx: iflib context 5811 * @event: event code to check 5812 * 5813 * Defaults to returning false for unknown events. 5814 * 5815 * @returns true if iflib needs to reinit the interface 5816 */ 5817 static bool 5818 em_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 5819 { 5820 switch (event) { 5821 case IFLIB_RESTART_VLAN_CONFIG: 5822 default: 5823 return (false); 5824 } 5825 } 5826 5827 /* Export a single 32-bit register via a read-only sysctl. */ 5828 static int 5829 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 5830 { 5831 struct e1000_softc *sc; 5832 u_int val; 5833 5834 sc = oidp->oid_arg1; 5835 val = E1000_READ_REG(&sc->hw, oidp->oid_arg2); 5836 return (sysctl_handle_int(oidp, &val, 0, req)); 5837 } 5838 5839 enum em_ring_register { 5840 EM_RING_HEAD, 5841 EM_RING_TAIL, 5842 }; 5843 5844 /* Queue register addresses can change when the PF enters IOV mode. */ 5845 static int 5846 em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS) 5847 { 5848 struct tx_ring *txr; 5849 u_int reg, val; 5850 5851 txr = oidp->oid_arg1; 5852 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) : 5853 E1000_TDT(txr->me); 5854 val = E1000_READ_REG(&txr->sc->hw, reg); 5855 return (sysctl_handle_int(oidp, &val, 0, req)); 5856 } 5857 5858 static int 5859 em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS) 5860 { 5861 struct rx_ring *rxr; 5862 u_int reg, val; 5863 5864 rxr = oidp->oid_arg1; 5865 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) : 5866 E1000_RDT(rxr->me); 5867 val = E1000_READ_REG(&rxr->sc->hw, reg); 5868 return (sysctl_handle_int(oidp, &val, 0, req)); 5869 } 5870 5871 /* Per queue holdoff interrupt rate handler */ 5872 static int 5873 em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 5874 { 5875 struct em_rx_queue *rque; 5876 struct em_tx_queue *tque; 5877 struct e1000_hw *hw; 5878 int error; 5879 u32 reg, usec, rate; 5880 5881 bool tx = oidp->oid_arg2; 5882 5883 if (tx) { 5884 tque = oidp->oid_arg1; 5885 hw = &tque->sc->hw; 5886 if (hw->mac.type >= igb_mac_min) 5887 reg = E1000_READ_REG(hw, E1000_EITR(tque->msix)); 5888 else if (hw->mac.type == e1000_82574 && 5889 tque->sc->intr_type == IFLIB_INTR_MSIX) 5890 reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->msix)); 5891 else 5892 reg = E1000_READ_REG(hw, E1000_ITR); 5893 } else { 5894 rque = oidp->oid_arg1; 5895 hw = &rque->sc->hw; 5896 if (hw->mac.type >= igb_mac_min) 5897 reg = E1000_READ_REG(hw, E1000_EITR(rque->msix)); 5898 else if (hw->mac.type == e1000_82574 && 5899 rque->sc->intr_type == IFLIB_INTR_MSIX) 5900 reg = E1000_READ_REG(hw, 5901 E1000_EITR_82574(rque->msix)); 5902 else 5903 reg = E1000_READ_REG(hw, E1000_ITR); 5904 } 5905 5906 if (hw->mac.type < igb_mac_min) { 5907 if (reg > 0) 5908 rate = EM_INTS_TO_ITR(reg); 5909 else 5910 rate = 0; 5911 } else { 5912 usec = (reg & IGB_QVECTOR_MASK); 5913 if (usec > 0) 5914 rate = IGB_EITR_TO_INTS(usec); 5915 else 5916 rate = 0; 5917 } 5918 5919 error = sysctl_handle_int(oidp, &rate, 0, req); 5920 if (error || !req->newptr) 5921 return error; 5922 return 0; 5923 } 5924 5925 /* 5926 * Add sysctl variables, one per statistic, to the system. 5927 */ 5928 static void 5929 em_add_hw_stats(struct e1000_softc *sc) 5930 { 5931 device_t dev = iflib_get_dev(sc->ctx); 5932 struct em_tx_queue *tx_que = sc->tx_queues; 5933 struct em_rx_queue *rx_que = sc->rx_queues; 5934 5935 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 5936 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 5937 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 5938 struct e1000_hw_stats *stats; 5939 5940 struct sysctl_oid *stat_node, *queue_node, *int_node; 5941 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 5942 5943 #define QUEUE_NAME_LEN 32 5944 char namebuf[QUEUE_NAME_LEN]; 5945 5946 /* Driver Statistics */ 5947 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5948 CTLFLAG_RD, &sc->dropped_pkts, 5949 "Driver dropped packets"); 5950 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 5951 CTLFLAG_RD, &sc->link_irq, 5952 "Link MSI-X IRQ Handled"); 5953 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 5954 CTLFLAG_RD, &sc->rx_overruns, 5955 "RX overruns"); 5956 if (!sc->vf_ifp) { 5957 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 5958 CTLTYPE_UINT | CTLFLAG_RD, 5959 sc, E1000_CTRL, em_sysctl_reg_handler, "IU", 5960 "Device Control Register"); 5961 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 5962 CTLTYPE_UINT | CTLFLAG_RD, 5963 sc, E1000_RCTL, em_sysctl_reg_handler, "IU", 5964 "Receiver Control Register"); 5965 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 5966 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 5967 "Flow Control High Watermark"); 5968 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 5969 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 5970 "Flow Control Low Watermark"); 5971 } 5972 5973 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 5974 struct tx_ring *txr = &tx_que->txr; 5975 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 5976 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5977 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 5978 queue_list = SYSCTL_CHILDREN(queue_node); 5979 5980 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 5981 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 5982 true, em_sysctl_interrupt_rate_handler, 5983 "IU", "Interrupt Rate"); 5984 5985 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 5986 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD, 5987 em_sysctl_tx_ring_handler, "IU", 5988 "Transmit Descriptor Head"); 5989 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 5990 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL, 5991 em_sysctl_tx_ring_handler, "IU", 5992 "Transmit Descriptor Tail"); 5993 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 5994 CTLFLAG_RD, &txr->tx_irq, 5995 "Queue MSI-X Transmit Interrupts"); 5996 } 5997 5998 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 5999 struct rx_ring *rxr = &rx_que->rxr; 6000 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 6001 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 6002 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 6003 queue_list = SYSCTL_CHILDREN(queue_node); 6004 6005 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 6006 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 6007 false, em_sysctl_interrupt_rate_handler, 6008 "IU", "Interrupt Rate"); 6009 6010 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 6011 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD, 6012 em_sysctl_rx_ring_handler, "IU", 6013 "Receive Descriptor Head"); 6014 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 6015 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL, 6016 em_sysctl_rx_ring_handler, "IU", 6017 "Receive Descriptor Tail"); 6018 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 6019 CTLFLAG_RD, &rxr->rx_irq, 6020 "Queue MSI-X Receive Interrupts"); 6021 } 6022 6023 /* MAC stats get their own sub node */ 6024 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 6025 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 6026 stat_list = SYSCTL_CHILDREN(stat_node); 6027 6028 /* 6029 ** VF adapter has a very limited set of stats 6030 ** since its not managing the metal, so to speak. 6031 */ 6032 if (sc->vf_ifp) { 6033 struct e1000_vf_stats *vfstats = &sc->ustats.vf_stats; 6034 6035 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 6036 CTLFLAG_RD, &vfstats->gprc, 6037 "Good Packets Received"); 6038 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 6039 CTLFLAG_RD, &vfstats->gptc, 6040 "Good Packets Transmitted"); 6041 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 6042 CTLFLAG_RD, &vfstats->gorc, 6043 "Good Octets Received"); 6044 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 6045 CTLFLAG_RD, &vfstats->gotc, 6046 "Good Octets Transmitted"); 6047 if (sc->hw.mac.type == e1000_vfadapt) { 6048 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 6049 "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc, 6050 "Multicast Packets Received"); 6051 } 6052 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 6053 "loopback_good_pkts_recvd", 6054 CTLFLAG_RD, &vfstats->gprlbc, 6055 "Good Loopback Packets Received"); 6056 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 6057 "loopback_good_pkts_txd", 6058 CTLFLAG_RD, &vfstats->gptlbc, 6059 "Good Loopback Packets Transmitted"); 6060 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 6061 "loopback_good_octets_recvd", 6062 CTLFLAG_RD, &vfstats->gorlbc, 6063 "Good Loopback Octets Received"); 6064 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 6065 "loopback_good_octets_txd", 6066 CTLFLAG_RD, &vfstats->gotlbc, 6067 "Good Loopback Octets Transmitted"); 6068 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 6069 "rx_csum_offload_good", 6070 CTLFLAG_RD, &sc->rx_csum_good, 6071 "Receive Checksum Offload Successes"); 6072 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 6073 "rx_csum_offload_errors", 6074 CTLFLAG_RD, &sc->rx_csum_errors, 6075 "Receive Checksum Offload Errors"); 6076 return; 6077 } 6078 6079 stats = &sc->ustats.stats; 6080 if (em_mac_has_eee(sc->hw.mac.type)) { 6081 struct sysctl_oid *eee_node; 6082 struct sysctl_oid_list *eee_list; 6083 6084 eee_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eee", 6085 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 6086 "Energy Efficient Ethernet statistics"); 6087 eee_list = SYSCTL_CHILDREN(eee_node); 6088 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "tx_lpi_count", 6089 CTLFLAG_RD, &stats->tlpic, "TX LPI event count"); 6090 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "rx_lpi_count", 6091 CTLFLAG_RD, &stats->rlpic, "RX LPI event count"); 6092 } 6093 6094 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 6095 CTLFLAG_RD, &stats->ecol, 6096 "Excessive collisions"); 6097 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 6098 CTLFLAG_RD, &stats->scc, 6099 "Single collisions"); 6100 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 6101 CTLFLAG_RD, &stats->mcc, 6102 "Multiple collisions"); 6103 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 6104 CTLFLAG_RD, &stats->latecol, 6105 "Late collisions"); 6106 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 6107 CTLFLAG_RD, &stats->colc, 6108 "Collision Count"); 6109 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 6110 CTLFLAG_RD, &stats->symerrs, 6111 "Symbol Errors"); 6112 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 6113 CTLFLAG_RD, &stats->sec, 6114 "Sequence Errors"); 6115 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 6116 CTLFLAG_RD, &stats->dc, 6117 "Defer Count"); 6118 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 6119 CTLFLAG_RD, &stats->mpc, 6120 "Missed Packets"); 6121 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 6122 CTLFLAG_RD, &stats->rlec, 6123 "Receive Length Errors"); 6124 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 6125 CTLFLAG_RD, &stats->rnbc, 6126 "Receive No Buffers"); 6127 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 6128 CTLFLAG_RD, &stats->ruc, 6129 "Receive Undersize"); 6130 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 6131 CTLFLAG_RD, &stats->rfc, 6132 "Fragmented Packets Received "); 6133 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 6134 CTLFLAG_RD, &stats->roc, 6135 "Oversized Packets Received"); 6136 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 6137 CTLFLAG_RD, &stats->rjc, 6138 "Recevied Jabber"); 6139 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 6140 CTLFLAG_RD, &stats->rxerrc, 6141 "Receive Errors"); 6142 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 6143 CTLFLAG_RD, &stats->crcerrs, 6144 "CRC errors"); 6145 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 6146 CTLFLAG_RD, &stats->algnerrc, 6147 "Alignment Errors"); 6148 /* On 82575 these are collision counts */ 6149 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 6150 CTLFLAG_RD, &stats->cexterr, 6151 "Collision/Carrier extension errors"); 6152 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 6153 CTLFLAG_RD, &stats->xonrxc, 6154 "XON Received"); 6155 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 6156 CTLFLAG_RD, &stats->xontxc, 6157 "XON Transmitted"); 6158 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 6159 CTLFLAG_RD, &stats->xoffrxc, 6160 "XOFF Received"); 6161 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 6162 CTLFLAG_RD, &stats->xofftxc, 6163 "XOFF Transmitted"); 6164 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 6165 CTLFLAG_RD, &stats->fcruc, 6166 "Unsupported Flow Control Received"); 6167 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 6168 CTLFLAG_RD, &stats->mgprc, 6169 "Management Packets Received"); 6170 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 6171 CTLFLAG_RD, &stats->mgpdc, 6172 "Management Packets Dropped"); 6173 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 6174 CTLFLAG_RD, &stats->mgptc, 6175 "Management Packets Transmitted"); 6176 6177 /* Packet Reception Stats */ 6178 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 6179 CTLFLAG_RD, &stats->tpr, 6180 "Total Packets Received "); 6181 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 6182 CTLFLAG_RD, &stats->gprc, 6183 "Good Packets Received"); 6184 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 6185 CTLFLAG_RD, &stats->bprc, 6186 "Broadcast Packets Received"); 6187 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 6188 CTLFLAG_RD, &stats->mprc, 6189 "Multicast Packets Received"); 6190 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 6191 CTLFLAG_RD, &stats->prc64, 6192 "64 byte frames received "); 6193 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 6194 CTLFLAG_RD, &stats->prc127, 6195 "65-127 byte frames received"); 6196 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 6197 CTLFLAG_RD, &stats->prc255, 6198 "128-255 byte frames received"); 6199 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 6200 CTLFLAG_RD, &stats->prc511, 6201 "256-511 byte frames received"); 6202 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 6203 CTLFLAG_RD, &stats->prc1023, 6204 "512-1023 byte frames received"); 6205 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 6206 CTLFLAG_RD, &stats->prc1522, 6207 "1023-1522 byte frames received"); 6208 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 6209 CTLFLAG_RD, &stats->gorc, 6210 "Good Octets Received"); 6211 6212 /* Packet Transmission Stats */ 6213 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 6214 CTLFLAG_RD, &stats->gotc, 6215 "Good Octets Transmitted"); 6216 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 6217 CTLFLAG_RD, &stats->tpt, 6218 "Total Packets Transmitted"); 6219 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 6220 CTLFLAG_RD, &stats->gptc, 6221 "Good Packets Transmitted"); 6222 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 6223 CTLFLAG_RD, &stats->bptc, 6224 "Broadcast Packets Transmitted"); 6225 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 6226 CTLFLAG_RD, &stats->mptc, 6227 "Multicast Packets Transmitted"); 6228 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 6229 CTLFLAG_RD, &stats->ptc64, 6230 "64 byte frames transmitted "); 6231 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 6232 CTLFLAG_RD, &stats->ptc127, 6233 "65-127 byte frames transmitted"); 6234 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 6235 CTLFLAG_RD, &stats->ptc255, 6236 "128-255 byte frames transmitted"); 6237 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 6238 CTLFLAG_RD, &stats->ptc511, 6239 "256-511 byte frames transmitted"); 6240 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 6241 CTLFLAG_RD, &stats->ptc1023, 6242 "512-1023 byte frames transmitted"); 6243 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 6244 CTLFLAG_RD, &stats->ptc1522, 6245 "1024-1522 byte frames transmitted"); 6246 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 6247 CTLFLAG_RD, &stats->tsctc, 6248 "TSO Contexts Transmitted"); 6249 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 6250 CTLFLAG_RD, &stats->tsctfc, 6251 "TSO Contexts Failed"); 6252 6253 /* Interrupt Stats */ 6254 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 6255 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 6256 int_list = SYSCTL_CHILDREN(int_node); 6257 6258 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 6259 CTLFLAG_RD, &stats->iac, 6260 "Interrupt Assertion Count"); 6261 6262 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 6263 CTLFLAG_RD, &stats->icrxptc, 6264 "Interrupt Cause Rx Pkt Timer Expire Count"); 6265 6266 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 6267 CTLFLAG_RD, &stats->icrxatc, 6268 "Interrupt Cause Rx Abs Timer Expire Count"); 6269 6270 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 6271 CTLFLAG_RD, &stats->ictxptc, 6272 "Interrupt Cause Tx Pkt Timer Expire Count"); 6273 6274 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 6275 CTLFLAG_RD, &stats->ictxatc, 6276 "Interrupt Cause Tx Abs Timer Expire Count"); 6277 6278 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 6279 CTLFLAG_RD, &stats->ictxqec, 6280 "Interrupt Cause Tx Queue Empty Count"); 6281 6282 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 6283 CTLFLAG_RD, &stats->ictxqmtc, 6284 "Interrupt Cause Tx Queue Min Thresh Count"); 6285 6286 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 6287 CTLFLAG_RD, &stats->icrxdmtc, 6288 "Interrupt Cause Rx Desc Min Thresh Count"); 6289 6290 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun", 6291 CTLFLAG_RD, &stats->icrxoc, 6292 "Interrupt Cause Receiver Overrun Count"); 6293 } 6294 6295 static void 6296 em_fw_version_locked(if_ctx_t ctx) 6297 { 6298 struct e1000_softc *sc = iflib_get_softc(ctx); 6299 struct e1000_hw *hw = &sc->hw; 6300 struct e1000_fw_version *fw_ver = &sc->fw_ver; 6301 uint16_t eep = 0; 6302 6303 /* 6304 * em_fw_version_locked() must run under the IFLIB_CTX_LOCK to meet 6305 * the NVM locking model, so we do it in em_if_attach_pre() and store 6306 * the info in the softc 6307 */ 6308 ASSERT_CTX_LOCK_HELD(hw); 6309 6310 *fw_ver = (struct e1000_fw_version){0}; 6311 6312 if (hw->mac.type >= igb_mac_min) { 6313 /* 6314 * Use the Shared Code for igb(4) 6315 */ 6316 e1000_get_fw_version(hw, fw_ver); 6317 } else { 6318 /* 6319 * Otherwise, EEPROM version should be present on (almost?) 6320 * all devices here 6321 */ 6322 if(e1000_read_nvm(hw, NVM_VERSION, 1, &eep)) { 6323 INIT_DEBUGOUT("can't get EEPROM version"); 6324 return; 6325 } 6326 6327 fw_ver->eep_major = (eep & NVM_MAJOR_MASK) >> NVM_MAJOR_SHIFT; 6328 fw_ver->eep_minor = (eep & NVM_MINOR_MASK) >> NVM_MINOR_SHIFT; 6329 fw_ver->eep_build = (eep & NVM_IMAGE_ID_MASK); 6330 } 6331 } 6332 6333 static void 6334 em_sbuf_fw_version(struct e1000_fw_version *fw_ver, struct sbuf *buf) 6335 { 6336 const char *space = ""; 6337 6338 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 6339 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 6340 fw_ver->eep_minor, fw_ver->eep_build); 6341 space = " "; 6342 } 6343 6344 if (fw_ver->invm_major || fw_ver->invm_minor || 6345 fw_ver->invm_img_type) { 6346 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 6347 space, fw_ver->invm_major, fw_ver->invm_minor, 6348 fw_ver->invm_img_type); 6349 space = " "; 6350 } 6351 6352 if (fw_ver->or_valid) { 6353 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 6354 space, fw_ver->or_major, fw_ver->or_build, 6355 fw_ver->or_patch); 6356 space = " "; 6357 } 6358 6359 if (fw_ver->etrack_id) 6360 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 6361 } 6362 6363 static void 6364 em_print_fw_version(struct e1000_softc *sc ) 6365 { 6366 device_t dev = sc->dev; 6367 struct sbuf *buf; 6368 int error = 0; 6369 6370 buf = sbuf_new_auto(); 6371 if (!buf) { 6372 device_printf(dev, "Could not allocate sbuf for output.\n"); 6373 return; 6374 } 6375 6376 em_sbuf_fw_version(&sc->fw_ver, buf); 6377 6378 error = sbuf_finish(buf); 6379 if (error) 6380 device_printf(dev, "Error finishing sbuf: %d\n", error); 6381 else if (sbuf_len(buf)) 6382 device_printf(dev, "%s\n", sbuf_data(buf)); 6383 6384 sbuf_delete(buf); 6385 } 6386 6387 static int 6388 em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 6389 { 6390 struct e1000_softc *sc = (struct e1000_softc *)arg1; 6391 device_t dev = sc->dev; 6392 struct sbuf *buf; 6393 int error = 0; 6394 6395 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 6396 if (!buf) { 6397 device_printf(dev, "Could not allocate sbuf for output.\n"); 6398 return (ENOMEM); 6399 } 6400 6401 em_sbuf_fw_version(&sc->fw_ver, buf); 6402 6403 error = sbuf_finish(buf); 6404 if (error) 6405 device_printf(dev, "Error finishing sbuf: %d\n", error); 6406 6407 sbuf_delete(buf); 6408 6409 return (0); 6410 } 6411 6412 /********************************************************************** 6413 * 6414 * This routine provides a way to dump out the adapter eeprom, 6415 * often a useful debug/service tool. This only dumps the first 6416 * 32 words, stuff that matters is in that extent. 6417 * 6418 **********************************************************************/ 6419 static int 6420 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 6421 { 6422 struct e1000_softc *sc = (struct e1000_softc *)arg1; 6423 int error; 6424 int result; 6425 6426 result = -1; 6427 error = sysctl_handle_int(oidp, &result, 0, req); 6428 6429 if (error || !req->newptr) 6430 return (error); 6431 6432 /* 6433 * This value will cause a hex dump of the 6434 * first 32 16-bit words of the EEPROM to 6435 * the screen. 6436 */ 6437 if (result == 1) 6438 em_print_nvm_info(sc); 6439 6440 return (error); 6441 } 6442 6443 static void 6444 em_print_nvm_info(struct e1000_softc *sc) 6445 { 6446 struct e1000_hw *hw = &sc->hw; 6447 struct sx *iflib_ctx_lock = iflib_ctx_lock_get(sc->ctx); 6448 u16 eeprom_data; 6449 int i, j, row = 0; 6450 6451 /* Its a bit crude, but it gets the job done */ 6452 printf("\nInterface EEPROM Dump:\n"); 6453 printf("Offset\n0x0000 "); 6454 6455 /* We rely on the IFLIB_CTX_LOCK as part of NVM locking model */ 6456 sx_xlock(iflib_ctx_lock); 6457 ASSERT_CTX_LOCK_HELD(hw); 6458 for (i = 0, j = 0; i < 32; i++, j++) { 6459 if (j == 8) { /* Make the offset block */ 6460 j = 0; ++row; 6461 printf("\n0x00%x0 ",row); 6462 } 6463 eeprom_data = 0; 6464 if (e1000_read_nvm(hw, i, 1, &eeprom_data) != 6465 E1000_SUCCESS) { 6466 printf("\nNVM read failed at offset %#x\n", i); 6467 break; 6468 } 6469 printf("%04x ", eeprom_data); 6470 } 6471 sx_xunlock(iflib_ctx_lock); 6472 printf("\n"); 6473 } 6474 6475 static int 6476 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 6477 { 6478 struct em_int_delay_info *info; 6479 struct e1000_softc *sc; 6480 u32 regval; 6481 int error, usecs, ticks; 6482 6483 info = (struct em_int_delay_info *) arg1; 6484 usecs = info->value; 6485 error = sysctl_handle_int(oidp, &usecs, 0, req); 6486 if (error != 0 || req->newptr == NULL) 6487 return (error); 6488 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 6489 return (EINVAL); 6490 info->value = usecs; 6491 ticks = EM_USECS_TO_TICKS(usecs); 6492 6493 sc = info->sc; 6494 6495 regval = E1000_READ_OFFSET(&sc->hw, info->offset); 6496 regval = (regval & ~0xffff) | (ticks & 0xffff); 6497 /* Handle a few special cases. */ 6498 switch (info->offset) { 6499 case E1000_RDTR: 6500 break; 6501 case E1000_TIDV: 6502 if (ticks == 0) { 6503 sc->txd_cmd &= ~E1000_TXD_CMD_IDE; 6504 /* Don't write 0 into the TIDV register. */ 6505 regval++; 6506 } else 6507 sc->txd_cmd |= E1000_TXD_CMD_IDE; 6508 break; 6509 } 6510 E1000_WRITE_OFFSET(&sc->hw, info->offset, regval); 6511 return (0); 6512 } 6513 6514 static int 6515 em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 6516 { 6517 struct e1000_softc *sc; 6518 u32 reg, val, shift; 6519 int error, mask; 6520 6521 sc = oidp->oid_arg1; 6522 switch (oidp->oid_arg2) { 6523 case 0: 6524 reg = E1000_DTXTCPFLGL; 6525 shift = 0; 6526 break; 6527 case 1: 6528 reg = E1000_DTXTCPFLGL; 6529 shift = 16; 6530 break; 6531 case 2: 6532 reg = E1000_DTXTCPFLGH; 6533 shift = 0; 6534 break; 6535 default: 6536 return (EINVAL); 6537 break; 6538 } 6539 val = E1000_READ_REG(&sc->hw, reg); 6540 mask = (val >> shift) & 0xfff; 6541 error = sysctl_handle_int(oidp, &mask, 0, req); 6542 if (error != 0 || req->newptr == NULL) 6543 return (error); 6544 if (mask < 0 || mask > 0xfff) 6545 return (EINVAL); 6546 val = (val & ~(0xfff << shift)) | (mask << shift); 6547 E1000_WRITE_REG(&sc->hw, reg, val); 6548 return (0); 6549 } 6550 6551 static void 6552 em_add_int_delay_sysctl(struct e1000_softc *sc, const char *name, 6553 const char *description, struct em_int_delay_info *info, int offset, 6554 int value) 6555 { 6556 info->sc = sc; 6557 info->offset = offset; 6558 info->value = value; 6559 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 6560 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 6561 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 6562 info, 0, em_sysctl_int_delay, "I", description); 6563 } 6564 6565 /* 6566 * Set flow control using sysctl: 6567 * Flow control values: 6568 * 0 - off 6569 * 1 - rx pause 6570 * 2 - tx pause 6571 * 3 - full 6572 */ 6573 static int 6574 em_set_flowcntl(SYSCTL_HANDLER_ARGS) 6575 { 6576 int error; 6577 static int input = 3; /* default is full */ 6578 struct e1000_softc *sc = (struct e1000_softc *) arg1; 6579 6580 error = sysctl_handle_int(oidp, &input, 0, req); 6581 6582 if ((error) || (req->newptr == NULL)) 6583 return (error); 6584 6585 if (input == sc->fc) /* no change? */ 6586 return (error); 6587 6588 switch (input) { 6589 case e1000_fc_rx_pause: 6590 case e1000_fc_tx_pause: 6591 case e1000_fc_full: 6592 case e1000_fc_none: 6593 sc->hw.fc.requested_mode = input; 6594 sc->fc = input; 6595 break; 6596 default: 6597 /* Do nothing */ 6598 return (error); 6599 } 6600 6601 sc->hw.fc.current_mode = sc->hw.fc.requested_mode; 6602 e1000_force_mac_fc(&sc->hw); 6603 return (error); 6604 } 6605 6606 static void 6607 em_sysctl_request_reinit(struct e1000_softc *sc) 6608 { 6609 if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) 6610 return; 6611 6612 iflib_request_reset(sc->ctx); 6613 iflib_admin_intr_deferred(sc->ctx); 6614 } 6615 6616 /* 6617 * Manage DMA Coalesce: 6618 * Control values: 6619 * 0/1 - off/on 6620 * Legal timer values are: 6621 * 250,500,1000-10000 in thousands 6622 */ 6623 static int 6624 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) 6625 { 6626 struct e1000_softc *sc = (struct e1000_softc *) arg1; 6627 int error; 6628 6629 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 6630 6631 if ((error) || (req->newptr == NULL)) 6632 return (error); 6633 6634 switch (sc->dmac) { 6635 case 0: 6636 /* Disabling */ 6637 break; 6638 case 1: /* Just enable and use default */ 6639 sc->dmac = 1000; 6640 break; 6641 case 250: 6642 case 500: 6643 case 1000: 6644 case 2000: 6645 case 3000: 6646 case 4000: 6647 case 5000: 6648 case 6000: 6649 case 7000: 6650 case 8000: 6651 case 9000: 6652 case 10000: 6653 /* Legal values - allow */ 6654 break; 6655 default: 6656 /* Do nothing, illegal value */ 6657 sc->dmac = 0; 6658 return (EINVAL); 6659 } 6660 /* Reinit the interface */ 6661 em_sysctl_request_reinit(sc); 6662 return (error); 6663 } 6664 6665 /* 6666 * Manage Energy Efficient Ethernet: 6667 * Control values: 6668 * 0/1 - enabled/disabled 6669 */ 6670 static int 6671 em_sysctl_eee(SYSCTL_HANDLER_ARGS) 6672 { 6673 struct e1000_softc *sc = (struct e1000_softc *) arg1; 6674 int error, value; 6675 6676 if (sc->hw.mac.type < igb_mac_min) 6677 value = sc->hw.dev_spec.ich8lan.eee_disable; 6678 else 6679 value = sc->hw.dev_spec._82575.eee_disable; 6680 error = sysctl_handle_int(oidp, &value, 0, req); 6681 if (error || req->newptr == NULL) 6682 return (error); 6683 if (sc->hw.mac.type < igb_mac_min) 6684 sc->hw.dev_spec.ich8lan.eee_disable = (value != 0); 6685 else 6686 sc->hw.dev_spec._82575.eee_disable = (value != 0); 6687 em_sysctl_request_reinit(sc); 6688 6689 return (0); 6690 } 6691 6692 static int 6693 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 6694 { 6695 struct e1000_softc *sc; 6696 int error; 6697 int result; 6698 6699 result = -1; 6700 error = sysctl_handle_int(oidp, &result, 0, req); 6701 6702 if (error || !req->newptr) 6703 return (error); 6704 6705 if (result == 1) { 6706 sc = (struct e1000_softc *) arg1; 6707 em_print_debug_info(sc); 6708 } 6709 6710 return (error); 6711 } 6712 6713 static int 6714 em_get_rs(SYSCTL_HANDLER_ARGS) 6715 { 6716 struct e1000_softc *sc = (struct e1000_softc *) arg1; 6717 int error; 6718 int result; 6719 6720 result = 0; 6721 error = sysctl_handle_int(oidp, &result, 0, req); 6722 6723 if (error || !req->newptr || result != 1) 6724 return (error); 6725 em_dump_rs(sc); 6726 6727 return (error); 6728 } 6729 6730 static void 6731 em_if_debug(if_ctx_t ctx) 6732 { 6733 em_dump_rs(iflib_get_softc(ctx)); 6734 } 6735 6736 /* 6737 * This routine is meant to be fluid, add whatever is 6738 * needed for debugging a problem. -jfv 6739 */ 6740 static void 6741 em_print_debug_info(struct e1000_softc *sc) 6742 { 6743 device_t dev = iflib_get_dev(sc->ctx); 6744 if_t ifp = iflib_get_ifp(sc->ctx); 6745 struct tx_ring *txr; 6746 struct rx_ring *rxr; 6747 6748 if (sc->tx_queues == NULL || sc->rx_queues == NULL) { 6749 device_printf(dev, "queue state is unavailable\n"); 6750 return; 6751 } 6752 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 6753 printf("Interface is RUNNING "); 6754 else 6755 printf("Interface is NOT RUNNING\n"); 6756 6757 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) 6758 printf("and INACTIVE\n"); 6759 else 6760 printf("and ACTIVE\n"); 6761 6762 for (int i = 0; i < sc->tx_num_queues; i++) { 6763 txr = &sc->tx_queues[i].txr; 6764 device_printf(dev, "TX Queue %d ------\n", i); 6765 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 6766 E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)), 6767 E1000_READ_REG(&sc->hw, E1000_TDT(txr->me))); 6768 6769 } 6770 for (int j = 0; j < sc->rx_num_queues; j++) { 6771 rxr = &sc->rx_queues[j].rxr; 6772 device_printf(dev, "RX Queue %d ------\n", j); 6773 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 6774 E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)), 6775 E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me))); 6776 } 6777 } 6778 6779 /* 6780 * 82574 only: 6781 * Write a new value to the EEPROM increasing the number of MSI-X 6782 * vectors from 3 to 5, for proper multiqueue support. 6783 */ 6784 static void 6785 em_enable_vectors_82574(if_ctx_t ctx) 6786 { 6787 struct e1000_softc *sc = iflib_get_softc(ctx); 6788 struct e1000_hw *hw = &sc->hw; 6789 device_t dev = iflib_get_dev(ctx); 6790 u16 edata; 6791 6792 e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 6793 if (bootverbose) 6794 device_printf(dev, "EM_NVM_PCIE_CTRL = %#06x\n", edata); 6795 if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { 6796 device_printf(dev, "Writing to eeprom: increasing " 6797 "reported MSI-X vectors from 3 to 5...\n"); 6798 edata &= ~(EM_NVM_MSIX_N_MASK); 6799 edata |= 4 << EM_NVM_MSIX_N_SHIFT; 6800 e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 6801 e1000_update_nvm_checksum(hw); 6802 device_printf(dev, "Writing to eeprom: done\n"); 6803 } 6804 } 6805