1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2024 Kevin Bowling <kbowling@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 #include "if_igb_iov.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #define em_mac_min e1000_82571 36 #define igb_mac_min e1000_82575 37 38 /********************************************************************* 39 * Driver version: 40 *********************************************************************/ 41 static const char em_driver_version[] = "7.7.8-fbsd"; 42 static const char igb_driver_version[] = "2.5.28-fbsd"; 43 44 /********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into e1000_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54 static const pci_vendor_info_t em_vendor_info_array[] = 55 { 56 /* Intel(R) - lem-class legacy devices */ 57 PVID(0x8086, E1000_DEV_ID_82540EM, 58 "Intel(R) Legacy PRO/1000 MT 82540EM"), 59 PVID(0x8086, E1000_DEV_ID_82540EM_LOM, 60 "Intel(R) Legacy PRO/1000 MT 82540EM (LOM)"), 61 PVID(0x8086, E1000_DEV_ID_82540EP, 62 "Intel(R) Legacy PRO/1000 MT 82540EP"), 63 PVID(0x8086, E1000_DEV_ID_82540EP_LOM, 64 "Intel(R) Legacy PRO/1000 MT 82540EP (LOM)"), 65 PVID(0x8086, E1000_DEV_ID_82540EP_LP, 66 "Intel(R) Legacy PRO/1000 MT 82540EP (Mobile)"), 67 68 PVID(0x8086, E1000_DEV_ID_82541EI, 69 "Intel(R) Legacy PRO/1000 MT 82541EI (Copper)"), 70 PVID(0x8086, E1000_DEV_ID_82541ER, 71 "Intel(R) Legacy PRO/1000 82541ER"), 72 PVID(0x8086, E1000_DEV_ID_82541ER_LOM, 73 "Intel(R) Legacy PRO/1000 MT 82541ER"), 74 PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, 75 "Intel(R) Legacy PRO/1000 MT 82541EI (Mobile)"), 76 PVID(0x8086, E1000_DEV_ID_82541GI, 77 "Intel(R) Legacy PRO/1000 MT 82541GI"), 78 PVID(0x8086, E1000_DEV_ID_82541GI_LF, 79 "Intel(R) Legacy PRO/1000 GT 82541PI"), 80 PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, 81 "Intel(R) Legacy PRO/1000 MT 82541GI (Mobile)"), 82 83 PVID(0x8086, E1000_DEV_ID_82542, 84 "Intel(R) Legacy PRO/1000 82542 (Fiber)"), 85 86 PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, 87 "Intel(R) Legacy PRO/1000 F 82543GC (Fiber)"), 88 PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, 89 "Intel(R) Legacy PRO/1000 T 82543GC (Copper)"), 90 91 PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, 92 "Intel(R) Legacy PRO/1000 XT 82544EI (Copper)"), 93 PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, 94 "Intel(R) Legacy PRO/1000 XF 82544EI (Fiber)"), 95 PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, 96 "Intel(R) Legacy PRO/1000 T 82544GC (Copper)"), 97 PVID(0x8086, E1000_DEV_ID_82544GC_LOM, 98 "Intel(R) Legacy PRO/1000 XT 82544GC (LOM)"), 99 100 PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, 101 "Intel(R) Legacy PRO/1000 MT 82545EM (Copper)"), 102 PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, 103 "Intel(R) Legacy PRO/1000 MF 82545EM (Fiber)"), 104 PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, 105 "Intel(R) Legacy PRO/1000 MT 82545GM (Copper)"), 106 PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, 107 "Intel(R) Legacy PRO/1000 MF 82545GM (Fiber)"), 108 PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, 109 "Intel(R) Legacy PRO/1000 MB 82545GM (SERDES)"), 110 111 PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, 112 "Intel(R) Legacy PRO/1000 MT 82546EB (Copper)"), 113 PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, 114 "Intel(R) Legacy PRO/1000 MF 82546EB (Fiber)"), 115 PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, 116 "Intel(R) Legacy PRO/1000 MT 82546EB (Quad Copper"), 117 PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, 118 "Intel(R) Legacy PRO/1000 MT 82546GB (Copper)"), 119 PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, 120 "Intel(R) Legacy PRO/1000 MF 82546GB (Fiber)"), 121 PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, 122 "Intel(R) Legacy PRO/1000 MB 82546GB (SERDES)"), 123 PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, 124 "Intel(R) Legacy PRO/1000 P 82546GB (PCIe)"), 125 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, 126 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 127 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 128 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 129 130 PVID(0x8086, E1000_DEV_ID_82547EI, 131 "Intel(R) Legacy PRO/1000 CT 82547EI"), 132 PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, 133 "Intel(R) Legacy PRO/1000 CT 82547EI (Mobile)"), 134 PVID(0x8086, E1000_DEV_ID_82547GI, 135 "Intel(R) Legacy PRO/1000 CT 82547GI"), 136 137 /* Intel(R) - em-class devices */ 138 PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, 139 "Intel(R) PRO/1000 PT 82571EB/82571GB (Copper)"), 140 PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, 141 "Intel(R) PRO/1000 PF 82571EB/82571GB (Fiber)"), 142 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, 143 "Intel(R) PRO/1000 PB 82571EB (SERDES)"), 144 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 145 "Intel(R) PRO/1000 82571EB (Dual Mezzanine)"), 146 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 147 "Intel(R) PRO/1000 82571EB (Quad Mezzanine)"), 148 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 149 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 150 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 151 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 152 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 153 "Intel(R) PRO/1000 PF 82571EB (Quad Fiber)"), 154 PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 155 "Intel(R) PRO/1000 PT 82571PT (Quad Copper)"), 156 PVID(0x8086, E1000_DEV_ID_82572EI, 157 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 158 PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, 159 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 160 PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, 161 "Intel(R) PRO/1000 PF 82572EI (Fiber)"), 162 PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, 163 "Intel(R) PRO/1000 82572EI (SERDES)"), 164 PVID(0x8086, E1000_DEV_ID_82573E, 165 "Intel(R) PRO/1000 82573E (Copper)"), 166 PVID(0x8086, E1000_DEV_ID_82573E_IAMT, 167 "Intel(R) PRO/1000 82573E AMT (Copper)"), 168 PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 82573L"), 169 PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) 82583V"), 170 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 171 "Intel(R) 80003ES2LAN (Copper)"), 172 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 173 "Intel(R) 80003ES2LAN (SERDES)"), 174 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 175 "Intel(R) 80003ES2LAN (Dual Copper)"), 176 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 177 "Intel(R) 80003ES2LAN (Dual SERDES)"), 178 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, 179 "Intel(R) 82566MM ICH8 AMT (Mobile)"), 180 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) 82566DM ICH8 AMT"), 181 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) 82566DC ICH8"), 182 PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) 82562V ICH8"), 183 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) 82562GT ICH8"), 184 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) 82562G ICH8"), 185 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) 82566MC ICH8"), 186 PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) 82567V-3 ICH8"), 187 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, 188 "Intel(R) 82567LM ICH9 AMT"), 189 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, 190 "Intel(R) 82566DM-2 ICH9 AMT"), 191 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) 82566DC-2 ICH9"), 192 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) 82567LF ICH9"), 193 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) 82567V ICH9"), 194 PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) 82562V-2 ICH9"), 195 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) 82562GT-2 ICH9"), 196 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) 82562G-2 ICH9"), 197 PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) 82567LM-4 ICH9"), 198 PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) Gigabit CT 82574L"), 199 PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) 82574L-Apple"), 200 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) 82567LM-2 ICH10"), 201 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) 82567LF-2 ICH10"), 202 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) 82567V-2 ICH10"), 203 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) 82567LM-3 ICH10"), 204 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) 82567LF-3 ICH10"), 205 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) 82567V-4 ICH10"), 206 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) 82577LM"), 207 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) 82577LC"), 208 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) 82578DM"), 209 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) 82578DC"), 210 PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) 82579LM"), 211 PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) 82579V"), 212 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) I217-LM LPT"), 213 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) I217-V LPT"), 214 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, 215 "Intel(R) I218-LM LPTLP"), 216 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) I218-V LPTLP"), 217 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) I218-LM (2)"), 218 PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) I218-V (2)"), 219 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) I218-LM (3)"), 220 PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) I218-V (3)"), 221 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) I219-LM SPT"), 222 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) I219-V SPT"), 223 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, 224 "Intel(R) I219-LM SPT-H(2)"), 225 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, 226 "Intel(R) I219-V SPT-H(2)"), 227 PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, 228 "Intel(R) I219-LM LBG(3)"), 229 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, 230 "Intel(R) I219-LM SPT(4)"), 231 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) I219-V SPT(4)"), 232 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, 233 "Intel(R) I219-LM SPT(5)"), 234 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) I219-V SPT(5)"), 235 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM6, 236 "Intel(R) I219-LM CNP(6)"), 237 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V6, "Intel(R) I219-V CNP(6)"), 238 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM7, 239 "Intel(R) I219-LM CNP(7)"), 240 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V7, "Intel(R) I219-V CNP(7)"), 241 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM8, 242 "Intel(R) I219-LM ICP(8)"), 243 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V8, "Intel(R) I219-V ICP(8)"), 244 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM9, 245 "Intel(R) I219-LM ICP(9)"), 246 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V9, "Intel(R) I219-V ICP(9)"), 247 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM10, 248 "Intel(R) I219-LM CMP(10)"), 249 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V10, 250 "Intel(R) I219-V CMP(10)"), 251 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM11, 252 "Intel(R) I219-LM CMP(11)"), 253 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V11, 254 "Intel(R) I219-V CMP(11)"), 255 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM12, 256 "Intel(R) I219-LM CMP(12)"), 257 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V12, 258 "Intel(R) I219-V CMP(12)"), 259 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM13, 260 "Intel(R) I219-LM TGP(13)"), 261 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V13, 262 "Intel(R) I219-V TGP(13)"), 263 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM14, 264 "Intel(R) I219-LM TGP(14)"), 265 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V14, 266 "Intel(R) I219-V GTP(14)"), 267 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM15, 268 "Intel(R) I219-LM TGP(15)"), 269 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V15, 270 "Intel(R) I219-V TGP(15)"), 271 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM16, 272 "Intel(R) I219-LM ADL(16)"), 273 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V16, 274 "Intel(R) I219-V ADL(16)"), 275 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM17, 276 "Intel(R) I219-LM ADL(17)"), 277 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V17, 278 "Intel(R) I219-V ADL(17)"), 279 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_LM18, 280 "Intel(R) I219-LM MTP(18)"), 281 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_V18, 282 "Intel(R) I219-V MTP(18)"), 283 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM19, 284 "Intel(R) I219-LM ADL(19)"), 285 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V19, 286 "Intel(R) I219-V ADL(19)"), 287 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM20, 288 "Intel(R) I219-LM LNL(20)"), 289 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V20, 290 "Intel(R) I219-V LNL(20)"), 291 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM21, 292 "Intel(R) I219-LM LNL(21)"), 293 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V21, 294 "Intel(R) I219-V LNL(21)"), 295 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM22, 296 "Intel(R) I219-LM RPL(22)"), 297 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V22, 298 "Intel(R) I219-V RPL(22)"), 299 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM23, 300 "Intel(R) I219-LM RPL(23)"), 301 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V23, 302 "Intel(R) I219-V RPL(23)"), 303 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_LM24, 304 "Intel(R) I219-LM ARL(24)"), 305 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_V24, 306 "Intel(R) I219-V ARL(24)"), 307 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM25, 308 "Intel(R) I219-LM PTP(25)"), 309 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V25, 310 "Intel(R) I219-V PTP(25)"), 311 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM26, 312 "Intel(R) I219-LM PTP(26)"), 313 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V26, 314 "Intel(R) I219-V PTP(26)"), 315 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM27, 316 "Intel(R) I219-LM PTP(27)"), 317 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V27, 318 "Intel(R) I219-V PTP(27)"), 319 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_LM29, 320 "Intel(R) I219-LM NVL(29)"), 321 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_V29, 322 "Intel(R) I219-V NVL(29)"), 323 /* required last entry */ 324 PVID_END 325 }; 326 327 static const pci_vendor_info_t igb_vendor_info_array[] = 328 { 329 /* Intel(R) - igb-class devices */ 330 PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, 331 "Intel(R) PRO/1000 82575EB (Copper)"), 332 PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 333 "Intel(R) PRO/1000 82575EB (SERDES)"), 334 PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 335 "Intel(R) PRO/1000 VT 82575GB (Quad Copper)"), 336 PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 82576"), 337 PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 82576NS"), 338 PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, 339 "Intel(R) PRO/1000 82576NS (SERDES)"), 340 PVID(0x8086, E1000_DEV_ID_82576_FIBER, 341 "Intel(R) PRO/1000 EF 82576 (Dual Fiber)"), 342 PVID(0x8086, E1000_DEV_ID_82576_SERDES, 343 "Intel(R) PRO/1000 82576 (Dual SERDES)"), 344 PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 345 "Intel(R) PRO/1000 ET 82576 (Quad SERDES)"), 346 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 347 "Intel(R) PRO/1000 ET 82576 (Quad Copper)"), 348 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 349 "Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"), 350 PVID(0x8086, E1000_DEV_ID_82580_COPPER, 351 "Intel(R) I340 82580 (Copper)"), 352 PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"), 353 PVID(0x8086, E1000_DEV_ID_82580_SERDES, 354 "Intel(R) I340 82580 (SERDES)"), 355 PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) I340 82580 (SGMII)"), 356 PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 357 "Intel(R) I340-T2 82580 (Dual Copper)"), 358 PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, 359 "Intel(R) I340-F4 82580 (Quad Fiber)"), 360 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, 361 "Intel(R) DH89XXCC (SERDES)"), 362 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, 363 "Intel(R) I347-AT4 DH89XXCC"), 364 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) DH89XXCC (SFP)"), 365 PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, 366 "Intel(R) DH89XXCC (Backplane)"), 367 PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) I350 (Copper)"), 368 PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"), 369 PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"), 370 PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"), 371 PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"), 372 PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, 373 "Intel(R) I210 IT (Copper)"), 374 PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) I210 (OEM)"), 375 PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, 376 "Intel(R) I210 Flashless (Copper)"), 377 PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, 378 "Intel(R) I210 Flashless (SERDES)"), 379 PVID(0x8086, E1000_DEV_ID_I210_SGMII_FLASHLESS, 380 "Intel(R) I210 Flashless (SGMII)"), 381 PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) I210 (Fiber)"), 382 PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) I210 (SERDES)"), 383 PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) I210 (SGMII)"), 384 PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) I211 (Copper)"), 385 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 386 "Intel(R) I354 (1.0 GbE Backplane)"), 387 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 388 "Intel(R) I354 (2.5 GbE Backplane)"), 389 PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) I354 (SGMII)"), 390 /* required last entry */ 391 PVID_END 392 }; 393 394 static const pci_vendor_info_t igbv_vendor_info_array[] = { 395 PVID(0x8086, E1000_DEV_ID_82576_VF, 396 "Intel(R) PRO/1000 82576 Virtual Function"), 397 PVID(0x8086, E1000_DEV_ID_82576_VF_HV, 398 "Intel(R) PRO/1000 82576 Hyper-V Virtual Function"), 399 PVID(0x8086, E1000_DEV_ID_I350_VF, 400 "Intel(R) I350 Virtual Function"), 401 PVID(0x8086, E1000_DEV_ID_I350_VF_HV, 402 "Intel(R) I350 Hyper-V Virtual Function"), 403 PVID_END 404 }; 405 406 /********************************************************************* 407 * Function prototypes 408 *********************************************************************/ 409 static void *em_register(device_t); 410 static void *igb_register(device_t); 411 static void *igbv_register(device_t); 412 static int igb_device_attach(device_t); 413 #ifdef PCI_IOV 414 static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *); 415 static void igb_device_iov_uninit(device_t); 416 #endif 417 static int em_if_detach(if_ctx_t); 418 static int em_if_shutdown(if_ctx_t); 419 static int em_if_suspend(if_ctx_t); 420 static int em_if_resume(if_ctx_t); 421 422 static int em_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 423 int); 424 static int em_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 425 int); 426 static void em_if_queues_free(if_ctx_t); 427 428 static uint64_t em_if_get_vf_counter(if_ctx_t, ift_counter); 429 static uint64_t em_if_get_counter(if_ctx_t, ift_counter); 430 static void em_if_init(if_ctx_t); 431 static void em_if_stop(if_ctx_t); 432 static void em_fence_pci_busmaster(struct e1000_softc *); 433 static int em_enable_pci_busmaster(struct e1000_softc *); 434 static void em_if_media_status(if_ctx_t, struct ifmediareq *); 435 static int em_if_media_change(if_ctx_t); 436 static int em_if_mtu_set(if_ctx_t, uint32_t); 437 static void em_if_timer(if_ctx_t, uint16_t); 438 static void em_if_vlan_register(if_ctx_t, u16); 439 static void em_if_vlan_unregister(if_ctx_t, u16); 440 static bool em_if_needs_restart(if_ctx_t, enum iflib_restart_event); 441 442 static void em_identify_hardware(if_ctx_t); 443 static int em_allocate_pci_resources(if_ctx_t); 444 static void em_free_pci_resources(if_ctx_t); 445 static int em_reset(if_ctx_t); 446 static int em_setup_interface(if_ctx_t); 447 static int em_setup_msix(if_ctx_t); 448 449 static void em_initialize_transmit_unit(if_ctx_t); 450 static void em_initialize_receive_unit(if_ctx_t); 451 452 static void em_if_intr_enable(if_ctx_t); 453 static void em_if_intr_disable(if_ctx_t); 454 static void igb_if_intr_enable(if_ctx_t); 455 static void igb_if_intr_disable(if_ctx_t); 456 static int em_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 457 static int em_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 458 static int igb_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 459 static int igb_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 460 static void em_handle_fatal_error_intr(struct e1000_softc *, u32); 461 static bool em_handle_fatal_error_admin(struct e1000_softc *); 462 static u32 igb_device_reset_intr_mask(struct e1000_softc *); 463 static bool igb_device_reset_pending(struct e1000_softc *); 464 static bool igb_handle_device_reset(struct e1000_softc *, u32); 465 static void igb_prepare_device_reset(struct e1000_softc *); 466 static bool igb_finish_device_reset(struct e1000_softc *, u32); 467 static void em_prepare_fatal_error_reset(struct e1000_softc *); 468 static void em_finish_fatal_error_reset(struct e1000_softc *); 469 static void em_configure_peind_memory_errors(struct e1000_softc *); 470 static void em_configure_82575_memory_errors(struct e1000_softc *); 471 static void em_configure_82580_memory_errors(struct e1000_softc *); 472 static void em_update_82580_ecc_stats(struct e1000_softc *, u32, u32, 473 u32); 474 static void em_if_multi_set(if_ctx_t); 475 static void em_if_update_admin_status(if_ctx_t); 476 static void em_if_debug(if_ctx_t); 477 static void em_initialize_vf_stats(struct e1000_softc *); 478 static void em_rebase_vf_stats(struct e1000_softc *); 479 static void em_update_vf_stats_counters(struct e1000_softc *); 480 static void em_add_hw_stats(struct e1000_softc *); 481 static bool em_mac_has_eee(enum e1000_mac_type); 482 static int em_if_set_promisc(if_ctx_t, int); 483 static bool em_if_defer_promisc(struct e1000_softc *); 484 static bool em_if_vlan_filter_capable(if_ctx_t); 485 static bool em_if_vlan_filter_used(if_ctx_t); 486 static void em_if_vlan_filter_enable(struct e1000_softc *); 487 static void em_if_vlan_filter_disable(struct e1000_softc *); 488 static void em_if_vlan_filter_write(struct e1000_softc *, int); 489 static void em_setup_vlan_hw_support(if_ctx_t ctx); 490 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 491 static void em_print_nvm_info(struct e1000_softc *); 492 static void em_fw_version_locked(if_ctx_t); 493 static void em_sbuf_fw_version(struct e1000_fw_version *, struct sbuf *); 494 static void em_print_fw_version(struct e1000_softc *); 495 static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 496 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 497 static int em_get_rs(SYSCTL_HANDLER_ARGS); 498 static void em_print_debug_info(struct e1000_softc *); 499 static void em_newitr(struct e1000_softc *, struct em_rx_queue *, 500 struct rx_ring *); 501 static bool em_automask_tso(if_ctx_t); 502 static int em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 503 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 504 static void em_add_int_delay_sysctl(struct e1000_softc *, const char *, 505 const char *, struct em_int_delay_info *, int, int); 506 /* Management and WOL Support */ 507 static void em_init_manageability(struct e1000_softc *); 508 static void em_release_manageability(struct e1000_softc *); 509 static void em_get_hw_control(struct e1000_softc *); 510 static void em_release_hw_control(struct e1000_softc *); 511 static void em_get_wakeup(if_ctx_t); 512 static void em_fill_wakeup_mta(struct e1000_hw *); 513 static int em_enable_wakeup(if_ctx_t); 514 static void em_configure_sx_low_power(struct e1000_softc *, u32); 515 static int em_enable_phy_wakeup(struct e1000_softc *, u32); 516 static int em_disable_phy_wakeup(struct e1000_softc *, u16 *); 517 static void em_power_up_wakeup_link(struct e1000_softc *); 518 static void em_power_down_wakeup_link(struct e1000_softc *); 519 static void em_disable_aspm(struct e1000_softc *); 520 521 int em_intr(void *); 522 523 enum em_fatal_error_state { 524 EM_FATAL_ERROR_NONE, 525 EM_FATAL_ERROR_CAPTURING, 526 EM_FATAL_ERROR_DETECTED, 527 EM_FATAL_ERROR_RESET_REQUESTED, 528 EM_FATAL_ERROR_RESET_PREPARED, 529 }; 530 531 enum igb_device_reset_state { 532 IGB_DEVICE_RESET_NONE, 533 IGB_DEVICE_RESET_DETECTED, 534 IGB_DEVICE_RESET_REQUESTED, 535 IGB_DEVICE_RESET_PREPARED, 536 }; 537 538 #define IGB_DEVICE_RESET_TIMEOUT_MS 100 539 540 /* MSI-X handlers */ 541 static int em_if_msix_intr_assign(if_ctx_t, int); 542 static int em_msix_link(void *); 543 static void em_handle_link(void *); 544 545 static void em_enable_vectors_82574(if_ctx_t); 546 547 static void em_sysctl_request_reinit(struct e1000_softc *); 548 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); 549 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); 550 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); 551 static void em_if_led_func(if_ctx_t, int); 552 553 static int em_get_regs(SYSCTL_HANDLER_ARGS); 554 static void lem_smartspeed(struct e1000_softc *); 555 static void igb_configure_queues(struct e1000_softc *); 556 static void igb_initialize_interrupt_rate(struct e1000_softc *); 557 static void em_flush_desc_rings(struct e1000_softc *); 558 559 560 /********************************************************************* 561 * FreeBSD Device Interface Entry Points 562 *********************************************************************/ 563 static device_method_t em_methods[] = { 564 /* Device interface */ 565 DEVMETHOD(device_register, em_register), 566 DEVMETHOD(device_probe, iflib_device_probe), 567 DEVMETHOD(device_attach, iflib_device_attach), 568 DEVMETHOD(device_detach, iflib_device_detach), 569 DEVMETHOD(device_shutdown, iflib_device_shutdown), 570 DEVMETHOD(device_suspend, iflib_device_suspend), 571 DEVMETHOD(device_resume, iflib_device_resume), 572 DEVMETHOD_END 573 }; 574 575 static device_method_t igb_methods[] = { 576 /* Device interface */ 577 DEVMETHOD(device_register, igb_register), 578 DEVMETHOD(device_probe, iflib_device_probe), 579 DEVMETHOD(device_attach, igb_device_attach), 580 DEVMETHOD(device_detach, iflib_device_detach), 581 DEVMETHOD(device_shutdown, iflib_device_shutdown), 582 DEVMETHOD(device_suspend, iflib_device_suspend), 583 DEVMETHOD(device_resume, iflib_device_resume), 584 #ifdef PCI_IOV 585 DEVMETHOD(pci_iov_init, igb_device_iov_init), 586 DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit), 587 DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), 588 #endif 589 DEVMETHOD_END 590 }; 591 592 static device_method_t igbv_methods[] = { 593 /* Device interface */ 594 DEVMETHOD(device_register, igbv_register), 595 DEVMETHOD(device_probe, iflib_device_probe), 596 DEVMETHOD(device_attach, iflib_device_attach), 597 DEVMETHOD(device_detach, iflib_device_detach), 598 DEVMETHOD(device_shutdown, iflib_device_shutdown), 599 DEVMETHOD(device_suspend, iflib_device_suspend), 600 DEVMETHOD(device_resume, iflib_device_resume), 601 DEVMETHOD_END 602 }; 603 604 605 static driver_t em_driver = { 606 "em", em_methods, sizeof(struct e1000_softc), 607 }; 608 609 DRIVER_MODULE(em, pci, em_driver, 0, 0); 610 611 MODULE_DEPEND(em, pci, 1, 1, 1); 612 MODULE_DEPEND(em, ether, 1, 1, 1); 613 MODULE_DEPEND(em, iflib, 1, 1, 1); 614 615 IFLIB_PNP_INFO(pci, em, em_vendor_info_array); 616 617 static driver_t igb_driver = { 618 "igb", igb_methods, sizeof(struct e1000_softc), 619 }; 620 621 DRIVER_MODULE(igb, pci, igb_driver, 0, 0); 622 623 MODULE_DEPEND(igb, pci, 1, 1, 1); 624 MODULE_DEPEND(igb, ether, 1, 1, 1); 625 MODULE_DEPEND(igb, iflib, 1, 1, 1); 626 627 IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array); 628 629 static driver_t igbv_driver = { 630 "igbv", igbv_methods, sizeof(struct e1000_softc), 631 }; 632 633 DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0); 634 635 MODULE_DEPEND(igbv, pci, 1, 1, 1); 636 MODULE_DEPEND(igbv, ether, 1, 1, 1); 637 MODULE_DEPEND(igbv, iflib, 1, 1, 1); 638 639 IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array); 640 641 static device_method_t em_if_methods[] = { 642 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 643 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 644 DEVMETHOD(ifdi_detach, em_if_detach), 645 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 646 DEVMETHOD(ifdi_suspend, em_if_suspend), 647 DEVMETHOD(ifdi_resume, em_if_resume), 648 DEVMETHOD(ifdi_init, em_if_init), 649 DEVMETHOD(ifdi_stop, em_if_stop), 650 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 651 DEVMETHOD(ifdi_intr_enable, em_if_intr_enable), 652 DEVMETHOD(ifdi_intr_disable, em_if_intr_disable), 653 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 654 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 655 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 656 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 657 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 658 DEVMETHOD(ifdi_media_status, em_if_media_status), 659 DEVMETHOD(ifdi_media_change, em_if_media_change), 660 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 661 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 662 DEVMETHOD(ifdi_timer, em_if_timer), 663 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 664 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 665 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 666 DEVMETHOD(ifdi_led_func, em_if_led_func), 667 DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), 668 DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), 669 DEVMETHOD(ifdi_debug, em_if_debug), 670 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 671 DEVMETHOD_END 672 }; 673 674 static driver_t em_if_driver = { 675 "em_if", em_if_methods, sizeof(struct e1000_softc) 676 }; 677 678 static device_method_t igb_if_methods[] = { 679 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 680 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 681 DEVMETHOD(ifdi_detach, em_if_detach), 682 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 683 DEVMETHOD(ifdi_suspend, em_if_suspend), 684 DEVMETHOD(ifdi_resume, em_if_resume), 685 DEVMETHOD(ifdi_init, em_if_init), 686 DEVMETHOD(ifdi_stop, em_if_stop), 687 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 688 DEVMETHOD(ifdi_intr_enable, igb_if_intr_enable), 689 DEVMETHOD(ifdi_intr_disable, igb_if_intr_disable), 690 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 691 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 692 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 693 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 694 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 695 DEVMETHOD(ifdi_media_status, em_if_media_status), 696 DEVMETHOD(ifdi_media_change, em_if_media_change), 697 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 698 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 699 DEVMETHOD(ifdi_timer, em_if_timer), 700 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 701 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 702 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 703 DEVMETHOD(ifdi_led_func, em_if_led_func), 704 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 705 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 706 DEVMETHOD(ifdi_debug, em_if_debug), 707 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 708 #ifdef PCI_IOV 709 DEVMETHOD(ifdi_iov_init, igb_if_iov_init), 710 DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit), 711 DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add), 712 DEVMETHOD(ifdi_vf_status, igb_if_vf_status), 713 #endif 714 DEVMETHOD_END 715 }; 716 717 static driver_t igb_if_driver = { 718 "igb_if", igb_if_methods, sizeof(struct e1000_softc) 719 }; 720 721 static device_method_t igbv_if_methods[] = { 722 DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre), 723 DEVMETHOD(ifdi_attach_post, igbv_if_attach_post), 724 DEVMETHOD(ifdi_detach, em_if_detach), 725 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 726 DEVMETHOD(ifdi_suspend, em_if_suspend), 727 DEVMETHOD(ifdi_resume, em_if_resume), 728 DEVMETHOD(ifdi_init, em_if_init), 729 DEVMETHOD(ifdi_stop, em_if_stop), 730 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 731 DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable), 732 DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable), 733 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 734 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 735 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 736 DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status), 737 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 738 DEVMETHOD(ifdi_media_status, em_if_media_status), 739 DEVMETHOD(ifdi_media_change, igbv_if_media_change), 740 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 741 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 742 DEVMETHOD(ifdi_timer, em_if_timer), 743 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 744 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 745 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 746 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 747 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 748 DEVMETHOD(ifdi_debug, em_if_debug), 749 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 750 DEVMETHOD_END 751 }; 752 753 static driver_t igbv_if_driver = { 754 "igbv_if", igbv_if_methods, sizeof(struct e1000_softc) 755 }; 756 757 /********************************************************************* 758 * Tunable default values. 759 *********************************************************************/ 760 761 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 762 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 763 764 /* Allow common code without TSO */ 765 #ifndef CSUM_TSO 766 #define CSUM_TSO 0 767 #endif 768 769 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 770 "EM driver parameters"); 771 772 static int em_disable_crc_stripping = 0; 773 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 774 &em_disable_crc_stripping, 0, "Disable CRC Stripping"); 775 776 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 777 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 778 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, 779 &em_tx_int_delay_dflt, 0, "Default transmit interrupt delay in usecs"); 780 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, 781 &em_rx_int_delay_dflt, 0, "Default receive interrupt delay in usecs"); 782 783 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 784 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 785 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN, 786 &em_tx_abs_int_delay_dflt, 0, 787 "Default transmit interrupt delay limit in usecs"); 788 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, 789 &em_rx_abs_int_delay_dflt, 0, 790 "Default receive interrupt delay limit in usecs"); 791 792 static int em_smart_pwr_down = false; 793 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 794 &em_smart_pwr_down, 795 0, "Set to true to leave smart power down enabled on newer adapters"); 796 797 static bool em_unsupported_tso = false; 798 SYSCTL_BOOL(_hw_em, OID_AUTO, unsupported_tso, CTLFLAG_RDTUN, 799 &em_unsupported_tso, 0, "Allow unsupported em(4) TSO configurations"); 800 801 /* Controls whether promiscuous also shows bad packets */ 802 static int em_debug_sbp = false; 803 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, 804 "Show bad packets in promiscuous mode"); 805 806 /* Energy efficient ethernet - default to OFF */ 807 static int eee_setting = 1; 808 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, 809 "Enable Energy Efficient Ethernet"); 810 811 /* 812 * AIM: Adaptive Interrupt Moderation 813 * which means that the interrupt rate is varied over time based on the 814 * traffic for that interrupt vector 815 */ 816 static int em_enable_aim = 1; 817 SYSCTL_INT(_hw_em, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &em_enable_aim, 818 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 819 820 /* 821 ** Tuneable Interrupt rate 822 */ 823 static int em_max_interrupt_rate = EM_INTS_DEFAULT; 824 SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 825 &em_max_interrupt_rate, 0, "Maximum interrupts per second"); 826 827 /* Global used in WOL setup with multiport cards */ 828 static int global_quad_port_a = 0; 829 830 extern struct if_txrx igb_txrx; 831 extern struct if_txrx em_txrx; 832 extern struct if_txrx lem_txrx; 833 834 static struct if_shared_ctx em_sctx_init = { 835 .isc_magic = IFLIB_MAGIC, 836 .isc_q_align = PAGE_SIZE, 837 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 838 .isc_tx_maxsegsize = PAGE_SIZE, 839 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 840 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 841 .isc_rx_maxsize = MJUM9BYTES, 842 .isc_rx_nsegments = 1, 843 .isc_rx_maxsegsize = MJUM9BYTES, 844 .isc_nfl = 1, 845 .isc_nrxqs = 1, 846 .isc_ntxqs = 1, 847 .isc_admin_intrcnt = 1, 848 .isc_vendor_info = em_vendor_info_array, 849 .isc_driver_version = em_driver_version, 850 .isc_driver = &em_if_driver, 851 .isc_flags = 852 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 853 854 .isc_nrxd_min = {EM_MIN_RXD}, 855 .isc_ntxd_min = {EM_MIN_TXD}, 856 .isc_nrxd_max = {EM_MAX_RXD}, 857 .isc_ntxd_max = {EM_MAX_TXD}, 858 .isc_nrxd_default = {EM_DEFAULT_RXD}, 859 .isc_ntxd_default = {EM_DEFAULT_TXD}, 860 }; 861 862 static struct if_shared_ctx igb_sctx_init = { 863 .isc_magic = IFLIB_MAGIC, 864 .isc_q_align = PAGE_SIZE, 865 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 866 .isc_tx_maxsegsize = PAGE_SIZE, 867 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 868 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 869 .isc_rx_maxsize = MJUM9BYTES, 870 .isc_rx_nsegments = 1, 871 .isc_rx_maxsegsize = MJUM9BYTES, 872 .isc_nfl = 1, 873 .isc_nrxqs = 1, 874 .isc_ntxqs = 1, 875 .isc_admin_intrcnt = 1, 876 .isc_vendor_info = igb_vendor_info_array, 877 .isc_driver_version = igb_driver_version, 878 .isc_driver = &igb_if_driver, 879 .isc_flags = 880 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 881 882 .isc_nrxd_min = {EM_MIN_RXD}, 883 .isc_ntxd_min = {EM_MIN_TXD}, 884 .isc_nrxd_max = {IGB_MAX_RXD}, 885 .isc_ntxd_max = {IGB_MAX_TXD}, 886 .isc_nrxd_default = {EM_DEFAULT_RXD}, 887 .isc_ntxd_default = {EM_DEFAULT_TXD}, 888 }; 889 890 /* 891 * igb PFs and igbv VFs share the common datapath implementation. Keep a 892 * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or 893 * interrupt modes. 894 */ 895 static struct if_shared_ctx igbv_sctx_init = { 896 .isc_magic = IFLIB_MAGIC, 897 .isc_q_align = PAGE_SIZE, 898 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 899 .isc_tx_maxsegsize = PAGE_SIZE, 900 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 901 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 902 .isc_rx_maxsize = MJUM9BYTES, 903 .isc_rx_nsegments = 1, 904 .isc_rx_maxsegsize = MJUM9BYTES, 905 .isc_nfl = 1, 906 .isc_nrxqs = 1, 907 .isc_ntxqs = 1, 908 .isc_admin_intrcnt = 1, 909 .isc_vendor_info = igbv_vendor_info_array, 910 .isc_driver_version = igb_driver_version, 911 .isc_driver = &igbv_if_driver, 912 .isc_flags = 913 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM | 914 IFLIB_IS_VF, 915 916 .isc_nrxd_min = {EM_MIN_RXD}, 917 .isc_ntxd_min = {EM_MIN_TXD}, 918 .isc_nrxd_max = {IGB_MAX_RXD}, 919 .isc_ntxd_max = {IGB_MAX_TXD}, 920 .isc_nrxd_default = {EM_DEFAULT_RXD}, 921 .isc_ntxd_default = {EM_DEFAULT_TXD}, 922 }; 923 924 /***************************************************************** 925 * 926 * Dump Registers 927 * 928 ****************************************************************/ 929 #define IGB_REGS_LEN 739 930 931 static int em_get_regs(SYSCTL_HANDLER_ARGS) 932 { 933 struct e1000_softc *sc = (struct e1000_softc *)arg1; 934 struct e1000_hw *hw = &sc->hw; 935 struct sbuf *sb; 936 u32 *regs_buff; 937 int rc; 938 uint32_t rxqid, txqid; 939 940 /* 941 * This sysctl is registered before iflib allocates the queue arrays, 942 * and remains registered while iflib tears them down. 943 */ 944 if (sc->rx_queues == NULL || sc->tx_queues == NULL) 945 return (ENXIO); 946 947 regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK); 948 memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); 949 rxqid = sc->rx_queues[0].rxr.me; 950 txqid = sc->tx_queues[0].txr.me; 951 952 rc = sysctl_wire_old_buffer(req, 0); 953 MPASS(rc == 0); 954 if (rc != 0) { 955 free(regs_buff, M_DEVBUF); 956 return (rc); 957 } 958 959 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 960 MPASS(sb != NULL); 961 if (sb == NULL) { 962 free(regs_buff, M_DEVBUF); 963 return (ENOMEM); 964 } 965 966 /* General Registers */ 967 regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); 968 regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); 969 regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); 970 regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); 971 regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); 972 regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid)); 973 regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid)); 974 regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid)); 975 regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid)); 976 regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid)); 977 regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid)); 978 regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); 979 regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid)); 980 regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid)); 981 regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid)); 982 regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid)); 983 regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid)); 984 regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid)); 985 regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); 986 regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); 987 regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); 988 regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); 989 990 sbuf_printf(sb, "General Registers\n"); 991 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 992 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 993 sbuf_printf(sb, "\tCTRL_EXT\t %08x\n\n", regs_buff[2]); 994 995 sbuf_printf(sb, "Interrupt Registers\n"); 996 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 997 998 sbuf_printf(sb, "RX Registers\n"); 999 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 1000 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 1001 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 1002 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 1003 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 1004 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 1005 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 1006 1007 sbuf_printf(sb, "TX Registers\n"); 1008 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 1009 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 1010 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 1011 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 1012 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 1013 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 1014 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 1015 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 1016 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 1017 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 1018 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 1019 1020 free(regs_buff, M_DEVBUF); 1021 1022 #ifdef DUMP_DESCS 1023 { 1024 if_softc_ctx_t scctx = sc->shared; 1025 struct rx_ring *rxr = &rx_que->rxr; 1026 struct tx_ring *txr = &tx_que->txr; 1027 int ntxd = scctx->isc_ntxd[0]; 1028 int nrxd = scctx->isc_nrxd[0]; 1029 int j; 1030 1031 for (j = 0; j < nrxd; j++) { 1032 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 1033 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 1034 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 1035 PRIx64 " Error:%d Length:%d\n", 1036 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 1037 } 1038 1039 for (j = 0; j < min(ntxd, 256); j++) { 1040 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 1041 1042 sbuf_printf(sb, 1043 "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x" 1044 " eop: %d DD=%d\n", 1045 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 1046 buf->eop != -1 ? 1047 txr->tx_base[buf->eop].upper.fields.status & 1048 E1000_TXD_STAT_DD : 0); 1049 1050 } 1051 } 1052 #endif 1053 1054 rc = sbuf_finish(sb); 1055 sbuf_delete(sb); 1056 return(rc); 1057 } 1058 1059 static void * 1060 em_register(device_t dev) 1061 { 1062 return (&em_sctx_init); 1063 } 1064 1065 static void * 1066 igb_register(device_t dev) 1067 { 1068 return (&igb_sctx_init); 1069 } 1070 1071 static void * 1072 igbv_register(device_t dev) 1073 { 1074 return (&igbv_sctx_init); 1075 } 1076 1077 static int 1078 igb_device_attach(device_t dev) 1079 { 1080 struct e1000_softc *sc; 1081 if_ctx_t ctx; 1082 int error; 1083 1084 error = iflib_device_attach(dev); 1085 if (error != 0) 1086 return (error); 1087 1088 ctx = device_get_softc(dev); 1089 sc = iflib_get_softc(ctx); 1090 (void)igb_iov_attach(sc); 1091 return (0); 1092 } 1093 1094 #ifdef PCI_IOV 1095 static int 1096 igb_device_iov_init(device_t dev, uint16_t num_vfs, 1097 const nvlist_t *params) 1098 { 1099 struct e1000_softc *sc; 1100 if_ctx_t ctx; 1101 int error; 1102 1103 ctx = device_get_softc(dev); 1104 sc = iflib_get_softc(ctx); 1105 error = igb_iov_validate(sc, num_vfs); 1106 if (error != 0) 1107 return (error); 1108 return (iflib_device_iov_init_restart(dev, num_vfs, params)); 1109 } 1110 1111 static void 1112 igb_device_iov_uninit(device_t dev) 1113 { 1114 struct e1000_softc *sc; 1115 if_ctx_t ctx; 1116 1117 ctx = device_get_softc(dev); 1118 sc = iflib_get_softc(ctx); 1119 /* 1120 * pci_iov(4) has already detached the VF devices. Tell the stop 1121 * half of iflib's restart transaction not to wait for acknowledgements 1122 * from VFs which can no longer service their mailbox vectors. 1123 */ 1124 atomic_store_rel_32(&sc->iov_teardown, 1); 1125 iflib_device_iov_uninit_restart(dev); 1126 } 1127 1128 #endif 1129 1130 static int 1131 em_set_num_queues(if_ctx_t ctx) 1132 { 1133 struct e1000_softc *sc = iflib_get_softc(ctx); 1134 int maxqueues; 1135 1136 /* Sanity check based on HW */ 1137 switch (sc->hw.mac.type) { 1138 case e1000_82576: 1139 case e1000_82580: 1140 case e1000_i350: 1141 case e1000_i354: 1142 maxqueues = 8; 1143 break; 1144 case e1000_i210: 1145 case e1000_82575: 1146 maxqueues = 4; 1147 break; 1148 case e1000_i211: 1149 case e1000_82574: 1150 maxqueues = 2; 1151 break; 1152 case e1000_vfadapt: 1153 /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */ 1154 case e1000_vfadapt_i350: 1155 maxqueues = 1; 1156 break; 1157 default: 1158 maxqueues = 1; 1159 break; 1160 } 1161 1162 return (maxqueues); 1163 } 1164 1165 #define LEM_CAPS ( \ 1166 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1167 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1168 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6) 1169 1170 #define EM_CAPS ( \ 1171 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1172 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1173 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1174 IFCAP_TSO6) 1175 1176 #define IGB_CAPS ( \ 1177 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1178 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1179 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1180 IFCAP_TSO6) 1181 1182 /* 1183 * VLAN filtering is an effective VF capability, but its policy is owned by 1184 * the PF and cannot be disabled from the VF. vlan(4) registration callbacks 1185 * are independent of this capability bit. 1186 */ 1187 #define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL) 1188 1189 void 1190 em_add_device_sysctls(struct e1000_softc *sc) 1191 { 1192 struct e1000_hw *hw; 1193 struct sysctl_oid_list *child; 1194 struct sysctl_ctx_list *ctx_list; 1195 1196 hw = &sc->hw; 1197 ctx_list = device_get_sysctl_ctx(sc->dev); 1198 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1199 1200 sc->enable_aim = em_enable_aim; 1201 SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", 1202 CTLFLAG_RW, &sc->enable_aim, 0, 1203 "Interrupt Moderation (1=normal, 2=lowlatency)"); 1204 1205 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug", 1206 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1207 em_sysctl_debug_info, "I", "Debug Information"); 1208 1209 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", 1210 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1211 em_get_rs, "I", "Dump RS indexes"); 1212 1213 if (sc->vf_ifp) { 1214 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1215 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1216 igbv_get_regs, "A", "Dump VF registers"); 1217 return; 1218 } 1219 1220 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", 1221 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1222 em_sysctl_nvm_info, "I", "NVM Information"); 1223 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", 1224 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 1225 em_sysctl_print_fw_version, "A", 1226 "Prints FW/NVM Versions"); 1227 sc->fc = e1000_fc_full; 1228 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc", 1229 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1230 em_set_flowcntl, "I", "Flow Control"); 1231 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1232 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1233 em_get_regs, "A", "Dump Registers"); 1234 1235 if (hw->mac.type >= e1000_i350 && hw->mac.type != e1000_i211) { 1236 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac", 1237 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1238 igb_sysctl_dmac, "I", "DMA Coalesce"); 1239 } 1240 1241 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1242 "tso_tcp_flags_mask_first_segment", 1243 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1244 sc, 0, em_sysctl_tso_tcp_flags_mask, "IU", 1245 "TSO TCP flags mask for first segment"); 1246 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1247 "tso_tcp_flags_mask_middle_segment", 1248 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1249 sc, 1, em_sysctl_tso_tcp_flags_mask, "IU", 1250 "TSO TCP flags mask for middle segment"); 1251 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1252 "tso_tcp_flags_mask_last_segment", 1253 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1254 sc, 2, em_sysctl_tso_tcp_flags_mask, "IU", 1255 "TSO TCP flags mask for last segment"); 1256 } 1257 1258 /********************************************************************* 1259 * Device initialization routine 1260 * 1261 * The attach entry point is called when the driver is being loaded. 1262 * This routine identifies the type of hardware, allocates all resources 1263 * and initializes the hardware. 1264 * 1265 * return 0 on success, positive on failure 1266 *********************************************************************/ 1267 int 1268 em_if_attach_pre(if_ctx_t ctx) 1269 { 1270 struct e1000_softc *sc; 1271 if_softc_ctx_t scctx; 1272 device_t dev; 1273 struct e1000_hw *hw; 1274 struct sysctl_oid_list *child; 1275 struct sysctl_ctx_list *ctx_list; 1276 int error = 0; 1277 1278 INIT_DEBUGOUT("em_if_attach_pre: begin"); 1279 dev = iflib_get_dev(ctx); 1280 sc = iflib_get_softc(ctx); 1281 1282 if (em_max_interrupt_rate <= 0) { 1283 device_printf(dev, 1284 "Invalid max_interrupt_rate %d; using default %d\n", 1285 em_max_interrupt_rate, EM_INTS_DEFAULT); 1286 em_max_interrupt_rate = EM_INTS_DEFAULT; 1287 } 1288 1289 sc->ctx = sc->osdep.ctx = ctx; 1290 sc->dev = sc->osdep.dev = dev; 1291 scctx = sc->shared = iflib_get_softc_ctx(ctx); 1292 sc->media = iflib_get_media(ctx); 1293 hw = &sc->hw; 1294 sc->vf_ifp = 1295 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0; 1296 sc->osdep.vf = sc->vf_ifp; 1297 1298 /* Determine hardware and mac info */ 1299 em_identify_hardware(ctx); 1300 sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt; 1301 1302 /* VF sysctls are deferred until attach-post confirms MSI-X. */ 1303 ctx_list = device_get_sysctl_ctx(dev); 1304 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 1305 if (!sc->vf_ifp) 1306 em_add_device_sysctls(sc); 1307 1308 scctx->isc_tx_nsegments = EM_MAX_SCATTER; 1309 scctx->isc_nrxqsets_max = 1310 scctx->isc_ntxqsets_max = em_set_num_queues(ctx); 1311 if (bootverbose) 1312 device_printf(dev, "attach_pre capping queues at %d\n", 1313 scctx->isc_ntxqsets_max); 1314 1315 if (hw->mac.type >= igb_mac_min) { 1316 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1317 sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); 1318 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1319 sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); 1320 scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); 1321 scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); 1322 scctx->isc_txrx = &igb_txrx; 1323 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1324 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1325 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1326 scctx->isc_capabilities = scctx->isc_capenable = 1327 sc->vf_ifp ? IGBV_CAPS : IGB_CAPS; 1328 if (igbv_is_hyperv(sc)) { 1329 /* The host owns VLAN filters and the receive-frame limit. */ 1330 scctx->isc_capabilities &= 1331 ~(IFCAP_VLAN_HWFILTER | IFCAP_JUMBO_MTU); 1332 scctx->isc_capenable = scctx->isc_capabilities; 1333 } 1334 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 1335 CSUM_IP6_TCP | CSUM_IP6_UDP; 1336 if (hw->mac.type != e1000_82575) 1337 scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; 1338 /* 1339 ** Some new devices, as with ixgbe, now may 1340 ** use a different BAR, so we need to keep 1341 ** track of which is used. 1342 */ 1343 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1344 } else if (hw->mac.type >= em_mac_min) { 1345 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1346 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1347 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1348 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); 1349 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1350 scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); 1351 scctx->isc_txrx = &em_txrx; 1352 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1353 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1354 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1355 scctx->isc_capabilities = scctx->isc_capenable = EM_CAPS; 1356 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1357 CSUM_IP6_TCP | CSUM_IP6_UDP; 1358 1359 /* Disable TSO on all em(4) until ring stalls are debugged */ 1360 scctx->isc_capenable &= ~IFCAP_TSO; 1361 1362 /* 1363 * Disable TSO on SPT due to errata that downclocks DMA 1364 * performance 1365 * i218-i219 Specification Update 1.5.4.5 1366 */ 1367 if (hw->mac.type == e1000_pch_spt) 1368 scctx->isc_capenable &= ~IFCAP_TSO; 1369 1370 /* 1371 * We support MSI-X with 82574 only, but indicate to iflib(4) 1372 * that it shall give MSI at least a try with other devices. 1373 */ 1374 if (hw->mac.type == e1000_82574) { 1375 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1376 } else { 1377 scctx->isc_msix_bar = -1; 1378 scctx->isc_disable_msix = 1; 1379 } 1380 } else { 1381 scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * 1382 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1383 scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * 1384 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 1385 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1386 scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); 1387 scctx->isc_txrx = &lem_txrx; 1388 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1389 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1390 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1391 scctx->isc_capabilities = scctx->isc_capenable = LEM_CAPS; 1392 if (em_unsupported_tso) 1393 scctx->isc_capabilities |= IFCAP_TSO6; 1394 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1395 CSUM_IP6_TCP | CSUM_IP6_UDP; 1396 1397 /* Disable TSO on all lem(4) until ring stalls debugged */ 1398 scctx->isc_capenable &= ~IFCAP_TSO; 1399 1400 /* 82541ER doesn't do HW tagging */ 1401 if (hw->device_id == E1000_DEV_ID_82541ER || 1402 hw->device_id == E1000_DEV_ID_82541ER_LOM) { 1403 scctx->isc_capabilities &= ~IFCAP_VLAN_HWTAGGING; 1404 scctx->isc_capenable = scctx->isc_capabilities; 1405 } 1406 /* This is the first e1000 chip and it does not do offloads */ 1407 if (hw->mac.type == e1000_82542) { 1408 scctx->isc_capabilities &= ~(IFCAP_HWCSUM | 1409 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6 | 1410 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER | 1411 IFCAP_TSO | IFCAP_VLAN_HWTSO); 1412 scctx->isc_capenable = scctx->isc_capabilities; 1413 } 1414 /* These can't do TSO for various reasons */ 1415 if (hw->mac.type < e1000_82544 || 1416 hw->mac.type == e1000_82547 || 1417 hw->mac.type == e1000_82547_rev_2) { 1418 scctx->isc_capabilities &= 1419 ~(IFCAP_TSO |IFCAP_VLAN_HWTSO); 1420 scctx->isc_capenable = scctx->isc_capabilities; 1421 } 1422 /* XXXKB: No IPv6 before this? */ 1423 if (hw->mac.type < e1000_82545){ 1424 scctx->isc_capabilities &= ~IFCAP_HWCSUM_IPV6; 1425 scctx->isc_capenable = scctx->isc_capabilities; 1426 } 1427 /* 1428 * "PCI/PCI-X SDM 4.0" page 33 (b): 1429 * FDX requirement on these chips 1430 */ 1431 if (hw->mac.type == e1000_82547 || 1432 hw->mac.type == e1000_82547_rev_2) 1433 scctx->isc_capenable &= ~(IFCAP_HWCSUM | 1434 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6); 1435 1436 /* INTx only */ 1437 scctx->isc_msix_bar = 0; 1438 } 1439 1440 /* Setup PCI resources */ 1441 if (em_allocate_pci_resources(ctx)) { 1442 device_printf(dev, "Allocation of PCI resources failed\n"); 1443 error = ENXIO; 1444 goto err_pci; 1445 } 1446 /* 1447 * A VF can retain queue enable bits and DMA addresses across VFLR. 1448 * Fence bus mastering before the first mailbox reset so state left by 1449 * a previous owner cannot issue DMA while the driver attaches. 1450 */ 1451 if (sc->vf_ifp) 1452 em_fence_pci_busmaster(sc); 1453 /* 1454 * 82579 can lose a host CSR write while the Management Engine owns 1455 * the PCIm2PCI arbiter. Enable the OS register write interlock before 1456 * shared code initialization performs any MAC writes. 1457 */ 1458 if (hw->mac.type == e1000_pch2lan && 1459 (E1000_READ_REG(hw, E1000_FWSM) & 1460 E1000_ICH_FWSM_FW_VALID) != 0) 1461 sc->osdep.pcim2pci_arbiter_wa = true; 1462 1463 /* 1464 ** For ICH8 and family we need to 1465 ** map the flash memory, and this 1466 ** must happen after the MAC is 1467 ** identified 1468 */ 1469 if ((hw->mac.type == e1000_ich8lan) || 1470 (hw->mac.type == e1000_ich9lan) || 1471 (hw->mac.type == e1000_ich10lan) || 1472 (hw->mac.type == e1000_pchlan) || 1473 (hw->mac.type == e1000_pch2lan) || 1474 (hw->mac.type == e1000_pch_lpt)) { 1475 int rid = EM_BAR_TYPE_FLASH; 1476 sc->flash = bus_alloc_resource_any(dev, 1477 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1478 if (sc->flash == NULL) { 1479 device_printf(dev, "Mapping of Flash failed\n"); 1480 error = ENXIO; 1481 goto err_pci; 1482 } 1483 /* This is used in the shared code */ 1484 hw->flash_address = (u8 *)sc->flash; 1485 sc->osdep.flash_bus_space_tag = 1486 rman_get_bustag(sc->flash); 1487 sc->osdep.flash_bus_space_handle = 1488 rman_get_bushandle(sc->flash); 1489 } 1490 /* 1491 ** In the new SPT device flash is not a 1492 ** separate BAR, rather it is also in BAR0, 1493 ** so use the same tag and an offset handle for the 1494 ** FLASH read/write macros in the shared code. 1495 */ 1496 else if (hw->mac.type >= e1000_pch_spt) { 1497 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag; 1498 sc->osdep.flash_bus_space_handle = 1499 sc->osdep.mem_bus_space_handle + E1000_FLASH_BASE_ADDR; 1500 } 1501 1502 /* Do Shared Code initialization */ 1503 error = e1000_setup_init_funcs(hw, true); 1504 if (error) { 1505 device_printf(dev, "Setup of Shared code failed, error %d\n", 1506 error); 1507 error = ENXIO; 1508 goto err_pci; 1509 } 1510 1511 if (igbv_is_hyperv(sc)) 1512 igbv_init_hv_ops(hw); 1513 1514 em_setup_msix(ctx); 1515 e1000_get_bus_info(hw); 1516 1517 /* 1518 * Some conventional PCI systems hang when e1000 devices use 1519 * DMA addresses above 4 GB. Keep PCI-mode DMA below that boundary 1520 * by default; PCI-X and PCIe retain 64-bit DMA. 1521 */ 1522 if (hw->bus.type == e1000_bus_type_pci) { 1523 SYSCTL_ADD_BOOL(ctx_list, child, OID_AUTO, "allow_64bit_dma", 1524 CTLFLAG_RDTUN, &sc->allow_64bit_dma, 0, 1525 "Allow 64-bit DMA in conventional PCI mode"); 1526 if (sc->allow_64bit_dma) 1527 device_printf(dev, "64-bit DMA in conventional PCI mode. " 1528 "Some chipsets are unstable.\n"); 1529 else { 1530 scctx->isc_dma_width = 32; 1531 device_printf(dev, "32-bit DMA in conventional PCI mode. " 1532 "Set dev.%s.%d.allow_64bit_dma=1 at boot to enable " 1533 "64-bit DMA if the chipset is stable with it.\n", 1534 device_get_name(dev), device_get_unit(dev)); 1535 } 1536 } 1537 1538 /* Set up some sysctls for the tunable interrupt delays */ 1539 if (hw->mac.type < igb_mac_min) { 1540 em_add_int_delay_sysctl(sc, "rx_int_delay", 1541 "receive interrupt delay in usecs", &sc->rx_int_delay, 1542 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); 1543 em_add_int_delay_sysctl(sc, "tx_int_delay", 1544 "transmit interrupt delay in usecs", &sc->tx_int_delay, 1545 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt); 1546 } 1547 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) { 1548 em_add_int_delay_sysctl(sc, "rx_abs_int_delay", 1549 "receive interrupt delay limit in usecs", 1550 &sc->rx_abs_int_delay, 1551 E1000_REGISTER(hw, E1000_RADV), em_rx_abs_int_delay_dflt); 1552 em_add_int_delay_sysctl(sc, "tx_abs_int_delay", 1553 "transmit interrupt delay limit in usecs", 1554 &sc->tx_abs_int_delay, 1555 E1000_REGISTER(hw, E1000_TADV), em_tx_abs_int_delay_dflt); 1556 } 1557 1558 hw->mac.autoneg = DO_AUTO_NEG; 1559 hw->phy.autoneg_wait_to_complete = false; 1560 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1561 1562 if (hw->mac.type < em_mac_min) { 1563 e1000_init_script_state_82541(hw, true); 1564 e1000_set_tbi_compatibility_82543(hw, true); 1565 } 1566 /* Copper options */ 1567 if (hw->phy.media_type == e1000_media_type_copper) { 1568 hw->phy.mdix = AUTO_ALL_MODES; 1569 hw->phy.disable_polarity_correction = false; 1570 hw->phy.ms_type = EM_MASTER_SLAVE; 1571 } 1572 1573 /* 1574 * Set the frame limits assuming 1575 * standard ethernet sized frames. 1576 */ 1577 scctx->isc_max_frame_size = hw->mac.max_frame_size = 1578 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 1579 1580 /* 1581 * This controls when hardware reports transmit completion 1582 * status. 1583 */ 1584 hw->mac.report_tx_early = 1; 1585 1586 /* Allocate multicast array memory. */ 1587 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 1588 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 1589 if (sc->mta == NULL) { 1590 device_printf(dev, 1591 "Can not allocate multicast setup array\n"); 1592 error = ENOMEM; 1593 goto err_late; 1594 } 1595 1596 /* Clear the IFCAP_TSO auto mask */ 1597 sc->tso_automasked = 0; 1598 1599 /* Check SOL/IDER usage on physical functions. */ 1600 if (!sc->vf_ifp && e1000_check_reset_block(hw)) 1601 device_printf(dev, 1602 "PHY reset is blocked due to SOL/IDER session.\n"); 1603 1604 /* Sysctl for setting Energy Efficient Ethernet */ 1605 if (!sc->vf_ifp) { 1606 if (hw->mac.type < igb_mac_min) 1607 hw->dev_spec.ich8lan.eee_disable = eee_setting; 1608 else 1609 hw->dev_spec._82575.eee_disable = eee_setting; 1610 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", 1611 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1612 em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); 1613 } 1614 1615 /* 1616 ** Start from a known state, this is 1617 ** important in reading the nvm and 1618 ** mac from that. 1619 */ 1620 error = e1000_reset_hw(hw); 1621 if (sc->vf_ifp) { 1622 atomic_store_rel_32(&sc->vf_mbx_ready, 1623 error == E1000_SUCCESS); 1624 if (error != E1000_SUCCESS) 1625 igbv_log_reset_failure(sc, error, true); 1626 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 1627 } else if (error != E1000_SUCCESS) { 1628 device_printf(dev, "Hardware reset failed: %d\n", error); 1629 error = EIO; 1630 goto err_late; 1631 } 1632 1633 /* Make sure a PF has a good EEPROM before we read from it. */ 1634 if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) { 1635 /* 1636 ** Some PCI-E parts fail the first check due to 1637 ** the link being in sleep state, call it again, 1638 ** if it fails a second time its a real issue. 1639 */ 1640 if (e1000_validate_nvm_checksum(hw) < 0) { 1641 device_printf(dev, 1642 "The EEPROM Checksum Is Not Valid\n"); 1643 error = EIO; 1644 goto err_late; 1645 } 1646 } 1647 1648 /* Copy the permanent MAC address out of the EEPROM */ 1649 if (e1000_read_mac_addr(hw) < 0) { 1650 device_printf(dev, 1651 "Unable to read MAC address\n"); 1652 error = EIO; 1653 goto err_late; 1654 } 1655 1656 if (!em_is_valid_ether_addr(hw->mac.addr)) { 1657 if (sc->vf_ifp) { 1658 device_printf(dev, 1659 "PF did not assign a MAC address; using a " 1660 "locally generated address\n"); 1661 ether_gen_addr(iflib_get_ifp(ctx), 1662 (struct ether_addr *)hw->mac.addr); 1663 } else { 1664 device_printf(dev, "Invalid MAC address\n"); 1665 error = EIO; 1666 goto err_late; 1667 } 1668 } 1669 1670 if (!sc->vf_ifp) { 1671 /* Save NVM versions while holding the IFLIB context lock. */ 1672 em_fw_version_locked(ctx); 1673 em_print_fw_version(sc); 1674 } 1675 1676 /* 1677 * Get Wake-on-Lan and Management info for later use 1678 */ 1679 if (!sc->vf_ifp) { 1680 em_get_wakeup(ctx); 1681 } 1682 1683 iflib_set_mac(ctx, hw->mac.addr); 1684 1685 return (0); 1686 1687 err_late: 1688 em_release_hw_control(sc); 1689 err_pci: 1690 em_free_pci_resources(ctx); 1691 free(sc->mta, M_DEVBUF); 1692 sc->mta = NULL; 1693 1694 return (error); 1695 } 1696 1697 int 1698 em_if_attach_post(if_ctx_t ctx) 1699 { 1700 struct e1000_softc *sc = iflib_get_softc(ctx); 1701 struct e1000_hw *hw = &sc->hw; 1702 int error = 0; 1703 1704 /* Setup OS specific network interface */ 1705 error = em_setup_interface(ctx); 1706 if (error != 0) { 1707 device_printf(sc->dev, "Interface setup failed: %d\n", error); 1708 goto err_late; 1709 } 1710 1711 if (sc->vf_ifp) { 1712 (void)igbv_reset(ctx); 1713 } else if (em_reset(ctx) != E1000_SUCCESS) { 1714 error = EIO; 1715 goto err_late; 1716 } 1717 1718 /* Initialize statistics */ 1719 if (sc->vf_ifp) 1720 em_initialize_vf_stats(sc); 1721 else 1722 sc->ustats.stats = (struct e1000_hw_stats){}; 1723 1724 em_update_stats_counters(sc); 1725 atomic_readandclear_32(&sc->stats_pending); 1726 hw->mac.get_link_status = 1; 1727 if (sc->vf_ifp) 1728 igbv_if_update_admin_status(ctx); 1729 else 1730 em_if_update_admin_status(ctx); 1731 em_add_hw_stats(sc); 1732 1733 /* Non-AMT based hardware can now take control from firmware */ 1734 if (sc->has_manage && !sc->has_amt) 1735 em_get_hw_control(sc); 1736 1737 INIT_DEBUGOUT("em_if_attach_post: end"); 1738 1739 return (0); 1740 1741 err_late: 1742 /* 1743 * Upon em_if_attach_post() error, iflib calls em_if_detach() to 1744 * free resources 1745 */ 1746 return (error); 1747 } 1748 1749 /********************************************************************* 1750 * Device removal routine 1751 * 1752 * The detach entry point is called when the driver is being removed. 1753 * This routine stops the adapter and deallocates all the resources 1754 * that were allocated for driver operation. 1755 * 1756 * return 0 on success, positive on failure 1757 *********************************************************************/ 1758 static int 1759 em_if_detach(if_ctx_t ctx) 1760 { 1761 struct e1000_softc *sc = iflib_get_softc(ctx); 1762 1763 INIT_DEBUGOUT("em_if_detach: begin"); 1764 1765 igb_iov_detach(sc); 1766 if (sc->vf_ifp) { 1767 igbv_queue_retry_detach(sc); 1768 igbv_mbx_retry_detach(sc); 1769 } else { 1770 e1000_phy_hw_reset(&sc->hw); 1771 } 1772 1773 em_release_manageability(sc); 1774 em_release_hw_control(sc); 1775 em_free_pci_resources(ctx); 1776 free(sc->mta, M_DEVBUF); 1777 sc->mta = NULL; 1778 1779 return (0); 1780 } 1781 1782 /********************************************************************* 1783 * 1784 * Shutdown entry point 1785 * 1786 **********************************************************************/ 1787 1788 static int 1789 em_if_shutdown(if_ctx_t ctx) 1790 { 1791 int error; 1792 1793 error = em_if_suspend(ctx); 1794 if (error != 0) 1795 device_printf(iflib_get_dev(ctx), 1796 "Wake configuration failed during shutdown: %d\n", error); 1797 return (0); 1798 } 1799 1800 /* 1801 * Suspend/resume device methods. 1802 */ 1803 static int 1804 em_if_suspend(if_ctx_t ctx) 1805 { 1806 struct e1000_softc *sc = iflib_get_softc(ctx); 1807 int error; 1808 1809 if (sc->vf_ifp) { 1810 igbv_queue_retry_stop(sc); 1811 igbv_mbx_retry_stop(sc); 1812 } 1813 error = em_enable_wakeup(ctx); 1814 em_release_manageability(sc); 1815 em_release_hw_control(sc); 1816 return (error); 1817 } 1818 1819 static int 1820 em_if_resume(if_ctx_t ctx) 1821 { 1822 struct e1000_softc *sc = iflib_get_softc(ctx); 1823 u32 wus; 1824 u16 phy_wus; 1825 int error; 1826 1827 if (sc->hw.mac.type >= e1000_pch2lan && 1828 sc->hw.mac.type < igb_mac_min) 1829 e1000_resume_workarounds_pchlan(&sc->hw); 1830 1831 if (sc->wol_phy_armed) { 1832 /* 1833 * The PHY wake sequence requires an LCD reset before host wake 1834 * ownership is cleared. Wake registers survive this reset. 1835 */ 1836 (void)e1000_phy_hw_reset(&sc->hw); 1837 error = em_disable_phy_wakeup(sc, &phy_wus); 1838 if (error != E1000_SUCCESS) 1839 device_printf(sc->dev, 1840 "Could not clear PHY wakeup state: %d\n", error); 1841 else if (phy_wus != 0) 1842 device_printf(sc->dev, "PHY wakeup status: %#06x\n", 1843 phy_wus); 1844 } 1845 if (!sc->vf_ifp && sc->hw.mac.type >= e1000_82544) { 1846 wus = E1000_READ_REG(&sc->hw, E1000_WUS); 1847 if (!sc->wol_phy_wakeup && wus != 0) 1848 device_printf(sc->dev, "MAC wakeup status: %#010x\n", 1849 wus); 1850 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 1851 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1852 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 1853 } 1854 /* Clear PME after its MAC or PHY wake source has been removed. */ 1855 pci_clear_pme(sc->dev); 1856 1857 return (0); 1858 } 1859 1860 static int 1861 em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1862 { 1863 int max_frame_size; 1864 struct e1000_softc *sc = iflib_get_softc(ctx); 1865 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1866 1867 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1868 1869 /* No native SET_LPE exchange is available with a Hyper-V PF. */ 1870 if (igbv_is_hyperv(sc) && mtu > ETHERMTU) 1871 return (EINVAL); 1872 1873 switch (sc->hw.mac.type) { 1874 case e1000_82571: 1875 case e1000_82572: 1876 case e1000_ich9lan: 1877 case e1000_ich10lan: 1878 case e1000_pch2lan: 1879 case e1000_pch_lpt: 1880 case e1000_pch_spt: 1881 case e1000_pch_cnp: 1882 case e1000_pch_tgp: 1883 case e1000_pch_adp: 1884 case e1000_pch_mtp: 1885 case e1000_pch_ptp: 1886 case e1000_pch_nvp: 1887 case e1000_82574: 1888 case e1000_82583: 1889 case e1000_80003es2lan: 1890 /* 9K Jumbo Frame size */ 1891 max_frame_size = 9234; 1892 break; 1893 case e1000_pchlan: 1894 max_frame_size = 4096; 1895 break; 1896 case e1000_82542: 1897 case e1000_ich8lan: 1898 /* Adapters that do not support jumbo frames */ 1899 max_frame_size = ETHER_MAX_LEN; 1900 break; 1901 default: 1902 if (sc->hw.mac.type >= igb_mac_min) 1903 max_frame_size = IGB_MAX_FRAME_SIZE; 1904 else /* lem */ 1905 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1906 } 1907 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 1908 return (EINVAL); 1909 } 1910 1911 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 1912 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1913 return (0); 1914 } 1915 1916 /********************************************************************* 1917 * Init entry point 1918 * 1919 * This routine is used in two ways. It is used by the stack as 1920 * init entry point in network interface structure. It is also used 1921 * by the driver as a hw/sw initialization routine to get to a 1922 * consistent state. 1923 * 1924 **********************************************************************/ 1925 static void 1926 em_if_init(if_ctx_t ctx) 1927 { 1928 struct e1000_softc *sc = iflib_get_softc(ctx); 1929 if_softc_ctx_t scctx = sc->shared; 1930 if_t ifp = iflib_get_ifp(ctx); 1931 struct em_tx_queue *tx_que; 1932 int i; 1933 1934 INIT_DEBUGOUT("em_if_init: begin"); 1935 if (sc->vf_ifp) { 1936 igbv_queue_retry_prepare(sc); 1937 igbv_mbx_retry_prepare(sc); 1938 sc->vf_reset_pending = true; 1939 } 1940 if (sc->suspend_link_powered_down) 1941 em_power_up_wakeup_link(sc); 1942 1943 /* Get the latest mac address, User can use a LAA */ 1944 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1945 1946 /* 1947 * A VF restores its address only after its reset handshake establishes 1948 * its PF-assigned state. The PF path programs RAR[0] directly here. 1949 */ 1950 if (!sc->vf_ifp) 1951 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1952 1953 /* 1954 * With the 82571 adapter, RAR[0] may be overwritten 1955 * when the other port is reset, we make a duplicate 1956 * in RAR[14] for that eventuality, this assures 1957 * the interface continues to function. 1958 */ 1959 if (sc->hw.mac.type == e1000_82571) { 1960 e1000_set_laa_state_82571(&sc->hw, true); 1961 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1962 E1000_RAR_ENTRIES - 1); 1963 } 1964 1965 /* Initialize the hardware */ 1966 igb_iov_reset_prepare(sc); 1967 if (sc->vf_ifp) { 1968 (void)igbv_reset(ctx); 1969 em_rebase_vf_stats(sc); 1970 } else if (em_reset(ctx) != E1000_SUCCESS) { 1971 em_fence_pci_busmaster(sc); 1972 iflib_init_failed(ctx); 1973 return; 1974 } 1975 if (sc->vf_ifp && !sc->vf_queues_sanitized) { 1976 /* 1977 * Do not program or enable rings while retained queue state 1978 * might still contain a previous VF owner's DMA address. A 1979 * bounded callout retries initialization after iflib leaves the 1980 * failed initialization stopped. 1981 */ 1982 em_fence_pci_busmaster(sc); 1983 igbv_queue_retry_failed(ctx); 1984 return; 1985 } 1986 if (sc->vf_ifp && 1987 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1988 igbv_mbx_retry_failed(ctx); 1989 return; 1990 } 1991 /* 1992 * Keep a fail-closed device fenced until reset and VF queue 1993 * sanitization have removed every stale DMA address. 1994 */ 1995 if (em_enable_pci_busmaster(sc) != 0) { 1996 device_printf(sc->dev, 1997 "Unable to enable PCI bus mastering\n"); 1998 iflib_init_failed(ctx); 1999 return; 2000 } 2001 if (sc->vf_ifp) 2002 igbv_reconcile_mac(sc, ifp); 2003 /* Re-arm a link-up transition deferred for this reset. */ 2004 if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING || 2005 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING) 2006 sc->link_state = EM_LINK_STATE_DOWN; 2007 if (sc->vf_ifp) 2008 igbv_if_update_admin_status(ctx); 2009 else 2010 em_if_update_admin_status(ctx); 2011 2012 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 2013 i++, tx_que++) { 2014 struct tx_ring *txr = &tx_que->txr; 2015 2016 txr->tx_rs_cidx = txr->tx_rs_pidx; 2017 2018 /* Initialize the last processed descriptor to be the end of 2019 * the ring, rather than the start, so that we avoid an 2020 * off-by-one error when calculating how many descriptors are 2021 * done in the credits_update function. 2022 */ 2023 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 2024 } 2025 2026 /* The VF VLAN EtherType is fixed and has no VET register. */ 2027 if (!sc->vf_ifp) 2028 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 2029 2030 /* Clear bad data from Rx FIFOs */ 2031 if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp) 2032 e1000_rx_fifo_flush_base(&sc->hw); 2033 2034 /* Configure for OS presence */ 2035 em_init_manageability(sc); 2036 2037 /* Prepare transmit descriptors and buffers */ 2038 if (sc->vf_ifp) 2039 igbv_initialize_transmit_unit(ctx); 2040 else 2041 em_initialize_transmit_unit(ctx); 2042 2043 /* 2044 * A failed VF reset has no CTS channel on which to restore mailbox 2045 * state. The reset detector schedules another complete init, which 2046 * replays these interface-owned lists after the handshake succeeds. 2047 */ 2048 em_if_multi_set(ctx); 2049 2050 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 2051 if (sc->vf_ifp) 2052 igbv_initialize_receive_unit(ctx); 2053 else 2054 em_initialize_receive_unit(ctx); 2055 2056 /* Set up VLAN support and filter. */ 2057 em_setup_vlan_hw_support(ctx); 2058 2059 /* Don't lose promiscuous settings. */ 2060 em_if_set_promisc_impl(ctx, if_getflags(ifp)); 2061 atomic_readandclear_32(&sc->promisc_pending); 2062 2063 /* Restore PF/VF pool configuration after the global reset. */ 2064 igb_iov_initialize(sc); 2065 2066 if (sc->hw.mac.ops.clear_hw_cntrs != NULL) 2067 sc->hw.mac.ops.clear_hw_cntrs(&sc->hw); 2068 2069 /* MSI-X configuration for 82574 */ 2070 if (sc->hw.mac.type == e1000_82574) { 2071 int tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 2072 2073 tmp |= E1000_CTRL_EXT_PBA_CLR; 2074 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 2075 /* Set the IVAR - interrupt vector routing. */ 2076 E1000_WRITE_REG(&sc->hw, E1000_IVAR, sc->ivars); 2077 } else if (sc->intr_type == IFLIB_INTR_MSIX) { 2078 /* Set up queue routing */ 2079 igb_configure_queues(sc); 2080 } 2081 if (sc->hw.mac.type >= igb_mac_min) 2082 igb_initialize_interrupt_rate(sc); 2083 2084 /* AMT based hardware can now take control from firmware */ 2085 if (sc->has_manage && sc->has_amt) 2086 em_get_hw_control(sc); 2087 2088 /* Set Energy Efficient Ethernet */ 2089 if (sc->hw.mac.type >= igb_mac_min && 2090 sc->hw.phy.media_type == e1000_media_type_copper) { 2091 if (sc->hw.mac.type == e1000_i354) 2092 e1000_set_eee_i354(&sc->hw, true, true); 2093 else 2094 e1000_set_eee_i350(&sc->hw, true, true); 2095 } 2096 em_configure_peind_memory_errors(sc); 2097 em_configure_82575_memory_errors(sc); 2098 em_configure_82580_memory_errors(sc); 2099 if (sc->vf_ifp) { 2100 sc->vf_reset_pending = false; 2101 } else { 2102 u32 icr; 2103 2104 /* 2105 * Drain stale causes only after register reconstruction is 2106 * complete. DRSTA and DEV_RST_SET together close the window in 2107 * which another device reset can arrive while interrupts are 2108 * masked. 2109 */ 2110 icr = E1000_READ_REG(&sc->hw, E1000_ICR); 2111 if (igb_finish_device_reset(sc, icr)) { 2112 iflib_init_failed(ctx); 2113 return; 2114 } 2115 E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); 2116 } 2117 } 2118 2119 /* 2120 * RX publishes its byte and packet counters as one snapshot when iflib 2121 * returns descriptors to hardware. This also covers watchdog-driven RX 2122 * processing, which can run while the interrupt vector is unmasked. 2123 */ 2124 static __inline void 2125 em_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 2126 { 2127 uint64_t snapshot; 2128 u32 now_bytes, now_packets; 2129 2130 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 2131 now_bytes = snapshot >> 32; 2132 now_packets = (u32)snapshot; 2133 *bytes = now_bytes - rxr->rx_bytes_last; 2134 *packets = now_packets - rxr->rx_packets_last; 2135 rxr->rx_bytes_last = now_bytes; 2136 rxr->rx_packets_last = now_packets; 2137 } 2138 2139 /* 2140 * TX publishes its byte and packet counters as one snapshot at the doorbell, 2141 * because encapsulation can overlap the interrupt filter. The two halves 2142 * remain independent free running u32 counters, so their deltas are correct 2143 * across wrap. 2144 */ 2145 static __inline void 2146 em_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 2147 { 2148 uint64_t snapshot; 2149 u32 now_bytes, now_packets; 2150 2151 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 2152 now_bytes = snapshot >> 32; 2153 now_packets = (u32)snapshot; 2154 *bytes = now_bytes - txr->tx_bytes_last; 2155 *packets = now_packets - txr->tx_packets_last; 2156 txr->tx_bytes_last = now_bytes; 2157 txr->tx_packets_last = now_packets; 2158 } 2159 2160 /********************************************************************* 2161 * 2162 * Do Adaptive Interrupt Moderation: 2163 * - Calculate based on average size over the last interval 2164 * 2165 * Returns interrupts per second rather than a register value, so that the 2166 * caller's EM_INTS_TO_ITR()/IGB_INTS_TO_EITR() conversion applies, or zero 2167 * if the interval carried no packet to measure. 2168 * 2169 *********************************************************************/ 2170 static u32 2171 em_ring_itr(struct e1000_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 2172 u32 txpackets) 2173 { 2174 u32 newitr = 0; 2175 2176 if (txbytes && txpackets) 2177 newitr = txbytes / txpackets; 2178 if (rxbytes && rxpackets) 2179 newitr = max(newitr, rxbytes / rxpackets); 2180 2181 /* 2182 * No packet was observed, so there is no size to work from. Report no 2183 * observation and let the caller keep the rate it already has. 2184 */ 2185 if (newitr == 0) 2186 return (0); 2187 2188 newitr += 24; /* account for hardware frame, crc */ 2189 /* set an upper boundary */ 2190 newitr = min(newitr, 3000); 2191 /* Be nice to the mid range */ 2192 if ((newitr > 300) && (newitr < 1200)) 2193 newitr = (newitr / 3); 2194 else 2195 newitr = (newitr / 2); 2196 2197 /* The value above was written straight to EITR; make it a rate */ 2198 newitr = EM_AIM_DIVIDEND / newitr; 2199 2200 /* 2201 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 2202 * into the low latency end. The original was unbounded and would ask 2203 * for ~95k ints/s on minimum sized frames. There is deliberately no 2204 * floor, so jumbo traffic settles near 2.7k ints/s. 2205 */ 2206 if (sc->enable_aim == 1) 2207 newitr = min(newitr, EM_INTS_20K); 2208 else 2209 newitr = min(newitr, EM_INTS_70K); 2210 2211 return (newitr); 2212 } 2213 2214 /********************************************************************* 2215 * 2216 * Helper to calculate next (E)ITR value for AIM 2217 * 2218 *********************************************************************/ 2219 static void 2220 em_newitr(struct e1000_softc *sc, struct em_rx_queue *que, 2221 struct rx_ring *rxr) 2222 { 2223 struct e1000_hw *hw = &sc->hw; 2224 struct em_tx_queue *tx_que; 2225 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 2226 u32 newitr; 2227 int i; 2228 2229 em_aim_rx_delta(rxr, &rxbytes, &rxpackets); 2230 2231 /* 2232 * A vector can service more than one TX ring when iflib is configured 2233 * with unequal RX and TX queue counts. Sample every ring routed to 2234 * this vector rather than treating the vector as a TX queue index. 2235 */ 2236 txbytes = txpackets = 0; 2237 for (i = 0; i < sc->tx_num_queues; i++) { 2238 tx_que = &sc->tx_queues[i]; 2239 if (tx_que->msix != que->msix) 2240 continue; 2241 em_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 2242 txbytes += ringbytes; 2243 txpackets += ringpackets; 2244 } 2245 2246 /* Idle, do nothing */ 2247 if (txbytes == 0 && rxbytes == 0) 2248 return; 2249 2250 if (sc->enable_aim == 0) { 2251 newitr = em_max_interrupt_rate; 2252 } else if (sc->link_speed < SPEED_1000) { 2253 /* Use half default (4K) ITR if sub-gig */ 2254 newitr = EM_INTS_4K; 2255 } else if (!sc->vf_ifp && 2256 sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 2257 /* Want at least enough packet buffer for two frames to AIM */ 2258 newitr = em_max_interrupt_rate; 2259 } else { 2260 newitr = em_ring_itr(sc, rxbytes, rxpackets, txbytes, 2261 txpackets); 2262 /* No usable observation; leave the rate where it is */ 2263 if (newitr == 0) 2264 return; 2265 } 2266 2267 if (hw->mac.type >= igb_mac_min) { 2268 newitr = IGB_INTS_TO_EITR(newitr); 2269 2270 if (hw->mac.type == e1000_82575) 2271 newitr |= newitr << 16; 2272 else 2273 newitr |= E1000_EITR_CNT_IGNR; 2274 2275 if (newitr != que->itr_setting) { 2276 que->itr_setting = newitr; 2277 E1000_WRITE_REG(hw, E1000_EITR(que->msix), 2278 que->itr_setting); 2279 } 2280 } else { 2281 newitr = EM_INTS_TO_ITR(newitr); 2282 2283 if (newitr != que->itr_setting) { 2284 que->itr_setting = newitr; 2285 if (hw->mac.type == e1000_82574 && 2286 sc->intr_type == IFLIB_INTR_MSIX) { 2287 E1000_WRITE_REG(hw, 2288 E1000_EITR_82574(que->msix), 2289 que->itr_setting); 2290 } else { 2291 E1000_WRITE_REG(hw, E1000_ITR, 2292 que->itr_setting); 2293 } 2294 } 2295 } 2296 } 2297 2298 static bool 2299 em_has_pch_ecc(const struct e1000_hw *hw) 2300 { 2301 2302 return (hw->mac.type >= e1000_pch_lpt && 2303 hw->mac.type < e1000_82575); 2304 } 2305 2306 static bool 2307 em_has_82571_ecc_stats(const struct e1000_hw *hw) 2308 { 2309 2310 return (hw->mac.type == e1000_82571); 2311 } 2312 2313 static bool 2314 em_has_82575_memory_errors(const struct e1000_hw *hw) 2315 { 2316 2317 return (hw->mac.type == e1000_82575); 2318 } 2319 2320 static void 2321 em_configure_82575_memory_errors(struct e1000_softc *sc) 2322 { 2323 struct e1000_hw *hw; 2324 u32 ctrl_ext; 2325 2326 hw = &sc->hw; 2327 if (!em_has_82575_memory_errors(hw)) 2328 return; 2329 2330 /* Discard pre-driver status before enabling the hardware reaction. */ 2331 (void)E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2332 (void)E1000_READ_REG(hw, E1000_RDHESTS_82575); 2333 (void)E1000_READ_REG(hw, E1000_TDHESTS_82575); 2334 E1000_WRITE_REG(hw, E1000_PBECCSTS_82575, 2335 E1000_ECC_82575_ENABLE); 2336 E1000_WRITE_REG(hw, E1000_RDHESTS_82575, 2337 E1000_ECC_82575_ENABLE); 2338 E1000_WRITE_REG(hw, E1000_TDHESTS_82575, 2339 E1000_ECC_82575_ENABLE); 2340 2341 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2342 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2343 ctrl_ext | E1000_CTRL_EXT_MEHE); 2344 E1000_WRITE_FLUSH(hw); 2345 } 2346 2347 static bool 2348 em_has_82576_memory_errors(const struct e1000_hw *hw) 2349 { 2350 2351 return (hw->mac.type == e1000_82576); 2352 } 2353 2354 static bool 2355 em_82576_has_ipsec(const struct e1000_hw *hw) 2356 { 2357 2358 return (hw->device_id != E1000_DEV_ID_82576_NS && 2359 hw->device_id != E1000_DEV_ID_82576_NS_SERDES); 2360 } 2361 2362 static void 2363 em_configure_82576_memory_errors(struct e1000_softc *sc) 2364 { 2365 struct e1000_hw *hw; 2366 u32 peindm, reactions; 2367 2368 hw = &sc->hw; 2369 if (!em_has_82576_memory_errors(hw)) 2370 return; 2371 2372 reactions = E1000_PEIND_82576_NONFATAL_MASK | 2373 E1000_PEIND_82576_FATAL_MASK | 2374 E1000_PEINDM_82576_PARITY_ENABLE; 2375 if (!em_82576_has_ipsec(hw)) 2376 reactions &= ~E1000_PEIND_82576_IPSEC_MASK; 2377 2378 /* Discard indications left by firmware before enabling reactions. */ 2379 (void)E1000_READ_REG(hw, E1000_PEIND); 2380 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2381 E1000_WRITE_REG(hw, E1000_PEINDM, peindm | reactions); 2382 E1000_WRITE_FLUSH(hw); 2383 } 2384 2385 static bool 2386 em_has_82580_memory_errors(const struct e1000_hw *hw) 2387 { 2388 2389 return (hw->mac.type == e1000_82580); 2390 } 2391 2392 static void 2393 em_clear_82580_memory_error_status(struct e1000_hw *hw, u32 reg) 2394 { 2395 u32 status; 2396 2397 status = E1000_READ_REG(hw, reg); 2398 if (status != 0) 2399 E1000_WRITE_REG(hw, reg, status); 2400 } 2401 2402 static void 2403 em_configure_82580_memory_errors(struct e1000_softc *sc) 2404 { 2405 struct e1000_hw *hw; 2406 u32 reg; 2407 2408 hw = &sc->hw; 2409 if (!em_has_82580_memory_errors(hw)) 2410 return; 2411 2412 /* Clear status left before the driver completed its memory tables. */ 2413 (void)E1000_READ_REG(hw, E1000_PEIND); 2414 em_clear_82580_memory_error_status(hw, E1000_DTPARS_82580); 2415 em_clear_82580_memory_error_status(hw, E1000_DRPARS_82580); 2416 em_clear_82580_memory_error_status(hw, E1000_DDPARS_82580); 2417 em_clear_82580_memory_error_status(hw, E1000_PCIEERRSTS); 2418 (void)E1000_READ_REG(hw, E1000_LANPERRSTS); 2419 em_update_82580_ecc_stats(sc, 2420 E1000_READ_REG(hw, E1000_RPBECCSTS), 2421 E1000_READ_REG(hw, E1000_TPBECCSTS), 2422 E1000_READ_REG(hw, E1000_PCIEECCSTS)); 2423 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2424 E1000_PBECCSTS_82580_ECC_ENABLE); 2425 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2426 E1000_PBECCSTS_82580_ECC_ENABLE); 2427 2428 reg = E1000_READ_REG(hw, E1000_DTPARC_82580); 2429 E1000_WRITE_REG(hw, E1000_DTPARC_82580, 2430 reg | E1000_DTPARC_82580_ENABLE_MASK); 2431 reg = E1000_READ_REG(hw, E1000_DRPARC_82580); 2432 E1000_WRITE_REG(hw, E1000_DRPARC_82580, 2433 reg | E1000_DRPARC_82580_ENABLE_MASK); 2434 reg = E1000_READ_REG(hw, E1000_DDPARC_82580); 2435 E1000_WRITE_REG(hw, E1000_DDPARC_82580, 2436 reg | E1000_DDPARC_82580_ENABLE_MASK); 2437 reg = E1000_READ_REG(hw, E1000_PCIEERRCTL_82580); 2438 E1000_WRITE_REG(hw, E1000_PCIEERRCTL_82580, 2439 reg | E1000_PCIEERRCTL_82580_ENABLE_MASK); 2440 reg = E1000_READ_REG(hw, E1000_PCIEECCCTL_82580); 2441 E1000_WRITE_REG(hw, E1000_PCIEECCCTL_82580, 2442 reg | E1000_PCIEECCCTL_82580_ENABLE_MASK); 2443 reg = E1000_READ_REG(hw, E1000_LANPERRCTL_82580); 2444 reg |= E1000_LANPERRCTL_82580_HOST_MASK; 2445 /* The RSS memory is initialized only for a multiqueue layout. */ 2446 if (sc->rx_num_queues <= 1) 2447 reg &= ~E1000_LANPERRCTL_82580_RSS_ENABLE; 2448 E1000_WRITE_REG(hw, E1000_LANPERRCTL_82580, reg); 2449 reg = E1000_READ_REG(hw, E1000_PEINDM); 2450 E1000_WRITE_REG(hw, E1000_PEINDM, 2451 reg | E1000_PEIND_FATAL_MASK); 2452 E1000_WRITE_FLUSH(hw); 2453 } 2454 2455 static bool 2456 em_has_i210_memory_errors(const struct e1000_hw *hw) 2457 { 2458 2459 return (hw->mac.type == e1000_i210 || 2460 hw->mac.type == e1000_i211); 2461 } 2462 2463 static bool 2464 em_has_i350_i354_memory_errors(const struct e1000_hw *hw) 2465 { 2466 2467 return (hw->mac.type == e1000_i350 || 2468 hw->mac.type == e1000_i354); 2469 } 2470 2471 static void 2472 em_configure_peind_memory_errors(struct e1000_softc *sc) 2473 { 2474 struct e1000_hw *hw; 2475 u32 peindm; 2476 2477 hw = &sc->hw; 2478 if (!em_has_i350_i354_memory_errors(hw) && 2479 !em_has_i210_memory_errors(hw)) 2480 return; 2481 2482 /* Discard indications left by firmware before enabling reactions. */ 2483 (void)E1000_READ_REG(hw, E1000_PEIND); 2484 /* Do not depend on firmware preserving the datasheet defaults. */ 2485 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2486 E1000_WRITE_REG(hw, E1000_PEINDM, 2487 peindm | E1000_PEIND_FATAL_MASK); 2488 E1000_WRITE_FLUSH(hw); 2489 } 2490 2491 static bool 2492 em_has_peind_memory_errors(const struct e1000_hw *hw) 2493 { 2494 2495 return (em_has_82580_memory_errors(hw) || 2496 em_has_i350_i354_memory_errors(hw) || 2497 em_has_i210_memory_errors(hw)); 2498 } 2499 2500 static u32 2501 em_pcie_fatal_error_mask(const struct e1000_hw *hw) 2502 { 2503 2504 if (em_has_82580_memory_errors(hw)) 2505 return (~0U); 2506 if (em_has_i350_i354_memory_errors(hw)) 2507 return (E1000_PCIEERRSTS_I350_I354_FATAL_MASK); 2508 if (em_has_i210_memory_errors(hw)) 2509 return (E1000_PCIEERRSTS_I210_FATAL_MASK); 2510 return (0); 2511 } 2512 2513 static u32 2514 em_memory_error_intr_mask(const struct e1000_hw *hw) 2515 { 2516 2517 if (em_has_82575_memory_errors(hw)) 2518 return (E1000_IMS_82575_MEMORY_ERROR_MASK); 2519 if (em_has_82576_memory_errors(hw)) 2520 return (E1000_IMS_FER | E1000_IMS_NFER); 2521 if (em_has_pch_ecc(hw) || em_has_peind_memory_errors(hw)) 2522 return (E1000_IMS_FER); 2523 return (0); 2524 } 2525 2526 static bool 2527 em_has_memory_errors(const struct e1000_hw *hw) 2528 { 2529 2530 return (em_memory_error_intr_mask(hw) != 0); 2531 } 2532 2533 static bool 2534 em_has_memory_error_stats(const struct e1000_hw *hw) 2535 { 2536 2537 return (em_has_82571_ecc_stats(hw) || em_has_memory_errors(hw)); 2538 } 2539 2540 static u32 2541 em_fatal_error_intr_mask(struct e1000_softc *sc) 2542 { 2543 if (!em_has_memory_errors(&sc->hw)) 2544 return (0); 2545 if (atomic_load_acq_32(&sc->fatal_error_state) != 2546 EM_FATAL_ERROR_NONE) 2547 return (0); 2548 return (em_memory_error_intr_mask(&sc->hw)); 2549 } 2550 2551 static void 2552 em_update_82580_ecc_stats(struct e1000_softc *sc, u32 rpbeccsts, 2553 u32 tpbeccsts, u32 pcieeccsts) 2554 { 2555 u32 status; 2556 2557 sc->corrected_error_packet_buffer_count += 2558 (rpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK) + 2559 (tpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK); 2560 status = pcieeccsts & E1000_PCIEECCSTS_82580_ERROR_MASK; 2561 sc->uncorrected_error_pcie_count += bitcount32(status); 2562 if (status != 0) 2563 E1000_WRITE_REG(&sc->hw, E1000_PCIEECCSTS, status); 2564 } 2565 2566 static void 2567 em_update_82575_ecc_stats(struct e1000_softc *sc, u32 pbeccsts, 2568 u32 rdhests, u32 tdhests) 2569 { 2570 2571 sc->corrected_error_packet_buffer_count += 2572 pbeccsts & E1000_ECC_82575_CORR_CNT_MASK; 2573 sc->uncorrected_error_packet_buffer_count += 2574 (pbeccsts & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2575 E1000_ECC_82575_UNCORR_CNT_SHIFT; 2576 sc->corrected_error_dma_count += 2577 (rdhests & E1000_ECC_82575_CORR_CNT_MASK) + 2578 (tdhests & E1000_ECC_82575_CORR_CNT_MASK); 2579 sc->uncorrected_error_dma_count += 2580 ((rdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2581 E1000_ECC_82575_UNCORR_CNT_SHIFT) + 2582 ((tdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2583 E1000_ECC_82575_UNCORR_CNT_SHIFT); 2584 } 2585 2586 static void 2587 em_update_82576_ecc_counter(struct e1000_softc *sc, u32 reg, 2588 u64 *corrected, u64 *uncorrected) 2589 { 2590 u32 status; 2591 2592 status = E1000_READ_REG(&sc->hw, reg); 2593 *corrected += status & E1000_ECC_82576_CORR_CNT_MASK; 2594 if (uncorrected != NULL) 2595 *uncorrected += 2596 (status & E1000_ECC_82576_UNCORR_CNT_MASK) >> 2597 E1000_ECC_82576_UNCORR_CNT_SHIFT; 2598 } 2599 2600 static void 2601 em_update_82576_ecc_stats(struct e1000_softc *sc) 2602 { 2603 2604 /* 2605 * These counters are clear-on-read. PRBESTS and PMSIXESTS are 2606 * controller-shared, so whichever LAN port samples them first owns 2607 * the software count. 2608 */ 2609 em_update_82576_ecc_counter(sc, E1000_RPBECCSTS, 2610 &sc->corrected_error_packet_buffer_count, 2611 &sc->uncorrected_error_packet_buffer_count); 2612 em_update_82576_ecc_counter(sc, E1000_TPBECCSTS, 2613 &sc->corrected_error_packet_buffer_count, 2614 &sc->uncorrected_error_packet_buffer_count); 2615 em_update_82576_ecc_counter(sc, E1000_SWPBECCSTS_82576, 2616 &sc->corrected_error_packet_buffer_count, 2617 &sc->uncorrected_error_packet_buffer_count); 2618 if (em_82576_has_ipsec(&sc->hw)) 2619 em_update_82576_ecc_counter(sc, E1000_IPPBECCSTS_82576, 2620 &sc->corrected_error_packet_buffer_count, 2621 &sc->uncorrected_error_packet_buffer_count); 2622 2623 em_update_82576_ecc_counter(sc, E1000_RDHESTS_82576, 2624 &sc->corrected_error_dma_count, 2625 &sc->uncorrected_error_dma_count); 2626 em_update_82576_ecc_counter(sc, E1000_TDHESTS_82576, 2627 &sc->corrected_error_dma_count, 2628 &sc->uncorrected_error_dma_count); 2629 2630 em_update_82576_ecc_counter(sc, E1000_PRBESTS_82576, 2631 &sc->corrected_error_pcie_retry_count, NULL); 2632 em_update_82576_ecc_counter(sc, E1000_PWBESTS_82576, 2633 &sc->corrected_error_pcie_tx_data_count, NULL); 2634 em_update_82576_ecc_counter(sc, E1000_PMSIXESTS_82576, 2635 &sc->corrected_error_pcie_other_count, NULL); 2636 } 2637 2638 static void 2639 em_update_pch_ecc_stats(struct e1000_softc *sc, u32 pbeccsts) 2640 { 2641 2642 sc->corrected_error_packet_buffer_count += 2643 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 2644 sc->uncorrected_error_packet_buffer_count += 2645 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 2646 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 2647 } 2648 2649 static void 2650 em_update_82571_ecc_stats(struct e1000_softc *sc) 2651 { 2652 struct e1000_hw *hw; 2653 u32 count, pba_ecc; 2654 2655 hw = &sc->hw; 2656 pba_ecc = E1000_READ_REG(hw, E1000_PBA_ECC); 2657 count = (pba_ecc & E1000_PBA_ECC_COUNTER_MASK) >> 2658 E1000_PBA_ECC_COUNTER_SHIFT; 2659 if (count == 0) 2660 return; 2661 sc->corrected_error_packet_buffer_count += count; 2662 /* Preserve correction and reserved state while clearing statistics. */ 2663 E1000_WRITE_REG(hw, E1000_PBA_ECC, 2664 pba_ecc | E1000_PBA_ECC_STAT_CLR); 2665 } 2666 2667 static void 2668 em_update_i210_ecc_stats(struct e1000_softc *sc) 2669 { 2670 struct e1000_hw *hw; 2671 u32 pbeccsts, pcieeccsts; 2672 2673 hw = &sc->hw; 2674 pbeccsts = E1000_READ_REG(hw, E1000_PBECCSTS_I210); 2675 if (pbeccsts & E1000_PBECCSTS_I210_CORR_ERR) { 2676 sc->corrected_error_dma_count++; 2677 /* Preserve the enable bit while clearing the RW1C status. */ 2678 E1000_WRITE_REG(hw, E1000_PBECCSTS_I210, 2679 pbeccsts & (E1000_PBECCSTS_I210_ECC_ENABLE | 2680 E1000_PBECCSTS_I210_CORR_ERR)); 2681 } 2682 2683 pcieeccsts = E1000_READ_REG(hw, E1000_PCIEECCSTS) & 2684 E1000_PCIEECCSTS_I210_CORR_MASK; 2685 if (pcieeccsts & E1000_PCIEECCSTS_TX_WR_DATA) 2686 sc->corrected_error_pcie_tx_data_count++; 2687 if (pcieeccsts & E1000_PCIEECCSTS_RETRY_BUF) 2688 sc->corrected_error_pcie_retry_count++; 2689 if (pcieeccsts != 0) 2690 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieeccsts); 2691 } 2692 2693 static void 2694 em_update_i350_i354_ecc_stats(struct e1000_softc *sc) 2695 { 2696 struct e1000_hw *hw; 2697 u32 pbeccsts, pcieecc_mask, status; 2698 2699 hw = &sc->hw; 2700 status = E1000_READ_REG(hw, E1000_DTPARS) & 2701 E1000_DTPARS_CORR_MASK; 2702 if (status != 0) { 2703 sc->corrected_error_dma_count += bitcount32(status); 2704 E1000_WRITE_REG(hw, E1000_DTPARS, status); 2705 } 2706 status = E1000_READ_REG(hw, E1000_DRPARS) & 2707 E1000_DRPARS_CORR_MASK; 2708 if (status != 0) { 2709 sc->corrected_error_dma_count += bitcount32(status); 2710 E1000_WRITE_REG(hw, E1000_DRPARS, status); 2711 } 2712 status = E1000_READ_REG(hw, E1000_DDECCS) & 2713 E1000_DDECCS_CORR_MASK; 2714 if (status != 0) { 2715 sc->corrected_error_dma_count += bitcount32(status); 2716 E1000_WRITE_REG(hw, E1000_DDECCS, status); 2717 } 2718 status = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2719 E1000_LANPERRSTS_MNG_FIFO_CORR; 2720 if (status != 0) { 2721 sc->corrected_error_lan_mng_fifo_count++; 2722 E1000_WRITE_REG(hw, E1000_LANPERRSTS, status); 2723 } 2724 2725 pbeccsts = E1000_READ_REG(hw, E1000_RPBECCSTS); 2726 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2727 if (status != 0) { 2728 sc->corrected_error_packet_buffer_count += bitcount32(status); 2729 /* Preserve the enable bits while clearing RW1C status. */ 2730 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2731 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2732 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2733 } 2734 pbeccsts = E1000_READ_REG(hw, E1000_TPBECCSTS); 2735 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2736 if (status != 0) { 2737 sc->corrected_error_packet_buffer_count += bitcount32(status); 2738 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2739 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2740 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2741 } 2742 2743 pcieecc_mask = hw->mac.type == e1000_i354 ? 2744 E1000_PCIEECCSTS_I354_CORR_MASK : 2745 E1000_PCIEECCSTS_I350_CORR_MASK; 2746 status = E1000_READ_REG(hw, E1000_PCIEECCSTS) & pcieecc_mask; 2747 if (status & E1000_PCIEECCSTS_TX_WR_DATA) 2748 sc->corrected_error_pcie_tx_data_count++; 2749 if (status & E1000_PCIEECCSTS_RETRY_BUF) 2750 sc->corrected_error_pcie_retry_count++; 2751 sc->corrected_error_pcie_other_count += bitcount32(status & 2752 E1000_PCIEECCSTS_I350_I354_OTHER_MASK); 2753 if (status != 0) 2754 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, status); 2755 } 2756 2757 /* 2758 * Internal-memory error causes are read-clear. Capture them before handing 2759 * fatal recovery or non-fatal acknowledgement to the iflib admin task. 2760 */ 2761 static void 2762 em_handle_fatal_error_intr(struct e1000_softc *sc, u32 icr) 2763 { 2764 struct e1000_hw *hw; 2765 u32 dma_host, dma_rx, dma_tx, error_mask, lanerr, pcieerr, peind; 2766 2767 error_mask = em_memory_error_intr_mask(&sc->hw); 2768 if (!em_has_memory_errors(&sc->hw) || 2769 (icr & error_mask) == 0) 2770 return; 2771 2772 hw = &sc->hw; 2773 E1000_WRITE_REG(hw, E1000_IMC, error_mask); 2774 if (!atomic_cmpset_32(&sc->fatal_error_state, 2775 EM_FATAL_ERROR_NONE, EM_FATAL_ERROR_CAPTURING)) 2776 return; 2777 2778 sc->fatal_error_icr = icr & error_mask; 2779 if (em_has_pch_ecc(hw)) { 2780 sc->fatal_error_pbeccsts = 2781 E1000_READ_REG(hw, E1000_PBECCSTS); 2782 } else if (em_has_82575_memory_errors(hw)) { 2783 sc->fatal_error_pbeccsts = 2784 E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2785 sc->fatal_error_dma_rx = 2786 E1000_READ_REG(hw, E1000_RDHESTS_82575); 2787 sc->fatal_error_dma_tx = 2788 E1000_READ_REG(hw, E1000_TDHESTS_82575); 2789 } else if (em_has_82576_memory_errors(hw)) { 2790 sc->fatal_error_peind = E1000_READ_REG(hw, E1000_PEIND); 2791 } else { 2792 peind = E1000_READ_REG(hw, E1000_PEIND) & 2793 E1000_PEIND_FATAL_MASK; 2794 pcieerr = E1000_READ_REG(hw, E1000_PCIEERRSTS) & 2795 em_pcie_fatal_error_mask(hw); 2796 dma_host = 0; 2797 if (em_has_82580_memory_errors(hw)) { 2798 /* 2799 * PEIND is visible through every function. Retain the 2800 * management indication, which has no subordinate status, 2801 * but attribute host-owned regions from this function's 2802 * status registers. 2803 */ 2804 peind &= E1000_PEIND_MNG_PARITY_FATAL; 2805 dma_tx = E1000_READ_REG(hw, E1000_DTPARS_82580); 2806 dma_rx = E1000_READ_REG(hw, E1000_DRPARS_82580); 2807 dma_host = E1000_READ_REG(hw, 2808 E1000_DDPARS_82580); 2809 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2810 E1000_LANPERRSTS_82580_ERROR_MASK; 2811 } else if (em_has_i350_i354_memory_errors(hw)) { 2812 dma_tx = E1000_READ_REG(hw, E1000_DTPARS) & 2813 E1000_DTPARS_FATAL_MASK; 2814 dma_rx = E1000_READ_REG(hw, E1000_DRPARS) & 2815 E1000_DRPARS_FATAL_MASK; 2816 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2817 E1000_LANPERRSTS_I350_I354_FATAL_MASK; 2818 } else { 2819 dma_tx = 0; 2820 dma_rx = 0; 2821 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2822 E1000_LANPERRSTS_RETX_BUF; 2823 } 2824 if (pcieerr != 0) 2825 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2826 if (lanerr != 0) 2827 peind |= E1000_PEIND_LANPORT_PARITY_FATAL; 2828 if (dma_tx != 0 || dma_rx != 0 || dma_host != 0) 2829 peind |= E1000_PEIND_DMA_PARITY_FATAL; 2830 sc->fatal_error_peind = peind; 2831 sc->fatal_error_pcie = pcieerr; 2832 sc->fatal_error_lan = lanerr; 2833 sc->fatal_error_dma_tx = dma_tx; 2834 sc->fatal_error_dma_rx = dma_rx; 2835 sc->fatal_error_dma_host = dma_host; 2836 } 2837 atomic_store_rel_32(&sc->fatal_error_state, 2838 EM_FATAL_ERROR_DETECTED); 2839 iflib_admin_intr_deferred(sc->ctx); 2840 } 2841 2842 static bool 2843 em_handle_fatal_error_admin(struct e1000_softc *sc) 2844 { 2845 u32 error_mask, pcieecc, peind; 2846 bool reset_required; 2847 2848 if (!atomic_cmpset_acq_32(&sc->fatal_error_state, 2849 EM_FATAL_ERROR_DETECTED, EM_FATAL_ERROR_RESET_REQUESTED)) 2850 return (atomic_load_acq_32(&sc->fatal_error_state) != 2851 EM_FATAL_ERROR_NONE); 2852 2853 if (em_has_pch_ecc(&sc->hw)) { 2854 em_update_pch_ecc_stats(sc, sc->fatal_error_pbeccsts); 2855 device_printf(sc->dev, 2856 "uncorrectable packet-buffer ECC error: " 2857 "PBECCSTS %#x; requesting reset\n", 2858 sc->fatal_error_pbeccsts); 2859 } else if (em_has_82575_memory_errors(&sc->hw)) { 2860 em_update_82575_ecc_stats(sc, sc->fatal_error_pbeccsts, 2861 sc->fatal_error_dma_rx, sc->fatal_error_dma_tx); 2862 device_printf(sc->dev, 2863 "unrecoverable internal memory ECC error: ICR %#x, " 2864 "PBECCSTS %#x, RDHESTS %#x, TDHESTS %#x; " 2865 "requesting reset\n", sc->fatal_error_icr, 2866 sc->fatal_error_pbeccsts, sc->fatal_error_dma_rx, 2867 sc->fatal_error_dma_tx); 2868 } else if (em_has_82576_memory_errors(&sc->hw)) { 2869 peind = sc->fatal_error_peind; 2870 em_update_82576_ecc_stats(sc); 2871 reset_required = 2872 (sc->fatal_error_icr & E1000_ICR_FER) != 0 || 2873 (peind & (E1000_PEIND_82576_FATAL_MASK | 2874 E1000_PEIND_82576_MEMORY_HANG)) != 0; 2875 if (!reset_required) { 2876 device_printf(sc->dev, 2877 "non-fatal internal memory error: PEIND %#x\n", 2878 peind); 2879 sc->fatal_error_icr = 0; 2880 sc->fatal_error_peind = 0; 2881 atomic_store_rel_32(&sc->fatal_error_state, 2882 EM_FATAL_ERROR_NONE); 2883 error_mask = E1000_IMS_FER | E1000_IMS_NFER; 2884 E1000_WRITE_REG(&sc->hw, E1000_IMS, error_mask); 2885 E1000_WRITE_FLUSH(&sc->hw); 2886 return (true); 2887 } 2888 if ((peind & (E1000_PEIND_82576_FATAL_MASK | 2889 E1000_PEIND_82576_MEMORY_HANG)) == 0) 2890 sc->fatal_error_unknown_count++; 2891 device_printf(sc->dev, 2892 "fatal internal memory error: PEIND %#x; " 2893 "requesting reset\n", peind); 2894 } else { 2895 peind = sc->fatal_error_peind; 2896 if (em_has_82580_memory_errors(&sc->hw)) { 2897 pcieecc = E1000_READ_REG(&sc->hw, 2898 E1000_PCIEECCSTS) & 2899 E1000_PCIEECCSTS_82580_ERROR_MASK; 2900 sc->fatal_error_pcie_ecc |= pcieecc; 2901 if (pcieecc != 0) { 2902 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2903 sc->fatal_error_peind = peind; 2904 } 2905 em_update_82580_ecc_stats(sc, 2906 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 2907 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 2908 pcieecc); 2909 } else if (em_has_i350_i354_memory_errors(&sc->hw)) 2910 em_update_i350_i354_ecc_stats(sc); 2911 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) 2912 sc->fatal_error_lan_count++; 2913 if (peind & E1000_PEIND_MNG_PARITY_FATAL) 2914 sc->fatal_error_mng_count++; 2915 if (peind & E1000_PEIND_PCIE_PARITY_FATAL) 2916 sc->fatal_error_pcie_count++; 2917 if (peind & E1000_PEIND_DMA_PARITY_FATAL) 2918 sc->fatal_error_dma_count++; 2919 if (peind == 0) 2920 sc->fatal_error_unknown_count++; 2921 if (em_has_82580_memory_errors(&sc->hw)) { 2922 device_printf(sc->dev, 2923 "fatal internal memory error: PEIND %#x, " 2924 "PCIEERRSTS %#x, PCIEECCSTS %#x, " 2925 "DTPARS %#x, DRPARS %#x, DDPARS %#x, " 2926 "LANPERRSTS %#x\n", peind, 2927 sc->fatal_error_pcie, 2928 sc->fatal_error_pcie_ecc, 2929 sc->fatal_error_dma_tx, 2930 sc->fatal_error_dma_rx, 2931 sc->fatal_error_dma_host, 2932 sc->fatal_error_lan); 2933 } else { 2934 device_printf(sc->dev, 2935 "fatal internal memory error: PEIND %#x, " 2936 "PCIEERRSTS %#x, DTPARS %#x, DRPARS %#x, " 2937 "LANPERRSTS %#x\n", peind, 2938 sc->fatal_error_pcie, 2939 sc->fatal_error_dma_tx, 2940 sc->fatal_error_dma_rx, 2941 sc->fatal_error_lan); 2942 } 2943 2944 reset_required = (peind & 2945 (E1000_PEIND_PCIE_PARITY_FATAL | 2946 E1000_PEIND_DMA_PARITY_FATAL)) != 0; 2947 if (peind == 0) 2948 reset_required = true; 2949 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) { 2950 if (!em_has_i350_i354_memory_errors(&sc->hw) || 2951 sc->fatal_error_lan == 0 || 2952 (sc->fatal_error_lan & 2953 E1000_LANPERRSTS_I350_I354_RESET_MASK) != 0) 2954 reset_required = true; 2955 } 2956 /* Management-memory recovery belongs to management firmware. */ 2957 if (!reset_required) { 2958 if (em_has_i350_i354_memory_errors(&sc->hw) && 2959 sc->fatal_error_lan != 0) 2960 E1000_WRITE_REG(&sc->hw, E1000_LANPERRSTS, 2961 sc->fatal_error_lan & 2962 E1000_LANPERRSTS_I350_I354_NO_RESET_MASK); 2963 sc->fatal_error_peind = 0; 2964 sc->fatal_error_pcie = 0; 2965 sc->fatal_error_pcie_ecc = 0; 2966 sc->fatal_error_lan = 0; 2967 sc->fatal_error_dma_tx = 0; 2968 sc->fatal_error_dma_rx = 0; 2969 sc->fatal_error_dma_host = 0; 2970 atomic_store_rel_32(&sc->fatal_error_state, 2971 EM_FATAL_ERROR_NONE); 2972 E1000_WRITE_REG(&sc->hw, E1000_IMS, 2973 E1000_IMS_FER); 2974 E1000_WRITE_FLUSH(&sc->hw); 2975 return (true); 2976 } 2977 device_printf(sc->dev, 2978 "requesting reset after memory error\n"); 2979 } 2980 sc->fatal_error_reset_count++; 2981 iflib_request_reset(sc->ctx); 2982 /* Re-enter the admin task so it observes the reset request. */ 2983 iflib_admin_intr_deferred(sc->ctx); 2984 return (true); 2985 } 2986 2987 /* 2988 * ICR bit 30 is reserved on 82575 and is the TCP timer on 82576. It becomes 2989 * the Device Reset Asserted interrupt starting with 82580. 2990 */ 2991 static u32 2992 igb_device_reset_intr_mask(struct e1000_softc *sc) 2993 { 2994 2995 return (sc->hw.mac.type >= e1000_82580 ? E1000_IMS_DRSTA : 0); 2996 } 2997 2998 /* Keep interrupt-side work quiesced until device-reset recovery completes. */ 2999 static bool 3000 igb_device_reset_pending(struct e1000_softc *sc) 3001 { 3002 3003 return (!sc->vf_ifp && igb_device_reset_intr_mask(sc) != 0 && 3004 atomic_load_acq_32(&sc->device_reset_state) != 3005 IGB_DEVICE_RESET_NONE); 3006 } 3007 3008 /* 3009 * CTRL.DEV_RST resets every port in the device. ICR.DRSTA tells the other 3010 * ports that their registers and descriptor rings must be reinitialized. 3011 */ 3012 static bool 3013 igb_handle_device_reset(struct e1000_softc *sc, u32 icr) 3014 { 3015 u32 state; 3016 3017 if (sc->vf_ifp || igb_device_reset_intr_mask(sc) == 0 || 3018 (icr & E1000_ICR_DRSTA) == 0) 3019 return (false); 3020 state = atomic_swap_32(&sc->device_reset_state, 3021 IGB_DEVICE_RESET_DETECTED); 3022 if (state == IGB_DEVICE_RESET_DETECTED) 3023 return (true); 3024 3025 iflib_admin_intr_deferred(sc->ctx); 3026 return (true); 3027 } 3028 3029 /* 3030 * A device reset can leave a sibling port accessible before its internal 3031 * reset and PCIe transactions have completed. For 82580 and newer parts, 3032 * wait for that device-wide reset to finish and acknowledge it before any 3033 * ordinary port register programming. I350 and newer parts also publish 3034 * explicit EEPROM autoload and PF-reset completion indications. 3035 * 3036 * The wait is bounded because the only useful fallback for a controller 3037 * that never completes the device reset is the port reset already requested 3038 * by the interrupt handler. 3039 */ 3040 static void 3041 igb_prepare_device_reset(struct e1000_softc *sc) 3042 { 3043 struct e1000_hw *hw; 3044 u32 state; 3045 u32 eecd, gcr, status; 3046 int i; 3047 3048 hw = &sc->hw; 3049 state = atomic_load_acq_32(&sc->device_reset_state); 3050 if (state != IGB_DEVICE_RESET_DETECTED && 3051 state != IGB_DEVICE_RESET_REQUESTED && 3052 hw->mac.type >= e1000_82580) { 3053 /* 3054 * A reset can start while this interface has interrupts disabled. 3055 * GCR is the documented gate before ordinary port accesses. STATUS 3056 * also detects a reset that completed while this interface was down 3057 * or after an earlier preparation pass. 3058 */ 3059 gcr = E1000_READ_REG(hw, E1000_GCR); 3060 if (gcr != 0xffffffff && 3061 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) { 3062 atomic_store_rel_32(&sc->device_reset_state, 3063 IGB_DEVICE_RESET_DETECTED); 3064 state = IGB_DEVICE_RESET_DETECTED; 3065 } else if (gcr != 0xffffffff) { 3066 status = E1000_READ_REG(hw, E1000_STATUS); 3067 if (status != 0xffffffff && 3068 (status & E1000_STAT_DEV_RST_SET) != 0) { 3069 atomic_store_rel_32(&sc->device_reset_state, 3070 IGB_DEVICE_RESET_DETECTED); 3071 state = IGB_DEVICE_RESET_DETECTED; 3072 } 3073 } 3074 } 3075 if (state != IGB_DEVICE_RESET_DETECTED && 3076 state != IGB_DEVICE_RESET_REQUESTED) 3077 return; 3078 3079 if (hw->mac.type >= e1000_82580) { 3080 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3081 gcr = E1000_READ_REG(hw, E1000_GCR); 3082 if (gcr != 0xffffffff && 3083 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) == 0) 3084 break; 3085 msec_delay(1); 3086 } 3087 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) { 3088 device_printf(sc->dev, 3089 "device-wide reset did not complete; " 3090 "attempting port reset\n"); 3091 goto prepared; 3092 } 3093 3094 /* STATUS.DEV_RST_SET is write-one-to-clear. */ 3095 E1000_WRITE_REG(hw, E1000_STATUS, E1000_STAT_DEV_RST_SET); 3096 3097 if (hw->mac.type >= e1000_i350) { 3098 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3099 eecd = E1000_READ_REG(hw, E1000_EECD); 3100 status = E1000_READ_REG(hw, E1000_STATUS); 3101 if (eecd != 0xffffffff && status != 0xffffffff && 3102 (eecd & E1000_EECD_AUTO_RD) != 0 && 3103 (status & E1000_STATUS_RST_DONE) != 0) 3104 break; 3105 msec_delay(1); 3106 } 3107 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) 3108 device_printf(sc->dev, 3109 "device-wide reset did not finish EEPROM " 3110 "autoload or port reset; attempting port " 3111 "reset\n"); 3112 } 3113 } 3114 3115 prepared: 3116 atomic_store_rel_32(&sc->device_reset_state, 3117 IGB_DEVICE_RESET_PREPARED); 3118 } 3119 3120 /* 3121 * A second device reset can arrive while the port is being initialized. 3122 * Leave its status latched for the next preparation pass and do not let 3123 * iflib publish this incomplete initialization as a running datapath. 3124 */ 3125 static bool 3126 igb_finish_device_reset(struct e1000_softc *sc, u32 icr) 3127 { 3128 bool reset_again; 3129 u32 gcr, state, status; 3130 3131 if (igb_device_reset_intr_mask(sc) == 0) 3132 return (false); 3133 3134 state = atomic_load_acq_32(&sc->device_reset_state); 3135 reset_again = icr != 0xffffffff && 3136 (icr & E1000_ICR_DRSTA) != 0; 3137 if (sc->hw.mac.type >= e1000_82580) { 3138 gcr = E1000_READ_REG(&sc->hw, E1000_GCR); 3139 if (gcr != 0xffffffff && 3140 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) 3141 reset_again = true; 3142 status = E1000_READ_REG(&sc->hw, E1000_STATUS); 3143 if (status == 0xffffffff && 3144 state != IGB_DEVICE_RESET_NONE) { 3145 /* 3146 * MMIO can disappear briefly while SR-IOV is changing, but 3147 * config space remains readable. If both are gone, retain the 3148 * stopped state without queueing an endless reset loop. 3149 */ 3150 if (pci_read_config(sc->dev, PCIR_VENDOR, 2) == 0xffff) { 3151 atomic_store_rel_32(&sc->device_reset_state, 3152 IGB_DEVICE_RESET_DETECTED); 3153 device_printf(sc->dev, 3154 "device unavailable after device-wide reset; " 3155 "leaving interface stopped\n"); 3156 return (true); 3157 } 3158 reset_again = true; 3159 } else if (status != 0xffffffff && 3160 (status & E1000_STAT_DEV_RST_SET) != 0) 3161 reset_again = true; 3162 } 3163 if (state == IGB_DEVICE_RESET_DETECTED || 3164 state == IGB_DEVICE_RESET_REQUESTED) 3165 reset_again = true; 3166 if (!reset_again) { 3167 if (state == IGB_DEVICE_RESET_PREPARED && 3168 !atomic_cmpset_rel_32(&sc->device_reset_state, 3169 IGB_DEVICE_RESET_PREPARED, IGB_DEVICE_RESET_NONE)) 3170 return (true); 3171 return (false); 3172 } 3173 3174 state = atomic_swap_32(&sc->device_reset_state, 3175 IGB_DEVICE_RESET_DETECTED); 3176 if (state != IGB_DEVICE_RESET_DETECTED) { 3177 iflib_request_reset_if_up(sc->ctx); 3178 iflib_admin_intr_deferred(sc->ctx); 3179 } 3180 return (true); 3181 } 3182 3183 /* 3184 * A PCIe-region parity failure stops PCIe and DMA traffic. I350, I354, 3185 * I210, and I211 require a port reset before master disable in this case. 3186 * 82580 stops PCIe traffic for a fatal error in any host-owned region, so use 3187 * the same order for every 82580 recovery. This differs from the normal 3188 * reset path, which disables the bus master first. 3189 * 3190 * Indications that relatch after admin accounting are discarded during 3191 * reset; sticky bits cannot distinguish them from the saved event. 3192 */ 3193 static void 3194 em_prepare_fatal_error_reset(struct e1000_softc *sc) 3195 { 3196 struct e1000_hw *hw; 3197 s32 error; 3198 u32 ctrl, pcieecc, pcieerr; 3199 int i; 3200 3201 if (!em_has_peind_memory_errors(&sc->hw) || 3202 atomic_load_acq_32(&sc->fatal_error_state) != 3203 EM_FATAL_ERROR_RESET_REQUESTED) 3204 return; 3205 3206 pcieerr = sc->fatal_error_pcie | 3207 (E1000_READ_REG(&sc->hw, E1000_PCIEERRSTS) & 3208 em_pcie_fatal_error_mask(&sc->hw)); 3209 pcieecc = sc->fatal_error_pcie_ecc; 3210 if (!em_has_82580_memory_errors(&sc->hw) && 3211 (sc->fatal_error_peind & E1000_PEIND_PCIE_PARITY_FATAL) == 0 && 3212 pcieerr == 0) 3213 return; 3214 3215 hw = &sc->hw; 3216 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3217 E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST); 3218 /* Do not access device registers for at least 3 ms after RST. */ 3219 msec_delay(3); 3220 for (i = 0; i < AUTO_READ_DONE_TIMEOUT; i++) { 3221 if ((E1000_READ_REG(hw, E1000_EECD) & 3222 E1000_EECD_AUTO_RD) != 0 && 3223 (em_has_82580_memory_errors(hw) || 3224 (E1000_READ_REG(hw, E1000_STATUS) & 3225 E1000_STATUS_RST_DONE) != 0)) 3226 break; 3227 msec_delay(1); 3228 } 3229 if (i == AUTO_READ_DONE_TIMEOUT) 3230 device_printf(sc->dev, 3231 "port reset did not complete during parity recovery\n"); 3232 error = e1000_disable_pcie_master_generic(hw); 3233 if (error != E1000_SUCCESS) 3234 device_printf(sc->dev, 3235 "PCIe master disable failed during parity recovery: %d\n", 3236 error); 3237 pcieerr |= E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3238 em_pcie_fatal_error_mask(hw); 3239 if (pcieerr != 0) 3240 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3241 if (em_has_82580_memory_errors(hw)) { 3242 pcieecc |= E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3243 E1000_PCIEECCSTS_82580_ERROR_MASK; 3244 if (pcieecc != 0) 3245 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3246 } 3247 atomic_store_rel_32(&sc->fatal_error_state, 3248 EM_FATAL_ERROR_RESET_PREPARED); 3249 } 3250 3251 static void 3252 em_finish_fatal_error_reset(struct e1000_softc *sc) 3253 { 3254 struct e1000_hw *hw; 3255 u32 dma_host, dma_rx, dma_tx, lanerr, pcieecc, pcieerr; 3256 u32 state; 3257 3258 state = atomic_load_acq_32(&sc->fatal_error_state); 3259 if (state != EM_FATAL_ERROR_RESET_REQUESTED && 3260 state != EM_FATAL_ERROR_RESET_PREPARED) 3261 return; 3262 3263 hw = &sc->hw; 3264 if (em_has_82575_memory_errors(hw)) { 3265 sc->fatal_error_dma_tx = 0; 3266 sc->fatal_error_dma_rx = 0; 3267 } else if (em_has_82576_memory_errors(hw)) { 3268 /* Drain any indication relatched while the port was resetting. */ 3269 (void)E1000_READ_REG(hw, E1000_PEIND); 3270 sc->fatal_error_peind = 0; 3271 } else if (em_has_82580_memory_errors(hw)) { 3272 pcieerr = sc->fatal_error_pcie | 3273 E1000_READ_REG(hw, E1000_PCIEERRSTS); 3274 if (pcieerr != 0) 3275 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3276 pcieecc = sc->fatal_error_pcie_ecc | 3277 (E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3278 E1000_PCIEECCSTS_82580_ERROR_MASK); 3279 if (pcieecc != 0) 3280 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3281 dma_tx = sc->fatal_error_dma_tx | 3282 E1000_READ_REG(hw, E1000_DTPARS_82580); 3283 if (dma_tx != 0) 3284 E1000_WRITE_REG(hw, E1000_DTPARS_82580, dma_tx); 3285 dma_rx = sc->fatal_error_dma_rx | 3286 E1000_READ_REG(hw, E1000_DRPARS_82580); 3287 if (dma_rx != 0) 3288 E1000_WRITE_REG(hw, E1000_DRPARS_82580, dma_rx); 3289 dma_host = sc->fatal_error_dma_host | 3290 E1000_READ_REG(hw, E1000_DDPARS_82580); 3291 if (dma_host != 0) 3292 E1000_WRITE_REG(hw, E1000_DDPARS_82580, dma_host); 3293 /* LANPERRSTS is read-only and is cleared by the port reset. */ 3294 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 3295 E1000_LANPERRSTS_82580_ERROR_MASK; 3296 if (lanerr != 0) 3297 device_printf(sc->dev, 3298 "LAN parity status remained set after reset: %#x\n", 3299 lanerr); 3300 (void)E1000_READ_REG(hw, E1000_PEIND); 3301 sc->fatal_error_peind = 0; 3302 sc->fatal_error_pcie = 0; 3303 sc->fatal_error_pcie_ecc = 0; 3304 sc->fatal_error_lan = 0; 3305 sc->fatal_error_dma_tx = 0; 3306 sc->fatal_error_dma_rx = 0; 3307 sc->fatal_error_dma_host = 0; 3308 } else if (em_has_peind_memory_errors(hw)) { 3309 pcieerr = sc->fatal_error_pcie | 3310 (E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3311 em_pcie_fatal_error_mask(hw)); 3312 if (pcieerr != 0) 3313 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3314 if (em_has_i350_i354_memory_errors(hw)) { 3315 dma_tx = sc->fatal_error_dma_tx | 3316 (E1000_READ_REG(hw, E1000_DTPARS) & 3317 E1000_DTPARS_FATAL_MASK); 3318 if (dma_tx != 0) 3319 E1000_WRITE_REG(hw, E1000_DTPARS, dma_tx); 3320 dma_rx = sc->fatal_error_dma_rx | 3321 (E1000_READ_REG(hw, E1000_DRPARS) & 3322 E1000_DRPARS_FATAL_MASK); 3323 if (dma_rx != 0) 3324 E1000_WRITE_REG(hw, E1000_DRPARS, dma_rx); 3325 lanerr = sc->fatal_error_lan | 3326 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3327 E1000_LANPERRSTS_I350_I354_FATAL_MASK); 3328 } else { 3329 lanerr = sc->fatal_error_lan | 3330 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3331 E1000_LANPERRSTS_RETX_BUF); 3332 } 3333 if (lanerr != 0) 3334 E1000_WRITE_REG(hw, E1000_LANPERRSTS, lanerr); 3335 /* 3336 * RST can relatch PEIND from a subordinate status register 3337 * before that register is cleared. Drain the recovered 3338 * indication before unmasking FER. 3339 */ 3340 (void)E1000_READ_REG(hw, E1000_PEIND); 3341 sc->fatal_error_peind = 0; 3342 sc->fatal_error_pcie = 0; 3343 sc->fatal_error_pcie_ecc = 0; 3344 sc->fatal_error_lan = 0; 3345 sc->fatal_error_dma_tx = 0; 3346 sc->fatal_error_dma_rx = 0; 3347 sc->fatal_error_dma_host = 0; 3348 } 3349 sc->fatal_error_icr = 0; 3350 sc->fatal_error_pbeccsts = 0; 3351 atomic_store_rel_32(&sc->fatal_error_state, EM_FATAL_ERROR_NONE); 3352 } 3353 3354 /********************************************************************* 3355 * 3356 * Fast Legacy/MSI Combined Interrupt Service routine 3357 * 3358 *********************************************************************/ 3359 int 3360 em_intr(void *arg) 3361 { 3362 struct e1000_softc *sc = arg; 3363 struct e1000_hw *hw = &sc->hw; 3364 struct em_rx_queue *que = &sc->rx_queues[0]; 3365 struct rx_ring *rxr = &que->rxr; 3366 if_ctx_t ctx = sc->ctx; 3367 u32 reg_icr; 3368 3369 reg_icr = E1000_READ_REG(hw, E1000_ICR); 3370 3371 /* Hot eject? */ 3372 if (reg_icr == 0xffffffff) 3373 return FILTER_STRAY; 3374 3375 /* Definitely not our interrupt. */ 3376 if (reg_icr == 0x0) 3377 return FILTER_STRAY; 3378 3379 /* 3380 * Starting with the 82571 chip, bit 31 should be used to 3381 * determine whether the interrupt belongs to us. 3382 */ 3383 if (hw->mac.type >= e1000_82571 && 3384 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 3385 return FILTER_STRAY; 3386 if (igb_handle_device_reset(sc, reg_icr)) 3387 return (FILTER_HANDLED); 3388 if (igb_device_reset_pending(sc)) 3389 return (FILTER_HANDLED); 3390 3391 /* 3392 * IAM auto-masks igb shared interrupts when ICR is read. Older em 3393 * hardware still needs an explicit disable, which also works around 3394 * MSI message reordering errata on certain systems. 3395 */ 3396 if (sc->vf_ifp || hw->mac.type < igb_mac_min) 3397 IFDI_INTR_DISABLE(ctx); 3398 3399 /* Link status change */ 3400 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3401 em_handle_link(ctx); 3402 3403 if (reg_icr & E1000_ICR_RXO) 3404 sc->rx_overruns++; 3405 3406 em_handle_fatal_error_intr(sc, reg_icr); 3407 3408 if (hw->mac.type >= e1000_82540) 3409 em_newitr(sc, que, rxr); 3410 3411 return (FILTER_SCHEDULE_THREAD); 3412 } 3413 3414 static int 3415 em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3416 { 3417 struct e1000_softc *sc = iflib_get_softc(ctx); 3418 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3419 3420 E1000_WRITE_REG(&sc->hw, E1000_IMS, rxq->eims); 3421 return (0); 3422 } 3423 3424 static int 3425 em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3426 { 3427 struct e1000_softc *sc = iflib_get_softc(ctx); 3428 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3429 3430 E1000_WRITE_REG(&sc->hw, E1000_IMS, txq->eims); 3431 return (0); 3432 } 3433 3434 static int 3435 igb_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3436 { 3437 struct e1000_softc *sc = iflib_get_softc(ctx); 3438 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3439 3440 if (igb_device_reset_pending(sc)) 3441 return (0); 3442 E1000_WRITE_REG(&sc->hw, E1000_EIMS, rxq->eims); 3443 return (0); 3444 } 3445 3446 static int 3447 igb_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3448 { 3449 struct e1000_softc *sc = iflib_get_softc(ctx); 3450 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3451 3452 if (igb_device_reset_pending(sc)) 3453 return (0); 3454 E1000_WRITE_REG(&sc->hw, E1000_EIMS, txq->eims); 3455 return (0); 3456 } 3457 3458 /********************************************************************* 3459 * 3460 * MSI-X RX Interrupt Service routine 3461 * 3462 **********************************************************************/ 3463 static int 3464 em_msix_que(void *arg) 3465 { 3466 struct em_rx_queue *que = arg; 3467 struct e1000_softc *sc = que->sc; 3468 struct rx_ring *rxr = &que->rxr; 3469 3470 ++que->irqs; 3471 3472 if (igb_device_reset_pending(sc)) 3473 return (FILTER_HANDLED); 3474 em_newitr(sc, que, rxr); 3475 3476 return (FILTER_SCHEDULE_THREAD); 3477 } 3478 3479 /********************************************************************* 3480 * 3481 * MSI-X Link Fast Interrupt Service routine 3482 * 3483 **********************************************************************/ 3484 static int 3485 em_msix_link(void *arg) 3486 { 3487 struct e1000_softc *sc = arg; 3488 u32 reg_icr; 3489 3490 ++sc->link_irq; 3491 MPASS(sc->hw.back != NULL); 3492 /* 3493 * The VF's admin vector represents mailbox and link activity. It has 3494 * no PF ICR at E1000_ICR, so process every admin-vector interrupt, 3495 * matching the igbvf misc-vector model. 3496 */ 3497 if (sc->vf_ifp) { 3498 sc->hw.mac.get_link_status = true; 3499 iflib_admin_intr_deferred(sc->ctx); 3500 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3501 return (FILTER_HANDLED); 3502 } 3503 3504 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 3505 if (igb_device_reset_pending(sc)) 3506 return (FILTER_HANDLED); 3507 3508 /* 3509 * Enabling or disabling SR-IOV can briefly make PF MMIO reads return 3510 * all ones. This is not an interrupt cause; in particular, do not 3511 * turn it into a malicious-driver event. 3512 */ 3513 if (__predict_false(reg_icr == 0xffffffff)) 3514 goto rearm; 3515 if (igb_handle_device_reset(sc, reg_icr)) 3516 return (FILTER_HANDLED); 3517 3518 if (reg_icr & E1000_ICR_RXO) 3519 sc->rx_overruns++; 3520 3521 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3522 em_handle_link(sc->ctx); 3523 if (reg_icr & E1000_ICR_MDDET) 3524 igb_iov_mdd_event(sc); 3525 if (reg_icr & E1000_ICR_VMMB) 3526 iflib_admin_intr_deferred(sc->ctx); 3527 em_handle_fatal_error_intr(sc, reg_icr); 3528 3529 rearm: 3530 /* Re-arm unconditionally */ 3531 if (sc->hw.mac.type >= igb_mac_min) { 3532 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3533 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 3534 igb_iov_intr_mask(sc) | 3535 em_fatal_error_intr_mask(sc)); 3536 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3537 } else if (sc->hw.mac.type == e1000_82574) { 3538 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3539 E1000_IMS_LSC | E1000_IMS_OTHER); 3540 /* 3541 * Because we must read the ICR for this interrupt it may 3542 * clear other causes using autoclear, for this reason we 3543 * simply create a soft interrupt for all these vectors. 3544 */ 3545 if (reg_icr) 3546 E1000_WRITE_REG(&sc->hw, E1000_ICS, sc->ims); 3547 } else 3548 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3549 E1000_IMS_LSC | em_fatal_error_intr_mask(sc)); 3550 3551 return (FILTER_HANDLED); 3552 } 3553 3554 static void 3555 em_handle_link(void *context) 3556 { 3557 if_ctx_t ctx = context; 3558 struct e1000_softc *sc = iflib_get_softc(ctx); 3559 3560 sc->hw.mac.get_link_status = 1; 3561 iflib_admin_intr_deferred(ctx); 3562 } 3563 3564 /********************************************************************* 3565 * 3566 * Media Ioctl callback 3567 * 3568 * This routine is called whenever the user queries the status of 3569 * the interface using ifconfig. 3570 * 3571 **********************************************************************/ 3572 static void 3573 em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 3574 { 3575 struct e1000_softc *sc = iflib_get_softc(ctx); 3576 u_char fiber_type = IFM_1000_SX; 3577 3578 INIT_DEBUGOUT("em_if_media_status: begin"); 3579 3580 iflib_admin_intr_deferred(ctx); 3581 3582 ifmr->ifm_status = IFM_AVALID; 3583 ifmr->ifm_active = IFM_ETHER; 3584 3585 if (sc->link_state == EM_LINK_STATE_DOWN || 3586 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING) { 3587 return; 3588 } 3589 3590 ifmr->ifm_status |= IFM_ACTIVE; 3591 3592 if ((sc->hw.phy.media_type == e1000_media_type_fiber) || 3593 (sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3594 if (sc->hw.mac.type == e1000_82545) 3595 fiber_type = IFM_1000_LX; 3596 ifmr->ifm_active |= fiber_type | IFM_FDX; 3597 } else { 3598 switch (sc->link_speed) { 3599 case 10: 3600 ifmr->ifm_active |= IFM_10_T; 3601 break; 3602 case 100: 3603 ifmr->ifm_active |= IFM_100_TX; 3604 break; 3605 case 1000: 3606 ifmr->ifm_active |= IFM_1000_T; 3607 break; 3608 } 3609 if (sc->link_duplex == FULL_DUPLEX) 3610 ifmr->ifm_active |= IFM_FDX; 3611 else 3612 ifmr->ifm_active |= IFM_HDX; 3613 } 3614 } 3615 3616 /********************************************************************* 3617 * 3618 * Media Ioctl callback 3619 * 3620 * This routine is called when the user changes speed/duplex using 3621 * media/mediopt option with ifconfig. 3622 * 3623 **********************************************************************/ 3624 static int 3625 em_if_media_change(if_ctx_t ctx) 3626 { 3627 struct e1000_softc *sc = iflib_get_softc(ctx); 3628 struct ifmedia *ifm = iflib_get_media(ctx); 3629 3630 INIT_DEBUGOUT("em_if_media_change: begin"); 3631 3632 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3633 return (EINVAL); 3634 3635 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3636 case IFM_AUTO: 3637 sc->hw.mac.autoneg = DO_AUTO_NEG; 3638 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 3639 break; 3640 case IFM_1000_LX: 3641 case IFM_1000_SX: 3642 case IFM_1000_T: 3643 sc->hw.mac.autoneg = DO_AUTO_NEG; 3644 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 3645 break; 3646 case IFM_100_TX: 3647 sc->hw.mac.autoneg = false; 3648 sc->hw.phy.autoneg_advertised = 0; 3649 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3650 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 3651 else 3652 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 3653 break; 3654 case IFM_10_T: 3655 sc->hw.mac.autoneg = false; 3656 sc->hw.phy.autoneg_advertised = 0; 3657 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3658 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 3659 else 3660 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 3661 break; 3662 default: 3663 device_printf(sc->dev, "Unsupported media type\n"); 3664 } 3665 3666 return (0); 3667 } 3668 3669 static int 3670 em_if_set_promisc(if_ctx_t ctx, int flags) 3671 { 3672 struct e1000_softc *sc; 3673 3674 sc = iflib_get_softc(ctx); 3675 if (em_if_defer_promisc(sc)) 3676 return (0); 3677 return (em_if_set_promisc_impl(ctx, flags)); 3678 } 3679 3680 static bool 3681 em_if_defer_promisc(struct e1000_softc *sc) 3682 { 3683 switch (sc->hw.mac.type) { 3684 case e1000_82576: 3685 case e1000_i350: 3686 case e1000_vfadapt: 3687 case e1000_vfadapt_i350: 3688 break; 3689 default: 3690 return (false); 3691 } 3692 3693 /* 3694 * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox 3695 * and IOV register operations later from the locked admin task. 3696 * A deferred VF mailbox rejection cannot be returned to ifconfig; the 3697 * admin task logs it instead. 3698 */ 3699 atomic_set_32(&sc->promisc_pending, 1); 3700 iflib_admin_intr_deferred(sc->ctx); 3701 return (true); 3702 } 3703 3704 int 3705 em_if_set_promisc_impl(if_ctx_t ctx, int flags) 3706 { 3707 struct e1000_softc *sc = iflib_get_softc(ctx); 3708 if_t ifp = iflib_get_ifp(ctx); 3709 enum e1000_promisc_type type; 3710 s32 error; 3711 u32 reg_rctl; 3712 int mcnt = 0; 3713 3714 /* Hyper-V receive-mode policy is configured through the host. */ 3715 if (igbv_is_hyperv(sc)) 3716 return (0); 3717 3718 if (sc->vf_ifp) { 3719 if (flags & IFF_PROMISC) 3720 type = e1000_promisc_enabled; 3721 else if (flags & IFF_ALLMULTI) 3722 type = e1000_promisc_multicast; 3723 else 3724 type = e1000_promisc_disabled; 3725 error = e1000_promisc_set_vf(&sc->hw, type); 3726 if (error != E1000_SUCCESS) { 3727 device_printf(sc->dev, 3728 "VF promiscuous-mode request failed\n"); 3729 return (EPERM); 3730 } 3731 return (0); 3732 } 3733 3734 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3735 reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE); 3736 if (flags & IFF_ALLMULTI) 3737 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 3738 else 3739 mcnt = min(if_llmaddr_count(ifp), 3740 MAX_NUM_MULTICAST_ADDRESSES); 3741 3742 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 3743 reg_rctl &= (~E1000_RCTL_MPE); 3744 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3745 3746 if (flags & IFF_PROMISC) { 3747 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3748 /* Turn this on if you want to see bad packets */ 3749 if (em_debug_sbp) 3750 reg_rctl |= E1000_RCTL_SBP; 3751 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3752 if (igb_iov_enabled(sc)) 3753 em_if_vlan_filter_enable(sc); 3754 else 3755 em_if_vlan_filter_disable(sc); 3756 } else { 3757 if (flags & IFF_ALLMULTI) { 3758 reg_rctl |= E1000_RCTL_MPE; 3759 reg_rctl &= ~E1000_RCTL_UPE; 3760 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3761 } 3762 if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx)) 3763 em_if_vlan_filter_enable(sc); 3764 } 3765 igb_iov_update_pf_vmolr(sc); 3766 igb_iov_rebuild_vlan(sc); 3767 return (0); 3768 } 3769 3770 static u_int 3771 em_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 3772 { 3773 u8 *mta = arg; 3774 3775 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 3776 return (0); 3777 3778 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3779 3780 return (1); 3781 } 3782 3783 /* Make every multicast hash eligible on parts whose wake matcher needs MTA. */ 3784 static void 3785 em_fill_wakeup_mta(struct e1000_hw *hw) 3786 { 3787 int i; 3788 3789 memset(hw->mac.mta_shadow, 0xff, sizeof(hw->mac.mta_shadow)); 3790 for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) 3791 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, 3792 hw->mac.mta_shadow[i]); 3793 E1000_WRITE_FLUSH(hw); 3794 } 3795 3796 /********************************************************************* 3797 * Multicast Update 3798 * 3799 * This routine is called whenever multicast address list is updated. 3800 * 3801 **********************************************************************/ 3802 static void 3803 em_if_multi_set(if_ctx_t ctx) 3804 { 3805 struct e1000_softc *sc = iflib_get_softc(ctx); 3806 if_t ifp = iflib_get_ifp(ctx); 3807 u8 *mta; /* Multicast array memory */ 3808 u32 reg_rctl = 0; 3809 int mcnt = 0; 3810 3811 IOCTL_DEBUGOUT("em_set_multi: begin"); 3812 3813 mta = sc->mta; 3814 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 3815 3816 if (sc->hw.mac.type == e1000_82542 && 3817 sc->hw.revision_id == E1000_REVISION_2) { 3818 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3819 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3820 e1000_pci_clear_mwi(&sc->hw); 3821 reg_rctl |= E1000_RCTL_RST; 3822 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3823 msec_delay(5); 3824 } 3825 3826 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta); 3827 3828 if (sc->vf_ifp) { 3829 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3830 igbv_update_uc_addr_list(sc, ifp); 3831 return; 3832 } 3833 3834 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES && 3835 !igb_iov_enabled(sc)) 3836 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3837 3838 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3839 3840 if (if_getflags(ifp) & IFF_PROMISC) 3841 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3842 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 3843 if_getflags(ifp) & IFF_ALLMULTI) { 3844 reg_rctl |= E1000_RCTL_MPE; 3845 reg_rctl &= ~E1000_RCTL_UPE; 3846 } else 3847 reg_rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3848 3849 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3850 3851 if (sc->hw.mac.type == e1000_82542 && 3852 sc->hw.revision_id == E1000_REVISION_2) { 3853 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3854 reg_rctl &= ~E1000_RCTL_RST; 3855 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3856 msec_delay(5); 3857 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3858 e1000_pci_set_mwi(&sc->hw); 3859 } 3860 igb_iov_rebuild_mta(sc); 3861 igb_iov_update_pf_vmolr(sc); 3862 } 3863 3864 /********************************************************************* 3865 * Timer routine 3866 * 3867 * This routine schedules em_if_update_admin_status() to check for 3868 * link status and to gather statistics as well as to perform some 3869 * controller-specific hardware patting. 3870 * 3871 **********************************************************************/ 3872 static void 3873 em_if_timer(if_ctx_t ctx, uint16_t qid) 3874 { 3875 struct e1000_softc *sc; 3876 3877 if (qid != 0) 3878 return; 3879 3880 sc = iflib_get_softc(ctx); 3881 atomic_set_32(&sc->stats_pending, 1); 3882 iflib_admin_intr_deferred(ctx); 3883 } 3884 3885 static void 3886 em_if_update_admin_status(if_ctx_t ctx) 3887 { 3888 struct e1000_softc *sc = iflib_get_softc(ctx); 3889 struct e1000_hw *hw = &sc->hw; 3890 device_t dev = iflib_get_dev(ctx); 3891 u32 link_check, thstat, ctrl; 3892 bool reset_requested = false; 3893 3894 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 3895 if (em_handle_fatal_error_admin(sc)) 3896 return; 3897 /* A sibling-port reset invalidated the registers and VF mailboxes. */ 3898 if (atomic_cmpset_acq_32(&sc->device_reset_state, 3899 IGB_DEVICE_RESET_DETECTED, IGB_DEVICE_RESET_REQUESTED)) { 3900 if (sc->link_state == EM_LINK_STATE_UP) 3901 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 3902 sc->link_speed = 0; 3903 sc->link_duplex = 0; 3904 sc->link_state = EM_LINK_STATE_DOWN_RESET_PENDING; 3905 /* Request the reset here; interrupt filters cannot take STATE_LOCK. */ 3906 iflib_request_reset_if_up(ctx); 3907 /* Re-enter the admin task so it observes the reset request. */ 3908 iflib_admin_intr_deferred(ctx); 3909 return; 3910 } 3911 if (atomic_load_acq_32(&sc->device_reset_state) != 3912 IGB_DEVICE_RESET_NONE) 3913 return; 3914 3915 if (atomic_readandclear_32(&sc->promisc_pending) != 0) 3916 (void)em_if_set_promisc_impl(ctx, 3917 if_getflags(iflib_get_ifp(ctx))); 3918 igb_iov_handle_mdd(sc); 3919 igb_iov_handle_mbx(sc); 3920 3921 link_check = thstat = ctrl = 0; 3922 /* Get the cached link value or read phy for real */ 3923 switch (hw->phy.media_type) { 3924 case e1000_media_type_copper: 3925 if (hw->mac.get_link_status) { 3926 if (hw->mac.type == e1000_pch_spt) 3927 msec_delay(50); 3928 /* Do the work to read phy */ 3929 e1000_check_for_link(hw); 3930 link_check = !hw->mac.get_link_status; 3931 if (link_check) /* ESB2 fix */ 3932 e1000_cfg_on_link_up(hw); 3933 } else { 3934 link_check = true; 3935 } 3936 break; 3937 case e1000_media_type_fiber: 3938 e1000_check_for_link(hw); 3939 link_check = 3940 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 3941 break; 3942 case e1000_media_type_internal_serdes: 3943 e1000_check_for_link(hw); 3944 link_check = hw->mac.serdes_has_link; 3945 break; 3946 default: 3947 break; 3948 } 3949 3950 /* Check for thermal downshift or shutdown */ 3951 if (hw->mac.type == e1000_i350) { 3952 thstat = E1000_READ_REG(hw, E1000_THSTAT); 3953 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); 3954 } 3955 3956 /* Now check for a transition */ 3957 if (link_check && 3958 (sc->link_state == EM_LINK_STATE_DOWN || 3959 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { 3960 bool reset_pending; 3961 3962 reset_pending = 3963 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING; 3964 e1000_get_speed_and_duplex(hw, &sc->link_speed, 3965 &sc->link_duplex); 3966 /* Check if we must disable SPEED_MODE bit on PCI-E */ 3967 if ((sc->link_speed != SPEED_1000) && 3968 ((hw->mac.type == e1000_82571) || 3969 (hw->mac.type == e1000_82572))) { 3970 int tarc0; 3971 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 3972 tarc0 &= ~TARC_SPEED_MODE_BIT; 3973 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 3974 } 3975 if (bootverbose) 3976 device_printf(dev, "Link is up %d Mbps %s\n", 3977 sc->link_speed, 3978 ((sc->link_duplex == FULL_DUPLEX) ? 3979 "Full Duplex" : "Half Duplex")); 3980 sc->link_state = EM_LINK_STATE_UP; 3981 sc->smartspeed = 0; 3982 if (hw->mac.type == e1000_i350 && 3983 (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == 3984 E1000_CTRL_EXT_LINK_MODE_GMII && 3985 (thstat & E1000_THSTAT_LINK_THROTTLE)) 3986 device_printf(dev, "Link: thermal downshift\n"); 3987 /* Delay Link Up for Phy update */ 3988 if (((hw->mac.type == e1000_i210) || 3989 (hw->mac.type == e1000_i211)) && 3990 (hw->phy.id == I210_I_PHY_ID)) 3991 msec_delay(I210_LINK_DELAY); 3992 /* Reset if the media type changed. */ 3993 if (hw->dev_spec._82575.media_changed && 3994 hw->mac.type >= igb_mac_min) { 3995 hw->dev_spec._82575.media_changed = false; 3996 sc->flags |= IGB_MEDIA_RESET; 3997 iflib_request_reset(ctx); 3998 iflib_admin_intr_deferred(ctx); 3999 reset_requested = true; 4000 } 4001 /* Only do TSO on gigabit for older chips due to errata */ 4002 if (hw->mac.type < igb_mac_min) 4003 reset_requested = em_automask_tso(ctx); 4004 4005 if (reset_pending || reset_requested) { 4006 /* 4007 * The PHY is up, but publish it only after the TSO 4008 * capability-change reset. 4009 */ 4010 sc->link_state = EM_LINK_STATE_UP_RESET_PENDING; 4011 } else { 4012 iflib_link_state_change(ctx, LINK_STATE_UP, 4013 IF_Mbps(sc->link_speed)); 4014 } 4015 igb_iov_ping_all_vfs(sc); 4016 } else if (!link_check && 4017 (sc->link_state == EM_LINK_STATE_UP || 4018 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { 4019 bool link_was_published; 4020 bool reset_pending; 4021 4022 link_was_published = sc->link_state == EM_LINK_STATE_UP; 4023 reset_pending = 4024 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING; 4025 sc->link_speed = 0; 4026 sc->link_duplex = 0; 4027 sc->link_state = reset_pending ? 4028 EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN; 4029 if (link_was_published) 4030 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4031 igb_iov_ping_all_vfs(sc); 4032 } 4033 /* 4034 * Mailbox, link, and timer events share this admin task. The PF 4035 * statistics sweep performs 66 MMIO reads, so run it only when the 4036 * ordinary iflib timer requests a sample rather than once per mailbox 4037 * message. Exported counters can consequently trail hardware by the 4038 * timer interval (normally 500 ms). 4039 */ 4040 if (atomic_readandclear_32(&sc->stats_pending) != 0) { 4041 em_update_stats_counters(sc); 4042 /* 4043 * The 82574 PHY can enter a state in which both its receive 4044 * error and idle error counters saturate. Require two 4045 * consecutive timer samples before resetting, matching Intel's 4046 * e1000e recovery policy and avoiding a reset on a transient 4047 * register sample. 4048 */ 4049 if (hw->mac.type == e1000_82574) { 4050 if (e1000_check_phy_82574(hw)) 4051 sc->phy_hang_count++; 4052 else 4053 sc->phy_hang_count = 0; 4054 if (sc->phy_hang_count > 1) { 4055 sc->phy_hang_count = 0; 4056 device_printf(dev, 4057 "PHY appears hung; requesting reset\n"); 4058 iflib_request_reset(ctx); 4059 iflib_admin_intr_deferred(ctx); 4060 return; 4061 } 4062 } 4063 } 4064 4065 /* Reset LAA into RAR[0] on 82571 */ 4066 if (hw->mac.type == e1000_82571 && e1000_get_laa_state_82571(hw)) 4067 e1000_rar_set(hw, hw->mac.addr, 0); 4068 4069 if (hw->mac.type < em_mac_min) 4070 lem_smartspeed(sc); 4071 } 4072 4073 /* 4074 * Last-resort DMA fence. iflib releases DMA mappings after the driver's 4075 * stop callback, so continuing with bus mastering still enabled would turn 4076 * a recoverable NIC failure into memory corruption. Treat failure of the 4077 * PCI command bit as a fail-stop invariant violation. 4078 */ 4079 static void 4080 em_fence_pci_busmaster(struct e1000_softc *sc) 4081 { 4082 device_t dev; 4083 u_int timeout; 4084 u16 command; 4085 int error; 4086 4087 dev = sc->dev; 4088 error = pci_disable_busmaster(dev); 4089 command = pci_read_config(dev, PCIR_COMMAND, 2); 4090 if (command != 0xffff && (command & PCIM_CMD_BUSMASTEREN) != 0) 4091 panic("%s: unable to fence device DMA (error %d)", 4092 device_get_nameunit(dev), error); 4093 if (error != 0 && command != 0xffff) 4094 device_printf(dev, 4095 "PCI bus-master disable returned %d; readback is disabled\n", 4096 error); 4097 4098 timeout = max(pcie_get_max_completion_timeout(dev) / 1000, 10); 4099 if (command != 0xffff && 4100 !pcie_wait_for_pending_transactions(dev, timeout)) { 4101 /* A function removed during the wait can no longer issue DMA. */ 4102 command = pci_read_config(dev, PCIR_COMMAND, 2); 4103 if (command != 0xffff) 4104 panic("%s: DMA transactions remain pending after fencing", 4105 device_get_nameunit(dev)); 4106 } 4107 } 4108 4109 static int 4110 em_enable_pci_busmaster(struct e1000_softc *sc) 4111 { 4112 device_t dev; 4113 u16 command; 4114 int error; 4115 4116 dev = sc->dev; 4117 command = pci_read_config(dev, PCIR_COMMAND, 2); 4118 if (command == 0xffff) 4119 return (ENXIO); 4120 if ((command & PCIM_CMD_BUSMASTEREN) != 0) 4121 return (0); 4122 4123 error = pci_enable_busmaster(dev); 4124 command = pci_read_config(dev, PCIR_COMMAND, 2); 4125 if (command == 0xffff) 4126 return (ENXIO); 4127 if ((command & PCIM_CMD_BUSMASTEREN) == 0) 4128 return (error != 0 ? error : EIO); 4129 return (0); 4130 } 4131 4132 /********************************************************************* 4133 * 4134 * This routine disables all traffic on the adapter by issuing a 4135 * global reset on the MAC. 4136 * 4137 **********************************************************************/ 4138 static void 4139 em_if_stop(if_ctx_t ctx) 4140 { 4141 struct e1000_softc *sc = iflib_get_softc(ctx); 4142 s32 error; 4143 4144 INIT_DEBUGOUT("em_if_stop: begin"); 4145 4146 if (sc->vf_ifp) { 4147 igbv_queue_retry_stop(sc); 4148 igbv_mbx_retry_stop(sc); 4149 } 4150 4151 /* I219 needs special flushing to avoid hangs */ 4152 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 4153 em_flush_desc_rings(sc); 4154 4155 igb_iov_reset_prepare(sc); 4156 if (!sc->vf_ifp || 4157 (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 && 4158 (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) { 4159 em_prepare_fatal_error_reset(sc); 4160 error = e1000_reset_hw(&sc->hw); 4161 if (!sc->vf_ifp && error != E1000_SUCCESS) { 4162 device_printf(sc->dev, "Hardware reset failed while " 4163 "stopping: %d\n", error); 4164 em_fence_pci_busmaster(sc); 4165 return; 4166 } 4167 } 4168 if (sc->vf_ifp) { 4169 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 4170 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 4171 if (!sc->vf_queues_sanitized) 4172 em_fence_pci_busmaster(sc); 4173 } 4174 if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp) 4175 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 4176 4177 if (!sc->vf_ifp) { 4178 e1000_led_off(&sc->hw); 4179 e1000_cleanup_led(&sc->hw); 4180 } else { 4181 sc->link_speed = 0; 4182 sc->link_duplex = 0; 4183 if (sc->link_state != EM_LINK_STATE_DOWN) { 4184 sc->link_state = EM_LINK_STATE_DOWN; 4185 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4186 } 4187 } 4188 } 4189 4190 /********************************************************************* 4191 * 4192 * Determine hardware revision. 4193 * 4194 **********************************************************************/ 4195 static void 4196 em_identify_hardware(if_ctx_t ctx) 4197 { 4198 device_t dev = iflib_get_dev(ctx); 4199 struct e1000_softc *sc = iflib_get_softc(ctx); 4200 4201 /* Make sure our PCI config space has the necessary stuff set */ 4202 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 4203 4204 /* Save off the information about this board */ 4205 sc->hw.vendor_id = pci_get_vendor(dev); 4206 sc->hw.device_id = pci_get_device(dev); 4207 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 4208 sc->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 4209 sc->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 4210 4211 /* Do Shared Code Init and Setup */ 4212 if (e1000_set_mac_type(&sc->hw)) { 4213 device_printf(dev, "Setup init failure\n"); 4214 return; 4215 } 4216 4217 /* 4218 * Function type comes from the selected iflib shared context, not from 4219 * enum ordering. Keep the detected MAC type as an independent check 4220 * that the igb/igbv probe tables selected the right policy. 4221 */ 4222 KASSERT(sc->vf_ifp == 4223 (sc->hw.mac.type == e1000_vfadapt || 4224 sc->hw.mac.type == e1000_vfadapt_i350), 4225 ("%s: iflib function type and MAC type disagree", __func__)); 4226 } 4227 4228 static int 4229 em_allocate_pci_resources(if_ctx_t ctx) 4230 { 4231 struct e1000_softc *sc = iflib_get_softc(ctx); 4232 device_t dev = iflib_get_dev(ctx); 4233 int rid, val; 4234 4235 rid = PCIR_BAR(0); 4236 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 4237 RF_ACTIVE); 4238 if (sc->memory == NULL) { 4239 device_printf(dev, 4240 "Unable to allocate bus resource: memory\n"); 4241 return (ENXIO); 4242 } 4243 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 4244 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 4245 #ifdef INVARIANTS 4246 sc->osdep.mem_bus_space_size = rman_get_size(sc->memory); 4247 #endif 4248 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 4249 4250 /* Only older adapters use IO mapping */ 4251 if (sc->hw.mac.type < em_mac_min && sc->hw.mac.type > e1000_82543) { 4252 /* Figure our where our IO BAR is ? */ 4253 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 4254 val = pci_read_config(dev, rid, 4); 4255 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 4256 break; 4257 } 4258 rid += 4; 4259 /* check for 64bit BAR */ 4260 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 4261 rid += 4; 4262 } 4263 if (rid >= PCIR_CIS) { 4264 device_printf(dev, "Unable to locate IO BAR\n"); 4265 return (ENXIO); 4266 } 4267 sc->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 4268 &rid, RF_ACTIVE); 4269 if (sc->ioport == NULL) { 4270 device_printf(dev, 4271 "Unable to allocate bus resource: ioport\n"); 4272 return (ENXIO); 4273 } 4274 sc->hw.io_base = 0; 4275 sc->osdep.io_bus_space_tag = 4276 rman_get_bustag(sc->ioport); 4277 sc->osdep.io_bus_space_handle = 4278 rman_get_bushandle(sc->ioport); 4279 } 4280 4281 sc->hw.back = &sc->osdep; 4282 4283 return (0); 4284 } 4285 4286 /********************************************************************* 4287 * 4288 * Set up the MSI-X Interrupt handlers 4289 * 4290 **********************************************************************/ 4291 static int 4292 em_if_msix_intr_assign(if_ctx_t ctx, int msix) 4293 { 4294 struct e1000_softc *sc = iflib_get_softc(ctx); 4295 struct em_rx_queue *rx_que = sc->rx_queues; 4296 struct em_tx_queue *tx_que = sc->tx_queues; 4297 int error, rid, i, vector = 0, rx_vectors; 4298 char buf[16]; 4299 4300 /* First set up ring resources */ 4301 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 4302 rid = vector + 1; 4303 snprintf(buf, sizeof(buf), "rxq%d", i); 4304 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 4305 IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); 4306 if (error) { 4307 device_printf(iflib_get_dev(ctx), 4308 "Failed to allocate que int %d err: %d", 4309 i, error); 4310 sc->rx_num_queues = i + 1; 4311 goto fail; 4312 } 4313 4314 rx_que->msix = vector; 4315 4316 /* 4317 * Set the bit to enable interrupt 4318 * in E1000_IMS -- bits 20 and 21 4319 * are for RX0 and RX1, note this has 4320 * NOTHING to do with the MSI-X vector 4321 */ 4322 if (sc->hw.mac.type == e1000_82574) { 4323 rx_que->eims = 1 << (20 + i); 4324 sc->ims |= rx_que->eims; 4325 sc->ivars |= (8 | rx_que->msix) << (i * 4); 4326 } else if (sc->hw.mac.type == e1000_82575) 4327 rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; 4328 else 4329 rx_que->eims = 1 << vector; 4330 } 4331 rx_vectors = vector; 4332 4333 vector = 0; 4334 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 4335 snprintf(buf, sizeof(buf), "txq%d", i); 4336 tx_que = &sc->tx_queues[i]; 4337 iflib_softirq_alloc_generic(ctx, 4338 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 4339 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 4340 4341 tx_que->msix = (vector % sc->rx_num_queues); 4342 4343 /* 4344 * Set the bit to enable interrupt 4345 * in E1000_IMS -- bits 22 and 23 4346 * are for TX0 and TX1, note this has 4347 * NOTHING to do with the MSI-X vector 4348 */ 4349 if (sc->hw.mac.type == e1000_82574) { 4350 tx_que->eims = 1 << (22 + i); 4351 sc->ims |= tx_que->eims; 4352 sc->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); 4353 } else if (sc->hw.mac.type == e1000_82575) { 4354 tx_que->eims = E1000_EICR_TX_QUEUE0 << i; 4355 } else { 4356 tx_que->eims = 1 << i; 4357 } 4358 } 4359 4360 /* Link interrupt */ 4361 rid = rx_vectors + 1; 4362 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 4363 em_msix_link, sc, 0, "aq"); 4364 4365 if (error) { 4366 device_printf(iflib_get_dev(ctx), 4367 "Failed to register admin handler"); 4368 goto fail; 4369 } 4370 sc->linkvec = rx_vectors; 4371 if (sc->hw.mac.type < igb_mac_min) { 4372 sc->ivars |= (8 | rx_vectors) << 16; 4373 sc->ivars |= 0x80000000; 4374 /* Enable the "Other" interrupt type for link status change */ 4375 sc->ims |= E1000_IMS_OTHER; 4376 } 4377 4378 return (0); 4379 fail: 4380 iflib_irq_free(ctx, &sc->irq); 4381 rx_que = sc->rx_queues; 4382 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 4383 iflib_irq_free(ctx, &rx_que->que_irq); 4384 return (error); 4385 } 4386 4387 static void 4388 igb_configure_queues(struct e1000_softc *sc) 4389 { 4390 struct e1000_hw *hw = &sc->hw; 4391 struct em_rx_queue *rx_que; 4392 struct em_tx_queue *tx_que; 4393 u32 tmp, ivar = 0; 4394 4395 /* 4396 * Queue ownership can change when SR-IOV is enabled or disabled. 4397 * Rebuild the interrupt mask for the current layout instead of 4398 * retaining vectors from a previous initialization. 4399 */ 4400 sc->que_mask = 0; 4401 sc->link_mask = 0; 4402 4403 /* GPIE controls the PF interrupt block and is not in the VF BAR. */ 4404 if (!sc->vf_ifp && hw->mac.type != e1000_82575) 4405 E1000_WRITE_REG(hw, E1000_GPIE, 4406 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 4407 E1000_GPIE_PBA | E1000_GPIE_NSICR); 4408 4409 /* Turn on MSI-X */ 4410 switch (hw->mac.type) { 4411 case e1000_82580: 4412 case e1000_i350: 4413 case e1000_i354: 4414 case e1000_i210: 4415 case e1000_i211: 4416 case e1000_vfadapt: 4417 case e1000_vfadapt_i350: 4418 /* RX entries */ 4419 for (int i = 0; i < sc->rx_num_queues; i++) { 4420 uint32_t index, qid; 4421 4422 rx_que = &sc->rx_queues[i]; 4423 qid = rx_que->rxr.me; 4424 index = qid >> 1; 4425 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4426 if (qid & 1) { 4427 ivar &= 0xFF00FFFF; 4428 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4429 16; 4430 } else { 4431 ivar &= 0xFFFFFF00; 4432 ivar |= rx_que->msix | E1000_IVAR_VALID; 4433 } 4434 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4435 sc->que_mask |= rx_que->eims; 4436 } 4437 /* TX entries */ 4438 for (int i = 0; i < sc->tx_num_queues; i++) { 4439 uint32_t index, qid; 4440 4441 tx_que = &sc->tx_queues[i]; 4442 qid = tx_que->txr.me; 4443 index = qid >> 1; 4444 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4445 if (qid & 1) { 4446 ivar &= 0x00FFFFFF; 4447 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4448 24; 4449 } else { 4450 ivar &= 0xFFFF00FF; 4451 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4452 8; 4453 } 4454 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4455 sc->que_mask |= tx_que->eims; 4456 } 4457 4458 /* And for the link interrupt */ 4459 if (sc->vf_ifp) { 4460 /* 4461 * VTIVAR_MISC maps the VF mailbox in bits 7:0. 4462 * The PF IVAR_MISC maps other causes in bits 15:8. 4463 */ 4464 ivar = sc->linkvec | E1000_IVAR_VALID; 4465 } else 4466 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4467 sc->link_mask = 1 << sc->linkvec; 4468 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4469 break; 4470 case e1000_82576: 4471 /* RX entries */ 4472 for (int i = 0; i < sc->rx_num_queues; i++) { 4473 uint32_t index, qid; 4474 4475 rx_que = &sc->rx_queues[i]; 4476 qid = rx_que->rxr.me; 4477 index = qid & 0x7; /* Each IVAR has two entries */ 4478 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4479 if (qid < 8) { 4480 ivar &= 0xFFFFFF00; 4481 ivar |= rx_que->msix | E1000_IVAR_VALID; 4482 } else { 4483 ivar &= 0xFF00FFFF; 4484 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4485 16; 4486 } 4487 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4488 sc->que_mask |= rx_que->eims; 4489 } 4490 /* TX entries */ 4491 for (int i = 0; i < sc->tx_num_queues; i++) { 4492 uint32_t index, qid; 4493 4494 tx_que = &sc->tx_queues[i]; 4495 qid = tx_que->txr.me; 4496 index = qid & 0x7; /* Each IVAR has two entries */ 4497 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4498 if (qid < 8) { 4499 ivar &= 0xFFFF00FF; 4500 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4501 8; 4502 } else { 4503 ivar &= 0x00FFFFFF; 4504 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4505 24; 4506 } 4507 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4508 sc->que_mask |= tx_que->eims; 4509 } 4510 4511 /* And for the link interrupt */ 4512 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4513 sc->link_mask = 1 << sc->linkvec; 4514 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4515 break; 4516 4517 case e1000_82575: 4518 /* enable MSI-X support*/ 4519 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 4520 tmp |= E1000_CTRL_EXT_PBA_CLR; 4521 /* Auto-Mask interrupts upon ICR read. */ 4522 tmp |= E1000_CTRL_EXT_EIAME; 4523 tmp |= E1000_CTRL_EXT_IRCA; 4524 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 4525 4526 /* Queues */ 4527 for (int i = 0; i < sc->rx_num_queues; i++) { 4528 rx_que = &sc->rx_queues[i]; 4529 tmp = E1000_EICR_RX_QUEUE0 << i; 4530 tmp |= E1000_EICR_TX_QUEUE0 << i; 4531 rx_que->eims = tmp; 4532 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), i, 4533 rx_que->eims); 4534 sc->que_mask |= rx_que->eims; 4535 } 4536 4537 /* Link */ 4538 E1000_WRITE_REG(hw, E1000_MSIXBM(sc->linkvec), 4539 E1000_EIMS_OTHER); 4540 sc->link_mask |= E1000_EIMS_OTHER; 4541 default: 4542 break; 4543 } 4544 4545 return; 4546 } 4547 4548 static void 4549 igb_initialize_interrupt_rate(struct e1000_softc *sc) 4550 { 4551 struct e1000_hw *hw = &sc->hw; 4552 struct em_rx_queue *rx_que; 4553 u32 newitr; 4554 4555 newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); 4556 if (hw->mac.type == e1000_82575) 4557 newitr |= newitr << 16; 4558 else 4559 newitr |= E1000_EITR_CNT_IGNR; 4560 4561 for (int i = 0; i < sc->rx_num_queues; i++) { 4562 rx_que = &sc->rx_queues[i]; 4563 rx_que->itr_setting = newitr; 4564 E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), 4565 rx_que->itr_setting); 4566 } 4567 if (sc->intr_type == IFLIB_INTR_MSIX) 4568 E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr); 4569 } 4570 4571 static void 4572 em_free_pci_resources(if_ctx_t ctx) 4573 { 4574 struct e1000_softc *sc = iflib_get_softc(ctx); 4575 struct em_rx_queue *que = sc->rx_queues; 4576 device_t dev = iflib_get_dev(ctx); 4577 4578 /* Release all MSI-X queue resources */ 4579 if (sc->intr_type == IFLIB_INTR_MSIX) 4580 iflib_irq_free(ctx, &sc->irq); 4581 4582 if (que != NULL) { 4583 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 4584 iflib_irq_free(ctx, &que->que_irq); 4585 } 4586 } 4587 4588 if (sc->memory != NULL) { 4589 bus_release_resource(dev, SYS_RES_MEMORY, 4590 rman_get_rid(sc->memory), sc->memory); 4591 sc->memory = NULL; 4592 } 4593 4594 if (sc->flash != NULL) { 4595 bus_release_resource(dev, SYS_RES_MEMORY, 4596 rman_get_rid(sc->flash), sc->flash); 4597 sc->flash = NULL; 4598 } 4599 4600 if (sc->ioport != NULL) { 4601 bus_release_resource(dev, SYS_RES_IOPORT, 4602 rman_get_rid(sc->ioport), sc->ioport); 4603 sc->ioport = NULL; 4604 } 4605 } 4606 4607 /* Set up MSI or MSI-X */ 4608 static int 4609 em_setup_msix(if_ctx_t ctx) 4610 { 4611 struct e1000_softc *sc = iflib_get_softc(ctx); 4612 4613 if (sc->hw.mac.type == e1000_82574) { 4614 em_enable_vectors_82574(ctx); 4615 } 4616 return (0); 4617 } 4618 4619 /********************************************************************* 4620 * 4621 * Workaround for SmartSpeed on 82541 and 82547 controllers 4622 * 4623 **********************************************************************/ 4624 static void 4625 lem_smartspeed(struct e1000_softc *sc) 4626 { 4627 u16 phy_tmp; 4628 4629 if (sc->link_state == EM_LINK_STATE_UP || 4630 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING || 4631 (sc->hw.phy.type != e1000_phy_igp) || 4632 sc->hw.mac.autoneg == 0 || 4633 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 4634 return; 4635 4636 if (sc->smartspeed == 0) { 4637 /* If Master/Slave config fault is asserted twice, 4638 * we assume back-to-back */ 4639 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4640 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 4641 return; 4642 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4643 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 4644 e1000_read_phy_reg(&sc->hw, 4645 PHY_1000T_CTRL, &phy_tmp); 4646 if(phy_tmp & CR_1000T_MS_ENABLE) { 4647 phy_tmp &= ~CR_1000T_MS_ENABLE; 4648 e1000_write_phy_reg(&sc->hw, 4649 PHY_1000T_CTRL, phy_tmp); 4650 sc->smartspeed++; 4651 if(sc->hw.mac.autoneg && 4652 !e1000_copper_link_autoneg(&sc->hw) && 4653 !e1000_read_phy_reg(&sc->hw, 4654 PHY_CONTROL, &phy_tmp)) { 4655 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4656 MII_CR_RESTART_AUTO_NEG); 4657 e1000_write_phy_reg(&sc->hw, 4658 PHY_CONTROL, phy_tmp); 4659 } 4660 } 4661 } 4662 return; 4663 } else if(sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 4664 /* If still no link, perhaps using 2/3 pair cable */ 4665 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 4666 phy_tmp |= CR_1000T_MS_ENABLE; 4667 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 4668 if(sc->hw.mac.autoneg && 4669 !e1000_copper_link_autoneg(&sc->hw) && 4670 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 4671 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4672 MII_CR_RESTART_AUTO_NEG); 4673 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 4674 } 4675 } 4676 /* Restart process after EM_SMARTSPEED_MAX iterations */ 4677 if(sc->smartspeed++ == EM_SMARTSPEED_MAX) 4678 sc->smartspeed = 0; 4679 } 4680 4681 static void 4682 igb_disable_dmac(struct e1000_hw *hw) 4683 { 4684 u32 reg; 4685 4686 reg = E1000_READ_REG(hw, E1000_DMACR); 4687 reg &= ~E1000_DMACR_DMAC_EN; 4688 /* Retain the documented Lx policy and I210 reserved encoding. */ 4689 reg |= E1000_DMACR_DMAC_LX_MASK; 4690 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4691 } 4692 4693 /********************************************************************* 4694 * 4695 * Initialize the DMA Coalescing feature 4696 * 4697 **********************************************************************/ 4698 static void 4699 igb_init_dmac(struct e1000_softc *sc, u32 pba) 4700 { 4701 device_t dev = sc->dev; 4702 struct e1000_hw *hw = &sc->hw; 4703 u32 dmac, dmacwt, reg, ttlx; 4704 u16 hwm; 4705 u16 max_frame_size; 4706 4707 KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF", 4708 __func__)); 4709 4710 if (hw->mac.type == e1000_i211) 4711 return; 4712 4713 /* 4714 * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve 4715 * the configured value so it can be restored after IOV is disabled. 4716 */ 4717 if (igb_iov_enabled(sc)) { 4718 if (hw->mac.type > e1000_82580) 4719 igb_disable_dmac(hw); 4720 return; 4721 } 4722 4723 max_frame_size = sc->shared->isc_max_frame_size; 4724 if (hw->mac.type > e1000_82580) { 4725 4726 if (sc->dmac == 0) { /* Disabling it */ 4727 igb_disable_dmac(hw); 4728 return; 4729 } else 4730 device_printf(dev, "DMA Coalescing enabled\n"); 4731 4732 /* Set starting threshold */ 4733 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); 4734 4735 hwm = 64 * pba - max_frame_size / 16; 4736 if (hwm < 64 * (pba - 6)) 4737 hwm = 64 * (pba - 6); 4738 reg = E1000_READ_REG(hw, E1000_FCRTC); 4739 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 4740 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 4741 & E1000_FCRTC_RTH_COAL_MASK); 4742 E1000_WRITE_REG(hw, E1000_FCRTC, reg); 4743 4744 4745 dmac = pba - max_frame_size / 512; 4746 if (dmac < pba - 10) 4747 dmac = pba - 10; 4748 reg = E1000_READ_REG(hw, E1000_DMACR); 4749 reg &= ~(E1000_DMACR_DMACWT_MASK | 4750 E1000_DMACR_DMACTHR_MASK | E1000_DMACR_DMAC_LX_MASK | 4751 E1000_DMACR_DMAC_EN | E1000_DMACR_DC_LPBKW_EN | 4752 E1000_DMACR_DC_BMC2OSW_EN); 4753 reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT) 4754 & E1000_DMACR_DMACTHR_MASK); 4755 4756 /* Transition to L0s or L1 if available. */ 4757 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 4758 4759 /* 4760 * The watchdog uses 12.8 usec units on an I354 2.5 Gb/s 4761 * backplane connection and 32 usec units otherwise. 4762 */ 4763 if (hw->mac.type == e1000_i354) { 4764 int status = E1000_READ_REG(hw, E1000_STATUS); 4765 if ((status & E1000_STATUS_2P5_SKU) && 4766 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4767 dmacwt = (sc->dmac * 5) >> 6; 4768 else 4769 dmacwt = sc->dmac >> 5; 4770 } else { 4771 dmacwt = sc->dmac >> 5; 4772 } 4773 reg |= dmacwt & E1000_DMACR_DMACWT_MASK; 4774 if (hw->mac.type == e1000_i350 || 4775 hw->mac.type == e1000_i354) 4776 reg |= E1000_DMACR_DC_LPBKW_EN; 4777 if (hw->mac.type == e1000_i354) 4778 reg |= E1000_DMACR_DC_BMC2OSW_EN; 4779 4780 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4781 4782 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); 4783 4784 /* Set the interval before transition. */ 4785 reg = E1000_READ_REG(hw, E1000_DMCTLX); 4786 reg &= ~E1000_DMCTLX_TTLX_MASK; 4787 if (hw->mac.type == e1000_i350) 4788 reg |= IGB_DMCTLX_DCFLUSH_DIS; 4789 /* 4790 * I210 documents TTLX as reserved with a required value of 0x20. 4791 * At 2.5 Gb/s the I354 unit is 0.4 usec, so ten ticks retain 4792 * the four usec interval used at other speeds. 4793 */ 4794 if (hw->mac.type == e1000_i210) { 4795 ttlx = 0x20; 4796 } else if (hw->mac.type == e1000_i354) { 4797 int status = E1000_READ_REG(hw, E1000_STATUS); 4798 if ((status & E1000_STATUS_2P5_SKU) && 4799 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4800 ttlx = 0xA; 4801 else 4802 ttlx = 0x4; 4803 } else { 4804 ttlx = 0x4; 4805 } 4806 reg |= ttlx & E1000_DMCTLX_TTLX_MASK; 4807 4808 E1000_WRITE_REG(hw, E1000_DMCTLX, reg); 4809 4810 /* free space in tx packet buffer to wake from DMA coal */ 4811 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - 4812 (2 * max_frame_size)) >> 6); 4813 4814 /* make low power state decision controlled by DMA coal */ 4815 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4816 reg |= E1000_PCIEMISC_LX_DECISION; 4817 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); 4818 4819 } else if (hw->mac.type == e1000_82580) { 4820 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4821 E1000_WRITE_REG(hw, E1000_PCIEMISC, 4822 reg & ~E1000_PCIEMISC_LX_DECISION); 4823 E1000_WRITE_REG(hw, E1000_DMACR, 0); 4824 } 4825 } 4826 /********************************************************************* 4827 * The 3 following flush routines are used as a workaround in the 4828 * I219 client parts and only for them. 4829 * 4830 * em_flush_tx_ring - remove all descriptors from the tx_ring 4831 * 4832 * We want to clear all pending descriptors from the TX ring. 4833 * zeroing happens when the HW reads the regs. We assign the ring itself as 4834 * the data of the next descriptor. We don't care about the data we are about 4835 * to reset the HW. 4836 **********************************************************************/ 4837 static void 4838 em_flush_tx_ring(struct e1000_softc *sc) 4839 { 4840 struct e1000_hw *hw = &sc->hw; 4841 struct tx_ring *txr = &sc->tx_queues->txr; 4842 struct e1000_tx_desc *txd; 4843 u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; 4844 u16 size = 512; 4845 4846 tctl = E1000_READ_REG(hw, E1000_TCTL); 4847 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); 4848 4849 txd = &txr->tx_base[txr->tx_cidx_processed]; 4850 4851 /* Just use the ring as a dummy buffer addr */ 4852 txd->buffer_addr = txr->tx_paddr; 4853 txd->lower.data = htole32(txd_lower | size); 4854 txd->upper.data = 0; 4855 4856 /* flush descriptors to memory before notifying the HW */ 4857 wmb(); 4858 4859 E1000_WRITE_REG(hw, E1000_TDT(0), txr->tx_cidx_processed); 4860 mb(); 4861 usec_delay(250); 4862 } 4863 4864 /********************************************************************* 4865 * em_flush_rx_ring - remove all descriptors from the rx_ring 4866 * 4867 * Mark all descriptors in the RX ring as consumed and disable the rx ring 4868 **********************************************************************/ 4869 static void 4870 em_flush_rx_ring(struct e1000_softc *sc) 4871 { 4872 struct e1000_hw *hw = &sc->hw; 4873 u32 rctl, rxdctl; 4874 4875 rctl = E1000_READ_REG(hw, E1000_RCTL); 4876 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4877 E1000_WRITE_FLUSH(hw); 4878 usec_delay(150); 4879 4880 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 4881 /* zero the lower 14 bits (prefetch and host thresholds) */ 4882 rxdctl &= 0xffffc000; 4883 /* 4884 * update thresholds: prefetch threshold to 31, host threshold to 1 4885 * and make sure the granularity is "descriptors" and not 4886 * "cache lines" 4887 */ 4888 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); 4889 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 4890 4891 /* momentarily enable the RX ring for the changes to take effect */ 4892 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); 4893 E1000_WRITE_FLUSH(hw); 4894 usec_delay(150); 4895 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4896 } 4897 4898 /********************************************************************* 4899 * em_flush_desc_rings - remove all descriptors from the descriptor rings 4900 * 4901 * In I219, the descriptor rings must be emptied before resetting the HW 4902 * or before changing the device state to D3 during runtime (runtime PM). 4903 * 4904 * Failure to do this will cause the HW to enter a unit hang state which can 4905 * only be released by PCI reset on the device 4906 * 4907 **********************************************************************/ 4908 static void 4909 em_flush_desc_rings(struct e1000_softc *sc) 4910 { 4911 struct e1000_hw *hw = &sc->hw; 4912 device_t dev = sc->dev; 4913 u16 hang_state; 4914 u32 fext_nvm11, tdlen; 4915 4916 /* First, disable MULR fix in FEXTNVM11 */ 4917 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); 4918 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 4919 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); 4920 4921 /* do nothing if we're not in faulty state, or the queue is empty */ 4922 tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); 4923 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4924 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 4925 return; 4926 em_flush_tx_ring(sc); 4927 4928 /* recheck, maybe the fault is caused by the rx ring */ 4929 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4930 if (hang_state & FLUSH_DESC_REQUIRED) 4931 em_flush_rx_ring(sc); 4932 } 4933 4934 4935 /********************************************************************* 4936 * 4937 * Initialize the hardware to a configuration as specified by the 4938 * sc structure. 4939 * 4940 **********************************************************************/ 4941 static int 4942 em_reset(if_ctx_t ctx) 4943 { 4944 device_t dev = iflib_get_dev(ctx); 4945 struct e1000_softc *sc = iflib_get_softc(ctx); 4946 if_t ifp = iflib_get_ifp(ctx); 4947 struct e1000_hw *hw = &sc->hw; 4948 u32 rx_buffer_size; 4949 u32 pba; 4950 s32 error; 4951 4952 INIT_DEBUGOUT("em_reset: begin"); 4953 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 4954 4955 /* Let the firmware know the OS is in control */ 4956 em_get_hw_control(sc); 4957 4958 /* Set up smart power down as default off on newer adapters. */ 4959 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 4960 hw->mac.type == e1000_82572)) { 4961 u16 phy_tmp = 0; 4962 4963 /* Speed up time to link by disabling smart power down. */ 4964 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 4965 phy_tmp &= ~IGP02E1000_PM_SPD; 4966 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 4967 } 4968 4969 /* 4970 * Packet Buffer Allocation (PBA) 4971 * Writing PBA sets the receive portion of the buffer 4972 * the remainder is used for the transmit buffer. 4973 */ 4974 switch (hw->mac.type) { 4975 /* 82547: Total Packet Buffer is 40K */ 4976 case e1000_82547: 4977 case e1000_82547_rev_2: 4978 if (hw->mac.max_frame_size > 8192) 4979 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4980 else 4981 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 4982 break; 4983 /* 82571/82572/80003es2lan: Total Packet Buffer is 48K */ 4984 case e1000_82571: 4985 case e1000_82572: 4986 case e1000_80003es2lan: 4987 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 4988 break; 4989 /* 82573: Total Packet Buffer is 32K */ 4990 case e1000_82573: 4991 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 4992 break; 4993 /* 82574/82583: Total Packet Buffer is 40K */ 4994 case e1000_82574: 4995 case e1000_82583: 4996 if (hw->mac.max_frame_size > 8192) 4997 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4998 else 4999 pba = E1000_PBA_32K; /* 32K for RX, 8K for Tx */ 5000 break; 5001 case e1000_ich8lan: 5002 pba = E1000_PBA_8K; 5003 break; 5004 case e1000_ich9lan: 5005 case e1000_ich10lan: 5006 /* Boost Receive side for jumbo frames */ 5007 if (hw->mac.max_frame_size > 4096) 5008 pba = E1000_PBA_14K; 5009 else 5010 pba = E1000_PBA_10K; 5011 break; 5012 case e1000_pchlan: 5013 case e1000_pch2lan: 5014 case e1000_pch_lpt: 5015 case e1000_pch_spt: 5016 case e1000_pch_cnp: 5017 case e1000_pch_tgp: 5018 case e1000_pch_adp: 5019 case e1000_pch_mtp: 5020 case e1000_pch_ptp: 5021 case e1000_pch_nvp: 5022 pba = E1000_PBA_26K; 5023 break; 5024 case e1000_82575: 5025 pba = E1000_PBA_32K; 5026 break; 5027 case e1000_82576: 5028 pba = E1000_READ_REG(hw, E1000_RXPBS); 5029 pba &= E1000_RXPBS_SIZE_MASK_82576; 5030 break; 5031 case e1000_82580: 5032 case e1000_i350: 5033 case e1000_i354: 5034 pba = E1000_READ_REG(hw, E1000_RXPBS); 5035 pba = e1000_rxpbs_adjust_82580(pba); 5036 break; 5037 case e1000_i210: 5038 case e1000_i211: 5039 pba = E1000_PBA_34K; 5040 break; 5041 default: 5042 /* Remaining devices assumed to have Packet Buffer of 64K. */ 5043 if (hw->mac.max_frame_size > 8192) 5044 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 5045 else 5046 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 5047 } 5048 5049 /* Special needs in case of Jumbo frames */ 5050 if ((hw->mac.type == e1000_82575) && (if_getmtu(ifp) > ETHERMTU)) { 5051 u32 tx_space, min_tx, min_rx; 5052 pba = E1000_READ_REG(hw, E1000_PBA); 5053 tx_space = pba >> 16; 5054 pba &= 0xffff; 5055 min_tx = (hw->mac.max_frame_size + 5056 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 5057 min_tx = roundup2(min_tx, 1024); 5058 min_tx >>= 10; 5059 min_rx = hw->mac.max_frame_size; 5060 min_rx = roundup2(min_rx, 1024); 5061 min_rx >>= 10; 5062 if (tx_space < min_tx && 5063 ((min_tx - tx_space) < pba)) { 5064 pba = pba - (min_tx - tx_space); 5065 /* 5066 * if short on rx space, rx wins 5067 * and must trump tx adjustment 5068 */ 5069 if (pba < min_rx) 5070 pba = min_rx; 5071 } 5072 E1000_WRITE_REG(hw, E1000_PBA, pba); 5073 } 5074 5075 if (hw->mac.type < igb_mac_min) 5076 E1000_WRITE_REG(hw, E1000_PBA, pba); 5077 5078 INIT_DEBUGOUT1("em_reset: pba=%dK", pba); 5079 5080 /* 5081 * These parameters control the automatic generation (Tx) and 5082 * response (Rx) to Ethernet PAUSE frames. 5083 * - High water mark should allow for at least two frames to be 5084 * received after sending an XOFF. 5085 * - Low water mark works best when it is very near the high water 5086 mark. 5087 * This allows the receiver to restart by sending XON when it has 5088 * drained a bit. Here we use an arbitrary value of 1500 which will 5089 * restart after one full frame is pulled from the buffer. There 5090 * could be several smaller frames in the buffer and if so they will 5091 * not trigger the XON until their total number reduces the buffer 5092 * by 1500. 5093 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 5094 */ 5095 rx_buffer_size = (pba & 0xffff) << 10; 5096 hw->fc.high_water = rx_buffer_size - 5097 roundup2(hw->mac.max_frame_size, 1024); 5098 hw->fc.low_water = hw->fc.high_water - 1500; 5099 5100 hw->fc.requested_mode = sc->fc; 5101 5102 if (hw->mac.type == e1000_80003es2lan) 5103 hw->fc.pause_time = 0xFFFF; 5104 else 5105 hw->fc.pause_time = EM_FC_PAUSE_TIME; 5106 5107 hw->fc.send_xon = true; 5108 5109 /* Device specific overrides/settings */ 5110 switch (hw->mac.type) { 5111 case e1000_pchlan: 5112 /* Workaround: no TX flow ctrl for PCH */ 5113 if (hw->fc.requested_mode == e1000_fc_full) 5114 hw->fc.requested_mode = e1000_fc_rx_pause; 5115 else if (hw->fc.requested_mode == e1000_fc_tx_pause) 5116 hw->fc.requested_mode = e1000_fc_none; 5117 hw->fc.pause_time = 0xFFFF; /* override */ 5118 if (if_getmtu(ifp) > ETHERMTU) { 5119 hw->fc.high_water = 0x3500; 5120 hw->fc.low_water = 0x1500; 5121 } else { 5122 hw->fc.high_water = 0x5000; 5123 hw->fc.low_water = 0x3000; 5124 } 5125 hw->fc.refresh_time = 0x1000; 5126 break; 5127 case e1000_pch2lan: 5128 case e1000_pch_lpt: 5129 case e1000_pch_spt: 5130 case e1000_pch_cnp: 5131 case e1000_pch_tgp: 5132 case e1000_pch_adp: 5133 case e1000_pch_mtp: 5134 case e1000_pch_ptp: 5135 case e1000_pch_nvp: 5136 hw->fc.high_water = 0x5C20; 5137 hw->fc.low_water = 0x5048; 5138 hw->fc.pause_time = 0xFFFF; 5139 hw->fc.refresh_time = 0xFFFF; 5140 /* Jumbos need adjusted PBA */ 5141 if (if_getmtu(ifp) > ETHERMTU) 5142 pba = E1000_PBA_12K; 5143 else 5144 pba = E1000_PBA_26K; 5145 E1000_WRITE_REG(hw, E1000_PBA, pba); 5146 break; 5147 case e1000_82575: 5148 case e1000_82576: 5149 /* 8-byte granularity */ 5150 hw->fc.low_water = hw->fc.high_water - 8; 5151 break; 5152 case e1000_82580: 5153 case e1000_i350: 5154 case e1000_i354: 5155 case e1000_i210: 5156 case e1000_i211: 5157 /* 16-byte granularity */ 5158 hw->fc.low_water = hw->fc.high_water - 16; 5159 break; 5160 case e1000_ich9lan: 5161 case e1000_ich10lan: 5162 if (if_getmtu(ifp) > ETHERMTU) { 5163 hw->fc.high_water = 0x2800; 5164 hw->fc.low_water = hw->fc.high_water - 8; 5165 break; 5166 } 5167 /* FALLTHROUGH */ 5168 default: 5169 if (hw->mac.type == e1000_80003es2lan) 5170 hw->fc.pause_time = 0xFFFF; 5171 break; 5172 } 5173 5174 /* I219 needs some special flushing to avoid hangs */ 5175 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 5176 em_flush_desc_rings(sc); 5177 5178 /* Issue a global reset */ 5179 em_prepare_fatal_error_reset(sc); 5180 error = e1000_reset_hw(hw); 5181 if (error != E1000_SUCCESS) { 5182 device_printf(dev, "Hardware reset failed: %d\n", error); 5183 return (error); 5184 } 5185 if (hw->mac.type >= igb_mac_min) { 5186 E1000_WRITE_REG(hw, E1000_WUC, 0); 5187 } else { 5188 E1000_WRITE_REG(hw, E1000_WUFC, 0); 5189 em_disable_aspm(sc); 5190 } 5191 if (sc->flags & IGB_MEDIA_RESET) { 5192 e1000_setup_init_funcs(hw, true); 5193 e1000_get_bus_info(hw); 5194 sc->flags &= ~IGB_MEDIA_RESET; 5195 } 5196 /* and a re-init */ 5197 error = e1000_init_hw(hw); 5198 if (error != E1000_SUCCESS) { 5199 device_printf(dev, "Hardware initialization failed: %d\n", 5200 error); 5201 return (error); 5202 } 5203 em_configure_82576_memory_errors(sc); 5204 em_finish_fatal_error_reset(sc); 5205 if (hw->mac.type >= igb_mac_min) 5206 igb_init_dmac(sc, pba); 5207 5208 /* Save the receive packet-buffer allocation for AIM. */ 5209 sc->pba = pba; 5210 5211 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 5212 e1000_get_phy_info(hw); 5213 e1000_check_for_link(hw); 5214 sc->phy_hang_count = 0; 5215 5216 return (E1000_SUCCESS); 5217 } 5218 5219 /* 5220 * Initialise the RSS mapping for NICs that support multiple transmit/ 5221 * receive rings. 5222 */ 5223 5224 #define RSSKEYLEN 10 5225 static void 5226 em_initialize_rss_mapping(struct e1000_softc *sc) 5227 { 5228 uint8_t rss_key[4 * RSSKEYLEN]; 5229 uint32_t reta = 0; 5230 struct e1000_hw *hw = &sc->hw; 5231 int i; 5232 5233 /* 5234 * Configure RSS key 5235 */ 5236 _Static_assert(sizeof(rss_key) == RSS_KEYSIZE, 5237 "RSS key size mismatch"); 5238 rss_getkey(rss_key); 5239 for (i = 0; i < RSSKEYLEN; ++i) { 5240 uint32_t rssrk = 0; 5241 5242 rssrk = le32dec(rss_key + i * sizeof(rssrk)); 5243 E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); 5244 } 5245 5246 /* 5247 * Configure RSS redirect table in following fashion: 5248 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 5249 */ 5250 for (i = 0; i < sizeof(reta); ++i) { 5251 uint32_t q; 5252 5253 q = (i % sc->rx_num_queues) << 7; 5254 reta |= q << (8 * i); 5255 } 5256 5257 for (i = 0; i < 32; ++i) 5258 E1000_WRITE_REG(hw, E1000_RETA(i), reta); 5259 5260 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | 5261 E1000_MRQC_RSS_FIELD_IPV4_TCP | 5262 E1000_MRQC_RSS_FIELD_IPV4 | 5263 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | 5264 E1000_MRQC_RSS_FIELD_IPV6_EX | 5265 E1000_MRQC_RSS_FIELD_IPV6); 5266 } 5267 5268 static void 5269 igb_initialize_rss_mapping(struct e1000_softc *sc) 5270 { 5271 struct e1000_hw *hw = &sc->hw; 5272 int i; 5273 int queue_id; 5274 u32 reta; 5275 u32 rss_key[10], mrqc, shift = 0; 5276 5277 /* XXX? */ 5278 if (hw->mac.type == e1000_82575) 5279 shift = 6; 5280 5281 /* 5282 * The redirection table controls which destination 5283 * queue each bucket redirects traffic to. 5284 * Each DWORD represents four queues, with the LSB 5285 * being the first queue in the DWORD. 5286 * 5287 * This just allocates buckets to queues using round-robin 5288 * allocation. 5289 * 5290 * NOTE: It Just Happens to line up with the default 5291 * RSS allocation method. 5292 */ 5293 5294 /* Warning FM follows */ 5295 reta = 0; 5296 for (i = 0; i < 128; i++) { 5297 #ifdef RSS 5298 queue_id = rss_get_indirection_to_bucket(i); 5299 /* 5300 * If we have more queues than buckets, we'll 5301 * end up mapping buckets to a subset of the 5302 * queues. 5303 * 5304 * If we have more buckets than queues, we'll 5305 * end up instead assigning multiple buckets 5306 * to queues. 5307 * 5308 * Both are suboptimal, but we need to handle 5309 * the case so we don't go out of bounds 5310 * indexing arrays and such. 5311 */ 5312 queue_id = queue_id % sc->rx_num_queues; 5313 #else 5314 queue_id = (i % sc->rx_num_queues); 5315 #endif 5316 /* Adjust if required */ 5317 queue_id = queue_id << shift; 5318 5319 /* 5320 * The low 8 bits are for hash value (n+0); 5321 * The next 8 bits are for hash value (n+1), etc. 5322 */ 5323 reta = reta >> 8; 5324 reta = reta | ( ((uint32_t) queue_id) << 24); 5325 if ((i & 3) == 3) { 5326 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 5327 reta = 0; 5328 } 5329 } 5330 5331 /* Now fill in hash table */ 5332 5333 /* 5334 * MRQC: Multiple Receive Queues Command 5335 * Set queuing to RSS control, number depends on the device. 5336 */ 5337 mrqc = E1000_MRQC_ENABLE_RSS_MQ; 5338 5339 /* XXX ew typecasting */ 5340 rss_getkey((uint8_t *) &rss_key); 5341 for (i = 0; i < 10; i++) 5342 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]); 5343 5344 /* 5345 * Configure the RSS fields to hash upon. 5346 */ 5347 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 5348 E1000_MRQC_RSS_FIELD_IPV4_TCP); 5349 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 5350 E1000_MRQC_RSS_FIELD_IPV6_TCP); 5351 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 5352 E1000_MRQC_RSS_FIELD_IPV6_UDP); 5353 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 5354 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 5355 5356 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 5357 } 5358 5359 /********************************************************************* 5360 * 5361 * Setup networking device structure and register interface media. 5362 * 5363 **********************************************************************/ 5364 static int 5365 em_setup_interface(if_ctx_t ctx) 5366 { 5367 if_t ifp = iflib_get_ifp(ctx); 5368 struct e1000_softc *sc = iflib_get_softc(ctx); 5369 if_softc_ctx_t scctx = sc->shared; 5370 5371 INIT_DEBUGOUT("em_setup_interface: begin"); 5372 5373 /* Single Queue */ 5374 if (sc->tx_num_queues == 1) { 5375 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 5376 if_setsendqready(ifp); 5377 } 5378 5379 /* 5380 * Specify the media types supported by this adapter and register 5381 * callbacks to update media and link information 5382 */ 5383 if (sc->vf_ifp) { 5384 ifmedia_add(sc->media, 5385 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5386 ifmedia_set(sc->media, 5387 IFM_ETHER | IFM_1000_T | IFM_FDX); 5388 return (0); 5389 } 5390 5391 if (sc->hw.phy.media_type == e1000_media_type_fiber || 5392 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 5393 u_char fiber_type = IFM_1000_SX; /* default type */ 5394 5395 if (sc->hw.mac.type == e1000_82545) 5396 fiber_type = IFM_1000_LX; 5397 ifmedia_add(sc->media, 5398 IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); 5399 ifmedia_add(sc->media, IFM_ETHER | fiber_type, 0, NULL); 5400 } else { 5401 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 5402 ifmedia_add(sc->media, 5403 IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 5404 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 5405 ifmedia_add(sc->media, 5406 IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 5407 if (sc->hw.phy.type != e1000_phy_ife) { 5408 ifmedia_add(sc->media, 5409 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5410 ifmedia_add(sc->media, 5411 IFM_ETHER | IFM_1000_T, 0, NULL); 5412 } 5413 } 5414 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 5415 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 5416 return (0); 5417 } 5418 5419 static int 5420 em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5421 int ntxqs, int ntxqsets) 5422 { 5423 struct e1000_softc *sc = iflib_get_softc(ctx); 5424 if_softc_ctx_t scctx = sc->shared; 5425 int error = E1000_SUCCESS; 5426 struct em_tx_queue *que; 5427 int i, j; 5428 5429 MPASS(sc->tx_num_queues > 0); 5430 MPASS(sc->tx_num_queues == ntxqsets); 5431 5432 /* First allocate the top level queue structs */ 5433 if (!(sc->tx_queues = 5434 (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * 5435 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5436 device_printf(iflib_get_dev(ctx), 5437 "Unable to allocate queue memory\n"); 5438 return(ENOMEM); 5439 } 5440 5441 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 5442 /* Set up some basics */ 5443 5444 struct tx_ring *txr = &que->txr; 5445 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 5446 ("%s: misaligned TX AIM snapshot %p", __func__, 5447 &txr->tx_aim_snapshot)); 5448 txr->sc = que->sc = sc; 5449 que->me = txr->me = i; 5450 5451 /* Allocate report status array */ 5452 if (!(txr->tx_rsq = 5453 (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], 5454 M_DEVBUF, M_NOWAIT | M_ZERO))) { 5455 device_printf(iflib_get_dev(ctx), 5456 "failed to allocate rs_idxs memory\n"); 5457 error = ENOMEM; 5458 goto fail; 5459 } 5460 for (j = 0; j < scctx->isc_ntxd[0]; j++) 5461 txr->tx_rsq[j] = QIDX_INVALID; 5462 /* get the virtual and physical address of hardware queues */ 5463 txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; 5464 txr->tx_paddr = paddrs[i*ntxqs]; 5465 } 5466 5467 if (bootverbose) 5468 device_printf(iflib_get_dev(ctx), 5469 "allocated for %d tx_queues\n", sc->tx_num_queues); 5470 return (0); 5471 fail: 5472 em_if_queues_free(ctx); 5473 return (error); 5474 } 5475 5476 static int 5477 em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5478 int nrxqs, int nrxqsets) 5479 { 5480 struct e1000_softc *sc = iflib_get_softc(ctx); 5481 int error = E1000_SUCCESS; 5482 struct em_rx_queue *que; 5483 int i; 5484 5485 MPASS(sc->rx_num_queues > 0); 5486 MPASS(sc->rx_num_queues == nrxqsets); 5487 5488 /* First allocate the top level queue structs */ 5489 if (!(sc->rx_queues = 5490 (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * 5491 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5492 device_printf(iflib_get_dev(ctx), 5493 "Unable to allocate queue memory\n"); 5494 error = ENOMEM; 5495 goto fail; 5496 } 5497 5498 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 5499 /* Set up some basics */ 5500 struct rx_ring *rxr = &que->rxr; 5501 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 5502 ("%s: misaligned RX AIM snapshot %p", __func__, 5503 &rxr->rx_aim_snapshot)); 5504 rxr->sc = que->sc = sc; 5505 rxr->que = que; 5506 que->me = rxr->me = i; 5507 5508 /* get the virtual and physical address of hardware queues */ 5509 rxr->rx_base = 5510 (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; 5511 rxr->rx_paddr = paddrs[i*nrxqs]; 5512 } 5513 5514 if (bootverbose) 5515 device_printf(iflib_get_dev(ctx), 5516 "allocated for %d rx_queues\n", sc->rx_num_queues); 5517 5518 return (0); 5519 fail: 5520 em_if_queues_free(ctx); 5521 return (error); 5522 } 5523 5524 static void 5525 em_if_queues_free(if_ctx_t ctx) 5526 { 5527 struct e1000_softc *sc = iflib_get_softc(ctx); 5528 struct em_tx_queue *tx_que = sc->tx_queues; 5529 struct em_rx_queue *rx_que = sc->rx_queues; 5530 5531 if (tx_que != NULL) { 5532 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 5533 struct tx_ring *txr = &tx_que->txr; 5534 if (txr->tx_rsq == NULL) 5535 break; 5536 5537 free(txr->tx_rsq, M_DEVBUF); 5538 txr->tx_rsq = NULL; 5539 } 5540 free(sc->tx_queues, M_DEVBUF); 5541 sc->tx_queues = NULL; 5542 } 5543 5544 if (rx_que != NULL) { 5545 free(sc->rx_queues, M_DEVBUF); 5546 sc->rx_queues = NULL; 5547 } 5548 } 5549 5550 static u32 5551 em_legacy_txdctl(struct e1000_hw *hw) 5552 { 5553 u32 txdctl; 5554 5555 /* 5556 * Start with the established full-descriptor writeback policy. 5557 * Several generations have descriptor-queue errata for which it is 5558 * a documented workaround. The unsafe early controllers are 5559 * overridden below. 5560 */ 5561 txdctl = EM_TX_PTHRESH | (EM_TX_HTHRESH << 8) | 5562 (EM_TX_WTHRESH << 16) | E1000_TXDCTL_GRAN; 5563 5564 switch (hw->mac.type) { 5565 case e1000_82571: 5566 case e1000_82572: 5567 case e1000_82573: 5568 case e1000_82574: 5569 case e1000_82583: 5570 case e1000_80003es2lan: 5571 /* Match the Intel shared-code policy for these families. */ 5572 txdctl |= E1000_TXDCTL_COUNT_DESC; 5573 break; 5574 case e1000_ich8lan: 5575 case e1000_ich9lan: 5576 case e1000_ich10lan: 5577 case e1000_pchlan: 5578 case e1000_pch2lan: 5579 case e1000_pch_lpt: 5580 case e1000_pch_spt: 5581 case e1000_pch_cnp: 5582 case e1000_pch_tgp: 5583 case e1000_pch_adp: 5584 case e1000_pch_mtp: 5585 case e1000_pch_ptp: 5586 case e1000_pch_nvp: 5587 /* Preserve the required bit set by the integrated shared code. */ 5588 txdctl |= (1U << 22); 5589 break; 5590 case e1000_82542: 5591 case e1000_82543: 5592 case e1000_82544: 5593 /* 5594 * 82543 erratum 35 and 82544 erratum 20 require 5595 * WTHRESH=0. Leave all descriptor-control thresholds at 5596 * their reset values on these early controllers. 5597 */ 5598 txdctl = 0; 5599 break; 5600 case e1000_82540: 5601 case e1000_82545: 5602 case e1000_82545_rev_3: 5603 case e1000_82546: 5604 case e1000_82546_rev_3: 5605 case e1000_82541: 5606 case e1000_82541_rev_2: 5607 case e1000_82547: 5608 case e1000_82547_rev_2: 5609 break; 5610 default: 5611 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5612 hw->mac.type)); 5613 break; 5614 } 5615 5616 return (txdctl); 5617 } 5618 5619 static u32 5620 igb_txdctl(struct e1000_hw *hw) 5621 { 5622 u32 pthresh; 5623 5624 switch (hw->mac.type) { 5625 case e1000_i354: 5626 pthresh = I354_TX_PTHRESH; 5627 break; 5628 case e1000_82575: 5629 case e1000_82576: 5630 case e1000_82580: 5631 case e1000_i350: 5632 case e1000_i210: 5633 case e1000_i211: 5634 case e1000_vfadapt: 5635 case e1000_vfadapt_i350: 5636 pthresh = IGB_TX_PTHRESH; 5637 break; 5638 default: 5639 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5640 hw->mac.type)); 5641 pthresh = IGB_TX_PTHRESH; 5642 break; 5643 } 5644 5645 return (pthresh | (IGB_TX_HTHRESH << 8) | 5646 E1000_TXDCTL_QUEUE_ENABLE); 5647 } 5648 5649 /********************************************************************* 5650 * 5651 * Enable transmit unit. 5652 * 5653 **********************************************************************/ 5654 void 5655 em_initialize_transmit_rings(if_ctx_t ctx) 5656 { 5657 struct e1000_softc *sc = iflib_get_softc(ctx); 5658 if_softc_ctx_t scctx = sc->shared; 5659 struct em_tx_queue *que; 5660 struct tx_ring *txr; 5661 struct e1000_hw *hw = &sc->hw; 5662 u32 txdctl; 5663 5664 for (int i = 0; i < sc->tx_num_queues; i++) { 5665 u64 bus_addr; 5666 caddr_t offp, endp; 5667 uint32_t qid; 5668 5669 que = &sc->tx_queues[i]; 5670 txr = &que->txr; 5671 qid = txr->me; 5672 bus_addr = txr->tx_paddr; 5673 5674 /* Clear checksum offload context. */ 5675 offp = (caddr_t)txr + offsetof(struct tx_ring, csum_flags); 5676 endp = (caddr_t)(txr + 1); 5677 memset(offp, 0, endp - offp); 5678 5679 if (hw->mac.type >= igb_mac_min) { 5680 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid)); 5681 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), 5682 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE); 5683 E1000_WRITE_FLUSH(hw); 5684 } 5685 5686 /* Base and Len of TX Ring */ 5687 E1000_WRITE_REG(hw, E1000_TDLEN(qid), 5688 scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); 5689 E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32)); 5690 E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr); 5691 /* Init the HEAD/TAIL indices */ 5692 E1000_WRITE_REG(hw, E1000_TDT(qid), 0); 5693 E1000_WRITE_REG(hw, E1000_TDH(qid), 0); 5694 5695 HW_DEBUGOUT2("Base = %x, Length = %x\n", 5696 E1000_READ_REG(hw, E1000_TDBAL(qid)), 5697 E1000_READ_REG(hw, E1000_TDLEN(qid))); 5698 5699 if (hw->mac.type < igb_mac_min) 5700 txdctl = em_legacy_txdctl(hw); 5701 else 5702 txdctl = igb_txdctl(hw); 5703 5704 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl); 5705 } 5706 } 5707 5708 static void 5709 em_initialize_transmit_unit(if_ctx_t ctx) 5710 { 5711 struct e1000_softc *sc = iflib_get_softc(ctx); 5712 struct e1000_hw *hw = &sc->hw; 5713 u32 tctl, tarc, tipg = 0; 5714 5715 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 5716 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5717 5718 em_initialize_transmit_rings(ctx); 5719 5720 /* Set the default values for the Tx Inter Packet Gap timer */ 5721 switch (hw->mac.type) { 5722 case e1000_80003es2lan: 5723 tipg = DEFAULT_82543_TIPG_IPGR1; 5724 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 5725 E1000_TIPG_IPGR2_SHIFT; 5726 break; 5727 case e1000_82542: 5728 tipg = DEFAULT_82542_TIPG_IPGT; 5729 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5730 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5731 break; 5732 default: 5733 if (hw->phy.media_type == e1000_media_type_fiber || 5734 hw->phy.media_type == e1000_media_type_internal_serdes) 5735 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 5736 else 5737 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 5738 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5739 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5740 } 5741 5742 if (hw->mac.type < igb_mac_min) { 5743 E1000_WRITE_REG(hw, E1000_TIPG, tipg); 5744 E1000_WRITE_REG(hw, E1000_TIDV, sc->tx_int_delay.value); 5745 5746 if (sc->tx_int_delay.value > 0) 5747 sc->txd_cmd |= E1000_TXD_CMD_IDE; 5748 } 5749 5750 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) 5751 E1000_WRITE_REG(hw, E1000_TADV, sc->tx_abs_int_delay.value); 5752 5753 if (hw->mac.type == e1000_82571 || hw->mac.type == e1000_82572) { 5754 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5755 tarc |= TARC_SPEED_MODE_BIT; 5756 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5757 } else if (hw->mac.type == e1000_80003es2lan) { 5758 /* errata: program both queues to unweighted RR */ 5759 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5760 tarc |= 1; 5761 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5762 tarc = E1000_READ_REG(hw, E1000_TARC(1)); 5763 tarc |= 1; 5764 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5765 } else if (hw->mac.type == e1000_82574) { 5766 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5767 tarc |= TARC_ERRATA_BIT; 5768 if ( sc->tx_num_queues > 1) { 5769 tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); 5770 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5771 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5772 } else 5773 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5774 } 5775 5776 /* Program the Transmit Control Register */ 5777 tctl = E1000_READ_REG(hw, E1000_TCTL); 5778 tctl &= ~E1000_TCTL_CT; 5779 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 5780 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 5781 5782 if (hw->mac.type >= e1000_82571 && hw->mac.type < igb_mac_min) 5783 tctl |= E1000_TCTL_MULR; 5784 5785 /* This write will effectively turn on the transmit unit. */ 5786 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 5787 5788 /* SPT and KBL errata workarounds */ 5789 if (hw->mac.type == e1000_pch_spt) { 5790 u32 reg; 5791 reg = E1000_READ_REG(hw, E1000_IOSFPC); 5792 reg |= E1000_RCTL_RDMTS_HEX; 5793 E1000_WRITE_REG(hw, E1000_IOSFPC, reg); 5794 /* i218-i219 Specification Update 1.5.4.5 */ 5795 reg = E1000_READ_REG(hw, E1000_TARC(0)); 5796 reg &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 5797 reg |= E1000_TARC0_CB_MULTIQ_2_REQ; 5798 E1000_WRITE_REG(hw, E1000_TARC(0), reg); 5799 } 5800 } 5801 5802 /********************************************************************* 5803 * 5804 * Enable receive unit. 5805 * 5806 **********************************************************************/ 5807 #define BSIZEPKT_ROUNDUP ((1<<E1000_SRRCTL_BSIZEPKT_SHIFT)-1) 5808 5809 static u32 5810 igb_rxdctl(struct e1000_softc *sc, u32 rxdctl) 5811 { 5812 struct e1000_hw *hw; 5813 u32 mask, pthresh, wthresh; 5814 5815 hw = &sc->hw; 5816 mask = IGB_RXDCTL_THRESH_MASK; 5817 switch (hw->mac.type) { 5818 case e1000_82575: 5819 mask = IGB_82575_RXDCTL_THRESH_MASK; 5820 pthresh = IGB_RX_PTHRESH; 5821 wthresh = IGB_RX_WTHRESH; 5822 break; 5823 case e1000_82576: 5824 pthresh = IGB_RX_PTHRESH; 5825 wthresh = sc->intr_type == IFLIB_INTR_MSIX ? 5826 IGB_82576_RX_WTHRESH : IGB_RX_WTHRESH; 5827 break; 5828 case e1000_vfadapt: 5829 /* 82576 VFs always need the MSI-X writeback workaround. */ 5830 pthresh = IGB_RX_PTHRESH; 5831 wthresh = IGB_82576_RX_WTHRESH; 5832 break; 5833 case e1000_i354: 5834 pthresh = I354_RX_PTHRESH; 5835 wthresh = IGB_RX_WTHRESH; 5836 break; 5837 case e1000_82580: 5838 case e1000_i350: 5839 case e1000_i210: 5840 case e1000_i211: 5841 case e1000_vfadapt_i350: 5842 pthresh = IGB_RX_PTHRESH; 5843 wthresh = IGB_RX_WTHRESH; 5844 break; 5845 default: 5846 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5847 hw->mac.type)); 5848 pthresh = IGB_RX_PTHRESH; 5849 wthresh = IGB_RX_WTHRESH; 5850 break; 5851 } 5852 5853 rxdctl &= ~mask; 5854 rxdctl |= pthresh | (IGB_RX_HTHRESH << 8) | 5855 (wthresh << 16) | E1000_RXDCTL_QUEUE_ENABLE; 5856 return (rxdctl); 5857 } 5858 5859 void 5860 igb_initialize_receive_rings(if_ctx_t ctx, bool drop) 5861 { 5862 struct e1000_softc *sc = iflib_get_softc(ctx); 5863 if_softc_ctx_t scctx = sc->shared; 5864 struct e1000_hw *hw = &sc->hw; 5865 struct em_rx_queue *que; 5866 u32 srrctl; 5867 5868 srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 5869 E1000_SRRCTL_BSIZEPKT_SHIFT; 5870 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 5871 if (drop) 5872 srrctl |= E1000_SRRCTL_DROP_EN; 5873 5874 for (int i = 0; i < sc->rx_num_queues; i++) { 5875 struct rx_ring *rxr; 5876 u64 bus_addr; 5877 u32 rxdctl; 5878 uint32_t qid; 5879 5880 que = &sc->rx_queues[i]; 5881 rxr = &que->rxr; 5882 bus_addr = rxr->rx_paddr; 5883 qid = rxr->me; 5884 5885 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid)); 5886 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), 5887 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE); 5888 E1000_WRITE_FLUSH(hw); 5889 5890 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 5891 scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); 5892 E1000_WRITE_REG(hw, E1000_RDBAH(qid), 5893 (uint32_t)(bus_addr >> 32)); 5894 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr); 5895 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 5896 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 5897 E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl); 5898 5899 rxdctl = igb_rxdctl(sc, rxdctl); 5900 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl); 5901 } 5902 } 5903 5904 static bool 5905 em_integrated_jumbo_rx(struct e1000_hw *hw) 5906 { 5907 switch (hw->mac.type) { 5908 case e1000_ich9lan: 5909 case e1000_ich10lan: 5910 case e1000_pchlan: 5911 case e1000_pch2lan: 5912 case e1000_pch_lpt: 5913 case e1000_pch_spt: 5914 case e1000_pch_cnp: 5915 case e1000_pch_tgp: 5916 case e1000_pch_adp: 5917 case e1000_pch_mtp: 5918 case e1000_pch_ptp: 5919 case e1000_pch_nvp: 5920 return (true); 5921 default: 5922 return (false); 5923 } 5924 } 5925 5926 static void 5927 em_initialize_receive_unit(if_ctx_t ctx) 5928 { 5929 struct e1000_softc *sc = iflib_get_softc(ctx); 5930 if_softc_ctx_t scctx = sc->shared; 5931 if_t ifp = iflib_get_ifp(ctx); 5932 struct e1000_hw *hw = &sc->hw; 5933 struct em_rx_queue *que; 5934 int i; 5935 uint32_t rctl, rxcsum; 5936 5937 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 5938 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5939 5940 /* 5941 * Make sure receives are disabled while setting up the descriptor 5942 * ring. 5943 */ 5944 rctl = E1000_READ_REG(hw, E1000_RCTL); 5945 /* Do not disable if ever enabled on this hardware. */ 5946 if (hw->mac.type != e1000_82574 && 5947 hw->mac.type != e1000_82583) 5948 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 5949 5950 /* Setup the Receive Control Register. */ 5951 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 5952 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 5953 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 5954 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 5955 rctl &= ~E1000_RCTL_SBP; 5956 5957 if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU) 5958 rctl |= E1000_RCTL_LPE; 5959 else 5960 rctl &= ~E1000_RCTL_LPE; 5961 if (!em_disable_crc_stripping) 5962 rctl |= E1000_RCTL_SECRC; 5963 5964 /* lem/em default interrupt moderation */ 5965 if (hw->mac.type < igb_mac_min) { 5966 if (hw->mac.type >= e1000_82540) { 5967 E1000_WRITE_REG(hw, E1000_RADV, 5968 sc->rx_abs_int_delay.value); 5969 5970 /* Set the default interrupt throttling rate */ 5971 E1000_WRITE_REG(hw, E1000_ITR, 5972 EM_INTS_TO_ITR(em_max_interrupt_rate)); 5973 5974 /* 5975 * The 82574 MSI-X EITR registers are programmed 5976 * with the same value further below. Either way 5977 * the hardware now holds the default rate, so seed 5978 * the software copy to match; otherwise a stale 5979 * itr_setting left over from AIM makes em_newitr() 5980 * skip the write that would restore it. 5981 */ 5982 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; 5983 i++, que++) 5984 que->itr_setting = 5985 EM_INTS_TO_ITR(em_max_interrupt_rate); 5986 } 5987 5988 /* XXX TEMPORARY WORKAROUND: on some systems with 82573 5989 * long latencies are observed, like Lenovo X60. This 5990 * change eliminates the problem, but since having positive 5991 * values in RDTR is a known source of problems on other 5992 * platforms another solution is being sought. 5993 */ 5994 if (hw->mac.type == e1000_82573) 5995 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 5996 else 5997 E1000_WRITE_REG(hw, E1000_RDTR, 5998 sc->rx_int_delay.value); 5999 } 6000 6001 if (hw->mac.type >= em_mac_min) { 6002 uint32_t rfctl; 6003 /* Use extended rx descriptor formats */ 6004 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 6005 rfctl |= E1000_RFCTL_EXTEN; 6006 6007 /* 6008 * When using MSI-X interrupts we need to throttle 6009 * using the EITR register (82574 only) 6010 */ 6011 if (hw->mac.type == e1000_82574) { 6012 for (int i = 0; i < 4; i++) 6013 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 6014 EM_INTS_TO_ITR(em_max_interrupt_rate)); 6015 /* Disable accelerated acknowledge */ 6016 rfctl |= E1000_RFCTL_ACK_DIS; 6017 } 6018 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 6019 } 6020 6021 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 6022 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 6023 rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; 6024 if (hw->mac.type > e1000_82575) 6025 rxcsum |= E1000_RXCSUM_CRCOFL; 6026 else if (hw->mac.type < em_mac_min && 6027 if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) 6028 rxcsum |= E1000_RXCSUM_IPV6OFL; 6029 } else { 6030 rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 6031 if (hw->mac.type > e1000_82575) 6032 rxcsum &= ~E1000_RXCSUM_CRCOFL; 6033 else if (hw->mac.type < em_mac_min) 6034 rxcsum &= ~E1000_RXCSUM_IPV6OFL; 6035 } 6036 6037 if (sc->rx_num_queues > 1) { 6038 /* RSS hash needed in the Rx descriptor */ 6039 rxcsum |= E1000_RXCSUM_PCSD; 6040 6041 if (hw->mac.type >= igb_mac_min) 6042 igb_initialize_rss_mapping(sc); 6043 else 6044 em_initialize_rss_mapping(sc); 6045 } 6046 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 6047 6048 for (i = 0, que = sc->rx_queues; 6049 hw->mac.type < igb_mac_min && i < sc->rx_num_queues; 6050 i++, que++) { 6051 struct rx_ring *rxr = &que->rxr; 6052 /* Setup the Base and Length of the Rx Descriptor Ring */ 6053 u64 bus_addr = rxr->rx_paddr; 6054 uint32_t qid = rxr->me; 6055 #if 0 6056 u32 rdt = sc->rx_num_queues -1; /* default */ 6057 #endif 6058 6059 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 6060 scctx->isc_nrxd[0] * 6061 sizeof(union e1000_rx_desc_extended)); 6062 E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32)); 6063 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr); 6064 /* Setup the Head and Tail Descriptor Pointers */ 6065 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 6066 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 6067 } 6068 6069 /* Increase receive-descriptor prefetching for integrated jumbo MACs. */ 6070 if (em_integrated_jumbo_rx(hw) && if_getmtu(ifp) > ETHERMTU) { 6071 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 6072 6073 rxdctl &= ~(EM_RXDCTL_PTHRESH_MASK | 6074 EM_RXDCTL_HTHRESH_MASK); 6075 rxdctl |= EM_JUMBO_RX_PTHRESH | 6076 (EM_JUMBO_RX_HTHRESH << 8); 6077 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 6078 } else if (hw->mac.type == e1000_82574) { 6079 /* RXDCTL(0) writes are mirrored to RXDCTL(1) on 82574. */ 6080 for (int i = 0; i < sc->rx_num_queues; i++) { 6081 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 6082 6083 rxdctl &= ~EM_RXDCTL_THRESH_MASK; 6084 rxdctl |= EM_82574_RX_PTHRESH | 6085 (EM_82574_RX_HTHRESH << 8) | 6086 (EM_82574_RX_WTHRESH << 16) | 6087 E1000_RXDCTL_THRESH_UNIT_DESC; 6088 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 6089 } 6090 } else if (hw->mac.type >= igb_mac_min) { 6091 bool drop; 6092 u32 psize; 6093 6094 if (igb_iov_enabled(sc)) { 6095 E1000_WRITE_REG(hw, E1000_RLPML, 6096 IGB_IOV_MAX_FRAME_SIZE); 6097 } else if (if_getmtu(ifp) > ETHERMTU) { 6098 psize = scctx->isc_max_frame_size; 6099 /* are we on a vlan? */ 6100 if (if_vlantrunkinuse(ifp)) 6101 psize += VLAN_TAG_SIZE; 6102 6103 E1000_WRITE_REG(hw, E1000_RLPML, psize); 6104 } 6105 6106 /* 6107 * If TX flow control is disabled and there's >1 queue 6108 * defined, enable DROP. 6109 * 6110 * This drops frames rather than hanging the RX MAC for all 6111 * queues. 6112 */ 6113 drop = igb_iov_enabled(sc) || 6114 ((sc->rx_num_queues > 1) && 6115 (sc->fc == e1000_fc_none || 6116 sc->fc == e1000_fc_rx_pause)); 6117 igb_initialize_receive_rings(ctx, drop); 6118 } else if (hw->mac.type >= e1000_pch2lan) { 6119 if (if_getmtu(ifp) > ETHERMTU) 6120 e1000_lv_jumbo_workaround_ich8lan(hw, true); 6121 else 6122 e1000_lv_jumbo_workaround_ich8lan(hw, false); 6123 } 6124 6125 /* Make sure VLAN Filters are off */ 6126 rctl &= ~E1000_RCTL_VFE; 6127 6128 /* Set up packet buffer size, overridden by per queue srrctl on igb */ 6129 if (hw->mac.type < igb_mac_min) { 6130 if (sc->rx_mbuf_sz > 2048 && sc->rx_mbuf_sz <= 4096) 6131 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 6132 else if (sc->rx_mbuf_sz > 4096 && sc->rx_mbuf_sz <= 8192) 6133 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 6134 else if (sc->rx_mbuf_sz > 8192) 6135 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX; 6136 else { 6137 rctl |= E1000_RCTL_SZ_2048; 6138 rctl &= ~E1000_RCTL_BSEX; 6139 } 6140 } else 6141 rctl |= E1000_RCTL_SZ_2048; 6142 6143 /* 6144 * rctl bits 11:10 are as follows 6145 * lem: reserved 6146 * em: DTYPE 6147 * igb: reserved 6148 * and should be 00 on all of the above 6149 */ 6150 rctl &= ~0x00000C00; 6151 6152 /* Write out the settings */ 6153 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 6154 6155 return; 6156 } 6157 6158 static void 6159 em_if_vlan_register(if_ctx_t ctx, u16 vtag) 6160 { 6161 struct e1000_softc *sc = iflib_get_softc(ctx); 6162 bool present; 6163 u32 index, mask; 6164 6165 /* Hyper-V supports host-assigned access VLANs, not guest VLANs. */ 6166 if (igbv_is_hyperv(sc)) 6167 return; 6168 6169 index = (vtag >> 5) & 0x7F; 6170 mask = 1U << (vtag & 0x1F); 6171 present = (sc->shadow_vfta[index] & mask) != 0; 6172 /* 6173 * On a VF, record registration intent for replay even if the PF is not 6174 * ready to accept it yet. 6175 */ 6176 sc->shadow_vfta[index] |= mask; 6177 sc->vf_vfta_stale[index] &= ~mask; 6178 if (!present) 6179 ++sc->num_vlans; 6180 if (sc->vf_ifp && 6181 e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { 6182 igbv_vlan_retry_add(sc, vtag); 6183 device_printf(sc->dev, 6184 "VF VLAN %u add request failed\n", vtag); 6185 } else if (sc->vf_ifp) 6186 igbv_vlan_retry_clear(sc, vtag); 6187 if (!sc->vf_ifp) { 6188 if (igb_iov_enabled(sc)) 6189 igb_iov_rebuild_vlan(sc); 6190 else 6191 em_if_vlan_filter_write(sc, index); 6192 } 6193 } 6194 6195 static void 6196 em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 6197 { 6198 struct e1000_softc *sc = iflib_get_softc(ctx); 6199 bool present; 6200 u32 index, mask; 6201 6202 if (igbv_is_hyperv(sc)) 6203 return; 6204 6205 index = (vtag >> 5) & 0x7F; 6206 mask = 1U << (vtag & 0x1F); 6207 present = (sc->shadow_vfta[index] & mask) != 0; 6208 if (sc->vf_ifp) 6209 igbv_vlan_retry_clear(sc, vtag); 6210 if (sc->vf_ifp && 6211 e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { 6212 device_printf(sc->dev, 6213 "VF VLAN %u remove request failed\n", vtag); 6214 /* 6215 * Hardware might still admit this VID. Preserve its receive 6216 * tag until a successful VF reset proves the stale filter gone. 6217 */ 6218 sc->vf_vfta_stale[index] |= mask; 6219 } else { 6220 sc->vf_vfta_stale[index] &= ~mask; 6221 } 6222 sc->shadow_vfta[index] &= ~mask; 6223 if (present) 6224 --sc->num_vlans; 6225 if (!sc->vf_ifp) { 6226 if (igb_iov_enabled(sc)) 6227 igb_iov_rebuild_vlan(sc); 6228 else 6229 em_if_vlan_filter_write(sc, index); 6230 } 6231 } 6232 6233 static bool 6234 em_if_vlan_filter_capable(if_ctx_t ctx) 6235 { 6236 if_t ifp = iflib_get_ifp(ctx); 6237 6238 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) && 6239 !em_disable_crc_stripping) 6240 return (true); 6241 6242 return (false); 6243 } 6244 6245 static bool 6246 em_if_vlan_filter_used(if_ctx_t ctx) 6247 { 6248 struct e1000_softc *sc = iflib_get_softc(ctx); 6249 6250 if (!em_if_vlan_filter_capable(ctx)) 6251 return (false); 6252 6253 for (int i = 0; i < EM_VFTA_SIZE; i++) 6254 if (sc->shadow_vfta[i] != 0) 6255 return (true); 6256 6257 return (false); 6258 } 6259 6260 static void 6261 em_if_vlan_filter_enable(struct e1000_softc *sc) 6262 { 6263 struct e1000_hw *hw = &sc->hw; 6264 u32 reg; 6265 6266 reg = E1000_READ_REG(hw, E1000_RCTL); 6267 reg &= ~E1000_RCTL_CFIEN; 6268 reg |= E1000_RCTL_VFE; 6269 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6270 } 6271 6272 static void 6273 em_if_vlan_filter_disable(struct e1000_softc *sc) 6274 { 6275 struct e1000_hw *hw = &sc->hw; 6276 u32 reg; 6277 6278 reg = E1000_READ_REG(hw, E1000_RCTL); 6279 reg &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 6280 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6281 } 6282 6283 static void 6284 em_if_vlan_filter_write(struct e1000_softc *sc, int changed_index) 6285 { 6286 struct e1000_hw *hw = &sc->hw; 6287 6288 KASSERT(!sc->vf_ifp, ("VLAN filter write on VF\n")); 6289 6290 /* Disable interrupts for lem(4) devices during the filter change */ 6291 if (hw->mac.type < em_mac_min) 6292 em_if_intr_disable(sc->ctx); 6293 6294 /* 6295 * Restore every retained VLAN after reset. Also write the changed 6296 * word when its final VLAN was removed so stale hardware membership 6297 * does not survive a zero shadow value. 6298 */ 6299 for (int i = 0; i < EM_VFTA_SIZE; i++) 6300 if (sc->shadow_vfta[i] != 0 || i == changed_index) 6301 e1000_write_vfta(hw, i, sc->shadow_vfta[i]); 6302 6303 /* Re-enable interrupts for lem-class devices */ 6304 if (hw->mac.type < em_mac_min) 6305 em_if_intr_enable(sc->ctx); 6306 } 6307 6308 static void 6309 em_setup_vlan_hw_support(if_ctx_t ctx) 6310 { 6311 struct e1000_softc *sc = iflib_get_softc(ctx); 6312 struct e1000_hw *hw = &sc->hw; 6313 if_t ifp = iflib_get_ifp(ctx); 6314 s32 error; 6315 u32 max_frame_size, reg; 6316 u16 vid; 6317 int restore_failures; 6318 6319 /* Hyper-V programs the VF's receive limit and VLAN membership. */ 6320 if (igbv_is_hyperv(sc)) 6321 return; 6322 6323 /* 6324 * Only PFs have control over VLAN HW filtering 6325 * configuration. VFs have to act as if it's always 6326 * enabled. 6327 */ 6328 if (sc->vf_ifp) { 6329 max_frame_size = min(sc->shared->isc_max_frame_size + 6330 VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); 6331 e1000_rlpml_set_vf(hw, max_frame_size); 6332 restore_failures = 0; 6333 for (vid = 0; vid < 4096; vid++) { 6334 if ((sc->shadow_vfta[vid >> 5] & 6335 (1U << (vid & 0x1f))) == 0) 6336 continue; 6337 /* 6338 * Desired state remains in shadow_vfta for the next 6339 * replay if the PF mailbox is absent during reset. 6340 */ 6341 error = e1000_vfta_set_vf(hw, vid, true); 6342 if (error != E1000_SUCCESS) { 6343 igbv_vlan_retry_add(sc, vid); 6344 restore_failures++; 6345 } else 6346 igbv_vlan_retry_clear(sc, vid); 6347 } 6348 if (restore_failures != 0) 6349 device_printf(sc->dev, 6350 "VF VLAN restore failed for %d VIDs; retrying\n", 6351 restore_failures); 6352 return; 6353 } 6354 6355 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 6356 !em_disable_crc_stripping) { 6357 reg = E1000_READ_REG(hw, E1000_CTRL); 6358 reg |= E1000_CTRL_VME; 6359 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6360 } else { 6361 reg = E1000_READ_REG(hw, E1000_CTRL); 6362 reg &= ~E1000_CTRL_VME; 6363 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6364 } 6365 6366 /* 6367 * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN 6368 * filtering is disabled, the IOV VLAN rebuild instead makes the PF 6369 * VLAN-promiscuous without disabling the global filter. 6370 */ 6371 if (!em_if_vlan_filter_capable(ctx)) { 6372 if (igb_iov_enabled(sc)) { 6373 #ifdef PCI_IOV 6374 sc->iov_pf_vlan_promisc = true; 6375 #endif 6376 em_if_vlan_filter_enable(sc); 6377 } else 6378 em_if_vlan_filter_disable(sc); 6379 return; 6380 } 6381 #ifdef PCI_IOV 6382 if (igb_iov_enabled(sc)) 6383 sc->iov_pf_vlan_promisc = false; 6384 #endif 6385 6386 /* 6387 * A soft reset zero's out the VFTA, so 6388 * we need to repopulate it now. 6389 * We also insert VLAN 0 in the filter list, so we pass VLAN 0 tagged 6390 * traffic through. This will write the entire table. 6391 */ 6392 em_if_vlan_register(ctx, 0); 6393 6394 /* Enable the Filter Table */ 6395 em_if_vlan_filter_enable(sc); 6396 } 6397 6398 static void 6399 em_if_intr_enable(if_ctx_t ctx) 6400 { 6401 struct e1000_softc *sc = iflib_get_softc(ctx); 6402 struct e1000_hw *hw = &sc->hw; 6403 u32 ims_mask = IMS_ENABLE_MASK | em_fatal_error_intr_mask(sc); 6404 6405 if (sc->intr_type == IFLIB_INTR_MSIX) { 6406 E1000_WRITE_REG(hw, EM_EIAC, sc->ims); 6407 ims_mask |= sc->ims; 6408 } 6409 6410 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 6411 E1000_WRITE_FLUSH(hw); 6412 } 6413 6414 static void 6415 em_if_intr_disable(if_ctx_t ctx) 6416 { 6417 struct e1000_softc *sc = iflib_get_softc(ctx); 6418 struct e1000_hw *hw = &sc->hw; 6419 6420 if (sc->intr_type == IFLIB_INTR_MSIX) 6421 E1000_WRITE_REG(hw, EM_EIAC, 0); 6422 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6423 E1000_WRITE_FLUSH(hw); 6424 } 6425 6426 static void 6427 igb_if_intr_enable(if_ctx_t ctx) 6428 { 6429 struct e1000_softc *sc = iflib_get_softc(ctx); 6430 struct e1000_hw *hw = &sc->hw; 6431 u32 mask, reg; 6432 6433 if (igb_device_reset_pending(sc)) 6434 return; 6435 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6436 mask = (sc->que_mask | sc->link_mask); 6437 /* 6438 * VF interrupt controls are also mapped into these registers. 6439 * Preserve them and change only the PF vectors we own. 6440 */ 6441 reg = E1000_READ_REG(hw, E1000_EIAC); 6442 E1000_WRITE_REG(hw, E1000_EIAC, reg | mask); 6443 reg = E1000_READ_REG(hw, E1000_EIAM); 6444 E1000_WRITE_REG(hw, E1000_EIAM, reg | mask); 6445 igb_iov_intr_drain_stale(sc); 6446 E1000_WRITE_REG(hw, E1000_EIMS, mask); 6447 E1000_WRITE_REG(hw, E1000_IMS, 6448 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 6449 igb_iov_intr_mask(sc) | 6450 em_fatal_error_intr_mask(sc)); 6451 } else { 6452 mask = IMS_ENABLE_MASK | igb_device_reset_intr_mask(sc) | 6453 em_fatal_error_intr_mask(sc); 6454 /* Reading ICR masks every shared interrupt before the filter runs. */ 6455 E1000_WRITE_REG(hw, E1000_IAM, mask); 6456 E1000_WRITE_REG(hw, E1000_IMS, mask); 6457 } 6458 E1000_WRITE_FLUSH(hw); 6459 } 6460 6461 static void 6462 igb_if_intr_disable(if_ctx_t ctx) 6463 { 6464 struct e1000_softc *sc = iflib_get_softc(ctx); 6465 struct e1000_hw *hw = &sc->hw; 6466 u32 mask, reg; 6467 6468 /* This is the first CTX-owned register access after ICR.DRSTA. */ 6469 igb_prepare_device_reset(sc); 6470 6471 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6472 /* 6473 * Do not use a blanket EIMC write here. VF interrupt controls 6474 * are mapped into the same PF register space, so clearing bits 6475 * we do not own can leave running VFs with interrupts masked. 6476 * Before initial queue configuration the owned mask is zero 6477 * because this driver has not enabled a vector yet. 6478 */ 6479 mask = (sc->que_mask | sc->link_mask); 6480 reg = E1000_READ_REG(hw, E1000_EIAM); 6481 E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask); 6482 E1000_WRITE_REG(hw, E1000_EIMC, mask); 6483 reg = E1000_READ_REG(hw, E1000_EIAC); 6484 E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask); 6485 } else 6486 E1000_WRITE_REG(hw, E1000_IAM, 0); 6487 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6488 E1000_WRITE_FLUSH(hw); 6489 } 6490 6491 /* 6492 * Bit of a misnomer, what this really means is 6493 * to enable OS management of the system... aka 6494 * to disable special hardware management features 6495 */ 6496 static void 6497 em_init_manageability(struct e1000_softc *sc) 6498 { 6499 /* A shared code workaround */ 6500 #define E1000_82542_MANC2H E1000_MANC2H 6501 if (sc->has_manage) { 6502 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 6503 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6504 6505 /* disable hardware interception of ARP */ 6506 manc &= ~(E1000_MANC_ARP_EN); 6507 6508 /* enable receiving management packets to the host */ 6509 manc |= E1000_MANC_EN_MNG2HOST; 6510 #define E1000_MNG2HOST_PORT_623 (1 << 5) 6511 #define E1000_MNG2HOST_PORT_664 (1 << 6) 6512 manc2h |= E1000_MNG2HOST_PORT_623; 6513 manc2h |= E1000_MNG2HOST_PORT_664; 6514 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 6515 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6516 } 6517 } 6518 6519 /* 6520 * Give control back to hardware management 6521 * controller if there is one. 6522 */ 6523 static void 6524 em_release_manageability(struct e1000_softc *sc) 6525 { 6526 if (sc->has_manage) { 6527 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6528 6529 /* re-enable hardware interception of ARP */ 6530 manc |= E1000_MANC_ARP_EN; 6531 manc &= ~E1000_MANC_EN_MNG2HOST; 6532 6533 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6534 } 6535 } 6536 6537 /* 6538 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 6539 * For ASF and Pass Through versions of f/w this means 6540 * that the driver is loaded. For AMT version type f/w 6541 * this means that the network i/f is open. 6542 */ 6543 static void 6544 em_get_hw_control(struct e1000_softc *sc) 6545 { 6546 u32 ctrl_ext, swsm; 6547 6548 if (sc->vf_ifp) 6549 return; 6550 6551 if (sc->hw.mac.type == e1000_82573) { 6552 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6553 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6554 swsm | E1000_SWSM_DRV_LOAD); 6555 return; 6556 } 6557 /* else */ 6558 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6559 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6560 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 6561 } 6562 6563 /* 6564 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 6565 * For ASF and Pass Through versions of f/w this means that 6566 * the driver is no longer loaded. For AMT versions of the 6567 * f/w this means that the network i/f is closed. 6568 */ 6569 static void 6570 em_release_hw_control(struct e1000_softc *sc) 6571 { 6572 u32 ctrl_ext, swsm; 6573 6574 if (!sc->has_manage) 6575 return; 6576 6577 if (sc->hw.mac.type == e1000_82573) { 6578 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6579 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6580 swsm & ~E1000_SWSM_DRV_LOAD); 6581 return; 6582 } 6583 /* else */ 6584 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6585 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6586 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 6587 return; 6588 } 6589 6590 bool 6591 em_is_valid_ether_addr(const u8 *addr) 6592 { 6593 static const u8 zero_addr[ETHER_ADDR_LEN]; 6594 6595 return (!ETHER_IS_MULTICAST(addr) && 6596 memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0); 6597 } 6598 6599 static bool 6600 em_automask_tso(if_ctx_t ctx) 6601 { 6602 struct e1000_softc *sc = iflib_get_softc(ctx); 6603 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6604 if_t ifp = iflib_get_ifp(ctx); 6605 bool reset_needed; 6606 6607 if (!em_unsupported_tso && sc->link_speed && 6608 sc->link_speed != SPEED_1000 && 6609 scctx->isc_capenable & IFCAP_TSO) { 6610 device_printf(sc->dev, 6611 "Disabling TSO for 10/100 Ethernet.\n"); 6612 sc->tso_automasked = scctx->isc_capenable & IFCAP_TSO; 6613 scctx->isc_capenable &= ~IFCAP_TSO; 6614 if_setcapenablebit(ifp, 0, IFCAP_TSO); 6615 } else if (sc->link_speed == SPEED_1000 && sc->tso_automasked) { 6616 device_printf(sc->dev, "Re-enabling TSO for GbE.\n"); 6617 scctx->isc_capenable |= sc->tso_automasked; 6618 if_setcapenablebit(ifp, sc->tso_automasked, 0); 6619 sc->tso_automasked = 0; 6620 } else { 6621 return (false); 6622 } 6623 6624 /* 6625 * Apply the new capabilities to a running or administratively-up 6626 * interface, including one whose initialization has not completed. 6627 * A stopped, administratively-down interface will apply them at the 6628 * next initialization; driver flags do not describe that intent. 6629 */ 6630 reset_needed = iflib_is_running(ctx) || 6631 (if_getflags(ifp) & IFF_UP) != 0; 6632 if (!reset_needed) 6633 return (false); 6634 6635 /* iflib_init_locked handles ifnet hwassistbits */ 6636 iflib_request_reset(ctx); 6637 return (true); 6638 } 6639 6640 /* 6641 ** Parse the interface capabilities with regard 6642 ** to both system management and wake-on-lan for 6643 ** later use. 6644 */ 6645 static void 6646 em_get_wakeup(if_ctx_t ctx) 6647 { 6648 struct e1000_softc *sc = iflib_get_softc(ctx); 6649 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6650 device_t dev = iflib_get_dev(ctx); 6651 u16 eeprom_data = 0, device_id, apme_mask; 6652 bool apme; 6653 int error, wol_capabilities; 6654 6655 sc->has_manage = e1000_enable_mng_pass_thru(&sc->hw); 6656 apme_mask = EM_EEPROM_APME_HIGH; 6657 error = E1000_SUCCESS; 6658 6659 switch (sc->hw.mac.type) { 6660 case e1000_82542: 6661 case e1000_82543: 6662 break; 6663 case e1000_82544: 6664 error = e1000_read_nvm(&sc->hw, 6665 NVM_INIT_CONTROL2_REG, 1, &eeprom_data); 6666 apme_mask = EM_EEPROM_APME_LOW; 6667 break; 6668 case e1000_82541: 6669 case e1000_82547: 6670 error = e1000_read_nvm(&sc->hw, 6671 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6672 /* The EI parts place APM Enable in the low byte. */ 6673 if (sc->hw.device_id != E1000_DEV_ID_82541ER_LOM) 6674 apme_mask = EM_EEPROM_APME_LOW; 6675 break; 6676 case e1000_82546: 6677 case e1000_82546_rev_3: 6678 if (sc->hw.bus.func == 1) { 6679 error = e1000_read_nvm(&sc->hw, 6680 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6681 break; 6682 } else 6683 error = e1000_read_nvm(&sc->hw, 6684 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6685 break; 6686 case e1000_82573: 6687 case e1000_82583: 6688 sc->has_amt = true; 6689 /* FALLTHROUGH */ 6690 case e1000_82571: 6691 case e1000_82572: 6692 case e1000_80003es2lan: 6693 if (sc->hw.bus.func == 1) { 6694 error = e1000_read_nvm(&sc->hw, 6695 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6696 break; 6697 } else 6698 error = e1000_read_nvm(&sc->hw, 6699 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6700 break; 6701 case e1000_ich8lan: 6702 case e1000_ich9lan: 6703 case e1000_ich10lan: 6704 case e1000_pchlan: 6705 case e1000_pch2lan: 6706 case e1000_pch_lpt: 6707 case e1000_pch_spt: 6708 case e1000_pch_cnp: 6709 case e1000_pch_tgp: 6710 case e1000_pch_adp: 6711 case e1000_pch_mtp: 6712 case e1000_pch_ptp: 6713 case e1000_pch_nvp: 6714 apme_mask = E1000_WUC_APME; 6715 sc->has_amt = true; 6716 eeprom_data = E1000_READ_REG(&sc->hw, E1000_WUC); 6717 if (sc->hw.mac.type > e1000_ich10lan && 6718 (eeprom_data & E1000_WUC_PHY_WAKE) != 0) 6719 sc->wol_phy_wakeup = true; 6720 break; 6721 case e1000_82575: 6722 case e1000_82576: 6723 if (sc->hw.bus.func == 1) 6724 error = e1000_read_nvm(&sc->hw, 6725 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6726 else 6727 error = e1000_read_nvm(&sc->hw, 6728 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6729 sc->has_amt = true; 6730 break; 6731 case e1000_82580: 6732 case e1000_i350: 6733 case e1000_i354: 6734 case e1000_i210: 6735 case e1000_i211: 6736 error = e1000_read_nvm(&sc->hw, 6737 NVM_INIT_CONTROL3_PORT_A + 6738 NVM_82580_LAN_FUNC_OFFSET(sc->hw.bus.func), 1, 6739 &eeprom_data); 6740 sc->has_amt = true; 6741 break; 6742 default: 6743 error = e1000_read_nvm(&sc->hw, 6744 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6745 break; 6746 } 6747 if (error != E1000_SUCCESS && bootverbose) 6748 device_printf(dev, "NVM read failed while checking WoL: %d\n", 6749 error); 6750 if ((sc->hw.mac.type == e1000_i210 || 6751 sc->hw.mac.type == e1000_i211) && 6752 sc->hw.nvm.type == e1000_nvm_invm) { 6753 /* The shared reader does not expose the optional iNVM word. */ 6754 apme = (E1000_READ_REG(&sc->hw, E1000_WUC) & 6755 E1000_WUC_APME) != 0; 6756 } else { 6757 apme = error == E1000_SUCCESS && 6758 (eeprom_data & apme_mask) != 0; 6759 } 6760 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6761 IFCAP_WOL : 0; 6762 if (sc->hw.mac.type == e1000_82542 || 6763 sc->hw.mac.type == e1000_82543) 6764 wol_capabilities = 0; 6765 6766 /* APME selects the default; board and port restrictions select support. */ 6767 device_id = pci_get_device(dev); 6768 switch (device_id) { 6769 case E1000_DEV_ID_82542: 6770 case E1000_DEV_ID_82543GC_FIBER: 6771 case E1000_DEV_ID_82543GC_COPPER: 6772 case E1000_DEV_ID_82541ER: 6773 case E1000_DEV_ID_82541ER_LOM: 6774 case E1000_DEV_ID_82544EI_FIBER: 6775 case E1000_DEV_ID_82545EM_COPPER: 6776 case E1000_DEV_ID_82545EM_FIBER: 6777 case E1000_DEV_ID_82546EB_QUAD_COPPER: 6778 case E1000_DEV_ID_82546GB_QUAD_COPPER: 6779 case E1000_DEV_ID_82546GB_PCIE: 6780 wol_capabilities = 0; 6781 break; 6782 case E1000_DEV_ID_82546EB_FIBER: 6783 case E1000_DEV_ID_82546GB_FIBER: 6784 /* 6785 * Wake events are supported only on port A for dual fiber, 6786 * regardless of the NVM setting. 6787 */ 6788 if (sc->hw.bus.func == 1) 6789 wol_capabilities = 0; 6790 break; 6791 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 6792 /* if quad port adapter, disable WoL on all but port A */ 6793 if (global_quad_port_a != 0) 6794 wol_capabilities = 0; 6795 else 6796 wol_capabilities &= ~IFCAP_WOL_UCAST; 6797 /* Reset for multiple quad port adapters */ 6798 if (++global_quad_port_a == 4) 6799 global_quad_port_a = 0; 6800 break; 6801 case E1000_DEV_ID_82571EB_COPPER: 6802 case E1000_DEV_ID_82571EB_FIBER: 6803 case E1000_DEV_ID_82571EB_SERDES: 6804 /* These dual-port adapters support wake only on port A. */ 6805 if (sc->hw.bus.func == 1) 6806 wol_capabilities = 0; 6807 break; 6808 case E1000_DEV_ID_82571EB_SERDES_QUAD: 6809 wol_capabilities = 0; 6810 break; 6811 case E1000_DEV_ID_82571EB_QUAD_COPPER: 6812 case E1000_DEV_ID_82571EB_QUAD_FIBER: 6813 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 6814 case E1000_DEV_ID_82571PT_QUAD_COPPER: 6815 /* if quad port adapter, disable WoL on all but port A */ 6816 if (global_quad_port_a != 0) 6817 wol_capabilities = 0; 6818 /* Reset for multiple quad port adapters */ 6819 if (++global_quad_port_a == 4) 6820 global_quad_port_a = 0; 6821 break; 6822 case E1000_DEV_ID_82575GB_QUAD_COPPER: 6823 wol_capabilities = 0; 6824 break; 6825 case E1000_DEV_ID_82575EB_FIBER_SERDES: 6826 case E1000_DEV_ID_82576_FIBER: 6827 case E1000_DEV_ID_82576_SERDES: 6828 if (sc->hw.bus.func == 1) 6829 wol_capabilities = 0; 6830 break; 6831 case E1000_DEV_ID_82576_QUAD_COPPER: 6832 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 6833 if (global_quad_port_a != 0) 6834 wol_capabilities = 0; 6835 if (++global_quad_port_a == 4) 6836 global_quad_port_a = 0; 6837 break; 6838 default: 6839 break; 6840 } 6841 /* Legacy and igb non-primary ports require an explicit NVM setting. */ 6842 if ((sc->hw.mac.type < e1000_82571 || 6843 sc->hw.mac.type >= igb_mac_min) && sc->hw.bus.func != 0 && 6844 !apme) 6845 wol_capabilities = 0; 6846 6847 /* Some I350-family systems expose wake support but default it off. */ 6848 if ((sc->hw.mac.type == e1000_i350 && 6849 pci_get_subvendor(dev) == EM_SUBVENDOR_HP) || 6850 ((sc->hw.mac.type == e1000_i350 || 6851 sc->hw.mac.type == e1000_i354) && 6852 pci_get_subvendor(dev) == EM_SUBVENDOR_DELL) || 6853 (sc->hw.mac.type == e1000_i350 && 6854 ((pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_2 || 6855 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_3) && 6856 sc->hw.bus.func == 0))) { 6857 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6858 IFCAP_WOL : 0; 6859 apme = false; 6860 } 6861 if (sc->hw.mac.type == e1000_i350 && 6862 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_1) 6863 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6864 IFCAP_WOL : 0; 6865 6866 scctx->isc_capabilities &= ~IFCAP_WOL; 6867 scctx->isc_capabilities |= wol_capabilities; 6868 scctx->isc_capenable &= ~IFCAP_WOL; 6869 if (wol_capabilities != 0 && apme) 6870 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 6871 } 6872 6873 /* Configure the requested PCI Wake-on-LAN filters for suspend. */ 6874 static int 6875 em_enable_wakeup(if_ctx_t ctx) 6876 { 6877 struct e1000_softc *sc = iflib_get_softc(ctx); 6878 device_t dev = iflib_get_dev(ctx); 6879 if_t ifp = iflib_get_ifp(ctx); 6880 int enabled, error = 0, master_error, mcnt; 6881 u32 ctrl, ctrl_ext, rctl, saved_rctl, wuc, wufc; 6882 bool manage, rctl_modified; 6883 6884 if (sc->vf_ifp) 6885 return (0); 6886 if (!pci_has_pme(dev, PCI_POWERSTATE_D3_HOT)) 6887 return (0); 6888 6889 enabled = if_getcapenable(ifp) & if_getcapabilities(ifp) & IFCAP_WOL; 6890 manage = e1000_enable_mng_pass_thru(&sc->hw); 6891 rctl_modified = false; 6892 wuc = 0; 6893 /* Early 82545EM/82546EB need APM clocks for D3 manageability. */ 6894 if (manage && (sc->hw.mac.type == e1000_82545 || 6895 sc->hw.mac.type == e1000_82546)) 6896 wuc = E1000_WUC_APME; 6897 wufc = 0; 6898 if ((enabled & IFCAP_WOL_MAGIC) != 0) 6899 wufc |= E1000_WUFC_MAG; 6900 if ((enabled & IFCAP_WOL_UCAST) != 0) 6901 wufc |= E1000_WUFC_EX; 6902 if ((enabled & IFCAP_WOL_MCAST) != 0) { 6903 wufc |= E1000_WUFC_MC; 6904 bzero(sc->mta, ETHER_ADDR_LEN * 6905 MAX_NUM_MULTICAST_ADDRESSES); 6906 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, sc->mta); 6907 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) { 6908 e1000_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 6909 } else { 6910 switch (sc->hw.mac.type) { 6911 case e1000_82544: 6912 case e1000_82540: 6913 case e1000_82545: 6914 case e1000_82545_rev_3: 6915 case e1000_82546: 6916 case e1000_82546_rev_3: 6917 case e1000_82541: 6918 case e1000_82541_rev_2: 6919 case e1000_82547: 6920 case e1000_82547_rev_2: 6921 case e1000_82575: 6922 case e1000_82576: 6923 case e1000_82580: 6924 /* These parts require an MTA hit for WUFC_MC. */ 6925 em_fill_wakeup_mta(&sc->hw); 6926 break; 6927 default: 6928 break; 6929 } 6930 } 6931 } 6932 6933 if (wufc == 0) { 6934 if (sc->hw.mac.type >= e1000_82544) { 6935 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 6936 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 6937 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6938 } 6939 if (sc->wol_phy_wakeup && sc->wol_phy_armed) 6940 (void)em_disable_phy_wakeup(sc, NULL); 6941 if (manage) { 6942 if (sc->suspend_link_powered_down) 6943 em_power_up_wakeup_link(sc); 6944 em_configure_sx_low_power(sc, 0); 6945 pci_enable_pme(dev); 6946 } else { 6947 em_power_down_wakeup_link(sc); 6948 pci_clear_pme(dev); 6949 } 6950 goto master_disable; 6951 } 6952 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 6953 error = e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 6954 if (error != E1000_SUCCESS) { 6955 device_printf(dev, 6956 "Could not restore unicast wake address: %d\n", error); 6957 goto pme; 6958 } 6959 saved_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 6960 rctl = saved_rctl; 6961 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_MO_3); 6962 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 6963 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 6964 if ((wufc & E1000_WUFC_MC) != 0) 6965 rctl |= E1000_RCTL_MPE; 6966 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 6967 rctl_modified = true; 6968 6969 /* Advertise the wakeup capability */ 6970 if (sc->hw.mac.type >= e1000_82540) { 6971 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 6972 ctrl |= E1000_CTRL_ADVD3WUC; 6973 if (sc->hw.mac.type < igb_mac_min && !sc->wol_phy_wakeup) 6974 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6975 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 6976 } 6977 6978 /* Keep the laser running on legacy fiber and SerDes adapters. */ 6979 if (sc->hw.mac.type < igb_mac_min && 6980 (sc->hw.phy.media_type == e1000_media_type_fiber || 6981 sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 6982 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6983 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6984 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, ctrl_ext); 6985 } 6986 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6987 em_power_up_wakeup_link(sc); 6988 6989 if (sc->hw.mac.type >= e1000_ich8lan && 6990 sc->hw.mac.type < igb_mac_min) 6991 e1000_suspend_workarounds_ich8lan(&sc->hw); 6992 6993 if (sc->wol_phy_wakeup) { 6994 error = em_enable_phy_wakeup(sc, wufc); 6995 if (error) 6996 goto pme; 6997 } else { 6998 /* Enable wakeup by the MAC */ 6999 E1000_WRITE_REG(&sc->hw, E1000_WUC, 7000 wuc | E1000_WUC_PME_EN); 7001 E1000_WRITE_REG(&sc->hw, E1000_WUFC, wufc); 7002 } 7003 7004 /* The IGP3 D3 power-down workaround is specific to the em family. */ 7005 if (sc->hw.mac.type < igb_mac_min && 7006 sc->hw.phy.type == e1000_phy_igp_3) 7007 e1000_igp3_phy_powerdown_workaround_ich8lan(&sc->hw); 7008 em_configure_sx_low_power(sc, wufc); 7009 7010 pme: 7011 if (!error) 7012 pci_enable_pme(dev); 7013 else { 7014 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 7015 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 7016 if (rctl_modified) 7017 E1000_WRITE_REG(&sc->hw, E1000_RCTL, saved_rctl); 7018 pci_clear_pme(dev); 7019 } 7020 7021 master_disable: 7022 master_error = e1000_disable_pcie_master(&sc->hw); 7023 if (master_error != E1000_SUCCESS) 7024 device_printf(dev, "PCIe master disable timed out: %d\n", 7025 master_error); 7026 master_error = pci_disable_busmaster(dev); 7027 if (master_error != 0) 7028 device_printf(dev, "PCI bus-master disable failed: %d\n", 7029 master_error); 7030 7031 return (error == E1000_SUCCESS ? 0 : EIO); 7032 } 7033 7034 /* Configure the PCH low-power link modes used while the system sleeps. */ 7035 static void 7036 em_configure_sx_low_power(struct e1000_softc *sc, u32 wufc) 7037 { 7038 struct e1000_hw *hw = &sc->hw; 7039 struct e1000_dev_spec_ich8lan *dev_spec; 7040 s32 error; 7041 u16 eee_advert, lpi_ctrl; 7042 7043 if (hw->mac.type < e1000_pch_lpt || hw->mac.type >= igb_mac_min || 7044 sc->suspend_link_powered_down) 7045 return; 7046 7047 if (wufc != 0 && 7048 (wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC)) == 0) { 7049 /* ULP cannot preserve directed or broad multicast wake. */ 7050 error = e1000_enable_ulp_lpt_lp(hw, true); 7051 if (error != E1000_SUCCESS) { 7052 device_printf(sc->dev, 7053 "Could not enter PHY ultra-low-power mode: %d\n", 7054 error); 7055 return; 7056 } 7057 } 7058 7059 dev_spec = &hw->dev_spec.ich8lan; 7060 if (hw->phy.type != e1000_phy_i217 || dev_spec->eee_disable || 7061 dev_spec->eee_lp_ability == 0) 7062 return; 7063 7064 error = hw->phy.ops.acquire(hw); 7065 if (error != E1000_SUCCESS) 7066 goto out; 7067 error = hw->phy.ops.read_reg_locked(hw, I82579_LPI_CTRL, &lpi_ctrl); 7068 if (error != E1000_SUCCESS) 7069 goto release; 7070 error = e1000_read_emi_reg_locked(hw, I217_EEE_ADVERTISEMENT, 7071 &eee_advert); 7072 if (error != E1000_SUCCESS) 7073 goto release; 7074 7075 if ((eee_advert & dev_spec->eee_lp_ability & 7076 I82579_EEE_100_SUPPORTED) != 0) 7077 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 7078 if ((eee_advert & dev_spec->eee_lp_ability & 7079 I82579_EEE_1000_SUPPORTED) != 0) 7080 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 7081 error = hw->phy.ops.write_reg_locked(hw, I82579_LPI_CTRL, lpi_ctrl); 7082 release: 7083 hw->phy.ops.release(hw); 7084 out: 7085 if (error != E1000_SUCCESS) 7086 device_printf(sc->dev, 7087 "Could not configure Energy Efficient Ethernet for sleep: %d\n", 7088 error); 7089 } 7090 7091 static void 7092 em_power_up_wakeup_link(struct e1000_softc *sc) 7093 { 7094 struct e1000_hw *hw = &sc->hw; 7095 7096 if (hw->mac.type < igb_mac_min) 7097 e1000_power_up_phy(hw); 7098 else if (hw->phy.media_type == e1000_media_type_copper) 7099 e1000_power_up_phy(hw); 7100 else { 7101 e1000_power_up_fiber_serdes_link(hw); 7102 (void)e1000_setup_link(hw); 7103 } 7104 sc->suspend_link_powered_down = false; 7105 } 7106 7107 /* Drop the unused suspend link through the controller's shared-code hook. */ 7108 static void 7109 em_power_down_wakeup_link(struct e1000_softc *sc) 7110 { 7111 struct e1000_hw *hw = &sc->hw; 7112 7113 if (hw->mac.type >= igb_mac_min && 7114 hw->phy.media_type != e1000_media_type_copper) 7115 e1000_shutdown_fiber_serdes_link(hw); 7116 else 7117 e1000_power_down_phy(hw); 7118 sc->suspend_link_powered_down = true; 7119 } 7120 7121 /* PCH PHY wake requires the MAC receive state on the BM wake page. */ 7122 static int 7123 em_enable_phy_wakeup(struct e1000_softc *sc, u32 wufc) 7124 { 7125 struct e1000_hw *hw = &sc->hw; 7126 u32 mreg, wuc; 7127 u16 preg, wuc_enable; 7128 s32 error, restore_error; 7129 7130 /* Copy MAC RARs to PHY RARs before selecting the BM wake page. */ 7131 error = e1000_copy_rx_addrs_to_phy_ich8lan(hw); 7132 if (error != E1000_SUCCESS) 7133 goto out; 7134 7135 error = hw->phy.ops.acquire(hw); 7136 if (error != E1000_SUCCESS) { 7137 device_printf(sc->dev, "Could not acquire PHY for wakeup\n"); 7138 goto out; 7139 } 7140 7141 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7142 if (error != E1000_SUCCESS) 7143 goto release; 7144 7145 /* Wake status is RW1C and survives controller reset. */ 7146 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7147 if (error != E1000_SUCCESS) 7148 goto restore; 7149 7150 /* copy MAC MTA to PHY MTA */ 7151 for (int i = 0; i < hw->mac.mta_reg_count; i++) { 7152 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 7153 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i), 7154 (u16)(mreg & 0xffff)); 7155 if (error != E1000_SUCCESS) 7156 goto restore; 7157 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 7158 (u16)(mreg >> 16)); 7159 if (error != E1000_SUCCESS) 7160 goto restore; 7161 } 7162 7163 /* configure PHY Rx Control register */ 7164 error = hw->phy.ops.read_reg_page(hw, BM_RCTL, &preg); 7165 if (error != E1000_SUCCESS) 7166 goto restore; 7167 mreg = E1000_READ_REG(hw, E1000_RCTL); 7168 if (mreg & E1000_RCTL_UPE) 7169 preg |= BM_RCTL_UPE; 7170 if (mreg & E1000_RCTL_MPE) 7171 preg |= BM_RCTL_MPE; 7172 preg &= ~(BM_RCTL_MO_MASK); 7173 if (mreg & E1000_RCTL_MO_3) 7174 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 7175 << BM_RCTL_MO_SHIFT); 7176 if (mreg & E1000_RCTL_BAM) 7177 preg |= BM_RCTL_BAM; 7178 if (mreg & E1000_RCTL_PMCF) 7179 preg |= BM_RCTL_PMCF; 7180 mreg = E1000_READ_REG(hw, E1000_CTRL); 7181 if (mreg & E1000_CTRL_RFCE) 7182 preg |= BM_RCTL_RFCE; 7183 error = hw->phy.ops.write_reg_page(hw, BM_RCTL, preg); 7184 if (error != E1000_SUCCESS) 7185 goto restore; 7186 7187 wuc = E1000_WUC_PME_EN; 7188 if ((wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) != 0) 7189 wuc |= E1000_WUC_APME; 7190 7191 /* enable PHY wakeup in MAC register */ 7192 E1000_WRITE_REG(hw, E1000_WUFC, wufc); 7193 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PHY_WAKE | 7194 E1000_WUC_APMPME | E1000_WUC_PME_STATUS | wuc); 7195 7196 /* configure and enable PHY wakeup in PHY registers */ 7197 error = hw->phy.ops.write_reg_page(hw, BM_WUFC, wufc); 7198 if (error != E1000_SUCCESS) 7199 goto restore; 7200 error = hw->phy.ops.write_reg_page(hw, BM_WUC, wuc); 7201 if (error != E1000_SUCCESS) 7202 goto restore; 7203 7204 restore: 7205 /* Restore the page selector and expose only a complete setup. */ 7206 if (error == E1000_SUCCESS) 7207 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 7208 else 7209 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7210 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7211 &wuc_enable); 7212 if (error == E1000_SUCCESS) 7213 error = restore_error; 7214 release: 7215 hw->phy.ops.release(hw); 7216 out: 7217 sc->wol_phy_armed = error == E1000_SUCCESS; 7218 if (error != E1000_SUCCESS) 7219 device_printf(sc->dev, "Could not configure PHY wakeup: %d\n", 7220 error); 7221 7222 return (error); 7223 } 7224 7225 /* Clear host ownership and sticky status without disturbing ME wake. */ 7226 static int 7227 em_disable_phy_wakeup(struct e1000_softc *sc, u16 *wus) 7228 { 7229 struct e1000_hw *hw = &sc->hw; 7230 s32 error, restore_error; 7231 u16 phy_wus, wuc_enable; 7232 7233 error = hw->phy.ops.acquire(hw); 7234 if (error != E1000_SUCCESS) 7235 return (error); 7236 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7237 if (error != E1000_SUCCESS) 7238 goto release; 7239 7240 error = hw->phy.ops.read_reg_page(hw, BM_WUS, &phy_wus); 7241 if (error == E1000_SUCCESS) 7242 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7243 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7244 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7245 &wuc_enable); 7246 if (error == E1000_SUCCESS) 7247 error = restore_error; 7248 release: 7249 hw->phy.ops.release(hw); 7250 if (error == E1000_SUCCESS) { 7251 sc->wol_phy_armed = false; 7252 if (wus != NULL) 7253 *wus = phy_wus; 7254 } 7255 return (error); 7256 } 7257 7258 static void 7259 em_if_led_func(if_ctx_t ctx, int onoff) 7260 { 7261 struct e1000_softc *sc = iflib_get_softc(ctx); 7262 7263 if (onoff) { 7264 e1000_setup_led(&sc->hw); 7265 if (sc->hw.phy.media_type == e1000_media_type_internal_serdes) 7266 e1000_blink_led(&sc->hw); 7267 else 7268 e1000_led_on(&sc->hw); 7269 } else { 7270 e1000_led_off(&sc->hw); 7271 e1000_cleanup_led(&sc->hw); 7272 } 7273 } 7274 7275 /* 7276 * Disable the L0S and L1 LINK states 7277 */ 7278 static void 7279 em_disable_aspm(struct e1000_softc *sc) 7280 { 7281 int base, reg; 7282 u16 link_cap,link_ctrl; 7283 device_t dev = sc->dev; 7284 7285 switch (sc->hw.mac.type) { 7286 case e1000_82573: 7287 case e1000_82574: 7288 case e1000_82583: 7289 break; 7290 default: 7291 return; 7292 } 7293 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) 7294 return; 7295 reg = base + PCIER_LINK_CAP; 7296 link_cap = pci_read_config(dev, reg, 2); 7297 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0) 7298 return; 7299 reg = base + PCIER_LINK_CTL; 7300 link_ctrl = pci_read_config(dev, reg, 2); 7301 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC; 7302 pci_write_config(dev, reg, link_ctrl, 2); 7303 return; 7304 } 7305 7306 /********************************************************************** 7307 * 7308 * Update the board statistics counters. 7309 * 7310 **********************************************************************/ 7311 void 7312 em_update_stats_counters(struct e1000_softc *sc) 7313 { 7314 struct e1000_hw_stats *stats; 7315 u64 prev_xoffrxc; 7316 7317 if (sc->vf_ifp) { 7318 em_update_vf_stats_counters(sc); 7319 return; 7320 } 7321 7322 stats = &sc->ustats.stats; 7323 prev_xoffrxc = stats->xoffrxc; 7324 7325 if(sc->hw.phy.media_type == e1000_media_type_copper || 7326 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 7327 stats->symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 7328 stats->sec += E1000_READ_REG(&sc->hw, E1000_SEC); 7329 } 7330 stats->crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 7331 stats->mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 7332 stats->scc += E1000_READ_REG(&sc->hw, E1000_SCC); 7333 stats->ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 7334 7335 stats->mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 7336 stats->latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 7337 stats->colc += E1000_READ_REG(&sc->hw, E1000_COLC); 7338 stats->dc += E1000_READ_REG(&sc->hw, E1000_DC); 7339 stats->rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 7340 stats->xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 7341 stats->xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 7342 stats->xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 7343 /* 7344 ** For watchdog management we need to know if we have been 7345 ** paused during the last interval, so capture that here. 7346 */ 7347 if (stats->xoffrxc != prev_xoffrxc) 7348 sc->shared->isc_pause_frames = 1; 7349 stats->xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 7350 stats->fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 7351 stats->prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 7352 stats->prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 7353 stats->prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 7354 stats->prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 7355 stats->prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 7356 stats->prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 7357 stats->gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 7358 stats->bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 7359 stats->mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 7360 stats->gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 7361 7362 /* For the 64-bit byte counters the low dword must be read first. */ 7363 /* Both registers clear on the read of the high dword */ 7364 7365 stats->gorc += E1000_READ_REG(&sc->hw, E1000_GORCL) + 7366 ((u64)E1000_READ_REG(&sc->hw, E1000_GORCH) << 32); 7367 stats->gotc += E1000_READ_REG(&sc->hw, E1000_GOTCL) + 7368 ((u64)E1000_READ_REG(&sc->hw, E1000_GOTCH) << 32); 7369 7370 stats->rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 7371 stats->ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 7372 stats->rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 7373 stats->roc += E1000_READ_REG(&sc->hw, E1000_ROC); 7374 stats->rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 7375 7376 stats->mgprc += E1000_READ_REG(&sc->hw, E1000_MGTPRC); 7377 stats->mgpdc += E1000_READ_REG(&sc->hw, E1000_MGTPDC); 7378 stats->mgptc += E1000_READ_REG(&sc->hw, E1000_MGTPTC); 7379 7380 stats->tor += E1000_READ_REG(&sc->hw, E1000_TORH); 7381 stats->tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 7382 7383 stats->tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 7384 stats->tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 7385 stats->ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 7386 stats->ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 7387 stats->ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 7388 stats->ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 7389 stats->ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 7390 stats->ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 7391 stats->mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 7392 stats->bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 7393 7394 /* TLPIC and RLPIC are clear-on-read. */ 7395 if (em_mac_has_eee(sc->hw.mac.type)) { 7396 stats->tlpic += E1000_READ_REG(&sc->hw, E1000_TLPIC); 7397 stats->rlpic += E1000_READ_REG(&sc->hw, E1000_RLPIC); 7398 } 7399 7400 /* Interrupt Counts */ 7401 7402 stats->iac += E1000_READ_REG(&sc->hw, E1000_IAC); 7403 stats->icrxptc += E1000_READ_REG(&sc->hw, E1000_ICRXPTC); 7404 stats->icrxatc += E1000_READ_REG(&sc->hw, E1000_ICRXATC); 7405 stats->ictxptc += E1000_READ_REG(&sc->hw, E1000_ICTXPTC); 7406 stats->ictxatc += E1000_READ_REG(&sc->hw, E1000_ICTXATC); 7407 stats->ictxqec += E1000_READ_REG(&sc->hw, E1000_ICTXQEC); 7408 stats->ictxqmtc += E1000_READ_REG(&sc->hw, E1000_ICTXQMTC); 7409 stats->icrxdmtc += E1000_READ_REG(&sc->hw, E1000_ICRXDMTC); 7410 stats->icrxoc += E1000_READ_REG(&sc->hw, E1000_ICRXOC); 7411 7412 if (sc->hw.mac.type >= e1000_82543) { 7413 stats->algnerrc += 7414 E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 7415 stats->rxerrc += 7416 E1000_READ_REG(&sc->hw, E1000_RXERRC); 7417 stats->tncrs += 7418 E1000_READ_REG(&sc->hw, E1000_TNCRS); 7419 stats->cexterr += 7420 E1000_READ_REG(&sc->hw, E1000_CEXTERR); 7421 stats->tsctc += 7422 E1000_READ_REG(&sc->hw, E1000_TSCTC); 7423 stats->tsctfc += 7424 E1000_READ_REG(&sc->hw, E1000_TSCTFC); 7425 } 7426 7427 if (em_has_82571_ecc_stats(&sc->hw)) 7428 em_update_82571_ecc_stats(sc); 7429 else if (em_has_pch_ecc(&sc->hw)) 7430 em_update_pch_ecc_stats(sc, 7431 E1000_READ_REG(&sc->hw, E1000_PBECCSTS)); 7432 else if (em_has_82575_memory_errors(&sc->hw)) 7433 em_update_82575_ecc_stats(sc, 7434 E1000_READ_REG(&sc->hw, E1000_PBECCSTS_82575), 7435 E1000_READ_REG(&sc->hw, E1000_RDHESTS_82575), 7436 E1000_READ_REG(&sc->hw, E1000_TDHESTS_82575)); 7437 else if (em_has_82576_memory_errors(&sc->hw)) 7438 em_update_82576_ecc_stats(sc); 7439 else if (em_has_82580_memory_errors(&sc->hw)) 7440 em_update_82580_ecc_stats(sc, 7441 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 7442 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 7443 E1000_READ_REG(&sc->hw, E1000_PCIEECCSTS)); 7444 else if (em_has_i350_i354_memory_errors(&sc->hw)) 7445 em_update_i350_i354_ecc_stats(sc); 7446 else if (em_has_i210_memory_errors(&sc->hw)) 7447 em_update_i210_ecc_stats(sc); 7448 } 7449 7450 static bool 7451 em_mac_has_eee(enum e1000_mac_type type) 7452 { 7453 7454 return ((type >= e1000_pch2lan && type < e1000_82575) || 7455 (type >= e1000_i350 && type <= e1000_i211)); 7456 } 7457 7458 static void 7459 em_initialize_vf_stats(struct e1000_softc *sc) 7460 { 7461 struct e1000_vf_stats *stats; 7462 7463 stats = &sc->ustats.vf_stats; 7464 *stats = (struct e1000_vf_stats){}; 7465 em_rebase_vf_stats(sc); 7466 } 7467 7468 static void 7469 em_rebase_vf_stats(struct e1000_softc *sc) 7470 { 7471 struct e1000_vf_stats *stats; 7472 bool hyperv = igbv_is_hyperv(sc); 7473 7474 sc->vf_stats_valid = false; 7475 if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) 7476 return; 7477 7478 /* 7479 * A PF reset starts a new VF counter epoch. Preserve the accumulated 7480 * totals while establishing a new raw baseline so the reset is not 7481 * mistaken for a 32-bit wrap. 7482 */ 7483 stats = &sc->ustats.vf_stats; 7484 #define INIT_VF_REG(reg, name) do { \ 7485 stats->last_##name = E1000_READ_REG(&sc->hw, reg); \ 7486 } while (0) 7487 INIT_VF_REG(E1000_VFGPRC, gprc); 7488 INIT_VF_REG(E1000_VFGORC, gorc); 7489 INIT_VF_REG(E1000_VFGPTC, gptc); 7490 INIT_VF_REG(E1000_VFGOTC, gotc); 7491 /* 7492 * I350 specification update erratum 31 says VFMPRC is not 7493 * accessible from VF memory. The 0xf3c register remains valid on 7494 * 82576 VFs, but must not be read on vfadapt_i350. 7495 */ 7496 if (sc->hw.mac.type == e1000_vfadapt) 7497 INIT_VF_REG(E1000_VFMPRC, mprc); 7498 else 7499 stats->last_mprc = 0; 7500 INIT_VF_REG(E1000_VFGOTLBC, gotlbc); 7501 INIT_VF_REG(E1000_VFGPTLBC, gptlbc); 7502 INIT_VF_REG(E1000_VFGORLBC, gorlbc); 7503 INIT_VF_REG(E1000_VFGPRLBC, gprlbc); 7504 #undef INIT_VF_REG 7505 sc->vf_stats_valid = !hyperv || 7506 E1000_READ_REG(&sc->hw, E1000_STATUS) != 0xffffffff; 7507 } 7508 7509 static void 7510 em_update_vf_stats_counters(struct e1000_softc *sc) 7511 { 7512 struct e1000_vf_stats sample; 7513 struct e1000_vf_stats *stats; 7514 bool hyperv, reset; 7515 7516 hyperv = igbv_is_hyperv(sc); 7517 if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) { 7518 sc->vf_stats_valid = false; 7519 return; 7520 } 7521 reset = hyperv && e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS; 7522 if (hyperv && !sc->vf_stats_valid) 7523 reset = true; 7524 if (hyperv && (E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) & 7525 E1000_TXDCTL_QUEUE_ENABLE) == 0) 7526 reset = true; 7527 sample = sc->ustats.vf_stats; 7528 stats = &sample; 7529 7530 /* 7531 * Internal VF loopback traffic can continue without physical link, 7532 * so sample the counters regardless of link state. 7533 */ 7534 UPDATE_VF_REG(E1000_VFGPRC, 7535 stats->last_gprc, stats->gprc); 7536 UPDATE_VF_REG(E1000_VFGORC, 7537 stats->last_gorc, stats->gorc); 7538 UPDATE_VF_REG(E1000_VFGPTC, 7539 stats->last_gptc, stats->gptc); 7540 UPDATE_VF_REG(E1000_VFGOTC, 7541 stats->last_gotc, stats->gotc); 7542 if (sc->hw.mac.type == e1000_vfadapt) 7543 UPDATE_VF_REG(E1000_VFMPRC, 7544 stats->last_mprc, stats->mprc); 7545 UPDATE_VF_REG(E1000_VFGOTLBC, 7546 stats->last_gotlbc, stats->gotlbc); 7547 UPDATE_VF_REG(E1000_VFGPTLBC, 7548 stats->last_gptlbc, stats->gptlbc); 7549 UPDATE_VF_REG(E1000_VFGORLBC, 7550 stats->last_gorlbc, stats->gorlbc); 7551 UPDATE_VF_REG(E1000_VFGPRLBC, 7552 stats->last_gprlbc, stats->gprlbc); 7553 /* 7554 * Hyper-V can reset the VF without a native mailbox handshake. Do not 7555 * count that counter clear as a 32-bit wrap, including a reset during 7556 * this sweep. The PF can also clear counters while blocking a queue 7557 * for MDD without leaving a reset indication. Hyper-V has one queue; 7558 * a disabled queue cannot supply a valid running counter epoch. 7559 */ 7560 if (hyperv) { 7561 if (e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS) 7562 reset = true; 7563 if ((E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) & 7564 E1000_TXDCTL_QUEUE_ENABLE) == 0) 7565 reset = true; 7566 if (E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) { 7567 /* Rebase on a good sample before accounting further deltas. */ 7568 sc->vf_stats_valid = false; 7569 return; 7570 } 7571 } 7572 if (reset) 7573 em_rebase_vf_stats(sc); 7574 else 7575 sc->ustats.vf_stats = sample; 7576 } 7577 7578 static uint64_t 7579 em_if_get_vf_counter(if_ctx_t ctx, ift_counter cnt) 7580 { 7581 struct e1000_softc *sc = iflib_get_softc(ctx); 7582 if_t ifp = iflib_get_ifp(ctx); 7583 7584 switch (cnt) { 7585 case IFCOUNTER_IERRORS: 7586 return sc->dropped_pkts; 7587 default: 7588 return (if_get_counter_default(ifp, cnt)); 7589 } 7590 } 7591 7592 static uint64_t 7593 em_if_get_counter(if_ctx_t ctx, ift_counter cnt) 7594 { 7595 struct e1000_softc *sc = iflib_get_softc(ctx); 7596 struct e1000_hw_stats *stats; 7597 if_t ifp = iflib_get_ifp(ctx); 7598 7599 if (sc->vf_ifp) 7600 return (em_if_get_vf_counter(ctx, cnt)); 7601 7602 stats = &sc->ustats.stats; 7603 7604 switch (cnt) { 7605 case IFCOUNTER_COLLISIONS: 7606 return (stats->colc); 7607 case IFCOUNTER_IERRORS: 7608 return (sc->dropped_pkts + stats->rxerrc + 7609 stats->crcerrs + stats->algnerrc + 7610 stats->ruc + stats->roc + 7611 stats->mpc + stats->cexterr); 7612 case IFCOUNTER_OERRORS: 7613 return (if_get_counter_default(ifp, cnt) + 7614 stats->ecol + stats->latecol); 7615 default: 7616 return (if_get_counter_default(ifp, cnt)); 7617 } 7618 } 7619 7620 /* em_if_needs_restart - Tell iflib when the driver needs to be reinitialized 7621 * @ctx: iflib context 7622 * @event: event code to check 7623 * 7624 * Defaults to returning false for unknown events. 7625 * 7626 * @returns true if iflib needs to reinit the interface 7627 */ 7628 static bool 7629 em_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 7630 { 7631 switch (event) { 7632 case IFLIB_RESTART_VLAN_CONFIG: 7633 default: 7634 return (false); 7635 } 7636 } 7637 7638 /* Export a single 32-bit register via a read-only sysctl. */ 7639 static int 7640 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 7641 { 7642 struct e1000_softc *sc; 7643 u_int val; 7644 7645 sc = oidp->oid_arg1; 7646 val = E1000_READ_REG(&sc->hw, oidp->oid_arg2); 7647 return (sysctl_handle_int(oidp, &val, 0, req)); 7648 } 7649 7650 enum em_ring_register { 7651 EM_RING_HEAD, 7652 EM_RING_TAIL, 7653 }; 7654 7655 /* Queue register addresses can change when the PF enters IOV mode. */ 7656 static int 7657 em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS) 7658 { 7659 struct tx_ring *txr; 7660 u_int reg, val; 7661 7662 txr = oidp->oid_arg1; 7663 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) : 7664 E1000_TDT(txr->me); 7665 val = E1000_READ_REG(&txr->sc->hw, reg); 7666 return (sysctl_handle_int(oidp, &val, 0, req)); 7667 } 7668 7669 static int 7670 em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS) 7671 { 7672 struct rx_ring *rxr; 7673 u_int reg, val; 7674 7675 rxr = oidp->oid_arg1; 7676 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) : 7677 E1000_RDT(rxr->me); 7678 val = E1000_READ_REG(&rxr->sc->hw, reg); 7679 return (sysctl_handle_int(oidp, &val, 0, req)); 7680 } 7681 7682 /* Per queue holdoff interrupt rate handler */ 7683 static int 7684 em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 7685 { 7686 struct em_rx_queue *rque; 7687 struct em_tx_queue *tque; 7688 struct e1000_hw *hw; 7689 int error; 7690 u32 reg, usec, rate; 7691 7692 bool tx = oidp->oid_arg2; 7693 7694 if (tx) { 7695 tque = oidp->oid_arg1; 7696 hw = &tque->sc->hw; 7697 if (hw->mac.type >= igb_mac_min) 7698 reg = E1000_READ_REG(hw, E1000_EITR(tque->msix)); 7699 else if (hw->mac.type == e1000_82574 && 7700 tque->sc->intr_type == IFLIB_INTR_MSIX) 7701 reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->msix)); 7702 else 7703 reg = E1000_READ_REG(hw, E1000_ITR); 7704 } else { 7705 rque = oidp->oid_arg1; 7706 hw = &rque->sc->hw; 7707 if (hw->mac.type >= igb_mac_min) 7708 reg = E1000_READ_REG(hw, E1000_EITR(rque->msix)); 7709 else if (hw->mac.type == e1000_82574 && 7710 rque->sc->intr_type == IFLIB_INTR_MSIX) 7711 reg = E1000_READ_REG(hw, 7712 E1000_EITR_82574(rque->msix)); 7713 else 7714 reg = E1000_READ_REG(hw, E1000_ITR); 7715 } 7716 7717 if (hw->mac.type < igb_mac_min) { 7718 if (reg > 0) 7719 rate = EM_INTS_TO_ITR(reg); 7720 else 7721 rate = 0; 7722 } else { 7723 usec = (reg & IGB_QVECTOR_MASK); 7724 if (usec > 0) 7725 rate = IGB_EITR_TO_INTS(usec); 7726 else 7727 rate = 0; 7728 } 7729 7730 error = sysctl_handle_int(oidp, &rate, 0, req); 7731 if (error || !req->newptr) 7732 return error; 7733 return 0; 7734 } 7735 7736 /* 7737 * Add sysctl variables, one per statistic, to the system. 7738 */ 7739 static void 7740 em_add_hw_stats(struct e1000_softc *sc) 7741 { 7742 device_t dev = iflib_get_dev(sc->ctx); 7743 struct em_tx_queue *tx_que = sc->tx_queues; 7744 struct em_rx_queue *rx_que = sc->rx_queues; 7745 7746 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 7747 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 7748 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 7749 struct e1000_hw_stats *stats; 7750 7751 struct sysctl_oid *stat_node, *queue_node, *int_node; 7752 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 7753 7754 #define QUEUE_NAME_LEN 32 7755 char namebuf[QUEUE_NAME_LEN]; 7756 7757 /* Driver Statistics */ 7758 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 7759 CTLFLAG_RD, &sc->dropped_pkts, 7760 "Driver dropped packets"); 7761 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 7762 CTLFLAG_RD, &sc->link_irq, 7763 "Link MSI-X IRQ Handled"); 7764 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 7765 CTLFLAG_RD, &sc->rx_overruns, 7766 "RX overruns"); 7767 if (!sc->vf_ifp) { 7768 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 7769 CTLTYPE_UINT | CTLFLAG_RD, 7770 sc, E1000_CTRL, em_sysctl_reg_handler, "IU", 7771 "Device Control Register"); 7772 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 7773 CTLTYPE_UINT | CTLFLAG_RD, 7774 sc, E1000_RCTL, em_sysctl_reg_handler, "IU", 7775 "Receiver Control Register"); 7776 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 7777 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 7778 "Flow Control High Watermark"); 7779 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 7780 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 7781 "Flow Control Low Watermark"); 7782 } 7783 7784 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 7785 struct tx_ring *txr = &tx_que->txr; 7786 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 7787 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7788 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 7789 queue_list = SYSCTL_CHILDREN(queue_node); 7790 7791 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7792 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 7793 true, em_sysctl_interrupt_rate_handler, 7794 "IU", "Interrupt Rate"); 7795 7796 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 7797 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD, 7798 em_sysctl_tx_ring_handler, "IU", 7799 "Transmit Descriptor Head"); 7800 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 7801 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL, 7802 em_sysctl_tx_ring_handler, "IU", 7803 "Transmit Descriptor Tail"); 7804 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 7805 CTLFLAG_RD, &txr->tx_irq, 7806 "Queue MSI-X Transmit Interrupts"); 7807 } 7808 7809 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 7810 struct rx_ring *rxr = &rx_que->rxr; 7811 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 7812 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7813 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 7814 queue_list = SYSCTL_CHILDREN(queue_node); 7815 7816 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7817 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 7818 false, em_sysctl_interrupt_rate_handler, 7819 "IU", "Interrupt Rate"); 7820 7821 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 7822 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD, 7823 em_sysctl_rx_ring_handler, "IU", 7824 "Receive Descriptor Head"); 7825 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 7826 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL, 7827 em_sysctl_rx_ring_handler, "IU", 7828 "Receive Descriptor Tail"); 7829 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 7830 CTLFLAG_RD, &rxr->rx_irq, 7831 "Queue MSI-X Receive Interrupts"); 7832 } 7833 7834 /* MAC stats get their own sub node */ 7835 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 7836 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 7837 stat_list = SYSCTL_CHILDREN(stat_node); 7838 7839 /* 7840 ** VF adapter has a very limited set of stats 7841 ** since its not managing the metal, so to speak. 7842 */ 7843 if (sc->vf_ifp) { 7844 struct e1000_vf_stats *vfstats = &sc->ustats.vf_stats; 7845 7846 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 7847 CTLFLAG_RD, &vfstats->gprc, 7848 "Good Packets Received"); 7849 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 7850 CTLFLAG_RD, &vfstats->gptc, 7851 "Good Packets Transmitted"); 7852 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 7853 CTLFLAG_RD, &vfstats->gorc, 7854 "Good Octets Received"); 7855 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 7856 CTLFLAG_RD, &vfstats->gotc, 7857 "Good Octets Transmitted"); 7858 if (sc->hw.mac.type == e1000_vfadapt) { 7859 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7860 "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc, 7861 "Multicast Packets Received"); 7862 } 7863 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7864 "loopback_good_pkts_recvd", 7865 CTLFLAG_RD, &vfstats->gprlbc, 7866 "Good Loopback Packets Received"); 7867 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7868 "loopback_good_pkts_txd", 7869 CTLFLAG_RD, &vfstats->gptlbc, 7870 "Good Loopback Packets Transmitted"); 7871 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7872 "loopback_good_octets_recvd", 7873 CTLFLAG_RD, &vfstats->gorlbc, 7874 "Good Loopback Octets Received"); 7875 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7876 "loopback_good_octets_txd", 7877 CTLFLAG_RD, &vfstats->gotlbc, 7878 "Good Loopback Octets Transmitted"); 7879 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7880 "rx_csum_offload_good", 7881 CTLFLAG_RD, &sc->rx_csum_good, 7882 "Receive Checksum Offload Successes"); 7883 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7884 "rx_csum_offload_errors", 7885 CTLFLAG_RD, &sc->rx_csum_errors, 7886 "Receive Checksum Offload Errors"); 7887 return; 7888 } 7889 7890 stats = &sc->ustats.stats; 7891 if (em_mac_has_eee(sc->hw.mac.type)) { 7892 struct sysctl_oid *eee_node; 7893 struct sysctl_oid_list *eee_list; 7894 7895 eee_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eee", 7896 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7897 "Energy Efficient Ethernet statistics"); 7898 eee_list = SYSCTL_CHILDREN(eee_node); 7899 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "tx_lpi_count", 7900 CTLFLAG_RD, &stats->tlpic, "TX LPI event count"); 7901 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "rx_lpi_count", 7902 CTLFLAG_RD, &stats->rlpic, "RX LPI event count"); 7903 } 7904 if (em_has_memory_error_stats(&sc->hw)) { 7905 struct sysctl_oid *memerr_node; 7906 struct sysctl_oid_list *memerr_list; 7907 7908 memerr_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 7909 "memory_errors", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7910 "Internal memory error indications"); 7911 memerr_list = SYSCTL_CHILDREN(memerr_node); 7912 if (em_has_memory_errors(&sc->hw)) 7913 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7914 "fatal_resets", CTLFLAG_RD, 7915 &sc->fatal_error_reset_count, 7916 "Resets requested for fatal internal memory errors"); 7917 if (em_has_82571_ecc_stats(&sc->hw)) { 7918 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7919 "detected_packet_buffer", CTLFLAG_RD, 7920 &sc->corrected_error_packet_buffer_count, 7921 "Detected packet-buffer ECC errors"); 7922 } else if (em_has_pch_ecc(&sc->hw)) { 7923 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7924 "corrected_packet_buffer", CTLFLAG_RD, 7925 &sc->corrected_error_packet_buffer_count, 7926 "Corrected packet-buffer ECC errors"); 7927 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7928 "uncorrected_packet_buffer", CTLFLAG_RD, 7929 &sc->uncorrected_error_packet_buffer_count, 7930 "Uncorrected packet-buffer ECC errors"); 7931 } else if (em_has_82575_memory_errors(&sc->hw)) { 7932 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7933 "corrected_packet_buffer", CTLFLAG_RD, 7934 &sc->corrected_error_packet_buffer_count, 7935 "Corrected packet-buffer ECC errors"); 7936 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7937 "uncorrected_packet_buffer", CTLFLAG_RD, 7938 &sc->uncorrected_error_packet_buffer_count, 7939 "Uncorrected packet-buffer ECC errors"); 7940 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7941 "corrected_descriptor_handler", CTLFLAG_RD, 7942 &sc->corrected_error_dma_count, 7943 "Corrected descriptor-handler ECC errors"); 7944 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7945 "uncorrected_descriptor_handler", CTLFLAG_RD, 7946 &sc->uncorrected_error_dma_count, 7947 "Uncorrected descriptor-handler ECC errors"); 7948 } else if (em_has_82576_memory_errors(&sc->hw)) { 7949 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7950 "fatal_unknown", CTLFLAG_RD, 7951 &sc->fatal_error_unknown_count, 7952 "Fatal memory errors without a reported source"); 7953 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7954 "corrected_packet_buffer", CTLFLAG_RD, 7955 &sc->corrected_error_packet_buffer_count, 7956 "Corrected packet and switch-buffer ECC errors"); 7957 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7958 "uncorrected_packet_buffer", CTLFLAG_RD, 7959 &sc->uncorrected_error_packet_buffer_count, 7960 "Uncorrected packet and switch-buffer ECC errors"); 7961 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7962 "corrected_descriptor_handler", CTLFLAG_RD, 7963 &sc->corrected_error_dma_count, 7964 "Corrected descriptor-handler ECC errors"); 7965 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7966 "uncorrected_descriptor_handler", CTLFLAG_RD, 7967 &sc->uncorrected_error_dma_count, 7968 "Uncorrected descriptor-handler ECC errors"); 7969 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7970 "corrected_pcie_write_buffer", CTLFLAG_RD, 7971 &sc->corrected_error_pcie_tx_data_count, 7972 "Corrected PCIe write-buffer ECC errors"); 7973 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7974 "corrected_pcie_retry_buffer", CTLFLAG_RD, 7975 &sc->corrected_error_pcie_retry_count, 7976 "Corrected controller-shared PCIe retry-buffer errors"); 7977 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7978 "corrected_pcie_msix", CTLFLAG_RD, 7979 &sc->corrected_error_pcie_other_count, 7980 "Corrected controller-shared PCIe MSI-X errors"); 7981 } else { 7982 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7983 "fatal_lan", CTLFLAG_RD, 7984 &sc->fatal_error_lan_count, 7985 "Fatal LAN-port memory error indications"); 7986 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7987 "fatal_management", CTLFLAG_RD, 7988 &sc->fatal_error_mng_count, 7989 "Fatal management-memory error indications"); 7990 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7991 "fatal_pcie", CTLFLAG_RD, 7992 &sc->fatal_error_pcie_count, 7993 "Fatal PCIe memory error indications"); 7994 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7995 "fatal_dma", CTLFLAG_RD, 7996 &sc->fatal_error_dma_count, 7997 "Fatal DMA memory error indications"); 7998 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7999 "fatal_unknown", CTLFLAG_RD, 8000 &sc->fatal_error_unknown_count, 8001 "Fatal memory errors without a reported region"); 8002 if (em_has_82580_memory_errors(&sc->hw)) { 8003 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8004 "corrected_packet_buffer", CTLFLAG_RD, 8005 &sc->corrected_error_packet_buffer_count, 8006 "Corrected packet-buffer ECC errors"); 8007 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8008 "uncorrected_pcie", CTLFLAG_RD, 8009 &sc->uncorrected_error_pcie_count, 8010 "Uncorrected PCIe command-memory ECC indications"); 8011 } else if (em_has_i210_memory_errors(&sc->hw)) { 8012 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8013 "corrected_dma", CTLFLAG_RD, 8014 &sc->corrected_error_dma_count, 8015 "Corrected DMA memory error indications"); 8016 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8017 "corrected_pcie_tx_data", CTLFLAG_RD, 8018 &sc->corrected_error_pcie_tx_data_count, 8019 "Corrected PCIe transmit-data memory indications"); 8020 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8021 "corrected_pcie_retry", CTLFLAG_RD, 8022 &sc->corrected_error_pcie_retry_count, 8023 "Corrected PCIe retry-buffer memory indications"); 8024 } else { 8025 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8026 "corrected_dma", CTLFLAG_RD, 8027 &sc->corrected_error_dma_count, 8028 "Corrected DMA memory indications"); 8029 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8030 "corrected_packet_buffer", CTLFLAG_RD, 8031 &sc->corrected_error_packet_buffer_count, 8032 "Corrected packet-buffer memory indications"); 8033 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8034 "corrected_lan_mng_fifo", CTLFLAG_RD, 8035 &sc->corrected_error_lan_mng_fifo_count, 8036 "Corrected LAN management transmit-FIFO ECC " 8037 "indications"); 8038 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8039 "corrected_pcie_tx_data", CTLFLAG_RD, 8040 &sc->corrected_error_pcie_tx_data_count, 8041 "Corrected PCIe transmit-data memory indications"); 8042 if (sc->hw.mac.type == e1000_i350) 8043 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8044 "corrected_pcie_retry", CTLFLAG_RD, 8045 &sc->corrected_error_pcie_retry_count, 8046 "Corrected PCIe retry-buffer memory " 8047 "indications"); 8048 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8049 "corrected_pcie_other", CTLFLAG_RD, 8050 &sc->corrected_error_pcie_other_count, 8051 "Other corrected PCIe memory indications"); 8052 } 8053 } 8054 } 8055 8056 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 8057 CTLFLAG_RD, &stats->ecol, 8058 "Excessive collisions"); 8059 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 8060 CTLFLAG_RD, &stats->scc, 8061 "Single collisions"); 8062 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 8063 CTLFLAG_RD, &stats->mcc, 8064 "Multiple collisions"); 8065 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 8066 CTLFLAG_RD, &stats->latecol, 8067 "Late collisions"); 8068 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 8069 CTLFLAG_RD, &stats->colc, 8070 "Collision Count"); 8071 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 8072 CTLFLAG_RD, &stats->symerrs, 8073 "Symbol Errors"); 8074 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 8075 CTLFLAG_RD, &stats->sec, 8076 "Sequence Errors"); 8077 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 8078 CTLFLAG_RD, &stats->dc, 8079 "Defer Count"); 8080 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 8081 CTLFLAG_RD, &stats->mpc, 8082 "Missed Packets"); 8083 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 8084 CTLFLAG_RD, &stats->rlec, 8085 "Receive Length Errors"); 8086 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 8087 CTLFLAG_RD, &stats->rnbc, 8088 "Receive No Buffers"); 8089 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 8090 CTLFLAG_RD, &stats->ruc, 8091 "Receive Undersize"); 8092 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 8093 CTLFLAG_RD, &stats->rfc, 8094 "Fragmented Packets Received "); 8095 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 8096 CTLFLAG_RD, &stats->roc, 8097 "Oversized Packets Received"); 8098 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 8099 CTLFLAG_RD, &stats->rjc, 8100 "Recevied Jabber"); 8101 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 8102 CTLFLAG_RD, &stats->rxerrc, 8103 "Receive Errors"); 8104 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 8105 CTLFLAG_RD, &stats->crcerrs, 8106 "CRC errors"); 8107 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 8108 CTLFLAG_RD, &stats->algnerrc, 8109 "Alignment Errors"); 8110 /* On 82575 these are collision counts */ 8111 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 8112 CTLFLAG_RD, &stats->cexterr, 8113 "Collision/Carrier extension errors"); 8114 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 8115 CTLFLAG_RD, &stats->xonrxc, 8116 "XON Received"); 8117 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 8118 CTLFLAG_RD, &stats->xontxc, 8119 "XON Transmitted"); 8120 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 8121 CTLFLAG_RD, &stats->xoffrxc, 8122 "XOFF Received"); 8123 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 8124 CTLFLAG_RD, &stats->xofftxc, 8125 "XOFF Transmitted"); 8126 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 8127 CTLFLAG_RD, &stats->fcruc, 8128 "Unsupported Flow Control Received"); 8129 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 8130 CTLFLAG_RD, &stats->mgprc, 8131 "Management Packets Received"); 8132 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 8133 CTLFLAG_RD, &stats->mgpdc, 8134 "Management Packets Dropped"); 8135 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 8136 CTLFLAG_RD, &stats->mgptc, 8137 "Management Packets Transmitted"); 8138 8139 /* Packet Reception Stats */ 8140 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 8141 CTLFLAG_RD, &stats->tpr, 8142 "Total Packets Received "); 8143 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 8144 CTLFLAG_RD, &stats->gprc, 8145 "Good Packets Received"); 8146 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 8147 CTLFLAG_RD, &stats->bprc, 8148 "Broadcast Packets Received"); 8149 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 8150 CTLFLAG_RD, &stats->mprc, 8151 "Multicast Packets Received"); 8152 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 8153 CTLFLAG_RD, &stats->prc64, 8154 "64 byte frames received "); 8155 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 8156 CTLFLAG_RD, &stats->prc127, 8157 "65-127 byte frames received"); 8158 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 8159 CTLFLAG_RD, &stats->prc255, 8160 "128-255 byte frames received"); 8161 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 8162 CTLFLAG_RD, &stats->prc511, 8163 "256-511 byte frames received"); 8164 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 8165 CTLFLAG_RD, &stats->prc1023, 8166 "512-1023 byte frames received"); 8167 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 8168 CTLFLAG_RD, &stats->prc1522, 8169 "1023-1522 byte frames received"); 8170 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 8171 CTLFLAG_RD, &stats->gorc, 8172 "Good Octets Received"); 8173 8174 /* Packet Transmission Stats */ 8175 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 8176 CTLFLAG_RD, &stats->gotc, 8177 "Good Octets Transmitted"); 8178 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 8179 CTLFLAG_RD, &stats->tpt, 8180 "Total Packets Transmitted"); 8181 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 8182 CTLFLAG_RD, &stats->gptc, 8183 "Good Packets Transmitted"); 8184 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 8185 CTLFLAG_RD, &stats->bptc, 8186 "Broadcast Packets Transmitted"); 8187 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 8188 CTLFLAG_RD, &stats->mptc, 8189 "Multicast Packets Transmitted"); 8190 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 8191 CTLFLAG_RD, &stats->ptc64, 8192 "64 byte frames transmitted "); 8193 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 8194 CTLFLAG_RD, &stats->ptc127, 8195 "65-127 byte frames transmitted"); 8196 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 8197 CTLFLAG_RD, &stats->ptc255, 8198 "128-255 byte frames transmitted"); 8199 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 8200 CTLFLAG_RD, &stats->ptc511, 8201 "256-511 byte frames transmitted"); 8202 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 8203 CTLFLAG_RD, &stats->ptc1023, 8204 "512-1023 byte frames transmitted"); 8205 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 8206 CTLFLAG_RD, &stats->ptc1522, 8207 "1024-1522 byte frames transmitted"); 8208 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 8209 CTLFLAG_RD, &stats->tsctc, 8210 "TSO Contexts Transmitted"); 8211 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 8212 CTLFLAG_RD, &stats->tsctfc, 8213 "TSO Contexts Failed"); 8214 8215 /* Interrupt Stats */ 8216 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 8217 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 8218 int_list = SYSCTL_CHILDREN(int_node); 8219 8220 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 8221 CTLFLAG_RD, &stats->iac, 8222 "Interrupt Assertion Count"); 8223 8224 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 8225 CTLFLAG_RD, &stats->icrxptc, 8226 "Interrupt Cause Rx Pkt Timer Expire Count"); 8227 8228 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 8229 CTLFLAG_RD, &stats->icrxatc, 8230 "Interrupt Cause Rx Abs Timer Expire Count"); 8231 8232 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 8233 CTLFLAG_RD, &stats->ictxptc, 8234 "Interrupt Cause Tx Pkt Timer Expire Count"); 8235 8236 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 8237 CTLFLAG_RD, &stats->ictxatc, 8238 "Interrupt Cause Tx Abs Timer Expire Count"); 8239 8240 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 8241 CTLFLAG_RD, &stats->ictxqec, 8242 "Interrupt Cause Tx Queue Empty Count"); 8243 8244 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 8245 CTLFLAG_RD, &stats->ictxqmtc, 8246 "Interrupt Cause Tx Queue Min Thresh Count"); 8247 8248 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 8249 CTLFLAG_RD, &stats->icrxdmtc, 8250 "Interrupt Cause Rx Desc Min Thresh Count"); 8251 8252 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun", 8253 CTLFLAG_RD, &stats->icrxoc, 8254 "Interrupt Cause Receiver Overrun Count"); 8255 } 8256 8257 static void 8258 em_fw_version_locked(if_ctx_t ctx) 8259 { 8260 struct e1000_softc *sc = iflib_get_softc(ctx); 8261 struct e1000_hw *hw = &sc->hw; 8262 struct e1000_fw_version *fw_ver = &sc->fw_ver; 8263 uint16_t eep = 0; 8264 8265 /* 8266 * em_fw_version_locked() must run under the IFLIB_CTX_LOCK to meet 8267 * the NVM locking model, so we do it in em_if_attach_pre() and store 8268 * the info in the softc 8269 */ 8270 ASSERT_CTX_LOCK_HELD(hw); 8271 8272 *fw_ver = (struct e1000_fw_version){0}; 8273 8274 if (hw->mac.type >= igb_mac_min) { 8275 /* 8276 * Use the Shared Code for igb(4) 8277 */ 8278 e1000_get_fw_version(hw, fw_ver); 8279 } else { 8280 /* 8281 * Otherwise, EEPROM version should be present on (almost?) 8282 * all devices here 8283 */ 8284 if(e1000_read_nvm(hw, NVM_VERSION, 1, &eep)) { 8285 INIT_DEBUGOUT("can't get EEPROM version"); 8286 return; 8287 } 8288 8289 fw_ver->eep_major = (eep & NVM_MAJOR_MASK) >> NVM_MAJOR_SHIFT; 8290 fw_ver->eep_minor = (eep & NVM_MINOR_MASK) >> NVM_MINOR_SHIFT; 8291 fw_ver->eep_build = (eep & NVM_IMAGE_ID_MASK); 8292 } 8293 } 8294 8295 static void 8296 em_sbuf_fw_version(struct e1000_fw_version *fw_ver, struct sbuf *buf) 8297 { 8298 const char *space = ""; 8299 8300 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 8301 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 8302 fw_ver->eep_minor, fw_ver->eep_build); 8303 space = " "; 8304 } 8305 8306 if (fw_ver->invm_major || fw_ver->invm_minor || 8307 fw_ver->invm_img_type) { 8308 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 8309 space, fw_ver->invm_major, fw_ver->invm_minor, 8310 fw_ver->invm_img_type); 8311 space = " "; 8312 } 8313 8314 if (fw_ver->or_valid) { 8315 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 8316 space, fw_ver->or_major, fw_ver->or_build, 8317 fw_ver->or_patch); 8318 space = " "; 8319 } 8320 8321 if (fw_ver->etrack_id) 8322 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 8323 } 8324 8325 static void 8326 em_print_fw_version(struct e1000_softc *sc ) 8327 { 8328 device_t dev = sc->dev; 8329 struct sbuf *buf; 8330 int error = 0; 8331 8332 buf = sbuf_new_auto(); 8333 if (!buf) { 8334 device_printf(dev, "Could not allocate sbuf for output.\n"); 8335 return; 8336 } 8337 8338 em_sbuf_fw_version(&sc->fw_ver, buf); 8339 8340 error = sbuf_finish(buf); 8341 if (error) 8342 device_printf(dev, "Error finishing sbuf: %d\n", error); 8343 else if (sbuf_len(buf)) 8344 device_printf(dev, "%s\n", sbuf_data(buf)); 8345 8346 sbuf_delete(buf); 8347 } 8348 8349 static int 8350 em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 8351 { 8352 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8353 device_t dev = sc->dev; 8354 struct sbuf *buf; 8355 int error = 0; 8356 8357 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8358 if (!buf) { 8359 device_printf(dev, "Could not allocate sbuf for output.\n"); 8360 return (ENOMEM); 8361 } 8362 8363 em_sbuf_fw_version(&sc->fw_ver, buf); 8364 8365 error = sbuf_finish(buf); 8366 if (error) 8367 device_printf(dev, "Error finishing sbuf: %d\n", error); 8368 8369 sbuf_delete(buf); 8370 8371 return (0); 8372 } 8373 8374 /********************************************************************** 8375 * 8376 * This routine provides a way to dump out the adapter eeprom, 8377 * often a useful debug/service tool. This only dumps the first 8378 * 32 words, stuff that matters is in that extent. 8379 * 8380 **********************************************************************/ 8381 static int 8382 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 8383 { 8384 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8385 int error; 8386 int result; 8387 8388 result = -1; 8389 error = sysctl_handle_int(oidp, &result, 0, req); 8390 8391 if (error || !req->newptr) 8392 return (error); 8393 8394 /* 8395 * This value will cause a hex dump of the 8396 * first 32 16-bit words of the EEPROM to 8397 * the screen. 8398 */ 8399 if (result == 1) 8400 em_print_nvm_info(sc); 8401 8402 return (error); 8403 } 8404 8405 static void 8406 em_print_nvm_info(struct e1000_softc *sc) 8407 { 8408 struct e1000_hw *hw = &sc->hw; 8409 struct sx *iflib_ctx_lock = iflib_ctx_lock_get(sc->ctx); 8410 u16 eeprom_data; 8411 int i, j, row = 0; 8412 8413 /* Its a bit crude, but it gets the job done */ 8414 printf("\nInterface EEPROM Dump:\n"); 8415 printf("Offset\n0x0000 "); 8416 8417 /* We rely on the IFLIB_CTX_LOCK as part of NVM locking model */ 8418 sx_xlock(iflib_ctx_lock); 8419 ASSERT_CTX_LOCK_HELD(hw); 8420 for (i = 0, j = 0; i < 32; i++, j++) { 8421 if (j == 8) { /* Make the offset block */ 8422 j = 0; ++row; 8423 printf("\n0x00%x0 ",row); 8424 } 8425 eeprom_data = 0; 8426 if (e1000_read_nvm(hw, i, 1, &eeprom_data) != 8427 E1000_SUCCESS) { 8428 printf("\nNVM read failed at offset %#x\n", i); 8429 break; 8430 } 8431 printf("%04x ", eeprom_data); 8432 } 8433 sx_xunlock(iflib_ctx_lock); 8434 printf("\n"); 8435 } 8436 8437 static int 8438 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 8439 { 8440 struct em_int_delay_info *info; 8441 struct e1000_softc *sc; 8442 u32 regval; 8443 int error, usecs, ticks; 8444 8445 info = (struct em_int_delay_info *) arg1; 8446 usecs = info->value; 8447 error = sysctl_handle_int(oidp, &usecs, 0, req); 8448 if (error != 0 || req->newptr == NULL) 8449 return (error); 8450 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 8451 return (EINVAL); 8452 info->value = usecs; 8453 ticks = EM_USECS_TO_TICKS(usecs); 8454 8455 sc = info->sc; 8456 8457 regval = E1000_READ_OFFSET(&sc->hw, info->offset); 8458 regval = (regval & ~0xffff) | (ticks & 0xffff); 8459 /* Handle a few special cases. */ 8460 switch (info->offset) { 8461 case E1000_RDTR: 8462 break; 8463 case E1000_TIDV: 8464 if (ticks == 0) { 8465 sc->txd_cmd &= ~E1000_TXD_CMD_IDE; 8466 /* Don't write 0 into the TIDV register. */ 8467 regval++; 8468 } else 8469 sc->txd_cmd |= E1000_TXD_CMD_IDE; 8470 break; 8471 } 8472 E1000_WRITE_OFFSET(&sc->hw, info->offset, regval); 8473 return (0); 8474 } 8475 8476 static int 8477 em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 8478 { 8479 struct e1000_softc *sc; 8480 u32 reg, val, shift; 8481 int error, mask; 8482 8483 sc = oidp->oid_arg1; 8484 switch (oidp->oid_arg2) { 8485 case 0: 8486 reg = E1000_DTXTCPFLGL; 8487 shift = 0; 8488 break; 8489 case 1: 8490 reg = E1000_DTXTCPFLGL; 8491 shift = 16; 8492 break; 8493 case 2: 8494 reg = E1000_DTXTCPFLGH; 8495 shift = 0; 8496 break; 8497 default: 8498 return (EINVAL); 8499 break; 8500 } 8501 val = E1000_READ_REG(&sc->hw, reg); 8502 mask = (val >> shift) & 0xfff; 8503 error = sysctl_handle_int(oidp, &mask, 0, req); 8504 if (error != 0 || req->newptr == NULL) 8505 return (error); 8506 if (mask < 0 || mask > 0xfff) 8507 return (EINVAL); 8508 val = (val & ~(0xfff << shift)) | (mask << shift); 8509 E1000_WRITE_REG(&sc->hw, reg, val); 8510 return (0); 8511 } 8512 8513 static void 8514 em_add_int_delay_sysctl(struct e1000_softc *sc, const char *name, 8515 const char *description, struct em_int_delay_info *info, int offset, 8516 int value) 8517 { 8518 info->sc = sc; 8519 info->offset = offset; 8520 info->value = value; 8521 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 8522 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 8523 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 8524 info, 0, em_sysctl_int_delay, "I", description); 8525 } 8526 8527 /* 8528 * Set flow control using sysctl: 8529 * Flow control values: 8530 * 0 - off 8531 * 1 - rx pause 8532 * 2 - tx pause 8533 * 3 - full 8534 */ 8535 static int 8536 em_set_flowcntl(SYSCTL_HANDLER_ARGS) 8537 { 8538 int error, input; 8539 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8540 8541 input = sc->fc; 8542 error = sysctl_handle_int(oidp, &input, 0, req); 8543 8544 if ((error) || (req->newptr == NULL)) 8545 return (error); 8546 8547 switch (input) { 8548 case e1000_fc_rx_pause: 8549 case e1000_fc_tx_pause: 8550 case e1000_fc_full: 8551 case e1000_fc_none: 8552 break; 8553 default: 8554 return (EINVAL); 8555 } 8556 8557 if (input == sc->fc) /* no change? */ 8558 return (0); 8559 8560 /* 8561 * The mode is advertised to the link partner and also decides the 8562 * pause thresholds and per-queue drop, so it takes a reinit to apply. 8563 */ 8564 sc->fc = input; 8565 em_sysctl_request_reinit(sc); 8566 return (0); 8567 } 8568 8569 static void 8570 em_sysctl_request_reinit(struct e1000_softc *sc) 8571 { 8572 if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) 8573 return; 8574 8575 iflib_request_reset(sc->ctx); 8576 iflib_admin_intr_deferred(sc->ctx); 8577 } 8578 8579 /* 8580 * Manage DMA Coalesce: 8581 * Control values: 8582 * 0/1 - off/on 8583 * Legal timer values are: 8584 * 250,500,1000-10000 in thousands 8585 */ 8586 static int 8587 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) 8588 { 8589 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8590 int error; 8591 8592 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 8593 8594 if ((error) || (req->newptr == NULL)) 8595 return (error); 8596 8597 switch (sc->dmac) { 8598 case 0: 8599 /* Disabling */ 8600 break; 8601 case 1: /* Just enable and use default */ 8602 sc->dmac = 1000; 8603 break; 8604 case 250: 8605 case 500: 8606 case 1000: 8607 case 2000: 8608 case 3000: 8609 case 4000: 8610 case 5000: 8611 case 6000: 8612 case 7000: 8613 case 8000: 8614 case 9000: 8615 case 10000: 8616 /* Legal values - allow */ 8617 break; 8618 default: 8619 /* Do nothing, illegal value */ 8620 sc->dmac = 0; 8621 return (EINVAL); 8622 } 8623 /* Reinit the interface */ 8624 em_sysctl_request_reinit(sc); 8625 return (error); 8626 } 8627 8628 /* 8629 * Manage Energy Efficient Ethernet: 8630 * Control values: 8631 * 0/1 - enabled/disabled 8632 */ 8633 static int 8634 em_sysctl_eee(SYSCTL_HANDLER_ARGS) 8635 { 8636 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8637 int error, value; 8638 8639 if (sc->hw.mac.type < igb_mac_min) 8640 value = sc->hw.dev_spec.ich8lan.eee_disable; 8641 else 8642 value = sc->hw.dev_spec._82575.eee_disable; 8643 error = sysctl_handle_int(oidp, &value, 0, req); 8644 if (error || req->newptr == NULL) 8645 return (error); 8646 if (sc->hw.mac.type < igb_mac_min) 8647 sc->hw.dev_spec.ich8lan.eee_disable = (value != 0); 8648 else 8649 sc->hw.dev_spec._82575.eee_disable = (value != 0); 8650 em_sysctl_request_reinit(sc); 8651 8652 return (0); 8653 } 8654 8655 static int 8656 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 8657 { 8658 struct e1000_softc *sc; 8659 int error; 8660 int result; 8661 8662 result = -1; 8663 error = sysctl_handle_int(oidp, &result, 0, req); 8664 8665 if (error || !req->newptr) 8666 return (error); 8667 8668 if (result == 1) { 8669 sc = (struct e1000_softc *) arg1; 8670 em_print_debug_info(sc); 8671 } 8672 8673 return (error); 8674 } 8675 8676 static int 8677 em_get_rs(SYSCTL_HANDLER_ARGS) 8678 { 8679 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8680 int error; 8681 int result; 8682 8683 result = 0; 8684 error = sysctl_handle_int(oidp, &result, 0, req); 8685 8686 if (error || !req->newptr || result != 1) 8687 return (error); 8688 em_dump_rs(sc); 8689 8690 return (error); 8691 } 8692 8693 static void 8694 em_if_debug(if_ctx_t ctx) 8695 { 8696 em_dump_rs(iflib_get_softc(ctx)); 8697 } 8698 8699 /* 8700 * This routine is meant to be fluid, add whatever is 8701 * needed for debugging a problem. -jfv 8702 */ 8703 static void 8704 em_print_debug_info(struct e1000_softc *sc) 8705 { 8706 device_t dev = iflib_get_dev(sc->ctx); 8707 struct tx_ring *txr; 8708 struct rx_ring *rxr; 8709 8710 if (sc->tx_queues == NULL || sc->rx_queues == NULL) { 8711 device_printf(dev, "queue state is unavailable\n"); 8712 return; 8713 } 8714 device_printf(dev, "iflib software admission: %s\n", 8715 iflib_is_running(sc->ctx) ? "open" : "closed"); 8716 8717 for (int i = 0; i < sc->tx_num_queues; i++) { 8718 txr = &sc->tx_queues[i].txr; 8719 device_printf(dev, "TX Queue %d ------\n", i); 8720 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 8721 E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)), 8722 E1000_READ_REG(&sc->hw, E1000_TDT(txr->me))); 8723 8724 } 8725 for (int j = 0; j < sc->rx_num_queues; j++) { 8726 rxr = &sc->rx_queues[j].rxr; 8727 device_printf(dev, "RX Queue %d ------\n", j); 8728 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 8729 E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)), 8730 E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me))); 8731 } 8732 } 8733 8734 /* 8735 * 82574 only: 8736 * Write a new value to the EEPROM increasing the number of MSI-X 8737 * vectors from 3 to 5, for proper multiqueue support. 8738 */ 8739 static void 8740 em_enable_vectors_82574(if_ctx_t ctx) 8741 { 8742 struct e1000_softc *sc = iflib_get_softc(ctx); 8743 struct e1000_hw *hw = &sc->hw; 8744 device_t dev = iflib_get_dev(ctx); 8745 u16 edata; 8746 8747 e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8748 if (bootverbose) 8749 device_printf(dev, "EM_NVM_PCIE_CTRL = %#06x\n", edata); 8750 if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { 8751 device_printf(dev, "Writing to eeprom: increasing " 8752 "reported MSI-X vectors from 3 to 5...\n"); 8753 edata &= ~(EM_NVM_MSIX_N_MASK); 8754 edata |= 4 << EM_NVM_MSIX_N_SHIFT; 8755 e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8756 e1000_update_nvm_checksum(hw); 8757 device_printf(dev, "Writing to eeprom: done\n"); 8758 } 8759 } 8760