1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2024 Kevin Bowling <kbowling@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 #include "if_igb_iov.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #define em_mac_min e1000_82571 36 #define igb_mac_min e1000_82575 37 38 /********************************************************************* 39 * Driver version: 40 *********************************************************************/ 41 static const char em_driver_version[] = "7.7.8-fbsd"; 42 static const char igb_driver_version[] = "2.5.28-fbsd"; 43 44 /********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into e1000_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54 static const pci_vendor_info_t em_vendor_info_array[] = 55 { 56 /* Intel(R) - lem-class legacy devices */ 57 PVID(0x8086, E1000_DEV_ID_82540EM, 58 "Intel(R) Legacy PRO/1000 MT 82540EM"), 59 PVID(0x8086, E1000_DEV_ID_82540EM_LOM, 60 "Intel(R) Legacy PRO/1000 MT 82540EM (LOM)"), 61 PVID(0x8086, E1000_DEV_ID_82540EP, 62 "Intel(R) Legacy PRO/1000 MT 82540EP"), 63 PVID(0x8086, E1000_DEV_ID_82540EP_LOM, 64 "Intel(R) Legacy PRO/1000 MT 82540EP (LOM)"), 65 PVID(0x8086, E1000_DEV_ID_82540EP_LP, 66 "Intel(R) Legacy PRO/1000 MT 82540EP (Mobile)"), 67 68 PVID(0x8086, E1000_DEV_ID_82541EI, 69 "Intel(R) Legacy PRO/1000 MT 82541EI (Copper)"), 70 PVID(0x8086, E1000_DEV_ID_82541ER, 71 "Intel(R) Legacy PRO/1000 82541ER"), 72 PVID(0x8086, E1000_DEV_ID_82541ER_LOM, 73 "Intel(R) Legacy PRO/1000 MT 82541ER"), 74 PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, 75 "Intel(R) Legacy PRO/1000 MT 82541EI (Mobile)"), 76 PVID(0x8086, E1000_DEV_ID_82541GI, 77 "Intel(R) Legacy PRO/1000 MT 82541GI"), 78 PVID(0x8086, E1000_DEV_ID_82541GI_LF, 79 "Intel(R) Legacy PRO/1000 GT 82541PI"), 80 PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, 81 "Intel(R) Legacy PRO/1000 MT 82541GI (Mobile)"), 82 83 PVID(0x8086, E1000_DEV_ID_82542, 84 "Intel(R) Legacy PRO/1000 82542 (Fiber)"), 85 86 PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, 87 "Intel(R) Legacy PRO/1000 F 82543GC (Fiber)"), 88 PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, 89 "Intel(R) Legacy PRO/1000 T 82543GC (Copper)"), 90 91 PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, 92 "Intel(R) Legacy PRO/1000 XT 82544EI (Copper)"), 93 PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, 94 "Intel(R) Legacy PRO/1000 XF 82544EI (Fiber)"), 95 PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, 96 "Intel(R) Legacy PRO/1000 T 82544GC (Copper)"), 97 PVID(0x8086, E1000_DEV_ID_82544GC_LOM, 98 "Intel(R) Legacy PRO/1000 XT 82544GC (LOM)"), 99 100 PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, 101 "Intel(R) Legacy PRO/1000 MT 82545EM (Copper)"), 102 PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, 103 "Intel(R) Legacy PRO/1000 MF 82545EM (Fiber)"), 104 PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, 105 "Intel(R) Legacy PRO/1000 MT 82545GM (Copper)"), 106 PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, 107 "Intel(R) Legacy PRO/1000 MF 82545GM (Fiber)"), 108 PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, 109 "Intel(R) Legacy PRO/1000 MB 82545GM (SERDES)"), 110 111 PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, 112 "Intel(R) Legacy PRO/1000 MT 82546EB (Copper)"), 113 PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, 114 "Intel(R) Legacy PRO/1000 MF 82546EB (Fiber)"), 115 PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, 116 "Intel(R) Legacy PRO/1000 MT 82546EB (Quad Copper"), 117 PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, 118 "Intel(R) Legacy PRO/1000 MT 82546GB (Copper)"), 119 PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, 120 "Intel(R) Legacy PRO/1000 MF 82546GB (Fiber)"), 121 PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, 122 "Intel(R) Legacy PRO/1000 MB 82546GB (SERDES)"), 123 PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, 124 "Intel(R) Legacy PRO/1000 P 82546GB (PCIe)"), 125 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, 126 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 127 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 128 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 129 130 PVID(0x8086, E1000_DEV_ID_82547EI, 131 "Intel(R) Legacy PRO/1000 CT 82547EI"), 132 PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, 133 "Intel(R) Legacy PRO/1000 CT 82547EI (Mobile)"), 134 PVID(0x8086, E1000_DEV_ID_82547GI, 135 "Intel(R) Legacy PRO/1000 CT 82547GI"), 136 137 /* Intel(R) - em-class devices */ 138 PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, 139 "Intel(R) PRO/1000 PT 82571EB/82571GB (Copper)"), 140 PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, 141 "Intel(R) PRO/1000 PF 82571EB/82571GB (Fiber)"), 142 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, 143 "Intel(R) PRO/1000 PB 82571EB (SERDES)"), 144 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 145 "Intel(R) PRO/1000 82571EB (Dual Mezzanine)"), 146 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 147 "Intel(R) PRO/1000 82571EB (Quad Mezzanine)"), 148 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 149 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 150 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 151 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 152 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 153 "Intel(R) PRO/1000 PF 82571EB (Quad Fiber)"), 154 PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 155 "Intel(R) PRO/1000 PT 82571PT (Quad Copper)"), 156 PVID(0x8086, E1000_DEV_ID_82572EI, 157 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 158 PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, 159 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 160 PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, 161 "Intel(R) PRO/1000 PF 82572EI (Fiber)"), 162 PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, 163 "Intel(R) PRO/1000 82572EI (SERDES)"), 164 PVID(0x8086, E1000_DEV_ID_82573E, 165 "Intel(R) PRO/1000 82573E (Copper)"), 166 PVID(0x8086, E1000_DEV_ID_82573E_IAMT, 167 "Intel(R) PRO/1000 82573E AMT (Copper)"), 168 PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 82573L"), 169 PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) 82583V"), 170 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 171 "Intel(R) 80003ES2LAN (Copper)"), 172 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 173 "Intel(R) 80003ES2LAN (SERDES)"), 174 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 175 "Intel(R) 80003ES2LAN (Dual Copper)"), 176 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 177 "Intel(R) 80003ES2LAN (Dual SERDES)"), 178 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, 179 "Intel(R) 82566MM ICH8 AMT (Mobile)"), 180 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) 82566DM ICH8 AMT"), 181 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) 82566DC ICH8"), 182 PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) 82562V ICH8"), 183 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) 82562GT ICH8"), 184 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) 82562G ICH8"), 185 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) 82566MC ICH8"), 186 PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) 82567V-3 ICH8"), 187 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, 188 "Intel(R) 82567LM ICH9 AMT"), 189 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, 190 "Intel(R) 82566DM-2 ICH9 AMT"), 191 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) 82566DC-2 ICH9"), 192 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) 82567LF ICH9"), 193 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) 82567V ICH9"), 194 PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) 82562V-2 ICH9"), 195 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) 82562GT-2 ICH9"), 196 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) 82562G-2 ICH9"), 197 PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) 82567LM-4 ICH9"), 198 PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) Gigabit CT 82574L"), 199 PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) 82574L-Apple"), 200 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) 82567LM-2 ICH10"), 201 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) 82567LF-2 ICH10"), 202 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) 82567V-2 ICH10"), 203 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) 82567LM-3 ICH10"), 204 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) 82567LF-3 ICH10"), 205 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) 82567V-4 ICH10"), 206 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) 82577LM"), 207 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) 82577LC"), 208 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) 82578DM"), 209 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) 82578DC"), 210 PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) 82579LM"), 211 PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) 82579V"), 212 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) I217-LM LPT"), 213 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) I217-V LPT"), 214 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, 215 "Intel(R) I218-LM LPTLP"), 216 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) I218-V LPTLP"), 217 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) I218-LM (2)"), 218 PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) I218-V (2)"), 219 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) I218-LM (3)"), 220 PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) I218-V (3)"), 221 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) I219-LM SPT"), 222 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) I219-V SPT"), 223 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, 224 "Intel(R) I219-LM SPT-H(2)"), 225 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, 226 "Intel(R) I219-V SPT-H(2)"), 227 PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, 228 "Intel(R) I219-LM LBG(3)"), 229 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, 230 "Intel(R) I219-LM SPT(4)"), 231 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) I219-V SPT(4)"), 232 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, 233 "Intel(R) I219-LM SPT(5)"), 234 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) I219-V SPT(5)"), 235 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM6, 236 "Intel(R) I219-LM CNP(6)"), 237 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V6, "Intel(R) I219-V CNP(6)"), 238 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM7, 239 "Intel(R) I219-LM CNP(7)"), 240 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V7, "Intel(R) I219-V CNP(7)"), 241 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM8, 242 "Intel(R) I219-LM ICP(8)"), 243 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V8, "Intel(R) I219-V ICP(8)"), 244 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM9, 245 "Intel(R) I219-LM ICP(9)"), 246 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V9, "Intel(R) I219-V ICP(9)"), 247 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM10, 248 "Intel(R) I219-LM CMP(10)"), 249 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V10, 250 "Intel(R) I219-V CMP(10)"), 251 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM11, 252 "Intel(R) I219-LM CMP(11)"), 253 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V11, 254 "Intel(R) I219-V CMP(11)"), 255 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM12, 256 "Intel(R) I219-LM CMP(12)"), 257 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V12, 258 "Intel(R) I219-V CMP(12)"), 259 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM13, 260 "Intel(R) I219-LM TGP(13)"), 261 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V13, 262 "Intel(R) I219-V TGP(13)"), 263 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM14, 264 "Intel(R) I219-LM TGP(14)"), 265 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V14, 266 "Intel(R) I219-V GTP(14)"), 267 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM15, 268 "Intel(R) I219-LM TGP(15)"), 269 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V15, 270 "Intel(R) I219-V TGP(15)"), 271 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM16, 272 "Intel(R) I219-LM ADL(16)"), 273 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V16, 274 "Intel(R) I219-V ADL(16)"), 275 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM17, 276 "Intel(R) I219-LM ADL(17)"), 277 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V17, 278 "Intel(R) I219-V ADL(17)"), 279 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_LM18, 280 "Intel(R) I219-LM MTP(18)"), 281 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_V18, 282 "Intel(R) I219-V MTP(18)"), 283 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM19, 284 "Intel(R) I219-LM ADL(19)"), 285 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V19, 286 "Intel(R) I219-V ADL(19)"), 287 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM20, 288 "Intel(R) I219-LM LNL(20)"), 289 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V20, 290 "Intel(R) I219-V LNL(20)"), 291 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM21, 292 "Intel(R) I219-LM LNL(21)"), 293 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V21, 294 "Intel(R) I219-V LNL(21)"), 295 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM22, 296 "Intel(R) I219-LM RPL(22)"), 297 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V22, 298 "Intel(R) I219-V RPL(22)"), 299 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM23, 300 "Intel(R) I219-LM RPL(23)"), 301 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V23, 302 "Intel(R) I219-V RPL(23)"), 303 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_LM24, 304 "Intel(R) I219-LM ARL(24)"), 305 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_V24, 306 "Intel(R) I219-V ARL(24)"), 307 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM25, 308 "Intel(R) I219-LM PTP(25)"), 309 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V25, 310 "Intel(R) I219-V PTP(25)"), 311 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM26, 312 "Intel(R) I219-LM PTP(26)"), 313 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V26, 314 "Intel(R) I219-V PTP(26)"), 315 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM27, 316 "Intel(R) I219-LM PTP(27)"), 317 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V27, 318 "Intel(R) I219-V PTP(27)"), 319 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_LM29, 320 "Intel(R) I219-LM NVL(29)"), 321 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_V29, 322 "Intel(R) I219-V NVL(29)"), 323 /* required last entry */ 324 PVID_END 325 }; 326 327 static const pci_vendor_info_t igb_vendor_info_array[] = 328 { 329 /* Intel(R) - igb-class devices */ 330 PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, 331 "Intel(R) PRO/1000 82575EB (Copper)"), 332 PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 333 "Intel(R) PRO/1000 82575EB (SERDES)"), 334 PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 335 "Intel(R) PRO/1000 VT 82575GB (Quad Copper)"), 336 PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 82576"), 337 PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 82576NS"), 338 PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, 339 "Intel(R) PRO/1000 82576NS (SERDES)"), 340 PVID(0x8086, E1000_DEV_ID_82576_FIBER, 341 "Intel(R) PRO/1000 EF 82576 (Dual Fiber)"), 342 PVID(0x8086, E1000_DEV_ID_82576_SERDES, 343 "Intel(R) PRO/1000 82576 (Dual SERDES)"), 344 PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 345 "Intel(R) PRO/1000 ET 82576 (Quad SERDES)"), 346 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 347 "Intel(R) PRO/1000 ET 82576 (Quad Copper)"), 348 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 349 "Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"), 350 PVID(0x8086, E1000_DEV_ID_82580_COPPER, 351 "Intel(R) I340 82580 (Copper)"), 352 PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"), 353 PVID(0x8086, E1000_DEV_ID_82580_SERDES, 354 "Intel(R) I340 82580 (SERDES)"), 355 PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) I340 82580 (SGMII)"), 356 PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 357 "Intel(R) I340-T2 82580 (Dual Copper)"), 358 PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, 359 "Intel(R) I340-F4 82580 (Quad Fiber)"), 360 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, 361 "Intel(R) DH89XXCC (SERDES)"), 362 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, 363 "Intel(R) I347-AT4 DH89XXCC"), 364 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) DH89XXCC (SFP)"), 365 PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, 366 "Intel(R) DH89XXCC (Backplane)"), 367 PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) I350 (Copper)"), 368 PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"), 369 PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"), 370 PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"), 371 PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"), 372 PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, 373 "Intel(R) I210 IT (Copper)"), 374 PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) I210 (OEM)"), 375 PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, 376 "Intel(R) I210 Flashless (Copper)"), 377 PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, 378 "Intel(R) I210 Flashless (SERDES)"), 379 PVID(0x8086, E1000_DEV_ID_I210_SGMII_FLASHLESS, 380 "Intel(R) I210 Flashless (SGMII)"), 381 PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) I210 (Fiber)"), 382 PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) I210 (SERDES)"), 383 PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) I210 (SGMII)"), 384 PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) I211 (Copper)"), 385 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 386 "Intel(R) I354 (1.0 GbE Backplane)"), 387 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 388 "Intel(R) I354 (2.5 GbE Backplane)"), 389 PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) I354 (SGMII)"), 390 /* required last entry */ 391 PVID_END 392 }; 393 394 static const pci_vendor_info_t igbv_vendor_info_array[] = { 395 PVID(0x8086, E1000_DEV_ID_82576_VF, 396 "Intel(R) PRO/1000 82576 Virtual Function"), 397 PVID(0x8086, E1000_DEV_ID_82576_VF_HV, 398 "Intel(R) PRO/1000 82576 Hyper-V Virtual Function"), 399 PVID(0x8086, E1000_DEV_ID_I350_VF, 400 "Intel(R) I350 Virtual Function"), 401 PVID(0x8086, E1000_DEV_ID_I350_VF_HV, 402 "Intel(R) I350 Hyper-V Virtual Function"), 403 PVID_END 404 }; 405 406 /********************************************************************* 407 * Function prototypes 408 *********************************************************************/ 409 static void *em_register(device_t); 410 static void *igb_register(device_t); 411 static void *igbv_register(device_t); 412 static int igb_device_attach(device_t); 413 #ifdef PCI_IOV 414 static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *); 415 static void igb_device_iov_uninit(device_t); 416 #endif 417 static int em_if_detach(if_ctx_t); 418 static int em_if_shutdown(if_ctx_t); 419 static int em_if_suspend(if_ctx_t); 420 static int em_if_resume(if_ctx_t); 421 422 static int em_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 423 int); 424 static int em_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 425 int); 426 static void em_if_queues_free(if_ctx_t); 427 428 static uint64_t em_if_get_vf_counter(if_ctx_t, ift_counter); 429 static uint64_t em_if_get_counter(if_ctx_t, ift_counter); 430 static void em_if_init(if_ctx_t); 431 static void em_if_stop(if_ctx_t); 432 static void em_fence_pci_busmaster(struct e1000_softc *); 433 static int em_enable_pci_busmaster(struct e1000_softc *); 434 static void em_if_media_status(if_ctx_t, struct ifmediareq *); 435 static int em_if_media_change(if_ctx_t); 436 static int em_if_mtu_set(if_ctx_t, uint32_t); 437 static void em_if_timer(if_ctx_t, uint16_t); 438 static void em_if_vlan_register(if_ctx_t, u16); 439 static void em_if_vlan_unregister(if_ctx_t, u16); 440 static bool em_if_needs_restart(if_ctx_t, enum iflib_restart_event); 441 442 static void em_identify_hardware(if_ctx_t); 443 static int em_allocate_pci_resources(if_ctx_t); 444 static void em_free_pci_resources(if_ctx_t); 445 static int em_reset(if_ctx_t); 446 static int em_setup_interface(if_ctx_t); 447 static int em_setup_msix(if_ctx_t); 448 449 static void em_initialize_transmit_unit(if_ctx_t); 450 static void em_initialize_receive_unit(if_ctx_t); 451 452 static void em_if_intr_enable(if_ctx_t); 453 static void em_if_intr_disable(if_ctx_t); 454 static void igb_if_intr_enable(if_ctx_t); 455 static void igb_if_intr_disable(if_ctx_t); 456 static int em_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 457 static int em_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 458 static int igb_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 459 static int igb_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 460 static void em_handle_fatal_error_intr(struct e1000_softc *, u32); 461 static bool em_handle_fatal_error_admin(struct e1000_softc *); 462 static u32 igb_device_reset_intr_mask(struct e1000_softc *); 463 static bool igb_device_reset_pending(struct e1000_softc *); 464 static bool igb_handle_device_reset(struct e1000_softc *, u32); 465 static void igb_prepare_device_reset(struct e1000_softc *); 466 static bool igb_finish_device_reset(struct e1000_softc *, u32); 467 static void em_prepare_fatal_error_reset(struct e1000_softc *); 468 static void em_finish_fatal_error_reset(struct e1000_softc *); 469 static void em_configure_peind_memory_errors(struct e1000_softc *); 470 static void em_configure_82575_memory_errors(struct e1000_softc *); 471 static void em_configure_82580_memory_errors(struct e1000_softc *); 472 static void em_update_82580_ecc_stats(struct e1000_softc *, u32, u32, 473 u32); 474 static void em_if_multi_set(if_ctx_t); 475 static void em_if_update_admin_status(if_ctx_t); 476 static void em_if_debug(if_ctx_t); 477 static void em_initialize_vf_stats(struct e1000_softc *); 478 static void em_rebase_vf_stats(struct e1000_softc *); 479 static void em_update_vf_stats_counters(struct e1000_softc *); 480 static void em_add_hw_stats(struct e1000_softc *); 481 static bool em_mac_has_eee(enum e1000_mac_type); 482 static int em_if_set_promisc(if_ctx_t, int); 483 static bool em_if_defer_promisc(struct e1000_softc *); 484 static bool em_if_vlan_filter_capable(if_ctx_t); 485 static bool em_if_vlan_filter_used(if_ctx_t); 486 static void em_if_vlan_filter_enable(struct e1000_softc *); 487 static void em_if_vlan_filter_disable(struct e1000_softc *); 488 static void em_if_vlan_filter_write(struct e1000_softc *, int); 489 static void em_setup_vlan_hw_support(if_ctx_t ctx); 490 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 491 static void em_print_nvm_info(struct e1000_softc *); 492 static void em_fw_version_locked(if_ctx_t); 493 static void em_sbuf_fw_version(struct e1000_fw_version *, struct sbuf *); 494 static void em_print_fw_version(struct e1000_softc *); 495 static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 496 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 497 static int em_get_rs(SYSCTL_HANDLER_ARGS); 498 static void em_print_debug_info(struct e1000_softc *); 499 static void em_newitr(struct e1000_softc *, struct em_rx_queue *, 500 struct rx_ring *); 501 static bool em_automask_tso(if_ctx_t); 502 static int em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 503 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 504 static void em_add_int_delay_sysctl(struct e1000_softc *, const char *, 505 const char *, struct em_int_delay_info *, int, int); 506 /* Management and WOL Support */ 507 static void em_init_manageability(struct e1000_softc *); 508 static void em_release_manageability(struct e1000_softc *); 509 static void em_get_hw_control(struct e1000_softc *); 510 static void em_release_hw_control(struct e1000_softc *); 511 static void em_get_wakeup(if_ctx_t); 512 static void em_fill_wakeup_mta(struct e1000_hw *); 513 static int em_enable_wakeup(if_ctx_t); 514 static void em_configure_sx_low_power(struct e1000_softc *, u32); 515 static int em_enable_phy_wakeup(struct e1000_softc *, u32); 516 static int em_disable_phy_wakeup(struct e1000_softc *, u16 *); 517 static void em_power_up_wakeup_link(struct e1000_softc *); 518 static void em_power_down_wakeup_link(struct e1000_softc *); 519 static void em_disable_aspm(struct e1000_softc *); 520 521 int em_intr(void *); 522 523 enum em_fatal_error_state { 524 EM_FATAL_ERROR_NONE, 525 EM_FATAL_ERROR_CAPTURING, 526 EM_FATAL_ERROR_DETECTED, 527 EM_FATAL_ERROR_RESET_REQUESTED, 528 EM_FATAL_ERROR_RESET_PREPARED, 529 }; 530 531 enum igb_device_reset_state { 532 IGB_DEVICE_RESET_NONE, 533 IGB_DEVICE_RESET_DETECTED, 534 IGB_DEVICE_RESET_REQUESTED, 535 IGB_DEVICE_RESET_PREPARED, 536 }; 537 538 #define IGB_DEVICE_RESET_TIMEOUT_MS 100 539 540 /* MSI-X handlers */ 541 static int em_if_msix_intr_assign(if_ctx_t, int); 542 static int em_msix_link(void *); 543 static void em_handle_link(void *); 544 545 static void em_enable_vectors_82574(if_ctx_t); 546 547 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); 548 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); 549 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); 550 static void em_if_led_func(if_ctx_t, int); 551 552 static int em_get_regs(SYSCTL_HANDLER_ARGS); 553 static void lem_smartspeed(struct e1000_softc *); 554 static void igb_configure_queues(struct e1000_softc *); 555 static void igb_initialize_interrupt_rate(struct e1000_softc *); 556 static void em_flush_desc_rings(struct e1000_softc *); 557 558 559 /********************************************************************* 560 * FreeBSD Device Interface Entry Points 561 *********************************************************************/ 562 static device_method_t em_methods[] = { 563 /* Device interface */ 564 DEVMETHOD(device_register, em_register), 565 DEVMETHOD(device_probe, iflib_device_probe), 566 DEVMETHOD(device_attach, iflib_device_attach), 567 DEVMETHOD(device_detach, iflib_device_detach), 568 DEVMETHOD(device_shutdown, iflib_device_shutdown), 569 DEVMETHOD(device_suspend, iflib_device_suspend), 570 DEVMETHOD(device_resume, iflib_device_resume), 571 DEVMETHOD_END 572 }; 573 574 static device_method_t igb_methods[] = { 575 /* Device interface */ 576 DEVMETHOD(device_register, igb_register), 577 DEVMETHOD(device_probe, iflib_device_probe), 578 DEVMETHOD(device_attach, igb_device_attach), 579 DEVMETHOD(device_detach, iflib_device_detach), 580 DEVMETHOD(device_shutdown, iflib_device_shutdown), 581 DEVMETHOD(device_suspend, iflib_device_suspend), 582 DEVMETHOD(device_resume, iflib_device_resume), 583 #ifdef PCI_IOV 584 DEVMETHOD(pci_iov_init, igb_device_iov_init), 585 DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit), 586 DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), 587 #endif 588 DEVMETHOD_END 589 }; 590 591 static device_method_t igbv_methods[] = { 592 /* Device interface */ 593 DEVMETHOD(device_register, igbv_register), 594 DEVMETHOD(device_probe, iflib_device_probe), 595 DEVMETHOD(device_attach, iflib_device_attach), 596 DEVMETHOD(device_detach, iflib_device_detach), 597 DEVMETHOD(device_shutdown, iflib_device_shutdown), 598 DEVMETHOD(device_suspend, iflib_device_suspend), 599 DEVMETHOD(device_resume, iflib_device_resume), 600 DEVMETHOD_END 601 }; 602 603 604 static driver_t em_driver = { 605 "em", em_methods, sizeof(struct e1000_softc), 606 }; 607 608 DRIVER_MODULE(em, pci, em_driver, 0, 0); 609 610 MODULE_DEPEND(em, pci, 1, 1, 1); 611 MODULE_DEPEND(em, ether, 1, 1, 1); 612 MODULE_DEPEND(em, iflib, 1, 1, 1); 613 614 IFLIB_PNP_INFO(pci, em, em_vendor_info_array); 615 616 static driver_t igb_driver = { 617 "igb", igb_methods, sizeof(struct e1000_softc), 618 }; 619 620 DRIVER_MODULE(igb, pci, igb_driver, 0, 0); 621 622 MODULE_DEPEND(igb, pci, 1, 1, 1); 623 MODULE_DEPEND(igb, ether, 1, 1, 1); 624 MODULE_DEPEND(igb, iflib, 1, 1, 1); 625 626 IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array); 627 628 static driver_t igbv_driver = { 629 "igbv", igbv_methods, sizeof(struct e1000_softc), 630 }; 631 632 DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0); 633 634 MODULE_DEPEND(igbv, pci, 1, 1, 1); 635 MODULE_DEPEND(igbv, ether, 1, 1, 1); 636 MODULE_DEPEND(igbv, iflib, 1, 1, 1); 637 638 IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array); 639 640 static device_method_t em_if_methods[] = { 641 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 642 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 643 DEVMETHOD(ifdi_detach, em_if_detach), 644 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 645 DEVMETHOD(ifdi_suspend, em_if_suspend), 646 DEVMETHOD(ifdi_resume, em_if_resume), 647 DEVMETHOD(ifdi_init, em_if_init), 648 DEVMETHOD(ifdi_stop, em_if_stop), 649 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 650 DEVMETHOD(ifdi_intr_enable, em_if_intr_enable), 651 DEVMETHOD(ifdi_intr_disable, em_if_intr_disable), 652 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 653 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 654 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 655 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 656 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 657 DEVMETHOD(ifdi_media_status, em_if_media_status), 658 DEVMETHOD(ifdi_media_change, em_if_media_change), 659 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 660 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 661 DEVMETHOD(ifdi_timer, em_if_timer), 662 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 663 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 664 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 665 DEVMETHOD(ifdi_led_func, em_if_led_func), 666 DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), 667 DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), 668 DEVMETHOD(ifdi_debug, em_if_debug), 669 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 670 DEVMETHOD_END 671 }; 672 673 static driver_t em_if_driver = { 674 "em_if", em_if_methods, sizeof(struct e1000_softc) 675 }; 676 677 static device_method_t igb_if_methods[] = { 678 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 679 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 680 DEVMETHOD(ifdi_detach, em_if_detach), 681 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 682 DEVMETHOD(ifdi_suspend, em_if_suspend), 683 DEVMETHOD(ifdi_resume, em_if_resume), 684 DEVMETHOD(ifdi_init, em_if_init), 685 DEVMETHOD(ifdi_stop, em_if_stop), 686 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 687 DEVMETHOD(ifdi_intr_enable, igb_if_intr_enable), 688 DEVMETHOD(ifdi_intr_disable, igb_if_intr_disable), 689 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 690 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 691 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 692 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 693 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 694 DEVMETHOD(ifdi_media_status, em_if_media_status), 695 DEVMETHOD(ifdi_media_change, em_if_media_change), 696 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 697 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 698 DEVMETHOD(ifdi_timer, em_if_timer), 699 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 700 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 701 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 702 DEVMETHOD(ifdi_led_func, em_if_led_func), 703 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 704 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 705 DEVMETHOD(ifdi_debug, em_if_debug), 706 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 707 #ifdef PCI_IOV 708 DEVMETHOD(ifdi_iov_init, igb_if_iov_init), 709 DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit), 710 DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add), 711 DEVMETHOD(ifdi_vf_status, igb_if_vf_status), 712 #endif 713 DEVMETHOD_END 714 }; 715 716 static driver_t igb_if_driver = { 717 "igb_if", igb_if_methods, sizeof(struct e1000_softc) 718 }; 719 720 static device_method_t igbv_if_methods[] = { 721 DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre), 722 DEVMETHOD(ifdi_attach_post, igbv_if_attach_post), 723 DEVMETHOD(ifdi_detach, em_if_detach), 724 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 725 DEVMETHOD(ifdi_suspend, em_if_suspend), 726 DEVMETHOD(ifdi_resume, em_if_resume), 727 DEVMETHOD(ifdi_init, em_if_init), 728 DEVMETHOD(ifdi_stop, em_if_stop), 729 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 730 DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable), 731 DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable), 732 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 733 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 734 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 735 DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status), 736 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 737 DEVMETHOD(ifdi_media_status, em_if_media_status), 738 DEVMETHOD(ifdi_media_change, igbv_if_media_change), 739 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 740 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 741 DEVMETHOD(ifdi_timer, em_if_timer), 742 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 743 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 744 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 745 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 746 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 747 DEVMETHOD(ifdi_debug, em_if_debug), 748 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 749 DEVMETHOD_END 750 }; 751 752 static driver_t igbv_if_driver = { 753 "igbv_if", igbv_if_methods, sizeof(struct e1000_softc) 754 }; 755 756 /********************************************************************* 757 * Tunable default values. 758 *********************************************************************/ 759 760 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 761 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 762 763 /* Allow common code without TSO */ 764 #ifndef CSUM_TSO 765 #define CSUM_TSO 0 766 #endif 767 768 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 769 "EM driver parameters"); 770 771 static int em_disable_crc_stripping = 0; 772 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 773 &em_disable_crc_stripping, 0, "Disable CRC Stripping"); 774 775 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 776 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 777 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, 778 &em_tx_int_delay_dflt, 0, "Default transmit interrupt delay in usecs"); 779 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, 780 &em_rx_int_delay_dflt, 0, "Default receive interrupt delay in usecs"); 781 782 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 783 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 784 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN, 785 &em_tx_abs_int_delay_dflt, 0, 786 "Default transmit interrupt delay limit in usecs"); 787 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, 788 &em_rx_abs_int_delay_dflt, 0, 789 "Default receive interrupt delay limit in usecs"); 790 791 static int em_smart_pwr_down = false; 792 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 793 &em_smart_pwr_down, 794 0, "Set to true to leave smart power down enabled on newer adapters"); 795 796 static bool em_unsupported_tso = false; 797 SYSCTL_BOOL(_hw_em, OID_AUTO, unsupported_tso, CTLFLAG_RDTUN, 798 &em_unsupported_tso, 0, "Allow unsupported em(4) TSO configurations"); 799 800 /* Controls whether promiscuous also shows bad packets */ 801 static int em_debug_sbp = false; 802 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, 803 "Show bad packets in promiscuous mode"); 804 805 /* Energy efficient ethernet - default to OFF */ 806 static int eee_setting = 1; 807 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, 808 "Enable Energy Efficient Ethernet"); 809 810 /* 811 * AIM: Adaptive Interrupt Moderation 812 * which means that the interrupt rate is varied over time based on the 813 * traffic for that interrupt vector 814 */ 815 static int em_enable_aim = 1; 816 SYSCTL_INT(_hw_em, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &em_enable_aim, 817 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 818 819 /* 820 ** Tuneable Interrupt rate 821 */ 822 static int em_max_interrupt_rate = EM_INTS_DEFAULT; 823 SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 824 &em_max_interrupt_rate, 0, "Maximum interrupts per second"); 825 826 /* Global used in WOL setup with multiport cards */ 827 static int global_quad_port_a = 0; 828 829 extern struct if_txrx igb_txrx; 830 extern struct if_txrx em_txrx; 831 extern struct if_txrx lem_txrx; 832 833 static struct if_shared_ctx em_sctx_init = { 834 .isc_magic = IFLIB_MAGIC, 835 .isc_q_align = PAGE_SIZE, 836 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 837 .isc_tx_maxsegsize = PAGE_SIZE, 838 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 839 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 840 .isc_rx_maxsize = MJUM9BYTES, 841 .isc_rx_nsegments = 1, 842 .isc_rx_maxsegsize = MJUM9BYTES, 843 .isc_nfl = 1, 844 .isc_nrxqs = 1, 845 .isc_ntxqs = 1, 846 .isc_admin_intrcnt = 1, 847 .isc_vendor_info = em_vendor_info_array, 848 .isc_driver_version = em_driver_version, 849 .isc_driver = &em_if_driver, 850 .isc_flags = 851 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 852 853 .isc_nrxd_min = {EM_MIN_RXD}, 854 .isc_ntxd_min = {EM_MIN_TXD}, 855 .isc_nrxd_max = {EM_MAX_RXD}, 856 .isc_ntxd_max = {EM_MAX_TXD}, 857 .isc_nrxd_default = {EM_DEFAULT_RXD}, 858 .isc_ntxd_default = {EM_DEFAULT_TXD}, 859 }; 860 861 static struct if_shared_ctx igb_sctx_init = { 862 .isc_magic = IFLIB_MAGIC, 863 .isc_q_align = PAGE_SIZE, 864 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 865 .isc_tx_maxsegsize = PAGE_SIZE, 866 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 867 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 868 .isc_rx_maxsize = MJUM9BYTES, 869 .isc_rx_nsegments = 1, 870 .isc_rx_maxsegsize = MJUM9BYTES, 871 .isc_nfl = 1, 872 .isc_nrxqs = 1, 873 .isc_ntxqs = 1, 874 .isc_admin_intrcnt = 1, 875 .isc_vendor_info = igb_vendor_info_array, 876 .isc_driver_version = igb_driver_version, 877 .isc_driver = &igb_if_driver, 878 .isc_flags = 879 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 880 881 .isc_nrxd_min = {EM_MIN_RXD}, 882 .isc_ntxd_min = {EM_MIN_TXD}, 883 .isc_nrxd_max = {IGB_MAX_RXD}, 884 .isc_ntxd_max = {IGB_MAX_TXD}, 885 .isc_nrxd_default = {EM_DEFAULT_RXD}, 886 .isc_ntxd_default = {EM_DEFAULT_TXD}, 887 }; 888 889 /* 890 * igb PFs and igbv VFs share the common datapath implementation. Keep a 891 * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or 892 * interrupt modes. 893 */ 894 static struct if_shared_ctx igbv_sctx_init = { 895 .isc_magic = IFLIB_MAGIC, 896 .isc_q_align = PAGE_SIZE, 897 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 898 .isc_tx_maxsegsize = PAGE_SIZE, 899 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 900 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 901 .isc_rx_maxsize = MJUM9BYTES, 902 .isc_rx_nsegments = 1, 903 .isc_rx_maxsegsize = MJUM9BYTES, 904 .isc_nfl = 1, 905 .isc_nrxqs = 1, 906 .isc_ntxqs = 1, 907 .isc_admin_intrcnt = 1, 908 .isc_vendor_info = igbv_vendor_info_array, 909 .isc_driver_version = igb_driver_version, 910 .isc_driver = &igbv_if_driver, 911 .isc_flags = 912 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM | 913 IFLIB_IS_VF, 914 915 .isc_nrxd_min = {EM_MIN_RXD}, 916 .isc_ntxd_min = {EM_MIN_TXD}, 917 .isc_nrxd_max = {IGB_MAX_RXD}, 918 .isc_ntxd_max = {IGB_MAX_TXD}, 919 .isc_nrxd_default = {EM_DEFAULT_RXD}, 920 .isc_ntxd_default = {EM_DEFAULT_TXD}, 921 }; 922 923 /***************************************************************** 924 * 925 * Dump Registers 926 * 927 ****************************************************************/ 928 #define IGB_REGS_LEN 739 929 930 static int em_get_regs(SYSCTL_HANDLER_ARGS) 931 { 932 struct e1000_softc *sc = (struct e1000_softc *)arg1; 933 struct e1000_hw *hw = &sc->hw; 934 struct sbuf *sb; 935 u32 *regs_buff; 936 int rc; 937 uint32_t rxqid, txqid; 938 939 /* 940 * This sysctl is registered before iflib allocates the queue arrays, 941 * and remains registered while iflib tears them down. 942 */ 943 if (sc->rx_queues == NULL || sc->tx_queues == NULL) 944 return (ENXIO); 945 946 regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK); 947 memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); 948 rxqid = sc->rx_queues[0].rxr.me; 949 txqid = sc->tx_queues[0].txr.me; 950 951 rc = sysctl_wire_old_buffer(req, 0); 952 MPASS(rc == 0); 953 if (rc != 0) { 954 free(regs_buff, M_DEVBUF); 955 return (rc); 956 } 957 958 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 959 MPASS(sb != NULL); 960 if (sb == NULL) { 961 free(regs_buff, M_DEVBUF); 962 return (ENOMEM); 963 } 964 965 /* General Registers */ 966 regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); 967 regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); 968 regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); 969 regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); 970 regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); 971 regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid)); 972 regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid)); 973 regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid)); 974 regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid)); 975 regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid)); 976 regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid)); 977 regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); 978 regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid)); 979 regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid)); 980 regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid)); 981 regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid)); 982 regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid)); 983 regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid)); 984 regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); 985 regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); 986 regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); 987 regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); 988 989 sbuf_printf(sb, "General Registers\n"); 990 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 991 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 992 sbuf_printf(sb, "\tCTRL_EXT\t %08x\n\n", regs_buff[2]); 993 994 sbuf_printf(sb, "Interrupt Registers\n"); 995 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 996 997 sbuf_printf(sb, "RX Registers\n"); 998 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 999 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 1000 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 1001 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 1002 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 1003 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 1004 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 1005 1006 sbuf_printf(sb, "TX Registers\n"); 1007 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 1008 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 1009 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 1010 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 1011 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 1012 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 1013 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 1014 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 1015 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 1016 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 1017 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 1018 1019 free(regs_buff, M_DEVBUF); 1020 1021 #ifdef DUMP_DESCS 1022 { 1023 if_softc_ctx_t scctx = sc->shared; 1024 struct rx_ring *rxr = &rx_que->rxr; 1025 struct tx_ring *txr = &tx_que->txr; 1026 int ntxd = scctx->isc_ntxd[0]; 1027 int nrxd = scctx->isc_nrxd[0]; 1028 int j; 1029 1030 for (j = 0; j < nrxd; j++) { 1031 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 1032 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 1033 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 1034 PRIx64 " Error:%d Length:%d\n", 1035 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 1036 } 1037 1038 for (j = 0; j < min(ntxd, 256); j++) { 1039 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 1040 1041 sbuf_printf(sb, 1042 "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x" 1043 " eop: %d DD=%d\n", 1044 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 1045 buf->eop != -1 ? 1046 txr->tx_base[buf->eop].upper.fields.status & 1047 E1000_TXD_STAT_DD : 0); 1048 1049 } 1050 } 1051 #endif 1052 1053 rc = sbuf_finish(sb); 1054 sbuf_delete(sb); 1055 return(rc); 1056 } 1057 1058 static void * 1059 em_register(device_t dev) 1060 { 1061 return (&em_sctx_init); 1062 } 1063 1064 static void * 1065 igb_register(device_t dev) 1066 { 1067 return (&igb_sctx_init); 1068 } 1069 1070 static void * 1071 igbv_register(device_t dev) 1072 { 1073 return (&igbv_sctx_init); 1074 } 1075 1076 static int 1077 igb_device_attach(device_t dev) 1078 { 1079 struct e1000_softc *sc; 1080 if_ctx_t ctx; 1081 int error; 1082 1083 error = iflib_device_attach(dev); 1084 if (error != 0) 1085 return (error); 1086 1087 ctx = device_get_softc(dev); 1088 sc = iflib_get_softc(ctx); 1089 (void)igb_iov_attach(sc); 1090 return (0); 1091 } 1092 1093 #ifdef PCI_IOV 1094 static int 1095 igb_device_iov_init(device_t dev, uint16_t num_vfs, 1096 const nvlist_t *params) 1097 { 1098 struct e1000_softc *sc; 1099 if_ctx_t ctx; 1100 int error; 1101 1102 ctx = device_get_softc(dev); 1103 sc = iflib_get_softc(ctx); 1104 error = igb_iov_validate(sc, num_vfs); 1105 if (error != 0) 1106 return (error); 1107 return (iflib_device_iov_init_restart(dev, num_vfs, params)); 1108 } 1109 1110 static void 1111 igb_device_iov_uninit(device_t dev) 1112 { 1113 struct e1000_softc *sc; 1114 if_ctx_t ctx; 1115 1116 ctx = device_get_softc(dev); 1117 sc = iflib_get_softc(ctx); 1118 /* 1119 * pci_iov(4) has already detached the VF devices. Tell the stop 1120 * half of iflib's restart transaction not to wait for acknowledgements 1121 * from VFs which can no longer service their mailbox vectors. 1122 */ 1123 atomic_store_rel_32(&sc->iov_teardown, 1); 1124 iflib_device_iov_uninit_restart(dev); 1125 } 1126 1127 #endif 1128 1129 static int 1130 em_set_num_queues(if_ctx_t ctx) 1131 { 1132 struct e1000_softc *sc = iflib_get_softc(ctx); 1133 int maxqueues; 1134 1135 /* Sanity check based on HW */ 1136 switch (sc->hw.mac.type) { 1137 case e1000_82576: 1138 case e1000_82580: 1139 case e1000_i350: 1140 case e1000_i354: 1141 maxqueues = 8; 1142 break; 1143 case e1000_i210: 1144 case e1000_82575: 1145 maxqueues = 4; 1146 break; 1147 case e1000_i211: 1148 case e1000_82574: 1149 maxqueues = 2; 1150 break; 1151 case e1000_vfadapt: 1152 /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */ 1153 case e1000_vfadapt_i350: 1154 maxqueues = 1; 1155 break; 1156 default: 1157 maxqueues = 1; 1158 break; 1159 } 1160 1161 return (maxqueues); 1162 } 1163 1164 #define LEM_CAPS ( \ 1165 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1166 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1167 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6) 1168 1169 #define EM_CAPS ( \ 1170 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1171 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1172 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1173 IFCAP_TSO6) 1174 1175 #define IGB_CAPS ( \ 1176 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1177 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1178 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1179 IFCAP_TSO6) 1180 1181 /* 1182 * VLAN filtering is an effective VF capability, but its policy is owned by 1183 * the PF and cannot be disabled from the VF. vlan(4) registration callbacks 1184 * are independent of this capability bit. 1185 */ 1186 #define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL) 1187 1188 void 1189 em_add_device_sysctls(struct e1000_softc *sc) 1190 { 1191 struct e1000_hw *hw; 1192 struct sysctl_oid_list *child; 1193 struct sysctl_ctx_list *ctx_list; 1194 1195 hw = &sc->hw; 1196 ctx_list = device_get_sysctl_ctx(sc->dev); 1197 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1198 1199 sc->enable_aim = em_enable_aim; 1200 SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", 1201 CTLFLAG_RW, &sc->enable_aim, 0, 1202 "Interrupt Moderation (1=normal, 2=lowlatency)"); 1203 1204 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug", 1205 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1206 em_sysctl_debug_info, "I", "Debug Information"); 1207 1208 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", 1209 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1210 em_get_rs, "I", "Dump RS indexes"); 1211 1212 if (sc->vf_ifp) { 1213 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1214 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1215 igbv_get_regs, "A", "Dump VF registers"); 1216 return; 1217 } 1218 1219 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", 1220 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1221 em_sysctl_nvm_info, "I", "NVM Information"); 1222 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", 1223 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 1224 em_sysctl_print_fw_version, "A", 1225 "Prints FW/NVM Versions"); 1226 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc", 1227 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1228 em_set_flowcntl, "I", "Flow Control"); 1229 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1230 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1231 em_get_regs, "A", "Dump Registers"); 1232 1233 if (hw->mac.type >= e1000_i350 && hw->mac.type != e1000_i211) { 1234 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac", 1235 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1236 igb_sysctl_dmac, "I", "DMA Coalesce"); 1237 } 1238 1239 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1240 "tso_tcp_flags_mask_first_segment", 1241 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1242 sc, 0, em_sysctl_tso_tcp_flags_mask, "IU", 1243 "TSO TCP flags mask for first segment"); 1244 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1245 "tso_tcp_flags_mask_middle_segment", 1246 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1247 sc, 1, em_sysctl_tso_tcp_flags_mask, "IU", 1248 "TSO TCP flags mask for middle segment"); 1249 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1250 "tso_tcp_flags_mask_last_segment", 1251 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1252 sc, 2, em_sysctl_tso_tcp_flags_mask, "IU", 1253 "TSO TCP flags mask for last segment"); 1254 } 1255 1256 /********************************************************************* 1257 * Device initialization routine 1258 * 1259 * The attach entry point is called when the driver is being loaded. 1260 * This routine identifies the type of hardware, allocates all resources 1261 * and initializes the hardware. 1262 * 1263 * return 0 on success, positive on failure 1264 *********************************************************************/ 1265 int 1266 em_if_attach_pre(if_ctx_t ctx) 1267 { 1268 struct e1000_softc *sc; 1269 if_softc_ctx_t scctx; 1270 device_t dev; 1271 struct e1000_hw *hw; 1272 struct sysctl_oid_list *child; 1273 struct sysctl_ctx_list *ctx_list; 1274 int error = 0; 1275 1276 INIT_DEBUGOUT("em_if_attach_pre: begin"); 1277 dev = iflib_get_dev(ctx); 1278 sc = iflib_get_softc(ctx); 1279 1280 if (em_max_interrupt_rate <= 0) { 1281 device_printf(dev, 1282 "Invalid max_interrupt_rate %d; using default %d\n", 1283 em_max_interrupt_rate, EM_INTS_DEFAULT); 1284 em_max_interrupt_rate = EM_INTS_DEFAULT; 1285 } 1286 1287 sc->ctx = sc->osdep.ctx = ctx; 1288 sc->dev = sc->osdep.dev = dev; 1289 scctx = sc->shared = iflib_get_softc_ctx(ctx); 1290 sc->media = iflib_get_media(ctx); 1291 hw = &sc->hw; 1292 sc->vf_ifp = 1293 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0; 1294 sc->osdep.vf = sc->vf_ifp; 1295 1296 /* Determine hardware and mac info */ 1297 em_identify_hardware(ctx); 1298 sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt; 1299 1300 /* VF sysctls are deferred until attach-post confirms MSI-X. */ 1301 ctx_list = device_get_sysctl_ctx(dev); 1302 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 1303 if (!sc->vf_ifp) 1304 em_add_device_sysctls(sc); 1305 1306 scctx->isc_tx_nsegments = EM_MAX_SCATTER; 1307 scctx->isc_nrxqsets_max = 1308 scctx->isc_ntxqsets_max = em_set_num_queues(ctx); 1309 if (bootverbose) 1310 device_printf(dev, "attach_pre capping queues at %d\n", 1311 scctx->isc_ntxqsets_max); 1312 1313 if (hw->mac.type >= igb_mac_min) { 1314 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1315 sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); 1316 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1317 sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); 1318 scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); 1319 scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); 1320 scctx->isc_txrx = &igb_txrx; 1321 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1322 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1323 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1324 scctx->isc_capabilities = scctx->isc_capenable = 1325 sc->vf_ifp ? IGBV_CAPS : IGB_CAPS; 1326 if (igbv_is_hyperv(sc)) { 1327 /* The host owns VLAN filters and the receive-frame limit. */ 1328 scctx->isc_capabilities &= 1329 ~(IFCAP_VLAN_HWFILTER | IFCAP_JUMBO_MTU); 1330 scctx->isc_capenable = scctx->isc_capabilities; 1331 } 1332 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 1333 CSUM_IP6_TCP | CSUM_IP6_UDP; 1334 if (hw->mac.type != e1000_82575) 1335 scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; 1336 /* 1337 ** Some new devices, as with ixgbe, now may 1338 ** use a different BAR, so we need to keep 1339 ** track of which is used. 1340 */ 1341 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1342 } else if (hw->mac.type >= em_mac_min) { 1343 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1344 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1345 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1346 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); 1347 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1348 scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); 1349 scctx->isc_txrx = &em_txrx; 1350 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1351 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1352 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1353 scctx->isc_capabilities = scctx->isc_capenable = EM_CAPS; 1354 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1355 CSUM_IP6_TCP | CSUM_IP6_UDP; 1356 1357 /* Disable TSO on all em(4) until ring stalls are debugged */ 1358 scctx->isc_capenable &= ~IFCAP_TSO; 1359 1360 /* 1361 * Disable TSO on SPT due to errata that downclocks DMA 1362 * performance 1363 * i218-i219 Specification Update 1.5.4.5 1364 */ 1365 if (hw->mac.type == e1000_pch_spt) 1366 scctx->isc_capenable &= ~IFCAP_TSO; 1367 1368 /* 1369 * We support MSI-X with 82574 only, but indicate to iflib(4) 1370 * that it shall give MSI at least a try with other devices. 1371 */ 1372 if (hw->mac.type == e1000_82574) { 1373 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1374 } else { 1375 scctx->isc_msix_bar = -1; 1376 scctx->isc_disable_msix = 1; 1377 } 1378 } else { 1379 scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * 1380 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1381 scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * 1382 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 1383 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1384 scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); 1385 scctx->isc_txrx = &lem_txrx; 1386 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1387 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1388 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1389 scctx->isc_capabilities = scctx->isc_capenable = LEM_CAPS; 1390 if (em_unsupported_tso) 1391 scctx->isc_capabilities |= IFCAP_TSO6; 1392 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1393 CSUM_IP6_TCP | CSUM_IP6_UDP; 1394 1395 /* Disable TSO on all lem(4) until ring stalls debugged */ 1396 scctx->isc_capenable &= ~IFCAP_TSO; 1397 1398 /* 82541ER doesn't do HW tagging */ 1399 if (hw->device_id == E1000_DEV_ID_82541ER || 1400 hw->device_id == E1000_DEV_ID_82541ER_LOM) { 1401 scctx->isc_capabilities &= ~IFCAP_VLAN_HWTAGGING; 1402 scctx->isc_capenable = scctx->isc_capabilities; 1403 } 1404 /* This is the first e1000 chip and it does not do offloads */ 1405 if (hw->mac.type == e1000_82542) { 1406 scctx->isc_capabilities &= ~(IFCAP_HWCSUM | 1407 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6 | 1408 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER | 1409 IFCAP_TSO | IFCAP_VLAN_HWTSO); 1410 scctx->isc_capenable = scctx->isc_capabilities; 1411 } 1412 /* These can't do TSO for various reasons */ 1413 if (hw->mac.type < e1000_82544 || 1414 hw->mac.type == e1000_82547 || 1415 hw->mac.type == e1000_82547_rev_2) { 1416 scctx->isc_capabilities &= 1417 ~(IFCAP_TSO |IFCAP_VLAN_HWTSO); 1418 scctx->isc_capenable = scctx->isc_capabilities; 1419 } 1420 /* XXXKB: No IPv6 before this? */ 1421 if (hw->mac.type < e1000_82545){ 1422 scctx->isc_capabilities &= ~IFCAP_HWCSUM_IPV6; 1423 scctx->isc_capenable = scctx->isc_capabilities; 1424 } 1425 /* 1426 * "PCI/PCI-X SDM 4.0" page 33 (b): 1427 * FDX requirement on these chips 1428 */ 1429 if (hw->mac.type == e1000_82547 || 1430 hw->mac.type == e1000_82547_rev_2) 1431 scctx->isc_capenable &= ~(IFCAP_HWCSUM | 1432 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6); 1433 1434 /* INTx only */ 1435 scctx->isc_msix_bar = 0; 1436 } 1437 1438 /* Setup PCI resources */ 1439 if (em_allocate_pci_resources(ctx)) { 1440 device_printf(dev, "Allocation of PCI resources failed\n"); 1441 error = ENXIO; 1442 goto err_pci; 1443 } 1444 /* 1445 * A VF can retain queue enable bits and DMA addresses across VFLR. 1446 * Fence bus mastering before the first mailbox reset so state left by 1447 * a previous owner cannot issue DMA while the driver attaches. 1448 */ 1449 if (sc->vf_ifp) 1450 em_fence_pci_busmaster(sc); 1451 /* 1452 * 82579 can lose a host CSR write while the Management Engine owns 1453 * the PCIm2PCI arbiter. Enable the OS register write interlock before 1454 * shared code initialization performs any MAC writes. 1455 */ 1456 if (hw->mac.type == e1000_pch2lan && 1457 (E1000_READ_REG(hw, E1000_FWSM) & 1458 E1000_ICH_FWSM_FW_VALID) != 0) 1459 sc->osdep.pcim2pci_arbiter_wa = true; 1460 1461 /* 1462 ** For ICH8 and family we need to 1463 ** map the flash memory, and this 1464 ** must happen after the MAC is 1465 ** identified 1466 */ 1467 if ((hw->mac.type == e1000_ich8lan) || 1468 (hw->mac.type == e1000_ich9lan) || 1469 (hw->mac.type == e1000_ich10lan) || 1470 (hw->mac.type == e1000_pchlan) || 1471 (hw->mac.type == e1000_pch2lan) || 1472 (hw->mac.type == e1000_pch_lpt)) { 1473 int rid = EM_BAR_TYPE_FLASH; 1474 sc->flash = bus_alloc_resource_any(dev, 1475 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1476 if (sc->flash == NULL) { 1477 device_printf(dev, "Mapping of Flash failed\n"); 1478 error = ENXIO; 1479 goto err_pci; 1480 } 1481 /* This is used in the shared code */ 1482 hw->flash_address = (u8 *)sc->flash; 1483 sc->osdep.flash_bus_space_tag = 1484 rman_get_bustag(sc->flash); 1485 sc->osdep.flash_bus_space_handle = 1486 rman_get_bushandle(sc->flash); 1487 } 1488 /* 1489 ** In the new SPT device flash is not a 1490 ** separate BAR, rather it is also in BAR0, 1491 ** so use the same tag and an offset handle for the 1492 ** FLASH read/write macros in the shared code. 1493 */ 1494 else if (hw->mac.type >= e1000_pch_spt) { 1495 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag; 1496 sc->osdep.flash_bus_space_handle = 1497 sc->osdep.mem_bus_space_handle + E1000_FLASH_BASE_ADDR; 1498 } 1499 1500 /* Do Shared Code initialization */ 1501 error = e1000_setup_init_funcs(hw, true); 1502 if (error) { 1503 device_printf(dev, "Setup of Shared code failed, error %d\n", 1504 error); 1505 error = ENXIO; 1506 goto err_pci; 1507 } 1508 1509 if (igbv_is_hyperv(sc)) 1510 igbv_init_hv_ops(hw); 1511 1512 em_setup_msix(ctx); 1513 e1000_get_bus_info(hw); 1514 1515 /* 1516 * Some conventional PCI systems hang when e1000 devices use 1517 * DMA addresses above 4 GB. Keep PCI-mode DMA below that boundary 1518 * by default; PCI-X and PCIe retain 64-bit DMA. 1519 */ 1520 if (hw->bus.type == e1000_bus_type_pci) { 1521 SYSCTL_ADD_BOOL(ctx_list, child, OID_AUTO, "allow_64bit_dma", 1522 CTLFLAG_RDTUN, &sc->allow_64bit_dma, 0, 1523 "Allow 64-bit DMA in conventional PCI mode"); 1524 if (sc->allow_64bit_dma) 1525 device_printf(dev, "64-bit DMA in conventional PCI mode. " 1526 "Some chipsets are unstable.\n"); 1527 else { 1528 scctx->isc_dma_width = 32; 1529 device_printf(dev, "32-bit DMA in conventional PCI mode. " 1530 "Set dev.%s.%d.allow_64bit_dma=1 at boot to enable " 1531 "64-bit DMA if the chipset is stable with it.\n", 1532 device_get_name(dev), device_get_unit(dev)); 1533 } 1534 } 1535 1536 /* Set up some sysctls for the tunable interrupt delays */ 1537 if (hw->mac.type < igb_mac_min) { 1538 em_add_int_delay_sysctl(sc, "rx_int_delay", 1539 "receive interrupt delay in usecs", &sc->rx_int_delay, 1540 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); 1541 em_add_int_delay_sysctl(sc, "tx_int_delay", 1542 "transmit interrupt delay in usecs", &sc->tx_int_delay, 1543 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt); 1544 } 1545 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) { 1546 em_add_int_delay_sysctl(sc, "rx_abs_int_delay", 1547 "receive interrupt delay limit in usecs", 1548 &sc->rx_abs_int_delay, 1549 E1000_REGISTER(hw, E1000_RADV), em_rx_abs_int_delay_dflt); 1550 em_add_int_delay_sysctl(sc, "tx_abs_int_delay", 1551 "transmit interrupt delay limit in usecs", 1552 &sc->tx_abs_int_delay, 1553 E1000_REGISTER(hw, E1000_TADV), em_tx_abs_int_delay_dflt); 1554 } 1555 1556 hw->mac.autoneg = DO_AUTO_NEG; 1557 hw->phy.autoneg_wait_to_complete = false; 1558 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1559 1560 if (hw->mac.type < em_mac_min) { 1561 e1000_init_script_state_82541(hw, true); 1562 e1000_set_tbi_compatibility_82543(hw, true); 1563 } 1564 /* Copper options */ 1565 if (hw->phy.media_type == e1000_media_type_copper) { 1566 hw->phy.mdix = AUTO_ALL_MODES; 1567 hw->phy.disable_polarity_correction = false; 1568 hw->phy.ms_type = EM_MASTER_SLAVE; 1569 } 1570 1571 /* 1572 * Set the frame limits assuming 1573 * standard ethernet sized frames. 1574 */ 1575 scctx->isc_max_frame_size = hw->mac.max_frame_size = 1576 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 1577 1578 /* 1579 * This controls when hardware reports transmit completion 1580 * status. 1581 */ 1582 hw->mac.report_tx_early = 1; 1583 1584 /* Allocate multicast array memory. */ 1585 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 1586 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 1587 if (sc->mta == NULL) { 1588 device_printf(dev, 1589 "Can not allocate multicast setup array\n"); 1590 error = ENOMEM; 1591 goto err_late; 1592 } 1593 1594 /* Clear the IFCAP_TSO auto mask */ 1595 sc->tso_automasked = 0; 1596 1597 /* Check SOL/IDER usage on physical functions. */ 1598 if (!sc->vf_ifp && e1000_check_reset_block(hw)) 1599 device_printf(dev, 1600 "PHY reset is blocked due to SOL/IDER session.\n"); 1601 1602 /* Sysctl for setting Energy Efficient Ethernet */ 1603 if (!sc->vf_ifp) { 1604 if (hw->mac.type < igb_mac_min) 1605 hw->dev_spec.ich8lan.eee_disable = eee_setting; 1606 else 1607 hw->dev_spec._82575.eee_disable = eee_setting; 1608 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", 1609 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1610 em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); 1611 } 1612 1613 /* 1614 ** Start from a known state, this is 1615 ** important in reading the nvm and 1616 ** mac from that. 1617 */ 1618 error = e1000_reset_hw(hw); 1619 if (sc->vf_ifp) { 1620 atomic_store_rel_32(&sc->vf_mbx_ready, 1621 error == E1000_SUCCESS); 1622 if (error != E1000_SUCCESS) 1623 igbv_log_reset_failure(sc, error, true); 1624 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 1625 } else if (error != E1000_SUCCESS) { 1626 device_printf(dev, "Hardware reset failed: %d\n", error); 1627 error = EIO; 1628 goto err_late; 1629 } 1630 1631 /* Make sure a PF has a good EEPROM before we read from it. */ 1632 if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) { 1633 /* 1634 ** Some PCI-E parts fail the first check due to 1635 ** the link being in sleep state, call it again, 1636 ** if it fails a second time its a real issue. 1637 */ 1638 if (e1000_validate_nvm_checksum(hw) < 0) { 1639 device_printf(dev, 1640 "The EEPROM Checksum Is Not Valid\n"); 1641 error = EIO; 1642 goto err_late; 1643 } 1644 } 1645 1646 /* Copy the permanent MAC address out of the EEPROM */ 1647 if (e1000_read_mac_addr(hw) < 0) { 1648 device_printf(dev, 1649 "Unable to read MAC address\n"); 1650 error = EIO; 1651 goto err_late; 1652 } 1653 1654 if (!em_is_valid_ether_addr(hw->mac.addr)) { 1655 if (sc->vf_ifp) { 1656 device_printf(dev, 1657 "PF did not assign a MAC address; using a " 1658 "locally generated address\n"); 1659 ether_gen_addr(iflib_get_ifp(ctx), 1660 (struct ether_addr *)hw->mac.addr); 1661 } else { 1662 device_printf(dev, "Invalid MAC address\n"); 1663 error = EIO; 1664 goto err_late; 1665 } 1666 } 1667 1668 if (!sc->vf_ifp) { 1669 /* Save NVM versions while holding the IFLIB context lock. */ 1670 em_fw_version_locked(ctx); 1671 em_print_fw_version(sc); 1672 } 1673 1674 /* 1675 * Get Wake-on-Lan and Management info for later use 1676 */ 1677 if (!sc->vf_ifp) { 1678 em_get_wakeup(ctx); 1679 } 1680 1681 iflib_set_mac(ctx, hw->mac.addr); 1682 1683 return (0); 1684 1685 err_late: 1686 em_release_hw_control(sc); 1687 err_pci: 1688 em_free_pci_resources(ctx); 1689 free(sc->mta, M_DEVBUF); 1690 sc->mta = NULL; 1691 1692 return (error); 1693 } 1694 1695 int 1696 em_if_attach_post(if_ctx_t ctx) 1697 { 1698 struct e1000_softc *sc = iflib_get_softc(ctx); 1699 struct e1000_hw *hw = &sc->hw; 1700 int error = 0; 1701 1702 /* Setup OS specific network interface */ 1703 error = em_setup_interface(ctx); 1704 if (error != 0) { 1705 device_printf(sc->dev, "Interface setup failed: %d\n", error); 1706 goto err_late; 1707 } 1708 1709 if (sc->vf_ifp) { 1710 (void)igbv_reset(ctx); 1711 } else if (em_reset(ctx) != E1000_SUCCESS) { 1712 error = EIO; 1713 goto err_late; 1714 } 1715 1716 /* Initialize statistics */ 1717 if (sc->vf_ifp) 1718 em_initialize_vf_stats(sc); 1719 else 1720 sc->ustats.stats = (struct e1000_hw_stats){}; 1721 1722 em_update_stats_counters(sc); 1723 atomic_readandclear_32(&sc->stats_pending); 1724 hw->mac.get_link_status = 1; 1725 if (sc->vf_ifp) 1726 igbv_if_update_admin_status(ctx); 1727 else 1728 em_if_update_admin_status(ctx); 1729 em_add_hw_stats(sc); 1730 1731 /* Non-AMT based hardware can now take control from firmware */ 1732 if (sc->has_manage && !sc->has_amt) 1733 em_get_hw_control(sc); 1734 1735 INIT_DEBUGOUT("em_if_attach_post: end"); 1736 1737 return (0); 1738 1739 err_late: 1740 /* 1741 * Upon em_if_attach_post() error, iflib calls em_if_detach() to 1742 * free resources 1743 */ 1744 return (error); 1745 } 1746 1747 /********************************************************************* 1748 * Device removal routine 1749 * 1750 * The detach entry point is called when the driver is being removed. 1751 * This routine stops the adapter and deallocates all the resources 1752 * that were allocated for driver operation. 1753 * 1754 * return 0 on success, positive on failure 1755 *********************************************************************/ 1756 static int 1757 em_if_detach(if_ctx_t ctx) 1758 { 1759 struct e1000_softc *sc = iflib_get_softc(ctx); 1760 1761 INIT_DEBUGOUT("em_if_detach: begin"); 1762 1763 igb_iov_detach(sc); 1764 if (sc->vf_ifp) { 1765 igbv_queue_retry_detach(sc); 1766 igbv_mbx_retry_detach(sc); 1767 } else { 1768 e1000_phy_hw_reset(&sc->hw); 1769 } 1770 1771 em_release_manageability(sc); 1772 em_release_hw_control(sc); 1773 em_free_pci_resources(ctx); 1774 free(sc->mta, M_DEVBUF); 1775 sc->mta = NULL; 1776 1777 return (0); 1778 } 1779 1780 /********************************************************************* 1781 * 1782 * Shutdown entry point 1783 * 1784 **********************************************************************/ 1785 1786 static int 1787 em_if_shutdown(if_ctx_t ctx) 1788 { 1789 int error; 1790 1791 error = em_if_suspend(ctx); 1792 if (error != 0) 1793 device_printf(iflib_get_dev(ctx), 1794 "Wake configuration failed during shutdown: %d\n", error); 1795 return (0); 1796 } 1797 1798 /* 1799 * Suspend/resume device methods. 1800 */ 1801 static int 1802 em_if_suspend(if_ctx_t ctx) 1803 { 1804 struct e1000_softc *sc = iflib_get_softc(ctx); 1805 int error; 1806 1807 if (sc->vf_ifp) { 1808 igbv_queue_retry_stop(sc); 1809 igbv_mbx_retry_stop(sc); 1810 } 1811 error = em_enable_wakeup(ctx); 1812 em_release_manageability(sc); 1813 em_release_hw_control(sc); 1814 return (error); 1815 } 1816 1817 static int 1818 em_if_resume(if_ctx_t ctx) 1819 { 1820 struct e1000_softc *sc = iflib_get_softc(ctx); 1821 u32 wus; 1822 u16 phy_wus; 1823 int error; 1824 1825 if (sc->hw.mac.type >= e1000_pch2lan && 1826 sc->hw.mac.type < igb_mac_min) 1827 e1000_resume_workarounds_pchlan(&sc->hw); 1828 1829 if (sc->wol_phy_armed) { 1830 /* 1831 * The PHY wake sequence requires an LCD reset before host wake 1832 * ownership is cleared. Wake registers survive this reset. 1833 */ 1834 (void)e1000_phy_hw_reset(&sc->hw); 1835 error = em_disable_phy_wakeup(sc, &phy_wus); 1836 if (error != E1000_SUCCESS) 1837 device_printf(sc->dev, 1838 "Could not clear PHY wakeup state: %d\n", error); 1839 else if (phy_wus != 0) 1840 device_printf(sc->dev, "PHY wakeup status: %#06x\n", 1841 phy_wus); 1842 } 1843 if (!sc->vf_ifp && sc->hw.mac.type >= e1000_82544) { 1844 wus = E1000_READ_REG(&sc->hw, E1000_WUS); 1845 if (!sc->wol_phy_wakeup && wus != 0) 1846 device_printf(sc->dev, "MAC wakeup status: %#010x\n", 1847 wus); 1848 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 1849 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1850 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 1851 } 1852 /* Clear PME after its MAC or PHY wake source has been removed. */ 1853 pci_clear_pme(sc->dev); 1854 1855 return (0); 1856 } 1857 1858 static int 1859 em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1860 { 1861 int max_frame_size; 1862 struct e1000_softc *sc = iflib_get_softc(ctx); 1863 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1864 1865 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1866 1867 /* No native SET_LPE exchange is available with a Hyper-V PF. */ 1868 if (igbv_is_hyperv(sc) && mtu > ETHERMTU) 1869 return (EINVAL); 1870 1871 switch (sc->hw.mac.type) { 1872 case e1000_82571: 1873 case e1000_82572: 1874 case e1000_ich9lan: 1875 case e1000_ich10lan: 1876 case e1000_pch2lan: 1877 case e1000_pch_lpt: 1878 case e1000_pch_spt: 1879 case e1000_pch_cnp: 1880 case e1000_pch_tgp: 1881 case e1000_pch_adp: 1882 case e1000_pch_mtp: 1883 case e1000_pch_ptp: 1884 case e1000_pch_nvp: 1885 case e1000_82574: 1886 case e1000_82583: 1887 case e1000_80003es2lan: 1888 /* 9K Jumbo Frame size */ 1889 max_frame_size = 9234; 1890 break; 1891 case e1000_pchlan: 1892 max_frame_size = 4096; 1893 break; 1894 case e1000_82542: 1895 case e1000_ich8lan: 1896 /* Adapters that do not support jumbo frames */ 1897 max_frame_size = ETHER_MAX_LEN; 1898 break; 1899 default: 1900 if (sc->hw.mac.type >= igb_mac_min) 1901 max_frame_size = IGB_MAX_FRAME_SIZE; 1902 else /* lem */ 1903 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1904 } 1905 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 1906 return (EINVAL); 1907 } 1908 1909 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 1910 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1911 return (0); 1912 } 1913 1914 /********************************************************************* 1915 * Init entry point 1916 * 1917 * This routine is used in two ways. It is used by the stack as 1918 * init entry point in network interface structure. It is also used 1919 * by the driver as a hw/sw initialization routine to get to a 1920 * consistent state. 1921 * 1922 **********************************************************************/ 1923 static void 1924 em_if_init(if_ctx_t ctx) 1925 { 1926 struct e1000_softc *sc = iflib_get_softc(ctx); 1927 if_softc_ctx_t scctx = sc->shared; 1928 if_t ifp = iflib_get_ifp(ctx); 1929 struct em_tx_queue *tx_que; 1930 int i; 1931 1932 INIT_DEBUGOUT("em_if_init: begin"); 1933 if (sc->vf_ifp) { 1934 igbv_queue_retry_prepare(sc); 1935 igbv_mbx_retry_prepare(sc); 1936 sc->vf_reset_pending = true; 1937 } 1938 if (sc->suspend_link_powered_down) 1939 em_power_up_wakeup_link(sc); 1940 1941 /* Get the latest mac address, User can use a LAA */ 1942 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1943 1944 /* 1945 * A VF restores its address only after its reset handshake establishes 1946 * its PF-assigned state. The PF path programs RAR[0] directly here. 1947 */ 1948 if (!sc->vf_ifp) 1949 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1950 1951 /* 1952 * With the 82571 adapter, RAR[0] may be overwritten 1953 * when the other port is reset, we make a duplicate 1954 * in RAR[14] for that eventuality, this assures 1955 * the interface continues to function. 1956 */ 1957 if (sc->hw.mac.type == e1000_82571) { 1958 e1000_set_laa_state_82571(&sc->hw, true); 1959 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1960 E1000_RAR_ENTRIES - 1); 1961 } 1962 1963 /* Initialize the hardware */ 1964 igb_iov_reset_prepare(sc); 1965 if (sc->vf_ifp) { 1966 (void)igbv_reset(ctx); 1967 em_rebase_vf_stats(sc); 1968 } else if (em_reset(ctx) != E1000_SUCCESS) { 1969 em_fence_pci_busmaster(sc); 1970 iflib_init_failed(ctx); 1971 return; 1972 } 1973 if (sc->vf_ifp && !sc->vf_queues_sanitized) { 1974 /* 1975 * Do not program or enable rings while retained queue state 1976 * might still contain a previous VF owner's DMA address. A 1977 * bounded callout retries initialization after iflib leaves the 1978 * failed initialization stopped. 1979 */ 1980 em_fence_pci_busmaster(sc); 1981 igbv_queue_retry_failed(ctx); 1982 return; 1983 } 1984 if (sc->vf_ifp && 1985 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1986 igbv_mbx_retry_failed(ctx); 1987 return; 1988 } 1989 /* 1990 * Keep a fail-closed device fenced until reset and VF queue 1991 * sanitization have removed every stale DMA address. 1992 */ 1993 if (em_enable_pci_busmaster(sc) != 0) { 1994 device_printf(sc->dev, 1995 "Unable to enable PCI bus mastering\n"); 1996 iflib_init_failed(ctx); 1997 return; 1998 } 1999 if (sc->vf_ifp) 2000 igbv_reconcile_mac(sc, ifp); 2001 /* Re-arm a link-up transition deferred for this reset. */ 2002 if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING || 2003 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING) 2004 sc->link_state = EM_LINK_STATE_DOWN; 2005 if (sc->vf_ifp) 2006 igbv_if_update_admin_status(ctx); 2007 else 2008 em_if_update_admin_status(ctx); 2009 2010 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 2011 i++, tx_que++) { 2012 struct tx_ring *txr = &tx_que->txr; 2013 2014 txr->tx_rs_cidx = txr->tx_rs_pidx; 2015 2016 /* Initialize the last processed descriptor to be the end of 2017 * the ring, rather than the start, so that we avoid an 2018 * off-by-one error when calculating how many descriptors are 2019 * done in the credits_update function. 2020 */ 2021 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 2022 } 2023 2024 /* The VF VLAN EtherType is fixed and has no VET register. */ 2025 if (!sc->vf_ifp) 2026 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 2027 2028 /* Clear bad data from Rx FIFOs */ 2029 if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp) 2030 e1000_rx_fifo_flush_base(&sc->hw); 2031 2032 /* Configure for OS presence */ 2033 em_init_manageability(sc); 2034 2035 /* Prepare transmit descriptors and buffers */ 2036 if (sc->vf_ifp) 2037 igbv_initialize_transmit_unit(ctx); 2038 else 2039 em_initialize_transmit_unit(ctx); 2040 2041 /* 2042 * A failed VF reset has no CTS channel on which to restore mailbox 2043 * state. The reset detector schedules another complete init, which 2044 * replays these interface-owned lists after the handshake succeeds. 2045 */ 2046 em_if_multi_set(ctx); 2047 2048 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 2049 if (sc->vf_ifp) 2050 igbv_initialize_receive_unit(ctx); 2051 else 2052 em_initialize_receive_unit(ctx); 2053 2054 /* Set up VLAN support and filter. */ 2055 em_setup_vlan_hw_support(ctx); 2056 2057 /* Don't lose promiscuous settings. */ 2058 em_if_set_promisc_impl(ctx, if_getflags(ifp)); 2059 atomic_readandclear_32(&sc->promisc_pending); 2060 2061 /* Restore PF/VF pool configuration after the global reset. */ 2062 igb_iov_initialize(sc); 2063 2064 if (sc->hw.mac.ops.clear_hw_cntrs != NULL) 2065 sc->hw.mac.ops.clear_hw_cntrs(&sc->hw); 2066 2067 /* MSI-X configuration for 82574 */ 2068 if (sc->hw.mac.type == e1000_82574) { 2069 int tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 2070 2071 tmp |= E1000_CTRL_EXT_PBA_CLR; 2072 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 2073 /* Set the IVAR - interrupt vector routing. */ 2074 E1000_WRITE_REG(&sc->hw, E1000_IVAR, sc->ivars); 2075 } else if (sc->intr_type == IFLIB_INTR_MSIX) { 2076 /* Set up queue routing */ 2077 igb_configure_queues(sc); 2078 } 2079 if (sc->hw.mac.type >= igb_mac_min) 2080 igb_initialize_interrupt_rate(sc); 2081 2082 /* AMT based hardware can now take control from firmware */ 2083 if (sc->has_manage && sc->has_amt) 2084 em_get_hw_control(sc); 2085 2086 /* Set Energy Efficient Ethernet */ 2087 if (sc->hw.mac.type >= igb_mac_min && 2088 sc->hw.phy.media_type == e1000_media_type_copper) { 2089 if (sc->hw.mac.type == e1000_i354) 2090 e1000_set_eee_i354(&sc->hw, true, true); 2091 else 2092 e1000_set_eee_i350(&sc->hw, true, true); 2093 } 2094 em_configure_peind_memory_errors(sc); 2095 em_configure_82575_memory_errors(sc); 2096 em_configure_82580_memory_errors(sc); 2097 if (sc->vf_ifp) { 2098 sc->vf_reset_pending = false; 2099 } else { 2100 u32 icr; 2101 2102 /* 2103 * Drain stale causes only after register reconstruction is 2104 * complete. DRSTA and DEV_RST_SET together close the window in 2105 * which another device reset can arrive while interrupts are 2106 * masked. 2107 */ 2108 icr = E1000_READ_REG(&sc->hw, E1000_ICR); 2109 if (igb_finish_device_reset(sc, icr)) { 2110 iflib_init_failed(ctx); 2111 return; 2112 } 2113 E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); 2114 } 2115 } 2116 2117 /* 2118 * RX publishes its byte and packet counters as one snapshot when iflib 2119 * returns descriptors to hardware. This also covers watchdog-driven RX 2120 * processing, which can run while the interrupt vector is unmasked. 2121 */ 2122 static __inline void 2123 em_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 2124 { 2125 uint64_t snapshot; 2126 u32 now_bytes, now_packets; 2127 2128 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 2129 now_bytes = snapshot >> 32; 2130 now_packets = (u32)snapshot; 2131 *bytes = now_bytes - rxr->rx_bytes_last; 2132 *packets = now_packets - rxr->rx_packets_last; 2133 rxr->rx_bytes_last = now_bytes; 2134 rxr->rx_packets_last = now_packets; 2135 } 2136 2137 /* 2138 * TX publishes its byte and packet counters as one snapshot at the doorbell, 2139 * because encapsulation can overlap the interrupt filter. The two halves 2140 * remain independent free running u32 counters, so their deltas are correct 2141 * across wrap. 2142 */ 2143 static __inline void 2144 em_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 2145 { 2146 uint64_t snapshot; 2147 u32 now_bytes, now_packets; 2148 2149 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 2150 now_bytes = snapshot >> 32; 2151 now_packets = (u32)snapshot; 2152 *bytes = now_bytes - txr->tx_bytes_last; 2153 *packets = now_packets - txr->tx_packets_last; 2154 txr->tx_bytes_last = now_bytes; 2155 txr->tx_packets_last = now_packets; 2156 } 2157 2158 /********************************************************************* 2159 * 2160 * Do Adaptive Interrupt Moderation: 2161 * - Calculate based on average size over the last interval 2162 * 2163 * Returns interrupts per second rather than a register value, so that the 2164 * caller's EM_INTS_TO_ITR()/IGB_INTS_TO_EITR() conversion applies, or zero 2165 * if the interval carried no packet to measure. 2166 * 2167 *********************************************************************/ 2168 static u32 2169 em_ring_itr(struct e1000_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 2170 u32 txpackets) 2171 { 2172 u32 newitr = 0; 2173 2174 if (txbytes && txpackets) 2175 newitr = txbytes / txpackets; 2176 if (rxbytes && rxpackets) 2177 newitr = max(newitr, rxbytes / rxpackets); 2178 2179 /* 2180 * No packet was observed, so there is no size to work from. Report no 2181 * observation and let the caller keep the rate it already has. 2182 */ 2183 if (newitr == 0) 2184 return (0); 2185 2186 newitr += 24; /* account for hardware frame, crc */ 2187 /* set an upper boundary */ 2188 newitr = min(newitr, 3000); 2189 /* Be nice to the mid range */ 2190 if ((newitr > 300) && (newitr < 1200)) 2191 newitr = (newitr / 3); 2192 else 2193 newitr = (newitr / 2); 2194 2195 /* The value above was written straight to EITR; make it a rate */ 2196 newitr = EM_AIM_DIVIDEND / newitr; 2197 2198 /* 2199 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 2200 * into the low latency end. The original was unbounded and would ask 2201 * for ~95k ints/s on minimum sized frames. There is deliberately no 2202 * floor, so jumbo traffic settles near 2.7k ints/s. 2203 */ 2204 if (sc->enable_aim == 1) 2205 newitr = min(newitr, EM_INTS_20K); 2206 else 2207 newitr = min(newitr, EM_INTS_70K); 2208 2209 return (newitr); 2210 } 2211 2212 /********************************************************************* 2213 * 2214 * Helper to calculate next (E)ITR value for AIM 2215 * 2216 *********************************************************************/ 2217 static void 2218 em_newitr(struct e1000_softc *sc, struct em_rx_queue *que, 2219 struct rx_ring *rxr) 2220 { 2221 struct e1000_hw *hw = &sc->hw; 2222 struct em_tx_queue *tx_que; 2223 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 2224 u32 newitr; 2225 int i; 2226 2227 em_aim_rx_delta(rxr, &rxbytes, &rxpackets); 2228 2229 /* 2230 * A vector can service more than one TX ring when iflib is configured 2231 * with unequal RX and TX queue counts. Sample every ring routed to 2232 * this vector rather than treating the vector as a TX queue index. 2233 */ 2234 txbytes = txpackets = 0; 2235 for (i = 0; i < sc->tx_num_queues; i++) { 2236 tx_que = &sc->tx_queues[i]; 2237 if (tx_que->msix != que->msix) 2238 continue; 2239 em_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 2240 txbytes += ringbytes; 2241 txpackets += ringpackets; 2242 } 2243 2244 /* Idle, do nothing */ 2245 if (txbytes == 0 && rxbytes == 0) 2246 return; 2247 2248 if (sc->enable_aim == 0) { 2249 newitr = em_max_interrupt_rate; 2250 } else if (sc->link_speed < SPEED_1000) { 2251 /* Use half default (4K) ITR if sub-gig */ 2252 newitr = EM_INTS_4K; 2253 } else if (!sc->vf_ifp && 2254 sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 2255 /* Want at least enough packet buffer for two frames to AIM */ 2256 newitr = em_max_interrupt_rate; 2257 } else { 2258 newitr = em_ring_itr(sc, rxbytes, rxpackets, txbytes, 2259 txpackets); 2260 /* No usable observation; leave the rate where it is */ 2261 if (newitr == 0) 2262 return; 2263 } 2264 2265 if (hw->mac.type >= igb_mac_min) { 2266 newitr = IGB_INTS_TO_EITR(newitr); 2267 2268 if (hw->mac.type == e1000_82575) 2269 newitr |= newitr << 16; 2270 else 2271 newitr |= E1000_EITR_CNT_IGNR; 2272 2273 if (newitr != que->itr_setting) { 2274 que->itr_setting = newitr; 2275 E1000_WRITE_REG(hw, E1000_EITR(que->msix), 2276 que->itr_setting); 2277 } 2278 } else { 2279 newitr = EM_INTS_TO_ITR(newitr); 2280 2281 if (newitr != que->itr_setting) { 2282 que->itr_setting = newitr; 2283 if (hw->mac.type == e1000_82574 && 2284 sc->intr_type == IFLIB_INTR_MSIX) { 2285 E1000_WRITE_REG(hw, 2286 E1000_EITR_82574(que->msix), 2287 que->itr_setting); 2288 } else { 2289 E1000_WRITE_REG(hw, E1000_ITR, 2290 que->itr_setting); 2291 } 2292 } 2293 } 2294 } 2295 2296 static bool 2297 em_has_pch_ecc(const struct e1000_hw *hw) 2298 { 2299 2300 return (hw->mac.type >= e1000_pch_lpt && 2301 hw->mac.type < e1000_82575); 2302 } 2303 2304 static bool 2305 em_has_82571_ecc_stats(const struct e1000_hw *hw) 2306 { 2307 2308 return (hw->mac.type == e1000_82571); 2309 } 2310 2311 static bool 2312 em_has_82575_memory_errors(const struct e1000_hw *hw) 2313 { 2314 2315 return (hw->mac.type == e1000_82575); 2316 } 2317 2318 static void 2319 em_configure_82575_memory_errors(struct e1000_softc *sc) 2320 { 2321 struct e1000_hw *hw; 2322 u32 ctrl_ext; 2323 2324 hw = &sc->hw; 2325 if (!em_has_82575_memory_errors(hw)) 2326 return; 2327 2328 /* Discard pre-driver status before enabling the hardware reaction. */ 2329 (void)E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2330 (void)E1000_READ_REG(hw, E1000_RDHESTS_82575); 2331 (void)E1000_READ_REG(hw, E1000_TDHESTS_82575); 2332 E1000_WRITE_REG(hw, E1000_PBECCSTS_82575, 2333 E1000_ECC_82575_ENABLE); 2334 E1000_WRITE_REG(hw, E1000_RDHESTS_82575, 2335 E1000_ECC_82575_ENABLE); 2336 E1000_WRITE_REG(hw, E1000_TDHESTS_82575, 2337 E1000_ECC_82575_ENABLE); 2338 2339 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2340 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2341 ctrl_ext | E1000_CTRL_EXT_MEHE); 2342 E1000_WRITE_FLUSH(hw); 2343 } 2344 2345 static bool 2346 em_has_82576_memory_errors(const struct e1000_hw *hw) 2347 { 2348 2349 return (hw->mac.type == e1000_82576); 2350 } 2351 2352 static bool 2353 em_82576_has_ipsec(const struct e1000_hw *hw) 2354 { 2355 2356 return (hw->device_id != E1000_DEV_ID_82576_NS && 2357 hw->device_id != E1000_DEV_ID_82576_NS_SERDES); 2358 } 2359 2360 static void 2361 em_configure_82576_memory_errors(struct e1000_softc *sc) 2362 { 2363 struct e1000_hw *hw; 2364 u32 peindm, reactions; 2365 2366 hw = &sc->hw; 2367 if (!em_has_82576_memory_errors(hw)) 2368 return; 2369 2370 reactions = E1000_PEIND_82576_NONFATAL_MASK | 2371 E1000_PEIND_82576_FATAL_MASK | 2372 E1000_PEINDM_82576_PARITY_ENABLE; 2373 if (!em_82576_has_ipsec(hw)) 2374 reactions &= ~E1000_PEIND_82576_IPSEC_MASK; 2375 2376 /* Discard indications left by firmware before enabling reactions. */ 2377 (void)E1000_READ_REG(hw, E1000_PEIND); 2378 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2379 E1000_WRITE_REG(hw, E1000_PEINDM, peindm | reactions); 2380 E1000_WRITE_FLUSH(hw); 2381 } 2382 2383 static bool 2384 em_has_82580_memory_errors(const struct e1000_hw *hw) 2385 { 2386 2387 return (hw->mac.type == e1000_82580); 2388 } 2389 2390 static void 2391 em_clear_82580_memory_error_status(struct e1000_hw *hw, u32 reg) 2392 { 2393 u32 status; 2394 2395 status = E1000_READ_REG(hw, reg); 2396 if (status != 0) 2397 E1000_WRITE_REG(hw, reg, status); 2398 } 2399 2400 static void 2401 em_configure_82580_memory_errors(struct e1000_softc *sc) 2402 { 2403 struct e1000_hw *hw; 2404 u32 reg; 2405 2406 hw = &sc->hw; 2407 if (!em_has_82580_memory_errors(hw)) 2408 return; 2409 2410 /* Clear status left before the driver completed its memory tables. */ 2411 (void)E1000_READ_REG(hw, E1000_PEIND); 2412 em_clear_82580_memory_error_status(hw, E1000_DTPARS_82580); 2413 em_clear_82580_memory_error_status(hw, E1000_DRPARS_82580); 2414 em_clear_82580_memory_error_status(hw, E1000_DDPARS_82580); 2415 em_clear_82580_memory_error_status(hw, E1000_PCIEERRSTS); 2416 (void)E1000_READ_REG(hw, E1000_LANPERRSTS); 2417 em_update_82580_ecc_stats(sc, 2418 E1000_READ_REG(hw, E1000_RPBECCSTS), 2419 E1000_READ_REG(hw, E1000_TPBECCSTS), 2420 E1000_READ_REG(hw, E1000_PCIEECCSTS)); 2421 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2422 E1000_PBECCSTS_82580_ECC_ENABLE); 2423 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2424 E1000_PBECCSTS_82580_ECC_ENABLE); 2425 2426 reg = E1000_READ_REG(hw, E1000_DTPARC_82580); 2427 E1000_WRITE_REG(hw, E1000_DTPARC_82580, 2428 reg | E1000_DTPARC_82580_ENABLE_MASK); 2429 reg = E1000_READ_REG(hw, E1000_DRPARC_82580); 2430 E1000_WRITE_REG(hw, E1000_DRPARC_82580, 2431 reg | E1000_DRPARC_82580_ENABLE_MASK); 2432 reg = E1000_READ_REG(hw, E1000_DDPARC_82580); 2433 E1000_WRITE_REG(hw, E1000_DDPARC_82580, 2434 reg | E1000_DDPARC_82580_ENABLE_MASK); 2435 reg = E1000_READ_REG(hw, E1000_PCIEERRCTL_82580); 2436 E1000_WRITE_REG(hw, E1000_PCIEERRCTL_82580, 2437 reg | E1000_PCIEERRCTL_82580_ENABLE_MASK); 2438 reg = E1000_READ_REG(hw, E1000_PCIEECCCTL_82580); 2439 E1000_WRITE_REG(hw, E1000_PCIEECCCTL_82580, 2440 reg | E1000_PCIEECCCTL_82580_ENABLE_MASK); 2441 reg = E1000_READ_REG(hw, E1000_LANPERRCTL_82580); 2442 reg |= E1000_LANPERRCTL_82580_HOST_MASK; 2443 /* The RSS memory is initialized only for a multiqueue layout. */ 2444 if (sc->rx_num_queues <= 1) 2445 reg &= ~E1000_LANPERRCTL_82580_RSS_ENABLE; 2446 E1000_WRITE_REG(hw, E1000_LANPERRCTL_82580, reg); 2447 reg = E1000_READ_REG(hw, E1000_PEINDM); 2448 E1000_WRITE_REG(hw, E1000_PEINDM, 2449 reg | E1000_PEIND_FATAL_MASK); 2450 E1000_WRITE_FLUSH(hw); 2451 } 2452 2453 static bool 2454 em_has_i210_memory_errors(const struct e1000_hw *hw) 2455 { 2456 2457 return (hw->mac.type == e1000_i210 || 2458 hw->mac.type == e1000_i211); 2459 } 2460 2461 static bool 2462 em_has_i350_i354_memory_errors(const struct e1000_hw *hw) 2463 { 2464 2465 return (hw->mac.type == e1000_i350 || 2466 hw->mac.type == e1000_i354); 2467 } 2468 2469 static void 2470 em_configure_peind_memory_errors(struct e1000_softc *sc) 2471 { 2472 struct e1000_hw *hw; 2473 u32 peindm; 2474 2475 hw = &sc->hw; 2476 if (!em_has_i350_i354_memory_errors(hw) && 2477 !em_has_i210_memory_errors(hw)) 2478 return; 2479 2480 /* Discard indications left by firmware before enabling reactions. */ 2481 (void)E1000_READ_REG(hw, E1000_PEIND); 2482 /* Do not depend on firmware preserving the datasheet defaults. */ 2483 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2484 E1000_WRITE_REG(hw, E1000_PEINDM, 2485 peindm | E1000_PEIND_FATAL_MASK); 2486 E1000_WRITE_FLUSH(hw); 2487 } 2488 2489 static bool 2490 em_has_peind_memory_errors(const struct e1000_hw *hw) 2491 { 2492 2493 return (em_has_82580_memory_errors(hw) || 2494 em_has_i350_i354_memory_errors(hw) || 2495 em_has_i210_memory_errors(hw)); 2496 } 2497 2498 static u32 2499 em_pcie_fatal_error_mask(const struct e1000_hw *hw) 2500 { 2501 2502 if (em_has_82580_memory_errors(hw)) 2503 return (~0U); 2504 if (em_has_i350_i354_memory_errors(hw)) 2505 return (E1000_PCIEERRSTS_I350_I354_FATAL_MASK); 2506 if (em_has_i210_memory_errors(hw)) 2507 return (E1000_PCIEERRSTS_I210_FATAL_MASK); 2508 return (0); 2509 } 2510 2511 static u32 2512 em_memory_error_intr_mask(const struct e1000_hw *hw) 2513 { 2514 2515 if (em_has_82575_memory_errors(hw)) 2516 return (E1000_IMS_82575_MEMORY_ERROR_MASK); 2517 if (em_has_82576_memory_errors(hw)) 2518 return (E1000_IMS_FER | E1000_IMS_NFER); 2519 if (em_has_pch_ecc(hw) || em_has_peind_memory_errors(hw)) 2520 return (E1000_IMS_FER); 2521 return (0); 2522 } 2523 2524 static bool 2525 em_has_memory_errors(const struct e1000_hw *hw) 2526 { 2527 2528 return (em_memory_error_intr_mask(hw) != 0); 2529 } 2530 2531 static bool 2532 em_has_memory_error_stats(const struct e1000_hw *hw) 2533 { 2534 2535 return (em_has_82571_ecc_stats(hw) || em_has_memory_errors(hw)); 2536 } 2537 2538 static u32 2539 em_fatal_error_intr_mask(struct e1000_softc *sc) 2540 { 2541 if (!em_has_memory_errors(&sc->hw)) 2542 return (0); 2543 if (atomic_load_acq_32(&sc->fatal_error_state) != 2544 EM_FATAL_ERROR_NONE) 2545 return (0); 2546 return (em_memory_error_intr_mask(&sc->hw)); 2547 } 2548 2549 static void 2550 em_update_82580_ecc_stats(struct e1000_softc *sc, u32 rpbeccsts, 2551 u32 tpbeccsts, u32 pcieeccsts) 2552 { 2553 u32 status; 2554 2555 sc->corrected_error_packet_buffer_count += 2556 (rpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK) + 2557 (tpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK); 2558 status = pcieeccsts & E1000_PCIEECCSTS_82580_ERROR_MASK; 2559 sc->uncorrected_error_pcie_count += bitcount32(status); 2560 if (status != 0) 2561 E1000_WRITE_REG(&sc->hw, E1000_PCIEECCSTS, status); 2562 } 2563 2564 static void 2565 em_update_82575_ecc_stats(struct e1000_softc *sc, u32 pbeccsts, 2566 u32 rdhests, u32 tdhests) 2567 { 2568 2569 sc->corrected_error_packet_buffer_count += 2570 pbeccsts & E1000_ECC_82575_CORR_CNT_MASK; 2571 sc->uncorrected_error_packet_buffer_count += 2572 (pbeccsts & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2573 E1000_ECC_82575_UNCORR_CNT_SHIFT; 2574 sc->corrected_error_dma_count += 2575 (rdhests & E1000_ECC_82575_CORR_CNT_MASK) + 2576 (tdhests & E1000_ECC_82575_CORR_CNT_MASK); 2577 sc->uncorrected_error_dma_count += 2578 ((rdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2579 E1000_ECC_82575_UNCORR_CNT_SHIFT) + 2580 ((tdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2581 E1000_ECC_82575_UNCORR_CNT_SHIFT); 2582 } 2583 2584 static void 2585 em_update_82576_ecc_counter(struct e1000_softc *sc, u32 reg, 2586 u64 *corrected, u64 *uncorrected) 2587 { 2588 u32 status; 2589 2590 status = E1000_READ_REG(&sc->hw, reg); 2591 *corrected += status & E1000_ECC_82576_CORR_CNT_MASK; 2592 if (uncorrected != NULL) 2593 *uncorrected += 2594 (status & E1000_ECC_82576_UNCORR_CNT_MASK) >> 2595 E1000_ECC_82576_UNCORR_CNT_SHIFT; 2596 } 2597 2598 static void 2599 em_update_82576_ecc_stats(struct e1000_softc *sc) 2600 { 2601 2602 /* 2603 * These counters are clear-on-read. PRBESTS and PMSIXESTS are 2604 * controller-shared, so whichever LAN port samples them first owns 2605 * the software count. 2606 */ 2607 em_update_82576_ecc_counter(sc, E1000_RPBECCSTS, 2608 &sc->corrected_error_packet_buffer_count, 2609 &sc->uncorrected_error_packet_buffer_count); 2610 em_update_82576_ecc_counter(sc, E1000_TPBECCSTS, 2611 &sc->corrected_error_packet_buffer_count, 2612 &sc->uncorrected_error_packet_buffer_count); 2613 em_update_82576_ecc_counter(sc, E1000_SWPBECCSTS_82576, 2614 &sc->corrected_error_packet_buffer_count, 2615 &sc->uncorrected_error_packet_buffer_count); 2616 if (em_82576_has_ipsec(&sc->hw)) 2617 em_update_82576_ecc_counter(sc, E1000_IPPBECCSTS_82576, 2618 &sc->corrected_error_packet_buffer_count, 2619 &sc->uncorrected_error_packet_buffer_count); 2620 2621 em_update_82576_ecc_counter(sc, E1000_RDHESTS_82576, 2622 &sc->corrected_error_dma_count, 2623 &sc->uncorrected_error_dma_count); 2624 em_update_82576_ecc_counter(sc, E1000_TDHESTS_82576, 2625 &sc->corrected_error_dma_count, 2626 &sc->uncorrected_error_dma_count); 2627 2628 em_update_82576_ecc_counter(sc, E1000_PRBESTS_82576, 2629 &sc->corrected_error_pcie_retry_count, NULL); 2630 em_update_82576_ecc_counter(sc, E1000_PWBESTS_82576, 2631 &sc->corrected_error_pcie_tx_data_count, NULL); 2632 em_update_82576_ecc_counter(sc, E1000_PMSIXESTS_82576, 2633 &sc->corrected_error_pcie_other_count, NULL); 2634 } 2635 2636 static void 2637 em_update_pch_ecc_stats(struct e1000_softc *sc, u32 pbeccsts) 2638 { 2639 2640 sc->corrected_error_packet_buffer_count += 2641 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 2642 sc->uncorrected_error_packet_buffer_count += 2643 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 2644 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 2645 } 2646 2647 static void 2648 em_update_82571_ecc_stats(struct e1000_softc *sc) 2649 { 2650 struct e1000_hw *hw; 2651 u32 count, pba_ecc; 2652 2653 hw = &sc->hw; 2654 pba_ecc = E1000_READ_REG(hw, E1000_PBA_ECC); 2655 count = (pba_ecc & E1000_PBA_ECC_COUNTER_MASK) >> 2656 E1000_PBA_ECC_COUNTER_SHIFT; 2657 if (count == 0) 2658 return; 2659 sc->corrected_error_packet_buffer_count += count; 2660 /* Preserve correction and reserved state while clearing statistics. */ 2661 E1000_WRITE_REG(hw, E1000_PBA_ECC, 2662 pba_ecc | E1000_PBA_ECC_STAT_CLR); 2663 } 2664 2665 static void 2666 em_update_i210_ecc_stats(struct e1000_softc *sc) 2667 { 2668 struct e1000_hw *hw; 2669 u32 pbeccsts, pcieeccsts; 2670 2671 hw = &sc->hw; 2672 pbeccsts = E1000_READ_REG(hw, E1000_PBECCSTS_I210); 2673 if (pbeccsts & E1000_PBECCSTS_I210_CORR_ERR) { 2674 sc->corrected_error_dma_count++; 2675 /* Preserve the enable bit while clearing the RW1C status. */ 2676 E1000_WRITE_REG(hw, E1000_PBECCSTS_I210, 2677 pbeccsts & (E1000_PBECCSTS_I210_ECC_ENABLE | 2678 E1000_PBECCSTS_I210_CORR_ERR)); 2679 } 2680 2681 pcieeccsts = E1000_READ_REG(hw, E1000_PCIEECCSTS) & 2682 E1000_PCIEECCSTS_I210_CORR_MASK; 2683 if (pcieeccsts & E1000_PCIEECCSTS_TX_WR_DATA) 2684 sc->corrected_error_pcie_tx_data_count++; 2685 if (pcieeccsts & E1000_PCIEECCSTS_RETRY_BUF) 2686 sc->corrected_error_pcie_retry_count++; 2687 if (pcieeccsts != 0) 2688 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieeccsts); 2689 } 2690 2691 static void 2692 em_update_i350_i354_ecc_stats(struct e1000_softc *sc) 2693 { 2694 struct e1000_hw *hw; 2695 u32 pbeccsts, pcieecc_mask, status; 2696 2697 hw = &sc->hw; 2698 status = E1000_READ_REG(hw, E1000_DTPARS) & 2699 E1000_DTPARS_CORR_MASK; 2700 if (status != 0) { 2701 sc->corrected_error_dma_count += bitcount32(status); 2702 E1000_WRITE_REG(hw, E1000_DTPARS, status); 2703 } 2704 status = E1000_READ_REG(hw, E1000_DRPARS) & 2705 E1000_DRPARS_CORR_MASK; 2706 if (status != 0) { 2707 sc->corrected_error_dma_count += bitcount32(status); 2708 E1000_WRITE_REG(hw, E1000_DRPARS, status); 2709 } 2710 status = E1000_READ_REG(hw, E1000_DDECCS) & 2711 E1000_DDECCS_CORR_MASK; 2712 if (status != 0) { 2713 sc->corrected_error_dma_count += bitcount32(status); 2714 E1000_WRITE_REG(hw, E1000_DDECCS, status); 2715 } 2716 status = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2717 E1000_LANPERRSTS_MNG_FIFO_CORR; 2718 if (status != 0) { 2719 sc->corrected_error_lan_mng_fifo_count++; 2720 E1000_WRITE_REG(hw, E1000_LANPERRSTS, status); 2721 } 2722 2723 pbeccsts = E1000_READ_REG(hw, E1000_RPBECCSTS); 2724 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2725 if (status != 0) { 2726 sc->corrected_error_packet_buffer_count += bitcount32(status); 2727 /* Preserve the enable bits while clearing RW1C status. */ 2728 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2729 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2730 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2731 } 2732 pbeccsts = E1000_READ_REG(hw, E1000_TPBECCSTS); 2733 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2734 if (status != 0) { 2735 sc->corrected_error_packet_buffer_count += bitcount32(status); 2736 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2737 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2738 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2739 } 2740 2741 pcieecc_mask = hw->mac.type == e1000_i354 ? 2742 E1000_PCIEECCSTS_I354_CORR_MASK : 2743 E1000_PCIEECCSTS_I350_CORR_MASK; 2744 status = E1000_READ_REG(hw, E1000_PCIEECCSTS) & pcieecc_mask; 2745 if (status & E1000_PCIEECCSTS_TX_WR_DATA) 2746 sc->corrected_error_pcie_tx_data_count++; 2747 if (status & E1000_PCIEECCSTS_RETRY_BUF) 2748 sc->corrected_error_pcie_retry_count++; 2749 sc->corrected_error_pcie_other_count += bitcount32(status & 2750 E1000_PCIEECCSTS_I350_I354_OTHER_MASK); 2751 if (status != 0) 2752 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, status); 2753 } 2754 2755 /* 2756 * Internal-memory error causes are read-clear. Capture them before handing 2757 * fatal recovery or non-fatal acknowledgement to the iflib admin task. 2758 */ 2759 static void 2760 em_handle_fatal_error_intr(struct e1000_softc *sc, u32 icr) 2761 { 2762 struct e1000_hw *hw; 2763 u32 dma_host, dma_rx, dma_tx, error_mask, lanerr, pcieerr, peind; 2764 2765 error_mask = em_memory_error_intr_mask(&sc->hw); 2766 if (!em_has_memory_errors(&sc->hw) || 2767 (icr & error_mask) == 0) 2768 return; 2769 2770 hw = &sc->hw; 2771 E1000_WRITE_REG(hw, E1000_IMC, error_mask); 2772 if (!atomic_cmpset_32(&sc->fatal_error_state, 2773 EM_FATAL_ERROR_NONE, EM_FATAL_ERROR_CAPTURING)) 2774 return; 2775 2776 sc->fatal_error_icr = icr & error_mask; 2777 if (em_has_pch_ecc(hw)) { 2778 sc->fatal_error_pbeccsts = 2779 E1000_READ_REG(hw, E1000_PBECCSTS); 2780 } else if (em_has_82575_memory_errors(hw)) { 2781 sc->fatal_error_pbeccsts = 2782 E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2783 sc->fatal_error_dma_rx = 2784 E1000_READ_REG(hw, E1000_RDHESTS_82575); 2785 sc->fatal_error_dma_tx = 2786 E1000_READ_REG(hw, E1000_TDHESTS_82575); 2787 } else if (em_has_82576_memory_errors(hw)) { 2788 sc->fatal_error_peind = E1000_READ_REG(hw, E1000_PEIND); 2789 } else { 2790 peind = E1000_READ_REG(hw, E1000_PEIND) & 2791 E1000_PEIND_FATAL_MASK; 2792 pcieerr = E1000_READ_REG(hw, E1000_PCIEERRSTS) & 2793 em_pcie_fatal_error_mask(hw); 2794 dma_host = 0; 2795 if (em_has_82580_memory_errors(hw)) { 2796 /* 2797 * PEIND is visible through every function. Retain the 2798 * management indication, which has no subordinate status, 2799 * but attribute host-owned regions from this function's 2800 * status registers. 2801 */ 2802 peind &= E1000_PEIND_MNG_PARITY_FATAL; 2803 dma_tx = E1000_READ_REG(hw, E1000_DTPARS_82580); 2804 dma_rx = E1000_READ_REG(hw, E1000_DRPARS_82580); 2805 dma_host = E1000_READ_REG(hw, 2806 E1000_DDPARS_82580); 2807 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2808 E1000_LANPERRSTS_82580_ERROR_MASK; 2809 } else if (em_has_i350_i354_memory_errors(hw)) { 2810 dma_tx = E1000_READ_REG(hw, E1000_DTPARS) & 2811 E1000_DTPARS_FATAL_MASK; 2812 dma_rx = E1000_READ_REG(hw, E1000_DRPARS) & 2813 E1000_DRPARS_FATAL_MASK; 2814 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2815 E1000_LANPERRSTS_I350_I354_FATAL_MASK; 2816 } else { 2817 dma_tx = 0; 2818 dma_rx = 0; 2819 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2820 E1000_LANPERRSTS_RETX_BUF; 2821 } 2822 if (pcieerr != 0) 2823 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2824 if (lanerr != 0) 2825 peind |= E1000_PEIND_LANPORT_PARITY_FATAL; 2826 if (dma_tx != 0 || dma_rx != 0 || dma_host != 0) 2827 peind |= E1000_PEIND_DMA_PARITY_FATAL; 2828 sc->fatal_error_peind = peind; 2829 sc->fatal_error_pcie = pcieerr; 2830 sc->fatal_error_lan = lanerr; 2831 sc->fatal_error_dma_tx = dma_tx; 2832 sc->fatal_error_dma_rx = dma_rx; 2833 sc->fatal_error_dma_host = dma_host; 2834 } 2835 atomic_store_rel_32(&sc->fatal_error_state, 2836 EM_FATAL_ERROR_DETECTED); 2837 iflib_admin_intr_deferred(sc->ctx); 2838 } 2839 2840 static bool 2841 em_handle_fatal_error_admin(struct e1000_softc *sc) 2842 { 2843 u32 error_mask, pcieecc, peind; 2844 bool reset_required; 2845 2846 if (!atomic_cmpset_acq_32(&sc->fatal_error_state, 2847 EM_FATAL_ERROR_DETECTED, EM_FATAL_ERROR_RESET_REQUESTED)) 2848 return (atomic_load_acq_32(&sc->fatal_error_state) != 2849 EM_FATAL_ERROR_NONE); 2850 2851 if (em_has_pch_ecc(&sc->hw)) { 2852 em_update_pch_ecc_stats(sc, sc->fatal_error_pbeccsts); 2853 device_printf(sc->dev, 2854 "uncorrectable packet-buffer ECC error: " 2855 "PBECCSTS %#x; requesting reset\n", 2856 sc->fatal_error_pbeccsts); 2857 } else if (em_has_82575_memory_errors(&sc->hw)) { 2858 em_update_82575_ecc_stats(sc, sc->fatal_error_pbeccsts, 2859 sc->fatal_error_dma_rx, sc->fatal_error_dma_tx); 2860 device_printf(sc->dev, 2861 "unrecoverable internal memory ECC error: ICR %#x, " 2862 "PBECCSTS %#x, RDHESTS %#x, TDHESTS %#x; " 2863 "requesting reset\n", sc->fatal_error_icr, 2864 sc->fatal_error_pbeccsts, sc->fatal_error_dma_rx, 2865 sc->fatal_error_dma_tx); 2866 } else if (em_has_82576_memory_errors(&sc->hw)) { 2867 peind = sc->fatal_error_peind; 2868 em_update_82576_ecc_stats(sc); 2869 reset_required = 2870 (sc->fatal_error_icr & E1000_ICR_FER) != 0 || 2871 (peind & (E1000_PEIND_82576_FATAL_MASK | 2872 E1000_PEIND_82576_MEMORY_HANG)) != 0; 2873 if (!reset_required) { 2874 device_printf(sc->dev, 2875 "non-fatal internal memory error: PEIND %#x\n", 2876 peind); 2877 sc->fatal_error_icr = 0; 2878 sc->fatal_error_peind = 0; 2879 atomic_store_rel_32(&sc->fatal_error_state, 2880 EM_FATAL_ERROR_NONE); 2881 error_mask = E1000_IMS_FER | E1000_IMS_NFER; 2882 E1000_WRITE_REG(&sc->hw, E1000_IMS, error_mask); 2883 E1000_WRITE_FLUSH(&sc->hw); 2884 return (true); 2885 } 2886 if ((peind & (E1000_PEIND_82576_FATAL_MASK | 2887 E1000_PEIND_82576_MEMORY_HANG)) == 0) 2888 sc->fatal_error_unknown_count++; 2889 device_printf(sc->dev, 2890 "fatal internal memory error: PEIND %#x; " 2891 "requesting reset\n", peind); 2892 } else { 2893 peind = sc->fatal_error_peind; 2894 if (em_has_82580_memory_errors(&sc->hw)) { 2895 pcieecc = E1000_READ_REG(&sc->hw, 2896 E1000_PCIEECCSTS) & 2897 E1000_PCIEECCSTS_82580_ERROR_MASK; 2898 sc->fatal_error_pcie_ecc |= pcieecc; 2899 if (pcieecc != 0) { 2900 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2901 sc->fatal_error_peind = peind; 2902 } 2903 em_update_82580_ecc_stats(sc, 2904 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 2905 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 2906 pcieecc); 2907 } else if (em_has_i350_i354_memory_errors(&sc->hw)) 2908 em_update_i350_i354_ecc_stats(sc); 2909 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) 2910 sc->fatal_error_lan_count++; 2911 if (peind & E1000_PEIND_MNG_PARITY_FATAL) 2912 sc->fatal_error_mng_count++; 2913 if (peind & E1000_PEIND_PCIE_PARITY_FATAL) 2914 sc->fatal_error_pcie_count++; 2915 if (peind & E1000_PEIND_DMA_PARITY_FATAL) 2916 sc->fatal_error_dma_count++; 2917 if (peind == 0) 2918 sc->fatal_error_unknown_count++; 2919 if (em_has_82580_memory_errors(&sc->hw)) { 2920 device_printf(sc->dev, 2921 "fatal internal memory error: PEIND %#x, " 2922 "PCIEERRSTS %#x, PCIEECCSTS %#x, " 2923 "DTPARS %#x, DRPARS %#x, DDPARS %#x, " 2924 "LANPERRSTS %#x\n", peind, 2925 sc->fatal_error_pcie, 2926 sc->fatal_error_pcie_ecc, 2927 sc->fatal_error_dma_tx, 2928 sc->fatal_error_dma_rx, 2929 sc->fatal_error_dma_host, 2930 sc->fatal_error_lan); 2931 } else { 2932 device_printf(sc->dev, 2933 "fatal internal memory error: PEIND %#x, " 2934 "PCIEERRSTS %#x, DTPARS %#x, DRPARS %#x, " 2935 "LANPERRSTS %#x\n", peind, 2936 sc->fatal_error_pcie, 2937 sc->fatal_error_dma_tx, 2938 sc->fatal_error_dma_rx, 2939 sc->fatal_error_lan); 2940 } 2941 2942 reset_required = (peind & 2943 (E1000_PEIND_PCIE_PARITY_FATAL | 2944 E1000_PEIND_DMA_PARITY_FATAL)) != 0; 2945 if (peind == 0) 2946 reset_required = true; 2947 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) { 2948 if (!em_has_i350_i354_memory_errors(&sc->hw) || 2949 sc->fatal_error_lan == 0 || 2950 (sc->fatal_error_lan & 2951 E1000_LANPERRSTS_I350_I354_RESET_MASK) != 0) 2952 reset_required = true; 2953 } 2954 /* Management-memory recovery belongs to management firmware. */ 2955 if (!reset_required) { 2956 if (em_has_i350_i354_memory_errors(&sc->hw) && 2957 sc->fatal_error_lan != 0) 2958 E1000_WRITE_REG(&sc->hw, E1000_LANPERRSTS, 2959 sc->fatal_error_lan & 2960 E1000_LANPERRSTS_I350_I354_NO_RESET_MASK); 2961 sc->fatal_error_peind = 0; 2962 sc->fatal_error_pcie = 0; 2963 sc->fatal_error_pcie_ecc = 0; 2964 sc->fatal_error_lan = 0; 2965 sc->fatal_error_dma_tx = 0; 2966 sc->fatal_error_dma_rx = 0; 2967 sc->fatal_error_dma_host = 0; 2968 atomic_store_rel_32(&sc->fatal_error_state, 2969 EM_FATAL_ERROR_NONE); 2970 E1000_WRITE_REG(&sc->hw, E1000_IMS, 2971 E1000_IMS_FER); 2972 E1000_WRITE_FLUSH(&sc->hw); 2973 return (true); 2974 } 2975 device_printf(sc->dev, 2976 "requesting reset after memory error\n"); 2977 } 2978 sc->fatal_error_reset_count++; 2979 iflib_request_reset(sc->ctx); 2980 /* Re-enter the admin task so it observes the reset request. */ 2981 iflib_admin_intr_deferred(sc->ctx); 2982 return (true); 2983 } 2984 2985 /* 2986 * ICR bit 30 is reserved on 82575 and is the TCP timer on 82576. It becomes 2987 * the Device Reset Asserted interrupt starting with 82580. 2988 */ 2989 static u32 2990 igb_device_reset_intr_mask(struct e1000_softc *sc) 2991 { 2992 2993 return (sc->hw.mac.type >= e1000_82580 ? E1000_IMS_DRSTA : 0); 2994 } 2995 2996 /* Keep interrupt-side work quiesced until device-reset recovery completes. */ 2997 static bool 2998 igb_device_reset_pending(struct e1000_softc *sc) 2999 { 3000 3001 return (!sc->vf_ifp && igb_device_reset_intr_mask(sc) != 0 && 3002 atomic_load_acq_32(&sc->device_reset_state) != 3003 IGB_DEVICE_RESET_NONE); 3004 } 3005 3006 /* 3007 * CTRL.DEV_RST resets every port in the device. ICR.DRSTA tells the other 3008 * ports that their registers and descriptor rings must be reinitialized. 3009 */ 3010 static bool 3011 igb_handle_device_reset(struct e1000_softc *sc, u32 icr) 3012 { 3013 u32 state; 3014 3015 if (sc->vf_ifp || igb_device_reset_intr_mask(sc) == 0 || 3016 (icr & E1000_ICR_DRSTA) == 0) 3017 return (false); 3018 state = atomic_swap_32(&sc->device_reset_state, 3019 IGB_DEVICE_RESET_DETECTED); 3020 if (state == IGB_DEVICE_RESET_DETECTED) 3021 return (true); 3022 3023 iflib_admin_intr_deferred(sc->ctx); 3024 return (true); 3025 } 3026 3027 /* 3028 * A device reset can leave a sibling port accessible before its internal 3029 * reset and PCIe transactions have completed. For 82580 and newer parts, 3030 * wait for that device-wide reset to finish and acknowledge it before any 3031 * ordinary port register programming. I350 and newer parts also publish 3032 * explicit EEPROM autoload and PF-reset completion indications. 3033 * 3034 * The wait is bounded because the only useful fallback for a controller 3035 * that never completes the device reset is the port reset already requested 3036 * by the interrupt handler. 3037 */ 3038 static void 3039 igb_prepare_device_reset(struct e1000_softc *sc) 3040 { 3041 struct e1000_hw *hw; 3042 u32 state; 3043 u32 eecd, gcr, status; 3044 int i; 3045 3046 hw = &sc->hw; 3047 state = atomic_load_acq_32(&sc->device_reset_state); 3048 if (state != IGB_DEVICE_RESET_DETECTED && 3049 state != IGB_DEVICE_RESET_REQUESTED && 3050 hw->mac.type >= e1000_82580) { 3051 /* 3052 * A reset can start while this interface has interrupts disabled. 3053 * GCR is the documented gate before ordinary port accesses. STATUS 3054 * also detects a reset that completed while this interface was down 3055 * or after an earlier preparation pass. 3056 */ 3057 gcr = E1000_READ_REG(hw, E1000_GCR); 3058 if (gcr != 0xffffffff && 3059 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) { 3060 atomic_store_rel_32(&sc->device_reset_state, 3061 IGB_DEVICE_RESET_DETECTED); 3062 state = IGB_DEVICE_RESET_DETECTED; 3063 } else if (gcr != 0xffffffff) { 3064 status = E1000_READ_REG(hw, E1000_STATUS); 3065 if (status != 0xffffffff && 3066 (status & E1000_STAT_DEV_RST_SET) != 0) { 3067 atomic_store_rel_32(&sc->device_reset_state, 3068 IGB_DEVICE_RESET_DETECTED); 3069 state = IGB_DEVICE_RESET_DETECTED; 3070 } 3071 } 3072 } 3073 if (state != IGB_DEVICE_RESET_DETECTED && 3074 state != IGB_DEVICE_RESET_REQUESTED) 3075 return; 3076 3077 if (hw->mac.type >= e1000_82580) { 3078 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3079 gcr = E1000_READ_REG(hw, E1000_GCR); 3080 if (gcr != 0xffffffff && 3081 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) == 0) 3082 break; 3083 msec_delay(1); 3084 } 3085 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) { 3086 device_printf(sc->dev, 3087 "device-wide reset did not complete; " 3088 "attempting port reset\n"); 3089 goto prepared; 3090 } 3091 3092 /* STATUS.DEV_RST_SET is write-one-to-clear. */ 3093 E1000_WRITE_REG(hw, E1000_STATUS, E1000_STAT_DEV_RST_SET); 3094 3095 if (hw->mac.type >= e1000_i350) { 3096 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3097 eecd = E1000_READ_REG(hw, E1000_EECD); 3098 status = E1000_READ_REG(hw, E1000_STATUS); 3099 if (eecd != 0xffffffff && status != 0xffffffff && 3100 (eecd & E1000_EECD_AUTO_RD) != 0 && 3101 (status & E1000_STATUS_RST_DONE) != 0) 3102 break; 3103 msec_delay(1); 3104 } 3105 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) 3106 device_printf(sc->dev, 3107 "device-wide reset did not finish EEPROM " 3108 "autoload or port reset; attempting port " 3109 "reset\n"); 3110 } 3111 } 3112 3113 prepared: 3114 atomic_store_rel_32(&sc->device_reset_state, 3115 IGB_DEVICE_RESET_PREPARED); 3116 } 3117 3118 /* 3119 * A second device reset can arrive while the port is being initialized. 3120 * Leave its status latched for the next preparation pass and do not let 3121 * iflib publish this incomplete initialization as a running datapath. 3122 */ 3123 static bool 3124 igb_finish_device_reset(struct e1000_softc *sc, u32 icr) 3125 { 3126 bool reset_again; 3127 u32 gcr, state, status; 3128 3129 if (igb_device_reset_intr_mask(sc) == 0) 3130 return (false); 3131 3132 state = atomic_load_acq_32(&sc->device_reset_state); 3133 reset_again = icr != 0xffffffff && 3134 (icr & E1000_ICR_DRSTA) != 0; 3135 if (sc->hw.mac.type >= e1000_82580) { 3136 gcr = E1000_READ_REG(&sc->hw, E1000_GCR); 3137 if (gcr != 0xffffffff && 3138 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) 3139 reset_again = true; 3140 status = E1000_READ_REG(&sc->hw, E1000_STATUS); 3141 if (status == 0xffffffff && 3142 state != IGB_DEVICE_RESET_NONE) { 3143 /* 3144 * MMIO can disappear briefly while SR-IOV is changing, but 3145 * config space remains readable. If both are gone, retain the 3146 * stopped state without queueing an endless reset loop. 3147 */ 3148 if (pci_read_config(sc->dev, PCIR_VENDOR, 2) == 0xffff) { 3149 atomic_store_rel_32(&sc->device_reset_state, 3150 IGB_DEVICE_RESET_DETECTED); 3151 device_printf(sc->dev, 3152 "device unavailable after device-wide reset; " 3153 "leaving interface stopped\n"); 3154 return (true); 3155 } 3156 reset_again = true; 3157 } else if (status != 0xffffffff && 3158 (status & E1000_STAT_DEV_RST_SET) != 0) 3159 reset_again = true; 3160 } 3161 if (state == IGB_DEVICE_RESET_DETECTED || 3162 state == IGB_DEVICE_RESET_REQUESTED) 3163 reset_again = true; 3164 if (!reset_again) { 3165 if (state == IGB_DEVICE_RESET_PREPARED && 3166 !atomic_cmpset_rel_32(&sc->device_reset_state, 3167 IGB_DEVICE_RESET_PREPARED, IGB_DEVICE_RESET_NONE)) 3168 return (true); 3169 return (false); 3170 } 3171 3172 state = atomic_swap_32(&sc->device_reset_state, 3173 IGB_DEVICE_RESET_DETECTED); 3174 if (state != IGB_DEVICE_RESET_DETECTED) { 3175 iflib_request_reset_if_up(sc->ctx); 3176 iflib_admin_intr_deferred(sc->ctx); 3177 } 3178 return (true); 3179 } 3180 3181 /* 3182 * A PCIe-region parity failure stops PCIe and DMA traffic. I350, I354, 3183 * I210, and I211 require a port reset before master disable in this case. 3184 * 82580 stops PCIe traffic for a fatal error in any host-owned region, so use 3185 * the same order for every 82580 recovery. This differs from the normal 3186 * reset path, which disables the bus master first. 3187 * 3188 * Indications that relatch after admin accounting are discarded during 3189 * reset; sticky bits cannot distinguish them from the saved event. 3190 */ 3191 static void 3192 em_prepare_fatal_error_reset(struct e1000_softc *sc) 3193 { 3194 struct e1000_hw *hw; 3195 s32 error; 3196 u32 ctrl, pcieecc, pcieerr; 3197 int i; 3198 3199 if (!em_has_peind_memory_errors(&sc->hw) || 3200 atomic_load_acq_32(&sc->fatal_error_state) != 3201 EM_FATAL_ERROR_RESET_REQUESTED) 3202 return; 3203 3204 pcieerr = sc->fatal_error_pcie | 3205 (E1000_READ_REG(&sc->hw, E1000_PCIEERRSTS) & 3206 em_pcie_fatal_error_mask(&sc->hw)); 3207 pcieecc = sc->fatal_error_pcie_ecc; 3208 if (!em_has_82580_memory_errors(&sc->hw) && 3209 (sc->fatal_error_peind & E1000_PEIND_PCIE_PARITY_FATAL) == 0 && 3210 pcieerr == 0) 3211 return; 3212 3213 hw = &sc->hw; 3214 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3215 E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST); 3216 /* Do not access device registers for at least 3 ms after RST. */ 3217 msec_delay(3); 3218 for (i = 0; i < AUTO_READ_DONE_TIMEOUT; i++) { 3219 if ((E1000_READ_REG(hw, E1000_EECD) & 3220 E1000_EECD_AUTO_RD) != 0 && 3221 (em_has_82580_memory_errors(hw) || 3222 (E1000_READ_REG(hw, E1000_STATUS) & 3223 E1000_STATUS_RST_DONE) != 0)) 3224 break; 3225 msec_delay(1); 3226 } 3227 if (i == AUTO_READ_DONE_TIMEOUT) 3228 device_printf(sc->dev, 3229 "port reset did not complete during parity recovery\n"); 3230 error = e1000_disable_pcie_master_generic(hw); 3231 if (error != E1000_SUCCESS) 3232 device_printf(sc->dev, 3233 "PCIe master disable failed during parity recovery: %d\n", 3234 error); 3235 pcieerr |= E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3236 em_pcie_fatal_error_mask(hw); 3237 if (pcieerr != 0) 3238 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3239 if (em_has_82580_memory_errors(hw)) { 3240 pcieecc |= E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3241 E1000_PCIEECCSTS_82580_ERROR_MASK; 3242 if (pcieecc != 0) 3243 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3244 } 3245 atomic_store_rel_32(&sc->fatal_error_state, 3246 EM_FATAL_ERROR_RESET_PREPARED); 3247 } 3248 3249 static void 3250 em_finish_fatal_error_reset(struct e1000_softc *sc) 3251 { 3252 struct e1000_hw *hw; 3253 u32 dma_host, dma_rx, dma_tx, lanerr, pcieecc, pcieerr; 3254 u32 state; 3255 3256 state = atomic_load_acq_32(&sc->fatal_error_state); 3257 if (state != EM_FATAL_ERROR_RESET_REQUESTED && 3258 state != EM_FATAL_ERROR_RESET_PREPARED) 3259 return; 3260 3261 hw = &sc->hw; 3262 if (em_has_82575_memory_errors(hw)) { 3263 sc->fatal_error_dma_tx = 0; 3264 sc->fatal_error_dma_rx = 0; 3265 } else if (em_has_82576_memory_errors(hw)) { 3266 /* Drain any indication relatched while the port was resetting. */ 3267 (void)E1000_READ_REG(hw, E1000_PEIND); 3268 sc->fatal_error_peind = 0; 3269 } else if (em_has_82580_memory_errors(hw)) { 3270 pcieerr = sc->fatal_error_pcie | 3271 E1000_READ_REG(hw, E1000_PCIEERRSTS); 3272 if (pcieerr != 0) 3273 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3274 pcieecc = sc->fatal_error_pcie_ecc | 3275 (E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3276 E1000_PCIEECCSTS_82580_ERROR_MASK); 3277 if (pcieecc != 0) 3278 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3279 dma_tx = sc->fatal_error_dma_tx | 3280 E1000_READ_REG(hw, E1000_DTPARS_82580); 3281 if (dma_tx != 0) 3282 E1000_WRITE_REG(hw, E1000_DTPARS_82580, dma_tx); 3283 dma_rx = sc->fatal_error_dma_rx | 3284 E1000_READ_REG(hw, E1000_DRPARS_82580); 3285 if (dma_rx != 0) 3286 E1000_WRITE_REG(hw, E1000_DRPARS_82580, dma_rx); 3287 dma_host = sc->fatal_error_dma_host | 3288 E1000_READ_REG(hw, E1000_DDPARS_82580); 3289 if (dma_host != 0) 3290 E1000_WRITE_REG(hw, E1000_DDPARS_82580, dma_host); 3291 /* LANPERRSTS is read-only and is cleared by the port reset. */ 3292 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 3293 E1000_LANPERRSTS_82580_ERROR_MASK; 3294 if (lanerr != 0) 3295 device_printf(sc->dev, 3296 "LAN parity status remained set after reset: %#x\n", 3297 lanerr); 3298 (void)E1000_READ_REG(hw, E1000_PEIND); 3299 sc->fatal_error_peind = 0; 3300 sc->fatal_error_pcie = 0; 3301 sc->fatal_error_pcie_ecc = 0; 3302 sc->fatal_error_lan = 0; 3303 sc->fatal_error_dma_tx = 0; 3304 sc->fatal_error_dma_rx = 0; 3305 sc->fatal_error_dma_host = 0; 3306 } else if (em_has_peind_memory_errors(hw)) { 3307 pcieerr = sc->fatal_error_pcie | 3308 (E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3309 em_pcie_fatal_error_mask(hw)); 3310 if (pcieerr != 0) 3311 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3312 if (em_has_i350_i354_memory_errors(hw)) { 3313 dma_tx = sc->fatal_error_dma_tx | 3314 (E1000_READ_REG(hw, E1000_DTPARS) & 3315 E1000_DTPARS_FATAL_MASK); 3316 if (dma_tx != 0) 3317 E1000_WRITE_REG(hw, E1000_DTPARS, dma_tx); 3318 dma_rx = sc->fatal_error_dma_rx | 3319 (E1000_READ_REG(hw, E1000_DRPARS) & 3320 E1000_DRPARS_FATAL_MASK); 3321 if (dma_rx != 0) 3322 E1000_WRITE_REG(hw, E1000_DRPARS, dma_rx); 3323 lanerr = sc->fatal_error_lan | 3324 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3325 E1000_LANPERRSTS_I350_I354_FATAL_MASK); 3326 } else { 3327 lanerr = sc->fatal_error_lan | 3328 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3329 E1000_LANPERRSTS_RETX_BUF); 3330 } 3331 if (lanerr != 0) 3332 E1000_WRITE_REG(hw, E1000_LANPERRSTS, lanerr); 3333 /* 3334 * RST can relatch PEIND from a subordinate status register 3335 * before that register is cleared. Drain the recovered 3336 * indication before unmasking FER. 3337 */ 3338 (void)E1000_READ_REG(hw, E1000_PEIND); 3339 sc->fatal_error_peind = 0; 3340 sc->fatal_error_pcie = 0; 3341 sc->fatal_error_pcie_ecc = 0; 3342 sc->fatal_error_lan = 0; 3343 sc->fatal_error_dma_tx = 0; 3344 sc->fatal_error_dma_rx = 0; 3345 sc->fatal_error_dma_host = 0; 3346 } 3347 sc->fatal_error_icr = 0; 3348 sc->fatal_error_pbeccsts = 0; 3349 atomic_store_rel_32(&sc->fatal_error_state, EM_FATAL_ERROR_NONE); 3350 } 3351 3352 /********************************************************************* 3353 * 3354 * Fast Legacy/MSI Combined Interrupt Service routine 3355 * 3356 *********************************************************************/ 3357 int 3358 em_intr(void *arg) 3359 { 3360 struct e1000_softc *sc = arg; 3361 struct e1000_hw *hw = &sc->hw; 3362 struct em_rx_queue *que = &sc->rx_queues[0]; 3363 struct rx_ring *rxr = &que->rxr; 3364 if_ctx_t ctx = sc->ctx; 3365 u32 reg_icr; 3366 3367 reg_icr = E1000_READ_REG(hw, E1000_ICR); 3368 3369 /* Hot eject? */ 3370 if (reg_icr == 0xffffffff) 3371 return FILTER_STRAY; 3372 3373 /* Definitely not our interrupt. */ 3374 if (reg_icr == 0x0) 3375 return FILTER_STRAY; 3376 3377 /* 3378 * Starting with the 82571 chip, bit 31 should be used to 3379 * determine whether the interrupt belongs to us. 3380 */ 3381 if (hw->mac.type >= e1000_82571 && 3382 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 3383 return FILTER_STRAY; 3384 if (igb_handle_device_reset(sc, reg_icr)) 3385 return (FILTER_HANDLED); 3386 if (igb_device_reset_pending(sc)) 3387 return (FILTER_HANDLED); 3388 3389 /* 3390 * IAM auto-masks igb shared interrupts when ICR is read. Older em 3391 * hardware still needs an explicit disable, which also works around 3392 * MSI message reordering errata on certain systems. 3393 */ 3394 if (sc->vf_ifp || hw->mac.type < igb_mac_min) 3395 IFDI_INTR_DISABLE(ctx); 3396 3397 /* Link status change */ 3398 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3399 em_handle_link(ctx); 3400 3401 if (reg_icr & E1000_ICR_RXO) 3402 sc->rx_overruns++; 3403 3404 em_handle_fatal_error_intr(sc, reg_icr); 3405 3406 if (hw->mac.type >= e1000_82540) 3407 em_newitr(sc, que, rxr); 3408 3409 return (FILTER_SCHEDULE_THREAD); 3410 } 3411 3412 static int 3413 em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3414 { 3415 struct e1000_softc *sc = iflib_get_softc(ctx); 3416 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3417 3418 E1000_WRITE_REG(&sc->hw, E1000_IMS, rxq->eims); 3419 return (0); 3420 } 3421 3422 static int 3423 em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3424 { 3425 struct e1000_softc *sc = iflib_get_softc(ctx); 3426 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3427 3428 E1000_WRITE_REG(&sc->hw, E1000_IMS, txq->eims); 3429 return (0); 3430 } 3431 3432 static int 3433 igb_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3434 { 3435 struct e1000_softc *sc = iflib_get_softc(ctx); 3436 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3437 3438 if (igb_device_reset_pending(sc)) 3439 return (0); 3440 E1000_WRITE_REG(&sc->hw, E1000_EIMS, rxq->eims); 3441 return (0); 3442 } 3443 3444 static int 3445 igb_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3446 { 3447 struct e1000_softc *sc = iflib_get_softc(ctx); 3448 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3449 3450 if (igb_device_reset_pending(sc)) 3451 return (0); 3452 E1000_WRITE_REG(&sc->hw, E1000_EIMS, txq->eims); 3453 return (0); 3454 } 3455 3456 /********************************************************************* 3457 * 3458 * MSI-X RX Interrupt Service routine 3459 * 3460 **********************************************************************/ 3461 static int 3462 em_msix_que(void *arg) 3463 { 3464 struct em_rx_queue *que = arg; 3465 struct e1000_softc *sc = que->sc; 3466 struct rx_ring *rxr = &que->rxr; 3467 3468 ++que->irqs; 3469 3470 if (igb_device_reset_pending(sc)) 3471 return (FILTER_HANDLED); 3472 em_newitr(sc, que, rxr); 3473 3474 return (FILTER_SCHEDULE_THREAD); 3475 } 3476 3477 /********************************************************************* 3478 * 3479 * MSI-X Link Fast Interrupt Service routine 3480 * 3481 **********************************************************************/ 3482 static int 3483 em_msix_link(void *arg) 3484 { 3485 struct e1000_softc *sc = arg; 3486 u32 reg_icr; 3487 3488 ++sc->link_irq; 3489 MPASS(sc->hw.back != NULL); 3490 /* 3491 * The VF's admin vector represents mailbox and link activity. It has 3492 * no PF ICR at E1000_ICR, so process every admin-vector interrupt, 3493 * matching the igbvf misc-vector model. 3494 */ 3495 if (sc->vf_ifp) { 3496 sc->hw.mac.get_link_status = true; 3497 iflib_admin_intr_deferred(sc->ctx); 3498 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3499 return (FILTER_HANDLED); 3500 } 3501 3502 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 3503 if (igb_device_reset_pending(sc)) 3504 return (FILTER_HANDLED); 3505 3506 /* 3507 * Enabling or disabling SR-IOV can briefly make PF MMIO reads return 3508 * all ones. This is not an interrupt cause; in particular, do not 3509 * turn it into a malicious-driver event. 3510 */ 3511 if (__predict_false(reg_icr == 0xffffffff)) 3512 goto rearm; 3513 if (igb_handle_device_reset(sc, reg_icr)) 3514 return (FILTER_HANDLED); 3515 3516 if (reg_icr & E1000_ICR_RXO) 3517 sc->rx_overruns++; 3518 3519 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3520 em_handle_link(sc->ctx); 3521 if (reg_icr & E1000_ICR_MDDET) 3522 igb_iov_mdd_event(sc); 3523 if (reg_icr & E1000_ICR_VMMB) 3524 iflib_admin_intr_deferred(sc->ctx); 3525 em_handle_fatal_error_intr(sc, reg_icr); 3526 3527 rearm: 3528 /* Re-arm unconditionally */ 3529 if (sc->hw.mac.type >= igb_mac_min) { 3530 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3531 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 3532 igb_iov_intr_mask(sc) | 3533 em_fatal_error_intr_mask(sc)); 3534 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3535 } else if (sc->hw.mac.type == e1000_82574) { 3536 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3537 E1000_IMS_LSC | E1000_IMS_OTHER); 3538 /* 3539 * Because we must read the ICR for this interrupt it may 3540 * clear other causes using autoclear, for this reason we 3541 * simply create a soft interrupt for all these vectors. 3542 */ 3543 if (reg_icr) 3544 E1000_WRITE_REG(&sc->hw, E1000_ICS, sc->ims); 3545 } else 3546 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3547 E1000_IMS_LSC | em_fatal_error_intr_mask(sc)); 3548 3549 return (FILTER_HANDLED); 3550 } 3551 3552 static void 3553 em_handle_link(void *context) 3554 { 3555 if_ctx_t ctx = context; 3556 struct e1000_softc *sc = iflib_get_softc(ctx); 3557 3558 sc->hw.mac.get_link_status = 1; 3559 iflib_admin_intr_deferred(ctx); 3560 } 3561 3562 /********************************************************************* 3563 * 3564 * Media Ioctl callback 3565 * 3566 * This routine is called whenever the user queries the status of 3567 * the interface using ifconfig. 3568 * 3569 **********************************************************************/ 3570 static void 3571 em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 3572 { 3573 struct e1000_softc *sc = iflib_get_softc(ctx); 3574 u_char fiber_type = IFM_1000_SX; 3575 3576 INIT_DEBUGOUT("em_if_media_status: begin"); 3577 3578 iflib_admin_intr_deferred(ctx); 3579 3580 ifmr->ifm_status = IFM_AVALID; 3581 ifmr->ifm_active = IFM_ETHER; 3582 3583 if (sc->link_state == EM_LINK_STATE_DOWN || 3584 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING) { 3585 return; 3586 } 3587 3588 ifmr->ifm_status |= IFM_ACTIVE; 3589 3590 if ((sc->hw.phy.media_type == e1000_media_type_fiber) || 3591 (sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3592 if (sc->hw.mac.type == e1000_82545) 3593 fiber_type = IFM_1000_LX; 3594 ifmr->ifm_active |= fiber_type | IFM_FDX; 3595 } else { 3596 switch (sc->link_speed) { 3597 case 10: 3598 ifmr->ifm_active |= IFM_10_T; 3599 break; 3600 case 100: 3601 ifmr->ifm_active |= IFM_100_TX; 3602 break; 3603 case 1000: 3604 ifmr->ifm_active |= IFM_1000_T; 3605 break; 3606 } 3607 if (sc->link_duplex == FULL_DUPLEX) 3608 ifmr->ifm_active |= IFM_FDX; 3609 else 3610 ifmr->ifm_active |= IFM_HDX; 3611 } 3612 } 3613 3614 /********************************************************************* 3615 * 3616 * Media Ioctl callback 3617 * 3618 * This routine is called when the user changes speed/duplex using 3619 * media/mediopt option with ifconfig. 3620 * 3621 **********************************************************************/ 3622 static int 3623 em_if_media_change(if_ctx_t ctx) 3624 { 3625 struct e1000_softc *sc = iflib_get_softc(ctx); 3626 struct ifmedia *ifm = iflib_get_media(ctx); 3627 3628 INIT_DEBUGOUT("em_if_media_change: begin"); 3629 3630 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3631 return (EINVAL); 3632 3633 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3634 case IFM_AUTO: 3635 sc->hw.mac.autoneg = DO_AUTO_NEG; 3636 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 3637 break; 3638 case IFM_1000_LX: 3639 case IFM_1000_SX: 3640 case IFM_1000_T: 3641 sc->hw.mac.autoneg = DO_AUTO_NEG; 3642 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 3643 break; 3644 case IFM_100_TX: 3645 sc->hw.mac.autoneg = false; 3646 sc->hw.phy.autoneg_advertised = 0; 3647 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3648 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 3649 else 3650 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 3651 break; 3652 case IFM_10_T: 3653 sc->hw.mac.autoneg = false; 3654 sc->hw.phy.autoneg_advertised = 0; 3655 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3656 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 3657 else 3658 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 3659 break; 3660 default: 3661 device_printf(sc->dev, "Unsupported media type\n"); 3662 } 3663 3664 return (0); 3665 } 3666 3667 static int 3668 em_if_set_promisc(if_ctx_t ctx, int flags) 3669 { 3670 struct e1000_softc *sc; 3671 3672 sc = iflib_get_softc(ctx); 3673 if (em_if_defer_promisc(sc)) 3674 return (0); 3675 return (em_if_set_promisc_impl(ctx, flags)); 3676 } 3677 3678 static bool 3679 em_if_defer_promisc(struct e1000_softc *sc) 3680 { 3681 switch (sc->hw.mac.type) { 3682 case e1000_82576: 3683 case e1000_i350: 3684 case e1000_vfadapt: 3685 case e1000_vfadapt_i350: 3686 break; 3687 default: 3688 return (false); 3689 } 3690 3691 /* 3692 * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox 3693 * and IOV register operations later from the locked admin task. 3694 * A deferred VF mailbox rejection cannot be returned to ifconfig; the 3695 * admin task logs it instead. 3696 */ 3697 atomic_set_32(&sc->promisc_pending, 1); 3698 iflib_admin_intr_deferred(sc->ctx); 3699 return (true); 3700 } 3701 3702 int 3703 em_if_set_promisc_impl(if_ctx_t ctx, int flags) 3704 { 3705 struct e1000_softc *sc = iflib_get_softc(ctx); 3706 if_t ifp = iflib_get_ifp(ctx); 3707 enum e1000_promisc_type type; 3708 s32 error; 3709 u32 reg_rctl; 3710 int mcnt = 0; 3711 3712 /* Hyper-V receive-mode policy is configured through the host. */ 3713 if (igbv_is_hyperv(sc)) 3714 return (0); 3715 3716 if (sc->vf_ifp) { 3717 if (flags & IFF_PROMISC) 3718 type = e1000_promisc_enabled; 3719 else if (flags & IFF_ALLMULTI) 3720 type = e1000_promisc_multicast; 3721 else 3722 type = e1000_promisc_disabled; 3723 error = e1000_promisc_set_vf(&sc->hw, type); 3724 if (error != E1000_SUCCESS) { 3725 device_printf(sc->dev, 3726 "VF promiscuous-mode request failed\n"); 3727 return (EPERM); 3728 } 3729 return (0); 3730 } 3731 3732 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3733 reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE); 3734 if (flags & IFF_ALLMULTI) 3735 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 3736 else 3737 mcnt = min(if_llmaddr_count(ifp), 3738 MAX_NUM_MULTICAST_ADDRESSES); 3739 3740 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 3741 reg_rctl &= (~E1000_RCTL_MPE); 3742 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3743 3744 if (flags & IFF_PROMISC) { 3745 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3746 /* Turn this on if you want to see bad packets */ 3747 if (em_debug_sbp) 3748 reg_rctl |= E1000_RCTL_SBP; 3749 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3750 if (igb_iov_enabled(sc)) 3751 em_if_vlan_filter_enable(sc); 3752 else 3753 em_if_vlan_filter_disable(sc); 3754 } else { 3755 if (flags & IFF_ALLMULTI) { 3756 reg_rctl |= E1000_RCTL_MPE; 3757 reg_rctl &= ~E1000_RCTL_UPE; 3758 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3759 } 3760 if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx)) 3761 em_if_vlan_filter_enable(sc); 3762 } 3763 igb_iov_update_pf_vmolr(sc); 3764 igb_iov_rebuild_vlan(sc); 3765 return (0); 3766 } 3767 3768 static u_int 3769 em_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 3770 { 3771 u8 *mta = arg; 3772 3773 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 3774 return (0); 3775 3776 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3777 3778 return (1); 3779 } 3780 3781 /* Make every multicast hash eligible on parts whose wake matcher needs MTA. */ 3782 static void 3783 em_fill_wakeup_mta(struct e1000_hw *hw) 3784 { 3785 int i; 3786 3787 memset(hw->mac.mta_shadow, 0xff, sizeof(hw->mac.mta_shadow)); 3788 for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) 3789 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, 3790 hw->mac.mta_shadow[i]); 3791 E1000_WRITE_FLUSH(hw); 3792 } 3793 3794 /********************************************************************* 3795 * Multicast Update 3796 * 3797 * This routine is called whenever multicast address list is updated. 3798 * 3799 **********************************************************************/ 3800 static void 3801 em_if_multi_set(if_ctx_t ctx) 3802 { 3803 struct e1000_softc *sc = iflib_get_softc(ctx); 3804 if_t ifp = iflib_get_ifp(ctx); 3805 u8 *mta; /* Multicast array memory */ 3806 u32 reg_rctl = 0; 3807 int mcnt = 0; 3808 3809 IOCTL_DEBUGOUT("em_set_multi: begin"); 3810 3811 mta = sc->mta; 3812 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 3813 3814 if (sc->hw.mac.type == e1000_82542 && 3815 sc->hw.revision_id == E1000_REVISION_2) { 3816 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3817 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3818 e1000_pci_clear_mwi(&sc->hw); 3819 reg_rctl |= E1000_RCTL_RST; 3820 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3821 msec_delay(5); 3822 } 3823 3824 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta); 3825 3826 if (sc->vf_ifp) { 3827 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3828 igbv_update_uc_addr_list(sc, ifp); 3829 return; 3830 } 3831 3832 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES && 3833 !igb_iov_enabled(sc)) 3834 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3835 3836 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3837 3838 if (if_getflags(ifp) & IFF_PROMISC) 3839 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3840 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 3841 if_getflags(ifp) & IFF_ALLMULTI) { 3842 reg_rctl |= E1000_RCTL_MPE; 3843 reg_rctl &= ~E1000_RCTL_UPE; 3844 } else 3845 reg_rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3846 3847 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3848 3849 if (sc->hw.mac.type == e1000_82542 && 3850 sc->hw.revision_id == E1000_REVISION_2) { 3851 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3852 reg_rctl &= ~E1000_RCTL_RST; 3853 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3854 msec_delay(5); 3855 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3856 e1000_pci_set_mwi(&sc->hw); 3857 } 3858 igb_iov_rebuild_mta(sc); 3859 igb_iov_update_pf_vmolr(sc); 3860 } 3861 3862 /********************************************************************* 3863 * Timer routine 3864 * 3865 * This routine schedules em_if_update_admin_status() to check for 3866 * link status and to gather statistics as well as to perform some 3867 * controller-specific hardware patting. 3868 * 3869 **********************************************************************/ 3870 static void 3871 em_if_timer(if_ctx_t ctx, uint16_t qid) 3872 { 3873 struct e1000_softc *sc; 3874 3875 if (qid != 0) 3876 return; 3877 3878 sc = iflib_get_softc(ctx); 3879 atomic_set_32(&sc->stats_pending, 1); 3880 iflib_admin_intr_deferred(ctx); 3881 } 3882 3883 static void 3884 em_if_update_admin_status(if_ctx_t ctx) 3885 { 3886 struct e1000_softc *sc = iflib_get_softc(ctx); 3887 struct e1000_hw *hw = &sc->hw; 3888 device_t dev = iflib_get_dev(ctx); 3889 u32 link_check, thstat, ctrl; 3890 bool reset_requested = false; 3891 3892 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 3893 if (em_handle_fatal_error_admin(sc)) 3894 return; 3895 /* A sibling-port reset invalidated the registers and VF mailboxes. */ 3896 if (atomic_cmpset_acq_32(&sc->device_reset_state, 3897 IGB_DEVICE_RESET_DETECTED, IGB_DEVICE_RESET_REQUESTED)) { 3898 if (sc->link_state == EM_LINK_STATE_UP) 3899 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 3900 sc->link_speed = 0; 3901 sc->link_duplex = 0; 3902 sc->link_state = EM_LINK_STATE_DOWN_RESET_PENDING; 3903 /* Request the reset here; interrupt filters cannot take STATE_LOCK. */ 3904 iflib_request_reset_if_up(ctx); 3905 /* Re-enter the admin task so it observes the reset request. */ 3906 iflib_admin_intr_deferred(ctx); 3907 return; 3908 } 3909 if (atomic_load_acq_32(&sc->device_reset_state) != 3910 IGB_DEVICE_RESET_NONE) 3911 return; 3912 3913 if (atomic_readandclear_32(&sc->promisc_pending) != 0) 3914 (void)em_if_set_promisc_impl(ctx, 3915 if_getflags(iflib_get_ifp(ctx))); 3916 igb_iov_handle_mdd(sc); 3917 igb_iov_handle_mbx(sc); 3918 3919 link_check = thstat = ctrl = 0; 3920 /* Get the cached link value or read phy for real */ 3921 switch (hw->phy.media_type) { 3922 case e1000_media_type_copper: 3923 if (hw->mac.get_link_status) { 3924 if (hw->mac.type == e1000_pch_spt) 3925 msec_delay(50); 3926 /* Do the work to read phy */ 3927 e1000_check_for_link(hw); 3928 link_check = !hw->mac.get_link_status; 3929 if (link_check) /* ESB2 fix */ 3930 e1000_cfg_on_link_up(hw); 3931 } else { 3932 link_check = true; 3933 } 3934 break; 3935 case e1000_media_type_fiber: 3936 e1000_check_for_link(hw); 3937 link_check = 3938 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 3939 break; 3940 case e1000_media_type_internal_serdes: 3941 e1000_check_for_link(hw); 3942 link_check = hw->mac.serdes_has_link; 3943 break; 3944 default: 3945 break; 3946 } 3947 3948 /* Check for thermal downshift or shutdown */ 3949 if (hw->mac.type == e1000_i350) { 3950 thstat = E1000_READ_REG(hw, E1000_THSTAT); 3951 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); 3952 } 3953 3954 /* Now check for a transition */ 3955 if (link_check && 3956 (sc->link_state == EM_LINK_STATE_DOWN || 3957 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { 3958 bool reset_pending; 3959 3960 reset_pending = 3961 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING; 3962 e1000_get_speed_and_duplex(hw, &sc->link_speed, 3963 &sc->link_duplex); 3964 /* Check if we must disable SPEED_MODE bit on PCI-E */ 3965 if ((sc->link_speed != SPEED_1000) && 3966 ((hw->mac.type == e1000_82571) || 3967 (hw->mac.type == e1000_82572))) { 3968 int tarc0; 3969 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 3970 tarc0 &= ~TARC_SPEED_MODE_BIT; 3971 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 3972 } 3973 if (bootverbose) 3974 device_printf(dev, "Link is up %d Mbps %s\n", 3975 sc->link_speed, 3976 ((sc->link_duplex == FULL_DUPLEX) ? 3977 "Full Duplex" : "Half Duplex")); 3978 sc->link_state = EM_LINK_STATE_UP; 3979 sc->smartspeed = 0; 3980 if (hw->mac.type == e1000_i350 && 3981 (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == 3982 E1000_CTRL_EXT_LINK_MODE_GMII && 3983 (thstat & E1000_THSTAT_LINK_THROTTLE)) 3984 device_printf(dev, "Link: thermal downshift\n"); 3985 /* Delay Link Up for Phy update */ 3986 if (((hw->mac.type == e1000_i210) || 3987 (hw->mac.type == e1000_i211)) && 3988 (hw->phy.id == I210_I_PHY_ID)) 3989 msec_delay(I210_LINK_DELAY); 3990 /* Reset if the media type changed. */ 3991 if (hw->dev_spec._82575.media_changed && 3992 hw->mac.type >= igb_mac_min) { 3993 hw->dev_spec._82575.media_changed = false; 3994 sc->flags |= IGB_MEDIA_RESET; 3995 iflib_request_reset(ctx); 3996 iflib_admin_intr_deferred(ctx); 3997 reset_requested = true; 3998 } 3999 /* Only do TSO on gigabit for older chips due to errata */ 4000 if (hw->mac.type < igb_mac_min) 4001 reset_requested = em_automask_tso(ctx); 4002 4003 if (reset_pending || reset_requested) { 4004 /* 4005 * The PHY is up, but publish it only after the TSO 4006 * capability-change reset. 4007 */ 4008 sc->link_state = EM_LINK_STATE_UP_RESET_PENDING; 4009 } else { 4010 iflib_link_state_change(ctx, LINK_STATE_UP, 4011 IF_Mbps(sc->link_speed)); 4012 } 4013 igb_iov_ping_all_vfs(sc); 4014 } else if (!link_check && 4015 (sc->link_state == EM_LINK_STATE_UP || 4016 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { 4017 bool link_was_published; 4018 bool reset_pending; 4019 4020 link_was_published = sc->link_state == EM_LINK_STATE_UP; 4021 reset_pending = 4022 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING; 4023 sc->link_speed = 0; 4024 sc->link_duplex = 0; 4025 sc->link_state = reset_pending ? 4026 EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN; 4027 if (link_was_published) 4028 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4029 igb_iov_ping_all_vfs(sc); 4030 } 4031 /* 4032 * Mailbox, link, and timer events share this admin task. The PF 4033 * statistics sweep performs 66 MMIO reads, so run it only when the 4034 * ordinary iflib timer requests a sample rather than once per mailbox 4035 * message. Exported counters can consequently trail hardware by the 4036 * timer interval (normally 500 ms). 4037 */ 4038 if (atomic_readandclear_32(&sc->stats_pending) != 0) { 4039 em_update_stats_counters(sc); 4040 /* 4041 * The 82574 PHY can enter a state in which both its receive 4042 * error and idle error counters saturate. Require two 4043 * consecutive timer samples before resetting, matching Intel's 4044 * e1000e recovery policy and avoiding a reset on a transient 4045 * register sample. 4046 */ 4047 if (hw->mac.type == e1000_82574) { 4048 if (e1000_check_phy_82574(hw)) 4049 sc->phy_hang_count++; 4050 else 4051 sc->phy_hang_count = 0; 4052 if (sc->phy_hang_count > 1) { 4053 sc->phy_hang_count = 0; 4054 device_printf(dev, 4055 "PHY appears hung; requesting reset\n"); 4056 iflib_request_reset(ctx); 4057 iflib_admin_intr_deferred(ctx); 4058 return; 4059 } 4060 } 4061 } 4062 4063 /* Reset LAA into RAR[0] on 82571 */ 4064 if (hw->mac.type == e1000_82571 && e1000_get_laa_state_82571(hw)) 4065 e1000_rar_set(hw, hw->mac.addr, 0); 4066 4067 if (hw->mac.type < em_mac_min) 4068 lem_smartspeed(sc); 4069 } 4070 4071 /* 4072 * Last-resort DMA fence. iflib releases DMA mappings after the driver's 4073 * stop callback, so continuing with bus mastering still enabled would turn 4074 * a recoverable NIC failure into memory corruption. Treat failure of the 4075 * PCI command bit as a fail-stop invariant violation. 4076 */ 4077 static void 4078 em_fence_pci_busmaster(struct e1000_softc *sc) 4079 { 4080 device_t dev; 4081 u_int timeout; 4082 u16 command; 4083 int error; 4084 4085 dev = sc->dev; 4086 error = pci_disable_busmaster(dev); 4087 command = pci_read_config(dev, PCIR_COMMAND, 2); 4088 if (command != 0xffff && (command & PCIM_CMD_BUSMASTEREN) != 0) 4089 panic("%s: unable to fence device DMA (error %d)", 4090 device_get_nameunit(dev), error); 4091 if (error != 0 && command != 0xffff) 4092 device_printf(dev, 4093 "PCI bus-master disable returned %d; readback is disabled\n", 4094 error); 4095 4096 timeout = max(pcie_get_max_completion_timeout(dev) / 1000, 10); 4097 if (command != 0xffff && 4098 !pcie_wait_for_pending_transactions(dev, timeout)) { 4099 /* A function removed during the wait can no longer issue DMA. */ 4100 command = pci_read_config(dev, PCIR_COMMAND, 2); 4101 if (command != 0xffff) 4102 panic("%s: DMA transactions remain pending after fencing", 4103 device_get_nameunit(dev)); 4104 } 4105 } 4106 4107 static int 4108 em_enable_pci_busmaster(struct e1000_softc *sc) 4109 { 4110 device_t dev; 4111 u16 command; 4112 int error; 4113 4114 dev = sc->dev; 4115 command = pci_read_config(dev, PCIR_COMMAND, 2); 4116 if (command == 0xffff) 4117 return (ENXIO); 4118 if ((command & PCIM_CMD_BUSMASTEREN) != 0) 4119 return (0); 4120 4121 error = pci_enable_busmaster(dev); 4122 command = pci_read_config(dev, PCIR_COMMAND, 2); 4123 if (command == 0xffff) 4124 return (ENXIO); 4125 if ((command & PCIM_CMD_BUSMASTEREN) == 0) 4126 return (error != 0 ? error : EIO); 4127 return (0); 4128 } 4129 4130 /********************************************************************* 4131 * 4132 * This routine disables all traffic on the adapter by issuing a 4133 * global reset on the MAC. 4134 * 4135 **********************************************************************/ 4136 static void 4137 em_if_stop(if_ctx_t ctx) 4138 { 4139 struct e1000_softc *sc = iflib_get_softc(ctx); 4140 s32 error; 4141 4142 INIT_DEBUGOUT("em_if_stop: begin"); 4143 4144 if (sc->vf_ifp) { 4145 igbv_queue_retry_stop(sc); 4146 igbv_mbx_retry_stop(sc); 4147 } 4148 4149 /* I219 needs special flushing to avoid hangs */ 4150 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 4151 em_flush_desc_rings(sc); 4152 4153 igb_iov_reset_prepare(sc); 4154 if (!sc->vf_ifp || 4155 (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 && 4156 (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) { 4157 em_prepare_fatal_error_reset(sc); 4158 error = e1000_reset_hw(&sc->hw); 4159 if (!sc->vf_ifp && error != E1000_SUCCESS) { 4160 device_printf(sc->dev, "Hardware reset failed while " 4161 "stopping: %d\n", error); 4162 em_fence_pci_busmaster(sc); 4163 return; 4164 } 4165 } 4166 if (sc->vf_ifp) { 4167 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 4168 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 4169 if (!sc->vf_queues_sanitized) 4170 em_fence_pci_busmaster(sc); 4171 } 4172 if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp) 4173 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 4174 4175 if (!sc->vf_ifp) { 4176 e1000_led_off(&sc->hw); 4177 e1000_cleanup_led(&sc->hw); 4178 } else { 4179 sc->link_speed = 0; 4180 sc->link_duplex = 0; 4181 if (sc->link_state != EM_LINK_STATE_DOWN) { 4182 sc->link_state = EM_LINK_STATE_DOWN; 4183 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4184 } 4185 } 4186 } 4187 4188 /********************************************************************* 4189 * 4190 * Determine hardware revision. 4191 * 4192 **********************************************************************/ 4193 static void 4194 em_identify_hardware(if_ctx_t ctx) 4195 { 4196 device_t dev = iflib_get_dev(ctx); 4197 struct e1000_softc *sc = iflib_get_softc(ctx); 4198 4199 /* Make sure our PCI config space has the necessary stuff set */ 4200 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 4201 4202 /* Save off the information about this board */ 4203 sc->hw.vendor_id = pci_get_vendor(dev); 4204 sc->hw.device_id = pci_get_device(dev); 4205 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 4206 sc->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 4207 sc->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 4208 4209 /* Do Shared Code Init and Setup */ 4210 if (e1000_set_mac_type(&sc->hw)) { 4211 device_printf(dev, "Setup init failure\n"); 4212 return; 4213 } 4214 4215 /* 4216 * Function type comes from the selected iflib shared context, not from 4217 * enum ordering. Keep the detected MAC type as an independent check 4218 * that the igb/igbv probe tables selected the right policy. 4219 */ 4220 KASSERT(sc->vf_ifp == 4221 (sc->hw.mac.type == e1000_vfadapt || 4222 sc->hw.mac.type == e1000_vfadapt_i350), 4223 ("%s: iflib function type and MAC type disagree", __func__)); 4224 } 4225 4226 static int 4227 em_allocate_pci_resources(if_ctx_t ctx) 4228 { 4229 struct e1000_softc *sc = iflib_get_softc(ctx); 4230 device_t dev = iflib_get_dev(ctx); 4231 int rid, val; 4232 4233 rid = PCIR_BAR(0); 4234 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 4235 RF_ACTIVE); 4236 if (sc->memory == NULL) { 4237 device_printf(dev, 4238 "Unable to allocate bus resource: memory\n"); 4239 return (ENXIO); 4240 } 4241 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 4242 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 4243 #ifdef INVARIANTS 4244 sc->osdep.mem_bus_space_size = rman_get_size(sc->memory); 4245 #endif 4246 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 4247 4248 /* Only older adapters use IO mapping */ 4249 if (sc->hw.mac.type < em_mac_min && sc->hw.mac.type > e1000_82543) { 4250 /* Figure our where our IO BAR is ? */ 4251 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 4252 val = pci_read_config(dev, rid, 4); 4253 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 4254 break; 4255 } 4256 rid += 4; 4257 /* check for 64bit BAR */ 4258 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 4259 rid += 4; 4260 } 4261 if (rid >= PCIR_CIS) { 4262 device_printf(dev, "Unable to locate IO BAR\n"); 4263 return (ENXIO); 4264 } 4265 sc->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 4266 &rid, RF_ACTIVE); 4267 if (sc->ioport == NULL) { 4268 device_printf(dev, 4269 "Unable to allocate bus resource: ioport\n"); 4270 return (ENXIO); 4271 } 4272 sc->hw.io_base = 0; 4273 sc->osdep.io_bus_space_tag = 4274 rman_get_bustag(sc->ioport); 4275 sc->osdep.io_bus_space_handle = 4276 rman_get_bushandle(sc->ioport); 4277 } 4278 4279 sc->hw.back = &sc->osdep; 4280 4281 return (0); 4282 } 4283 4284 /********************************************************************* 4285 * 4286 * Set up the MSI-X Interrupt handlers 4287 * 4288 **********************************************************************/ 4289 static int 4290 em_if_msix_intr_assign(if_ctx_t ctx, int msix) 4291 { 4292 struct e1000_softc *sc = iflib_get_softc(ctx); 4293 struct em_rx_queue *rx_que = sc->rx_queues; 4294 struct em_tx_queue *tx_que = sc->tx_queues; 4295 int error, rid, i, vector = 0, rx_vectors; 4296 char buf[16]; 4297 4298 /* First set up ring resources */ 4299 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 4300 rid = vector + 1; 4301 snprintf(buf, sizeof(buf), "rxq%d", i); 4302 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 4303 IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); 4304 if (error) { 4305 device_printf(iflib_get_dev(ctx), 4306 "Failed to allocate que int %d err: %d", 4307 i, error); 4308 sc->rx_num_queues = i + 1; 4309 goto fail; 4310 } 4311 4312 rx_que->msix = vector; 4313 4314 /* 4315 * Set the bit to enable interrupt 4316 * in E1000_IMS -- bits 20 and 21 4317 * are for RX0 and RX1, note this has 4318 * NOTHING to do with the MSI-X vector 4319 */ 4320 if (sc->hw.mac.type == e1000_82574) { 4321 rx_que->eims = 1 << (20 + i); 4322 sc->ims |= rx_que->eims; 4323 sc->ivars |= (8 | rx_que->msix) << (i * 4); 4324 } else if (sc->hw.mac.type == e1000_82575) 4325 rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; 4326 else 4327 rx_que->eims = 1 << vector; 4328 } 4329 rx_vectors = vector; 4330 4331 vector = 0; 4332 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 4333 snprintf(buf, sizeof(buf), "txq%d", i); 4334 tx_que = &sc->tx_queues[i]; 4335 iflib_softirq_alloc_generic(ctx, 4336 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 4337 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 4338 4339 tx_que->msix = (vector % sc->rx_num_queues); 4340 4341 /* 4342 * Set the bit to enable interrupt 4343 * in E1000_IMS -- bits 22 and 23 4344 * are for TX0 and TX1, note this has 4345 * NOTHING to do with the MSI-X vector 4346 */ 4347 if (sc->hw.mac.type == e1000_82574) { 4348 tx_que->eims = 1 << (22 + i); 4349 sc->ims |= tx_que->eims; 4350 sc->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); 4351 } else if (sc->hw.mac.type == e1000_82575) { 4352 tx_que->eims = E1000_EICR_TX_QUEUE0 << i; 4353 } else { 4354 tx_que->eims = 1 << i; 4355 } 4356 } 4357 4358 /* Link interrupt */ 4359 rid = rx_vectors + 1; 4360 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 4361 em_msix_link, sc, 0, "aq"); 4362 4363 if (error) { 4364 device_printf(iflib_get_dev(ctx), 4365 "Failed to register admin handler"); 4366 goto fail; 4367 } 4368 sc->linkvec = rx_vectors; 4369 if (sc->hw.mac.type < igb_mac_min) { 4370 sc->ivars |= (8 | rx_vectors) << 16; 4371 sc->ivars |= 0x80000000; 4372 /* Enable the "Other" interrupt type for link status change */ 4373 sc->ims |= E1000_IMS_OTHER; 4374 } 4375 4376 return (0); 4377 fail: 4378 iflib_irq_free(ctx, &sc->irq); 4379 rx_que = sc->rx_queues; 4380 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 4381 iflib_irq_free(ctx, &rx_que->que_irq); 4382 return (error); 4383 } 4384 4385 static void 4386 igb_configure_queues(struct e1000_softc *sc) 4387 { 4388 struct e1000_hw *hw = &sc->hw; 4389 struct em_rx_queue *rx_que; 4390 struct em_tx_queue *tx_que; 4391 u32 tmp, ivar = 0; 4392 4393 /* 4394 * Queue ownership can change when SR-IOV is enabled or disabled. 4395 * Rebuild the interrupt mask for the current layout instead of 4396 * retaining vectors from a previous initialization. 4397 */ 4398 sc->que_mask = 0; 4399 sc->link_mask = 0; 4400 4401 /* GPIE controls the PF interrupt block and is not in the VF BAR. */ 4402 if (!sc->vf_ifp && hw->mac.type != e1000_82575) 4403 E1000_WRITE_REG(hw, E1000_GPIE, 4404 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 4405 E1000_GPIE_PBA | E1000_GPIE_NSICR); 4406 4407 /* Turn on MSI-X */ 4408 switch (hw->mac.type) { 4409 case e1000_82580: 4410 case e1000_i350: 4411 case e1000_i354: 4412 case e1000_i210: 4413 case e1000_i211: 4414 case e1000_vfadapt: 4415 case e1000_vfadapt_i350: 4416 /* RX entries */ 4417 for (int i = 0; i < sc->rx_num_queues; i++) { 4418 uint32_t index, qid; 4419 4420 rx_que = &sc->rx_queues[i]; 4421 qid = rx_que->rxr.me; 4422 index = qid >> 1; 4423 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4424 if (qid & 1) { 4425 ivar &= 0xFF00FFFF; 4426 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4427 16; 4428 } else { 4429 ivar &= 0xFFFFFF00; 4430 ivar |= rx_que->msix | E1000_IVAR_VALID; 4431 } 4432 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4433 sc->que_mask |= rx_que->eims; 4434 } 4435 /* TX entries */ 4436 for (int i = 0; i < sc->tx_num_queues; i++) { 4437 uint32_t index, qid; 4438 4439 tx_que = &sc->tx_queues[i]; 4440 qid = tx_que->txr.me; 4441 index = qid >> 1; 4442 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4443 if (qid & 1) { 4444 ivar &= 0x00FFFFFF; 4445 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4446 24; 4447 } else { 4448 ivar &= 0xFFFF00FF; 4449 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4450 8; 4451 } 4452 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4453 sc->que_mask |= tx_que->eims; 4454 } 4455 4456 /* And for the link interrupt */ 4457 if (sc->vf_ifp) { 4458 /* 4459 * VTIVAR_MISC maps the VF mailbox in bits 7:0. 4460 * The PF IVAR_MISC maps other causes in bits 15:8. 4461 */ 4462 ivar = sc->linkvec | E1000_IVAR_VALID; 4463 } else 4464 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4465 sc->link_mask = 1 << sc->linkvec; 4466 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4467 break; 4468 case e1000_82576: 4469 /* RX entries */ 4470 for (int i = 0; i < sc->rx_num_queues; i++) { 4471 uint32_t index, qid; 4472 4473 rx_que = &sc->rx_queues[i]; 4474 qid = rx_que->rxr.me; 4475 index = qid & 0x7; /* Each IVAR has two entries */ 4476 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4477 if (qid < 8) { 4478 ivar &= 0xFFFFFF00; 4479 ivar |= rx_que->msix | E1000_IVAR_VALID; 4480 } else { 4481 ivar &= 0xFF00FFFF; 4482 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4483 16; 4484 } 4485 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4486 sc->que_mask |= rx_que->eims; 4487 } 4488 /* TX entries */ 4489 for (int i = 0; i < sc->tx_num_queues; i++) { 4490 uint32_t index, qid; 4491 4492 tx_que = &sc->tx_queues[i]; 4493 qid = tx_que->txr.me; 4494 index = qid & 0x7; /* Each IVAR has two entries */ 4495 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4496 if (qid < 8) { 4497 ivar &= 0xFFFF00FF; 4498 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4499 8; 4500 } else { 4501 ivar &= 0x00FFFFFF; 4502 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4503 24; 4504 } 4505 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4506 sc->que_mask |= tx_que->eims; 4507 } 4508 4509 /* And for the link interrupt */ 4510 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4511 sc->link_mask = 1 << sc->linkvec; 4512 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4513 break; 4514 4515 case e1000_82575: 4516 /* enable MSI-X support*/ 4517 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 4518 tmp |= E1000_CTRL_EXT_PBA_CLR; 4519 /* Auto-Mask interrupts upon ICR read. */ 4520 tmp |= E1000_CTRL_EXT_EIAME; 4521 tmp |= E1000_CTRL_EXT_IRCA; 4522 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 4523 4524 /* Queues */ 4525 for (int i = 0; i < sc->rx_num_queues; i++) { 4526 rx_que = &sc->rx_queues[i]; 4527 tmp = E1000_EICR_RX_QUEUE0 << i; 4528 tmp |= E1000_EICR_TX_QUEUE0 << i; 4529 rx_que->eims = tmp; 4530 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), i, 4531 rx_que->eims); 4532 sc->que_mask |= rx_que->eims; 4533 } 4534 4535 /* Link */ 4536 E1000_WRITE_REG(hw, E1000_MSIXBM(sc->linkvec), 4537 E1000_EIMS_OTHER); 4538 sc->link_mask |= E1000_EIMS_OTHER; 4539 default: 4540 break; 4541 } 4542 4543 return; 4544 } 4545 4546 static void 4547 igb_initialize_interrupt_rate(struct e1000_softc *sc) 4548 { 4549 struct e1000_hw *hw = &sc->hw; 4550 struct em_rx_queue *rx_que; 4551 u32 newitr; 4552 4553 newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); 4554 if (hw->mac.type == e1000_82575) 4555 newitr |= newitr << 16; 4556 else 4557 newitr |= E1000_EITR_CNT_IGNR; 4558 4559 for (int i = 0; i < sc->rx_num_queues; i++) { 4560 rx_que = &sc->rx_queues[i]; 4561 rx_que->itr_setting = newitr; 4562 E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), 4563 rx_que->itr_setting); 4564 } 4565 if (sc->intr_type == IFLIB_INTR_MSIX) 4566 E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr); 4567 } 4568 4569 static void 4570 em_free_pci_resources(if_ctx_t ctx) 4571 { 4572 struct e1000_softc *sc = iflib_get_softc(ctx); 4573 struct em_rx_queue *que = sc->rx_queues; 4574 device_t dev = iflib_get_dev(ctx); 4575 4576 /* Release all MSI-X queue resources */ 4577 if (sc->intr_type == IFLIB_INTR_MSIX) 4578 iflib_irq_free(ctx, &sc->irq); 4579 4580 if (que != NULL) { 4581 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 4582 iflib_irq_free(ctx, &que->que_irq); 4583 } 4584 } 4585 4586 if (sc->memory != NULL) { 4587 bus_release_resource(dev, SYS_RES_MEMORY, 4588 rman_get_rid(sc->memory), sc->memory); 4589 sc->memory = NULL; 4590 } 4591 4592 if (sc->flash != NULL) { 4593 bus_release_resource(dev, SYS_RES_MEMORY, 4594 rman_get_rid(sc->flash), sc->flash); 4595 sc->flash = NULL; 4596 } 4597 4598 if (sc->ioport != NULL) { 4599 bus_release_resource(dev, SYS_RES_IOPORT, 4600 rman_get_rid(sc->ioport), sc->ioport); 4601 sc->ioport = NULL; 4602 } 4603 } 4604 4605 /* Set up MSI or MSI-X */ 4606 static int 4607 em_setup_msix(if_ctx_t ctx) 4608 { 4609 struct e1000_softc *sc = iflib_get_softc(ctx); 4610 4611 if (sc->hw.mac.type == e1000_82574) { 4612 em_enable_vectors_82574(ctx); 4613 } 4614 return (0); 4615 } 4616 4617 /********************************************************************* 4618 * 4619 * Workaround for SmartSpeed on 82541 and 82547 controllers 4620 * 4621 **********************************************************************/ 4622 static void 4623 lem_smartspeed(struct e1000_softc *sc) 4624 { 4625 u16 phy_tmp; 4626 4627 if (sc->link_state == EM_LINK_STATE_UP || 4628 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING || 4629 (sc->hw.phy.type != e1000_phy_igp) || 4630 sc->hw.mac.autoneg == 0 || 4631 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 4632 return; 4633 4634 if (sc->smartspeed == 0) { 4635 /* If Master/Slave config fault is asserted twice, 4636 * we assume back-to-back */ 4637 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4638 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 4639 return; 4640 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4641 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 4642 e1000_read_phy_reg(&sc->hw, 4643 PHY_1000T_CTRL, &phy_tmp); 4644 if(phy_tmp & CR_1000T_MS_ENABLE) { 4645 phy_tmp &= ~CR_1000T_MS_ENABLE; 4646 e1000_write_phy_reg(&sc->hw, 4647 PHY_1000T_CTRL, phy_tmp); 4648 sc->smartspeed++; 4649 if(sc->hw.mac.autoneg && 4650 !e1000_copper_link_autoneg(&sc->hw) && 4651 !e1000_read_phy_reg(&sc->hw, 4652 PHY_CONTROL, &phy_tmp)) { 4653 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4654 MII_CR_RESTART_AUTO_NEG); 4655 e1000_write_phy_reg(&sc->hw, 4656 PHY_CONTROL, phy_tmp); 4657 } 4658 } 4659 } 4660 return; 4661 } else if(sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 4662 /* If still no link, perhaps using 2/3 pair cable */ 4663 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 4664 phy_tmp |= CR_1000T_MS_ENABLE; 4665 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 4666 if(sc->hw.mac.autoneg && 4667 !e1000_copper_link_autoneg(&sc->hw) && 4668 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 4669 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4670 MII_CR_RESTART_AUTO_NEG); 4671 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 4672 } 4673 } 4674 /* Restart process after EM_SMARTSPEED_MAX iterations */ 4675 if(sc->smartspeed++ == EM_SMARTSPEED_MAX) 4676 sc->smartspeed = 0; 4677 } 4678 4679 static void 4680 igb_disable_dmac(struct e1000_hw *hw) 4681 { 4682 u32 reg; 4683 4684 reg = E1000_READ_REG(hw, E1000_DMACR); 4685 reg &= ~E1000_DMACR_DMAC_EN; 4686 /* Retain the documented Lx policy and I210 reserved encoding. */ 4687 reg |= E1000_DMACR_DMAC_LX_MASK; 4688 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4689 } 4690 4691 /********************************************************************* 4692 * 4693 * Initialize the DMA Coalescing feature 4694 * 4695 **********************************************************************/ 4696 static void 4697 igb_init_dmac(struct e1000_softc *sc, u32 pba) 4698 { 4699 device_t dev = sc->dev; 4700 struct e1000_hw *hw = &sc->hw; 4701 u32 dmac, dmacwt, reg, ttlx; 4702 u16 hwm; 4703 u16 max_frame_size; 4704 4705 KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF", 4706 __func__)); 4707 4708 if (hw->mac.type == e1000_i211) 4709 return; 4710 4711 /* 4712 * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve 4713 * the configured value so it can be restored after IOV is disabled. 4714 */ 4715 if (igb_iov_enabled(sc)) { 4716 if (hw->mac.type > e1000_82580) 4717 igb_disable_dmac(hw); 4718 return; 4719 } 4720 4721 max_frame_size = sc->shared->isc_max_frame_size; 4722 if (hw->mac.type > e1000_82580) { 4723 4724 if (sc->dmac == 0) { /* Disabling it */ 4725 igb_disable_dmac(hw); 4726 return; 4727 } else 4728 device_printf(dev, "DMA Coalescing enabled\n"); 4729 4730 /* Set starting threshold */ 4731 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); 4732 4733 hwm = 64 * pba - max_frame_size / 16; 4734 if (hwm < 64 * (pba - 6)) 4735 hwm = 64 * (pba - 6); 4736 reg = E1000_READ_REG(hw, E1000_FCRTC); 4737 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 4738 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 4739 & E1000_FCRTC_RTH_COAL_MASK); 4740 E1000_WRITE_REG(hw, E1000_FCRTC, reg); 4741 4742 4743 dmac = pba - max_frame_size / 512; 4744 if (dmac < pba - 10) 4745 dmac = pba - 10; 4746 reg = E1000_READ_REG(hw, E1000_DMACR); 4747 reg &= ~(E1000_DMACR_DMACWT_MASK | 4748 E1000_DMACR_DMACTHR_MASK | E1000_DMACR_DMAC_LX_MASK | 4749 E1000_DMACR_DMAC_EN | E1000_DMACR_DC_LPBKW_EN | 4750 E1000_DMACR_DC_BMC2OSW_EN); 4751 reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT) 4752 & E1000_DMACR_DMACTHR_MASK); 4753 4754 /* Transition to L0s or L1 if available. */ 4755 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 4756 4757 /* 4758 * The watchdog uses 12.8 usec units on an I354 2.5 Gb/s 4759 * backplane connection and 32 usec units otherwise. 4760 */ 4761 if (hw->mac.type == e1000_i354) { 4762 int status = E1000_READ_REG(hw, E1000_STATUS); 4763 if ((status & E1000_STATUS_2P5_SKU) && 4764 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4765 dmacwt = (sc->dmac * 5) >> 6; 4766 else 4767 dmacwt = sc->dmac >> 5; 4768 } else { 4769 dmacwt = sc->dmac >> 5; 4770 } 4771 reg |= dmacwt & E1000_DMACR_DMACWT_MASK; 4772 if (hw->mac.type == e1000_i350 || 4773 hw->mac.type == e1000_i354) 4774 reg |= E1000_DMACR_DC_LPBKW_EN; 4775 if (hw->mac.type == e1000_i354) 4776 reg |= E1000_DMACR_DC_BMC2OSW_EN; 4777 4778 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4779 4780 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); 4781 4782 /* Set the interval before transition. */ 4783 reg = E1000_READ_REG(hw, E1000_DMCTLX); 4784 reg &= ~E1000_DMCTLX_TTLX_MASK; 4785 if (hw->mac.type == e1000_i350) 4786 reg |= IGB_DMCTLX_DCFLUSH_DIS; 4787 /* 4788 * I210 documents TTLX as reserved with a required value of 0x20. 4789 * At 2.5 Gb/s the I354 unit is 0.4 usec, so ten ticks retain 4790 * the four usec interval used at other speeds. 4791 */ 4792 if (hw->mac.type == e1000_i210) { 4793 ttlx = 0x20; 4794 } else if (hw->mac.type == e1000_i354) { 4795 int status = E1000_READ_REG(hw, E1000_STATUS); 4796 if ((status & E1000_STATUS_2P5_SKU) && 4797 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4798 ttlx = 0xA; 4799 else 4800 ttlx = 0x4; 4801 } else { 4802 ttlx = 0x4; 4803 } 4804 reg |= ttlx & E1000_DMCTLX_TTLX_MASK; 4805 4806 E1000_WRITE_REG(hw, E1000_DMCTLX, reg); 4807 4808 /* free space in tx packet buffer to wake from DMA coal */ 4809 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - 4810 (2 * max_frame_size)) >> 6); 4811 4812 /* make low power state decision controlled by DMA coal */ 4813 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4814 reg |= E1000_PCIEMISC_LX_DECISION; 4815 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); 4816 4817 } else if (hw->mac.type == e1000_82580) { 4818 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4819 E1000_WRITE_REG(hw, E1000_PCIEMISC, 4820 reg & ~E1000_PCIEMISC_LX_DECISION); 4821 E1000_WRITE_REG(hw, E1000_DMACR, 0); 4822 } 4823 } 4824 /********************************************************************* 4825 * The 3 following flush routines are used as a workaround in the 4826 * I219 client parts and only for them. 4827 * 4828 * em_flush_tx_ring - remove all descriptors from the tx_ring 4829 * 4830 * We want to clear all pending descriptors from the TX ring. 4831 * zeroing happens when the HW reads the regs. We assign the ring itself as 4832 * the data of the next descriptor. We don't care about the data we are about 4833 * to reset the HW. 4834 **********************************************************************/ 4835 static void 4836 em_flush_tx_ring(struct e1000_softc *sc) 4837 { 4838 struct e1000_hw *hw = &sc->hw; 4839 struct tx_ring *txr = &sc->tx_queues->txr; 4840 struct e1000_tx_desc *txd; 4841 u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; 4842 u16 size = 512; 4843 4844 tctl = E1000_READ_REG(hw, E1000_TCTL); 4845 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); 4846 4847 txd = &txr->tx_base[txr->tx_cidx_processed]; 4848 4849 /* Just use the ring as a dummy buffer addr */ 4850 txd->buffer_addr = txr->tx_paddr; 4851 txd->lower.data = htole32(txd_lower | size); 4852 txd->upper.data = 0; 4853 4854 /* flush descriptors to memory before notifying the HW */ 4855 wmb(); 4856 4857 E1000_WRITE_REG(hw, E1000_TDT(0), txr->tx_cidx_processed); 4858 mb(); 4859 usec_delay(250); 4860 } 4861 4862 /********************************************************************* 4863 * em_flush_rx_ring - remove all descriptors from the rx_ring 4864 * 4865 * Mark all descriptors in the RX ring as consumed and disable the rx ring 4866 **********************************************************************/ 4867 static void 4868 em_flush_rx_ring(struct e1000_softc *sc) 4869 { 4870 struct e1000_hw *hw = &sc->hw; 4871 u32 rctl, rxdctl; 4872 4873 rctl = E1000_READ_REG(hw, E1000_RCTL); 4874 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4875 E1000_WRITE_FLUSH(hw); 4876 usec_delay(150); 4877 4878 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 4879 /* zero the lower 14 bits (prefetch and host thresholds) */ 4880 rxdctl &= 0xffffc000; 4881 /* 4882 * update thresholds: prefetch threshold to 31, host threshold to 1 4883 * and make sure the granularity is "descriptors" and not 4884 * "cache lines" 4885 */ 4886 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); 4887 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 4888 4889 /* momentarily enable the RX ring for the changes to take effect */ 4890 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); 4891 E1000_WRITE_FLUSH(hw); 4892 usec_delay(150); 4893 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4894 } 4895 4896 /********************************************************************* 4897 * em_flush_desc_rings - remove all descriptors from the descriptor rings 4898 * 4899 * In I219, the descriptor rings must be emptied before resetting the HW 4900 * or before changing the device state to D3 during runtime (runtime PM). 4901 * 4902 * Failure to do this will cause the HW to enter a unit hang state which can 4903 * only be released by PCI reset on the device 4904 * 4905 **********************************************************************/ 4906 static void 4907 em_flush_desc_rings(struct e1000_softc *sc) 4908 { 4909 struct e1000_hw *hw = &sc->hw; 4910 device_t dev = sc->dev; 4911 u16 hang_state; 4912 u32 fext_nvm11, tdlen; 4913 4914 /* First, disable MULR fix in FEXTNVM11 */ 4915 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); 4916 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 4917 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); 4918 4919 /* do nothing if we're not in faulty state, or the queue is empty */ 4920 tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); 4921 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4922 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 4923 return; 4924 em_flush_tx_ring(sc); 4925 4926 /* recheck, maybe the fault is caused by the rx ring */ 4927 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4928 if (hang_state & FLUSH_DESC_REQUIRED) 4929 em_flush_rx_ring(sc); 4930 } 4931 4932 4933 /********************************************************************* 4934 * 4935 * Initialize the hardware to a configuration as specified by the 4936 * sc structure. 4937 * 4938 **********************************************************************/ 4939 static int 4940 em_reset(if_ctx_t ctx) 4941 { 4942 device_t dev = iflib_get_dev(ctx); 4943 struct e1000_softc *sc = iflib_get_softc(ctx); 4944 if_t ifp = iflib_get_ifp(ctx); 4945 struct e1000_hw *hw = &sc->hw; 4946 u32 rx_buffer_size; 4947 u32 pba; 4948 s32 error; 4949 4950 INIT_DEBUGOUT("em_reset: begin"); 4951 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 4952 4953 /* Let the firmware know the OS is in control */ 4954 em_get_hw_control(sc); 4955 4956 /* Set up smart power down as default off on newer adapters. */ 4957 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 4958 hw->mac.type == e1000_82572)) { 4959 u16 phy_tmp = 0; 4960 4961 /* Speed up time to link by disabling smart power down. */ 4962 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 4963 phy_tmp &= ~IGP02E1000_PM_SPD; 4964 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 4965 } 4966 4967 /* 4968 * Packet Buffer Allocation (PBA) 4969 * Writing PBA sets the receive portion of the buffer 4970 * the remainder is used for the transmit buffer. 4971 */ 4972 switch (hw->mac.type) { 4973 /* 82547: Total Packet Buffer is 40K */ 4974 case e1000_82547: 4975 case e1000_82547_rev_2: 4976 if (hw->mac.max_frame_size > 8192) 4977 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4978 else 4979 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 4980 break; 4981 /* 82571/82572/80003es2lan: Total Packet Buffer is 48K */ 4982 case e1000_82571: 4983 case e1000_82572: 4984 case e1000_80003es2lan: 4985 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 4986 break; 4987 /* 82573: Total Packet Buffer is 32K */ 4988 case e1000_82573: 4989 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 4990 break; 4991 /* 82574/82583: Total Packet Buffer is 40K */ 4992 case e1000_82574: 4993 case e1000_82583: 4994 if (hw->mac.max_frame_size > 8192) 4995 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4996 else 4997 pba = E1000_PBA_32K; /* 32K for RX, 8K for Tx */ 4998 break; 4999 case e1000_ich8lan: 5000 pba = E1000_PBA_8K; 5001 break; 5002 case e1000_ich9lan: 5003 case e1000_ich10lan: 5004 /* Boost Receive side for jumbo frames */ 5005 if (hw->mac.max_frame_size > 4096) 5006 pba = E1000_PBA_14K; 5007 else 5008 pba = E1000_PBA_10K; 5009 break; 5010 case e1000_pchlan: 5011 case e1000_pch2lan: 5012 case e1000_pch_lpt: 5013 case e1000_pch_spt: 5014 case e1000_pch_cnp: 5015 case e1000_pch_tgp: 5016 case e1000_pch_adp: 5017 case e1000_pch_mtp: 5018 case e1000_pch_ptp: 5019 case e1000_pch_nvp: 5020 pba = E1000_PBA_26K; 5021 break; 5022 case e1000_82575: 5023 pba = E1000_PBA_32K; 5024 break; 5025 case e1000_82576: 5026 pba = E1000_READ_REG(hw, E1000_RXPBS); 5027 pba &= E1000_RXPBS_SIZE_MASK_82576; 5028 break; 5029 case e1000_82580: 5030 case e1000_i350: 5031 case e1000_i354: 5032 pba = E1000_READ_REG(hw, E1000_RXPBS); 5033 pba = e1000_rxpbs_adjust_82580(pba); 5034 break; 5035 case e1000_i210: 5036 case e1000_i211: 5037 pba = E1000_PBA_34K; 5038 break; 5039 default: 5040 /* Remaining devices assumed to have Packet Buffer of 64K. */ 5041 if (hw->mac.max_frame_size > 8192) 5042 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 5043 else 5044 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 5045 } 5046 5047 /* Special needs in case of Jumbo frames */ 5048 if ((hw->mac.type == e1000_82575) && (if_getmtu(ifp) > ETHERMTU)) { 5049 u32 tx_space, min_tx, min_rx; 5050 pba = E1000_READ_REG(hw, E1000_PBA); 5051 tx_space = pba >> 16; 5052 pba &= 0xffff; 5053 min_tx = (hw->mac.max_frame_size + 5054 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 5055 min_tx = roundup2(min_tx, 1024); 5056 min_tx >>= 10; 5057 min_rx = hw->mac.max_frame_size; 5058 min_rx = roundup2(min_rx, 1024); 5059 min_rx >>= 10; 5060 if (tx_space < min_tx && 5061 ((min_tx - tx_space) < pba)) { 5062 pba = pba - (min_tx - tx_space); 5063 /* 5064 * if short on rx space, rx wins 5065 * and must trump tx adjustment 5066 */ 5067 if (pba < min_rx) 5068 pba = min_rx; 5069 } 5070 E1000_WRITE_REG(hw, E1000_PBA, pba); 5071 } 5072 5073 if (hw->mac.type < igb_mac_min) 5074 E1000_WRITE_REG(hw, E1000_PBA, pba); 5075 5076 INIT_DEBUGOUT1("em_reset: pba=%dK", pba); 5077 5078 /* 5079 * These parameters control the automatic generation (Tx) and 5080 * response (Rx) to Ethernet PAUSE frames. 5081 * - High water mark should allow for at least two frames to be 5082 * received after sending an XOFF. 5083 * - Low water mark works best when it is very near the high water 5084 mark. 5085 * This allows the receiver to restart by sending XON when it has 5086 * drained a bit. Here we use an arbitrary value of 1500 which will 5087 * restart after one full frame is pulled from the buffer. There 5088 * could be several smaller frames in the buffer and if so they will 5089 * not trigger the XON until their total number reduces the buffer 5090 * by 1500. 5091 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 5092 */ 5093 rx_buffer_size = (pba & 0xffff) << 10; 5094 hw->fc.high_water = rx_buffer_size - 5095 roundup2(hw->mac.max_frame_size, 1024); 5096 hw->fc.low_water = hw->fc.high_water - 1500; 5097 5098 if (sc->fc) /* locally set flow control value? */ 5099 hw->fc.requested_mode = sc->fc; 5100 else 5101 hw->fc.requested_mode = e1000_fc_full; 5102 5103 if (hw->mac.type == e1000_80003es2lan) 5104 hw->fc.pause_time = 0xFFFF; 5105 else 5106 hw->fc.pause_time = EM_FC_PAUSE_TIME; 5107 5108 hw->fc.send_xon = true; 5109 5110 /* Device specific overrides/settings */ 5111 switch (hw->mac.type) { 5112 case e1000_pchlan: 5113 /* Workaround: no TX flow ctrl for PCH */ 5114 hw->fc.requested_mode = e1000_fc_rx_pause; 5115 hw->fc.pause_time = 0xFFFF; /* override */ 5116 if (if_getmtu(ifp) > ETHERMTU) { 5117 hw->fc.high_water = 0x3500; 5118 hw->fc.low_water = 0x1500; 5119 } else { 5120 hw->fc.high_water = 0x5000; 5121 hw->fc.low_water = 0x3000; 5122 } 5123 hw->fc.refresh_time = 0x1000; 5124 break; 5125 case e1000_pch2lan: 5126 case e1000_pch_lpt: 5127 case e1000_pch_spt: 5128 case e1000_pch_cnp: 5129 case e1000_pch_tgp: 5130 case e1000_pch_adp: 5131 case e1000_pch_mtp: 5132 case e1000_pch_ptp: 5133 case e1000_pch_nvp: 5134 hw->fc.high_water = 0x5C20; 5135 hw->fc.low_water = 0x5048; 5136 hw->fc.pause_time = 0xFFFF; 5137 hw->fc.refresh_time = 0xFFFF; 5138 /* Jumbos need adjusted PBA */ 5139 if (if_getmtu(ifp) > ETHERMTU) 5140 pba = E1000_PBA_12K; 5141 else 5142 pba = E1000_PBA_26K; 5143 E1000_WRITE_REG(hw, E1000_PBA, pba); 5144 break; 5145 case e1000_82575: 5146 case e1000_82576: 5147 /* 8-byte granularity */ 5148 hw->fc.low_water = hw->fc.high_water - 8; 5149 break; 5150 case e1000_82580: 5151 case e1000_i350: 5152 case e1000_i354: 5153 case e1000_i210: 5154 case e1000_i211: 5155 /* 16-byte granularity */ 5156 hw->fc.low_water = hw->fc.high_water - 16; 5157 break; 5158 case e1000_ich9lan: 5159 case e1000_ich10lan: 5160 if (if_getmtu(ifp) > ETHERMTU) { 5161 hw->fc.high_water = 0x2800; 5162 hw->fc.low_water = hw->fc.high_water - 8; 5163 break; 5164 } 5165 /* FALLTHROUGH */ 5166 default: 5167 if (hw->mac.type == e1000_80003es2lan) 5168 hw->fc.pause_time = 0xFFFF; 5169 break; 5170 } 5171 5172 /* I219 needs some special flushing to avoid hangs */ 5173 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 5174 em_flush_desc_rings(sc); 5175 5176 /* Issue a global reset */ 5177 em_prepare_fatal_error_reset(sc); 5178 error = e1000_reset_hw(hw); 5179 if (error != E1000_SUCCESS) { 5180 device_printf(dev, "Hardware reset failed: %d\n", error); 5181 return (error); 5182 } 5183 if (hw->mac.type >= igb_mac_min) { 5184 E1000_WRITE_REG(hw, E1000_WUC, 0); 5185 } else { 5186 E1000_WRITE_REG(hw, E1000_WUFC, 0); 5187 em_disable_aspm(sc); 5188 } 5189 if (sc->flags & IGB_MEDIA_RESET) { 5190 e1000_setup_init_funcs(hw, true); 5191 e1000_get_bus_info(hw); 5192 sc->flags &= ~IGB_MEDIA_RESET; 5193 } 5194 /* and a re-init */ 5195 error = e1000_init_hw(hw); 5196 if (error != E1000_SUCCESS) { 5197 device_printf(dev, "Hardware initialization failed: %d\n", 5198 error); 5199 return (error); 5200 } 5201 em_configure_82576_memory_errors(sc); 5202 em_finish_fatal_error_reset(sc); 5203 if (hw->mac.type >= igb_mac_min) 5204 igb_init_dmac(sc, pba); 5205 5206 /* Save the receive packet-buffer allocation for AIM. */ 5207 sc->pba = pba; 5208 5209 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 5210 e1000_get_phy_info(hw); 5211 e1000_check_for_link(hw); 5212 sc->phy_hang_count = 0; 5213 5214 return (E1000_SUCCESS); 5215 } 5216 5217 /* 5218 * Initialise the RSS mapping for NICs that support multiple transmit/ 5219 * receive rings. 5220 */ 5221 5222 #define RSSKEYLEN 10 5223 static void 5224 em_initialize_rss_mapping(struct e1000_softc *sc) 5225 { 5226 uint8_t rss_key[4 * RSSKEYLEN]; 5227 uint32_t reta = 0; 5228 struct e1000_hw *hw = &sc->hw; 5229 int i; 5230 5231 /* 5232 * Configure RSS key 5233 */ 5234 _Static_assert(sizeof(rss_key) == RSS_KEYSIZE, 5235 "RSS key size mismatch"); 5236 rss_getkey(rss_key); 5237 for (i = 0; i < RSSKEYLEN; ++i) { 5238 uint32_t rssrk = 0; 5239 5240 rssrk = le32dec(rss_key + i * sizeof(rssrk)); 5241 E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); 5242 } 5243 5244 /* 5245 * Configure RSS redirect table in following fashion: 5246 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 5247 */ 5248 for (i = 0; i < sizeof(reta); ++i) { 5249 uint32_t q; 5250 5251 q = (i % sc->rx_num_queues) << 7; 5252 reta |= q << (8 * i); 5253 } 5254 5255 for (i = 0; i < 32; ++i) 5256 E1000_WRITE_REG(hw, E1000_RETA(i), reta); 5257 5258 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | 5259 E1000_MRQC_RSS_FIELD_IPV4_TCP | 5260 E1000_MRQC_RSS_FIELD_IPV4 | 5261 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | 5262 E1000_MRQC_RSS_FIELD_IPV6_EX | 5263 E1000_MRQC_RSS_FIELD_IPV6); 5264 } 5265 5266 static void 5267 igb_initialize_rss_mapping(struct e1000_softc *sc) 5268 { 5269 struct e1000_hw *hw = &sc->hw; 5270 int i; 5271 int queue_id; 5272 u32 reta; 5273 u32 rss_key[10], mrqc, shift = 0; 5274 5275 /* XXX? */ 5276 if (hw->mac.type == e1000_82575) 5277 shift = 6; 5278 5279 /* 5280 * The redirection table controls which destination 5281 * queue each bucket redirects traffic to. 5282 * Each DWORD represents four queues, with the LSB 5283 * being the first queue in the DWORD. 5284 * 5285 * This just allocates buckets to queues using round-robin 5286 * allocation. 5287 * 5288 * NOTE: It Just Happens to line up with the default 5289 * RSS allocation method. 5290 */ 5291 5292 /* Warning FM follows */ 5293 reta = 0; 5294 for (i = 0; i < 128; i++) { 5295 #ifdef RSS 5296 queue_id = rss_get_indirection_to_bucket(i); 5297 /* 5298 * If we have more queues than buckets, we'll 5299 * end up mapping buckets to a subset of the 5300 * queues. 5301 * 5302 * If we have more buckets than queues, we'll 5303 * end up instead assigning multiple buckets 5304 * to queues. 5305 * 5306 * Both are suboptimal, but we need to handle 5307 * the case so we don't go out of bounds 5308 * indexing arrays and such. 5309 */ 5310 queue_id = queue_id % sc->rx_num_queues; 5311 #else 5312 queue_id = (i % sc->rx_num_queues); 5313 #endif 5314 /* Adjust if required */ 5315 queue_id = queue_id << shift; 5316 5317 /* 5318 * The low 8 bits are for hash value (n+0); 5319 * The next 8 bits are for hash value (n+1), etc. 5320 */ 5321 reta = reta >> 8; 5322 reta = reta | ( ((uint32_t) queue_id) << 24); 5323 if ((i & 3) == 3) { 5324 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 5325 reta = 0; 5326 } 5327 } 5328 5329 /* Now fill in hash table */ 5330 5331 /* 5332 * MRQC: Multiple Receive Queues Command 5333 * Set queuing to RSS control, number depends on the device. 5334 */ 5335 mrqc = E1000_MRQC_ENABLE_RSS_MQ; 5336 5337 /* XXX ew typecasting */ 5338 rss_getkey((uint8_t *) &rss_key); 5339 for (i = 0; i < 10; i++) 5340 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]); 5341 5342 /* 5343 * Configure the RSS fields to hash upon. 5344 */ 5345 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 5346 E1000_MRQC_RSS_FIELD_IPV4_TCP); 5347 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 5348 E1000_MRQC_RSS_FIELD_IPV6_TCP); 5349 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 5350 E1000_MRQC_RSS_FIELD_IPV6_UDP); 5351 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 5352 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 5353 5354 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 5355 } 5356 5357 /********************************************************************* 5358 * 5359 * Setup networking device structure and register interface media. 5360 * 5361 **********************************************************************/ 5362 static int 5363 em_setup_interface(if_ctx_t ctx) 5364 { 5365 if_t ifp = iflib_get_ifp(ctx); 5366 struct e1000_softc *sc = iflib_get_softc(ctx); 5367 if_softc_ctx_t scctx = sc->shared; 5368 5369 INIT_DEBUGOUT("em_setup_interface: begin"); 5370 5371 /* Single Queue */ 5372 if (sc->tx_num_queues == 1) { 5373 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 5374 if_setsendqready(ifp); 5375 } 5376 5377 /* 5378 * Specify the media types supported by this adapter and register 5379 * callbacks to update media and link information 5380 */ 5381 if (sc->vf_ifp) { 5382 ifmedia_add(sc->media, 5383 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5384 ifmedia_set(sc->media, 5385 IFM_ETHER | IFM_1000_T | IFM_FDX); 5386 return (0); 5387 } 5388 5389 if (sc->hw.phy.media_type == e1000_media_type_fiber || 5390 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 5391 u_char fiber_type = IFM_1000_SX; /* default type */ 5392 5393 if (sc->hw.mac.type == e1000_82545) 5394 fiber_type = IFM_1000_LX; 5395 ifmedia_add(sc->media, 5396 IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); 5397 ifmedia_add(sc->media, IFM_ETHER | fiber_type, 0, NULL); 5398 } else { 5399 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 5400 ifmedia_add(sc->media, 5401 IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 5402 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 5403 ifmedia_add(sc->media, 5404 IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 5405 if (sc->hw.phy.type != e1000_phy_ife) { 5406 ifmedia_add(sc->media, 5407 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5408 ifmedia_add(sc->media, 5409 IFM_ETHER | IFM_1000_T, 0, NULL); 5410 } 5411 } 5412 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 5413 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 5414 return (0); 5415 } 5416 5417 static int 5418 em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5419 int ntxqs, int ntxqsets) 5420 { 5421 struct e1000_softc *sc = iflib_get_softc(ctx); 5422 if_softc_ctx_t scctx = sc->shared; 5423 int error = E1000_SUCCESS; 5424 struct em_tx_queue *que; 5425 int i, j; 5426 5427 MPASS(sc->tx_num_queues > 0); 5428 MPASS(sc->tx_num_queues == ntxqsets); 5429 5430 /* First allocate the top level queue structs */ 5431 if (!(sc->tx_queues = 5432 (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * 5433 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5434 device_printf(iflib_get_dev(ctx), 5435 "Unable to allocate queue memory\n"); 5436 return(ENOMEM); 5437 } 5438 5439 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 5440 /* Set up some basics */ 5441 5442 struct tx_ring *txr = &que->txr; 5443 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 5444 ("%s: misaligned TX AIM snapshot %p", __func__, 5445 &txr->tx_aim_snapshot)); 5446 txr->sc = que->sc = sc; 5447 que->me = txr->me = i; 5448 5449 /* Allocate report status array */ 5450 if (!(txr->tx_rsq = 5451 (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], 5452 M_DEVBUF, M_NOWAIT | M_ZERO))) { 5453 device_printf(iflib_get_dev(ctx), 5454 "failed to allocate rs_idxs memory\n"); 5455 error = ENOMEM; 5456 goto fail; 5457 } 5458 for (j = 0; j < scctx->isc_ntxd[0]; j++) 5459 txr->tx_rsq[j] = QIDX_INVALID; 5460 /* get the virtual and physical address of hardware queues */ 5461 txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; 5462 txr->tx_paddr = paddrs[i*ntxqs]; 5463 } 5464 5465 if (bootverbose) 5466 device_printf(iflib_get_dev(ctx), 5467 "allocated for %d tx_queues\n", sc->tx_num_queues); 5468 return (0); 5469 fail: 5470 em_if_queues_free(ctx); 5471 return (error); 5472 } 5473 5474 static int 5475 em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5476 int nrxqs, int nrxqsets) 5477 { 5478 struct e1000_softc *sc = iflib_get_softc(ctx); 5479 int error = E1000_SUCCESS; 5480 struct em_rx_queue *que; 5481 int i; 5482 5483 MPASS(sc->rx_num_queues > 0); 5484 MPASS(sc->rx_num_queues == nrxqsets); 5485 5486 /* First allocate the top level queue structs */ 5487 if (!(sc->rx_queues = 5488 (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * 5489 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5490 device_printf(iflib_get_dev(ctx), 5491 "Unable to allocate queue memory\n"); 5492 error = ENOMEM; 5493 goto fail; 5494 } 5495 5496 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 5497 /* Set up some basics */ 5498 struct rx_ring *rxr = &que->rxr; 5499 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 5500 ("%s: misaligned RX AIM snapshot %p", __func__, 5501 &rxr->rx_aim_snapshot)); 5502 rxr->sc = que->sc = sc; 5503 rxr->que = que; 5504 que->me = rxr->me = i; 5505 5506 /* get the virtual and physical address of hardware queues */ 5507 rxr->rx_base = 5508 (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; 5509 rxr->rx_paddr = paddrs[i*nrxqs]; 5510 } 5511 5512 if (bootverbose) 5513 device_printf(iflib_get_dev(ctx), 5514 "allocated for %d rx_queues\n", sc->rx_num_queues); 5515 5516 return (0); 5517 fail: 5518 em_if_queues_free(ctx); 5519 return (error); 5520 } 5521 5522 static void 5523 em_if_queues_free(if_ctx_t ctx) 5524 { 5525 struct e1000_softc *sc = iflib_get_softc(ctx); 5526 struct em_tx_queue *tx_que = sc->tx_queues; 5527 struct em_rx_queue *rx_que = sc->rx_queues; 5528 5529 if (tx_que != NULL) { 5530 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 5531 struct tx_ring *txr = &tx_que->txr; 5532 if (txr->tx_rsq == NULL) 5533 break; 5534 5535 free(txr->tx_rsq, M_DEVBUF); 5536 txr->tx_rsq = NULL; 5537 } 5538 free(sc->tx_queues, M_DEVBUF); 5539 sc->tx_queues = NULL; 5540 } 5541 5542 if (rx_que != NULL) { 5543 free(sc->rx_queues, M_DEVBUF); 5544 sc->rx_queues = NULL; 5545 } 5546 } 5547 5548 static u32 5549 em_legacy_txdctl(struct e1000_hw *hw) 5550 { 5551 u32 txdctl; 5552 5553 /* 5554 * Start with the established full-descriptor writeback policy. 5555 * Several generations have descriptor-queue errata for which it is 5556 * a documented workaround. The unsafe early controllers are 5557 * overridden below. 5558 */ 5559 txdctl = EM_TX_PTHRESH | (EM_TX_HTHRESH << 8) | 5560 (EM_TX_WTHRESH << 16) | E1000_TXDCTL_GRAN; 5561 5562 switch (hw->mac.type) { 5563 case e1000_82571: 5564 case e1000_82572: 5565 case e1000_82573: 5566 case e1000_82574: 5567 case e1000_82583: 5568 case e1000_80003es2lan: 5569 /* Match the Intel shared-code policy for these families. */ 5570 txdctl |= E1000_TXDCTL_COUNT_DESC; 5571 break; 5572 case e1000_ich8lan: 5573 case e1000_ich9lan: 5574 case e1000_ich10lan: 5575 case e1000_pchlan: 5576 case e1000_pch2lan: 5577 case e1000_pch_lpt: 5578 case e1000_pch_spt: 5579 case e1000_pch_cnp: 5580 case e1000_pch_tgp: 5581 case e1000_pch_adp: 5582 case e1000_pch_mtp: 5583 case e1000_pch_ptp: 5584 case e1000_pch_nvp: 5585 /* Preserve the required bit set by the integrated shared code. */ 5586 txdctl |= (1U << 22); 5587 break; 5588 case e1000_82542: 5589 case e1000_82543: 5590 case e1000_82544: 5591 /* 5592 * 82543 erratum 35 and 82544 erratum 20 require 5593 * WTHRESH=0. Leave all descriptor-control thresholds at 5594 * their reset values on these early controllers. 5595 */ 5596 txdctl = 0; 5597 break; 5598 case e1000_82540: 5599 case e1000_82545: 5600 case e1000_82545_rev_3: 5601 case e1000_82546: 5602 case e1000_82546_rev_3: 5603 case e1000_82541: 5604 case e1000_82541_rev_2: 5605 case e1000_82547: 5606 case e1000_82547_rev_2: 5607 break; 5608 default: 5609 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5610 hw->mac.type)); 5611 break; 5612 } 5613 5614 return (txdctl); 5615 } 5616 5617 static u32 5618 igb_txdctl(struct e1000_hw *hw) 5619 { 5620 u32 pthresh; 5621 5622 switch (hw->mac.type) { 5623 case e1000_i354: 5624 pthresh = I354_TX_PTHRESH; 5625 break; 5626 case e1000_82575: 5627 case e1000_82576: 5628 case e1000_82580: 5629 case e1000_i350: 5630 case e1000_i210: 5631 case e1000_i211: 5632 case e1000_vfadapt: 5633 case e1000_vfadapt_i350: 5634 pthresh = IGB_TX_PTHRESH; 5635 break; 5636 default: 5637 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5638 hw->mac.type)); 5639 pthresh = IGB_TX_PTHRESH; 5640 break; 5641 } 5642 5643 return (pthresh | (IGB_TX_HTHRESH << 8) | 5644 E1000_TXDCTL_QUEUE_ENABLE); 5645 } 5646 5647 /********************************************************************* 5648 * 5649 * Enable transmit unit. 5650 * 5651 **********************************************************************/ 5652 void 5653 em_initialize_transmit_rings(if_ctx_t ctx) 5654 { 5655 struct e1000_softc *sc = iflib_get_softc(ctx); 5656 if_softc_ctx_t scctx = sc->shared; 5657 struct em_tx_queue *que; 5658 struct tx_ring *txr; 5659 struct e1000_hw *hw = &sc->hw; 5660 u32 txdctl; 5661 5662 for (int i = 0; i < sc->tx_num_queues; i++) { 5663 u64 bus_addr; 5664 caddr_t offp, endp; 5665 uint32_t qid; 5666 5667 que = &sc->tx_queues[i]; 5668 txr = &que->txr; 5669 qid = txr->me; 5670 bus_addr = txr->tx_paddr; 5671 5672 /* Clear checksum offload context. */ 5673 offp = (caddr_t)txr + offsetof(struct tx_ring, csum_flags); 5674 endp = (caddr_t)(txr + 1); 5675 memset(offp, 0, endp - offp); 5676 5677 if (hw->mac.type >= igb_mac_min) { 5678 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid)); 5679 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), 5680 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE); 5681 E1000_WRITE_FLUSH(hw); 5682 } 5683 5684 /* Base and Len of TX Ring */ 5685 E1000_WRITE_REG(hw, E1000_TDLEN(qid), 5686 scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); 5687 E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32)); 5688 E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr); 5689 /* Init the HEAD/TAIL indices */ 5690 E1000_WRITE_REG(hw, E1000_TDT(qid), 0); 5691 E1000_WRITE_REG(hw, E1000_TDH(qid), 0); 5692 5693 HW_DEBUGOUT2("Base = %x, Length = %x\n", 5694 E1000_READ_REG(hw, E1000_TDBAL(qid)), 5695 E1000_READ_REG(hw, E1000_TDLEN(qid))); 5696 5697 if (hw->mac.type < igb_mac_min) 5698 txdctl = em_legacy_txdctl(hw); 5699 else 5700 txdctl = igb_txdctl(hw); 5701 5702 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl); 5703 } 5704 } 5705 5706 static void 5707 em_initialize_transmit_unit(if_ctx_t ctx) 5708 { 5709 struct e1000_softc *sc = iflib_get_softc(ctx); 5710 struct e1000_hw *hw = &sc->hw; 5711 u32 tctl, tarc, tipg = 0; 5712 5713 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 5714 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5715 5716 em_initialize_transmit_rings(ctx); 5717 5718 /* Set the default values for the Tx Inter Packet Gap timer */ 5719 switch (hw->mac.type) { 5720 case e1000_80003es2lan: 5721 tipg = DEFAULT_82543_TIPG_IPGR1; 5722 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 5723 E1000_TIPG_IPGR2_SHIFT; 5724 break; 5725 case e1000_82542: 5726 tipg = DEFAULT_82542_TIPG_IPGT; 5727 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5728 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5729 break; 5730 default: 5731 if (hw->phy.media_type == e1000_media_type_fiber || 5732 hw->phy.media_type == e1000_media_type_internal_serdes) 5733 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 5734 else 5735 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 5736 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5737 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5738 } 5739 5740 if (hw->mac.type < igb_mac_min) { 5741 E1000_WRITE_REG(hw, E1000_TIPG, tipg); 5742 E1000_WRITE_REG(hw, E1000_TIDV, sc->tx_int_delay.value); 5743 5744 if (sc->tx_int_delay.value > 0) 5745 sc->txd_cmd |= E1000_TXD_CMD_IDE; 5746 } 5747 5748 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) 5749 E1000_WRITE_REG(hw, E1000_TADV, sc->tx_abs_int_delay.value); 5750 5751 if (hw->mac.type == e1000_82571 || hw->mac.type == e1000_82572) { 5752 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5753 tarc |= TARC_SPEED_MODE_BIT; 5754 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5755 } else if (hw->mac.type == e1000_80003es2lan) { 5756 /* errata: program both queues to unweighted RR */ 5757 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5758 tarc |= 1; 5759 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5760 tarc = E1000_READ_REG(hw, E1000_TARC(1)); 5761 tarc |= 1; 5762 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5763 } else if (hw->mac.type == e1000_82574) { 5764 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5765 tarc |= TARC_ERRATA_BIT; 5766 if ( sc->tx_num_queues > 1) { 5767 tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); 5768 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5769 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5770 } else 5771 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5772 } 5773 5774 /* Program the Transmit Control Register */ 5775 tctl = E1000_READ_REG(hw, E1000_TCTL); 5776 tctl &= ~E1000_TCTL_CT; 5777 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 5778 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 5779 5780 if (hw->mac.type >= e1000_82571 && hw->mac.type < igb_mac_min) 5781 tctl |= E1000_TCTL_MULR; 5782 5783 /* This write will effectively turn on the transmit unit. */ 5784 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 5785 5786 /* SPT and KBL errata workarounds */ 5787 if (hw->mac.type == e1000_pch_spt) { 5788 u32 reg; 5789 reg = E1000_READ_REG(hw, E1000_IOSFPC); 5790 reg |= E1000_RCTL_RDMTS_HEX; 5791 E1000_WRITE_REG(hw, E1000_IOSFPC, reg); 5792 /* i218-i219 Specification Update 1.5.4.5 */ 5793 reg = E1000_READ_REG(hw, E1000_TARC(0)); 5794 reg &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 5795 reg |= E1000_TARC0_CB_MULTIQ_2_REQ; 5796 E1000_WRITE_REG(hw, E1000_TARC(0), reg); 5797 } 5798 } 5799 5800 /********************************************************************* 5801 * 5802 * Enable receive unit. 5803 * 5804 **********************************************************************/ 5805 #define BSIZEPKT_ROUNDUP ((1<<E1000_SRRCTL_BSIZEPKT_SHIFT)-1) 5806 5807 static u32 5808 igb_rxdctl(struct e1000_softc *sc, u32 rxdctl) 5809 { 5810 struct e1000_hw *hw; 5811 u32 mask, pthresh, wthresh; 5812 5813 hw = &sc->hw; 5814 mask = IGB_RXDCTL_THRESH_MASK; 5815 switch (hw->mac.type) { 5816 case e1000_82575: 5817 mask = IGB_82575_RXDCTL_THRESH_MASK; 5818 pthresh = IGB_RX_PTHRESH; 5819 wthresh = IGB_RX_WTHRESH; 5820 break; 5821 case e1000_82576: 5822 pthresh = IGB_RX_PTHRESH; 5823 wthresh = sc->intr_type == IFLIB_INTR_MSIX ? 5824 IGB_82576_RX_WTHRESH : IGB_RX_WTHRESH; 5825 break; 5826 case e1000_vfadapt: 5827 /* 82576 VFs always need the MSI-X writeback workaround. */ 5828 pthresh = IGB_RX_PTHRESH; 5829 wthresh = IGB_82576_RX_WTHRESH; 5830 break; 5831 case e1000_i354: 5832 pthresh = I354_RX_PTHRESH; 5833 wthresh = IGB_RX_WTHRESH; 5834 break; 5835 case e1000_82580: 5836 case e1000_i350: 5837 case e1000_i210: 5838 case e1000_i211: 5839 case e1000_vfadapt_i350: 5840 pthresh = IGB_RX_PTHRESH; 5841 wthresh = IGB_RX_WTHRESH; 5842 break; 5843 default: 5844 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5845 hw->mac.type)); 5846 pthresh = IGB_RX_PTHRESH; 5847 wthresh = IGB_RX_WTHRESH; 5848 break; 5849 } 5850 5851 rxdctl &= ~mask; 5852 rxdctl |= pthresh | (IGB_RX_HTHRESH << 8) | 5853 (wthresh << 16) | E1000_RXDCTL_QUEUE_ENABLE; 5854 return (rxdctl); 5855 } 5856 5857 void 5858 igb_initialize_receive_rings(if_ctx_t ctx, bool drop) 5859 { 5860 struct e1000_softc *sc = iflib_get_softc(ctx); 5861 if_softc_ctx_t scctx = sc->shared; 5862 struct e1000_hw *hw = &sc->hw; 5863 struct em_rx_queue *que; 5864 u32 srrctl; 5865 5866 srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 5867 E1000_SRRCTL_BSIZEPKT_SHIFT; 5868 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 5869 if (drop) 5870 srrctl |= E1000_SRRCTL_DROP_EN; 5871 5872 for (int i = 0; i < sc->rx_num_queues; i++) { 5873 struct rx_ring *rxr; 5874 u64 bus_addr; 5875 u32 rxdctl; 5876 uint32_t qid; 5877 5878 que = &sc->rx_queues[i]; 5879 rxr = &que->rxr; 5880 bus_addr = rxr->rx_paddr; 5881 qid = rxr->me; 5882 5883 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid)); 5884 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), 5885 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE); 5886 E1000_WRITE_FLUSH(hw); 5887 5888 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 5889 scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); 5890 E1000_WRITE_REG(hw, E1000_RDBAH(qid), 5891 (uint32_t)(bus_addr >> 32)); 5892 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr); 5893 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 5894 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 5895 E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl); 5896 5897 rxdctl = igb_rxdctl(sc, rxdctl); 5898 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl); 5899 } 5900 } 5901 5902 static bool 5903 em_integrated_jumbo_rx(struct e1000_hw *hw) 5904 { 5905 switch (hw->mac.type) { 5906 case e1000_ich9lan: 5907 case e1000_ich10lan: 5908 case e1000_pchlan: 5909 case e1000_pch2lan: 5910 case e1000_pch_lpt: 5911 case e1000_pch_spt: 5912 case e1000_pch_cnp: 5913 case e1000_pch_tgp: 5914 case e1000_pch_adp: 5915 case e1000_pch_mtp: 5916 case e1000_pch_ptp: 5917 case e1000_pch_nvp: 5918 return (true); 5919 default: 5920 return (false); 5921 } 5922 } 5923 5924 static void 5925 em_initialize_receive_unit(if_ctx_t ctx) 5926 { 5927 struct e1000_softc *sc = iflib_get_softc(ctx); 5928 if_softc_ctx_t scctx = sc->shared; 5929 if_t ifp = iflib_get_ifp(ctx); 5930 struct e1000_hw *hw = &sc->hw; 5931 struct em_rx_queue *que; 5932 int i; 5933 uint32_t rctl, rxcsum; 5934 5935 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 5936 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5937 5938 /* 5939 * Make sure receives are disabled while setting up the descriptor 5940 * ring. 5941 */ 5942 rctl = E1000_READ_REG(hw, E1000_RCTL); 5943 /* Do not disable if ever enabled on this hardware. */ 5944 if (hw->mac.type != e1000_82574 && 5945 hw->mac.type != e1000_82583) 5946 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 5947 5948 /* Setup the Receive Control Register. */ 5949 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 5950 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 5951 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 5952 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 5953 rctl &= ~E1000_RCTL_SBP; 5954 5955 if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU) 5956 rctl |= E1000_RCTL_LPE; 5957 else 5958 rctl &= ~E1000_RCTL_LPE; 5959 if (!em_disable_crc_stripping) 5960 rctl |= E1000_RCTL_SECRC; 5961 5962 /* lem/em default interrupt moderation */ 5963 if (hw->mac.type < igb_mac_min) { 5964 if (hw->mac.type >= e1000_82540) { 5965 E1000_WRITE_REG(hw, E1000_RADV, 5966 sc->rx_abs_int_delay.value); 5967 5968 /* Set the default interrupt throttling rate */ 5969 E1000_WRITE_REG(hw, E1000_ITR, 5970 EM_INTS_TO_ITR(em_max_interrupt_rate)); 5971 5972 /* 5973 * The 82574 MSI-X EITR registers are programmed 5974 * with the same value further below. Either way 5975 * the hardware now holds the default rate, so seed 5976 * the software copy to match; otherwise a stale 5977 * itr_setting left over from AIM makes em_newitr() 5978 * skip the write that would restore it. 5979 */ 5980 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; 5981 i++, que++) 5982 que->itr_setting = 5983 EM_INTS_TO_ITR(em_max_interrupt_rate); 5984 } 5985 5986 /* XXX TEMPORARY WORKAROUND: on some systems with 82573 5987 * long latencies are observed, like Lenovo X60. This 5988 * change eliminates the problem, but since having positive 5989 * values in RDTR is a known source of problems on other 5990 * platforms another solution is being sought. 5991 */ 5992 if (hw->mac.type == e1000_82573) 5993 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 5994 else 5995 E1000_WRITE_REG(hw, E1000_RDTR, 5996 sc->rx_int_delay.value); 5997 } 5998 5999 if (hw->mac.type >= em_mac_min) { 6000 uint32_t rfctl; 6001 /* Use extended rx descriptor formats */ 6002 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 6003 rfctl |= E1000_RFCTL_EXTEN; 6004 6005 /* 6006 * When using MSI-X interrupts we need to throttle 6007 * using the EITR register (82574 only) 6008 */ 6009 if (hw->mac.type == e1000_82574) { 6010 for (int i = 0; i < 4; i++) 6011 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 6012 EM_INTS_TO_ITR(em_max_interrupt_rate)); 6013 /* Disable accelerated acknowledge */ 6014 rfctl |= E1000_RFCTL_ACK_DIS; 6015 } 6016 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 6017 } 6018 6019 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 6020 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 6021 rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; 6022 if (hw->mac.type > e1000_82575) 6023 rxcsum |= E1000_RXCSUM_CRCOFL; 6024 else if (hw->mac.type < em_mac_min && 6025 if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) 6026 rxcsum |= E1000_RXCSUM_IPV6OFL; 6027 } else { 6028 rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 6029 if (hw->mac.type > e1000_82575) 6030 rxcsum &= ~E1000_RXCSUM_CRCOFL; 6031 else if (hw->mac.type < em_mac_min) 6032 rxcsum &= ~E1000_RXCSUM_IPV6OFL; 6033 } 6034 6035 if (sc->rx_num_queues > 1) { 6036 /* RSS hash needed in the Rx descriptor */ 6037 rxcsum |= E1000_RXCSUM_PCSD; 6038 6039 if (hw->mac.type >= igb_mac_min) 6040 igb_initialize_rss_mapping(sc); 6041 else 6042 em_initialize_rss_mapping(sc); 6043 } 6044 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 6045 6046 for (i = 0, que = sc->rx_queues; 6047 hw->mac.type < igb_mac_min && i < sc->rx_num_queues; 6048 i++, que++) { 6049 struct rx_ring *rxr = &que->rxr; 6050 /* Setup the Base and Length of the Rx Descriptor Ring */ 6051 u64 bus_addr = rxr->rx_paddr; 6052 uint32_t qid = rxr->me; 6053 #if 0 6054 u32 rdt = sc->rx_num_queues -1; /* default */ 6055 #endif 6056 6057 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 6058 scctx->isc_nrxd[0] * 6059 sizeof(union e1000_rx_desc_extended)); 6060 E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32)); 6061 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr); 6062 /* Setup the Head and Tail Descriptor Pointers */ 6063 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 6064 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 6065 } 6066 6067 /* Increase receive-descriptor prefetching for integrated jumbo MACs. */ 6068 if (em_integrated_jumbo_rx(hw) && if_getmtu(ifp) > ETHERMTU) { 6069 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 6070 6071 rxdctl &= ~(EM_RXDCTL_PTHRESH_MASK | 6072 EM_RXDCTL_HTHRESH_MASK); 6073 rxdctl |= EM_JUMBO_RX_PTHRESH | 6074 (EM_JUMBO_RX_HTHRESH << 8); 6075 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 6076 } else if (hw->mac.type == e1000_82574) { 6077 /* RXDCTL(0) writes are mirrored to RXDCTL(1) on 82574. */ 6078 for (int i = 0; i < sc->rx_num_queues; i++) { 6079 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 6080 6081 rxdctl &= ~EM_RXDCTL_THRESH_MASK; 6082 rxdctl |= EM_82574_RX_PTHRESH | 6083 (EM_82574_RX_HTHRESH << 8) | 6084 (EM_82574_RX_WTHRESH << 16) | 6085 E1000_RXDCTL_THRESH_UNIT_DESC; 6086 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 6087 } 6088 } else if (hw->mac.type >= igb_mac_min) { 6089 bool drop; 6090 u32 psize; 6091 6092 if (igb_iov_enabled(sc)) { 6093 E1000_WRITE_REG(hw, E1000_RLPML, 6094 IGB_IOV_MAX_FRAME_SIZE); 6095 } else if (if_getmtu(ifp) > ETHERMTU) { 6096 psize = scctx->isc_max_frame_size; 6097 /* are we on a vlan? */ 6098 if (if_vlantrunkinuse(ifp)) 6099 psize += VLAN_TAG_SIZE; 6100 6101 E1000_WRITE_REG(hw, E1000_RLPML, psize); 6102 } 6103 6104 /* 6105 * If TX flow control is disabled and there's >1 queue 6106 * defined, enable DROP. 6107 * 6108 * This drops frames rather than hanging the RX MAC for all 6109 * queues. 6110 */ 6111 drop = igb_iov_enabled(sc) || 6112 ((sc->rx_num_queues > 1) && 6113 (sc->fc == e1000_fc_none || 6114 sc->fc == e1000_fc_rx_pause)); 6115 igb_initialize_receive_rings(ctx, drop); 6116 } else if (hw->mac.type >= e1000_pch2lan) { 6117 if (if_getmtu(ifp) > ETHERMTU) 6118 e1000_lv_jumbo_workaround_ich8lan(hw, true); 6119 else 6120 e1000_lv_jumbo_workaround_ich8lan(hw, false); 6121 } 6122 6123 /* Make sure VLAN Filters are off */ 6124 rctl &= ~E1000_RCTL_VFE; 6125 6126 /* Set up packet buffer size, overridden by per queue srrctl on igb */ 6127 if (hw->mac.type < igb_mac_min) { 6128 if (sc->rx_mbuf_sz > 2048 && sc->rx_mbuf_sz <= 4096) 6129 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 6130 else if (sc->rx_mbuf_sz > 4096 && sc->rx_mbuf_sz <= 8192) 6131 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 6132 else if (sc->rx_mbuf_sz > 8192) 6133 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX; 6134 else { 6135 rctl |= E1000_RCTL_SZ_2048; 6136 rctl &= ~E1000_RCTL_BSEX; 6137 } 6138 } else 6139 rctl |= E1000_RCTL_SZ_2048; 6140 6141 /* 6142 * rctl bits 11:10 are as follows 6143 * lem: reserved 6144 * em: DTYPE 6145 * igb: reserved 6146 * and should be 00 on all of the above 6147 */ 6148 rctl &= ~0x00000C00; 6149 6150 /* Write out the settings */ 6151 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 6152 6153 return; 6154 } 6155 6156 static void 6157 em_if_vlan_register(if_ctx_t ctx, u16 vtag) 6158 { 6159 struct e1000_softc *sc = iflib_get_softc(ctx); 6160 bool present; 6161 u32 index, mask; 6162 6163 /* Hyper-V supports host-assigned access VLANs, not guest VLANs. */ 6164 if (igbv_is_hyperv(sc)) 6165 return; 6166 6167 index = (vtag >> 5) & 0x7F; 6168 mask = 1U << (vtag & 0x1F); 6169 present = (sc->shadow_vfta[index] & mask) != 0; 6170 /* 6171 * On a VF, record registration intent for replay even if the PF is not 6172 * ready to accept it yet. 6173 */ 6174 sc->shadow_vfta[index] |= mask; 6175 sc->vf_vfta_stale[index] &= ~mask; 6176 if (!present) 6177 ++sc->num_vlans; 6178 if (sc->vf_ifp && 6179 e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { 6180 igbv_vlan_retry_add(sc, vtag); 6181 device_printf(sc->dev, 6182 "VF VLAN %u add request failed\n", vtag); 6183 } else if (sc->vf_ifp) 6184 igbv_vlan_retry_clear(sc, vtag); 6185 if (!sc->vf_ifp) { 6186 if (igb_iov_enabled(sc)) 6187 igb_iov_rebuild_vlan(sc); 6188 else 6189 em_if_vlan_filter_write(sc, index); 6190 } 6191 } 6192 6193 static void 6194 em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 6195 { 6196 struct e1000_softc *sc = iflib_get_softc(ctx); 6197 bool present; 6198 u32 index, mask; 6199 6200 if (igbv_is_hyperv(sc)) 6201 return; 6202 6203 index = (vtag >> 5) & 0x7F; 6204 mask = 1U << (vtag & 0x1F); 6205 present = (sc->shadow_vfta[index] & mask) != 0; 6206 if (sc->vf_ifp) 6207 igbv_vlan_retry_clear(sc, vtag); 6208 if (sc->vf_ifp && 6209 e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { 6210 device_printf(sc->dev, 6211 "VF VLAN %u remove request failed\n", vtag); 6212 /* 6213 * Hardware might still admit this VID. Preserve its receive 6214 * tag until a successful VF reset proves the stale filter gone. 6215 */ 6216 sc->vf_vfta_stale[index] |= mask; 6217 } else { 6218 sc->vf_vfta_stale[index] &= ~mask; 6219 } 6220 sc->shadow_vfta[index] &= ~mask; 6221 if (present) 6222 --sc->num_vlans; 6223 if (!sc->vf_ifp) { 6224 if (igb_iov_enabled(sc)) 6225 igb_iov_rebuild_vlan(sc); 6226 else 6227 em_if_vlan_filter_write(sc, index); 6228 } 6229 } 6230 6231 static bool 6232 em_if_vlan_filter_capable(if_ctx_t ctx) 6233 { 6234 if_t ifp = iflib_get_ifp(ctx); 6235 6236 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) && 6237 !em_disable_crc_stripping) 6238 return (true); 6239 6240 return (false); 6241 } 6242 6243 static bool 6244 em_if_vlan_filter_used(if_ctx_t ctx) 6245 { 6246 struct e1000_softc *sc = iflib_get_softc(ctx); 6247 6248 if (!em_if_vlan_filter_capable(ctx)) 6249 return (false); 6250 6251 for (int i = 0; i < EM_VFTA_SIZE; i++) 6252 if (sc->shadow_vfta[i] != 0) 6253 return (true); 6254 6255 return (false); 6256 } 6257 6258 static void 6259 em_if_vlan_filter_enable(struct e1000_softc *sc) 6260 { 6261 struct e1000_hw *hw = &sc->hw; 6262 u32 reg; 6263 6264 reg = E1000_READ_REG(hw, E1000_RCTL); 6265 reg &= ~E1000_RCTL_CFIEN; 6266 reg |= E1000_RCTL_VFE; 6267 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6268 } 6269 6270 static void 6271 em_if_vlan_filter_disable(struct e1000_softc *sc) 6272 { 6273 struct e1000_hw *hw = &sc->hw; 6274 u32 reg; 6275 6276 reg = E1000_READ_REG(hw, E1000_RCTL); 6277 reg &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 6278 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6279 } 6280 6281 static void 6282 em_if_vlan_filter_write(struct e1000_softc *sc, int changed_index) 6283 { 6284 struct e1000_hw *hw = &sc->hw; 6285 6286 KASSERT(!sc->vf_ifp, ("VLAN filter write on VF\n")); 6287 6288 /* Disable interrupts for lem(4) devices during the filter change */ 6289 if (hw->mac.type < em_mac_min) 6290 em_if_intr_disable(sc->ctx); 6291 6292 /* 6293 * Restore every retained VLAN after reset. Also write the changed 6294 * word when its final VLAN was removed so stale hardware membership 6295 * does not survive a zero shadow value. 6296 */ 6297 for (int i = 0; i < EM_VFTA_SIZE; i++) 6298 if (sc->shadow_vfta[i] != 0 || i == changed_index) 6299 e1000_write_vfta(hw, i, sc->shadow_vfta[i]); 6300 6301 /* Re-enable interrupts for lem-class devices */ 6302 if (hw->mac.type < em_mac_min) 6303 em_if_intr_enable(sc->ctx); 6304 } 6305 6306 static void 6307 em_setup_vlan_hw_support(if_ctx_t ctx) 6308 { 6309 struct e1000_softc *sc = iflib_get_softc(ctx); 6310 struct e1000_hw *hw = &sc->hw; 6311 if_t ifp = iflib_get_ifp(ctx); 6312 s32 error; 6313 u32 max_frame_size, reg; 6314 u16 vid; 6315 int restore_failures; 6316 6317 /* Hyper-V programs the VF's receive limit and VLAN membership. */ 6318 if (igbv_is_hyperv(sc)) 6319 return; 6320 6321 /* 6322 * Only PFs have control over VLAN HW filtering 6323 * configuration. VFs have to act as if it's always 6324 * enabled. 6325 */ 6326 if (sc->vf_ifp) { 6327 max_frame_size = min(sc->shared->isc_max_frame_size + 6328 VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); 6329 e1000_rlpml_set_vf(hw, max_frame_size); 6330 restore_failures = 0; 6331 for (vid = 0; vid < 4096; vid++) { 6332 if ((sc->shadow_vfta[vid >> 5] & 6333 (1U << (vid & 0x1f))) == 0) 6334 continue; 6335 /* 6336 * Desired state remains in shadow_vfta for the next 6337 * replay if the PF mailbox is absent during reset. 6338 */ 6339 error = e1000_vfta_set_vf(hw, vid, true); 6340 if (error != E1000_SUCCESS) { 6341 igbv_vlan_retry_add(sc, vid); 6342 restore_failures++; 6343 } else 6344 igbv_vlan_retry_clear(sc, vid); 6345 } 6346 if (restore_failures != 0) 6347 device_printf(sc->dev, 6348 "VF VLAN restore failed for %d VIDs; retrying\n", 6349 restore_failures); 6350 return; 6351 } 6352 6353 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 6354 !em_disable_crc_stripping) { 6355 reg = E1000_READ_REG(hw, E1000_CTRL); 6356 reg |= E1000_CTRL_VME; 6357 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6358 } else { 6359 reg = E1000_READ_REG(hw, E1000_CTRL); 6360 reg &= ~E1000_CTRL_VME; 6361 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6362 } 6363 6364 /* 6365 * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN 6366 * filtering is disabled, the IOV VLAN rebuild instead makes the PF 6367 * VLAN-promiscuous without disabling the global filter. 6368 */ 6369 if (!em_if_vlan_filter_capable(ctx)) { 6370 if (igb_iov_enabled(sc)) { 6371 #ifdef PCI_IOV 6372 sc->iov_pf_vlan_promisc = true; 6373 #endif 6374 em_if_vlan_filter_enable(sc); 6375 } else 6376 em_if_vlan_filter_disable(sc); 6377 return; 6378 } 6379 #ifdef PCI_IOV 6380 if (igb_iov_enabled(sc)) 6381 sc->iov_pf_vlan_promisc = false; 6382 #endif 6383 6384 /* 6385 * A soft reset zero's out the VFTA, so 6386 * we need to repopulate it now. 6387 * We also insert VLAN 0 in the filter list, so we pass VLAN 0 tagged 6388 * traffic through. This will write the entire table. 6389 */ 6390 em_if_vlan_register(ctx, 0); 6391 6392 /* Enable the Filter Table */ 6393 em_if_vlan_filter_enable(sc); 6394 } 6395 6396 static void 6397 em_if_intr_enable(if_ctx_t ctx) 6398 { 6399 struct e1000_softc *sc = iflib_get_softc(ctx); 6400 struct e1000_hw *hw = &sc->hw; 6401 u32 ims_mask = IMS_ENABLE_MASK | em_fatal_error_intr_mask(sc); 6402 6403 if (sc->intr_type == IFLIB_INTR_MSIX) { 6404 E1000_WRITE_REG(hw, EM_EIAC, sc->ims); 6405 ims_mask |= sc->ims; 6406 } 6407 6408 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 6409 E1000_WRITE_FLUSH(hw); 6410 } 6411 6412 static void 6413 em_if_intr_disable(if_ctx_t ctx) 6414 { 6415 struct e1000_softc *sc = iflib_get_softc(ctx); 6416 struct e1000_hw *hw = &sc->hw; 6417 6418 if (sc->intr_type == IFLIB_INTR_MSIX) 6419 E1000_WRITE_REG(hw, EM_EIAC, 0); 6420 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6421 E1000_WRITE_FLUSH(hw); 6422 } 6423 6424 static void 6425 igb_if_intr_enable(if_ctx_t ctx) 6426 { 6427 struct e1000_softc *sc = iflib_get_softc(ctx); 6428 struct e1000_hw *hw = &sc->hw; 6429 u32 mask, reg; 6430 6431 if (igb_device_reset_pending(sc)) 6432 return; 6433 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6434 mask = (sc->que_mask | sc->link_mask); 6435 /* 6436 * VF interrupt controls are also mapped into these registers. 6437 * Preserve them and change only the PF vectors we own. 6438 */ 6439 reg = E1000_READ_REG(hw, E1000_EIAC); 6440 E1000_WRITE_REG(hw, E1000_EIAC, reg | mask); 6441 reg = E1000_READ_REG(hw, E1000_EIAM); 6442 E1000_WRITE_REG(hw, E1000_EIAM, reg | mask); 6443 igb_iov_intr_drain_stale(sc); 6444 E1000_WRITE_REG(hw, E1000_EIMS, mask); 6445 E1000_WRITE_REG(hw, E1000_IMS, 6446 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 6447 igb_iov_intr_mask(sc) | 6448 em_fatal_error_intr_mask(sc)); 6449 } else { 6450 mask = IMS_ENABLE_MASK | igb_device_reset_intr_mask(sc) | 6451 em_fatal_error_intr_mask(sc); 6452 /* Reading ICR masks every shared interrupt before the filter runs. */ 6453 E1000_WRITE_REG(hw, E1000_IAM, mask); 6454 E1000_WRITE_REG(hw, E1000_IMS, mask); 6455 } 6456 E1000_WRITE_FLUSH(hw); 6457 } 6458 6459 static void 6460 igb_if_intr_disable(if_ctx_t ctx) 6461 { 6462 struct e1000_softc *sc = iflib_get_softc(ctx); 6463 struct e1000_hw *hw = &sc->hw; 6464 u32 mask, reg; 6465 6466 /* This is the first CTX-owned register access after ICR.DRSTA. */ 6467 igb_prepare_device_reset(sc); 6468 6469 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6470 /* 6471 * Do not use a blanket EIMC write here. VF interrupt controls 6472 * are mapped into the same PF register space, so clearing bits 6473 * we do not own can leave running VFs with interrupts masked. 6474 * Before initial queue configuration the owned mask is zero 6475 * because this driver has not enabled a vector yet. 6476 */ 6477 mask = (sc->que_mask | sc->link_mask); 6478 reg = E1000_READ_REG(hw, E1000_EIAM); 6479 E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask); 6480 E1000_WRITE_REG(hw, E1000_EIMC, mask); 6481 reg = E1000_READ_REG(hw, E1000_EIAC); 6482 E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask); 6483 } else 6484 E1000_WRITE_REG(hw, E1000_IAM, 0); 6485 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6486 E1000_WRITE_FLUSH(hw); 6487 } 6488 6489 /* 6490 * Bit of a misnomer, what this really means is 6491 * to enable OS management of the system... aka 6492 * to disable special hardware management features 6493 */ 6494 static void 6495 em_init_manageability(struct e1000_softc *sc) 6496 { 6497 /* A shared code workaround */ 6498 #define E1000_82542_MANC2H E1000_MANC2H 6499 if (sc->has_manage) { 6500 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 6501 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6502 6503 /* disable hardware interception of ARP */ 6504 manc &= ~(E1000_MANC_ARP_EN); 6505 6506 /* enable receiving management packets to the host */ 6507 manc |= E1000_MANC_EN_MNG2HOST; 6508 #define E1000_MNG2HOST_PORT_623 (1 << 5) 6509 #define E1000_MNG2HOST_PORT_664 (1 << 6) 6510 manc2h |= E1000_MNG2HOST_PORT_623; 6511 manc2h |= E1000_MNG2HOST_PORT_664; 6512 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 6513 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6514 } 6515 } 6516 6517 /* 6518 * Give control back to hardware management 6519 * controller if there is one. 6520 */ 6521 static void 6522 em_release_manageability(struct e1000_softc *sc) 6523 { 6524 if (sc->has_manage) { 6525 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6526 6527 /* re-enable hardware interception of ARP */ 6528 manc |= E1000_MANC_ARP_EN; 6529 manc &= ~E1000_MANC_EN_MNG2HOST; 6530 6531 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6532 } 6533 } 6534 6535 /* 6536 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 6537 * For ASF and Pass Through versions of f/w this means 6538 * that the driver is loaded. For AMT version type f/w 6539 * this means that the network i/f is open. 6540 */ 6541 static void 6542 em_get_hw_control(struct e1000_softc *sc) 6543 { 6544 u32 ctrl_ext, swsm; 6545 6546 if (sc->vf_ifp) 6547 return; 6548 6549 if (sc->hw.mac.type == e1000_82573) { 6550 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6551 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6552 swsm | E1000_SWSM_DRV_LOAD); 6553 return; 6554 } 6555 /* else */ 6556 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6557 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6558 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 6559 } 6560 6561 /* 6562 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 6563 * For ASF and Pass Through versions of f/w this means that 6564 * the driver is no longer loaded. For AMT versions of the 6565 * f/w this means that the network i/f is closed. 6566 */ 6567 static void 6568 em_release_hw_control(struct e1000_softc *sc) 6569 { 6570 u32 ctrl_ext, swsm; 6571 6572 if (!sc->has_manage) 6573 return; 6574 6575 if (sc->hw.mac.type == e1000_82573) { 6576 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6577 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6578 swsm & ~E1000_SWSM_DRV_LOAD); 6579 return; 6580 } 6581 /* else */ 6582 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6583 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6584 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 6585 return; 6586 } 6587 6588 bool 6589 em_is_valid_ether_addr(const u8 *addr) 6590 { 6591 static const u8 zero_addr[ETHER_ADDR_LEN]; 6592 6593 return (!ETHER_IS_MULTICAST(addr) && 6594 memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0); 6595 } 6596 6597 static bool 6598 em_automask_tso(if_ctx_t ctx) 6599 { 6600 struct e1000_softc *sc = iflib_get_softc(ctx); 6601 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6602 if_t ifp = iflib_get_ifp(ctx); 6603 bool reset_needed; 6604 6605 if (!em_unsupported_tso && sc->link_speed && 6606 sc->link_speed != SPEED_1000 && 6607 scctx->isc_capenable & IFCAP_TSO) { 6608 device_printf(sc->dev, 6609 "Disabling TSO for 10/100 Ethernet.\n"); 6610 sc->tso_automasked = scctx->isc_capenable & IFCAP_TSO; 6611 scctx->isc_capenable &= ~IFCAP_TSO; 6612 if_setcapenablebit(ifp, 0, IFCAP_TSO); 6613 } else if (sc->link_speed == SPEED_1000 && sc->tso_automasked) { 6614 device_printf(sc->dev, "Re-enabling TSO for GbE.\n"); 6615 scctx->isc_capenable |= sc->tso_automasked; 6616 if_setcapenablebit(ifp, sc->tso_automasked, 0); 6617 sc->tso_automasked = 0; 6618 } else { 6619 return (false); 6620 } 6621 6622 /* 6623 * Apply the new capabilities to a running or administratively-up 6624 * interface, including one whose initialization has not completed. 6625 * A stopped, administratively-down interface will apply them at the 6626 * next initialization; driver flags do not describe that intent. 6627 */ 6628 reset_needed = iflib_is_running(ctx) || 6629 (if_getflags(ifp) & IFF_UP) != 0; 6630 if (!reset_needed) 6631 return (false); 6632 6633 /* iflib_init_locked handles ifnet hwassistbits */ 6634 iflib_request_reset(ctx); 6635 return (true); 6636 } 6637 6638 /* 6639 ** Parse the interface capabilities with regard 6640 ** to both system management and wake-on-lan for 6641 ** later use. 6642 */ 6643 static void 6644 em_get_wakeup(if_ctx_t ctx) 6645 { 6646 struct e1000_softc *sc = iflib_get_softc(ctx); 6647 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6648 device_t dev = iflib_get_dev(ctx); 6649 u16 eeprom_data = 0, device_id, apme_mask; 6650 bool apme; 6651 int error, wol_capabilities; 6652 6653 sc->has_manage = e1000_enable_mng_pass_thru(&sc->hw); 6654 apme_mask = EM_EEPROM_APME_HIGH; 6655 error = E1000_SUCCESS; 6656 6657 switch (sc->hw.mac.type) { 6658 case e1000_82542: 6659 case e1000_82543: 6660 break; 6661 case e1000_82544: 6662 error = e1000_read_nvm(&sc->hw, 6663 NVM_INIT_CONTROL2_REG, 1, &eeprom_data); 6664 apme_mask = EM_EEPROM_APME_LOW; 6665 break; 6666 case e1000_82541: 6667 case e1000_82547: 6668 error = e1000_read_nvm(&sc->hw, 6669 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6670 /* The EI parts place APM Enable in the low byte. */ 6671 if (sc->hw.device_id != E1000_DEV_ID_82541ER_LOM) 6672 apme_mask = EM_EEPROM_APME_LOW; 6673 break; 6674 case e1000_82546: 6675 case e1000_82546_rev_3: 6676 if (sc->hw.bus.func == 1) { 6677 error = e1000_read_nvm(&sc->hw, 6678 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6679 break; 6680 } else 6681 error = e1000_read_nvm(&sc->hw, 6682 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6683 break; 6684 case e1000_82573: 6685 case e1000_82583: 6686 sc->has_amt = true; 6687 /* FALLTHROUGH */ 6688 case e1000_82571: 6689 case e1000_82572: 6690 case e1000_80003es2lan: 6691 if (sc->hw.bus.func == 1) { 6692 error = e1000_read_nvm(&sc->hw, 6693 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6694 break; 6695 } else 6696 error = e1000_read_nvm(&sc->hw, 6697 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6698 break; 6699 case e1000_ich8lan: 6700 case e1000_ich9lan: 6701 case e1000_ich10lan: 6702 case e1000_pchlan: 6703 case e1000_pch2lan: 6704 case e1000_pch_lpt: 6705 case e1000_pch_spt: 6706 case e1000_pch_cnp: 6707 case e1000_pch_tgp: 6708 case e1000_pch_adp: 6709 case e1000_pch_mtp: 6710 case e1000_pch_ptp: 6711 case e1000_pch_nvp: 6712 apme_mask = E1000_WUC_APME; 6713 sc->has_amt = true; 6714 eeprom_data = E1000_READ_REG(&sc->hw, E1000_WUC); 6715 if (sc->hw.mac.type > e1000_ich10lan && 6716 (eeprom_data & E1000_WUC_PHY_WAKE) != 0) 6717 sc->wol_phy_wakeup = true; 6718 break; 6719 case e1000_82575: 6720 case e1000_82576: 6721 if (sc->hw.bus.func == 1) 6722 error = e1000_read_nvm(&sc->hw, 6723 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6724 else 6725 error = e1000_read_nvm(&sc->hw, 6726 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6727 sc->has_amt = true; 6728 break; 6729 case e1000_82580: 6730 case e1000_i350: 6731 case e1000_i354: 6732 case e1000_i210: 6733 case e1000_i211: 6734 error = e1000_read_nvm(&sc->hw, 6735 NVM_INIT_CONTROL3_PORT_A + 6736 NVM_82580_LAN_FUNC_OFFSET(sc->hw.bus.func), 1, 6737 &eeprom_data); 6738 sc->has_amt = true; 6739 break; 6740 default: 6741 error = e1000_read_nvm(&sc->hw, 6742 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6743 break; 6744 } 6745 if (error != E1000_SUCCESS && bootverbose) 6746 device_printf(dev, "NVM read failed while checking WoL: %d\n", 6747 error); 6748 if ((sc->hw.mac.type == e1000_i210 || 6749 sc->hw.mac.type == e1000_i211) && 6750 sc->hw.nvm.type == e1000_nvm_invm) { 6751 /* The shared reader does not expose the optional iNVM word. */ 6752 apme = (E1000_READ_REG(&sc->hw, E1000_WUC) & 6753 E1000_WUC_APME) != 0; 6754 } else { 6755 apme = error == E1000_SUCCESS && 6756 (eeprom_data & apme_mask) != 0; 6757 } 6758 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6759 IFCAP_WOL : 0; 6760 if (sc->hw.mac.type == e1000_82542 || 6761 sc->hw.mac.type == e1000_82543) 6762 wol_capabilities = 0; 6763 6764 /* APME selects the default; board and port restrictions select support. */ 6765 device_id = pci_get_device(dev); 6766 switch (device_id) { 6767 case E1000_DEV_ID_82542: 6768 case E1000_DEV_ID_82543GC_FIBER: 6769 case E1000_DEV_ID_82543GC_COPPER: 6770 case E1000_DEV_ID_82541ER: 6771 case E1000_DEV_ID_82541ER_LOM: 6772 case E1000_DEV_ID_82544EI_FIBER: 6773 case E1000_DEV_ID_82545EM_COPPER: 6774 case E1000_DEV_ID_82545EM_FIBER: 6775 case E1000_DEV_ID_82546EB_QUAD_COPPER: 6776 case E1000_DEV_ID_82546GB_QUAD_COPPER: 6777 case E1000_DEV_ID_82546GB_PCIE: 6778 wol_capabilities = 0; 6779 break; 6780 case E1000_DEV_ID_82546EB_FIBER: 6781 case E1000_DEV_ID_82546GB_FIBER: 6782 /* 6783 * Wake events are supported only on port A for dual fiber, 6784 * regardless of the NVM setting. 6785 */ 6786 if (sc->hw.bus.func == 1) 6787 wol_capabilities = 0; 6788 break; 6789 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 6790 /* if quad port adapter, disable WoL on all but port A */ 6791 if (global_quad_port_a != 0) 6792 wol_capabilities = 0; 6793 else 6794 wol_capabilities &= ~IFCAP_WOL_UCAST; 6795 /* Reset for multiple quad port adapters */ 6796 if (++global_quad_port_a == 4) 6797 global_quad_port_a = 0; 6798 break; 6799 case E1000_DEV_ID_82571EB_COPPER: 6800 case E1000_DEV_ID_82571EB_FIBER: 6801 case E1000_DEV_ID_82571EB_SERDES: 6802 /* These dual-port adapters support wake only on port A. */ 6803 if (sc->hw.bus.func == 1) 6804 wol_capabilities = 0; 6805 break; 6806 case E1000_DEV_ID_82571EB_SERDES_QUAD: 6807 wol_capabilities = 0; 6808 break; 6809 case E1000_DEV_ID_82571EB_QUAD_COPPER: 6810 case E1000_DEV_ID_82571EB_QUAD_FIBER: 6811 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 6812 case E1000_DEV_ID_82571PT_QUAD_COPPER: 6813 /* if quad port adapter, disable WoL on all but port A */ 6814 if (global_quad_port_a != 0) 6815 wol_capabilities = 0; 6816 /* Reset for multiple quad port adapters */ 6817 if (++global_quad_port_a == 4) 6818 global_quad_port_a = 0; 6819 break; 6820 case E1000_DEV_ID_82575GB_QUAD_COPPER: 6821 wol_capabilities = 0; 6822 break; 6823 case E1000_DEV_ID_82575EB_FIBER_SERDES: 6824 case E1000_DEV_ID_82576_FIBER: 6825 case E1000_DEV_ID_82576_SERDES: 6826 if (sc->hw.bus.func == 1) 6827 wol_capabilities = 0; 6828 break; 6829 case E1000_DEV_ID_82576_QUAD_COPPER: 6830 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 6831 if (global_quad_port_a != 0) 6832 wol_capabilities = 0; 6833 if (++global_quad_port_a == 4) 6834 global_quad_port_a = 0; 6835 break; 6836 default: 6837 break; 6838 } 6839 /* Legacy and igb non-primary ports require an explicit NVM setting. */ 6840 if ((sc->hw.mac.type < e1000_82571 || 6841 sc->hw.mac.type >= igb_mac_min) && sc->hw.bus.func != 0 && 6842 !apme) 6843 wol_capabilities = 0; 6844 6845 /* Some I350-family systems expose wake support but default it off. */ 6846 if ((sc->hw.mac.type == e1000_i350 && 6847 pci_get_subvendor(dev) == EM_SUBVENDOR_HP) || 6848 ((sc->hw.mac.type == e1000_i350 || 6849 sc->hw.mac.type == e1000_i354) && 6850 pci_get_subvendor(dev) == EM_SUBVENDOR_DELL) || 6851 (sc->hw.mac.type == e1000_i350 && 6852 ((pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_2 || 6853 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_3) && 6854 sc->hw.bus.func == 0))) { 6855 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6856 IFCAP_WOL : 0; 6857 apme = false; 6858 } 6859 if (sc->hw.mac.type == e1000_i350 && 6860 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_1) 6861 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6862 IFCAP_WOL : 0; 6863 6864 scctx->isc_capabilities &= ~IFCAP_WOL; 6865 scctx->isc_capabilities |= wol_capabilities; 6866 scctx->isc_capenable &= ~IFCAP_WOL; 6867 if (wol_capabilities != 0 && apme) 6868 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 6869 } 6870 6871 /* Configure the requested PCI Wake-on-LAN filters for suspend. */ 6872 static int 6873 em_enable_wakeup(if_ctx_t ctx) 6874 { 6875 struct e1000_softc *sc = iflib_get_softc(ctx); 6876 device_t dev = iflib_get_dev(ctx); 6877 if_t ifp = iflib_get_ifp(ctx); 6878 int enabled, error = 0, master_error, mcnt; 6879 u32 ctrl, ctrl_ext, rctl, saved_rctl, wuc, wufc; 6880 bool manage, rctl_modified; 6881 6882 if (sc->vf_ifp) 6883 return (0); 6884 if (!pci_has_pme(dev, PCI_POWERSTATE_D3_HOT)) 6885 return (0); 6886 6887 enabled = if_getcapenable(ifp) & if_getcapabilities(ifp) & IFCAP_WOL; 6888 manage = e1000_enable_mng_pass_thru(&sc->hw); 6889 rctl_modified = false; 6890 wuc = 0; 6891 /* Early 82545EM/82546EB need APM clocks for D3 manageability. */ 6892 if (manage && (sc->hw.mac.type == e1000_82545 || 6893 sc->hw.mac.type == e1000_82546)) 6894 wuc = E1000_WUC_APME; 6895 wufc = 0; 6896 if ((enabled & IFCAP_WOL_MAGIC) != 0) 6897 wufc |= E1000_WUFC_MAG; 6898 if ((enabled & IFCAP_WOL_UCAST) != 0) 6899 wufc |= E1000_WUFC_EX; 6900 if ((enabled & IFCAP_WOL_MCAST) != 0) { 6901 wufc |= E1000_WUFC_MC; 6902 bzero(sc->mta, ETHER_ADDR_LEN * 6903 MAX_NUM_MULTICAST_ADDRESSES); 6904 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, sc->mta); 6905 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) { 6906 e1000_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 6907 } else { 6908 switch (sc->hw.mac.type) { 6909 case e1000_82544: 6910 case e1000_82540: 6911 case e1000_82545: 6912 case e1000_82545_rev_3: 6913 case e1000_82546: 6914 case e1000_82546_rev_3: 6915 case e1000_82541: 6916 case e1000_82541_rev_2: 6917 case e1000_82547: 6918 case e1000_82547_rev_2: 6919 case e1000_82575: 6920 case e1000_82576: 6921 case e1000_82580: 6922 /* These parts require an MTA hit for WUFC_MC. */ 6923 em_fill_wakeup_mta(&sc->hw); 6924 break; 6925 default: 6926 break; 6927 } 6928 } 6929 } 6930 6931 if (wufc == 0) { 6932 if (sc->hw.mac.type >= e1000_82544) { 6933 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 6934 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 6935 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6936 } 6937 if (sc->wol_phy_wakeup && sc->wol_phy_armed) 6938 (void)em_disable_phy_wakeup(sc, NULL); 6939 if (manage) { 6940 if (sc->suspend_link_powered_down) 6941 em_power_up_wakeup_link(sc); 6942 em_configure_sx_low_power(sc, 0); 6943 pci_enable_pme(dev); 6944 } else { 6945 em_power_down_wakeup_link(sc); 6946 pci_clear_pme(dev); 6947 } 6948 goto master_disable; 6949 } 6950 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 6951 error = e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 6952 if (error != E1000_SUCCESS) { 6953 device_printf(dev, 6954 "Could not restore unicast wake address: %d\n", error); 6955 goto pme; 6956 } 6957 saved_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 6958 rctl = saved_rctl; 6959 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_MO_3); 6960 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 6961 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 6962 if ((wufc & E1000_WUFC_MC) != 0) 6963 rctl |= E1000_RCTL_MPE; 6964 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 6965 rctl_modified = true; 6966 6967 /* Advertise the wakeup capability */ 6968 if (sc->hw.mac.type >= e1000_82540) { 6969 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 6970 ctrl |= E1000_CTRL_ADVD3WUC; 6971 if (sc->hw.mac.type < igb_mac_min && !sc->wol_phy_wakeup) 6972 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6973 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 6974 } 6975 6976 /* Keep the laser running on legacy fiber and SerDes adapters. */ 6977 if (sc->hw.mac.type < igb_mac_min && 6978 (sc->hw.phy.media_type == e1000_media_type_fiber || 6979 sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 6980 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6981 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6982 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, ctrl_ext); 6983 } 6984 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6985 em_power_up_wakeup_link(sc); 6986 6987 if (sc->hw.mac.type >= e1000_ich8lan && 6988 sc->hw.mac.type < igb_mac_min) 6989 e1000_suspend_workarounds_ich8lan(&sc->hw); 6990 6991 if (sc->wol_phy_wakeup) { 6992 error = em_enable_phy_wakeup(sc, wufc); 6993 if (error) 6994 goto pme; 6995 } else { 6996 /* Enable wakeup by the MAC */ 6997 E1000_WRITE_REG(&sc->hw, E1000_WUC, 6998 wuc | E1000_WUC_PME_EN); 6999 E1000_WRITE_REG(&sc->hw, E1000_WUFC, wufc); 7000 } 7001 7002 /* The IGP3 D3 power-down workaround is specific to the em family. */ 7003 if (sc->hw.mac.type < igb_mac_min && 7004 sc->hw.phy.type == e1000_phy_igp_3) 7005 e1000_igp3_phy_powerdown_workaround_ich8lan(&sc->hw); 7006 em_configure_sx_low_power(sc, wufc); 7007 7008 pme: 7009 if (!error) 7010 pci_enable_pme(dev); 7011 else { 7012 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 7013 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 7014 if (rctl_modified) 7015 E1000_WRITE_REG(&sc->hw, E1000_RCTL, saved_rctl); 7016 pci_clear_pme(dev); 7017 } 7018 7019 master_disable: 7020 master_error = e1000_disable_pcie_master(&sc->hw); 7021 if (master_error != E1000_SUCCESS) 7022 device_printf(dev, "PCIe master disable timed out: %d\n", 7023 master_error); 7024 master_error = pci_disable_busmaster(dev); 7025 if (master_error != 0) 7026 device_printf(dev, "PCI bus-master disable failed: %d\n", 7027 master_error); 7028 7029 return (error == E1000_SUCCESS ? 0 : EIO); 7030 } 7031 7032 /* Configure the PCH low-power link modes used while the system sleeps. */ 7033 static void 7034 em_configure_sx_low_power(struct e1000_softc *sc, u32 wufc) 7035 { 7036 struct e1000_hw *hw = &sc->hw; 7037 struct e1000_dev_spec_ich8lan *dev_spec; 7038 s32 error; 7039 u16 eee_advert, lpi_ctrl; 7040 7041 if (hw->mac.type < e1000_pch_lpt || hw->mac.type >= igb_mac_min || 7042 sc->suspend_link_powered_down) 7043 return; 7044 7045 if (wufc != 0 && 7046 (wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC)) == 0) { 7047 /* ULP cannot preserve directed or broad multicast wake. */ 7048 error = e1000_enable_ulp_lpt_lp(hw, true); 7049 if (error != E1000_SUCCESS) { 7050 device_printf(sc->dev, 7051 "Could not enter PHY ultra-low-power mode: %d\n", 7052 error); 7053 return; 7054 } 7055 } 7056 7057 dev_spec = &hw->dev_spec.ich8lan; 7058 if (hw->phy.type != e1000_phy_i217 || dev_spec->eee_disable || 7059 dev_spec->eee_lp_ability == 0) 7060 return; 7061 7062 error = hw->phy.ops.acquire(hw); 7063 if (error != E1000_SUCCESS) 7064 goto out; 7065 error = hw->phy.ops.read_reg_locked(hw, I82579_LPI_CTRL, &lpi_ctrl); 7066 if (error != E1000_SUCCESS) 7067 goto release; 7068 error = e1000_read_emi_reg_locked(hw, I217_EEE_ADVERTISEMENT, 7069 &eee_advert); 7070 if (error != E1000_SUCCESS) 7071 goto release; 7072 7073 if ((eee_advert & dev_spec->eee_lp_ability & 7074 I82579_EEE_100_SUPPORTED) != 0) 7075 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 7076 if ((eee_advert & dev_spec->eee_lp_ability & 7077 I82579_EEE_1000_SUPPORTED) != 0) 7078 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 7079 error = hw->phy.ops.write_reg_locked(hw, I82579_LPI_CTRL, lpi_ctrl); 7080 release: 7081 hw->phy.ops.release(hw); 7082 out: 7083 if (error != E1000_SUCCESS) 7084 device_printf(sc->dev, 7085 "Could not configure Energy Efficient Ethernet for sleep: %d\n", 7086 error); 7087 } 7088 7089 static void 7090 em_power_up_wakeup_link(struct e1000_softc *sc) 7091 { 7092 struct e1000_hw *hw = &sc->hw; 7093 7094 if (hw->mac.type < igb_mac_min) 7095 e1000_power_up_phy(hw); 7096 else if (hw->phy.media_type == e1000_media_type_copper) 7097 e1000_power_up_phy(hw); 7098 else { 7099 e1000_power_up_fiber_serdes_link(hw); 7100 (void)e1000_setup_link(hw); 7101 } 7102 sc->suspend_link_powered_down = false; 7103 } 7104 7105 /* Drop the unused suspend link through the controller's shared-code hook. */ 7106 static void 7107 em_power_down_wakeup_link(struct e1000_softc *sc) 7108 { 7109 struct e1000_hw *hw = &sc->hw; 7110 7111 if (hw->mac.type >= igb_mac_min && 7112 hw->phy.media_type != e1000_media_type_copper) 7113 e1000_shutdown_fiber_serdes_link(hw); 7114 else 7115 e1000_power_down_phy(hw); 7116 sc->suspend_link_powered_down = true; 7117 } 7118 7119 /* PCH PHY wake requires the MAC receive state on the BM wake page. */ 7120 static int 7121 em_enable_phy_wakeup(struct e1000_softc *sc, u32 wufc) 7122 { 7123 struct e1000_hw *hw = &sc->hw; 7124 u32 mreg, wuc; 7125 u16 preg, wuc_enable; 7126 s32 error, restore_error; 7127 7128 /* Copy MAC RARs to PHY RARs before selecting the BM wake page. */ 7129 error = e1000_copy_rx_addrs_to_phy_ich8lan(hw); 7130 if (error != E1000_SUCCESS) 7131 goto out; 7132 7133 error = hw->phy.ops.acquire(hw); 7134 if (error != E1000_SUCCESS) { 7135 device_printf(sc->dev, "Could not acquire PHY for wakeup\n"); 7136 goto out; 7137 } 7138 7139 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7140 if (error != E1000_SUCCESS) 7141 goto release; 7142 7143 /* Wake status is RW1C and survives controller reset. */ 7144 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7145 if (error != E1000_SUCCESS) 7146 goto restore; 7147 7148 /* copy MAC MTA to PHY MTA */ 7149 for (int i = 0; i < hw->mac.mta_reg_count; i++) { 7150 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 7151 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i), 7152 (u16)(mreg & 0xffff)); 7153 if (error != E1000_SUCCESS) 7154 goto restore; 7155 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 7156 (u16)(mreg >> 16)); 7157 if (error != E1000_SUCCESS) 7158 goto restore; 7159 } 7160 7161 /* configure PHY Rx Control register */ 7162 error = hw->phy.ops.read_reg_page(hw, BM_RCTL, &preg); 7163 if (error != E1000_SUCCESS) 7164 goto restore; 7165 mreg = E1000_READ_REG(hw, E1000_RCTL); 7166 if (mreg & E1000_RCTL_UPE) 7167 preg |= BM_RCTL_UPE; 7168 if (mreg & E1000_RCTL_MPE) 7169 preg |= BM_RCTL_MPE; 7170 preg &= ~(BM_RCTL_MO_MASK); 7171 if (mreg & E1000_RCTL_MO_3) 7172 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 7173 << BM_RCTL_MO_SHIFT); 7174 if (mreg & E1000_RCTL_BAM) 7175 preg |= BM_RCTL_BAM; 7176 if (mreg & E1000_RCTL_PMCF) 7177 preg |= BM_RCTL_PMCF; 7178 mreg = E1000_READ_REG(hw, E1000_CTRL); 7179 if (mreg & E1000_CTRL_RFCE) 7180 preg |= BM_RCTL_RFCE; 7181 error = hw->phy.ops.write_reg_page(hw, BM_RCTL, preg); 7182 if (error != E1000_SUCCESS) 7183 goto restore; 7184 7185 wuc = E1000_WUC_PME_EN; 7186 if ((wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) != 0) 7187 wuc |= E1000_WUC_APME; 7188 7189 /* enable PHY wakeup in MAC register */ 7190 E1000_WRITE_REG(hw, E1000_WUFC, wufc); 7191 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PHY_WAKE | 7192 E1000_WUC_APMPME | E1000_WUC_PME_STATUS | wuc); 7193 7194 /* configure and enable PHY wakeup in PHY registers */ 7195 error = hw->phy.ops.write_reg_page(hw, BM_WUFC, wufc); 7196 if (error != E1000_SUCCESS) 7197 goto restore; 7198 error = hw->phy.ops.write_reg_page(hw, BM_WUC, wuc); 7199 if (error != E1000_SUCCESS) 7200 goto restore; 7201 7202 restore: 7203 /* Restore the page selector and expose only a complete setup. */ 7204 if (error == E1000_SUCCESS) 7205 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 7206 else 7207 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7208 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7209 &wuc_enable); 7210 if (error == E1000_SUCCESS) 7211 error = restore_error; 7212 release: 7213 hw->phy.ops.release(hw); 7214 out: 7215 sc->wol_phy_armed = error == E1000_SUCCESS; 7216 if (error != E1000_SUCCESS) 7217 device_printf(sc->dev, "Could not configure PHY wakeup: %d\n", 7218 error); 7219 7220 return (error); 7221 } 7222 7223 /* Clear host ownership and sticky status without disturbing ME wake. */ 7224 static int 7225 em_disable_phy_wakeup(struct e1000_softc *sc, u16 *wus) 7226 { 7227 struct e1000_hw *hw = &sc->hw; 7228 s32 error, restore_error; 7229 u16 phy_wus, wuc_enable; 7230 7231 error = hw->phy.ops.acquire(hw); 7232 if (error != E1000_SUCCESS) 7233 return (error); 7234 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7235 if (error != E1000_SUCCESS) 7236 goto release; 7237 7238 error = hw->phy.ops.read_reg_page(hw, BM_WUS, &phy_wus); 7239 if (error == E1000_SUCCESS) 7240 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7241 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7242 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7243 &wuc_enable); 7244 if (error == E1000_SUCCESS) 7245 error = restore_error; 7246 release: 7247 hw->phy.ops.release(hw); 7248 if (error == E1000_SUCCESS) { 7249 sc->wol_phy_armed = false; 7250 if (wus != NULL) 7251 *wus = phy_wus; 7252 } 7253 return (error); 7254 } 7255 7256 static void 7257 em_if_led_func(if_ctx_t ctx, int onoff) 7258 { 7259 struct e1000_softc *sc = iflib_get_softc(ctx); 7260 7261 if (onoff) { 7262 e1000_setup_led(&sc->hw); 7263 if (sc->hw.phy.media_type == e1000_media_type_internal_serdes) 7264 e1000_blink_led(&sc->hw); 7265 else 7266 e1000_led_on(&sc->hw); 7267 } else { 7268 e1000_led_off(&sc->hw); 7269 e1000_cleanup_led(&sc->hw); 7270 } 7271 } 7272 7273 /* 7274 * Disable the L0S and L1 LINK states 7275 */ 7276 static void 7277 em_disable_aspm(struct e1000_softc *sc) 7278 { 7279 int base, reg; 7280 u16 link_cap,link_ctrl; 7281 device_t dev = sc->dev; 7282 7283 switch (sc->hw.mac.type) { 7284 case e1000_82573: 7285 case e1000_82574: 7286 case e1000_82583: 7287 break; 7288 default: 7289 return; 7290 } 7291 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) 7292 return; 7293 reg = base + PCIER_LINK_CAP; 7294 link_cap = pci_read_config(dev, reg, 2); 7295 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0) 7296 return; 7297 reg = base + PCIER_LINK_CTL; 7298 link_ctrl = pci_read_config(dev, reg, 2); 7299 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC; 7300 pci_write_config(dev, reg, link_ctrl, 2); 7301 return; 7302 } 7303 7304 /********************************************************************** 7305 * 7306 * Update the board statistics counters. 7307 * 7308 **********************************************************************/ 7309 void 7310 em_update_stats_counters(struct e1000_softc *sc) 7311 { 7312 struct e1000_hw_stats *stats; 7313 u64 prev_xoffrxc; 7314 7315 if (sc->vf_ifp) { 7316 em_update_vf_stats_counters(sc); 7317 return; 7318 } 7319 7320 stats = &sc->ustats.stats; 7321 prev_xoffrxc = stats->xoffrxc; 7322 7323 if(sc->hw.phy.media_type == e1000_media_type_copper || 7324 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 7325 stats->symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 7326 stats->sec += E1000_READ_REG(&sc->hw, E1000_SEC); 7327 } 7328 stats->crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 7329 stats->mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 7330 stats->scc += E1000_READ_REG(&sc->hw, E1000_SCC); 7331 stats->ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 7332 7333 stats->mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 7334 stats->latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 7335 stats->colc += E1000_READ_REG(&sc->hw, E1000_COLC); 7336 stats->dc += E1000_READ_REG(&sc->hw, E1000_DC); 7337 stats->rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 7338 stats->xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 7339 stats->xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 7340 stats->xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 7341 /* 7342 ** For watchdog management we need to know if we have been 7343 ** paused during the last interval, so capture that here. 7344 */ 7345 if (stats->xoffrxc != prev_xoffrxc) 7346 sc->shared->isc_pause_frames = 1; 7347 stats->xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 7348 stats->fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 7349 stats->prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 7350 stats->prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 7351 stats->prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 7352 stats->prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 7353 stats->prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 7354 stats->prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 7355 stats->gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 7356 stats->bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 7357 stats->mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 7358 stats->gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 7359 7360 /* For the 64-bit byte counters the low dword must be read first. */ 7361 /* Both registers clear on the read of the high dword */ 7362 7363 stats->gorc += E1000_READ_REG(&sc->hw, E1000_GORCL) + 7364 ((u64)E1000_READ_REG(&sc->hw, E1000_GORCH) << 32); 7365 stats->gotc += E1000_READ_REG(&sc->hw, E1000_GOTCL) + 7366 ((u64)E1000_READ_REG(&sc->hw, E1000_GOTCH) << 32); 7367 7368 stats->rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 7369 stats->ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 7370 stats->rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 7371 stats->roc += E1000_READ_REG(&sc->hw, E1000_ROC); 7372 stats->rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 7373 7374 stats->mgprc += E1000_READ_REG(&sc->hw, E1000_MGTPRC); 7375 stats->mgpdc += E1000_READ_REG(&sc->hw, E1000_MGTPDC); 7376 stats->mgptc += E1000_READ_REG(&sc->hw, E1000_MGTPTC); 7377 7378 stats->tor += E1000_READ_REG(&sc->hw, E1000_TORH); 7379 stats->tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 7380 7381 stats->tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 7382 stats->tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 7383 stats->ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 7384 stats->ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 7385 stats->ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 7386 stats->ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 7387 stats->ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 7388 stats->ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 7389 stats->mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 7390 stats->bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 7391 7392 /* TLPIC and RLPIC are clear-on-read. */ 7393 if (em_mac_has_eee(sc->hw.mac.type)) { 7394 stats->tlpic += E1000_READ_REG(&sc->hw, E1000_TLPIC); 7395 stats->rlpic += E1000_READ_REG(&sc->hw, E1000_RLPIC); 7396 } 7397 7398 /* Interrupt Counts */ 7399 7400 stats->iac += E1000_READ_REG(&sc->hw, E1000_IAC); 7401 stats->icrxptc += E1000_READ_REG(&sc->hw, E1000_ICRXPTC); 7402 stats->icrxatc += E1000_READ_REG(&sc->hw, E1000_ICRXATC); 7403 stats->ictxptc += E1000_READ_REG(&sc->hw, E1000_ICTXPTC); 7404 stats->ictxatc += E1000_READ_REG(&sc->hw, E1000_ICTXATC); 7405 stats->ictxqec += E1000_READ_REG(&sc->hw, E1000_ICTXQEC); 7406 stats->ictxqmtc += E1000_READ_REG(&sc->hw, E1000_ICTXQMTC); 7407 stats->icrxdmtc += E1000_READ_REG(&sc->hw, E1000_ICRXDMTC); 7408 stats->icrxoc += E1000_READ_REG(&sc->hw, E1000_ICRXOC); 7409 7410 if (sc->hw.mac.type >= e1000_82543) { 7411 stats->algnerrc += 7412 E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 7413 stats->rxerrc += 7414 E1000_READ_REG(&sc->hw, E1000_RXERRC); 7415 stats->tncrs += 7416 E1000_READ_REG(&sc->hw, E1000_TNCRS); 7417 stats->cexterr += 7418 E1000_READ_REG(&sc->hw, E1000_CEXTERR); 7419 stats->tsctc += 7420 E1000_READ_REG(&sc->hw, E1000_TSCTC); 7421 stats->tsctfc += 7422 E1000_READ_REG(&sc->hw, E1000_TSCTFC); 7423 } 7424 7425 if (em_has_82571_ecc_stats(&sc->hw)) 7426 em_update_82571_ecc_stats(sc); 7427 else if (em_has_pch_ecc(&sc->hw)) 7428 em_update_pch_ecc_stats(sc, 7429 E1000_READ_REG(&sc->hw, E1000_PBECCSTS)); 7430 else if (em_has_82575_memory_errors(&sc->hw)) 7431 em_update_82575_ecc_stats(sc, 7432 E1000_READ_REG(&sc->hw, E1000_PBECCSTS_82575), 7433 E1000_READ_REG(&sc->hw, E1000_RDHESTS_82575), 7434 E1000_READ_REG(&sc->hw, E1000_TDHESTS_82575)); 7435 else if (em_has_82576_memory_errors(&sc->hw)) 7436 em_update_82576_ecc_stats(sc); 7437 else if (em_has_82580_memory_errors(&sc->hw)) 7438 em_update_82580_ecc_stats(sc, 7439 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 7440 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 7441 E1000_READ_REG(&sc->hw, E1000_PCIEECCSTS)); 7442 else if (em_has_i350_i354_memory_errors(&sc->hw)) 7443 em_update_i350_i354_ecc_stats(sc); 7444 else if (em_has_i210_memory_errors(&sc->hw)) 7445 em_update_i210_ecc_stats(sc); 7446 } 7447 7448 static bool 7449 em_mac_has_eee(enum e1000_mac_type type) 7450 { 7451 7452 return ((type >= e1000_pch2lan && type < e1000_82575) || 7453 (type >= e1000_i350 && type <= e1000_i211)); 7454 } 7455 7456 static void 7457 em_initialize_vf_stats(struct e1000_softc *sc) 7458 { 7459 struct e1000_vf_stats *stats; 7460 7461 stats = &sc->ustats.vf_stats; 7462 *stats = (struct e1000_vf_stats){}; 7463 em_rebase_vf_stats(sc); 7464 } 7465 7466 static void 7467 em_rebase_vf_stats(struct e1000_softc *sc) 7468 { 7469 struct e1000_vf_stats *stats; 7470 bool hyperv = igbv_is_hyperv(sc); 7471 7472 sc->vf_stats_valid = false; 7473 if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) 7474 return; 7475 7476 /* 7477 * A PF reset starts a new VF counter epoch. Preserve the accumulated 7478 * totals while establishing a new raw baseline so the reset is not 7479 * mistaken for a 32-bit wrap. 7480 */ 7481 stats = &sc->ustats.vf_stats; 7482 #define INIT_VF_REG(reg, name) do { \ 7483 stats->last_##name = E1000_READ_REG(&sc->hw, reg); \ 7484 } while (0) 7485 INIT_VF_REG(E1000_VFGPRC, gprc); 7486 INIT_VF_REG(E1000_VFGORC, gorc); 7487 INIT_VF_REG(E1000_VFGPTC, gptc); 7488 INIT_VF_REG(E1000_VFGOTC, gotc); 7489 /* 7490 * I350 specification update erratum 31 says VFMPRC is not 7491 * accessible from VF memory. The 0xf3c register remains valid on 7492 * 82576 VFs, but must not be read on vfadapt_i350. 7493 */ 7494 if (sc->hw.mac.type == e1000_vfadapt) 7495 INIT_VF_REG(E1000_VFMPRC, mprc); 7496 else 7497 stats->last_mprc = 0; 7498 INIT_VF_REG(E1000_VFGOTLBC, gotlbc); 7499 INIT_VF_REG(E1000_VFGPTLBC, gptlbc); 7500 INIT_VF_REG(E1000_VFGORLBC, gorlbc); 7501 INIT_VF_REG(E1000_VFGPRLBC, gprlbc); 7502 #undef INIT_VF_REG 7503 sc->vf_stats_valid = !hyperv || 7504 E1000_READ_REG(&sc->hw, E1000_STATUS) != 0xffffffff; 7505 } 7506 7507 static void 7508 em_update_vf_stats_counters(struct e1000_softc *sc) 7509 { 7510 struct e1000_vf_stats sample; 7511 struct e1000_vf_stats *stats; 7512 bool hyperv, reset; 7513 7514 hyperv = igbv_is_hyperv(sc); 7515 if (hyperv && E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) { 7516 sc->vf_stats_valid = false; 7517 return; 7518 } 7519 reset = hyperv && e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS; 7520 if (hyperv && !sc->vf_stats_valid) 7521 reset = true; 7522 if (hyperv && (E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) & 7523 E1000_TXDCTL_QUEUE_ENABLE) == 0) 7524 reset = true; 7525 sample = sc->ustats.vf_stats; 7526 stats = &sample; 7527 7528 /* 7529 * Internal VF loopback traffic can continue without physical link, 7530 * so sample the counters regardless of link state. 7531 */ 7532 UPDATE_VF_REG(E1000_VFGPRC, 7533 stats->last_gprc, stats->gprc); 7534 UPDATE_VF_REG(E1000_VFGORC, 7535 stats->last_gorc, stats->gorc); 7536 UPDATE_VF_REG(E1000_VFGPTC, 7537 stats->last_gptc, stats->gptc); 7538 UPDATE_VF_REG(E1000_VFGOTC, 7539 stats->last_gotc, stats->gotc); 7540 if (sc->hw.mac.type == e1000_vfadapt) 7541 UPDATE_VF_REG(E1000_VFMPRC, 7542 stats->last_mprc, stats->mprc); 7543 UPDATE_VF_REG(E1000_VFGOTLBC, 7544 stats->last_gotlbc, stats->gotlbc); 7545 UPDATE_VF_REG(E1000_VFGPTLBC, 7546 stats->last_gptlbc, stats->gptlbc); 7547 UPDATE_VF_REG(E1000_VFGORLBC, 7548 stats->last_gorlbc, stats->gorlbc); 7549 UPDATE_VF_REG(E1000_VFGPRLBC, 7550 stats->last_gprlbc, stats->gprlbc); 7551 /* 7552 * Hyper-V can reset the VF without a native mailbox handshake. Do not 7553 * count that counter clear as a 32-bit wrap, including a reset during 7554 * this sweep. The PF can also clear counters while blocking a queue 7555 * for MDD without leaving a reset indication. Hyper-V has one queue; 7556 * a disabled queue cannot supply a valid running counter epoch. 7557 */ 7558 if (hyperv) { 7559 if (e1000_check_for_rst(&sc->hw, 0) == E1000_SUCCESS) 7560 reset = true; 7561 if ((E1000_READ_REG(&sc->hw, E1000_TXDCTL(0)) & 7562 E1000_TXDCTL_QUEUE_ENABLE) == 0) 7563 reset = true; 7564 if (E1000_READ_REG(&sc->hw, E1000_STATUS) == 0xffffffff) { 7565 /* Rebase on a good sample before accounting further deltas. */ 7566 sc->vf_stats_valid = false; 7567 return; 7568 } 7569 } 7570 if (reset) 7571 em_rebase_vf_stats(sc); 7572 else 7573 sc->ustats.vf_stats = sample; 7574 } 7575 7576 static uint64_t 7577 em_if_get_vf_counter(if_ctx_t ctx, ift_counter cnt) 7578 { 7579 struct e1000_softc *sc = iflib_get_softc(ctx); 7580 if_t ifp = iflib_get_ifp(ctx); 7581 7582 switch (cnt) { 7583 case IFCOUNTER_IERRORS: 7584 return sc->dropped_pkts; 7585 default: 7586 return (if_get_counter_default(ifp, cnt)); 7587 } 7588 } 7589 7590 static uint64_t 7591 em_if_get_counter(if_ctx_t ctx, ift_counter cnt) 7592 { 7593 struct e1000_softc *sc = iflib_get_softc(ctx); 7594 struct e1000_hw_stats *stats; 7595 if_t ifp = iflib_get_ifp(ctx); 7596 7597 if (sc->vf_ifp) 7598 return (em_if_get_vf_counter(ctx, cnt)); 7599 7600 stats = &sc->ustats.stats; 7601 7602 switch (cnt) { 7603 case IFCOUNTER_COLLISIONS: 7604 return (stats->colc); 7605 case IFCOUNTER_IERRORS: 7606 return (sc->dropped_pkts + stats->rxerrc + 7607 stats->crcerrs + stats->algnerrc + 7608 stats->ruc + stats->roc + 7609 stats->mpc + stats->cexterr); 7610 case IFCOUNTER_OERRORS: 7611 return (if_get_counter_default(ifp, cnt) + 7612 stats->ecol + stats->latecol); 7613 default: 7614 return (if_get_counter_default(ifp, cnt)); 7615 } 7616 } 7617 7618 /* em_if_needs_restart - Tell iflib when the driver needs to be reinitialized 7619 * @ctx: iflib context 7620 * @event: event code to check 7621 * 7622 * Defaults to returning false for unknown events. 7623 * 7624 * @returns true if iflib needs to reinit the interface 7625 */ 7626 static bool 7627 em_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 7628 { 7629 switch (event) { 7630 case IFLIB_RESTART_VLAN_CONFIG: 7631 default: 7632 return (false); 7633 } 7634 } 7635 7636 /* Export a single 32-bit register via a read-only sysctl. */ 7637 static int 7638 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 7639 { 7640 struct e1000_softc *sc; 7641 u_int val; 7642 7643 sc = oidp->oid_arg1; 7644 val = E1000_READ_REG(&sc->hw, oidp->oid_arg2); 7645 return (sysctl_handle_int(oidp, &val, 0, req)); 7646 } 7647 7648 enum em_ring_register { 7649 EM_RING_HEAD, 7650 EM_RING_TAIL, 7651 }; 7652 7653 /* Queue register addresses can change when the PF enters IOV mode. */ 7654 static int 7655 em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS) 7656 { 7657 struct tx_ring *txr; 7658 u_int reg, val; 7659 7660 txr = oidp->oid_arg1; 7661 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) : 7662 E1000_TDT(txr->me); 7663 val = E1000_READ_REG(&txr->sc->hw, reg); 7664 return (sysctl_handle_int(oidp, &val, 0, req)); 7665 } 7666 7667 static int 7668 em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS) 7669 { 7670 struct rx_ring *rxr; 7671 u_int reg, val; 7672 7673 rxr = oidp->oid_arg1; 7674 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) : 7675 E1000_RDT(rxr->me); 7676 val = E1000_READ_REG(&rxr->sc->hw, reg); 7677 return (sysctl_handle_int(oidp, &val, 0, req)); 7678 } 7679 7680 /* Per queue holdoff interrupt rate handler */ 7681 static int 7682 em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 7683 { 7684 struct em_rx_queue *rque; 7685 struct em_tx_queue *tque; 7686 struct e1000_hw *hw; 7687 int error; 7688 u32 reg, usec, rate; 7689 7690 bool tx = oidp->oid_arg2; 7691 7692 if (tx) { 7693 tque = oidp->oid_arg1; 7694 hw = &tque->sc->hw; 7695 if (hw->mac.type >= igb_mac_min) 7696 reg = E1000_READ_REG(hw, E1000_EITR(tque->msix)); 7697 else if (hw->mac.type == e1000_82574 && 7698 tque->sc->intr_type == IFLIB_INTR_MSIX) 7699 reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->msix)); 7700 else 7701 reg = E1000_READ_REG(hw, E1000_ITR); 7702 } else { 7703 rque = oidp->oid_arg1; 7704 hw = &rque->sc->hw; 7705 if (hw->mac.type >= igb_mac_min) 7706 reg = E1000_READ_REG(hw, E1000_EITR(rque->msix)); 7707 else if (hw->mac.type == e1000_82574 && 7708 rque->sc->intr_type == IFLIB_INTR_MSIX) 7709 reg = E1000_READ_REG(hw, 7710 E1000_EITR_82574(rque->msix)); 7711 else 7712 reg = E1000_READ_REG(hw, E1000_ITR); 7713 } 7714 7715 if (hw->mac.type < igb_mac_min) { 7716 if (reg > 0) 7717 rate = EM_INTS_TO_ITR(reg); 7718 else 7719 rate = 0; 7720 } else { 7721 usec = (reg & IGB_QVECTOR_MASK); 7722 if (usec > 0) 7723 rate = IGB_EITR_TO_INTS(usec); 7724 else 7725 rate = 0; 7726 } 7727 7728 error = sysctl_handle_int(oidp, &rate, 0, req); 7729 if (error || !req->newptr) 7730 return error; 7731 return 0; 7732 } 7733 7734 /* 7735 * Add sysctl variables, one per statistic, to the system. 7736 */ 7737 static void 7738 em_add_hw_stats(struct e1000_softc *sc) 7739 { 7740 device_t dev = iflib_get_dev(sc->ctx); 7741 struct em_tx_queue *tx_que = sc->tx_queues; 7742 struct em_rx_queue *rx_que = sc->rx_queues; 7743 7744 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 7745 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 7746 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 7747 struct e1000_hw_stats *stats; 7748 7749 struct sysctl_oid *stat_node, *queue_node, *int_node; 7750 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 7751 7752 #define QUEUE_NAME_LEN 32 7753 char namebuf[QUEUE_NAME_LEN]; 7754 7755 /* Driver Statistics */ 7756 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 7757 CTLFLAG_RD, &sc->dropped_pkts, 7758 "Driver dropped packets"); 7759 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 7760 CTLFLAG_RD, &sc->link_irq, 7761 "Link MSI-X IRQ Handled"); 7762 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 7763 CTLFLAG_RD, &sc->rx_overruns, 7764 "RX overruns"); 7765 if (!sc->vf_ifp) { 7766 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 7767 CTLTYPE_UINT | CTLFLAG_RD, 7768 sc, E1000_CTRL, em_sysctl_reg_handler, "IU", 7769 "Device Control Register"); 7770 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 7771 CTLTYPE_UINT | CTLFLAG_RD, 7772 sc, E1000_RCTL, em_sysctl_reg_handler, "IU", 7773 "Receiver Control Register"); 7774 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 7775 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 7776 "Flow Control High Watermark"); 7777 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 7778 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 7779 "Flow Control Low Watermark"); 7780 } 7781 7782 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 7783 struct tx_ring *txr = &tx_que->txr; 7784 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 7785 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7786 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 7787 queue_list = SYSCTL_CHILDREN(queue_node); 7788 7789 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7790 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 7791 true, em_sysctl_interrupt_rate_handler, 7792 "IU", "Interrupt Rate"); 7793 7794 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 7795 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD, 7796 em_sysctl_tx_ring_handler, "IU", 7797 "Transmit Descriptor Head"); 7798 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 7799 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL, 7800 em_sysctl_tx_ring_handler, "IU", 7801 "Transmit Descriptor Tail"); 7802 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 7803 CTLFLAG_RD, &txr->tx_irq, 7804 "Queue MSI-X Transmit Interrupts"); 7805 } 7806 7807 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 7808 struct rx_ring *rxr = &rx_que->rxr; 7809 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 7810 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7811 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 7812 queue_list = SYSCTL_CHILDREN(queue_node); 7813 7814 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7815 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 7816 false, em_sysctl_interrupt_rate_handler, 7817 "IU", "Interrupt Rate"); 7818 7819 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 7820 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD, 7821 em_sysctl_rx_ring_handler, "IU", 7822 "Receive Descriptor Head"); 7823 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 7824 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL, 7825 em_sysctl_rx_ring_handler, "IU", 7826 "Receive Descriptor Tail"); 7827 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 7828 CTLFLAG_RD, &rxr->rx_irq, 7829 "Queue MSI-X Receive Interrupts"); 7830 } 7831 7832 /* MAC stats get their own sub node */ 7833 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 7834 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 7835 stat_list = SYSCTL_CHILDREN(stat_node); 7836 7837 /* 7838 ** VF adapter has a very limited set of stats 7839 ** since its not managing the metal, so to speak. 7840 */ 7841 if (sc->vf_ifp) { 7842 struct e1000_vf_stats *vfstats = &sc->ustats.vf_stats; 7843 7844 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 7845 CTLFLAG_RD, &vfstats->gprc, 7846 "Good Packets Received"); 7847 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 7848 CTLFLAG_RD, &vfstats->gptc, 7849 "Good Packets Transmitted"); 7850 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 7851 CTLFLAG_RD, &vfstats->gorc, 7852 "Good Octets Received"); 7853 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 7854 CTLFLAG_RD, &vfstats->gotc, 7855 "Good Octets Transmitted"); 7856 if (sc->hw.mac.type == e1000_vfadapt) { 7857 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7858 "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc, 7859 "Multicast Packets Received"); 7860 } 7861 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7862 "loopback_good_pkts_recvd", 7863 CTLFLAG_RD, &vfstats->gprlbc, 7864 "Good Loopback Packets Received"); 7865 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7866 "loopback_good_pkts_txd", 7867 CTLFLAG_RD, &vfstats->gptlbc, 7868 "Good Loopback Packets Transmitted"); 7869 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7870 "loopback_good_octets_recvd", 7871 CTLFLAG_RD, &vfstats->gorlbc, 7872 "Good Loopback Octets Received"); 7873 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7874 "loopback_good_octets_txd", 7875 CTLFLAG_RD, &vfstats->gotlbc, 7876 "Good Loopback Octets Transmitted"); 7877 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7878 "rx_csum_offload_good", 7879 CTLFLAG_RD, &sc->rx_csum_good, 7880 "Receive Checksum Offload Successes"); 7881 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7882 "rx_csum_offload_errors", 7883 CTLFLAG_RD, &sc->rx_csum_errors, 7884 "Receive Checksum Offload Errors"); 7885 return; 7886 } 7887 7888 stats = &sc->ustats.stats; 7889 if (em_mac_has_eee(sc->hw.mac.type)) { 7890 struct sysctl_oid *eee_node; 7891 struct sysctl_oid_list *eee_list; 7892 7893 eee_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eee", 7894 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7895 "Energy Efficient Ethernet statistics"); 7896 eee_list = SYSCTL_CHILDREN(eee_node); 7897 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "tx_lpi_count", 7898 CTLFLAG_RD, &stats->tlpic, "TX LPI event count"); 7899 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "rx_lpi_count", 7900 CTLFLAG_RD, &stats->rlpic, "RX LPI event count"); 7901 } 7902 if (em_has_memory_error_stats(&sc->hw)) { 7903 struct sysctl_oid *memerr_node; 7904 struct sysctl_oid_list *memerr_list; 7905 7906 memerr_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 7907 "memory_errors", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7908 "Internal memory error indications"); 7909 memerr_list = SYSCTL_CHILDREN(memerr_node); 7910 if (em_has_memory_errors(&sc->hw)) 7911 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7912 "fatal_resets", CTLFLAG_RD, 7913 &sc->fatal_error_reset_count, 7914 "Resets requested for fatal internal memory errors"); 7915 if (em_has_82571_ecc_stats(&sc->hw)) { 7916 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7917 "detected_packet_buffer", CTLFLAG_RD, 7918 &sc->corrected_error_packet_buffer_count, 7919 "Detected packet-buffer ECC errors"); 7920 } else if (em_has_pch_ecc(&sc->hw)) { 7921 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7922 "corrected_packet_buffer", CTLFLAG_RD, 7923 &sc->corrected_error_packet_buffer_count, 7924 "Corrected packet-buffer ECC errors"); 7925 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7926 "uncorrected_packet_buffer", CTLFLAG_RD, 7927 &sc->uncorrected_error_packet_buffer_count, 7928 "Uncorrected packet-buffer ECC errors"); 7929 } else if (em_has_82575_memory_errors(&sc->hw)) { 7930 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7931 "corrected_packet_buffer", CTLFLAG_RD, 7932 &sc->corrected_error_packet_buffer_count, 7933 "Corrected packet-buffer ECC errors"); 7934 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7935 "uncorrected_packet_buffer", CTLFLAG_RD, 7936 &sc->uncorrected_error_packet_buffer_count, 7937 "Uncorrected packet-buffer ECC errors"); 7938 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7939 "corrected_descriptor_handler", CTLFLAG_RD, 7940 &sc->corrected_error_dma_count, 7941 "Corrected descriptor-handler ECC errors"); 7942 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7943 "uncorrected_descriptor_handler", CTLFLAG_RD, 7944 &sc->uncorrected_error_dma_count, 7945 "Uncorrected descriptor-handler ECC errors"); 7946 } else if (em_has_82576_memory_errors(&sc->hw)) { 7947 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7948 "fatal_unknown", CTLFLAG_RD, 7949 &sc->fatal_error_unknown_count, 7950 "Fatal memory errors without a reported source"); 7951 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7952 "corrected_packet_buffer", CTLFLAG_RD, 7953 &sc->corrected_error_packet_buffer_count, 7954 "Corrected packet and switch-buffer ECC errors"); 7955 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7956 "uncorrected_packet_buffer", CTLFLAG_RD, 7957 &sc->uncorrected_error_packet_buffer_count, 7958 "Uncorrected packet and switch-buffer ECC errors"); 7959 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7960 "corrected_descriptor_handler", CTLFLAG_RD, 7961 &sc->corrected_error_dma_count, 7962 "Corrected descriptor-handler ECC errors"); 7963 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7964 "uncorrected_descriptor_handler", CTLFLAG_RD, 7965 &sc->uncorrected_error_dma_count, 7966 "Uncorrected descriptor-handler ECC errors"); 7967 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7968 "corrected_pcie_write_buffer", CTLFLAG_RD, 7969 &sc->corrected_error_pcie_tx_data_count, 7970 "Corrected PCIe write-buffer ECC errors"); 7971 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7972 "corrected_pcie_retry_buffer", CTLFLAG_RD, 7973 &sc->corrected_error_pcie_retry_count, 7974 "Corrected controller-shared PCIe retry-buffer errors"); 7975 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7976 "corrected_pcie_msix", CTLFLAG_RD, 7977 &sc->corrected_error_pcie_other_count, 7978 "Corrected controller-shared PCIe MSI-X errors"); 7979 } else { 7980 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7981 "fatal_lan", CTLFLAG_RD, 7982 &sc->fatal_error_lan_count, 7983 "Fatal LAN-port memory error indications"); 7984 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7985 "fatal_management", CTLFLAG_RD, 7986 &sc->fatal_error_mng_count, 7987 "Fatal management-memory error indications"); 7988 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7989 "fatal_pcie", CTLFLAG_RD, 7990 &sc->fatal_error_pcie_count, 7991 "Fatal PCIe memory error indications"); 7992 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7993 "fatal_dma", CTLFLAG_RD, 7994 &sc->fatal_error_dma_count, 7995 "Fatal DMA memory error indications"); 7996 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7997 "fatal_unknown", CTLFLAG_RD, 7998 &sc->fatal_error_unknown_count, 7999 "Fatal memory errors without a reported region"); 8000 if (em_has_82580_memory_errors(&sc->hw)) { 8001 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8002 "corrected_packet_buffer", CTLFLAG_RD, 8003 &sc->corrected_error_packet_buffer_count, 8004 "Corrected packet-buffer ECC errors"); 8005 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8006 "uncorrected_pcie", CTLFLAG_RD, 8007 &sc->uncorrected_error_pcie_count, 8008 "Uncorrected PCIe command-memory ECC indications"); 8009 } else if (em_has_i210_memory_errors(&sc->hw)) { 8010 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8011 "corrected_dma", CTLFLAG_RD, 8012 &sc->corrected_error_dma_count, 8013 "Corrected DMA memory error indications"); 8014 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8015 "corrected_pcie_tx_data", CTLFLAG_RD, 8016 &sc->corrected_error_pcie_tx_data_count, 8017 "Corrected PCIe transmit-data memory indications"); 8018 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8019 "corrected_pcie_retry", CTLFLAG_RD, 8020 &sc->corrected_error_pcie_retry_count, 8021 "Corrected PCIe retry-buffer memory indications"); 8022 } else { 8023 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8024 "corrected_dma", CTLFLAG_RD, 8025 &sc->corrected_error_dma_count, 8026 "Corrected DMA memory indications"); 8027 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8028 "corrected_packet_buffer", CTLFLAG_RD, 8029 &sc->corrected_error_packet_buffer_count, 8030 "Corrected packet-buffer memory indications"); 8031 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8032 "corrected_lan_mng_fifo", CTLFLAG_RD, 8033 &sc->corrected_error_lan_mng_fifo_count, 8034 "Corrected LAN management transmit-FIFO ECC " 8035 "indications"); 8036 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8037 "corrected_pcie_tx_data", CTLFLAG_RD, 8038 &sc->corrected_error_pcie_tx_data_count, 8039 "Corrected PCIe transmit-data memory indications"); 8040 if (sc->hw.mac.type == e1000_i350) 8041 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8042 "corrected_pcie_retry", CTLFLAG_RD, 8043 &sc->corrected_error_pcie_retry_count, 8044 "Corrected PCIe retry-buffer memory " 8045 "indications"); 8046 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 8047 "corrected_pcie_other", CTLFLAG_RD, 8048 &sc->corrected_error_pcie_other_count, 8049 "Other corrected PCIe memory indications"); 8050 } 8051 } 8052 } 8053 8054 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 8055 CTLFLAG_RD, &stats->ecol, 8056 "Excessive collisions"); 8057 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 8058 CTLFLAG_RD, &stats->scc, 8059 "Single collisions"); 8060 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 8061 CTLFLAG_RD, &stats->mcc, 8062 "Multiple collisions"); 8063 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 8064 CTLFLAG_RD, &stats->latecol, 8065 "Late collisions"); 8066 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 8067 CTLFLAG_RD, &stats->colc, 8068 "Collision Count"); 8069 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 8070 CTLFLAG_RD, &stats->symerrs, 8071 "Symbol Errors"); 8072 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 8073 CTLFLAG_RD, &stats->sec, 8074 "Sequence Errors"); 8075 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 8076 CTLFLAG_RD, &stats->dc, 8077 "Defer Count"); 8078 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 8079 CTLFLAG_RD, &stats->mpc, 8080 "Missed Packets"); 8081 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 8082 CTLFLAG_RD, &stats->rlec, 8083 "Receive Length Errors"); 8084 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 8085 CTLFLAG_RD, &stats->rnbc, 8086 "Receive No Buffers"); 8087 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 8088 CTLFLAG_RD, &stats->ruc, 8089 "Receive Undersize"); 8090 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 8091 CTLFLAG_RD, &stats->rfc, 8092 "Fragmented Packets Received "); 8093 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 8094 CTLFLAG_RD, &stats->roc, 8095 "Oversized Packets Received"); 8096 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 8097 CTLFLAG_RD, &stats->rjc, 8098 "Recevied Jabber"); 8099 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 8100 CTLFLAG_RD, &stats->rxerrc, 8101 "Receive Errors"); 8102 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 8103 CTLFLAG_RD, &stats->crcerrs, 8104 "CRC errors"); 8105 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 8106 CTLFLAG_RD, &stats->algnerrc, 8107 "Alignment Errors"); 8108 /* On 82575 these are collision counts */ 8109 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 8110 CTLFLAG_RD, &stats->cexterr, 8111 "Collision/Carrier extension errors"); 8112 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 8113 CTLFLAG_RD, &stats->xonrxc, 8114 "XON Received"); 8115 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 8116 CTLFLAG_RD, &stats->xontxc, 8117 "XON Transmitted"); 8118 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 8119 CTLFLAG_RD, &stats->xoffrxc, 8120 "XOFF Received"); 8121 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 8122 CTLFLAG_RD, &stats->xofftxc, 8123 "XOFF Transmitted"); 8124 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 8125 CTLFLAG_RD, &stats->fcruc, 8126 "Unsupported Flow Control Received"); 8127 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 8128 CTLFLAG_RD, &stats->mgprc, 8129 "Management Packets Received"); 8130 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 8131 CTLFLAG_RD, &stats->mgpdc, 8132 "Management Packets Dropped"); 8133 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 8134 CTLFLAG_RD, &stats->mgptc, 8135 "Management Packets Transmitted"); 8136 8137 /* Packet Reception Stats */ 8138 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 8139 CTLFLAG_RD, &stats->tpr, 8140 "Total Packets Received "); 8141 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 8142 CTLFLAG_RD, &stats->gprc, 8143 "Good Packets Received"); 8144 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 8145 CTLFLAG_RD, &stats->bprc, 8146 "Broadcast Packets Received"); 8147 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 8148 CTLFLAG_RD, &stats->mprc, 8149 "Multicast Packets Received"); 8150 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 8151 CTLFLAG_RD, &stats->prc64, 8152 "64 byte frames received "); 8153 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 8154 CTLFLAG_RD, &stats->prc127, 8155 "65-127 byte frames received"); 8156 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 8157 CTLFLAG_RD, &stats->prc255, 8158 "128-255 byte frames received"); 8159 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 8160 CTLFLAG_RD, &stats->prc511, 8161 "256-511 byte frames received"); 8162 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 8163 CTLFLAG_RD, &stats->prc1023, 8164 "512-1023 byte frames received"); 8165 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 8166 CTLFLAG_RD, &stats->prc1522, 8167 "1023-1522 byte frames received"); 8168 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 8169 CTLFLAG_RD, &stats->gorc, 8170 "Good Octets Received"); 8171 8172 /* Packet Transmission Stats */ 8173 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 8174 CTLFLAG_RD, &stats->gotc, 8175 "Good Octets Transmitted"); 8176 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 8177 CTLFLAG_RD, &stats->tpt, 8178 "Total Packets Transmitted"); 8179 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 8180 CTLFLAG_RD, &stats->gptc, 8181 "Good Packets Transmitted"); 8182 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 8183 CTLFLAG_RD, &stats->bptc, 8184 "Broadcast Packets Transmitted"); 8185 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 8186 CTLFLAG_RD, &stats->mptc, 8187 "Multicast Packets Transmitted"); 8188 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 8189 CTLFLAG_RD, &stats->ptc64, 8190 "64 byte frames transmitted "); 8191 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 8192 CTLFLAG_RD, &stats->ptc127, 8193 "65-127 byte frames transmitted"); 8194 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 8195 CTLFLAG_RD, &stats->ptc255, 8196 "128-255 byte frames transmitted"); 8197 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 8198 CTLFLAG_RD, &stats->ptc511, 8199 "256-511 byte frames transmitted"); 8200 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 8201 CTLFLAG_RD, &stats->ptc1023, 8202 "512-1023 byte frames transmitted"); 8203 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 8204 CTLFLAG_RD, &stats->ptc1522, 8205 "1024-1522 byte frames transmitted"); 8206 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 8207 CTLFLAG_RD, &stats->tsctc, 8208 "TSO Contexts Transmitted"); 8209 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 8210 CTLFLAG_RD, &stats->tsctfc, 8211 "TSO Contexts Failed"); 8212 8213 /* Interrupt Stats */ 8214 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 8215 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 8216 int_list = SYSCTL_CHILDREN(int_node); 8217 8218 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 8219 CTLFLAG_RD, &stats->iac, 8220 "Interrupt Assertion Count"); 8221 8222 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 8223 CTLFLAG_RD, &stats->icrxptc, 8224 "Interrupt Cause Rx Pkt Timer Expire Count"); 8225 8226 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 8227 CTLFLAG_RD, &stats->icrxatc, 8228 "Interrupt Cause Rx Abs Timer Expire Count"); 8229 8230 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 8231 CTLFLAG_RD, &stats->ictxptc, 8232 "Interrupt Cause Tx Pkt Timer Expire Count"); 8233 8234 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 8235 CTLFLAG_RD, &stats->ictxatc, 8236 "Interrupt Cause Tx Abs Timer Expire Count"); 8237 8238 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 8239 CTLFLAG_RD, &stats->ictxqec, 8240 "Interrupt Cause Tx Queue Empty Count"); 8241 8242 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 8243 CTLFLAG_RD, &stats->ictxqmtc, 8244 "Interrupt Cause Tx Queue Min Thresh Count"); 8245 8246 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 8247 CTLFLAG_RD, &stats->icrxdmtc, 8248 "Interrupt Cause Rx Desc Min Thresh Count"); 8249 8250 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun", 8251 CTLFLAG_RD, &stats->icrxoc, 8252 "Interrupt Cause Receiver Overrun Count"); 8253 } 8254 8255 static void 8256 em_fw_version_locked(if_ctx_t ctx) 8257 { 8258 struct e1000_softc *sc = iflib_get_softc(ctx); 8259 struct e1000_hw *hw = &sc->hw; 8260 struct e1000_fw_version *fw_ver = &sc->fw_ver; 8261 uint16_t eep = 0; 8262 8263 /* 8264 * em_fw_version_locked() must run under the IFLIB_CTX_LOCK to meet 8265 * the NVM locking model, so we do it in em_if_attach_pre() and store 8266 * the info in the softc 8267 */ 8268 ASSERT_CTX_LOCK_HELD(hw); 8269 8270 *fw_ver = (struct e1000_fw_version){0}; 8271 8272 if (hw->mac.type >= igb_mac_min) { 8273 /* 8274 * Use the Shared Code for igb(4) 8275 */ 8276 e1000_get_fw_version(hw, fw_ver); 8277 } else { 8278 /* 8279 * Otherwise, EEPROM version should be present on (almost?) 8280 * all devices here 8281 */ 8282 if(e1000_read_nvm(hw, NVM_VERSION, 1, &eep)) { 8283 INIT_DEBUGOUT("can't get EEPROM version"); 8284 return; 8285 } 8286 8287 fw_ver->eep_major = (eep & NVM_MAJOR_MASK) >> NVM_MAJOR_SHIFT; 8288 fw_ver->eep_minor = (eep & NVM_MINOR_MASK) >> NVM_MINOR_SHIFT; 8289 fw_ver->eep_build = (eep & NVM_IMAGE_ID_MASK); 8290 } 8291 } 8292 8293 static void 8294 em_sbuf_fw_version(struct e1000_fw_version *fw_ver, struct sbuf *buf) 8295 { 8296 const char *space = ""; 8297 8298 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 8299 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 8300 fw_ver->eep_minor, fw_ver->eep_build); 8301 space = " "; 8302 } 8303 8304 if (fw_ver->invm_major || fw_ver->invm_minor || 8305 fw_ver->invm_img_type) { 8306 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 8307 space, fw_ver->invm_major, fw_ver->invm_minor, 8308 fw_ver->invm_img_type); 8309 space = " "; 8310 } 8311 8312 if (fw_ver->or_valid) { 8313 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 8314 space, fw_ver->or_major, fw_ver->or_build, 8315 fw_ver->or_patch); 8316 space = " "; 8317 } 8318 8319 if (fw_ver->etrack_id) 8320 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 8321 } 8322 8323 static void 8324 em_print_fw_version(struct e1000_softc *sc ) 8325 { 8326 device_t dev = sc->dev; 8327 struct sbuf *buf; 8328 int error = 0; 8329 8330 buf = sbuf_new_auto(); 8331 if (!buf) { 8332 device_printf(dev, "Could not allocate sbuf for output.\n"); 8333 return; 8334 } 8335 8336 em_sbuf_fw_version(&sc->fw_ver, buf); 8337 8338 error = sbuf_finish(buf); 8339 if (error) 8340 device_printf(dev, "Error finishing sbuf: %d\n", error); 8341 else if (sbuf_len(buf)) 8342 device_printf(dev, "%s\n", sbuf_data(buf)); 8343 8344 sbuf_delete(buf); 8345 } 8346 8347 static int 8348 em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 8349 { 8350 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8351 device_t dev = sc->dev; 8352 struct sbuf *buf; 8353 int error = 0; 8354 8355 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8356 if (!buf) { 8357 device_printf(dev, "Could not allocate sbuf for output.\n"); 8358 return (ENOMEM); 8359 } 8360 8361 em_sbuf_fw_version(&sc->fw_ver, buf); 8362 8363 error = sbuf_finish(buf); 8364 if (error) 8365 device_printf(dev, "Error finishing sbuf: %d\n", error); 8366 8367 sbuf_delete(buf); 8368 8369 return (0); 8370 } 8371 8372 /********************************************************************** 8373 * 8374 * This routine provides a way to dump out the adapter eeprom, 8375 * often a useful debug/service tool. This only dumps the first 8376 * 32 words, stuff that matters is in that extent. 8377 * 8378 **********************************************************************/ 8379 static int 8380 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 8381 { 8382 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8383 int error; 8384 int result; 8385 8386 result = -1; 8387 error = sysctl_handle_int(oidp, &result, 0, req); 8388 8389 if (error || !req->newptr) 8390 return (error); 8391 8392 /* 8393 * This value will cause a hex dump of the 8394 * first 32 16-bit words of the EEPROM to 8395 * the screen. 8396 */ 8397 if (result == 1) 8398 em_print_nvm_info(sc); 8399 8400 return (error); 8401 } 8402 8403 static void 8404 em_print_nvm_info(struct e1000_softc *sc) 8405 { 8406 struct e1000_hw *hw = &sc->hw; 8407 struct sx *iflib_ctx_lock = iflib_ctx_lock_get(sc->ctx); 8408 u16 eeprom_data; 8409 int i, j, row = 0; 8410 8411 /* Its a bit crude, but it gets the job done */ 8412 printf("\nInterface EEPROM Dump:\n"); 8413 printf("Offset\n0x0000 "); 8414 8415 /* We rely on the IFLIB_CTX_LOCK as part of NVM locking model */ 8416 sx_xlock(iflib_ctx_lock); 8417 ASSERT_CTX_LOCK_HELD(hw); 8418 for (i = 0, j = 0; i < 32; i++, j++) { 8419 if (j == 8) { /* Make the offset block */ 8420 j = 0; ++row; 8421 printf("\n0x00%x0 ",row); 8422 } 8423 eeprom_data = 0; 8424 if (e1000_read_nvm(hw, i, 1, &eeprom_data) != 8425 E1000_SUCCESS) { 8426 printf("\nNVM read failed at offset %#x\n", i); 8427 break; 8428 } 8429 printf("%04x ", eeprom_data); 8430 } 8431 sx_xunlock(iflib_ctx_lock); 8432 printf("\n"); 8433 } 8434 8435 static int 8436 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 8437 { 8438 struct em_int_delay_info *info; 8439 struct e1000_softc *sc; 8440 u32 regval; 8441 int error, usecs, ticks; 8442 8443 info = (struct em_int_delay_info *) arg1; 8444 usecs = info->value; 8445 error = sysctl_handle_int(oidp, &usecs, 0, req); 8446 if (error != 0 || req->newptr == NULL) 8447 return (error); 8448 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 8449 return (EINVAL); 8450 info->value = usecs; 8451 ticks = EM_USECS_TO_TICKS(usecs); 8452 8453 sc = info->sc; 8454 8455 regval = E1000_READ_OFFSET(&sc->hw, info->offset); 8456 regval = (regval & ~0xffff) | (ticks & 0xffff); 8457 /* Handle a few special cases. */ 8458 switch (info->offset) { 8459 case E1000_RDTR: 8460 break; 8461 case E1000_TIDV: 8462 if (ticks == 0) { 8463 sc->txd_cmd &= ~E1000_TXD_CMD_IDE; 8464 /* Don't write 0 into the TIDV register. */ 8465 regval++; 8466 } else 8467 sc->txd_cmd |= E1000_TXD_CMD_IDE; 8468 break; 8469 } 8470 E1000_WRITE_OFFSET(&sc->hw, info->offset, regval); 8471 return (0); 8472 } 8473 8474 static int 8475 em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 8476 { 8477 struct e1000_softc *sc; 8478 u32 reg, val, shift; 8479 int error, mask; 8480 8481 sc = oidp->oid_arg1; 8482 switch (oidp->oid_arg2) { 8483 case 0: 8484 reg = E1000_DTXTCPFLGL; 8485 shift = 0; 8486 break; 8487 case 1: 8488 reg = E1000_DTXTCPFLGL; 8489 shift = 16; 8490 break; 8491 case 2: 8492 reg = E1000_DTXTCPFLGH; 8493 shift = 0; 8494 break; 8495 default: 8496 return (EINVAL); 8497 break; 8498 } 8499 val = E1000_READ_REG(&sc->hw, reg); 8500 mask = (val >> shift) & 0xfff; 8501 error = sysctl_handle_int(oidp, &mask, 0, req); 8502 if (error != 0 || req->newptr == NULL) 8503 return (error); 8504 if (mask < 0 || mask > 0xfff) 8505 return (EINVAL); 8506 val = (val & ~(0xfff << shift)) | (mask << shift); 8507 E1000_WRITE_REG(&sc->hw, reg, val); 8508 return (0); 8509 } 8510 8511 static void 8512 em_add_int_delay_sysctl(struct e1000_softc *sc, const char *name, 8513 const char *description, struct em_int_delay_info *info, int offset, 8514 int value) 8515 { 8516 info->sc = sc; 8517 info->offset = offset; 8518 info->value = value; 8519 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 8520 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 8521 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 8522 info, 0, em_sysctl_int_delay, "I", description); 8523 } 8524 8525 /* 8526 * Set flow control using sysctl: 8527 * Flow control values: 8528 * 0 - off 8529 * 1 - rx pause 8530 * 2 - tx pause 8531 * 3 - full 8532 */ 8533 static int 8534 em_set_flowcntl(SYSCTL_HANDLER_ARGS) 8535 { 8536 int error; 8537 static int input = 3; /* default is full */ 8538 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8539 8540 error = sysctl_handle_int(oidp, &input, 0, req); 8541 8542 if ((error) || (req->newptr == NULL)) 8543 return (error); 8544 8545 if (input == sc->fc) /* no change? */ 8546 return (error); 8547 8548 switch (input) { 8549 case e1000_fc_rx_pause: 8550 case e1000_fc_tx_pause: 8551 case e1000_fc_full: 8552 case e1000_fc_none: 8553 sc->hw.fc.requested_mode = input; 8554 sc->fc = input; 8555 break; 8556 default: 8557 /* Do nothing */ 8558 return (error); 8559 } 8560 8561 sc->hw.fc.current_mode = sc->hw.fc.requested_mode; 8562 e1000_force_mac_fc(&sc->hw); 8563 return (error); 8564 } 8565 8566 static void 8567 em_sysctl_request_reinit(struct e1000_softc *sc) 8568 { 8569 if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) 8570 return; 8571 8572 iflib_request_reset(sc->ctx); 8573 iflib_admin_intr_deferred(sc->ctx); 8574 } 8575 8576 /* 8577 * Manage DMA Coalesce: 8578 * Control values: 8579 * 0/1 - off/on 8580 * Legal timer values are: 8581 * 250,500,1000-10000 in thousands 8582 */ 8583 static int 8584 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) 8585 { 8586 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8587 int error; 8588 8589 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 8590 8591 if ((error) || (req->newptr == NULL)) 8592 return (error); 8593 8594 switch (sc->dmac) { 8595 case 0: 8596 /* Disabling */ 8597 break; 8598 case 1: /* Just enable and use default */ 8599 sc->dmac = 1000; 8600 break; 8601 case 250: 8602 case 500: 8603 case 1000: 8604 case 2000: 8605 case 3000: 8606 case 4000: 8607 case 5000: 8608 case 6000: 8609 case 7000: 8610 case 8000: 8611 case 9000: 8612 case 10000: 8613 /* Legal values - allow */ 8614 break; 8615 default: 8616 /* Do nothing, illegal value */ 8617 sc->dmac = 0; 8618 return (EINVAL); 8619 } 8620 /* Reinit the interface */ 8621 em_sysctl_request_reinit(sc); 8622 return (error); 8623 } 8624 8625 /* 8626 * Manage Energy Efficient Ethernet: 8627 * Control values: 8628 * 0/1 - enabled/disabled 8629 */ 8630 static int 8631 em_sysctl_eee(SYSCTL_HANDLER_ARGS) 8632 { 8633 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8634 int error, value; 8635 8636 if (sc->hw.mac.type < igb_mac_min) 8637 value = sc->hw.dev_spec.ich8lan.eee_disable; 8638 else 8639 value = sc->hw.dev_spec._82575.eee_disable; 8640 error = sysctl_handle_int(oidp, &value, 0, req); 8641 if (error || req->newptr == NULL) 8642 return (error); 8643 if (sc->hw.mac.type < igb_mac_min) 8644 sc->hw.dev_spec.ich8lan.eee_disable = (value != 0); 8645 else 8646 sc->hw.dev_spec._82575.eee_disable = (value != 0); 8647 em_sysctl_request_reinit(sc); 8648 8649 return (0); 8650 } 8651 8652 static int 8653 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 8654 { 8655 struct e1000_softc *sc; 8656 int error; 8657 int result; 8658 8659 result = -1; 8660 error = sysctl_handle_int(oidp, &result, 0, req); 8661 8662 if (error || !req->newptr) 8663 return (error); 8664 8665 if (result == 1) { 8666 sc = (struct e1000_softc *) arg1; 8667 em_print_debug_info(sc); 8668 } 8669 8670 return (error); 8671 } 8672 8673 static int 8674 em_get_rs(SYSCTL_HANDLER_ARGS) 8675 { 8676 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8677 int error; 8678 int result; 8679 8680 result = 0; 8681 error = sysctl_handle_int(oidp, &result, 0, req); 8682 8683 if (error || !req->newptr || result != 1) 8684 return (error); 8685 em_dump_rs(sc); 8686 8687 return (error); 8688 } 8689 8690 static void 8691 em_if_debug(if_ctx_t ctx) 8692 { 8693 em_dump_rs(iflib_get_softc(ctx)); 8694 } 8695 8696 /* 8697 * This routine is meant to be fluid, add whatever is 8698 * needed for debugging a problem. -jfv 8699 */ 8700 static void 8701 em_print_debug_info(struct e1000_softc *sc) 8702 { 8703 device_t dev = iflib_get_dev(sc->ctx); 8704 struct tx_ring *txr; 8705 struct rx_ring *rxr; 8706 8707 if (sc->tx_queues == NULL || sc->rx_queues == NULL) { 8708 device_printf(dev, "queue state is unavailable\n"); 8709 return; 8710 } 8711 device_printf(dev, "iflib software admission: %s\n", 8712 iflib_is_running(sc->ctx) ? "open" : "closed"); 8713 8714 for (int i = 0; i < sc->tx_num_queues; i++) { 8715 txr = &sc->tx_queues[i].txr; 8716 device_printf(dev, "TX Queue %d ------\n", i); 8717 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 8718 E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)), 8719 E1000_READ_REG(&sc->hw, E1000_TDT(txr->me))); 8720 8721 } 8722 for (int j = 0; j < sc->rx_num_queues; j++) { 8723 rxr = &sc->rx_queues[j].rxr; 8724 device_printf(dev, "RX Queue %d ------\n", j); 8725 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 8726 E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)), 8727 E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me))); 8728 } 8729 } 8730 8731 /* 8732 * 82574 only: 8733 * Write a new value to the EEPROM increasing the number of MSI-X 8734 * vectors from 3 to 5, for proper multiqueue support. 8735 */ 8736 static void 8737 em_enable_vectors_82574(if_ctx_t ctx) 8738 { 8739 struct e1000_softc *sc = iflib_get_softc(ctx); 8740 struct e1000_hw *hw = &sc->hw; 8741 device_t dev = iflib_get_dev(ctx); 8742 u16 edata; 8743 8744 e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8745 if (bootverbose) 8746 device_printf(dev, "EM_NVM_PCIE_CTRL = %#06x\n", edata); 8747 if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { 8748 device_printf(dev, "Writing to eeprom: increasing " 8749 "reported MSI-X vectors from 3 to 5...\n"); 8750 edata &= ~(EM_NVM_MSIX_N_MASK); 8751 edata |= 4 << EM_NVM_MSIX_N_SHIFT; 8752 e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8753 e1000_update_nvm_checksum(hw); 8754 device_printf(dev, "Writing to eeprom: done\n"); 8755 } 8756 } 8757