1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2024 Kevin Bowling <kbowling@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 #include "if_igb_iov.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #define em_mac_min e1000_82571 36 #define igb_mac_min e1000_82575 37 38 /********************************************************************* 39 * Driver version: 40 *********************************************************************/ 41 static const char em_driver_version[] = "7.7.8-fbsd"; 42 static const char igb_driver_version[] = "2.5.28-fbsd"; 43 44 /********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into e1000_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54 static const pci_vendor_info_t em_vendor_info_array[] = 55 { 56 /* Intel(R) - lem-class legacy devices */ 57 PVID(0x8086, E1000_DEV_ID_82540EM, 58 "Intel(R) Legacy PRO/1000 MT 82540EM"), 59 PVID(0x8086, E1000_DEV_ID_82540EM_LOM, 60 "Intel(R) Legacy PRO/1000 MT 82540EM (LOM)"), 61 PVID(0x8086, E1000_DEV_ID_82540EP, 62 "Intel(R) Legacy PRO/1000 MT 82540EP"), 63 PVID(0x8086, E1000_DEV_ID_82540EP_LOM, 64 "Intel(R) Legacy PRO/1000 MT 82540EP (LOM)"), 65 PVID(0x8086, E1000_DEV_ID_82540EP_LP, 66 "Intel(R) Legacy PRO/1000 MT 82540EP (Mobile)"), 67 68 PVID(0x8086, E1000_DEV_ID_82541EI, 69 "Intel(R) Legacy PRO/1000 MT 82541EI (Copper)"), 70 PVID(0x8086, E1000_DEV_ID_82541ER, 71 "Intel(R) Legacy PRO/1000 82541ER"), 72 PVID(0x8086, E1000_DEV_ID_82541ER_LOM, 73 "Intel(R) Legacy PRO/1000 MT 82541ER"), 74 PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, 75 "Intel(R) Legacy PRO/1000 MT 82541EI (Mobile)"), 76 PVID(0x8086, E1000_DEV_ID_82541GI, 77 "Intel(R) Legacy PRO/1000 MT 82541GI"), 78 PVID(0x8086, E1000_DEV_ID_82541GI_LF, 79 "Intel(R) Legacy PRO/1000 GT 82541PI"), 80 PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, 81 "Intel(R) Legacy PRO/1000 MT 82541GI (Mobile)"), 82 83 PVID(0x8086, E1000_DEV_ID_82542, 84 "Intel(R) Legacy PRO/1000 82542 (Fiber)"), 85 86 PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, 87 "Intel(R) Legacy PRO/1000 F 82543GC (Fiber)"), 88 PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, 89 "Intel(R) Legacy PRO/1000 T 82543GC (Copper)"), 90 91 PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, 92 "Intel(R) Legacy PRO/1000 XT 82544EI (Copper)"), 93 PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, 94 "Intel(R) Legacy PRO/1000 XF 82544EI (Fiber)"), 95 PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, 96 "Intel(R) Legacy PRO/1000 T 82544GC (Copper)"), 97 PVID(0x8086, E1000_DEV_ID_82544GC_LOM, 98 "Intel(R) Legacy PRO/1000 XT 82544GC (LOM)"), 99 100 PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, 101 "Intel(R) Legacy PRO/1000 MT 82545EM (Copper)"), 102 PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, 103 "Intel(R) Legacy PRO/1000 MF 82545EM (Fiber)"), 104 PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, 105 "Intel(R) Legacy PRO/1000 MT 82545GM (Copper)"), 106 PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, 107 "Intel(R) Legacy PRO/1000 MF 82545GM (Fiber)"), 108 PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, 109 "Intel(R) Legacy PRO/1000 MB 82545GM (SERDES)"), 110 111 PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, 112 "Intel(R) Legacy PRO/1000 MT 82546EB (Copper)"), 113 PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, 114 "Intel(R) Legacy PRO/1000 MF 82546EB (Fiber)"), 115 PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, 116 "Intel(R) Legacy PRO/1000 MT 82546EB (Quad Copper"), 117 PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, 118 "Intel(R) Legacy PRO/1000 MT 82546GB (Copper)"), 119 PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, 120 "Intel(R) Legacy PRO/1000 MF 82546GB (Fiber)"), 121 PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, 122 "Intel(R) Legacy PRO/1000 MB 82546GB (SERDES)"), 123 PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, 124 "Intel(R) Legacy PRO/1000 P 82546GB (PCIe)"), 125 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, 126 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 127 PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 128 "Intel(R) Legacy PRO/1000 GT 82546GB (Quad Copper)"), 129 130 PVID(0x8086, E1000_DEV_ID_82547EI, 131 "Intel(R) Legacy PRO/1000 CT 82547EI"), 132 PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, 133 "Intel(R) Legacy PRO/1000 CT 82547EI (Mobile)"), 134 PVID(0x8086, E1000_DEV_ID_82547GI, 135 "Intel(R) Legacy PRO/1000 CT 82547GI"), 136 137 /* Intel(R) - em-class devices */ 138 PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, 139 "Intel(R) PRO/1000 PT 82571EB/82571GB (Copper)"), 140 PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, 141 "Intel(R) PRO/1000 PF 82571EB/82571GB (Fiber)"), 142 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, 143 "Intel(R) PRO/1000 PB 82571EB (SERDES)"), 144 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 145 "Intel(R) PRO/1000 82571EB (Dual Mezzanine)"), 146 PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 147 "Intel(R) PRO/1000 82571EB (Quad Mezzanine)"), 148 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 149 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 150 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 151 "Intel(R) PRO/1000 PT 82571EB/82571GB (Quad Copper)"), 152 PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 153 "Intel(R) PRO/1000 PF 82571EB (Quad Fiber)"), 154 PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 155 "Intel(R) PRO/1000 PT 82571PT (Quad Copper)"), 156 PVID(0x8086, E1000_DEV_ID_82572EI, 157 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 158 PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, 159 "Intel(R) PRO/1000 PT 82572EI (Copper)"), 160 PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, 161 "Intel(R) PRO/1000 PF 82572EI (Fiber)"), 162 PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, 163 "Intel(R) PRO/1000 82572EI (SERDES)"), 164 PVID(0x8086, E1000_DEV_ID_82573E, 165 "Intel(R) PRO/1000 82573E (Copper)"), 166 PVID(0x8086, E1000_DEV_ID_82573E_IAMT, 167 "Intel(R) PRO/1000 82573E AMT (Copper)"), 168 PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 82573L"), 169 PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) 82583V"), 170 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 171 "Intel(R) 80003ES2LAN (Copper)"), 172 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 173 "Intel(R) 80003ES2LAN (SERDES)"), 174 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 175 "Intel(R) 80003ES2LAN (Dual Copper)"), 176 PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 177 "Intel(R) 80003ES2LAN (Dual SERDES)"), 178 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, 179 "Intel(R) 82566MM ICH8 AMT (Mobile)"), 180 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) 82566DM ICH8 AMT"), 181 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) 82566DC ICH8"), 182 PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) 82562V ICH8"), 183 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) 82562GT ICH8"), 184 PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) 82562G ICH8"), 185 PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) 82566MC ICH8"), 186 PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) 82567V-3 ICH8"), 187 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, 188 "Intel(R) 82567LM ICH9 AMT"), 189 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, 190 "Intel(R) 82566DM-2 ICH9 AMT"), 191 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) 82566DC-2 ICH9"), 192 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) 82567LF ICH9"), 193 PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) 82567V ICH9"), 194 PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) 82562V-2 ICH9"), 195 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) 82562GT-2 ICH9"), 196 PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) 82562G-2 ICH9"), 197 PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) 82567LM-4 ICH9"), 198 PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) Gigabit CT 82574L"), 199 PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) 82574L-Apple"), 200 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) 82567LM-2 ICH10"), 201 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) 82567LF-2 ICH10"), 202 PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) 82567V-2 ICH10"), 203 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) 82567LM-3 ICH10"), 204 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) 82567LF-3 ICH10"), 205 PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) 82567V-4 ICH10"), 206 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) 82577LM"), 207 PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) 82577LC"), 208 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) 82578DM"), 209 PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) 82578DC"), 210 PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) 82579LM"), 211 PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) 82579V"), 212 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) I217-LM LPT"), 213 PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) I217-V LPT"), 214 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, 215 "Intel(R) I218-LM LPTLP"), 216 PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) I218-V LPTLP"), 217 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) I218-LM (2)"), 218 PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) I218-V (2)"), 219 PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) I218-LM (3)"), 220 PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) I218-V (3)"), 221 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) I219-LM SPT"), 222 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) I219-V SPT"), 223 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, 224 "Intel(R) I219-LM SPT-H(2)"), 225 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, 226 "Intel(R) I219-V SPT-H(2)"), 227 PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, 228 "Intel(R) I219-LM LBG(3)"), 229 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, 230 "Intel(R) I219-LM SPT(4)"), 231 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) I219-V SPT(4)"), 232 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, 233 "Intel(R) I219-LM SPT(5)"), 234 PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) I219-V SPT(5)"), 235 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM6, 236 "Intel(R) I219-LM CNP(6)"), 237 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V6, "Intel(R) I219-V CNP(6)"), 238 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM7, 239 "Intel(R) I219-LM CNP(7)"), 240 PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V7, "Intel(R) I219-V CNP(7)"), 241 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM8, 242 "Intel(R) I219-LM ICP(8)"), 243 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V8, "Intel(R) I219-V ICP(8)"), 244 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM9, 245 "Intel(R) I219-LM ICP(9)"), 246 PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V9, "Intel(R) I219-V ICP(9)"), 247 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM10, 248 "Intel(R) I219-LM CMP(10)"), 249 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V10, 250 "Intel(R) I219-V CMP(10)"), 251 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM11, 252 "Intel(R) I219-LM CMP(11)"), 253 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V11, 254 "Intel(R) I219-V CMP(11)"), 255 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_LM12, 256 "Intel(R) I219-LM CMP(12)"), 257 PVID(0x8086, E1000_DEV_ID_PCH_CMP_I219_V12, 258 "Intel(R) I219-V CMP(12)"), 259 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM13, 260 "Intel(R) I219-LM TGP(13)"), 261 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V13, 262 "Intel(R) I219-V TGP(13)"), 263 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM14, 264 "Intel(R) I219-LM TGP(14)"), 265 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V14, 266 "Intel(R) I219-V GTP(14)"), 267 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_LM15, 268 "Intel(R) I219-LM TGP(15)"), 269 PVID(0x8086, E1000_DEV_ID_PCH_TGP_I219_V15, 270 "Intel(R) I219-V TGP(15)"), 271 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM16, 272 "Intel(R) I219-LM ADL(16)"), 273 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V16, 274 "Intel(R) I219-V ADL(16)"), 275 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM17, 276 "Intel(R) I219-LM ADL(17)"), 277 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V17, 278 "Intel(R) I219-V ADL(17)"), 279 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_LM18, 280 "Intel(R) I219-LM MTP(18)"), 281 PVID(0x8086, E1000_DEV_ID_PCH_MTP_I219_V18, 282 "Intel(R) I219-V MTP(18)"), 283 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_LM19, 284 "Intel(R) I219-LM ADL(19)"), 285 PVID(0x8086, E1000_DEV_ID_PCH_ADL_I219_V19, 286 "Intel(R) I219-V ADL(19)"), 287 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM20, 288 "Intel(R) I219-LM LNL(20)"), 289 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V20, 290 "Intel(R) I219-V LNL(20)"), 291 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_LM21, 292 "Intel(R) I219-LM LNL(21)"), 293 PVID(0x8086, E1000_DEV_ID_PCH_LNL_I219_V21, 294 "Intel(R) I219-V LNL(21)"), 295 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM22, 296 "Intel(R) I219-LM RPL(22)"), 297 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V22, 298 "Intel(R) I219-V RPL(22)"), 299 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_LM23, 300 "Intel(R) I219-LM RPL(23)"), 301 PVID(0x8086, E1000_DEV_ID_PCH_RPL_I219_V23, 302 "Intel(R) I219-V RPL(23)"), 303 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_LM24, 304 "Intel(R) I219-LM ARL(24)"), 305 PVID(0x8086, E1000_DEV_ID_PCH_ARL_I219_V24, 306 "Intel(R) I219-V ARL(24)"), 307 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM25, 308 "Intel(R) I219-LM PTP(25)"), 309 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V25, 310 "Intel(R) I219-V PTP(25)"), 311 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM26, 312 "Intel(R) I219-LM PTP(26)"), 313 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V26, 314 "Intel(R) I219-V PTP(26)"), 315 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_LM27, 316 "Intel(R) I219-LM PTP(27)"), 317 PVID(0x8086, E1000_DEV_ID_PCH_PTP_I219_V27, 318 "Intel(R) I219-V PTP(27)"), 319 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_LM29, 320 "Intel(R) I219-LM NVL(29)"), 321 PVID(0x8086, E1000_DEV_ID_PCH_NVL_I219_V29, 322 "Intel(R) I219-V NVL(29)"), 323 /* required last entry */ 324 PVID_END 325 }; 326 327 static const pci_vendor_info_t igb_vendor_info_array[] = 328 { 329 /* Intel(R) - igb-class devices */ 330 PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, 331 "Intel(R) PRO/1000 82575EB (Copper)"), 332 PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 333 "Intel(R) PRO/1000 82575EB (SERDES)"), 334 PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 335 "Intel(R) PRO/1000 VT 82575GB (Quad Copper)"), 336 PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 82576"), 337 PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 82576NS"), 338 PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, 339 "Intel(R) PRO/1000 82576NS (SERDES)"), 340 PVID(0x8086, E1000_DEV_ID_82576_FIBER, 341 "Intel(R) PRO/1000 EF 82576 (Dual Fiber)"), 342 PVID(0x8086, E1000_DEV_ID_82576_SERDES, 343 "Intel(R) PRO/1000 82576 (Dual SERDES)"), 344 PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 345 "Intel(R) PRO/1000 ET 82576 (Quad SERDES)"), 346 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 347 "Intel(R) PRO/1000 ET 82576 (Quad Copper)"), 348 PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 349 "Intel(R) PRO/1000 ET(2) 82576 (Quad Copper)"), 350 PVID(0x8086, E1000_DEV_ID_82580_COPPER, 351 "Intel(R) I340 82580 (Copper)"), 352 PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) I340 82580 (Fiber)"), 353 PVID(0x8086, E1000_DEV_ID_82580_SERDES, 354 "Intel(R) I340 82580 (SERDES)"), 355 PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) I340 82580 (SGMII)"), 356 PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 357 "Intel(R) I340-T2 82580 (Dual Copper)"), 358 PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, 359 "Intel(R) I340-F4 82580 (Quad Fiber)"), 360 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, 361 "Intel(R) DH89XXCC (SERDES)"), 362 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, 363 "Intel(R) I347-AT4 DH89XXCC"), 364 PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) DH89XXCC (SFP)"), 365 PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, 366 "Intel(R) DH89XXCC (Backplane)"), 367 PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) I350 (Copper)"), 368 PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) I350 (Fiber)"), 369 PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) I350 (SERDES)"), 370 PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) I350 (SGMII)"), 371 PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) I210 (Copper)"), 372 PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, 373 "Intel(R) I210 IT (Copper)"), 374 PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) I210 (OEM)"), 375 PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, 376 "Intel(R) I210 Flashless (Copper)"), 377 PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, 378 "Intel(R) I210 Flashless (SERDES)"), 379 PVID(0x8086, E1000_DEV_ID_I210_SGMII_FLASHLESS, 380 "Intel(R) I210 Flashless (SGMII)"), 381 PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) I210 (Fiber)"), 382 PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) I210 (SERDES)"), 383 PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) I210 (SGMII)"), 384 PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) I211 (Copper)"), 385 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 386 "Intel(R) I354 (1.0 GbE Backplane)"), 387 PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 388 "Intel(R) I354 (2.5 GbE Backplane)"), 389 PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) I354 (SGMII)"), 390 /* required last entry */ 391 PVID_END 392 }; 393 394 static const pci_vendor_info_t igbv_vendor_info_array[] = { 395 PVID(0x8086, E1000_DEV_ID_82576_VF, 396 "Intel(R) PRO/1000 82576 Virtual Function"), 397 PVID(0x8086, E1000_DEV_ID_82576_VF_HV, 398 "Intel(R) PRO/1000 82576 Virtual Function"), 399 PVID(0x8086, E1000_DEV_ID_I350_VF, 400 "Intel(R) I350 Virtual Function"), 401 PVID(0x8086, E1000_DEV_ID_I350_VF_HV, 402 "Intel(R) I350 Virtual Function"), 403 PVID_END 404 }; 405 406 /********************************************************************* 407 * Function prototypes 408 *********************************************************************/ 409 static void *em_register(device_t); 410 static void *igb_register(device_t); 411 static void *igbv_register(device_t); 412 static int igb_device_attach(device_t); 413 #ifdef PCI_IOV 414 static int igb_device_iov_init(device_t, uint16_t, const nvlist_t *); 415 static void igb_device_iov_uninit(device_t); 416 #endif 417 static int em_if_detach(if_ctx_t); 418 static int em_if_shutdown(if_ctx_t); 419 static int em_if_suspend(if_ctx_t); 420 static int em_if_resume(if_ctx_t); 421 422 static int em_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 423 int); 424 static int em_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 425 int); 426 static void em_if_queues_free(if_ctx_t); 427 428 static uint64_t em_if_get_vf_counter(if_ctx_t, ift_counter); 429 static uint64_t em_if_get_counter(if_ctx_t, ift_counter); 430 static void em_if_init(if_ctx_t); 431 static void em_if_stop(if_ctx_t); 432 static void em_fence_pci_busmaster(struct e1000_softc *); 433 static int em_enable_pci_busmaster(struct e1000_softc *); 434 static void em_if_media_status(if_ctx_t, struct ifmediareq *); 435 static int em_if_media_change(if_ctx_t); 436 static int em_if_mtu_set(if_ctx_t, uint32_t); 437 static void em_if_timer(if_ctx_t, uint16_t); 438 static void em_if_vlan_register(if_ctx_t, u16); 439 static void em_if_vlan_unregister(if_ctx_t, u16); 440 static bool em_if_needs_restart(if_ctx_t, enum iflib_restart_event); 441 442 static void em_identify_hardware(if_ctx_t); 443 static int em_allocate_pci_resources(if_ctx_t); 444 static void em_free_pci_resources(if_ctx_t); 445 static int em_reset(if_ctx_t); 446 static int em_setup_interface(if_ctx_t); 447 static int em_setup_msix(if_ctx_t); 448 449 static void em_initialize_transmit_unit(if_ctx_t); 450 static void em_initialize_receive_unit(if_ctx_t); 451 452 static void em_if_intr_enable(if_ctx_t); 453 static void em_if_intr_disable(if_ctx_t); 454 static void igb_if_intr_enable(if_ctx_t); 455 static void igb_if_intr_disable(if_ctx_t); 456 static int em_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 457 static int em_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 458 static int igb_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 459 static int igb_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 460 static void em_handle_fatal_error_intr(struct e1000_softc *, u32); 461 static bool em_handle_fatal_error_admin(struct e1000_softc *); 462 static u32 igb_device_reset_intr_mask(struct e1000_softc *); 463 static bool igb_device_reset_pending(struct e1000_softc *); 464 static bool igb_handle_device_reset(struct e1000_softc *, u32); 465 static void igb_prepare_device_reset(struct e1000_softc *); 466 static bool igb_finish_device_reset(struct e1000_softc *, u32); 467 static void em_prepare_fatal_error_reset(struct e1000_softc *); 468 static void em_finish_fatal_error_reset(struct e1000_softc *); 469 static void em_configure_peind_memory_errors(struct e1000_softc *); 470 static void em_configure_82575_memory_errors(struct e1000_softc *); 471 static void em_configure_82580_memory_errors(struct e1000_softc *); 472 static void em_update_82580_ecc_stats(struct e1000_softc *, u32, u32, 473 u32); 474 static void em_if_multi_set(if_ctx_t); 475 static void em_if_update_admin_status(if_ctx_t); 476 static void em_if_debug(if_ctx_t); 477 static void em_initialize_vf_stats(struct e1000_softc *); 478 static void em_rebase_vf_stats(struct e1000_softc *); 479 static void em_update_vf_stats_counters(struct e1000_softc *); 480 static void em_add_hw_stats(struct e1000_softc *); 481 static bool em_mac_has_eee(enum e1000_mac_type); 482 static int em_if_set_promisc(if_ctx_t, int); 483 static bool em_if_defer_promisc(struct e1000_softc *); 484 static bool em_if_vlan_filter_capable(if_ctx_t); 485 static bool em_if_vlan_filter_used(if_ctx_t); 486 static void em_if_vlan_filter_enable(struct e1000_softc *); 487 static void em_if_vlan_filter_disable(struct e1000_softc *); 488 static void em_if_vlan_filter_write(struct e1000_softc *, int); 489 static void em_setup_vlan_hw_support(if_ctx_t ctx); 490 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 491 static void em_print_nvm_info(struct e1000_softc *); 492 static void em_fw_version_locked(if_ctx_t); 493 static void em_sbuf_fw_version(struct e1000_fw_version *, struct sbuf *); 494 static void em_print_fw_version(struct e1000_softc *); 495 static int em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 496 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 497 static int em_get_rs(SYSCTL_HANDLER_ARGS); 498 static void em_print_debug_info(struct e1000_softc *); 499 static void em_newitr(struct e1000_softc *, struct em_rx_queue *, 500 struct rx_ring *); 501 static bool em_automask_tso(if_ctx_t); 502 static int em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 503 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 504 static void em_add_int_delay_sysctl(struct e1000_softc *, const char *, 505 const char *, struct em_int_delay_info *, int, int); 506 /* Management and WOL Support */ 507 static void em_init_manageability(struct e1000_softc *); 508 static void em_release_manageability(struct e1000_softc *); 509 static void em_get_hw_control(struct e1000_softc *); 510 static void em_release_hw_control(struct e1000_softc *); 511 static void em_get_wakeup(if_ctx_t); 512 static void em_fill_wakeup_mta(struct e1000_hw *); 513 static int em_enable_wakeup(if_ctx_t); 514 static void em_configure_sx_low_power(struct e1000_softc *, u32); 515 static int em_enable_phy_wakeup(struct e1000_softc *, u32); 516 static int em_disable_phy_wakeup(struct e1000_softc *, u16 *); 517 static void em_power_up_wakeup_link(struct e1000_softc *); 518 static void em_power_down_wakeup_link(struct e1000_softc *); 519 static void em_disable_aspm(struct e1000_softc *); 520 521 int em_intr(void *); 522 523 enum em_fatal_error_state { 524 EM_FATAL_ERROR_NONE, 525 EM_FATAL_ERROR_CAPTURING, 526 EM_FATAL_ERROR_DETECTED, 527 EM_FATAL_ERROR_RESET_REQUESTED, 528 EM_FATAL_ERROR_RESET_PREPARED, 529 }; 530 531 enum igb_device_reset_state { 532 IGB_DEVICE_RESET_NONE, 533 IGB_DEVICE_RESET_DETECTED, 534 IGB_DEVICE_RESET_REQUESTED, 535 IGB_DEVICE_RESET_PREPARED, 536 }; 537 538 #define IGB_DEVICE_RESET_TIMEOUT_MS 100 539 540 /* MSI-X handlers */ 541 static int em_if_msix_intr_assign(if_ctx_t, int); 542 static int em_msix_link(void *); 543 static void em_handle_link(void *); 544 545 static void em_enable_vectors_82574(if_ctx_t); 546 547 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); 548 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); 549 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); 550 static void em_if_led_func(if_ctx_t, int); 551 552 static int em_get_regs(SYSCTL_HANDLER_ARGS); 553 static void lem_smartspeed(struct e1000_softc *); 554 static void igb_configure_queues(struct e1000_softc *); 555 static void igb_initialize_interrupt_rate(struct e1000_softc *); 556 static void em_flush_desc_rings(struct e1000_softc *); 557 558 559 /********************************************************************* 560 * FreeBSD Device Interface Entry Points 561 *********************************************************************/ 562 static device_method_t em_methods[] = { 563 /* Device interface */ 564 DEVMETHOD(device_register, em_register), 565 DEVMETHOD(device_probe, iflib_device_probe), 566 DEVMETHOD(device_attach, iflib_device_attach), 567 DEVMETHOD(device_detach, iflib_device_detach), 568 DEVMETHOD(device_shutdown, iflib_device_shutdown), 569 DEVMETHOD(device_suspend, iflib_device_suspend), 570 DEVMETHOD(device_resume, iflib_device_resume), 571 DEVMETHOD_END 572 }; 573 574 static device_method_t igb_methods[] = { 575 /* Device interface */ 576 DEVMETHOD(device_register, igb_register), 577 DEVMETHOD(device_probe, iflib_device_probe), 578 DEVMETHOD(device_attach, igb_device_attach), 579 DEVMETHOD(device_detach, iflib_device_detach), 580 DEVMETHOD(device_shutdown, iflib_device_shutdown), 581 DEVMETHOD(device_suspend, iflib_device_suspend), 582 DEVMETHOD(device_resume, iflib_device_resume), 583 #ifdef PCI_IOV 584 DEVMETHOD(pci_iov_init, igb_device_iov_init), 585 DEVMETHOD(pci_iov_uninit, igb_device_iov_uninit), 586 DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), 587 #endif 588 DEVMETHOD_END 589 }; 590 591 static device_method_t igbv_methods[] = { 592 /* Device interface */ 593 DEVMETHOD(device_register, igbv_register), 594 DEVMETHOD(device_probe, iflib_device_probe), 595 DEVMETHOD(device_attach, iflib_device_attach), 596 DEVMETHOD(device_detach, iflib_device_detach), 597 DEVMETHOD(device_shutdown, iflib_device_shutdown), 598 DEVMETHOD(device_suspend, iflib_device_suspend), 599 DEVMETHOD(device_resume, iflib_device_resume), 600 DEVMETHOD_END 601 }; 602 603 604 static driver_t em_driver = { 605 "em", em_methods, sizeof(struct e1000_softc), 606 }; 607 608 DRIVER_MODULE(em, pci, em_driver, 0, 0); 609 610 MODULE_DEPEND(em, pci, 1, 1, 1); 611 MODULE_DEPEND(em, ether, 1, 1, 1); 612 MODULE_DEPEND(em, iflib, 1, 1, 1); 613 614 IFLIB_PNP_INFO(pci, em, em_vendor_info_array); 615 616 static driver_t igb_driver = { 617 "igb", igb_methods, sizeof(struct e1000_softc), 618 }; 619 620 DRIVER_MODULE(igb, pci, igb_driver, 0, 0); 621 622 MODULE_DEPEND(igb, pci, 1, 1, 1); 623 MODULE_DEPEND(igb, ether, 1, 1, 1); 624 MODULE_DEPEND(igb, iflib, 1, 1, 1); 625 626 IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array); 627 628 static driver_t igbv_driver = { 629 "igbv", igbv_methods, sizeof(struct e1000_softc), 630 }; 631 632 DRIVER_MODULE(igbv, pci, igbv_driver, 0, 0); 633 634 MODULE_DEPEND(igbv, pci, 1, 1, 1); 635 MODULE_DEPEND(igbv, ether, 1, 1, 1); 636 MODULE_DEPEND(igbv, iflib, 1, 1, 1); 637 638 IFLIB_PNP_INFO(pci, igbv_driver, igbv_vendor_info_array); 639 640 static device_method_t em_if_methods[] = { 641 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 642 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 643 DEVMETHOD(ifdi_detach, em_if_detach), 644 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 645 DEVMETHOD(ifdi_suspend, em_if_suspend), 646 DEVMETHOD(ifdi_resume, em_if_resume), 647 DEVMETHOD(ifdi_init, em_if_init), 648 DEVMETHOD(ifdi_stop, em_if_stop), 649 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 650 DEVMETHOD(ifdi_intr_enable, em_if_intr_enable), 651 DEVMETHOD(ifdi_intr_disable, em_if_intr_disable), 652 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 653 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 654 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 655 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 656 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 657 DEVMETHOD(ifdi_media_status, em_if_media_status), 658 DEVMETHOD(ifdi_media_change, em_if_media_change), 659 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 660 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 661 DEVMETHOD(ifdi_timer, em_if_timer), 662 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 663 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 664 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 665 DEVMETHOD(ifdi_led_func, em_if_led_func), 666 DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), 667 DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), 668 DEVMETHOD(ifdi_debug, em_if_debug), 669 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 670 DEVMETHOD_END 671 }; 672 673 static driver_t em_if_driver = { 674 "em_if", em_if_methods, sizeof(struct e1000_softc) 675 }; 676 677 static device_method_t igb_if_methods[] = { 678 DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), 679 DEVMETHOD(ifdi_attach_post, em_if_attach_post), 680 DEVMETHOD(ifdi_detach, em_if_detach), 681 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 682 DEVMETHOD(ifdi_suspend, em_if_suspend), 683 DEVMETHOD(ifdi_resume, em_if_resume), 684 DEVMETHOD(ifdi_init, em_if_init), 685 DEVMETHOD(ifdi_stop, em_if_stop), 686 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 687 DEVMETHOD(ifdi_intr_enable, igb_if_intr_enable), 688 DEVMETHOD(ifdi_intr_disable, igb_if_intr_disable), 689 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 690 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 691 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 692 DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), 693 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 694 DEVMETHOD(ifdi_media_status, em_if_media_status), 695 DEVMETHOD(ifdi_media_change, em_if_media_change), 696 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 697 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 698 DEVMETHOD(ifdi_timer, em_if_timer), 699 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 700 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 701 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 702 DEVMETHOD(ifdi_led_func, em_if_led_func), 703 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 704 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 705 DEVMETHOD(ifdi_debug, em_if_debug), 706 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 707 #ifdef PCI_IOV 708 DEVMETHOD(ifdi_iov_init, igb_if_iov_init), 709 DEVMETHOD(ifdi_iov_uninit, igb_if_iov_uninit), 710 DEVMETHOD(ifdi_iov_vf_add, igb_if_iov_vf_add), 711 DEVMETHOD(ifdi_vf_status, igb_if_vf_status), 712 #endif 713 DEVMETHOD_END 714 }; 715 716 static driver_t igb_if_driver = { 717 "igb_if", igb_if_methods, sizeof(struct e1000_softc) 718 }; 719 720 static device_method_t igbv_if_methods[] = { 721 DEVMETHOD(ifdi_attach_pre, igbv_if_attach_pre), 722 DEVMETHOD(ifdi_attach_post, igbv_if_attach_post), 723 DEVMETHOD(ifdi_detach, em_if_detach), 724 DEVMETHOD(ifdi_shutdown, em_if_shutdown), 725 DEVMETHOD(ifdi_suspend, em_if_suspend), 726 DEVMETHOD(ifdi_resume, em_if_resume), 727 DEVMETHOD(ifdi_init, em_if_init), 728 DEVMETHOD(ifdi_stop, em_if_stop), 729 DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), 730 DEVMETHOD(ifdi_intr_enable, igbv_if_intr_enable), 731 DEVMETHOD(ifdi_intr_disable, igbv_if_intr_disable), 732 DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), 733 DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), 734 DEVMETHOD(ifdi_queues_free, em_if_queues_free), 735 DEVMETHOD(ifdi_update_admin_status, igbv_if_update_admin_status), 736 DEVMETHOD(ifdi_multi_set, em_if_multi_set), 737 DEVMETHOD(ifdi_media_status, em_if_media_status), 738 DEVMETHOD(ifdi_media_change, igbv_if_media_change), 739 DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), 740 DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), 741 DEVMETHOD(ifdi_timer, em_if_timer), 742 DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), 743 DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), 744 DEVMETHOD(ifdi_get_counter, em_if_get_counter), 745 DEVMETHOD(ifdi_rx_queue_intr_enable, igb_if_rx_queue_intr_enable), 746 DEVMETHOD(ifdi_tx_queue_intr_enable, igb_if_tx_queue_intr_enable), 747 DEVMETHOD(ifdi_debug, em_if_debug), 748 DEVMETHOD(ifdi_needs_restart, em_if_needs_restart), 749 DEVMETHOD_END 750 }; 751 752 static driver_t igbv_if_driver = { 753 "igbv_if", igbv_if_methods, sizeof(struct e1000_softc) 754 }; 755 756 /********************************************************************* 757 * Tunable default values. 758 *********************************************************************/ 759 760 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 761 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 762 763 /* Allow common code without TSO */ 764 #ifndef CSUM_TSO 765 #define CSUM_TSO 0 766 #endif 767 768 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 769 "EM driver parameters"); 770 771 static int em_disable_crc_stripping = 0; 772 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 773 &em_disable_crc_stripping, 0, "Disable CRC Stripping"); 774 775 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 776 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 777 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, 778 &em_tx_int_delay_dflt, 0, "Default transmit interrupt delay in usecs"); 779 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, 780 &em_rx_int_delay_dflt, 0, "Default receive interrupt delay in usecs"); 781 782 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 783 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 784 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN, 785 &em_tx_abs_int_delay_dflt, 0, 786 "Default transmit interrupt delay limit in usecs"); 787 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, 788 &em_rx_abs_int_delay_dflt, 0, 789 "Default receive interrupt delay limit in usecs"); 790 791 static int em_smart_pwr_down = false; 792 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 793 &em_smart_pwr_down, 794 0, "Set to true to leave smart power down enabled on newer adapters"); 795 796 static bool em_unsupported_tso = false; 797 SYSCTL_BOOL(_hw_em, OID_AUTO, unsupported_tso, CTLFLAG_RDTUN, 798 &em_unsupported_tso, 0, "Allow unsupported em(4) TSO configurations"); 799 800 /* Controls whether promiscuous also shows bad packets */ 801 static int em_debug_sbp = false; 802 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, 803 "Show bad packets in promiscuous mode"); 804 805 /* Energy efficient ethernet - default to OFF */ 806 static int eee_setting = 1; 807 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, 808 "Enable Energy Efficient Ethernet"); 809 810 /* 811 * AIM: Adaptive Interrupt Moderation 812 * which means that the interrupt rate is varied over time based on the 813 * traffic for that interrupt vector 814 */ 815 static int em_enable_aim = 1; 816 SYSCTL_INT(_hw_em, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &em_enable_aim, 817 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 818 819 /* 820 ** Tuneable Interrupt rate 821 */ 822 static int em_max_interrupt_rate = EM_INTS_DEFAULT; 823 SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 824 &em_max_interrupt_rate, 0, "Maximum interrupts per second"); 825 826 /* Global used in WOL setup with multiport cards */ 827 static int global_quad_port_a = 0; 828 829 extern struct if_txrx igb_txrx; 830 extern struct if_txrx em_txrx; 831 extern struct if_txrx lem_txrx; 832 833 static struct if_shared_ctx em_sctx_init = { 834 .isc_magic = IFLIB_MAGIC, 835 .isc_q_align = PAGE_SIZE, 836 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 837 .isc_tx_maxsegsize = PAGE_SIZE, 838 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 839 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 840 .isc_rx_maxsize = MJUM9BYTES, 841 .isc_rx_nsegments = 1, 842 .isc_rx_maxsegsize = MJUM9BYTES, 843 .isc_nfl = 1, 844 .isc_nrxqs = 1, 845 .isc_ntxqs = 1, 846 .isc_admin_intrcnt = 1, 847 .isc_vendor_info = em_vendor_info_array, 848 .isc_driver_version = em_driver_version, 849 .isc_driver = &em_if_driver, 850 .isc_flags = 851 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 852 853 .isc_nrxd_min = {EM_MIN_RXD}, 854 .isc_ntxd_min = {EM_MIN_TXD}, 855 .isc_nrxd_max = {EM_MAX_RXD}, 856 .isc_ntxd_max = {EM_MAX_TXD}, 857 .isc_nrxd_default = {EM_DEFAULT_RXD}, 858 .isc_ntxd_default = {EM_DEFAULT_TXD}, 859 }; 860 861 static struct if_shared_ctx igb_sctx_init = { 862 .isc_magic = IFLIB_MAGIC, 863 .isc_q_align = PAGE_SIZE, 864 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 865 .isc_tx_maxsegsize = PAGE_SIZE, 866 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 867 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 868 .isc_rx_maxsize = MJUM9BYTES, 869 .isc_rx_nsegments = 1, 870 .isc_rx_maxsegsize = MJUM9BYTES, 871 .isc_nfl = 1, 872 .isc_nrxqs = 1, 873 .isc_ntxqs = 1, 874 .isc_admin_intrcnt = 1, 875 .isc_vendor_info = igb_vendor_info_array, 876 .isc_driver_version = igb_driver_version, 877 .isc_driver = &igb_if_driver, 878 .isc_flags = 879 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 880 881 .isc_nrxd_min = {EM_MIN_RXD}, 882 .isc_ntxd_min = {EM_MIN_TXD}, 883 .isc_nrxd_max = {IGB_MAX_RXD}, 884 .isc_ntxd_max = {IGB_MAX_TXD}, 885 .isc_nrxd_default = {EM_DEFAULT_RXD}, 886 .isc_ntxd_default = {EM_DEFAULT_TXD}, 887 }; 888 889 /* 890 * igb PFs and igbv VFs share the common datapath implementation. Keep a 891 * separate ifdi policy for VFs so they cannot inherit PF-only callbacks or 892 * interrupt modes. 893 */ 894 static struct if_shared_ctx igbv_sctx_init = { 895 .isc_magic = IFLIB_MAGIC, 896 .isc_q_align = PAGE_SIZE, 897 .isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 898 .isc_tx_maxsegsize = PAGE_SIZE, 899 .isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header), 900 .isc_tso_maxsegsize = EM_TSO_SEG_SIZE, 901 .isc_rx_maxsize = MJUM9BYTES, 902 .isc_rx_nsegments = 1, 903 .isc_rx_maxsegsize = MJUM9BYTES, 904 .isc_nfl = 1, 905 .isc_nrxqs = 1, 906 .isc_ntxqs = 1, 907 .isc_admin_intrcnt = 1, 908 .isc_vendor_info = igbv_vendor_info_array, 909 .isc_driver_version = igb_driver_version, 910 .isc_driver = &igbv_if_driver, 911 .isc_flags = 912 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM | 913 IFLIB_IS_VF, 914 915 .isc_nrxd_min = {EM_MIN_RXD}, 916 .isc_ntxd_min = {EM_MIN_TXD}, 917 .isc_nrxd_max = {IGB_MAX_RXD}, 918 .isc_ntxd_max = {IGB_MAX_TXD}, 919 .isc_nrxd_default = {EM_DEFAULT_RXD}, 920 .isc_ntxd_default = {EM_DEFAULT_TXD}, 921 }; 922 923 /***************************************************************** 924 * 925 * Dump Registers 926 * 927 ****************************************************************/ 928 #define IGB_REGS_LEN 739 929 930 static int em_get_regs(SYSCTL_HANDLER_ARGS) 931 { 932 struct e1000_softc *sc = (struct e1000_softc *)arg1; 933 struct e1000_hw *hw = &sc->hw; 934 struct sbuf *sb; 935 u32 *regs_buff; 936 int rc; 937 uint32_t rxqid, txqid; 938 939 /* 940 * This sysctl is registered before iflib allocates the queue arrays, 941 * and remains registered while iflib tears them down. 942 */ 943 if (sc->rx_queues == NULL || sc->tx_queues == NULL) 944 return (ENXIO); 945 946 regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK); 947 memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); 948 rxqid = sc->rx_queues[0].rxr.me; 949 txqid = sc->tx_queues[0].txr.me; 950 951 rc = sysctl_wire_old_buffer(req, 0); 952 MPASS(rc == 0); 953 if (rc != 0) { 954 free(regs_buff, M_DEVBUF); 955 return (rc); 956 } 957 958 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 959 MPASS(sb != NULL); 960 if (sb == NULL) { 961 free(regs_buff, M_DEVBUF); 962 return (ENOMEM); 963 } 964 965 /* General Registers */ 966 regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); 967 regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); 968 regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); 969 regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); 970 regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); 971 regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(rxqid)); 972 regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(rxqid)); 973 regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(rxqid)); 974 regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(rxqid)); 975 regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(rxqid)); 976 regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(rxqid)); 977 regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); 978 regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(txqid)); 979 regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(txqid)); 980 regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(txqid)); 981 regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(txqid)); 982 regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(txqid)); 983 regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(txqid)); 984 regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); 985 regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); 986 regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); 987 regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); 988 989 sbuf_printf(sb, "General Registers\n"); 990 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 991 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 992 sbuf_printf(sb, "\tCTRL_EXT\t %08x\n\n", regs_buff[2]); 993 994 sbuf_printf(sb, "Interrupt Registers\n"); 995 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 996 997 sbuf_printf(sb, "RX Registers\n"); 998 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 999 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 1000 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 1001 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 1002 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 1003 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 1004 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 1005 1006 sbuf_printf(sb, "TX Registers\n"); 1007 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 1008 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 1009 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 1010 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 1011 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 1012 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 1013 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 1014 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 1015 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 1016 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 1017 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 1018 1019 free(regs_buff, M_DEVBUF); 1020 1021 #ifdef DUMP_DESCS 1022 { 1023 if_softc_ctx_t scctx = sc->shared; 1024 struct rx_ring *rxr = &rx_que->rxr; 1025 struct tx_ring *txr = &tx_que->txr; 1026 int ntxd = scctx->isc_ntxd[0]; 1027 int nrxd = scctx->isc_nrxd[0]; 1028 int j; 1029 1030 for (j = 0; j < nrxd; j++) { 1031 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 1032 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 1033 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 1034 PRIx64 " Error:%d Length:%d\n", 1035 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 1036 } 1037 1038 for (j = 0; j < min(ntxd, 256); j++) { 1039 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 1040 1041 sbuf_printf(sb, 1042 "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x" 1043 " eop: %d DD=%d\n", 1044 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 1045 buf->eop != -1 ? 1046 txr->tx_base[buf->eop].upper.fields.status & 1047 E1000_TXD_STAT_DD : 0); 1048 1049 } 1050 } 1051 #endif 1052 1053 rc = sbuf_finish(sb); 1054 sbuf_delete(sb); 1055 return(rc); 1056 } 1057 1058 static void * 1059 em_register(device_t dev) 1060 { 1061 return (&em_sctx_init); 1062 } 1063 1064 static void * 1065 igb_register(device_t dev) 1066 { 1067 return (&igb_sctx_init); 1068 } 1069 1070 static void * 1071 igbv_register(device_t dev) 1072 { 1073 return (&igbv_sctx_init); 1074 } 1075 1076 static int 1077 igb_device_attach(device_t dev) 1078 { 1079 struct e1000_softc *sc; 1080 if_ctx_t ctx; 1081 int error; 1082 1083 error = iflib_device_attach(dev); 1084 if (error != 0) 1085 return (error); 1086 1087 ctx = device_get_softc(dev); 1088 sc = iflib_get_softc(ctx); 1089 (void)igb_iov_attach(sc); 1090 return (0); 1091 } 1092 1093 #ifdef PCI_IOV 1094 static int 1095 igb_device_iov_init(device_t dev, uint16_t num_vfs, 1096 const nvlist_t *params) 1097 { 1098 struct e1000_softc *sc; 1099 if_ctx_t ctx; 1100 int error; 1101 1102 ctx = device_get_softc(dev); 1103 sc = iflib_get_softc(ctx); 1104 error = igb_iov_validate(sc, num_vfs); 1105 if (error != 0) 1106 return (error); 1107 return (iflib_device_iov_init_restart(dev, num_vfs, params)); 1108 } 1109 1110 static void 1111 igb_device_iov_uninit(device_t dev) 1112 { 1113 struct e1000_softc *sc; 1114 if_ctx_t ctx; 1115 1116 ctx = device_get_softc(dev); 1117 sc = iflib_get_softc(ctx); 1118 /* 1119 * pci_iov(4) has already detached the VF devices. Tell the stop 1120 * half of iflib's restart transaction not to wait for acknowledgements 1121 * from VFs which can no longer service their mailbox vectors. 1122 */ 1123 atomic_store_rel_32(&sc->iov_teardown, 1); 1124 iflib_device_iov_uninit_restart(dev); 1125 } 1126 1127 #endif 1128 1129 static int 1130 em_set_num_queues(if_ctx_t ctx) 1131 { 1132 struct e1000_softc *sc = iflib_get_softc(ctx); 1133 int maxqueues; 1134 1135 /* Sanity check based on HW */ 1136 switch (sc->hw.mac.type) { 1137 case e1000_82576: 1138 case e1000_82580: 1139 case e1000_i350: 1140 case e1000_i354: 1141 maxqueues = 8; 1142 break; 1143 case e1000_i210: 1144 case e1000_82575: 1145 maxqueues = 4; 1146 break; 1147 case e1000_i211: 1148 case e1000_82574: 1149 maxqueues = 2; 1150 break; 1151 case e1000_vfadapt: 1152 /* Keep 82576 VFs at one RX/TX queue for mixed-driver safety. */ 1153 case e1000_vfadapt_i350: 1154 maxqueues = 1; 1155 break; 1156 default: 1157 maxqueues = 1; 1158 break; 1159 } 1160 1161 return (maxqueues); 1162 } 1163 1164 #define LEM_CAPS ( \ 1165 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1166 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1167 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6) 1168 1169 #define EM_CAPS ( \ 1170 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1171 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1172 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1173 IFCAP_TSO6) 1174 1175 #define IGB_CAPS ( \ 1176 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 1177 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 | \ 1178 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | \ 1179 IFCAP_TSO6) 1180 1181 /* 1182 * VLAN filtering is an effective VF capability, but its policy is owned by 1183 * the PF and cannot be disabled from the VF. vlan(4) registration callbacks 1184 * are independent of this capability bit. 1185 */ 1186 #define IGBV_CAPS (IGB_CAPS & ~IFCAP_WOL) 1187 1188 void 1189 em_add_device_sysctls(struct e1000_softc *sc) 1190 { 1191 struct e1000_hw *hw; 1192 struct sysctl_oid_list *child; 1193 struct sysctl_ctx_list *ctx_list; 1194 1195 hw = &sc->hw; 1196 ctx_list = device_get_sysctl_ctx(sc->dev); 1197 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1198 1199 sc->enable_aim = em_enable_aim; 1200 SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", 1201 CTLFLAG_RW, &sc->enable_aim, 0, 1202 "Interrupt Moderation (1=normal, 2=lowlatency)"); 1203 1204 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "debug", 1205 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1206 em_sysctl_debug_info, "I", "Debug Information"); 1207 1208 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "rs_dump", 1209 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1210 em_get_rs, "I", "Dump RS indexes"); 1211 1212 if (sc->vf_ifp) { 1213 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1214 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1215 igbv_get_regs, "A", "Dump VF registers"); 1216 return; 1217 } 1218 1219 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "nvm", 1220 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1221 em_sysctl_nvm_info, "I", "NVM Information"); 1222 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version", 1223 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 1224 em_sysctl_print_fw_version, "A", 1225 "Prints FW/NVM Versions"); 1226 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc", 1227 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1228 em_set_flowcntl, "I", "Flow Control"); 1229 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "reg_dump", 1230 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 1231 em_get_regs, "A", "Dump Registers"); 1232 1233 if (hw->mac.type >= e1000_i350 && hw->mac.type != e1000_i211) { 1234 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac", 1235 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 1236 igb_sysctl_dmac, "I", "DMA Coalesce"); 1237 } 1238 1239 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1240 "tso_tcp_flags_mask_first_segment", 1241 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1242 sc, 0, em_sysctl_tso_tcp_flags_mask, "IU", 1243 "TSO TCP flags mask for first segment"); 1244 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1245 "tso_tcp_flags_mask_middle_segment", 1246 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1247 sc, 1, em_sysctl_tso_tcp_flags_mask, "IU", 1248 "TSO TCP flags mask for middle segment"); 1249 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, 1250 "tso_tcp_flags_mask_last_segment", 1251 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 1252 sc, 2, em_sysctl_tso_tcp_flags_mask, "IU", 1253 "TSO TCP flags mask for last segment"); 1254 } 1255 1256 /********************************************************************* 1257 * Device initialization routine 1258 * 1259 * The attach entry point is called when the driver is being loaded. 1260 * This routine identifies the type of hardware, allocates all resources 1261 * and initializes the hardware. 1262 * 1263 * return 0 on success, positive on failure 1264 *********************************************************************/ 1265 int 1266 em_if_attach_pre(if_ctx_t ctx) 1267 { 1268 struct e1000_softc *sc; 1269 if_softc_ctx_t scctx; 1270 device_t dev; 1271 struct e1000_hw *hw; 1272 struct sysctl_oid_list *child; 1273 struct sysctl_ctx_list *ctx_list; 1274 int error = 0; 1275 1276 INIT_DEBUGOUT("em_if_attach_pre: begin"); 1277 dev = iflib_get_dev(ctx); 1278 sc = iflib_get_softc(ctx); 1279 1280 if (em_max_interrupt_rate <= 0) { 1281 device_printf(dev, 1282 "Invalid max_interrupt_rate %d; using default %d\n", 1283 em_max_interrupt_rate, EM_INTS_DEFAULT); 1284 em_max_interrupt_rate = EM_INTS_DEFAULT; 1285 } 1286 1287 sc->ctx = sc->osdep.ctx = ctx; 1288 sc->dev = sc->osdep.dev = dev; 1289 scctx = sc->shared = iflib_get_softc_ctx(ctx); 1290 sc->media = iflib_get_media(ctx); 1291 hw = &sc->hw; 1292 sc->vf_ifp = 1293 (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0; 1294 sc->osdep.vf = sc->vf_ifp; 1295 1296 /* Determine hardware and mac info */ 1297 em_identify_hardware(ctx); 1298 sc->osdep.vf_82576 = sc->hw.mac.type == e1000_vfadapt; 1299 1300 /* VF sysctls are deferred until attach-post confirms MSI-X. */ 1301 ctx_list = device_get_sysctl_ctx(dev); 1302 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 1303 if (!sc->vf_ifp) 1304 em_add_device_sysctls(sc); 1305 1306 scctx->isc_tx_nsegments = EM_MAX_SCATTER; 1307 scctx->isc_nrxqsets_max = 1308 scctx->isc_ntxqsets_max = em_set_num_queues(ctx); 1309 if (bootverbose) 1310 device_printf(dev, "attach_pre capping queues at %d\n", 1311 scctx->isc_ntxqsets_max); 1312 1313 if (hw->mac.type >= igb_mac_min) { 1314 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1315 sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); 1316 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1317 sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); 1318 scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); 1319 scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); 1320 scctx->isc_txrx = &igb_txrx; 1321 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1322 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1323 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1324 scctx->isc_capabilities = scctx->isc_capenable = 1325 sc->vf_ifp ? IGBV_CAPS : IGB_CAPS; 1326 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 1327 CSUM_IP6_TCP | CSUM_IP6_UDP; 1328 if (hw->mac.type != e1000_82575) 1329 scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; 1330 /* 1331 ** Some new devices, as with ixgbe, now may 1332 ** use a different BAR, so we need to keep 1333 ** track of which is used. 1334 */ 1335 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1336 } else if (hw->mac.type >= em_mac_min) { 1337 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 1338 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1339 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 1340 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); 1341 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1342 scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); 1343 scctx->isc_txrx = &em_txrx; 1344 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1345 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1346 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1347 scctx->isc_capabilities = scctx->isc_capenable = EM_CAPS; 1348 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1349 CSUM_IP6_TCP | CSUM_IP6_UDP; 1350 1351 /* Disable TSO on all em(4) until ring stalls are debugged */ 1352 scctx->isc_capenable &= ~IFCAP_TSO; 1353 1354 /* 1355 * Disable TSO on SPT due to errata that downclocks DMA 1356 * performance 1357 * i218-i219 Specification Update 1.5.4.5 1358 */ 1359 if (hw->mac.type == e1000_pch_spt) 1360 scctx->isc_capenable &= ~IFCAP_TSO; 1361 1362 /* 1363 * We support MSI-X with 82574 only, but indicate to iflib(4) 1364 * that it shall give MSI at least a try with other devices. 1365 */ 1366 if (hw->mac.type == e1000_82574) { 1367 scctx->isc_msix_bar = pci_msix_table_bar(dev); 1368 } else { 1369 scctx->isc_msix_bar = -1; 1370 scctx->isc_disable_msix = 1; 1371 } 1372 } else { 1373 scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * 1374 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 1375 scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * 1376 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 1377 scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); 1378 scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); 1379 scctx->isc_txrx = &lem_txrx; 1380 scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER; 1381 scctx->isc_tx_tso_size_max = EM_TSO_SIZE; 1382 scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; 1383 scctx->isc_capabilities = scctx->isc_capenable = LEM_CAPS; 1384 if (em_unsupported_tso) 1385 scctx->isc_capabilities |= IFCAP_TSO6; 1386 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO | 1387 CSUM_IP6_TCP | CSUM_IP6_UDP; 1388 1389 /* Disable TSO on all lem(4) until ring stalls debugged */ 1390 scctx->isc_capenable &= ~IFCAP_TSO; 1391 1392 /* 82541ER doesn't do HW tagging */ 1393 if (hw->device_id == E1000_DEV_ID_82541ER || 1394 hw->device_id == E1000_DEV_ID_82541ER_LOM) { 1395 scctx->isc_capabilities &= ~IFCAP_VLAN_HWTAGGING; 1396 scctx->isc_capenable = scctx->isc_capabilities; 1397 } 1398 /* This is the first e1000 chip and it does not do offloads */ 1399 if (hw->mac.type == e1000_82542) { 1400 scctx->isc_capabilities &= ~(IFCAP_HWCSUM | 1401 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6 | 1402 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER | 1403 IFCAP_TSO | IFCAP_VLAN_HWTSO); 1404 scctx->isc_capenable = scctx->isc_capabilities; 1405 } 1406 /* These can't do TSO for various reasons */ 1407 if (hw->mac.type < e1000_82544 || 1408 hw->mac.type == e1000_82547 || 1409 hw->mac.type == e1000_82547_rev_2) { 1410 scctx->isc_capabilities &= 1411 ~(IFCAP_TSO |IFCAP_VLAN_HWTSO); 1412 scctx->isc_capenable = scctx->isc_capabilities; 1413 } 1414 /* XXXKB: No IPv6 before this? */ 1415 if (hw->mac.type < e1000_82545){ 1416 scctx->isc_capabilities &= ~IFCAP_HWCSUM_IPV6; 1417 scctx->isc_capenable = scctx->isc_capabilities; 1418 } 1419 /* 1420 * "PCI/PCI-X SDM 4.0" page 33 (b): 1421 * FDX requirement on these chips 1422 */ 1423 if (hw->mac.type == e1000_82547 || 1424 hw->mac.type == e1000_82547_rev_2) 1425 scctx->isc_capenable &= ~(IFCAP_HWCSUM | 1426 IFCAP_VLAN_HWCSUM | IFCAP_HWCSUM_IPV6); 1427 1428 /* INTx only */ 1429 scctx->isc_msix_bar = 0; 1430 } 1431 1432 /* Setup PCI resources */ 1433 if (em_allocate_pci_resources(ctx)) { 1434 device_printf(dev, "Allocation of PCI resources failed\n"); 1435 error = ENXIO; 1436 goto err_pci; 1437 } 1438 /* 1439 * A VF can retain queue enable bits and DMA addresses across VFLR. 1440 * Fence bus mastering before the first mailbox reset so state left by 1441 * a previous owner cannot issue DMA while the driver attaches. 1442 */ 1443 if (sc->vf_ifp) 1444 em_fence_pci_busmaster(sc); 1445 /* 1446 * 82579 can lose a host CSR write while the Management Engine owns 1447 * the PCIm2PCI arbiter. Enable the OS register write interlock before 1448 * shared code initialization performs any MAC writes. 1449 */ 1450 if (hw->mac.type == e1000_pch2lan && 1451 (E1000_READ_REG(hw, E1000_FWSM) & 1452 E1000_ICH_FWSM_FW_VALID) != 0) 1453 sc->osdep.pcim2pci_arbiter_wa = true; 1454 1455 /* 1456 ** For ICH8 and family we need to 1457 ** map the flash memory, and this 1458 ** must happen after the MAC is 1459 ** identified 1460 */ 1461 if ((hw->mac.type == e1000_ich8lan) || 1462 (hw->mac.type == e1000_ich9lan) || 1463 (hw->mac.type == e1000_ich10lan) || 1464 (hw->mac.type == e1000_pchlan) || 1465 (hw->mac.type == e1000_pch2lan) || 1466 (hw->mac.type == e1000_pch_lpt)) { 1467 int rid = EM_BAR_TYPE_FLASH; 1468 sc->flash = bus_alloc_resource_any(dev, 1469 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1470 if (sc->flash == NULL) { 1471 device_printf(dev, "Mapping of Flash failed\n"); 1472 error = ENXIO; 1473 goto err_pci; 1474 } 1475 /* This is used in the shared code */ 1476 hw->flash_address = (u8 *)sc->flash; 1477 sc->osdep.flash_bus_space_tag = 1478 rman_get_bustag(sc->flash); 1479 sc->osdep.flash_bus_space_handle = 1480 rman_get_bushandle(sc->flash); 1481 } 1482 /* 1483 ** In the new SPT device flash is not a 1484 ** separate BAR, rather it is also in BAR0, 1485 ** so use the same tag and an offset handle for the 1486 ** FLASH read/write macros in the shared code. 1487 */ 1488 else if (hw->mac.type >= e1000_pch_spt) { 1489 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag; 1490 sc->osdep.flash_bus_space_handle = 1491 sc->osdep.mem_bus_space_handle + E1000_FLASH_BASE_ADDR; 1492 } 1493 1494 /* Do Shared Code initialization */ 1495 error = e1000_setup_init_funcs(hw, true); 1496 if (error) { 1497 device_printf(dev, "Setup of Shared code failed, error %d\n", 1498 error); 1499 error = ENXIO; 1500 goto err_pci; 1501 } 1502 1503 em_setup_msix(ctx); 1504 e1000_get_bus_info(hw); 1505 1506 /* 1507 * Some conventional PCI systems hang when e1000 devices use 1508 * DMA addresses above 4 GB. Keep PCI-mode DMA below that boundary 1509 * by default; PCI-X and PCIe retain 64-bit DMA. 1510 */ 1511 if (hw->bus.type == e1000_bus_type_pci) { 1512 SYSCTL_ADD_BOOL(ctx_list, child, OID_AUTO, "allow_64bit_dma", 1513 CTLFLAG_RDTUN, &sc->allow_64bit_dma, 0, 1514 "Allow 64-bit DMA in conventional PCI mode"); 1515 if (sc->allow_64bit_dma) 1516 device_printf(dev, "64-bit DMA in conventional PCI mode. " 1517 "Some chipsets are unstable.\n"); 1518 else { 1519 scctx->isc_dma_width = 32; 1520 device_printf(dev, "32-bit DMA in conventional PCI mode. " 1521 "Set dev.%s.%d.allow_64bit_dma=1 at boot to enable " 1522 "64-bit DMA if the chipset is stable with it.\n", 1523 device_get_name(dev), device_get_unit(dev)); 1524 } 1525 } 1526 1527 /* Set up some sysctls for the tunable interrupt delays */ 1528 if (hw->mac.type < igb_mac_min) { 1529 em_add_int_delay_sysctl(sc, "rx_int_delay", 1530 "receive interrupt delay in usecs", &sc->rx_int_delay, 1531 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt); 1532 em_add_int_delay_sysctl(sc, "tx_int_delay", 1533 "transmit interrupt delay in usecs", &sc->tx_int_delay, 1534 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt); 1535 } 1536 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) { 1537 em_add_int_delay_sysctl(sc, "rx_abs_int_delay", 1538 "receive interrupt delay limit in usecs", 1539 &sc->rx_abs_int_delay, 1540 E1000_REGISTER(hw, E1000_RADV), em_rx_abs_int_delay_dflt); 1541 em_add_int_delay_sysctl(sc, "tx_abs_int_delay", 1542 "transmit interrupt delay limit in usecs", 1543 &sc->tx_abs_int_delay, 1544 E1000_REGISTER(hw, E1000_TADV), em_tx_abs_int_delay_dflt); 1545 } 1546 1547 hw->mac.autoneg = DO_AUTO_NEG; 1548 hw->phy.autoneg_wait_to_complete = false; 1549 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1550 1551 if (hw->mac.type < em_mac_min) { 1552 e1000_init_script_state_82541(hw, true); 1553 e1000_set_tbi_compatibility_82543(hw, true); 1554 } 1555 /* Copper options */ 1556 if (hw->phy.media_type == e1000_media_type_copper) { 1557 hw->phy.mdix = AUTO_ALL_MODES; 1558 hw->phy.disable_polarity_correction = false; 1559 hw->phy.ms_type = EM_MASTER_SLAVE; 1560 } 1561 1562 /* 1563 * Set the frame limits assuming 1564 * standard ethernet sized frames. 1565 */ 1566 scctx->isc_max_frame_size = hw->mac.max_frame_size = 1567 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 1568 1569 /* 1570 * This controls when hardware reports transmit completion 1571 * status. 1572 */ 1573 hw->mac.report_tx_early = 1; 1574 1575 /* Allocate multicast array memory. */ 1576 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 1577 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 1578 if (sc->mta == NULL) { 1579 device_printf(dev, 1580 "Can not allocate multicast setup array\n"); 1581 error = ENOMEM; 1582 goto err_late; 1583 } 1584 1585 /* Clear the IFCAP_TSO auto mask */ 1586 sc->tso_automasked = 0; 1587 1588 /* Check SOL/IDER usage on physical functions. */ 1589 if (!sc->vf_ifp && e1000_check_reset_block(hw)) 1590 device_printf(dev, 1591 "PHY reset is blocked due to SOL/IDER session.\n"); 1592 1593 /* Sysctl for setting Energy Efficient Ethernet */ 1594 if (!sc->vf_ifp) { 1595 if (hw->mac.type < igb_mac_min) 1596 hw->dev_spec.ich8lan.eee_disable = eee_setting; 1597 else 1598 hw->dev_spec._82575.eee_disable = eee_setting; 1599 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_control", 1600 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 1601 em_sysctl_eee, "I", "Disable Energy Efficient Ethernet"); 1602 } 1603 1604 /* 1605 ** Start from a known state, this is 1606 ** important in reading the nvm and 1607 ** mac from that. 1608 */ 1609 error = e1000_reset_hw(hw); 1610 if (sc->vf_ifp) { 1611 atomic_store_rel_32(&sc->vf_mbx_ready, 1612 error == E1000_SUCCESS); 1613 if (error != E1000_SUCCESS) 1614 igbv_log_reset_failure(sc, error, true); 1615 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 1616 } else if (error != E1000_SUCCESS) { 1617 device_printf(dev, "Hardware reset failed: %d\n", error); 1618 error = EIO; 1619 goto err_late; 1620 } 1621 1622 /* Make sure a PF has a good EEPROM before we read from it. */ 1623 if (!sc->vf_ifp && e1000_validate_nvm_checksum(hw) < 0) { 1624 /* 1625 ** Some PCI-E parts fail the first check due to 1626 ** the link being in sleep state, call it again, 1627 ** if it fails a second time its a real issue. 1628 */ 1629 if (e1000_validate_nvm_checksum(hw) < 0) { 1630 device_printf(dev, 1631 "The EEPROM Checksum Is Not Valid\n"); 1632 error = EIO; 1633 goto err_late; 1634 } 1635 } 1636 1637 /* Copy the permanent MAC address out of the EEPROM */ 1638 if (e1000_read_mac_addr(hw) < 0) { 1639 device_printf(dev, 1640 "EEPROM read error while reading MAC address\n"); 1641 error = EIO; 1642 goto err_late; 1643 } 1644 1645 if (!em_is_valid_ether_addr(hw->mac.addr)) { 1646 if (sc->vf_ifp) { 1647 device_printf(dev, 1648 "PF did not assign a MAC address; using a " 1649 "locally generated address\n"); 1650 ether_gen_addr(iflib_get_ifp(ctx), 1651 (struct ether_addr *)hw->mac.addr); 1652 } else { 1653 device_printf(dev, "Invalid MAC address\n"); 1654 error = EIO; 1655 goto err_late; 1656 } 1657 } 1658 1659 if (!sc->vf_ifp) { 1660 /* Save NVM versions while holding the IFLIB context lock. */ 1661 em_fw_version_locked(ctx); 1662 em_print_fw_version(sc); 1663 } 1664 1665 /* 1666 * Get Wake-on-Lan and Management info for later use 1667 */ 1668 if (!sc->vf_ifp) { 1669 em_get_wakeup(ctx); 1670 } 1671 1672 iflib_set_mac(ctx, hw->mac.addr); 1673 1674 return (0); 1675 1676 err_late: 1677 em_release_hw_control(sc); 1678 err_pci: 1679 em_free_pci_resources(ctx); 1680 free(sc->mta, M_DEVBUF); 1681 sc->mta = NULL; 1682 1683 return (error); 1684 } 1685 1686 int 1687 em_if_attach_post(if_ctx_t ctx) 1688 { 1689 struct e1000_softc *sc = iflib_get_softc(ctx); 1690 struct e1000_hw *hw = &sc->hw; 1691 int error = 0; 1692 1693 /* Setup OS specific network interface */ 1694 error = em_setup_interface(ctx); 1695 if (error != 0) { 1696 device_printf(sc->dev, "Interface setup failed: %d\n", error); 1697 goto err_late; 1698 } 1699 1700 if (sc->vf_ifp) { 1701 (void)igbv_reset(ctx); 1702 } else if (em_reset(ctx) != E1000_SUCCESS) { 1703 error = EIO; 1704 goto err_late; 1705 } 1706 1707 /* Initialize statistics */ 1708 if (sc->vf_ifp) 1709 em_initialize_vf_stats(sc); 1710 else 1711 sc->ustats.stats = (struct e1000_hw_stats){}; 1712 1713 em_update_stats_counters(sc); 1714 atomic_readandclear_32(&sc->stats_pending); 1715 hw->mac.get_link_status = 1; 1716 if (sc->vf_ifp) 1717 igbv_if_update_admin_status(ctx); 1718 else 1719 em_if_update_admin_status(ctx); 1720 em_add_hw_stats(sc); 1721 1722 /* Non-AMT based hardware can now take control from firmware */ 1723 if (sc->has_manage && !sc->has_amt) 1724 em_get_hw_control(sc); 1725 1726 INIT_DEBUGOUT("em_if_attach_post: end"); 1727 1728 return (0); 1729 1730 err_late: 1731 /* 1732 * Upon em_if_attach_post() error, iflib calls em_if_detach() to 1733 * free resources 1734 */ 1735 return (error); 1736 } 1737 1738 /********************************************************************* 1739 * Device removal routine 1740 * 1741 * The detach entry point is called when the driver is being removed. 1742 * This routine stops the adapter and deallocates all the resources 1743 * that were allocated for driver operation. 1744 * 1745 * return 0 on success, positive on failure 1746 *********************************************************************/ 1747 static int 1748 em_if_detach(if_ctx_t ctx) 1749 { 1750 struct e1000_softc *sc = iflib_get_softc(ctx); 1751 1752 INIT_DEBUGOUT("em_if_detach: begin"); 1753 1754 igb_iov_detach(sc); 1755 if (sc->vf_ifp) { 1756 igbv_queue_retry_detach(sc); 1757 igbv_mbx_retry_detach(sc); 1758 } else { 1759 e1000_phy_hw_reset(&sc->hw); 1760 } 1761 1762 em_release_manageability(sc); 1763 em_release_hw_control(sc); 1764 em_free_pci_resources(ctx); 1765 free(sc->mta, M_DEVBUF); 1766 sc->mta = NULL; 1767 1768 return (0); 1769 } 1770 1771 /********************************************************************* 1772 * 1773 * Shutdown entry point 1774 * 1775 **********************************************************************/ 1776 1777 static int 1778 em_if_shutdown(if_ctx_t ctx) 1779 { 1780 int error; 1781 1782 error = em_if_suspend(ctx); 1783 if (error != 0) 1784 device_printf(iflib_get_dev(ctx), 1785 "Wake configuration failed during shutdown: %d\n", error); 1786 return (0); 1787 } 1788 1789 /* 1790 * Suspend/resume device methods. 1791 */ 1792 static int 1793 em_if_suspend(if_ctx_t ctx) 1794 { 1795 struct e1000_softc *sc = iflib_get_softc(ctx); 1796 int error; 1797 1798 if (sc->vf_ifp) { 1799 igbv_queue_retry_stop(sc); 1800 igbv_mbx_retry_stop(sc); 1801 } 1802 error = em_enable_wakeup(ctx); 1803 em_release_manageability(sc); 1804 em_release_hw_control(sc); 1805 return (error); 1806 } 1807 1808 static int 1809 em_if_resume(if_ctx_t ctx) 1810 { 1811 struct e1000_softc *sc = iflib_get_softc(ctx); 1812 u32 wus; 1813 u16 phy_wus; 1814 int error; 1815 1816 if (sc->hw.mac.type >= e1000_pch2lan && 1817 sc->hw.mac.type < igb_mac_min) 1818 e1000_resume_workarounds_pchlan(&sc->hw); 1819 1820 if (sc->wol_phy_armed) { 1821 /* 1822 * The PHY wake sequence requires an LCD reset before host wake 1823 * ownership is cleared. Wake registers survive this reset. 1824 */ 1825 (void)e1000_phy_hw_reset(&sc->hw); 1826 error = em_disable_phy_wakeup(sc, &phy_wus); 1827 if (error != E1000_SUCCESS) 1828 device_printf(sc->dev, 1829 "Could not clear PHY wakeup state: %d\n", error); 1830 else if (phy_wus != 0) 1831 device_printf(sc->dev, "PHY wakeup status: %#06x\n", 1832 phy_wus); 1833 } 1834 if (!sc->vf_ifp && sc->hw.mac.type >= e1000_82544) { 1835 wus = E1000_READ_REG(&sc->hw, E1000_WUS); 1836 if (!sc->wol_phy_wakeup && wus != 0) 1837 device_printf(sc->dev, "MAC wakeup status: %#010x\n", 1838 wus); 1839 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 1840 E1000_WRITE_REG(&sc->hw, E1000_WUC, 0); 1841 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 1842 } 1843 /* Clear PME after its MAC or PHY wake source has been removed. */ 1844 pci_clear_pme(sc->dev); 1845 1846 return (0); 1847 } 1848 1849 static int 1850 em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1851 { 1852 int max_frame_size; 1853 struct e1000_softc *sc = iflib_get_softc(ctx); 1854 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1855 1856 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1857 1858 switch (sc->hw.mac.type) { 1859 case e1000_82571: 1860 case e1000_82572: 1861 case e1000_ich9lan: 1862 case e1000_ich10lan: 1863 case e1000_pch2lan: 1864 case e1000_pch_lpt: 1865 case e1000_pch_spt: 1866 case e1000_pch_cnp: 1867 case e1000_pch_tgp: 1868 case e1000_pch_adp: 1869 case e1000_pch_mtp: 1870 case e1000_pch_ptp: 1871 case e1000_pch_nvp: 1872 case e1000_82574: 1873 case e1000_82583: 1874 case e1000_80003es2lan: 1875 /* 9K Jumbo Frame size */ 1876 max_frame_size = 9234; 1877 break; 1878 case e1000_pchlan: 1879 max_frame_size = 4096; 1880 break; 1881 case e1000_82542: 1882 case e1000_ich8lan: 1883 /* Adapters that do not support jumbo frames */ 1884 max_frame_size = ETHER_MAX_LEN; 1885 break; 1886 default: 1887 if (sc->hw.mac.type >= igb_mac_min) 1888 max_frame_size = IGB_MAX_FRAME_SIZE; 1889 else /* lem */ 1890 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1891 } 1892 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 1893 return (EINVAL); 1894 } 1895 1896 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 1897 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1898 return (0); 1899 } 1900 1901 /********************************************************************* 1902 * Init entry point 1903 * 1904 * This routine is used in two ways. It is used by the stack as 1905 * init entry point in network interface structure. It is also used 1906 * by the driver as a hw/sw initialization routine to get to a 1907 * consistent state. 1908 * 1909 **********************************************************************/ 1910 static void 1911 em_if_init(if_ctx_t ctx) 1912 { 1913 struct e1000_softc *sc = iflib_get_softc(ctx); 1914 if_softc_ctx_t scctx = sc->shared; 1915 if_t ifp = iflib_get_ifp(ctx); 1916 struct em_tx_queue *tx_que; 1917 int i; 1918 1919 INIT_DEBUGOUT("em_if_init: begin"); 1920 if (sc->vf_ifp) { 1921 igbv_queue_retry_prepare(sc); 1922 igbv_mbx_retry_prepare(sc); 1923 sc->vf_reset_pending = true; 1924 } 1925 if (sc->suspend_link_powered_down) 1926 em_power_up_wakeup_link(sc); 1927 1928 /* Get the latest mac address, User can use a LAA */ 1929 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 1930 1931 /* 1932 * A VF restores its address only after its reset handshake establishes 1933 * CTS. The PF path programs RAR[0] directly here. 1934 */ 1935 if (!sc->vf_ifp) 1936 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1937 1938 /* 1939 * With the 82571 adapter, RAR[0] may be overwritten 1940 * when the other port is reset, we make a duplicate 1941 * in RAR[14] for that eventuality, this assures 1942 * the interface continues to function. 1943 */ 1944 if (sc->hw.mac.type == e1000_82571) { 1945 e1000_set_laa_state_82571(&sc->hw, true); 1946 e1000_rar_set(&sc->hw, sc->hw.mac.addr, 1947 E1000_RAR_ENTRIES - 1); 1948 } 1949 1950 /* Initialize the hardware */ 1951 igb_iov_reset_prepare(sc); 1952 if (sc->vf_ifp) { 1953 (void)igbv_reset(ctx); 1954 em_rebase_vf_stats(sc); 1955 } else if (em_reset(ctx) != E1000_SUCCESS) { 1956 em_fence_pci_busmaster(sc); 1957 iflib_init_failed(ctx); 1958 return; 1959 } 1960 if (sc->vf_ifp && !sc->vf_queues_sanitized) { 1961 /* 1962 * Do not program or enable rings while retained queue state 1963 * might still contain a previous VF owner's DMA address. A 1964 * bounded callout retries initialization after iflib leaves the 1965 * failed initialization stopped. 1966 */ 1967 em_fence_pci_busmaster(sc); 1968 igbv_queue_retry_failed(ctx); 1969 return; 1970 } 1971 if (sc->vf_ifp && 1972 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1973 igbv_mbx_retry_failed(ctx); 1974 return; 1975 } 1976 /* 1977 * Keep a fail-closed device fenced until reset and VF queue 1978 * sanitization have removed every stale DMA address. 1979 */ 1980 if (em_enable_pci_busmaster(sc) != 0) { 1981 device_printf(sc->dev, 1982 "Unable to enable PCI bus mastering\n"); 1983 iflib_init_failed(ctx); 1984 return; 1985 } 1986 if (sc->vf_ifp) 1987 igbv_reconcile_mac(sc, ifp); 1988 /* Re-arm a link-up transition deferred for this reset. */ 1989 if (sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING || 1990 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING) 1991 sc->link_state = EM_LINK_STATE_DOWN; 1992 if (sc->vf_ifp) 1993 igbv_if_update_admin_status(ctx); 1994 else 1995 em_if_update_admin_status(ctx); 1996 1997 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 1998 i++, tx_que++) { 1999 struct tx_ring *txr = &tx_que->txr; 2000 2001 txr->tx_rs_cidx = txr->tx_rs_pidx; 2002 2003 /* Initialize the last processed descriptor to be the end of 2004 * the ring, rather than the start, so that we avoid an 2005 * off-by-one error when calculating how many descriptors are 2006 * done in the credits_update function. 2007 */ 2008 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 2009 } 2010 2011 /* The VF VLAN EtherType is fixed and has no VET register. */ 2012 if (!sc->vf_ifp) 2013 E1000_WRITE_REG(&sc->hw, E1000_VET, ETHERTYPE_VLAN); 2014 2015 /* Clear bad data from Rx FIFOs */ 2016 if (sc->hw.mac.type >= igb_mac_min && !sc->vf_ifp) 2017 e1000_rx_fifo_flush_base(&sc->hw); 2018 2019 /* Configure for OS presence */ 2020 em_init_manageability(sc); 2021 2022 /* Prepare transmit descriptors and buffers */ 2023 if (sc->vf_ifp) 2024 igbv_initialize_transmit_unit(ctx); 2025 else 2026 em_initialize_transmit_unit(ctx); 2027 2028 /* 2029 * A failed VF reset has no CTS channel on which to restore mailbox 2030 * state. The reset detector schedules another complete init, which 2031 * replays these interface-owned lists after the handshake succeeds. 2032 */ 2033 em_if_multi_set(ctx); 2034 2035 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 2036 if (sc->vf_ifp) 2037 igbv_initialize_receive_unit(ctx); 2038 else 2039 em_initialize_receive_unit(ctx); 2040 2041 /* Set up VLAN support and filter. */ 2042 em_setup_vlan_hw_support(ctx); 2043 2044 /* Don't lose promiscuous settings. */ 2045 em_if_set_promisc_impl(ctx, if_getflags(ifp)); 2046 atomic_readandclear_32(&sc->promisc_pending); 2047 2048 /* Restore PF/VF pool configuration after the global reset. */ 2049 igb_iov_initialize(sc); 2050 2051 if (sc->hw.mac.ops.clear_hw_cntrs != NULL) 2052 sc->hw.mac.ops.clear_hw_cntrs(&sc->hw); 2053 2054 /* MSI-X configuration for 82574 */ 2055 if (sc->hw.mac.type == e1000_82574) { 2056 int tmp = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 2057 2058 tmp |= E1000_CTRL_EXT_PBA_CLR; 2059 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, tmp); 2060 /* Set the IVAR - interrupt vector routing. */ 2061 E1000_WRITE_REG(&sc->hw, E1000_IVAR, sc->ivars); 2062 } else if (sc->intr_type == IFLIB_INTR_MSIX) { 2063 /* Set up queue routing */ 2064 igb_configure_queues(sc); 2065 } 2066 if (sc->hw.mac.type >= igb_mac_min) 2067 igb_initialize_interrupt_rate(sc); 2068 2069 /* AMT based hardware can now take control from firmware */ 2070 if (sc->has_manage && sc->has_amt) 2071 em_get_hw_control(sc); 2072 2073 /* Set Energy Efficient Ethernet */ 2074 if (sc->hw.mac.type >= igb_mac_min && 2075 sc->hw.phy.media_type == e1000_media_type_copper) { 2076 if (sc->hw.mac.type == e1000_i354) 2077 e1000_set_eee_i354(&sc->hw, true, true); 2078 else 2079 e1000_set_eee_i350(&sc->hw, true, true); 2080 } 2081 em_configure_peind_memory_errors(sc); 2082 em_configure_82575_memory_errors(sc); 2083 em_configure_82580_memory_errors(sc); 2084 if (sc->vf_ifp) { 2085 sc->vf_reset_pending = false; 2086 } else { 2087 u32 icr; 2088 2089 /* 2090 * Drain stale causes only after register reconstruction is 2091 * complete. DRSTA and DEV_RST_SET together close the window in 2092 * which another device reset can arrive while interrupts are 2093 * masked. 2094 */ 2095 icr = E1000_READ_REG(&sc->hw, E1000_ICR); 2096 if (igb_finish_device_reset(sc, icr)) { 2097 iflib_init_failed(ctx); 2098 return; 2099 } 2100 E1000_WRITE_REG(&sc->hw, E1000_ICS, E1000_ICS_LSC); 2101 } 2102 } 2103 2104 /* 2105 * RX publishes its byte and packet counters as one snapshot when iflib 2106 * returns descriptors to hardware. This also covers watchdog-driven RX 2107 * processing, which can run while the interrupt vector is unmasked. 2108 */ 2109 static __inline void 2110 em_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 2111 { 2112 uint64_t snapshot; 2113 u32 now_bytes, now_packets; 2114 2115 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 2116 now_bytes = snapshot >> 32; 2117 now_packets = (u32)snapshot; 2118 *bytes = now_bytes - rxr->rx_bytes_last; 2119 *packets = now_packets - rxr->rx_packets_last; 2120 rxr->rx_bytes_last = now_bytes; 2121 rxr->rx_packets_last = now_packets; 2122 } 2123 2124 /* 2125 * TX publishes its byte and packet counters as one snapshot at the doorbell, 2126 * because encapsulation can overlap the interrupt filter. The two halves 2127 * remain independent free running u32 counters, so their deltas are correct 2128 * across wrap. 2129 */ 2130 static __inline void 2131 em_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 2132 { 2133 uint64_t snapshot; 2134 u32 now_bytes, now_packets; 2135 2136 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 2137 now_bytes = snapshot >> 32; 2138 now_packets = (u32)snapshot; 2139 *bytes = now_bytes - txr->tx_bytes_last; 2140 *packets = now_packets - txr->tx_packets_last; 2141 txr->tx_bytes_last = now_bytes; 2142 txr->tx_packets_last = now_packets; 2143 } 2144 2145 /********************************************************************* 2146 * 2147 * Do Adaptive Interrupt Moderation: 2148 * - Calculate based on average size over the last interval 2149 * 2150 * Returns interrupts per second rather than a register value, so that the 2151 * caller's EM_INTS_TO_ITR()/IGB_INTS_TO_EITR() conversion applies, or zero 2152 * if the interval carried no packet to measure. 2153 * 2154 *********************************************************************/ 2155 static u32 2156 em_ring_itr(struct e1000_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 2157 u32 txpackets) 2158 { 2159 u32 newitr = 0; 2160 2161 if (txbytes && txpackets) 2162 newitr = txbytes / txpackets; 2163 if (rxbytes && rxpackets) 2164 newitr = max(newitr, rxbytes / rxpackets); 2165 2166 /* 2167 * No packet was observed, so there is no size to work from. Report no 2168 * observation and let the caller keep the rate it already has. 2169 */ 2170 if (newitr == 0) 2171 return (0); 2172 2173 newitr += 24; /* account for hardware frame, crc */ 2174 /* set an upper boundary */ 2175 newitr = min(newitr, 3000); 2176 /* Be nice to the mid range */ 2177 if ((newitr > 300) && (newitr < 1200)) 2178 newitr = (newitr / 3); 2179 else 2180 newitr = (newitr / 2); 2181 2182 /* The value above was written straight to EITR; make it a rate */ 2183 newitr = EM_AIM_DIVIDEND / newitr; 2184 2185 /* 2186 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 2187 * into the low latency end. The original was unbounded and would ask 2188 * for ~95k ints/s on minimum sized frames. There is deliberately no 2189 * floor, so jumbo traffic settles near 2.7k ints/s. 2190 */ 2191 if (sc->enable_aim == 1) 2192 newitr = min(newitr, EM_INTS_20K); 2193 else 2194 newitr = min(newitr, EM_INTS_70K); 2195 2196 return (newitr); 2197 } 2198 2199 /********************************************************************* 2200 * 2201 * Helper to calculate next (E)ITR value for AIM 2202 * 2203 *********************************************************************/ 2204 static void 2205 em_newitr(struct e1000_softc *sc, struct em_rx_queue *que, 2206 struct rx_ring *rxr) 2207 { 2208 struct e1000_hw *hw = &sc->hw; 2209 struct em_tx_queue *tx_que; 2210 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 2211 u32 newitr; 2212 int i; 2213 2214 em_aim_rx_delta(rxr, &rxbytes, &rxpackets); 2215 2216 /* 2217 * A vector can service more than one TX ring when iflib is configured 2218 * with unequal RX and TX queue counts. Sample every ring routed to 2219 * this vector rather than treating the vector as a TX queue index. 2220 */ 2221 txbytes = txpackets = 0; 2222 for (i = 0; i < sc->tx_num_queues; i++) { 2223 tx_que = &sc->tx_queues[i]; 2224 if (tx_que->msix != que->msix) 2225 continue; 2226 em_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 2227 txbytes += ringbytes; 2228 txpackets += ringpackets; 2229 } 2230 2231 /* Idle, do nothing */ 2232 if (txbytes == 0 && rxbytes == 0) 2233 return; 2234 2235 if (sc->enable_aim == 0) { 2236 newitr = em_max_interrupt_rate; 2237 } else if (sc->link_speed < SPEED_1000) { 2238 /* Use half default (4K) ITR if sub-gig */ 2239 newitr = EM_INTS_4K; 2240 } else if (!sc->vf_ifp && 2241 sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 2242 /* Want at least enough packet buffer for two frames to AIM */ 2243 newitr = em_max_interrupt_rate; 2244 } else { 2245 newitr = em_ring_itr(sc, rxbytes, rxpackets, txbytes, 2246 txpackets); 2247 /* No usable observation; leave the rate where it is */ 2248 if (newitr == 0) 2249 return; 2250 } 2251 2252 if (hw->mac.type >= igb_mac_min) { 2253 newitr = IGB_INTS_TO_EITR(newitr); 2254 2255 if (hw->mac.type == e1000_82575) 2256 newitr |= newitr << 16; 2257 else 2258 newitr |= E1000_EITR_CNT_IGNR; 2259 2260 if (newitr != que->itr_setting) { 2261 que->itr_setting = newitr; 2262 E1000_WRITE_REG(hw, E1000_EITR(que->msix), 2263 que->itr_setting); 2264 } 2265 } else { 2266 newitr = EM_INTS_TO_ITR(newitr); 2267 2268 if (newitr != que->itr_setting) { 2269 que->itr_setting = newitr; 2270 if (hw->mac.type == e1000_82574 && 2271 sc->intr_type == IFLIB_INTR_MSIX) { 2272 E1000_WRITE_REG(hw, 2273 E1000_EITR_82574(que->msix), 2274 que->itr_setting); 2275 } else { 2276 E1000_WRITE_REG(hw, E1000_ITR, 2277 que->itr_setting); 2278 } 2279 } 2280 } 2281 } 2282 2283 static bool 2284 em_has_pch_ecc(const struct e1000_hw *hw) 2285 { 2286 2287 return (hw->mac.type >= e1000_pch_lpt && 2288 hw->mac.type < e1000_82575); 2289 } 2290 2291 static bool 2292 em_has_82571_ecc_stats(const struct e1000_hw *hw) 2293 { 2294 2295 return (hw->mac.type == e1000_82571); 2296 } 2297 2298 static bool 2299 em_has_82575_memory_errors(const struct e1000_hw *hw) 2300 { 2301 2302 return (hw->mac.type == e1000_82575); 2303 } 2304 2305 static void 2306 em_configure_82575_memory_errors(struct e1000_softc *sc) 2307 { 2308 struct e1000_hw *hw; 2309 u32 ctrl_ext; 2310 2311 hw = &sc->hw; 2312 if (!em_has_82575_memory_errors(hw)) 2313 return; 2314 2315 /* Discard pre-driver status before enabling the hardware reaction. */ 2316 (void)E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2317 (void)E1000_READ_REG(hw, E1000_RDHESTS_82575); 2318 (void)E1000_READ_REG(hw, E1000_TDHESTS_82575); 2319 E1000_WRITE_REG(hw, E1000_PBECCSTS_82575, 2320 E1000_ECC_82575_ENABLE); 2321 E1000_WRITE_REG(hw, E1000_RDHESTS_82575, 2322 E1000_ECC_82575_ENABLE); 2323 E1000_WRITE_REG(hw, E1000_TDHESTS_82575, 2324 E1000_ECC_82575_ENABLE); 2325 2326 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2327 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2328 ctrl_ext | E1000_CTRL_EXT_MEHE); 2329 E1000_WRITE_FLUSH(hw); 2330 } 2331 2332 static bool 2333 em_has_82576_memory_errors(const struct e1000_hw *hw) 2334 { 2335 2336 return (hw->mac.type == e1000_82576); 2337 } 2338 2339 static bool 2340 em_82576_has_ipsec(const struct e1000_hw *hw) 2341 { 2342 2343 return (hw->device_id != E1000_DEV_ID_82576_NS && 2344 hw->device_id != E1000_DEV_ID_82576_NS_SERDES); 2345 } 2346 2347 static void 2348 em_configure_82576_memory_errors(struct e1000_softc *sc) 2349 { 2350 struct e1000_hw *hw; 2351 u32 peindm, reactions; 2352 2353 hw = &sc->hw; 2354 if (!em_has_82576_memory_errors(hw)) 2355 return; 2356 2357 reactions = E1000_PEIND_82576_NONFATAL_MASK | 2358 E1000_PEIND_82576_FATAL_MASK | 2359 E1000_PEINDM_82576_PARITY_ENABLE; 2360 if (!em_82576_has_ipsec(hw)) 2361 reactions &= ~E1000_PEIND_82576_IPSEC_MASK; 2362 2363 /* Discard indications left by firmware before enabling reactions. */ 2364 (void)E1000_READ_REG(hw, E1000_PEIND); 2365 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2366 E1000_WRITE_REG(hw, E1000_PEINDM, peindm | reactions); 2367 E1000_WRITE_FLUSH(hw); 2368 } 2369 2370 static bool 2371 em_has_82580_memory_errors(const struct e1000_hw *hw) 2372 { 2373 2374 return (hw->mac.type == e1000_82580); 2375 } 2376 2377 static void 2378 em_clear_82580_memory_error_status(struct e1000_hw *hw, u32 reg) 2379 { 2380 u32 status; 2381 2382 status = E1000_READ_REG(hw, reg); 2383 if (status != 0) 2384 E1000_WRITE_REG(hw, reg, status); 2385 } 2386 2387 static void 2388 em_configure_82580_memory_errors(struct e1000_softc *sc) 2389 { 2390 struct e1000_hw *hw; 2391 u32 reg; 2392 2393 hw = &sc->hw; 2394 if (!em_has_82580_memory_errors(hw)) 2395 return; 2396 2397 /* Clear status left before the driver completed its memory tables. */ 2398 (void)E1000_READ_REG(hw, E1000_PEIND); 2399 em_clear_82580_memory_error_status(hw, E1000_DTPARS_82580); 2400 em_clear_82580_memory_error_status(hw, E1000_DRPARS_82580); 2401 em_clear_82580_memory_error_status(hw, E1000_DDPARS_82580); 2402 em_clear_82580_memory_error_status(hw, E1000_PCIEERRSTS); 2403 (void)E1000_READ_REG(hw, E1000_LANPERRSTS); 2404 em_update_82580_ecc_stats(sc, 2405 E1000_READ_REG(hw, E1000_RPBECCSTS), 2406 E1000_READ_REG(hw, E1000_TPBECCSTS), 2407 E1000_READ_REG(hw, E1000_PCIEECCSTS)); 2408 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2409 E1000_PBECCSTS_82580_ECC_ENABLE); 2410 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2411 E1000_PBECCSTS_82580_ECC_ENABLE); 2412 2413 reg = E1000_READ_REG(hw, E1000_DTPARC_82580); 2414 E1000_WRITE_REG(hw, E1000_DTPARC_82580, 2415 reg | E1000_DTPARC_82580_ENABLE_MASK); 2416 reg = E1000_READ_REG(hw, E1000_DRPARC_82580); 2417 E1000_WRITE_REG(hw, E1000_DRPARC_82580, 2418 reg | E1000_DRPARC_82580_ENABLE_MASK); 2419 reg = E1000_READ_REG(hw, E1000_DDPARC_82580); 2420 E1000_WRITE_REG(hw, E1000_DDPARC_82580, 2421 reg | E1000_DDPARC_82580_ENABLE_MASK); 2422 reg = E1000_READ_REG(hw, E1000_PCIEERRCTL_82580); 2423 E1000_WRITE_REG(hw, E1000_PCIEERRCTL_82580, 2424 reg | E1000_PCIEERRCTL_82580_ENABLE_MASK); 2425 reg = E1000_READ_REG(hw, E1000_PCIEECCCTL_82580); 2426 E1000_WRITE_REG(hw, E1000_PCIEECCCTL_82580, 2427 reg | E1000_PCIEECCCTL_82580_ENABLE_MASK); 2428 reg = E1000_READ_REG(hw, E1000_LANPERRCTL_82580); 2429 reg |= E1000_LANPERRCTL_82580_HOST_MASK; 2430 /* The RSS memory is initialized only for a multiqueue layout. */ 2431 if (sc->rx_num_queues <= 1) 2432 reg &= ~E1000_LANPERRCTL_82580_RSS_ENABLE; 2433 E1000_WRITE_REG(hw, E1000_LANPERRCTL_82580, reg); 2434 reg = E1000_READ_REG(hw, E1000_PEINDM); 2435 E1000_WRITE_REG(hw, E1000_PEINDM, 2436 reg | E1000_PEIND_FATAL_MASK); 2437 E1000_WRITE_FLUSH(hw); 2438 } 2439 2440 static bool 2441 em_has_i210_memory_errors(const struct e1000_hw *hw) 2442 { 2443 2444 return (hw->mac.type == e1000_i210 || 2445 hw->mac.type == e1000_i211); 2446 } 2447 2448 static bool 2449 em_has_i350_i354_memory_errors(const struct e1000_hw *hw) 2450 { 2451 2452 return (hw->mac.type == e1000_i350 || 2453 hw->mac.type == e1000_i354); 2454 } 2455 2456 static void 2457 em_configure_peind_memory_errors(struct e1000_softc *sc) 2458 { 2459 struct e1000_hw *hw; 2460 u32 peindm; 2461 2462 hw = &sc->hw; 2463 if (!em_has_i350_i354_memory_errors(hw) && 2464 !em_has_i210_memory_errors(hw)) 2465 return; 2466 2467 /* Discard indications left by firmware before enabling reactions. */ 2468 (void)E1000_READ_REG(hw, E1000_PEIND); 2469 /* Do not depend on firmware preserving the datasheet defaults. */ 2470 peindm = E1000_READ_REG(hw, E1000_PEINDM); 2471 E1000_WRITE_REG(hw, E1000_PEINDM, 2472 peindm | E1000_PEIND_FATAL_MASK); 2473 E1000_WRITE_FLUSH(hw); 2474 } 2475 2476 static bool 2477 em_has_peind_memory_errors(const struct e1000_hw *hw) 2478 { 2479 2480 return (em_has_82580_memory_errors(hw) || 2481 em_has_i350_i354_memory_errors(hw) || 2482 em_has_i210_memory_errors(hw)); 2483 } 2484 2485 static u32 2486 em_pcie_fatal_error_mask(const struct e1000_hw *hw) 2487 { 2488 2489 if (em_has_82580_memory_errors(hw)) 2490 return (~0U); 2491 if (em_has_i350_i354_memory_errors(hw)) 2492 return (E1000_PCIEERRSTS_I350_I354_FATAL_MASK); 2493 if (em_has_i210_memory_errors(hw)) 2494 return (E1000_PCIEERRSTS_I210_FATAL_MASK); 2495 return (0); 2496 } 2497 2498 static u32 2499 em_memory_error_intr_mask(const struct e1000_hw *hw) 2500 { 2501 2502 if (em_has_82575_memory_errors(hw)) 2503 return (E1000_IMS_82575_MEMORY_ERROR_MASK); 2504 if (em_has_82576_memory_errors(hw)) 2505 return (E1000_IMS_FER | E1000_IMS_NFER); 2506 if (em_has_pch_ecc(hw) || em_has_peind_memory_errors(hw)) 2507 return (E1000_IMS_FER); 2508 return (0); 2509 } 2510 2511 static bool 2512 em_has_memory_errors(const struct e1000_hw *hw) 2513 { 2514 2515 return (em_memory_error_intr_mask(hw) != 0); 2516 } 2517 2518 static bool 2519 em_has_memory_error_stats(const struct e1000_hw *hw) 2520 { 2521 2522 return (em_has_82571_ecc_stats(hw) || em_has_memory_errors(hw)); 2523 } 2524 2525 static u32 2526 em_fatal_error_intr_mask(struct e1000_softc *sc) 2527 { 2528 if (!em_has_memory_errors(&sc->hw)) 2529 return (0); 2530 if (atomic_load_acq_32(&sc->fatal_error_state) != 2531 EM_FATAL_ERROR_NONE) 2532 return (0); 2533 return (em_memory_error_intr_mask(&sc->hw)); 2534 } 2535 2536 static void 2537 em_update_82580_ecc_stats(struct e1000_softc *sc, u32 rpbeccsts, 2538 u32 tpbeccsts, u32 pcieeccsts) 2539 { 2540 u32 status; 2541 2542 sc->corrected_error_packet_buffer_count += 2543 (rpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK) + 2544 (tpbeccsts & E1000_PBECCSTS_82580_CORR_CNT_MASK); 2545 status = pcieeccsts & E1000_PCIEECCSTS_82580_ERROR_MASK; 2546 sc->uncorrected_error_pcie_count += bitcount32(status); 2547 if (status != 0) 2548 E1000_WRITE_REG(&sc->hw, E1000_PCIEECCSTS, status); 2549 } 2550 2551 static void 2552 em_update_82575_ecc_stats(struct e1000_softc *sc, u32 pbeccsts, 2553 u32 rdhests, u32 tdhests) 2554 { 2555 2556 sc->corrected_error_packet_buffer_count += 2557 pbeccsts & E1000_ECC_82575_CORR_CNT_MASK; 2558 sc->uncorrected_error_packet_buffer_count += 2559 (pbeccsts & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2560 E1000_ECC_82575_UNCORR_CNT_SHIFT; 2561 sc->corrected_error_dma_count += 2562 (rdhests & E1000_ECC_82575_CORR_CNT_MASK) + 2563 (tdhests & E1000_ECC_82575_CORR_CNT_MASK); 2564 sc->uncorrected_error_dma_count += 2565 ((rdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2566 E1000_ECC_82575_UNCORR_CNT_SHIFT) + 2567 ((tdhests & E1000_ECC_82575_UNCORR_CNT_MASK) >> 2568 E1000_ECC_82575_UNCORR_CNT_SHIFT); 2569 } 2570 2571 static void 2572 em_update_82576_ecc_counter(struct e1000_softc *sc, u32 reg, 2573 u64 *corrected, u64 *uncorrected) 2574 { 2575 u32 status; 2576 2577 status = E1000_READ_REG(&sc->hw, reg); 2578 *corrected += status & E1000_ECC_82576_CORR_CNT_MASK; 2579 if (uncorrected != NULL) 2580 *uncorrected += 2581 (status & E1000_ECC_82576_UNCORR_CNT_MASK) >> 2582 E1000_ECC_82576_UNCORR_CNT_SHIFT; 2583 } 2584 2585 static void 2586 em_update_82576_ecc_stats(struct e1000_softc *sc) 2587 { 2588 2589 /* 2590 * These counters are clear-on-read. PRBESTS and PMSIXESTS are 2591 * controller-shared, so whichever LAN port samples them first owns 2592 * the software count. 2593 */ 2594 em_update_82576_ecc_counter(sc, E1000_RPBECCSTS, 2595 &sc->corrected_error_packet_buffer_count, 2596 &sc->uncorrected_error_packet_buffer_count); 2597 em_update_82576_ecc_counter(sc, E1000_TPBECCSTS, 2598 &sc->corrected_error_packet_buffer_count, 2599 &sc->uncorrected_error_packet_buffer_count); 2600 em_update_82576_ecc_counter(sc, E1000_SWPBECCSTS_82576, 2601 &sc->corrected_error_packet_buffer_count, 2602 &sc->uncorrected_error_packet_buffer_count); 2603 if (em_82576_has_ipsec(&sc->hw)) 2604 em_update_82576_ecc_counter(sc, E1000_IPPBECCSTS_82576, 2605 &sc->corrected_error_packet_buffer_count, 2606 &sc->uncorrected_error_packet_buffer_count); 2607 2608 em_update_82576_ecc_counter(sc, E1000_RDHESTS_82576, 2609 &sc->corrected_error_dma_count, 2610 &sc->uncorrected_error_dma_count); 2611 em_update_82576_ecc_counter(sc, E1000_TDHESTS_82576, 2612 &sc->corrected_error_dma_count, 2613 &sc->uncorrected_error_dma_count); 2614 2615 em_update_82576_ecc_counter(sc, E1000_PRBESTS_82576, 2616 &sc->corrected_error_pcie_retry_count, NULL); 2617 em_update_82576_ecc_counter(sc, E1000_PWBESTS_82576, 2618 &sc->corrected_error_pcie_tx_data_count, NULL); 2619 em_update_82576_ecc_counter(sc, E1000_PMSIXESTS_82576, 2620 &sc->corrected_error_pcie_other_count, NULL); 2621 } 2622 2623 static void 2624 em_update_pch_ecc_stats(struct e1000_softc *sc, u32 pbeccsts) 2625 { 2626 2627 sc->corrected_error_packet_buffer_count += 2628 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 2629 sc->uncorrected_error_packet_buffer_count += 2630 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >> 2631 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT; 2632 } 2633 2634 static void 2635 em_update_82571_ecc_stats(struct e1000_softc *sc) 2636 { 2637 struct e1000_hw *hw; 2638 u32 count, pba_ecc; 2639 2640 hw = &sc->hw; 2641 pba_ecc = E1000_READ_REG(hw, E1000_PBA_ECC); 2642 count = (pba_ecc & E1000_PBA_ECC_COUNTER_MASK) >> 2643 E1000_PBA_ECC_COUNTER_SHIFT; 2644 if (count == 0) 2645 return; 2646 sc->corrected_error_packet_buffer_count += count; 2647 /* Preserve correction and reserved state while clearing statistics. */ 2648 E1000_WRITE_REG(hw, E1000_PBA_ECC, 2649 pba_ecc | E1000_PBA_ECC_STAT_CLR); 2650 } 2651 2652 static void 2653 em_update_i210_ecc_stats(struct e1000_softc *sc) 2654 { 2655 struct e1000_hw *hw; 2656 u32 pbeccsts, pcieeccsts; 2657 2658 hw = &sc->hw; 2659 pbeccsts = E1000_READ_REG(hw, E1000_PBECCSTS_I210); 2660 if (pbeccsts & E1000_PBECCSTS_I210_CORR_ERR) { 2661 sc->corrected_error_dma_count++; 2662 /* Preserve the enable bit while clearing the RW1C status. */ 2663 E1000_WRITE_REG(hw, E1000_PBECCSTS_I210, 2664 pbeccsts & (E1000_PBECCSTS_I210_ECC_ENABLE | 2665 E1000_PBECCSTS_I210_CORR_ERR)); 2666 } 2667 2668 pcieeccsts = E1000_READ_REG(hw, E1000_PCIEECCSTS) & 2669 E1000_PCIEECCSTS_I210_CORR_MASK; 2670 if (pcieeccsts & E1000_PCIEECCSTS_TX_WR_DATA) 2671 sc->corrected_error_pcie_tx_data_count++; 2672 if (pcieeccsts & E1000_PCIEECCSTS_RETRY_BUF) 2673 sc->corrected_error_pcie_retry_count++; 2674 if (pcieeccsts != 0) 2675 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieeccsts); 2676 } 2677 2678 static void 2679 em_update_i350_i354_ecc_stats(struct e1000_softc *sc) 2680 { 2681 struct e1000_hw *hw; 2682 u32 pbeccsts, pcieecc_mask, status; 2683 2684 hw = &sc->hw; 2685 status = E1000_READ_REG(hw, E1000_DTPARS) & 2686 E1000_DTPARS_CORR_MASK; 2687 if (status != 0) { 2688 sc->corrected_error_dma_count += bitcount32(status); 2689 E1000_WRITE_REG(hw, E1000_DTPARS, status); 2690 } 2691 status = E1000_READ_REG(hw, E1000_DRPARS) & 2692 E1000_DRPARS_CORR_MASK; 2693 if (status != 0) { 2694 sc->corrected_error_dma_count += bitcount32(status); 2695 E1000_WRITE_REG(hw, E1000_DRPARS, status); 2696 } 2697 status = E1000_READ_REG(hw, E1000_DDECCS) & 2698 E1000_DDECCS_CORR_MASK; 2699 if (status != 0) { 2700 sc->corrected_error_dma_count += bitcount32(status); 2701 E1000_WRITE_REG(hw, E1000_DDECCS, status); 2702 } 2703 status = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2704 E1000_LANPERRSTS_MNG_FIFO_CORR; 2705 if (status != 0) { 2706 sc->corrected_error_lan_mng_fifo_count++; 2707 E1000_WRITE_REG(hw, E1000_LANPERRSTS, status); 2708 } 2709 2710 pbeccsts = E1000_READ_REG(hw, E1000_RPBECCSTS); 2711 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2712 if (status != 0) { 2713 sc->corrected_error_packet_buffer_count += bitcount32(status); 2714 /* Preserve the enable bits while clearing RW1C status. */ 2715 E1000_WRITE_REG(hw, E1000_RPBECCSTS, 2716 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2717 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2718 } 2719 pbeccsts = E1000_READ_REG(hw, E1000_TPBECCSTS); 2720 status = pbeccsts & E1000_PBECCSTS_I350_I354_CORR_MASK; 2721 if (status != 0) { 2722 sc->corrected_error_packet_buffer_count += bitcount32(status); 2723 E1000_WRITE_REG(hw, E1000_TPBECCSTS, 2724 pbeccsts & (E1000_PBECCSTS_I350_I354_ENABLE_MASK | 2725 E1000_PBECCSTS_I350_I354_CORR_MASK)); 2726 } 2727 2728 pcieecc_mask = hw->mac.type == e1000_i354 ? 2729 E1000_PCIEECCSTS_I354_CORR_MASK : 2730 E1000_PCIEECCSTS_I350_CORR_MASK; 2731 status = E1000_READ_REG(hw, E1000_PCIEECCSTS) & pcieecc_mask; 2732 if (status & E1000_PCIEECCSTS_TX_WR_DATA) 2733 sc->corrected_error_pcie_tx_data_count++; 2734 if (status & E1000_PCIEECCSTS_RETRY_BUF) 2735 sc->corrected_error_pcie_retry_count++; 2736 sc->corrected_error_pcie_other_count += bitcount32(status & 2737 E1000_PCIEECCSTS_I350_I354_OTHER_MASK); 2738 if (status != 0) 2739 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, status); 2740 } 2741 2742 /* 2743 * Internal-memory error causes are read-clear. Capture them before handing 2744 * fatal recovery or non-fatal acknowledgement to the iflib admin task. 2745 */ 2746 static void 2747 em_handle_fatal_error_intr(struct e1000_softc *sc, u32 icr) 2748 { 2749 struct e1000_hw *hw; 2750 u32 dma_host, dma_rx, dma_tx, error_mask, lanerr, pcieerr, peind; 2751 2752 error_mask = em_memory_error_intr_mask(&sc->hw); 2753 if (!em_has_memory_errors(&sc->hw) || 2754 (icr & error_mask) == 0) 2755 return; 2756 2757 hw = &sc->hw; 2758 E1000_WRITE_REG(hw, E1000_IMC, error_mask); 2759 if (!atomic_cmpset_32(&sc->fatal_error_state, 2760 EM_FATAL_ERROR_NONE, EM_FATAL_ERROR_CAPTURING)) 2761 return; 2762 2763 sc->fatal_error_icr = icr & error_mask; 2764 if (em_has_pch_ecc(hw)) { 2765 sc->fatal_error_pbeccsts = 2766 E1000_READ_REG(hw, E1000_PBECCSTS); 2767 } else if (em_has_82575_memory_errors(hw)) { 2768 sc->fatal_error_pbeccsts = 2769 E1000_READ_REG(hw, E1000_PBECCSTS_82575); 2770 sc->fatal_error_dma_rx = 2771 E1000_READ_REG(hw, E1000_RDHESTS_82575); 2772 sc->fatal_error_dma_tx = 2773 E1000_READ_REG(hw, E1000_TDHESTS_82575); 2774 } else if (em_has_82576_memory_errors(hw)) { 2775 sc->fatal_error_peind = E1000_READ_REG(hw, E1000_PEIND); 2776 } else { 2777 peind = E1000_READ_REG(hw, E1000_PEIND) & 2778 E1000_PEIND_FATAL_MASK; 2779 pcieerr = E1000_READ_REG(hw, E1000_PCIEERRSTS) & 2780 em_pcie_fatal_error_mask(hw); 2781 dma_host = 0; 2782 if (em_has_82580_memory_errors(hw)) { 2783 /* 2784 * PEIND is visible through every function. Retain the 2785 * management indication, which has no subordinate status, 2786 * but attribute host-owned regions from this function's 2787 * status registers. 2788 */ 2789 peind &= E1000_PEIND_MNG_PARITY_FATAL; 2790 dma_tx = E1000_READ_REG(hw, E1000_DTPARS_82580); 2791 dma_rx = E1000_READ_REG(hw, E1000_DRPARS_82580); 2792 dma_host = E1000_READ_REG(hw, 2793 E1000_DDPARS_82580); 2794 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2795 E1000_LANPERRSTS_82580_ERROR_MASK; 2796 } else if (em_has_i350_i354_memory_errors(hw)) { 2797 dma_tx = E1000_READ_REG(hw, E1000_DTPARS) & 2798 E1000_DTPARS_FATAL_MASK; 2799 dma_rx = E1000_READ_REG(hw, E1000_DRPARS) & 2800 E1000_DRPARS_FATAL_MASK; 2801 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2802 E1000_LANPERRSTS_I350_I354_FATAL_MASK; 2803 } else { 2804 dma_tx = 0; 2805 dma_rx = 0; 2806 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 2807 E1000_LANPERRSTS_RETX_BUF; 2808 } 2809 if (pcieerr != 0) 2810 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2811 if (lanerr != 0) 2812 peind |= E1000_PEIND_LANPORT_PARITY_FATAL; 2813 if (dma_tx != 0 || dma_rx != 0 || dma_host != 0) 2814 peind |= E1000_PEIND_DMA_PARITY_FATAL; 2815 sc->fatal_error_peind = peind; 2816 sc->fatal_error_pcie = pcieerr; 2817 sc->fatal_error_lan = lanerr; 2818 sc->fatal_error_dma_tx = dma_tx; 2819 sc->fatal_error_dma_rx = dma_rx; 2820 sc->fatal_error_dma_host = dma_host; 2821 } 2822 atomic_store_rel_32(&sc->fatal_error_state, 2823 EM_FATAL_ERROR_DETECTED); 2824 iflib_admin_intr_deferred(sc->ctx); 2825 } 2826 2827 static bool 2828 em_handle_fatal_error_admin(struct e1000_softc *sc) 2829 { 2830 u32 error_mask, pcieecc, peind; 2831 bool reset_required; 2832 2833 if (!atomic_cmpset_acq_32(&sc->fatal_error_state, 2834 EM_FATAL_ERROR_DETECTED, EM_FATAL_ERROR_RESET_REQUESTED)) 2835 return (atomic_load_acq_32(&sc->fatal_error_state) != 2836 EM_FATAL_ERROR_NONE); 2837 2838 if (em_has_pch_ecc(&sc->hw)) { 2839 em_update_pch_ecc_stats(sc, sc->fatal_error_pbeccsts); 2840 device_printf(sc->dev, 2841 "uncorrectable packet-buffer ECC error: " 2842 "PBECCSTS %#x; requesting reset\n", 2843 sc->fatal_error_pbeccsts); 2844 } else if (em_has_82575_memory_errors(&sc->hw)) { 2845 em_update_82575_ecc_stats(sc, sc->fatal_error_pbeccsts, 2846 sc->fatal_error_dma_rx, sc->fatal_error_dma_tx); 2847 device_printf(sc->dev, 2848 "unrecoverable internal memory ECC error: ICR %#x, " 2849 "PBECCSTS %#x, RDHESTS %#x, TDHESTS %#x; " 2850 "requesting reset\n", sc->fatal_error_icr, 2851 sc->fatal_error_pbeccsts, sc->fatal_error_dma_rx, 2852 sc->fatal_error_dma_tx); 2853 } else if (em_has_82576_memory_errors(&sc->hw)) { 2854 peind = sc->fatal_error_peind; 2855 em_update_82576_ecc_stats(sc); 2856 reset_required = 2857 (sc->fatal_error_icr & E1000_ICR_FER) != 0 || 2858 (peind & (E1000_PEIND_82576_FATAL_MASK | 2859 E1000_PEIND_82576_MEMORY_HANG)) != 0; 2860 if (!reset_required) { 2861 device_printf(sc->dev, 2862 "non-fatal internal memory error: PEIND %#x\n", 2863 peind); 2864 sc->fatal_error_icr = 0; 2865 sc->fatal_error_peind = 0; 2866 atomic_store_rel_32(&sc->fatal_error_state, 2867 EM_FATAL_ERROR_NONE); 2868 error_mask = E1000_IMS_FER | E1000_IMS_NFER; 2869 E1000_WRITE_REG(&sc->hw, E1000_IMS, error_mask); 2870 E1000_WRITE_FLUSH(&sc->hw); 2871 return (true); 2872 } 2873 if ((peind & (E1000_PEIND_82576_FATAL_MASK | 2874 E1000_PEIND_82576_MEMORY_HANG)) == 0) 2875 sc->fatal_error_unknown_count++; 2876 device_printf(sc->dev, 2877 "fatal internal memory error: PEIND %#x; " 2878 "requesting reset\n", peind); 2879 } else { 2880 peind = sc->fatal_error_peind; 2881 if (em_has_82580_memory_errors(&sc->hw)) { 2882 pcieecc = E1000_READ_REG(&sc->hw, 2883 E1000_PCIEECCSTS) & 2884 E1000_PCIEECCSTS_82580_ERROR_MASK; 2885 sc->fatal_error_pcie_ecc |= pcieecc; 2886 if (pcieecc != 0) { 2887 peind |= E1000_PEIND_PCIE_PARITY_FATAL; 2888 sc->fatal_error_peind = peind; 2889 } 2890 em_update_82580_ecc_stats(sc, 2891 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 2892 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 2893 pcieecc); 2894 } else if (em_has_i350_i354_memory_errors(&sc->hw)) 2895 em_update_i350_i354_ecc_stats(sc); 2896 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) 2897 sc->fatal_error_lan_count++; 2898 if (peind & E1000_PEIND_MNG_PARITY_FATAL) 2899 sc->fatal_error_mng_count++; 2900 if (peind & E1000_PEIND_PCIE_PARITY_FATAL) 2901 sc->fatal_error_pcie_count++; 2902 if (peind & E1000_PEIND_DMA_PARITY_FATAL) 2903 sc->fatal_error_dma_count++; 2904 if (peind == 0) 2905 sc->fatal_error_unknown_count++; 2906 if (em_has_82580_memory_errors(&sc->hw)) { 2907 device_printf(sc->dev, 2908 "fatal internal memory error: PEIND %#x, " 2909 "PCIEERRSTS %#x, PCIEECCSTS %#x, " 2910 "DTPARS %#x, DRPARS %#x, DDPARS %#x, " 2911 "LANPERRSTS %#x\n", peind, 2912 sc->fatal_error_pcie, 2913 sc->fatal_error_pcie_ecc, 2914 sc->fatal_error_dma_tx, 2915 sc->fatal_error_dma_rx, 2916 sc->fatal_error_dma_host, 2917 sc->fatal_error_lan); 2918 } else { 2919 device_printf(sc->dev, 2920 "fatal internal memory error: PEIND %#x, " 2921 "PCIEERRSTS %#x, DTPARS %#x, DRPARS %#x, " 2922 "LANPERRSTS %#x\n", peind, 2923 sc->fatal_error_pcie, 2924 sc->fatal_error_dma_tx, 2925 sc->fatal_error_dma_rx, 2926 sc->fatal_error_lan); 2927 } 2928 2929 reset_required = (peind & 2930 (E1000_PEIND_PCIE_PARITY_FATAL | 2931 E1000_PEIND_DMA_PARITY_FATAL)) != 0; 2932 if (peind == 0) 2933 reset_required = true; 2934 if (peind & E1000_PEIND_LANPORT_PARITY_FATAL) { 2935 if (!em_has_i350_i354_memory_errors(&sc->hw) || 2936 sc->fatal_error_lan == 0 || 2937 (sc->fatal_error_lan & 2938 E1000_LANPERRSTS_I350_I354_RESET_MASK) != 0) 2939 reset_required = true; 2940 } 2941 /* Management-memory recovery belongs to management firmware. */ 2942 if (!reset_required) { 2943 if (em_has_i350_i354_memory_errors(&sc->hw) && 2944 sc->fatal_error_lan != 0) 2945 E1000_WRITE_REG(&sc->hw, E1000_LANPERRSTS, 2946 sc->fatal_error_lan & 2947 E1000_LANPERRSTS_I350_I354_NO_RESET_MASK); 2948 sc->fatal_error_peind = 0; 2949 sc->fatal_error_pcie = 0; 2950 sc->fatal_error_pcie_ecc = 0; 2951 sc->fatal_error_lan = 0; 2952 sc->fatal_error_dma_tx = 0; 2953 sc->fatal_error_dma_rx = 0; 2954 sc->fatal_error_dma_host = 0; 2955 atomic_store_rel_32(&sc->fatal_error_state, 2956 EM_FATAL_ERROR_NONE); 2957 E1000_WRITE_REG(&sc->hw, E1000_IMS, 2958 E1000_IMS_FER); 2959 E1000_WRITE_FLUSH(&sc->hw); 2960 return (true); 2961 } 2962 device_printf(sc->dev, 2963 "requesting reset after memory error\n"); 2964 } 2965 sc->fatal_error_reset_count++; 2966 iflib_request_reset(sc->ctx); 2967 /* Re-enter the admin task so it observes the reset request. */ 2968 iflib_admin_intr_deferred(sc->ctx); 2969 return (true); 2970 } 2971 2972 /* 2973 * ICR bit 30 is reserved on 82575 and is the TCP timer on 82576. It becomes 2974 * the Device Reset Asserted interrupt starting with 82580. 2975 */ 2976 static u32 2977 igb_device_reset_intr_mask(struct e1000_softc *sc) 2978 { 2979 2980 return (sc->hw.mac.type >= e1000_82580 ? E1000_IMS_DRSTA : 0); 2981 } 2982 2983 /* Keep interrupt-side work quiesced until device-reset recovery completes. */ 2984 static bool 2985 igb_device_reset_pending(struct e1000_softc *sc) 2986 { 2987 2988 return (!sc->vf_ifp && igb_device_reset_intr_mask(sc) != 0 && 2989 atomic_load_acq_32(&sc->device_reset_state) != 2990 IGB_DEVICE_RESET_NONE); 2991 } 2992 2993 /* 2994 * CTRL.DEV_RST resets every port in the device. ICR.DRSTA tells the other 2995 * ports that their registers and descriptor rings must be reinitialized. 2996 */ 2997 static bool 2998 igb_handle_device_reset(struct e1000_softc *sc, u32 icr) 2999 { 3000 u32 state; 3001 3002 if (sc->vf_ifp || igb_device_reset_intr_mask(sc) == 0 || 3003 (icr & E1000_ICR_DRSTA) == 0) 3004 return (false); 3005 state = atomic_swap_32(&sc->device_reset_state, 3006 IGB_DEVICE_RESET_DETECTED); 3007 if (state == IGB_DEVICE_RESET_DETECTED) 3008 return (true); 3009 3010 iflib_admin_intr_deferred(sc->ctx); 3011 return (true); 3012 } 3013 3014 /* 3015 * A device reset can leave a sibling port accessible before its internal 3016 * reset and PCIe transactions have completed. For 82580 and newer parts, 3017 * wait for that device-wide reset to finish and acknowledge it before any 3018 * ordinary port register programming. I350 and newer parts also publish 3019 * explicit EEPROM autoload and PF-reset completion indications. 3020 * 3021 * The wait is bounded because the only useful fallback for a controller 3022 * that never completes the device reset is the port reset already requested 3023 * by the interrupt handler. 3024 */ 3025 static void 3026 igb_prepare_device_reset(struct e1000_softc *sc) 3027 { 3028 struct e1000_hw *hw; 3029 u32 state; 3030 u32 eecd, gcr, status; 3031 int i; 3032 3033 hw = &sc->hw; 3034 state = atomic_load_acq_32(&sc->device_reset_state); 3035 if (state != IGB_DEVICE_RESET_DETECTED && 3036 state != IGB_DEVICE_RESET_REQUESTED && 3037 hw->mac.type >= e1000_82580) { 3038 /* 3039 * A reset can start while this interface has interrupts disabled. 3040 * GCR is the documented gate before ordinary port accesses. STATUS 3041 * also detects a reset that completed while this interface was down 3042 * or after an earlier preparation pass. 3043 */ 3044 gcr = E1000_READ_REG(hw, E1000_GCR); 3045 if (gcr != 0xffffffff && 3046 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) { 3047 atomic_store_rel_32(&sc->device_reset_state, 3048 IGB_DEVICE_RESET_DETECTED); 3049 state = IGB_DEVICE_RESET_DETECTED; 3050 } else if (gcr != 0xffffffff) { 3051 status = E1000_READ_REG(hw, E1000_STATUS); 3052 if (status != 0xffffffff && 3053 (status & E1000_STAT_DEV_RST_SET) != 0) { 3054 atomic_store_rel_32(&sc->device_reset_state, 3055 IGB_DEVICE_RESET_DETECTED); 3056 state = IGB_DEVICE_RESET_DETECTED; 3057 } 3058 } 3059 } 3060 if (state != IGB_DEVICE_RESET_DETECTED && 3061 state != IGB_DEVICE_RESET_REQUESTED) 3062 return; 3063 3064 if (hw->mac.type >= e1000_82580) { 3065 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3066 gcr = E1000_READ_REG(hw, E1000_GCR); 3067 if (gcr != 0xffffffff && 3068 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) == 0) 3069 break; 3070 msec_delay(1); 3071 } 3072 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) { 3073 device_printf(sc->dev, 3074 "device-wide reset did not complete; " 3075 "attempting port reset\n"); 3076 goto prepared; 3077 } 3078 3079 /* STATUS.DEV_RST_SET is write-one-to-clear. */ 3080 E1000_WRITE_REG(hw, E1000_STATUS, E1000_STAT_DEV_RST_SET); 3081 3082 if (hw->mac.type >= e1000_i350) { 3083 for (i = 0; i < IGB_DEVICE_RESET_TIMEOUT_MS; i++) { 3084 eecd = E1000_READ_REG(hw, E1000_EECD); 3085 status = E1000_READ_REG(hw, E1000_STATUS); 3086 if (eecd != 0xffffffff && status != 0xffffffff && 3087 (eecd & E1000_EECD_AUTO_RD) != 0 && 3088 (status & E1000_STATUS_RST_DONE) != 0) 3089 break; 3090 msec_delay(1); 3091 } 3092 if (i == IGB_DEVICE_RESET_TIMEOUT_MS) 3093 device_printf(sc->dev, 3094 "device-wide reset did not finish EEPROM " 3095 "autoload or port reset; attempting port " 3096 "reset\n"); 3097 } 3098 } 3099 3100 prepared: 3101 atomic_store_rel_32(&sc->device_reset_state, 3102 IGB_DEVICE_RESET_PREPARED); 3103 } 3104 3105 /* 3106 * A second device reset can arrive while the port is being initialized. 3107 * Leave its status latched for the next preparation pass and do not let 3108 * iflib publish this incomplete initialization as a running datapath. 3109 */ 3110 static bool 3111 igb_finish_device_reset(struct e1000_softc *sc, u32 icr) 3112 { 3113 bool reset_again; 3114 u32 gcr, state, status; 3115 3116 if (igb_device_reset_intr_mask(sc) == 0) 3117 return (false); 3118 3119 state = atomic_load_acq_32(&sc->device_reset_state); 3120 reset_again = icr != 0xffffffff && 3121 (icr & E1000_ICR_DRSTA) != 0; 3122 if (sc->hw.mac.type >= e1000_82580) { 3123 gcr = E1000_READ_REG(&sc->hw, E1000_GCR); 3124 if (gcr != 0xffffffff && 3125 (gcr & E1000_GCR_DEV_RST_IN_PROGRESS) != 0) 3126 reset_again = true; 3127 status = E1000_READ_REG(&sc->hw, E1000_STATUS); 3128 if (status == 0xffffffff && 3129 state != IGB_DEVICE_RESET_NONE) { 3130 /* 3131 * MMIO can disappear briefly while SR-IOV is changing, but 3132 * config space remains readable. If both are gone, retain the 3133 * stopped state without queueing an endless reset loop. 3134 */ 3135 if (pci_read_config(sc->dev, PCIR_VENDOR, 2) == 0xffff) { 3136 atomic_store_rel_32(&sc->device_reset_state, 3137 IGB_DEVICE_RESET_DETECTED); 3138 device_printf(sc->dev, 3139 "device unavailable after device-wide reset; " 3140 "leaving interface stopped\n"); 3141 return (true); 3142 } 3143 reset_again = true; 3144 } else if (status != 0xffffffff && 3145 (status & E1000_STAT_DEV_RST_SET) != 0) 3146 reset_again = true; 3147 } 3148 if (state == IGB_DEVICE_RESET_DETECTED || 3149 state == IGB_DEVICE_RESET_REQUESTED) 3150 reset_again = true; 3151 if (!reset_again) { 3152 if (state == IGB_DEVICE_RESET_PREPARED && 3153 !atomic_cmpset_rel_32(&sc->device_reset_state, 3154 IGB_DEVICE_RESET_PREPARED, IGB_DEVICE_RESET_NONE)) 3155 return (true); 3156 return (false); 3157 } 3158 3159 state = atomic_swap_32(&sc->device_reset_state, 3160 IGB_DEVICE_RESET_DETECTED); 3161 if (state != IGB_DEVICE_RESET_DETECTED) { 3162 iflib_request_reset_if_up(sc->ctx); 3163 iflib_admin_intr_deferred(sc->ctx); 3164 } 3165 return (true); 3166 } 3167 3168 /* 3169 * A PCIe-region parity failure stops PCIe and DMA traffic. I350, I354, 3170 * I210, and I211 require a port reset before master disable in this case. 3171 * 82580 stops PCIe traffic for a fatal error in any host-owned region, so use 3172 * the same order for every 82580 recovery. This differs from the normal 3173 * reset path, which disables the bus master first. 3174 * 3175 * Indications that relatch after admin accounting are discarded during 3176 * reset; sticky bits cannot distinguish them from the saved event. 3177 */ 3178 static void 3179 em_prepare_fatal_error_reset(struct e1000_softc *sc) 3180 { 3181 struct e1000_hw *hw; 3182 s32 error; 3183 u32 ctrl, pcieecc, pcieerr; 3184 int i; 3185 3186 if (!em_has_peind_memory_errors(&sc->hw) || 3187 atomic_load_acq_32(&sc->fatal_error_state) != 3188 EM_FATAL_ERROR_RESET_REQUESTED) 3189 return; 3190 3191 pcieerr = sc->fatal_error_pcie | 3192 (E1000_READ_REG(&sc->hw, E1000_PCIEERRSTS) & 3193 em_pcie_fatal_error_mask(&sc->hw)); 3194 pcieecc = sc->fatal_error_pcie_ecc; 3195 if (!em_has_82580_memory_errors(&sc->hw) && 3196 (sc->fatal_error_peind & E1000_PEIND_PCIE_PARITY_FATAL) == 0 && 3197 pcieerr == 0) 3198 return; 3199 3200 hw = &sc->hw; 3201 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3202 E1000_WRITE_REG(hw, E1000_CTRL, ctrl | E1000_CTRL_RST); 3203 /* Do not access device registers for at least 3 ms after RST. */ 3204 msec_delay(3); 3205 for (i = 0; i < AUTO_READ_DONE_TIMEOUT; i++) { 3206 if ((E1000_READ_REG(hw, E1000_EECD) & 3207 E1000_EECD_AUTO_RD) != 0 && 3208 (em_has_82580_memory_errors(hw) || 3209 (E1000_READ_REG(hw, E1000_STATUS) & 3210 E1000_STATUS_RST_DONE) != 0)) 3211 break; 3212 msec_delay(1); 3213 } 3214 if (i == AUTO_READ_DONE_TIMEOUT) 3215 device_printf(sc->dev, 3216 "port reset did not complete during parity recovery\n"); 3217 error = e1000_disable_pcie_master_generic(hw); 3218 if (error != E1000_SUCCESS) 3219 device_printf(sc->dev, 3220 "PCIe master disable failed during parity recovery: %d\n", 3221 error); 3222 pcieerr |= E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3223 em_pcie_fatal_error_mask(hw); 3224 if (pcieerr != 0) 3225 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3226 if (em_has_82580_memory_errors(hw)) { 3227 pcieecc |= E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3228 E1000_PCIEECCSTS_82580_ERROR_MASK; 3229 if (pcieecc != 0) 3230 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3231 } 3232 atomic_store_rel_32(&sc->fatal_error_state, 3233 EM_FATAL_ERROR_RESET_PREPARED); 3234 } 3235 3236 static void 3237 em_finish_fatal_error_reset(struct e1000_softc *sc) 3238 { 3239 struct e1000_hw *hw; 3240 u32 dma_host, dma_rx, dma_tx, lanerr, pcieecc, pcieerr; 3241 u32 state; 3242 3243 state = atomic_load_acq_32(&sc->fatal_error_state); 3244 if (state != EM_FATAL_ERROR_RESET_REQUESTED && 3245 state != EM_FATAL_ERROR_RESET_PREPARED) 3246 return; 3247 3248 hw = &sc->hw; 3249 if (em_has_82575_memory_errors(hw)) { 3250 sc->fatal_error_dma_tx = 0; 3251 sc->fatal_error_dma_rx = 0; 3252 } else if (em_has_82576_memory_errors(hw)) { 3253 /* Drain any indication relatched while the port was resetting. */ 3254 (void)E1000_READ_REG(hw, E1000_PEIND); 3255 sc->fatal_error_peind = 0; 3256 } else if (em_has_82580_memory_errors(hw)) { 3257 pcieerr = sc->fatal_error_pcie | 3258 E1000_READ_REG(hw, E1000_PCIEERRSTS); 3259 if (pcieerr != 0) 3260 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3261 pcieecc = sc->fatal_error_pcie_ecc | 3262 (E1000_READ_REG(hw, E1000_PCIEECCSTS) & 3263 E1000_PCIEECCSTS_82580_ERROR_MASK); 3264 if (pcieecc != 0) 3265 E1000_WRITE_REG(hw, E1000_PCIEECCSTS, pcieecc); 3266 dma_tx = sc->fatal_error_dma_tx | 3267 E1000_READ_REG(hw, E1000_DTPARS_82580); 3268 if (dma_tx != 0) 3269 E1000_WRITE_REG(hw, E1000_DTPARS_82580, dma_tx); 3270 dma_rx = sc->fatal_error_dma_rx | 3271 E1000_READ_REG(hw, E1000_DRPARS_82580); 3272 if (dma_rx != 0) 3273 E1000_WRITE_REG(hw, E1000_DRPARS_82580, dma_rx); 3274 dma_host = sc->fatal_error_dma_host | 3275 E1000_READ_REG(hw, E1000_DDPARS_82580); 3276 if (dma_host != 0) 3277 E1000_WRITE_REG(hw, E1000_DDPARS_82580, dma_host); 3278 /* LANPERRSTS is read-only and is cleared by the port reset. */ 3279 lanerr = E1000_READ_REG(hw, E1000_LANPERRSTS) & 3280 E1000_LANPERRSTS_82580_ERROR_MASK; 3281 if (lanerr != 0) 3282 device_printf(sc->dev, 3283 "LAN parity status remained set after reset: %#x\n", 3284 lanerr); 3285 (void)E1000_READ_REG(hw, E1000_PEIND); 3286 sc->fatal_error_peind = 0; 3287 sc->fatal_error_pcie = 0; 3288 sc->fatal_error_pcie_ecc = 0; 3289 sc->fatal_error_lan = 0; 3290 sc->fatal_error_dma_tx = 0; 3291 sc->fatal_error_dma_rx = 0; 3292 sc->fatal_error_dma_host = 0; 3293 } else if (em_has_peind_memory_errors(hw)) { 3294 pcieerr = sc->fatal_error_pcie | 3295 (E1000_READ_REG(hw, E1000_PCIEERRSTS) & 3296 em_pcie_fatal_error_mask(hw)); 3297 if (pcieerr != 0) 3298 E1000_WRITE_REG(hw, E1000_PCIEERRSTS, pcieerr); 3299 if (em_has_i350_i354_memory_errors(hw)) { 3300 dma_tx = sc->fatal_error_dma_tx | 3301 (E1000_READ_REG(hw, E1000_DTPARS) & 3302 E1000_DTPARS_FATAL_MASK); 3303 if (dma_tx != 0) 3304 E1000_WRITE_REG(hw, E1000_DTPARS, dma_tx); 3305 dma_rx = sc->fatal_error_dma_rx | 3306 (E1000_READ_REG(hw, E1000_DRPARS) & 3307 E1000_DRPARS_FATAL_MASK); 3308 if (dma_rx != 0) 3309 E1000_WRITE_REG(hw, E1000_DRPARS, dma_rx); 3310 lanerr = sc->fatal_error_lan | 3311 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3312 E1000_LANPERRSTS_I350_I354_FATAL_MASK); 3313 } else { 3314 lanerr = sc->fatal_error_lan | 3315 (E1000_READ_REG(hw, E1000_LANPERRSTS) & 3316 E1000_LANPERRSTS_RETX_BUF); 3317 } 3318 if (lanerr != 0) 3319 E1000_WRITE_REG(hw, E1000_LANPERRSTS, lanerr); 3320 /* 3321 * RST can relatch PEIND from a subordinate status register 3322 * before that register is cleared. Drain the recovered 3323 * indication before unmasking FER. 3324 */ 3325 (void)E1000_READ_REG(hw, E1000_PEIND); 3326 sc->fatal_error_peind = 0; 3327 sc->fatal_error_pcie = 0; 3328 sc->fatal_error_pcie_ecc = 0; 3329 sc->fatal_error_lan = 0; 3330 sc->fatal_error_dma_tx = 0; 3331 sc->fatal_error_dma_rx = 0; 3332 sc->fatal_error_dma_host = 0; 3333 } 3334 sc->fatal_error_icr = 0; 3335 sc->fatal_error_pbeccsts = 0; 3336 atomic_store_rel_32(&sc->fatal_error_state, EM_FATAL_ERROR_NONE); 3337 } 3338 3339 /********************************************************************* 3340 * 3341 * Fast Legacy/MSI Combined Interrupt Service routine 3342 * 3343 *********************************************************************/ 3344 int 3345 em_intr(void *arg) 3346 { 3347 struct e1000_softc *sc = arg; 3348 struct e1000_hw *hw = &sc->hw; 3349 struct em_rx_queue *que = &sc->rx_queues[0]; 3350 struct rx_ring *rxr = &que->rxr; 3351 if_ctx_t ctx = sc->ctx; 3352 u32 reg_icr; 3353 3354 reg_icr = E1000_READ_REG(hw, E1000_ICR); 3355 3356 /* Hot eject? */ 3357 if (reg_icr == 0xffffffff) 3358 return FILTER_STRAY; 3359 3360 /* Definitely not our interrupt. */ 3361 if (reg_icr == 0x0) 3362 return FILTER_STRAY; 3363 3364 /* 3365 * Starting with the 82571 chip, bit 31 should be used to 3366 * determine whether the interrupt belongs to us. 3367 */ 3368 if (hw->mac.type >= e1000_82571 && 3369 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 3370 return FILTER_STRAY; 3371 if (igb_handle_device_reset(sc, reg_icr)) 3372 return (FILTER_HANDLED); 3373 if (igb_device_reset_pending(sc)) 3374 return (FILTER_HANDLED); 3375 3376 /* 3377 * IAM auto-masks igb shared interrupts when ICR is read. Older em 3378 * hardware still needs an explicit disable, which also works around 3379 * MSI message reordering errata on certain systems. 3380 */ 3381 if (sc->vf_ifp || hw->mac.type < igb_mac_min) 3382 IFDI_INTR_DISABLE(ctx); 3383 3384 /* Link status change */ 3385 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3386 em_handle_link(ctx); 3387 3388 if (reg_icr & E1000_ICR_RXO) 3389 sc->rx_overruns++; 3390 3391 em_handle_fatal_error_intr(sc, reg_icr); 3392 3393 if (hw->mac.type >= e1000_82540) 3394 em_newitr(sc, que, rxr); 3395 3396 return (FILTER_SCHEDULE_THREAD); 3397 } 3398 3399 static int 3400 em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3401 { 3402 struct e1000_softc *sc = iflib_get_softc(ctx); 3403 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3404 3405 E1000_WRITE_REG(&sc->hw, E1000_IMS, rxq->eims); 3406 return (0); 3407 } 3408 3409 static int 3410 em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3411 { 3412 struct e1000_softc *sc = iflib_get_softc(ctx); 3413 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3414 3415 E1000_WRITE_REG(&sc->hw, E1000_IMS, txq->eims); 3416 return (0); 3417 } 3418 3419 static int 3420 igb_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 3421 { 3422 struct e1000_softc *sc = iflib_get_softc(ctx); 3423 struct em_rx_queue *rxq = &sc->rx_queues[rxqid]; 3424 3425 if (igb_device_reset_pending(sc)) 3426 return (0); 3427 E1000_WRITE_REG(&sc->hw, E1000_EIMS, rxq->eims); 3428 return (0); 3429 } 3430 3431 static int 3432 igb_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 3433 { 3434 struct e1000_softc *sc = iflib_get_softc(ctx); 3435 struct em_tx_queue *txq = &sc->tx_queues[txqid]; 3436 3437 if (igb_device_reset_pending(sc)) 3438 return (0); 3439 E1000_WRITE_REG(&sc->hw, E1000_EIMS, txq->eims); 3440 return (0); 3441 } 3442 3443 /********************************************************************* 3444 * 3445 * MSI-X RX Interrupt Service routine 3446 * 3447 **********************************************************************/ 3448 static int 3449 em_msix_que(void *arg) 3450 { 3451 struct em_rx_queue *que = arg; 3452 struct e1000_softc *sc = que->sc; 3453 struct rx_ring *rxr = &que->rxr; 3454 3455 ++que->irqs; 3456 3457 if (igb_device_reset_pending(sc)) 3458 return (FILTER_HANDLED); 3459 em_newitr(sc, que, rxr); 3460 3461 return (FILTER_SCHEDULE_THREAD); 3462 } 3463 3464 /********************************************************************* 3465 * 3466 * MSI-X Link Fast Interrupt Service routine 3467 * 3468 **********************************************************************/ 3469 static int 3470 em_msix_link(void *arg) 3471 { 3472 struct e1000_softc *sc = arg; 3473 u32 reg_icr; 3474 3475 ++sc->link_irq; 3476 MPASS(sc->hw.back != NULL); 3477 /* 3478 * The VF's admin vector represents mailbox and link activity. It has 3479 * no PF ICR at E1000_ICR, so process every admin-vector interrupt, 3480 * matching the igbvf misc-vector model. 3481 */ 3482 if (sc->vf_ifp) { 3483 sc->hw.mac.get_link_status = true; 3484 iflib_admin_intr_deferred(sc->ctx); 3485 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3486 return (FILTER_HANDLED); 3487 } 3488 3489 reg_icr = E1000_READ_REG(&sc->hw, E1000_ICR); 3490 if (igb_device_reset_pending(sc)) 3491 return (FILTER_HANDLED); 3492 3493 /* 3494 * Enabling or disabling SR-IOV can briefly make PF MMIO reads return 3495 * all ones. This is not an interrupt cause; in particular, do not 3496 * turn it into a malicious-driver event. 3497 */ 3498 if (__predict_false(reg_icr == 0xffffffff)) 3499 goto rearm; 3500 if (igb_handle_device_reset(sc, reg_icr)) 3501 return (FILTER_HANDLED); 3502 3503 if (reg_icr & E1000_ICR_RXO) 3504 sc->rx_overruns++; 3505 3506 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 3507 em_handle_link(sc->ctx); 3508 if (reg_icr & E1000_ICR_MDDET) 3509 igb_iov_mdd_event(sc); 3510 if (reg_icr & E1000_ICR_VMMB) 3511 iflib_admin_intr_deferred(sc->ctx); 3512 em_handle_fatal_error_intr(sc, reg_icr); 3513 3514 rearm: 3515 /* Re-arm unconditionally */ 3516 if (sc->hw.mac.type >= igb_mac_min) { 3517 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3518 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 3519 igb_iov_intr_mask(sc) | 3520 em_fatal_error_intr_mask(sc)); 3521 E1000_WRITE_REG(&sc->hw, E1000_EIMS, sc->link_mask); 3522 } else if (sc->hw.mac.type == e1000_82574) { 3523 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3524 E1000_IMS_LSC | E1000_IMS_OTHER); 3525 /* 3526 * Because we must read the ICR for this interrupt it may 3527 * clear other causes using autoclear, for this reason we 3528 * simply create a soft interrupt for all these vectors. 3529 */ 3530 if (reg_icr) 3531 E1000_WRITE_REG(&sc->hw, E1000_ICS, sc->ims); 3532 } else 3533 E1000_WRITE_REG(&sc->hw, E1000_IMS, 3534 E1000_IMS_LSC | em_fatal_error_intr_mask(sc)); 3535 3536 return (FILTER_HANDLED); 3537 } 3538 3539 static void 3540 em_handle_link(void *context) 3541 { 3542 if_ctx_t ctx = context; 3543 struct e1000_softc *sc = iflib_get_softc(ctx); 3544 3545 sc->hw.mac.get_link_status = 1; 3546 iflib_admin_intr_deferred(ctx); 3547 } 3548 3549 /********************************************************************* 3550 * 3551 * Media Ioctl callback 3552 * 3553 * This routine is called whenever the user queries the status of 3554 * the interface using ifconfig. 3555 * 3556 **********************************************************************/ 3557 static void 3558 em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 3559 { 3560 struct e1000_softc *sc = iflib_get_softc(ctx); 3561 u_char fiber_type = IFM_1000_SX; 3562 3563 INIT_DEBUGOUT("em_if_media_status: begin"); 3564 3565 iflib_admin_intr_deferred(ctx); 3566 3567 ifmr->ifm_status = IFM_AVALID; 3568 ifmr->ifm_active = IFM_ETHER; 3569 3570 if (sc->link_state == EM_LINK_STATE_DOWN || 3571 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING) { 3572 return; 3573 } 3574 3575 ifmr->ifm_status |= IFM_ACTIVE; 3576 3577 if ((sc->hw.phy.media_type == e1000_media_type_fiber) || 3578 (sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3579 if (sc->hw.mac.type == e1000_82545) 3580 fiber_type = IFM_1000_LX; 3581 ifmr->ifm_active |= fiber_type | IFM_FDX; 3582 } else { 3583 switch (sc->link_speed) { 3584 case 10: 3585 ifmr->ifm_active |= IFM_10_T; 3586 break; 3587 case 100: 3588 ifmr->ifm_active |= IFM_100_TX; 3589 break; 3590 case 1000: 3591 ifmr->ifm_active |= IFM_1000_T; 3592 break; 3593 } 3594 if (sc->link_duplex == FULL_DUPLEX) 3595 ifmr->ifm_active |= IFM_FDX; 3596 else 3597 ifmr->ifm_active |= IFM_HDX; 3598 } 3599 } 3600 3601 /********************************************************************* 3602 * 3603 * Media Ioctl callback 3604 * 3605 * This routine is called when the user changes speed/duplex using 3606 * media/mediopt option with ifconfig. 3607 * 3608 **********************************************************************/ 3609 static int 3610 em_if_media_change(if_ctx_t ctx) 3611 { 3612 struct e1000_softc *sc = iflib_get_softc(ctx); 3613 struct ifmedia *ifm = iflib_get_media(ctx); 3614 3615 INIT_DEBUGOUT("em_if_media_change: begin"); 3616 3617 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3618 return (EINVAL); 3619 3620 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3621 case IFM_AUTO: 3622 sc->hw.mac.autoneg = DO_AUTO_NEG; 3623 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 3624 break; 3625 case IFM_1000_LX: 3626 case IFM_1000_SX: 3627 case IFM_1000_T: 3628 sc->hw.mac.autoneg = DO_AUTO_NEG; 3629 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 3630 break; 3631 case IFM_100_TX: 3632 sc->hw.mac.autoneg = false; 3633 sc->hw.phy.autoneg_advertised = 0; 3634 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3635 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 3636 else 3637 sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 3638 break; 3639 case IFM_10_T: 3640 sc->hw.mac.autoneg = false; 3641 sc->hw.phy.autoneg_advertised = 0; 3642 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 3643 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 3644 else 3645 sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 3646 break; 3647 default: 3648 device_printf(sc->dev, "Unsupported media type\n"); 3649 } 3650 3651 return (0); 3652 } 3653 3654 static int 3655 em_if_set_promisc(if_ctx_t ctx, int flags) 3656 { 3657 struct e1000_softc *sc; 3658 3659 sc = iflib_get_softc(ctx); 3660 if (em_if_defer_promisc(sc)) 3661 return (0); 3662 return (em_if_set_promisc_impl(ctx, flags)); 3663 } 3664 3665 static bool 3666 em_if_defer_promisc(struct e1000_softc *sc) 3667 { 3668 switch (sc->hw.mac.type) { 3669 case e1000_82576: 3670 case e1000_i350: 3671 case e1000_vfadapt: 3672 case e1000_vfadapt_i350: 3673 break; 3674 default: 3675 return (false); 3676 } 3677 3678 /* 3679 * iflib drops its context lock around IFDI_PROMISC_SET. Run mailbox 3680 * and IOV register operations later from the locked admin task. 3681 * A deferred VF mailbox rejection cannot be returned to ifconfig; the 3682 * admin task logs it instead. 3683 */ 3684 atomic_set_32(&sc->promisc_pending, 1); 3685 iflib_admin_intr_deferred(sc->ctx); 3686 return (true); 3687 } 3688 3689 int 3690 em_if_set_promisc_impl(if_ctx_t ctx, int flags) 3691 { 3692 struct e1000_softc *sc = iflib_get_softc(ctx); 3693 if_t ifp = iflib_get_ifp(ctx); 3694 enum e1000_promisc_type type; 3695 s32 error; 3696 u32 reg_rctl; 3697 int mcnt = 0; 3698 3699 if (sc->vf_ifp) { 3700 if (flags & IFF_PROMISC) 3701 type = e1000_promisc_enabled; 3702 else if (flags & IFF_ALLMULTI) 3703 type = e1000_promisc_multicast; 3704 else 3705 type = e1000_promisc_disabled; 3706 error = e1000_promisc_set_vf(&sc->hw, type); 3707 if (error != E1000_SUCCESS) { 3708 device_printf(sc->dev, 3709 "VF promiscuous-mode request failed\n"); 3710 return (EPERM); 3711 } 3712 return (0); 3713 } 3714 3715 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3716 reg_rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_UPE); 3717 if (flags & IFF_ALLMULTI) 3718 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 3719 else 3720 mcnt = min(if_llmaddr_count(ifp), 3721 MAX_NUM_MULTICAST_ADDRESSES); 3722 3723 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 3724 reg_rctl &= (~E1000_RCTL_MPE); 3725 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3726 3727 if (flags & IFF_PROMISC) { 3728 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3729 /* Turn this on if you want to see bad packets */ 3730 if (em_debug_sbp) 3731 reg_rctl |= E1000_RCTL_SBP; 3732 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3733 if (igb_iov_enabled(sc)) 3734 em_if_vlan_filter_enable(sc); 3735 else 3736 em_if_vlan_filter_disable(sc); 3737 } else { 3738 if (flags & IFF_ALLMULTI) { 3739 reg_rctl |= E1000_RCTL_MPE; 3740 reg_rctl &= ~E1000_RCTL_UPE; 3741 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3742 } 3743 if (igb_iov_enabled(sc) || em_if_vlan_filter_used(ctx)) 3744 em_if_vlan_filter_enable(sc); 3745 } 3746 igb_iov_update_pf_vmolr(sc); 3747 igb_iov_rebuild_vlan(sc); 3748 return (0); 3749 } 3750 3751 static u_int 3752 em_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 3753 { 3754 u8 *mta = arg; 3755 3756 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 3757 return (0); 3758 3759 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3760 3761 return (1); 3762 } 3763 3764 /* Make every multicast hash eligible on parts whose wake matcher needs MTA. */ 3765 static void 3766 em_fill_wakeup_mta(struct e1000_hw *hw) 3767 { 3768 int i; 3769 3770 memset(hw->mac.mta_shadow, 0xff, sizeof(hw->mac.mta_shadow)); 3771 for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) 3772 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, 3773 hw->mac.mta_shadow[i]); 3774 E1000_WRITE_FLUSH(hw); 3775 } 3776 3777 /********************************************************************* 3778 * Multicast Update 3779 * 3780 * This routine is called whenever multicast address list is updated. 3781 * 3782 **********************************************************************/ 3783 static void 3784 em_if_multi_set(if_ctx_t ctx) 3785 { 3786 struct e1000_softc *sc = iflib_get_softc(ctx); 3787 if_t ifp = iflib_get_ifp(ctx); 3788 u8 *mta; /* Multicast array memory */ 3789 u32 reg_rctl = 0; 3790 int mcnt = 0; 3791 3792 IOCTL_DEBUGOUT("em_set_multi: begin"); 3793 3794 mta = sc->mta; 3795 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 3796 3797 if (sc->hw.mac.type == e1000_82542 && 3798 sc->hw.revision_id == E1000_REVISION_2) { 3799 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3800 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3801 e1000_pci_clear_mwi(&sc->hw); 3802 reg_rctl |= E1000_RCTL_RST; 3803 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3804 msec_delay(5); 3805 } 3806 3807 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, mta); 3808 3809 if (sc->vf_ifp) { 3810 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3811 igbv_update_uc_addr_list(sc, ifp); 3812 return; 3813 } 3814 3815 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES && 3816 !igb_iov_enabled(sc)) 3817 e1000_update_mc_addr_list(&sc->hw, mta, mcnt); 3818 3819 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3820 3821 if (if_getflags(ifp) & IFF_PROMISC) 3822 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3823 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 3824 if_getflags(ifp) & IFF_ALLMULTI) { 3825 reg_rctl |= E1000_RCTL_MPE; 3826 reg_rctl &= ~E1000_RCTL_UPE; 3827 } else 3828 reg_rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3829 3830 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3831 3832 if (sc->hw.mac.type == e1000_82542 && 3833 sc->hw.revision_id == E1000_REVISION_2) { 3834 reg_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 3835 reg_rctl &= ~E1000_RCTL_RST; 3836 E1000_WRITE_REG(&sc->hw, E1000_RCTL, reg_rctl); 3837 msec_delay(5); 3838 if (sc->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 3839 e1000_pci_set_mwi(&sc->hw); 3840 } 3841 igb_iov_rebuild_mta(sc); 3842 igb_iov_update_pf_vmolr(sc); 3843 } 3844 3845 /********************************************************************* 3846 * Timer routine 3847 * 3848 * This routine schedules em_if_update_admin_status() to check for 3849 * link status and to gather statistics as well as to perform some 3850 * controller-specific hardware patting. 3851 * 3852 **********************************************************************/ 3853 static void 3854 em_if_timer(if_ctx_t ctx, uint16_t qid) 3855 { 3856 struct e1000_softc *sc; 3857 3858 if (qid != 0) 3859 return; 3860 3861 sc = iflib_get_softc(ctx); 3862 atomic_set_32(&sc->stats_pending, 1); 3863 iflib_admin_intr_deferred(ctx); 3864 } 3865 3866 static void 3867 em_if_update_admin_status(if_ctx_t ctx) 3868 { 3869 struct e1000_softc *sc = iflib_get_softc(ctx); 3870 struct e1000_hw *hw = &sc->hw; 3871 device_t dev = iflib_get_dev(ctx); 3872 u32 link_check, thstat, ctrl; 3873 bool reset_requested = false; 3874 3875 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 3876 if (em_handle_fatal_error_admin(sc)) 3877 return; 3878 /* A sibling-port reset invalidated the registers and VF mailboxes. */ 3879 if (atomic_cmpset_acq_32(&sc->device_reset_state, 3880 IGB_DEVICE_RESET_DETECTED, IGB_DEVICE_RESET_REQUESTED)) { 3881 if (sc->link_state == EM_LINK_STATE_UP) 3882 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 3883 sc->link_speed = 0; 3884 sc->link_duplex = 0; 3885 sc->link_state = EM_LINK_STATE_DOWN_RESET_PENDING; 3886 /* Request the reset here; interrupt filters cannot take STATE_LOCK. */ 3887 iflib_request_reset_if_up(ctx); 3888 /* Re-enter the admin task so it observes the reset request. */ 3889 iflib_admin_intr_deferred(ctx); 3890 return; 3891 } 3892 if (atomic_load_acq_32(&sc->device_reset_state) != 3893 IGB_DEVICE_RESET_NONE) 3894 return; 3895 3896 if (atomic_readandclear_32(&sc->promisc_pending) != 0) 3897 (void)em_if_set_promisc_impl(ctx, 3898 if_getflags(iflib_get_ifp(ctx))); 3899 igb_iov_handle_mdd(sc); 3900 igb_iov_handle_mbx(sc); 3901 3902 link_check = thstat = ctrl = 0; 3903 /* Get the cached link value or read phy for real */ 3904 switch (hw->phy.media_type) { 3905 case e1000_media_type_copper: 3906 if (hw->mac.get_link_status) { 3907 if (hw->mac.type == e1000_pch_spt) 3908 msec_delay(50); 3909 /* Do the work to read phy */ 3910 e1000_check_for_link(hw); 3911 link_check = !hw->mac.get_link_status; 3912 if (link_check) /* ESB2 fix */ 3913 e1000_cfg_on_link_up(hw); 3914 } else { 3915 link_check = true; 3916 } 3917 break; 3918 case e1000_media_type_fiber: 3919 e1000_check_for_link(hw); 3920 link_check = 3921 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 3922 break; 3923 case e1000_media_type_internal_serdes: 3924 e1000_check_for_link(hw); 3925 link_check = hw->mac.serdes_has_link; 3926 break; 3927 default: 3928 break; 3929 } 3930 3931 /* Check for thermal downshift or shutdown */ 3932 if (hw->mac.type == e1000_i350) { 3933 thstat = E1000_READ_REG(hw, E1000_THSTAT); 3934 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); 3935 } 3936 3937 /* Now check for a transition */ 3938 if (link_check && 3939 (sc->link_state == EM_LINK_STATE_DOWN || 3940 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { 3941 bool reset_pending; 3942 3943 reset_pending = 3944 sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING; 3945 e1000_get_speed_and_duplex(hw, &sc->link_speed, 3946 &sc->link_duplex); 3947 /* Check if we must disable SPEED_MODE bit on PCI-E */ 3948 if ((sc->link_speed != SPEED_1000) && 3949 ((hw->mac.type == e1000_82571) || 3950 (hw->mac.type == e1000_82572))) { 3951 int tarc0; 3952 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 3953 tarc0 &= ~TARC_SPEED_MODE_BIT; 3954 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 3955 } 3956 if (bootverbose) 3957 device_printf(dev, "Link is up %d Mbps %s\n", 3958 sc->link_speed, 3959 ((sc->link_duplex == FULL_DUPLEX) ? 3960 "Full Duplex" : "Half Duplex")); 3961 sc->link_state = EM_LINK_STATE_UP; 3962 sc->smartspeed = 0; 3963 if (hw->mac.type == e1000_i350 && 3964 (ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) == 3965 E1000_CTRL_EXT_LINK_MODE_GMII && 3966 (thstat & E1000_THSTAT_LINK_THROTTLE)) 3967 device_printf(dev, "Link: thermal downshift\n"); 3968 /* Delay Link Up for Phy update */ 3969 if (((hw->mac.type == e1000_i210) || 3970 (hw->mac.type == e1000_i211)) && 3971 (hw->phy.id == I210_I_PHY_ID)) 3972 msec_delay(I210_LINK_DELAY); 3973 /* Reset if the media type changed. */ 3974 if (hw->dev_spec._82575.media_changed && 3975 hw->mac.type >= igb_mac_min) { 3976 hw->dev_spec._82575.media_changed = false; 3977 sc->flags |= IGB_MEDIA_RESET; 3978 iflib_request_reset(ctx); 3979 iflib_admin_intr_deferred(ctx); 3980 reset_requested = true; 3981 } 3982 /* Only do TSO on gigabit for older chips due to errata */ 3983 if (hw->mac.type < igb_mac_min) 3984 reset_requested = em_automask_tso(ctx); 3985 3986 if (reset_pending || reset_requested) { 3987 /* 3988 * The PHY is up, but publish it only after the TSO 3989 * capability-change reset. 3990 */ 3991 sc->link_state = EM_LINK_STATE_UP_RESET_PENDING; 3992 } else { 3993 iflib_link_state_change(ctx, LINK_STATE_UP, 3994 IF_Mbps(sc->link_speed)); 3995 } 3996 igb_iov_ping_all_vfs(sc); 3997 } else if (!link_check && 3998 (sc->link_state == EM_LINK_STATE_UP || 3999 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { 4000 bool link_was_published; 4001 bool reset_pending; 4002 4003 link_was_published = sc->link_state == EM_LINK_STATE_UP; 4004 reset_pending = 4005 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING; 4006 sc->link_speed = 0; 4007 sc->link_duplex = 0; 4008 sc->link_state = reset_pending ? 4009 EM_LINK_STATE_DOWN_RESET_PENDING : EM_LINK_STATE_DOWN; 4010 if (link_was_published) 4011 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4012 igb_iov_ping_all_vfs(sc); 4013 } 4014 /* 4015 * Mailbox, link, and timer events share this admin task. The PF 4016 * statistics sweep performs 66 MMIO reads, so run it only when the 4017 * ordinary iflib timer requests a sample rather than once per mailbox 4018 * message. Exported counters can consequently trail hardware by the 4019 * timer interval (normally 500 ms). 4020 */ 4021 if (atomic_readandclear_32(&sc->stats_pending) != 0) { 4022 em_update_stats_counters(sc); 4023 /* 4024 * The 82574 PHY can enter a state in which both its receive 4025 * error and idle error counters saturate. Require two 4026 * consecutive timer samples before resetting, matching Intel's 4027 * e1000e recovery policy and avoiding a reset on a transient 4028 * register sample. 4029 */ 4030 if (hw->mac.type == e1000_82574) { 4031 if (e1000_check_phy_82574(hw)) 4032 sc->phy_hang_count++; 4033 else 4034 sc->phy_hang_count = 0; 4035 if (sc->phy_hang_count > 1) { 4036 sc->phy_hang_count = 0; 4037 device_printf(dev, 4038 "PHY appears hung; requesting reset\n"); 4039 iflib_request_reset(ctx); 4040 iflib_admin_intr_deferred(ctx); 4041 return; 4042 } 4043 } 4044 } 4045 4046 /* Reset LAA into RAR[0] on 82571 */ 4047 if (hw->mac.type == e1000_82571 && e1000_get_laa_state_82571(hw)) 4048 e1000_rar_set(hw, hw->mac.addr, 0); 4049 4050 if (hw->mac.type < em_mac_min) 4051 lem_smartspeed(sc); 4052 } 4053 4054 /* 4055 * Last-resort DMA fence. iflib releases DMA mappings after the driver's 4056 * stop callback, so continuing with bus mastering still enabled would turn 4057 * a recoverable NIC failure into memory corruption. Treat failure of the 4058 * PCI command bit as a fail-stop invariant violation. 4059 */ 4060 static void 4061 em_fence_pci_busmaster(struct e1000_softc *sc) 4062 { 4063 device_t dev; 4064 u_int timeout; 4065 u16 command; 4066 int error; 4067 4068 dev = sc->dev; 4069 error = pci_disable_busmaster(dev); 4070 command = pci_read_config(dev, PCIR_COMMAND, 2); 4071 if (command != 0xffff && (command & PCIM_CMD_BUSMASTEREN) != 0) 4072 panic("%s: unable to fence device DMA (error %d)", 4073 device_get_nameunit(dev), error); 4074 if (error != 0 && command != 0xffff) 4075 device_printf(dev, 4076 "PCI bus-master disable returned %d; readback is disabled\n", 4077 error); 4078 4079 timeout = max(pcie_get_max_completion_timeout(dev) / 1000, 10); 4080 if (command != 0xffff && 4081 !pcie_wait_for_pending_transactions(dev, timeout)) { 4082 /* A function removed during the wait can no longer issue DMA. */ 4083 command = pci_read_config(dev, PCIR_COMMAND, 2); 4084 if (command != 0xffff) 4085 panic("%s: DMA transactions remain pending after fencing", 4086 device_get_nameunit(dev)); 4087 } 4088 } 4089 4090 static int 4091 em_enable_pci_busmaster(struct e1000_softc *sc) 4092 { 4093 device_t dev; 4094 u16 command; 4095 int error; 4096 4097 dev = sc->dev; 4098 command = pci_read_config(dev, PCIR_COMMAND, 2); 4099 if (command == 0xffff) 4100 return (ENXIO); 4101 if ((command & PCIM_CMD_BUSMASTEREN) != 0) 4102 return (0); 4103 4104 error = pci_enable_busmaster(dev); 4105 command = pci_read_config(dev, PCIR_COMMAND, 2); 4106 if (command == 0xffff) 4107 return (ENXIO); 4108 if ((command & PCIM_CMD_BUSMASTEREN) == 0) 4109 return (error != 0 ? error : EIO); 4110 return (0); 4111 } 4112 4113 /********************************************************************* 4114 * 4115 * This routine disables all traffic on the adapter by issuing a 4116 * global reset on the MAC. 4117 * 4118 **********************************************************************/ 4119 static void 4120 em_if_stop(if_ctx_t ctx) 4121 { 4122 struct e1000_softc *sc = iflib_get_softc(ctx); 4123 s32 error; 4124 4125 INIT_DEBUGOUT("em_if_stop: begin"); 4126 4127 if (sc->vf_ifp) { 4128 igbv_queue_retry_stop(sc); 4129 igbv_mbx_retry_stop(sc); 4130 } 4131 4132 /* I219 needs special flushing to avoid hangs */ 4133 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 4134 em_flush_desc_rings(sc); 4135 4136 igb_iov_reset_prepare(sc); 4137 if (!sc->vf_ifp || 4138 (atomic_load_acq_32(&sc->vf_mbx_ready) != 0 && 4139 (if_getflags(iflib_get_ifp(ctx)) & IFF_UP) == 0)) { 4140 em_prepare_fatal_error_reset(sc); 4141 error = e1000_reset_hw(&sc->hw); 4142 if (!sc->vf_ifp && error != E1000_SUCCESS) { 4143 device_printf(sc->dev, "Hardware reset failed while " 4144 "stopping: %d\n", error); 4145 em_fence_pci_busmaster(sc); 4146 return; 4147 } 4148 } 4149 if (sc->vf_ifp) { 4150 sc->vf_queues_sanitized = igbv_sanitize_queues(sc); 4151 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 4152 if (!sc->vf_queues_sanitized) 4153 em_fence_pci_busmaster(sc); 4154 } 4155 if (sc->hw.mac.type >= e1000_82544 && !sc->vf_ifp) 4156 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 4157 4158 if (!sc->vf_ifp) { 4159 e1000_led_off(&sc->hw); 4160 e1000_cleanup_led(&sc->hw); 4161 } else { 4162 sc->link_speed = 0; 4163 sc->link_duplex = 0; 4164 if (sc->link_state != EM_LINK_STATE_DOWN) { 4165 sc->link_state = EM_LINK_STATE_DOWN; 4166 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 4167 } 4168 } 4169 } 4170 4171 /********************************************************************* 4172 * 4173 * Determine hardware revision. 4174 * 4175 **********************************************************************/ 4176 static void 4177 em_identify_hardware(if_ctx_t ctx) 4178 { 4179 device_t dev = iflib_get_dev(ctx); 4180 struct e1000_softc *sc = iflib_get_softc(ctx); 4181 4182 /* Make sure our PCI config space has the necessary stuff set */ 4183 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 4184 4185 /* Save off the information about this board */ 4186 sc->hw.vendor_id = pci_get_vendor(dev); 4187 sc->hw.device_id = pci_get_device(dev); 4188 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 4189 sc->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); 4190 sc->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); 4191 4192 /* Do Shared Code Init and Setup */ 4193 if (e1000_set_mac_type(&sc->hw)) { 4194 device_printf(dev, "Setup init failure\n"); 4195 return; 4196 } 4197 4198 /* 4199 * Function type comes from the selected iflib shared context, not from 4200 * enum ordering. Keep the detected MAC type as an independent check 4201 * that the igb/igbv probe tables selected the right policy. 4202 */ 4203 KASSERT(sc->vf_ifp == 4204 (sc->hw.mac.type == e1000_vfadapt || 4205 sc->hw.mac.type == e1000_vfadapt_i350), 4206 ("%s: iflib function type and MAC type disagree", __func__)); 4207 } 4208 4209 static int 4210 em_allocate_pci_resources(if_ctx_t ctx) 4211 { 4212 struct e1000_softc *sc = iflib_get_softc(ctx); 4213 device_t dev = iflib_get_dev(ctx); 4214 int rid, val; 4215 4216 rid = PCIR_BAR(0); 4217 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 4218 RF_ACTIVE); 4219 if (sc->memory == NULL) { 4220 device_printf(dev, 4221 "Unable to allocate bus resource: memory\n"); 4222 return (ENXIO); 4223 } 4224 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 4225 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->memory); 4226 #ifdef INVARIANTS 4227 sc->osdep.mem_bus_space_size = rman_get_size(sc->memory); 4228 #endif 4229 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 4230 4231 /* Only older adapters use IO mapping */ 4232 if (sc->hw.mac.type < em_mac_min && sc->hw.mac.type > e1000_82543) { 4233 /* Figure our where our IO BAR is ? */ 4234 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 4235 val = pci_read_config(dev, rid, 4); 4236 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 4237 break; 4238 } 4239 rid += 4; 4240 /* check for 64bit BAR */ 4241 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 4242 rid += 4; 4243 } 4244 if (rid >= PCIR_CIS) { 4245 device_printf(dev, "Unable to locate IO BAR\n"); 4246 return (ENXIO); 4247 } 4248 sc->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 4249 &rid, RF_ACTIVE); 4250 if (sc->ioport == NULL) { 4251 device_printf(dev, 4252 "Unable to allocate bus resource: ioport\n"); 4253 return (ENXIO); 4254 } 4255 sc->hw.io_base = 0; 4256 sc->osdep.io_bus_space_tag = 4257 rman_get_bustag(sc->ioport); 4258 sc->osdep.io_bus_space_handle = 4259 rman_get_bushandle(sc->ioport); 4260 } 4261 4262 sc->hw.back = &sc->osdep; 4263 4264 return (0); 4265 } 4266 4267 /********************************************************************* 4268 * 4269 * Set up the MSI-X Interrupt handlers 4270 * 4271 **********************************************************************/ 4272 static int 4273 em_if_msix_intr_assign(if_ctx_t ctx, int msix) 4274 { 4275 struct e1000_softc *sc = iflib_get_softc(ctx); 4276 struct em_rx_queue *rx_que = sc->rx_queues; 4277 struct em_tx_queue *tx_que = sc->tx_queues; 4278 int error, rid, i, vector = 0, rx_vectors; 4279 char buf[16]; 4280 4281 /* First set up ring resources */ 4282 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 4283 rid = vector + 1; 4284 snprintf(buf, sizeof(buf), "rxq%d", i); 4285 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 4286 IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); 4287 if (error) { 4288 device_printf(iflib_get_dev(ctx), 4289 "Failed to allocate que int %d err: %d", 4290 i, error); 4291 sc->rx_num_queues = i + 1; 4292 goto fail; 4293 } 4294 4295 rx_que->msix = vector; 4296 4297 /* 4298 * Set the bit to enable interrupt 4299 * in E1000_IMS -- bits 20 and 21 4300 * are for RX0 and RX1, note this has 4301 * NOTHING to do with the MSI-X vector 4302 */ 4303 if (sc->hw.mac.type == e1000_82574) { 4304 rx_que->eims = 1 << (20 + i); 4305 sc->ims |= rx_que->eims; 4306 sc->ivars |= (8 | rx_que->msix) << (i * 4); 4307 } else if (sc->hw.mac.type == e1000_82575) 4308 rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; 4309 else 4310 rx_que->eims = 1 << vector; 4311 } 4312 rx_vectors = vector; 4313 4314 vector = 0; 4315 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 4316 snprintf(buf, sizeof(buf), "txq%d", i); 4317 tx_que = &sc->tx_queues[i]; 4318 iflib_softirq_alloc_generic(ctx, 4319 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 4320 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 4321 4322 tx_que->msix = (vector % sc->rx_num_queues); 4323 4324 /* 4325 * Set the bit to enable interrupt 4326 * in E1000_IMS -- bits 22 and 23 4327 * are for TX0 and TX1, note this has 4328 * NOTHING to do with the MSI-X vector 4329 */ 4330 if (sc->hw.mac.type == e1000_82574) { 4331 tx_que->eims = 1 << (22 + i); 4332 sc->ims |= tx_que->eims; 4333 sc->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); 4334 } else if (sc->hw.mac.type == e1000_82575) { 4335 tx_que->eims = E1000_EICR_TX_QUEUE0 << i; 4336 } else { 4337 tx_que->eims = 1 << i; 4338 } 4339 } 4340 4341 /* Link interrupt */ 4342 rid = rx_vectors + 1; 4343 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 4344 em_msix_link, sc, 0, "aq"); 4345 4346 if (error) { 4347 device_printf(iflib_get_dev(ctx), 4348 "Failed to register admin handler"); 4349 goto fail; 4350 } 4351 sc->linkvec = rx_vectors; 4352 if (sc->hw.mac.type < igb_mac_min) { 4353 sc->ivars |= (8 | rx_vectors) << 16; 4354 sc->ivars |= 0x80000000; 4355 /* Enable the "Other" interrupt type for link status change */ 4356 sc->ims |= E1000_IMS_OTHER; 4357 } 4358 4359 return (0); 4360 fail: 4361 iflib_irq_free(ctx, &sc->irq); 4362 rx_que = sc->rx_queues; 4363 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 4364 iflib_irq_free(ctx, &rx_que->que_irq); 4365 return (error); 4366 } 4367 4368 static void 4369 igb_configure_queues(struct e1000_softc *sc) 4370 { 4371 struct e1000_hw *hw = &sc->hw; 4372 struct em_rx_queue *rx_que; 4373 struct em_tx_queue *tx_que; 4374 u32 tmp, ivar = 0; 4375 4376 /* 4377 * Queue ownership can change when SR-IOV is enabled or disabled. 4378 * Rebuild the interrupt mask for the current layout instead of 4379 * retaining vectors from a previous initialization. 4380 */ 4381 sc->que_mask = 0; 4382 sc->link_mask = 0; 4383 4384 /* GPIE controls the PF interrupt block and is not in the VF BAR. */ 4385 if (!sc->vf_ifp && hw->mac.type != e1000_82575) 4386 E1000_WRITE_REG(hw, E1000_GPIE, 4387 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 4388 E1000_GPIE_PBA | E1000_GPIE_NSICR); 4389 4390 /* Turn on MSI-X */ 4391 switch (hw->mac.type) { 4392 case e1000_82580: 4393 case e1000_i350: 4394 case e1000_i354: 4395 case e1000_i210: 4396 case e1000_i211: 4397 case e1000_vfadapt: 4398 case e1000_vfadapt_i350: 4399 /* RX entries */ 4400 for (int i = 0; i < sc->rx_num_queues; i++) { 4401 uint32_t index, qid; 4402 4403 rx_que = &sc->rx_queues[i]; 4404 qid = rx_que->rxr.me; 4405 index = qid >> 1; 4406 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4407 if (qid & 1) { 4408 ivar &= 0xFF00FFFF; 4409 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4410 16; 4411 } else { 4412 ivar &= 0xFFFFFF00; 4413 ivar |= rx_que->msix | E1000_IVAR_VALID; 4414 } 4415 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4416 sc->que_mask |= rx_que->eims; 4417 } 4418 /* TX entries */ 4419 for (int i = 0; i < sc->tx_num_queues; i++) { 4420 uint32_t index, qid; 4421 4422 tx_que = &sc->tx_queues[i]; 4423 qid = tx_que->txr.me; 4424 index = qid >> 1; 4425 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4426 if (qid & 1) { 4427 ivar &= 0x00FFFFFF; 4428 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4429 24; 4430 } else { 4431 ivar &= 0xFFFF00FF; 4432 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4433 8; 4434 } 4435 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4436 sc->que_mask |= tx_que->eims; 4437 } 4438 4439 /* And for the link interrupt */ 4440 if (sc->vf_ifp) { 4441 /* 4442 * VTIVAR_MISC maps the VF mailbox in bits 7:0. 4443 * The PF IVAR_MISC maps other causes in bits 15:8. 4444 */ 4445 ivar = sc->linkvec | E1000_IVAR_VALID; 4446 } else 4447 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4448 sc->link_mask = 1 << sc->linkvec; 4449 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4450 break; 4451 case e1000_82576: 4452 /* RX entries */ 4453 for (int i = 0; i < sc->rx_num_queues; i++) { 4454 uint32_t index, qid; 4455 4456 rx_que = &sc->rx_queues[i]; 4457 qid = rx_que->rxr.me; 4458 index = qid & 0x7; /* Each IVAR has two entries */ 4459 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4460 if (qid < 8) { 4461 ivar &= 0xFFFFFF00; 4462 ivar |= rx_que->msix | E1000_IVAR_VALID; 4463 } else { 4464 ivar &= 0xFF00FFFF; 4465 ivar |= (rx_que->msix | E1000_IVAR_VALID) << 4466 16; 4467 } 4468 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4469 sc->que_mask |= rx_que->eims; 4470 } 4471 /* TX entries */ 4472 for (int i = 0; i < sc->tx_num_queues; i++) { 4473 uint32_t index, qid; 4474 4475 tx_que = &sc->tx_queues[i]; 4476 qid = tx_que->txr.me; 4477 index = qid & 0x7; /* Each IVAR has two entries */ 4478 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4479 if (qid < 8) { 4480 ivar &= 0xFFFF00FF; 4481 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4482 8; 4483 } else { 4484 ivar &= 0x00FFFFFF; 4485 ivar |= (tx_que->msix | E1000_IVAR_VALID) << 4486 24; 4487 } 4488 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4489 sc->que_mask |= tx_que->eims; 4490 } 4491 4492 /* And for the link interrupt */ 4493 ivar = (sc->linkvec | E1000_IVAR_VALID) << 8; 4494 sc->link_mask = 1 << sc->linkvec; 4495 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4496 break; 4497 4498 case e1000_82575: 4499 /* enable MSI-X support*/ 4500 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 4501 tmp |= E1000_CTRL_EXT_PBA_CLR; 4502 /* Auto-Mask interrupts upon ICR read. */ 4503 tmp |= E1000_CTRL_EXT_EIAME; 4504 tmp |= E1000_CTRL_EXT_IRCA; 4505 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 4506 4507 /* Queues */ 4508 for (int i = 0; i < sc->rx_num_queues; i++) { 4509 rx_que = &sc->rx_queues[i]; 4510 tmp = E1000_EICR_RX_QUEUE0 << i; 4511 tmp |= E1000_EICR_TX_QUEUE0 << i; 4512 rx_que->eims = tmp; 4513 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), i, 4514 rx_que->eims); 4515 sc->que_mask |= rx_que->eims; 4516 } 4517 4518 /* Link */ 4519 E1000_WRITE_REG(hw, E1000_MSIXBM(sc->linkvec), 4520 E1000_EIMS_OTHER); 4521 sc->link_mask |= E1000_EIMS_OTHER; 4522 default: 4523 break; 4524 } 4525 4526 return; 4527 } 4528 4529 static void 4530 igb_initialize_interrupt_rate(struct e1000_softc *sc) 4531 { 4532 struct e1000_hw *hw = &sc->hw; 4533 struct em_rx_queue *rx_que; 4534 u32 newitr; 4535 4536 newitr = IGB_INTS_TO_EITR(em_max_interrupt_rate); 4537 if (hw->mac.type == e1000_82575) 4538 newitr |= newitr << 16; 4539 else 4540 newitr |= E1000_EITR_CNT_IGNR; 4541 4542 for (int i = 0; i < sc->rx_num_queues; i++) { 4543 rx_que = &sc->rx_queues[i]; 4544 rx_que->itr_setting = newitr; 4545 E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), 4546 rx_que->itr_setting); 4547 } 4548 if (sc->intr_type == IFLIB_INTR_MSIX) 4549 E1000_WRITE_REG(hw, E1000_EITR(sc->linkvec), newitr); 4550 } 4551 4552 static void 4553 em_free_pci_resources(if_ctx_t ctx) 4554 { 4555 struct e1000_softc *sc = iflib_get_softc(ctx); 4556 struct em_rx_queue *que = sc->rx_queues; 4557 device_t dev = iflib_get_dev(ctx); 4558 4559 /* Release all MSI-X queue resources */ 4560 if (sc->intr_type == IFLIB_INTR_MSIX) 4561 iflib_irq_free(ctx, &sc->irq); 4562 4563 if (que != NULL) { 4564 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 4565 iflib_irq_free(ctx, &que->que_irq); 4566 } 4567 } 4568 4569 if (sc->memory != NULL) { 4570 bus_release_resource(dev, SYS_RES_MEMORY, 4571 rman_get_rid(sc->memory), sc->memory); 4572 sc->memory = NULL; 4573 } 4574 4575 if (sc->flash != NULL) { 4576 bus_release_resource(dev, SYS_RES_MEMORY, 4577 rman_get_rid(sc->flash), sc->flash); 4578 sc->flash = NULL; 4579 } 4580 4581 if (sc->ioport != NULL) { 4582 bus_release_resource(dev, SYS_RES_IOPORT, 4583 rman_get_rid(sc->ioport), sc->ioport); 4584 sc->ioport = NULL; 4585 } 4586 } 4587 4588 /* Set up MSI or MSI-X */ 4589 static int 4590 em_setup_msix(if_ctx_t ctx) 4591 { 4592 struct e1000_softc *sc = iflib_get_softc(ctx); 4593 4594 if (sc->hw.mac.type == e1000_82574) { 4595 em_enable_vectors_82574(ctx); 4596 } 4597 return (0); 4598 } 4599 4600 /********************************************************************* 4601 * 4602 * Workaround for SmartSpeed on 82541 and 82547 controllers 4603 * 4604 **********************************************************************/ 4605 static void 4606 lem_smartspeed(struct e1000_softc *sc) 4607 { 4608 u16 phy_tmp; 4609 4610 if (sc->link_state == EM_LINK_STATE_UP || 4611 sc->link_state == EM_LINK_STATE_UP_RESET_PENDING || 4612 (sc->hw.phy.type != e1000_phy_igp) || 4613 sc->hw.mac.autoneg == 0 || 4614 (sc->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 4615 return; 4616 4617 if (sc->smartspeed == 0) { 4618 /* If Master/Slave config fault is asserted twice, 4619 * we assume back-to-back */ 4620 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4621 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 4622 return; 4623 e1000_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp); 4624 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 4625 e1000_read_phy_reg(&sc->hw, 4626 PHY_1000T_CTRL, &phy_tmp); 4627 if(phy_tmp & CR_1000T_MS_ENABLE) { 4628 phy_tmp &= ~CR_1000T_MS_ENABLE; 4629 e1000_write_phy_reg(&sc->hw, 4630 PHY_1000T_CTRL, phy_tmp); 4631 sc->smartspeed++; 4632 if(sc->hw.mac.autoneg && 4633 !e1000_copper_link_autoneg(&sc->hw) && 4634 !e1000_read_phy_reg(&sc->hw, 4635 PHY_CONTROL, &phy_tmp)) { 4636 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4637 MII_CR_RESTART_AUTO_NEG); 4638 e1000_write_phy_reg(&sc->hw, 4639 PHY_CONTROL, phy_tmp); 4640 } 4641 } 4642 } 4643 return; 4644 } else if(sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 4645 /* If still no link, perhaps using 2/3 pair cable */ 4646 e1000_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp); 4647 phy_tmp |= CR_1000T_MS_ENABLE; 4648 e1000_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp); 4649 if(sc->hw.mac.autoneg && 4650 !e1000_copper_link_autoneg(&sc->hw) && 4651 !e1000_read_phy_reg(&sc->hw, PHY_CONTROL, &phy_tmp)) { 4652 phy_tmp |= (MII_CR_AUTO_NEG_EN | 4653 MII_CR_RESTART_AUTO_NEG); 4654 e1000_write_phy_reg(&sc->hw, PHY_CONTROL, phy_tmp); 4655 } 4656 } 4657 /* Restart process after EM_SMARTSPEED_MAX iterations */ 4658 if(sc->smartspeed++ == EM_SMARTSPEED_MAX) 4659 sc->smartspeed = 0; 4660 } 4661 4662 static void 4663 igb_disable_dmac(struct e1000_hw *hw) 4664 { 4665 u32 reg; 4666 4667 reg = E1000_READ_REG(hw, E1000_DMACR); 4668 reg &= ~E1000_DMACR_DMAC_EN; 4669 /* Retain the documented Lx policy and I210 reserved encoding. */ 4670 reg |= E1000_DMACR_DMAC_LX_MASK; 4671 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4672 } 4673 4674 /********************************************************************* 4675 * 4676 * Initialize the DMA Coalescing feature 4677 * 4678 **********************************************************************/ 4679 static void 4680 igb_init_dmac(struct e1000_softc *sc, u32 pba) 4681 { 4682 device_t dev = sc->dev; 4683 struct e1000_hw *hw = &sc->hw; 4684 u32 dmac, dmacwt, reg, ttlx; 4685 u16 hwm; 4686 u16 max_frame_size; 4687 4688 KASSERT(!sc->vf_ifp, ("%s: DMA coalescing requested for a VF", 4689 __func__)); 4690 4691 if (hw->mac.type == e1000_i211) 4692 return; 4693 4694 /* 4695 * I350 DMA coalescing and SR-IOV are mutually exclusive. Preserve 4696 * the configured value so it can be restored after IOV is disabled. 4697 */ 4698 if (igb_iov_enabled(sc)) { 4699 if (hw->mac.type > e1000_82580) 4700 igb_disable_dmac(hw); 4701 return; 4702 } 4703 4704 max_frame_size = sc->shared->isc_max_frame_size; 4705 if (hw->mac.type > e1000_82580) { 4706 4707 if (sc->dmac == 0) { /* Disabling it */ 4708 igb_disable_dmac(hw); 4709 return; 4710 } else 4711 device_printf(dev, "DMA Coalescing enabled\n"); 4712 4713 /* Set starting threshold */ 4714 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); 4715 4716 hwm = 64 * pba - max_frame_size / 16; 4717 if (hwm < 64 * (pba - 6)) 4718 hwm = 64 * (pba - 6); 4719 reg = E1000_READ_REG(hw, E1000_FCRTC); 4720 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 4721 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 4722 & E1000_FCRTC_RTH_COAL_MASK); 4723 E1000_WRITE_REG(hw, E1000_FCRTC, reg); 4724 4725 4726 dmac = pba - max_frame_size / 512; 4727 if (dmac < pba - 10) 4728 dmac = pba - 10; 4729 reg = E1000_READ_REG(hw, E1000_DMACR); 4730 reg &= ~(E1000_DMACR_DMACWT_MASK | 4731 E1000_DMACR_DMACTHR_MASK | E1000_DMACR_DMAC_LX_MASK | 4732 E1000_DMACR_DMAC_EN | E1000_DMACR_DC_LPBKW_EN | 4733 E1000_DMACR_DC_BMC2OSW_EN); 4734 reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT) 4735 & E1000_DMACR_DMACTHR_MASK); 4736 4737 /* Transition to L0s or L1 if available. */ 4738 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 4739 4740 /* 4741 * The watchdog uses 12.8 usec units on an I354 2.5 Gb/s 4742 * backplane connection and 32 usec units otherwise. 4743 */ 4744 if (hw->mac.type == e1000_i354) { 4745 int status = E1000_READ_REG(hw, E1000_STATUS); 4746 if ((status & E1000_STATUS_2P5_SKU) && 4747 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4748 dmacwt = (sc->dmac * 5) >> 6; 4749 else 4750 dmacwt = sc->dmac >> 5; 4751 } else { 4752 dmacwt = sc->dmac >> 5; 4753 } 4754 reg |= dmacwt & E1000_DMACR_DMACWT_MASK; 4755 if (hw->mac.type == e1000_i350 || 4756 hw->mac.type == e1000_i354) 4757 reg |= E1000_DMACR_DC_LPBKW_EN; 4758 if (hw->mac.type == e1000_i354) 4759 reg |= E1000_DMACR_DC_BMC2OSW_EN; 4760 4761 E1000_WRITE_REG(hw, E1000_DMACR, reg); 4762 4763 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); 4764 4765 /* Set the interval before transition. */ 4766 reg = E1000_READ_REG(hw, E1000_DMCTLX); 4767 reg &= ~E1000_DMCTLX_TTLX_MASK; 4768 if (hw->mac.type == e1000_i350) 4769 reg |= IGB_DMCTLX_DCFLUSH_DIS; 4770 /* 4771 * I210 documents TTLX as reserved with a required value of 0x20. 4772 * At 2.5 Gb/s the I354 unit is 0.4 usec, so ten ticks retain 4773 * the four usec interval used at other speeds. 4774 */ 4775 if (hw->mac.type == e1000_i210) { 4776 ttlx = 0x20; 4777 } else if (hw->mac.type == e1000_i354) { 4778 int status = E1000_READ_REG(hw, E1000_STATUS); 4779 if ((status & E1000_STATUS_2P5_SKU) && 4780 (!(status & E1000_STATUS_2P5_SKU_OVER))) 4781 ttlx = 0xA; 4782 else 4783 ttlx = 0x4; 4784 } else { 4785 ttlx = 0x4; 4786 } 4787 reg |= ttlx & E1000_DMCTLX_TTLX_MASK; 4788 4789 E1000_WRITE_REG(hw, E1000_DMCTLX, reg); 4790 4791 /* free space in tx packet buffer to wake from DMA coal */ 4792 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - 4793 (2 * max_frame_size)) >> 6); 4794 4795 /* make low power state decision controlled by DMA coal */ 4796 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4797 reg |= E1000_PCIEMISC_LX_DECISION; 4798 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); 4799 4800 } else if (hw->mac.type == e1000_82580) { 4801 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 4802 E1000_WRITE_REG(hw, E1000_PCIEMISC, 4803 reg & ~E1000_PCIEMISC_LX_DECISION); 4804 E1000_WRITE_REG(hw, E1000_DMACR, 0); 4805 } 4806 } 4807 /********************************************************************* 4808 * The 3 following flush routines are used as a workaround in the 4809 * I219 client parts and only for them. 4810 * 4811 * em_flush_tx_ring - remove all descriptors from the tx_ring 4812 * 4813 * We want to clear all pending descriptors from the TX ring. 4814 * zeroing happens when the HW reads the regs. We assign the ring itself as 4815 * the data of the next descriptor. We don't care about the data we are about 4816 * to reset the HW. 4817 **********************************************************************/ 4818 static void 4819 em_flush_tx_ring(struct e1000_softc *sc) 4820 { 4821 struct e1000_hw *hw = &sc->hw; 4822 struct tx_ring *txr = &sc->tx_queues->txr; 4823 struct e1000_tx_desc *txd; 4824 u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; 4825 u16 size = 512; 4826 4827 tctl = E1000_READ_REG(hw, E1000_TCTL); 4828 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); 4829 4830 txd = &txr->tx_base[txr->tx_cidx_processed]; 4831 4832 /* Just use the ring as a dummy buffer addr */ 4833 txd->buffer_addr = txr->tx_paddr; 4834 txd->lower.data = htole32(txd_lower | size); 4835 txd->upper.data = 0; 4836 4837 /* flush descriptors to memory before notifying the HW */ 4838 wmb(); 4839 4840 E1000_WRITE_REG(hw, E1000_TDT(0), txr->tx_cidx_processed); 4841 mb(); 4842 usec_delay(250); 4843 } 4844 4845 /********************************************************************* 4846 * em_flush_rx_ring - remove all descriptors from the rx_ring 4847 * 4848 * Mark all descriptors in the RX ring as consumed and disable the rx ring 4849 **********************************************************************/ 4850 static void 4851 em_flush_rx_ring(struct e1000_softc *sc) 4852 { 4853 struct e1000_hw *hw = &sc->hw; 4854 u32 rctl, rxdctl; 4855 4856 rctl = E1000_READ_REG(hw, E1000_RCTL); 4857 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4858 E1000_WRITE_FLUSH(hw); 4859 usec_delay(150); 4860 4861 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 4862 /* zero the lower 14 bits (prefetch and host thresholds) */ 4863 rxdctl &= 0xffffc000; 4864 /* 4865 * update thresholds: prefetch threshold to 31, host threshold to 1 4866 * and make sure the granularity is "descriptors" and not 4867 * "cache lines" 4868 */ 4869 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); 4870 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 4871 4872 /* momentarily enable the RX ring for the changes to take effect */ 4873 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); 4874 E1000_WRITE_FLUSH(hw); 4875 usec_delay(150); 4876 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4877 } 4878 4879 /********************************************************************* 4880 * em_flush_desc_rings - remove all descriptors from the descriptor rings 4881 * 4882 * In I219, the descriptor rings must be emptied before resetting the HW 4883 * or before changing the device state to D3 during runtime (runtime PM). 4884 * 4885 * Failure to do this will cause the HW to enter a unit hang state which can 4886 * only be released by PCI reset on the device 4887 * 4888 **********************************************************************/ 4889 static void 4890 em_flush_desc_rings(struct e1000_softc *sc) 4891 { 4892 struct e1000_hw *hw = &sc->hw; 4893 device_t dev = sc->dev; 4894 u16 hang_state; 4895 u32 fext_nvm11, tdlen; 4896 4897 /* First, disable MULR fix in FEXTNVM11 */ 4898 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); 4899 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 4900 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); 4901 4902 /* do nothing if we're not in faulty state, or the queue is empty */ 4903 tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); 4904 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4905 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 4906 return; 4907 em_flush_tx_ring(sc); 4908 4909 /* recheck, maybe the fault is caused by the rx ring */ 4910 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); 4911 if (hang_state & FLUSH_DESC_REQUIRED) 4912 em_flush_rx_ring(sc); 4913 } 4914 4915 4916 /********************************************************************* 4917 * 4918 * Initialize the hardware to a configuration as specified by the 4919 * sc structure. 4920 * 4921 **********************************************************************/ 4922 static int 4923 em_reset(if_ctx_t ctx) 4924 { 4925 device_t dev = iflib_get_dev(ctx); 4926 struct e1000_softc *sc = iflib_get_softc(ctx); 4927 if_t ifp = iflib_get_ifp(ctx); 4928 struct e1000_hw *hw = &sc->hw; 4929 u32 rx_buffer_size; 4930 u32 pba; 4931 s32 error; 4932 4933 INIT_DEBUGOUT("em_reset: begin"); 4934 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 4935 4936 /* Let the firmware know the OS is in control */ 4937 em_get_hw_control(sc); 4938 4939 /* Set up smart power down as default off on newer adapters. */ 4940 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 4941 hw->mac.type == e1000_82572)) { 4942 u16 phy_tmp = 0; 4943 4944 /* Speed up time to link by disabling smart power down. */ 4945 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 4946 phy_tmp &= ~IGP02E1000_PM_SPD; 4947 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 4948 } 4949 4950 /* 4951 * Packet Buffer Allocation (PBA) 4952 * Writing PBA sets the receive portion of the buffer 4953 * the remainder is used for the transmit buffer. 4954 */ 4955 switch (hw->mac.type) { 4956 /* 82547: Total Packet Buffer is 40K */ 4957 case e1000_82547: 4958 case e1000_82547_rev_2: 4959 if (hw->mac.max_frame_size > 8192) 4960 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4961 else 4962 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 4963 break; 4964 /* 82571/82572/80003es2lan: Total Packet Buffer is 48K */ 4965 case e1000_82571: 4966 case e1000_82572: 4967 case e1000_80003es2lan: 4968 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 4969 break; 4970 /* 82573: Total Packet Buffer is 32K */ 4971 case e1000_82573: 4972 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 4973 break; 4974 /* 82574/82583: Total Packet Buffer is 40K */ 4975 case e1000_82574: 4976 case e1000_82583: 4977 if (hw->mac.max_frame_size > 8192) 4978 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 4979 else 4980 pba = E1000_PBA_32K; /* 32K for RX, 8K for Tx */ 4981 break; 4982 case e1000_ich8lan: 4983 pba = E1000_PBA_8K; 4984 break; 4985 case e1000_ich9lan: 4986 case e1000_ich10lan: 4987 /* Boost Receive side for jumbo frames */ 4988 if (hw->mac.max_frame_size > 4096) 4989 pba = E1000_PBA_14K; 4990 else 4991 pba = E1000_PBA_10K; 4992 break; 4993 case e1000_pchlan: 4994 case e1000_pch2lan: 4995 case e1000_pch_lpt: 4996 case e1000_pch_spt: 4997 case e1000_pch_cnp: 4998 case e1000_pch_tgp: 4999 case e1000_pch_adp: 5000 case e1000_pch_mtp: 5001 case e1000_pch_ptp: 5002 case e1000_pch_nvp: 5003 pba = E1000_PBA_26K; 5004 break; 5005 case e1000_82575: 5006 pba = E1000_PBA_32K; 5007 break; 5008 case e1000_82576: 5009 pba = E1000_READ_REG(hw, E1000_RXPBS); 5010 pba &= E1000_RXPBS_SIZE_MASK_82576; 5011 break; 5012 case e1000_82580: 5013 case e1000_i350: 5014 case e1000_i354: 5015 pba = E1000_READ_REG(hw, E1000_RXPBS); 5016 pba = e1000_rxpbs_adjust_82580(pba); 5017 break; 5018 case e1000_i210: 5019 case e1000_i211: 5020 pba = E1000_PBA_34K; 5021 break; 5022 default: 5023 /* Remaining devices assumed to have Packet Buffer of 64K. */ 5024 if (hw->mac.max_frame_size > 8192) 5025 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 5026 else 5027 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 5028 } 5029 5030 /* Special needs in case of Jumbo frames */ 5031 if ((hw->mac.type == e1000_82575) && (if_getmtu(ifp) > ETHERMTU)) { 5032 u32 tx_space, min_tx, min_rx; 5033 pba = E1000_READ_REG(hw, E1000_PBA); 5034 tx_space = pba >> 16; 5035 pba &= 0xffff; 5036 min_tx = (hw->mac.max_frame_size + 5037 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 5038 min_tx = roundup2(min_tx, 1024); 5039 min_tx >>= 10; 5040 min_rx = hw->mac.max_frame_size; 5041 min_rx = roundup2(min_rx, 1024); 5042 min_rx >>= 10; 5043 if (tx_space < min_tx && 5044 ((min_tx - tx_space) < pba)) { 5045 pba = pba - (min_tx - tx_space); 5046 /* 5047 * if short on rx space, rx wins 5048 * and must trump tx adjustment 5049 */ 5050 if (pba < min_rx) 5051 pba = min_rx; 5052 } 5053 E1000_WRITE_REG(hw, E1000_PBA, pba); 5054 } 5055 5056 if (hw->mac.type < igb_mac_min) 5057 E1000_WRITE_REG(hw, E1000_PBA, pba); 5058 5059 INIT_DEBUGOUT1("em_reset: pba=%dK", pba); 5060 5061 /* 5062 * These parameters control the automatic generation (Tx) and 5063 * response (Rx) to Ethernet PAUSE frames. 5064 * - High water mark should allow for at least two frames to be 5065 * received after sending an XOFF. 5066 * - Low water mark works best when it is very near the high water 5067 mark. 5068 * This allows the receiver to restart by sending XON when it has 5069 * drained a bit. Here we use an arbitrary value of 1500 which will 5070 * restart after one full frame is pulled from the buffer. There 5071 * could be several smaller frames in the buffer and if so they will 5072 * not trigger the XON until their total number reduces the buffer 5073 * by 1500. 5074 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 5075 */ 5076 rx_buffer_size = (pba & 0xffff) << 10; 5077 hw->fc.high_water = rx_buffer_size - 5078 roundup2(hw->mac.max_frame_size, 1024); 5079 hw->fc.low_water = hw->fc.high_water - 1500; 5080 5081 if (sc->fc) /* locally set flow control value? */ 5082 hw->fc.requested_mode = sc->fc; 5083 else 5084 hw->fc.requested_mode = e1000_fc_full; 5085 5086 if (hw->mac.type == e1000_80003es2lan) 5087 hw->fc.pause_time = 0xFFFF; 5088 else 5089 hw->fc.pause_time = EM_FC_PAUSE_TIME; 5090 5091 hw->fc.send_xon = true; 5092 5093 /* Device specific overrides/settings */ 5094 switch (hw->mac.type) { 5095 case e1000_pchlan: 5096 /* Workaround: no TX flow ctrl for PCH */ 5097 hw->fc.requested_mode = e1000_fc_rx_pause; 5098 hw->fc.pause_time = 0xFFFF; /* override */ 5099 if (if_getmtu(ifp) > ETHERMTU) { 5100 hw->fc.high_water = 0x3500; 5101 hw->fc.low_water = 0x1500; 5102 } else { 5103 hw->fc.high_water = 0x5000; 5104 hw->fc.low_water = 0x3000; 5105 } 5106 hw->fc.refresh_time = 0x1000; 5107 break; 5108 case e1000_pch2lan: 5109 case e1000_pch_lpt: 5110 case e1000_pch_spt: 5111 case e1000_pch_cnp: 5112 case e1000_pch_tgp: 5113 case e1000_pch_adp: 5114 case e1000_pch_mtp: 5115 case e1000_pch_ptp: 5116 case e1000_pch_nvp: 5117 hw->fc.high_water = 0x5C20; 5118 hw->fc.low_water = 0x5048; 5119 hw->fc.pause_time = 0xFFFF; 5120 hw->fc.refresh_time = 0xFFFF; 5121 /* Jumbos need adjusted PBA */ 5122 if (if_getmtu(ifp) > ETHERMTU) 5123 pba = E1000_PBA_12K; 5124 else 5125 pba = E1000_PBA_26K; 5126 E1000_WRITE_REG(hw, E1000_PBA, pba); 5127 break; 5128 case e1000_82575: 5129 case e1000_82576: 5130 /* 8-byte granularity */ 5131 hw->fc.low_water = hw->fc.high_water - 8; 5132 break; 5133 case e1000_82580: 5134 case e1000_i350: 5135 case e1000_i354: 5136 case e1000_i210: 5137 case e1000_i211: 5138 /* 16-byte granularity */ 5139 hw->fc.low_water = hw->fc.high_water - 16; 5140 break; 5141 case e1000_ich9lan: 5142 case e1000_ich10lan: 5143 if (if_getmtu(ifp) > ETHERMTU) { 5144 hw->fc.high_water = 0x2800; 5145 hw->fc.low_water = hw->fc.high_water - 8; 5146 break; 5147 } 5148 /* FALLTHROUGH */ 5149 default: 5150 if (hw->mac.type == e1000_80003es2lan) 5151 hw->fc.pause_time = 0xFFFF; 5152 break; 5153 } 5154 5155 /* I219 needs some special flushing to avoid hangs */ 5156 if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min) 5157 em_flush_desc_rings(sc); 5158 5159 /* Issue a global reset */ 5160 em_prepare_fatal_error_reset(sc); 5161 error = e1000_reset_hw(hw); 5162 if (error != E1000_SUCCESS) { 5163 device_printf(dev, "Hardware reset failed: %d\n", error); 5164 return (error); 5165 } 5166 if (hw->mac.type >= igb_mac_min) { 5167 E1000_WRITE_REG(hw, E1000_WUC, 0); 5168 } else { 5169 E1000_WRITE_REG(hw, E1000_WUFC, 0); 5170 em_disable_aspm(sc); 5171 } 5172 if (sc->flags & IGB_MEDIA_RESET) { 5173 e1000_setup_init_funcs(hw, true); 5174 e1000_get_bus_info(hw); 5175 sc->flags &= ~IGB_MEDIA_RESET; 5176 } 5177 /* and a re-init */ 5178 error = e1000_init_hw(hw); 5179 if (error != E1000_SUCCESS) { 5180 device_printf(dev, "Hardware initialization failed: %d\n", 5181 error); 5182 return (error); 5183 } 5184 em_configure_82576_memory_errors(sc); 5185 em_finish_fatal_error_reset(sc); 5186 if (hw->mac.type >= igb_mac_min) 5187 igb_init_dmac(sc, pba); 5188 5189 /* Save the receive packet-buffer allocation for AIM. */ 5190 sc->pba = pba; 5191 5192 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 5193 e1000_get_phy_info(hw); 5194 e1000_check_for_link(hw); 5195 sc->phy_hang_count = 0; 5196 5197 return (E1000_SUCCESS); 5198 } 5199 5200 /* 5201 * Initialise the RSS mapping for NICs that support multiple transmit/ 5202 * receive rings. 5203 */ 5204 5205 #define RSSKEYLEN 10 5206 static void 5207 em_initialize_rss_mapping(struct e1000_softc *sc) 5208 { 5209 uint8_t rss_key[4 * RSSKEYLEN]; 5210 uint32_t reta = 0; 5211 struct e1000_hw *hw = &sc->hw; 5212 int i; 5213 5214 /* 5215 * Configure RSS key 5216 */ 5217 arc4rand(rss_key, sizeof(rss_key), 0); 5218 for (i = 0; i < RSSKEYLEN; ++i) { 5219 uint32_t rssrk = 0; 5220 5221 rssrk = EM_RSSRK_VAL(rss_key, i); 5222 E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); 5223 } 5224 5225 /* 5226 * Configure RSS redirect table in following fashion: 5227 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] 5228 */ 5229 for (i = 0; i < sizeof(reta); ++i) { 5230 uint32_t q; 5231 5232 q = (i % sc->rx_num_queues) << 7; 5233 reta |= q << (8 * i); 5234 } 5235 5236 for (i = 0; i < 32; ++i) 5237 E1000_WRITE_REG(hw, E1000_RETA(i), reta); 5238 5239 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | 5240 E1000_MRQC_RSS_FIELD_IPV4_TCP | 5241 E1000_MRQC_RSS_FIELD_IPV4 | 5242 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | 5243 E1000_MRQC_RSS_FIELD_IPV6_EX | 5244 E1000_MRQC_RSS_FIELD_IPV6); 5245 } 5246 5247 static void 5248 igb_initialize_rss_mapping(struct e1000_softc *sc) 5249 { 5250 struct e1000_hw *hw = &sc->hw; 5251 int i; 5252 int queue_id; 5253 u32 reta; 5254 u32 rss_key[10], mrqc, shift = 0; 5255 5256 /* XXX? */ 5257 if (hw->mac.type == e1000_82575) 5258 shift = 6; 5259 5260 /* 5261 * The redirection table controls which destination 5262 * queue each bucket redirects traffic to. 5263 * Each DWORD represents four queues, with the LSB 5264 * being the first queue in the DWORD. 5265 * 5266 * This just allocates buckets to queues using round-robin 5267 * allocation. 5268 * 5269 * NOTE: It Just Happens to line up with the default 5270 * RSS allocation method. 5271 */ 5272 5273 /* Warning FM follows */ 5274 reta = 0; 5275 for (i = 0; i < 128; i++) { 5276 #ifdef RSS 5277 queue_id = rss_get_indirection_to_bucket(i); 5278 /* 5279 * If we have more queues than buckets, we'll 5280 * end up mapping buckets to a subset of the 5281 * queues. 5282 * 5283 * If we have more buckets than queues, we'll 5284 * end up instead assigning multiple buckets 5285 * to queues. 5286 * 5287 * Both are suboptimal, but we need to handle 5288 * the case so we don't go out of bounds 5289 * indexing arrays and such. 5290 */ 5291 queue_id = queue_id % sc->rx_num_queues; 5292 #else 5293 queue_id = (i % sc->rx_num_queues); 5294 #endif 5295 /* Adjust if required */ 5296 queue_id = queue_id << shift; 5297 5298 /* 5299 * The low 8 bits are for hash value (n+0); 5300 * The next 8 bits are for hash value (n+1), etc. 5301 */ 5302 reta = reta >> 8; 5303 reta = reta | ( ((uint32_t) queue_id) << 24); 5304 if ((i & 3) == 3) { 5305 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 5306 reta = 0; 5307 } 5308 } 5309 5310 /* Now fill in hash table */ 5311 5312 /* 5313 * MRQC: Multiple Receive Queues Command 5314 * Set queuing to RSS control, number depends on the device. 5315 */ 5316 mrqc = E1000_MRQC_ENABLE_RSS_MQ; 5317 5318 /* XXX ew typecasting */ 5319 rss_getkey((uint8_t *) &rss_key); 5320 for (i = 0; i < 10; i++) 5321 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]); 5322 5323 /* 5324 * Configure the RSS fields to hash upon. 5325 */ 5326 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 5327 E1000_MRQC_RSS_FIELD_IPV4_TCP); 5328 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 5329 E1000_MRQC_RSS_FIELD_IPV6_TCP); 5330 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 5331 E1000_MRQC_RSS_FIELD_IPV6_UDP); 5332 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 5333 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 5334 5335 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 5336 } 5337 5338 /********************************************************************* 5339 * 5340 * Setup networking device structure and register interface media. 5341 * 5342 **********************************************************************/ 5343 static int 5344 em_setup_interface(if_ctx_t ctx) 5345 { 5346 if_t ifp = iflib_get_ifp(ctx); 5347 struct e1000_softc *sc = iflib_get_softc(ctx); 5348 if_softc_ctx_t scctx = sc->shared; 5349 5350 INIT_DEBUGOUT("em_setup_interface: begin"); 5351 5352 /* Single Queue */ 5353 if (sc->tx_num_queues == 1) { 5354 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 5355 if_setsendqready(ifp); 5356 } 5357 5358 /* 5359 * Specify the media types supported by this adapter and register 5360 * callbacks to update media and link information 5361 */ 5362 if (sc->vf_ifp) { 5363 ifmedia_add(sc->media, 5364 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5365 ifmedia_set(sc->media, 5366 IFM_ETHER | IFM_1000_T | IFM_FDX); 5367 return (0); 5368 } 5369 5370 if (sc->hw.phy.media_type == e1000_media_type_fiber || 5371 sc->hw.phy.media_type == e1000_media_type_internal_serdes) { 5372 u_char fiber_type = IFM_1000_SX; /* default type */ 5373 5374 if (sc->hw.mac.type == e1000_82545) 5375 fiber_type = IFM_1000_LX; 5376 ifmedia_add(sc->media, 5377 IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); 5378 ifmedia_add(sc->media, IFM_ETHER | fiber_type, 0, NULL); 5379 } else { 5380 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 5381 ifmedia_add(sc->media, 5382 IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 5383 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 5384 ifmedia_add(sc->media, 5385 IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 5386 if (sc->hw.phy.type != e1000_phy_ife) { 5387 ifmedia_add(sc->media, 5388 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 5389 ifmedia_add(sc->media, 5390 IFM_ETHER | IFM_1000_T, 0, NULL); 5391 } 5392 } 5393 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 5394 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 5395 return (0); 5396 } 5397 5398 static int 5399 em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5400 int ntxqs, int ntxqsets) 5401 { 5402 struct e1000_softc *sc = iflib_get_softc(ctx); 5403 if_softc_ctx_t scctx = sc->shared; 5404 int error = E1000_SUCCESS; 5405 struct em_tx_queue *que; 5406 int i, j; 5407 5408 MPASS(sc->tx_num_queues > 0); 5409 MPASS(sc->tx_num_queues == ntxqsets); 5410 5411 /* First allocate the top level queue structs */ 5412 if (!(sc->tx_queues = 5413 (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * 5414 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5415 device_printf(iflib_get_dev(ctx), 5416 "Unable to allocate queue memory\n"); 5417 return(ENOMEM); 5418 } 5419 5420 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 5421 /* Set up some basics */ 5422 5423 struct tx_ring *txr = &que->txr; 5424 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 5425 ("%s: misaligned TX AIM snapshot %p", __func__, 5426 &txr->tx_aim_snapshot)); 5427 txr->sc = que->sc = sc; 5428 que->me = txr->me = i; 5429 5430 /* Allocate report status array */ 5431 if (!(txr->tx_rsq = 5432 (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], 5433 M_DEVBUF, M_NOWAIT | M_ZERO))) { 5434 device_printf(iflib_get_dev(ctx), 5435 "failed to allocate rs_idxs memory\n"); 5436 error = ENOMEM; 5437 goto fail; 5438 } 5439 for (j = 0; j < scctx->isc_ntxd[0]; j++) 5440 txr->tx_rsq[j] = QIDX_INVALID; 5441 /* get the virtual and physical address of hardware queues */ 5442 txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; 5443 txr->tx_paddr = paddrs[i*ntxqs]; 5444 } 5445 5446 if (bootverbose) 5447 device_printf(iflib_get_dev(ctx), 5448 "allocated for %d tx_queues\n", sc->tx_num_queues); 5449 return (0); 5450 fail: 5451 em_if_queues_free(ctx); 5452 return (error); 5453 } 5454 5455 static int 5456 em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 5457 int nrxqs, int nrxqsets) 5458 { 5459 struct e1000_softc *sc = iflib_get_softc(ctx); 5460 int error = E1000_SUCCESS; 5461 struct em_rx_queue *que; 5462 int i; 5463 5464 MPASS(sc->rx_num_queues > 0); 5465 MPASS(sc->rx_num_queues == nrxqsets); 5466 5467 /* First allocate the top level queue structs */ 5468 if (!(sc->rx_queues = 5469 (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * 5470 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 5471 device_printf(iflib_get_dev(ctx), 5472 "Unable to allocate queue memory\n"); 5473 error = ENOMEM; 5474 goto fail; 5475 } 5476 5477 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 5478 /* Set up some basics */ 5479 struct rx_ring *rxr = &que->rxr; 5480 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 5481 ("%s: misaligned RX AIM snapshot %p", __func__, 5482 &rxr->rx_aim_snapshot)); 5483 rxr->sc = que->sc = sc; 5484 rxr->que = que; 5485 que->me = rxr->me = i; 5486 5487 /* get the virtual and physical address of hardware queues */ 5488 rxr->rx_base = 5489 (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; 5490 rxr->rx_paddr = paddrs[i*nrxqs]; 5491 } 5492 5493 if (bootverbose) 5494 device_printf(iflib_get_dev(ctx), 5495 "allocated for %d rx_queues\n", sc->rx_num_queues); 5496 5497 return (0); 5498 fail: 5499 em_if_queues_free(ctx); 5500 return (error); 5501 } 5502 5503 static void 5504 em_if_queues_free(if_ctx_t ctx) 5505 { 5506 struct e1000_softc *sc = iflib_get_softc(ctx); 5507 struct em_tx_queue *tx_que = sc->tx_queues; 5508 struct em_rx_queue *rx_que = sc->rx_queues; 5509 5510 if (tx_que != NULL) { 5511 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 5512 struct tx_ring *txr = &tx_que->txr; 5513 if (txr->tx_rsq == NULL) 5514 break; 5515 5516 free(txr->tx_rsq, M_DEVBUF); 5517 txr->tx_rsq = NULL; 5518 } 5519 free(sc->tx_queues, M_DEVBUF); 5520 sc->tx_queues = NULL; 5521 } 5522 5523 if (rx_que != NULL) { 5524 free(sc->rx_queues, M_DEVBUF); 5525 sc->rx_queues = NULL; 5526 } 5527 } 5528 5529 static u32 5530 em_legacy_txdctl(struct e1000_hw *hw) 5531 { 5532 u32 txdctl; 5533 5534 /* 5535 * Start with the established full-descriptor writeback policy. 5536 * Several generations have descriptor-queue errata for which it is 5537 * a documented workaround. The unsafe early controllers are 5538 * overridden below. 5539 */ 5540 txdctl = EM_TX_PTHRESH | (EM_TX_HTHRESH << 8) | 5541 (EM_TX_WTHRESH << 16) | E1000_TXDCTL_GRAN; 5542 5543 switch (hw->mac.type) { 5544 case e1000_82571: 5545 case e1000_82572: 5546 case e1000_82573: 5547 case e1000_82574: 5548 case e1000_82583: 5549 case e1000_80003es2lan: 5550 /* Match the Intel shared-code policy for these families. */ 5551 txdctl |= E1000_TXDCTL_COUNT_DESC; 5552 break; 5553 case e1000_ich8lan: 5554 case e1000_ich9lan: 5555 case e1000_ich10lan: 5556 case e1000_pchlan: 5557 case e1000_pch2lan: 5558 case e1000_pch_lpt: 5559 case e1000_pch_spt: 5560 case e1000_pch_cnp: 5561 case e1000_pch_tgp: 5562 case e1000_pch_adp: 5563 case e1000_pch_mtp: 5564 case e1000_pch_ptp: 5565 case e1000_pch_nvp: 5566 /* Preserve the required bit set by the integrated shared code. */ 5567 txdctl |= (1U << 22); 5568 break; 5569 case e1000_82542: 5570 case e1000_82543: 5571 case e1000_82544: 5572 /* 5573 * 82543 erratum 35 and 82544 erratum 20 require 5574 * WTHRESH=0. Leave all descriptor-control thresholds at 5575 * their reset values on these early controllers. 5576 */ 5577 txdctl = 0; 5578 break; 5579 case e1000_82540: 5580 case e1000_82545: 5581 case e1000_82545_rev_3: 5582 case e1000_82546: 5583 case e1000_82546_rev_3: 5584 case e1000_82541: 5585 case e1000_82541_rev_2: 5586 case e1000_82547: 5587 case e1000_82547_rev_2: 5588 break; 5589 default: 5590 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5591 hw->mac.type)); 5592 break; 5593 } 5594 5595 return (txdctl); 5596 } 5597 5598 static u32 5599 igb_txdctl(struct e1000_hw *hw) 5600 { 5601 u32 pthresh; 5602 5603 switch (hw->mac.type) { 5604 case e1000_i354: 5605 pthresh = I354_TX_PTHRESH; 5606 break; 5607 case e1000_82575: 5608 case e1000_82576: 5609 case e1000_82580: 5610 case e1000_i350: 5611 case e1000_i210: 5612 case e1000_i211: 5613 case e1000_vfadapt: 5614 case e1000_vfadapt_i350: 5615 pthresh = IGB_TX_PTHRESH; 5616 break; 5617 default: 5618 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5619 hw->mac.type)); 5620 pthresh = IGB_TX_PTHRESH; 5621 break; 5622 } 5623 5624 return (pthresh | (IGB_TX_HTHRESH << 8) | 5625 E1000_TXDCTL_QUEUE_ENABLE); 5626 } 5627 5628 /********************************************************************* 5629 * 5630 * Enable transmit unit. 5631 * 5632 **********************************************************************/ 5633 void 5634 em_initialize_transmit_rings(if_ctx_t ctx) 5635 { 5636 struct e1000_softc *sc = iflib_get_softc(ctx); 5637 if_softc_ctx_t scctx = sc->shared; 5638 struct em_tx_queue *que; 5639 struct tx_ring *txr; 5640 struct e1000_hw *hw = &sc->hw; 5641 u32 txdctl; 5642 5643 for (int i = 0; i < sc->tx_num_queues; i++) { 5644 u64 bus_addr; 5645 caddr_t offp, endp; 5646 uint32_t qid; 5647 5648 que = &sc->tx_queues[i]; 5649 txr = &que->txr; 5650 qid = txr->me; 5651 bus_addr = txr->tx_paddr; 5652 5653 /* Clear checksum offload context. */ 5654 offp = (caddr_t)txr + offsetof(struct tx_ring, csum_flags); 5655 endp = (caddr_t)(txr + 1); 5656 memset(offp, 0, endp - offp); 5657 5658 if (hw->mac.type >= igb_mac_min) { 5659 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid)); 5660 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), 5661 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE); 5662 E1000_WRITE_FLUSH(hw); 5663 } 5664 5665 /* Base and Len of TX Ring */ 5666 E1000_WRITE_REG(hw, E1000_TDLEN(qid), 5667 scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); 5668 E1000_WRITE_REG(hw, E1000_TDBAH(qid), (u32)(bus_addr >> 32)); 5669 E1000_WRITE_REG(hw, E1000_TDBAL(qid), (u32)bus_addr); 5670 /* Init the HEAD/TAIL indices */ 5671 E1000_WRITE_REG(hw, E1000_TDT(qid), 0); 5672 E1000_WRITE_REG(hw, E1000_TDH(qid), 0); 5673 5674 HW_DEBUGOUT2("Base = %x, Length = %x\n", 5675 E1000_READ_REG(hw, E1000_TDBAL(qid)), 5676 E1000_READ_REG(hw, E1000_TDLEN(qid))); 5677 5678 if (hw->mac.type < igb_mac_min) 5679 txdctl = em_legacy_txdctl(hw); 5680 else 5681 txdctl = igb_txdctl(hw); 5682 5683 E1000_WRITE_REG(hw, E1000_TXDCTL(qid), txdctl); 5684 } 5685 } 5686 5687 static void 5688 em_initialize_transmit_unit(if_ctx_t ctx) 5689 { 5690 struct e1000_softc *sc = iflib_get_softc(ctx); 5691 struct e1000_hw *hw = &sc->hw; 5692 u32 tctl, tarc, tipg = 0; 5693 5694 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 5695 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5696 5697 em_initialize_transmit_rings(ctx); 5698 5699 /* Set the default values for the Tx Inter Packet Gap timer */ 5700 switch (hw->mac.type) { 5701 case e1000_80003es2lan: 5702 tipg = DEFAULT_82543_TIPG_IPGR1; 5703 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 5704 E1000_TIPG_IPGR2_SHIFT; 5705 break; 5706 case e1000_82542: 5707 tipg = DEFAULT_82542_TIPG_IPGT; 5708 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5709 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5710 break; 5711 default: 5712 if (hw->phy.media_type == e1000_media_type_fiber || 5713 hw->phy.media_type == e1000_media_type_internal_serdes) 5714 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 5715 else 5716 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 5717 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 5718 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 5719 } 5720 5721 if (hw->mac.type < igb_mac_min) { 5722 E1000_WRITE_REG(hw, E1000_TIPG, tipg); 5723 E1000_WRITE_REG(hw, E1000_TIDV, sc->tx_int_delay.value); 5724 5725 if (sc->tx_int_delay.value > 0) 5726 sc->txd_cmd |= E1000_TXD_CMD_IDE; 5727 } 5728 5729 if (hw->mac.type >= e1000_82540 && hw->mac.type < igb_mac_min) 5730 E1000_WRITE_REG(hw, E1000_TADV, sc->tx_abs_int_delay.value); 5731 5732 if (hw->mac.type == e1000_82571 || hw->mac.type == e1000_82572) { 5733 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5734 tarc |= TARC_SPEED_MODE_BIT; 5735 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5736 } else if (hw->mac.type == e1000_80003es2lan) { 5737 /* errata: program both queues to unweighted RR */ 5738 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5739 tarc |= 1; 5740 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5741 tarc = E1000_READ_REG(hw, E1000_TARC(1)); 5742 tarc |= 1; 5743 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5744 } else if (hw->mac.type == e1000_82574) { 5745 tarc = E1000_READ_REG(hw, E1000_TARC(0)); 5746 tarc |= TARC_ERRATA_BIT; 5747 if ( sc->tx_num_queues > 1) { 5748 tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); 5749 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5750 E1000_WRITE_REG(hw, E1000_TARC(1), tarc); 5751 } else 5752 E1000_WRITE_REG(hw, E1000_TARC(0), tarc); 5753 } 5754 5755 /* Program the Transmit Control Register */ 5756 tctl = E1000_READ_REG(hw, E1000_TCTL); 5757 tctl &= ~E1000_TCTL_CT; 5758 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 5759 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 5760 5761 if (hw->mac.type >= e1000_82571 && hw->mac.type < igb_mac_min) 5762 tctl |= E1000_TCTL_MULR; 5763 5764 /* This write will effectively turn on the transmit unit. */ 5765 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 5766 5767 /* SPT and KBL errata workarounds */ 5768 if (hw->mac.type == e1000_pch_spt) { 5769 u32 reg; 5770 reg = E1000_READ_REG(hw, E1000_IOSFPC); 5771 reg |= E1000_RCTL_RDMTS_HEX; 5772 E1000_WRITE_REG(hw, E1000_IOSFPC, reg); 5773 /* i218-i219 Specification Update 1.5.4.5 */ 5774 reg = E1000_READ_REG(hw, E1000_TARC(0)); 5775 reg &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 5776 reg |= E1000_TARC0_CB_MULTIQ_2_REQ; 5777 E1000_WRITE_REG(hw, E1000_TARC(0), reg); 5778 } 5779 } 5780 5781 /********************************************************************* 5782 * 5783 * Enable receive unit. 5784 * 5785 **********************************************************************/ 5786 #define BSIZEPKT_ROUNDUP ((1<<E1000_SRRCTL_BSIZEPKT_SHIFT)-1) 5787 5788 static u32 5789 igb_rxdctl(struct e1000_softc *sc, u32 rxdctl) 5790 { 5791 struct e1000_hw *hw; 5792 u32 mask, pthresh, wthresh; 5793 5794 hw = &sc->hw; 5795 mask = IGB_RXDCTL_THRESH_MASK; 5796 switch (hw->mac.type) { 5797 case e1000_82575: 5798 mask = IGB_82575_RXDCTL_THRESH_MASK; 5799 pthresh = IGB_RX_PTHRESH; 5800 wthresh = IGB_RX_WTHRESH; 5801 break; 5802 case e1000_82576: 5803 pthresh = IGB_RX_PTHRESH; 5804 wthresh = sc->intr_type == IFLIB_INTR_MSIX ? 5805 IGB_82576_RX_WTHRESH : IGB_RX_WTHRESH; 5806 break; 5807 case e1000_vfadapt: 5808 /* 82576 VFs always need the MSI-X writeback workaround. */ 5809 pthresh = IGB_RX_PTHRESH; 5810 wthresh = IGB_82576_RX_WTHRESH; 5811 break; 5812 case e1000_i354: 5813 pthresh = I354_RX_PTHRESH; 5814 wthresh = IGB_RX_WTHRESH; 5815 break; 5816 case e1000_82580: 5817 case e1000_i350: 5818 case e1000_i210: 5819 case e1000_i211: 5820 case e1000_vfadapt_i350: 5821 pthresh = IGB_RX_PTHRESH; 5822 wthresh = IGB_RX_WTHRESH; 5823 break; 5824 default: 5825 KASSERT(0, ("%s: unsupported MAC type %d", __func__, 5826 hw->mac.type)); 5827 pthresh = IGB_RX_PTHRESH; 5828 wthresh = IGB_RX_WTHRESH; 5829 break; 5830 } 5831 5832 rxdctl &= ~mask; 5833 rxdctl |= pthresh | (IGB_RX_HTHRESH << 8) | 5834 (wthresh << 16) | E1000_RXDCTL_QUEUE_ENABLE; 5835 return (rxdctl); 5836 } 5837 5838 void 5839 igb_initialize_receive_rings(if_ctx_t ctx, bool drop) 5840 { 5841 struct e1000_softc *sc = iflib_get_softc(ctx); 5842 if_softc_ctx_t scctx = sc->shared; 5843 struct e1000_hw *hw = &sc->hw; 5844 struct em_rx_queue *que; 5845 u32 srrctl; 5846 5847 srrctl = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 5848 E1000_SRRCTL_BSIZEPKT_SHIFT; 5849 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 5850 if (drop) 5851 srrctl |= E1000_SRRCTL_DROP_EN; 5852 5853 for (int i = 0; i < sc->rx_num_queues; i++) { 5854 struct rx_ring *rxr; 5855 u64 bus_addr; 5856 u32 rxdctl; 5857 uint32_t qid; 5858 5859 que = &sc->rx_queues[i]; 5860 rxr = &que->rxr; 5861 bus_addr = rxr->rx_paddr; 5862 qid = rxr->me; 5863 5864 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid)); 5865 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), 5866 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE); 5867 E1000_WRITE_FLUSH(hw); 5868 5869 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 5870 scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); 5871 E1000_WRITE_REG(hw, E1000_RDBAH(qid), 5872 (uint32_t)(bus_addr >> 32)); 5873 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (uint32_t)bus_addr); 5874 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 5875 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 5876 E1000_WRITE_REG(hw, E1000_SRRCTL(qid), srrctl); 5877 5878 rxdctl = igb_rxdctl(sc, rxdctl); 5879 E1000_WRITE_REG(hw, E1000_RXDCTL(qid), rxdctl); 5880 } 5881 } 5882 5883 static bool 5884 em_integrated_jumbo_rx(struct e1000_hw *hw) 5885 { 5886 switch (hw->mac.type) { 5887 case e1000_ich9lan: 5888 case e1000_ich10lan: 5889 case e1000_pchlan: 5890 case e1000_pch2lan: 5891 case e1000_pch_lpt: 5892 case e1000_pch_spt: 5893 case e1000_pch_cnp: 5894 case e1000_pch_tgp: 5895 case e1000_pch_adp: 5896 case e1000_pch_mtp: 5897 case e1000_pch_ptp: 5898 case e1000_pch_nvp: 5899 return (true); 5900 default: 5901 return (false); 5902 } 5903 } 5904 5905 static void 5906 em_initialize_receive_unit(if_ctx_t ctx) 5907 { 5908 struct e1000_softc *sc = iflib_get_softc(ctx); 5909 if_softc_ctx_t scctx = sc->shared; 5910 if_t ifp = iflib_get_ifp(ctx); 5911 struct e1000_hw *hw = &sc->hw; 5912 struct em_rx_queue *que; 5913 int i; 5914 uint32_t rctl, rxcsum; 5915 5916 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 5917 KASSERT(!sc->vf_ifp, ("%s called for a VF", __func__)); 5918 5919 /* 5920 * Make sure receives are disabled while setting up the descriptor 5921 * ring. 5922 */ 5923 rctl = E1000_READ_REG(hw, E1000_RCTL); 5924 /* Do not disable if ever enabled on this hardware. */ 5925 if (hw->mac.type != e1000_82574 && 5926 hw->mac.type != e1000_82583) 5927 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 5928 5929 /* Setup the Receive Control Register. */ 5930 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 5931 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 5932 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 5933 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 5934 rctl &= ~E1000_RCTL_SBP; 5935 5936 if (igb_iov_enabled(sc) || if_getmtu(ifp) > ETHERMTU) 5937 rctl |= E1000_RCTL_LPE; 5938 else 5939 rctl &= ~E1000_RCTL_LPE; 5940 if (!em_disable_crc_stripping) 5941 rctl |= E1000_RCTL_SECRC; 5942 5943 /* lem/em default interrupt moderation */ 5944 if (hw->mac.type < igb_mac_min) { 5945 if (hw->mac.type >= e1000_82540) { 5946 E1000_WRITE_REG(hw, E1000_RADV, 5947 sc->rx_abs_int_delay.value); 5948 5949 /* Set the default interrupt throttling rate */ 5950 E1000_WRITE_REG(hw, E1000_ITR, 5951 EM_INTS_TO_ITR(em_max_interrupt_rate)); 5952 5953 /* 5954 * The 82574 MSI-X EITR registers are programmed 5955 * with the same value further below. Either way 5956 * the hardware now holds the default rate, so seed 5957 * the software copy to match; otherwise a stale 5958 * itr_setting left over from AIM makes em_newitr() 5959 * skip the write that would restore it. 5960 */ 5961 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; 5962 i++, que++) 5963 que->itr_setting = 5964 EM_INTS_TO_ITR(em_max_interrupt_rate); 5965 } 5966 5967 /* XXX TEMPORARY WORKAROUND: on some systems with 82573 5968 * long latencies are observed, like Lenovo X60. This 5969 * change eliminates the problem, but since having positive 5970 * values in RDTR is a known source of problems on other 5971 * platforms another solution is being sought. 5972 */ 5973 if (hw->mac.type == e1000_82573) 5974 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 5975 else 5976 E1000_WRITE_REG(hw, E1000_RDTR, 5977 sc->rx_int_delay.value); 5978 } 5979 5980 if (hw->mac.type >= em_mac_min) { 5981 uint32_t rfctl; 5982 /* Use extended rx descriptor formats */ 5983 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 5984 rfctl |= E1000_RFCTL_EXTEN; 5985 5986 /* 5987 * When using MSI-X interrupts we need to throttle 5988 * using the EITR register (82574 only) 5989 */ 5990 if (hw->mac.type == e1000_82574) { 5991 for (int i = 0; i < 4; i++) 5992 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 5993 EM_INTS_TO_ITR(em_max_interrupt_rate)); 5994 /* Disable accelerated acknowledge */ 5995 rfctl |= E1000_RFCTL_ACK_DIS; 5996 } 5997 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 5998 } 5999 6000 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 6001 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 6002 rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; 6003 if (hw->mac.type > e1000_82575) 6004 rxcsum |= E1000_RXCSUM_CRCOFL; 6005 else if (hw->mac.type < em_mac_min && 6006 if_getcapenable(ifp) & IFCAP_HWCSUM_IPV6) 6007 rxcsum |= E1000_RXCSUM_IPV6OFL; 6008 } else { 6009 rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 6010 if (hw->mac.type > e1000_82575) 6011 rxcsum &= ~E1000_RXCSUM_CRCOFL; 6012 else if (hw->mac.type < em_mac_min) 6013 rxcsum &= ~E1000_RXCSUM_IPV6OFL; 6014 } 6015 6016 if (sc->rx_num_queues > 1) { 6017 /* RSS hash needed in the Rx descriptor */ 6018 rxcsum |= E1000_RXCSUM_PCSD; 6019 6020 if (hw->mac.type >= igb_mac_min) 6021 igb_initialize_rss_mapping(sc); 6022 else 6023 em_initialize_rss_mapping(sc); 6024 } 6025 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 6026 6027 for (i = 0, que = sc->rx_queues; 6028 hw->mac.type < igb_mac_min && i < sc->rx_num_queues; 6029 i++, que++) { 6030 struct rx_ring *rxr = &que->rxr; 6031 /* Setup the Base and Length of the Rx Descriptor Ring */ 6032 u64 bus_addr = rxr->rx_paddr; 6033 uint32_t qid = rxr->me; 6034 #if 0 6035 u32 rdt = sc->rx_num_queues -1; /* default */ 6036 #endif 6037 6038 E1000_WRITE_REG(hw, E1000_RDLEN(qid), 6039 scctx->isc_nrxd[0] * 6040 sizeof(union e1000_rx_desc_extended)); 6041 E1000_WRITE_REG(hw, E1000_RDBAH(qid), (u32)(bus_addr >> 32)); 6042 E1000_WRITE_REG(hw, E1000_RDBAL(qid), (u32)bus_addr); 6043 /* Setup the Head and Tail Descriptor Pointers */ 6044 E1000_WRITE_REG(hw, E1000_RDH(qid), 0); 6045 E1000_WRITE_REG(hw, E1000_RDT(qid), 0); 6046 } 6047 6048 /* Increase receive-descriptor prefetching for integrated jumbo MACs. */ 6049 if (em_integrated_jumbo_rx(hw) && if_getmtu(ifp) > ETHERMTU) { 6050 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 6051 6052 rxdctl &= ~(EM_RXDCTL_PTHRESH_MASK | 6053 EM_RXDCTL_HTHRESH_MASK); 6054 rxdctl |= EM_JUMBO_RX_PTHRESH | 6055 (EM_JUMBO_RX_HTHRESH << 8); 6056 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); 6057 } else if (hw->mac.type == e1000_82574) { 6058 /* RXDCTL(0) writes are mirrored to RXDCTL(1) on 82574. */ 6059 for (int i = 0; i < sc->rx_num_queues; i++) { 6060 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 6061 6062 rxdctl &= ~EM_RXDCTL_THRESH_MASK; 6063 rxdctl |= EM_82574_RX_PTHRESH | 6064 (EM_82574_RX_HTHRESH << 8) | 6065 (EM_82574_RX_WTHRESH << 16) | 6066 E1000_RXDCTL_THRESH_UNIT_DESC; 6067 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 6068 } 6069 } else if (hw->mac.type >= igb_mac_min) { 6070 bool drop; 6071 u32 psize; 6072 6073 if (igb_iov_enabled(sc)) { 6074 E1000_WRITE_REG(hw, E1000_RLPML, 6075 IGB_IOV_MAX_FRAME_SIZE); 6076 } else if (if_getmtu(ifp) > ETHERMTU) { 6077 psize = scctx->isc_max_frame_size; 6078 /* are we on a vlan? */ 6079 if (if_vlantrunkinuse(ifp)) 6080 psize += VLAN_TAG_SIZE; 6081 6082 E1000_WRITE_REG(hw, E1000_RLPML, psize); 6083 } 6084 6085 /* 6086 * If TX flow control is disabled and there's >1 queue 6087 * defined, enable DROP. 6088 * 6089 * This drops frames rather than hanging the RX MAC for all 6090 * queues. 6091 */ 6092 drop = igb_iov_enabled(sc) || 6093 ((sc->rx_num_queues > 1) && 6094 (sc->fc == e1000_fc_none || 6095 sc->fc == e1000_fc_rx_pause)); 6096 igb_initialize_receive_rings(ctx, drop); 6097 } else if (hw->mac.type >= e1000_pch2lan) { 6098 if (if_getmtu(ifp) > ETHERMTU) 6099 e1000_lv_jumbo_workaround_ich8lan(hw, true); 6100 else 6101 e1000_lv_jumbo_workaround_ich8lan(hw, false); 6102 } 6103 6104 /* Make sure VLAN Filters are off */ 6105 rctl &= ~E1000_RCTL_VFE; 6106 6107 /* Set up packet buffer size, overridden by per queue srrctl on igb */ 6108 if (hw->mac.type < igb_mac_min) { 6109 if (sc->rx_mbuf_sz > 2048 && sc->rx_mbuf_sz <= 4096) 6110 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 6111 else if (sc->rx_mbuf_sz > 4096 && sc->rx_mbuf_sz <= 8192) 6112 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 6113 else if (sc->rx_mbuf_sz > 8192) 6114 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX; 6115 else { 6116 rctl |= E1000_RCTL_SZ_2048; 6117 rctl &= ~E1000_RCTL_BSEX; 6118 } 6119 } else 6120 rctl |= E1000_RCTL_SZ_2048; 6121 6122 /* 6123 * rctl bits 11:10 are as follows 6124 * lem: reserved 6125 * em: DTYPE 6126 * igb: reserved 6127 * and should be 00 on all of the above 6128 */ 6129 rctl &= ~0x00000C00; 6130 6131 /* Write out the settings */ 6132 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 6133 6134 return; 6135 } 6136 6137 static void 6138 em_if_vlan_register(if_ctx_t ctx, u16 vtag) 6139 { 6140 struct e1000_softc *sc = iflib_get_softc(ctx); 6141 bool present; 6142 u32 index, mask; 6143 6144 index = (vtag >> 5) & 0x7F; 6145 mask = 1U << (vtag & 0x1F); 6146 present = (sc->shadow_vfta[index] & mask) != 0; 6147 /* 6148 * On a VF, record registration intent for replay even if the PF is not 6149 * ready to accept it yet. 6150 */ 6151 sc->shadow_vfta[index] |= mask; 6152 sc->vf_vfta_stale[index] &= ~mask; 6153 if (!present) 6154 ++sc->num_vlans; 6155 if (sc->vf_ifp && 6156 e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) { 6157 igbv_vlan_retry_add(sc, vtag); 6158 device_printf(sc->dev, 6159 "VF VLAN %u add request failed\n", vtag); 6160 } else if (sc->vf_ifp) 6161 igbv_vlan_retry_clear(sc, vtag); 6162 if (!sc->vf_ifp) { 6163 if (igb_iov_enabled(sc)) 6164 igb_iov_rebuild_vlan(sc); 6165 else 6166 em_if_vlan_filter_write(sc, index); 6167 } 6168 } 6169 6170 static void 6171 em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 6172 { 6173 struct e1000_softc *sc = iflib_get_softc(ctx); 6174 bool present; 6175 u32 index, mask; 6176 6177 index = (vtag >> 5) & 0x7F; 6178 mask = 1U << (vtag & 0x1F); 6179 present = (sc->shadow_vfta[index] & mask) != 0; 6180 if (sc->vf_ifp) 6181 igbv_vlan_retry_clear(sc, vtag); 6182 if (sc->vf_ifp && 6183 e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) { 6184 device_printf(sc->dev, 6185 "VF VLAN %u remove request failed\n", vtag); 6186 /* 6187 * Hardware might still admit this VID. Preserve its receive 6188 * tag until a successful VF reset proves the stale filter gone. 6189 */ 6190 sc->vf_vfta_stale[index] |= mask; 6191 } else { 6192 sc->vf_vfta_stale[index] &= ~mask; 6193 } 6194 sc->shadow_vfta[index] &= ~mask; 6195 if (present) 6196 --sc->num_vlans; 6197 if (!sc->vf_ifp) { 6198 if (igb_iov_enabled(sc)) 6199 igb_iov_rebuild_vlan(sc); 6200 else 6201 em_if_vlan_filter_write(sc, index); 6202 } 6203 } 6204 6205 static bool 6206 em_if_vlan_filter_capable(if_ctx_t ctx) 6207 { 6208 if_t ifp = iflib_get_ifp(ctx); 6209 6210 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) && 6211 !em_disable_crc_stripping) 6212 return (true); 6213 6214 return (false); 6215 } 6216 6217 static bool 6218 em_if_vlan_filter_used(if_ctx_t ctx) 6219 { 6220 struct e1000_softc *sc = iflib_get_softc(ctx); 6221 6222 if (!em_if_vlan_filter_capable(ctx)) 6223 return (false); 6224 6225 for (int i = 0; i < EM_VFTA_SIZE; i++) 6226 if (sc->shadow_vfta[i] != 0) 6227 return (true); 6228 6229 return (false); 6230 } 6231 6232 static void 6233 em_if_vlan_filter_enable(struct e1000_softc *sc) 6234 { 6235 struct e1000_hw *hw = &sc->hw; 6236 u32 reg; 6237 6238 reg = E1000_READ_REG(hw, E1000_RCTL); 6239 reg &= ~E1000_RCTL_CFIEN; 6240 reg |= E1000_RCTL_VFE; 6241 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6242 } 6243 6244 static void 6245 em_if_vlan_filter_disable(struct e1000_softc *sc) 6246 { 6247 struct e1000_hw *hw = &sc->hw; 6248 u32 reg; 6249 6250 reg = E1000_READ_REG(hw, E1000_RCTL); 6251 reg &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 6252 E1000_WRITE_REG(hw, E1000_RCTL, reg); 6253 } 6254 6255 static void 6256 em_if_vlan_filter_write(struct e1000_softc *sc, int changed_index) 6257 { 6258 struct e1000_hw *hw = &sc->hw; 6259 6260 KASSERT(!sc->vf_ifp, ("VLAN filter write on VF\n")); 6261 6262 /* Disable interrupts for lem(4) devices during the filter change */ 6263 if (hw->mac.type < em_mac_min) 6264 em_if_intr_disable(sc->ctx); 6265 6266 /* 6267 * Restore every retained VLAN after reset. Also write the changed 6268 * word when its final VLAN was removed so stale hardware membership 6269 * does not survive a zero shadow value. 6270 */ 6271 for (int i = 0; i < EM_VFTA_SIZE; i++) 6272 if (sc->shadow_vfta[i] != 0 || i == changed_index) 6273 e1000_write_vfta(hw, i, sc->shadow_vfta[i]); 6274 6275 /* Re-enable interrupts for lem-class devices */ 6276 if (hw->mac.type < em_mac_min) 6277 em_if_intr_enable(sc->ctx); 6278 } 6279 6280 static void 6281 em_setup_vlan_hw_support(if_ctx_t ctx) 6282 { 6283 struct e1000_softc *sc = iflib_get_softc(ctx); 6284 struct e1000_hw *hw = &sc->hw; 6285 if_t ifp = iflib_get_ifp(ctx); 6286 s32 error; 6287 u32 max_frame_size, reg; 6288 u16 vid; 6289 int restore_failures; 6290 6291 /* 6292 * Only PFs have control over VLAN HW filtering 6293 * configuration. VFs have to act as if it's always 6294 * enabled. 6295 */ 6296 if (sc->vf_ifp) { 6297 max_frame_size = min(sc->shared->isc_max_frame_size + 6298 VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE); 6299 e1000_rlpml_set_vf(hw, max_frame_size); 6300 restore_failures = 0; 6301 for (vid = 0; vid < 4096; vid++) { 6302 if ((sc->shadow_vfta[vid >> 5] & 6303 (1U << (vid & 0x1f))) == 0) 6304 continue; 6305 /* 6306 * Desired state remains in shadow_vfta for the next 6307 * replay if the PF mailbox is absent during reset. 6308 */ 6309 error = e1000_vfta_set_vf(hw, vid, true); 6310 if (error != E1000_SUCCESS) { 6311 igbv_vlan_retry_add(sc, vid); 6312 restore_failures++; 6313 } else 6314 igbv_vlan_retry_clear(sc, vid); 6315 } 6316 if (restore_failures != 0) 6317 device_printf(sc->dev, 6318 "VF VLAN restore failed for %d VIDs; retrying\n", 6319 restore_failures); 6320 return; 6321 } 6322 6323 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 6324 !em_disable_crc_stripping) { 6325 reg = E1000_READ_REG(hw, E1000_CTRL); 6326 reg |= E1000_CTRL_VME; 6327 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6328 } else { 6329 reg = E1000_READ_REG(hw, E1000_CTRL); 6330 reg &= ~E1000_CTRL_VME; 6331 E1000_WRITE_REG(hw, E1000_CTRL, reg); 6332 } 6333 6334 /* 6335 * SR-IOV always needs VFE for VF isolation. When PF hardware VLAN 6336 * filtering is disabled, the IOV VLAN rebuild instead makes the PF 6337 * VLAN-promiscuous without disabling the global filter. 6338 */ 6339 if (!em_if_vlan_filter_capable(ctx)) { 6340 if (igb_iov_enabled(sc)) { 6341 #ifdef PCI_IOV 6342 sc->iov_pf_vlan_promisc = true; 6343 #endif 6344 em_if_vlan_filter_enable(sc); 6345 } else 6346 em_if_vlan_filter_disable(sc); 6347 return; 6348 } 6349 #ifdef PCI_IOV 6350 if (igb_iov_enabled(sc)) 6351 sc->iov_pf_vlan_promisc = false; 6352 #endif 6353 6354 /* 6355 * A soft reset zero's out the VFTA, so 6356 * we need to repopulate it now. 6357 * We also insert VLAN 0 in the filter list, so we pass VLAN 0 tagged 6358 * traffic through. This will write the entire table. 6359 */ 6360 em_if_vlan_register(ctx, 0); 6361 6362 /* Enable the Filter Table */ 6363 em_if_vlan_filter_enable(sc); 6364 } 6365 6366 static void 6367 em_if_intr_enable(if_ctx_t ctx) 6368 { 6369 struct e1000_softc *sc = iflib_get_softc(ctx); 6370 struct e1000_hw *hw = &sc->hw; 6371 u32 ims_mask = IMS_ENABLE_MASK | em_fatal_error_intr_mask(sc); 6372 6373 if (sc->intr_type == IFLIB_INTR_MSIX) { 6374 E1000_WRITE_REG(hw, EM_EIAC, sc->ims); 6375 ims_mask |= sc->ims; 6376 } 6377 6378 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 6379 E1000_WRITE_FLUSH(hw); 6380 } 6381 6382 static void 6383 em_if_intr_disable(if_ctx_t ctx) 6384 { 6385 struct e1000_softc *sc = iflib_get_softc(ctx); 6386 struct e1000_hw *hw = &sc->hw; 6387 6388 if (sc->intr_type == IFLIB_INTR_MSIX) 6389 E1000_WRITE_REG(hw, EM_EIAC, 0); 6390 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6391 E1000_WRITE_FLUSH(hw); 6392 } 6393 6394 static void 6395 igb_if_intr_enable(if_ctx_t ctx) 6396 { 6397 struct e1000_softc *sc = iflib_get_softc(ctx); 6398 struct e1000_hw *hw = &sc->hw; 6399 u32 mask, reg; 6400 6401 if (igb_device_reset_pending(sc)) 6402 return; 6403 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6404 mask = (sc->que_mask | sc->link_mask); 6405 /* 6406 * VF interrupt controls are also mapped into these registers. 6407 * Preserve them and change only the PF vectors we own. 6408 */ 6409 reg = E1000_READ_REG(hw, E1000_EIAC); 6410 E1000_WRITE_REG(hw, E1000_EIAC, reg | mask); 6411 reg = E1000_READ_REG(hw, E1000_EIAM); 6412 E1000_WRITE_REG(hw, E1000_EIAM, reg | mask); 6413 igb_iov_intr_drain_stale(sc); 6414 E1000_WRITE_REG(hw, E1000_EIMS, mask); 6415 E1000_WRITE_REG(hw, E1000_IMS, 6416 E1000_IMS_LSC | igb_device_reset_intr_mask(sc) | 6417 igb_iov_intr_mask(sc) | 6418 em_fatal_error_intr_mask(sc)); 6419 } else { 6420 mask = IMS_ENABLE_MASK | igb_device_reset_intr_mask(sc) | 6421 em_fatal_error_intr_mask(sc); 6422 /* Reading ICR masks every shared interrupt before the filter runs. */ 6423 E1000_WRITE_REG(hw, E1000_IAM, mask); 6424 E1000_WRITE_REG(hw, E1000_IMS, mask); 6425 } 6426 E1000_WRITE_FLUSH(hw); 6427 } 6428 6429 static void 6430 igb_if_intr_disable(if_ctx_t ctx) 6431 { 6432 struct e1000_softc *sc = iflib_get_softc(ctx); 6433 struct e1000_hw *hw = &sc->hw; 6434 u32 mask, reg; 6435 6436 /* This is the first CTX-owned register access after ICR.DRSTA. */ 6437 igb_prepare_device_reset(sc); 6438 6439 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 6440 /* 6441 * Do not use a blanket EIMC write here. VF interrupt controls 6442 * are mapped into the same PF register space, so clearing bits 6443 * we do not own can leave running VFs with interrupts masked. 6444 * Before initial queue configuration the owned mask is zero 6445 * because this driver has not enabled a vector yet. 6446 */ 6447 mask = (sc->que_mask | sc->link_mask); 6448 reg = E1000_READ_REG(hw, E1000_EIAM); 6449 E1000_WRITE_REG(hw, E1000_EIAM, reg & ~mask); 6450 E1000_WRITE_REG(hw, E1000_EIMC, mask); 6451 reg = E1000_READ_REG(hw, E1000_EIAC); 6452 E1000_WRITE_REG(hw, E1000_EIAC, reg & ~mask); 6453 } else 6454 E1000_WRITE_REG(hw, E1000_IAM, 0); 6455 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); 6456 E1000_WRITE_FLUSH(hw); 6457 } 6458 6459 /* 6460 * Bit of a misnomer, what this really means is 6461 * to enable OS management of the system... aka 6462 * to disable special hardware management features 6463 */ 6464 static void 6465 em_init_manageability(struct e1000_softc *sc) 6466 { 6467 /* A shared code workaround */ 6468 #define E1000_82542_MANC2H E1000_MANC2H 6469 if (sc->has_manage) { 6470 int manc2h = E1000_READ_REG(&sc->hw, E1000_MANC2H); 6471 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6472 6473 /* disable hardware interception of ARP */ 6474 manc &= ~(E1000_MANC_ARP_EN); 6475 6476 /* enable receiving management packets to the host */ 6477 manc |= E1000_MANC_EN_MNG2HOST; 6478 #define E1000_MNG2HOST_PORT_623 (1 << 5) 6479 #define E1000_MNG2HOST_PORT_664 (1 << 6) 6480 manc2h |= E1000_MNG2HOST_PORT_623; 6481 manc2h |= E1000_MNG2HOST_PORT_664; 6482 E1000_WRITE_REG(&sc->hw, E1000_MANC2H, manc2h); 6483 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6484 } 6485 } 6486 6487 /* 6488 * Give control back to hardware management 6489 * controller if there is one. 6490 */ 6491 static void 6492 em_release_manageability(struct e1000_softc *sc) 6493 { 6494 if (sc->has_manage) { 6495 int manc = E1000_READ_REG(&sc->hw, E1000_MANC); 6496 6497 /* re-enable hardware interception of ARP */ 6498 manc |= E1000_MANC_ARP_EN; 6499 manc &= ~E1000_MANC_EN_MNG2HOST; 6500 6501 E1000_WRITE_REG(&sc->hw, E1000_MANC, manc); 6502 } 6503 } 6504 6505 /* 6506 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 6507 * For ASF and Pass Through versions of f/w this means 6508 * that the driver is loaded. For AMT version type f/w 6509 * this means that the network i/f is open. 6510 */ 6511 static void 6512 em_get_hw_control(struct e1000_softc *sc) 6513 { 6514 u32 ctrl_ext, swsm; 6515 6516 if (sc->vf_ifp) 6517 return; 6518 6519 if (sc->hw.mac.type == e1000_82573) { 6520 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6521 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6522 swsm | E1000_SWSM_DRV_LOAD); 6523 return; 6524 } 6525 /* else */ 6526 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6527 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6528 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 6529 } 6530 6531 /* 6532 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 6533 * For ASF and Pass Through versions of f/w this means that 6534 * the driver is no longer loaded. For AMT versions of the 6535 * f/w this means that the network i/f is closed. 6536 */ 6537 static void 6538 em_release_hw_control(struct e1000_softc *sc) 6539 { 6540 u32 ctrl_ext, swsm; 6541 6542 if (!sc->has_manage) 6543 return; 6544 6545 if (sc->hw.mac.type == e1000_82573) { 6546 swsm = E1000_READ_REG(&sc->hw, E1000_SWSM); 6547 E1000_WRITE_REG(&sc->hw, E1000_SWSM, 6548 swsm & ~E1000_SWSM_DRV_LOAD); 6549 return; 6550 } 6551 /* else */ 6552 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6553 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, 6554 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 6555 return; 6556 } 6557 6558 bool 6559 em_is_valid_ether_addr(const u8 *addr) 6560 { 6561 static const u8 zero_addr[ETHER_ADDR_LEN]; 6562 6563 return (!ETHER_IS_MULTICAST(addr) && 6564 memcmp(addr, zero_addr, ETHER_ADDR_LEN) != 0); 6565 } 6566 6567 static bool 6568 em_automask_tso(if_ctx_t ctx) 6569 { 6570 struct e1000_softc *sc = iflib_get_softc(ctx); 6571 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6572 if_t ifp = iflib_get_ifp(ctx); 6573 bool reset_needed; 6574 int drvflags; 6575 6576 if (!em_unsupported_tso && sc->link_speed && 6577 sc->link_speed != SPEED_1000 && 6578 scctx->isc_capenable & IFCAP_TSO) { 6579 device_printf(sc->dev, 6580 "Disabling TSO for 10/100 Ethernet.\n"); 6581 sc->tso_automasked = scctx->isc_capenable & IFCAP_TSO; 6582 scctx->isc_capenable &= ~IFCAP_TSO; 6583 if_setcapenablebit(ifp, 0, IFCAP_TSO); 6584 } else if (sc->link_speed == SPEED_1000 && sc->tso_automasked) { 6585 device_printf(sc->dev, "Re-enabling TSO for GbE.\n"); 6586 scctx->isc_capenable |= sc->tso_automasked; 6587 if_setcapenablebit(ifp, sc->tso_automasked, 0); 6588 sc->tso_automasked = 0; 6589 } else { 6590 return (false); 6591 } 6592 6593 /* 6594 * Reset a running interface, or one being initialized while 6595 * administratively up. OACTIVE remains set after iflib_stop(), so 6596 * it alone cannot distinguish initialization from an interface that 6597 * is down. In other states, the next initialization will apply the 6598 * updated capabilities. 6599 */ 6600 drvflags = if_getdrvflags(ifp); 6601 reset_needed = (drvflags & IFF_DRV_RUNNING) != 0 || 6602 ((drvflags & IFF_DRV_OACTIVE) != 0 && 6603 (if_getflags(ifp) & IFF_UP) != 0); 6604 if (!reset_needed) 6605 return (false); 6606 6607 /* iflib_init_locked handles ifnet hwassistbits */ 6608 iflib_request_reset(ctx); 6609 return (true); 6610 } 6611 6612 /* 6613 ** Parse the interface capabilities with regard 6614 ** to both system management and wake-on-lan for 6615 ** later use. 6616 */ 6617 static void 6618 em_get_wakeup(if_ctx_t ctx) 6619 { 6620 struct e1000_softc *sc = iflib_get_softc(ctx); 6621 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 6622 device_t dev = iflib_get_dev(ctx); 6623 u16 eeprom_data = 0, device_id, apme_mask; 6624 bool apme; 6625 int error, wol_capabilities; 6626 6627 sc->has_manage = e1000_enable_mng_pass_thru(&sc->hw); 6628 apme_mask = EM_EEPROM_APME_HIGH; 6629 error = E1000_SUCCESS; 6630 6631 switch (sc->hw.mac.type) { 6632 case e1000_82542: 6633 case e1000_82543: 6634 break; 6635 case e1000_82544: 6636 error = e1000_read_nvm(&sc->hw, 6637 NVM_INIT_CONTROL2_REG, 1, &eeprom_data); 6638 apme_mask = EM_EEPROM_APME_LOW; 6639 break; 6640 case e1000_82541: 6641 case e1000_82547: 6642 error = e1000_read_nvm(&sc->hw, 6643 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6644 /* The EI parts place APM Enable in the low byte. */ 6645 if (sc->hw.device_id != E1000_DEV_ID_82541ER_LOM) 6646 apme_mask = EM_EEPROM_APME_LOW; 6647 break; 6648 case e1000_82546: 6649 case e1000_82546_rev_3: 6650 if (sc->hw.bus.func == 1) { 6651 error = e1000_read_nvm(&sc->hw, 6652 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6653 break; 6654 } else 6655 error = e1000_read_nvm(&sc->hw, 6656 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6657 break; 6658 case e1000_82573: 6659 case e1000_82583: 6660 sc->has_amt = true; 6661 /* FALLTHROUGH */ 6662 case e1000_82571: 6663 case e1000_82572: 6664 case e1000_80003es2lan: 6665 if (sc->hw.bus.func == 1) { 6666 error = e1000_read_nvm(&sc->hw, 6667 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6668 break; 6669 } else 6670 error = e1000_read_nvm(&sc->hw, 6671 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6672 break; 6673 case e1000_ich8lan: 6674 case e1000_ich9lan: 6675 case e1000_ich10lan: 6676 case e1000_pchlan: 6677 case e1000_pch2lan: 6678 case e1000_pch_lpt: 6679 case e1000_pch_spt: 6680 case e1000_pch_cnp: 6681 case e1000_pch_tgp: 6682 case e1000_pch_adp: 6683 case e1000_pch_mtp: 6684 case e1000_pch_ptp: 6685 case e1000_pch_nvp: 6686 apme_mask = E1000_WUC_APME; 6687 sc->has_amt = true; 6688 eeprom_data = E1000_READ_REG(&sc->hw, E1000_WUC); 6689 if (sc->hw.mac.type > e1000_ich10lan && 6690 (eeprom_data & E1000_WUC_PHY_WAKE) != 0) 6691 sc->wol_phy_wakeup = true; 6692 break; 6693 case e1000_82575: 6694 case e1000_82576: 6695 if (sc->hw.bus.func == 1) 6696 error = e1000_read_nvm(&sc->hw, 6697 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 6698 else 6699 error = e1000_read_nvm(&sc->hw, 6700 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6701 sc->has_amt = true; 6702 break; 6703 case e1000_82580: 6704 case e1000_i350: 6705 case e1000_i354: 6706 case e1000_i210: 6707 case e1000_i211: 6708 error = e1000_read_nvm(&sc->hw, 6709 NVM_INIT_CONTROL3_PORT_A + 6710 NVM_82580_LAN_FUNC_OFFSET(sc->hw.bus.func), 1, 6711 &eeprom_data); 6712 sc->has_amt = true; 6713 break; 6714 default: 6715 error = e1000_read_nvm(&sc->hw, 6716 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 6717 break; 6718 } 6719 if (error != E1000_SUCCESS && bootverbose) 6720 device_printf(dev, "NVM read failed while checking WoL: %d\n", 6721 error); 6722 if ((sc->hw.mac.type == e1000_i210 || 6723 sc->hw.mac.type == e1000_i211) && 6724 sc->hw.nvm.type == e1000_nvm_invm) { 6725 /* The shared reader does not expose the optional iNVM word. */ 6726 apme = (E1000_READ_REG(&sc->hw, E1000_WUC) & 6727 E1000_WUC_APME) != 0; 6728 } else { 6729 apme = error == E1000_SUCCESS && 6730 (eeprom_data & apme_mask) != 0; 6731 } 6732 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6733 IFCAP_WOL : 0; 6734 if (sc->hw.mac.type == e1000_82542 || 6735 sc->hw.mac.type == e1000_82543) 6736 wol_capabilities = 0; 6737 6738 /* APME selects the default; board and port restrictions select support. */ 6739 device_id = pci_get_device(dev); 6740 switch (device_id) { 6741 case E1000_DEV_ID_82542: 6742 case E1000_DEV_ID_82543GC_FIBER: 6743 case E1000_DEV_ID_82543GC_COPPER: 6744 case E1000_DEV_ID_82541ER: 6745 case E1000_DEV_ID_82541ER_LOM: 6746 case E1000_DEV_ID_82544EI_FIBER: 6747 case E1000_DEV_ID_82545EM_COPPER: 6748 case E1000_DEV_ID_82545EM_FIBER: 6749 case E1000_DEV_ID_82546EB_QUAD_COPPER: 6750 case E1000_DEV_ID_82546GB_QUAD_COPPER: 6751 case E1000_DEV_ID_82546GB_PCIE: 6752 wol_capabilities = 0; 6753 break; 6754 case E1000_DEV_ID_82546EB_FIBER: 6755 case E1000_DEV_ID_82546GB_FIBER: 6756 /* 6757 * Wake events are supported only on port A for dual fiber, 6758 * regardless of the NVM setting. 6759 */ 6760 if (sc->hw.bus.func == 1) 6761 wol_capabilities = 0; 6762 break; 6763 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 6764 /* if quad port adapter, disable WoL on all but port A */ 6765 if (global_quad_port_a != 0) 6766 wol_capabilities = 0; 6767 else 6768 wol_capabilities &= ~IFCAP_WOL_UCAST; 6769 /* Reset for multiple quad port adapters */ 6770 if (++global_quad_port_a == 4) 6771 global_quad_port_a = 0; 6772 break; 6773 case E1000_DEV_ID_82571EB_COPPER: 6774 case E1000_DEV_ID_82571EB_FIBER: 6775 case E1000_DEV_ID_82571EB_SERDES: 6776 /* These dual-port adapters support wake only on port A. */ 6777 if (sc->hw.bus.func == 1) 6778 wol_capabilities = 0; 6779 break; 6780 case E1000_DEV_ID_82571EB_SERDES_QUAD: 6781 wol_capabilities = 0; 6782 break; 6783 case E1000_DEV_ID_82571EB_QUAD_COPPER: 6784 case E1000_DEV_ID_82571EB_QUAD_FIBER: 6785 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 6786 case E1000_DEV_ID_82571PT_QUAD_COPPER: 6787 /* if quad port adapter, disable WoL on all but port A */ 6788 if (global_quad_port_a != 0) 6789 wol_capabilities = 0; 6790 /* Reset for multiple quad port adapters */ 6791 if (++global_quad_port_a == 4) 6792 global_quad_port_a = 0; 6793 break; 6794 case E1000_DEV_ID_82575GB_QUAD_COPPER: 6795 wol_capabilities = 0; 6796 break; 6797 case E1000_DEV_ID_82575EB_FIBER_SERDES: 6798 case E1000_DEV_ID_82576_FIBER: 6799 case E1000_DEV_ID_82576_SERDES: 6800 if (sc->hw.bus.func == 1) 6801 wol_capabilities = 0; 6802 break; 6803 case E1000_DEV_ID_82576_QUAD_COPPER: 6804 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 6805 if (global_quad_port_a != 0) 6806 wol_capabilities = 0; 6807 if (++global_quad_port_a == 4) 6808 global_quad_port_a = 0; 6809 break; 6810 default: 6811 break; 6812 } 6813 /* Legacy and igb non-primary ports require an explicit NVM setting. */ 6814 if ((sc->hw.mac.type < e1000_82571 || 6815 sc->hw.mac.type >= igb_mac_min) && sc->hw.bus.func != 0 && 6816 !apme) 6817 wol_capabilities = 0; 6818 6819 /* Some I350-family systems expose wake support but default it off. */ 6820 if ((sc->hw.mac.type == e1000_i350 && 6821 pci_get_subvendor(dev) == EM_SUBVENDOR_HP) || 6822 ((sc->hw.mac.type == e1000_i350 || 6823 sc->hw.mac.type == e1000_i354) && 6824 pci_get_subvendor(dev) == EM_SUBVENDOR_DELL) || 6825 (sc->hw.mac.type == e1000_i350 && 6826 ((pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_2 || 6827 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_3) && 6828 sc->hw.bus.func == 0))) { 6829 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6830 IFCAP_WOL : 0; 6831 apme = false; 6832 } 6833 if (sc->hw.mac.type == e1000_i350 && 6834 pci_get_subdevice(dev) == EM_I350_SUBDEVICE_WOL_1) 6835 wol_capabilities = pci_has_pme(dev, PCI_POWERSTATE_D3_HOT) ? 6836 IFCAP_WOL : 0; 6837 6838 scctx->isc_capabilities &= ~IFCAP_WOL; 6839 scctx->isc_capabilities |= wol_capabilities; 6840 scctx->isc_capenable &= ~IFCAP_WOL; 6841 if (wol_capabilities != 0 && apme) 6842 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 6843 } 6844 6845 /* Configure the requested PCI Wake-on-LAN filters for suspend. */ 6846 static int 6847 em_enable_wakeup(if_ctx_t ctx) 6848 { 6849 struct e1000_softc *sc = iflib_get_softc(ctx); 6850 device_t dev = iflib_get_dev(ctx); 6851 if_t ifp = iflib_get_ifp(ctx); 6852 int enabled, error = 0, master_error, mcnt; 6853 u32 ctrl, ctrl_ext, rctl, saved_rctl, wuc, wufc; 6854 bool manage, rctl_modified; 6855 6856 if (sc->vf_ifp) 6857 return (0); 6858 if (!pci_has_pme(dev, PCI_POWERSTATE_D3_HOT)) 6859 return (0); 6860 6861 enabled = if_getcapenable(ifp) & if_getcapabilities(ifp) & IFCAP_WOL; 6862 manage = e1000_enable_mng_pass_thru(&sc->hw); 6863 rctl_modified = false; 6864 wuc = 0; 6865 /* Early 82545EM/82546EB need APM clocks for D3 manageability. */ 6866 if (manage && (sc->hw.mac.type == e1000_82545 || 6867 sc->hw.mac.type == e1000_82546)) 6868 wuc = E1000_WUC_APME; 6869 wufc = 0; 6870 if ((enabled & IFCAP_WOL_MAGIC) != 0) 6871 wufc |= E1000_WUFC_MAG; 6872 if ((enabled & IFCAP_WOL_UCAST) != 0) 6873 wufc |= E1000_WUFC_EX; 6874 if ((enabled & IFCAP_WOL_MCAST) != 0) { 6875 wufc |= E1000_WUFC_MC; 6876 bzero(sc->mta, ETHER_ADDR_LEN * 6877 MAX_NUM_MULTICAST_ADDRESSES); 6878 mcnt = if_foreach_llmaddr(ifp, em_copy_maddr, sc->mta); 6879 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) { 6880 e1000_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 6881 } else { 6882 switch (sc->hw.mac.type) { 6883 case e1000_82544: 6884 case e1000_82540: 6885 case e1000_82545: 6886 case e1000_82545_rev_3: 6887 case e1000_82546: 6888 case e1000_82546_rev_3: 6889 case e1000_82541: 6890 case e1000_82541_rev_2: 6891 case e1000_82547: 6892 case e1000_82547_rev_2: 6893 case e1000_82575: 6894 case e1000_82576: 6895 case e1000_82580: 6896 /* These parts require an MTA hit for WUFC_MC. */ 6897 em_fill_wakeup_mta(&sc->hw); 6898 break; 6899 default: 6900 break; 6901 } 6902 } 6903 } 6904 6905 if (wufc == 0) { 6906 if (sc->hw.mac.type >= e1000_82544) { 6907 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 6908 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 6909 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6910 } 6911 if (sc->wol_phy_wakeup && sc->wol_phy_armed) 6912 (void)em_disable_phy_wakeup(sc, NULL); 6913 if (manage) { 6914 if (sc->suspend_link_powered_down) 6915 em_power_up_wakeup_link(sc); 6916 em_configure_sx_low_power(sc, 0); 6917 pci_enable_pme(dev); 6918 } else { 6919 em_power_down_wakeup_link(sc); 6920 pci_clear_pme(dev); 6921 } 6922 goto master_disable; 6923 } 6924 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, ETHER_ADDR_LEN); 6925 error = e1000_rar_set(&sc->hw, sc->hw.mac.addr, 0); 6926 if (error != E1000_SUCCESS) { 6927 device_printf(dev, 6928 "Could not restore unicast wake address: %d\n", error); 6929 goto pme; 6930 } 6931 saved_rctl = E1000_READ_REG(&sc->hw, E1000_RCTL); 6932 rctl = saved_rctl; 6933 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_MO_3); 6934 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 6935 (sc->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 6936 if ((wufc & E1000_WUFC_MC) != 0) 6937 rctl |= E1000_RCTL_MPE; 6938 E1000_WRITE_REG(&sc->hw, E1000_RCTL, rctl); 6939 rctl_modified = true; 6940 6941 /* Advertise the wakeup capability */ 6942 if (sc->hw.mac.type >= e1000_82540) { 6943 ctrl = E1000_READ_REG(&sc->hw, E1000_CTRL); 6944 ctrl |= E1000_CTRL_ADVD3WUC; 6945 if (sc->hw.mac.type < igb_mac_min && !sc->wol_phy_wakeup) 6946 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6947 E1000_WRITE_REG(&sc->hw, E1000_CTRL, ctrl); 6948 } 6949 6950 /* Keep the laser running on legacy fiber and SerDes adapters. */ 6951 if (sc->hw.mac.type < igb_mac_min && 6952 (sc->hw.phy.media_type == e1000_media_type_fiber || 6953 sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { 6954 ctrl_ext = E1000_READ_REG(&sc->hw, E1000_CTRL_EXT); 6955 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6956 E1000_WRITE_REG(&sc->hw, E1000_CTRL_EXT, ctrl_ext); 6957 } 6958 E1000_WRITE_REG(&sc->hw, E1000_WUS, ~0U); 6959 em_power_up_wakeup_link(sc); 6960 6961 if (sc->hw.mac.type >= e1000_ich8lan && 6962 sc->hw.mac.type < igb_mac_min) 6963 e1000_suspend_workarounds_ich8lan(&sc->hw); 6964 6965 if (sc->wol_phy_wakeup) { 6966 error = em_enable_phy_wakeup(sc, wufc); 6967 if (error) 6968 goto pme; 6969 } else { 6970 /* Enable wakeup by the MAC */ 6971 E1000_WRITE_REG(&sc->hw, E1000_WUC, 6972 wuc | E1000_WUC_PME_EN); 6973 E1000_WRITE_REG(&sc->hw, E1000_WUFC, wufc); 6974 } 6975 6976 /* The IGP3 D3 power-down workaround is specific to the em family. */ 6977 if (sc->hw.mac.type < igb_mac_min && 6978 sc->hw.phy.type == e1000_phy_igp_3) 6979 e1000_igp3_phy_powerdown_workaround_ich8lan(&sc->hw); 6980 em_configure_sx_low_power(sc, wufc); 6981 6982 pme: 6983 if (!error) 6984 pci_enable_pme(dev); 6985 else { 6986 E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0); 6987 E1000_WRITE_REG(&sc->hw, E1000_WUC, wuc); 6988 if (rctl_modified) 6989 E1000_WRITE_REG(&sc->hw, E1000_RCTL, saved_rctl); 6990 pci_clear_pme(dev); 6991 } 6992 6993 master_disable: 6994 master_error = e1000_disable_pcie_master(&sc->hw); 6995 if (master_error != E1000_SUCCESS) 6996 device_printf(dev, "PCIe master disable timed out: %d\n", 6997 master_error); 6998 master_error = pci_disable_busmaster(dev); 6999 if (master_error != 0) 7000 device_printf(dev, "PCI bus-master disable failed: %d\n", 7001 master_error); 7002 7003 return (error == E1000_SUCCESS ? 0 : EIO); 7004 } 7005 7006 /* Configure the PCH low-power link modes used while the system sleeps. */ 7007 static void 7008 em_configure_sx_low_power(struct e1000_softc *sc, u32 wufc) 7009 { 7010 struct e1000_hw *hw = &sc->hw; 7011 struct e1000_dev_spec_ich8lan *dev_spec; 7012 s32 error; 7013 u16 eee_advert, lpi_ctrl; 7014 7015 if (hw->mac.type < e1000_pch_lpt || hw->mac.type >= igb_mac_min || 7016 sc->suspend_link_powered_down) 7017 return; 7018 7019 if (wufc != 0 && 7020 (wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC)) == 0) { 7021 /* ULP cannot preserve directed or broad multicast wake. */ 7022 error = e1000_enable_ulp_lpt_lp(hw, true); 7023 if (error != E1000_SUCCESS) { 7024 device_printf(sc->dev, 7025 "Could not enter PHY ultra-low-power mode: %d\n", 7026 error); 7027 return; 7028 } 7029 } 7030 7031 dev_spec = &hw->dev_spec.ich8lan; 7032 if (hw->phy.type != e1000_phy_i217 || dev_spec->eee_disable || 7033 dev_spec->eee_lp_ability == 0) 7034 return; 7035 7036 error = hw->phy.ops.acquire(hw); 7037 if (error != E1000_SUCCESS) 7038 goto out; 7039 error = hw->phy.ops.read_reg_locked(hw, I82579_LPI_CTRL, &lpi_ctrl); 7040 if (error != E1000_SUCCESS) 7041 goto release; 7042 error = e1000_read_emi_reg_locked(hw, I217_EEE_ADVERTISEMENT, 7043 &eee_advert); 7044 if (error != E1000_SUCCESS) 7045 goto release; 7046 7047 if ((eee_advert & dev_spec->eee_lp_ability & 7048 I82579_EEE_100_SUPPORTED) != 0) 7049 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 7050 if ((eee_advert & dev_spec->eee_lp_ability & 7051 I82579_EEE_1000_SUPPORTED) != 0) 7052 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 7053 error = hw->phy.ops.write_reg_locked(hw, I82579_LPI_CTRL, lpi_ctrl); 7054 release: 7055 hw->phy.ops.release(hw); 7056 out: 7057 if (error != E1000_SUCCESS) 7058 device_printf(sc->dev, 7059 "Could not configure Energy Efficient Ethernet for sleep: %d\n", 7060 error); 7061 } 7062 7063 static void 7064 em_power_up_wakeup_link(struct e1000_softc *sc) 7065 { 7066 struct e1000_hw *hw = &sc->hw; 7067 7068 if (hw->mac.type < igb_mac_min) 7069 e1000_power_up_phy(hw); 7070 else if (hw->phy.media_type == e1000_media_type_copper) 7071 e1000_power_up_phy(hw); 7072 else { 7073 e1000_power_up_fiber_serdes_link(hw); 7074 (void)e1000_setup_link(hw); 7075 } 7076 sc->suspend_link_powered_down = false; 7077 } 7078 7079 /* Drop the unused suspend link through the controller's shared-code hook. */ 7080 static void 7081 em_power_down_wakeup_link(struct e1000_softc *sc) 7082 { 7083 struct e1000_hw *hw = &sc->hw; 7084 7085 if (hw->mac.type >= igb_mac_min && 7086 hw->phy.media_type != e1000_media_type_copper) 7087 e1000_shutdown_fiber_serdes_link(hw); 7088 else 7089 e1000_power_down_phy(hw); 7090 sc->suspend_link_powered_down = true; 7091 } 7092 7093 /* PCH PHY wake requires the MAC receive state on the BM wake page. */ 7094 static int 7095 em_enable_phy_wakeup(struct e1000_softc *sc, u32 wufc) 7096 { 7097 struct e1000_hw *hw = &sc->hw; 7098 u32 mreg, wuc; 7099 u16 preg, wuc_enable; 7100 s32 error, restore_error; 7101 7102 /* Copy MAC RARs to PHY RARs before selecting the BM wake page. */ 7103 error = e1000_copy_rx_addrs_to_phy_ich8lan(hw); 7104 if (error != E1000_SUCCESS) 7105 goto out; 7106 7107 error = hw->phy.ops.acquire(hw); 7108 if (error != E1000_SUCCESS) { 7109 device_printf(sc->dev, "Could not acquire PHY for wakeup\n"); 7110 goto out; 7111 } 7112 7113 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7114 if (error != E1000_SUCCESS) 7115 goto release; 7116 7117 /* Wake status is RW1C and survives controller reset. */ 7118 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7119 if (error != E1000_SUCCESS) 7120 goto restore; 7121 7122 /* copy MAC MTA to PHY MTA */ 7123 for (int i = 0; i < hw->mac.mta_reg_count; i++) { 7124 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 7125 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i), 7126 (u16)(mreg & 0xffff)); 7127 if (error != E1000_SUCCESS) 7128 goto restore; 7129 error = hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 7130 (u16)(mreg >> 16)); 7131 if (error != E1000_SUCCESS) 7132 goto restore; 7133 } 7134 7135 /* configure PHY Rx Control register */ 7136 error = hw->phy.ops.read_reg_page(hw, BM_RCTL, &preg); 7137 if (error != E1000_SUCCESS) 7138 goto restore; 7139 mreg = E1000_READ_REG(hw, E1000_RCTL); 7140 if (mreg & E1000_RCTL_UPE) 7141 preg |= BM_RCTL_UPE; 7142 if (mreg & E1000_RCTL_MPE) 7143 preg |= BM_RCTL_MPE; 7144 preg &= ~(BM_RCTL_MO_MASK); 7145 if (mreg & E1000_RCTL_MO_3) 7146 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 7147 << BM_RCTL_MO_SHIFT); 7148 if (mreg & E1000_RCTL_BAM) 7149 preg |= BM_RCTL_BAM; 7150 if (mreg & E1000_RCTL_PMCF) 7151 preg |= BM_RCTL_PMCF; 7152 mreg = E1000_READ_REG(hw, E1000_CTRL); 7153 if (mreg & E1000_CTRL_RFCE) 7154 preg |= BM_RCTL_RFCE; 7155 error = hw->phy.ops.write_reg_page(hw, BM_RCTL, preg); 7156 if (error != E1000_SUCCESS) 7157 goto restore; 7158 7159 wuc = E1000_WUC_PME_EN; 7160 if ((wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) != 0) 7161 wuc |= E1000_WUC_APME; 7162 7163 /* enable PHY wakeup in MAC register */ 7164 E1000_WRITE_REG(hw, E1000_WUFC, wufc); 7165 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PHY_WAKE | 7166 E1000_WUC_APMPME | E1000_WUC_PME_STATUS | wuc); 7167 7168 /* configure and enable PHY wakeup in PHY registers */ 7169 error = hw->phy.ops.write_reg_page(hw, BM_WUFC, wufc); 7170 if (error != E1000_SUCCESS) 7171 goto restore; 7172 error = hw->phy.ops.write_reg_page(hw, BM_WUC, wuc); 7173 if (error != E1000_SUCCESS) 7174 goto restore; 7175 7176 restore: 7177 /* Restore the page selector and expose only a complete setup. */ 7178 if (error == E1000_SUCCESS) 7179 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 7180 else 7181 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7182 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7183 &wuc_enable); 7184 if (error == E1000_SUCCESS) 7185 error = restore_error; 7186 release: 7187 hw->phy.ops.release(hw); 7188 out: 7189 sc->wol_phy_armed = error == E1000_SUCCESS; 7190 if (error != E1000_SUCCESS) 7191 device_printf(sc->dev, "Could not configure PHY wakeup: %d\n", 7192 error); 7193 7194 return (error); 7195 } 7196 7197 /* Clear host ownership and sticky status without disturbing ME wake. */ 7198 static int 7199 em_disable_phy_wakeup(struct e1000_softc *sc, u16 *wus) 7200 { 7201 struct e1000_hw *hw = &sc->hw; 7202 s32 error, restore_error; 7203 u16 phy_wus, wuc_enable; 7204 7205 error = hw->phy.ops.acquire(hw); 7206 if (error != E1000_SUCCESS) 7207 return (error); 7208 error = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 7209 if (error != E1000_SUCCESS) 7210 goto release; 7211 7212 error = hw->phy.ops.read_reg_page(hw, BM_WUS, &phy_wus); 7213 if (error == E1000_SUCCESS) 7214 error = hw->phy.ops.write_reg_page(hw, BM_WUS, 0xffff); 7215 wuc_enable &= ~BM_WUC_HOST_WU_BIT; 7216 restore_error = e1000_disable_phy_wakeup_reg_access_bm(hw, 7217 &wuc_enable); 7218 if (error == E1000_SUCCESS) 7219 error = restore_error; 7220 release: 7221 hw->phy.ops.release(hw); 7222 if (error == E1000_SUCCESS) { 7223 sc->wol_phy_armed = false; 7224 if (wus != NULL) 7225 *wus = phy_wus; 7226 } 7227 return (error); 7228 } 7229 7230 static void 7231 em_if_led_func(if_ctx_t ctx, int onoff) 7232 { 7233 struct e1000_softc *sc = iflib_get_softc(ctx); 7234 7235 if (onoff) { 7236 e1000_setup_led(&sc->hw); 7237 if (sc->hw.phy.media_type == e1000_media_type_internal_serdes) 7238 e1000_blink_led(&sc->hw); 7239 else 7240 e1000_led_on(&sc->hw); 7241 } else { 7242 e1000_led_off(&sc->hw); 7243 e1000_cleanup_led(&sc->hw); 7244 } 7245 } 7246 7247 /* 7248 * Disable the L0S and L1 LINK states 7249 */ 7250 static void 7251 em_disable_aspm(struct e1000_softc *sc) 7252 { 7253 int base, reg; 7254 u16 link_cap,link_ctrl; 7255 device_t dev = sc->dev; 7256 7257 switch (sc->hw.mac.type) { 7258 case e1000_82573: 7259 case e1000_82574: 7260 case e1000_82583: 7261 break; 7262 default: 7263 return; 7264 } 7265 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) 7266 return; 7267 reg = base + PCIER_LINK_CAP; 7268 link_cap = pci_read_config(dev, reg, 2); 7269 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0) 7270 return; 7271 reg = base + PCIER_LINK_CTL; 7272 link_ctrl = pci_read_config(dev, reg, 2); 7273 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC; 7274 pci_write_config(dev, reg, link_ctrl, 2); 7275 return; 7276 } 7277 7278 /********************************************************************** 7279 * 7280 * Update the board statistics counters. 7281 * 7282 **********************************************************************/ 7283 void 7284 em_update_stats_counters(struct e1000_softc *sc) 7285 { 7286 struct e1000_hw_stats *stats; 7287 u64 prev_xoffrxc; 7288 7289 if (sc->vf_ifp) { 7290 em_update_vf_stats_counters(sc); 7291 return; 7292 } 7293 7294 stats = &sc->ustats.stats; 7295 prev_xoffrxc = stats->xoffrxc; 7296 7297 if(sc->hw.phy.media_type == e1000_media_type_copper || 7298 (E1000_READ_REG(&sc->hw, E1000_STATUS) & E1000_STATUS_LU)) { 7299 stats->symerrs += E1000_READ_REG(&sc->hw, E1000_SYMERRS); 7300 stats->sec += E1000_READ_REG(&sc->hw, E1000_SEC); 7301 } 7302 stats->crcerrs += E1000_READ_REG(&sc->hw, E1000_CRCERRS); 7303 stats->mpc += E1000_READ_REG(&sc->hw, E1000_MPC); 7304 stats->scc += E1000_READ_REG(&sc->hw, E1000_SCC); 7305 stats->ecol += E1000_READ_REG(&sc->hw, E1000_ECOL); 7306 7307 stats->mcc += E1000_READ_REG(&sc->hw, E1000_MCC); 7308 stats->latecol += E1000_READ_REG(&sc->hw, E1000_LATECOL); 7309 stats->colc += E1000_READ_REG(&sc->hw, E1000_COLC); 7310 stats->dc += E1000_READ_REG(&sc->hw, E1000_DC); 7311 stats->rlec += E1000_READ_REG(&sc->hw, E1000_RLEC); 7312 stats->xonrxc += E1000_READ_REG(&sc->hw, E1000_XONRXC); 7313 stats->xontxc += E1000_READ_REG(&sc->hw, E1000_XONTXC); 7314 stats->xoffrxc += E1000_READ_REG(&sc->hw, E1000_XOFFRXC); 7315 /* 7316 ** For watchdog management we need to know if we have been 7317 ** paused during the last interval, so capture that here. 7318 */ 7319 if (stats->xoffrxc != prev_xoffrxc) 7320 sc->shared->isc_pause_frames = 1; 7321 stats->xofftxc += E1000_READ_REG(&sc->hw, E1000_XOFFTXC); 7322 stats->fcruc += E1000_READ_REG(&sc->hw, E1000_FCRUC); 7323 stats->prc64 += E1000_READ_REG(&sc->hw, E1000_PRC64); 7324 stats->prc127 += E1000_READ_REG(&sc->hw, E1000_PRC127); 7325 stats->prc255 += E1000_READ_REG(&sc->hw, E1000_PRC255); 7326 stats->prc511 += E1000_READ_REG(&sc->hw, E1000_PRC511); 7327 stats->prc1023 += E1000_READ_REG(&sc->hw, E1000_PRC1023); 7328 stats->prc1522 += E1000_READ_REG(&sc->hw, E1000_PRC1522); 7329 stats->gprc += E1000_READ_REG(&sc->hw, E1000_GPRC); 7330 stats->bprc += E1000_READ_REG(&sc->hw, E1000_BPRC); 7331 stats->mprc += E1000_READ_REG(&sc->hw, E1000_MPRC); 7332 stats->gptc += E1000_READ_REG(&sc->hw, E1000_GPTC); 7333 7334 /* For the 64-bit byte counters the low dword must be read first. */ 7335 /* Both registers clear on the read of the high dword */ 7336 7337 stats->gorc += E1000_READ_REG(&sc->hw, E1000_GORCL) + 7338 ((u64)E1000_READ_REG(&sc->hw, E1000_GORCH) << 32); 7339 stats->gotc += E1000_READ_REG(&sc->hw, E1000_GOTCL) + 7340 ((u64)E1000_READ_REG(&sc->hw, E1000_GOTCH) << 32); 7341 7342 stats->rnbc += E1000_READ_REG(&sc->hw, E1000_RNBC); 7343 stats->ruc += E1000_READ_REG(&sc->hw, E1000_RUC); 7344 stats->rfc += E1000_READ_REG(&sc->hw, E1000_RFC); 7345 stats->roc += E1000_READ_REG(&sc->hw, E1000_ROC); 7346 stats->rjc += E1000_READ_REG(&sc->hw, E1000_RJC); 7347 7348 stats->mgprc += E1000_READ_REG(&sc->hw, E1000_MGTPRC); 7349 stats->mgpdc += E1000_READ_REG(&sc->hw, E1000_MGTPDC); 7350 stats->mgptc += E1000_READ_REG(&sc->hw, E1000_MGTPTC); 7351 7352 stats->tor += E1000_READ_REG(&sc->hw, E1000_TORH); 7353 stats->tot += E1000_READ_REG(&sc->hw, E1000_TOTH); 7354 7355 stats->tpr += E1000_READ_REG(&sc->hw, E1000_TPR); 7356 stats->tpt += E1000_READ_REG(&sc->hw, E1000_TPT); 7357 stats->ptc64 += E1000_READ_REG(&sc->hw, E1000_PTC64); 7358 stats->ptc127 += E1000_READ_REG(&sc->hw, E1000_PTC127); 7359 stats->ptc255 += E1000_READ_REG(&sc->hw, E1000_PTC255); 7360 stats->ptc511 += E1000_READ_REG(&sc->hw, E1000_PTC511); 7361 stats->ptc1023 += E1000_READ_REG(&sc->hw, E1000_PTC1023); 7362 stats->ptc1522 += E1000_READ_REG(&sc->hw, E1000_PTC1522); 7363 stats->mptc += E1000_READ_REG(&sc->hw, E1000_MPTC); 7364 stats->bptc += E1000_READ_REG(&sc->hw, E1000_BPTC); 7365 7366 /* TLPIC and RLPIC are clear-on-read. */ 7367 if (em_mac_has_eee(sc->hw.mac.type)) { 7368 stats->tlpic += E1000_READ_REG(&sc->hw, E1000_TLPIC); 7369 stats->rlpic += E1000_READ_REG(&sc->hw, E1000_RLPIC); 7370 } 7371 7372 /* Interrupt Counts */ 7373 7374 stats->iac += E1000_READ_REG(&sc->hw, E1000_IAC); 7375 stats->icrxptc += E1000_READ_REG(&sc->hw, E1000_ICRXPTC); 7376 stats->icrxatc += E1000_READ_REG(&sc->hw, E1000_ICRXATC); 7377 stats->ictxptc += E1000_READ_REG(&sc->hw, E1000_ICTXPTC); 7378 stats->ictxatc += E1000_READ_REG(&sc->hw, E1000_ICTXATC); 7379 stats->ictxqec += E1000_READ_REG(&sc->hw, E1000_ICTXQEC); 7380 stats->ictxqmtc += E1000_READ_REG(&sc->hw, E1000_ICTXQMTC); 7381 stats->icrxdmtc += E1000_READ_REG(&sc->hw, E1000_ICRXDMTC); 7382 stats->icrxoc += E1000_READ_REG(&sc->hw, E1000_ICRXOC); 7383 7384 if (sc->hw.mac.type >= e1000_82543) { 7385 stats->algnerrc += 7386 E1000_READ_REG(&sc->hw, E1000_ALGNERRC); 7387 stats->rxerrc += 7388 E1000_READ_REG(&sc->hw, E1000_RXERRC); 7389 stats->tncrs += 7390 E1000_READ_REG(&sc->hw, E1000_TNCRS); 7391 stats->cexterr += 7392 E1000_READ_REG(&sc->hw, E1000_CEXTERR); 7393 stats->tsctc += 7394 E1000_READ_REG(&sc->hw, E1000_TSCTC); 7395 stats->tsctfc += 7396 E1000_READ_REG(&sc->hw, E1000_TSCTFC); 7397 } 7398 7399 if (em_has_82571_ecc_stats(&sc->hw)) 7400 em_update_82571_ecc_stats(sc); 7401 else if (em_has_pch_ecc(&sc->hw)) 7402 em_update_pch_ecc_stats(sc, 7403 E1000_READ_REG(&sc->hw, E1000_PBECCSTS)); 7404 else if (em_has_82575_memory_errors(&sc->hw)) 7405 em_update_82575_ecc_stats(sc, 7406 E1000_READ_REG(&sc->hw, E1000_PBECCSTS_82575), 7407 E1000_READ_REG(&sc->hw, E1000_RDHESTS_82575), 7408 E1000_READ_REG(&sc->hw, E1000_TDHESTS_82575)); 7409 else if (em_has_82576_memory_errors(&sc->hw)) 7410 em_update_82576_ecc_stats(sc); 7411 else if (em_has_82580_memory_errors(&sc->hw)) 7412 em_update_82580_ecc_stats(sc, 7413 E1000_READ_REG(&sc->hw, E1000_RPBECCSTS), 7414 E1000_READ_REG(&sc->hw, E1000_TPBECCSTS), 7415 E1000_READ_REG(&sc->hw, E1000_PCIEECCSTS)); 7416 else if (em_has_i350_i354_memory_errors(&sc->hw)) 7417 em_update_i350_i354_ecc_stats(sc); 7418 else if (em_has_i210_memory_errors(&sc->hw)) 7419 em_update_i210_ecc_stats(sc); 7420 } 7421 7422 static bool 7423 em_mac_has_eee(enum e1000_mac_type type) 7424 { 7425 7426 return ((type >= e1000_pch2lan && type < e1000_82575) || 7427 (type >= e1000_i350 && type <= e1000_i211)); 7428 } 7429 7430 static void 7431 em_initialize_vf_stats(struct e1000_softc *sc) 7432 { 7433 struct e1000_vf_stats *stats; 7434 7435 stats = &sc->ustats.vf_stats; 7436 *stats = (struct e1000_vf_stats){}; 7437 em_rebase_vf_stats(sc); 7438 } 7439 7440 static void 7441 em_rebase_vf_stats(struct e1000_softc *sc) 7442 { 7443 struct e1000_vf_stats *stats; 7444 7445 /* 7446 * A PF reset starts a new VF counter epoch. Preserve the accumulated 7447 * totals while establishing a new raw baseline so the reset is not 7448 * mistaken for a 32-bit wrap. 7449 */ 7450 stats = &sc->ustats.vf_stats; 7451 #define INIT_VF_REG(reg, name) do { \ 7452 stats->last_##name = E1000_READ_REG(&sc->hw, reg); \ 7453 } while (0) 7454 INIT_VF_REG(E1000_VFGPRC, gprc); 7455 INIT_VF_REG(E1000_VFGORC, gorc); 7456 INIT_VF_REG(E1000_VFGPTC, gptc); 7457 INIT_VF_REG(E1000_VFGOTC, gotc); 7458 /* 7459 * I350 specification update erratum 31 says VFMPRC is not 7460 * accessible from VF memory. The 0xf3c register remains valid on 7461 * 82576 VFs, but must not be read on vfadapt_i350. 7462 */ 7463 if (sc->hw.mac.type == e1000_vfadapt) 7464 INIT_VF_REG(E1000_VFMPRC, mprc); 7465 else 7466 stats->last_mprc = 0; 7467 INIT_VF_REG(E1000_VFGOTLBC, gotlbc); 7468 INIT_VF_REG(E1000_VFGPTLBC, gptlbc); 7469 INIT_VF_REG(E1000_VFGORLBC, gorlbc); 7470 INIT_VF_REG(E1000_VFGPRLBC, gprlbc); 7471 #undef INIT_VF_REG 7472 } 7473 7474 static void 7475 em_update_vf_stats_counters(struct e1000_softc *sc) 7476 { 7477 struct e1000_vf_stats *stats; 7478 7479 stats = &sc->ustats.vf_stats; 7480 7481 /* 7482 * Internal VF loopback traffic can continue without physical link, 7483 * so sample the counters regardless of link state. 7484 */ 7485 UPDATE_VF_REG(E1000_VFGPRC, 7486 stats->last_gprc, stats->gprc); 7487 UPDATE_VF_REG(E1000_VFGORC, 7488 stats->last_gorc, stats->gorc); 7489 UPDATE_VF_REG(E1000_VFGPTC, 7490 stats->last_gptc, stats->gptc); 7491 UPDATE_VF_REG(E1000_VFGOTC, 7492 stats->last_gotc, stats->gotc); 7493 if (sc->hw.mac.type == e1000_vfadapt) 7494 UPDATE_VF_REG(E1000_VFMPRC, 7495 stats->last_mprc, stats->mprc); 7496 UPDATE_VF_REG(E1000_VFGOTLBC, 7497 stats->last_gotlbc, stats->gotlbc); 7498 UPDATE_VF_REG(E1000_VFGPTLBC, 7499 stats->last_gptlbc, stats->gptlbc); 7500 UPDATE_VF_REG(E1000_VFGORLBC, 7501 stats->last_gorlbc, stats->gorlbc); 7502 UPDATE_VF_REG(E1000_VFGPRLBC, 7503 stats->last_gprlbc, stats->gprlbc); 7504 } 7505 7506 static uint64_t 7507 em_if_get_vf_counter(if_ctx_t ctx, ift_counter cnt) 7508 { 7509 struct e1000_softc *sc = iflib_get_softc(ctx); 7510 if_t ifp = iflib_get_ifp(ctx); 7511 7512 switch (cnt) { 7513 case IFCOUNTER_IERRORS: 7514 return sc->dropped_pkts; 7515 default: 7516 return (if_get_counter_default(ifp, cnt)); 7517 } 7518 } 7519 7520 static uint64_t 7521 em_if_get_counter(if_ctx_t ctx, ift_counter cnt) 7522 { 7523 struct e1000_softc *sc = iflib_get_softc(ctx); 7524 struct e1000_hw_stats *stats; 7525 if_t ifp = iflib_get_ifp(ctx); 7526 7527 if (sc->vf_ifp) 7528 return (em_if_get_vf_counter(ctx, cnt)); 7529 7530 stats = &sc->ustats.stats; 7531 7532 switch (cnt) { 7533 case IFCOUNTER_COLLISIONS: 7534 return (stats->colc); 7535 case IFCOUNTER_IERRORS: 7536 return (sc->dropped_pkts + stats->rxerrc + 7537 stats->crcerrs + stats->algnerrc + 7538 stats->ruc + stats->roc + 7539 stats->mpc + stats->cexterr); 7540 case IFCOUNTER_OERRORS: 7541 return (if_get_counter_default(ifp, cnt) + 7542 stats->ecol + stats->latecol); 7543 default: 7544 return (if_get_counter_default(ifp, cnt)); 7545 } 7546 } 7547 7548 /* em_if_needs_restart - Tell iflib when the driver needs to be reinitialized 7549 * @ctx: iflib context 7550 * @event: event code to check 7551 * 7552 * Defaults to returning false for unknown events. 7553 * 7554 * @returns true if iflib needs to reinit the interface 7555 */ 7556 static bool 7557 em_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 7558 { 7559 switch (event) { 7560 case IFLIB_RESTART_VLAN_CONFIG: 7561 default: 7562 return (false); 7563 } 7564 } 7565 7566 /* Export a single 32-bit register via a read-only sysctl. */ 7567 static int 7568 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 7569 { 7570 struct e1000_softc *sc; 7571 u_int val; 7572 7573 sc = oidp->oid_arg1; 7574 val = E1000_READ_REG(&sc->hw, oidp->oid_arg2); 7575 return (sysctl_handle_int(oidp, &val, 0, req)); 7576 } 7577 7578 enum em_ring_register { 7579 EM_RING_HEAD, 7580 EM_RING_TAIL, 7581 }; 7582 7583 /* Queue register addresses can change when the PF enters IOV mode. */ 7584 static int 7585 em_sysctl_tx_ring_handler(SYSCTL_HANDLER_ARGS) 7586 { 7587 struct tx_ring *txr; 7588 u_int reg, val; 7589 7590 txr = oidp->oid_arg1; 7591 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_TDH(txr->me) : 7592 E1000_TDT(txr->me); 7593 val = E1000_READ_REG(&txr->sc->hw, reg); 7594 return (sysctl_handle_int(oidp, &val, 0, req)); 7595 } 7596 7597 static int 7598 em_sysctl_rx_ring_handler(SYSCTL_HANDLER_ARGS) 7599 { 7600 struct rx_ring *rxr; 7601 u_int reg, val; 7602 7603 rxr = oidp->oid_arg1; 7604 reg = oidp->oid_arg2 == EM_RING_HEAD ? E1000_RDH(rxr->me) : 7605 E1000_RDT(rxr->me); 7606 val = E1000_READ_REG(&rxr->sc->hw, reg); 7607 return (sysctl_handle_int(oidp, &val, 0, req)); 7608 } 7609 7610 /* Per queue holdoff interrupt rate handler */ 7611 static int 7612 em_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 7613 { 7614 struct em_rx_queue *rque; 7615 struct em_tx_queue *tque; 7616 struct e1000_hw *hw; 7617 int error; 7618 u32 reg, usec, rate; 7619 7620 bool tx = oidp->oid_arg2; 7621 7622 if (tx) { 7623 tque = oidp->oid_arg1; 7624 hw = &tque->sc->hw; 7625 if (hw->mac.type >= igb_mac_min) 7626 reg = E1000_READ_REG(hw, E1000_EITR(tque->msix)); 7627 else if (hw->mac.type == e1000_82574 && 7628 tque->sc->intr_type == IFLIB_INTR_MSIX) 7629 reg = E1000_READ_REG(hw, E1000_EITR_82574(tque->msix)); 7630 else 7631 reg = E1000_READ_REG(hw, E1000_ITR); 7632 } else { 7633 rque = oidp->oid_arg1; 7634 hw = &rque->sc->hw; 7635 if (hw->mac.type >= igb_mac_min) 7636 reg = E1000_READ_REG(hw, E1000_EITR(rque->msix)); 7637 else if (hw->mac.type == e1000_82574 && 7638 rque->sc->intr_type == IFLIB_INTR_MSIX) 7639 reg = E1000_READ_REG(hw, 7640 E1000_EITR_82574(rque->msix)); 7641 else 7642 reg = E1000_READ_REG(hw, E1000_ITR); 7643 } 7644 7645 if (hw->mac.type < igb_mac_min) { 7646 if (reg > 0) 7647 rate = EM_INTS_TO_ITR(reg); 7648 else 7649 rate = 0; 7650 } else { 7651 usec = (reg & IGB_QVECTOR_MASK); 7652 if (usec > 0) 7653 rate = IGB_EITR_TO_INTS(usec); 7654 else 7655 rate = 0; 7656 } 7657 7658 error = sysctl_handle_int(oidp, &rate, 0, req); 7659 if (error || !req->newptr) 7660 return error; 7661 return 0; 7662 } 7663 7664 /* 7665 * Add sysctl variables, one per statistic, to the system. 7666 */ 7667 static void 7668 em_add_hw_stats(struct e1000_softc *sc) 7669 { 7670 device_t dev = iflib_get_dev(sc->ctx); 7671 struct em_tx_queue *tx_que = sc->tx_queues; 7672 struct em_rx_queue *rx_que = sc->rx_queues; 7673 7674 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 7675 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 7676 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 7677 struct e1000_hw_stats *stats; 7678 7679 struct sysctl_oid *stat_node, *queue_node, *int_node; 7680 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 7681 7682 #define QUEUE_NAME_LEN 32 7683 char namebuf[QUEUE_NAME_LEN]; 7684 7685 /* Driver Statistics */ 7686 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 7687 CTLFLAG_RD, &sc->dropped_pkts, 7688 "Driver dropped packets"); 7689 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 7690 CTLFLAG_RD, &sc->link_irq, 7691 "Link MSI-X IRQ Handled"); 7692 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 7693 CTLFLAG_RD, &sc->rx_overruns, 7694 "RX overruns"); 7695 if (!sc->vf_ifp) { 7696 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 7697 CTLTYPE_UINT | CTLFLAG_RD, 7698 sc, E1000_CTRL, em_sysctl_reg_handler, "IU", 7699 "Device Control Register"); 7700 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 7701 CTLTYPE_UINT | CTLFLAG_RD, 7702 sc, E1000_RCTL, em_sysctl_reg_handler, "IU", 7703 "Receiver Control Register"); 7704 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 7705 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 7706 "Flow Control High Watermark"); 7707 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 7708 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 7709 "Flow Control Low Watermark"); 7710 } 7711 7712 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 7713 struct tx_ring *txr = &tx_que->txr; 7714 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 7715 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7716 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 7717 queue_list = SYSCTL_CHILDREN(queue_node); 7718 7719 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7720 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 7721 true, em_sysctl_interrupt_rate_handler, 7722 "IU", "Interrupt Rate"); 7723 7724 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 7725 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_HEAD, 7726 em_sysctl_tx_ring_handler, "IU", 7727 "Transmit Descriptor Head"); 7728 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 7729 CTLTYPE_UINT | CTLFLAG_RD, txr, EM_RING_TAIL, 7730 em_sysctl_tx_ring_handler, "IU", 7731 "Transmit Descriptor Tail"); 7732 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 7733 CTLFLAG_RD, &txr->tx_irq, 7734 "Queue MSI-X Transmit Interrupts"); 7735 } 7736 7737 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 7738 struct rx_ring *rxr = &rx_que->rxr; 7739 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 7740 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 7741 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 7742 queue_list = SYSCTL_CHILDREN(queue_node); 7743 7744 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 7745 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 7746 false, em_sysctl_interrupt_rate_handler, 7747 "IU", "Interrupt Rate"); 7748 7749 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 7750 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_HEAD, 7751 em_sysctl_rx_ring_handler, "IU", 7752 "Receive Descriptor Head"); 7753 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 7754 CTLTYPE_UINT | CTLFLAG_RD, rxr, EM_RING_TAIL, 7755 em_sysctl_rx_ring_handler, "IU", 7756 "Receive Descriptor Tail"); 7757 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 7758 CTLFLAG_RD, &rxr->rx_irq, 7759 "Queue MSI-X Receive Interrupts"); 7760 } 7761 7762 /* MAC stats get their own sub node */ 7763 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 7764 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 7765 stat_list = SYSCTL_CHILDREN(stat_node); 7766 7767 /* 7768 ** VF adapter has a very limited set of stats 7769 ** since its not managing the metal, so to speak. 7770 */ 7771 if (sc->vf_ifp) { 7772 struct e1000_vf_stats *vfstats = &sc->ustats.vf_stats; 7773 7774 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 7775 CTLFLAG_RD, &vfstats->gprc, 7776 "Good Packets Received"); 7777 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 7778 CTLFLAG_RD, &vfstats->gptc, 7779 "Good Packets Transmitted"); 7780 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 7781 CTLFLAG_RD, &vfstats->gorc, 7782 "Good Octets Received"); 7783 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 7784 CTLFLAG_RD, &vfstats->gotc, 7785 "Good Octets Transmitted"); 7786 if (sc->hw.mac.type == e1000_vfadapt) { 7787 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7788 "mcast_pkts_recvd", CTLFLAG_RD, &vfstats->mprc, 7789 "Multicast Packets Received"); 7790 } 7791 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7792 "loopback_good_pkts_recvd", 7793 CTLFLAG_RD, &vfstats->gprlbc, 7794 "Good Loopback Packets Received"); 7795 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7796 "loopback_good_pkts_txd", 7797 CTLFLAG_RD, &vfstats->gptlbc, 7798 "Good Loopback Packets Transmitted"); 7799 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7800 "loopback_good_octets_recvd", 7801 CTLFLAG_RD, &vfstats->gorlbc, 7802 "Good Loopback Octets Received"); 7803 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, 7804 "loopback_good_octets_txd", 7805 CTLFLAG_RD, &vfstats->gotlbc, 7806 "Good Loopback Octets Transmitted"); 7807 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7808 "rx_csum_offload_good", 7809 CTLFLAG_RD, &sc->rx_csum_good, 7810 "Receive Checksum Offload Successes"); 7811 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, 7812 "rx_csum_offload_errors", 7813 CTLFLAG_RD, &sc->rx_csum_errors, 7814 "Receive Checksum Offload Errors"); 7815 return; 7816 } 7817 7818 stats = &sc->ustats.stats; 7819 if (em_mac_has_eee(sc->hw.mac.type)) { 7820 struct sysctl_oid *eee_node; 7821 struct sysctl_oid_list *eee_list; 7822 7823 eee_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eee", 7824 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7825 "Energy Efficient Ethernet statistics"); 7826 eee_list = SYSCTL_CHILDREN(eee_node); 7827 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "tx_lpi_count", 7828 CTLFLAG_RD, &stats->tlpic, "TX LPI event count"); 7829 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "rx_lpi_count", 7830 CTLFLAG_RD, &stats->rlpic, "RX LPI event count"); 7831 } 7832 if (em_has_memory_error_stats(&sc->hw)) { 7833 struct sysctl_oid *memerr_node; 7834 struct sysctl_oid_list *memerr_list; 7835 7836 memerr_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 7837 "memory_errors", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 7838 "Internal memory error indications"); 7839 memerr_list = SYSCTL_CHILDREN(memerr_node); 7840 if (em_has_memory_errors(&sc->hw)) 7841 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7842 "fatal_resets", CTLFLAG_RD, 7843 &sc->fatal_error_reset_count, 7844 "Resets requested for fatal internal memory errors"); 7845 if (em_has_82571_ecc_stats(&sc->hw)) { 7846 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7847 "detected_packet_buffer", CTLFLAG_RD, 7848 &sc->corrected_error_packet_buffer_count, 7849 "Detected packet-buffer ECC errors"); 7850 } else if (em_has_pch_ecc(&sc->hw)) { 7851 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7852 "corrected_packet_buffer", CTLFLAG_RD, 7853 &sc->corrected_error_packet_buffer_count, 7854 "Corrected packet-buffer ECC errors"); 7855 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7856 "uncorrected_packet_buffer", CTLFLAG_RD, 7857 &sc->uncorrected_error_packet_buffer_count, 7858 "Uncorrected packet-buffer ECC errors"); 7859 } else if (em_has_82575_memory_errors(&sc->hw)) { 7860 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7861 "corrected_packet_buffer", CTLFLAG_RD, 7862 &sc->corrected_error_packet_buffer_count, 7863 "Corrected packet-buffer ECC errors"); 7864 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7865 "uncorrected_packet_buffer", CTLFLAG_RD, 7866 &sc->uncorrected_error_packet_buffer_count, 7867 "Uncorrected packet-buffer ECC errors"); 7868 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7869 "corrected_descriptor_handler", CTLFLAG_RD, 7870 &sc->corrected_error_dma_count, 7871 "Corrected descriptor-handler ECC errors"); 7872 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7873 "uncorrected_descriptor_handler", CTLFLAG_RD, 7874 &sc->uncorrected_error_dma_count, 7875 "Uncorrected descriptor-handler ECC errors"); 7876 } else if (em_has_82576_memory_errors(&sc->hw)) { 7877 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7878 "fatal_unknown", CTLFLAG_RD, 7879 &sc->fatal_error_unknown_count, 7880 "Fatal memory errors without a reported source"); 7881 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7882 "corrected_packet_buffer", CTLFLAG_RD, 7883 &sc->corrected_error_packet_buffer_count, 7884 "Corrected packet and switch-buffer ECC errors"); 7885 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7886 "uncorrected_packet_buffer", CTLFLAG_RD, 7887 &sc->uncorrected_error_packet_buffer_count, 7888 "Uncorrected packet and switch-buffer ECC errors"); 7889 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7890 "corrected_descriptor_handler", CTLFLAG_RD, 7891 &sc->corrected_error_dma_count, 7892 "Corrected descriptor-handler ECC errors"); 7893 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7894 "uncorrected_descriptor_handler", CTLFLAG_RD, 7895 &sc->uncorrected_error_dma_count, 7896 "Uncorrected descriptor-handler ECC errors"); 7897 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7898 "corrected_pcie_write_buffer", CTLFLAG_RD, 7899 &sc->corrected_error_pcie_tx_data_count, 7900 "Corrected PCIe write-buffer ECC errors"); 7901 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7902 "corrected_pcie_retry_buffer", CTLFLAG_RD, 7903 &sc->corrected_error_pcie_retry_count, 7904 "Corrected controller-shared PCIe retry-buffer errors"); 7905 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7906 "corrected_pcie_msix", CTLFLAG_RD, 7907 &sc->corrected_error_pcie_other_count, 7908 "Corrected controller-shared PCIe MSI-X errors"); 7909 } else { 7910 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7911 "fatal_lan", CTLFLAG_RD, 7912 &sc->fatal_error_lan_count, 7913 "Fatal LAN-port memory error indications"); 7914 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7915 "fatal_management", CTLFLAG_RD, 7916 &sc->fatal_error_mng_count, 7917 "Fatal management-memory error indications"); 7918 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7919 "fatal_pcie", CTLFLAG_RD, 7920 &sc->fatal_error_pcie_count, 7921 "Fatal PCIe memory error indications"); 7922 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7923 "fatal_dma", CTLFLAG_RD, 7924 &sc->fatal_error_dma_count, 7925 "Fatal DMA memory error indications"); 7926 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7927 "fatal_unknown", CTLFLAG_RD, 7928 &sc->fatal_error_unknown_count, 7929 "Fatal memory errors without a reported region"); 7930 if (em_has_82580_memory_errors(&sc->hw)) { 7931 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7932 "corrected_packet_buffer", CTLFLAG_RD, 7933 &sc->corrected_error_packet_buffer_count, 7934 "Corrected packet-buffer ECC errors"); 7935 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7936 "uncorrected_pcie", CTLFLAG_RD, 7937 &sc->uncorrected_error_pcie_count, 7938 "Uncorrected PCIe command-memory ECC indications"); 7939 } else if (em_has_i210_memory_errors(&sc->hw)) { 7940 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7941 "corrected_dma", CTLFLAG_RD, 7942 &sc->corrected_error_dma_count, 7943 "Corrected DMA memory error indications"); 7944 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7945 "corrected_pcie_tx_data", CTLFLAG_RD, 7946 &sc->corrected_error_pcie_tx_data_count, 7947 "Corrected PCIe transmit-data memory indications"); 7948 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7949 "corrected_pcie_retry", CTLFLAG_RD, 7950 &sc->corrected_error_pcie_retry_count, 7951 "Corrected PCIe retry-buffer memory indications"); 7952 } else { 7953 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7954 "corrected_dma", CTLFLAG_RD, 7955 &sc->corrected_error_dma_count, 7956 "Corrected DMA memory indications"); 7957 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7958 "corrected_packet_buffer", CTLFLAG_RD, 7959 &sc->corrected_error_packet_buffer_count, 7960 "Corrected packet-buffer memory indications"); 7961 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7962 "corrected_lan_mng_fifo", CTLFLAG_RD, 7963 &sc->corrected_error_lan_mng_fifo_count, 7964 "Corrected LAN management transmit-FIFO ECC " 7965 "indications"); 7966 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7967 "corrected_pcie_tx_data", CTLFLAG_RD, 7968 &sc->corrected_error_pcie_tx_data_count, 7969 "Corrected PCIe transmit-data memory indications"); 7970 if (sc->hw.mac.type == e1000_i350) 7971 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7972 "corrected_pcie_retry", CTLFLAG_RD, 7973 &sc->corrected_error_pcie_retry_count, 7974 "Corrected PCIe retry-buffer memory " 7975 "indications"); 7976 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 7977 "corrected_pcie_other", CTLFLAG_RD, 7978 &sc->corrected_error_pcie_other_count, 7979 "Other corrected PCIe memory indications"); 7980 } 7981 } 7982 } 7983 7984 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 7985 CTLFLAG_RD, &stats->ecol, 7986 "Excessive collisions"); 7987 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 7988 CTLFLAG_RD, &stats->scc, 7989 "Single collisions"); 7990 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 7991 CTLFLAG_RD, &stats->mcc, 7992 "Multiple collisions"); 7993 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 7994 CTLFLAG_RD, &stats->latecol, 7995 "Late collisions"); 7996 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 7997 CTLFLAG_RD, &stats->colc, 7998 "Collision Count"); 7999 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 8000 CTLFLAG_RD, &stats->symerrs, 8001 "Symbol Errors"); 8002 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 8003 CTLFLAG_RD, &stats->sec, 8004 "Sequence Errors"); 8005 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 8006 CTLFLAG_RD, &stats->dc, 8007 "Defer Count"); 8008 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 8009 CTLFLAG_RD, &stats->mpc, 8010 "Missed Packets"); 8011 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 8012 CTLFLAG_RD, &stats->rlec, 8013 "Receive Length Errors"); 8014 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 8015 CTLFLAG_RD, &stats->rnbc, 8016 "Receive No Buffers"); 8017 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 8018 CTLFLAG_RD, &stats->ruc, 8019 "Receive Undersize"); 8020 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 8021 CTLFLAG_RD, &stats->rfc, 8022 "Fragmented Packets Received "); 8023 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 8024 CTLFLAG_RD, &stats->roc, 8025 "Oversized Packets Received"); 8026 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 8027 CTLFLAG_RD, &stats->rjc, 8028 "Recevied Jabber"); 8029 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 8030 CTLFLAG_RD, &stats->rxerrc, 8031 "Receive Errors"); 8032 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 8033 CTLFLAG_RD, &stats->crcerrs, 8034 "CRC errors"); 8035 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 8036 CTLFLAG_RD, &stats->algnerrc, 8037 "Alignment Errors"); 8038 /* On 82575 these are collision counts */ 8039 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 8040 CTLFLAG_RD, &stats->cexterr, 8041 "Collision/Carrier extension errors"); 8042 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 8043 CTLFLAG_RD, &stats->xonrxc, 8044 "XON Received"); 8045 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 8046 CTLFLAG_RD, &stats->xontxc, 8047 "XON Transmitted"); 8048 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 8049 CTLFLAG_RD, &stats->xoffrxc, 8050 "XOFF Received"); 8051 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 8052 CTLFLAG_RD, &stats->xofftxc, 8053 "XOFF Transmitted"); 8054 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 8055 CTLFLAG_RD, &stats->fcruc, 8056 "Unsupported Flow Control Received"); 8057 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 8058 CTLFLAG_RD, &stats->mgprc, 8059 "Management Packets Received"); 8060 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 8061 CTLFLAG_RD, &stats->mgpdc, 8062 "Management Packets Dropped"); 8063 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 8064 CTLFLAG_RD, &stats->mgptc, 8065 "Management Packets Transmitted"); 8066 8067 /* Packet Reception Stats */ 8068 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 8069 CTLFLAG_RD, &stats->tpr, 8070 "Total Packets Received "); 8071 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 8072 CTLFLAG_RD, &stats->gprc, 8073 "Good Packets Received"); 8074 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 8075 CTLFLAG_RD, &stats->bprc, 8076 "Broadcast Packets Received"); 8077 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 8078 CTLFLAG_RD, &stats->mprc, 8079 "Multicast Packets Received"); 8080 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 8081 CTLFLAG_RD, &stats->prc64, 8082 "64 byte frames received "); 8083 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 8084 CTLFLAG_RD, &stats->prc127, 8085 "65-127 byte frames received"); 8086 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 8087 CTLFLAG_RD, &stats->prc255, 8088 "128-255 byte frames received"); 8089 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 8090 CTLFLAG_RD, &stats->prc511, 8091 "256-511 byte frames received"); 8092 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 8093 CTLFLAG_RD, &stats->prc1023, 8094 "512-1023 byte frames received"); 8095 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 8096 CTLFLAG_RD, &stats->prc1522, 8097 "1023-1522 byte frames received"); 8098 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 8099 CTLFLAG_RD, &stats->gorc, 8100 "Good Octets Received"); 8101 8102 /* Packet Transmission Stats */ 8103 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 8104 CTLFLAG_RD, &stats->gotc, 8105 "Good Octets Transmitted"); 8106 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 8107 CTLFLAG_RD, &stats->tpt, 8108 "Total Packets Transmitted"); 8109 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 8110 CTLFLAG_RD, &stats->gptc, 8111 "Good Packets Transmitted"); 8112 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 8113 CTLFLAG_RD, &stats->bptc, 8114 "Broadcast Packets Transmitted"); 8115 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 8116 CTLFLAG_RD, &stats->mptc, 8117 "Multicast Packets Transmitted"); 8118 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 8119 CTLFLAG_RD, &stats->ptc64, 8120 "64 byte frames transmitted "); 8121 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 8122 CTLFLAG_RD, &stats->ptc127, 8123 "65-127 byte frames transmitted"); 8124 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 8125 CTLFLAG_RD, &stats->ptc255, 8126 "128-255 byte frames transmitted"); 8127 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 8128 CTLFLAG_RD, &stats->ptc511, 8129 "256-511 byte frames transmitted"); 8130 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 8131 CTLFLAG_RD, &stats->ptc1023, 8132 "512-1023 byte frames transmitted"); 8133 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 8134 CTLFLAG_RD, &stats->ptc1522, 8135 "1024-1522 byte frames transmitted"); 8136 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 8137 CTLFLAG_RD, &stats->tsctc, 8138 "TSO Contexts Transmitted"); 8139 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 8140 CTLFLAG_RD, &stats->tsctfc, 8141 "TSO Contexts Failed"); 8142 8143 /* Interrupt Stats */ 8144 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 8145 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 8146 int_list = SYSCTL_CHILDREN(int_node); 8147 8148 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 8149 CTLFLAG_RD, &stats->iac, 8150 "Interrupt Assertion Count"); 8151 8152 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 8153 CTLFLAG_RD, &stats->icrxptc, 8154 "Interrupt Cause Rx Pkt Timer Expire Count"); 8155 8156 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 8157 CTLFLAG_RD, &stats->icrxatc, 8158 "Interrupt Cause Rx Abs Timer Expire Count"); 8159 8160 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 8161 CTLFLAG_RD, &stats->ictxptc, 8162 "Interrupt Cause Tx Pkt Timer Expire Count"); 8163 8164 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 8165 CTLFLAG_RD, &stats->ictxatc, 8166 "Interrupt Cause Tx Abs Timer Expire Count"); 8167 8168 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 8169 CTLFLAG_RD, &stats->ictxqec, 8170 "Interrupt Cause Tx Queue Empty Count"); 8171 8172 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 8173 CTLFLAG_RD, &stats->ictxqmtc, 8174 "Interrupt Cause Tx Queue Min Thresh Count"); 8175 8176 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 8177 CTLFLAG_RD, &stats->icrxdmtc, 8178 "Interrupt Cause Rx Desc Min Thresh Count"); 8179 8180 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun", 8181 CTLFLAG_RD, &stats->icrxoc, 8182 "Interrupt Cause Receiver Overrun Count"); 8183 } 8184 8185 static void 8186 em_fw_version_locked(if_ctx_t ctx) 8187 { 8188 struct e1000_softc *sc = iflib_get_softc(ctx); 8189 struct e1000_hw *hw = &sc->hw; 8190 struct e1000_fw_version *fw_ver = &sc->fw_ver; 8191 uint16_t eep = 0; 8192 8193 /* 8194 * em_fw_version_locked() must run under the IFLIB_CTX_LOCK to meet 8195 * the NVM locking model, so we do it in em_if_attach_pre() and store 8196 * the info in the softc 8197 */ 8198 ASSERT_CTX_LOCK_HELD(hw); 8199 8200 *fw_ver = (struct e1000_fw_version){0}; 8201 8202 if (hw->mac.type >= igb_mac_min) { 8203 /* 8204 * Use the Shared Code for igb(4) 8205 */ 8206 e1000_get_fw_version(hw, fw_ver); 8207 } else { 8208 /* 8209 * Otherwise, EEPROM version should be present on (almost?) 8210 * all devices here 8211 */ 8212 if(e1000_read_nvm(hw, NVM_VERSION, 1, &eep)) { 8213 INIT_DEBUGOUT("can't get EEPROM version"); 8214 return; 8215 } 8216 8217 fw_ver->eep_major = (eep & NVM_MAJOR_MASK) >> NVM_MAJOR_SHIFT; 8218 fw_ver->eep_minor = (eep & NVM_MINOR_MASK) >> NVM_MINOR_SHIFT; 8219 fw_ver->eep_build = (eep & NVM_IMAGE_ID_MASK); 8220 } 8221 } 8222 8223 static void 8224 em_sbuf_fw_version(struct e1000_fw_version *fw_ver, struct sbuf *buf) 8225 { 8226 const char *space = ""; 8227 8228 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 8229 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 8230 fw_ver->eep_minor, fw_ver->eep_build); 8231 space = " "; 8232 } 8233 8234 if (fw_ver->invm_major || fw_ver->invm_minor || 8235 fw_ver->invm_img_type) { 8236 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 8237 space, fw_ver->invm_major, fw_ver->invm_minor, 8238 fw_ver->invm_img_type); 8239 space = " "; 8240 } 8241 8242 if (fw_ver->or_valid) { 8243 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 8244 space, fw_ver->or_major, fw_ver->or_build, 8245 fw_ver->or_patch); 8246 space = " "; 8247 } 8248 8249 if (fw_ver->etrack_id) 8250 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 8251 } 8252 8253 static void 8254 em_print_fw_version(struct e1000_softc *sc ) 8255 { 8256 device_t dev = sc->dev; 8257 struct sbuf *buf; 8258 int error = 0; 8259 8260 buf = sbuf_new_auto(); 8261 if (!buf) { 8262 device_printf(dev, "Could not allocate sbuf for output.\n"); 8263 return; 8264 } 8265 8266 em_sbuf_fw_version(&sc->fw_ver, buf); 8267 8268 error = sbuf_finish(buf); 8269 if (error) 8270 device_printf(dev, "Error finishing sbuf: %d\n", error); 8271 else if (sbuf_len(buf)) 8272 device_printf(dev, "%s\n", sbuf_data(buf)); 8273 8274 sbuf_delete(buf); 8275 } 8276 8277 static int 8278 em_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 8279 { 8280 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8281 device_t dev = sc->dev; 8282 struct sbuf *buf; 8283 int error = 0; 8284 8285 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 8286 if (!buf) { 8287 device_printf(dev, "Could not allocate sbuf for output.\n"); 8288 return (ENOMEM); 8289 } 8290 8291 em_sbuf_fw_version(&sc->fw_ver, buf); 8292 8293 error = sbuf_finish(buf); 8294 if (error) 8295 device_printf(dev, "Error finishing sbuf: %d\n", error); 8296 8297 sbuf_delete(buf); 8298 8299 return (0); 8300 } 8301 8302 /********************************************************************** 8303 * 8304 * This routine provides a way to dump out the adapter eeprom, 8305 * often a useful debug/service tool. This only dumps the first 8306 * 32 words, stuff that matters is in that extent. 8307 * 8308 **********************************************************************/ 8309 static int 8310 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 8311 { 8312 struct e1000_softc *sc = (struct e1000_softc *)arg1; 8313 int error; 8314 int result; 8315 8316 result = -1; 8317 error = sysctl_handle_int(oidp, &result, 0, req); 8318 8319 if (error || !req->newptr) 8320 return (error); 8321 8322 /* 8323 * This value will cause a hex dump of the 8324 * first 32 16-bit words of the EEPROM to 8325 * the screen. 8326 */ 8327 if (result == 1) 8328 em_print_nvm_info(sc); 8329 8330 return (error); 8331 } 8332 8333 static void 8334 em_print_nvm_info(struct e1000_softc *sc) 8335 { 8336 struct e1000_hw *hw = &sc->hw; 8337 struct sx *iflib_ctx_lock = iflib_ctx_lock_get(sc->ctx); 8338 u16 eeprom_data; 8339 int i, j, row = 0; 8340 8341 /* Its a bit crude, but it gets the job done */ 8342 printf("\nInterface EEPROM Dump:\n"); 8343 printf("Offset\n0x0000 "); 8344 8345 /* We rely on the IFLIB_CTX_LOCK as part of NVM locking model */ 8346 sx_xlock(iflib_ctx_lock); 8347 ASSERT_CTX_LOCK_HELD(hw); 8348 for (i = 0, j = 0; i < 32; i++, j++) { 8349 if (j == 8) { /* Make the offset block */ 8350 j = 0; ++row; 8351 printf("\n0x00%x0 ",row); 8352 } 8353 eeprom_data = 0; 8354 if (e1000_read_nvm(hw, i, 1, &eeprom_data) != 8355 E1000_SUCCESS) { 8356 printf("\nNVM read failed at offset %#x\n", i); 8357 break; 8358 } 8359 printf("%04x ", eeprom_data); 8360 } 8361 sx_xunlock(iflib_ctx_lock); 8362 printf("\n"); 8363 } 8364 8365 static int 8366 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 8367 { 8368 struct em_int_delay_info *info; 8369 struct e1000_softc *sc; 8370 u32 regval; 8371 int error, usecs, ticks; 8372 8373 info = (struct em_int_delay_info *) arg1; 8374 usecs = info->value; 8375 error = sysctl_handle_int(oidp, &usecs, 0, req); 8376 if (error != 0 || req->newptr == NULL) 8377 return (error); 8378 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 8379 return (EINVAL); 8380 info->value = usecs; 8381 ticks = EM_USECS_TO_TICKS(usecs); 8382 8383 sc = info->sc; 8384 8385 regval = E1000_READ_OFFSET(&sc->hw, info->offset); 8386 regval = (regval & ~0xffff) | (ticks & 0xffff); 8387 /* Handle a few special cases. */ 8388 switch (info->offset) { 8389 case E1000_RDTR: 8390 break; 8391 case E1000_TIDV: 8392 if (ticks == 0) { 8393 sc->txd_cmd &= ~E1000_TXD_CMD_IDE; 8394 /* Don't write 0 into the TIDV register. */ 8395 regval++; 8396 } else 8397 sc->txd_cmd |= E1000_TXD_CMD_IDE; 8398 break; 8399 } 8400 E1000_WRITE_OFFSET(&sc->hw, info->offset, regval); 8401 return (0); 8402 } 8403 8404 static int 8405 em_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 8406 { 8407 struct e1000_softc *sc; 8408 u32 reg, val, shift; 8409 int error, mask; 8410 8411 sc = oidp->oid_arg1; 8412 switch (oidp->oid_arg2) { 8413 case 0: 8414 reg = E1000_DTXTCPFLGL; 8415 shift = 0; 8416 break; 8417 case 1: 8418 reg = E1000_DTXTCPFLGL; 8419 shift = 16; 8420 break; 8421 case 2: 8422 reg = E1000_DTXTCPFLGH; 8423 shift = 0; 8424 break; 8425 default: 8426 return (EINVAL); 8427 break; 8428 } 8429 val = E1000_READ_REG(&sc->hw, reg); 8430 mask = (val >> shift) & 0xfff; 8431 error = sysctl_handle_int(oidp, &mask, 0, req); 8432 if (error != 0 || req->newptr == NULL) 8433 return (error); 8434 if (mask < 0 || mask > 0xfff) 8435 return (EINVAL); 8436 val = (val & ~(0xfff << shift)) | (mask << shift); 8437 E1000_WRITE_REG(&sc->hw, reg, val); 8438 return (0); 8439 } 8440 8441 static void 8442 em_add_int_delay_sysctl(struct e1000_softc *sc, const char *name, 8443 const char *description, struct em_int_delay_info *info, int offset, 8444 int value) 8445 { 8446 info->sc = sc; 8447 info->offset = offset; 8448 info->value = value; 8449 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev), 8450 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), 8451 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 8452 info, 0, em_sysctl_int_delay, "I", description); 8453 } 8454 8455 /* 8456 * Set flow control using sysctl: 8457 * Flow control values: 8458 * 0 - off 8459 * 1 - rx pause 8460 * 2 - tx pause 8461 * 3 - full 8462 */ 8463 static int 8464 em_set_flowcntl(SYSCTL_HANDLER_ARGS) 8465 { 8466 int error; 8467 static int input = 3; /* default is full */ 8468 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8469 8470 error = sysctl_handle_int(oidp, &input, 0, req); 8471 8472 if ((error) || (req->newptr == NULL)) 8473 return (error); 8474 8475 if (input == sc->fc) /* no change? */ 8476 return (error); 8477 8478 switch (input) { 8479 case e1000_fc_rx_pause: 8480 case e1000_fc_tx_pause: 8481 case e1000_fc_full: 8482 case e1000_fc_none: 8483 sc->hw.fc.requested_mode = input; 8484 sc->fc = input; 8485 break; 8486 default: 8487 /* Do nothing */ 8488 return (error); 8489 } 8490 8491 sc->hw.fc.current_mode = sc->hw.fc.requested_mode; 8492 e1000_force_mac_fc(&sc->hw); 8493 return (error); 8494 } 8495 8496 static void 8497 em_sysctl_request_reinit(struct e1000_softc *sc) 8498 { 8499 if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) 8500 return; 8501 8502 iflib_request_reset(sc->ctx); 8503 iflib_admin_intr_deferred(sc->ctx); 8504 } 8505 8506 /* 8507 * Manage DMA Coalesce: 8508 * Control values: 8509 * 0/1 - off/on 8510 * Legal timer values are: 8511 * 250,500,1000-10000 in thousands 8512 */ 8513 static int 8514 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) 8515 { 8516 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8517 int error; 8518 8519 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 8520 8521 if ((error) || (req->newptr == NULL)) 8522 return (error); 8523 8524 switch (sc->dmac) { 8525 case 0: 8526 /* Disabling */ 8527 break; 8528 case 1: /* Just enable and use default */ 8529 sc->dmac = 1000; 8530 break; 8531 case 250: 8532 case 500: 8533 case 1000: 8534 case 2000: 8535 case 3000: 8536 case 4000: 8537 case 5000: 8538 case 6000: 8539 case 7000: 8540 case 8000: 8541 case 9000: 8542 case 10000: 8543 /* Legal values - allow */ 8544 break; 8545 default: 8546 /* Do nothing, illegal value */ 8547 sc->dmac = 0; 8548 return (EINVAL); 8549 } 8550 /* Reinit the interface */ 8551 em_sysctl_request_reinit(sc); 8552 return (error); 8553 } 8554 8555 /* 8556 * Manage Energy Efficient Ethernet: 8557 * Control values: 8558 * 0/1 - enabled/disabled 8559 */ 8560 static int 8561 em_sysctl_eee(SYSCTL_HANDLER_ARGS) 8562 { 8563 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8564 int error, value; 8565 8566 if (sc->hw.mac.type < igb_mac_min) 8567 value = sc->hw.dev_spec.ich8lan.eee_disable; 8568 else 8569 value = sc->hw.dev_spec._82575.eee_disable; 8570 error = sysctl_handle_int(oidp, &value, 0, req); 8571 if (error || req->newptr == NULL) 8572 return (error); 8573 if (sc->hw.mac.type < igb_mac_min) 8574 sc->hw.dev_spec.ich8lan.eee_disable = (value != 0); 8575 else 8576 sc->hw.dev_spec._82575.eee_disable = (value != 0); 8577 em_sysctl_request_reinit(sc); 8578 8579 return (0); 8580 } 8581 8582 static int 8583 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 8584 { 8585 struct e1000_softc *sc; 8586 int error; 8587 int result; 8588 8589 result = -1; 8590 error = sysctl_handle_int(oidp, &result, 0, req); 8591 8592 if (error || !req->newptr) 8593 return (error); 8594 8595 if (result == 1) { 8596 sc = (struct e1000_softc *) arg1; 8597 em_print_debug_info(sc); 8598 } 8599 8600 return (error); 8601 } 8602 8603 static int 8604 em_get_rs(SYSCTL_HANDLER_ARGS) 8605 { 8606 struct e1000_softc *sc = (struct e1000_softc *) arg1; 8607 int error; 8608 int result; 8609 8610 result = 0; 8611 error = sysctl_handle_int(oidp, &result, 0, req); 8612 8613 if (error || !req->newptr || result != 1) 8614 return (error); 8615 em_dump_rs(sc); 8616 8617 return (error); 8618 } 8619 8620 static void 8621 em_if_debug(if_ctx_t ctx) 8622 { 8623 em_dump_rs(iflib_get_softc(ctx)); 8624 } 8625 8626 /* 8627 * This routine is meant to be fluid, add whatever is 8628 * needed for debugging a problem. -jfv 8629 */ 8630 static void 8631 em_print_debug_info(struct e1000_softc *sc) 8632 { 8633 device_t dev = iflib_get_dev(sc->ctx); 8634 if_t ifp = iflib_get_ifp(sc->ctx); 8635 struct tx_ring *txr; 8636 struct rx_ring *rxr; 8637 8638 if (sc->tx_queues == NULL || sc->rx_queues == NULL) { 8639 device_printf(dev, "queue state is unavailable\n"); 8640 return; 8641 } 8642 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 8643 printf("Interface is RUNNING "); 8644 else 8645 printf("Interface is NOT RUNNING\n"); 8646 8647 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) 8648 printf("and INACTIVE\n"); 8649 else 8650 printf("and ACTIVE\n"); 8651 8652 for (int i = 0; i < sc->tx_num_queues; i++) { 8653 txr = &sc->tx_queues[i].txr; 8654 device_printf(dev, "TX Queue %d ------\n", i); 8655 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 8656 E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)), 8657 E1000_READ_REG(&sc->hw, E1000_TDT(txr->me))); 8658 8659 } 8660 for (int j = 0; j < sc->rx_num_queues; j++) { 8661 rxr = &sc->rx_queues[j].rxr; 8662 device_printf(dev, "RX Queue %d ------\n", j); 8663 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 8664 E1000_READ_REG(&sc->hw, E1000_RDH(rxr->me)), 8665 E1000_READ_REG(&sc->hw, E1000_RDT(rxr->me))); 8666 } 8667 } 8668 8669 /* 8670 * 82574 only: 8671 * Write a new value to the EEPROM increasing the number of MSI-X 8672 * vectors from 3 to 5, for proper multiqueue support. 8673 */ 8674 static void 8675 em_enable_vectors_82574(if_ctx_t ctx) 8676 { 8677 struct e1000_softc *sc = iflib_get_softc(ctx); 8678 struct e1000_hw *hw = &sc->hw; 8679 device_t dev = iflib_get_dev(ctx); 8680 u16 edata; 8681 8682 e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8683 if (bootverbose) 8684 device_printf(dev, "EM_NVM_PCIE_CTRL = %#06x\n", edata); 8685 if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { 8686 device_printf(dev, "Writing to eeprom: increasing " 8687 "reported MSI-X vectors from 3 to 5...\n"); 8688 edata &= ~(EM_NVM_MSIX_N_MASK); 8689 edata |= 4 << EM_NVM_MSIX_N_SHIFT; 8690 e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); 8691 e1000_update_nvm_checksum(hw); 8692 device_printf(dev, "Writing to eeprom: done\n"); 8693 } 8694 } 8695