1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/gro.h> 41 #include <net/ip.h> 42 #include <net/tcp.h> 43 #include <net/udp.h> 44 #include <net/checksum.h> 45 #include <net/ip6_checksum.h> 46 #include <net/udp_tunnel.h> 47 #include <linux/workqueue.h> 48 #include <linux/prefetch.h> 49 #include <linux/cache.h> 50 #include <linux/log2.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <net/page_pool/helpers.h> 56 #include <linux/align.h> 57 #include <net/netdev_queues.h> 58 59 #include "bnxt_hsi.h" 60 #include "bnxt.h" 61 #include "bnxt_hwrm.h" 62 #include "bnxt_ulp.h" 63 #include "bnxt_sriov.h" 64 #include "bnxt_ethtool.h" 65 #include "bnxt_dcb.h" 66 #include "bnxt_xdp.h" 67 #include "bnxt_ptp.h" 68 #include "bnxt_vfr.h" 69 #include "bnxt_tc.h" 70 #include "bnxt_devlink.h" 71 #include "bnxt_debugfs.h" 72 #include "bnxt_hwmon.h" 73 74 #define BNXT_TX_TIMEOUT (5 * HZ) 75 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 76 NETIF_MSG_TX_ERR) 77 78 MODULE_LICENSE("GPL"); 79 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 80 81 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 82 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 83 #define BNXT_RX_COPY_THRESH 256 84 85 #define BNXT_TX_PUSH_THRESH 164 86 87 /* indexed by enum board_idx */ 88 static const struct { 89 char *name; 90 } board_info[] = { 91 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 92 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 93 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 94 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 95 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 96 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 97 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 98 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 99 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 100 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 101 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 102 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 103 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 104 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 105 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 106 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 108 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 109 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 110 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 111 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 112 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 113 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 114 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 115 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 116 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 117 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 118 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 119 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 120 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 121 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 122 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 123 [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" }, 124 [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 125 [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" }, 126 [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" }, 127 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 128 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 129 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 130 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 131 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 132 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 133 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 134 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 135 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 136 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 137 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 138 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 139 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 140 }; 141 142 static const struct pci_device_id bnxt_pci_tbl[] = { 143 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 144 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 145 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 146 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 147 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 148 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 149 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 150 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 151 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 152 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 153 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 154 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 155 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 156 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 157 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 158 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 159 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 160 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 161 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 162 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 163 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 165 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 166 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 167 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 169 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 170 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 171 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 173 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 174 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 175 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 176 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 177 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 178 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 179 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 180 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 181 { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 }, 182 { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 }, 183 { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 }, 184 { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 }, 185 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR }, 186 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 187 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR }, 188 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR }, 189 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 190 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR }, 191 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 192 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 193 #ifdef CONFIG_BNXT_SRIOV 194 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 195 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 196 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 197 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 198 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 199 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 200 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 201 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 202 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 203 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 204 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 205 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 207 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 208 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 209 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 210 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 211 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 212 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 213 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 214 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 215 #endif 216 { 0 } 217 }; 218 219 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 220 221 static const u16 bnxt_vf_req_snif[] = { 222 HWRM_FUNC_CFG, 223 HWRM_FUNC_VF_CFG, 224 HWRM_PORT_PHY_QCFG, 225 HWRM_CFA_L2_FILTER_ALLOC, 226 }; 227 228 static const u16 bnxt_async_events_arr[] = { 229 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 230 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 231 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 232 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 233 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 234 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 235 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 236 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 237 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 238 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 239 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 240 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 241 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 242 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 243 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 244 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, 245 }; 246 247 static struct workqueue_struct *bnxt_pf_wq; 248 249 #define BNXT_IPV6_MASK_ALL {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, \ 250 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }}} 251 #define BNXT_IPV6_MASK_NONE {{{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}} 252 253 const struct bnxt_flow_masks BNXT_FLOW_MASK_NONE = { 254 .ports = { 255 .src = 0, 256 .dst = 0, 257 }, 258 .addrs = { 259 .v6addrs = { 260 .src = BNXT_IPV6_MASK_NONE, 261 .dst = BNXT_IPV6_MASK_NONE, 262 }, 263 }, 264 }; 265 266 const struct bnxt_flow_masks BNXT_FLOW_IPV6_MASK_ALL = { 267 .ports = { 268 .src = cpu_to_be16(0xffff), 269 .dst = cpu_to_be16(0xffff), 270 }, 271 .addrs = { 272 .v6addrs = { 273 .src = BNXT_IPV6_MASK_ALL, 274 .dst = BNXT_IPV6_MASK_ALL, 275 }, 276 }, 277 }; 278 279 const struct bnxt_flow_masks BNXT_FLOW_IPV4_MASK_ALL = { 280 .ports = { 281 .src = cpu_to_be16(0xffff), 282 .dst = cpu_to_be16(0xffff), 283 }, 284 .addrs = { 285 .v4addrs = { 286 .src = cpu_to_be32(0xffffffff), 287 .dst = cpu_to_be32(0xffffffff), 288 }, 289 }, 290 }; 291 292 static bool bnxt_vf_pciid(enum board_idx idx) 293 { 294 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 295 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 296 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 297 idx == NETXTREME_E_P5_VF_HV); 298 } 299 300 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 301 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 302 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 303 304 #define BNXT_CP_DB_IRQ_DIS(db) \ 305 writel(DB_CP_IRQ_DIS_FLAGS, db) 306 307 #define BNXT_DB_CQ(db, idx) \ 308 writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell) 309 310 #define BNXT_DB_NQ_P5(db, idx) \ 311 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\ 312 (db)->doorbell) 313 314 #define BNXT_DB_NQ_P7(db, idx) \ 315 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK | \ 316 DB_RING_IDX(db, idx), (db)->doorbell) 317 318 #define BNXT_DB_CQ_ARM(db, idx) \ 319 writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell) 320 321 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 322 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | \ 323 DB_RING_IDX(db, idx), (db)->doorbell) 324 325 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 326 { 327 if (bp->flags & BNXT_FLAG_CHIP_P7) 328 BNXT_DB_NQ_P7(db, idx); 329 else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 330 BNXT_DB_NQ_P5(db, idx); 331 else 332 BNXT_DB_CQ(db, idx); 333 } 334 335 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 336 { 337 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 338 BNXT_DB_NQ_ARM_P5(db, idx); 339 else 340 BNXT_DB_CQ_ARM(db, idx); 341 } 342 343 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 344 { 345 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 346 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 347 DB_RING_IDX(db, idx), db->doorbell); 348 else 349 BNXT_DB_CQ(db, idx); 350 } 351 352 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 353 { 354 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 355 return; 356 357 if (BNXT_PF(bp)) 358 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 359 else 360 schedule_delayed_work(&bp->fw_reset_task, delay); 361 } 362 363 static void __bnxt_queue_sp_work(struct bnxt *bp) 364 { 365 if (BNXT_PF(bp)) 366 queue_work(bnxt_pf_wq, &bp->sp_task); 367 else 368 schedule_work(&bp->sp_task); 369 } 370 371 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event) 372 { 373 set_bit(event, &bp->sp_event); 374 __bnxt_queue_sp_work(bp); 375 } 376 377 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 378 { 379 if (!rxr->bnapi->in_reset) { 380 rxr->bnapi->in_reset = true; 381 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 382 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 383 else 384 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 385 __bnxt_queue_sp_work(bp); 386 } 387 rxr->rx_next_cons = 0xffff; 388 } 389 390 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 391 u16 curr) 392 { 393 struct bnxt_napi *bnapi = txr->bnapi; 394 395 if (bnapi->tx_fault) 396 return; 397 398 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)", 399 txr->txq_index, txr->tx_hw_cons, 400 txr->tx_cons, txr->tx_prod, curr); 401 WARN_ON_ONCE(1); 402 bnapi->tx_fault = 1; 403 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 404 } 405 406 const u16 bnxt_lhint_arr[] = { 407 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 408 TX_BD_FLAGS_LHINT_512_TO_1023, 409 TX_BD_FLAGS_LHINT_1024_TO_2047, 410 TX_BD_FLAGS_LHINT_1024_TO_2047, 411 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 412 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 413 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 414 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 415 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 416 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 417 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 418 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 419 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 420 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 421 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 422 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 423 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 424 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 425 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 426 }; 427 428 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 429 { 430 struct metadata_dst *md_dst = skb_metadata_dst(skb); 431 432 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 433 return 0; 434 435 return md_dst->u.port_info.port_id; 436 } 437 438 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 439 u16 prod) 440 { 441 /* Sync BD data before updating doorbell */ 442 wmb(); 443 bnxt_db_write(bp, &txr->tx_db, prod); 444 txr->kick_pending = 0; 445 } 446 447 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 448 { 449 struct bnxt *bp = netdev_priv(dev); 450 struct tx_bd *txbd, *txbd0; 451 struct tx_bd_ext *txbd1; 452 struct netdev_queue *txq; 453 int i; 454 dma_addr_t mapping; 455 unsigned int length, pad = 0; 456 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 457 u16 prod, last_frag; 458 struct pci_dev *pdev = bp->pdev; 459 struct bnxt_tx_ring_info *txr; 460 struct bnxt_sw_tx_bd *tx_buf; 461 __le32 lflags = 0; 462 463 i = skb_get_queue_mapping(skb); 464 if (unlikely(i >= bp->tx_nr_rings)) { 465 dev_kfree_skb_any(skb); 466 dev_core_stats_tx_dropped_inc(dev); 467 return NETDEV_TX_OK; 468 } 469 470 txq = netdev_get_tx_queue(dev, i); 471 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 472 prod = txr->tx_prod; 473 474 free_size = bnxt_tx_avail(bp, txr); 475 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 476 /* We must have raced with NAPI cleanup */ 477 if (net_ratelimit() && txr->kick_pending) 478 netif_warn(bp, tx_err, dev, 479 "bnxt: ring busy w/ flush pending!\n"); 480 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 481 bp->tx_wake_thresh)) 482 return NETDEV_TX_BUSY; 483 } 484 485 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 486 goto tx_free; 487 488 length = skb->len; 489 len = skb_headlen(skb); 490 last_frag = skb_shinfo(skb)->nr_frags; 491 492 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 493 494 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 495 tx_buf->skb = skb; 496 tx_buf->nr_frags = last_frag; 497 498 vlan_tag_flags = 0; 499 cfa_action = bnxt_xmit_get_cfa_action(skb); 500 if (skb_vlan_tag_present(skb)) { 501 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 502 skb_vlan_tag_get(skb); 503 /* Currently supports 8021Q, 8021AD vlan offloads 504 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 505 */ 506 if (skb->vlan_proto == htons(ETH_P_8021Q)) 507 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 508 } 509 510 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 511 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 512 513 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && 514 atomic_dec_if_positive(&ptp->tx_avail) >= 0) { 515 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid, 516 &ptp->tx_hdr_off)) { 517 if (vlan_tag_flags) 518 ptp->tx_hdr_off += VLAN_HLEN; 519 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 520 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 521 } else { 522 atomic_inc(&bp->ptp_cfg->tx_avail); 523 } 524 } 525 } 526 527 if (unlikely(skb->no_fcs)) 528 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 529 530 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 531 !lflags) { 532 struct tx_push_buffer *tx_push_buf = txr->tx_push; 533 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 534 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 535 void __iomem *db = txr->tx_db.doorbell; 536 void *pdata = tx_push_buf->data; 537 u64 *end; 538 int j, push_len; 539 540 /* Set COAL_NOW to be ready quickly for the next push */ 541 tx_push->tx_bd_len_flags_type = 542 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 543 TX_BD_TYPE_LONG_TX_BD | 544 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 545 TX_BD_FLAGS_COAL_NOW | 546 TX_BD_FLAGS_PACKET_END | 547 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 548 549 if (skb->ip_summed == CHECKSUM_PARTIAL) 550 tx_push1->tx_bd_hsize_lflags = 551 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 552 else 553 tx_push1->tx_bd_hsize_lflags = 0; 554 555 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 556 tx_push1->tx_bd_cfa_action = 557 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 558 559 end = pdata + length; 560 end = PTR_ALIGN(end, 8) - 1; 561 *end = 0; 562 563 skb_copy_from_linear_data(skb, pdata, len); 564 pdata += len; 565 for (j = 0; j < last_frag; j++) { 566 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 567 void *fptr; 568 569 fptr = skb_frag_address_safe(frag); 570 if (!fptr) 571 goto normal_tx; 572 573 memcpy(pdata, fptr, skb_frag_size(frag)); 574 pdata += skb_frag_size(frag); 575 } 576 577 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 578 txbd->tx_bd_haddr = txr->data_mapping; 579 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2); 580 prod = NEXT_TX(prod); 581 tx_push->tx_bd_opaque = txbd->tx_bd_opaque; 582 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 583 memcpy(txbd, tx_push1, sizeof(*txbd)); 584 prod = NEXT_TX(prod); 585 tx_push->doorbell = 586 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | 587 DB_RING_IDX(&txr->tx_db, prod)); 588 WRITE_ONCE(txr->tx_prod, prod); 589 590 tx_buf->is_push = 1; 591 netdev_tx_sent_queue(txq, skb->len); 592 wmb(); /* Sync is_push and byte queue before pushing data */ 593 594 push_len = (length + sizeof(*tx_push) + 7) / 8; 595 if (push_len > 16) { 596 __iowrite64_copy(db, tx_push_buf, 16); 597 __iowrite32_copy(db + 4, tx_push_buf + 1, 598 (push_len - 16) << 1); 599 } else { 600 __iowrite64_copy(db, tx_push_buf, push_len); 601 } 602 603 goto tx_done; 604 } 605 606 normal_tx: 607 if (length < BNXT_MIN_PKT_SIZE) { 608 pad = BNXT_MIN_PKT_SIZE - length; 609 if (skb_pad(skb, pad)) 610 /* SKB already freed. */ 611 goto tx_kick_pending; 612 length = BNXT_MIN_PKT_SIZE; 613 } 614 615 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 616 617 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 618 goto tx_free; 619 620 dma_unmap_addr_set(tx_buf, mapping, mapping); 621 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 622 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 623 624 txbd->tx_bd_haddr = cpu_to_le64(mapping); 625 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag); 626 627 prod = NEXT_TX(prod); 628 txbd1 = (struct tx_bd_ext *) 629 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 630 631 txbd1->tx_bd_hsize_lflags = lflags; 632 if (skb_is_gso(skb)) { 633 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4); 634 u32 hdr_len; 635 636 if (skb->encapsulation) { 637 if (udp_gso) 638 hdr_len = skb_inner_transport_offset(skb) + 639 sizeof(struct udphdr); 640 else 641 hdr_len = skb_inner_tcp_all_headers(skb); 642 } else if (udp_gso) { 643 hdr_len = skb_transport_offset(skb) + 644 sizeof(struct udphdr); 645 } else { 646 hdr_len = skb_tcp_all_headers(skb); 647 } 648 649 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 650 TX_BD_FLAGS_T_IPID | 651 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 652 length = skb_shinfo(skb)->gso_size; 653 txbd1->tx_bd_mss = cpu_to_le32(length); 654 length += hdr_len; 655 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 656 txbd1->tx_bd_hsize_lflags |= 657 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 658 txbd1->tx_bd_mss = 0; 659 } 660 661 length >>= 9; 662 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 663 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 664 skb->len); 665 i = 0; 666 goto tx_dma_error; 667 } 668 flags |= bnxt_lhint_arr[length]; 669 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 670 671 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 672 txbd1->tx_bd_cfa_action = 673 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 674 txbd0 = txbd; 675 for (i = 0; i < last_frag; i++) { 676 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 677 678 prod = NEXT_TX(prod); 679 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 680 681 len = skb_frag_size(frag); 682 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 683 DMA_TO_DEVICE); 684 685 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 686 goto tx_dma_error; 687 688 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 689 dma_unmap_addr_set(tx_buf, mapping, mapping); 690 691 txbd->tx_bd_haddr = cpu_to_le64(mapping); 692 693 flags = len << TX_BD_LEN_SHIFT; 694 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 695 } 696 697 flags &= ~TX_BD_LEN; 698 txbd->tx_bd_len_flags_type = 699 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 700 TX_BD_FLAGS_PACKET_END); 701 702 netdev_tx_sent_queue(txq, skb->len); 703 704 skb_tx_timestamp(skb); 705 706 prod = NEXT_TX(prod); 707 WRITE_ONCE(txr->tx_prod, prod); 708 709 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) { 710 bnxt_txr_db_kick(bp, txr, prod); 711 } else { 712 if (free_size >= bp->tx_wake_thresh) 713 txbd0->tx_bd_len_flags_type |= 714 cpu_to_le32(TX_BD_FLAGS_NO_CMPL); 715 txr->kick_pending = 1; 716 } 717 718 tx_done: 719 720 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 721 if (netdev_xmit_more() && !tx_buf->is_push) { 722 txbd0->tx_bd_len_flags_type &= 723 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); 724 bnxt_txr_db_kick(bp, txr, prod); 725 } 726 727 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 728 bp->tx_wake_thresh); 729 } 730 return NETDEV_TX_OK; 731 732 tx_dma_error: 733 if (BNXT_TX_PTP_IS_SET(lflags)) 734 atomic_inc(&bp->ptp_cfg->tx_avail); 735 736 last_frag = i; 737 738 /* start back at beginning and unmap skb */ 739 prod = txr->tx_prod; 740 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 741 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 742 skb_headlen(skb), DMA_TO_DEVICE); 743 prod = NEXT_TX(prod); 744 745 /* unmap remaining mapped pages */ 746 for (i = 0; i < last_frag; i++) { 747 prod = NEXT_TX(prod); 748 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 749 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 750 skb_frag_size(&skb_shinfo(skb)->frags[i]), 751 DMA_TO_DEVICE); 752 } 753 754 tx_free: 755 dev_kfree_skb_any(skb); 756 tx_kick_pending: 757 if (txr->kick_pending) 758 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 759 txr->tx_buf_ring[txr->tx_prod].skb = NULL; 760 dev_core_stats_tx_dropped_inc(dev); 761 return NETDEV_TX_OK; 762 } 763 764 static void __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 765 int budget) 766 { 767 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 768 struct pci_dev *pdev = bp->pdev; 769 u16 hw_cons = txr->tx_hw_cons; 770 unsigned int tx_bytes = 0; 771 u16 cons = txr->tx_cons; 772 int tx_pkts = 0; 773 774 while (RING_TX(bp, cons) != hw_cons) { 775 struct bnxt_sw_tx_bd *tx_buf; 776 struct sk_buff *skb; 777 int j, last; 778 779 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; 780 cons = NEXT_TX(cons); 781 skb = tx_buf->skb; 782 tx_buf->skb = NULL; 783 784 if (unlikely(!skb)) { 785 bnxt_sched_reset_txr(bp, txr, cons); 786 return; 787 } 788 789 tx_pkts++; 790 tx_bytes += skb->len; 791 792 if (tx_buf->is_push) { 793 tx_buf->is_push = 0; 794 goto next_tx_int; 795 } 796 797 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 798 skb_headlen(skb), DMA_TO_DEVICE); 799 last = tx_buf->nr_frags; 800 801 for (j = 0; j < last; j++) { 802 cons = NEXT_TX(cons); 803 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; 804 dma_unmap_page( 805 &pdev->dev, 806 dma_unmap_addr(tx_buf, mapping), 807 skb_frag_size(&skb_shinfo(skb)->frags[j]), 808 DMA_TO_DEVICE); 809 } 810 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 811 if (BNXT_CHIP_P5(bp)) { 812 /* PTP worker takes ownership of the skb */ 813 if (!bnxt_get_tx_ts_p5(bp, skb)) 814 skb = NULL; 815 else 816 atomic_inc(&bp->ptp_cfg->tx_avail); 817 } 818 } 819 820 next_tx_int: 821 cons = NEXT_TX(cons); 822 823 dev_consume_skb_any(skb); 824 } 825 826 WRITE_ONCE(txr->tx_cons, cons); 827 828 __netif_txq_completed_wake(txq, tx_pkts, tx_bytes, 829 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh, 830 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING); 831 } 832 833 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 834 { 835 struct bnxt_tx_ring_info *txr; 836 int i; 837 838 bnxt_for_each_napi_tx(i, bnapi, txr) { 839 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons)) 840 __bnxt_tx_int(bp, txr, budget); 841 } 842 bnapi->events &= ~BNXT_TX_CMP_EVENT; 843 } 844 845 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 846 struct bnxt_rx_ring_info *rxr, 847 unsigned int *offset, 848 gfp_t gfp) 849 { 850 struct page *page; 851 852 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 853 page = page_pool_dev_alloc_frag(rxr->page_pool, offset, 854 BNXT_RX_PAGE_SIZE); 855 } else { 856 page = page_pool_dev_alloc_pages(rxr->page_pool); 857 *offset = 0; 858 } 859 if (!page) 860 return NULL; 861 862 *mapping = page_pool_get_dma_addr(page) + *offset; 863 return page; 864 } 865 866 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 867 gfp_t gfp) 868 { 869 u8 *data; 870 struct pci_dev *pdev = bp->pdev; 871 872 if (gfp == GFP_ATOMIC) 873 data = napi_alloc_frag(bp->rx_buf_size); 874 else 875 data = netdev_alloc_frag(bp->rx_buf_size); 876 if (!data) 877 return NULL; 878 879 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 880 bp->rx_buf_use_size, bp->rx_dir, 881 DMA_ATTR_WEAK_ORDERING); 882 883 if (dma_mapping_error(&pdev->dev, *mapping)) { 884 skb_free_frag(data); 885 data = NULL; 886 } 887 return data; 888 } 889 890 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 891 u16 prod, gfp_t gfp) 892 { 893 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 894 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 895 dma_addr_t mapping; 896 897 if (BNXT_RX_PAGE_MODE(bp)) { 898 unsigned int offset; 899 struct page *page = 900 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); 901 902 if (!page) 903 return -ENOMEM; 904 905 mapping += bp->rx_dma_offset; 906 rx_buf->data = page; 907 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; 908 } else { 909 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); 910 911 if (!data) 912 return -ENOMEM; 913 914 rx_buf->data = data; 915 rx_buf->data_ptr = data + bp->rx_offset; 916 } 917 rx_buf->mapping = mapping; 918 919 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 920 return 0; 921 } 922 923 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 924 { 925 u16 prod = rxr->rx_prod; 926 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 927 struct bnxt *bp = rxr->bnapi->bp; 928 struct rx_bd *cons_bd, *prod_bd; 929 930 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 931 cons_rx_buf = &rxr->rx_buf_ring[cons]; 932 933 prod_rx_buf->data = data; 934 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 935 936 prod_rx_buf->mapping = cons_rx_buf->mapping; 937 938 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 939 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; 940 941 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 942 } 943 944 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 945 { 946 u16 next, max = rxr->rx_agg_bmap_size; 947 948 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 949 if (next >= max) 950 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 951 return next; 952 } 953 954 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 955 struct bnxt_rx_ring_info *rxr, 956 u16 prod, gfp_t gfp) 957 { 958 struct rx_bd *rxbd = 959 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; 960 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 961 struct page *page; 962 dma_addr_t mapping; 963 u16 sw_prod = rxr->rx_sw_agg_prod; 964 unsigned int offset = 0; 965 966 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); 967 968 if (!page) 969 return -ENOMEM; 970 971 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 972 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 973 974 __set_bit(sw_prod, rxr->rx_agg_bmap); 975 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 976 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod)); 977 978 rx_agg_buf->page = page; 979 rx_agg_buf->offset = offset; 980 rx_agg_buf->mapping = mapping; 981 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 982 rxbd->rx_bd_opaque = sw_prod; 983 return 0; 984 } 985 986 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 987 struct bnxt_cp_ring_info *cpr, 988 u16 cp_cons, u16 curr) 989 { 990 struct rx_agg_cmp *agg; 991 992 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 993 agg = (struct rx_agg_cmp *) 994 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 995 return agg; 996 } 997 998 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 999 struct bnxt_rx_ring_info *rxr, 1000 u16 agg_id, u16 curr) 1001 { 1002 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 1003 1004 return &tpa_info->agg_arr[curr]; 1005 } 1006 1007 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 1008 u16 start, u32 agg_bufs, bool tpa) 1009 { 1010 struct bnxt_napi *bnapi = cpr->bnapi; 1011 struct bnxt *bp = bnapi->bp; 1012 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1013 u16 prod = rxr->rx_agg_prod; 1014 u16 sw_prod = rxr->rx_sw_agg_prod; 1015 bool p5_tpa = false; 1016 u32 i; 1017 1018 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa) 1019 p5_tpa = true; 1020 1021 for (i = 0; i < agg_bufs; i++) { 1022 u16 cons; 1023 struct rx_agg_cmp *agg; 1024 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 1025 struct rx_bd *prod_bd; 1026 struct page *page; 1027 1028 if (p5_tpa) 1029 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 1030 else 1031 agg = bnxt_get_agg(bp, cpr, idx, start + i); 1032 cons = agg->rx_agg_cmp_opaque; 1033 __clear_bit(cons, rxr->rx_agg_bmap); 1034 1035 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1036 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 1037 1038 __set_bit(sw_prod, rxr->rx_agg_bmap); 1039 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 1040 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1041 1042 /* It is possible for sw_prod to be equal to cons, so 1043 * set cons_rx_buf->page to NULL first. 1044 */ 1045 page = cons_rx_buf->page; 1046 cons_rx_buf->page = NULL; 1047 prod_rx_buf->page = page; 1048 prod_rx_buf->offset = cons_rx_buf->offset; 1049 1050 prod_rx_buf->mapping = cons_rx_buf->mapping; 1051 1052 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; 1053 1054 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 1055 prod_bd->rx_bd_opaque = sw_prod; 1056 1057 prod = NEXT_RX_AGG(prod); 1058 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod)); 1059 } 1060 rxr->rx_agg_prod = prod; 1061 rxr->rx_sw_agg_prod = sw_prod; 1062 } 1063 1064 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, 1065 struct bnxt_rx_ring_info *rxr, 1066 u16 cons, void *data, u8 *data_ptr, 1067 dma_addr_t dma_addr, 1068 unsigned int offset_and_len) 1069 { 1070 unsigned int len = offset_and_len & 0xffff; 1071 struct page *page = data; 1072 u16 prod = rxr->rx_prod; 1073 struct sk_buff *skb; 1074 int err; 1075 1076 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1077 if (unlikely(err)) { 1078 bnxt_reuse_rx_data(rxr, cons, data); 1079 return NULL; 1080 } 1081 dma_addr -= bp->rx_dma_offset; 1082 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 1083 bp->rx_dir); 1084 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE); 1085 if (!skb) { 1086 page_pool_recycle_direct(rxr->page_pool, page); 1087 return NULL; 1088 } 1089 skb_mark_for_recycle(skb); 1090 skb_reserve(skb, bp->rx_offset); 1091 __skb_put(skb, len); 1092 1093 return skb; 1094 } 1095 1096 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 1097 struct bnxt_rx_ring_info *rxr, 1098 u16 cons, void *data, u8 *data_ptr, 1099 dma_addr_t dma_addr, 1100 unsigned int offset_and_len) 1101 { 1102 unsigned int payload = offset_and_len >> 16; 1103 unsigned int len = offset_and_len & 0xffff; 1104 skb_frag_t *frag; 1105 struct page *page = data; 1106 u16 prod = rxr->rx_prod; 1107 struct sk_buff *skb; 1108 int off, err; 1109 1110 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1111 if (unlikely(err)) { 1112 bnxt_reuse_rx_data(rxr, cons, data); 1113 return NULL; 1114 } 1115 dma_addr -= bp->rx_dma_offset; 1116 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 1117 bp->rx_dir); 1118 1119 if (unlikely(!payload)) 1120 payload = eth_get_headlen(bp->dev, data_ptr, len); 1121 1122 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 1123 if (!skb) { 1124 page_pool_recycle_direct(rxr->page_pool, page); 1125 return NULL; 1126 } 1127 1128 skb_mark_for_recycle(skb); 1129 off = (void *)data_ptr - page_address(page); 1130 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE); 1131 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1132 payload + NET_IP_ALIGN); 1133 1134 frag = &skb_shinfo(skb)->frags[0]; 1135 skb_frag_size_sub(frag, payload); 1136 skb_frag_off_add(frag, payload); 1137 skb->data_len -= payload; 1138 skb->tail += payload; 1139 1140 return skb; 1141 } 1142 1143 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1144 struct bnxt_rx_ring_info *rxr, u16 cons, 1145 void *data, u8 *data_ptr, 1146 dma_addr_t dma_addr, 1147 unsigned int offset_and_len) 1148 { 1149 u16 prod = rxr->rx_prod; 1150 struct sk_buff *skb; 1151 int err; 1152 1153 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1154 if (unlikely(err)) { 1155 bnxt_reuse_rx_data(rxr, cons, data); 1156 return NULL; 1157 } 1158 1159 skb = napi_build_skb(data, bp->rx_buf_size); 1160 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1161 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 1162 if (!skb) { 1163 skb_free_frag(data); 1164 return NULL; 1165 } 1166 1167 skb_reserve(skb, bp->rx_offset); 1168 skb_put(skb, offset_and_len & 0xffff); 1169 return skb; 1170 } 1171 1172 static u32 __bnxt_rx_agg_pages(struct bnxt *bp, 1173 struct bnxt_cp_ring_info *cpr, 1174 struct skb_shared_info *shinfo, 1175 u16 idx, u32 agg_bufs, bool tpa, 1176 struct xdp_buff *xdp) 1177 { 1178 struct bnxt_napi *bnapi = cpr->bnapi; 1179 struct pci_dev *pdev = bp->pdev; 1180 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1181 u16 prod = rxr->rx_agg_prod; 1182 u32 i, total_frag_len = 0; 1183 bool p5_tpa = false; 1184 1185 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa) 1186 p5_tpa = true; 1187 1188 for (i = 0; i < agg_bufs; i++) { 1189 skb_frag_t *frag = &shinfo->frags[i]; 1190 u16 cons, frag_len; 1191 struct rx_agg_cmp *agg; 1192 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1193 struct page *page; 1194 dma_addr_t mapping; 1195 1196 if (p5_tpa) 1197 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1198 else 1199 agg = bnxt_get_agg(bp, cpr, idx, i); 1200 cons = agg->rx_agg_cmp_opaque; 1201 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1202 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1203 1204 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1205 skb_frag_fill_page_desc(frag, cons_rx_buf->page, 1206 cons_rx_buf->offset, frag_len); 1207 shinfo->nr_frags = i + 1; 1208 __clear_bit(cons, rxr->rx_agg_bmap); 1209 1210 /* It is possible for bnxt_alloc_rx_page() to allocate 1211 * a sw_prod index that equals the cons index, so we 1212 * need to clear the cons entry now. 1213 */ 1214 mapping = cons_rx_buf->mapping; 1215 page = cons_rx_buf->page; 1216 cons_rx_buf->page = NULL; 1217 1218 if (xdp && page_is_pfmemalloc(page)) 1219 xdp_buff_set_frag_pfmemalloc(xdp); 1220 1221 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1222 --shinfo->nr_frags; 1223 cons_rx_buf->page = page; 1224 1225 /* Update prod since possibly some pages have been 1226 * allocated already. 1227 */ 1228 rxr->rx_agg_prod = prod; 1229 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1230 return 0; 1231 } 1232 1233 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1234 bp->rx_dir); 1235 1236 total_frag_len += frag_len; 1237 prod = NEXT_RX_AGG(prod); 1238 } 1239 rxr->rx_agg_prod = prod; 1240 return total_frag_len; 1241 } 1242 1243 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp, 1244 struct bnxt_cp_ring_info *cpr, 1245 struct sk_buff *skb, u16 idx, 1246 u32 agg_bufs, bool tpa) 1247 { 1248 struct skb_shared_info *shinfo = skb_shinfo(skb); 1249 u32 total_frag_len = 0; 1250 1251 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx, 1252 agg_bufs, tpa, NULL); 1253 if (!total_frag_len) { 1254 skb_mark_for_recycle(skb); 1255 dev_kfree_skb(skb); 1256 return NULL; 1257 } 1258 1259 skb->data_len += total_frag_len; 1260 skb->len += total_frag_len; 1261 skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs; 1262 return skb; 1263 } 1264 1265 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp, 1266 struct bnxt_cp_ring_info *cpr, 1267 struct xdp_buff *xdp, u16 idx, 1268 u32 agg_bufs, bool tpa) 1269 { 1270 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp); 1271 u32 total_frag_len = 0; 1272 1273 if (!xdp_buff_has_frags(xdp)) 1274 shinfo->nr_frags = 0; 1275 1276 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, 1277 idx, agg_bufs, tpa, xdp); 1278 if (total_frag_len) { 1279 xdp_buff_set_frags_flag(xdp); 1280 shinfo->nr_frags = agg_bufs; 1281 shinfo->xdp_frags_size = total_frag_len; 1282 } 1283 return total_frag_len; 1284 } 1285 1286 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1287 u8 agg_bufs, u32 *raw_cons) 1288 { 1289 u16 last; 1290 struct rx_agg_cmp *agg; 1291 1292 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1293 last = RING_CMP(*raw_cons); 1294 agg = (struct rx_agg_cmp *) 1295 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1296 return RX_AGG_CMP_VALID(agg, *raw_cons); 1297 } 1298 1299 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1300 unsigned int len, 1301 dma_addr_t mapping) 1302 { 1303 struct bnxt *bp = bnapi->bp; 1304 struct pci_dev *pdev = bp->pdev; 1305 struct sk_buff *skb; 1306 1307 skb = napi_alloc_skb(&bnapi->napi, len); 1308 if (!skb) 1309 return NULL; 1310 1311 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1312 bp->rx_dir); 1313 1314 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1315 len + NET_IP_ALIGN); 1316 1317 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1318 bp->rx_dir); 1319 1320 skb_put(skb, len); 1321 return skb; 1322 } 1323 1324 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1325 u32 *raw_cons, void *cmp) 1326 { 1327 struct rx_cmp *rxcmp = cmp; 1328 u32 tmp_raw_cons = *raw_cons; 1329 u8 cmp_type, agg_bufs = 0; 1330 1331 cmp_type = RX_CMP_TYPE(rxcmp); 1332 1333 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1334 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1335 RX_CMP_AGG_BUFS) >> 1336 RX_CMP_AGG_BUFS_SHIFT; 1337 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1338 struct rx_tpa_end_cmp *tpa_end = cmp; 1339 1340 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 1341 return 0; 1342 1343 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1344 } 1345 1346 if (agg_bufs) { 1347 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1348 return -EBUSY; 1349 } 1350 *raw_cons = tmp_raw_cons; 1351 return 0; 1352 } 1353 1354 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1355 { 1356 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1357 u16 idx = agg_id & MAX_TPA_P5_MASK; 1358 1359 if (test_bit(idx, map->agg_idx_bmap)) 1360 idx = find_first_zero_bit(map->agg_idx_bmap, 1361 BNXT_AGG_IDX_BMAP_SIZE); 1362 __set_bit(idx, map->agg_idx_bmap); 1363 map->agg_id_tbl[agg_id] = idx; 1364 return idx; 1365 } 1366 1367 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1368 { 1369 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1370 1371 __clear_bit(idx, map->agg_idx_bmap); 1372 } 1373 1374 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1375 { 1376 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1377 1378 return map->agg_id_tbl[agg_id]; 1379 } 1380 1381 static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info, 1382 struct rx_tpa_start_cmp *tpa_start, 1383 struct rx_tpa_start_cmp_ext *tpa_start1) 1384 { 1385 tpa_info->cfa_code_valid = 1; 1386 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1387 tpa_info->vlan_valid = 0; 1388 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) { 1389 tpa_info->vlan_valid = 1; 1390 tpa_info->metadata = 1391 le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1392 } 1393 } 1394 1395 static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info, 1396 struct rx_tpa_start_cmp *tpa_start, 1397 struct rx_tpa_start_cmp_ext *tpa_start1) 1398 { 1399 tpa_info->vlan_valid = 0; 1400 if (TPA_START_VLAN_VALID(tpa_start)) { 1401 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start); 1402 u32 vlan_proto = ETH_P_8021Q; 1403 1404 tpa_info->vlan_valid = 1; 1405 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD) 1406 vlan_proto = ETH_P_8021AD; 1407 tpa_info->metadata = vlan_proto << 16 | 1408 TPA_START_METADATA0_TCI(tpa_start1); 1409 } 1410 } 1411 1412 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1413 u8 cmp_type, struct rx_tpa_start_cmp *tpa_start, 1414 struct rx_tpa_start_cmp_ext *tpa_start1) 1415 { 1416 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1417 struct bnxt_tpa_info *tpa_info; 1418 u16 cons, prod, agg_id; 1419 struct rx_bd *prod_bd; 1420 dma_addr_t mapping; 1421 1422 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 1423 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1424 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1425 } else { 1426 agg_id = TPA_START_AGG_ID(tpa_start); 1427 } 1428 cons = tpa_start->rx_tpa_start_cmp_opaque; 1429 prod = rxr->rx_prod; 1430 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1431 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 1432 tpa_info = &rxr->rx_tpa[agg_id]; 1433 1434 if (unlikely(cons != rxr->rx_next_cons || 1435 TPA_START_ERROR(tpa_start))) { 1436 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1437 cons, rxr->rx_next_cons, 1438 TPA_START_ERROR_CODE(tpa_start1)); 1439 bnxt_sched_reset_rxr(bp, rxr); 1440 return; 1441 } 1442 prod_rx_buf->data = tpa_info->data; 1443 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1444 1445 mapping = tpa_info->mapping; 1446 prod_rx_buf->mapping = mapping; 1447 1448 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 1449 1450 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1451 1452 tpa_info->data = cons_rx_buf->data; 1453 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1454 cons_rx_buf->data = NULL; 1455 tpa_info->mapping = cons_rx_buf->mapping; 1456 1457 tpa_info->len = 1458 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1459 RX_TPA_START_CMP_LEN_SHIFT; 1460 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1461 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1462 tpa_info->gso_type = SKB_GSO_TCPV4; 1463 if (TPA_START_IS_IPV6(tpa_start1)) 1464 tpa_info->gso_type = SKB_GSO_TCPV6; 1465 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1466 else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP && 1467 TPA_START_HASH_TYPE(tpa_start) == 3) 1468 tpa_info->gso_type = SKB_GSO_TCPV6; 1469 tpa_info->rss_hash = 1470 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1471 } else { 1472 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1473 tpa_info->gso_type = 0; 1474 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1475 } 1476 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1477 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1478 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) 1479 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1); 1480 else 1481 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1); 1482 tpa_info->agg_count = 0; 1483 1484 rxr->rx_prod = NEXT_RX(prod); 1485 cons = RING_RX(bp, NEXT_RX(cons)); 1486 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); 1487 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1488 1489 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1490 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1491 cons_rx_buf->data = NULL; 1492 } 1493 1494 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1495 { 1496 if (agg_bufs) 1497 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1498 } 1499 1500 #ifdef CONFIG_INET 1501 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1502 { 1503 struct udphdr *uh = NULL; 1504 1505 if (ip_proto == htons(ETH_P_IP)) { 1506 struct iphdr *iph = (struct iphdr *)skb->data; 1507 1508 if (iph->protocol == IPPROTO_UDP) 1509 uh = (struct udphdr *)(iph + 1); 1510 } else { 1511 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1512 1513 if (iph->nexthdr == IPPROTO_UDP) 1514 uh = (struct udphdr *)(iph + 1); 1515 } 1516 if (uh) { 1517 if (uh->check) 1518 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1519 else 1520 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1521 } 1522 } 1523 #endif 1524 1525 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1526 int payload_off, int tcp_ts, 1527 struct sk_buff *skb) 1528 { 1529 #ifdef CONFIG_INET 1530 struct tcphdr *th; 1531 int len, nw_off; 1532 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1533 u32 hdr_info = tpa_info->hdr_info; 1534 bool loopback = false; 1535 1536 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1537 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1538 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1539 1540 /* If the packet is an internal loopback packet, the offsets will 1541 * have an extra 4 bytes. 1542 */ 1543 if (inner_mac_off == 4) { 1544 loopback = true; 1545 } else if (inner_mac_off > 4) { 1546 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1547 ETH_HLEN - 2)); 1548 1549 /* We only support inner iPv4/ipv6. If we don't see the 1550 * correct protocol ID, it must be a loopback packet where 1551 * the offsets are off by 4. 1552 */ 1553 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1554 loopback = true; 1555 } 1556 if (loopback) { 1557 /* internal loopback packet, subtract all offsets by 4 */ 1558 inner_ip_off -= 4; 1559 inner_mac_off -= 4; 1560 outer_ip_off -= 4; 1561 } 1562 1563 nw_off = inner_ip_off - ETH_HLEN; 1564 skb_set_network_header(skb, nw_off); 1565 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1566 struct ipv6hdr *iph = ipv6_hdr(skb); 1567 1568 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1569 len = skb->len - skb_transport_offset(skb); 1570 th = tcp_hdr(skb); 1571 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1572 } else { 1573 struct iphdr *iph = ip_hdr(skb); 1574 1575 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1576 len = skb->len - skb_transport_offset(skb); 1577 th = tcp_hdr(skb); 1578 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1579 } 1580 1581 if (inner_mac_off) { /* tunnel */ 1582 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1583 ETH_HLEN - 2)); 1584 1585 bnxt_gro_tunnel(skb, proto); 1586 } 1587 #endif 1588 return skb; 1589 } 1590 1591 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1592 int payload_off, int tcp_ts, 1593 struct sk_buff *skb) 1594 { 1595 #ifdef CONFIG_INET 1596 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1597 u32 hdr_info = tpa_info->hdr_info; 1598 int iphdr_len, nw_off; 1599 1600 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1601 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1602 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1603 1604 nw_off = inner_ip_off - ETH_HLEN; 1605 skb_set_network_header(skb, nw_off); 1606 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1607 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1608 skb_set_transport_header(skb, nw_off + iphdr_len); 1609 1610 if (inner_mac_off) { /* tunnel */ 1611 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1612 ETH_HLEN - 2)); 1613 1614 bnxt_gro_tunnel(skb, proto); 1615 } 1616 #endif 1617 return skb; 1618 } 1619 1620 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1621 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1622 1623 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1624 int payload_off, int tcp_ts, 1625 struct sk_buff *skb) 1626 { 1627 #ifdef CONFIG_INET 1628 struct tcphdr *th; 1629 int len, nw_off, tcp_opt_len = 0; 1630 1631 if (tcp_ts) 1632 tcp_opt_len = 12; 1633 1634 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1635 struct iphdr *iph; 1636 1637 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1638 ETH_HLEN; 1639 skb_set_network_header(skb, nw_off); 1640 iph = ip_hdr(skb); 1641 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1642 len = skb->len - skb_transport_offset(skb); 1643 th = tcp_hdr(skb); 1644 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1645 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1646 struct ipv6hdr *iph; 1647 1648 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1649 ETH_HLEN; 1650 skb_set_network_header(skb, nw_off); 1651 iph = ipv6_hdr(skb); 1652 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1653 len = skb->len - skb_transport_offset(skb); 1654 th = tcp_hdr(skb); 1655 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1656 } else { 1657 dev_kfree_skb_any(skb); 1658 return NULL; 1659 } 1660 1661 if (nw_off) /* tunnel */ 1662 bnxt_gro_tunnel(skb, skb->protocol); 1663 #endif 1664 return skb; 1665 } 1666 1667 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1668 struct bnxt_tpa_info *tpa_info, 1669 struct rx_tpa_end_cmp *tpa_end, 1670 struct rx_tpa_end_cmp_ext *tpa_end1, 1671 struct sk_buff *skb) 1672 { 1673 #ifdef CONFIG_INET 1674 int payload_off; 1675 u16 segs; 1676 1677 segs = TPA_END_TPA_SEGS(tpa_end); 1678 if (segs == 1) 1679 return skb; 1680 1681 NAPI_GRO_CB(skb)->count = segs; 1682 skb_shinfo(skb)->gso_size = 1683 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1684 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1685 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 1686 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1687 else 1688 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1689 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1690 if (likely(skb)) 1691 tcp_gro_complete(skb); 1692 #endif 1693 return skb; 1694 } 1695 1696 /* Given the cfa_code of a received packet determine which 1697 * netdev (vf-rep or PF) the packet is destined to. 1698 */ 1699 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1700 { 1701 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1702 1703 /* if vf-rep dev is NULL, the must belongs to the PF */ 1704 return dev ? dev : bp->dev; 1705 } 1706 1707 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1708 struct bnxt_cp_ring_info *cpr, 1709 u32 *raw_cons, 1710 struct rx_tpa_end_cmp *tpa_end, 1711 struct rx_tpa_end_cmp_ext *tpa_end1, 1712 u8 *event) 1713 { 1714 struct bnxt_napi *bnapi = cpr->bnapi; 1715 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1716 struct net_device *dev = bp->dev; 1717 u8 *data_ptr, agg_bufs; 1718 unsigned int len; 1719 struct bnxt_tpa_info *tpa_info; 1720 dma_addr_t mapping; 1721 struct sk_buff *skb; 1722 u16 idx = 0, agg_id; 1723 void *data; 1724 bool gro; 1725 1726 if (unlikely(bnapi->in_reset)) { 1727 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1728 1729 if (rc < 0) 1730 return ERR_PTR(-EBUSY); 1731 return NULL; 1732 } 1733 1734 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 1735 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1736 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1737 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1738 tpa_info = &rxr->rx_tpa[agg_id]; 1739 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1740 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1741 agg_bufs, tpa_info->agg_count); 1742 agg_bufs = tpa_info->agg_count; 1743 } 1744 tpa_info->agg_count = 0; 1745 *event |= BNXT_AGG_EVENT; 1746 bnxt_free_agg_idx(rxr, agg_id); 1747 idx = agg_id; 1748 gro = !!(bp->flags & BNXT_FLAG_GRO); 1749 } else { 1750 agg_id = TPA_END_AGG_ID(tpa_end); 1751 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1752 tpa_info = &rxr->rx_tpa[agg_id]; 1753 idx = RING_CMP(*raw_cons); 1754 if (agg_bufs) { 1755 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1756 return ERR_PTR(-EBUSY); 1757 1758 *event |= BNXT_AGG_EVENT; 1759 idx = NEXT_CMP(idx); 1760 } 1761 gro = !!TPA_END_GRO(tpa_end); 1762 } 1763 data = tpa_info->data; 1764 data_ptr = tpa_info->data_ptr; 1765 prefetch(data_ptr); 1766 len = tpa_info->len; 1767 mapping = tpa_info->mapping; 1768 1769 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1770 bnxt_abort_tpa(cpr, idx, agg_bufs); 1771 if (agg_bufs > MAX_SKB_FRAGS) 1772 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1773 agg_bufs, (int)MAX_SKB_FRAGS); 1774 return NULL; 1775 } 1776 1777 if (len <= bp->rx_copy_thresh) { 1778 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1779 if (!skb) { 1780 bnxt_abort_tpa(cpr, idx, agg_bufs); 1781 cpr->sw_stats.rx.rx_oom_discards += 1; 1782 return NULL; 1783 } 1784 } else { 1785 u8 *new_data; 1786 dma_addr_t new_mapping; 1787 1788 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); 1789 if (!new_data) { 1790 bnxt_abort_tpa(cpr, idx, agg_bufs); 1791 cpr->sw_stats.rx.rx_oom_discards += 1; 1792 return NULL; 1793 } 1794 1795 tpa_info->data = new_data; 1796 tpa_info->data_ptr = new_data + bp->rx_offset; 1797 tpa_info->mapping = new_mapping; 1798 1799 skb = napi_build_skb(data, bp->rx_buf_size); 1800 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1801 bp->rx_buf_use_size, bp->rx_dir, 1802 DMA_ATTR_WEAK_ORDERING); 1803 1804 if (!skb) { 1805 skb_free_frag(data); 1806 bnxt_abort_tpa(cpr, idx, agg_bufs); 1807 cpr->sw_stats.rx.rx_oom_discards += 1; 1808 return NULL; 1809 } 1810 skb_reserve(skb, bp->rx_offset); 1811 skb_put(skb, len); 1812 } 1813 1814 if (agg_bufs) { 1815 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true); 1816 if (!skb) { 1817 /* Page reuse already handled by bnxt_rx_pages(). */ 1818 cpr->sw_stats.rx.rx_oom_discards += 1; 1819 return NULL; 1820 } 1821 } 1822 1823 if (tpa_info->cfa_code_valid) 1824 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code); 1825 skb->protocol = eth_type_trans(skb, dev); 1826 1827 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1828 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1829 1830 if (tpa_info->vlan_valid && 1831 (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1832 __be16 vlan_proto = htons(tpa_info->metadata >> 1833 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1834 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1835 1836 if (eth_type_vlan(vlan_proto)) { 1837 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1838 } else { 1839 dev_kfree_skb(skb); 1840 return NULL; 1841 } 1842 } 1843 1844 skb_checksum_none_assert(skb); 1845 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1846 skb->ip_summed = CHECKSUM_UNNECESSARY; 1847 skb->csum_level = 1848 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1849 } 1850 1851 if (gro) 1852 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1853 1854 return skb; 1855 } 1856 1857 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1858 struct rx_agg_cmp *rx_agg) 1859 { 1860 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1861 struct bnxt_tpa_info *tpa_info; 1862 1863 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1864 tpa_info = &rxr->rx_tpa[agg_id]; 1865 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1866 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1867 } 1868 1869 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1870 struct sk_buff *skb) 1871 { 1872 skb_mark_for_recycle(skb); 1873 1874 if (skb->dev != bp->dev) { 1875 /* this packet belongs to a vf-rep */ 1876 bnxt_vf_rep_rx(bp, skb); 1877 return; 1878 } 1879 skb_record_rx_queue(skb, bnapi->index); 1880 napi_gro_receive(&bnapi->napi, skb); 1881 } 1882 1883 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags, 1884 struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts) 1885 { 1886 u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 1887 1888 if (BNXT_PTP_RX_TS_VALID(flags)) 1889 goto ts_valid; 1890 if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags)) 1891 return false; 1892 1893 ts_valid: 1894 *cmpl_ts = ts; 1895 return true; 1896 } 1897 1898 static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type, 1899 struct rx_cmp *rxcmp, 1900 struct rx_cmp_ext *rxcmp1) 1901 { 1902 __be16 vlan_proto; 1903 u16 vtag; 1904 1905 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1906 __le32 flags2 = rxcmp1->rx_cmp_flags2; 1907 u32 meta_data; 1908 1909 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN))) 1910 return skb; 1911 1912 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1913 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1914 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT); 1915 if (eth_type_vlan(vlan_proto)) 1916 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1917 else 1918 goto vlan_err; 1919 } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 1920 if (RX_CMP_VLAN_VALID(rxcmp)) { 1921 u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp); 1922 1923 if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q) 1924 vlan_proto = htons(ETH_P_8021Q); 1925 else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD) 1926 vlan_proto = htons(ETH_P_8021AD); 1927 else 1928 goto vlan_err; 1929 vtag = RX_CMP_METADATA0_TCI(rxcmp1); 1930 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1931 } 1932 } 1933 return skb; 1934 vlan_err: 1935 dev_kfree_skb(skb); 1936 return NULL; 1937 } 1938 1939 static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp, 1940 struct rx_cmp *rxcmp) 1941 { 1942 u8 ext_op; 1943 1944 ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp); 1945 switch (ext_op) { 1946 case EXT_OP_INNER_4: 1947 case EXT_OP_OUTER_4: 1948 case EXT_OP_INNFL_3: 1949 case EXT_OP_OUTFL_3: 1950 return PKT_HASH_TYPE_L4; 1951 default: 1952 return PKT_HASH_TYPE_L3; 1953 } 1954 } 1955 1956 /* returns the following: 1957 * 1 - 1 packet successfully received 1958 * 0 - successful TPA_START, packet not completed yet 1959 * -EBUSY - completion ring does not have all the agg buffers yet 1960 * -ENOMEM - packet aborted due to out of memory 1961 * -EIO - packet aborted due to hw error indicated in BD 1962 */ 1963 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1964 u32 *raw_cons, u8 *event) 1965 { 1966 struct bnxt_napi *bnapi = cpr->bnapi; 1967 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1968 struct net_device *dev = bp->dev; 1969 struct rx_cmp *rxcmp; 1970 struct rx_cmp_ext *rxcmp1; 1971 u32 tmp_raw_cons = *raw_cons; 1972 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1973 struct bnxt_sw_rx_bd *rx_buf; 1974 unsigned int len; 1975 u8 *data_ptr, agg_bufs, cmp_type; 1976 bool xdp_active = false; 1977 dma_addr_t dma_addr; 1978 struct sk_buff *skb; 1979 struct xdp_buff xdp; 1980 u32 flags, misc; 1981 u32 cmpl_ts; 1982 void *data; 1983 int rc = 0; 1984 1985 rxcmp = (struct rx_cmp *) 1986 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1987 1988 cmp_type = RX_CMP_TYPE(rxcmp); 1989 1990 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1991 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1992 goto next_rx_no_prod_no_len; 1993 } 1994 1995 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1996 cp_cons = RING_CMP(tmp_raw_cons); 1997 rxcmp1 = (struct rx_cmp_ext *) 1998 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1999 2000 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2001 return -EBUSY; 2002 2003 /* The valid test of the entry must be done first before 2004 * reading any further. 2005 */ 2006 dma_rmb(); 2007 prod = rxr->rx_prod; 2008 2009 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP || 2010 cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) { 2011 bnxt_tpa_start(bp, rxr, cmp_type, 2012 (struct rx_tpa_start_cmp *)rxcmp, 2013 (struct rx_tpa_start_cmp_ext *)rxcmp1); 2014 2015 *event |= BNXT_RX_EVENT; 2016 goto next_rx_no_prod_no_len; 2017 2018 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2019 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 2020 (struct rx_tpa_end_cmp *)rxcmp, 2021 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 2022 2023 if (IS_ERR(skb)) 2024 return -EBUSY; 2025 2026 rc = -ENOMEM; 2027 if (likely(skb)) { 2028 bnxt_deliver_skb(bp, bnapi, skb); 2029 rc = 1; 2030 } 2031 *event |= BNXT_RX_EVENT; 2032 goto next_rx_no_prod_no_len; 2033 } 2034 2035 cons = rxcmp->rx_cmp_opaque; 2036 if (unlikely(cons != rxr->rx_next_cons)) { 2037 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 2038 2039 /* 0xffff is forced error, don't print it */ 2040 if (rxr->rx_next_cons != 0xffff) 2041 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 2042 cons, rxr->rx_next_cons); 2043 bnxt_sched_reset_rxr(bp, rxr); 2044 if (rc1) 2045 return rc1; 2046 goto next_rx_no_prod_no_len; 2047 } 2048 rx_buf = &rxr->rx_buf_ring[cons]; 2049 data = rx_buf->data; 2050 data_ptr = rx_buf->data_ptr; 2051 prefetch(data_ptr); 2052 2053 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 2054 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 2055 2056 if (agg_bufs) { 2057 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 2058 return -EBUSY; 2059 2060 cp_cons = NEXT_CMP(cp_cons); 2061 *event |= BNXT_AGG_EVENT; 2062 } 2063 *event |= BNXT_RX_EVENT; 2064 2065 rx_buf->data = NULL; 2066 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 2067 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 2068 2069 bnxt_reuse_rx_data(rxr, cons, data); 2070 if (agg_bufs) 2071 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 2072 false); 2073 2074 rc = -EIO; 2075 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 2076 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 2077 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 2078 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 2079 netdev_warn_once(bp->dev, "RX buffer error %x\n", 2080 rx_err); 2081 bnxt_sched_reset_rxr(bp, rxr); 2082 } 2083 } 2084 goto next_rx_no_len; 2085 } 2086 2087 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 2088 len = flags >> RX_CMP_LEN_SHIFT; 2089 dma_addr = rx_buf->mapping; 2090 2091 if (bnxt_xdp_attached(bp, rxr)) { 2092 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); 2093 if (agg_bufs) { 2094 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp, 2095 cp_cons, agg_bufs, 2096 false); 2097 if (!frag_len) { 2098 cpr->sw_stats.rx.rx_oom_discards += 1; 2099 rc = -ENOMEM; 2100 goto next_rx; 2101 } 2102 } 2103 xdp_active = true; 2104 } 2105 2106 if (xdp_active) { 2107 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) { 2108 rc = 1; 2109 goto next_rx; 2110 } 2111 } 2112 2113 if (len <= bp->rx_copy_thresh) { 2114 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 2115 bnxt_reuse_rx_data(rxr, cons, data); 2116 if (!skb) { 2117 if (agg_bufs) { 2118 if (!xdp_active) 2119 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 2120 agg_bufs, false); 2121 else 2122 bnxt_xdp_buff_frags_free(rxr, &xdp); 2123 } 2124 cpr->sw_stats.rx.rx_oom_discards += 1; 2125 rc = -ENOMEM; 2126 goto next_rx; 2127 } 2128 } else { 2129 u32 payload; 2130 2131 if (rx_buf->data_ptr == data_ptr) 2132 payload = misc & RX_CMP_PAYLOAD_OFFSET; 2133 else 2134 payload = 0; 2135 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 2136 payload | len); 2137 if (!skb) { 2138 cpr->sw_stats.rx.rx_oom_discards += 1; 2139 rc = -ENOMEM; 2140 goto next_rx; 2141 } 2142 } 2143 2144 if (agg_bufs) { 2145 if (!xdp_active) { 2146 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false); 2147 if (!skb) { 2148 cpr->sw_stats.rx.rx_oom_discards += 1; 2149 rc = -ENOMEM; 2150 goto next_rx; 2151 } 2152 } else { 2153 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); 2154 if (!skb) { 2155 /* we should be able to free the old skb here */ 2156 bnxt_xdp_buff_frags_free(rxr, &xdp); 2157 cpr->sw_stats.rx.rx_oom_discards += 1; 2158 rc = -ENOMEM; 2159 goto next_rx; 2160 } 2161 } 2162 } 2163 2164 if (RX_CMP_HASH_VALID(rxcmp)) { 2165 enum pkt_hash_types type; 2166 2167 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 2168 type = bnxt_rss_ext_op(bp, rxcmp); 2169 } else { 2170 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 2171 2172 /* RSS profiles 1 and 3 with extract code 0 for inner 2173 * 4-tuple 2174 */ 2175 if (hash_type != 1 && hash_type != 3) 2176 type = PKT_HASH_TYPE_L3; 2177 else 2178 type = PKT_HASH_TYPE_L4; 2179 } 2180 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 2181 } 2182 2183 if (cmp_type == CMP_TYPE_RX_L2_CMP) 2184 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1)); 2185 skb->protocol = eth_type_trans(skb, dev); 2186 2187 if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) { 2188 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1); 2189 if (!skb) 2190 goto next_rx; 2191 } 2192 2193 skb_checksum_none_assert(skb); 2194 if (RX_CMP_L4_CS_OK(rxcmp1)) { 2195 if (dev->features & NETIF_F_RXCSUM) { 2196 skb->ip_summed = CHECKSUM_UNNECESSARY; 2197 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 2198 } 2199 } else { 2200 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 2201 if (dev->features & NETIF_F_RXCSUM) 2202 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 2203 } 2204 } 2205 2206 if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) { 2207 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 2208 u64 ns, ts; 2209 2210 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 2211 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2212 2213 spin_lock_bh(&ptp->ptp_lock); 2214 ns = timecounter_cyc2time(&ptp->tc, ts); 2215 spin_unlock_bh(&ptp->ptp_lock); 2216 memset(skb_hwtstamps(skb), 0, 2217 sizeof(*skb_hwtstamps(skb))); 2218 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 2219 } 2220 } 2221 } 2222 bnxt_deliver_skb(bp, bnapi, skb); 2223 rc = 1; 2224 2225 next_rx: 2226 cpr->rx_packets += 1; 2227 cpr->rx_bytes += len; 2228 2229 next_rx_no_len: 2230 rxr->rx_prod = NEXT_RX(prod); 2231 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); 2232 2233 next_rx_no_prod_no_len: 2234 *raw_cons = tmp_raw_cons; 2235 2236 return rc; 2237 } 2238 2239 /* In netpoll mode, if we are using a combined completion ring, we need to 2240 * discard the rx packets and recycle the buffers. 2241 */ 2242 static int bnxt_force_rx_discard(struct bnxt *bp, 2243 struct bnxt_cp_ring_info *cpr, 2244 u32 *raw_cons, u8 *event) 2245 { 2246 u32 tmp_raw_cons = *raw_cons; 2247 struct rx_cmp_ext *rxcmp1; 2248 struct rx_cmp *rxcmp; 2249 u16 cp_cons; 2250 u8 cmp_type; 2251 int rc; 2252 2253 cp_cons = RING_CMP(tmp_raw_cons); 2254 rxcmp = (struct rx_cmp *) 2255 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2256 2257 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 2258 cp_cons = RING_CMP(tmp_raw_cons); 2259 rxcmp1 = (struct rx_cmp_ext *) 2260 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2261 2262 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2263 return -EBUSY; 2264 2265 /* The valid test of the entry must be done first before 2266 * reading any further. 2267 */ 2268 dma_rmb(); 2269 cmp_type = RX_CMP_TYPE(rxcmp); 2270 if (cmp_type == CMP_TYPE_RX_L2_CMP || 2271 cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 2272 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2273 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2274 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2275 struct rx_tpa_end_cmp_ext *tpa_end1; 2276 2277 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 2278 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 2279 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 2280 } 2281 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2282 if (rc && rc != -EBUSY) 2283 cpr->sw_stats.rx.rx_netpoll_discards += 1; 2284 return rc; 2285 } 2286 2287 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2288 { 2289 struct bnxt_fw_health *fw_health = bp->fw_health; 2290 u32 reg = fw_health->regs[reg_idx]; 2291 u32 reg_type, reg_off, val = 0; 2292 2293 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2294 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2295 switch (reg_type) { 2296 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2297 pci_read_config_dword(bp->pdev, reg_off, &val); 2298 break; 2299 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2300 reg_off = fw_health->mapped_regs[reg_idx]; 2301 fallthrough; 2302 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2303 val = readl(bp->bar0 + reg_off); 2304 break; 2305 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2306 val = readl(bp->bar1 + reg_off); 2307 break; 2308 } 2309 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2310 val &= fw_health->fw_reset_inprog_reg_mask; 2311 return val; 2312 } 2313 2314 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2315 { 2316 int i; 2317 2318 for (i = 0; i < bp->rx_nr_rings; i++) { 2319 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2320 struct bnxt_ring_grp_info *grp_info; 2321 2322 grp_info = &bp->grp_info[grp_idx]; 2323 if (grp_info->agg_fw_ring_id == ring_id) 2324 return grp_idx; 2325 } 2326 return INVALID_HW_RING_ID; 2327 } 2328 2329 static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info) 2330 { 2331 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2332 2333 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) 2334 return link_info->force_link_speed2; 2335 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4) 2336 return link_info->force_pam4_link_speed; 2337 return link_info->force_link_speed; 2338 } 2339 2340 static void bnxt_set_force_speed(struct bnxt_link_info *link_info) 2341 { 2342 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2343 2344 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2345 link_info->req_link_speed = link_info->force_link_speed2; 2346 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2347 switch (link_info->req_link_speed) { 2348 case BNXT_LINK_SPEED_50GB_PAM4: 2349 case BNXT_LINK_SPEED_100GB_PAM4: 2350 case BNXT_LINK_SPEED_200GB_PAM4: 2351 case BNXT_LINK_SPEED_400GB_PAM4: 2352 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 2353 break; 2354 case BNXT_LINK_SPEED_100GB_PAM4_112: 2355 case BNXT_LINK_SPEED_200GB_PAM4_112: 2356 case BNXT_LINK_SPEED_400GB_PAM4_112: 2357 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112; 2358 break; 2359 default: 2360 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2361 } 2362 return; 2363 } 2364 link_info->req_link_speed = link_info->force_link_speed; 2365 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2366 if (link_info->force_pam4_link_speed) { 2367 link_info->req_link_speed = link_info->force_pam4_link_speed; 2368 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 2369 } 2370 } 2371 2372 static void bnxt_set_auto_speed(struct bnxt_link_info *link_info) 2373 { 2374 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2375 2376 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2377 link_info->advertising = link_info->auto_link_speeds2; 2378 return; 2379 } 2380 link_info->advertising = link_info->auto_link_speeds; 2381 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 2382 } 2383 2384 static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info) 2385 { 2386 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2387 2388 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2389 if (link_info->req_link_speed != link_info->force_link_speed2) 2390 return true; 2391 return false; 2392 } 2393 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 2394 link_info->req_link_speed != link_info->force_link_speed) 2395 return true; 2396 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 2397 link_info->req_link_speed != link_info->force_pam4_link_speed) 2398 return true; 2399 return false; 2400 } 2401 2402 static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info) 2403 { 2404 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2405 2406 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2407 if (link_info->advertising != link_info->auto_link_speeds2) 2408 return true; 2409 return false; 2410 } 2411 if (link_info->advertising != link_info->auto_link_speeds || 2412 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 2413 return true; 2414 return false; 2415 } 2416 2417 #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \ 2418 ((data2) & \ 2419 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK) 2420 2421 #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \ 2422 (((data2) & \ 2423 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\ 2424 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT) 2425 2426 #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \ 2427 ((data1) & \ 2428 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK) 2429 2430 #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \ 2431 (((data1) & \ 2432 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\ 2433 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING) 2434 2435 /* Return true if the workqueue has to be scheduled */ 2436 static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2437 { 2438 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2439 2440 switch (err_type) { 2441 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2442 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2443 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2444 break; 2445 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2446 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2447 break; 2448 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2449 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2450 break; 2451 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: { 2452 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1); 2453 char *threshold_type; 2454 bool notify = false; 2455 char *dir_str; 2456 2457 switch (type) { 2458 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN: 2459 threshold_type = "warning"; 2460 break; 2461 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL: 2462 threshold_type = "critical"; 2463 break; 2464 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL: 2465 threshold_type = "fatal"; 2466 break; 2467 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN: 2468 threshold_type = "shutdown"; 2469 break; 2470 default: 2471 netdev_err(bp->dev, "Unknown Thermal threshold type event\n"); 2472 return false; 2473 } 2474 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) { 2475 dir_str = "above"; 2476 notify = true; 2477 } else { 2478 dir_str = "below"; 2479 } 2480 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n", 2481 dir_str, threshold_type); 2482 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n", 2483 BNXT_EVENT_THERMAL_CURRENT_TEMP(data2), 2484 BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)); 2485 if (notify) { 2486 bp->thermal_threshold_type = type; 2487 set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event); 2488 return true; 2489 } 2490 return false; 2491 } 2492 default: 2493 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2494 err_type); 2495 break; 2496 } 2497 return false; 2498 } 2499 2500 #define BNXT_GET_EVENT_PORT(data) \ 2501 ((data) & \ 2502 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2503 2504 #define BNXT_EVENT_RING_TYPE(data2) \ 2505 ((data2) & \ 2506 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2507 2508 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 2509 (BNXT_EVENT_RING_TYPE(data2) == \ 2510 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2511 2512 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \ 2513 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\ 2514 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT) 2515 2516 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \ 2517 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\ 2518 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT) 2519 2520 #define BNXT_PHC_BITS 48 2521 2522 static int bnxt_async_event_process(struct bnxt *bp, 2523 struct hwrm_async_event_cmpl *cmpl) 2524 { 2525 u16 event_id = le16_to_cpu(cmpl->event_id); 2526 u32 data1 = le32_to_cpu(cmpl->event_data1); 2527 u32 data2 = le32_to_cpu(cmpl->event_data2); 2528 2529 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2530 event_id, data1, data2); 2531 2532 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2533 switch (event_id) { 2534 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2535 struct bnxt_link_info *link_info = &bp->link_info; 2536 2537 if (BNXT_VF(bp)) 2538 goto async_event_process_exit; 2539 2540 /* print unsupported speed warning in forced speed mode only */ 2541 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2542 (data1 & 0x20000)) { 2543 u16 fw_speed = bnxt_get_force_speed(link_info); 2544 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2545 2546 if (speed != SPEED_UNKNOWN) 2547 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2548 speed); 2549 } 2550 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2551 } 2552 fallthrough; 2553 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2554 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2555 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2556 fallthrough; 2557 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2558 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2559 break; 2560 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2561 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2562 break; 2563 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2564 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2565 2566 if (BNXT_VF(bp)) 2567 break; 2568 2569 if (bp->pf.port_id != port_id) 2570 break; 2571 2572 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2573 break; 2574 } 2575 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2576 if (BNXT_PF(bp)) 2577 goto async_event_process_exit; 2578 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2579 break; 2580 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2581 char *type_str = "Solicited"; 2582 2583 if (!bp->fw_health) 2584 goto async_event_process_exit; 2585 2586 bp->fw_reset_timestamp = jiffies; 2587 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2588 if (!bp->fw_reset_min_dsecs) 2589 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2590 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2591 if (!bp->fw_reset_max_dsecs) 2592 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2593 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2594 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2595 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2596 type_str = "Fatal"; 2597 bp->fw_health->fatalities++; 2598 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2599 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2600 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2601 type_str = "Non-fatal"; 2602 bp->fw_health->survivals++; 2603 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2604 } 2605 netif_warn(bp, hw, bp->dev, 2606 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2607 type_str, data1, data2, 2608 bp->fw_reset_min_dsecs * 100, 2609 bp->fw_reset_max_dsecs * 100); 2610 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2611 break; 2612 } 2613 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2614 struct bnxt_fw_health *fw_health = bp->fw_health; 2615 char *status_desc = "healthy"; 2616 u32 status; 2617 2618 if (!fw_health) 2619 goto async_event_process_exit; 2620 2621 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2622 fw_health->enabled = false; 2623 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2624 break; 2625 } 2626 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2627 fw_health->tmr_multiplier = 2628 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2629 bp->current_interval * 10); 2630 fw_health->tmr_counter = fw_health->tmr_multiplier; 2631 if (!fw_health->enabled) 2632 fw_health->last_fw_heartbeat = 2633 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2634 fw_health->last_fw_reset_cnt = 2635 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2636 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2637 if (status != BNXT_FW_STATUS_HEALTHY) 2638 status_desc = "unhealthy"; 2639 netif_info(bp, drv, bp->dev, 2640 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2641 fw_health->primary ? "primary" : "backup", status, 2642 status_desc, fw_health->last_fw_reset_cnt); 2643 if (!fw_health->enabled) { 2644 /* Make sure tmr_counter is set and visible to 2645 * bnxt_health_check() before setting enabled to true. 2646 */ 2647 smp_wmb(); 2648 fw_health->enabled = true; 2649 } 2650 goto async_event_process_exit; 2651 } 2652 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2653 netif_notice(bp, hw, bp->dev, 2654 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2655 data1, data2); 2656 goto async_event_process_exit; 2657 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2658 struct bnxt_rx_ring_info *rxr; 2659 u16 grp_idx; 2660 2661 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 2662 goto async_event_process_exit; 2663 2664 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2665 BNXT_EVENT_RING_TYPE(data2), data1); 2666 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2667 goto async_event_process_exit; 2668 2669 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2670 if (grp_idx == INVALID_HW_RING_ID) { 2671 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2672 data1); 2673 goto async_event_process_exit; 2674 } 2675 rxr = bp->bnapi[grp_idx]->rx_ring; 2676 bnxt_sched_reset_rxr(bp, rxr); 2677 goto async_event_process_exit; 2678 } 2679 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2680 struct bnxt_fw_health *fw_health = bp->fw_health; 2681 2682 netif_notice(bp, hw, bp->dev, 2683 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2684 data1, data2); 2685 if (fw_health) { 2686 fw_health->echo_req_data1 = data1; 2687 fw_health->echo_req_data2 = data2; 2688 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2689 break; 2690 } 2691 goto async_event_process_exit; 2692 } 2693 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2694 bnxt_ptp_pps_event(bp, data1, data2); 2695 goto async_event_process_exit; 2696 } 2697 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2698 if (bnxt_event_error_report(bp, data1, data2)) 2699 break; 2700 goto async_event_process_exit; 2701 } 2702 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: { 2703 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) { 2704 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE: 2705 if (BNXT_PTP_USE_RTC(bp)) { 2706 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2707 u64 ns; 2708 2709 if (!ptp) 2710 goto async_event_process_exit; 2711 2712 spin_lock_bh(&ptp->ptp_lock); 2713 bnxt_ptp_update_current_time(bp); 2714 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << 2715 BNXT_PHC_BITS) | ptp->current_time); 2716 bnxt_ptp_rtc_timecounter_init(ptp, ns); 2717 spin_unlock_bh(&ptp->ptp_lock); 2718 } 2719 break; 2720 } 2721 goto async_event_process_exit; 2722 } 2723 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2724 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2725 2726 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2727 goto async_event_process_exit; 2728 } 2729 default: 2730 goto async_event_process_exit; 2731 } 2732 __bnxt_queue_sp_work(bp); 2733 async_event_process_exit: 2734 return 0; 2735 } 2736 2737 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2738 { 2739 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2740 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2741 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2742 (struct hwrm_fwd_req_cmpl *)txcmp; 2743 2744 switch (cmpl_type) { 2745 case CMPL_BASE_TYPE_HWRM_DONE: 2746 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2747 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2748 break; 2749 2750 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2751 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2752 2753 if ((vf_id < bp->pf.first_vf_id) || 2754 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2755 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2756 vf_id); 2757 return -EINVAL; 2758 } 2759 2760 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2761 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT); 2762 break; 2763 2764 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2765 bnxt_async_event_process(bp, 2766 (struct hwrm_async_event_cmpl *)txcmp); 2767 break; 2768 2769 default: 2770 break; 2771 } 2772 2773 return 0; 2774 } 2775 2776 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2777 { 2778 struct bnxt_napi *bnapi = dev_instance; 2779 struct bnxt *bp = bnapi->bp; 2780 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2781 u32 cons = RING_CMP(cpr->cp_raw_cons); 2782 2783 cpr->event_ctr++; 2784 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2785 napi_schedule(&bnapi->napi); 2786 return IRQ_HANDLED; 2787 } 2788 2789 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2790 { 2791 u32 raw_cons = cpr->cp_raw_cons; 2792 u16 cons = RING_CMP(raw_cons); 2793 struct tx_cmp *txcmp; 2794 2795 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2796 2797 return TX_CMP_VALID(txcmp, raw_cons); 2798 } 2799 2800 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2801 { 2802 struct bnxt_napi *bnapi = dev_instance; 2803 struct bnxt *bp = bnapi->bp; 2804 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2805 u32 cons = RING_CMP(cpr->cp_raw_cons); 2806 u32 int_status; 2807 2808 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2809 2810 if (!bnxt_has_work(bp, cpr)) { 2811 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2812 /* return if erroneous interrupt */ 2813 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2814 return IRQ_NONE; 2815 } 2816 2817 /* disable ring IRQ */ 2818 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2819 2820 /* Return here if interrupt is shared and is disabled. */ 2821 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2822 return IRQ_HANDLED; 2823 2824 napi_schedule(&bnapi->napi); 2825 return IRQ_HANDLED; 2826 } 2827 2828 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2829 int budget) 2830 { 2831 struct bnxt_napi *bnapi = cpr->bnapi; 2832 u32 raw_cons = cpr->cp_raw_cons; 2833 u32 cons; 2834 int rx_pkts = 0; 2835 u8 event = 0; 2836 struct tx_cmp *txcmp; 2837 2838 cpr->has_more_work = 0; 2839 cpr->had_work_done = 1; 2840 while (1) { 2841 u8 cmp_type; 2842 int rc; 2843 2844 cons = RING_CMP(raw_cons); 2845 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2846 2847 if (!TX_CMP_VALID(txcmp, raw_cons)) 2848 break; 2849 2850 /* The valid test of the entry must be done first before 2851 * reading any further. 2852 */ 2853 dma_rmb(); 2854 cmp_type = TX_CMP_TYPE(txcmp); 2855 if (cmp_type == CMP_TYPE_TX_L2_CMP || 2856 cmp_type == CMP_TYPE_TX_L2_COAL_CMP) { 2857 u32 opaque = txcmp->tx_cmp_opaque; 2858 struct bnxt_tx_ring_info *txr; 2859 u16 tx_freed; 2860 2861 txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)]; 2862 event |= BNXT_TX_CMP_EVENT; 2863 if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP) 2864 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp); 2865 else 2866 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque); 2867 tx_freed = (txr->tx_hw_cons - txr->tx_cons) & 2868 bp->tx_ring_mask; 2869 /* return full budget so NAPI will complete. */ 2870 if (unlikely(tx_freed >= bp->tx_wake_thresh)) { 2871 rx_pkts = budget; 2872 raw_cons = NEXT_RAW_CMP(raw_cons); 2873 if (budget) 2874 cpr->has_more_work = 1; 2875 break; 2876 } 2877 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP && 2878 cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) { 2879 if (likely(budget)) 2880 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2881 else 2882 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2883 &event); 2884 if (likely(rc >= 0)) 2885 rx_pkts += rc; 2886 /* Increment rx_pkts when rc is -ENOMEM to count towards 2887 * the NAPI budget. Otherwise, we may potentially loop 2888 * here forever if we consistently cannot allocate 2889 * buffers. 2890 */ 2891 else if (rc == -ENOMEM && budget) 2892 rx_pkts++; 2893 else if (rc == -EBUSY) /* partial completion */ 2894 break; 2895 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE || 2896 cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ || 2897 cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) { 2898 bnxt_hwrm_handler(bp, txcmp); 2899 } 2900 raw_cons = NEXT_RAW_CMP(raw_cons); 2901 2902 if (rx_pkts && rx_pkts == budget) { 2903 cpr->has_more_work = 1; 2904 break; 2905 } 2906 } 2907 2908 if (event & BNXT_REDIRECT_EVENT) 2909 xdp_do_flush(); 2910 2911 if (event & BNXT_TX_EVENT) { 2912 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0]; 2913 u16 prod = txr->tx_prod; 2914 2915 /* Sync BD data before updating doorbell */ 2916 wmb(); 2917 2918 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2919 } 2920 2921 cpr->cp_raw_cons = raw_cons; 2922 bnapi->events |= event; 2923 return rx_pkts; 2924 } 2925 2926 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2927 int budget) 2928 { 2929 if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault) 2930 bnapi->tx_int(bp, bnapi, budget); 2931 2932 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2933 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2934 2935 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2936 } 2937 if (bnapi->events & BNXT_AGG_EVENT) { 2938 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2939 2940 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2941 } 2942 bnapi->events &= BNXT_TX_CMP_EVENT; 2943 } 2944 2945 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2946 int budget) 2947 { 2948 struct bnxt_napi *bnapi = cpr->bnapi; 2949 int rx_pkts; 2950 2951 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2952 2953 /* ACK completion ring before freeing tx ring and producing new 2954 * buffers in rx/agg rings to prevent overflowing the completion 2955 * ring. 2956 */ 2957 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2958 2959 __bnxt_poll_work_done(bp, bnapi, budget); 2960 return rx_pkts; 2961 } 2962 2963 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2964 { 2965 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2966 struct bnxt *bp = bnapi->bp; 2967 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2968 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2969 struct tx_cmp *txcmp; 2970 struct rx_cmp_ext *rxcmp1; 2971 u32 cp_cons, tmp_raw_cons; 2972 u32 raw_cons = cpr->cp_raw_cons; 2973 bool flush_xdp = false; 2974 u32 rx_pkts = 0; 2975 u8 event = 0; 2976 2977 while (1) { 2978 int rc; 2979 2980 cp_cons = RING_CMP(raw_cons); 2981 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2982 2983 if (!TX_CMP_VALID(txcmp, raw_cons)) 2984 break; 2985 2986 /* The valid test of the entry must be done first before 2987 * reading any further. 2988 */ 2989 dma_rmb(); 2990 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2991 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2992 cp_cons = RING_CMP(tmp_raw_cons); 2993 rxcmp1 = (struct rx_cmp_ext *) 2994 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2995 2996 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2997 break; 2998 2999 /* force an error to recycle the buffer */ 3000 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 3001 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 3002 3003 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 3004 if (likely(rc == -EIO) && budget) 3005 rx_pkts++; 3006 else if (rc == -EBUSY) /* partial completion */ 3007 break; 3008 if (event & BNXT_REDIRECT_EVENT) 3009 flush_xdp = true; 3010 } else if (unlikely(TX_CMP_TYPE(txcmp) == 3011 CMPL_BASE_TYPE_HWRM_DONE)) { 3012 bnxt_hwrm_handler(bp, txcmp); 3013 } else { 3014 netdev_err(bp->dev, 3015 "Invalid completion received on special ring\n"); 3016 } 3017 raw_cons = NEXT_RAW_CMP(raw_cons); 3018 3019 if (rx_pkts == budget) 3020 break; 3021 } 3022 3023 cpr->cp_raw_cons = raw_cons; 3024 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 3025 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 3026 3027 if (event & BNXT_AGG_EVENT) 3028 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 3029 if (flush_xdp) 3030 xdp_do_flush(); 3031 3032 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 3033 napi_complete_done(napi, rx_pkts); 3034 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3035 } 3036 return rx_pkts; 3037 } 3038 3039 static int bnxt_poll(struct napi_struct *napi, int budget) 3040 { 3041 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 3042 struct bnxt *bp = bnapi->bp; 3043 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3044 int work_done = 0; 3045 3046 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 3047 napi_complete(napi); 3048 return 0; 3049 } 3050 while (1) { 3051 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 3052 3053 if (work_done >= budget) { 3054 if (!budget) 3055 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3056 break; 3057 } 3058 3059 if (!bnxt_has_work(bp, cpr)) { 3060 if (napi_complete_done(napi, work_done)) 3061 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3062 break; 3063 } 3064 } 3065 if (bp->flags & BNXT_FLAG_DIM) { 3066 struct dim_sample dim_sample = {}; 3067 3068 dim_update_sample(cpr->event_ctr, 3069 cpr->rx_packets, 3070 cpr->rx_bytes, 3071 &dim_sample); 3072 net_dim(&cpr->dim, dim_sample); 3073 } 3074 return work_done; 3075 } 3076 3077 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 3078 { 3079 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3080 int i, work_done = 0; 3081 3082 for (i = 0; i < cpr->cp_ring_count; i++) { 3083 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i]; 3084 3085 if (cpr2->had_nqe_notify) { 3086 work_done += __bnxt_poll_work(bp, cpr2, 3087 budget - work_done); 3088 cpr->has_more_work |= cpr2->has_more_work; 3089 } 3090 } 3091 return work_done; 3092 } 3093 3094 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 3095 u64 dbr_type, int budget) 3096 { 3097 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3098 int i; 3099 3100 for (i = 0; i < cpr->cp_ring_count; i++) { 3101 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i]; 3102 struct bnxt_db_info *db; 3103 3104 if (cpr2->had_work_done) { 3105 u32 tgl = 0; 3106 3107 if (dbr_type == DBR_TYPE_CQ_ARMALL) { 3108 cpr2->had_nqe_notify = 0; 3109 tgl = cpr2->toggle; 3110 } 3111 db = &cpr2->cp_db; 3112 bnxt_writeq(bp, 3113 db->db_key64 | dbr_type | DB_TOGGLE(tgl) | 3114 DB_RING_IDX(db, cpr2->cp_raw_cons), 3115 db->doorbell); 3116 cpr2->had_work_done = 0; 3117 } 3118 } 3119 __bnxt_poll_work_done(bp, bnapi, budget); 3120 } 3121 3122 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 3123 { 3124 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 3125 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3126 struct bnxt_cp_ring_info *cpr_rx; 3127 u32 raw_cons = cpr->cp_raw_cons; 3128 struct bnxt *bp = bnapi->bp; 3129 struct nqe_cn *nqcmp; 3130 int work_done = 0; 3131 u32 cons; 3132 3133 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 3134 napi_complete(napi); 3135 return 0; 3136 } 3137 if (cpr->has_more_work) { 3138 cpr->has_more_work = 0; 3139 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 3140 } 3141 while (1) { 3142 u16 type; 3143 3144 cons = RING_CMP(raw_cons); 3145 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 3146 3147 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 3148 if (cpr->has_more_work) 3149 break; 3150 3151 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, 3152 budget); 3153 cpr->cp_raw_cons = raw_cons; 3154 if (napi_complete_done(napi, work_done)) 3155 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 3156 cpr->cp_raw_cons); 3157 goto poll_done; 3158 } 3159 3160 /* The valid test of the entry must be done first before 3161 * reading any further. 3162 */ 3163 dma_rmb(); 3164 3165 type = le16_to_cpu(nqcmp->type); 3166 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) { 3167 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 3168 u32 cq_type = BNXT_NQ_HDL_TYPE(idx); 3169 struct bnxt_cp_ring_info *cpr2; 3170 3171 /* No more budget for RX work */ 3172 if (budget && work_done >= budget && 3173 cq_type == BNXT_NQ_HDL_TYPE_RX) 3174 break; 3175 3176 idx = BNXT_NQ_HDL_IDX(idx); 3177 cpr2 = &cpr->cp_ring_arr[idx]; 3178 cpr2->had_nqe_notify = 1; 3179 cpr2->toggle = NQE_CN_TOGGLE(type); 3180 work_done += __bnxt_poll_work(bp, cpr2, 3181 budget - work_done); 3182 cpr->has_more_work |= cpr2->has_more_work; 3183 } else { 3184 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 3185 } 3186 raw_cons = NEXT_RAW_CMP(raw_cons); 3187 } 3188 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget); 3189 if (raw_cons != cpr->cp_raw_cons) { 3190 cpr->cp_raw_cons = raw_cons; 3191 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 3192 } 3193 poll_done: 3194 cpr_rx = &cpr->cp_ring_arr[0]; 3195 if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX && 3196 (bp->flags & BNXT_FLAG_DIM)) { 3197 struct dim_sample dim_sample = {}; 3198 3199 dim_update_sample(cpr->event_ctr, 3200 cpr_rx->rx_packets, 3201 cpr_rx->rx_bytes, 3202 &dim_sample); 3203 net_dim(&cpr->dim, dim_sample); 3204 } 3205 return work_done; 3206 } 3207 3208 static void bnxt_free_tx_skbs(struct bnxt *bp) 3209 { 3210 int i, max_idx; 3211 struct pci_dev *pdev = bp->pdev; 3212 3213 if (!bp->tx_ring) 3214 return; 3215 3216 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 3217 for (i = 0; i < bp->tx_nr_rings; i++) { 3218 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3219 int j; 3220 3221 if (!txr->tx_buf_ring) 3222 continue; 3223 3224 for (j = 0; j < max_idx;) { 3225 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 3226 struct sk_buff *skb; 3227 int k, last; 3228 3229 if (i < bp->tx_nr_rings_xdp && 3230 tx_buf->action == XDP_REDIRECT) { 3231 dma_unmap_single(&pdev->dev, 3232 dma_unmap_addr(tx_buf, mapping), 3233 dma_unmap_len(tx_buf, len), 3234 DMA_TO_DEVICE); 3235 xdp_return_frame(tx_buf->xdpf); 3236 tx_buf->action = 0; 3237 tx_buf->xdpf = NULL; 3238 j++; 3239 continue; 3240 } 3241 3242 skb = tx_buf->skb; 3243 if (!skb) { 3244 j++; 3245 continue; 3246 } 3247 3248 tx_buf->skb = NULL; 3249 3250 if (tx_buf->is_push) { 3251 dev_kfree_skb(skb); 3252 j += 2; 3253 continue; 3254 } 3255 3256 dma_unmap_single(&pdev->dev, 3257 dma_unmap_addr(tx_buf, mapping), 3258 skb_headlen(skb), 3259 DMA_TO_DEVICE); 3260 3261 last = tx_buf->nr_frags; 3262 j += 2; 3263 for (k = 0; k < last; k++, j++) { 3264 int ring_idx = j & bp->tx_ring_mask; 3265 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 3266 3267 tx_buf = &txr->tx_buf_ring[ring_idx]; 3268 dma_unmap_page( 3269 &pdev->dev, 3270 dma_unmap_addr(tx_buf, mapping), 3271 skb_frag_size(frag), DMA_TO_DEVICE); 3272 } 3273 dev_kfree_skb(skb); 3274 } 3275 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 3276 } 3277 } 3278 3279 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 3280 { 3281 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3282 struct pci_dev *pdev = bp->pdev; 3283 struct bnxt_tpa_idx_map *map; 3284 int i, max_idx, max_agg_idx; 3285 3286 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 3287 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 3288 if (!rxr->rx_tpa) 3289 goto skip_rx_tpa_free; 3290 3291 for (i = 0; i < bp->max_tpa; i++) { 3292 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 3293 u8 *data = tpa_info->data; 3294 3295 if (!data) 3296 continue; 3297 3298 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 3299 bp->rx_buf_use_size, bp->rx_dir, 3300 DMA_ATTR_WEAK_ORDERING); 3301 3302 tpa_info->data = NULL; 3303 3304 skb_free_frag(data); 3305 } 3306 3307 skip_rx_tpa_free: 3308 if (!rxr->rx_buf_ring) 3309 goto skip_rx_buf_free; 3310 3311 for (i = 0; i < max_idx; i++) { 3312 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 3313 dma_addr_t mapping = rx_buf->mapping; 3314 void *data = rx_buf->data; 3315 3316 if (!data) 3317 continue; 3318 3319 rx_buf->data = NULL; 3320 if (BNXT_RX_PAGE_MODE(bp)) { 3321 page_pool_recycle_direct(rxr->page_pool, data); 3322 } else { 3323 dma_unmap_single_attrs(&pdev->dev, mapping, 3324 bp->rx_buf_use_size, bp->rx_dir, 3325 DMA_ATTR_WEAK_ORDERING); 3326 skb_free_frag(data); 3327 } 3328 } 3329 3330 skip_rx_buf_free: 3331 if (!rxr->rx_agg_ring) 3332 goto skip_rx_agg_free; 3333 3334 for (i = 0; i < max_agg_idx; i++) { 3335 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 3336 struct page *page = rx_agg_buf->page; 3337 3338 if (!page) 3339 continue; 3340 3341 rx_agg_buf->page = NULL; 3342 __clear_bit(i, rxr->rx_agg_bmap); 3343 3344 page_pool_recycle_direct(rxr->page_pool, page); 3345 } 3346 3347 skip_rx_agg_free: 3348 map = rxr->rx_tpa_idx_map; 3349 if (map) 3350 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 3351 } 3352 3353 static void bnxt_free_rx_skbs(struct bnxt *bp) 3354 { 3355 int i; 3356 3357 if (!bp->rx_ring) 3358 return; 3359 3360 for (i = 0; i < bp->rx_nr_rings; i++) 3361 bnxt_free_one_rx_ring_skbs(bp, i); 3362 } 3363 3364 static void bnxt_free_skbs(struct bnxt *bp) 3365 { 3366 bnxt_free_tx_skbs(bp); 3367 bnxt_free_rx_skbs(bp); 3368 } 3369 3370 static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len) 3371 { 3372 u8 init_val = ctxm->init_value; 3373 u16 offset = ctxm->init_offset; 3374 u8 *p2 = p; 3375 int i; 3376 3377 if (!init_val) 3378 return; 3379 if (offset == BNXT_CTX_INIT_INVALID_OFFSET) { 3380 memset(p, init_val, len); 3381 return; 3382 } 3383 for (i = 0; i < len; i += ctxm->entry_size) 3384 *(p2 + i + offset) = init_val; 3385 } 3386 3387 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3388 { 3389 struct pci_dev *pdev = bp->pdev; 3390 int i; 3391 3392 if (!rmem->pg_arr) 3393 goto skip_pages; 3394 3395 for (i = 0; i < rmem->nr_pages; i++) { 3396 if (!rmem->pg_arr[i]) 3397 continue; 3398 3399 dma_free_coherent(&pdev->dev, rmem->page_size, 3400 rmem->pg_arr[i], rmem->dma_arr[i]); 3401 3402 rmem->pg_arr[i] = NULL; 3403 } 3404 skip_pages: 3405 if (rmem->pg_tbl) { 3406 size_t pg_tbl_size = rmem->nr_pages * 8; 3407 3408 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3409 pg_tbl_size = rmem->page_size; 3410 dma_free_coherent(&pdev->dev, pg_tbl_size, 3411 rmem->pg_tbl, rmem->pg_tbl_map); 3412 rmem->pg_tbl = NULL; 3413 } 3414 if (rmem->vmem_size && *rmem->vmem) { 3415 vfree(*rmem->vmem); 3416 *rmem->vmem = NULL; 3417 } 3418 } 3419 3420 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3421 { 3422 struct pci_dev *pdev = bp->pdev; 3423 u64 valid_bit = 0; 3424 int i; 3425 3426 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 3427 valid_bit = PTU_PTE_VALID; 3428 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 3429 size_t pg_tbl_size = rmem->nr_pages * 8; 3430 3431 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3432 pg_tbl_size = rmem->page_size; 3433 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 3434 &rmem->pg_tbl_map, 3435 GFP_KERNEL); 3436 if (!rmem->pg_tbl) 3437 return -ENOMEM; 3438 } 3439 3440 for (i = 0; i < rmem->nr_pages; i++) { 3441 u64 extra_bits = valid_bit; 3442 3443 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 3444 rmem->page_size, 3445 &rmem->dma_arr[i], 3446 GFP_KERNEL); 3447 if (!rmem->pg_arr[i]) 3448 return -ENOMEM; 3449 3450 if (rmem->ctx_mem) 3451 bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i], 3452 rmem->page_size); 3453 if (rmem->nr_pages > 1 || rmem->depth > 0) { 3454 if (i == rmem->nr_pages - 2 && 3455 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3456 extra_bits |= PTU_PTE_NEXT_TO_LAST; 3457 else if (i == rmem->nr_pages - 1 && 3458 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3459 extra_bits |= PTU_PTE_LAST; 3460 rmem->pg_tbl[i] = 3461 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 3462 } 3463 } 3464 3465 if (rmem->vmem_size) { 3466 *rmem->vmem = vzalloc(rmem->vmem_size); 3467 if (!(*rmem->vmem)) 3468 return -ENOMEM; 3469 } 3470 return 0; 3471 } 3472 3473 static void bnxt_free_tpa_info(struct bnxt *bp) 3474 { 3475 int i, j; 3476 3477 for (i = 0; i < bp->rx_nr_rings; i++) { 3478 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3479 3480 kfree(rxr->rx_tpa_idx_map); 3481 rxr->rx_tpa_idx_map = NULL; 3482 if (rxr->rx_tpa) { 3483 for (j = 0; j < bp->max_tpa; j++) { 3484 kfree(rxr->rx_tpa[j].agg_arr); 3485 rxr->rx_tpa[j].agg_arr = NULL; 3486 } 3487 } 3488 kfree(rxr->rx_tpa); 3489 rxr->rx_tpa = NULL; 3490 } 3491 } 3492 3493 static int bnxt_alloc_tpa_info(struct bnxt *bp) 3494 { 3495 int i, j; 3496 3497 bp->max_tpa = MAX_TPA; 3498 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 3499 if (!bp->max_tpa_v2) 3500 return 0; 3501 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3502 } 3503 3504 for (i = 0; i < bp->rx_nr_rings; i++) { 3505 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3506 struct rx_agg_cmp *agg; 3507 3508 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3509 GFP_KERNEL); 3510 if (!rxr->rx_tpa) 3511 return -ENOMEM; 3512 3513 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 3514 continue; 3515 for (j = 0; j < bp->max_tpa; j++) { 3516 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); 3517 if (!agg) 3518 return -ENOMEM; 3519 rxr->rx_tpa[j].agg_arr = agg; 3520 } 3521 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3522 GFP_KERNEL); 3523 if (!rxr->rx_tpa_idx_map) 3524 return -ENOMEM; 3525 } 3526 return 0; 3527 } 3528 3529 static void bnxt_free_rx_rings(struct bnxt *bp) 3530 { 3531 int i; 3532 3533 if (!bp->rx_ring) 3534 return; 3535 3536 bnxt_free_tpa_info(bp); 3537 for (i = 0; i < bp->rx_nr_rings; i++) { 3538 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3539 struct bnxt_ring_struct *ring; 3540 3541 if (rxr->xdp_prog) 3542 bpf_prog_put(rxr->xdp_prog); 3543 3544 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3545 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3546 3547 page_pool_destroy(rxr->page_pool); 3548 rxr->page_pool = NULL; 3549 3550 kfree(rxr->rx_agg_bmap); 3551 rxr->rx_agg_bmap = NULL; 3552 3553 ring = &rxr->rx_ring_struct; 3554 bnxt_free_ring(bp, &ring->ring_mem); 3555 3556 ring = &rxr->rx_agg_ring_struct; 3557 bnxt_free_ring(bp, &ring->ring_mem); 3558 } 3559 } 3560 3561 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3562 struct bnxt_rx_ring_info *rxr) 3563 { 3564 struct page_pool_params pp = { 0 }; 3565 3566 pp.pool_size = bp->rx_agg_ring_size; 3567 if (BNXT_RX_PAGE_MODE(bp)) 3568 pp.pool_size += bp->rx_ring_size; 3569 pp.nid = dev_to_node(&bp->pdev->dev); 3570 pp.napi = &rxr->bnapi->napi; 3571 pp.netdev = bp->dev; 3572 pp.dev = &bp->pdev->dev; 3573 pp.dma_dir = bp->rx_dir; 3574 pp.max_len = PAGE_SIZE; 3575 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 3576 3577 rxr->page_pool = page_pool_create(&pp); 3578 if (IS_ERR(rxr->page_pool)) { 3579 int err = PTR_ERR(rxr->page_pool); 3580 3581 rxr->page_pool = NULL; 3582 return err; 3583 } 3584 return 0; 3585 } 3586 3587 static int bnxt_alloc_rx_rings(struct bnxt *bp) 3588 { 3589 int i, rc = 0, agg_rings = 0; 3590 3591 if (!bp->rx_ring) 3592 return -ENOMEM; 3593 3594 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3595 agg_rings = 1; 3596 3597 for (i = 0; i < bp->rx_nr_rings; i++) { 3598 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3599 struct bnxt_ring_struct *ring; 3600 3601 ring = &rxr->rx_ring_struct; 3602 3603 rc = bnxt_alloc_rx_page_pool(bp, rxr); 3604 if (rc) 3605 return rc; 3606 3607 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3608 if (rc < 0) 3609 return rc; 3610 3611 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3612 MEM_TYPE_PAGE_POOL, 3613 rxr->page_pool); 3614 if (rc) { 3615 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3616 return rc; 3617 } 3618 3619 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3620 if (rc) 3621 return rc; 3622 3623 ring->grp_idx = i; 3624 if (agg_rings) { 3625 u16 mem_size; 3626 3627 ring = &rxr->rx_agg_ring_struct; 3628 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3629 if (rc) 3630 return rc; 3631 3632 ring->grp_idx = i; 3633 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3634 mem_size = rxr->rx_agg_bmap_size / 8; 3635 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3636 if (!rxr->rx_agg_bmap) 3637 return -ENOMEM; 3638 } 3639 } 3640 if (bp->flags & BNXT_FLAG_TPA) 3641 rc = bnxt_alloc_tpa_info(bp); 3642 return rc; 3643 } 3644 3645 static void bnxt_free_tx_rings(struct bnxt *bp) 3646 { 3647 int i; 3648 struct pci_dev *pdev = bp->pdev; 3649 3650 if (!bp->tx_ring) 3651 return; 3652 3653 for (i = 0; i < bp->tx_nr_rings; i++) { 3654 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3655 struct bnxt_ring_struct *ring; 3656 3657 if (txr->tx_push) { 3658 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3659 txr->tx_push, txr->tx_push_mapping); 3660 txr->tx_push = NULL; 3661 } 3662 3663 ring = &txr->tx_ring_struct; 3664 3665 bnxt_free_ring(bp, &ring->ring_mem); 3666 } 3667 } 3668 3669 #define BNXT_TC_TO_RING_BASE(bp, tc) \ 3670 ((tc) * (bp)->tx_nr_rings_per_tc) 3671 3672 #define BNXT_RING_TO_TC_OFF(bp, tx) \ 3673 ((tx) % (bp)->tx_nr_rings_per_tc) 3674 3675 #define BNXT_RING_TO_TC(bp, tx) \ 3676 ((tx) / (bp)->tx_nr_rings_per_tc) 3677 3678 static int bnxt_alloc_tx_rings(struct bnxt *bp) 3679 { 3680 int i, j, rc; 3681 struct pci_dev *pdev = bp->pdev; 3682 3683 bp->tx_push_size = 0; 3684 if (bp->tx_push_thresh) { 3685 int push_size; 3686 3687 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3688 bp->tx_push_thresh); 3689 3690 if (push_size > 256) { 3691 push_size = 0; 3692 bp->tx_push_thresh = 0; 3693 } 3694 3695 bp->tx_push_size = push_size; 3696 } 3697 3698 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3699 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3700 struct bnxt_ring_struct *ring; 3701 u8 qidx; 3702 3703 ring = &txr->tx_ring_struct; 3704 3705 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3706 if (rc) 3707 return rc; 3708 3709 ring->grp_idx = txr->bnapi->index; 3710 if (bp->tx_push_size) { 3711 dma_addr_t mapping; 3712 3713 /* One pre-allocated DMA buffer to backup 3714 * TX push operation 3715 */ 3716 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3717 bp->tx_push_size, 3718 &txr->tx_push_mapping, 3719 GFP_KERNEL); 3720 3721 if (!txr->tx_push) 3722 return -ENOMEM; 3723 3724 mapping = txr->tx_push_mapping + 3725 sizeof(struct tx_push_bd); 3726 txr->data_mapping = cpu_to_le64(mapping); 3727 } 3728 qidx = bp->tc_to_qidx[j]; 3729 ring->queue_id = bp->q_info[qidx].queue_id; 3730 spin_lock_init(&txr->xdp_tx_lock); 3731 if (i < bp->tx_nr_rings_xdp) 3732 continue; 3733 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1)) 3734 j++; 3735 } 3736 return 0; 3737 } 3738 3739 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 3740 { 3741 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3742 3743 kfree(cpr->cp_desc_ring); 3744 cpr->cp_desc_ring = NULL; 3745 ring->ring_mem.pg_arr = NULL; 3746 kfree(cpr->cp_desc_mapping); 3747 cpr->cp_desc_mapping = NULL; 3748 ring->ring_mem.dma_arr = NULL; 3749 } 3750 3751 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 3752 { 3753 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 3754 if (!cpr->cp_desc_ring) 3755 return -ENOMEM; 3756 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 3757 GFP_KERNEL); 3758 if (!cpr->cp_desc_mapping) 3759 return -ENOMEM; 3760 return 0; 3761 } 3762 3763 static void bnxt_free_all_cp_arrays(struct bnxt *bp) 3764 { 3765 int i; 3766 3767 if (!bp->bnapi) 3768 return; 3769 for (i = 0; i < bp->cp_nr_rings; i++) { 3770 struct bnxt_napi *bnapi = bp->bnapi[i]; 3771 3772 if (!bnapi) 3773 continue; 3774 bnxt_free_cp_arrays(&bnapi->cp_ring); 3775 } 3776 } 3777 3778 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 3779 { 3780 int i, n = bp->cp_nr_pages; 3781 3782 for (i = 0; i < bp->cp_nr_rings; i++) { 3783 struct bnxt_napi *bnapi = bp->bnapi[i]; 3784 int rc; 3785 3786 if (!bnapi) 3787 continue; 3788 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 3789 if (rc) 3790 return rc; 3791 } 3792 return 0; 3793 } 3794 3795 static void bnxt_free_cp_rings(struct bnxt *bp) 3796 { 3797 int i; 3798 3799 if (!bp->bnapi) 3800 return; 3801 3802 for (i = 0; i < bp->cp_nr_rings; i++) { 3803 struct bnxt_napi *bnapi = bp->bnapi[i]; 3804 struct bnxt_cp_ring_info *cpr; 3805 struct bnxt_ring_struct *ring; 3806 int j; 3807 3808 if (!bnapi) 3809 continue; 3810 3811 cpr = &bnapi->cp_ring; 3812 ring = &cpr->cp_ring_struct; 3813 3814 bnxt_free_ring(bp, &ring->ring_mem); 3815 3816 if (!cpr->cp_ring_arr) 3817 continue; 3818 3819 for (j = 0; j < cpr->cp_ring_count; j++) { 3820 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 3821 3822 ring = &cpr2->cp_ring_struct; 3823 bnxt_free_ring(bp, &ring->ring_mem); 3824 bnxt_free_cp_arrays(cpr2); 3825 } 3826 kfree(cpr->cp_ring_arr); 3827 cpr->cp_ring_arr = NULL; 3828 cpr->cp_ring_count = 0; 3829 } 3830 } 3831 3832 static int bnxt_alloc_cp_sub_ring(struct bnxt *bp, 3833 struct bnxt_cp_ring_info *cpr) 3834 { 3835 struct bnxt_ring_mem_info *rmem; 3836 struct bnxt_ring_struct *ring; 3837 int rc; 3838 3839 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 3840 if (rc) { 3841 bnxt_free_cp_arrays(cpr); 3842 return -ENOMEM; 3843 } 3844 ring = &cpr->cp_ring_struct; 3845 rmem = &ring->ring_mem; 3846 rmem->nr_pages = bp->cp_nr_pages; 3847 rmem->page_size = HW_CMPD_RING_SIZE; 3848 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3849 rmem->dma_arr = cpr->cp_desc_mapping; 3850 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3851 rc = bnxt_alloc_ring(bp, rmem); 3852 if (rc) { 3853 bnxt_free_ring(bp, rmem); 3854 bnxt_free_cp_arrays(cpr); 3855 } 3856 return rc; 3857 } 3858 3859 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3860 { 3861 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3862 int i, j, rc, ulp_base_vec, ulp_msix; 3863 int tcs = bp->num_tc; 3864 3865 if (!tcs) 3866 tcs = 1; 3867 ulp_msix = bnxt_get_ulp_msix_num(bp); 3868 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3869 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 3870 struct bnxt_napi *bnapi = bp->bnapi[i]; 3871 struct bnxt_cp_ring_info *cpr, *cpr2; 3872 struct bnxt_ring_struct *ring; 3873 int cp_count = 0, k; 3874 int rx = 0, tx = 0; 3875 3876 if (!bnapi) 3877 continue; 3878 3879 cpr = &bnapi->cp_ring; 3880 cpr->bnapi = bnapi; 3881 ring = &cpr->cp_ring_struct; 3882 3883 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3884 if (rc) 3885 return rc; 3886 3887 if (ulp_msix && i >= ulp_base_vec) 3888 ring->map_idx = i + ulp_msix; 3889 else 3890 ring->map_idx = i; 3891 3892 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 3893 continue; 3894 3895 if (i < bp->rx_nr_rings) { 3896 cp_count++; 3897 rx = 1; 3898 } 3899 if (i < bp->tx_nr_rings_xdp) { 3900 cp_count++; 3901 tx = 1; 3902 } else if ((sh && i < bp->tx_nr_rings) || 3903 (!sh && i >= bp->rx_nr_rings)) { 3904 cp_count += tcs; 3905 tx = 1; 3906 } 3907 3908 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr), 3909 GFP_KERNEL); 3910 if (!cpr->cp_ring_arr) 3911 return -ENOMEM; 3912 cpr->cp_ring_count = cp_count; 3913 3914 for (k = 0; k < cp_count; k++) { 3915 cpr2 = &cpr->cp_ring_arr[k]; 3916 rc = bnxt_alloc_cp_sub_ring(bp, cpr2); 3917 if (rc) 3918 return rc; 3919 cpr2->bnapi = bnapi; 3920 cpr2->cp_idx = k; 3921 if (!k && rx) { 3922 bp->rx_ring[i].rx_cpr = cpr2; 3923 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX; 3924 } else { 3925 int n, tc = k - rx; 3926 3927 n = BNXT_TC_TO_RING_BASE(bp, tc) + j; 3928 bp->tx_ring[n].tx_cpr = cpr2; 3929 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX; 3930 } 3931 } 3932 if (tx) 3933 j++; 3934 } 3935 return 0; 3936 } 3937 3938 static void bnxt_init_ring_struct(struct bnxt *bp) 3939 { 3940 int i, j; 3941 3942 for (i = 0; i < bp->cp_nr_rings; i++) { 3943 struct bnxt_napi *bnapi = bp->bnapi[i]; 3944 struct bnxt_ring_mem_info *rmem; 3945 struct bnxt_cp_ring_info *cpr; 3946 struct bnxt_rx_ring_info *rxr; 3947 struct bnxt_tx_ring_info *txr; 3948 struct bnxt_ring_struct *ring; 3949 3950 if (!bnapi) 3951 continue; 3952 3953 cpr = &bnapi->cp_ring; 3954 ring = &cpr->cp_ring_struct; 3955 rmem = &ring->ring_mem; 3956 rmem->nr_pages = bp->cp_nr_pages; 3957 rmem->page_size = HW_CMPD_RING_SIZE; 3958 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3959 rmem->dma_arr = cpr->cp_desc_mapping; 3960 rmem->vmem_size = 0; 3961 3962 rxr = bnapi->rx_ring; 3963 if (!rxr) 3964 goto skip_rx; 3965 3966 ring = &rxr->rx_ring_struct; 3967 rmem = &ring->ring_mem; 3968 rmem->nr_pages = bp->rx_nr_pages; 3969 rmem->page_size = HW_RXBD_RING_SIZE; 3970 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3971 rmem->dma_arr = rxr->rx_desc_mapping; 3972 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3973 rmem->vmem = (void **)&rxr->rx_buf_ring; 3974 3975 ring = &rxr->rx_agg_ring_struct; 3976 rmem = &ring->ring_mem; 3977 rmem->nr_pages = bp->rx_agg_nr_pages; 3978 rmem->page_size = HW_RXBD_RING_SIZE; 3979 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3980 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3981 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3982 rmem->vmem = (void **)&rxr->rx_agg_ring; 3983 3984 skip_rx: 3985 bnxt_for_each_napi_tx(j, bnapi, txr) { 3986 ring = &txr->tx_ring_struct; 3987 rmem = &ring->ring_mem; 3988 rmem->nr_pages = bp->tx_nr_pages; 3989 rmem->page_size = HW_TXBD_RING_SIZE; 3990 rmem->pg_arr = (void **)txr->tx_desc_ring; 3991 rmem->dma_arr = txr->tx_desc_mapping; 3992 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3993 rmem->vmem = (void **)&txr->tx_buf_ring; 3994 } 3995 } 3996 } 3997 3998 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3999 { 4000 int i; 4001 u32 prod; 4002 struct rx_bd **rx_buf_ring; 4003 4004 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 4005 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 4006 int j; 4007 struct rx_bd *rxbd; 4008 4009 rxbd = rx_buf_ring[i]; 4010 if (!rxbd) 4011 continue; 4012 4013 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 4014 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 4015 rxbd->rx_bd_opaque = prod; 4016 } 4017 } 4018 } 4019 4020 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 4021 { 4022 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 4023 struct net_device *dev = bp->dev; 4024 u32 prod; 4025 int i; 4026 4027 prod = rxr->rx_prod; 4028 for (i = 0; i < bp->rx_ring_size; i++) { 4029 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 4030 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 4031 ring_nr, i, bp->rx_ring_size); 4032 break; 4033 } 4034 prod = NEXT_RX(prod); 4035 } 4036 rxr->rx_prod = prod; 4037 4038 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 4039 return 0; 4040 4041 prod = rxr->rx_agg_prod; 4042 for (i = 0; i < bp->rx_agg_ring_size; i++) { 4043 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 4044 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 4045 ring_nr, i, bp->rx_ring_size); 4046 break; 4047 } 4048 prod = NEXT_RX_AGG(prod); 4049 } 4050 rxr->rx_agg_prod = prod; 4051 4052 if (rxr->rx_tpa) { 4053 dma_addr_t mapping; 4054 u8 *data; 4055 4056 for (i = 0; i < bp->max_tpa; i++) { 4057 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); 4058 if (!data) 4059 return -ENOMEM; 4060 4061 rxr->rx_tpa[i].data = data; 4062 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 4063 rxr->rx_tpa[i].mapping = mapping; 4064 } 4065 } 4066 return 0; 4067 } 4068 4069 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 4070 { 4071 struct bnxt_rx_ring_info *rxr; 4072 struct bnxt_ring_struct *ring; 4073 u32 type; 4074 4075 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 4076 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 4077 4078 if (NET_IP_ALIGN == 2) 4079 type |= RX_BD_FLAGS_SOP; 4080 4081 rxr = &bp->rx_ring[ring_nr]; 4082 ring = &rxr->rx_ring_struct; 4083 bnxt_init_rxbd_pages(ring, type); 4084 4085 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX, 4086 &rxr->bnapi->napi); 4087 4088 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 4089 bpf_prog_add(bp->xdp_prog, 1); 4090 rxr->xdp_prog = bp->xdp_prog; 4091 } 4092 ring->fw_ring_id = INVALID_HW_RING_ID; 4093 4094 ring = &rxr->rx_agg_ring_struct; 4095 ring->fw_ring_id = INVALID_HW_RING_ID; 4096 4097 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 4098 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 4099 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 4100 4101 bnxt_init_rxbd_pages(ring, type); 4102 } 4103 4104 return bnxt_alloc_one_rx_ring(bp, ring_nr); 4105 } 4106 4107 static void bnxt_init_cp_rings(struct bnxt *bp) 4108 { 4109 int i, j; 4110 4111 for (i = 0; i < bp->cp_nr_rings; i++) { 4112 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 4113 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4114 4115 ring->fw_ring_id = INVALID_HW_RING_ID; 4116 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 4117 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 4118 if (!cpr->cp_ring_arr) 4119 continue; 4120 for (j = 0; j < cpr->cp_ring_count; j++) { 4121 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 4122 4123 ring = &cpr2->cp_ring_struct; 4124 ring->fw_ring_id = INVALID_HW_RING_ID; 4125 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 4126 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 4127 } 4128 } 4129 } 4130 4131 static int bnxt_init_rx_rings(struct bnxt *bp) 4132 { 4133 int i, rc = 0; 4134 4135 if (BNXT_RX_PAGE_MODE(bp)) { 4136 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 4137 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 4138 } else { 4139 bp->rx_offset = BNXT_RX_OFFSET; 4140 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 4141 } 4142 4143 for (i = 0; i < bp->rx_nr_rings; i++) { 4144 rc = bnxt_init_one_rx_ring(bp, i); 4145 if (rc) 4146 break; 4147 } 4148 4149 return rc; 4150 } 4151 4152 static int bnxt_init_tx_rings(struct bnxt *bp) 4153 { 4154 u16 i; 4155 4156 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 4157 BNXT_MIN_TX_DESC_CNT); 4158 4159 for (i = 0; i < bp->tx_nr_rings; i++) { 4160 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4161 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4162 4163 ring->fw_ring_id = INVALID_HW_RING_ID; 4164 4165 if (i >= bp->tx_nr_rings_xdp) 4166 netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp, 4167 NETDEV_QUEUE_TYPE_TX, 4168 &txr->bnapi->napi); 4169 } 4170 4171 return 0; 4172 } 4173 4174 static void bnxt_free_ring_grps(struct bnxt *bp) 4175 { 4176 kfree(bp->grp_info); 4177 bp->grp_info = NULL; 4178 } 4179 4180 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 4181 { 4182 int i; 4183 4184 if (irq_re_init) { 4185 bp->grp_info = kcalloc(bp->cp_nr_rings, 4186 sizeof(struct bnxt_ring_grp_info), 4187 GFP_KERNEL); 4188 if (!bp->grp_info) 4189 return -ENOMEM; 4190 } 4191 for (i = 0; i < bp->cp_nr_rings; i++) { 4192 if (irq_re_init) 4193 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 4194 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4195 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 4196 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 4197 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4198 } 4199 return 0; 4200 } 4201 4202 static void bnxt_free_vnics(struct bnxt *bp) 4203 { 4204 kfree(bp->vnic_info); 4205 bp->vnic_info = NULL; 4206 bp->nr_vnics = 0; 4207 } 4208 4209 static int bnxt_alloc_vnics(struct bnxt *bp) 4210 { 4211 int num_vnics = 1; 4212 4213 #ifdef CONFIG_RFS_ACCEL 4214 if (bp->flags & BNXT_FLAG_RFS) { 4215 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 4216 num_vnics++; 4217 else if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 4218 num_vnics += bp->rx_nr_rings; 4219 } 4220 #endif 4221 4222 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4223 num_vnics++; 4224 4225 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 4226 GFP_KERNEL); 4227 if (!bp->vnic_info) 4228 return -ENOMEM; 4229 4230 bp->nr_vnics = num_vnics; 4231 return 0; 4232 } 4233 4234 static void bnxt_init_vnics(struct bnxt *bp) 4235 { 4236 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 4237 int i; 4238 4239 for (i = 0; i < bp->nr_vnics; i++) { 4240 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4241 int j; 4242 4243 vnic->fw_vnic_id = INVALID_HW_RING_ID; 4244 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 4245 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 4246 4247 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 4248 4249 if (bp->vnic_info[i].rss_hash_key) { 4250 if (i == BNXT_VNIC_DEFAULT) { 4251 u8 *key = (void *)vnic->rss_hash_key; 4252 int k; 4253 4254 if (!bp->rss_hash_key_valid && 4255 !bp->rss_hash_key_updated) { 4256 get_random_bytes(bp->rss_hash_key, 4257 HW_HASH_KEY_SIZE); 4258 bp->rss_hash_key_updated = true; 4259 } 4260 4261 memcpy(vnic->rss_hash_key, bp->rss_hash_key, 4262 HW_HASH_KEY_SIZE); 4263 4264 if (!bp->rss_hash_key_updated) 4265 continue; 4266 4267 bp->rss_hash_key_updated = false; 4268 bp->rss_hash_key_valid = true; 4269 4270 bp->toeplitz_prefix = 0; 4271 for (k = 0; k < 8; k++) { 4272 bp->toeplitz_prefix <<= 8; 4273 bp->toeplitz_prefix |= key[k]; 4274 } 4275 } else { 4276 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 4277 HW_HASH_KEY_SIZE); 4278 } 4279 } 4280 } 4281 } 4282 4283 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 4284 { 4285 int pages; 4286 4287 pages = ring_size / desc_per_pg; 4288 4289 if (!pages) 4290 return 1; 4291 4292 pages++; 4293 4294 while (pages & (pages - 1)) 4295 pages++; 4296 4297 return pages; 4298 } 4299 4300 void bnxt_set_tpa_flags(struct bnxt *bp) 4301 { 4302 bp->flags &= ~BNXT_FLAG_TPA; 4303 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 4304 return; 4305 if (bp->dev->features & NETIF_F_LRO) 4306 bp->flags |= BNXT_FLAG_LRO; 4307 else if (bp->dev->features & NETIF_F_GRO_HW) 4308 bp->flags |= BNXT_FLAG_GRO; 4309 } 4310 4311 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 4312 * be set on entry. 4313 */ 4314 void bnxt_set_ring_params(struct bnxt *bp) 4315 { 4316 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 4317 u32 agg_factor = 0, agg_ring_size = 0; 4318 4319 /* 8 for CRC and VLAN */ 4320 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 4321 4322 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) + 4323 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4324 4325 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 4326 ring_size = bp->rx_ring_size; 4327 bp->rx_agg_ring_size = 0; 4328 bp->rx_agg_nr_pages = 0; 4329 4330 if (bp->flags & BNXT_FLAG_TPA) 4331 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 4332 4333 bp->flags &= ~BNXT_FLAG_JUMBO; 4334 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 4335 u32 jumbo_factor; 4336 4337 bp->flags |= BNXT_FLAG_JUMBO; 4338 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 4339 if (jumbo_factor > agg_factor) 4340 agg_factor = jumbo_factor; 4341 } 4342 if (agg_factor) { 4343 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 4344 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 4345 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 4346 bp->rx_ring_size, ring_size); 4347 bp->rx_ring_size = ring_size; 4348 } 4349 agg_ring_size = ring_size * agg_factor; 4350 4351 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 4352 RX_DESC_CNT); 4353 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 4354 u32 tmp = agg_ring_size; 4355 4356 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 4357 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 4358 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 4359 tmp, agg_ring_size); 4360 } 4361 bp->rx_agg_ring_size = agg_ring_size; 4362 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 4363 4364 if (BNXT_RX_PAGE_MODE(bp)) { 4365 rx_space = PAGE_SIZE; 4366 rx_size = PAGE_SIZE - 4367 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) - 4368 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4369 } else { 4370 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 4371 rx_space = rx_size + NET_SKB_PAD + 4372 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4373 } 4374 } 4375 4376 bp->rx_buf_use_size = rx_size; 4377 bp->rx_buf_size = rx_space; 4378 4379 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 4380 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 4381 4382 ring_size = bp->tx_ring_size; 4383 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 4384 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 4385 4386 max_rx_cmpl = bp->rx_ring_size; 4387 /* MAX TPA needs to be added because TPA_START completions are 4388 * immediately recycled, so the TPA completions are not bound by 4389 * the RX ring size. 4390 */ 4391 if (bp->flags & BNXT_FLAG_TPA) 4392 max_rx_cmpl += bp->max_tpa; 4393 /* RX and TPA completions are 32-byte, all others are 16-byte */ 4394 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 4395 bp->cp_ring_size = ring_size; 4396 4397 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 4398 if (bp->cp_nr_pages > MAX_CP_PAGES) { 4399 bp->cp_nr_pages = MAX_CP_PAGES; 4400 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 4401 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 4402 ring_size, bp->cp_ring_size); 4403 } 4404 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 4405 bp->cp_ring_mask = bp->cp_bit - 1; 4406 } 4407 4408 /* Changing allocation mode of RX rings. 4409 * TODO: Update when extending xdp_rxq_info to support allocation modes. 4410 */ 4411 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 4412 { 4413 struct net_device *dev = bp->dev; 4414 4415 if (page_mode) { 4416 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 4417 bp->flags |= BNXT_FLAG_RX_PAGE_MODE; 4418 4419 if (bp->xdp_prog->aux->xdp_has_frags) 4420 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU); 4421 else 4422 dev->max_mtu = 4423 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 4424 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { 4425 bp->flags |= BNXT_FLAG_JUMBO; 4426 bp->rx_skb_func = bnxt_rx_multi_page_skb; 4427 } else { 4428 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4429 bp->rx_skb_func = bnxt_rx_page_skb; 4430 } 4431 bp->rx_dir = DMA_BIDIRECTIONAL; 4432 /* Disable LRO or GRO_HW */ 4433 netdev_update_features(dev); 4434 } else { 4435 dev->max_mtu = bp->max_mtu; 4436 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 4437 bp->rx_dir = DMA_FROM_DEVICE; 4438 bp->rx_skb_func = bnxt_rx_skb; 4439 } 4440 return 0; 4441 } 4442 4443 static void bnxt_free_vnic_attributes(struct bnxt *bp) 4444 { 4445 int i; 4446 struct bnxt_vnic_info *vnic; 4447 struct pci_dev *pdev = bp->pdev; 4448 4449 if (!bp->vnic_info) 4450 return; 4451 4452 for (i = 0; i < bp->nr_vnics; i++) { 4453 vnic = &bp->vnic_info[i]; 4454 4455 kfree(vnic->fw_grp_ids); 4456 vnic->fw_grp_ids = NULL; 4457 4458 kfree(vnic->uc_list); 4459 vnic->uc_list = NULL; 4460 4461 if (vnic->mc_list) { 4462 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 4463 vnic->mc_list, vnic->mc_list_mapping); 4464 vnic->mc_list = NULL; 4465 } 4466 4467 if (vnic->rss_table) { 4468 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 4469 vnic->rss_table, 4470 vnic->rss_table_dma_addr); 4471 vnic->rss_table = NULL; 4472 } 4473 4474 vnic->rss_hash_key = NULL; 4475 vnic->flags = 0; 4476 } 4477 } 4478 4479 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 4480 { 4481 int i, rc = 0, size; 4482 struct bnxt_vnic_info *vnic; 4483 struct pci_dev *pdev = bp->pdev; 4484 int max_rings; 4485 4486 for (i = 0; i < bp->nr_vnics; i++) { 4487 vnic = &bp->vnic_info[i]; 4488 4489 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 4490 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 4491 4492 if (mem_size > 0) { 4493 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 4494 if (!vnic->uc_list) { 4495 rc = -ENOMEM; 4496 goto out; 4497 } 4498 } 4499 } 4500 4501 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 4502 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 4503 vnic->mc_list = 4504 dma_alloc_coherent(&pdev->dev, 4505 vnic->mc_list_size, 4506 &vnic->mc_list_mapping, 4507 GFP_KERNEL); 4508 if (!vnic->mc_list) { 4509 rc = -ENOMEM; 4510 goto out; 4511 } 4512 } 4513 4514 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 4515 goto vnic_skip_grps; 4516 4517 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4518 max_rings = bp->rx_nr_rings; 4519 else 4520 max_rings = 1; 4521 4522 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 4523 if (!vnic->fw_grp_ids) { 4524 rc = -ENOMEM; 4525 goto out; 4526 } 4527 vnic_skip_grps: 4528 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && 4529 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 4530 continue; 4531 4532 /* Allocate rss table and hash key */ 4533 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 4534 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 4535 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 4536 4537 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 4538 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 4539 vnic->rss_table_size, 4540 &vnic->rss_table_dma_addr, 4541 GFP_KERNEL); 4542 if (!vnic->rss_table) { 4543 rc = -ENOMEM; 4544 goto out; 4545 } 4546 4547 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 4548 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 4549 } 4550 return 0; 4551 4552 out: 4553 return rc; 4554 } 4555 4556 static void bnxt_free_hwrm_resources(struct bnxt *bp) 4557 { 4558 struct bnxt_hwrm_wait_token *token; 4559 4560 dma_pool_destroy(bp->hwrm_dma_pool); 4561 bp->hwrm_dma_pool = NULL; 4562 4563 rcu_read_lock(); 4564 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 4565 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 4566 rcu_read_unlock(); 4567 } 4568 4569 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 4570 { 4571 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 4572 BNXT_HWRM_DMA_SIZE, 4573 BNXT_HWRM_DMA_ALIGN, 0); 4574 if (!bp->hwrm_dma_pool) 4575 return -ENOMEM; 4576 4577 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 4578 4579 return 0; 4580 } 4581 4582 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 4583 { 4584 kfree(stats->hw_masks); 4585 stats->hw_masks = NULL; 4586 kfree(stats->sw_stats); 4587 stats->sw_stats = NULL; 4588 if (stats->hw_stats) { 4589 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 4590 stats->hw_stats_map); 4591 stats->hw_stats = NULL; 4592 } 4593 } 4594 4595 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 4596 bool alloc_masks) 4597 { 4598 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 4599 &stats->hw_stats_map, GFP_KERNEL); 4600 if (!stats->hw_stats) 4601 return -ENOMEM; 4602 4603 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 4604 if (!stats->sw_stats) 4605 goto stats_mem_err; 4606 4607 if (alloc_masks) { 4608 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 4609 if (!stats->hw_masks) 4610 goto stats_mem_err; 4611 } 4612 return 0; 4613 4614 stats_mem_err: 4615 bnxt_free_stats_mem(bp, stats); 4616 return -ENOMEM; 4617 } 4618 4619 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 4620 { 4621 int i; 4622 4623 for (i = 0; i < count; i++) 4624 mask_arr[i] = mask; 4625 } 4626 4627 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 4628 { 4629 int i; 4630 4631 for (i = 0; i < count; i++) 4632 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 4633 } 4634 4635 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 4636 struct bnxt_stats_mem *stats) 4637 { 4638 struct hwrm_func_qstats_ext_output *resp; 4639 struct hwrm_func_qstats_ext_input *req; 4640 __le64 *hw_masks; 4641 int rc; 4642 4643 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 4644 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 4645 return -EOPNOTSUPP; 4646 4647 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 4648 if (rc) 4649 return rc; 4650 4651 req->fid = cpu_to_le16(0xffff); 4652 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4653 4654 resp = hwrm_req_hold(bp, req); 4655 rc = hwrm_req_send(bp, req); 4656 if (!rc) { 4657 hw_masks = &resp->rx_ucast_pkts; 4658 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 4659 } 4660 hwrm_req_drop(bp, req); 4661 return rc; 4662 } 4663 4664 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 4665 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 4666 4667 static void bnxt_init_stats(struct bnxt *bp) 4668 { 4669 struct bnxt_napi *bnapi = bp->bnapi[0]; 4670 struct bnxt_cp_ring_info *cpr; 4671 struct bnxt_stats_mem *stats; 4672 __le64 *rx_stats, *tx_stats; 4673 int rc, rx_count, tx_count; 4674 u64 *rx_masks, *tx_masks; 4675 u64 mask; 4676 u8 flags; 4677 4678 cpr = &bnapi->cp_ring; 4679 stats = &cpr->stats; 4680 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 4681 if (rc) { 4682 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 4683 mask = (1ULL << 48) - 1; 4684 else 4685 mask = -1ULL; 4686 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 4687 } 4688 if (bp->flags & BNXT_FLAG_PORT_STATS) { 4689 stats = &bp->port_stats; 4690 rx_stats = stats->hw_stats; 4691 rx_masks = stats->hw_masks; 4692 rx_count = sizeof(struct rx_port_stats) / 8; 4693 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4694 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4695 tx_count = sizeof(struct tx_port_stats) / 8; 4696 4697 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 4698 rc = bnxt_hwrm_port_qstats(bp, flags); 4699 if (rc) { 4700 mask = (1ULL << 40) - 1; 4701 4702 bnxt_fill_masks(rx_masks, mask, rx_count); 4703 bnxt_fill_masks(tx_masks, mask, tx_count); 4704 } else { 4705 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4706 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 4707 bnxt_hwrm_port_qstats(bp, 0); 4708 } 4709 } 4710 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 4711 stats = &bp->rx_port_stats_ext; 4712 rx_stats = stats->hw_stats; 4713 rx_masks = stats->hw_masks; 4714 rx_count = sizeof(struct rx_port_stats_ext) / 8; 4715 stats = &bp->tx_port_stats_ext; 4716 tx_stats = stats->hw_stats; 4717 tx_masks = stats->hw_masks; 4718 tx_count = sizeof(struct tx_port_stats_ext) / 8; 4719 4720 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4721 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 4722 if (rc) { 4723 mask = (1ULL << 40) - 1; 4724 4725 bnxt_fill_masks(rx_masks, mask, rx_count); 4726 if (tx_stats) 4727 bnxt_fill_masks(tx_masks, mask, tx_count); 4728 } else { 4729 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4730 if (tx_stats) 4731 bnxt_copy_hw_masks(tx_masks, tx_stats, 4732 tx_count); 4733 bnxt_hwrm_port_qstats_ext(bp, 0); 4734 } 4735 } 4736 } 4737 4738 static void bnxt_free_port_stats(struct bnxt *bp) 4739 { 4740 bp->flags &= ~BNXT_FLAG_PORT_STATS; 4741 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 4742 4743 bnxt_free_stats_mem(bp, &bp->port_stats); 4744 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 4745 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 4746 } 4747 4748 static void bnxt_free_ring_stats(struct bnxt *bp) 4749 { 4750 int i; 4751 4752 if (!bp->bnapi) 4753 return; 4754 4755 for (i = 0; i < bp->cp_nr_rings; i++) { 4756 struct bnxt_napi *bnapi = bp->bnapi[i]; 4757 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4758 4759 bnxt_free_stats_mem(bp, &cpr->stats); 4760 } 4761 } 4762 4763 static int bnxt_alloc_stats(struct bnxt *bp) 4764 { 4765 u32 size, i; 4766 int rc; 4767 4768 size = bp->hw_ring_stats_size; 4769 4770 for (i = 0; i < bp->cp_nr_rings; i++) { 4771 struct bnxt_napi *bnapi = bp->bnapi[i]; 4772 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4773 4774 cpr->stats.len = size; 4775 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4776 if (rc) 4777 return rc; 4778 4779 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4780 } 4781 4782 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4783 return 0; 4784 4785 if (bp->port_stats.hw_stats) 4786 goto alloc_ext_stats; 4787 4788 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4789 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4790 if (rc) 4791 return rc; 4792 4793 bp->flags |= BNXT_FLAG_PORT_STATS; 4794 4795 alloc_ext_stats: 4796 /* Display extended statistics only if FW supports it */ 4797 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4798 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4799 return 0; 4800 4801 if (bp->rx_port_stats_ext.hw_stats) 4802 goto alloc_tx_ext_stats; 4803 4804 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4805 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4806 /* Extended stats are optional */ 4807 if (rc) 4808 return 0; 4809 4810 alloc_tx_ext_stats: 4811 if (bp->tx_port_stats_ext.hw_stats) 4812 return 0; 4813 4814 if (bp->hwrm_spec_code >= 0x10902 || 4815 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4816 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4817 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4818 /* Extended stats are optional */ 4819 if (rc) 4820 return 0; 4821 } 4822 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4823 return 0; 4824 } 4825 4826 static void bnxt_clear_ring_indices(struct bnxt *bp) 4827 { 4828 int i, j; 4829 4830 if (!bp->bnapi) 4831 return; 4832 4833 for (i = 0; i < bp->cp_nr_rings; i++) { 4834 struct bnxt_napi *bnapi = bp->bnapi[i]; 4835 struct bnxt_cp_ring_info *cpr; 4836 struct bnxt_rx_ring_info *rxr; 4837 struct bnxt_tx_ring_info *txr; 4838 4839 if (!bnapi) 4840 continue; 4841 4842 cpr = &bnapi->cp_ring; 4843 cpr->cp_raw_cons = 0; 4844 4845 bnxt_for_each_napi_tx(j, bnapi, txr) { 4846 txr->tx_prod = 0; 4847 txr->tx_cons = 0; 4848 txr->tx_hw_cons = 0; 4849 } 4850 4851 rxr = bnapi->rx_ring; 4852 if (rxr) { 4853 rxr->rx_prod = 0; 4854 rxr->rx_agg_prod = 0; 4855 rxr->rx_sw_agg_prod = 0; 4856 rxr->rx_next_cons = 0; 4857 } 4858 bnapi->events = 0; 4859 } 4860 } 4861 4862 void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 4863 { 4864 u8 type = fltr->type, flags = fltr->flags; 4865 4866 INIT_LIST_HEAD(&fltr->list); 4867 if ((type == BNXT_FLTR_TYPE_L2 && flags & BNXT_ACT_RING_DST) || 4868 (type == BNXT_FLTR_TYPE_NTUPLE && flags & BNXT_ACT_NO_AGING)) 4869 list_add_tail(&fltr->list, &bp->usr_fltr_list); 4870 } 4871 4872 void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 4873 { 4874 if (!list_empty(&fltr->list)) 4875 list_del_init(&fltr->list); 4876 } 4877 4878 void bnxt_clear_usr_fltrs(struct bnxt *bp, bool all) 4879 { 4880 struct bnxt_filter_base *usr_fltr, *tmp; 4881 4882 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) { 4883 if (!all && usr_fltr->type == BNXT_FLTR_TYPE_L2) 4884 continue; 4885 bnxt_del_one_usr_fltr(bp, usr_fltr); 4886 } 4887 } 4888 4889 static void bnxt_del_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 4890 { 4891 hlist_del(&fltr->hash); 4892 bnxt_del_one_usr_fltr(bp, fltr); 4893 if (fltr->flags) { 4894 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 4895 bp->ntp_fltr_count--; 4896 } 4897 kfree(fltr); 4898 } 4899 4900 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all) 4901 { 4902 int i; 4903 4904 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4905 * safe to delete the hash table. 4906 */ 4907 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4908 struct hlist_head *head; 4909 struct hlist_node *tmp; 4910 struct bnxt_ntuple_filter *fltr; 4911 4912 head = &bp->ntp_fltr_hash_tbl[i]; 4913 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 4914 bnxt_del_l2_filter(bp, fltr->l2_fltr); 4915 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) || 4916 !list_empty(&fltr->base.list))) 4917 continue; 4918 bnxt_del_fltr(bp, &fltr->base); 4919 } 4920 } 4921 if (!all) 4922 return; 4923 4924 bitmap_free(bp->ntp_fltr_bmap); 4925 bp->ntp_fltr_bmap = NULL; 4926 bp->ntp_fltr_count = 0; 4927 } 4928 4929 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4930 { 4931 int i, rc = 0; 4932 4933 if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap) 4934 return 0; 4935 4936 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4937 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4938 4939 bp->ntp_fltr_count = 0; 4940 bp->ntp_fltr_bmap = bitmap_zalloc(bp->max_fltr, GFP_KERNEL); 4941 4942 if (!bp->ntp_fltr_bmap) 4943 rc = -ENOMEM; 4944 4945 return rc; 4946 } 4947 4948 static void bnxt_free_l2_filters(struct bnxt *bp, bool all) 4949 { 4950 int i; 4951 4952 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) { 4953 struct hlist_head *head; 4954 struct hlist_node *tmp; 4955 struct bnxt_l2_filter *fltr; 4956 4957 head = &bp->l2_fltr_hash_tbl[i]; 4958 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 4959 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) || 4960 !list_empty(&fltr->base.list))) 4961 continue; 4962 bnxt_del_fltr(bp, &fltr->base); 4963 } 4964 } 4965 } 4966 4967 static void bnxt_init_l2_fltr_tbl(struct bnxt *bp) 4968 { 4969 int i; 4970 4971 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) 4972 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]); 4973 get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed)); 4974 } 4975 4976 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4977 { 4978 bnxt_free_vnic_attributes(bp); 4979 bnxt_free_tx_rings(bp); 4980 bnxt_free_rx_rings(bp); 4981 bnxt_free_cp_rings(bp); 4982 bnxt_free_all_cp_arrays(bp); 4983 bnxt_free_ntp_fltrs(bp, false); 4984 bnxt_free_l2_filters(bp, false); 4985 if (irq_re_init) { 4986 bnxt_free_ring_stats(bp); 4987 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 4988 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4989 bnxt_free_port_stats(bp); 4990 bnxt_free_ring_grps(bp); 4991 bnxt_free_vnics(bp); 4992 kfree(bp->tx_ring_map); 4993 bp->tx_ring_map = NULL; 4994 kfree(bp->tx_ring); 4995 bp->tx_ring = NULL; 4996 kfree(bp->rx_ring); 4997 bp->rx_ring = NULL; 4998 kfree(bp->bnapi); 4999 bp->bnapi = NULL; 5000 } else { 5001 bnxt_clear_ring_indices(bp); 5002 } 5003 } 5004 5005 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 5006 { 5007 int i, j, rc, size, arr_size; 5008 void *bnapi; 5009 5010 if (irq_re_init) { 5011 /* Allocate bnapi mem pointer array and mem block for 5012 * all queues 5013 */ 5014 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 5015 bp->cp_nr_rings); 5016 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 5017 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 5018 if (!bnapi) 5019 return -ENOMEM; 5020 5021 bp->bnapi = bnapi; 5022 bnapi += arr_size; 5023 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 5024 bp->bnapi[i] = bnapi; 5025 bp->bnapi[i]->index = i; 5026 bp->bnapi[i]->bp = bp; 5027 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 5028 struct bnxt_cp_ring_info *cpr = 5029 &bp->bnapi[i]->cp_ring; 5030 5031 cpr->cp_ring_struct.ring_mem.flags = 5032 BNXT_RMEM_RING_PTE_FLAG; 5033 } 5034 } 5035 5036 bp->rx_ring = kcalloc(bp->rx_nr_rings, 5037 sizeof(struct bnxt_rx_ring_info), 5038 GFP_KERNEL); 5039 if (!bp->rx_ring) 5040 return -ENOMEM; 5041 5042 for (i = 0; i < bp->rx_nr_rings; i++) { 5043 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5044 5045 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 5046 rxr->rx_ring_struct.ring_mem.flags = 5047 BNXT_RMEM_RING_PTE_FLAG; 5048 rxr->rx_agg_ring_struct.ring_mem.flags = 5049 BNXT_RMEM_RING_PTE_FLAG; 5050 } else { 5051 rxr->rx_cpr = &bp->bnapi[i]->cp_ring; 5052 } 5053 rxr->bnapi = bp->bnapi[i]; 5054 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 5055 } 5056 5057 bp->tx_ring = kcalloc(bp->tx_nr_rings, 5058 sizeof(struct bnxt_tx_ring_info), 5059 GFP_KERNEL); 5060 if (!bp->tx_ring) 5061 return -ENOMEM; 5062 5063 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 5064 GFP_KERNEL); 5065 5066 if (!bp->tx_ring_map) 5067 return -ENOMEM; 5068 5069 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5070 j = 0; 5071 else 5072 j = bp->rx_nr_rings; 5073 5074 for (i = 0; i < bp->tx_nr_rings; i++) { 5075 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5076 struct bnxt_napi *bnapi2; 5077 5078 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 5079 txr->tx_ring_struct.ring_mem.flags = 5080 BNXT_RMEM_RING_PTE_FLAG; 5081 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 5082 if (i >= bp->tx_nr_rings_xdp) { 5083 int k = j + BNXT_RING_TO_TC_OFF(bp, i); 5084 5085 bnapi2 = bp->bnapi[k]; 5086 txr->txq_index = i - bp->tx_nr_rings_xdp; 5087 txr->tx_napi_idx = 5088 BNXT_RING_TO_TC(bp, txr->txq_index); 5089 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 5090 bnapi2->tx_int = bnxt_tx_int; 5091 } else { 5092 bnapi2 = bp->bnapi[j]; 5093 bnapi2->flags |= BNXT_NAPI_FLAG_XDP; 5094 bnapi2->tx_ring[0] = txr; 5095 bnapi2->tx_int = bnxt_tx_int_xdp; 5096 j++; 5097 } 5098 txr->bnapi = bnapi2; 5099 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 5100 txr->tx_cpr = &bnapi2->cp_ring; 5101 } 5102 5103 rc = bnxt_alloc_stats(bp); 5104 if (rc) 5105 goto alloc_mem_err; 5106 bnxt_init_stats(bp); 5107 5108 rc = bnxt_alloc_ntp_fltrs(bp); 5109 if (rc) 5110 goto alloc_mem_err; 5111 5112 rc = bnxt_alloc_vnics(bp); 5113 if (rc) 5114 goto alloc_mem_err; 5115 } 5116 5117 rc = bnxt_alloc_all_cp_arrays(bp); 5118 if (rc) 5119 goto alloc_mem_err; 5120 5121 bnxt_init_ring_struct(bp); 5122 5123 rc = bnxt_alloc_rx_rings(bp); 5124 if (rc) 5125 goto alloc_mem_err; 5126 5127 rc = bnxt_alloc_tx_rings(bp); 5128 if (rc) 5129 goto alloc_mem_err; 5130 5131 rc = bnxt_alloc_cp_rings(bp); 5132 if (rc) 5133 goto alloc_mem_err; 5134 5135 bp->vnic_info[BNXT_VNIC_DEFAULT].flags |= BNXT_VNIC_RSS_FLAG | 5136 BNXT_VNIC_MCAST_FLAG | 5137 BNXT_VNIC_UCAST_FLAG; 5138 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp) && (bp->flags & BNXT_FLAG_RFS)) 5139 bp->vnic_info[BNXT_VNIC_NTUPLE].flags |= 5140 BNXT_VNIC_RSS_FLAG | BNXT_VNIC_NTUPLE_FLAG; 5141 5142 rc = bnxt_alloc_vnic_attributes(bp); 5143 if (rc) 5144 goto alloc_mem_err; 5145 return 0; 5146 5147 alloc_mem_err: 5148 bnxt_free_mem(bp, true); 5149 return rc; 5150 } 5151 5152 static void bnxt_disable_int(struct bnxt *bp) 5153 { 5154 int i; 5155 5156 if (!bp->bnapi) 5157 return; 5158 5159 for (i = 0; i < bp->cp_nr_rings; i++) { 5160 struct bnxt_napi *bnapi = bp->bnapi[i]; 5161 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5162 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5163 5164 if (ring->fw_ring_id != INVALID_HW_RING_ID) 5165 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5166 } 5167 } 5168 5169 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 5170 { 5171 struct bnxt_napi *bnapi = bp->bnapi[n]; 5172 struct bnxt_cp_ring_info *cpr; 5173 5174 cpr = &bnapi->cp_ring; 5175 return cpr->cp_ring_struct.map_idx; 5176 } 5177 5178 static void bnxt_disable_int_sync(struct bnxt *bp) 5179 { 5180 int i; 5181 5182 if (!bp->irq_tbl) 5183 return; 5184 5185 atomic_inc(&bp->intr_sem); 5186 5187 bnxt_disable_int(bp); 5188 for (i = 0; i < bp->cp_nr_rings; i++) { 5189 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5190 5191 synchronize_irq(bp->irq_tbl[map_idx].vector); 5192 } 5193 } 5194 5195 static void bnxt_enable_int(struct bnxt *bp) 5196 { 5197 int i; 5198 5199 atomic_set(&bp->intr_sem, 0); 5200 for (i = 0; i < bp->cp_nr_rings; i++) { 5201 struct bnxt_napi *bnapi = bp->bnapi[i]; 5202 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5203 5204 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 5205 } 5206 } 5207 5208 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 5209 bool async_only) 5210 { 5211 DECLARE_BITMAP(async_events_bmap, 256); 5212 u32 *events = (u32 *)async_events_bmap; 5213 struct hwrm_func_drv_rgtr_output *resp; 5214 struct hwrm_func_drv_rgtr_input *req; 5215 u32 flags; 5216 int rc, i; 5217 5218 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 5219 if (rc) 5220 return rc; 5221 5222 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 5223 FUNC_DRV_RGTR_REQ_ENABLES_VER | 5224 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 5225 5226 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 5227 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 5228 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 5229 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 5230 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 5231 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 5232 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 5233 req->flags = cpu_to_le32(flags); 5234 req->ver_maj_8b = DRV_VER_MAJ; 5235 req->ver_min_8b = DRV_VER_MIN; 5236 req->ver_upd_8b = DRV_VER_UPD; 5237 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 5238 req->ver_min = cpu_to_le16(DRV_VER_MIN); 5239 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 5240 5241 if (BNXT_PF(bp)) { 5242 u32 data[8]; 5243 int i; 5244 5245 memset(data, 0, sizeof(data)); 5246 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 5247 u16 cmd = bnxt_vf_req_snif[i]; 5248 unsigned int bit, idx; 5249 5250 idx = cmd / 32; 5251 bit = cmd % 32; 5252 data[idx] |= 1 << bit; 5253 } 5254 5255 for (i = 0; i < 8; i++) 5256 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 5257 5258 req->enables |= 5259 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 5260 } 5261 5262 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 5263 req->flags |= cpu_to_le32( 5264 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 5265 5266 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 5267 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 5268 u16 event_id = bnxt_async_events_arr[i]; 5269 5270 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 5271 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 5272 continue; 5273 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE && 5274 !bp->ptp_cfg) 5275 continue; 5276 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 5277 } 5278 if (bmap && bmap_size) { 5279 for (i = 0; i < bmap_size; i++) { 5280 if (test_bit(i, bmap)) 5281 __set_bit(i, async_events_bmap); 5282 } 5283 } 5284 for (i = 0; i < 8; i++) 5285 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 5286 5287 if (async_only) 5288 req->enables = 5289 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 5290 5291 resp = hwrm_req_hold(bp, req); 5292 rc = hwrm_req_send(bp, req); 5293 if (!rc) { 5294 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 5295 if (resp->flags & 5296 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 5297 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 5298 } 5299 hwrm_req_drop(bp, req); 5300 return rc; 5301 } 5302 5303 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 5304 { 5305 struct hwrm_func_drv_unrgtr_input *req; 5306 int rc; 5307 5308 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 5309 return 0; 5310 5311 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 5312 if (rc) 5313 return rc; 5314 return hwrm_req_send(bp, req); 5315 } 5316 5317 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa); 5318 5319 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 5320 { 5321 struct hwrm_tunnel_dst_port_free_input *req; 5322 int rc; 5323 5324 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 5325 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 5326 return 0; 5327 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 5328 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 5329 return 0; 5330 5331 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 5332 if (rc) 5333 return rc; 5334 5335 req->tunnel_type = tunnel_type; 5336 5337 switch (tunnel_type) { 5338 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 5339 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 5340 bp->vxlan_port = 0; 5341 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 5342 break; 5343 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 5344 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 5345 bp->nge_port = 0; 5346 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 5347 break; 5348 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE: 5349 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id); 5350 bp->vxlan_gpe_port = 0; 5351 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID; 5352 break; 5353 default: 5354 break; 5355 } 5356 5357 rc = hwrm_req_send(bp, req); 5358 if (rc) 5359 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 5360 rc); 5361 if (bp->flags & BNXT_FLAG_TPA) 5362 bnxt_set_tpa(bp, true); 5363 return rc; 5364 } 5365 5366 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 5367 u8 tunnel_type) 5368 { 5369 struct hwrm_tunnel_dst_port_alloc_output *resp; 5370 struct hwrm_tunnel_dst_port_alloc_input *req; 5371 int rc; 5372 5373 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 5374 if (rc) 5375 return rc; 5376 5377 req->tunnel_type = tunnel_type; 5378 req->tunnel_dst_port_val = port; 5379 5380 resp = hwrm_req_hold(bp, req); 5381 rc = hwrm_req_send(bp, req); 5382 if (rc) { 5383 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 5384 rc); 5385 goto err_out; 5386 } 5387 5388 switch (tunnel_type) { 5389 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 5390 bp->vxlan_port = port; 5391 bp->vxlan_fw_dst_port_id = 5392 le16_to_cpu(resp->tunnel_dst_port_id); 5393 break; 5394 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 5395 bp->nge_port = port; 5396 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 5397 break; 5398 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE: 5399 bp->vxlan_gpe_port = port; 5400 bp->vxlan_gpe_fw_dst_port_id = 5401 le16_to_cpu(resp->tunnel_dst_port_id); 5402 break; 5403 default: 5404 break; 5405 } 5406 if (bp->flags & BNXT_FLAG_TPA) 5407 bnxt_set_tpa(bp, true); 5408 5409 err_out: 5410 hwrm_req_drop(bp, req); 5411 return rc; 5412 } 5413 5414 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 5415 { 5416 struct hwrm_cfa_l2_set_rx_mask_input *req; 5417 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5418 int rc; 5419 5420 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 5421 if (rc) 5422 return rc; 5423 5424 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5425 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 5426 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 5427 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 5428 } 5429 req->mask = cpu_to_le32(vnic->rx_mask); 5430 return hwrm_req_send_silent(bp, req); 5431 } 5432 5433 void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr) 5434 { 5435 if (!atomic_dec_and_test(&fltr->refcnt)) 5436 return; 5437 spin_lock_bh(&bp->ntp_fltr_lock); 5438 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) { 5439 spin_unlock_bh(&bp->ntp_fltr_lock); 5440 return; 5441 } 5442 hlist_del_rcu(&fltr->base.hash); 5443 bnxt_del_one_usr_fltr(bp, &fltr->base); 5444 if (fltr->base.flags) { 5445 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap); 5446 bp->ntp_fltr_count--; 5447 } 5448 spin_unlock_bh(&bp->ntp_fltr_lock); 5449 kfree_rcu(fltr, base.rcu); 5450 } 5451 5452 static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp, 5453 struct bnxt_l2_key *key, 5454 u32 idx) 5455 { 5456 struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx]; 5457 struct bnxt_l2_filter *fltr; 5458 5459 hlist_for_each_entry_rcu(fltr, head, base.hash) { 5460 struct bnxt_l2_key *l2_key = &fltr->l2_key; 5461 5462 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 5463 l2_key->vlan == key->vlan) 5464 return fltr; 5465 } 5466 return NULL; 5467 } 5468 5469 static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp, 5470 struct bnxt_l2_key *key, 5471 u32 idx) 5472 { 5473 struct bnxt_l2_filter *fltr = NULL; 5474 5475 rcu_read_lock(); 5476 fltr = __bnxt_lookup_l2_filter(bp, key, idx); 5477 if (fltr) 5478 atomic_inc(&fltr->refcnt); 5479 rcu_read_unlock(); 5480 return fltr; 5481 } 5482 5483 #define BNXT_IPV4_4TUPLE(bp, fkeys) \ 5484 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \ 5485 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) || \ 5486 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \ 5487 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4)) 5488 5489 #define BNXT_IPV6_4TUPLE(bp, fkeys) \ 5490 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \ 5491 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) || \ 5492 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \ 5493 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6)) 5494 5495 static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys) 5496 { 5497 if (fkeys->basic.n_proto == htons(ETH_P_IP)) { 5498 if (BNXT_IPV4_4TUPLE(bp, fkeys)) 5499 return sizeof(fkeys->addrs.v4addrs) + 5500 sizeof(fkeys->ports); 5501 5502 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4) 5503 return sizeof(fkeys->addrs.v4addrs); 5504 } 5505 5506 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) { 5507 if (BNXT_IPV6_4TUPLE(bp, fkeys)) 5508 return sizeof(fkeys->addrs.v6addrs) + 5509 sizeof(fkeys->ports); 5510 5511 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6) 5512 return sizeof(fkeys->addrs.v6addrs); 5513 } 5514 5515 return 0; 5516 } 5517 5518 static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys, 5519 const unsigned char *key) 5520 { 5521 u64 prefix = bp->toeplitz_prefix, hash = 0; 5522 struct bnxt_ipv4_tuple tuple4; 5523 struct bnxt_ipv6_tuple tuple6; 5524 int i, j, len = 0; 5525 u8 *four_tuple; 5526 5527 len = bnxt_get_rss_flow_tuple_len(bp, fkeys); 5528 if (!len) 5529 return 0; 5530 5531 if (fkeys->basic.n_proto == htons(ETH_P_IP)) { 5532 tuple4.v4addrs = fkeys->addrs.v4addrs; 5533 tuple4.ports = fkeys->ports; 5534 four_tuple = (unsigned char *)&tuple4; 5535 } else { 5536 tuple6.v6addrs = fkeys->addrs.v6addrs; 5537 tuple6.ports = fkeys->ports; 5538 four_tuple = (unsigned char *)&tuple6; 5539 } 5540 5541 for (i = 0, j = 8; i < len; i++, j++) { 5542 u8 byte = four_tuple[i]; 5543 int bit; 5544 5545 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) { 5546 if (byte & 0x80) 5547 hash ^= prefix; 5548 } 5549 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0; 5550 } 5551 5552 /* The valid part of the hash is in the upper 32 bits. */ 5553 return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK; 5554 } 5555 5556 #ifdef CONFIG_RFS_ACCEL 5557 static struct bnxt_l2_filter * 5558 bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key) 5559 { 5560 struct bnxt_l2_filter *fltr; 5561 u32 idx; 5562 5563 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 5564 BNXT_L2_FLTR_HASH_MASK; 5565 fltr = bnxt_lookup_l2_filter(bp, key, idx); 5566 return fltr; 5567 } 5568 #endif 5569 5570 static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr, 5571 struct bnxt_l2_key *key, u32 idx) 5572 { 5573 struct hlist_head *head; 5574 5575 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 5576 fltr->l2_key.vlan = key->vlan; 5577 fltr->base.type = BNXT_FLTR_TYPE_L2; 5578 if (fltr->base.flags) { 5579 int bit_id; 5580 5581 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 5582 bp->max_fltr, 0); 5583 if (bit_id < 0) 5584 return -ENOMEM; 5585 fltr->base.sw_id = (u16)bit_id; 5586 bp->ntp_fltr_count++; 5587 } 5588 head = &bp->l2_fltr_hash_tbl[idx]; 5589 hlist_add_head_rcu(&fltr->base.hash, head); 5590 bnxt_insert_usr_fltr(bp, &fltr->base); 5591 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state); 5592 atomic_set(&fltr->refcnt, 1); 5593 return 0; 5594 } 5595 5596 static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp, 5597 struct bnxt_l2_key *key, 5598 gfp_t gfp) 5599 { 5600 struct bnxt_l2_filter *fltr; 5601 u32 idx; 5602 int rc; 5603 5604 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 5605 BNXT_L2_FLTR_HASH_MASK; 5606 fltr = bnxt_lookup_l2_filter(bp, key, idx); 5607 if (fltr) 5608 return fltr; 5609 5610 fltr = kzalloc(sizeof(*fltr), gfp); 5611 if (!fltr) 5612 return ERR_PTR(-ENOMEM); 5613 spin_lock_bh(&bp->ntp_fltr_lock); 5614 rc = bnxt_init_l2_filter(bp, fltr, key, idx); 5615 spin_unlock_bh(&bp->ntp_fltr_lock); 5616 if (rc) { 5617 bnxt_del_l2_filter(bp, fltr); 5618 fltr = ERR_PTR(rc); 5619 } 5620 return fltr; 5621 } 5622 5623 struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp, 5624 struct bnxt_l2_key *key, 5625 u16 flags) 5626 { 5627 struct bnxt_l2_filter *fltr; 5628 u32 idx; 5629 int rc; 5630 5631 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 5632 BNXT_L2_FLTR_HASH_MASK; 5633 spin_lock_bh(&bp->ntp_fltr_lock); 5634 fltr = __bnxt_lookup_l2_filter(bp, key, idx); 5635 if (fltr) { 5636 fltr = ERR_PTR(-EEXIST); 5637 goto l2_filter_exit; 5638 } 5639 fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC); 5640 if (!fltr) { 5641 fltr = ERR_PTR(-ENOMEM); 5642 goto l2_filter_exit; 5643 } 5644 fltr->base.flags = flags; 5645 rc = bnxt_init_l2_filter(bp, fltr, key, idx); 5646 if (rc) { 5647 spin_unlock_bh(&bp->ntp_fltr_lock); 5648 bnxt_del_l2_filter(bp, fltr); 5649 return ERR_PTR(rc); 5650 } 5651 5652 l2_filter_exit: 5653 spin_unlock_bh(&bp->ntp_fltr_lock); 5654 return fltr; 5655 } 5656 5657 static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx) 5658 { 5659 #ifdef CONFIG_BNXT_SRIOV 5660 struct bnxt_vf_info *vf = &pf->vf[vf_idx]; 5661 5662 return vf->fw_fid; 5663 #else 5664 return INVALID_HW_RING_ID; 5665 #endif 5666 } 5667 5668 int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr) 5669 { 5670 struct hwrm_cfa_l2_filter_free_input *req; 5671 u16 target_id = 0xffff; 5672 int rc; 5673 5674 if (fltr->base.flags & BNXT_ACT_FUNC_DST) { 5675 struct bnxt_pf_info *pf = &bp->pf; 5676 5677 if (fltr->base.vf_idx >= pf->active_vfs) 5678 return -EINVAL; 5679 5680 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx); 5681 if (target_id == INVALID_HW_RING_ID) 5682 return -EINVAL; 5683 } 5684 5685 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 5686 if (rc) 5687 return rc; 5688 5689 req->target_id = cpu_to_le16(target_id); 5690 req->l2_filter_id = fltr->base.filter_id; 5691 return hwrm_req_send(bp, req); 5692 } 5693 5694 int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr) 5695 { 5696 struct hwrm_cfa_l2_filter_alloc_output *resp; 5697 struct hwrm_cfa_l2_filter_alloc_input *req; 5698 u16 target_id = 0xffff; 5699 int rc; 5700 5701 if (fltr->base.flags & BNXT_ACT_FUNC_DST) { 5702 struct bnxt_pf_info *pf = &bp->pf; 5703 5704 if (fltr->base.vf_idx >= pf->active_vfs) 5705 return -EINVAL; 5706 5707 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx); 5708 } 5709 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 5710 if (rc) 5711 return rc; 5712 5713 req->target_id = cpu_to_le16(target_id); 5714 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 5715 5716 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 5717 req->flags |= 5718 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 5719 req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id); 5720 req->enables = 5721 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 5722 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 5723 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 5724 ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr); 5725 eth_broadcast_addr(req->l2_addr_mask); 5726 5727 if (fltr->l2_key.vlan) { 5728 req->enables |= 5729 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN | 5730 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK | 5731 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS); 5732 req->num_vlans = 1; 5733 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan); 5734 req->l2_ivlan_mask = cpu_to_le16(0xfff); 5735 } 5736 5737 resp = hwrm_req_hold(bp, req); 5738 rc = hwrm_req_send(bp, req); 5739 if (!rc) { 5740 fltr->base.filter_id = resp->l2_filter_id; 5741 set_bit(BNXT_FLTR_VALID, &fltr->base.state); 5742 } 5743 hwrm_req_drop(bp, req); 5744 return rc; 5745 } 5746 5747 int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 5748 struct bnxt_ntuple_filter *fltr) 5749 { 5750 struct hwrm_cfa_ntuple_filter_free_input *req; 5751 int rc; 5752 5753 set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); 5754 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 5755 if (rc) 5756 return rc; 5757 5758 req->ntuple_filter_id = fltr->base.filter_id; 5759 return hwrm_req_send(bp, req); 5760 } 5761 5762 #define BNXT_NTP_FLTR_FLAGS \ 5763 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 5764 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 5765 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 5766 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 5767 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 5768 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 5769 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 5770 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 5771 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 5772 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 5773 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 5774 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 5775 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 5776 5777 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 5778 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 5779 5780 void bnxt_fill_ipv6_mask(__be32 mask[4]) 5781 { 5782 int i; 5783 5784 for (i = 0; i < 4; i++) 5785 mask[i] = cpu_to_be32(~0); 5786 } 5787 5788 static void 5789 bnxt_cfg_rfs_ring_tbl_idx(struct bnxt *bp, 5790 struct hwrm_cfa_ntuple_filter_alloc_input *req, 5791 u16 rxq) 5792 { 5793 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) { 5794 struct bnxt_vnic_info *vnic; 5795 u32 enables; 5796 5797 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE]; 5798 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 5799 enables = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX; 5800 req->enables |= cpu_to_le32(enables); 5801 req->rfs_ring_tbl_idx = cpu_to_le16(rxq); 5802 } else { 5803 u32 flags; 5804 5805 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 5806 req->flags |= cpu_to_le32(flags); 5807 req->dst_id = cpu_to_le16(rxq); 5808 } 5809 } 5810 5811 int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 5812 struct bnxt_ntuple_filter *fltr) 5813 { 5814 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 5815 struct hwrm_cfa_ntuple_filter_alloc_input *req; 5816 struct bnxt_flow_masks *masks = &fltr->fmasks; 5817 struct flow_keys *keys = &fltr->fkeys; 5818 struct bnxt_l2_filter *l2_fltr; 5819 struct bnxt_vnic_info *vnic; 5820 int rc; 5821 5822 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 5823 if (rc) 5824 return rc; 5825 5826 l2_fltr = fltr->l2_fltr; 5827 req->l2_filter_id = l2_fltr->base.filter_id; 5828 5829 if (fltr->base.flags & BNXT_ACT_DROP) { 5830 req->flags = 5831 cpu_to_le32(CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP); 5832 } else if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 5833 bnxt_cfg_rfs_ring_tbl_idx(bp, req, fltr->base.rxq); 5834 } else { 5835 vnic = &bp->vnic_info[fltr->base.rxq + 1]; 5836 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 5837 } 5838 req->enables |= cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 5839 5840 req->ethertype = htons(ETH_P_IP); 5841 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 5842 req->ip_protocol = keys->basic.ip_proto; 5843 5844 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 5845 req->ethertype = htons(ETH_P_IPV6); 5846 req->ip_addr_type = 5847 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 5848 *(struct in6_addr *)&req->src_ipaddr[0] = keys->addrs.v6addrs.src; 5849 *(struct in6_addr *)&req->src_ipaddr_mask[0] = masks->addrs.v6addrs.src; 5850 *(struct in6_addr *)&req->dst_ipaddr[0] = keys->addrs.v6addrs.dst; 5851 *(struct in6_addr *)&req->dst_ipaddr_mask[0] = masks->addrs.v6addrs.dst; 5852 } else { 5853 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 5854 req->src_ipaddr_mask[0] = masks->addrs.v4addrs.src; 5855 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 5856 req->dst_ipaddr_mask[0] = masks->addrs.v4addrs.dst; 5857 } 5858 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 5859 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 5860 req->tunnel_type = 5861 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 5862 } 5863 5864 req->src_port = keys->ports.src; 5865 req->src_port_mask = masks->ports.src; 5866 req->dst_port = keys->ports.dst; 5867 req->dst_port_mask = masks->ports.dst; 5868 5869 resp = hwrm_req_hold(bp, req); 5870 rc = hwrm_req_send(bp, req); 5871 if (!rc) 5872 fltr->base.filter_id = resp->ntuple_filter_id; 5873 hwrm_req_drop(bp, req); 5874 return rc; 5875 } 5876 5877 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 5878 const u8 *mac_addr) 5879 { 5880 struct bnxt_l2_filter *fltr; 5881 struct bnxt_l2_key key; 5882 int rc; 5883 5884 ether_addr_copy(key.dst_mac_addr, mac_addr); 5885 key.vlan = 0; 5886 fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL); 5887 if (IS_ERR(fltr)) 5888 return PTR_ERR(fltr); 5889 5890 fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id; 5891 rc = bnxt_hwrm_l2_filter_alloc(bp, fltr); 5892 if (rc) 5893 bnxt_del_l2_filter(bp, fltr); 5894 else 5895 bp->vnic_info[vnic_id].l2_filters[idx] = fltr; 5896 return rc; 5897 } 5898 5899 static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 5900 { 5901 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 5902 5903 /* Any associated ntuple filters will also be cleared by firmware. */ 5904 for (i = 0; i < num_of_vnics; i++) { 5905 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5906 5907 for (j = 0; j < vnic->uc_filter_count; j++) { 5908 struct bnxt_l2_filter *fltr = vnic->l2_filters[j]; 5909 5910 bnxt_hwrm_l2_filter_free(bp, fltr); 5911 bnxt_del_l2_filter(bp, fltr); 5912 } 5913 vnic->uc_filter_count = 0; 5914 } 5915 } 5916 5917 #define BNXT_DFLT_TUNL_TPA_BMAP \ 5918 (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \ 5919 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \ 5920 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6) 5921 5922 static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp, 5923 struct hwrm_vnic_tpa_cfg_input *req) 5924 { 5925 u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP; 5926 5927 if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA)) 5928 return; 5929 5930 if (bp->vxlan_port) 5931 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN; 5932 if (bp->vxlan_gpe_port) 5933 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE; 5934 if (bp->nge_port) 5935 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE; 5936 5937 req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN); 5938 req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap); 5939 } 5940 5941 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 5942 { 5943 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5944 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 5945 struct hwrm_vnic_tpa_cfg_input *req; 5946 int rc; 5947 5948 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 5949 return 0; 5950 5951 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 5952 if (rc) 5953 return rc; 5954 5955 if (tpa_flags) { 5956 u16 mss = bp->dev->mtu - 40; 5957 u32 nsegs, n, segs = 0, flags; 5958 5959 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 5960 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 5961 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 5962 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 5963 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 5964 if (tpa_flags & BNXT_FLAG_GRO) 5965 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 5966 5967 req->flags = cpu_to_le32(flags); 5968 5969 req->enables = 5970 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 5971 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 5972 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 5973 5974 /* Number of segs are log2 units, and first packet is not 5975 * included as part of this units. 5976 */ 5977 if (mss <= BNXT_RX_PAGE_SIZE) { 5978 n = BNXT_RX_PAGE_SIZE / mss; 5979 nsegs = (MAX_SKB_FRAGS - 1) * n; 5980 } else { 5981 n = mss / BNXT_RX_PAGE_SIZE; 5982 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 5983 n++; 5984 nsegs = (MAX_SKB_FRAGS - n) / n; 5985 } 5986 5987 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 5988 segs = MAX_TPA_SEGS_P5; 5989 max_aggs = bp->max_tpa; 5990 } else { 5991 segs = ilog2(nsegs); 5992 } 5993 req->max_agg_segs = cpu_to_le16(segs); 5994 req->max_aggs = cpu_to_le16(max_aggs); 5995 5996 req->min_agg_len = cpu_to_le32(512); 5997 bnxt_hwrm_vnic_update_tunl_tpa(bp, req); 5998 } 5999 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6000 6001 return hwrm_req_send(bp, req); 6002 } 6003 6004 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 6005 { 6006 struct bnxt_ring_grp_info *grp_info; 6007 6008 grp_info = &bp->grp_info[ring->grp_idx]; 6009 return grp_info->cp_fw_ring_id; 6010 } 6011 6012 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 6013 { 6014 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6015 return rxr->rx_cpr->cp_ring_struct.fw_ring_id; 6016 else 6017 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 6018 } 6019 6020 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 6021 { 6022 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6023 return txr->tx_cpr->cp_ring_struct.fw_ring_id; 6024 else 6025 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 6026 } 6027 6028 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 6029 { 6030 int entries; 6031 6032 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6033 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 6034 else 6035 entries = HW_HASH_INDEX_SIZE; 6036 6037 bp->rss_indir_tbl_entries = entries; 6038 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 6039 GFP_KERNEL); 6040 if (!bp->rss_indir_tbl) 6041 return -ENOMEM; 6042 return 0; 6043 } 6044 6045 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 6046 { 6047 u16 max_rings, max_entries, pad, i; 6048 6049 if (!bp->rx_nr_rings) 6050 return; 6051 6052 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6053 max_rings = bp->rx_nr_rings - 1; 6054 else 6055 max_rings = bp->rx_nr_rings; 6056 6057 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 6058 6059 for (i = 0; i < max_entries; i++) 6060 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 6061 6062 pad = bp->rss_indir_tbl_entries - max_entries; 6063 if (pad) 6064 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 6065 } 6066 6067 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 6068 { 6069 u16 i, tbl_size, max_ring = 0; 6070 6071 if (!bp->rss_indir_tbl) 6072 return 0; 6073 6074 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 6075 for (i = 0; i < tbl_size; i++) 6076 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 6077 return max_ring; 6078 } 6079 6080 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 6081 { 6082 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6083 if (!rx_rings) 6084 return 0; 6085 return bnxt_calc_nr_ring_pages(rx_rings - 1, 6086 BNXT_RSS_TABLE_ENTRIES_P5); 6087 } 6088 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6089 return 2; 6090 return 1; 6091 } 6092 6093 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 6094 { 6095 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 6096 u16 i, j; 6097 6098 /* Fill the RSS indirection table with ring group ids */ 6099 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 6100 if (!no_rss) 6101 j = bp->rss_indir_tbl[i]; 6102 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 6103 } 6104 } 6105 6106 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 6107 struct bnxt_vnic_info *vnic) 6108 { 6109 __le16 *ring_tbl = vnic->rss_table; 6110 struct bnxt_rx_ring_info *rxr; 6111 u16 tbl_size, i; 6112 6113 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 6114 6115 for (i = 0; i < tbl_size; i++) { 6116 u16 ring_id, j; 6117 6118 if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG) 6119 j = ethtool_rxfh_indir_default(i, bp->rx_nr_rings); 6120 else 6121 j = bp->rss_indir_tbl[i]; 6122 rxr = &bp->rx_ring[j]; 6123 6124 ring_id = rxr->rx_ring_struct.fw_ring_id; 6125 *ring_tbl++ = cpu_to_le16(ring_id); 6126 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6127 *ring_tbl++ = cpu_to_le16(ring_id); 6128 } 6129 } 6130 6131 static void 6132 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, 6133 struct bnxt_vnic_info *vnic) 6134 { 6135 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6136 bnxt_fill_hw_rss_tbl_p5(bp, vnic); 6137 if (bp->flags & BNXT_FLAG_CHIP_P7) 6138 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; 6139 } else { 6140 bnxt_fill_hw_rss_tbl(bp, vnic); 6141 } 6142 6143 if (bp->rss_hash_delta) { 6144 req->hash_type = cpu_to_le32(bp->rss_hash_delta); 6145 if (bp->rss_hash_cfg & bp->rss_hash_delta) 6146 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE; 6147 else 6148 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE; 6149 } else { 6150 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 6151 } 6152 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 6153 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 6154 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 6155 } 6156 6157 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 6158 { 6159 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 6160 struct hwrm_vnic_rss_cfg_input *req; 6161 int rc; 6162 6163 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) || 6164 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 6165 return 0; 6166 6167 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 6168 if (rc) 6169 return rc; 6170 6171 if (set_rss) 6172 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 6173 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6174 return hwrm_req_send(bp, req); 6175 } 6176 6177 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 6178 { 6179 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 6180 struct hwrm_vnic_rss_cfg_input *req; 6181 dma_addr_t ring_tbl_map; 6182 u32 i, nr_ctxs; 6183 int rc; 6184 6185 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 6186 if (rc) 6187 return rc; 6188 6189 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6190 if (!set_rss) 6191 return hwrm_req_send(bp, req); 6192 6193 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 6194 ring_tbl_map = vnic->rss_table_dma_addr; 6195 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 6196 6197 hwrm_req_hold(bp, req); 6198 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 6199 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 6200 req->ring_table_pair_index = i; 6201 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 6202 rc = hwrm_req_send(bp, req); 6203 if (rc) 6204 goto exit; 6205 } 6206 6207 exit: 6208 hwrm_req_drop(bp, req); 6209 return rc; 6210 } 6211 6212 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp) 6213 { 6214 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 6215 struct hwrm_vnic_rss_qcfg_output *resp; 6216 struct hwrm_vnic_rss_qcfg_input *req; 6217 6218 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG)) 6219 return; 6220 6221 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6222 /* all contexts configured to same hash_type, zero always exists */ 6223 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6224 resp = hwrm_req_hold(bp, req); 6225 if (!hwrm_req_send(bp, req)) { 6226 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg; 6227 bp->rss_hash_delta = 0; 6228 } 6229 hwrm_req_drop(bp, req); 6230 } 6231 6232 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 6233 { 6234 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 6235 struct hwrm_vnic_plcmodes_cfg_input *req; 6236 int rc; 6237 6238 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 6239 if (rc) 6240 return rc; 6241 6242 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT); 6243 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID); 6244 6245 if (BNXT_RX_PAGE_MODE(bp)) { 6246 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size); 6247 } else { 6248 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 6249 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 6250 req->enables |= 6251 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 6252 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 6253 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 6254 } 6255 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 6256 return hwrm_req_send(bp, req); 6257 } 6258 6259 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 6260 u16 ctx_idx) 6261 { 6262 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 6263 6264 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 6265 return; 6266 6267 req->rss_cos_lb_ctx_id = 6268 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 6269 6270 hwrm_req_send(bp, req); 6271 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 6272 } 6273 6274 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 6275 { 6276 int i, j; 6277 6278 for (i = 0; i < bp->nr_vnics; i++) { 6279 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 6280 6281 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 6282 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 6283 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 6284 } 6285 } 6286 bp->rsscos_nr_ctxs = 0; 6287 } 6288 6289 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 6290 { 6291 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 6292 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 6293 int rc; 6294 6295 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 6296 if (rc) 6297 return rc; 6298 6299 resp = hwrm_req_hold(bp, req); 6300 rc = hwrm_req_send(bp, req); 6301 if (!rc) 6302 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 6303 le16_to_cpu(resp->rss_cos_lb_ctx_id); 6304 hwrm_req_drop(bp, req); 6305 6306 return rc; 6307 } 6308 6309 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 6310 { 6311 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 6312 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 6313 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 6314 } 6315 6316 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 6317 { 6318 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 6319 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 6320 struct hwrm_vnic_cfg_input *req; 6321 unsigned int ring = 0, grp_idx; 6322 u16 def_vlan = 0; 6323 int rc; 6324 6325 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 6326 if (rc) 6327 return rc; 6328 6329 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6330 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 6331 6332 req->default_rx_ring_id = 6333 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 6334 req->default_cmpl_ring_id = 6335 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 6336 req->enables = 6337 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 6338 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 6339 goto vnic_mru; 6340 } 6341 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 6342 /* Only RSS support for now TBD: COS & LB */ 6343 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 6344 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6345 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 6346 VNIC_CFG_REQ_ENABLES_MRU); 6347 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 6348 req->rss_rule = cpu_to_le16(vnic0->fw_rss_cos_lb_ctx[0]); 6349 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 6350 VNIC_CFG_REQ_ENABLES_MRU); 6351 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 6352 } else { 6353 req->rss_rule = cpu_to_le16(0xffff); 6354 } 6355 6356 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 6357 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 6358 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 6359 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 6360 } else { 6361 req->cos_rule = cpu_to_le16(0xffff); 6362 } 6363 6364 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 6365 ring = 0; 6366 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 6367 ring = vnic_id - 1; 6368 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 6369 ring = bp->rx_nr_rings - 1; 6370 6371 grp_idx = bp->rx_ring[ring].bnapi->index; 6372 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 6373 req->lb_rule = cpu_to_le16(0xffff); 6374 vnic_mru: 6375 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 6376 6377 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6378 #ifdef CONFIG_BNXT_SRIOV 6379 if (BNXT_VF(bp)) 6380 def_vlan = bp->vf.vlan; 6381 #endif 6382 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 6383 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 6384 if (!vnic_id && bnxt_ulp_registered(bp->edev)) 6385 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 6386 6387 return hwrm_req_send(bp, req); 6388 } 6389 6390 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 6391 { 6392 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 6393 struct hwrm_vnic_free_input *req; 6394 6395 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 6396 return; 6397 6398 req->vnic_id = 6399 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 6400 6401 hwrm_req_send(bp, req); 6402 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 6403 } 6404 } 6405 6406 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 6407 { 6408 u16 i; 6409 6410 for (i = 0; i < bp->nr_vnics; i++) 6411 bnxt_hwrm_vnic_free_one(bp, i); 6412 } 6413 6414 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 6415 unsigned int start_rx_ring_idx, 6416 unsigned int nr_rings) 6417 { 6418 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 6419 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 6420 struct hwrm_vnic_alloc_output *resp; 6421 struct hwrm_vnic_alloc_input *req; 6422 int rc; 6423 6424 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 6425 if (rc) 6426 return rc; 6427 6428 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6429 goto vnic_no_ring_grps; 6430 6431 /* map ring groups to this vnic */ 6432 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 6433 grp_idx = bp->rx_ring[i].bnapi->index; 6434 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 6435 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 6436 j, nr_rings); 6437 break; 6438 } 6439 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 6440 } 6441 6442 vnic_no_ring_grps: 6443 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 6444 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 6445 if (vnic_id == BNXT_VNIC_DEFAULT) 6446 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 6447 6448 resp = hwrm_req_hold(bp, req); 6449 rc = hwrm_req_send(bp, req); 6450 if (!rc) 6451 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 6452 hwrm_req_drop(bp, req); 6453 return rc; 6454 } 6455 6456 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 6457 { 6458 struct hwrm_vnic_qcaps_output *resp; 6459 struct hwrm_vnic_qcaps_input *req; 6460 int rc; 6461 6462 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 6463 bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP; 6464 bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP; 6465 if (bp->hwrm_spec_code < 0x10600) 6466 return 0; 6467 6468 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 6469 if (rc) 6470 return rc; 6471 6472 resp = hwrm_req_hold(bp, req); 6473 rc = hwrm_req_send(bp, req); 6474 if (!rc) { 6475 u32 flags = le32_to_cpu(resp->flags); 6476 6477 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 6478 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 6479 bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP; 6480 if (flags & 6481 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 6482 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 6483 6484 /* Older P5 fw before EXT_HW_STATS support did not set 6485 * VLAN_STRIP_CAP properly. 6486 */ 6487 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 6488 (BNXT_CHIP_P5(bp) && 6489 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 6490 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 6491 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP) 6492 bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA; 6493 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED) 6494 bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM; 6495 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 6496 if (bp->max_tpa_v2) { 6497 if (BNXT_CHIP_P5(bp)) 6498 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 6499 else 6500 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7; 6501 } 6502 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP) 6503 bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA; 6504 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP) 6505 bp->rss_cap |= BNXT_RSS_CAP_AH_V4_RSS_CAP; 6506 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP) 6507 bp->rss_cap |= BNXT_RSS_CAP_AH_V6_RSS_CAP; 6508 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP) 6509 bp->rss_cap |= BNXT_RSS_CAP_ESP_V4_RSS_CAP; 6510 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP) 6511 bp->rss_cap |= BNXT_RSS_CAP_ESP_V6_RSS_CAP; 6512 } 6513 hwrm_req_drop(bp, req); 6514 return rc; 6515 } 6516 6517 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 6518 { 6519 struct hwrm_ring_grp_alloc_output *resp; 6520 struct hwrm_ring_grp_alloc_input *req; 6521 int rc; 6522 u16 i; 6523 6524 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6525 return 0; 6526 6527 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 6528 if (rc) 6529 return rc; 6530 6531 resp = hwrm_req_hold(bp, req); 6532 for (i = 0; i < bp->rx_nr_rings; i++) { 6533 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 6534 6535 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 6536 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 6537 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 6538 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 6539 6540 rc = hwrm_req_send(bp, req); 6541 6542 if (rc) 6543 break; 6544 6545 bp->grp_info[grp_idx].fw_grp_id = 6546 le32_to_cpu(resp->ring_group_id); 6547 } 6548 hwrm_req_drop(bp, req); 6549 return rc; 6550 } 6551 6552 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 6553 { 6554 struct hwrm_ring_grp_free_input *req; 6555 u16 i; 6556 6557 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 6558 return; 6559 6560 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 6561 return; 6562 6563 hwrm_req_hold(bp, req); 6564 for (i = 0; i < bp->cp_nr_rings; i++) { 6565 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 6566 continue; 6567 req->ring_group_id = 6568 cpu_to_le32(bp->grp_info[i].fw_grp_id); 6569 6570 hwrm_req_send(bp, req); 6571 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 6572 } 6573 hwrm_req_drop(bp, req); 6574 } 6575 6576 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 6577 struct bnxt_ring_struct *ring, 6578 u32 ring_type, u32 map_index) 6579 { 6580 struct hwrm_ring_alloc_output *resp; 6581 struct hwrm_ring_alloc_input *req; 6582 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 6583 struct bnxt_ring_grp_info *grp_info; 6584 int rc, err = 0; 6585 u16 ring_id; 6586 6587 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 6588 if (rc) 6589 goto exit; 6590 6591 req->enables = 0; 6592 if (rmem->nr_pages > 1) { 6593 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 6594 /* Page size is in log2 units */ 6595 req->page_size = BNXT_PAGE_SHIFT; 6596 req->page_tbl_depth = 1; 6597 } else { 6598 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 6599 } 6600 req->fbo = 0; 6601 /* Association of ring index with doorbell index and MSIX number */ 6602 req->logical_id = cpu_to_le16(map_index); 6603 6604 switch (ring_type) { 6605 case HWRM_RING_ALLOC_TX: { 6606 struct bnxt_tx_ring_info *txr; 6607 6608 txr = container_of(ring, struct bnxt_tx_ring_info, 6609 tx_ring_struct); 6610 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 6611 /* Association of transmit ring with completion ring */ 6612 grp_info = &bp->grp_info[ring->grp_idx]; 6613 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 6614 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 6615 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 6616 req->queue_id = cpu_to_le16(ring->queue_id); 6617 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL) 6618 req->cmpl_coal_cnt = 6619 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64; 6620 break; 6621 } 6622 case HWRM_RING_ALLOC_RX: 6623 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 6624 req->length = cpu_to_le32(bp->rx_ring_mask + 1); 6625 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6626 u16 flags = 0; 6627 6628 /* Association of rx ring with stats context */ 6629 grp_info = &bp->grp_info[ring->grp_idx]; 6630 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 6631 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 6632 req->enables |= cpu_to_le32( 6633 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 6634 if (NET_IP_ALIGN == 2) 6635 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 6636 req->flags = cpu_to_le16(flags); 6637 } 6638 break; 6639 case HWRM_RING_ALLOC_AGG: 6640 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6641 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 6642 /* Association of agg ring with rx ring */ 6643 grp_info = &bp->grp_info[ring->grp_idx]; 6644 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 6645 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 6646 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 6647 req->enables |= cpu_to_le32( 6648 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 6649 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 6650 } else { 6651 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 6652 } 6653 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 6654 break; 6655 case HWRM_RING_ALLOC_CMPL: 6656 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 6657 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 6658 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6659 /* Association of cp ring with nq */ 6660 grp_info = &bp->grp_info[map_index]; 6661 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 6662 req->cq_handle = cpu_to_le64(ring->handle); 6663 req->enables |= cpu_to_le32( 6664 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 6665 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 6666 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 6667 } 6668 break; 6669 case HWRM_RING_ALLOC_NQ: 6670 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 6671 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 6672 if (bp->flags & BNXT_FLAG_USING_MSIX) 6673 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 6674 break; 6675 default: 6676 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 6677 ring_type); 6678 return -1; 6679 } 6680 6681 resp = hwrm_req_hold(bp, req); 6682 rc = hwrm_req_send(bp, req); 6683 err = le16_to_cpu(resp->error_code); 6684 ring_id = le16_to_cpu(resp->ring_id); 6685 hwrm_req_drop(bp, req); 6686 6687 exit: 6688 if (rc || err) { 6689 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 6690 ring_type, rc, err); 6691 return -EIO; 6692 } 6693 ring->fw_ring_id = ring_id; 6694 return rc; 6695 } 6696 6697 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 6698 { 6699 int rc; 6700 6701 if (BNXT_PF(bp)) { 6702 struct hwrm_func_cfg_input *req; 6703 6704 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 6705 if (rc) 6706 return rc; 6707 6708 req->fid = cpu_to_le16(0xffff); 6709 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 6710 req->async_event_cr = cpu_to_le16(idx); 6711 return hwrm_req_send(bp, req); 6712 } else { 6713 struct hwrm_func_vf_cfg_input *req; 6714 6715 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 6716 if (rc) 6717 return rc; 6718 6719 req->enables = 6720 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 6721 req->async_event_cr = cpu_to_le16(idx); 6722 return hwrm_req_send(bp, req); 6723 } 6724 } 6725 6726 static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db, 6727 u32 ring_type) 6728 { 6729 switch (ring_type) { 6730 case HWRM_RING_ALLOC_TX: 6731 db->db_ring_mask = bp->tx_ring_mask; 6732 break; 6733 case HWRM_RING_ALLOC_RX: 6734 db->db_ring_mask = bp->rx_ring_mask; 6735 break; 6736 case HWRM_RING_ALLOC_AGG: 6737 db->db_ring_mask = bp->rx_agg_ring_mask; 6738 break; 6739 case HWRM_RING_ALLOC_CMPL: 6740 case HWRM_RING_ALLOC_NQ: 6741 db->db_ring_mask = bp->cp_ring_mask; 6742 break; 6743 } 6744 if (bp->flags & BNXT_FLAG_CHIP_P7) { 6745 db->db_epoch_mask = db->db_ring_mask + 1; 6746 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 6747 } 6748 } 6749 6750 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 6751 u32 map_idx, u32 xid) 6752 { 6753 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6754 switch (ring_type) { 6755 case HWRM_RING_ALLOC_TX: 6756 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 6757 break; 6758 case HWRM_RING_ALLOC_RX: 6759 case HWRM_RING_ALLOC_AGG: 6760 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 6761 break; 6762 case HWRM_RING_ALLOC_CMPL: 6763 db->db_key64 = DBR_PATH_L2; 6764 break; 6765 case HWRM_RING_ALLOC_NQ: 6766 db->db_key64 = DBR_PATH_L2; 6767 break; 6768 } 6769 db->db_key64 |= (u64)xid << DBR_XID_SFT; 6770 6771 if (bp->flags & BNXT_FLAG_CHIP_P7) 6772 db->db_key64 |= DBR_VALID; 6773 6774 db->doorbell = bp->bar1 + bp->db_offset; 6775 } else { 6776 db->doorbell = bp->bar1 + map_idx * 0x80; 6777 switch (ring_type) { 6778 case HWRM_RING_ALLOC_TX: 6779 db->db_key32 = DB_KEY_TX; 6780 break; 6781 case HWRM_RING_ALLOC_RX: 6782 case HWRM_RING_ALLOC_AGG: 6783 db->db_key32 = DB_KEY_RX; 6784 break; 6785 case HWRM_RING_ALLOC_CMPL: 6786 db->db_key32 = DB_KEY_CP; 6787 break; 6788 } 6789 } 6790 bnxt_set_db_mask(bp, db, ring_type); 6791 } 6792 6793 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 6794 { 6795 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 6796 int i, rc = 0; 6797 u32 type; 6798 6799 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6800 type = HWRM_RING_ALLOC_NQ; 6801 else 6802 type = HWRM_RING_ALLOC_CMPL; 6803 for (i = 0; i < bp->cp_nr_rings; i++) { 6804 struct bnxt_napi *bnapi = bp->bnapi[i]; 6805 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6806 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 6807 u32 map_idx = ring->map_idx; 6808 unsigned int vector; 6809 6810 vector = bp->irq_tbl[map_idx].vector; 6811 disable_irq_nosync(vector); 6812 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6813 if (rc) { 6814 enable_irq(vector); 6815 goto err_out; 6816 } 6817 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 6818 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 6819 enable_irq(vector); 6820 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 6821 6822 if (!i) { 6823 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 6824 if (rc) 6825 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 6826 } 6827 } 6828 6829 type = HWRM_RING_ALLOC_TX; 6830 for (i = 0; i < bp->tx_nr_rings; i++) { 6831 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 6832 struct bnxt_ring_struct *ring; 6833 u32 map_idx; 6834 6835 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6836 struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr; 6837 struct bnxt_napi *bnapi = txr->bnapi; 6838 u32 type2 = HWRM_RING_ALLOC_CMPL; 6839 6840 ring = &cpr2->cp_ring_struct; 6841 ring->handle = BNXT_SET_NQ_HDL(cpr2); 6842 map_idx = bnapi->index; 6843 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 6844 if (rc) 6845 goto err_out; 6846 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 6847 ring->fw_ring_id); 6848 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 6849 } 6850 ring = &txr->tx_ring_struct; 6851 map_idx = i; 6852 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6853 if (rc) 6854 goto err_out; 6855 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 6856 } 6857 6858 type = HWRM_RING_ALLOC_RX; 6859 for (i = 0; i < bp->rx_nr_rings; i++) { 6860 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6861 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 6862 struct bnxt_napi *bnapi = rxr->bnapi; 6863 u32 map_idx = bnapi->index; 6864 6865 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6866 if (rc) 6867 goto err_out; 6868 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 6869 /* If we have agg rings, post agg buffers first. */ 6870 if (!agg_rings) 6871 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 6872 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 6873 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6874 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr; 6875 u32 type2 = HWRM_RING_ALLOC_CMPL; 6876 6877 ring = &cpr2->cp_ring_struct; 6878 ring->handle = BNXT_SET_NQ_HDL(cpr2); 6879 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 6880 if (rc) 6881 goto err_out; 6882 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 6883 ring->fw_ring_id); 6884 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 6885 } 6886 } 6887 6888 if (agg_rings) { 6889 type = HWRM_RING_ALLOC_AGG; 6890 for (i = 0; i < bp->rx_nr_rings; i++) { 6891 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6892 struct bnxt_ring_struct *ring = 6893 &rxr->rx_agg_ring_struct; 6894 u32 grp_idx = ring->grp_idx; 6895 u32 map_idx = grp_idx + bp->rx_nr_rings; 6896 6897 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 6898 if (rc) 6899 goto err_out; 6900 6901 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 6902 ring->fw_ring_id); 6903 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 6904 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 6905 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 6906 } 6907 } 6908 err_out: 6909 return rc; 6910 } 6911 6912 static int hwrm_ring_free_send_msg(struct bnxt *bp, 6913 struct bnxt_ring_struct *ring, 6914 u32 ring_type, int cmpl_ring_id) 6915 { 6916 struct hwrm_ring_free_output *resp; 6917 struct hwrm_ring_free_input *req; 6918 u16 error_code = 0; 6919 int rc; 6920 6921 if (BNXT_NO_FW_ACCESS(bp)) 6922 return 0; 6923 6924 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 6925 if (rc) 6926 goto exit; 6927 6928 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 6929 req->ring_type = ring_type; 6930 req->ring_id = cpu_to_le16(ring->fw_ring_id); 6931 6932 resp = hwrm_req_hold(bp, req); 6933 rc = hwrm_req_send(bp, req); 6934 error_code = le16_to_cpu(resp->error_code); 6935 hwrm_req_drop(bp, req); 6936 exit: 6937 if (rc || error_code) { 6938 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 6939 ring_type, rc, error_code); 6940 return -EIO; 6941 } 6942 return 0; 6943 } 6944 6945 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 6946 { 6947 u32 type; 6948 int i; 6949 6950 if (!bp->bnapi) 6951 return; 6952 6953 for (i = 0; i < bp->tx_nr_rings; i++) { 6954 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 6955 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 6956 6957 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6958 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 6959 6960 hwrm_ring_free_send_msg(bp, ring, 6961 RING_FREE_REQ_RING_TYPE_TX, 6962 close_path ? cmpl_ring_id : 6963 INVALID_HW_RING_ID); 6964 ring->fw_ring_id = INVALID_HW_RING_ID; 6965 } 6966 } 6967 6968 for (i = 0; i < bp->rx_nr_rings; i++) { 6969 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6970 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 6971 u32 grp_idx = rxr->bnapi->index; 6972 6973 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6974 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6975 6976 hwrm_ring_free_send_msg(bp, ring, 6977 RING_FREE_REQ_RING_TYPE_RX, 6978 close_path ? cmpl_ring_id : 6979 INVALID_HW_RING_ID); 6980 ring->fw_ring_id = INVALID_HW_RING_ID; 6981 bp->grp_info[grp_idx].rx_fw_ring_id = 6982 INVALID_HW_RING_ID; 6983 } 6984 } 6985 6986 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6987 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 6988 else 6989 type = RING_FREE_REQ_RING_TYPE_RX; 6990 for (i = 0; i < bp->rx_nr_rings; i++) { 6991 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 6992 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 6993 u32 grp_idx = rxr->bnapi->index; 6994 6995 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 6996 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6997 6998 hwrm_ring_free_send_msg(bp, ring, type, 6999 close_path ? cmpl_ring_id : 7000 INVALID_HW_RING_ID); 7001 ring->fw_ring_id = INVALID_HW_RING_ID; 7002 bp->grp_info[grp_idx].agg_fw_ring_id = 7003 INVALID_HW_RING_ID; 7004 } 7005 } 7006 7007 /* The completion rings are about to be freed. After that the 7008 * IRQ doorbell will not work anymore. So we need to disable 7009 * IRQ here. 7010 */ 7011 bnxt_disable_int_sync(bp); 7012 7013 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7014 type = RING_FREE_REQ_RING_TYPE_NQ; 7015 else 7016 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 7017 for (i = 0; i < bp->cp_nr_rings; i++) { 7018 struct bnxt_napi *bnapi = bp->bnapi[i]; 7019 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7020 struct bnxt_ring_struct *ring; 7021 int j; 7022 7023 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) { 7024 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 7025 7026 ring = &cpr2->cp_ring_struct; 7027 if (ring->fw_ring_id == INVALID_HW_RING_ID) 7028 continue; 7029 hwrm_ring_free_send_msg(bp, ring, 7030 RING_FREE_REQ_RING_TYPE_L2_CMPL, 7031 INVALID_HW_RING_ID); 7032 ring->fw_ring_id = INVALID_HW_RING_ID; 7033 } 7034 ring = &cpr->cp_ring_struct; 7035 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7036 hwrm_ring_free_send_msg(bp, ring, type, 7037 INVALID_HW_RING_ID); 7038 ring->fw_ring_id = INVALID_HW_RING_ID; 7039 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 7040 } 7041 } 7042 } 7043 7044 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 7045 bool shared); 7046 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 7047 bool shared); 7048 7049 static int bnxt_hwrm_get_rings(struct bnxt *bp) 7050 { 7051 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7052 struct hwrm_func_qcfg_output *resp; 7053 struct hwrm_func_qcfg_input *req; 7054 int rc; 7055 7056 if (bp->hwrm_spec_code < 0x10601) 7057 return 0; 7058 7059 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 7060 if (rc) 7061 return rc; 7062 7063 req->fid = cpu_to_le16(0xffff); 7064 resp = hwrm_req_hold(bp, req); 7065 rc = hwrm_req_send(bp, req); 7066 if (rc) { 7067 hwrm_req_drop(bp, req); 7068 return rc; 7069 } 7070 7071 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 7072 if (BNXT_NEW_RM(bp)) { 7073 u16 cp, stats; 7074 7075 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 7076 hw_resc->resv_hw_ring_grps = 7077 le32_to_cpu(resp->alloc_hw_ring_grps); 7078 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 7079 hw_resc->resv_rsscos_ctxs = le16_to_cpu(resp->alloc_rsscos_ctx); 7080 cp = le16_to_cpu(resp->alloc_cmpl_rings); 7081 stats = le16_to_cpu(resp->alloc_stat_ctx); 7082 hw_resc->resv_irqs = cp; 7083 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7084 int rx = hw_resc->resv_rx_rings; 7085 int tx = hw_resc->resv_tx_rings; 7086 7087 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7088 rx >>= 1; 7089 if (cp < (rx + tx)) { 7090 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false); 7091 if (rc) 7092 goto get_rings_exit; 7093 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7094 rx <<= 1; 7095 hw_resc->resv_rx_rings = rx; 7096 hw_resc->resv_tx_rings = tx; 7097 } 7098 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 7099 hw_resc->resv_hw_ring_grps = rx; 7100 } 7101 hw_resc->resv_cp_rings = cp; 7102 hw_resc->resv_stat_ctxs = stats; 7103 } 7104 get_rings_exit: 7105 hwrm_req_drop(bp, req); 7106 return rc; 7107 } 7108 7109 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 7110 { 7111 struct hwrm_func_qcfg_output *resp; 7112 struct hwrm_func_qcfg_input *req; 7113 int rc; 7114 7115 if (bp->hwrm_spec_code < 0x10601) 7116 return 0; 7117 7118 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 7119 if (rc) 7120 return rc; 7121 7122 req->fid = cpu_to_le16(fid); 7123 resp = hwrm_req_hold(bp, req); 7124 rc = hwrm_req_send(bp, req); 7125 if (!rc) 7126 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 7127 7128 hwrm_req_drop(bp, req); 7129 return rc; 7130 } 7131 7132 static bool bnxt_rfs_supported(struct bnxt *bp); 7133 7134 static struct hwrm_func_cfg_input * 7135 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7136 { 7137 struct hwrm_func_cfg_input *req; 7138 u32 enables = 0; 7139 7140 if (bnxt_hwrm_func_cfg_short_req_init(bp, &req)) 7141 return NULL; 7142 7143 req->fid = cpu_to_le16(0xffff); 7144 enables |= hwr->tx ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 7145 req->num_tx_rings = cpu_to_le16(hwr->tx); 7146 if (BNXT_NEW_RM(bp)) { 7147 enables |= hwr->rx ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 7148 enables |= hwr->stat ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 7149 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7150 enables |= hwr->cp ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 7151 enables |= hwr->cp_p5 ? 7152 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7153 } else { 7154 enables |= hwr->cp ? 7155 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7156 enables |= hwr->grp ? 7157 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 7158 } 7159 enables |= hwr->vnic ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 7160 enables |= hwr->rss_ctx ? FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 7161 0; 7162 req->num_rx_rings = cpu_to_le16(hwr->rx); 7163 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx); 7164 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7165 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5); 7166 req->num_msix = cpu_to_le16(hwr->cp); 7167 } else { 7168 req->num_cmpl_rings = cpu_to_le16(hwr->cp); 7169 req->num_hw_ring_grps = cpu_to_le16(hwr->grp); 7170 } 7171 req->num_stat_ctxs = cpu_to_le16(hwr->stat); 7172 req->num_vnics = cpu_to_le16(hwr->vnic); 7173 } 7174 req->enables = cpu_to_le32(enables); 7175 return req; 7176 } 7177 7178 static struct hwrm_func_vf_cfg_input * 7179 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7180 { 7181 struct hwrm_func_vf_cfg_input *req; 7182 u32 enables = 0; 7183 7184 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 7185 return NULL; 7186 7187 enables |= hwr->tx ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 7188 enables |= hwr->rx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 7189 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 7190 enables |= hwr->stat ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 7191 enables |= hwr->rss_ctx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 7192 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7193 enables |= hwr->cp_p5 ? 7194 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7195 } else { 7196 enables |= hwr->cp ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7197 enables |= hwr->grp ? 7198 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 7199 } 7200 enables |= hwr->vnic ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 7201 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 7202 7203 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 7204 req->num_tx_rings = cpu_to_le16(hwr->tx); 7205 req->num_rx_rings = cpu_to_le16(hwr->rx); 7206 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx); 7207 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7208 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5); 7209 } else { 7210 req->num_cmpl_rings = cpu_to_le16(hwr->cp); 7211 req->num_hw_ring_grps = cpu_to_le16(hwr->grp); 7212 } 7213 req->num_stat_ctxs = cpu_to_le16(hwr->stat); 7214 req->num_vnics = cpu_to_le16(hwr->vnic); 7215 7216 req->enables = cpu_to_le32(enables); 7217 return req; 7218 } 7219 7220 static int 7221 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7222 { 7223 struct hwrm_func_cfg_input *req; 7224 int rc; 7225 7226 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr); 7227 if (!req) 7228 return -ENOMEM; 7229 7230 if (!req->enables) { 7231 hwrm_req_drop(bp, req); 7232 return 0; 7233 } 7234 7235 rc = hwrm_req_send(bp, req); 7236 if (rc) 7237 return rc; 7238 7239 if (bp->hwrm_spec_code < 0x10601) 7240 bp->hw_resc.resv_tx_rings = hwr->tx; 7241 7242 return bnxt_hwrm_get_rings(bp); 7243 } 7244 7245 static int 7246 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7247 { 7248 struct hwrm_func_vf_cfg_input *req; 7249 int rc; 7250 7251 if (!BNXT_NEW_RM(bp)) { 7252 bp->hw_resc.resv_tx_rings = hwr->tx; 7253 return 0; 7254 } 7255 7256 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr); 7257 if (!req) 7258 return -ENOMEM; 7259 7260 rc = hwrm_req_send(bp, req); 7261 if (rc) 7262 return rc; 7263 7264 return bnxt_hwrm_get_rings(bp); 7265 } 7266 7267 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7268 { 7269 if (BNXT_PF(bp)) 7270 return bnxt_hwrm_reserve_pf_rings(bp, hwr); 7271 else 7272 return bnxt_hwrm_reserve_vf_rings(bp, hwr); 7273 } 7274 7275 int bnxt_nq_rings_in_use(struct bnxt *bp) 7276 { 7277 int cp = bp->cp_nr_rings; 7278 int ulp_msix, ulp_base; 7279 7280 ulp_msix = bnxt_get_ulp_msix_num(bp); 7281 if (ulp_msix) { 7282 ulp_base = bnxt_get_ulp_msix_base(bp); 7283 cp += ulp_msix; 7284 if ((ulp_base + ulp_msix) > cp) 7285 cp = ulp_base + ulp_msix; 7286 } 7287 return cp; 7288 } 7289 7290 static int bnxt_cp_rings_in_use(struct bnxt *bp) 7291 { 7292 int cp; 7293 7294 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7295 return bnxt_nq_rings_in_use(bp); 7296 7297 cp = bp->tx_nr_rings + bp->rx_nr_rings; 7298 return cp; 7299 } 7300 7301 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 7302 { 7303 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 7304 int cp = bp->cp_nr_rings; 7305 7306 if (!ulp_stat) 7307 return cp; 7308 7309 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 7310 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 7311 7312 return cp + ulp_stat; 7313 } 7314 7315 static int bnxt_get_total_rss_ctxs(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7316 { 7317 if (!hwr->grp) 7318 return 0; 7319 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7320 int rss_ctx = bnxt_get_nr_rss_ctxs(bp, hwr->grp); 7321 7322 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 7323 rss_ctx *= hwr->vnic; 7324 return rss_ctx; 7325 } 7326 if (BNXT_VF(bp)) 7327 return BNXT_VF_MAX_RSS_CTX; 7328 if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && bnxt_rfs_supported(bp)) 7329 return hwr->grp + 1; 7330 return 1; 7331 } 7332 7333 /* Check if a default RSS map needs to be setup. This function is only 7334 * used on older firmware that does not require reserving RX rings. 7335 */ 7336 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 7337 { 7338 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7339 7340 /* The RSS map is valid for RX rings set to resv_rx_rings */ 7341 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 7342 hw_resc->resv_rx_rings = bp->rx_nr_rings; 7343 if (!netif_is_rxfh_configured(bp->dev)) 7344 bnxt_set_dflt_rss_indir_tbl(bp); 7345 } 7346 } 7347 7348 static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings) 7349 { 7350 if (bp->flags & BNXT_FLAG_RFS) { 7351 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 7352 return 2; 7353 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7354 return rx_rings + 1; 7355 } 7356 return 1; 7357 } 7358 7359 static bool bnxt_need_reserve_rings(struct bnxt *bp) 7360 { 7361 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7362 int cp = bnxt_cp_rings_in_use(bp); 7363 int nq = bnxt_nq_rings_in_use(bp); 7364 int rx = bp->rx_nr_rings, stat; 7365 int vnic, grp = rx; 7366 7367 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 7368 bp->hwrm_spec_code >= 0x10601) 7369 return true; 7370 7371 /* Old firmware does not need RX ring reservations but we still 7372 * need to setup a default RSS map when needed. With new firmware 7373 * we go through RX ring reservations first and then set up the 7374 * RSS map for the successfully reserved RX rings when needed. 7375 */ 7376 if (!BNXT_NEW_RM(bp)) { 7377 bnxt_check_rss_tbl_no_rmgr(bp); 7378 return false; 7379 } 7380 7381 vnic = bnxt_get_total_vnics(bp, rx); 7382 7383 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7384 rx <<= 1; 7385 stat = bnxt_get_func_stat_ctxs(bp); 7386 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 7387 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 7388 (hw_resc->resv_hw_ring_grps != grp && 7389 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))) 7390 return true; 7391 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) && 7392 hw_resc->resv_irqs != nq) 7393 return true; 7394 return false; 7395 } 7396 7397 static void bnxt_copy_reserved_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7398 { 7399 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7400 7401 hwr->tx = hw_resc->resv_tx_rings; 7402 if (BNXT_NEW_RM(bp)) { 7403 hwr->rx = hw_resc->resv_rx_rings; 7404 hwr->cp = hw_resc->resv_irqs; 7405 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7406 hwr->cp_p5 = hw_resc->resv_cp_rings; 7407 hwr->grp = hw_resc->resv_hw_ring_grps; 7408 hwr->vnic = hw_resc->resv_vnics; 7409 hwr->stat = hw_resc->resv_stat_ctxs; 7410 hwr->rss_ctx = hw_resc->resv_rsscos_ctxs; 7411 } 7412 } 7413 7414 static bool bnxt_rings_ok(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7415 { 7416 return hwr->tx && hwr->rx && hwr->cp && hwr->grp && hwr->vnic && 7417 hwr->stat && (hwr->cp_p5 || !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)); 7418 } 7419 7420 static int __bnxt_reserve_rings(struct bnxt *bp) 7421 { 7422 struct bnxt_hw_rings hwr = {0}; 7423 int rx_rings, rc; 7424 bool sh = false; 7425 int tx_cp; 7426 7427 if (!bnxt_need_reserve_rings(bp)) 7428 return 0; 7429 7430 hwr.cp = bnxt_nq_rings_in_use(bp); 7431 hwr.tx = bp->tx_nr_rings; 7432 hwr.rx = bp->rx_nr_rings; 7433 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 7434 sh = true; 7435 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7436 hwr.cp_p5 = hwr.rx + hwr.tx; 7437 7438 hwr.vnic = bnxt_get_total_vnics(bp, hwr.rx); 7439 7440 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7441 hwr.rx <<= 1; 7442 hwr.grp = bp->rx_nr_rings; 7443 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr); 7444 hwr.stat = bnxt_get_func_stat_ctxs(bp); 7445 7446 rc = bnxt_hwrm_reserve_rings(bp, &hwr); 7447 if (rc) 7448 return rc; 7449 7450 bnxt_copy_reserved_rings(bp, &hwr); 7451 7452 rx_rings = hwr.rx; 7453 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 7454 if (hwr.rx >= 2) { 7455 rx_rings = hwr.rx >> 1; 7456 } else { 7457 if (netif_running(bp->dev)) 7458 return -ENOMEM; 7459 7460 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 7461 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 7462 bp->dev->hw_features &= ~NETIF_F_LRO; 7463 bp->dev->features &= ~NETIF_F_LRO; 7464 bnxt_set_ring_params(bp); 7465 } 7466 } 7467 rx_rings = min_t(int, rx_rings, hwr.grp); 7468 hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings); 7469 if (hwr.stat > bnxt_get_ulp_stat_ctxs(bp)) 7470 hwr.stat -= bnxt_get_ulp_stat_ctxs(bp); 7471 hwr.cp = min_t(int, hwr.cp, hwr.stat); 7472 rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh); 7473 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7474 hwr.rx = rx_rings << 1; 7475 tx_cp = bnxt_num_tx_to_cp(bp, hwr.tx); 7476 hwr.cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings; 7477 bp->tx_nr_rings = hwr.tx; 7478 7479 /* If we cannot reserve all the RX rings, reset the RSS map only 7480 * if absolutely necessary 7481 */ 7482 if (rx_rings != bp->rx_nr_rings) { 7483 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 7484 rx_rings, bp->rx_nr_rings); 7485 if (netif_is_rxfh_configured(bp->dev) && 7486 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 7487 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 7488 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 7489 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 7490 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 7491 } 7492 } 7493 bp->rx_nr_rings = rx_rings; 7494 bp->cp_nr_rings = hwr.cp; 7495 7496 if (!bnxt_rings_ok(bp, &hwr)) 7497 return -ENOMEM; 7498 7499 if (!netif_is_rxfh_configured(bp->dev)) 7500 bnxt_set_dflt_rss_indir_tbl(bp); 7501 7502 return rc; 7503 } 7504 7505 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7506 { 7507 struct hwrm_func_vf_cfg_input *req; 7508 u32 flags; 7509 7510 if (!BNXT_NEW_RM(bp)) 7511 return 0; 7512 7513 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr); 7514 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 7515 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 7516 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 7517 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 7518 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 7519 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 7520 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7521 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 7522 7523 req->flags = cpu_to_le32(flags); 7524 return hwrm_req_send_silent(bp, req); 7525 } 7526 7527 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7528 { 7529 struct hwrm_func_cfg_input *req; 7530 u32 flags; 7531 7532 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr); 7533 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 7534 if (BNXT_NEW_RM(bp)) { 7535 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 7536 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 7537 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 7538 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 7539 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7540 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 7541 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 7542 else 7543 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 7544 } 7545 7546 req->flags = cpu_to_le32(flags); 7547 return hwrm_req_send_silent(bp, req); 7548 } 7549 7550 static int bnxt_hwrm_check_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7551 { 7552 if (bp->hwrm_spec_code < 0x10801) 7553 return 0; 7554 7555 if (BNXT_PF(bp)) 7556 return bnxt_hwrm_check_pf_rings(bp, hwr); 7557 7558 return bnxt_hwrm_check_vf_rings(bp, hwr); 7559 } 7560 7561 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 7562 { 7563 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 7564 struct hwrm_ring_aggint_qcaps_output *resp; 7565 struct hwrm_ring_aggint_qcaps_input *req; 7566 int rc; 7567 7568 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 7569 coal_cap->num_cmpl_dma_aggr_max = 63; 7570 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 7571 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 7572 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 7573 coal_cap->int_lat_tmr_min_max = 65535; 7574 coal_cap->int_lat_tmr_max_max = 65535; 7575 coal_cap->num_cmpl_aggr_int_max = 65535; 7576 coal_cap->timer_units = 80; 7577 7578 if (bp->hwrm_spec_code < 0x10902) 7579 return; 7580 7581 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 7582 return; 7583 7584 resp = hwrm_req_hold(bp, req); 7585 rc = hwrm_req_send_silent(bp, req); 7586 if (!rc) { 7587 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 7588 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 7589 coal_cap->num_cmpl_dma_aggr_max = 7590 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 7591 coal_cap->num_cmpl_dma_aggr_during_int_max = 7592 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 7593 coal_cap->cmpl_aggr_dma_tmr_max = 7594 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 7595 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 7596 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 7597 coal_cap->int_lat_tmr_min_max = 7598 le16_to_cpu(resp->int_lat_tmr_min_max); 7599 coal_cap->int_lat_tmr_max_max = 7600 le16_to_cpu(resp->int_lat_tmr_max_max); 7601 coal_cap->num_cmpl_aggr_int_max = 7602 le16_to_cpu(resp->num_cmpl_aggr_int_max); 7603 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 7604 } 7605 hwrm_req_drop(bp, req); 7606 } 7607 7608 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 7609 { 7610 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 7611 7612 return usec * 1000 / coal_cap->timer_units; 7613 } 7614 7615 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 7616 struct bnxt_coal *hw_coal, 7617 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 7618 { 7619 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 7620 u16 val, tmr, max, flags = hw_coal->flags; 7621 u32 cmpl_params = coal_cap->cmpl_params; 7622 7623 max = hw_coal->bufs_per_record * 128; 7624 if (hw_coal->budget) 7625 max = hw_coal->bufs_per_record * hw_coal->budget; 7626 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 7627 7628 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 7629 req->num_cmpl_aggr_int = cpu_to_le16(val); 7630 7631 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 7632 req->num_cmpl_dma_aggr = cpu_to_le16(val); 7633 7634 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 7635 coal_cap->num_cmpl_dma_aggr_during_int_max); 7636 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 7637 7638 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 7639 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 7640 req->int_lat_tmr_max = cpu_to_le16(tmr); 7641 7642 /* min timer set to 1/2 of interrupt timer */ 7643 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 7644 val = tmr / 2; 7645 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 7646 req->int_lat_tmr_min = cpu_to_le16(val); 7647 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 7648 } 7649 7650 /* buf timer set to 1/4 of interrupt timer */ 7651 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 7652 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 7653 7654 if (cmpl_params & 7655 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 7656 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 7657 val = clamp_t(u16, tmr, 1, 7658 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 7659 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 7660 req->enables |= 7661 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 7662 } 7663 7664 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 7665 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 7666 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 7667 req->flags = cpu_to_le16(flags); 7668 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 7669 } 7670 7671 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 7672 struct bnxt_coal *hw_coal) 7673 { 7674 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 7675 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7676 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 7677 u32 nq_params = coal_cap->nq_params; 7678 u16 tmr; 7679 int rc; 7680 7681 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 7682 return 0; 7683 7684 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 7685 if (rc) 7686 return rc; 7687 7688 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 7689 req->flags = 7690 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 7691 7692 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 7693 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 7694 req->int_lat_tmr_min = cpu_to_le16(tmr); 7695 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 7696 return hwrm_req_send(bp, req); 7697 } 7698 7699 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 7700 { 7701 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 7702 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7703 struct bnxt_coal coal; 7704 int rc; 7705 7706 /* Tick values in micro seconds. 7707 * 1 coal_buf x bufs_per_record = 1 completion record. 7708 */ 7709 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 7710 7711 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 7712 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 7713 7714 if (!bnapi->rx_ring) 7715 return -ENODEV; 7716 7717 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 7718 if (rc) 7719 return rc; 7720 7721 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 7722 7723 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 7724 7725 return hwrm_req_send(bp, req_rx); 7726 } 7727 7728 static int 7729 bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi, 7730 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 7731 { 7732 u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 7733 7734 req->ring_id = cpu_to_le16(ring_id); 7735 return hwrm_req_send(bp, req); 7736 } 7737 7738 static int 7739 bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi, 7740 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 7741 { 7742 struct bnxt_tx_ring_info *txr; 7743 int i, rc; 7744 7745 bnxt_for_each_napi_tx(i, bnapi, txr) { 7746 u16 ring_id; 7747 7748 ring_id = bnxt_cp_ring_for_tx(bp, txr); 7749 req->ring_id = cpu_to_le16(ring_id); 7750 rc = hwrm_req_send(bp, req); 7751 if (rc) 7752 return rc; 7753 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7754 return 0; 7755 } 7756 return 0; 7757 } 7758 7759 int bnxt_hwrm_set_coal(struct bnxt *bp) 7760 { 7761 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx; 7762 int i, rc; 7763 7764 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 7765 if (rc) 7766 return rc; 7767 7768 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 7769 if (rc) { 7770 hwrm_req_drop(bp, req_rx); 7771 return rc; 7772 } 7773 7774 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 7775 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 7776 7777 hwrm_req_hold(bp, req_rx); 7778 hwrm_req_hold(bp, req_tx); 7779 for (i = 0; i < bp->cp_nr_rings; i++) { 7780 struct bnxt_napi *bnapi = bp->bnapi[i]; 7781 struct bnxt_coal *hw_coal; 7782 7783 if (!bnapi->rx_ring) 7784 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx); 7785 else 7786 rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx); 7787 if (rc) 7788 break; 7789 7790 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7791 continue; 7792 7793 if (bnapi->rx_ring && bnapi->tx_ring[0]) { 7794 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx); 7795 if (rc) 7796 break; 7797 } 7798 if (bnapi->rx_ring) 7799 hw_coal = &bp->rx_coal; 7800 else 7801 hw_coal = &bp->tx_coal; 7802 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 7803 } 7804 hwrm_req_drop(bp, req_rx); 7805 hwrm_req_drop(bp, req_tx); 7806 return rc; 7807 } 7808 7809 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 7810 { 7811 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 7812 struct hwrm_stat_ctx_free_input *req; 7813 int i; 7814 7815 if (!bp->bnapi) 7816 return; 7817 7818 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 7819 return; 7820 7821 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 7822 return; 7823 if (BNXT_FW_MAJ(bp) <= 20) { 7824 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 7825 hwrm_req_drop(bp, req); 7826 return; 7827 } 7828 hwrm_req_hold(bp, req0); 7829 } 7830 hwrm_req_hold(bp, req); 7831 for (i = 0; i < bp->cp_nr_rings; i++) { 7832 struct bnxt_napi *bnapi = bp->bnapi[i]; 7833 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7834 7835 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 7836 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 7837 if (req0) { 7838 req0->stat_ctx_id = req->stat_ctx_id; 7839 hwrm_req_send(bp, req0); 7840 } 7841 hwrm_req_send(bp, req); 7842 7843 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 7844 } 7845 } 7846 hwrm_req_drop(bp, req); 7847 if (req0) 7848 hwrm_req_drop(bp, req0); 7849 } 7850 7851 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 7852 { 7853 struct hwrm_stat_ctx_alloc_output *resp; 7854 struct hwrm_stat_ctx_alloc_input *req; 7855 int rc, i; 7856 7857 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 7858 return 0; 7859 7860 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 7861 if (rc) 7862 return rc; 7863 7864 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 7865 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 7866 7867 resp = hwrm_req_hold(bp, req); 7868 for (i = 0; i < bp->cp_nr_rings; i++) { 7869 struct bnxt_napi *bnapi = bp->bnapi[i]; 7870 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7871 7872 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 7873 7874 rc = hwrm_req_send(bp, req); 7875 if (rc) 7876 break; 7877 7878 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 7879 7880 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 7881 } 7882 hwrm_req_drop(bp, req); 7883 return rc; 7884 } 7885 7886 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 7887 { 7888 struct hwrm_func_qcfg_output *resp; 7889 struct hwrm_func_qcfg_input *req; 7890 u16 flags; 7891 int rc; 7892 7893 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 7894 if (rc) 7895 return rc; 7896 7897 req->fid = cpu_to_le16(0xffff); 7898 resp = hwrm_req_hold(bp, req); 7899 rc = hwrm_req_send(bp, req); 7900 if (rc) 7901 goto func_qcfg_exit; 7902 7903 #ifdef CONFIG_BNXT_SRIOV 7904 if (BNXT_VF(bp)) { 7905 struct bnxt_vf_info *vf = &bp->vf; 7906 7907 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 7908 } else { 7909 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 7910 } 7911 #endif 7912 flags = le16_to_cpu(resp->flags); 7913 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 7914 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 7915 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 7916 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 7917 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 7918 } 7919 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 7920 bp->flags |= BNXT_FLAG_MULTI_HOST; 7921 7922 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 7923 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 7924 7925 switch (resp->port_partition_type) { 7926 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 7927 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 7928 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 7929 bp->port_partition_type = resp->port_partition_type; 7930 break; 7931 } 7932 if (bp->hwrm_spec_code < 0x10707 || 7933 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 7934 bp->br_mode = BRIDGE_MODE_VEB; 7935 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 7936 bp->br_mode = BRIDGE_MODE_VEPA; 7937 else 7938 bp->br_mode = BRIDGE_MODE_UNDEF; 7939 7940 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 7941 if (!bp->max_mtu) 7942 bp->max_mtu = BNXT_MAX_MTU; 7943 7944 if (bp->db_size) 7945 goto func_qcfg_exit; 7946 7947 bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024; 7948 if (BNXT_CHIP_P5(bp)) { 7949 if (BNXT_PF(bp)) 7950 bp->db_offset = DB_PF_OFFSET_P5; 7951 else 7952 bp->db_offset = DB_VF_OFFSET_P5; 7953 } 7954 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 7955 1024); 7956 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 7957 bp->db_size <= bp->db_offset) 7958 bp->db_size = pci_resource_len(bp->pdev, 2); 7959 7960 func_qcfg_exit: 7961 hwrm_req_drop(bp, req); 7962 return rc; 7963 } 7964 7965 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm, 7966 u8 init_val, u8 init_offset, 7967 bool init_mask_set) 7968 { 7969 ctxm->init_value = init_val; 7970 ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET; 7971 if (init_mask_set) 7972 ctxm->init_offset = init_offset * 4; 7973 else 7974 ctxm->init_value = 0; 7975 } 7976 7977 static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max) 7978 { 7979 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7980 u16 type; 7981 7982 for (type = 0; type < ctx_max; type++) { 7983 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 7984 int n = 1; 7985 7986 if (!ctxm->max_entries) 7987 continue; 7988 7989 if (ctxm->instance_bmap) 7990 n = hweight32(ctxm->instance_bmap); 7991 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL); 7992 if (!ctxm->pg_info) 7993 return -ENOMEM; 7994 } 7995 return 0; 7996 } 7997 7998 #define BNXT_CTX_INIT_VALID(flags) \ 7999 (!!((flags) & \ 8000 FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT)) 8001 8002 static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) 8003 { 8004 struct hwrm_func_backing_store_qcaps_v2_output *resp; 8005 struct hwrm_func_backing_store_qcaps_v2_input *req; 8006 struct bnxt_ctx_mem_info *ctx; 8007 u16 type; 8008 int rc; 8009 8010 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); 8011 if (rc) 8012 return rc; 8013 8014 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 8015 if (!ctx) 8016 return -ENOMEM; 8017 bp->ctx = ctx; 8018 8019 resp = hwrm_req_hold(bp, req); 8020 8021 for (type = 0; type < BNXT_CTX_V2_MAX; ) { 8022 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 8023 u8 init_val, init_off, i; 8024 __le32 *p; 8025 u32 flags; 8026 8027 req->type = cpu_to_le16(type); 8028 rc = hwrm_req_send(bp, req); 8029 if (rc) 8030 goto ctx_done; 8031 flags = le32_to_cpu(resp->flags); 8032 type = le16_to_cpu(resp->next_valid_type); 8033 if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID)) 8034 continue; 8035 8036 ctxm->type = le16_to_cpu(resp->type); 8037 ctxm->entry_size = le16_to_cpu(resp->entry_size); 8038 ctxm->flags = flags; 8039 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); 8040 ctxm->entry_multiple = resp->entry_multiple; 8041 ctxm->max_entries = le32_to_cpu(resp->max_num_entries); 8042 ctxm->min_entries = le32_to_cpu(resp->min_num_entries); 8043 init_val = resp->ctx_init_value; 8044 init_off = resp->ctx_init_offset; 8045 bnxt_init_ctx_initializer(ctxm, init_val, init_off, 8046 BNXT_CTX_INIT_VALID(flags)); 8047 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt, 8048 BNXT_MAX_SPLIT_ENTRY); 8049 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt; 8050 i++, p++) 8051 ctxm->split[i] = le32_to_cpu(*p); 8052 } 8053 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX); 8054 8055 ctx_done: 8056 hwrm_req_drop(bp, req); 8057 return rc; 8058 } 8059 8060 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 8061 { 8062 struct hwrm_func_backing_store_qcaps_output *resp; 8063 struct hwrm_func_backing_store_qcaps_input *req; 8064 int rc; 8065 8066 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 8067 return 0; 8068 8069 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) 8070 return bnxt_hwrm_func_backing_store_qcaps_v2(bp); 8071 8072 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 8073 if (rc) 8074 return rc; 8075 8076 resp = hwrm_req_hold(bp, req); 8077 rc = hwrm_req_send_silent(bp, req); 8078 if (!rc) { 8079 struct bnxt_ctx_mem_type *ctxm; 8080 struct bnxt_ctx_mem_info *ctx; 8081 u8 init_val, init_idx = 0; 8082 u16 init_mask; 8083 8084 ctx = bp->ctx; 8085 if (!ctx) { 8086 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 8087 if (!ctx) { 8088 rc = -ENOMEM; 8089 goto ctx_err; 8090 } 8091 bp->ctx = ctx; 8092 } 8093 init_val = resp->ctx_kind_initializer; 8094 init_mask = le16_to_cpu(resp->ctx_init_mask); 8095 8096 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8097 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries); 8098 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 8099 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 8100 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries); 8101 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size); 8102 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset, 8103 (init_mask & (1 << init_idx++)) != 0); 8104 8105 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8106 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 8107 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries); 8108 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size); 8109 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset, 8110 (init_mask & (1 << init_idx++)) != 0); 8111 8112 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 8113 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 8114 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries); 8115 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size); 8116 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset, 8117 (init_mask & (1 << init_idx++)) != 0); 8118 8119 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 8120 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries); 8121 ctxm->max_entries = ctxm->vnic_entries + 8122 le16_to_cpu(resp->vnic_max_ring_table_entries); 8123 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size); 8124 bnxt_init_ctx_initializer(ctxm, init_val, 8125 resp->vnic_init_offset, 8126 (init_mask & (1 << init_idx++)) != 0); 8127 8128 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 8129 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries); 8130 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size); 8131 bnxt_init_ctx_initializer(ctxm, init_val, 8132 resp->stat_init_offset, 8133 (init_mask & (1 << init_idx++)) != 0); 8134 8135 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 8136 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size); 8137 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring); 8138 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring); 8139 ctxm->entry_multiple = resp->tqm_entries_multiple; 8140 if (!ctxm->entry_multiple) 8141 ctxm->entry_multiple = 1; 8142 8143 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm)); 8144 8145 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 8146 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries); 8147 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size); 8148 ctxm->mrav_num_entries_units = 8149 le16_to_cpu(resp->mrav_num_entries_units); 8150 bnxt_init_ctx_initializer(ctxm, init_val, 8151 resp->mrav_init_offset, 8152 (init_mask & (1 << init_idx++)) != 0); 8153 8154 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 8155 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size); 8156 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries); 8157 8158 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 8159 if (!ctx->tqm_fp_rings_count) 8160 ctx->tqm_fp_rings_count = bp->max_q; 8161 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 8162 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 8163 8164 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; 8165 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm)); 8166 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1; 8167 8168 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX); 8169 } else { 8170 rc = 0; 8171 } 8172 ctx_err: 8173 hwrm_req_drop(bp, req); 8174 return rc; 8175 } 8176 8177 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 8178 __le64 *pg_dir) 8179 { 8180 if (!rmem->nr_pages) 8181 return; 8182 8183 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 8184 if (rmem->depth >= 1) { 8185 if (rmem->depth == 2) 8186 *pg_attr |= 2; 8187 else 8188 *pg_attr |= 1; 8189 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 8190 } else { 8191 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 8192 } 8193 } 8194 8195 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 8196 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 8197 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 8198 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 8199 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 8200 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 8201 8202 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 8203 { 8204 struct hwrm_func_backing_store_cfg_input *req; 8205 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8206 struct bnxt_ctx_pg_info *ctx_pg; 8207 struct bnxt_ctx_mem_type *ctxm; 8208 void **__req = (void **)&req; 8209 u32 req_len = sizeof(*req); 8210 __le32 *num_entries; 8211 __le64 *pg_dir; 8212 u32 flags = 0; 8213 u8 *pg_attr; 8214 u32 ena; 8215 int rc; 8216 int i; 8217 8218 if (!ctx) 8219 return 0; 8220 8221 if (req_len > bp->hwrm_max_ext_req_len) 8222 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 8223 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 8224 if (rc) 8225 return rc; 8226 8227 req->enables = cpu_to_le32(enables); 8228 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 8229 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8230 ctx_pg = ctxm->pg_info; 8231 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 8232 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries); 8233 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries); 8234 req->qp_entry_size = cpu_to_le16(ctxm->entry_size); 8235 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8236 &req->qpc_pg_size_qpc_lvl, 8237 &req->qpc_page_dir); 8238 8239 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD) 8240 req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries); 8241 } 8242 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 8243 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8244 ctx_pg = ctxm->pg_info; 8245 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 8246 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries); 8247 req->srq_entry_size = cpu_to_le16(ctxm->entry_size); 8248 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8249 &req->srq_pg_size_srq_lvl, 8250 &req->srq_page_dir); 8251 } 8252 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 8253 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 8254 ctx_pg = ctxm->pg_info; 8255 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 8256 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries); 8257 req->cq_entry_size = cpu_to_le16(ctxm->entry_size); 8258 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8259 &req->cq_pg_size_cq_lvl, 8260 &req->cq_page_dir); 8261 } 8262 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 8263 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 8264 ctx_pg = ctxm->pg_info; 8265 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries); 8266 req->vnic_num_ring_table_entries = 8267 cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries); 8268 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size); 8269 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8270 &req->vnic_pg_size_vnic_lvl, 8271 &req->vnic_page_dir); 8272 } 8273 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 8274 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 8275 ctx_pg = ctxm->pg_info; 8276 req->stat_num_entries = cpu_to_le32(ctxm->max_entries); 8277 req->stat_entry_size = cpu_to_le16(ctxm->entry_size); 8278 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8279 &req->stat_pg_size_stat_lvl, 8280 &req->stat_page_dir); 8281 } 8282 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 8283 u32 units; 8284 8285 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 8286 ctx_pg = ctxm->pg_info; 8287 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 8288 units = ctxm->mrav_num_entries_units; 8289 if (units) { 8290 u32 num_mr, num_ah = ctxm->mrav_av_entries; 8291 u32 entries; 8292 8293 num_mr = ctx_pg->entries - num_ah; 8294 entries = ((num_mr / units) << 16) | (num_ah / units); 8295 req->mrav_num_entries = cpu_to_le32(entries); 8296 flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 8297 } 8298 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size); 8299 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8300 &req->mrav_pg_size_mrav_lvl, 8301 &req->mrav_page_dir); 8302 } 8303 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 8304 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 8305 ctx_pg = ctxm->pg_info; 8306 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 8307 req->tim_entry_size = cpu_to_le16(ctxm->entry_size); 8308 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8309 &req->tim_pg_size_tim_lvl, 8310 &req->tim_page_dir); 8311 } 8312 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 8313 for (i = 0, num_entries = &req->tqm_sp_num_entries, 8314 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 8315 pg_dir = &req->tqm_sp_page_dir, 8316 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP, 8317 ctx_pg = ctxm->pg_info; 8318 i < BNXT_MAX_TQM_RINGS; 8319 ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i], 8320 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 8321 if (!(enables & ena)) 8322 continue; 8323 8324 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size); 8325 *num_entries = cpu_to_le32(ctx_pg->entries); 8326 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 8327 } 8328 req->flags = cpu_to_le32(flags); 8329 return hwrm_req_send(bp, req); 8330 } 8331 8332 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 8333 struct bnxt_ctx_pg_info *ctx_pg) 8334 { 8335 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 8336 8337 rmem->page_size = BNXT_PAGE_SIZE; 8338 rmem->pg_arr = ctx_pg->ctx_pg_arr; 8339 rmem->dma_arr = ctx_pg->ctx_dma_arr; 8340 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 8341 if (rmem->depth >= 1) 8342 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 8343 return bnxt_alloc_ring(bp, rmem); 8344 } 8345 8346 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 8347 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 8348 u8 depth, struct bnxt_ctx_mem_type *ctxm) 8349 { 8350 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 8351 int rc; 8352 8353 if (!mem_size) 8354 return -EINVAL; 8355 8356 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 8357 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 8358 ctx_pg->nr_pages = 0; 8359 return -EINVAL; 8360 } 8361 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 8362 int nr_tbls, i; 8363 8364 rmem->depth = 2; 8365 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 8366 GFP_KERNEL); 8367 if (!ctx_pg->ctx_pg_tbl) 8368 return -ENOMEM; 8369 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 8370 rmem->nr_pages = nr_tbls; 8371 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 8372 if (rc) 8373 return rc; 8374 for (i = 0; i < nr_tbls; i++) { 8375 struct bnxt_ctx_pg_info *pg_tbl; 8376 8377 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 8378 if (!pg_tbl) 8379 return -ENOMEM; 8380 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 8381 rmem = &pg_tbl->ring_mem; 8382 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 8383 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 8384 rmem->depth = 1; 8385 rmem->nr_pages = MAX_CTX_PAGES; 8386 rmem->ctx_mem = ctxm; 8387 if (i == (nr_tbls - 1)) { 8388 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 8389 8390 if (rem) 8391 rmem->nr_pages = rem; 8392 } 8393 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 8394 if (rc) 8395 break; 8396 } 8397 } else { 8398 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 8399 if (rmem->nr_pages > 1 || depth) 8400 rmem->depth = 1; 8401 rmem->ctx_mem = ctxm; 8402 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 8403 } 8404 return rc; 8405 } 8406 8407 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 8408 struct bnxt_ctx_pg_info *ctx_pg) 8409 { 8410 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 8411 8412 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 8413 ctx_pg->ctx_pg_tbl) { 8414 int i, nr_tbls = rmem->nr_pages; 8415 8416 for (i = 0; i < nr_tbls; i++) { 8417 struct bnxt_ctx_pg_info *pg_tbl; 8418 struct bnxt_ring_mem_info *rmem2; 8419 8420 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 8421 if (!pg_tbl) 8422 continue; 8423 rmem2 = &pg_tbl->ring_mem; 8424 bnxt_free_ring(bp, rmem2); 8425 ctx_pg->ctx_pg_arr[i] = NULL; 8426 kfree(pg_tbl); 8427 ctx_pg->ctx_pg_tbl[i] = NULL; 8428 } 8429 kfree(ctx_pg->ctx_pg_tbl); 8430 ctx_pg->ctx_pg_tbl = NULL; 8431 } 8432 bnxt_free_ring(bp, rmem); 8433 ctx_pg->nr_pages = 0; 8434 } 8435 8436 static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp, 8437 struct bnxt_ctx_mem_type *ctxm, u32 entries, 8438 u8 pg_lvl) 8439 { 8440 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; 8441 int i, rc = 0, n = 1; 8442 u32 mem_size; 8443 8444 if (!ctxm->entry_size || !ctx_pg) 8445 return -EINVAL; 8446 if (ctxm->instance_bmap) 8447 n = hweight32(ctxm->instance_bmap); 8448 if (ctxm->entry_multiple) 8449 entries = roundup(entries, ctxm->entry_multiple); 8450 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries); 8451 mem_size = entries * ctxm->entry_size; 8452 for (i = 0; i < n && !rc; i++) { 8453 ctx_pg[i].entries = entries; 8454 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl, 8455 ctxm->init_value ? ctxm : NULL); 8456 } 8457 return rc; 8458 } 8459 8460 static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp, 8461 struct bnxt_ctx_mem_type *ctxm, 8462 bool last) 8463 { 8464 struct hwrm_func_backing_store_cfg_v2_input *req; 8465 u32 instance_bmap = ctxm->instance_bmap; 8466 int i, j, rc = 0, n = 1; 8467 __le32 *p; 8468 8469 if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info) 8470 return 0; 8471 8472 if (instance_bmap) 8473 n = hweight32(ctxm->instance_bmap); 8474 else 8475 instance_bmap = 1; 8476 8477 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2); 8478 if (rc) 8479 return rc; 8480 hwrm_req_hold(bp, req); 8481 req->type = cpu_to_le16(ctxm->type); 8482 req->entry_size = cpu_to_le16(ctxm->entry_size); 8483 req->subtype_valid_cnt = ctxm->split_entry_cnt; 8484 for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++) 8485 p[i] = cpu_to_le32(ctxm->split[i]); 8486 for (i = 0, j = 0; j < n && !rc; i++) { 8487 struct bnxt_ctx_pg_info *ctx_pg; 8488 8489 if (!(instance_bmap & (1 << i))) 8490 continue; 8491 req->instance = cpu_to_le16(i); 8492 ctx_pg = &ctxm->pg_info[j++]; 8493 if (!ctx_pg->entries) 8494 continue; 8495 req->num_entries = cpu_to_le32(ctx_pg->entries); 8496 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8497 &req->page_size_pbl_level, 8498 &req->page_dir); 8499 if (last && j == n) 8500 req->flags = 8501 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE); 8502 rc = hwrm_req_send(bp, req); 8503 } 8504 hwrm_req_drop(bp, req); 8505 return rc; 8506 } 8507 8508 static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena) 8509 { 8510 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8511 struct bnxt_ctx_mem_type *ctxm; 8512 u16 last_type; 8513 int rc = 0; 8514 u16 type; 8515 8516 if (!ena) 8517 return 0; 8518 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) 8519 last_type = BNXT_CTX_MAX - 1; 8520 else 8521 last_type = BNXT_CTX_L2_MAX - 1; 8522 ctx->ctx_arr[last_type].last = 1; 8523 8524 for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) { 8525 ctxm = &ctx->ctx_arr[type]; 8526 8527 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last); 8528 if (rc) 8529 return rc; 8530 } 8531 return 0; 8532 } 8533 8534 void bnxt_free_ctx_mem(struct bnxt *bp) 8535 { 8536 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8537 u16 type; 8538 8539 if (!ctx) 8540 return; 8541 8542 for (type = 0; type < BNXT_CTX_V2_MAX; type++) { 8543 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 8544 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; 8545 int i, n = 1; 8546 8547 if (!ctx_pg) 8548 continue; 8549 if (ctxm->instance_bmap) 8550 n = hweight32(ctxm->instance_bmap); 8551 for (i = 0; i < n; i++) 8552 bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]); 8553 8554 kfree(ctx_pg); 8555 ctxm->pg_info = NULL; 8556 } 8557 8558 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 8559 kfree(ctx); 8560 bp->ctx = NULL; 8561 } 8562 8563 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 8564 { 8565 struct bnxt_ctx_mem_type *ctxm; 8566 struct bnxt_ctx_mem_info *ctx; 8567 u32 l2_qps, qp1_qps, max_qps; 8568 u32 ena, entries_sp, entries; 8569 u32 srqs, max_srqs, min; 8570 u32 num_mr, num_ah; 8571 u32 extra_srqs = 0; 8572 u32 extra_qps = 0; 8573 u32 fast_qpmd_qps; 8574 u8 pg_lvl = 1; 8575 int i, rc; 8576 8577 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 8578 if (rc) { 8579 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 8580 rc); 8581 return rc; 8582 } 8583 ctx = bp->ctx; 8584 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 8585 return 0; 8586 8587 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8588 l2_qps = ctxm->qp_l2_entries; 8589 qp1_qps = ctxm->qp_qp1_entries; 8590 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries; 8591 max_qps = ctxm->max_entries; 8592 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8593 srqs = ctxm->srq_l2_entries; 8594 max_srqs = ctxm->max_entries; 8595 ena = 0; 8596 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 8597 pg_lvl = 2; 8598 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps); 8599 /* allocate extra qps if fw supports RoCE fast qp destroy feature */ 8600 extra_qps += fast_qpmd_qps; 8601 extra_srqs = min_t(u32, 8192, max_srqs - srqs); 8602 if (fast_qpmd_qps) 8603 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD; 8604 } 8605 8606 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8607 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 8608 pg_lvl); 8609 if (rc) 8610 return rc; 8611 8612 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8613 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl); 8614 if (rc) 8615 return rc; 8616 8617 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 8618 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries + 8619 extra_qps * 2, pg_lvl); 8620 if (rc) 8621 return rc; 8622 8623 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 8624 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1); 8625 if (rc) 8626 return rc; 8627 8628 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 8629 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1); 8630 if (rc) 8631 return rc; 8632 8633 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 8634 goto skip_rdma; 8635 8636 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 8637 /* 128K extra is needed to accommodate static AH context 8638 * allocation by f/w. 8639 */ 8640 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); 8641 num_ah = min_t(u32, num_mr, 1024 * 128); 8642 ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1; 8643 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah) 8644 ctxm->mrav_av_entries = num_ah; 8645 8646 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2); 8647 if (rc) 8648 return rc; 8649 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 8650 8651 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 8652 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1); 8653 if (rc) 8654 return rc; 8655 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 8656 8657 skip_rdma: 8658 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 8659 min = ctxm->min_entries; 8660 entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps + 8661 2 * (extra_qps + qp1_qps) + min; 8662 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2); 8663 if (rc) 8664 return rc; 8665 8666 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; 8667 entries = l2_qps + 2 * (extra_qps + qp1_qps); 8668 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2); 8669 if (rc) 8670 return rc; 8671 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 8672 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 8673 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 8674 8675 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) 8676 rc = bnxt_backing_store_cfg_v2(bp, ena); 8677 else 8678 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 8679 if (rc) { 8680 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 8681 rc); 8682 return rc; 8683 } 8684 ctx->flags |= BNXT_CTX_FLAG_INITED; 8685 return 0; 8686 } 8687 8688 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 8689 { 8690 struct hwrm_func_resource_qcaps_output *resp; 8691 struct hwrm_func_resource_qcaps_input *req; 8692 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8693 int rc; 8694 8695 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 8696 if (rc) 8697 return rc; 8698 8699 req->fid = cpu_to_le16(0xffff); 8700 resp = hwrm_req_hold(bp, req); 8701 rc = hwrm_req_send_silent(bp, req); 8702 if (rc) 8703 goto hwrm_func_resc_qcaps_exit; 8704 8705 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 8706 if (!all) 8707 goto hwrm_func_resc_qcaps_exit; 8708 8709 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 8710 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 8711 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 8712 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 8713 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 8714 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 8715 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 8716 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 8717 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 8718 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 8719 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 8720 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 8721 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 8722 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 8723 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 8724 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 8725 8726 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 8727 u16 max_msix = le16_to_cpu(resp->max_msix); 8728 8729 hw_resc->max_nqs = max_msix; 8730 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 8731 } 8732 8733 if (BNXT_PF(bp)) { 8734 struct bnxt_pf_info *pf = &bp->pf; 8735 8736 pf->vf_resv_strategy = 8737 le16_to_cpu(resp->vf_reservation_strategy); 8738 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 8739 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 8740 } 8741 hwrm_func_resc_qcaps_exit: 8742 hwrm_req_drop(bp, req); 8743 return rc; 8744 } 8745 8746 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 8747 { 8748 struct hwrm_port_mac_ptp_qcfg_output *resp; 8749 struct hwrm_port_mac_ptp_qcfg_input *req; 8750 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 8751 bool phc_cfg; 8752 u8 flags; 8753 int rc; 8754 8755 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5(bp)) { 8756 rc = -ENODEV; 8757 goto no_ptp; 8758 } 8759 8760 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 8761 if (rc) 8762 goto no_ptp; 8763 8764 req->port_id = cpu_to_le16(bp->pf.port_id); 8765 resp = hwrm_req_hold(bp, req); 8766 rc = hwrm_req_send(bp, req); 8767 if (rc) 8768 goto exit; 8769 8770 flags = resp->flags; 8771 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 8772 rc = -ENODEV; 8773 goto exit; 8774 } 8775 if (!ptp) { 8776 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 8777 if (!ptp) { 8778 rc = -ENOMEM; 8779 goto exit; 8780 } 8781 ptp->bp = bp; 8782 bp->ptp_cfg = ptp; 8783 } 8784 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { 8785 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 8786 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 8787 } else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 8788 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 8789 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 8790 } else { 8791 rc = -ENODEV; 8792 goto exit; 8793 } 8794 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; 8795 rc = bnxt_ptp_init(bp, phc_cfg); 8796 if (rc) 8797 netdev_warn(bp->dev, "PTP initialization failed.\n"); 8798 exit: 8799 hwrm_req_drop(bp, req); 8800 if (!rc) 8801 return 0; 8802 8803 no_ptp: 8804 bnxt_ptp_clear(bp); 8805 kfree(ptp); 8806 bp->ptp_cfg = NULL; 8807 return rc; 8808 } 8809 8810 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 8811 { 8812 struct hwrm_func_qcaps_output *resp; 8813 struct hwrm_func_qcaps_input *req; 8814 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8815 u32 flags, flags_ext, flags_ext2; 8816 int rc; 8817 8818 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 8819 if (rc) 8820 return rc; 8821 8822 req->fid = cpu_to_le16(0xffff); 8823 resp = hwrm_req_hold(bp, req); 8824 rc = hwrm_req_send(bp, req); 8825 if (rc) 8826 goto hwrm_func_qcaps_exit; 8827 8828 flags = le32_to_cpu(resp->flags); 8829 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 8830 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 8831 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 8832 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 8833 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 8834 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 8835 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 8836 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 8837 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 8838 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 8839 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 8840 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 8841 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 8842 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 8843 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 8844 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 8845 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 8846 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 8847 8848 flags_ext = le32_to_cpu(resp->flags_ext); 8849 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 8850 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 8851 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 8852 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 8853 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) 8854 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; 8855 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 8856 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 8857 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 8858 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 8859 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED) 8860 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2; 8861 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP) 8862 bp->flags |= BNXT_FLAG_TX_COAL_CMPL; 8863 8864 flags_ext2 = le32_to_cpu(resp->flags_ext2); 8865 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED) 8866 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS; 8867 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED) 8868 bp->flags |= BNXT_FLAG_UDP_GSO_CAP; 8869 8870 bp->tx_push_thresh = 0; 8871 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 8872 BNXT_FW_MAJ(bp) > 217) 8873 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 8874 8875 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 8876 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 8877 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 8878 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 8879 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 8880 if (!hw_resc->max_hw_ring_grps) 8881 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 8882 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 8883 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 8884 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 8885 8886 hw_resc->max_encap_records = le32_to_cpu(resp->max_encap_records); 8887 hw_resc->max_decap_records = le32_to_cpu(resp->max_decap_records); 8888 hw_resc->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 8889 hw_resc->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 8890 hw_resc->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 8891 hw_resc->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 8892 8893 if (BNXT_PF(bp)) { 8894 struct bnxt_pf_info *pf = &bp->pf; 8895 8896 pf->fw_fid = le16_to_cpu(resp->fid); 8897 pf->port_id = le16_to_cpu(resp->port_id); 8898 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 8899 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 8900 pf->max_vfs = le16_to_cpu(resp->max_vfs); 8901 bp->flags &= ~BNXT_FLAG_WOL_CAP; 8902 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 8903 bp->flags |= BNXT_FLAG_WOL_CAP; 8904 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 8905 bp->fw_cap |= BNXT_FW_CAP_PTP; 8906 } else { 8907 bnxt_ptp_clear(bp); 8908 kfree(bp->ptp_cfg); 8909 bp->ptp_cfg = NULL; 8910 } 8911 } else { 8912 #ifdef CONFIG_BNXT_SRIOV 8913 struct bnxt_vf_info *vf = &bp->vf; 8914 8915 vf->fw_fid = le16_to_cpu(resp->fid); 8916 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 8917 #endif 8918 } 8919 8920 hwrm_func_qcaps_exit: 8921 hwrm_req_drop(bp, req); 8922 return rc; 8923 } 8924 8925 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 8926 { 8927 struct hwrm_dbg_qcaps_output *resp; 8928 struct hwrm_dbg_qcaps_input *req; 8929 int rc; 8930 8931 bp->fw_dbg_cap = 0; 8932 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 8933 return; 8934 8935 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 8936 if (rc) 8937 return; 8938 8939 req->fid = cpu_to_le16(0xffff); 8940 resp = hwrm_req_hold(bp, req); 8941 rc = hwrm_req_send(bp, req); 8942 if (rc) 8943 goto hwrm_dbg_qcaps_exit; 8944 8945 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 8946 8947 hwrm_dbg_qcaps_exit: 8948 hwrm_req_drop(bp, req); 8949 } 8950 8951 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 8952 8953 int bnxt_hwrm_func_qcaps(struct bnxt *bp) 8954 { 8955 int rc; 8956 8957 rc = __bnxt_hwrm_func_qcaps(bp); 8958 if (rc) 8959 return rc; 8960 8961 bnxt_hwrm_dbg_qcaps(bp); 8962 8963 rc = bnxt_hwrm_queue_qportcfg(bp); 8964 if (rc) { 8965 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 8966 return rc; 8967 } 8968 if (bp->hwrm_spec_code >= 0x10803) { 8969 rc = bnxt_alloc_ctx_mem(bp); 8970 if (rc) 8971 return rc; 8972 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 8973 if (!rc) 8974 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 8975 } 8976 return 0; 8977 } 8978 8979 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 8980 { 8981 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 8982 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 8983 u32 flags; 8984 int rc; 8985 8986 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 8987 return 0; 8988 8989 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 8990 if (rc) 8991 return rc; 8992 8993 resp = hwrm_req_hold(bp, req); 8994 rc = hwrm_req_send(bp, req); 8995 if (rc) 8996 goto hwrm_cfa_adv_qcaps_exit; 8997 8998 flags = le32_to_cpu(resp->flags); 8999 if (flags & 9000 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 9001 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 9002 9003 if (flags & 9004 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED) 9005 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3; 9006 9007 if (flags & 9008 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED) 9009 bp->fw_cap |= BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO; 9010 9011 hwrm_cfa_adv_qcaps_exit: 9012 hwrm_req_drop(bp, req); 9013 return rc; 9014 } 9015 9016 static int __bnxt_alloc_fw_health(struct bnxt *bp) 9017 { 9018 if (bp->fw_health) 9019 return 0; 9020 9021 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 9022 if (!bp->fw_health) 9023 return -ENOMEM; 9024 9025 mutex_init(&bp->fw_health->lock); 9026 return 0; 9027 } 9028 9029 static int bnxt_alloc_fw_health(struct bnxt *bp) 9030 { 9031 int rc; 9032 9033 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 9034 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 9035 return 0; 9036 9037 rc = __bnxt_alloc_fw_health(bp); 9038 if (rc) { 9039 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 9040 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 9041 return rc; 9042 } 9043 9044 return 0; 9045 } 9046 9047 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 9048 { 9049 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 9050 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 9051 BNXT_FW_HEALTH_WIN_MAP_OFF); 9052 } 9053 9054 static void bnxt_inv_fw_health_reg(struct bnxt *bp) 9055 { 9056 struct bnxt_fw_health *fw_health = bp->fw_health; 9057 u32 reg_type; 9058 9059 if (!fw_health) 9060 return; 9061 9062 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 9063 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 9064 fw_health->status_reliable = false; 9065 9066 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 9067 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 9068 fw_health->resets_reliable = false; 9069 } 9070 9071 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 9072 { 9073 void __iomem *hs; 9074 u32 status_loc; 9075 u32 reg_type; 9076 u32 sig; 9077 9078 if (bp->fw_health) 9079 bp->fw_health->status_reliable = false; 9080 9081 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 9082 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 9083 9084 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 9085 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 9086 if (!bp->chip_num) { 9087 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 9088 bp->chip_num = readl(bp->bar0 + 9089 BNXT_FW_HEALTH_WIN_BASE + 9090 BNXT_GRC_REG_CHIP_NUM); 9091 } 9092 if (!BNXT_CHIP_P5(bp)) 9093 return; 9094 9095 status_loc = BNXT_GRC_REG_STATUS_P5 | 9096 BNXT_FW_HEALTH_REG_TYPE_BAR0; 9097 } else { 9098 status_loc = readl(hs + offsetof(struct hcomm_status, 9099 fw_status_loc)); 9100 } 9101 9102 if (__bnxt_alloc_fw_health(bp)) { 9103 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 9104 return; 9105 } 9106 9107 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 9108 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 9109 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 9110 __bnxt_map_fw_health_reg(bp, status_loc); 9111 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 9112 BNXT_FW_HEALTH_WIN_OFF(status_loc); 9113 } 9114 9115 bp->fw_health->status_reliable = true; 9116 } 9117 9118 static int bnxt_map_fw_health_regs(struct bnxt *bp) 9119 { 9120 struct bnxt_fw_health *fw_health = bp->fw_health; 9121 u32 reg_base = 0xffffffff; 9122 int i; 9123 9124 bp->fw_health->status_reliable = false; 9125 bp->fw_health->resets_reliable = false; 9126 /* Only pre-map the monitoring GRC registers using window 3 */ 9127 for (i = 0; i < 4; i++) { 9128 u32 reg = fw_health->regs[i]; 9129 9130 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 9131 continue; 9132 if (reg_base == 0xffffffff) 9133 reg_base = reg & BNXT_GRC_BASE_MASK; 9134 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 9135 return -ERANGE; 9136 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 9137 } 9138 bp->fw_health->status_reliable = true; 9139 bp->fw_health->resets_reliable = true; 9140 if (reg_base == 0xffffffff) 9141 return 0; 9142 9143 __bnxt_map_fw_health_reg(bp, reg_base); 9144 return 0; 9145 } 9146 9147 static void bnxt_remap_fw_health_regs(struct bnxt *bp) 9148 { 9149 if (!bp->fw_health) 9150 return; 9151 9152 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 9153 bp->fw_health->status_reliable = true; 9154 bp->fw_health->resets_reliable = true; 9155 } else { 9156 bnxt_try_map_fw_health_reg(bp); 9157 } 9158 } 9159 9160 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 9161 { 9162 struct bnxt_fw_health *fw_health = bp->fw_health; 9163 struct hwrm_error_recovery_qcfg_output *resp; 9164 struct hwrm_error_recovery_qcfg_input *req; 9165 int rc, i; 9166 9167 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 9168 return 0; 9169 9170 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 9171 if (rc) 9172 return rc; 9173 9174 resp = hwrm_req_hold(bp, req); 9175 rc = hwrm_req_send(bp, req); 9176 if (rc) 9177 goto err_recovery_out; 9178 fw_health->flags = le32_to_cpu(resp->flags); 9179 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 9180 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 9181 rc = -EINVAL; 9182 goto err_recovery_out; 9183 } 9184 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 9185 fw_health->master_func_wait_dsecs = 9186 le32_to_cpu(resp->master_func_wait_period); 9187 fw_health->normal_func_wait_dsecs = 9188 le32_to_cpu(resp->normal_func_wait_period); 9189 fw_health->post_reset_wait_dsecs = 9190 le32_to_cpu(resp->master_func_wait_period_after_reset); 9191 fw_health->post_reset_max_wait_dsecs = 9192 le32_to_cpu(resp->max_bailout_time_after_reset); 9193 fw_health->regs[BNXT_FW_HEALTH_REG] = 9194 le32_to_cpu(resp->fw_health_status_reg); 9195 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 9196 le32_to_cpu(resp->fw_heartbeat_reg); 9197 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 9198 le32_to_cpu(resp->fw_reset_cnt_reg); 9199 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 9200 le32_to_cpu(resp->reset_inprogress_reg); 9201 fw_health->fw_reset_inprog_reg_mask = 9202 le32_to_cpu(resp->reset_inprogress_reg_mask); 9203 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 9204 if (fw_health->fw_reset_seq_cnt >= 16) { 9205 rc = -EINVAL; 9206 goto err_recovery_out; 9207 } 9208 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 9209 fw_health->fw_reset_seq_regs[i] = 9210 le32_to_cpu(resp->reset_reg[i]); 9211 fw_health->fw_reset_seq_vals[i] = 9212 le32_to_cpu(resp->reset_reg_val[i]); 9213 fw_health->fw_reset_seq_delay_msec[i] = 9214 resp->delay_after_reset[i]; 9215 } 9216 err_recovery_out: 9217 hwrm_req_drop(bp, req); 9218 if (!rc) 9219 rc = bnxt_map_fw_health_regs(bp); 9220 if (rc) 9221 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 9222 return rc; 9223 } 9224 9225 static int bnxt_hwrm_func_reset(struct bnxt *bp) 9226 { 9227 struct hwrm_func_reset_input *req; 9228 int rc; 9229 9230 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 9231 if (rc) 9232 return rc; 9233 9234 req->enables = 0; 9235 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 9236 return hwrm_req_send(bp, req); 9237 } 9238 9239 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 9240 { 9241 struct hwrm_nvm_get_dev_info_output nvm_info; 9242 9243 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 9244 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 9245 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 9246 nvm_info.nvm_cfg_ver_upd); 9247 } 9248 9249 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 9250 { 9251 struct hwrm_queue_qportcfg_output *resp; 9252 struct hwrm_queue_qportcfg_input *req; 9253 u8 i, j, *qptr; 9254 bool no_rdma; 9255 int rc = 0; 9256 9257 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 9258 if (rc) 9259 return rc; 9260 9261 resp = hwrm_req_hold(bp, req); 9262 rc = hwrm_req_send(bp, req); 9263 if (rc) 9264 goto qportcfg_exit; 9265 9266 if (!resp->max_configurable_queues) { 9267 rc = -EINVAL; 9268 goto qportcfg_exit; 9269 } 9270 bp->max_tc = resp->max_configurable_queues; 9271 bp->max_lltc = resp->max_configurable_lossless_queues; 9272 if (bp->max_tc > BNXT_MAX_QUEUE) 9273 bp->max_tc = BNXT_MAX_QUEUE; 9274 9275 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 9276 qptr = &resp->queue_id0; 9277 for (i = 0, j = 0; i < bp->max_tc; i++) { 9278 bp->q_info[j].queue_id = *qptr; 9279 bp->q_ids[i] = *qptr++; 9280 bp->q_info[j].queue_profile = *qptr++; 9281 bp->tc_to_qidx[j] = j; 9282 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 9283 (no_rdma && BNXT_PF(bp))) 9284 j++; 9285 } 9286 bp->max_q = bp->max_tc; 9287 bp->max_tc = max_t(u8, j, 1); 9288 9289 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 9290 bp->max_tc = 1; 9291 9292 if (bp->max_lltc > bp->max_tc) 9293 bp->max_lltc = bp->max_tc; 9294 9295 qportcfg_exit: 9296 hwrm_req_drop(bp, req); 9297 return rc; 9298 } 9299 9300 static int bnxt_hwrm_poll(struct bnxt *bp) 9301 { 9302 struct hwrm_ver_get_input *req; 9303 int rc; 9304 9305 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 9306 if (rc) 9307 return rc; 9308 9309 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 9310 req->hwrm_intf_min = HWRM_VERSION_MINOR; 9311 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 9312 9313 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 9314 rc = hwrm_req_send(bp, req); 9315 return rc; 9316 } 9317 9318 static int bnxt_hwrm_ver_get(struct bnxt *bp) 9319 { 9320 struct hwrm_ver_get_output *resp; 9321 struct hwrm_ver_get_input *req; 9322 u16 fw_maj, fw_min, fw_bld, fw_rsv; 9323 u32 dev_caps_cfg, hwrm_ver; 9324 int rc, len; 9325 9326 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 9327 if (rc) 9328 return rc; 9329 9330 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 9331 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 9332 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 9333 req->hwrm_intf_min = HWRM_VERSION_MINOR; 9334 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 9335 9336 resp = hwrm_req_hold(bp, req); 9337 rc = hwrm_req_send(bp, req); 9338 if (rc) 9339 goto hwrm_ver_get_exit; 9340 9341 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 9342 9343 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 9344 resp->hwrm_intf_min_8b << 8 | 9345 resp->hwrm_intf_upd_8b; 9346 if (resp->hwrm_intf_maj_8b < 1) { 9347 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 9348 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 9349 resp->hwrm_intf_upd_8b); 9350 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 9351 } 9352 9353 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 9354 HWRM_VERSION_UPDATE; 9355 9356 if (bp->hwrm_spec_code > hwrm_ver) 9357 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 9358 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 9359 HWRM_VERSION_UPDATE); 9360 else 9361 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 9362 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 9363 resp->hwrm_intf_upd_8b); 9364 9365 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 9366 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 9367 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 9368 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 9369 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 9370 len = FW_VER_STR_LEN; 9371 } else { 9372 fw_maj = resp->hwrm_fw_maj_8b; 9373 fw_min = resp->hwrm_fw_min_8b; 9374 fw_bld = resp->hwrm_fw_bld_8b; 9375 fw_rsv = resp->hwrm_fw_rsvd_8b; 9376 len = BC_HWRM_STR_LEN; 9377 } 9378 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 9379 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 9380 fw_rsv); 9381 9382 if (strlen(resp->active_pkg_name)) { 9383 int fw_ver_len = strlen(bp->fw_ver_str); 9384 9385 snprintf(bp->fw_ver_str + fw_ver_len, 9386 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 9387 resp->active_pkg_name); 9388 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 9389 } 9390 9391 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 9392 if (!bp->hwrm_cmd_timeout) 9393 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 9394 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 9395 if (!bp->hwrm_cmd_max_timeout) 9396 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 9397 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT) 9398 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n", 9399 bp->hwrm_cmd_max_timeout / 1000); 9400 9401 if (resp->hwrm_intf_maj_8b >= 1) { 9402 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 9403 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 9404 } 9405 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 9406 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 9407 9408 bp->chip_num = le16_to_cpu(resp->chip_num); 9409 bp->chip_rev = resp->chip_rev; 9410 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 9411 !resp->chip_metal) 9412 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 9413 9414 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 9415 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 9416 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 9417 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 9418 9419 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 9420 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 9421 9422 if (dev_caps_cfg & 9423 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 9424 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 9425 9426 if (dev_caps_cfg & 9427 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 9428 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 9429 9430 if (dev_caps_cfg & 9431 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 9432 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 9433 9434 hwrm_ver_get_exit: 9435 hwrm_req_drop(bp, req); 9436 return rc; 9437 } 9438 9439 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 9440 { 9441 struct hwrm_fw_set_time_input *req; 9442 struct tm tm; 9443 time64_t now = ktime_get_real_seconds(); 9444 int rc; 9445 9446 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 9447 bp->hwrm_spec_code < 0x10400) 9448 return -EOPNOTSUPP; 9449 9450 time64_to_tm(now, 0, &tm); 9451 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 9452 if (rc) 9453 return rc; 9454 9455 req->year = cpu_to_le16(1900 + tm.tm_year); 9456 req->month = 1 + tm.tm_mon; 9457 req->day = tm.tm_mday; 9458 req->hour = tm.tm_hour; 9459 req->minute = tm.tm_min; 9460 req->second = tm.tm_sec; 9461 return hwrm_req_send(bp, req); 9462 } 9463 9464 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 9465 { 9466 u64 sw_tmp; 9467 9468 hw &= mask; 9469 sw_tmp = (*sw & ~mask) | hw; 9470 if (hw < (*sw & mask)) 9471 sw_tmp += mask + 1; 9472 WRITE_ONCE(*sw, sw_tmp); 9473 } 9474 9475 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 9476 int count, bool ignore_zero) 9477 { 9478 int i; 9479 9480 for (i = 0; i < count; i++) { 9481 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 9482 9483 if (ignore_zero && !hw) 9484 continue; 9485 9486 if (masks[i] == -1ULL) 9487 sw_stats[i] = hw; 9488 else 9489 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 9490 } 9491 } 9492 9493 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 9494 { 9495 if (!stats->hw_stats) 9496 return; 9497 9498 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 9499 stats->hw_masks, stats->len / 8, false); 9500 } 9501 9502 static void bnxt_accumulate_all_stats(struct bnxt *bp) 9503 { 9504 struct bnxt_stats_mem *ring0_stats; 9505 bool ignore_zero = false; 9506 int i; 9507 9508 /* Chip bug. Counter intermittently becomes 0. */ 9509 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 9510 ignore_zero = true; 9511 9512 for (i = 0; i < bp->cp_nr_rings; i++) { 9513 struct bnxt_napi *bnapi = bp->bnapi[i]; 9514 struct bnxt_cp_ring_info *cpr; 9515 struct bnxt_stats_mem *stats; 9516 9517 cpr = &bnapi->cp_ring; 9518 stats = &cpr->stats; 9519 if (!i) 9520 ring0_stats = stats; 9521 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 9522 ring0_stats->hw_masks, 9523 ring0_stats->len / 8, ignore_zero); 9524 } 9525 if (bp->flags & BNXT_FLAG_PORT_STATS) { 9526 struct bnxt_stats_mem *stats = &bp->port_stats; 9527 __le64 *hw_stats = stats->hw_stats; 9528 u64 *sw_stats = stats->sw_stats; 9529 u64 *masks = stats->hw_masks; 9530 int cnt; 9531 9532 cnt = sizeof(struct rx_port_stats) / 8; 9533 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 9534 9535 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 9536 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 9537 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 9538 cnt = sizeof(struct tx_port_stats) / 8; 9539 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 9540 } 9541 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 9542 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 9543 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 9544 } 9545 } 9546 9547 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 9548 { 9549 struct hwrm_port_qstats_input *req; 9550 struct bnxt_pf_info *pf = &bp->pf; 9551 int rc; 9552 9553 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 9554 return 0; 9555 9556 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 9557 return -EOPNOTSUPP; 9558 9559 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 9560 if (rc) 9561 return rc; 9562 9563 req->flags = flags; 9564 req->port_id = cpu_to_le16(pf->port_id); 9565 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 9566 BNXT_TX_PORT_STATS_BYTE_OFFSET); 9567 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 9568 return hwrm_req_send(bp, req); 9569 } 9570 9571 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 9572 { 9573 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 9574 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 9575 struct hwrm_port_qstats_ext_output *resp_qs; 9576 struct hwrm_port_qstats_ext_input *req_qs; 9577 struct bnxt_pf_info *pf = &bp->pf; 9578 u32 tx_stat_size; 9579 int rc; 9580 9581 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 9582 return 0; 9583 9584 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 9585 return -EOPNOTSUPP; 9586 9587 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 9588 if (rc) 9589 return rc; 9590 9591 req_qs->flags = flags; 9592 req_qs->port_id = cpu_to_le16(pf->port_id); 9593 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 9594 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 9595 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 9596 sizeof(struct tx_port_stats_ext) : 0; 9597 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 9598 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 9599 resp_qs = hwrm_req_hold(bp, req_qs); 9600 rc = hwrm_req_send(bp, req_qs); 9601 if (!rc) { 9602 bp->fw_rx_stats_ext_size = 9603 le16_to_cpu(resp_qs->rx_stat_size) / 8; 9604 if (BNXT_FW_MAJ(bp) < 220 && 9605 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 9606 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 9607 9608 bp->fw_tx_stats_ext_size = tx_stat_size ? 9609 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 9610 } else { 9611 bp->fw_rx_stats_ext_size = 0; 9612 bp->fw_tx_stats_ext_size = 0; 9613 } 9614 hwrm_req_drop(bp, req_qs); 9615 9616 if (flags) 9617 return rc; 9618 9619 if (bp->fw_tx_stats_ext_size <= 9620 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 9621 bp->pri2cos_valid = 0; 9622 return rc; 9623 } 9624 9625 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 9626 if (rc) 9627 return rc; 9628 9629 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 9630 9631 resp_qc = hwrm_req_hold(bp, req_qc); 9632 rc = hwrm_req_send(bp, req_qc); 9633 if (!rc) { 9634 u8 *pri2cos; 9635 int i, j; 9636 9637 pri2cos = &resp_qc->pri0_cos_queue_id; 9638 for (i = 0; i < 8; i++) { 9639 u8 queue_id = pri2cos[i]; 9640 u8 queue_idx; 9641 9642 /* Per port queue IDs start from 0, 10, 20, etc */ 9643 queue_idx = queue_id % 10; 9644 if (queue_idx > BNXT_MAX_QUEUE) { 9645 bp->pri2cos_valid = false; 9646 hwrm_req_drop(bp, req_qc); 9647 return rc; 9648 } 9649 for (j = 0; j < bp->max_q; j++) { 9650 if (bp->q_ids[j] == queue_id) 9651 bp->pri2cos_idx[i] = queue_idx; 9652 } 9653 } 9654 bp->pri2cos_valid = true; 9655 } 9656 hwrm_req_drop(bp, req_qc); 9657 9658 return rc; 9659 } 9660 9661 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 9662 { 9663 bnxt_hwrm_tunnel_dst_port_free(bp, 9664 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 9665 bnxt_hwrm_tunnel_dst_port_free(bp, 9666 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 9667 } 9668 9669 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 9670 { 9671 int rc, i; 9672 u32 tpa_flags = 0; 9673 9674 if (set_tpa) 9675 tpa_flags = bp->flags & BNXT_FLAG_TPA; 9676 else if (BNXT_NO_FW_ACCESS(bp)) 9677 return 0; 9678 for (i = 0; i < bp->nr_vnics; i++) { 9679 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 9680 if (rc) { 9681 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 9682 i, rc); 9683 return rc; 9684 } 9685 } 9686 return 0; 9687 } 9688 9689 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 9690 { 9691 int i; 9692 9693 for (i = 0; i < bp->nr_vnics; i++) 9694 bnxt_hwrm_vnic_set_rss(bp, i, false); 9695 } 9696 9697 static void bnxt_clear_vnic(struct bnxt *bp) 9698 { 9699 if (!bp->vnic_info) 9700 return; 9701 9702 bnxt_hwrm_clear_vnic_filter(bp); 9703 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) { 9704 /* clear all RSS setting before free vnic ctx */ 9705 bnxt_hwrm_clear_vnic_rss(bp); 9706 bnxt_hwrm_vnic_ctx_free(bp); 9707 } 9708 /* before free the vnic, undo the vnic tpa settings */ 9709 if (bp->flags & BNXT_FLAG_TPA) 9710 bnxt_set_tpa(bp, false); 9711 bnxt_hwrm_vnic_free(bp); 9712 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 9713 bnxt_hwrm_vnic_ctx_free(bp); 9714 } 9715 9716 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 9717 bool irq_re_init) 9718 { 9719 bnxt_clear_vnic(bp); 9720 bnxt_hwrm_ring_free(bp, close_path); 9721 bnxt_hwrm_ring_grp_free(bp); 9722 if (irq_re_init) { 9723 bnxt_hwrm_stat_ctx_free(bp); 9724 bnxt_hwrm_free_tunnel_ports(bp); 9725 } 9726 } 9727 9728 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 9729 { 9730 struct hwrm_func_cfg_input *req; 9731 u8 evb_mode; 9732 int rc; 9733 9734 if (br_mode == BRIDGE_MODE_VEB) 9735 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 9736 else if (br_mode == BRIDGE_MODE_VEPA) 9737 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 9738 else 9739 return -EINVAL; 9740 9741 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 9742 if (rc) 9743 return rc; 9744 9745 req->fid = cpu_to_le16(0xffff); 9746 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 9747 req->evb_mode = evb_mode; 9748 return hwrm_req_send(bp, req); 9749 } 9750 9751 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 9752 { 9753 struct hwrm_func_cfg_input *req; 9754 int rc; 9755 9756 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 9757 return 0; 9758 9759 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 9760 if (rc) 9761 return rc; 9762 9763 req->fid = cpu_to_le16(0xffff); 9764 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 9765 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 9766 if (size == 128) 9767 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 9768 9769 return hwrm_req_send(bp, req); 9770 } 9771 9772 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 9773 { 9774 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 9775 int rc; 9776 9777 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 9778 goto skip_rss_ctx; 9779 9780 /* allocate context for vnic */ 9781 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 9782 if (rc) { 9783 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 9784 vnic_id, rc); 9785 goto vnic_setup_err; 9786 } 9787 bp->rsscos_nr_ctxs++; 9788 9789 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 9790 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 9791 if (rc) { 9792 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 9793 vnic_id, rc); 9794 goto vnic_setup_err; 9795 } 9796 bp->rsscos_nr_ctxs++; 9797 } 9798 9799 skip_rss_ctx: 9800 /* configure default vnic, ring grp */ 9801 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 9802 if (rc) { 9803 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 9804 vnic_id, rc); 9805 goto vnic_setup_err; 9806 } 9807 9808 /* Enable RSS hashing on vnic */ 9809 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 9810 if (rc) { 9811 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 9812 vnic_id, rc); 9813 goto vnic_setup_err; 9814 } 9815 9816 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 9817 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 9818 if (rc) { 9819 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 9820 vnic_id, rc); 9821 } 9822 } 9823 9824 vnic_setup_err: 9825 return rc; 9826 } 9827 9828 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 9829 { 9830 int rc, i, nr_ctxs; 9831 9832 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 9833 for (i = 0; i < nr_ctxs; i++) { 9834 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 9835 if (rc) { 9836 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 9837 vnic_id, i, rc); 9838 break; 9839 } 9840 bp->rsscos_nr_ctxs++; 9841 } 9842 if (i < nr_ctxs) 9843 return -ENOMEM; 9844 9845 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 9846 if (rc) { 9847 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 9848 vnic_id, rc); 9849 return rc; 9850 } 9851 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 9852 if (rc) { 9853 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 9854 vnic_id, rc); 9855 return rc; 9856 } 9857 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 9858 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 9859 if (rc) { 9860 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 9861 vnic_id, rc); 9862 } 9863 } 9864 return rc; 9865 } 9866 9867 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 9868 { 9869 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 9870 return __bnxt_setup_vnic_p5(bp, vnic_id); 9871 else 9872 return __bnxt_setup_vnic(bp, vnic_id); 9873 } 9874 9875 static int bnxt_alloc_and_setup_vnic(struct bnxt *bp, u16 vnic_id, 9876 u16 start_rx_ring_idx, int rx_rings) 9877 { 9878 int rc; 9879 9880 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, start_rx_ring_idx, rx_rings); 9881 if (rc) { 9882 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 9883 vnic_id, rc); 9884 return rc; 9885 } 9886 return bnxt_setup_vnic(bp, vnic_id); 9887 } 9888 9889 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 9890 { 9891 int i, rc = 0; 9892 9893 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 9894 return bnxt_alloc_and_setup_vnic(bp, BNXT_VNIC_NTUPLE, 0, 9895 bp->rx_nr_rings); 9896 9897 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 9898 return 0; 9899 9900 for (i = 0; i < bp->rx_nr_rings; i++) { 9901 struct bnxt_vnic_info *vnic; 9902 u16 vnic_id = i + 1; 9903 u16 ring_id = i; 9904 9905 if (vnic_id >= bp->nr_vnics) 9906 break; 9907 9908 vnic = &bp->vnic_info[vnic_id]; 9909 vnic->flags |= BNXT_VNIC_RFS_FLAG; 9910 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) 9911 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 9912 if (bnxt_alloc_and_setup_vnic(bp, vnic_id, ring_id, 1)) 9913 break; 9914 } 9915 return rc; 9916 } 9917 9918 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 9919 static bool bnxt_promisc_ok(struct bnxt *bp) 9920 { 9921 #ifdef CONFIG_BNXT_SRIOV 9922 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 9923 return false; 9924 #endif 9925 return true; 9926 } 9927 9928 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 9929 { 9930 unsigned int rc = 0; 9931 9932 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 9933 if (rc) { 9934 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 9935 rc); 9936 return rc; 9937 } 9938 9939 rc = bnxt_hwrm_vnic_cfg(bp, 1); 9940 if (rc) { 9941 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 9942 rc); 9943 return rc; 9944 } 9945 return rc; 9946 } 9947 9948 static int bnxt_cfg_rx_mode(struct bnxt *); 9949 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 9950 9951 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 9952 { 9953 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 9954 int rc = 0; 9955 unsigned int rx_nr_rings = bp->rx_nr_rings; 9956 9957 if (irq_re_init) { 9958 rc = bnxt_hwrm_stat_ctx_alloc(bp); 9959 if (rc) { 9960 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 9961 rc); 9962 goto err_out; 9963 } 9964 } 9965 9966 rc = bnxt_hwrm_ring_alloc(bp); 9967 if (rc) { 9968 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 9969 goto err_out; 9970 } 9971 9972 rc = bnxt_hwrm_ring_grp_alloc(bp); 9973 if (rc) { 9974 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 9975 goto err_out; 9976 } 9977 9978 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 9979 rx_nr_rings--; 9980 9981 /* default vnic 0 */ 9982 rc = bnxt_hwrm_vnic_alloc(bp, BNXT_VNIC_DEFAULT, 0, rx_nr_rings); 9983 if (rc) { 9984 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 9985 goto err_out; 9986 } 9987 9988 if (BNXT_VF(bp)) 9989 bnxt_hwrm_func_qcfg(bp); 9990 9991 rc = bnxt_setup_vnic(bp, BNXT_VNIC_DEFAULT); 9992 if (rc) 9993 goto err_out; 9994 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA) 9995 bnxt_hwrm_update_rss_hash_cfg(bp); 9996 9997 if (bp->flags & BNXT_FLAG_RFS) { 9998 rc = bnxt_alloc_rfs_vnics(bp); 9999 if (rc) 10000 goto err_out; 10001 } 10002 10003 if (bp->flags & BNXT_FLAG_TPA) { 10004 rc = bnxt_set_tpa(bp, true); 10005 if (rc) 10006 goto err_out; 10007 } 10008 10009 if (BNXT_VF(bp)) 10010 bnxt_update_vf_mac(bp); 10011 10012 /* Filter for default vnic 0 */ 10013 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 10014 if (rc) { 10015 if (BNXT_VF(bp) && rc == -ENODEV) 10016 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 10017 else 10018 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 10019 goto err_out; 10020 } 10021 vnic->uc_filter_count = 1; 10022 10023 vnic->rx_mask = 0; 10024 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 10025 goto skip_rx_mask; 10026 10027 if (bp->dev->flags & IFF_BROADCAST) 10028 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10029 10030 if (bp->dev->flags & IFF_PROMISC) 10031 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10032 10033 if (bp->dev->flags & IFF_ALLMULTI) { 10034 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10035 vnic->mc_list_count = 0; 10036 } else if (bp->dev->flags & IFF_MULTICAST) { 10037 u32 mask = 0; 10038 10039 bnxt_mc_list_updated(bp, &mask); 10040 vnic->rx_mask |= mask; 10041 } 10042 10043 rc = bnxt_cfg_rx_mode(bp); 10044 if (rc) 10045 goto err_out; 10046 10047 skip_rx_mask: 10048 rc = bnxt_hwrm_set_coal(bp); 10049 if (rc) 10050 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 10051 rc); 10052 10053 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 10054 rc = bnxt_setup_nitroa0_vnic(bp); 10055 if (rc) 10056 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 10057 rc); 10058 } 10059 10060 if (BNXT_VF(bp)) { 10061 bnxt_hwrm_func_qcfg(bp); 10062 netdev_update_features(bp->dev); 10063 } 10064 10065 return 0; 10066 10067 err_out: 10068 bnxt_hwrm_resource_free(bp, 0, true); 10069 10070 return rc; 10071 } 10072 10073 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 10074 { 10075 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 10076 return 0; 10077 } 10078 10079 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 10080 { 10081 bnxt_init_cp_rings(bp); 10082 bnxt_init_rx_rings(bp); 10083 bnxt_init_tx_rings(bp); 10084 bnxt_init_ring_grps(bp, irq_re_init); 10085 bnxt_init_vnics(bp); 10086 10087 return bnxt_init_chip(bp, irq_re_init); 10088 } 10089 10090 static int bnxt_set_real_num_queues(struct bnxt *bp) 10091 { 10092 int rc; 10093 struct net_device *dev = bp->dev; 10094 10095 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 10096 bp->tx_nr_rings_xdp); 10097 if (rc) 10098 return rc; 10099 10100 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 10101 if (rc) 10102 return rc; 10103 10104 #ifdef CONFIG_RFS_ACCEL 10105 if (bp->flags & BNXT_FLAG_RFS) 10106 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 10107 #endif 10108 10109 return rc; 10110 } 10111 10112 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 10113 bool shared) 10114 { 10115 int _rx = *rx, _tx = *tx; 10116 10117 if (shared) { 10118 *rx = min_t(int, _rx, max); 10119 *tx = min_t(int, _tx, max); 10120 } else { 10121 if (max < 2) 10122 return -ENOMEM; 10123 10124 while (_rx + _tx > max) { 10125 if (_rx > _tx && _rx > 1) 10126 _rx--; 10127 else if (_tx > 1) 10128 _tx--; 10129 } 10130 *rx = _rx; 10131 *tx = _tx; 10132 } 10133 return 0; 10134 } 10135 10136 static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp) 10137 { 10138 return (tx - tx_xdp) / tx_sets + tx_xdp; 10139 } 10140 10141 int bnxt_num_tx_to_cp(struct bnxt *bp, int tx) 10142 { 10143 int tcs = bp->num_tc; 10144 10145 if (!tcs) 10146 tcs = 1; 10147 return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp); 10148 } 10149 10150 static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp) 10151 { 10152 int tcs = bp->num_tc; 10153 10154 return (tx_cp - bp->tx_nr_rings_xdp) * tcs + 10155 bp->tx_nr_rings_xdp; 10156 } 10157 10158 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 10159 bool sh) 10160 { 10161 int tx_cp = bnxt_num_tx_to_cp(bp, *tx); 10162 10163 if (tx_cp != *tx) { 10164 int tx_saved = tx_cp, rc; 10165 10166 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh); 10167 if (rc) 10168 return rc; 10169 if (tx_cp != tx_saved) 10170 *tx = bnxt_num_cp_to_tx(bp, tx_cp); 10171 return 0; 10172 } 10173 return __bnxt_trim_rings(bp, rx, tx, max, sh); 10174 } 10175 10176 static void bnxt_setup_msix(struct bnxt *bp) 10177 { 10178 const int len = sizeof(bp->irq_tbl[0].name); 10179 struct net_device *dev = bp->dev; 10180 int tcs, i; 10181 10182 tcs = bp->num_tc; 10183 if (tcs) { 10184 int i, off, count; 10185 10186 for (i = 0; i < tcs; i++) { 10187 count = bp->tx_nr_rings_per_tc; 10188 off = BNXT_TC_TO_RING_BASE(bp, i); 10189 netdev_set_tc_queue(dev, i, count, off); 10190 } 10191 } 10192 10193 for (i = 0; i < bp->cp_nr_rings; i++) { 10194 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 10195 char *attr; 10196 10197 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 10198 attr = "TxRx"; 10199 else if (i < bp->rx_nr_rings) 10200 attr = "rx"; 10201 else 10202 attr = "tx"; 10203 10204 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 10205 attr, i); 10206 bp->irq_tbl[map_idx].handler = bnxt_msix; 10207 } 10208 } 10209 10210 static void bnxt_setup_inta(struct bnxt *bp) 10211 { 10212 const int len = sizeof(bp->irq_tbl[0].name); 10213 10214 if (bp->num_tc) { 10215 netdev_reset_tc(bp->dev); 10216 bp->num_tc = 0; 10217 } 10218 10219 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 10220 0); 10221 bp->irq_tbl[0].handler = bnxt_inta; 10222 } 10223 10224 static int bnxt_init_int_mode(struct bnxt *bp); 10225 10226 static int bnxt_setup_int_mode(struct bnxt *bp) 10227 { 10228 int rc; 10229 10230 if (!bp->irq_tbl) { 10231 rc = bnxt_init_int_mode(bp); 10232 if (rc || !bp->irq_tbl) 10233 return rc ?: -ENODEV; 10234 } 10235 10236 if (bp->flags & BNXT_FLAG_USING_MSIX) 10237 bnxt_setup_msix(bp); 10238 else 10239 bnxt_setup_inta(bp); 10240 10241 rc = bnxt_set_real_num_queues(bp); 10242 return rc; 10243 } 10244 10245 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 10246 { 10247 return bp->hw_resc.max_rsscos_ctxs; 10248 } 10249 10250 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 10251 { 10252 return bp->hw_resc.max_vnics; 10253 } 10254 10255 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 10256 { 10257 return bp->hw_resc.max_stat_ctxs; 10258 } 10259 10260 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 10261 { 10262 return bp->hw_resc.max_cp_rings; 10263 } 10264 10265 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 10266 { 10267 unsigned int cp = bp->hw_resc.max_cp_rings; 10268 10269 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 10270 cp -= bnxt_get_ulp_msix_num(bp); 10271 10272 return cp; 10273 } 10274 10275 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 10276 { 10277 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10278 10279 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10280 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 10281 10282 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 10283 } 10284 10285 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 10286 { 10287 bp->hw_resc.max_irqs = max_irqs; 10288 } 10289 10290 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 10291 { 10292 unsigned int cp; 10293 10294 cp = bnxt_get_max_func_cp_rings_for_en(bp); 10295 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10296 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 10297 else 10298 return cp - bp->cp_nr_rings; 10299 } 10300 10301 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 10302 { 10303 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 10304 } 10305 10306 int bnxt_get_avail_msix(struct bnxt *bp, int num) 10307 { 10308 int max_cp = bnxt_get_max_func_cp_rings(bp); 10309 int max_irq = bnxt_get_max_func_irqs(bp); 10310 int total_req = bp->cp_nr_rings + num; 10311 int max_idx, avail_msix; 10312 10313 max_idx = bp->total_irqs; 10314 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 10315 max_idx = min_t(int, bp->total_irqs, max_cp); 10316 avail_msix = max_idx - bp->cp_nr_rings; 10317 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 10318 return avail_msix; 10319 10320 if (max_irq < total_req) { 10321 num = max_irq - bp->cp_nr_rings; 10322 if (num <= 0) 10323 return 0; 10324 } 10325 return num; 10326 } 10327 10328 static int bnxt_get_num_msix(struct bnxt *bp) 10329 { 10330 if (!BNXT_NEW_RM(bp)) 10331 return bnxt_get_max_func_irqs(bp); 10332 10333 return bnxt_nq_rings_in_use(bp); 10334 } 10335 10336 static int bnxt_init_msix(struct bnxt *bp) 10337 { 10338 int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp; 10339 struct msix_entry *msix_ent; 10340 10341 total_vecs = bnxt_get_num_msix(bp); 10342 max = bnxt_get_max_func_irqs(bp); 10343 if (total_vecs > max) 10344 total_vecs = max; 10345 10346 if (!total_vecs) 10347 return 0; 10348 10349 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 10350 if (!msix_ent) 10351 return -ENOMEM; 10352 10353 for (i = 0; i < total_vecs; i++) { 10354 msix_ent[i].entry = i; 10355 msix_ent[i].vector = 0; 10356 } 10357 10358 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 10359 min = 2; 10360 10361 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 10362 ulp_msix = bnxt_get_ulp_msix_num(bp); 10363 if (total_vecs < 0 || total_vecs < ulp_msix) { 10364 rc = -ENODEV; 10365 goto msix_setup_exit; 10366 } 10367 10368 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 10369 if (bp->irq_tbl) { 10370 for (i = 0; i < total_vecs; i++) 10371 bp->irq_tbl[i].vector = msix_ent[i].vector; 10372 10373 bp->total_irqs = total_vecs; 10374 /* Trim rings based upon num of vectors allocated */ 10375 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 10376 total_vecs - ulp_msix, min == 1); 10377 if (rc) 10378 goto msix_setup_exit; 10379 10380 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); 10381 bp->cp_nr_rings = (min == 1) ? 10382 max_t(int, tx_cp, bp->rx_nr_rings) : 10383 tx_cp + bp->rx_nr_rings; 10384 10385 } else { 10386 rc = -ENOMEM; 10387 goto msix_setup_exit; 10388 } 10389 bp->flags |= BNXT_FLAG_USING_MSIX; 10390 kfree(msix_ent); 10391 return 0; 10392 10393 msix_setup_exit: 10394 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 10395 kfree(bp->irq_tbl); 10396 bp->irq_tbl = NULL; 10397 pci_disable_msix(bp->pdev); 10398 kfree(msix_ent); 10399 return rc; 10400 } 10401 10402 static int bnxt_init_inta(struct bnxt *bp) 10403 { 10404 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 10405 if (!bp->irq_tbl) 10406 return -ENOMEM; 10407 10408 bp->total_irqs = 1; 10409 bp->rx_nr_rings = 1; 10410 bp->tx_nr_rings = 1; 10411 bp->cp_nr_rings = 1; 10412 bp->flags |= BNXT_FLAG_SHARED_RINGS; 10413 bp->irq_tbl[0].vector = bp->pdev->irq; 10414 return 0; 10415 } 10416 10417 static int bnxt_init_int_mode(struct bnxt *bp) 10418 { 10419 int rc = -ENODEV; 10420 10421 if (bp->flags & BNXT_FLAG_MSIX_CAP) 10422 rc = bnxt_init_msix(bp); 10423 10424 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 10425 /* fallback to INTA */ 10426 rc = bnxt_init_inta(bp); 10427 } 10428 return rc; 10429 } 10430 10431 static void bnxt_clear_int_mode(struct bnxt *bp) 10432 { 10433 if (bp->flags & BNXT_FLAG_USING_MSIX) 10434 pci_disable_msix(bp->pdev); 10435 10436 kfree(bp->irq_tbl); 10437 bp->irq_tbl = NULL; 10438 bp->flags &= ~BNXT_FLAG_USING_MSIX; 10439 } 10440 10441 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 10442 { 10443 bool irq_cleared = false; 10444 int tcs = bp->num_tc; 10445 int rc; 10446 10447 if (!bnxt_need_reserve_rings(bp)) 10448 return 0; 10449 10450 if (irq_re_init && BNXT_NEW_RM(bp) && 10451 bnxt_get_num_msix(bp) != bp->total_irqs) { 10452 bnxt_ulp_irq_stop(bp); 10453 bnxt_clear_int_mode(bp); 10454 irq_cleared = true; 10455 } 10456 rc = __bnxt_reserve_rings(bp); 10457 if (irq_cleared) { 10458 if (!rc) 10459 rc = bnxt_init_int_mode(bp); 10460 bnxt_ulp_irq_restart(bp, rc); 10461 } 10462 if (rc) { 10463 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 10464 return rc; 10465 } 10466 if (tcs && (bp->tx_nr_rings_per_tc * tcs != 10467 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) { 10468 netdev_err(bp->dev, "tx ring reservation failure\n"); 10469 netdev_reset_tc(bp->dev); 10470 bp->num_tc = 0; 10471 if (bp->tx_nr_rings_xdp) 10472 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp; 10473 else 10474 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 10475 return -ENOMEM; 10476 } 10477 return 0; 10478 } 10479 10480 static void bnxt_free_irq(struct bnxt *bp) 10481 { 10482 struct bnxt_irq *irq; 10483 int i; 10484 10485 #ifdef CONFIG_RFS_ACCEL 10486 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 10487 bp->dev->rx_cpu_rmap = NULL; 10488 #endif 10489 if (!bp->irq_tbl || !bp->bnapi) 10490 return; 10491 10492 for (i = 0; i < bp->cp_nr_rings; i++) { 10493 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 10494 10495 irq = &bp->irq_tbl[map_idx]; 10496 if (irq->requested) { 10497 if (irq->have_cpumask) { 10498 irq_set_affinity_hint(irq->vector, NULL); 10499 free_cpumask_var(irq->cpu_mask); 10500 irq->have_cpumask = 0; 10501 } 10502 free_irq(irq->vector, bp->bnapi[i]); 10503 } 10504 10505 irq->requested = 0; 10506 } 10507 } 10508 10509 static int bnxt_request_irq(struct bnxt *bp) 10510 { 10511 int i, j, rc = 0; 10512 unsigned long flags = 0; 10513 #ifdef CONFIG_RFS_ACCEL 10514 struct cpu_rmap *rmap; 10515 #endif 10516 10517 rc = bnxt_setup_int_mode(bp); 10518 if (rc) { 10519 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 10520 rc); 10521 return rc; 10522 } 10523 #ifdef CONFIG_RFS_ACCEL 10524 rmap = bp->dev->rx_cpu_rmap; 10525 #endif 10526 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 10527 flags = IRQF_SHARED; 10528 10529 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 10530 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 10531 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 10532 10533 #ifdef CONFIG_RFS_ACCEL 10534 if (rmap && bp->bnapi[i]->rx_ring) { 10535 rc = irq_cpu_rmap_add(rmap, irq->vector); 10536 if (rc) 10537 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 10538 j); 10539 j++; 10540 } 10541 #endif 10542 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 10543 bp->bnapi[i]); 10544 if (rc) 10545 break; 10546 10547 netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector); 10548 irq->requested = 1; 10549 10550 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 10551 int numa_node = dev_to_node(&bp->pdev->dev); 10552 10553 irq->have_cpumask = 1; 10554 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 10555 irq->cpu_mask); 10556 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 10557 if (rc) { 10558 netdev_warn(bp->dev, 10559 "Set affinity failed, IRQ = %d\n", 10560 irq->vector); 10561 break; 10562 } 10563 } 10564 } 10565 return rc; 10566 } 10567 10568 static void bnxt_del_napi(struct bnxt *bp) 10569 { 10570 int i; 10571 10572 if (!bp->bnapi) 10573 return; 10574 10575 for (i = 0; i < bp->rx_nr_rings; i++) 10576 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL); 10577 for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++) 10578 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL); 10579 10580 for (i = 0; i < bp->cp_nr_rings; i++) { 10581 struct bnxt_napi *bnapi = bp->bnapi[i]; 10582 10583 __netif_napi_del(&bnapi->napi); 10584 } 10585 /* We called __netif_napi_del(), we need 10586 * to respect an RCU grace period before freeing napi structures. 10587 */ 10588 synchronize_net(); 10589 } 10590 10591 static void bnxt_init_napi(struct bnxt *bp) 10592 { 10593 int i; 10594 unsigned int cp_nr_rings = bp->cp_nr_rings; 10595 struct bnxt_napi *bnapi; 10596 10597 if (bp->flags & BNXT_FLAG_USING_MSIX) { 10598 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 10599 10600 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10601 poll_fn = bnxt_poll_p5; 10602 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 10603 cp_nr_rings--; 10604 for (i = 0; i < cp_nr_rings; i++) { 10605 bnapi = bp->bnapi[i]; 10606 netif_napi_add(bp->dev, &bnapi->napi, poll_fn); 10607 } 10608 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 10609 bnapi = bp->bnapi[cp_nr_rings]; 10610 netif_napi_add(bp->dev, &bnapi->napi, 10611 bnxt_poll_nitroa0); 10612 } 10613 } else { 10614 bnapi = bp->bnapi[0]; 10615 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll); 10616 } 10617 } 10618 10619 static void bnxt_disable_napi(struct bnxt *bp) 10620 { 10621 int i; 10622 10623 if (!bp->bnapi || 10624 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 10625 return; 10626 10627 for (i = 0; i < bp->cp_nr_rings; i++) { 10628 struct bnxt_napi *bnapi = bp->bnapi[i]; 10629 struct bnxt_cp_ring_info *cpr; 10630 10631 cpr = &bnapi->cp_ring; 10632 if (bnapi->tx_fault) 10633 cpr->sw_stats.tx.tx_resets++; 10634 if (bnapi->in_reset) 10635 cpr->sw_stats.rx.rx_resets++; 10636 napi_disable(&bnapi->napi); 10637 if (bnapi->rx_ring) 10638 cancel_work_sync(&cpr->dim.work); 10639 } 10640 } 10641 10642 static void bnxt_enable_napi(struct bnxt *bp) 10643 { 10644 int i; 10645 10646 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 10647 for (i = 0; i < bp->cp_nr_rings; i++) { 10648 struct bnxt_napi *bnapi = bp->bnapi[i]; 10649 struct bnxt_cp_ring_info *cpr; 10650 10651 bnapi->tx_fault = 0; 10652 10653 cpr = &bnapi->cp_ring; 10654 bnapi->in_reset = false; 10655 10656 if (bnapi->rx_ring) { 10657 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 10658 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 10659 } 10660 napi_enable(&bnapi->napi); 10661 } 10662 } 10663 10664 void bnxt_tx_disable(struct bnxt *bp) 10665 { 10666 int i; 10667 struct bnxt_tx_ring_info *txr; 10668 10669 if (bp->tx_ring) { 10670 for (i = 0; i < bp->tx_nr_rings; i++) { 10671 txr = &bp->tx_ring[i]; 10672 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 10673 } 10674 } 10675 /* Make sure napi polls see @dev_state change */ 10676 synchronize_net(); 10677 /* Drop carrier first to prevent TX timeout */ 10678 netif_carrier_off(bp->dev); 10679 /* Stop all TX queues */ 10680 netif_tx_disable(bp->dev); 10681 } 10682 10683 void bnxt_tx_enable(struct bnxt *bp) 10684 { 10685 int i; 10686 struct bnxt_tx_ring_info *txr; 10687 10688 for (i = 0; i < bp->tx_nr_rings; i++) { 10689 txr = &bp->tx_ring[i]; 10690 WRITE_ONCE(txr->dev_state, 0); 10691 } 10692 /* Make sure napi polls see @dev_state change */ 10693 synchronize_net(); 10694 netif_tx_wake_all_queues(bp->dev); 10695 if (BNXT_LINK_IS_UP(bp)) 10696 netif_carrier_on(bp->dev); 10697 } 10698 10699 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 10700 { 10701 u8 active_fec = link_info->active_fec_sig_mode & 10702 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 10703 10704 switch (active_fec) { 10705 default: 10706 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 10707 return "None"; 10708 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 10709 return "Clause 74 BaseR"; 10710 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 10711 return "Clause 91 RS(528,514)"; 10712 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 10713 return "Clause 91 RS544_1XN"; 10714 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 10715 return "Clause 91 RS(544,514)"; 10716 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 10717 return "Clause 91 RS272_1XN"; 10718 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 10719 return "Clause 91 RS(272,257)"; 10720 } 10721 } 10722 10723 void bnxt_report_link(struct bnxt *bp) 10724 { 10725 if (BNXT_LINK_IS_UP(bp)) { 10726 const char *signal = ""; 10727 const char *flow_ctrl; 10728 const char *duplex; 10729 u32 speed; 10730 u16 fec; 10731 10732 netif_carrier_on(bp->dev); 10733 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 10734 if (speed == SPEED_UNKNOWN) { 10735 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 10736 return; 10737 } 10738 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 10739 duplex = "full"; 10740 else 10741 duplex = "half"; 10742 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 10743 flow_ctrl = "ON - receive & transmit"; 10744 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 10745 flow_ctrl = "ON - transmit"; 10746 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 10747 flow_ctrl = "ON - receive"; 10748 else 10749 flow_ctrl = "none"; 10750 if (bp->link_info.phy_qcfg_resp.option_flags & 10751 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 10752 u8 sig_mode = bp->link_info.active_fec_sig_mode & 10753 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 10754 switch (sig_mode) { 10755 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 10756 signal = "(NRZ) "; 10757 break; 10758 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 10759 signal = "(PAM4 56Gbps) "; 10760 break; 10761 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112: 10762 signal = "(PAM4 112Gbps) "; 10763 break; 10764 default: 10765 break; 10766 } 10767 } 10768 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 10769 speed, signal, duplex, flow_ctrl); 10770 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 10771 netdev_info(bp->dev, "EEE is %s\n", 10772 bp->eee.eee_active ? "active" : 10773 "not active"); 10774 fec = bp->link_info.fec_cfg; 10775 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 10776 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 10777 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 10778 bnxt_report_fec(&bp->link_info)); 10779 } else { 10780 netif_carrier_off(bp->dev); 10781 netdev_err(bp->dev, "NIC Link is Down\n"); 10782 } 10783 } 10784 10785 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 10786 { 10787 if (!resp->supported_speeds_auto_mode && 10788 !resp->supported_speeds_force_mode && 10789 !resp->supported_pam4_speeds_auto_mode && 10790 !resp->supported_pam4_speeds_force_mode && 10791 !resp->supported_speeds2_auto_mode && 10792 !resp->supported_speeds2_force_mode) 10793 return true; 10794 return false; 10795 } 10796 10797 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 10798 { 10799 struct bnxt_link_info *link_info = &bp->link_info; 10800 struct hwrm_port_phy_qcaps_output *resp; 10801 struct hwrm_port_phy_qcaps_input *req; 10802 int rc = 0; 10803 10804 if (bp->hwrm_spec_code < 0x10201) 10805 return 0; 10806 10807 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 10808 if (rc) 10809 return rc; 10810 10811 resp = hwrm_req_hold(bp, req); 10812 rc = hwrm_req_send(bp, req); 10813 if (rc) 10814 goto hwrm_phy_qcaps_exit; 10815 10816 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8); 10817 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 10818 struct ethtool_keee *eee = &bp->eee; 10819 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 10820 10821 _bnxt_fw_to_linkmode(eee->supported, fw_speeds); 10822 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 10823 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 10824 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 10825 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 10826 } 10827 10828 if (bp->hwrm_spec_code >= 0x10a01) { 10829 if (bnxt_phy_qcaps_no_speed(resp)) { 10830 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 10831 netdev_warn(bp->dev, "Ethernet link disabled\n"); 10832 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 10833 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 10834 netdev_info(bp->dev, "Ethernet link enabled\n"); 10835 /* Phy re-enabled, reprobe the speeds */ 10836 link_info->support_auto_speeds = 0; 10837 link_info->support_pam4_auto_speeds = 0; 10838 link_info->support_auto_speeds2 = 0; 10839 } 10840 } 10841 if (resp->supported_speeds_auto_mode) 10842 link_info->support_auto_speeds = 10843 le16_to_cpu(resp->supported_speeds_auto_mode); 10844 if (resp->supported_pam4_speeds_auto_mode) 10845 link_info->support_pam4_auto_speeds = 10846 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 10847 if (resp->supported_speeds2_auto_mode) 10848 link_info->support_auto_speeds2 = 10849 le16_to_cpu(resp->supported_speeds2_auto_mode); 10850 10851 bp->port_count = resp->port_cnt; 10852 10853 hwrm_phy_qcaps_exit: 10854 hwrm_req_drop(bp, req); 10855 return rc; 10856 } 10857 10858 static bool bnxt_support_dropped(u16 advertising, u16 supported) 10859 { 10860 u16 diff = advertising ^ supported; 10861 10862 return ((supported | diff) != supported); 10863 } 10864 10865 static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info) 10866 { 10867 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 10868 10869 /* Check if any advertised speeds are no longer supported. The caller 10870 * holds the link_lock mutex, so we can modify link_info settings. 10871 */ 10872 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 10873 if (bnxt_support_dropped(link_info->advertising, 10874 link_info->support_auto_speeds2)) { 10875 link_info->advertising = link_info->support_auto_speeds2; 10876 return true; 10877 } 10878 return false; 10879 } 10880 if (bnxt_support_dropped(link_info->advertising, 10881 link_info->support_auto_speeds)) { 10882 link_info->advertising = link_info->support_auto_speeds; 10883 return true; 10884 } 10885 if (bnxt_support_dropped(link_info->advertising_pam4, 10886 link_info->support_pam4_auto_speeds)) { 10887 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 10888 return true; 10889 } 10890 return false; 10891 } 10892 10893 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 10894 { 10895 struct bnxt_link_info *link_info = &bp->link_info; 10896 struct hwrm_port_phy_qcfg_output *resp; 10897 struct hwrm_port_phy_qcfg_input *req; 10898 u8 link_state = link_info->link_state; 10899 bool support_changed; 10900 int rc; 10901 10902 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 10903 if (rc) 10904 return rc; 10905 10906 resp = hwrm_req_hold(bp, req); 10907 rc = hwrm_req_send(bp, req); 10908 if (rc) { 10909 hwrm_req_drop(bp, req); 10910 if (BNXT_VF(bp) && rc == -ENODEV) { 10911 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 10912 rc = 0; 10913 } 10914 return rc; 10915 } 10916 10917 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 10918 link_info->phy_link_status = resp->link; 10919 link_info->duplex = resp->duplex_cfg; 10920 if (bp->hwrm_spec_code >= 0x10800) 10921 link_info->duplex = resp->duplex_state; 10922 link_info->pause = resp->pause; 10923 link_info->auto_mode = resp->auto_mode; 10924 link_info->auto_pause_setting = resp->auto_pause; 10925 link_info->lp_pause = resp->link_partner_adv_pause; 10926 link_info->force_pause_setting = resp->force_pause; 10927 link_info->duplex_setting = resp->duplex_cfg; 10928 if (link_info->phy_link_status == BNXT_LINK_LINK) { 10929 link_info->link_speed = le16_to_cpu(resp->link_speed); 10930 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) 10931 link_info->active_lanes = resp->active_lanes; 10932 } else { 10933 link_info->link_speed = 0; 10934 link_info->active_lanes = 0; 10935 } 10936 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 10937 link_info->force_pam4_link_speed = 10938 le16_to_cpu(resp->force_pam4_link_speed); 10939 link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2); 10940 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 10941 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 10942 link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2); 10943 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 10944 link_info->auto_pam4_link_speeds = 10945 le16_to_cpu(resp->auto_pam4_link_speed_mask); 10946 link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2); 10947 link_info->lp_auto_link_speeds = 10948 le16_to_cpu(resp->link_partner_adv_speeds); 10949 link_info->lp_auto_pam4_link_speeds = 10950 resp->link_partner_pam4_adv_speeds; 10951 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 10952 link_info->phy_ver[0] = resp->phy_maj; 10953 link_info->phy_ver[1] = resp->phy_min; 10954 link_info->phy_ver[2] = resp->phy_bld; 10955 link_info->media_type = resp->media_type; 10956 link_info->phy_type = resp->phy_type; 10957 link_info->transceiver = resp->xcvr_pkg_type; 10958 link_info->phy_addr = resp->eee_config_phy_addr & 10959 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 10960 link_info->module_status = resp->module_status; 10961 10962 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 10963 struct ethtool_keee *eee = &bp->eee; 10964 u16 fw_speeds; 10965 10966 eee->eee_active = 0; 10967 if (resp->eee_config_phy_addr & 10968 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 10969 eee->eee_active = 1; 10970 fw_speeds = le16_to_cpu( 10971 resp->link_partner_adv_eee_link_speed_mask); 10972 _bnxt_fw_to_linkmode(eee->lp_advertised, fw_speeds); 10973 } 10974 10975 /* Pull initial EEE config */ 10976 if (!chng_link_state) { 10977 if (resp->eee_config_phy_addr & 10978 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 10979 eee->eee_enabled = 1; 10980 10981 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 10982 _bnxt_fw_to_linkmode(eee->advertised, fw_speeds); 10983 10984 if (resp->eee_config_phy_addr & 10985 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 10986 __le32 tmr; 10987 10988 eee->tx_lpi_enabled = 1; 10989 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 10990 eee->tx_lpi_timer = le32_to_cpu(tmr) & 10991 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 10992 } 10993 } 10994 } 10995 10996 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 10997 if (bp->hwrm_spec_code >= 0x10504) { 10998 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 10999 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 11000 } 11001 /* TODO: need to add more logic to report VF link */ 11002 if (chng_link_state) { 11003 if (link_info->phy_link_status == BNXT_LINK_LINK) 11004 link_info->link_state = BNXT_LINK_STATE_UP; 11005 else 11006 link_info->link_state = BNXT_LINK_STATE_DOWN; 11007 if (link_state != link_info->link_state) 11008 bnxt_report_link(bp); 11009 } else { 11010 /* always link down if not require to update link state */ 11011 link_info->link_state = BNXT_LINK_STATE_DOWN; 11012 } 11013 hwrm_req_drop(bp, req); 11014 11015 if (!BNXT_PHY_CFG_ABLE(bp)) 11016 return 0; 11017 11018 support_changed = bnxt_support_speed_dropped(link_info); 11019 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 11020 bnxt_hwrm_set_link_setting(bp, true, false); 11021 return 0; 11022 } 11023 11024 static void bnxt_get_port_module_status(struct bnxt *bp) 11025 { 11026 struct bnxt_link_info *link_info = &bp->link_info; 11027 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 11028 u8 module_status; 11029 11030 if (bnxt_update_link(bp, true)) 11031 return; 11032 11033 module_status = link_info->module_status; 11034 switch (module_status) { 11035 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 11036 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 11037 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 11038 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 11039 bp->pf.port_id); 11040 if (bp->hwrm_spec_code >= 0x10201) { 11041 netdev_warn(bp->dev, "Module part number %s\n", 11042 resp->phy_vendor_partnumber); 11043 } 11044 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 11045 netdev_warn(bp->dev, "TX is disabled\n"); 11046 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 11047 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 11048 } 11049 } 11050 11051 static void 11052 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 11053 { 11054 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 11055 if (bp->hwrm_spec_code >= 0x10201) 11056 req->auto_pause = 11057 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 11058 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 11059 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 11060 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 11061 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 11062 req->enables |= 11063 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 11064 } else { 11065 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 11066 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 11067 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 11068 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 11069 req->enables |= 11070 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 11071 if (bp->hwrm_spec_code >= 0x10201) { 11072 req->auto_pause = req->force_pause; 11073 req->enables |= cpu_to_le32( 11074 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 11075 } 11076 } 11077 } 11078 11079 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 11080 { 11081 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 11082 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 11083 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 11084 req->enables |= 11085 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK); 11086 req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising); 11087 } else if (bp->link_info.advertising) { 11088 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 11089 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 11090 } 11091 if (bp->link_info.advertising_pam4) { 11092 req->enables |= 11093 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 11094 req->auto_link_pam4_speed_mask = 11095 cpu_to_le16(bp->link_info.advertising_pam4); 11096 } 11097 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 11098 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 11099 } else { 11100 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 11101 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 11102 req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed); 11103 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2); 11104 netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n", 11105 (u32)bp->link_info.req_link_speed); 11106 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 11107 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 11108 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 11109 } else { 11110 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 11111 } 11112 } 11113 11114 /* tell chimp that the setting takes effect immediately */ 11115 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 11116 } 11117 11118 int bnxt_hwrm_set_pause(struct bnxt *bp) 11119 { 11120 struct hwrm_port_phy_cfg_input *req; 11121 int rc; 11122 11123 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 11124 if (rc) 11125 return rc; 11126 11127 bnxt_hwrm_set_pause_common(bp, req); 11128 11129 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 11130 bp->link_info.force_link_chng) 11131 bnxt_hwrm_set_link_common(bp, req); 11132 11133 rc = hwrm_req_send(bp, req); 11134 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 11135 /* since changing of pause setting doesn't trigger any link 11136 * change event, the driver needs to update the current pause 11137 * result upon successfully return of the phy_cfg command 11138 */ 11139 bp->link_info.pause = 11140 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 11141 bp->link_info.auto_pause_setting = 0; 11142 if (!bp->link_info.force_link_chng) 11143 bnxt_report_link(bp); 11144 } 11145 bp->link_info.force_link_chng = false; 11146 return rc; 11147 } 11148 11149 static void bnxt_hwrm_set_eee(struct bnxt *bp, 11150 struct hwrm_port_phy_cfg_input *req) 11151 { 11152 struct ethtool_keee *eee = &bp->eee; 11153 11154 if (eee->eee_enabled) { 11155 u16 eee_speeds; 11156 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 11157 11158 if (eee->tx_lpi_enabled) 11159 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 11160 else 11161 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 11162 11163 req->flags |= cpu_to_le32(flags); 11164 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 11165 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 11166 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 11167 } else { 11168 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 11169 } 11170 } 11171 11172 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 11173 { 11174 struct hwrm_port_phy_cfg_input *req; 11175 int rc; 11176 11177 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 11178 if (rc) 11179 return rc; 11180 11181 if (set_pause) 11182 bnxt_hwrm_set_pause_common(bp, req); 11183 11184 bnxt_hwrm_set_link_common(bp, req); 11185 11186 if (set_eee) 11187 bnxt_hwrm_set_eee(bp, req); 11188 return hwrm_req_send(bp, req); 11189 } 11190 11191 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 11192 { 11193 struct hwrm_port_phy_cfg_input *req; 11194 int rc; 11195 11196 if (!BNXT_SINGLE_PF(bp)) 11197 return 0; 11198 11199 if (pci_num_vf(bp->pdev) && 11200 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 11201 return 0; 11202 11203 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 11204 if (rc) 11205 return rc; 11206 11207 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 11208 rc = hwrm_req_send(bp, req); 11209 if (!rc) { 11210 mutex_lock(&bp->link_lock); 11211 /* Device is not obliged link down in certain scenarios, even 11212 * when forced. Setting the state unknown is consistent with 11213 * driver startup and will force link state to be reported 11214 * during subsequent open based on PORT_PHY_QCFG. 11215 */ 11216 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN; 11217 mutex_unlock(&bp->link_lock); 11218 } 11219 return rc; 11220 } 11221 11222 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 11223 { 11224 #ifdef CONFIG_TEE_BNXT_FW 11225 int rc = tee_bnxt_fw_load(); 11226 11227 if (rc) 11228 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 11229 11230 return rc; 11231 #else 11232 netdev_err(bp->dev, "OP-TEE not supported\n"); 11233 return -ENODEV; 11234 #endif 11235 } 11236 11237 static int bnxt_try_recover_fw(struct bnxt *bp) 11238 { 11239 if (bp->fw_health && bp->fw_health->status_reliable) { 11240 int retry = 0, rc; 11241 u32 sts; 11242 11243 do { 11244 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 11245 rc = bnxt_hwrm_poll(bp); 11246 if (!BNXT_FW_IS_BOOTING(sts) && 11247 !BNXT_FW_IS_RECOVERING(sts)) 11248 break; 11249 retry++; 11250 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 11251 11252 if (!BNXT_FW_IS_HEALTHY(sts)) { 11253 netdev_err(bp->dev, 11254 "Firmware not responding, status: 0x%x\n", 11255 sts); 11256 rc = -ENODEV; 11257 } 11258 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 11259 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 11260 return bnxt_fw_reset_via_optee(bp); 11261 } 11262 return rc; 11263 } 11264 11265 return -ENODEV; 11266 } 11267 11268 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset) 11269 { 11270 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 11271 11272 if (!BNXT_NEW_RM(bp)) 11273 return; /* no resource reservations required */ 11274 11275 hw_resc->resv_cp_rings = 0; 11276 hw_resc->resv_stat_ctxs = 0; 11277 hw_resc->resv_irqs = 0; 11278 hw_resc->resv_tx_rings = 0; 11279 hw_resc->resv_rx_rings = 0; 11280 hw_resc->resv_hw_ring_grps = 0; 11281 hw_resc->resv_vnics = 0; 11282 hw_resc->resv_rsscos_ctxs = 0; 11283 if (!fw_reset) { 11284 bp->tx_nr_rings = 0; 11285 bp->rx_nr_rings = 0; 11286 } 11287 } 11288 11289 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 11290 { 11291 int rc; 11292 11293 if (!BNXT_NEW_RM(bp)) 11294 return 0; /* no resource reservations required */ 11295 11296 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 11297 if (rc) 11298 netdev_err(bp->dev, "resc_qcaps failed\n"); 11299 11300 bnxt_clear_reservations(bp, fw_reset); 11301 11302 return rc; 11303 } 11304 11305 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 11306 { 11307 struct hwrm_func_drv_if_change_output *resp; 11308 struct hwrm_func_drv_if_change_input *req; 11309 bool fw_reset = !bp->irq_tbl; 11310 bool resc_reinit = false; 11311 int rc, retry = 0; 11312 u32 flags = 0; 11313 11314 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 11315 return 0; 11316 11317 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 11318 if (rc) 11319 return rc; 11320 11321 if (up) 11322 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 11323 resp = hwrm_req_hold(bp, req); 11324 11325 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 11326 while (retry < BNXT_FW_IF_RETRY) { 11327 rc = hwrm_req_send(bp, req); 11328 if (rc != -EAGAIN) 11329 break; 11330 11331 msleep(50); 11332 retry++; 11333 } 11334 11335 if (rc == -EAGAIN) { 11336 hwrm_req_drop(bp, req); 11337 return rc; 11338 } else if (!rc) { 11339 flags = le32_to_cpu(resp->flags); 11340 } else if (up) { 11341 rc = bnxt_try_recover_fw(bp); 11342 fw_reset = true; 11343 } 11344 hwrm_req_drop(bp, req); 11345 if (rc) 11346 return rc; 11347 11348 if (!up) { 11349 bnxt_inv_fw_health_reg(bp); 11350 return 0; 11351 } 11352 11353 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 11354 resc_reinit = true; 11355 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE || 11356 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) 11357 fw_reset = true; 11358 else 11359 bnxt_remap_fw_health_regs(bp); 11360 11361 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 11362 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 11363 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 11364 return -ENODEV; 11365 } 11366 if (resc_reinit || fw_reset) { 11367 if (fw_reset) { 11368 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 11369 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11370 bnxt_ulp_stop(bp); 11371 bnxt_free_ctx_mem(bp); 11372 bnxt_dcb_free(bp); 11373 rc = bnxt_fw_init_one(bp); 11374 if (rc) { 11375 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 11376 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 11377 return rc; 11378 } 11379 bnxt_clear_int_mode(bp); 11380 rc = bnxt_init_int_mode(bp); 11381 if (rc) { 11382 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 11383 netdev_err(bp->dev, "init int mode failed\n"); 11384 return rc; 11385 } 11386 } 11387 rc = bnxt_cancel_reservations(bp, fw_reset); 11388 } 11389 return rc; 11390 } 11391 11392 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 11393 { 11394 struct hwrm_port_led_qcaps_output *resp; 11395 struct hwrm_port_led_qcaps_input *req; 11396 struct bnxt_pf_info *pf = &bp->pf; 11397 int rc; 11398 11399 bp->num_leds = 0; 11400 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 11401 return 0; 11402 11403 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 11404 if (rc) 11405 return rc; 11406 11407 req->port_id = cpu_to_le16(pf->port_id); 11408 resp = hwrm_req_hold(bp, req); 11409 rc = hwrm_req_send(bp, req); 11410 if (rc) { 11411 hwrm_req_drop(bp, req); 11412 return rc; 11413 } 11414 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 11415 int i; 11416 11417 bp->num_leds = resp->num_leds; 11418 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 11419 bp->num_leds); 11420 for (i = 0; i < bp->num_leds; i++) { 11421 struct bnxt_led_info *led = &bp->leds[i]; 11422 __le16 caps = led->led_state_caps; 11423 11424 if (!led->led_group_id || 11425 !BNXT_LED_ALT_BLINK_CAP(caps)) { 11426 bp->num_leds = 0; 11427 break; 11428 } 11429 } 11430 } 11431 hwrm_req_drop(bp, req); 11432 return 0; 11433 } 11434 11435 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 11436 { 11437 struct hwrm_wol_filter_alloc_output *resp; 11438 struct hwrm_wol_filter_alloc_input *req; 11439 int rc; 11440 11441 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 11442 if (rc) 11443 return rc; 11444 11445 req->port_id = cpu_to_le16(bp->pf.port_id); 11446 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 11447 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 11448 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 11449 11450 resp = hwrm_req_hold(bp, req); 11451 rc = hwrm_req_send(bp, req); 11452 if (!rc) 11453 bp->wol_filter_id = resp->wol_filter_id; 11454 hwrm_req_drop(bp, req); 11455 return rc; 11456 } 11457 11458 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 11459 { 11460 struct hwrm_wol_filter_free_input *req; 11461 int rc; 11462 11463 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 11464 if (rc) 11465 return rc; 11466 11467 req->port_id = cpu_to_le16(bp->pf.port_id); 11468 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 11469 req->wol_filter_id = bp->wol_filter_id; 11470 11471 return hwrm_req_send(bp, req); 11472 } 11473 11474 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 11475 { 11476 struct hwrm_wol_filter_qcfg_output *resp; 11477 struct hwrm_wol_filter_qcfg_input *req; 11478 u16 next_handle = 0; 11479 int rc; 11480 11481 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 11482 if (rc) 11483 return rc; 11484 11485 req->port_id = cpu_to_le16(bp->pf.port_id); 11486 req->handle = cpu_to_le16(handle); 11487 resp = hwrm_req_hold(bp, req); 11488 rc = hwrm_req_send(bp, req); 11489 if (!rc) { 11490 next_handle = le16_to_cpu(resp->next_handle); 11491 if (next_handle != 0) { 11492 if (resp->wol_type == 11493 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 11494 bp->wol = 1; 11495 bp->wol_filter_id = resp->wol_filter_id; 11496 } 11497 } 11498 } 11499 hwrm_req_drop(bp, req); 11500 return next_handle; 11501 } 11502 11503 static void bnxt_get_wol_settings(struct bnxt *bp) 11504 { 11505 u16 handle = 0; 11506 11507 bp->wol = 0; 11508 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 11509 return; 11510 11511 do { 11512 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 11513 } while (handle && handle != 0xffff); 11514 } 11515 11516 static bool bnxt_eee_config_ok(struct bnxt *bp) 11517 { 11518 struct ethtool_keee *eee = &bp->eee; 11519 struct bnxt_link_info *link_info = &bp->link_info; 11520 11521 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 11522 return true; 11523 11524 if (eee->eee_enabled) { 11525 __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); 11526 __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp); 11527 11528 _bnxt_fw_to_linkmode(advertising, link_info->advertising); 11529 11530 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 11531 eee->eee_enabled = 0; 11532 return false; 11533 } 11534 if (linkmode_andnot(tmp, eee->advertised, advertising)) { 11535 linkmode_and(eee->advertised, advertising, 11536 eee->supported); 11537 return false; 11538 } 11539 } 11540 return true; 11541 } 11542 11543 static int bnxt_update_phy_setting(struct bnxt *bp) 11544 { 11545 int rc; 11546 bool update_link = false; 11547 bool update_pause = false; 11548 bool update_eee = false; 11549 struct bnxt_link_info *link_info = &bp->link_info; 11550 11551 rc = bnxt_update_link(bp, true); 11552 if (rc) { 11553 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 11554 rc); 11555 return rc; 11556 } 11557 if (!BNXT_SINGLE_PF(bp)) 11558 return 0; 11559 11560 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 11561 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 11562 link_info->req_flow_ctrl) 11563 update_pause = true; 11564 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 11565 link_info->force_pause_setting != link_info->req_flow_ctrl) 11566 update_pause = true; 11567 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 11568 if (BNXT_AUTO_MODE(link_info->auto_mode)) 11569 update_link = true; 11570 if (bnxt_force_speed_updated(link_info)) 11571 update_link = true; 11572 if (link_info->req_duplex != link_info->duplex_setting) 11573 update_link = true; 11574 } else { 11575 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 11576 update_link = true; 11577 if (bnxt_auto_speed_updated(link_info)) 11578 update_link = true; 11579 } 11580 11581 /* The last close may have shutdown the link, so need to call 11582 * PHY_CFG to bring it back up. 11583 */ 11584 if (!BNXT_LINK_IS_UP(bp)) 11585 update_link = true; 11586 11587 if (!bnxt_eee_config_ok(bp)) 11588 update_eee = true; 11589 11590 if (update_link) 11591 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 11592 else if (update_pause) 11593 rc = bnxt_hwrm_set_pause(bp); 11594 if (rc) { 11595 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 11596 rc); 11597 return rc; 11598 } 11599 11600 return rc; 11601 } 11602 11603 /* Common routine to pre-map certain register block to different GRC window. 11604 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 11605 * in PF and 3 windows in VF that can be customized to map in different 11606 * register blocks. 11607 */ 11608 static void bnxt_preset_reg_win(struct bnxt *bp) 11609 { 11610 if (BNXT_PF(bp)) { 11611 /* CAG registers map to GRC window #4 */ 11612 writel(BNXT_CAG_REG_BASE, 11613 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 11614 } 11615 } 11616 11617 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 11618 11619 static int bnxt_reinit_after_abort(struct bnxt *bp) 11620 { 11621 int rc; 11622 11623 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11624 return -EBUSY; 11625 11626 if (bp->dev->reg_state == NETREG_UNREGISTERED) 11627 return -ENODEV; 11628 11629 rc = bnxt_fw_init_one(bp); 11630 if (!rc) { 11631 bnxt_clear_int_mode(bp); 11632 rc = bnxt_init_int_mode(bp); 11633 if (!rc) { 11634 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 11635 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 11636 } 11637 } 11638 return rc; 11639 } 11640 11641 static void bnxt_cfg_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 11642 { 11643 struct bnxt_ntuple_filter *ntp_fltr; 11644 struct bnxt_l2_filter *l2_fltr; 11645 11646 if (list_empty(&fltr->list)) 11647 return; 11648 11649 if (fltr->type == BNXT_FLTR_TYPE_NTUPLE) { 11650 ntp_fltr = container_of(fltr, struct bnxt_ntuple_filter, base); 11651 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0]; 11652 atomic_inc(&l2_fltr->refcnt); 11653 ntp_fltr->l2_fltr = l2_fltr; 11654 if (bnxt_hwrm_cfa_ntuple_filter_alloc(bp, ntp_fltr)) { 11655 bnxt_del_ntp_filter(bp, ntp_fltr); 11656 netdev_err(bp->dev, "restoring previously configured ntuple filter id %d failed\n", 11657 fltr->sw_id); 11658 } 11659 } else if (fltr->type == BNXT_FLTR_TYPE_L2) { 11660 l2_fltr = container_of(fltr, struct bnxt_l2_filter, base); 11661 if (bnxt_hwrm_l2_filter_alloc(bp, l2_fltr)) { 11662 bnxt_del_l2_filter(bp, l2_fltr); 11663 netdev_err(bp->dev, "restoring previously configured l2 filter id %d failed\n", 11664 fltr->sw_id); 11665 } 11666 } 11667 } 11668 11669 static void bnxt_cfg_usr_fltrs(struct bnxt *bp) 11670 { 11671 struct bnxt_filter_base *usr_fltr, *tmp; 11672 11673 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) 11674 bnxt_cfg_one_usr_fltr(bp, usr_fltr); 11675 } 11676 11677 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 11678 { 11679 int rc = 0; 11680 11681 bnxt_preset_reg_win(bp); 11682 netif_carrier_off(bp->dev); 11683 if (irq_re_init) { 11684 /* Reserve rings now if none were reserved at driver probe. */ 11685 rc = bnxt_init_dflt_ring_mode(bp); 11686 if (rc) { 11687 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 11688 return rc; 11689 } 11690 } 11691 rc = bnxt_reserve_rings(bp, irq_re_init); 11692 if (rc) 11693 return rc; 11694 if ((bp->flags & BNXT_FLAG_RFS) && 11695 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 11696 /* disable RFS if falling back to INTA */ 11697 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 11698 bp->flags &= ~BNXT_FLAG_RFS; 11699 } 11700 11701 rc = bnxt_alloc_mem(bp, irq_re_init); 11702 if (rc) { 11703 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 11704 goto open_err_free_mem; 11705 } 11706 11707 if (irq_re_init) { 11708 bnxt_init_napi(bp); 11709 rc = bnxt_request_irq(bp); 11710 if (rc) { 11711 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 11712 goto open_err_irq; 11713 } 11714 } 11715 11716 rc = bnxt_init_nic(bp, irq_re_init); 11717 if (rc) { 11718 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 11719 goto open_err_irq; 11720 } 11721 11722 bnxt_enable_napi(bp); 11723 bnxt_debug_dev_init(bp); 11724 11725 if (link_re_init) { 11726 mutex_lock(&bp->link_lock); 11727 rc = bnxt_update_phy_setting(bp); 11728 mutex_unlock(&bp->link_lock); 11729 if (rc) { 11730 netdev_warn(bp->dev, "failed to update phy settings\n"); 11731 if (BNXT_SINGLE_PF(bp)) { 11732 bp->link_info.phy_retry = true; 11733 bp->link_info.phy_retry_expires = 11734 jiffies + 5 * HZ; 11735 } 11736 } 11737 } 11738 11739 if (irq_re_init) 11740 udp_tunnel_nic_reset_ntf(bp->dev); 11741 11742 if (bp->tx_nr_rings_xdp < num_possible_cpus()) { 11743 if (!static_key_enabled(&bnxt_xdp_locking_key)) 11744 static_branch_enable(&bnxt_xdp_locking_key); 11745 } else if (static_key_enabled(&bnxt_xdp_locking_key)) { 11746 static_branch_disable(&bnxt_xdp_locking_key); 11747 } 11748 set_bit(BNXT_STATE_OPEN, &bp->state); 11749 bnxt_enable_int(bp); 11750 /* Enable TX queues */ 11751 bnxt_tx_enable(bp); 11752 mod_timer(&bp->timer, jiffies + bp->current_interval); 11753 /* Poll link status and check for SFP+ module status */ 11754 mutex_lock(&bp->link_lock); 11755 bnxt_get_port_module_status(bp); 11756 mutex_unlock(&bp->link_lock); 11757 11758 /* VF-reps may need to be re-opened after the PF is re-opened */ 11759 if (BNXT_PF(bp)) 11760 bnxt_vf_reps_open(bp); 11761 bnxt_ptp_init_rtc(bp, true); 11762 bnxt_ptp_cfg_tstamp_filters(bp); 11763 bnxt_cfg_usr_fltrs(bp); 11764 return 0; 11765 11766 open_err_irq: 11767 bnxt_del_napi(bp); 11768 11769 open_err_free_mem: 11770 bnxt_free_skbs(bp); 11771 bnxt_free_irq(bp); 11772 bnxt_free_mem(bp, true); 11773 return rc; 11774 } 11775 11776 /* rtnl_lock held */ 11777 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 11778 { 11779 int rc = 0; 11780 11781 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 11782 rc = -EIO; 11783 if (!rc) 11784 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 11785 if (rc) { 11786 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 11787 dev_close(bp->dev); 11788 } 11789 return rc; 11790 } 11791 11792 /* rtnl_lock held, open the NIC half way by allocating all resources, but 11793 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 11794 * self tests. 11795 */ 11796 int bnxt_half_open_nic(struct bnxt *bp) 11797 { 11798 int rc = 0; 11799 11800 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 11801 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 11802 rc = -ENODEV; 11803 goto half_open_err; 11804 } 11805 11806 rc = bnxt_alloc_mem(bp, true); 11807 if (rc) { 11808 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 11809 goto half_open_err; 11810 } 11811 bnxt_init_napi(bp); 11812 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 11813 rc = bnxt_init_nic(bp, true); 11814 if (rc) { 11815 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 11816 bnxt_del_napi(bp); 11817 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 11818 goto half_open_err; 11819 } 11820 return 0; 11821 11822 half_open_err: 11823 bnxt_free_skbs(bp); 11824 bnxt_free_mem(bp, true); 11825 dev_close(bp->dev); 11826 return rc; 11827 } 11828 11829 /* rtnl_lock held, this call can only be made after a previous successful 11830 * call to bnxt_half_open_nic(). 11831 */ 11832 void bnxt_half_close_nic(struct bnxt *bp) 11833 { 11834 bnxt_hwrm_resource_free(bp, false, true); 11835 bnxt_del_napi(bp); 11836 bnxt_free_skbs(bp); 11837 bnxt_free_mem(bp, true); 11838 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 11839 } 11840 11841 void bnxt_reenable_sriov(struct bnxt *bp) 11842 { 11843 if (BNXT_PF(bp)) { 11844 struct bnxt_pf_info *pf = &bp->pf; 11845 int n = pf->active_vfs; 11846 11847 if (n) 11848 bnxt_cfg_hw_sriov(bp, &n, true); 11849 } 11850 } 11851 11852 static int bnxt_open(struct net_device *dev) 11853 { 11854 struct bnxt *bp = netdev_priv(dev); 11855 int rc; 11856 11857 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 11858 rc = bnxt_reinit_after_abort(bp); 11859 if (rc) { 11860 if (rc == -EBUSY) 11861 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 11862 else 11863 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 11864 return -ENODEV; 11865 } 11866 } 11867 11868 rc = bnxt_hwrm_if_change(bp, true); 11869 if (rc) 11870 return rc; 11871 11872 rc = __bnxt_open_nic(bp, true, true); 11873 if (rc) { 11874 bnxt_hwrm_if_change(bp, false); 11875 } else { 11876 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 11877 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11878 bnxt_ulp_start(bp, 0); 11879 bnxt_reenable_sriov(bp); 11880 } 11881 } 11882 } 11883 11884 return rc; 11885 } 11886 11887 static bool bnxt_drv_busy(struct bnxt *bp) 11888 { 11889 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 11890 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 11891 } 11892 11893 static void bnxt_get_ring_stats(struct bnxt *bp, 11894 struct rtnl_link_stats64 *stats); 11895 11896 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 11897 bool link_re_init) 11898 { 11899 /* Close the VF-reps before closing PF */ 11900 if (BNXT_PF(bp)) 11901 bnxt_vf_reps_close(bp); 11902 11903 /* Change device state to avoid TX queue wake up's */ 11904 bnxt_tx_disable(bp); 11905 11906 clear_bit(BNXT_STATE_OPEN, &bp->state); 11907 smp_mb__after_atomic(); 11908 while (bnxt_drv_busy(bp)) 11909 msleep(20); 11910 11911 /* Flush rings and disable interrupts */ 11912 bnxt_shutdown_nic(bp, irq_re_init); 11913 11914 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 11915 11916 bnxt_debug_dev_exit(bp); 11917 bnxt_disable_napi(bp); 11918 del_timer_sync(&bp->timer); 11919 bnxt_free_skbs(bp); 11920 11921 /* Save ring stats before shutdown */ 11922 if (bp->bnapi && irq_re_init) { 11923 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 11924 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev); 11925 } 11926 if (irq_re_init) { 11927 bnxt_free_irq(bp); 11928 bnxt_del_napi(bp); 11929 } 11930 bnxt_free_mem(bp, irq_re_init); 11931 } 11932 11933 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 11934 { 11935 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11936 /* If we get here, it means firmware reset is in progress 11937 * while we are trying to close. We can safely proceed with 11938 * the close because we are holding rtnl_lock(). Some firmware 11939 * messages may fail as we proceed to close. We set the 11940 * ABORT_ERR flag here so that the FW reset thread will later 11941 * abort when it gets the rtnl_lock() and sees the flag. 11942 */ 11943 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 11944 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 11945 } 11946 11947 #ifdef CONFIG_BNXT_SRIOV 11948 if (bp->sriov_cfg) { 11949 int rc; 11950 11951 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 11952 !bp->sriov_cfg, 11953 BNXT_SRIOV_CFG_WAIT_TMO); 11954 if (!rc) 11955 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n"); 11956 else if (rc < 0) 11957 netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n"); 11958 } 11959 #endif 11960 __bnxt_close_nic(bp, irq_re_init, link_re_init); 11961 } 11962 11963 static int bnxt_close(struct net_device *dev) 11964 { 11965 struct bnxt *bp = netdev_priv(dev); 11966 11967 bnxt_close_nic(bp, true, true); 11968 bnxt_hwrm_shutdown_link(bp); 11969 bnxt_hwrm_if_change(bp, false); 11970 return 0; 11971 } 11972 11973 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 11974 u16 *val) 11975 { 11976 struct hwrm_port_phy_mdio_read_output *resp; 11977 struct hwrm_port_phy_mdio_read_input *req; 11978 int rc; 11979 11980 if (bp->hwrm_spec_code < 0x10a00) 11981 return -EOPNOTSUPP; 11982 11983 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 11984 if (rc) 11985 return rc; 11986 11987 req->port_id = cpu_to_le16(bp->pf.port_id); 11988 req->phy_addr = phy_addr; 11989 req->reg_addr = cpu_to_le16(reg & 0x1f); 11990 if (mdio_phy_id_is_c45(phy_addr)) { 11991 req->cl45_mdio = 1; 11992 req->phy_addr = mdio_phy_id_prtad(phy_addr); 11993 req->dev_addr = mdio_phy_id_devad(phy_addr); 11994 req->reg_addr = cpu_to_le16(reg); 11995 } 11996 11997 resp = hwrm_req_hold(bp, req); 11998 rc = hwrm_req_send(bp, req); 11999 if (!rc) 12000 *val = le16_to_cpu(resp->reg_data); 12001 hwrm_req_drop(bp, req); 12002 return rc; 12003 } 12004 12005 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 12006 u16 val) 12007 { 12008 struct hwrm_port_phy_mdio_write_input *req; 12009 int rc; 12010 12011 if (bp->hwrm_spec_code < 0x10a00) 12012 return -EOPNOTSUPP; 12013 12014 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 12015 if (rc) 12016 return rc; 12017 12018 req->port_id = cpu_to_le16(bp->pf.port_id); 12019 req->phy_addr = phy_addr; 12020 req->reg_addr = cpu_to_le16(reg & 0x1f); 12021 if (mdio_phy_id_is_c45(phy_addr)) { 12022 req->cl45_mdio = 1; 12023 req->phy_addr = mdio_phy_id_prtad(phy_addr); 12024 req->dev_addr = mdio_phy_id_devad(phy_addr); 12025 req->reg_addr = cpu_to_le16(reg); 12026 } 12027 req->reg_data = cpu_to_le16(val); 12028 12029 return hwrm_req_send(bp, req); 12030 } 12031 12032 /* rtnl_lock held */ 12033 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 12034 { 12035 struct mii_ioctl_data *mdio = if_mii(ifr); 12036 struct bnxt *bp = netdev_priv(dev); 12037 int rc; 12038 12039 switch (cmd) { 12040 case SIOCGMIIPHY: 12041 mdio->phy_id = bp->link_info.phy_addr; 12042 12043 fallthrough; 12044 case SIOCGMIIREG: { 12045 u16 mii_regval = 0; 12046 12047 if (!netif_running(dev)) 12048 return -EAGAIN; 12049 12050 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 12051 &mii_regval); 12052 mdio->val_out = mii_regval; 12053 return rc; 12054 } 12055 12056 case SIOCSMIIREG: 12057 if (!netif_running(dev)) 12058 return -EAGAIN; 12059 12060 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 12061 mdio->val_in); 12062 12063 case SIOCSHWTSTAMP: 12064 return bnxt_hwtstamp_set(dev, ifr); 12065 12066 case SIOCGHWTSTAMP: 12067 return bnxt_hwtstamp_get(dev, ifr); 12068 12069 default: 12070 /* do nothing */ 12071 break; 12072 } 12073 return -EOPNOTSUPP; 12074 } 12075 12076 static void bnxt_get_ring_stats(struct bnxt *bp, 12077 struct rtnl_link_stats64 *stats) 12078 { 12079 int i; 12080 12081 for (i = 0; i < bp->cp_nr_rings; i++) { 12082 struct bnxt_napi *bnapi = bp->bnapi[i]; 12083 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 12084 u64 *sw = cpr->stats.sw_stats; 12085 12086 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 12087 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 12088 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 12089 12090 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 12091 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 12092 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 12093 12094 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 12095 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 12096 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 12097 12098 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 12099 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 12100 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 12101 12102 stats->rx_missed_errors += 12103 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 12104 12105 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 12106 12107 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 12108 12109 stats->rx_dropped += 12110 cpr->sw_stats.rx.rx_netpoll_discards + 12111 cpr->sw_stats.rx.rx_oom_discards; 12112 } 12113 } 12114 12115 static void bnxt_add_prev_stats(struct bnxt *bp, 12116 struct rtnl_link_stats64 *stats) 12117 { 12118 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 12119 12120 stats->rx_packets += prev_stats->rx_packets; 12121 stats->tx_packets += prev_stats->tx_packets; 12122 stats->rx_bytes += prev_stats->rx_bytes; 12123 stats->tx_bytes += prev_stats->tx_bytes; 12124 stats->rx_missed_errors += prev_stats->rx_missed_errors; 12125 stats->multicast += prev_stats->multicast; 12126 stats->rx_dropped += prev_stats->rx_dropped; 12127 stats->tx_dropped += prev_stats->tx_dropped; 12128 } 12129 12130 static void 12131 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 12132 { 12133 struct bnxt *bp = netdev_priv(dev); 12134 12135 set_bit(BNXT_STATE_READ_STATS, &bp->state); 12136 /* Make sure bnxt_close_nic() sees that we are reading stats before 12137 * we check the BNXT_STATE_OPEN flag. 12138 */ 12139 smp_mb__after_atomic(); 12140 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 12141 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 12142 *stats = bp->net_stats_prev; 12143 return; 12144 } 12145 12146 bnxt_get_ring_stats(bp, stats); 12147 bnxt_add_prev_stats(bp, stats); 12148 12149 if (bp->flags & BNXT_FLAG_PORT_STATS) { 12150 u64 *rx = bp->port_stats.sw_stats; 12151 u64 *tx = bp->port_stats.sw_stats + 12152 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 12153 12154 stats->rx_crc_errors = 12155 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 12156 stats->rx_frame_errors = 12157 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 12158 stats->rx_length_errors = 12159 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 12160 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 12161 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 12162 stats->rx_errors = 12163 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 12164 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 12165 stats->collisions = 12166 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 12167 stats->tx_fifo_errors = 12168 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 12169 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 12170 } 12171 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 12172 } 12173 12174 static void bnxt_get_one_ring_err_stats(struct bnxt *bp, 12175 struct bnxt_total_ring_err_stats *stats, 12176 struct bnxt_cp_ring_info *cpr) 12177 { 12178 struct bnxt_sw_stats *sw_stats = &cpr->sw_stats; 12179 u64 *hw_stats = cpr->stats.sw_stats; 12180 12181 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors; 12182 stats->rx_total_resets += sw_stats->rx.rx_resets; 12183 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors; 12184 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards; 12185 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards; 12186 stats->rx_total_ring_discards += 12187 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts); 12188 stats->tx_total_resets += sw_stats->tx.tx_resets; 12189 stats->tx_total_ring_discards += 12190 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts); 12191 stats->total_missed_irqs += sw_stats->cmn.missed_irqs; 12192 } 12193 12194 void bnxt_get_ring_err_stats(struct bnxt *bp, 12195 struct bnxt_total_ring_err_stats *stats) 12196 { 12197 int i; 12198 12199 for (i = 0; i < bp->cp_nr_rings; i++) 12200 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring); 12201 } 12202 12203 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 12204 { 12205 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 12206 struct net_device *dev = bp->dev; 12207 struct netdev_hw_addr *ha; 12208 u8 *haddr; 12209 int mc_count = 0; 12210 bool update = false; 12211 int off = 0; 12212 12213 netdev_for_each_mc_addr(ha, dev) { 12214 if (mc_count >= BNXT_MAX_MC_ADDRS) { 12215 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 12216 vnic->mc_list_count = 0; 12217 return false; 12218 } 12219 haddr = ha->addr; 12220 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 12221 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 12222 update = true; 12223 } 12224 off += ETH_ALEN; 12225 mc_count++; 12226 } 12227 if (mc_count) 12228 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 12229 12230 if (mc_count != vnic->mc_list_count) { 12231 vnic->mc_list_count = mc_count; 12232 update = true; 12233 } 12234 return update; 12235 } 12236 12237 static bool bnxt_uc_list_updated(struct bnxt *bp) 12238 { 12239 struct net_device *dev = bp->dev; 12240 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 12241 struct netdev_hw_addr *ha; 12242 int off = 0; 12243 12244 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 12245 return true; 12246 12247 netdev_for_each_uc_addr(ha, dev) { 12248 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 12249 return true; 12250 12251 off += ETH_ALEN; 12252 } 12253 return false; 12254 } 12255 12256 static void bnxt_set_rx_mode(struct net_device *dev) 12257 { 12258 struct bnxt *bp = netdev_priv(dev); 12259 struct bnxt_vnic_info *vnic; 12260 bool mc_update = false; 12261 bool uc_update; 12262 u32 mask; 12263 12264 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 12265 return; 12266 12267 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 12268 mask = vnic->rx_mask; 12269 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 12270 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 12271 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 12272 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 12273 12274 if (dev->flags & IFF_PROMISC) 12275 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 12276 12277 uc_update = bnxt_uc_list_updated(bp); 12278 12279 if (dev->flags & IFF_BROADCAST) 12280 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 12281 if (dev->flags & IFF_ALLMULTI) { 12282 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 12283 vnic->mc_list_count = 0; 12284 } else if (dev->flags & IFF_MULTICAST) { 12285 mc_update = bnxt_mc_list_updated(bp, &mask); 12286 } 12287 12288 if (mask != vnic->rx_mask || uc_update || mc_update) { 12289 vnic->rx_mask = mask; 12290 12291 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 12292 } 12293 } 12294 12295 static int bnxt_cfg_rx_mode(struct bnxt *bp) 12296 { 12297 struct net_device *dev = bp->dev; 12298 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 12299 struct netdev_hw_addr *ha; 12300 int i, off = 0, rc; 12301 bool uc_update; 12302 12303 netif_addr_lock_bh(dev); 12304 uc_update = bnxt_uc_list_updated(bp); 12305 netif_addr_unlock_bh(dev); 12306 12307 if (!uc_update) 12308 goto skip_uc; 12309 12310 for (i = 1; i < vnic->uc_filter_count; i++) { 12311 struct bnxt_l2_filter *fltr = vnic->l2_filters[i]; 12312 12313 bnxt_hwrm_l2_filter_free(bp, fltr); 12314 bnxt_del_l2_filter(bp, fltr); 12315 } 12316 12317 vnic->uc_filter_count = 1; 12318 12319 netif_addr_lock_bh(dev); 12320 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 12321 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 12322 } else { 12323 netdev_for_each_uc_addr(ha, dev) { 12324 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 12325 off += ETH_ALEN; 12326 vnic->uc_filter_count++; 12327 } 12328 } 12329 netif_addr_unlock_bh(dev); 12330 12331 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 12332 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 12333 if (rc) { 12334 if (BNXT_VF(bp) && rc == -ENODEV) { 12335 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 12336 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 12337 else 12338 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 12339 rc = 0; 12340 } else { 12341 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 12342 } 12343 vnic->uc_filter_count = i; 12344 return rc; 12345 } 12346 } 12347 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 12348 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 12349 12350 skip_uc: 12351 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 12352 !bnxt_promisc_ok(bp)) 12353 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 12354 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 12355 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 12356 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 12357 rc); 12358 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 12359 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 12360 vnic->mc_list_count = 0; 12361 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 12362 } 12363 if (rc) 12364 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 12365 rc); 12366 12367 return rc; 12368 } 12369 12370 static bool bnxt_can_reserve_rings(struct bnxt *bp) 12371 { 12372 #ifdef CONFIG_BNXT_SRIOV 12373 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 12374 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 12375 12376 /* No minimum rings were provisioned by the PF. Don't 12377 * reserve rings by default when device is down. 12378 */ 12379 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 12380 return true; 12381 12382 if (!netif_running(bp->dev)) 12383 return false; 12384 } 12385 #endif 12386 return true; 12387 } 12388 12389 /* If the chip and firmware supports RFS */ 12390 static bool bnxt_rfs_supported(struct bnxt *bp) 12391 { 12392 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 12393 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 12394 return true; 12395 return false; 12396 } 12397 /* 212 firmware is broken for aRFS */ 12398 if (BNXT_FW_MAJ(bp) == 212) 12399 return false; 12400 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 12401 return true; 12402 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) 12403 return true; 12404 return false; 12405 } 12406 12407 /* If runtime conditions support RFS */ 12408 static bool bnxt_rfs_capable(struct bnxt *bp) 12409 { 12410 struct bnxt_hw_rings hwr = {0}; 12411 int max_vnics, max_rss_ctxs; 12412 12413 hwr.rss_ctx = 1; 12414 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) { 12415 /* 2 VNICS: default + Ntuple */ 12416 hwr.vnic = 2; 12417 hwr.rss_ctx = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) * 12418 hwr.vnic; 12419 goto check_reserve_vnic; 12420 } 12421 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 12422 return bnxt_rfs_supported(bp); 12423 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings) 12424 return false; 12425 12426 hwr.vnic = 1 + bp->rx_nr_rings; 12427 check_reserve_vnic: 12428 max_vnics = bnxt_get_max_func_vnics(bp); 12429 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 12430 12431 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 12432 !(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)) 12433 hwr.rss_ctx = hwr.vnic; 12434 12435 if (hwr.vnic > max_vnics || hwr.rss_ctx > max_rss_ctxs) { 12436 if (bp->rx_nr_rings > 1) 12437 netdev_warn(bp->dev, 12438 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 12439 min(max_rss_ctxs - 1, max_vnics - 1)); 12440 return false; 12441 } 12442 12443 if (!BNXT_NEW_RM(bp)) 12444 return true; 12445 12446 if (hwr.vnic == bp->hw_resc.resv_vnics && 12447 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs) 12448 return true; 12449 12450 bnxt_hwrm_reserve_rings(bp, &hwr); 12451 if (hwr.vnic <= bp->hw_resc.resv_vnics && 12452 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs) 12453 return true; 12454 12455 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 12456 hwr.vnic = 1; 12457 hwr.rss_ctx = 0; 12458 bnxt_hwrm_reserve_rings(bp, &hwr); 12459 return false; 12460 } 12461 12462 static netdev_features_t bnxt_fix_features(struct net_device *dev, 12463 netdev_features_t features) 12464 { 12465 struct bnxt *bp = netdev_priv(dev); 12466 netdev_features_t vlan_features; 12467 12468 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 12469 features &= ~NETIF_F_NTUPLE; 12470 12471 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) 12472 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 12473 12474 if (!(features & NETIF_F_GRO)) 12475 features &= ~NETIF_F_GRO_HW; 12476 12477 if (features & NETIF_F_GRO_HW) 12478 features &= ~NETIF_F_LRO; 12479 12480 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 12481 * turned on or off together. 12482 */ 12483 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 12484 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 12485 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12486 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 12487 else if (vlan_features) 12488 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 12489 } 12490 #ifdef CONFIG_BNXT_SRIOV 12491 if (BNXT_VF(bp) && bp->vf.vlan) 12492 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 12493 #endif 12494 return features; 12495 } 12496 12497 static int bnxt_reinit_features(struct bnxt *bp, bool irq_re_init, 12498 bool link_re_init, u32 flags, bool update_tpa) 12499 { 12500 bnxt_close_nic(bp, irq_re_init, link_re_init); 12501 bp->flags = flags; 12502 if (update_tpa) 12503 bnxt_set_ring_params(bp); 12504 return bnxt_open_nic(bp, irq_re_init, link_re_init); 12505 } 12506 12507 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 12508 { 12509 bool update_tpa = false, update_ntuple = false; 12510 struct bnxt *bp = netdev_priv(dev); 12511 u32 flags = bp->flags; 12512 u32 changes; 12513 int rc = 0; 12514 bool re_init = false; 12515 12516 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 12517 if (features & NETIF_F_GRO_HW) 12518 flags |= BNXT_FLAG_GRO; 12519 else if (features & NETIF_F_LRO) 12520 flags |= BNXT_FLAG_LRO; 12521 12522 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 12523 flags &= ~BNXT_FLAG_TPA; 12524 12525 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12526 flags |= BNXT_FLAG_STRIP_VLAN; 12527 12528 if (features & NETIF_F_NTUPLE) 12529 flags |= BNXT_FLAG_RFS; 12530 else 12531 bnxt_clear_usr_fltrs(bp, true); 12532 12533 changes = flags ^ bp->flags; 12534 if (changes & BNXT_FLAG_TPA) { 12535 update_tpa = true; 12536 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 12537 (flags & BNXT_FLAG_TPA) == 0 || 12538 (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 12539 re_init = true; 12540 } 12541 12542 if (changes & ~BNXT_FLAG_TPA) 12543 re_init = true; 12544 12545 if (changes & BNXT_FLAG_RFS) 12546 update_ntuple = true; 12547 12548 if (flags != bp->flags) { 12549 u32 old_flags = bp->flags; 12550 12551 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 12552 bp->flags = flags; 12553 if (update_tpa) 12554 bnxt_set_ring_params(bp); 12555 return rc; 12556 } 12557 12558 if (update_ntuple) 12559 return bnxt_reinit_features(bp, true, false, flags, update_tpa); 12560 12561 if (re_init) 12562 return bnxt_reinit_features(bp, false, false, flags, update_tpa); 12563 12564 if (update_tpa) { 12565 bp->flags = flags; 12566 rc = bnxt_set_tpa(bp, 12567 (flags & BNXT_FLAG_TPA) ? 12568 true : false); 12569 if (rc) 12570 bp->flags = old_flags; 12571 } 12572 } 12573 return rc; 12574 } 12575 12576 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 12577 u8 **nextp) 12578 { 12579 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 12580 struct hop_jumbo_hdr *jhdr; 12581 int hdr_count = 0; 12582 u8 *nexthdr; 12583 int start; 12584 12585 /* Check that there are at most 2 IPv6 extension headers, no 12586 * fragment header, and each is <= 64 bytes. 12587 */ 12588 start = nw_off + sizeof(*ip6h); 12589 nexthdr = &ip6h->nexthdr; 12590 while (ipv6_ext_hdr(*nexthdr)) { 12591 struct ipv6_opt_hdr *hp; 12592 int hdrlen; 12593 12594 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 12595 *nexthdr == NEXTHDR_FRAGMENT) 12596 return false; 12597 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 12598 skb_headlen(skb), NULL); 12599 if (!hp) 12600 return false; 12601 if (*nexthdr == NEXTHDR_AUTH) 12602 hdrlen = ipv6_authlen(hp); 12603 else 12604 hdrlen = ipv6_optlen(hp); 12605 12606 if (hdrlen > 64) 12607 return false; 12608 12609 /* The ext header may be a hop-by-hop header inserted for 12610 * big TCP purposes. This will be removed before sending 12611 * from NIC, so do not count it. 12612 */ 12613 if (*nexthdr == NEXTHDR_HOP) { 12614 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) 12615 goto increment_hdr; 12616 12617 jhdr = (struct hop_jumbo_hdr *)hp; 12618 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || 12619 jhdr->nexthdr != IPPROTO_TCP) 12620 goto increment_hdr; 12621 12622 goto next_hdr; 12623 } 12624 increment_hdr: 12625 hdr_count++; 12626 next_hdr: 12627 nexthdr = &hp->nexthdr; 12628 start += hdrlen; 12629 } 12630 if (nextp) { 12631 /* Caller will check inner protocol */ 12632 if (skb->encapsulation) { 12633 *nextp = nexthdr; 12634 return true; 12635 } 12636 *nextp = NULL; 12637 } 12638 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 12639 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 12640 } 12641 12642 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 12643 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 12644 { 12645 struct udphdr *uh = udp_hdr(skb); 12646 __be16 udp_port = uh->dest; 12647 12648 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port && 12649 udp_port != bp->vxlan_gpe_port) 12650 return false; 12651 if (skb->inner_protocol == htons(ETH_P_TEB)) { 12652 struct ethhdr *eh = inner_eth_hdr(skb); 12653 12654 switch (eh->h_proto) { 12655 case htons(ETH_P_IP): 12656 return true; 12657 case htons(ETH_P_IPV6): 12658 return bnxt_exthdr_check(bp, skb, 12659 skb_inner_network_offset(skb), 12660 NULL); 12661 } 12662 } else if (skb->inner_protocol == htons(ETH_P_IP)) { 12663 return true; 12664 } else if (skb->inner_protocol == htons(ETH_P_IPV6)) { 12665 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 12666 NULL); 12667 } 12668 return false; 12669 } 12670 12671 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 12672 { 12673 switch (l4_proto) { 12674 case IPPROTO_UDP: 12675 return bnxt_udp_tunl_check(bp, skb); 12676 case IPPROTO_IPIP: 12677 return true; 12678 case IPPROTO_GRE: { 12679 switch (skb->inner_protocol) { 12680 default: 12681 return false; 12682 case htons(ETH_P_IP): 12683 return true; 12684 case htons(ETH_P_IPV6): 12685 fallthrough; 12686 } 12687 } 12688 case IPPROTO_IPV6: 12689 /* Check ext headers of inner ipv6 */ 12690 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 12691 NULL); 12692 } 12693 return false; 12694 } 12695 12696 static netdev_features_t bnxt_features_check(struct sk_buff *skb, 12697 struct net_device *dev, 12698 netdev_features_t features) 12699 { 12700 struct bnxt *bp = netdev_priv(dev); 12701 u8 *l4_proto; 12702 12703 features = vlan_features_check(skb, features); 12704 switch (vlan_get_protocol(skb)) { 12705 case htons(ETH_P_IP): 12706 if (!skb->encapsulation) 12707 return features; 12708 l4_proto = &ip_hdr(skb)->protocol; 12709 if (bnxt_tunl_check(bp, skb, *l4_proto)) 12710 return features; 12711 break; 12712 case htons(ETH_P_IPV6): 12713 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 12714 &l4_proto)) 12715 break; 12716 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 12717 return features; 12718 break; 12719 } 12720 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 12721 } 12722 12723 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 12724 u32 *reg_buf) 12725 { 12726 struct hwrm_dbg_read_direct_output *resp; 12727 struct hwrm_dbg_read_direct_input *req; 12728 __le32 *dbg_reg_buf; 12729 dma_addr_t mapping; 12730 int rc, i; 12731 12732 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 12733 if (rc) 12734 return rc; 12735 12736 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 12737 &mapping); 12738 if (!dbg_reg_buf) { 12739 rc = -ENOMEM; 12740 goto dbg_rd_reg_exit; 12741 } 12742 12743 req->host_dest_addr = cpu_to_le64(mapping); 12744 12745 resp = hwrm_req_hold(bp, req); 12746 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 12747 req->read_len32 = cpu_to_le32(num_words); 12748 12749 rc = hwrm_req_send(bp, req); 12750 if (rc || resp->error_code) { 12751 rc = -EIO; 12752 goto dbg_rd_reg_exit; 12753 } 12754 for (i = 0; i < num_words; i++) 12755 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 12756 12757 dbg_rd_reg_exit: 12758 hwrm_req_drop(bp, req); 12759 return rc; 12760 } 12761 12762 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 12763 u32 ring_id, u32 *prod, u32 *cons) 12764 { 12765 struct hwrm_dbg_ring_info_get_output *resp; 12766 struct hwrm_dbg_ring_info_get_input *req; 12767 int rc; 12768 12769 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 12770 if (rc) 12771 return rc; 12772 12773 req->ring_type = ring_type; 12774 req->fw_ring_id = cpu_to_le32(ring_id); 12775 resp = hwrm_req_hold(bp, req); 12776 rc = hwrm_req_send(bp, req); 12777 if (!rc) { 12778 *prod = le32_to_cpu(resp->producer_index); 12779 *cons = le32_to_cpu(resp->consumer_index); 12780 } 12781 hwrm_req_drop(bp, req); 12782 return rc; 12783 } 12784 12785 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 12786 { 12787 struct bnxt_tx_ring_info *txr; 12788 int i = bnapi->index, j; 12789 12790 bnxt_for_each_napi_tx(j, bnapi, txr) 12791 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 12792 i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 12793 txr->tx_cons); 12794 } 12795 12796 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 12797 { 12798 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 12799 int i = bnapi->index; 12800 12801 if (!rxr) 12802 return; 12803 12804 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 12805 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 12806 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 12807 rxr->rx_sw_agg_prod); 12808 } 12809 12810 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 12811 { 12812 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 12813 int i = bnapi->index; 12814 12815 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 12816 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 12817 } 12818 12819 static void bnxt_dbg_dump_states(struct bnxt *bp) 12820 { 12821 int i; 12822 struct bnxt_napi *bnapi; 12823 12824 for (i = 0; i < bp->cp_nr_rings; i++) { 12825 bnapi = bp->bnapi[i]; 12826 if (netif_msg_drv(bp)) { 12827 bnxt_dump_tx_sw_state(bnapi); 12828 bnxt_dump_rx_sw_state(bnapi); 12829 bnxt_dump_cp_sw_state(bnapi); 12830 } 12831 } 12832 } 12833 12834 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 12835 { 12836 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 12837 struct hwrm_ring_reset_input *req; 12838 struct bnxt_napi *bnapi = rxr->bnapi; 12839 struct bnxt_cp_ring_info *cpr; 12840 u16 cp_ring_id; 12841 int rc; 12842 12843 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 12844 if (rc) 12845 return rc; 12846 12847 cpr = &bnapi->cp_ring; 12848 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 12849 req->cmpl_ring = cpu_to_le16(cp_ring_id); 12850 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 12851 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 12852 return hwrm_req_send_silent(bp, req); 12853 } 12854 12855 static void bnxt_reset_task(struct bnxt *bp, bool silent) 12856 { 12857 if (!silent) 12858 bnxt_dbg_dump_states(bp); 12859 if (netif_running(bp->dev)) { 12860 int rc; 12861 12862 if (silent) { 12863 bnxt_close_nic(bp, false, false); 12864 bnxt_open_nic(bp, false, false); 12865 } else { 12866 bnxt_ulp_stop(bp); 12867 bnxt_close_nic(bp, true, false); 12868 rc = bnxt_open_nic(bp, true, false); 12869 bnxt_ulp_start(bp, rc); 12870 } 12871 } 12872 } 12873 12874 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 12875 { 12876 struct bnxt *bp = netdev_priv(dev); 12877 12878 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 12879 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 12880 } 12881 12882 static void bnxt_fw_health_check(struct bnxt *bp) 12883 { 12884 struct bnxt_fw_health *fw_health = bp->fw_health; 12885 struct pci_dev *pdev = bp->pdev; 12886 u32 val; 12887 12888 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 12889 return; 12890 12891 /* Make sure it is enabled before checking the tmr_counter. */ 12892 smp_rmb(); 12893 if (fw_health->tmr_counter) { 12894 fw_health->tmr_counter--; 12895 return; 12896 } 12897 12898 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 12899 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) { 12900 fw_health->arrests++; 12901 goto fw_reset; 12902 } 12903 12904 fw_health->last_fw_heartbeat = val; 12905 12906 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 12907 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) { 12908 fw_health->discoveries++; 12909 goto fw_reset; 12910 } 12911 12912 fw_health->tmr_counter = fw_health->tmr_multiplier; 12913 return; 12914 12915 fw_reset: 12916 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT); 12917 } 12918 12919 static void bnxt_timer(struct timer_list *t) 12920 { 12921 struct bnxt *bp = from_timer(bp, t, timer); 12922 struct net_device *dev = bp->dev; 12923 12924 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 12925 return; 12926 12927 if (atomic_read(&bp->intr_sem) != 0) 12928 goto bnxt_restart_timer; 12929 12930 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 12931 bnxt_fw_health_check(bp); 12932 12933 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) 12934 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT); 12935 12936 if (bnxt_tc_flower_enabled(bp)) 12937 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT); 12938 12939 #ifdef CONFIG_RFS_ACCEL 12940 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) 12941 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 12942 #endif /*CONFIG_RFS_ACCEL*/ 12943 12944 if (bp->link_info.phy_retry) { 12945 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 12946 bp->link_info.phy_retry = false; 12947 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 12948 } else { 12949 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT); 12950 } 12951 } 12952 12953 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 12954 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 12955 12956 if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev)) 12957 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT); 12958 12959 bnxt_restart_timer: 12960 mod_timer(&bp->timer, jiffies + bp->current_interval); 12961 } 12962 12963 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 12964 { 12965 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 12966 * set. If the device is being closed, bnxt_close() may be holding 12967 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 12968 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 12969 */ 12970 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12971 rtnl_lock(); 12972 } 12973 12974 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 12975 { 12976 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 12977 rtnl_unlock(); 12978 } 12979 12980 /* Only called from bnxt_sp_task() */ 12981 static void bnxt_reset(struct bnxt *bp, bool silent) 12982 { 12983 bnxt_rtnl_lock_sp(bp); 12984 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 12985 bnxt_reset_task(bp, silent); 12986 bnxt_rtnl_unlock_sp(bp); 12987 } 12988 12989 /* Only called from bnxt_sp_task() */ 12990 static void bnxt_rx_ring_reset(struct bnxt *bp) 12991 { 12992 int i; 12993 12994 bnxt_rtnl_lock_sp(bp); 12995 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 12996 bnxt_rtnl_unlock_sp(bp); 12997 return; 12998 } 12999 /* Disable and flush TPA before resetting the RX ring */ 13000 if (bp->flags & BNXT_FLAG_TPA) 13001 bnxt_set_tpa(bp, false); 13002 for (i = 0; i < bp->rx_nr_rings; i++) { 13003 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 13004 struct bnxt_cp_ring_info *cpr; 13005 int rc; 13006 13007 if (!rxr->bnapi->in_reset) 13008 continue; 13009 13010 rc = bnxt_hwrm_rx_ring_reset(bp, i); 13011 if (rc) { 13012 if (rc == -EINVAL || rc == -EOPNOTSUPP) 13013 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 13014 else 13015 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 13016 rc); 13017 bnxt_reset_task(bp, true); 13018 break; 13019 } 13020 bnxt_free_one_rx_ring_skbs(bp, i); 13021 rxr->rx_prod = 0; 13022 rxr->rx_agg_prod = 0; 13023 rxr->rx_sw_agg_prod = 0; 13024 rxr->rx_next_cons = 0; 13025 rxr->bnapi->in_reset = false; 13026 bnxt_alloc_one_rx_ring(bp, i); 13027 cpr = &rxr->bnapi->cp_ring; 13028 cpr->sw_stats.rx.rx_resets++; 13029 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13030 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 13031 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 13032 } 13033 if (bp->flags & BNXT_FLAG_TPA) 13034 bnxt_set_tpa(bp, true); 13035 bnxt_rtnl_unlock_sp(bp); 13036 } 13037 13038 static void bnxt_fw_reset_close(struct bnxt *bp) 13039 { 13040 bnxt_ulp_stop(bp); 13041 /* When firmware is in fatal state, quiesce device and disable 13042 * bus master to prevent any potential bad DMAs before freeing 13043 * kernel memory. 13044 */ 13045 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 13046 u16 val = 0; 13047 13048 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 13049 if (val == 0xffff) 13050 bp->fw_reset_min_dsecs = 0; 13051 bnxt_tx_disable(bp); 13052 bnxt_disable_napi(bp); 13053 bnxt_disable_int_sync(bp); 13054 bnxt_free_irq(bp); 13055 bnxt_clear_int_mode(bp); 13056 pci_disable_device(bp->pdev); 13057 } 13058 __bnxt_close_nic(bp, true, false); 13059 bnxt_vf_reps_free(bp); 13060 bnxt_clear_int_mode(bp); 13061 bnxt_hwrm_func_drv_unrgtr(bp); 13062 if (pci_is_enabled(bp->pdev)) 13063 pci_disable_device(bp->pdev); 13064 bnxt_free_ctx_mem(bp); 13065 } 13066 13067 static bool is_bnxt_fw_ok(struct bnxt *bp) 13068 { 13069 struct bnxt_fw_health *fw_health = bp->fw_health; 13070 bool no_heartbeat = false, has_reset = false; 13071 u32 val; 13072 13073 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 13074 if (val == fw_health->last_fw_heartbeat) 13075 no_heartbeat = true; 13076 13077 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 13078 if (val != fw_health->last_fw_reset_cnt) 13079 has_reset = true; 13080 13081 if (!no_heartbeat && has_reset) 13082 return true; 13083 13084 return false; 13085 } 13086 13087 /* rtnl_lock is acquired before calling this function */ 13088 static void bnxt_force_fw_reset(struct bnxt *bp) 13089 { 13090 struct bnxt_fw_health *fw_health = bp->fw_health; 13091 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 13092 u32 wait_dsecs; 13093 13094 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 13095 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 13096 return; 13097 13098 if (ptp) { 13099 spin_lock_bh(&ptp->ptp_lock); 13100 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13101 spin_unlock_bh(&ptp->ptp_lock); 13102 } else { 13103 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13104 } 13105 bnxt_fw_reset_close(bp); 13106 wait_dsecs = fw_health->master_func_wait_dsecs; 13107 if (fw_health->primary) { 13108 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 13109 wait_dsecs = 0; 13110 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 13111 } else { 13112 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 13113 wait_dsecs = fw_health->normal_func_wait_dsecs; 13114 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 13115 } 13116 13117 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 13118 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 13119 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 13120 } 13121 13122 void bnxt_fw_exception(struct bnxt *bp) 13123 { 13124 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 13125 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 13126 bnxt_rtnl_lock_sp(bp); 13127 bnxt_force_fw_reset(bp); 13128 bnxt_rtnl_unlock_sp(bp); 13129 } 13130 13131 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 13132 * < 0 on error. 13133 */ 13134 static int bnxt_get_registered_vfs(struct bnxt *bp) 13135 { 13136 #ifdef CONFIG_BNXT_SRIOV 13137 int rc; 13138 13139 if (!BNXT_PF(bp)) 13140 return 0; 13141 13142 rc = bnxt_hwrm_func_qcfg(bp); 13143 if (rc) { 13144 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 13145 return rc; 13146 } 13147 if (bp->pf.registered_vfs) 13148 return bp->pf.registered_vfs; 13149 if (bp->sriov_cfg) 13150 return 1; 13151 #endif 13152 return 0; 13153 } 13154 13155 void bnxt_fw_reset(struct bnxt *bp) 13156 { 13157 bnxt_rtnl_lock_sp(bp); 13158 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 13159 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 13160 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 13161 int n = 0, tmo; 13162 13163 if (ptp) { 13164 spin_lock_bh(&ptp->ptp_lock); 13165 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13166 spin_unlock_bh(&ptp->ptp_lock); 13167 } else { 13168 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13169 } 13170 if (bp->pf.active_vfs && 13171 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 13172 n = bnxt_get_registered_vfs(bp); 13173 if (n < 0) { 13174 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 13175 n); 13176 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13177 dev_close(bp->dev); 13178 goto fw_reset_exit; 13179 } else if (n > 0) { 13180 u16 vf_tmo_dsecs = n * 10; 13181 13182 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 13183 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 13184 bp->fw_reset_state = 13185 BNXT_FW_RESET_STATE_POLL_VF; 13186 bnxt_queue_fw_reset_work(bp, HZ / 10); 13187 goto fw_reset_exit; 13188 } 13189 bnxt_fw_reset_close(bp); 13190 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 13191 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 13192 tmo = HZ / 10; 13193 } else { 13194 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 13195 tmo = bp->fw_reset_min_dsecs * HZ / 10; 13196 } 13197 bnxt_queue_fw_reset_work(bp, tmo); 13198 } 13199 fw_reset_exit: 13200 bnxt_rtnl_unlock_sp(bp); 13201 } 13202 13203 static void bnxt_chk_missed_irq(struct bnxt *bp) 13204 { 13205 int i; 13206 13207 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 13208 return; 13209 13210 for (i = 0; i < bp->cp_nr_rings; i++) { 13211 struct bnxt_napi *bnapi = bp->bnapi[i]; 13212 struct bnxt_cp_ring_info *cpr; 13213 u32 fw_ring_id; 13214 int j; 13215 13216 if (!bnapi) 13217 continue; 13218 13219 cpr = &bnapi->cp_ring; 13220 for (j = 0; j < cpr->cp_ring_count; j++) { 13221 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 13222 u32 val[2]; 13223 13224 if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2)) 13225 continue; 13226 13227 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 13228 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 13229 continue; 13230 } 13231 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 13232 bnxt_dbg_hwrm_ring_info_get(bp, 13233 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 13234 fw_ring_id, &val[0], &val[1]); 13235 cpr->sw_stats.cmn.missed_irqs++; 13236 } 13237 } 13238 } 13239 13240 static void bnxt_cfg_ntp_filters(struct bnxt *); 13241 13242 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 13243 { 13244 struct bnxt_link_info *link_info = &bp->link_info; 13245 13246 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 13247 link_info->autoneg = BNXT_AUTONEG_SPEED; 13248 if (bp->hwrm_spec_code >= 0x10201) { 13249 if (link_info->auto_pause_setting & 13250 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 13251 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 13252 } else { 13253 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 13254 } 13255 bnxt_set_auto_speed(link_info); 13256 } else { 13257 bnxt_set_force_speed(link_info); 13258 link_info->req_duplex = link_info->duplex_setting; 13259 } 13260 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 13261 link_info->req_flow_ctrl = 13262 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 13263 else 13264 link_info->req_flow_ctrl = link_info->force_pause_setting; 13265 } 13266 13267 static void bnxt_fw_echo_reply(struct bnxt *bp) 13268 { 13269 struct bnxt_fw_health *fw_health = bp->fw_health; 13270 struct hwrm_func_echo_response_input *req; 13271 int rc; 13272 13273 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 13274 if (rc) 13275 return; 13276 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 13277 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 13278 hwrm_req_send(bp, req); 13279 } 13280 13281 static void bnxt_sp_task(struct work_struct *work) 13282 { 13283 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 13284 13285 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 13286 smp_mb__after_atomic(); 13287 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 13288 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 13289 return; 13290 } 13291 13292 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 13293 bnxt_cfg_rx_mode(bp); 13294 13295 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 13296 bnxt_cfg_ntp_filters(bp); 13297 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 13298 bnxt_hwrm_exec_fwd_req(bp); 13299 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 13300 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 13301 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 13302 bnxt_hwrm_port_qstats(bp, 0); 13303 bnxt_hwrm_port_qstats_ext(bp, 0); 13304 bnxt_accumulate_all_stats(bp); 13305 } 13306 13307 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 13308 int rc; 13309 13310 mutex_lock(&bp->link_lock); 13311 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 13312 &bp->sp_event)) 13313 bnxt_hwrm_phy_qcaps(bp); 13314 13315 rc = bnxt_update_link(bp, true); 13316 if (rc) 13317 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 13318 rc); 13319 13320 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 13321 &bp->sp_event)) 13322 bnxt_init_ethtool_link_settings(bp); 13323 mutex_unlock(&bp->link_lock); 13324 } 13325 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 13326 int rc; 13327 13328 mutex_lock(&bp->link_lock); 13329 rc = bnxt_update_phy_setting(bp); 13330 mutex_unlock(&bp->link_lock); 13331 if (rc) { 13332 netdev_warn(bp->dev, "update phy settings retry failed\n"); 13333 } else { 13334 bp->link_info.phy_retry = false; 13335 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 13336 } 13337 } 13338 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 13339 mutex_lock(&bp->link_lock); 13340 bnxt_get_port_module_status(bp); 13341 mutex_unlock(&bp->link_lock); 13342 } 13343 13344 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 13345 bnxt_tc_flow_stats_work(bp); 13346 13347 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 13348 bnxt_chk_missed_irq(bp); 13349 13350 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 13351 bnxt_fw_echo_reply(bp); 13352 13353 if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event)) 13354 bnxt_hwmon_notify_event(bp); 13355 13356 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 13357 * must be the last functions to be called before exiting. 13358 */ 13359 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 13360 bnxt_reset(bp, false); 13361 13362 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 13363 bnxt_reset(bp, true); 13364 13365 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 13366 bnxt_rx_ring_reset(bp); 13367 13368 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 13369 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 13370 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 13371 bnxt_devlink_health_fw_report(bp); 13372 else 13373 bnxt_fw_reset(bp); 13374 } 13375 13376 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 13377 if (!is_bnxt_fw_ok(bp)) 13378 bnxt_devlink_health_fw_report(bp); 13379 } 13380 13381 smp_mb__before_atomic(); 13382 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 13383 } 13384 13385 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13386 int *max_cp); 13387 13388 /* Under rtnl_lock */ 13389 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 13390 int tx_xdp) 13391 { 13392 int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp; 13393 struct bnxt_hw_rings hwr = {0}; 13394 int rx_rings = rx; 13395 13396 if (tcs) 13397 tx_sets = tcs; 13398 13399 _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp); 13400 13401 if (max_rx < rx_rings) 13402 return -ENOMEM; 13403 13404 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13405 rx_rings <<= 1; 13406 13407 hwr.rx = rx_rings; 13408 hwr.tx = tx * tx_sets + tx_xdp; 13409 if (max_tx < hwr.tx) 13410 return -ENOMEM; 13411 13412 hwr.vnic = bnxt_get_total_vnics(bp, rx); 13413 13414 tx_cp = __bnxt_num_tx_to_cp(bp, hwr.tx, tx_sets, tx_xdp); 13415 hwr.cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx; 13416 if (max_cp < hwr.cp) 13417 return -ENOMEM; 13418 hwr.stat = hwr.cp; 13419 if (BNXT_NEW_RM(bp)) { 13420 hwr.cp += bnxt_get_ulp_msix_num(bp); 13421 hwr.stat += bnxt_get_ulp_stat_ctxs(bp); 13422 hwr.grp = rx; 13423 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr); 13424 } 13425 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 13426 hwr.cp_p5 = hwr.tx + rx; 13427 return bnxt_hwrm_check_rings(bp, &hwr); 13428 } 13429 13430 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 13431 { 13432 if (bp->bar2) { 13433 pci_iounmap(pdev, bp->bar2); 13434 bp->bar2 = NULL; 13435 } 13436 13437 if (bp->bar1) { 13438 pci_iounmap(pdev, bp->bar1); 13439 bp->bar1 = NULL; 13440 } 13441 13442 if (bp->bar0) { 13443 pci_iounmap(pdev, bp->bar0); 13444 bp->bar0 = NULL; 13445 } 13446 } 13447 13448 static void bnxt_cleanup_pci(struct bnxt *bp) 13449 { 13450 bnxt_unmap_bars(bp, bp->pdev); 13451 pci_release_regions(bp->pdev); 13452 if (pci_is_enabled(bp->pdev)) 13453 pci_disable_device(bp->pdev); 13454 } 13455 13456 static void bnxt_init_dflt_coal(struct bnxt *bp) 13457 { 13458 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 13459 struct bnxt_coal *coal; 13460 u16 flags = 0; 13461 13462 if (coal_cap->cmpl_params & 13463 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 13464 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 13465 13466 /* Tick values in micro seconds. 13467 * 1 coal_buf x bufs_per_record = 1 completion record. 13468 */ 13469 coal = &bp->rx_coal; 13470 coal->coal_ticks = 10; 13471 coal->coal_bufs = 30; 13472 coal->coal_ticks_irq = 1; 13473 coal->coal_bufs_irq = 2; 13474 coal->idle_thresh = 50; 13475 coal->bufs_per_record = 2; 13476 coal->budget = 64; /* NAPI budget */ 13477 coal->flags = flags; 13478 13479 coal = &bp->tx_coal; 13480 coal->coal_ticks = 28; 13481 coal->coal_bufs = 30; 13482 coal->coal_ticks_irq = 2; 13483 coal->coal_bufs_irq = 2; 13484 coal->bufs_per_record = 1; 13485 coal->flags = flags; 13486 13487 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 13488 } 13489 13490 /* FW that pre-reserves 1 VNIC per function */ 13491 static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp) 13492 { 13493 u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp); 13494 13495 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 13496 (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18))) 13497 return true; 13498 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 13499 (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172))) 13500 return true; 13501 return false; 13502 } 13503 13504 static int bnxt_fw_init_one_p1(struct bnxt *bp) 13505 { 13506 int rc; 13507 13508 bp->fw_cap = 0; 13509 rc = bnxt_hwrm_ver_get(bp); 13510 /* FW may be unresponsive after FLR. FLR must complete within 100 msec 13511 * so wait before continuing with recovery. 13512 */ 13513 if (rc) 13514 msleep(100); 13515 bnxt_try_map_fw_health_reg(bp); 13516 if (rc) { 13517 rc = bnxt_try_recover_fw(bp); 13518 if (rc) 13519 return rc; 13520 rc = bnxt_hwrm_ver_get(bp); 13521 if (rc) 13522 return rc; 13523 } 13524 13525 bnxt_nvm_cfg_ver_get(bp); 13526 13527 rc = bnxt_hwrm_func_reset(bp); 13528 if (rc) 13529 return -ENODEV; 13530 13531 bnxt_hwrm_fw_set_time(bp); 13532 return 0; 13533 } 13534 13535 static int bnxt_fw_init_one_p2(struct bnxt *bp) 13536 { 13537 int rc; 13538 13539 /* Get the MAX capabilities for this function */ 13540 rc = bnxt_hwrm_func_qcaps(bp); 13541 if (rc) { 13542 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 13543 rc); 13544 return -ENODEV; 13545 } 13546 13547 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 13548 if (rc) 13549 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 13550 rc); 13551 13552 if (bnxt_alloc_fw_health(bp)) { 13553 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 13554 } else { 13555 rc = bnxt_hwrm_error_recovery_qcfg(bp); 13556 if (rc) 13557 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 13558 rc); 13559 } 13560 13561 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 13562 if (rc) 13563 return -ENODEV; 13564 13565 if (bnxt_fw_pre_resv_vnics(bp)) 13566 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS; 13567 13568 bnxt_hwrm_func_qcfg(bp); 13569 bnxt_hwrm_vnic_qcaps(bp); 13570 bnxt_hwrm_port_led_qcaps(bp); 13571 bnxt_ethtool_init(bp); 13572 if (bp->fw_cap & BNXT_FW_CAP_PTP) 13573 __bnxt_hwrm_ptp_qcfg(bp); 13574 bnxt_dcb_init(bp); 13575 bnxt_hwmon_init(bp); 13576 return 0; 13577 } 13578 13579 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 13580 { 13581 bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP; 13582 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 13583 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 13584 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 13585 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 13586 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA) 13587 bp->rss_hash_delta = bp->rss_hash_cfg; 13588 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 13589 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP; 13590 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 13591 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 13592 } 13593 } 13594 13595 static void bnxt_set_dflt_rfs(struct bnxt *bp) 13596 { 13597 struct net_device *dev = bp->dev; 13598 13599 dev->hw_features &= ~NETIF_F_NTUPLE; 13600 dev->features &= ~NETIF_F_NTUPLE; 13601 bp->flags &= ~BNXT_FLAG_RFS; 13602 if (bnxt_rfs_supported(bp)) { 13603 dev->hw_features |= NETIF_F_NTUPLE; 13604 if (bnxt_rfs_capable(bp)) { 13605 bp->flags |= BNXT_FLAG_RFS; 13606 dev->features |= NETIF_F_NTUPLE; 13607 } 13608 } 13609 } 13610 13611 static void bnxt_fw_init_one_p3(struct bnxt *bp) 13612 { 13613 struct pci_dev *pdev = bp->pdev; 13614 13615 bnxt_set_dflt_rss_hash_type(bp); 13616 bnxt_set_dflt_rfs(bp); 13617 13618 bnxt_get_wol_settings(bp); 13619 if (bp->flags & BNXT_FLAG_WOL_CAP) 13620 device_set_wakeup_enable(&pdev->dev, bp->wol); 13621 else 13622 device_set_wakeup_capable(&pdev->dev, false); 13623 13624 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 13625 bnxt_hwrm_coal_params_qcaps(bp); 13626 } 13627 13628 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 13629 13630 int bnxt_fw_init_one(struct bnxt *bp) 13631 { 13632 int rc; 13633 13634 rc = bnxt_fw_init_one_p1(bp); 13635 if (rc) { 13636 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 13637 return rc; 13638 } 13639 rc = bnxt_fw_init_one_p2(bp); 13640 if (rc) { 13641 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 13642 return rc; 13643 } 13644 rc = bnxt_probe_phy(bp, false); 13645 if (rc) 13646 return rc; 13647 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 13648 if (rc) 13649 return rc; 13650 13651 bnxt_fw_init_one_p3(bp); 13652 return 0; 13653 } 13654 13655 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 13656 { 13657 struct bnxt_fw_health *fw_health = bp->fw_health; 13658 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 13659 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 13660 u32 reg_type, reg_off, delay_msecs; 13661 13662 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 13663 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 13664 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 13665 switch (reg_type) { 13666 case BNXT_FW_HEALTH_REG_TYPE_CFG: 13667 pci_write_config_dword(bp->pdev, reg_off, val); 13668 break; 13669 case BNXT_FW_HEALTH_REG_TYPE_GRC: 13670 writel(reg_off & BNXT_GRC_BASE_MASK, 13671 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 13672 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 13673 fallthrough; 13674 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 13675 writel(val, bp->bar0 + reg_off); 13676 break; 13677 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 13678 writel(val, bp->bar1 + reg_off); 13679 break; 13680 } 13681 if (delay_msecs) { 13682 pci_read_config_dword(bp->pdev, 0, &val); 13683 msleep(delay_msecs); 13684 } 13685 } 13686 13687 bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 13688 { 13689 struct hwrm_func_qcfg_output *resp; 13690 struct hwrm_func_qcfg_input *req; 13691 bool result = true; /* firmware will enforce if unknown */ 13692 13693 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 13694 return result; 13695 13696 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 13697 return result; 13698 13699 req->fid = cpu_to_le16(0xffff); 13700 resp = hwrm_req_hold(bp, req); 13701 if (!hwrm_req_send(bp, req)) 13702 result = !!(le16_to_cpu(resp->flags) & 13703 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 13704 hwrm_req_drop(bp, req); 13705 return result; 13706 } 13707 13708 static void bnxt_reset_all(struct bnxt *bp) 13709 { 13710 struct bnxt_fw_health *fw_health = bp->fw_health; 13711 int i, rc; 13712 13713 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 13714 bnxt_fw_reset_via_optee(bp); 13715 bp->fw_reset_timestamp = jiffies; 13716 return; 13717 } 13718 13719 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 13720 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 13721 bnxt_fw_reset_writel(bp, i); 13722 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 13723 struct hwrm_fw_reset_input *req; 13724 13725 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 13726 if (!rc) { 13727 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 13728 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 13729 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 13730 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 13731 rc = hwrm_req_send(bp, req); 13732 } 13733 if (rc != -ENODEV) 13734 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 13735 } 13736 bp->fw_reset_timestamp = jiffies; 13737 } 13738 13739 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 13740 { 13741 return time_after(jiffies, bp->fw_reset_timestamp + 13742 (bp->fw_reset_max_dsecs * HZ / 10)); 13743 } 13744 13745 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 13746 { 13747 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13748 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { 13749 bnxt_ulp_start(bp, rc); 13750 bnxt_dl_health_fw_status_update(bp, false); 13751 } 13752 bp->fw_reset_state = 0; 13753 dev_close(bp->dev); 13754 } 13755 13756 static void bnxt_fw_reset_task(struct work_struct *work) 13757 { 13758 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 13759 int rc = 0; 13760 13761 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 13762 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 13763 return; 13764 } 13765 13766 switch (bp->fw_reset_state) { 13767 case BNXT_FW_RESET_STATE_POLL_VF: { 13768 int n = bnxt_get_registered_vfs(bp); 13769 int tmo; 13770 13771 if (n < 0) { 13772 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 13773 n, jiffies_to_msecs(jiffies - 13774 bp->fw_reset_timestamp)); 13775 goto fw_reset_abort; 13776 } else if (n > 0) { 13777 if (bnxt_fw_reset_timeout(bp)) { 13778 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13779 bp->fw_reset_state = 0; 13780 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 13781 n); 13782 return; 13783 } 13784 bnxt_queue_fw_reset_work(bp, HZ / 10); 13785 return; 13786 } 13787 bp->fw_reset_timestamp = jiffies; 13788 rtnl_lock(); 13789 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 13790 bnxt_fw_reset_abort(bp, rc); 13791 rtnl_unlock(); 13792 return; 13793 } 13794 bnxt_fw_reset_close(bp); 13795 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 13796 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 13797 tmo = HZ / 10; 13798 } else { 13799 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 13800 tmo = bp->fw_reset_min_dsecs * HZ / 10; 13801 } 13802 rtnl_unlock(); 13803 bnxt_queue_fw_reset_work(bp, tmo); 13804 return; 13805 } 13806 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 13807 u32 val; 13808 13809 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 13810 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 13811 !bnxt_fw_reset_timeout(bp)) { 13812 bnxt_queue_fw_reset_work(bp, HZ / 5); 13813 return; 13814 } 13815 13816 if (!bp->fw_health->primary) { 13817 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 13818 13819 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 13820 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 13821 return; 13822 } 13823 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 13824 } 13825 fallthrough; 13826 case BNXT_FW_RESET_STATE_RESET_FW: 13827 bnxt_reset_all(bp); 13828 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 13829 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 13830 return; 13831 case BNXT_FW_RESET_STATE_ENABLE_DEV: 13832 bnxt_inv_fw_health_reg(bp); 13833 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 13834 !bp->fw_reset_min_dsecs) { 13835 u16 val; 13836 13837 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 13838 if (val == 0xffff) { 13839 if (bnxt_fw_reset_timeout(bp)) { 13840 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 13841 rc = -ETIMEDOUT; 13842 goto fw_reset_abort; 13843 } 13844 bnxt_queue_fw_reset_work(bp, HZ / 1000); 13845 return; 13846 } 13847 } 13848 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 13849 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 13850 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 13851 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 13852 bnxt_dl_remote_reload(bp); 13853 if (pci_enable_device(bp->pdev)) { 13854 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 13855 rc = -ENODEV; 13856 goto fw_reset_abort; 13857 } 13858 pci_set_master(bp->pdev); 13859 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 13860 fallthrough; 13861 case BNXT_FW_RESET_STATE_POLL_FW: 13862 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 13863 rc = bnxt_hwrm_poll(bp); 13864 if (rc) { 13865 if (bnxt_fw_reset_timeout(bp)) { 13866 netdev_err(bp->dev, "Firmware reset aborted\n"); 13867 goto fw_reset_abort_status; 13868 } 13869 bnxt_queue_fw_reset_work(bp, HZ / 5); 13870 return; 13871 } 13872 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 13873 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 13874 fallthrough; 13875 case BNXT_FW_RESET_STATE_OPENING: 13876 while (!rtnl_trylock()) { 13877 bnxt_queue_fw_reset_work(bp, HZ / 10); 13878 return; 13879 } 13880 rc = bnxt_open(bp->dev); 13881 if (rc) { 13882 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 13883 bnxt_fw_reset_abort(bp, rc); 13884 rtnl_unlock(); 13885 return; 13886 } 13887 13888 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 13889 bp->fw_health->enabled) { 13890 bp->fw_health->last_fw_reset_cnt = 13891 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 13892 } 13893 bp->fw_reset_state = 0; 13894 /* Make sure fw_reset_state is 0 before clearing the flag */ 13895 smp_mb__before_atomic(); 13896 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 13897 bnxt_ulp_start(bp, 0); 13898 bnxt_reenable_sriov(bp); 13899 bnxt_vf_reps_alloc(bp); 13900 bnxt_vf_reps_open(bp); 13901 bnxt_ptp_reapply_pps(bp); 13902 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 13903 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 13904 bnxt_dl_health_fw_recovery_done(bp); 13905 bnxt_dl_health_fw_status_update(bp, true); 13906 } 13907 rtnl_unlock(); 13908 break; 13909 } 13910 return; 13911 13912 fw_reset_abort_status: 13913 if (bp->fw_health->status_reliable || 13914 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 13915 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 13916 13917 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 13918 } 13919 fw_reset_abort: 13920 rtnl_lock(); 13921 bnxt_fw_reset_abort(bp, rc); 13922 rtnl_unlock(); 13923 } 13924 13925 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 13926 { 13927 int rc; 13928 struct bnxt *bp = netdev_priv(dev); 13929 13930 SET_NETDEV_DEV(dev, &pdev->dev); 13931 13932 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 13933 rc = pci_enable_device(pdev); 13934 if (rc) { 13935 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 13936 goto init_err; 13937 } 13938 13939 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 13940 dev_err(&pdev->dev, 13941 "Cannot find PCI device base address, aborting\n"); 13942 rc = -ENODEV; 13943 goto init_err_disable; 13944 } 13945 13946 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 13947 if (rc) { 13948 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 13949 goto init_err_disable; 13950 } 13951 13952 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 13953 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 13954 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 13955 rc = -EIO; 13956 goto init_err_release; 13957 } 13958 13959 pci_set_master(pdev); 13960 13961 bp->dev = dev; 13962 bp->pdev = pdev; 13963 13964 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 13965 * determines the BAR size. 13966 */ 13967 bp->bar0 = pci_ioremap_bar(pdev, 0); 13968 if (!bp->bar0) { 13969 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 13970 rc = -ENOMEM; 13971 goto init_err_release; 13972 } 13973 13974 bp->bar2 = pci_ioremap_bar(pdev, 4); 13975 if (!bp->bar2) { 13976 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 13977 rc = -ENOMEM; 13978 goto init_err_release; 13979 } 13980 13981 INIT_WORK(&bp->sp_task, bnxt_sp_task); 13982 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 13983 13984 spin_lock_init(&bp->ntp_fltr_lock); 13985 #if BITS_PER_LONG == 32 13986 spin_lock_init(&bp->db_lock); 13987 #endif 13988 13989 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 13990 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 13991 13992 timer_setup(&bp->timer, bnxt_timer, 0); 13993 bp->current_interval = BNXT_TIMER_INTERVAL; 13994 13995 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 13996 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 13997 13998 clear_bit(BNXT_STATE_OPEN, &bp->state); 13999 return 0; 14000 14001 init_err_release: 14002 bnxt_unmap_bars(bp, pdev); 14003 pci_release_regions(pdev); 14004 14005 init_err_disable: 14006 pci_disable_device(pdev); 14007 14008 init_err: 14009 return rc; 14010 } 14011 14012 /* rtnl_lock held */ 14013 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 14014 { 14015 struct sockaddr *addr = p; 14016 struct bnxt *bp = netdev_priv(dev); 14017 int rc = 0; 14018 14019 if (!is_valid_ether_addr(addr->sa_data)) 14020 return -EADDRNOTAVAIL; 14021 14022 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 14023 return 0; 14024 14025 rc = bnxt_approve_mac(bp, addr->sa_data, true); 14026 if (rc) 14027 return rc; 14028 14029 eth_hw_addr_set(dev, addr->sa_data); 14030 bnxt_clear_usr_fltrs(bp, true); 14031 if (netif_running(dev)) { 14032 bnxt_close_nic(bp, false, false); 14033 rc = bnxt_open_nic(bp, false, false); 14034 } 14035 14036 return rc; 14037 } 14038 14039 /* rtnl_lock held */ 14040 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 14041 { 14042 struct bnxt *bp = netdev_priv(dev); 14043 14044 if (netif_running(dev)) 14045 bnxt_close_nic(bp, true, false); 14046 14047 dev->mtu = new_mtu; 14048 bnxt_set_ring_params(bp); 14049 14050 if (netif_running(dev)) 14051 return bnxt_open_nic(bp, true, false); 14052 14053 return 0; 14054 } 14055 14056 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 14057 { 14058 struct bnxt *bp = netdev_priv(dev); 14059 bool sh = false; 14060 int rc, tx_cp; 14061 14062 if (tc > bp->max_tc) { 14063 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 14064 tc, bp->max_tc); 14065 return -EINVAL; 14066 } 14067 14068 if (bp->num_tc == tc) 14069 return 0; 14070 14071 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 14072 sh = true; 14073 14074 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 14075 sh, tc, bp->tx_nr_rings_xdp); 14076 if (rc) 14077 return rc; 14078 14079 /* Needs to close the device and do hw resource re-allocations */ 14080 if (netif_running(bp->dev)) 14081 bnxt_close_nic(bp, true, false); 14082 14083 if (tc) { 14084 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 14085 netdev_set_num_tc(dev, tc); 14086 bp->num_tc = tc; 14087 } else { 14088 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 14089 netdev_reset_tc(dev); 14090 bp->num_tc = 0; 14091 } 14092 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 14093 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); 14094 bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : 14095 tx_cp + bp->rx_nr_rings; 14096 14097 if (netif_running(bp->dev)) 14098 return bnxt_open_nic(bp, true, false); 14099 14100 return 0; 14101 } 14102 14103 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 14104 void *cb_priv) 14105 { 14106 struct bnxt *bp = cb_priv; 14107 14108 if (!bnxt_tc_flower_enabled(bp) || 14109 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 14110 return -EOPNOTSUPP; 14111 14112 switch (type) { 14113 case TC_SETUP_CLSFLOWER: 14114 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 14115 default: 14116 return -EOPNOTSUPP; 14117 } 14118 } 14119 14120 LIST_HEAD(bnxt_block_cb_list); 14121 14122 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 14123 void *type_data) 14124 { 14125 struct bnxt *bp = netdev_priv(dev); 14126 14127 switch (type) { 14128 case TC_SETUP_BLOCK: 14129 return flow_block_cb_setup_simple(type_data, 14130 &bnxt_block_cb_list, 14131 bnxt_setup_tc_block_cb, 14132 bp, bp, true); 14133 case TC_SETUP_QDISC_MQPRIO: { 14134 struct tc_mqprio_qopt *mqprio = type_data; 14135 14136 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 14137 14138 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 14139 } 14140 default: 14141 return -EOPNOTSUPP; 14142 } 14143 } 14144 14145 u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys, 14146 const struct sk_buff *skb) 14147 { 14148 struct bnxt_vnic_info *vnic; 14149 14150 if (skb) 14151 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 14152 14153 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 14154 return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key); 14155 } 14156 14157 int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr, 14158 u32 idx) 14159 { 14160 struct hlist_head *head; 14161 int bit_id; 14162 14163 spin_lock_bh(&bp->ntp_fltr_lock); 14164 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, bp->max_fltr, 0); 14165 if (bit_id < 0) { 14166 spin_unlock_bh(&bp->ntp_fltr_lock); 14167 return -ENOMEM; 14168 } 14169 14170 fltr->base.sw_id = (u16)bit_id; 14171 fltr->base.type = BNXT_FLTR_TYPE_NTUPLE; 14172 fltr->base.flags |= BNXT_ACT_RING_DST; 14173 head = &bp->ntp_fltr_hash_tbl[idx]; 14174 hlist_add_head_rcu(&fltr->base.hash, head); 14175 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state); 14176 bnxt_insert_usr_fltr(bp, &fltr->base); 14177 bp->ntp_fltr_count++; 14178 spin_unlock_bh(&bp->ntp_fltr_lock); 14179 return 0; 14180 } 14181 14182 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 14183 struct bnxt_ntuple_filter *f2) 14184 { 14185 struct bnxt_flow_masks *masks1 = &f1->fmasks; 14186 struct bnxt_flow_masks *masks2 = &f2->fmasks; 14187 struct flow_keys *keys1 = &f1->fkeys; 14188 struct flow_keys *keys2 = &f2->fkeys; 14189 14190 if (keys1->basic.n_proto != keys2->basic.n_proto || 14191 keys1->basic.ip_proto != keys2->basic.ip_proto) 14192 return false; 14193 14194 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 14195 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 14196 masks1->addrs.v4addrs.src != masks2->addrs.v4addrs.src || 14197 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst || 14198 masks1->addrs.v4addrs.dst != masks2->addrs.v4addrs.dst) 14199 return false; 14200 } else { 14201 if (!ipv6_addr_equal(&keys1->addrs.v6addrs.src, 14202 &keys2->addrs.v6addrs.src) || 14203 !ipv6_addr_equal(&masks1->addrs.v6addrs.src, 14204 &masks2->addrs.v6addrs.src) || 14205 !ipv6_addr_equal(&keys1->addrs.v6addrs.dst, 14206 &keys2->addrs.v6addrs.dst) || 14207 !ipv6_addr_equal(&masks1->addrs.v6addrs.dst, 14208 &masks2->addrs.v6addrs.dst)) 14209 return false; 14210 } 14211 14212 return keys1->ports.src == keys2->ports.src && 14213 masks1->ports.src == masks2->ports.src && 14214 keys1->ports.dst == keys2->ports.dst && 14215 masks1->ports.dst == masks2->ports.dst && 14216 keys1->control.flags == keys2->control.flags && 14217 f1->l2_fltr == f2->l2_fltr; 14218 } 14219 14220 struct bnxt_ntuple_filter * 14221 bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp, 14222 struct bnxt_ntuple_filter *fltr, u32 idx) 14223 { 14224 struct bnxt_ntuple_filter *f; 14225 struct hlist_head *head; 14226 14227 head = &bp->ntp_fltr_hash_tbl[idx]; 14228 hlist_for_each_entry_rcu(f, head, base.hash) { 14229 if (bnxt_fltr_match(f, fltr)) 14230 return f; 14231 } 14232 return NULL; 14233 } 14234 14235 #ifdef CONFIG_RFS_ACCEL 14236 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 14237 u16 rxq_index, u32 flow_id) 14238 { 14239 struct bnxt *bp = netdev_priv(dev); 14240 struct bnxt_ntuple_filter *fltr, *new_fltr; 14241 struct flow_keys *fkeys; 14242 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 14243 struct bnxt_l2_filter *l2_fltr; 14244 int rc = 0, idx; 14245 u32 flags; 14246 14247 if (ether_addr_equal(dev->dev_addr, eth->h_dest)) { 14248 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0]; 14249 atomic_inc(&l2_fltr->refcnt); 14250 } else { 14251 struct bnxt_l2_key key; 14252 14253 ether_addr_copy(key.dst_mac_addr, eth->h_dest); 14254 key.vlan = 0; 14255 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key); 14256 if (!l2_fltr) 14257 return -EINVAL; 14258 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) { 14259 bnxt_del_l2_filter(bp, l2_fltr); 14260 return -EINVAL; 14261 } 14262 } 14263 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 14264 if (!new_fltr) { 14265 bnxt_del_l2_filter(bp, l2_fltr); 14266 return -ENOMEM; 14267 } 14268 14269 fkeys = &new_fltr->fkeys; 14270 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 14271 rc = -EPROTONOSUPPORT; 14272 goto err_free; 14273 } 14274 14275 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 14276 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 14277 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 14278 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 14279 rc = -EPROTONOSUPPORT; 14280 goto err_free; 14281 } 14282 new_fltr->fmasks = BNXT_FLOW_IPV4_MASK_ALL; 14283 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) { 14284 if (bp->hwrm_spec_code < 0x10601) { 14285 rc = -EPROTONOSUPPORT; 14286 goto err_free; 14287 } 14288 new_fltr->fmasks = BNXT_FLOW_IPV6_MASK_ALL; 14289 } 14290 flags = fkeys->control.flags; 14291 if (((flags & FLOW_DIS_ENCAPSULATION) && 14292 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 14293 rc = -EPROTONOSUPPORT; 14294 goto err_free; 14295 } 14296 new_fltr->l2_fltr = l2_fltr; 14297 14298 idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb); 14299 rcu_read_lock(); 14300 fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx); 14301 if (fltr) { 14302 rc = fltr->base.sw_id; 14303 rcu_read_unlock(); 14304 goto err_free; 14305 } 14306 rcu_read_unlock(); 14307 14308 new_fltr->flow_id = flow_id; 14309 new_fltr->base.rxq = rxq_index; 14310 rc = bnxt_insert_ntp_filter(bp, new_fltr, idx); 14311 if (!rc) { 14312 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 14313 return new_fltr->base.sw_id; 14314 } 14315 14316 err_free: 14317 bnxt_del_l2_filter(bp, l2_fltr); 14318 kfree(new_fltr); 14319 return rc; 14320 } 14321 #endif 14322 14323 void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr) 14324 { 14325 spin_lock_bh(&bp->ntp_fltr_lock); 14326 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) { 14327 spin_unlock_bh(&bp->ntp_fltr_lock); 14328 return; 14329 } 14330 hlist_del_rcu(&fltr->base.hash); 14331 bnxt_del_one_usr_fltr(bp, &fltr->base); 14332 bp->ntp_fltr_count--; 14333 spin_unlock_bh(&bp->ntp_fltr_lock); 14334 bnxt_del_l2_filter(bp, fltr->l2_fltr); 14335 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap); 14336 kfree_rcu(fltr, base.rcu); 14337 } 14338 14339 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 14340 { 14341 #ifdef CONFIG_RFS_ACCEL 14342 int i; 14343 14344 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 14345 struct hlist_head *head; 14346 struct hlist_node *tmp; 14347 struct bnxt_ntuple_filter *fltr; 14348 int rc; 14349 14350 head = &bp->ntp_fltr_hash_tbl[i]; 14351 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 14352 bool del = false; 14353 14354 if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) { 14355 if (fltr->base.flags & BNXT_ACT_NO_AGING) 14356 continue; 14357 if (rps_may_expire_flow(bp->dev, fltr->base.rxq, 14358 fltr->flow_id, 14359 fltr->base.sw_id)) { 14360 bnxt_hwrm_cfa_ntuple_filter_free(bp, 14361 fltr); 14362 del = true; 14363 } 14364 } else { 14365 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 14366 fltr); 14367 if (rc) 14368 del = true; 14369 else 14370 set_bit(BNXT_FLTR_VALID, &fltr->base.state); 14371 } 14372 14373 if (del) 14374 bnxt_del_ntp_filter(bp, fltr); 14375 } 14376 } 14377 #endif 14378 } 14379 14380 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, 14381 unsigned int entry, struct udp_tunnel_info *ti) 14382 { 14383 struct bnxt *bp = netdev_priv(netdev); 14384 unsigned int cmd; 14385 14386 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 14387 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN; 14388 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE) 14389 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE; 14390 else 14391 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE; 14392 14393 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd); 14394 } 14395 14396 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, 14397 unsigned int entry, struct udp_tunnel_info *ti) 14398 { 14399 struct bnxt *bp = netdev_priv(netdev); 14400 unsigned int cmd; 14401 14402 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 14403 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 14404 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE) 14405 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 14406 else 14407 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE; 14408 14409 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 14410 } 14411 14412 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 14413 .set_port = bnxt_udp_tunnel_set_port, 14414 .unset_port = bnxt_udp_tunnel_unset_port, 14415 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 14416 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 14417 .tables = { 14418 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 14419 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 14420 }, 14421 }, bnxt_udp_tunnels_p7 = { 14422 .set_port = bnxt_udp_tunnel_set_port, 14423 .unset_port = bnxt_udp_tunnel_unset_port, 14424 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 14425 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 14426 .tables = { 14427 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 14428 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 14429 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, }, 14430 }, 14431 }; 14432 14433 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 14434 struct net_device *dev, u32 filter_mask, 14435 int nlflags) 14436 { 14437 struct bnxt *bp = netdev_priv(dev); 14438 14439 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 14440 nlflags, filter_mask, NULL); 14441 } 14442 14443 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 14444 u16 flags, struct netlink_ext_ack *extack) 14445 { 14446 struct bnxt *bp = netdev_priv(dev); 14447 struct nlattr *attr, *br_spec; 14448 int rem, rc = 0; 14449 14450 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 14451 return -EOPNOTSUPP; 14452 14453 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 14454 if (!br_spec) 14455 return -EINVAL; 14456 14457 nla_for_each_nested(attr, br_spec, rem) { 14458 u16 mode; 14459 14460 if (nla_type(attr) != IFLA_BRIDGE_MODE) 14461 continue; 14462 14463 mode = nla_get_u16(attr); 14464 if (mode == bp->br_mode) 14465 break; 14466 14467 rc = bnxt_hwrm_set_br_mode(bp, mode); 14468 if (!rc) 14469 bp->br_mode = mode; 14470 break; 14471 } 14472 return rc; 14473 } 14474 14475 int bnxt_get_port_parent_id(struct net_device *dev, 14476 struct netdev_phys_item_id *ppid) 14477 { 14478 struct bnxt *bp = netdev_priv(dev); 14479 14480 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 14481 return -EOPNOTSUPP; 14482 14483 /* The PF and it's VF-reps only support the switchdev framework */ 14484 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 14485 return -EOPNOTSUPP; 14486 14487 ppid->id_len = sizeof(bp->dsn); 14488 memcpy(ppid->id, bp->dsn, ppid->id_len); 14489 14490 return 0; 14491 } 14492 14493 static const struct net_device_ops bnxt_netdev_ops = { 14494 .ndo_open = bnxt_open, 14495 .ndo_start_xmit = bnxt_start_xmit, 14496 .ndo_stop = bnxt_close, 14497 .ndo_get_stats64 = bnxt_get_stats64, 14498 .ndo_set_rx_mode = bnxt_set_rx_mode, 14499 .ndo_eth_ioctl = bnxt_ioctl, 14500 .ndo_validate_addr = eth_validate_addr, 14501 .ndo_set_mac_address = bnxt_change_mac_addr, 14502 .ndo_change_mtu = bnxt_change_mtu, 14503 .ndo_fix_features = bnxt_fix_features, 14504 .ndo_set_features = bnxt_set_features, 14505 .ndo_features_check = bnxt_features_check, 14506 .ndo_tx_timeout = bnxt_tx_timeout, 14507 #ifdef CONFIG_BNXT_SRIOV 14508 .ndo_get_vf_config = bnxt_get_vf_config, 14509 .ndo_set_vf_mac = bnxt_set_vf_mac, 14510 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 14511 .ndo_set_vf_rate = bnxt_set_vf_bw, 14512 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 14513 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 14514 .ndo_set_vf_trust = bnxt_set_vf_trust, 14515 #endif 14516 .ndo_setup_tc = bnxt_setup_tc, 14517 #ifdef CONFIG_RFS_ACCEL 14518 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 14519 #endif 14520 .ndo_bpf = bnxt_xdp, 14521 .ndo_xdp_xmit = bnxt_xdp_xmit, 14522 .ndo_bridge_getlink = bnxt_bridge_getlink, 14523 .ndo_bridge_setlink = bnxt_bridge_setlink, 14524 }; 14525 14526 static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, 14527 struct netdev_queue_stats_rx *stats) 14528 { 14529 struct bnxt *bp = netdev_priv(dev); 14530 struct bnxt_cp_ring_info *cpr; 14531 u64 *sw; 14532 14533 cpr = &bp->bnapi[i]->cp_ring; 14534 sw = cpr->stats.sw_stats; 14535 14536 stats->packets = 0; 14537 stats->packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 14538 stats->packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 14539 stats->packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 14540 14541 stats->bytes = 0; 14542 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 14543 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 14544 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 14545 14546 stats->alloc_fail = cpr->sw_stats.rx.rx_oom_discards; 14547 } 14548 14549 static void bnxt_get_queue_stats_tx(struct net_device *dev, int i, 14550 struct netdev_queue_stats_tx *stats) 14551 { 14552 struct bnxt *bp = netdev_priv(dev); 14553 struct bnxt_napi *bnapi; 14554 u64 *sw; 14555 14556 bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi; 14557 sw = bnapi->cp_ring.stats.sw_stats; 14558 14559 stats->packets = 0; 14560 stats->packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 14561 stats->packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 14562 stats->packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 14563 14564 stats->bytes = 0; 14565 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 14566 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 14567 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 14568 } 14569 14570 static void bnxt_get_base_stats(struct net_device *dev, 14571 struct netdev_queue_stats_rx *rx, 14572 struct netdev_queue_stats_tx *tx) 14573 { 14574 struct bnxt *bp = netdev_priv(dev); 14575 14576 rx->packets = bp->net_stats_prev.rx_packets; 14577 rx->bytes = bp->net_stats_prev.rx_bytes; 14578 rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards; 14579 14580 tx->packets = bp->net_stats_prev.tx_packets; 14581 tx->bytes = bp->net_stats_prev.tx_bytes; 14582 } 14583 14584 static const struct netdev_stat_ops bnxt_stat_ops = { 14585 .get_queue_stats_rx = bnxt_get_queue_stats_rx, 14586 .get_queue_stats_tx = bnxt_get_queue_stats_tx, 14587 .get_base_stats = bnxt_get_base_stats, 14588 }; 14589 14590 static void bnxt_remove_one(struct pci_dev *pdev) 14591 { 14592 struct net_device *dev = pci_get_drvdata(pdev); 14593 struct bnxt *bp = netdev_priv(dev); 14594 14595 if (BNXT_PF(bp)) 14596 bnxt_sriov_disable(bp); 14597 14598 bnxt_rdma_aux_device_uninit(bp); 14599 14600 bnxt_ptp_clear(bp); 14601 unregister_netdev(dev); 14602 bnxt_free_l2_filters(bp, true); 14603 bnxt_free_ntp_fltrs(bp, true); 14604 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14605 /* Flush any pending tasks */ 14606 cancel_work_sync(&bp->sp_task); 14607 cancel_delayed_work_sync(&bp->fw_reset_task); 14608 bp->sp_event = 0; 14609 14610 bnxt_dl_fw_reporters_destroy(bp); 14611 bnxt_dl_unregister(bp); 14612 bnxt_shutdown_tc(bp); 14613 14614 bnxt_clear_int_mode(bp); 14615 bnxt_hwrm_func_drv_unrgtr(bp); 14616 bnxt_free_hwrm_resources(bp); 14617 bnxt_hwmon_uninit(bp); 14618 bnxt_ethtool_free(bp); 14619 bnxt_dcb_free(bp); 14620 kfree(bp->ptp_cfg); 14621 bp->ptp_cfg = NULL; 14622 kfree(bp->fw_health); 14623 bp->fw_health = NULL; 14624 bnxt_cleanup_pci(bp); 14625 bnxt_free_ctx_mem(bp); 14626 kfree(bp->rss_indir_tbl); 14627 bp->rss_indir_tbl = NULL; 14628 bnxt_free_port_stats(bp); 14629 free_netdev(dev); 14630 } 14631 14632 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 14633 { 14634 int rc = 0; 14635 struct bnxt_link_info *link_info = &bp->link_info; 14636 14637 bp->phy_flags = 0; 14638 rc = bnxt_hwrm_phy_qcaps(bp); 14639 if (rc) { 14640 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 14641 rc); 14642 return rc; 14643 } 14644 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 14645 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 14646 else 14647 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 14648 if (!fw_dflt) 14649 return 0; 14650 14651 mutex_lock(&bp->link_lock); 14652 rc = bnxt_update_link(bp, false); 14653 if (rc) { 14654 mutex_unlock(&bp->link_lock); 14655 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 14656 rc); 14657 return rc; 14658 } 14659 14660 /* Older firmware does not have supported_auto_speeds, so assume 14661 * that all supported speeds can be autonegotiated. 14662 */ 14663 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 14664 link_info->support_auto_speeds = link_info->support_speeds; 14665 14666 bnxt_init_ethtool_link_settings(bp); 14667 mutex_unlock(&bp->link_lock); 14668 return 0; 14669 } 14670 14671 static int bnxt_get_max_irq(struct pci_dev *pdev) 14672 { 14673 u16 ctrl; 14674 14675 if (!pdev->msix_cap) 14676 return 1; 14677 14678 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 14679 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 14680 } 14681 14682 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 14683 int *max_cp) 14684 { 14685 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 14686 int max_ring_grps = 0, max_irq; 14687 14688 *max_tx = hw_resc->max_tx_rings; 14689 *max_rx = hw_resc->max_rx_rings; 14690 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 14691 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 14692 bnxt_get_ulp_msix_num(bp), 14693 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 14694 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 14695 *max_cp = min_t(int, *max_cp, max_irq); 14696 max_ring_grps = hw_resc->max_hw_ring_grps; 14697 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 14698 *max_cp -= 1; 14699 *max_rx -= 2; 14700 } 14701 if (bp->flags & BNXT_FLAG_AGG_RINGS) 14702 *max_rx >>= 1; 14703 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 14704 int rc; 14705 14706 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 14707 if (rc) { 14708 *max_rx = 0; 14709 *max_tx = 0; 14710 } 14711 /* On P5 chips, max_cp output param should be available NQs */ 14712 *max_cp = max_irq; 14713 } 14714 *max_rx = min_t(int, *max_rx, max_ring_grps); 14715 } 14716 14717 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 14718 { 14719 int rx, tx, cp; 14720 14721 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 14722 *max_rx = rx; 14723 *max_tx = tx; 14724 if (!rx || !tx || !cp) 14725 return -ENOMEM; 14726 14727 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 14728 } 14729 14730 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 14731 bool shared) 14732 { 14733 int rc; 14734 14735 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 14736 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 14737 /* Not enough rings, try disabling agg rings. */ 14738 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 14739 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 14740 if (rc) { 14741 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 14742 bp->flags |= BNXT_FLAG_AGG_RINGS; 14743 return rc; 14744 } 14745 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 14746 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 14747 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 14748 bnxt_set_ring_params(bp); 14749 } 14750 14751 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 14752 int max_cp, max_stat, max_irq; 14753 14754 /* Reserve minimum resources for RoCE */ 14755 max_cp = bnxt_get_max_func_cp_rings(bp); 14756 max_stat = bnxt_get_max_func_stat_ctxs(bp); 14757 max_irq = bnxt_get_max_func_irqs(bp); 14758 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 14759 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 14760 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 14761 return 0; 14762 14763 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 14764 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 14765 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 14766 max_cp = min_t(int, max_cp, max_irq); 14767 max_cp = min_t(int, max_cp, max_stat); 14768 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 14769 if (rc) 14770 rc = 0; 14771 } 14772 return rc; 14773 } 14774 14775 /* In initial default shared ring setting, each shared ring must have a 14776 * RX/TX ring pair. 14777 */ 14778 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 14779 { 14780 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 14781 bp->rx_nr_rings = bp->cp_nr_rings; 14782 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 14783 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 14784 } 14785 14786 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 14787 { 14788 int dflt_rings, max_rx_rings, max_tx_rings, rc; 14789 14790 if (!bnxt_can_reserve_rings(bp)) 14791 return 0; 14792 14793 if (sh) 14794 bp->flags |= BNXT_FLAG_SHARED_RINGS; 14795 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 14796 /* Reduce default rings on multi-port cards so that total default 14797 * rings do not exceed CPU count. 14798 */ 14799 if (bp->port_count > 1) { 14800 int max_rings = 14801 max_t(int, num_online_cpus() / bp->port_count, 1); 14802 14803 dflt_rings = min_t(int, dflt_rings, max_rings); 14804 } 14805 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 14806 if (rc) 14807 return rc; 14808 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 14809 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 14810 if (sh) 14811 bnxt_trim_dflt_sh_rings(bp); 14812 else 14813 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 14814 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 14815 14816 rc = __bnxt_reserve_rings(bp); 14817 if (rc && rc != -ENODEV) 14818 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 14819 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 14820 if (sh) 14821 bnxt_trim_dflt_sh_rings(bp); 14822 14823 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 14824 if (bnxt_need_reserve_rings(bp)) { 14825 rc = __bnxt_reserve_rings(bp); 14826 if (rc && rc != -ENODEV) 14827 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 14828 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 14829 } 14830 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 14831 bp->rx_nr_rings++; 14832 bp->cp_nr_rings++; 14833 } 14834 if (rc) { 14835 bp->tx_nr_rings = 0; 14836 bp->rx_nr_rings = 0; 14837 } 14838 return rc; 14839 } 14840 14841 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 14842 { 14843 int rc; 14844 14845 if (bp->tx_nr_rings) 14846 return 0; 14847 14848 bnxt_ulp_irq_stop(bp); 14849 bnxt_clear_int_mode(bp); 14850 rc = bnxt_set_dflt_rings(bp, true); 14851 if (rc) { 14852 if (BNXT_VF(bp) && rc == -ENODEV) 14853 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 14854 else 14855 netdev_err(bp->dev, "Not enough rings available.\n"); 14856 goto init_dflt_ring_err; 14857 } 14858 rc = bnxt_init_int_mode(bp); 14859 if (rc) 14860 goto init_dflt_ring_err; 14861 14862 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 14863 14864 bnxt_set_dflt_rfs(bp); 14865 14866 init_dflt_ring_err: 14867 bnxt_ulp_irq_restart(bp, rc); 14868 return rc; 14869 } 14870 14871 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 14872 { 14873 int rc; 14874 14875 ASSERT_RTNL(); 14876 bnxt_hwrm_func_qcaps(bp); 14877 14878 if (netif_running(bp->dev)) 14879 __bnxt_close_nic(bp, true, false); 14880 14881 bnxt_ulp_irq_stop(bp); 14882 bnxt_clear_int_mode(bp); 14883 rc = bnxt_init_int_mode(bp); 14884 bnxt_ulp_irq_restart(bp, rc); 14885 14886 if (netif_running(bp->dev)) { 14887 if (rc) 14888 dev_close(bp->dev); 14889 else 14890 rc = bnxt_open_nic(bp, true, false); 14891 } 14892 14893 return rc; 14894 } 14895 14896 static int bnxt_init_mac_addr(struct bnxt *bp) 14897 { 14898 int rc = 0; 14899 14900 if (BNXT_PF(bp)) { 14901 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 14902 } else { 14903 #ifdef CONFIG_BNXT_SRIOV 14904 struct bnxt_vf_info *vf = &bp->vf; 14905 bool strict_approval = true; 14906 14907 if (is_valid_ether_addr(vf->mac_addr)) { 14908 /* overwrite netdev dev_addr with admin VF MAC */ 14909 eth_hw_addr_set(bp->dev, vf->mac_addr); 14910 /* Older PF driver or firmware may not approve this 14911 * correctly. 14912 */ 14913 strict_approval = false; 14914 } else { 14915 eth_hw_addr_random(bp->dev); 14916 } 14917 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 14918 #endif 14919 } 14920 return rc; 14921 } 14922 14923 static void bnxt_vpd_read_info(struct bnxt *bp) 14924 { 14925 struct pci_dev *pdev = bp->pdev; 14926 unsigned int vpd_size, kw_len; 14927 int pos, size; 14928 u8 *vpd_data; 14929 14930 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 14931 if (IS_ERR(vpd_data)) { 14932 pci_warn(pdev, "Unable to read VPD\n"); 14933 return; 14934 } 14935 14936 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 14937 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 14938 if (pos < 0) 14939 goto read_sn; 14940 14941 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 14942 memcpy(bp->board_partno, &vpd_data[pos], size); 14943 14944 read_sn: 14945 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 14946 PCI_VPD_RO_KEYWORD_SERIALNO, 14947 &kw_len); 14948 if (pos < 0) 14949 goto exit; 14950 14951 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 14952 memcpy(bp->board_serialno, &vpd_data[pos], size); 14953 exit: 14954 kfree(vpd_data); 14955 } 14956 14957 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 14958 { 14959 struct pci_dev *pdev = bp->pdev; 14960 u64 qword; 14961 14962 qword = pci_get_dsn(pdev); 14963 if (!qword) { 14964 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 14965 return -EOPNOTSUPP; 14966 } 14967 14968 put_unaligned_le64(qword, dsn); 14969 14970 bp->flags |= BNXT_FLAG_DSN_VALID; 14971 return 0; 14972 } 14973 14974 static int bnxt_map_db_bar(struct bnxt *bp) 14975 { 14976 if (!bp->db_size) 14977 return -ENODEV; 14978 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 14979 if (!bp->bar1) 14980 return -ENOMEM; 14981 return 0; 14982 } 14983 14984 void bnxt_print_device_info(struct bnxt *bp) 14985 { 14986 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 14987 board_info[bp->board_idx].name, 14988 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 14989 14990 pcie_print_link_status(bp->pdev); 14991 } 14992 14993 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 14994 { 14995 struct bnxt_hw_resc *hw_resc; 14996 struct net_device *dev; 14997 struct bnxt *bp; 14998 int rc, max_irqs; 14999 15000 if (pci_is_bridge(pdev)) 15001 return -ENODEV; 15002 15003 /* Clear any pending DMA transactions from crash kernel 15004 * while loading driver in capture kernel. 15005 */ 15006 if (is_kdump_kernel()) { 15007 pci_clear_master(pdev); 15008 pcie_flr(pdev); 15009 } 15010 15011 max_irqs = bnxt_get_max_irq(pdev); 15012 dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE, 15013 max_irqs); 15014 if (!dev) 15015 return -ENOMEM; 15016 15017 bp = netdev_priv(dev); 15018 bp->board_idx = ent->driver_data; 15019 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 15020 bnxt_set_max_func_irqs(bp, max_irqs); 15021 15022 if (bnxt_vf_pciid(bp->board_idx)) 15023 bp->flags |= BNXT_FLAG_VF; 15024 15025 /* No devlink port registration in case of a VF */ 15026 if (BNXT_PF(bp)) 15027 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port); 15028 15029 if (pdev->msix_cap) 15030 bp->flags |= BNXT_FLAG_MSIX_CAP; 15031 15032 rc = bnxt_init_board(pdev, dev); 15033 if (rc < 0) 15034 goto init_err_free; 15035 15036 dev->netdev_ops = &bnxt_netdev_ops; 15037 dev->stat_ops = &bnxt_stat_ops; 15038 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 15039 dev->ethtool_ops = &bnxt_ethtool_ops; 15040 pci_set_drvdata(pdev, dev); 15041 15042 rc = bnxt_alloc_hwrm_resources(bp); 15043 if (rc) 15044 goto init_err_pci_clean; 15045 15046 mutex_init(&bp->hwrm_cmd_lock); 15047 mutex_init(&bp->link_lock); 15048 15049 rc = bnxt_fw_init_one_p1(bp); 15050 if (rc) 15051 goto init_err_pci_clean; 15052 15053 if (BNXT_PF(bp)) 15054 bnxt_vpd_read_info(bp); 15055 15056 if (BNXT_CHIP_P5_PLUS(bp)) { 15057 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS; 15058 if (BNXT_CHIP_P7(bp)) 15059 bp->flags |= BNXT_FLAG_CHIP_P7; 15060 } 15061 15062 rc = bnxt_alloc_rss_indir_tbl(bp); 15063 if (rc) 15064 goto init_err_pci_clean; 15065 15066 rc = bnxt_fw_init_one_p2(bp); 15067 if (rc) 15068 goto init_err_pci_clean; 15069 15070 rc = bnxt_map_db_bar(bp); 15071 if (rc) { 15072 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 15073 rc); 15074 goto init_err_pci_clean; 15075 } 15076 15077 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 15078 NETIF_F_TSO | NETIF_F_TSO6 | 15079 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 15080 NETIF_F_GSO_IPXIP4 | 15081 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 15082 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 15083 NETIF_F_RXCSUM | NETIF_F_GRO; 15084 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) 15085 dev->hw_features |= NETIF_F_GSO_UDP_L4; 15086 15087 if (BNXT_SUPPORTS_TPA(bp)) 15088 dev->hw_features |= NETIF_F_LRO; 15089 15090 dev->hw_enc_features = 15091 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 15092 NETIF_F_TSO | NETIF_F_TSO6 | 15093 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 15094 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 15095 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 15096 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) 15097 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 15098 if (bp->flags & BNXT_FLAG_CHIP_P7) 15099 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7; 15100 else 15101 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 15102 15103 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 15104 NETIF_F_GSO_GRE_CSUM; 15105 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 15106 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 15107 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 15108 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 15109 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 15110 if (BNXT_SUPPORTS_TPA(bp)) 15111 dev->hw_features |= NETIF_F_GRO_HW; 15112 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 15113 if (dev->features & NETIF_F_GRO_HW) 15114 dev->features &= ~NETIF_F_LRO; 15115 dev->priv_flags |= IFF_UNICAST_FLT; 15116 15117 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 15118 15119 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 15120 NETDEV_XDP_ACT_RX_SG; 15121 15122 #ifdef CONFIG_BNXT_SRIOV 15123 init_waitqueue_head(&bp->sriov_cfg_wait); 15124 #endif 15125 if (BNXT_SUPPORTS_TPA(bp)) { 15126 bp->gro_func = bnxt_gro_func_5730x; 15127 if (BNXT_CHIP_P4(bp)) 15128 bp->gro_func = bnxt_gro_func_5731x; 15129 else if (BNXT_CHIP_P5_PLUS(bp)) 15130 bp->gro_func = bnxt_gro_func_5750x; 15131 } 15132 if (!BNXT_CHIP_P4_PLUS(bp)) 15133 bp->flags |= BNXT_FLAG_DOUBLE_DB; 15134 15135 rc = bnxt_init_mac_addr(bp); 15136 if (rc) { 15137 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 15138 rc = -EADDRNOTAVAIL; 15139 goto init_err_pci_clean; 15140 } 15141 15142 if (BNXT_PF(bp)) { 15143 /* Read the adapter's DSN to use as the eswitch switch_id */ 15144 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 15145 } 15146 15147 /* MTU range: 60 - FW defined max */ 15148 dev->min_mtu = ETH_ZLEN; 15149 dev->max_mtu = bp->max_mtu; 15150 15151 rc = bnxt_probe_phy(bp, true); 15152 if (rc) 15153 goto init_err_pci_clean; 15154 15155 hw_resc = &bp->hw_resc; 15156 bp->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows + 15157 BNXT_L2_FLTR_MAX_FLTR; 15158 /* Older firmware may not report these filters properly */ 15159 if (bp->max_fltr < BNXT_MAX_FLTR) 15160 bp->max_fltr = BNXT_MAX_FLTR; 15161 bnxt_init_l2_fltr_tbl(bp); 15162 bnxt_set_rx_skb_mode(bp, false); 15163 bnxt_set_tpa_flags(bp); 15164 bnxt_set_ring_params(bp); 15165 rc = bnxt_set_dflt_rings(bp, true); 15166 if (rc) { 15167 if (BNXT_VF(bp) && rc == -ENODEV) { 15168 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 15169 } else { 15170 netdev_err(bp->dev, "Not enough rings available.\n"); 15171 rc = -ENOMEM; 15172 } 15173 goto init_err_pci_clean; 15174 } 15175 15176 bnxt_fw_init_one_p3(bp); 15177 15178 bnxt_init_dflt_coal(bp); 15179 15180 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 15181 bp->flags |= BNXT_FLAG_STRIP_VLAN; 15182 15183 rc = bnxt_init_int_mode(bp); 15184 if (rc) 15185 goto init_err_pci_clean; 15186 15187 /* No TC has been set yet and rings may have been trimmed due to 15188 * limited MSIX, so we re-initialize the TX rings per TC. 15189 */ 15190 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 15191 15192 if (BNXT_PF(bp)) { 15193 if (!bnxt_pf_wq) { 15194 bnxt_pf_wq = 15195 create_singlethread_workqueue("bnxt_pf_wq"); 15196 if (!bnxt_pf_wq) { 15197 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 15198 rc = -ENOMEM; 15199 goto init_err_pci_clean; 15200 } 15201 } 15202 rc = bnxt_init_tc(bp); 15203 if (rc) 15204 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 15205 rc); 15206 } 15207 15208 bnxt_inv_fw_health_reg(bp); 15209 rc = bnxt_dl_register(bp); 15210 if (rc) 15211 goto init_err_dl; 15212 15213 INIT_LIST_HEAD(&bp->usr_fltr_list); 15214 15215 rc = register_netdev(dev); 15216 if (rc) 15217 goto init_err_cleanup; 15218 15219 bnxt_dl_fw_reporters_create(bp); 15220 15221 bnxt_rdma_aux_device_init(bp); 15222 15223 bnxt_print_device_info(bp); 15224 15225 pci_save_state(pdev); 15226 15227 return 0; 15228 init_err_cleanup: 15229 bnxt_dl_unregister(bp); 15230 init_err_dl: 15231 bnxt_shutdown_tc(bp); 15232 bnxt_clear_int_mode(bp); 15233 15234 init_err_pci_clean: 15235 bnxt_hwrm_func_drv_unrgtr(bp); 15236 bnxt_free_hwrm_resources(bp); 15237 bnxt_hwmon_uninit(bp); 15238 bnxt_ethtool_free(bp); 15239 bnxt_ptp_clear(bp); 15240 kfree(bp->ptp_cfg); 15241 bp->ptp_cfg = NULL; 15242 kfree(bp->fw_health); 15243 bp->fw_health = NULL; 15244 bnxt_cleanup_pci(bp); 15245 bnxt_free_ctx_mem(bp); 15246 kfree(bp->rss_indir_tbl); 15247 bp->rss_indir_tbl = NULL; 15248 15249 init_err_free: 15250 free_netdev(dev); 15251 return rc; 15252 } 15253 15254 static void bnxt_shutdown(struct pci_dev *pdev) 15255 { 15256 struct net_device *dev = pci_get_drvdata(pdev); 15257 struct bnxt *bp; 15258 15259 if (!dev) 15260 return; 15261 15262 rtnl_lock(); 15263 bp = netdev_priv(dev); 15264 if (!bp) 15265 goto shutdown_exit; 15266 15267 if (netif_running(dev)) 15268 dev_close(dev); 15269 15270 bnxt_clear_int_mode(bp); 15271 pci_disable_device(pdev); 15272 15273 if (system_state == SYSTEM_POWER_OFF) { 15274 pci_wake_from_d3(pdev, bp->wol); 15275 pci_set_power_state(pdev, PCI_D3hot); 15276 } 15277 15278 shutdown_exit: 15279 rtnl_unlock(); 15280 } 15281 15282 #ifdef CONFIG_PM_SLEEP 15283 static int bnxt_suspend(struct device *device) 15284 { 15285 struct net_device *dev = dev_get_drvdata(device); 15286 struct bnxt *bp = netdev_priv(dev); 15287 int rc = 0; 15288 15289 rtnl_lock(); 15290 bnxt_ulp_stop(bp); 15291 if (netif_running(dev)) { 15292 netif_device_detach(dev); 15293 rc = bnxt_close(dev); 15294 } 15295 bnxt_hwrm_func_drv_unrgtr(bp); 15296 pci_disable_device(bp->pdev); 15297 bnxt_free_ctx_mem(bp); 15298 rtnl_unlock(); 15299 return rc; 15300 } 15301 15302 static int bnxt_resume(struct device *device) 15303 { 15304 struct net_device *dev = dev_get_drvdata(device); 15305 struct bnxt *bp = netdev_priv(dev); 15306 int rc = 0; 15307 15308 rtnl_lock(); 15309 rc = pci_enable_device(bp->pdev); 15310 if (rc) { 15311 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 15312 rc); 15313 goto resume_exit; 15314 } 15315 pci_set_master(bp->pdev); 15316 if (bnxt_hwrm_ver_get(bp)) { 15317 rc = -ENODEV; 15318 goto resume_exit; 15319 } 15320 rc = bnxt_hwrm_func_reset(bp); 15321 if (rc) { 15322 rc = -EBUSY; 15323 goto resume_exit; 15324 } 15325 15326 rc = bnxt_hwrm_func_qcaps(bp); 15327 if (rc) 15328 goto resume_exit; 15329 15330 bnxt_clear_reservations(bp, true); 15331 15332 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 15333 rc = -ENODEV; 15334 goto resume_exit; 15335 } 15336 15337 bnxt_get_wol_settings(bp); 15338 if (netif_running(dev)) { 15339 rc = bnxt_open(dev); 15340 if (!rc) 15341 netif_device_attach(dev); 15342 } 15343 15344 resume_exit: 15345 bnxt_ulp_start(bp, rc); 15346 if (!rc) 15347 bnxt_reenable_sriov(bp); 15348 rtnl_unlock(); 15349 return rc; 15350 } 15351 15352 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 15353 #define BNXT_PM_OPS (&bnxt_pm_ops) 15354 15355 #else 15356 15357 #define BNXT_PM_OPS NULL 15358 15359 #endif /* CONFIG_PM_SLEEP */ 15360 15361 /** 15362 * bnxt_io_error_detected - called when PCI error is detected 15363 * @pdev: Pointer to PCI device 15364 * @state: The current pci connection state 15365 * 15366 * This function is called after a PCI bus error affecting 15367 * this device has been detected. 15368 */ 15369 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 15370 pci_channel_state_t state) 15371 { 15372 struct net_device *netdev = pci_get_drvdata(pdev); 15373 struct bnxt *bp = netdev_priv(netdev); 15374 15375 netdev_info(netdev, "PCI I/O error detected\n"); 15376 15377 rtnl_lock(); 15378 netif_device_detach(netdev); 15379 15380 bnxt_ulp_stop(bp); 15381 15382 if (state == pci_channel_io_perm_failure) { 15383 rtnl_unlock(); 15384 return PCI_ERS_RESULT_DISCONNECT; 15385 } 15386 15387 if (state == pci_channel_io_frozen) 15388 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 15389 15390 if (netif_running(netdev)) 15391 bnxt_close(netdev); 15392 15393 if (pci_is_enabled(pdev)) 15394 pci_disable_device(pdev); 15395 bnxt_free_ctx_mem(bp); 15396 rtnl_unlock(); 15397 15398 /* Request a slot slot reset. */ 15399 return PCI_ERS_RESULT_NEED_RESET; 15400 } 15401 15402 /** 15403 * bnxt_io_slot_reset - called after the pci bus has been reset. 15404 * @pdev: Pointer to PCI device 15405 * 15406 * Restart the card from scratch, as if from a cold-boot. 15407 * At this point, the card has exprienced a hard reset, 15408 * followed by fixups by BIOS, and has its config space 15409 * set up identically to what it was at cold boot. 15410 */ 15411 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 15412 { 15413 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 15414 struct net_device *netdev = pci_get_drvdata(pdev); 15415 struct bnxt *bp = netdev_priv(netdev); 15416 int retry = 0; 15417 int err = 0; 15418 int off; 15419 15420 netdev_info(bp->dev, "PCI Slot Reset\n"); 15421 15422 rtnl_lock(); 15423 15424 if (pci_enable_device(pdev)) { 15425 dev_err(&pdev->dev, 15426 "Cannot re-enable PCI device after reset.\n"); 15427 } else { 15428 pci_set_master(pdev); 15429 /* Upon fatal error, our device internal logic that latches to 15430 * BAR value is getting reset and will restore only upon 15431 * rewritting the BARs. 15432 * 15433 * As pci_restore_state() does not re-write the BARs if the 15434 * value is same as saved value earlier, driver needs to 15435 * write the BARs to 0 to force restore, in case of fatal error. 15436 */ 15437 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 15438 &bp->state)) { 15439 for (off = PCI_BASE_ADDRESS_0; 15440 off <= PCI_BASE_ADDRESS_5; off += 4) 15441 pci_write_config_dword(bp->pdev, off, 0); 15442 } 15443 pci_restore_state(pdev); 15444 pci_save_state(pdev); 15445 15446 bnxt_inv_fw_health_reg(bp); 15447 bnxt_try_map_fw_health_reg(bp); 15448 15449 /* In some PCIe AER scenarios, firmware may take up to 15450 * 10 seconds to become ready in the worst case. 15451 */ 15452 do { 15453 err = bnxt_try_recover_fw(bp); 15454 if (!err) 15455 break; 15456 retry++; 15457 } while (retry < BNXT_FW_SLOT_RESET_RETRY); 15458 15459 if (err) { 15460 dev_err(&pdev->dev, "Firmware not ready\n"); 15461 goto reset_exit; 15462 } 15463 15464 err = bnxt_hwrm_func_reset(bp); 15465 if (!err) 15466 result = PCI_ERS_RESULT_RECOVERED; 15467 15468 bnxt_ulp_irq_stop(bp); 15469 bnxt_clear_int_mode(bp); 15470 err = bnxt_init_int_mode(bp); 15471 bnxt_ulp_irq_restart(bp, err); 15472 } 15473 15474 reset_exit: 15475 bnxt_clear_reservations(bp, true); 15476 rtnl_unlock(); 15477 15478 return result; 15479 } 15480 15481 /** 15482 * bnxt_io_resume - called when traffic can start flowing again. 15483 * @pdev: Pointer to PCI device 15484 * 15485 * This callback is called when the error recovery driver tells 15486 * us that its OK to resume normal operation. 15487 */ 15488 static void bnxt_io_resume(struct pci_dev *pdev) 15489 { 15490 struct net_device *netdev = pci_get_drvdata(pdev); 15491 struct bnxt *bp = netdev_priv(netdev); 15492 int err; 15493 15494 netdev_info(bp->dev, "PCI Slot Resume\n"); 15495 rtnl_lock(); 15496 15497 err = bnxt_hwrm_func_qcaps(bp); 15498 if (!err && netif_running(netdev)) 15499 err = bnxt_open(netdev); 15500 15501 bnxt_ulp_start(bp, err); 15502 if (!err) { 15503 bnxt_reenable_sriov(bp); 15504 netif_device_attach(netdev); 15505 } 15506 15507 rtnl_unlock(); 15508 } 15509 15510 static const struct pci_error_handlers bnxt_err_handler = { 15511 .error_detected = bnxt_io_error_detected, 15512 .slot_reset = bnxt_io_slot_reset, 15513 .resume = bnxt_io_resume 15514 }; 15515 15516 static struct pci_driver bnxt_pci_driver = { 15517 .name = DRV_MODULE_NAME, 15518 .id_table = bnxt_pci_tbl, 15519 .probe = bnxt_init_one, 15520 .remove = bnxt_remove_one, 15521 .shutdown = bnxt_shutdown, 15522 .driver.pm = BNXT_PM_OPS, 15523 .err_handler = &bnxt_err_handler, 15524 #if defined(CONFIG_BNXT_SRIOV) 15525 .sriov_configure = bnxt_sriov_configure, 15526 #endif 15527 }; 15528 15529 static int __init bnxt_init(void) 15530 { 15531 int err; 15532 15533 bnxt_debug_init(); 15534 err = pci_register_driver(&bnxt_pci_driver); 15535 if (err) { 15536 bnxt_debug_exit(); 15537 return err; 15538 } 15539 15540 return 0; 15541 } 15542 15543 static void __exit bnxt_exit(void) 15544 { 15545 pci_unregister_driver(&bnxt_pci_driver); 15546 if (bnxt_pf_wq) 15547 destroy_workqueue(bnxt_pf_wq); 15548 bnxt_debug_exit(); 15549 } 15550 15551 module_init(bnxt_init); 15552 module_exit(bnxt_exit); 15553