1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/gro.h> 41 #include <net/ip.h> 42 #include <net/tcp.h> 43 #include <net/udp.h> 44 #include <net/checksum.h> 45 #include <net/ip6_checksum.h> 46 #include <net/udp_tunnel.h> 47 #include <linux/workqueue.h> 48 #include <linux/prefetch.h> 49 #include <linux/cache.h> 50 #include <linux/log2.h> 51 #include <linux/aer.h> 52 #include <linux/bitmap.h> 53 #include <linux/cpu_rmap.h> 54 #include <linux/cpumask.h> 55 #include <net/pkt_cls.h> 56 #include <linux/hwmon.h> 57 #include <linux/hwmon-sysfs.h> 58 #include <net/page_pool.h> 59 60 #include "bnxt_hsi.h" 61 #include "bnxt.h" 62 #include "bnxt_hwrm.h" 63 #include "bnxt_ulp.h" 64 #include "bnxt_sriov.h" 65 #include "bnxt_ethtool.h" 66 #include "bnxt_dcb.h" 67 #include "bnxt_xdp.h" 68 #include "bnxt_ptp.h" 69 #include "bnxt_vfr.h" 70 #include "bnxt_tc.h" 71 #include "bnxt_devlink.h" 72 #include "bnxt_debugfs.h" 73 74 #define BNXT_TX_TIMEOUT (5 * HZ) 75 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 76 NETIF_MSG_TX_ERR) 77 78 MODULE_LICENSE("GPL"); 79 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 80 81 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 82 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 83 #define BNXT_RX_COPY_THRESH 256 84 85 #define BNXT_TX_PUSH_THRESH 164 86 87 /* indexed by enum board_idx */ 88 static const struct { 89 char *name; 90 } board_info[] = { 91 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 92 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 93 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 94 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 95 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 96 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 97 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 98 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 99 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 100 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 101 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 102 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 103 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 104 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 105 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 106 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 108 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 109 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 110 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 111 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 112 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 113 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 114 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 115 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 116 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 117 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 118 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 119 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 120 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 121 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 122 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 123 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 124 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 125 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 126 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 127 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 128 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 129 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 130 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 131 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 132 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 133 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 134 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 135 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 136 }; 137 138 static const struct pci_device_id bnxt_pci_tbl[] = { 139 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 140 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 141 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 142 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 143 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 144 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 145 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 146 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 147 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 148 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 149 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 150 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 151 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 152 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 153 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 154 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 155 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 156 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 157 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 158 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 159 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 160 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 161 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 162 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 163 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 165 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 166 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 167 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 168 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 169 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 171 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 172 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 173 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 174 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 175 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 176 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 177 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 182 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 183 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 184 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 185 #ifdef CONFIG_BNXT_SRIOV 186 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 187 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 188 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 189 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 190 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 191 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 192 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 193 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 194 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 195 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 196 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 197 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 198 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 199 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 200 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 202 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 203 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 205 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 206 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 207 #endif 208 { 0 } 209 }; 210 211 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 212 213 static const u16 bnxt_vf_req_snif[] = { 214 HWRM_FUNC_CFG, 215 HWRM_FUNC_VF_CFG, 216 HWRM_PORT_PHY_QCFG, 217 HWRM_CFA_L2_FILTER_ALLOC, 218 }; 219 220 static const u16 bnxt_async_events_arr[] = { 221 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 222 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 223 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 224 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 225 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 226 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 227 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 228 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 229 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 230 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 231 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 232 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 233 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 234 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 235 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 236 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, 237 }; 238 239 static struct workqueue_struct *bnxt_pf_wq; 240 241 static bool bnxt_vf_pciid(enum board_idx idx) 242 { 243 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 244 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 245 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 246 idx == NETXTREME_E_P5_VF_HV); 247 } 248 249 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 250 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 251 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 252 253 #define BNXT_CP_DB_IRQ_DIS(db) \ 254 writel(DB_CP_IRQ_DIS_FLAGS, db) 255 256 #define BNXT_DB_CQ(db, idx) \ 257 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 258 259 #define BNXT_DB_NQ_P5(db, idx) \ 260 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), \ 261 (db)->doorbell) 262 263 #define BNXT_DB_CQ_ARM(db, idx) \ 264 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 265 266 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 267 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\ 268 (db)->doorbell) 269 270 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 271 { 272 if (bp->flags & BNXT_FLAG_CHIP_P5) 273 BNXT_DB_NQ_P5(db, idx); 274 else 275 BNXT_DB_CQ(db, idx); 276 } 277 278 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 279 { 280 if (bp->flags & BNXT_FLAG_CHIP_P5) 281 BNXT_DB_NQ_ARM_P5(db, idx); 282 else 283 BNXT_DB_CQ_ARM(db, idx); 284 } 285 286 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 287 { 288 if (bp->flags & BNXT_FLAG_CHIP_P5) 289 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 290 RING_CMP(idx), db->doorbell); 291 else 292 BNXT_DB_CQ(db, idx); 293 } 294 295 const u16 bnxt_lhint_arr[] = { 296 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 297 TX_BD_FLAGS_LHINT_512_TO_1023, 298 TX_BD_FLAGS_LHINT_1024_TO_2047, 299 TX_BD_FLAGS_LHINT_1024_TO_2047, 300 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 301 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 302 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 303 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 304 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 305 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 306 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 307 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 308 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 309 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 310 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 311 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 312 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 313 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 314 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 315 }; 316 317 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 318 { 319 struct metadata_dst *md_dst = skb_metadata_dst(skb); 320 321 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 322 return 0; 323 324 return md_dst->u.port_info.port_id; 325 } 326 327 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 328 u16 prod) 329 { 330 bnxt_db_write(bp, &txr->tx_db, prod); 331 txr->kick_pending = 0; 332 } 333 334 static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp, 335 struct bnxt_tx_ring_info *txr, 336 struct netdev_queue *txq) 337 { 338 netif_tx_stop_queue(txq); 339 340 /* netif_tx_stop_queue() must be done before checking 341 * tx index in bnxt_tx_avail() below, because in 342 * bnxt_tx_int(), we update tx index before checking for 343 * netif_tx_queue_stopped(). 344 */ 345 smp_mb(); 346 if (bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh) { 347 netif_tx_wake_queue(txq); 348 return false; 349 } 350 351 return true; 352 } 353 354 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 355 { 356 struct bnxt *bp = netdev_priv(dev); 357 struct tx_bd *txbd; 358 struct tx_bd_ext *txbd1; 359 struct netdev_queue *txq; 360 int i; 361 dma_addr_t mapping; 362 unsigned int length, pad = 0; 363 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 364 u16 prod, last_frag; 365 struct pci_dev *pdev = bp->pdev; 366 struct bnxt_tx_ring_info *txr; 367 struct bnxt_sw_tx_bd *tx_buf; 368 __le32 lflags = 0; 369 370 i = skb_get_queue_mapping(skb); 371 if (unlikely(i >= bp->tx_nr_rings)) { 372 dev_kfree_skb_any(skb); 373 atomic_long_inc(&dev->tx_dropped); 374 return NETDEV_TX_OK; 375 } 376 377 txq = netdev_get_tx_queue(dev, i); 378 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 379 prod = txr->tx_prod; 380 381 free_size = bnxt_tx_avail(bp, txr); 382 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 383 /* We must have raced with NAPI cleanup */ 384 if (net_ratelimit() && txr->kick_pending) 385 netif_warn(bp, tx_err, dev, 386 "bnxt: ring busy w/ flush pending!\n"); 387 if (bnxt_txr_netif_try_stop_queue(bp, txr, txq)) 388 return NETDEV_TX_BUSY; 389 } 390 391 length = skb->len; 392 len = skb_headlen(skb); 393 last_frag = skb_shinfo(skb)->nr_frags; 394 395 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 396 397 txbd->tx_bd_opaque = prod; 398 399 tx_buf = &txr->tx_buf_ring[prod]; 400 tx_buf->skb = skb; 401 tx_buf->nr_frags = last_frag; 402 403 vlan_tag_flags = 0; 404 cfa_action = bnxt_xmit_get_cfa_action(skb); 405 if (skb_vlan_tag_present(skb)) { 406 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 407 skb_vlan_tag_get(skb); 408 /* Currently supports 8021Q, 8021AD vlan offloads 409 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 410 */ 411 if (skb->vlan_proto == htons(ETH_P_8021Q)) 412 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 413 } 414 415 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 416 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 417 418 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && 419 atomic_dec_if_positive(&ptp->tx_avail) >= 0) { 420 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid, 421 &ptp->tx_hdr_off)) { 422 if (vlan_tag_flags) 423 ptp->tx_hdr_off += VLAN_HLEN; 424 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 425 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 426 } else { 427 atomic_inc(&bp->ptp_cfg->tx_avail); 428 } 429 } 430 } 431 432 if (unlikely(skb->no_fcs)) 433 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 434 435 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 436 !lflags) { 437 struct tx_push_buffer *tx_push_buf = txr->tx_push; 438 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 439 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 440 void __iomem *db = txr->tx_db.doorbell; 441 void *pdata = tx_push_buf->data; 442 u64 *end; 443 int j, push_len; 444 445 /* Set COAL_NOW to be ready quickly for the next push */ 446 tx_push->tx_bd_len_flags_type = 447 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 448 TX_BD_TYPE_LONG_TX_BD | 449 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 450 TX_BD_FLAGS_COAL_NOW | 451 TX_BD_FLAGS_PACKET_END | 452 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 453 454 if (skb->ip_summed == CHECKSUM_PARTIAL) 455 tx_push1->tx_bd_hsize_lflags = 456 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 457 else 458 tx_push1->tx_bd_hsize_lflags = 0; 459 460 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 461 tx_push1->tx_bd_cfa_action = 462 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 463 464 end = pdata + length; 465 end = PTR_ALIGN(end, 8) - 1; 466 *end = 0; 467 468 skb_copy_from_linear_data(skb, pdata, len); 469 pdata += len; 470 for (j = 0; j < last_frag; j++) { 471 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 472 void *fptr; 473 474 fptr = skb_frag_address_safe(frag); 475 if (!fptr) 476 goto normal_tx; 477 478 memcpy(pdata, fptr, skb_frag_size(frag)); 479 pdata += skb_frag_size(frag); 480 } 481 482 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 483 txbd->tx_bd_haddr = txr->data_mapping; 484 prod = NEXT_TX(prod); 485 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 486 memcpy(txbd, tx_push1, sizeof(*txbd)); 487 prod = NEXT_TX(prod); 488 tx_push->doorbell = 489 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 490 txr->tx_prod = prod; 491 492 tx_buf->is_push = 1; 493 netdev_tx_sent_queue(txq, skb->len); 494 wmb(); /* Sync is_push and byte queue before pushing data */ 495 496 push_len = (length + sizeof(*tx_push) + 7) / 8; 497 if (push_len > 16) { 498 __iowrite64_copy(db, tx_push_buf, 16); 499 __iowrite32_copy(db + 4, tx_push_buf + 1, 500 (push_len - 16) << 1); 501 } else { 502 __iowrite64_copy(db, tx_push_buf, push_len); 503 } 504 505 goto tx_done; 506 } 507 508 normal_tx: 509 if (length < BNXT_MIN_PKT_SIZE) { 510 pad = BNXT_MIN_PKT_SIZE - length; 511 if (skb_pad(skb, pad)) 512 /* SKB already freed. */ 513 goto tx_kick_pending; 514 length = BNXT_MIN_PKT_SIZE; 515 } 516 517 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 518 519 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 520 goto tx_free; 521 522 dma_unmap_addr_set(tx_buf, mapping, mapping); 523 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 524 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 525 526 txbd->tx_bd_haddr = cpu_to_le64(mapping); 527 528 prod = NEXT_TX(prod); 529 txbd1 = (struct tx_bd_ext *) 530 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 531 532 txbd1->tx_bd_hsize_lflags = lflags; 533 if (skb_is_gso(skb)) { 534 u32 hdr_len; 535 536 if (skb->encapsulation) 537 hdr_len = skb_inner_network_offset(skb) + 538 skb_inner_network_header_len(skb) + 539 inner_tcp_hdrlen(skb); 540 else 541 hdr_len = skb_transport_offset(skb) + 542 tcp_hdrlen(skb); 543 544 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 545 TX_BD_FLAGS_T_IPID | 546 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 547 length = skb_shinfo(skb)->gso_size; 548 txbd1->tx_bd_mss = cpu_to_le32(length); 549 length += hdr_len; 550 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 551 txbd1->tx_bd_hsize_lflags |= 552 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 553 txbd1->tx_bd_mss = 0; 554 } 555 556 length >>= 9; 557 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 558 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 559 skb->len); 560 i = 0; 561 goto tx_dma_error; 562 } 563 flags |= bnxt_lhint_arr[length]; 564 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 565 566 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 567 txbd1->tx_bd_cfa_action = 568 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 569 for (i = 0; i < last_frag; i++) { 570 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 571 572 prod = NEXT_TX(prod); 573 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 574 575 len = skb_frag_size(frag); 576 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 577 DMA_TO_DEVICE); 578 579 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 580 goto tx_dma_error; 581 582 tx_buf = &txr->tx_buf_ring[prod]; 583 dma_unmap_addr_set(tx_buf, mapping, mapping); 584 585 txbd->tx_bd_haddr = cpu_to_le64(mapping); 586 587 flags = len << TX_BD_LEN_SHIFT; 588 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 589 } 590 591 flags &= ~TX_BD_LEN; 592 txbd->tx_bd_len_flags_type = 593 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 594 TX_BD_FLAGS_PACKET_END); 595 596 netdev_tx_sent_queue(txq, skb->len); 597 598 skb_tx_timestamp(skb); 599 600 /* Sync BD data before updating doorbell */ 601 wmb(); 602 603 prod = NEXT_TX(prod); 604 txr->tx_prod = prod; 605 606 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 607 bnxt_txr_db_kick(bp, txr, prod); 608 else 609 txr->kick_pending = 1; 610 611 tx_done: 612 613 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 614 if (netdev_xmit_more() && !tx_buf->is_push) 615 bnxt_txr_db_kick(bp, txr, prod); 616 617 bnxt_txr_netif_try_stop_queue(bp, txr, txq); 618 } 619 return NETDEV_TX_OK; 620 621 tx_dma_error: 622 if (BNXT_TX_PTP_IS_SET(lflags)) 623 atomic_inc(&bp->ptp_cfg->tx_avail); 624 625 last_frag = i; 626 627 /* start back at beginning and unmap skb */ 628 prod = txr->tx_prod; 629 tx_buf = &txr->tx_buf_ring[prod]; 630 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 631 skb_headlen(skb), DMA_TO_DEVICE); 632 prod = NEXT_TX(prod); 633 634 /* unmap remaining mapped pages */ 635 for (i = 0; i < last_frag; i++) { 636 prod = NEXT_TX(prod); 637 tx_buf = &txr->tx_buf_ring[prod]; 638 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 639 skb_frag_size(&skb_shinfo(skb)->frags[i]), 640 DMA_TO_DEVICE); 641 } 642 643 tx_free: 644 dev_kfree_skb_any(skb); 645 tx_kick_pending: 646 if (txr->kick_pending) 647 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 648 txr->tx_buf_ring[txr->tx_prod].skb = NULL; 649 atomic_long_inc(&dev->tx_dropped); 650 return NETDEV_TX_OK; 651 } 652 653 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 654 { 655 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 656 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 657 u16 cons = txr->tx_cons; 658 struct pci_dev *pdev = bp->pdev; 659 int i; 660 unsigned int tx_bytes = 0; 661 662 for (i = 0; i < nr_pkts; i++) { 663 struct bnxt_sw_tx_bd *tx_buf; 664 bool compl_deferred = false; 665 struct sk_buff *skb; 666 int j, last; 667 668 tx_buf = &txr->tx_buf_ring[cons]; 669 cons = NEXT_TX(cons); 670 skb = tx_buf->skb; 671 tx_buf->skb = NULL; 672 673 if (tx_buf->is_push) { 674 tx_buf->is_push = 0; 675 goto next_tx_int; 676 } 677 678 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 679 skb_headlen(skb), DMA_TO_DEVICE); 680 last = tx_buf->nr_frags; 681 682 for (j = 0; j < last; j++) { 683 cons = NEXT_TX(cons); 684 tx_buf = &txr->tx_buf_ring[cons]; 685 dma_unmap_page( 686 &pdev->dev, 687 dma_unmap_addr(tx_buf, mapping), 688 skb_frag_size(&skb_shinfo(skb)->frags[j]), 689 DMA_TO_DEVICE); 690 } 691 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 692 if (bp->flags & BNXT_FLAG_CHIP_P5) { 693 if (!bnxt_get_tx_ts_p5(bp, skb)) 694 compl_deferred = true; 695 else 696 atomic_inc(&bp->ptp_cfg->tx_avail); 697 } 698 } 699 700 next_tx_int: 701 cons = NEXT_TX(cons); 702 703 tx_bytes += skb->len; 704 if (!compl_deferred) 705 dev_kfree_skb_any(skb); 706 } 707 708 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 709 txr->tx_cons = cons; 710 711 /* Need to make the tx_cons update visible to bnxt_start_xmit() 712 * before checking for netif_tx_queue_stopped(). Without the 713 * memory barrier, there is a small possibility that bnxt_start_xmit() 714 * will miss it and cause the queue to be stopped forever. 715 */ 716 smp_mb(); 717 718 if (unlikely(netif_tx_queue_stopped(txq)) && 719 bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh && 720 READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING) 721 netif_tx_wake_queue(txq); 722 } 723 724 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 725 struct bnxt_rx_ring_info *rxr, 726 gfp_t gfp) 727 { 728 struct device *dev = &bp->pdev->dev; 729 struct page *page; 730 731 page = page_pool_dev_alloc_pages(rxr->page_pool); 732 if (!page) 733 return NULL; 734 735 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 736 DMA_ATTR_WEAK_ORDERING); 737 if (dma_mapping_error(dev, *mapping)) { 738 page_pool_recycle_direct(rxr->page_pool, page); 739 return NULL; 740 } 741 *mapping += bp->rx_dma_offset; 742 return page; 743 } 744 745 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 746 gfp_t gfp) 747 { 748 u8 *data; 749 struct pci_dev *pdev = bp->pdev; 750 751 if (gfp == GFP_ATOMIC) 752 data = napi_alloc_frag(bp->rx_buf_size); 753 else 754 data = netdev_alloc_frag(bp->rx_buf_size); 755 if (!data) 756 return NULL; 757 758 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 759 bp->rx_buf_use_size, bp->rx_dir, 760 DMA_ATTR_WEAK_ORDERING); 761 762 if (dma_mapping_error(&pdev->dev, *mapping)) { 763 skb_free_frag(data); 764 data = NULL; 765 } 766 return data; 767 } 768 769 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 770 u16 prod, gfp_t gfp) 771 { 772 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 773 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 774 dma_addr_t mapping; 775 776 if (BNXT_RX_PAGE_MODE(bp)) { 777 struct page *page = 778 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 779 780 if (!page) 781 return -ENOMEM; 782 783 rx_buf->data = page; 784 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 785 } else { 786 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); 787 788 if (!data) 789 return -ENOMEM; 790 791 rx_buf->data = data; 792 rx_buf->data_ptr = data + bp->rx_offset; 793 } 794 rx_buf->mapping = mapping; 795 796 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 797 return 0; 798 } 799 800 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 801 { 802 u16 prod = rxr->rx_prod; 803 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 804 struct rx_bd *cons_bd, *prod_bd; 805 806 prod_rx_buf = &rxr->rx_buf_ring[prod]; 807 cons_rx_buf = &rxr->rx_buf_ring[cons]; 808 809 prod_rx_buf->data = data; 810 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 811 812 prod_rx_buf->mapping = cons_rx_buf->mapping; 813 814 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 815 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 816 817 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 818 } 819 820 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 821 { 822 u16 next, max = rxr->rx_agg_bmap_size; 823 824 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 825 if (next >= max) 826 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 827 return next; 828 } 829 830 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 831 struct bnxt_rx_ring_info *rxr, 832 u16 prod, gfp_t gfp) 833 { 834 struct rx_bd *rxbd = 835 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 836 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 837 struct pci_dev *pdev = bp->pdev; 838 struct page *page; 839 dma_addr_t mapping; 840 u16 sw_prod = rxr->rx_sw_agg_prod; 841 unsigned int offset = 0; 842 843 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 844 page = rxr->rx_page; 845 if (!page) { 846 page = alloc_page(gfp); 847 if (!page) 848 return -ENOMEM; 849 rxr->rx_page = page; 850 rxr->rx_page_offset = 0; 851 } 852 offset = rxr->rx_page_offset; 853 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 854 if (rxr->rx_page_offset == PAGE_SIZE) 855 rxr->rx_page = NULL; 856 else 857 get_page(page); 858 } else { 859 page = alloc_page(gfp); 860 if (!page) 861 return -ENOMEM; 862 } 863 864 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 865 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 866 DMA_ATTR_WEAK_ORDERING); 867 if (dma_mapping_error(&pdev->dev, mapping)) { 868 __free_page(page); 869 return -EIO; 870 } 871 872 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 873 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 874 875 __set_bit(sw_prod, rxr->rx_agg_bmap); 876 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 877 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 878 879 rx_agg_buf->page = page; 880 rx_agg_buf->offset = offset; 881 rx_agg_buf->mapping = mapping; 882 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 883 rxbd->rx_bd_opaque = sw_prod; 884 return 0; 885 } 886 887 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 888 struct bnxt_cp_ring_info *cpr, 889 u16 cp_cons, u16 curr) 890 { 891 struct rx_agg_cmp *agg; 892 893 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 894 agg = (struct rx_agg_cmp *) 895 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 896 return agg; 897 } 898 899 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 900 struct bnxt_rx_ring_info *rxr, 901 u16 agg_id, u16 curr) 902 { 903 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 904 905 return &tpa_info->agg_arr[curr]; 906 } 907 908 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 909 u16 start, u32 agg_bufs, bool tpa) 910 { 911 struct bnxt_napi *bnapi = cpr->bnapi; 912 struct bnxt *bp = bnapi->bp; 913 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 914 u16 prod = rxr->rx_agg_prod; 915 u16 sw_prod = rxr->rx_sw_agg_prod; 916 bool p5_tpa = false; 917 u32 i; 918 919 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 920 p5_tpa = true; 921 922 for (i = 0; i < agg_bufs; i++) { 923 u16 cons; 924 struct rx_agg_cmp *agg; 925 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 926 struct rx_bd *prod_bd; 927 struct page *page; 928 929 if (p5_tpa) 930 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 931 else 932 agg = bnxt_get_agg(bp, cpr, idx, start + i); 933 cons = agg->rx_agg_cmp_opaque; 934 __clear_bit(cons, rxr->rx_agg_bmap); 935 936 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 937 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 938 939 __set_bit(sw_prod, rxr->rx_agg_bmap); 940 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 941 cons_rx_buf = &rxr->rx_agg_ring[cons]; 942 943 /* It is possible for sw_prod to be equal to cons, so 944 * set cons_rx_buf->page to NULL first. 945 */ 946 page = cons_rx_buf->page; 947 cons_rx_buf->page = NULL; 948 prod_rx_buf->page = page; 949 prod_rx_buf->offset = cons_rx_buf->offset; 950 951 prod_rx_buf->mapping = cons_rx_buf->mapping; 952 953 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 954 955 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 956 prod_bd->rx_bd_opaque = sw_prod; 957 958 prod = NEXT_RX_AGG(prod); 959 sw_prod = NEXT_RX_AGG(sw_prod); 960 } 961 rxr->rx_agg_prod = prod; 962 rxr->rx_sw_agg_prod = sw_prod; 963 } 964 965 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 966 struct bnxt_rx_ring_info *rxr, 967 u16 cons, void *data, u8 *data_ptr, 968 dma_addr_t dma_addr, 969 unsigned int offset_and_len) 970 { 971 unsigned int payload = offset_and_len >> 16; 972 unsigned int len = offset_and_len & 0xffff; 973 skb_frag_t *frag; 974 struct page *page = data; 975 u16 prod = rxr->rx_prod; 976 struct sk_buff *skb; 977 int off, err; 978 979 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 980 if (unlikely(err)) { 981 bnxt_reuse_rx_data(rxr, cons, data); 982 return NULL; 983 } 984 dma_addr -= bp->rx_dma_offset; 985 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 986 DMA_ATTR_WEAK_ORDERING); 987 page_pool_release_page(rxr->page_pool, page); 988 989 if (unlikely(!payload)) 990 payload = eth_get_headlen(bp->dev, data_ptr, len); 991 992 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 993 if (!skb) { 994 __free_page(page); 995 return NULL; 996 } 997 998 off = (void *)data_ptr - page_address(page); 999 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 1000 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1001 payload + NET_IP_ALIGN); 1002 1003 frag = &skb_shinfo(skb)->frags[0]; 1004 skb_frag_size_sub(frag, payload); 1005 skb_frag_off_add(frag, payload); 1006 skb->data_len -= payload; 1007 skb->tail += payload; 1008 1009 return skb; 1010 } 1011 1012 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1013 struct bnxt_rx_ring_info *rxr, u16 cons, 1014 void *data, u8 *data_ptr, 1015 dma_addr_t dma_addr, 1016 unsigned int offset_and_len) 1017 { 1018 u16 prod = rxr->rx_prod; 1019 struct sk_buff *skb; 1020 int err; 1021 1022 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1023 if (unlikely(err)) { 1024 bnxt_reuse_rx_data(rxr, cons, data); 1025 return NULL; 1026 } 1027 1028 skb = build_skb(data, bp->rx_buf_size); 1029 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1030 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 1031 if (!skb) { 1032 skb_free_frag(data); 1033 return NULL; 1034 } 1035 1036 skb_reserve(skb, bp->rx_offset); 1037 skb_put(skb, offset_and_len & 0xffff); 1038 return skb; 1039 } 1040 1041 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 1042 struct bnxt_cp_ring_info *cpr, 1043 struct sk_buff *skb, u16 idx, 1044 u32 agg_bufs, bool tpa) 1045 { 1046 struct bnxt_napi *bnapi = cpr->bnapi; 1047 struct pci_dev *pdev = bp->pdev; 1048 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1049 u16 prod = rxr->rx_agg_prod; 1050 bool p5_tpa = false; 1051 u32 i; 1052 1053 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1054 p5_tpa = true; 1055 1056 for (i = 0; i < agg_bufs; i++) { 1057 u16 cons, frag_len; 1058 struct rx_agg_cmp *agg; 1059 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1060 struct page *page; 1061 dma_addr_t mapping; 1062 1063 if (p5_tpa) 1064 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1065 else 1066 agg = bnxt_get_agg(bp, cpr, idx, i); 1067 cons = agg->rx_agg_cmp_opaque; 1068 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1069 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1070 1071 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1072 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1073 cons_rx_buf->offset, frag_len); 1074 __clear_bit(cons, rxr->rx_agg_bmap); 1075 1076 /* It is possible for bnxt_alloc_rx_page() to allocate 1077 * a sw_prod index that equals the cons index, so we 1078 * need to clear the cons entry now. 1079 */ 1080 mapping = cons_rx_buf->mapping; 1081 page = cons_rx_buf->page; 1082 cons_rx_buf->page = NULL; 1083 1084 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1085 struct skb_shared_info *shinfo; 1086 unsigned int nr_frags; 1087 1088 shinfo = skb_shinfo(skb); 1089 nr_frags = --shinfo->nr_frags; 1090 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1091 1092 dev_kfree_skb(skb); 1093 1094 cons_rx_buf->page = page; 1095 1096 /* Update prod since possibly some pages have been 1097 * allocated already. 1098 */ 1099 rxr->rx_agg_prod = prod; 1100 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1101 return NULL; 1102 } 1103 1104 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1105 DMA_FROM_DEVICE, 1106 DMA_ATTR_WEAK_ORDERING); 1107 1108 skb->data_len += frag_len; 1109 skb->len += frag_len; 1110 skb->truesize += PAGE_SIZE; 1111 1112 prod = NEXT_RX_AGG(prod); 1113 } 1114 rxr->rx_agg_prod = prod; 1115 return skb; 1116 } 1117 1118 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1119 u8 agg_bufs, u32 *raw_cons) 1120 { 1121 u16 last; 1122 struct rx_agg_cmp *agg; 1123 1124 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1125 last = RING_CMP(*raw_cons); 1126 agg = (struct rx_agg_cmp *) 1127 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1128 return RX_AGG_CMP_VALID(agg, *raw_cons); 1129 } 1130 1131 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1132 unsigned int len, 1133 dma_addr_t mapping) 1134 { 1135 struct bnxt *bp = bnapi->bp; 1136 struct pci_dev *pdev = bp->pdev; 1137 struct sk_buff *skb; 1138 1139 skb = napi_alloc_skb(&bnapi->napi, len); 1140 if (!skb) 1141 return NULL; 1142 1143 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1144 bp->rx_dir); 1145 1146 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1147 len + NET_IP_ALIGN); 1148 1149 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1150 bp->rx_dir); 1151 1152 skb_put(skb, len); 1153 return skb; 1154 } 1155 1156 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1157 u32 *raw_cons, void *cmp) 1158 { 1159 struct rx_cmp *rxcmp = cmp; 1160 u32 tmp_raw_cons = *raw_cons; 1161 u8 cmp_type, agg_bufs = 0; 1162 1163 cmp_type = RX_CMP_TYPE(rxcmp); 1164 1165 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1166 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1167 RX_CMP_AGG_BUFS) >> 1168 RX_CMP_AGG_BUFS_SHIFT; 1169 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1170 struct rx_tpa_end_cmp *tpa_end = cmp; 1171 1172 if (bp->flags & BNXT_FLAG_CHIP_P5) 1173 return 0; 1174 1175 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1176 } 1177 1178 if (agg_bufs) { 1179 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1180 return -EBUSY; 1181 } 1182 *raw_cons = tmp_raw_cons; 1183 return 0; 1184 } 1185 1186 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1187 { 1188 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 1189 return; 1190 1191 if (BNXT_PF(bp)) 1192 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1193 else 1194 schedule_delayed_work(&bp->fw_reset_task, delay); 1195 } 1196 1197 static void bnxt_queue_sp_work(struct bnxt *bp) 1198 { 1199 if (BNXT_PF(bp)) 1200 queue_work(bnxt_pf_wq, &bp->sp_task); 1201 else 1202 schedule_work(&bp->sp_task); 1203 } 1204 1205 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1206 { 1207 if (!rxr->bnapi->in_reset) { 1208 rxr->bnapi->in_reset = true; 1209 if (bp->flags & BNXT_FLAG_CHIP_P5) 1210 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1211 else 1212 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 1213 bnxt_queue_sp_work(bp); 1214 } 1215 rxr->rx_next_cons = 0xffff; 1216 } 1217 1218 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1219 { 1220 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1221 u16 idx = agg_id & MAX_TPA_P5_MASK; 1222 1223 if (test_bit(idx, map->agg_idx_bmap)) 1224 idx = find_first_zero_bit(map->agg_idx_bmap, 1225 BNXT_AGG_IDX_BMAP_SIZE); 1226 __set_bit(idx, map->agg_idx_bmap); 1227 map->agg_id_tbl[agg_id] = idx; 1228 return idx; 1229 } 1230 1231 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1232 { 1233 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1234 1235 __clear_bit(idx, map->agg_idx_bmap); 1236 } 1237 1238 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1239 { 1240 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1241 1242 return map->agg_id_tbl[agg_id]; 1243 } 1244 1245 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1246 struct rx_tpa_start_cmp *tpa_start, 1247 struct rx_tpa_start_cmp_ext *tpa_start1) 1248 { 1249 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1250 struct bnxt_tpa_info *tpa_info; 1251 u16 cons, prod, agg_id; 1252 struct rx_bd *prod_bd; 1253 dma_addr_t mapping; 1254 1255 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1256 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1257 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1258 } else { 1259 agg_id = TPA_START_AGG_ID(tpa_start); 1260 } 1261 cons = tpa_start->rx_tpa_start_cmp_opaque; 1262 prod = rxr->rx_prod; 1263 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1264 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1265 tpa_info = &rxr->rx_tpa[agg_id]; 1266 1267 if (unlikely(cons != rxr->rx_next_cons || 1268 TPA_START_ERROR(tpa_start))) { 1269 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1270 cons, rxr->rx_next_cons, 1271 TPA_START_ERROR_CODE(tpa_start1)); 1272 bnxt_sched_reset(bp, rxr); 1273 return; 1274 } 1275 /* Store cfa_code in tpa_info to use in tpa_end 1276 * completion processing. 1277 */ 1278 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1279 prod_rx_buf->data = tpa_info->data; 1280 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1281 1282 mapping = tpa_info->mapping; 1283 prod_rx_buf->mapping = mapping; 1284 1285 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1286 1287 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1288 1289 tpa_info->data = cons_rx_buf->data; 1290 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1291 cons_rx_buf->data = NULL; 1292 tpa_info->mapping = cons_rx_buf->mapping; 1293 1294 tpa_info->len = 1295 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1296 RX_TPA_START_CMP_LEN_SHIFT; 1297 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1298 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1299 1300 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1301 tpa_info->gso_type = SKB_GSO_TCPV4; 1302 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1303 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1304 tpa_info->gso_type = SKB_GSO_TCPV6; 1305 tpa_info->rss_hash = 1306 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1307 } else { 1308 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1309 tpa_info->gso_type = 0; 1310 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1311 } 1312 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1313 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1314 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1315 tpa_info->agg_count = 0; 1316 1317 rxr->rx_prod = NEXT_RX(prod); 1318 cons = NEXT_RX(cons); 1319 rxr->rx_next_cons = NEXT_RX(cons); 1320 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1321 1322 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1323 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1324 cons_rx_buf->data = NULL; 1325 } 1326 1327 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1328 { 1329 if (agg_bufs) 1330 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1331 } 1332 1333 #ifdef CONFIG_INET 1334 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1335 { 1336 struct udphdr *uh = NULL; 1337 1338 if (ip_proto == htons(ETH_P_IP)) { 1339 struct iphdr *iph = (struct iphdr *)skb->data; 1340 1341 if (iph->protocol == IPPROTO_UDP) 1342 uh = (struct udphdr *)(iph + 1); 1343 } else { 1344 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1345 1346 if (iph->nexthdr == IPPROTO_UDP) 1347 uh = (struct udphdr *)(iph + 1); 1348 } 1349 if (uh) { 1350 if (uh->check) 1351 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1352 else 1353 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1354 } 1355 } 1356 #endif 1357 1358 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1359 int payload_off, int tcp_ts, 1360 struct sk_buff *skb) 1361 { 1362 #ifdef CONFIG_INET 1363 struct tcphdr *th; 1364 int len, nw_off; 1365 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1366 u32 hdr_info = tpa_info->hdr_info; 1367 bool loopback = false; 1368 1369 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1370 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1371 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1372 1373 /* If the packet is an internal loopback packet, the offsets will 1374 * have an extra 4 bytes. 1375 */ 1376 if (inner_mac_off == 4) { 1377 loopback = true; 1378 } else if (inner_mac_off > 4) { 1379 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1380 ETH_HLEN - 2)); 1381 1382 /* We only support inner iPv4/ipv6. If we don't see the 1383 * correct protocol ID, it must be a loopback packet where 1384 * the offsets are off by 4. 1385 */ 1386 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1387 loopback = true; 1388 } 1389 if (loopback) { 1390 /* internal loopback packet, subtract all offsets by 4 */ 1391 inner_ip_off -= 4; 1392 inner_mac_off -= 4; 1393 outer_ip_off -= 4; 1394 } 1395 1396 nw_off = inner_ip_off - ETH_HLEN; 1397 skb_set_network_header(skb, nw_off); 1398 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1399 struct ipv6hdr *iph = ipv6_hdr(skb); 1400 1401 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1402 len = skb->len - skb_transport_offset(skb); 1403 th = tcp_hdr(skb); 1404 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1405 } else { 1406 struct iphdr *iph = ip_hdr(skb); 1407 1408 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1409 len = skb->len - skb_transport_offset(skb); 1410 th = tcp_hdr(skb); 1411 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1412 } 1413 1414 if (inner_mac_off) { /* tunnel */ 1415 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1416 ETH_HLEN - 2)); 1417 1418 bnxt_gro_tunnel(skb, proto); 1419 } 1420 #endif 1421 return skb; 1422 } 1423 1424 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1425 int payload_off, int tcp_ts, 1426 struct sk_buff *skb) 1427 { 1428 #ifdef CONFIG_INET 1429 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1430 u32 hdr_info = tpa_info->hdr_info; 1431 int iphdr_len, nw_off; 1432 1433 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1434 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1435 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1436 1437 nw_off = inner_ip_off - ETH_HLEN; 1438 skb_set_network_header(skb, nw_off); 1439 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1440 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1441 skb_set_transport_header(skb, nw_off + iphdr_len); 1442 1443 if (inner_mac_off) { /* tunnel */ 1444 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1445 ETH_HLEN - 2)); 1446 1447 bnxt_gro_tunnel(skb, proto); 1448 } 1449 #endif 1450 return skb; 1451 } 1452 1453 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1454 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1455 1456 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1457 int payload_off, int tcp_ts, 1458 struct sk_buff *skb) 1459 { 1460 #ifdef CONFIG_INET 1461 struct tcphdr *th; 1462 int len, nw_off, tcp_opt_len = 0; 1463 1464 if (tcp_ts) 1465 tcp_opt_len = 12; 1466 1467 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1468 struct iphdr *iph; 1469 1470 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1471 ETH_HLEN; 1472 skb_set_network_header(skb, nw_off); 1473 iph = ip_hdr(skb); 1474 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1475 len = skb->len - skb_transport_offset(skb); 1476 th = tcp_hdr(skb); 1477 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1478 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1479 struct ipv6hdr *iph; 1480 1481 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1482 ETH_HLEN; 1483 skb_set_network_header(skb, nw_off); 1484 iph = ipv6_hdr(skb); 1485 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1486 len = skb->len - skb_transport_offset(skb); 1487 th = tcp_hdr(skb); 1488 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1489 } else { 1490 dev_kfree_skb_any(skb); 1491 return NULL; 1492 } 1493 1494 if (nw_off) /* tunnel */ 1495 bnxt_gro_tunnel(skb, skb->protocol); 1496 #endif 1497 return skb; 1498 } 1499 1500 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1501 struct bnxt_tpa_info *tpa_info, 1502 struct rx_tpa_end_cmp *tpa_end, 1503 struct rx_tpa_end_cmp_ext *tpa_end1, 1504 struct sk_buff *skb) 1505 { 1506 #ifdef CONFIG_INET 1507 int payload_off; 1508 u16 segs; 1509 1510 segs = TPA_END_TPA_SEGS(tpa_end); 1511 if (segs == 1) 1512 return skb; 1513 1514 NAPI_GRO_CB(skb)->count = segs; 1515 skb_shinfo(skb)->gso_size = 1516 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1517 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1518 if (bp->flags & BNXT_FLAG_CHIP_P5) 1519 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1520 else 1521 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1522 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1523 if (likely(skb)) 1524 tcp_gro_complete(skb); 1525 #endif 1526 return skb; 1527 } 1528 1529 /* Given the cfa_code of a received packet determine which 1530 * netdev (vf-rep or PF) the packet is destined to. 1531 */ 1532 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1533 { 1534 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1535 1536 /* if vf-rep dev is NULL, the must belongs to the PF */ 1537 return dev ? dev : bp->dev; 1538 } 1539 1540 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1541 struct bnxt_cp_ring_info *cpr, 1542 u32 *raw_cons, 1543 struct rx_tpa_end_cmp *tpa_end, 1544 struct rx_tpa_end_cmp_ext *tpa_end1, 1545 u8 *event) 1546 { 1547 struct bnxt_napi *bnapi = cpr->bnapi; 1548 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1549 u8 *data_ptr, agg_bufs; 1550 unsigned int len; 1551 struct bnxt_tpa_info *tpa_info; 1552 dma_addr_t mapping; 1553 struct sk_buff *skb; 1554 u16 idx = 0, agg_id; 1555 void *data; 1556 bool gro; 1557 1558 if (unlikely(bnapi->in_reset)) { 1559 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1560 1561 if (rc < 0) 1562 return ERR_PTR(-EBUSY); 1563 return NULL; 1564 } 1565 1566 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1567 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1568 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1569 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1570 tpa_info = &rxr->rx_tpa[agg_id]; 1571 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1572 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1573 agg_bufs, tpa_info->agg_count); 1574 agg_bufs = tpa_info->agg_count; 1575 } 1576 tpa_info->agg_count = 0; 1577 *event |= BNXT_AGG_EVENT; 1578 bnxt_free_agg_idx(rxr, agg_id); 1579 idx = agg_id; 1580 gro = !!(bp->flags & BNXT_FLAG_GRO); 1581 } else { 1582 agg_id = TPA_END_AGG_ID(tpa_end); 1583 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1584 tpa_info = &rxr->rx_tpa[agg_id]; 1585 idx = RING_CMP(*raw_cons); 1586 if (agg_bufs) { 1587 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1588 return ERR_PTR(-EBUSY); 1589 1590 *event |= BNXT_AGG_EVENT; 1591 idx = NEXT_CMP(idx); 1592 } 1593 gro = !!TPA_END_GRO(tpa_end); 1594 } 1595 data = tpa_info->data; 1596 data_ptr = tpa_info->data_ptr; 1597 prefetch(data_ptr); 1598 len = tpa_info->len; 1599 mapping = tpa_info->mapping; 1600 1601 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1602 bnxt_abort_tpa(cpr, idx, agg_bufs); 1603 if (agg_bufs > MAX_SKB_FRAGS) 1604 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1605 agg_bufs, (int)MAX_SKB_FRAGS); 1606 return NULL; 1607 } 1608 1609 if (len <= bp->rx_copy_thresh) { 1610 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1611 if (!skb) { 1612 bnxt_abort_tpa(cpr, idx, agg_bufs); 1613 cpr->sw_stats.rx.rx_oom_discards += 1; 1614 return NULL; 1615 } 1616 } else { 1617 u8 *new_data; 1618 dma_addr_t new_mapping; 1619 1620 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); 1621 if (!new_data) { 1622 bnxt_abort_tpa(cpr, idx, agg_bufs); 1623 cpr->sw_stats.rx.rx_oom_discards += 1; 1624 return NULL; 1625 } 1626 1627 tpa_info->data = new_data; 1628 tpa_info->data_ptr = new_data + bp->rx_offset; 1629 tpa_info->mapping = new_mapping; 1630 1631 skb = build_skb(data, bp->rx_buf_size); 1632 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1633 bp->rx_buf_use_size, bp->rx_dir, 1634 DMA_ATTR_WEAK_ORDERING); 1635 1636 if (!skb) { 1637 skb_free_frag(data); 1638 bnxt_abort_tpa(cpr, idx, agg_bufs); 1639 cpr->sw_stats.rx.rx_oom_discards += 1; 1640 return NULL; 1641 } 1642 skb_reserve(skb, bp->rx_offset); 1643 skb_put(skb, len); 1644 } 1645 1646 if (agg_bufs) { 1647 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1648 if (!skb) { 1649 /* Page reuse already handled by bnxt_rx_pages(). */ 1650 cpr->sw_stats.rx.rx_oom_discards += 1; 1651 return NULL; 1652 } 1653 } 1654 1655 skb->protocol = 1656 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1657 1658 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1659 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1660 1661 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1662 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1663 __be16 vlan_proto = htons(tpa_info->metadata >> 1664 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1665 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1666 1667 if (eth_type_vlan(vlan_proto)) { 1668 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1669 } else { 1670 dev_kfree_skb(skb); 1671 return NULL; 1672 } 1673 } 1674 1675 skb_checksum_none_assert(skb); 1676 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1677 skb->ip_summed = CHECKSUM_UNNECESSARY; 1678 skb->csum_level = 1679 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1680 } 1681 1682 if (gro) 1683 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1684 1685 return skb; 1686 } 1687 1688 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1689 struct rx_agg_cmp *rx_agg) 1690 { 1691 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1692 struct bnxt_tpa_info *tpa_info; 1693 1694 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1695 tpa_info = &rxr->rx_tpa[agg_id]; 1696 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1697 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1698 } 1699 1700 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1701 struct sk_buff *skb) 1702 { 1703 if (skb->dev != bp->dev) { 1704 /* this packet belongs to a vf-rep */ 1705 bnxt_vf_rep_rx(bp, skb); 1706 return; 1707 } 1708 skb_record_rx_queue(skb, bnapi->index); 1709 napi_gro_receive(&bnapi->napi, skb); 1710 } 1711 1712 /* returns the following: 1713 * 1 - 1 packet successfully received 1714 * 0 - successful TPA_START, packet not completed yet 1715 * -EBUSY - completion ring does not have all the agg buffers yet 1716 * -ENOMEM - packet aborted due to out of memory 1717 * -EIO - packet aborted due to hw error indicated in BD 1718 */ 1719 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1720 u32 *raw_cons, u8 *event) 1721 { 1722 struct bnxt_napi *bnapi = cpr->bnapi; 1723 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1724 struct net_device *dev = bp->dev; 1725 struct rx_cmp *rxcmp; 1726 struct rx_cmp_ext *rxcmp1; 1727 u32 tmp_raw_cons = *raw_cons; 1728 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1729 struct bnxt_sw_rx_bd *rx_buf; 1730 unsigned int len; 1731 u8 *data_ptr, agg_bufs, cmp_type; 1732 dma_addr_t dma_addr; 1733 struct sk_buff *skb; 1734 u32 flags, misc; 1735 void *data; 1736 int rc = 0; 1737 1738 rxcmp = (struct rx_cmp *) 1739 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1740 1741 cmp_type = RX_CMP_TYPE(rxcmp); 1742 1743 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1744 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1745 goto next_rx_no_prod_no_len; 1746 } 1747 1748 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1749 cp_cons = RING_CMP(tmp_raw_cons); 1750 rxcmp1 = (struct rx_cmp_ext *) 1751 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1752 1753 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1754 return -EBUSY; 1755 1756 /* The valid test of the entry must be done first before 1757 * reading any further. 1758 */ 1759 dma_rmb(); 1760 prod = rxr->rx_prod; 1761 1762 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1763 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1764 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1765 1766 *event |= BNXT_RX_EVENT; 1767 goto next_rx_no_prod_no_len; 1768 1769 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1770 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1771 (struct rx_tpa_end_cmp *)rxcmp, 1772 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1773 1774 if (IS_ERR(skb)) 1775 return -EBUSY; 1776 1777 rc = -ENOMEM; 1778 if (likely(skb)) { 1779 bnxt_deliver_skb(bp, bnapi, skb); 1780 rc = 1; 1781 } 1782 *event |= BNXT_RX_EVENT; 1783 goto next_rx_no_prod_no_len; 1784 } 1785 1786 cons = rxcmp->rx_cmp_opaque; 1787 if (unlikely(cons != rxr->rx_next_cons)) { 1788 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 1789 1790 /* 0xffff is forced error, don't print it */ 1791 if (rxr->rx_next_cons != 0xffff) 1792 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1793 cons, rxr->rx_next_cons); 1794 bnxt_sched_reset(bp, rxr); 1795 if (rc1) 1796 return rc1; 1797 goto next_rx_no_prod_no_len; 1798 } 1799 rx_buf = &rxr->rx_buf_ring[cons]; 1800 data = rx_buf->data; 1801 data_ptr = rx_buf->data_ptr; 1802 prefetch(data_ptr); 1803 1804 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1805 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1806 1807 if (agg_bufs) { 1808 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1809 return -EBUSY; 1810 1811 cp_cons = NEXT_CMP(cp_cons); 1812 *event |= BNXT_AGG_EVENT; 1813 } 1814 *event |= BNXT_RX_EVENT; 1815 1816 rx_buf->data = NULL; 1817 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1818 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1819 1820 bnxt_reuse_rx_data(rxr, cons, data); 1821 if (agg_bufs) 1822 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1823 false); 1824 1825 rc = -EIO; 1826 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1827 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1828 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 1829 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 1830 netdev_warn_once(bp->dev, "RX buffer error %x\n", 1831 rx_err); 1832 bnxt_sched_reset(bp, rxr); 1833 } 1834 } 1835 goto next_rx_no_len; 1836 } 1837 1838 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 1839 len = flags >> RX_CMP_LEN_SHIFT; 1840 dma_addr = rx_buf->mapping; 1841 1842 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1843 rc = 1; 1844 goto next_rx; 1845 } 1846 1847 if (len <= bp->rx_copy_thresh) { 1848 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1849 bnxt_reuse_rx_data(rxr, cons, data); 1850 if (!skb) { 1851 if (agg_bufs) 1852 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1853 agg_bufs, false); 1854 cpr->sw_stats.rx.rx_oom_discards += 1; 1855 rc = -ENOMEM; 1856 goto next_rx; 1857 } 1858 } else { 1859 u32 payload; 1860 1861 if (rx_buf->data_ptr == data_ptr) 1862 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1863 else 1864 payload = 0; 1865 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1866 payload | len); 1867 if (!skb) { 1868 cpr->sw_stats.rx.rx_oom_discards += 1; 1869 rc = -ENOMEM; 1870 goto next_rx; 1871 } 1872 } 1873 1874 if (agg_bufs) { 1875 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1876 if (!skb) { 1877 cpr->sw_stats.rx.rx_oom_discards += 1; 1878 rc = -ENOMEM; 1879 goto next_rx; 1880 } 1881 } 1882 1883 if (RX_CMP_HASH_VALID(rxcmp)) { 1884 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1885 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1886 1887 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1888 if (hash_type != 1 && hash_type != 3) 1889 type = PKT_HASH_TYPE_L3; 1890 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1891 } 1892 1893 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1894 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1895 1896 if ((rxcmp1->rx_cmp_flags2 & 1897 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1898 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1899 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1900 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1901 __be16 vlan_proto = htons(meta_data >> 1902 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1903 1904 if (eth_type_vlan(vlan_proto)) { 1905 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1906 } else { 1907 dev_kfree_skb(skb); 1908 goto next_rx; 1909 } 1910 } 1911 1912 skb_checksum_none_assert(skb); 1913 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1914 if (dev->features & NETIF_F_RXCSUM) { 1915 skb->ip_summed = CHECKSUM_UNNECESSARY; 1916 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1917 } 1918 } else { 1919 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1920 if (dev->features & NETIF_F_RXCSUM) 1921 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1922 } 1923 } 1924 1925 if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) == 1926 RX_CMP_FLAGS_ITYPE_PTP_W_TS)) { 1927 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1928 u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 1929 u64 ns, ts; 1930 1931 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 1932 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 1933 1934 spin_lock_bh(&ptp->ptp_lock); 1935 ns = timecounter_cyc2time(&ptp->tc, ts); 1936 spin_unlock_bh(&ptp->ptp_lock); 1937 memset(skb_hwtstamps(skb), 0, 1938 sizeof(*skb_hwtstamps(skb))); 1939 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 1940 } 1941 } 1942 } 1943 bnxt_deliver_skb(bp, bnapi, skb); 1944 rc = 1; 1945 1946 next_rx: 1947 cpr->rx_packets += 1; 1948 cpr->rx_bytes += len; 1949 1950 next_rx_no_len: 1951 rxr->rx_prod = NEXT_RX(prod); 1952 rxr->rx_next_cons = NEXT_RX(cons); 1953 1954 next_rx_no_prod_no_len: 1955 *raw_cons = tmp_raw_cons; 1956 1957 return rc; 1958 } 1959 1960 /* In netpoll mode, if we are using a combined completion ring, we need to 1961 * discard the rx packets and recycle the buffers. 1962 */ 1963 static int bnxt_force_rx_discard(struct bnxt *bp, 1964 struct bnxt_cp_ring_info *cpr, 1965 u32 *raw_cons, u8 *event) 1966 { 1967 u32 tmp_raw_cons = *raw_cons; 1968 struct rx_cmp_ext *rxcmp1; 1969 struct rx_cmp *rxcmp; 1970 u16 cp_cons; 1971 u8 cmp_type; 1972 int rc; 1973 1974 cp_cons = RING_CMP(tmp_raw_cons); 1975 rxcmp = (struct rx_cmp *) 1976 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1977 1978 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1979 cp_cons = RING_CMP(tmp_raw_cons); 1980 rxcmp1 = (struct rx_cmp_ext *) 1981 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1982 1983 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1984 return -EBUSY; 1985 1986 /* The valid test of the entry must be done first before 1987 * reading any further. 1988 */ 1989 dma_rmb(); 1990 cmp_type = RX_CMP_TYPE(rxcmp); 1991 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1992 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1993 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1994 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1995 struct rx_tpa_end_cmp_ext *tpa_end1; 1996 1997 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1998 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1999 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 2000 } 2001 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2002 if (rc && rc != -EBUSY) 2003 cpr->sw_stats.rx.rx_netpoll_discards += 1; 2004 return rc; 2005 } 2006 2007 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2008 { 2009 struct bnxt_fw_health *fw_health = bp->fw_health; 2010 u32 reg = fw_health->regs[reg_idx]; 2011 u32 reg_type, reg_off, val = 0; 2012 2013 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2014 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2015 switch (reg_type) { 2016 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2017 pci_read_config_dword(bp->pdev, reg_off, &val); 2018 break; 2019 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2020 reg_off = fw_health->mapped_regs[reg_idx]; 2021 fallthrough; 2022 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2023 val = readl(bp->bar0 + reg_off); 2024 break; 2025 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2026 val = readl(bp->bar1 + reg_off); 2027 break; 2028 } 2029 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2030 val &= fw_health->fw_reset_inprog_reg_mask; 2031 return val; 2032 } 2033 2034 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2035 { 2036 int i; 2037 2038 for (i = 0; i < bp->rx_nr_rings; i++) { 2039 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2040 struct bnxt_ring_grp_info *grp_info; 2041 2042 grp_info = &bp->grp_info[grp_idx]; 2043 if (grp_info->agg_fw_ring_id == ring_id) 2044 return grp_idx; 2045 } 2046 return INVALID_HW_RING_ID; 2047 } 2048 2049 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2050 { 2051 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2052 2053 switch (err_type) { 2054 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2055 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2056 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2057 break; 2058 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2059 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2060 break; 2061 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2062 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2063 break; 2064 default: 2065 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2066 err_type); 2067 break; 2068 } 2069 } 2070 2071 #define BNXT_GET_EVENT_PORT(data) \ 2072 ((data) & \ 2073 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2074 2075 #define BNXT_EVENT_RING_TYPE(data2) \ 2076 ((data2) & \ 2077 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2078 2079 #define BNXT_EVENT_RING_TYPE_RX(data2) \ 2080 (BNXT_EVENT_RING_TYPE(data2) == \ 2081 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2082 2083 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \ 2084 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\ 2085 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT) 2086 2087 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \ 2088 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\ 2089 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT) 2090 2091 #define BNXT_PHC_BITS 48 2092 2093 static int bnxt_async_event_process(struct bnxt *bp, 2094 struct hwrm_async_event_cmpl *cmpl) 2095 { 2096 u16 event_id = le16_to_cpu(cmpl->event_id); 2097 u32 data1 = le32_to_cpu(cmpl->event_data1); 2098 u32 data2 = le32_to_cpu(cmpl->event_data2); 2099 2100 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2101 event_id, data1, data2); 2102 2103 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2104 switch (event_id) { 2105 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2106 struct bnxt_link_info *link_info = &bp->link_info; 2107 2108 if (BNXT_VF(bp)) 2109 goto async_event_process_exit; 2110 2111 /* print unsupported speed warning in forced speed mode only */ 2112 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2113 (data1 & 0x20000)) { 2114 u16 fw_speed = link_info->force_link_speed; 2115 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2116 2117 if (speed != SPEED_UNKNOWN) 2118 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2119 speed); 2120 } 2121 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2122 } 2123 fallthrough; 2124 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2125 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2126 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2127 fallthrough; 2128 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2129 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2130 break; 2131 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2132 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2133 break; 2134 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2135 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2136 2137 if (BNXT_VF(bp)) 2138 break; 2139 2140 if (bp->pf.port_id != port_id) 2141 break; 2142 2143 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2144 break; 2145 } 2146 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2147 if (BNXT_PF(bp)) 2148 goto async_event_process_exit; 2149 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2150 break; 2151 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2152 char *type_str = "Solicited"; 2153 2154 if (!bp->fw_health) 2155 goto async_event_process_exit; 2156 2157 bp->fw_reset_timestamp = jiffies; 2158 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2159 if (!bp->fw_reset_min_dsecs) 2160 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2161 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2162 if (!bp->fw_reset_max_dsecs) 2163 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2164 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2165 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2166 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2167 type_str = "Fatal"; 2168 bp->fw_health->fatalities++; 2169 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2170 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2171 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2172 type_str = "Non-fatal"; 2173 bp->fw_health->survivals++; 2174 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2175 } 2176 netif_warn(bp, hw, bp->dev, 2177 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2178 type_str, data1, data2, 2179 bp->fw_reset_min_dsecs * 100, 2180 bp->fw_reset_max_dsecs * 100); 2181 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2182 break; 2183 } 2184 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2185 struct bnxt_fw_health *fw_health = bp->fw_health; 2186 char *status_desc = "healthy"; 2187 u32 status; 2188 2189 if (!fw_health) 2190 goto async_event_process_exit; 2191 2192 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2193 fw_health->enabled = false; 2194 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2195 break; 2196 } 2197 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2198 fw_health->tmr_multiplier = 2199 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2200 bp->current_interval * 10); 2201 fw_health->tmr_counter = fw_health->tmr_multiplier; 2202 if (!fw_health->enabled) 2203 fw_health->last_fw_heartbeat = 2204 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2205 fw_health->last_fw_reset_cnt = 2206 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2207 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2208 if (status != BNXT_FW_STATUS_HEALTHY) 2209 status_desc = "unhealthy"; 2210 netif_info(bp, drv, bp->dev, 2211 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2212 fw_health->primary ? "primary" : "backup", status, 2213 status_desc, fw_health->last_fw_reset_cnt); 2214 if (!fw_health->enabled) { 2215 /* Make sure tmr_counter is set and visible to 2216 * bnxt_health_check() before setting enabled to true. 2217 */ 2218 smp_wmb(); 2219 fw_health->enabled = true; 2220 } 2221 goto async_event_process_exit; 2222 } 2223 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2224 netif_notice(bp, hw, bp->dev, 2225 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2226 data1, data2); 2227 goto async_event_process_exit; 2228 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2229 struct bnxt_rx_ring_info *rxr; 2230 u16 grp_idx; 2231 2232 if (bp->flags & BNXT_FLAG_CHIP_P5) 2233 goto async_event_process_exit; 2234 2235 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2236 BNXT_EVENT_RING_TYPE(data2), data1); 2237 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2238 goto async_event_process_exit; 2239 2240 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2241 if (grp_idx == INVALID_HW_RING_ID) { 2242 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2243 data1); 2244 goto async_event_process_exit; 2245 } 2246 rxr = bp->bnapi[grp_idx]->rx_ring; 2247 bnxt_sched_reset(bp, rxr); 2248 goto async_event_process_exit; 2249 } 2250 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2251 struct bnxt_fw_health *fw_health = bp->fw_health; 2252 2253 netif_notice(bp, hw, bp->dev, 2254 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2255 data1, data2); 2256 if (fw_health) { 2257 fw_health->echo_req_data1 = data1; 2258 fw_health->echo_req_data2 = data2; 2259 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2260 break; 2261 } 2262 goto async_event_process_exit; 2263 } 2264 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2265 bnxt_ptp_pps_event(bp, data1, data2); 2266 goto async_event_process_exit; 2267 } 2268 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2269 bnxt_event_error_report(bp, data1, data2); 2270 goto async_event_process_exit; 2271 } 2272 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: { 2273 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) { 2274 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE: 2275 if (bp->fw_cap & BNXT_FW_CAP_PTP_RTC) { 2276 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2277 u64 ns; 2278 2279 spin_lock_bh(&ptp->ptp_lock); 2280 bnxt_ptp_update_current_time(bp); 2281 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << 2282 BNXT_PHC_BITS) | ptp->current_time); 2283 bnxt_ptp_rtc_timecounter_init(ptp, ns); 2284 spin_unlock_bh(&ptp->ptp_lock); 2285 } 2286 break; 2287 } 2288 goto async_event_process_exit; 2289 } 2290 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2291 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2292 2293 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2294 goto async_event_process_exit; 2295 } 2296 default: 2297 goto async_event_process_exit; 2298 } 2299 bnxt_queue_sp_work(bp); 2300 async_event_process_exit: 2301 bnxt_ulp_async_events(bp, cmpl); 2302 return 0; 2303 } 2304 2305 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2306 { 2307 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2308 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2309 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2310 (struct hwrm_fwd_req_cmpl *)txcmp; 2311 2312 switch (cmpl_type) { 2313 case CMPL_BASE_TYPE_HWRM_DONE: 2314 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2315 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2316 break; 2317 2318 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2319 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2320 2321 if ((vf_id < bp->pf.first_vf_id) || 2322 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2323 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2324 vf_id); 2325 return -EINVAL; 2326 } 2327 2328 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2329 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2330 bnxt_queue_sp_work(bp); 2331 break; 2332 2333 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2334 bnxt_async_event_process(bp, 2335 (struct hwrm_async_event_cmpl *)txcmp); 2336 break; 2337 2338 default: 2339 break; 2340 } 2341 2342 return 0; 2343 } 2344 2345 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2346 { 2347 struct bnxt_napi *bnapi = dev_instance; 2348 struct bnxt *bp = bnapi->bp; 2349 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2350 u32 cons = RING_CMP(cpr->cp_raw_cons); 2351 2352 cpr->event_ctr++; 2353 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2354 napi_schedule(&bnapi->napi); 2355 return IRQ_HANDLED; 2356 } 2357 2358 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2359 { 2360 u32 raw_cons = cpr->cp_raw_cons; 2361 u16 cons = RING_CMP(raw_cons); 2362 struct tx_cmp *txcmp; 2363 2364 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2365 2366 return TX_CMP_VALID(txcmp, raw_cons); 2367 } 2368 2369 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2370 { 2371 struct bnxt_napi *bnapi = dev_instance; 2372 struct bnxt *bp = bnapi->bp; 2373 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2374 u32 cons = RING_CMP(cpr->cp_raw_cons); 2375 u32 int_status; 2376 2377 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2378 2379 if (!bnxt_has_work(bp, cpr)) { 2380 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2381 /* return if erroneous interrupt */ 2382 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2383 return IRQ_NONE; 2384 } 2385 2386 /* disable ring IRQ */ 2387 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2388 2389 /* Return here if interrupt is shared and is disabled. */ 2390 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2391 return IRQ_HANDLED; 2392 2393 napi_schedule(&bnapi->napi); 2394 return IRQ_HANDLED; 2395 } 2396 2397 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2398 int budget) 2399 { 2400 struct bnxt_napi *bnapi = cpr->bnapi; 2401 u32 raw_cons = cpr->cp_raw_cons; 2402 u32 cons; 2403 int tx_pkts = 0; 2404 int rx_pkts = 0; 2405 u8 event = 0; 2406 struct tx_cmp *txcmp; 2407 2408 cpr->has_more_work = 0; 2409 cpr->had_work_done = 1; 2410 while (1) { 2411 int rc; 2412 2413 cons = RING_CMP(raw_cons); 2414 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2415 2416 if (!TX_CMP_VALID(txcmp, raw_cons)) 2417 break; 2418 2419 /* The valid test of the entry must be done first before 2420 * reading any further. 2421 */ 2422 dma_rmb(); 2423 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2424 tx_pkts++; 2425 /* return full budget so NAPI will complete. */ 2426 if (unlikely(tx_pkts >= bp->tx_wake_thresh)) { 2427 rx_pkts = budget; 2428 raw_cons = NEXT_RAW_CMP(raw_cons); 2429 if (budget) 2430 cpr->has_more_work = 1; 2431 break; 2432 } 2433 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2434 if (likely(budget)) 2435 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2436 else 2437 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2438 &event); 2439 if (likely(rc >= 0)) 2440 rx_pkts += rc; 2441 /* Increment rx_pkts when rc is -ENOMEM to count towards 2442 * the NAPI budget. Otherwise, we may potentially loop 2443 * here forever if we consistently cannot allocate 2444 * buffers. 2445 */ 2446 else if (rc == -ENOMEM && budget) 2447 rx_pkts++; 2448 else if (rc == -EBUSY) /* partial completion */ 2449 break; 2450 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2451 CMPL_BASE_TYPE_HWRM_DONE) || 2452 (TX_CMP_TYPE(txcmp) == 2453 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2454 (TX_CMP_TYPE(txcmp) == 2455 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2456 bnxt_hwrm_handler(bp, txcmp); 2457 } 2458 raw_cons = NEXT_RAW_CMP(raw_cons); 2459 2460 if (rx_pkts && rx_pkts == budget) { 2461 cpr->has_more_work = 1; 2462 break; 2463 } 2464 } 2465 2466 if (event & BNXT_REDIRECT_EVENT) 2467 xdp_do_flush(); 2468 2469 if (event & BNXT_TX_EVENT) { 2470 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2471 u16 prod = txr->tx_prod; 2472 2473 /* Sync BD data before updating doorbell */ 2474 wmb(); 2475 2476 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2477 } 2478 2479 cpr->cp_raw_cons = raw_cons; 2480 bnapi->tx_pkts += tx_pkts; 2481 bnapi->events |= event; 2482 return rx_pkts; 2483 } 2484 2485 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2486 { 2487 if (bnapi->tx_pkts) { 2488 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2489 bnapi->tx_pkts = 0; 2490 } 2491 2492 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 2493 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2494 2495 if (bnapi->events & BNXT_AGG_EVENT) 2496 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2497 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2498 } 2499 bnapi->events = 0; 2500 } 2501 2502 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2503 int budget) 2504 { 2505 struct bnxt_napi *bnapi = cpr->bnapi; 2506 int rx_pkts; 2507 2508 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2509 2510 /* ACK completion ring before freeing tx ring and producing new 2511 * buffers in rx/agg rings to prevent overflowing the completion 2512 * ring. 2513 */ 2514 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2515 2516 __bnxt_poll_work_done(bp, bnapi); 2517 return rx_pkts; 2518 } 2519 2520 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2521 { 2522 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2523 struct bnxt *bp = bnapi->bp; 2524 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2525 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2526 struct tx_cmp *txcmp; 2527 struct rx_cmp_ext *rxcmp1; 2528 u32 cp_cons, tmp_raw_cons; 2529 u32 raw_cons = cpr->cp_raw_cons; 2530 u32 rx_pkts = 0; 2531 u8 event = 0; 2532 2533 while (1) { 2534 int rc; 2535 2536 cp_cons = RING_CMP(raw_cons); 2537 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2538 2539 if (!TX_CMP_VALID(txcmp, raw_cons)) 2540 break; 2541 2542 /* The valid test of the entry must be done first before 2543 * reading any further. 2544 */ 2545 dma_rmb(); 2546 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2547 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2548 cp_cons = RING_CMP(tmp_raw_cons); 2549 rxcmp1 = (struct rx_cmp_ext *) 2550 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2551 2552 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2553 break; 2554 2555 /* force an error to recycle the buffer */ 2556 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2557 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2558 2559 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2560 if (likely(rc == -EIO) && budget) 2561 rx_pkts++; 2562 else if (rc == -EBUSY) /* partial completion */ 2563 break; 2564 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2565 CMPL_BASE_TYPE_HWRM_DONE)) { 2566 bnxt_hwrm_handler(bp, txcmp); 2567 } else { 2568 netdev_err(bp->dev, 2569 "Invalid completion received on special ring\n"); 2570 } 2571 raw_cons = NEXT_RAW_CMP(raw_cons); 2572 2573 if (rx_pkts == budget) 2574 break; 2575 } 2576 2577 cpr->cp_raw_cons = raw_cons; 2578 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2579 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2580 2581 if (event & BNXT_AGG_EVENT) 2582 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2583 2584 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2585 napi_complete_done(napi, rx_pkts); 2586 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2587 } 2588 return rx_pkts; 2589 } 2590 2591 static int bnxt_poll(struct napi_struct *napi, int budget) 2592 { 2593 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2594 struct bnxt *bp = bnapi->bp; 2595 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2596 int work_done = 0; 2597 2598 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2599 napi_complete(napi); 2600 return 0; 2601 } 2602 while (1) { 2603 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2604 2605 if (work_done >= budget) { 2606 if (!budget) 2607 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2608 break; 2609 } 2610 2611 if (!bnxt_has_work(bp, cpr)) { 2612 if (napi_complete_done(napi, work_done)) 2613 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2614 break; 2615 } 2616 } 2617 if (bp->flags & BNXT_FLAG_DIM) { 2618 struct dim_sample dim_sample = {}; 2619 2620 dim_update_sample(cpr->event_ctr, 2621 cpr->rx_packets, 2622 cpr->rx_bytes, 2623 &dim_sample); 2624 net_dim(&cpr->dim, dim_sample); 2625 } 2626 return work_done; 2627 } 2628 2629 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2630 { 2631 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2632 int i, work_done = 0; 2633 2634 for (i = 0; i < 2; i++) { 2635 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2636 2637 if (cpr2) { 2638 work_done += __bnxt_poll_work(bp, cpr2, 2639 budget - work_done); 2640 cpr->has_more_work |= cpr2->has_more_work; 2641 } 2642 } 2643 return work_done; 2644 } 2645 2646 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2647 u64 dbr_type) 2648 { 2649 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2650 int i; 2651 2652 for (i = 0; i < 2; i++) { 2653 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2654 struct bnxt_db_info *db; 2655 2656 if (cpr2 && cpr2->had_work_done) { 2657 db = &cpr2->cp_db; 2658 bnxt_writeq(bp, db->db_key64 | dbr_type | 2659 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2660 cpr2->had_work_done = 0; 2661 } 2662 } 2663 __bnxt_poll_work_done(bp, bnapi); 2664 } 2665 2666 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2667 { 2668 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2669 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2670 struct bnxt_cp_ring_info *cpr_rx; 2671 u32 raw_cons = cpr->cp_raw_cons; 2672 struct bnxt *bp = bnapi->bp; 2673 struct nqe_cn *nqcmp; 2674 int work_done = 0; 2675 u32 cons; 2676 2677 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 2678 napi_complete(napi); 2679 return 0; 2680 } 2681 if (cpr->has_more_work) { 2682 cpr->has_more_work = 0; 2683 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2684 } 2685 while (1) { 2686 cons = RING_CMP(raw_cons); 2687 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2688 2689 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2690 if (cpr->has_more_work) 2691 break; 2692 2693 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2694 cpr->cp_raw_cons = raw_cons; 2695 if (napi_complete_done(napi, work_done)) 2696 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2697 cpr->cp_raw_cons); 2698 goto poll_done; 2699 } 2700 2701 /* The valid test of the entry must be done first before 2702 * reading any further. 2703 */ 2704 dma_rmb(); 2705 2706 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2707 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2708 struct bnxt_cp_ring_info *cpr2; 2709 2710 cpr2 = cpr->cp_ring_arr[idx]; 2711 work_done += __bnxt_poll_work(bp, cpr2, 2712 budget - work_done); 2713 cpr->has_more_work |= cpr2->has_more_work; 2714 } else { 2715 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2716 } 2717 raw_cons = NEXT_RAW_CMP(raw_cons); 2718 } 2719 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2720 if (raw_cons != cpr->cp_raw_cons) { 2721 cpr->cp_raw_cons = raw_cons; 2722 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2723 } 2724 poll_done: 2725 cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL]; 2726 if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) { 2727 struct dim_sample dim_sample = {}; 2728 2729 dim_update_sample(cpr->event_ctr, 2730 cpr_rx->rx_packets, 2731 cpr_rx->rx_bytes, 2732 &dim_sample); 2733 net_dim(&cpr->dim, dim_sample); 2734 } 2735 return work_done; 2736 } 2737 2738 static void bnxt_free_tx_skbs(struct bnxt *bp) 2739 { 2740 int i, max_idx; 2741 struct pci_dev *pdev = bp->pdev; 2742 2743 if (!bp->tx_ring) 2744 return; 2745 2746 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2747 for (i = 0; i < bp->tx_nr_rings; i++) { 2748 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2749 int j; 2750 2751 if (!txr->tx_buf_ring) 2752 continue; 2753 2754 for (j = 0; j < max_idx;) { 2755 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2756 struct sk_buff *skb; 2757 int k, last; 2758 2759 if (i < bp->tx_nr_rings_xdp && 2760 tx_buf->action == XDP_REDIRECT) { 2761 dma_unmap_single(&pdev->dev, 2762 dma_unmap_addr(tx_buf, mapping), 2763 dma_unmap_len(tx_buf, len), 2764 DMA_TO_DEVICE); 2765 xdp_return_frame(tx_buf->xdpf); 2766 tx_buf->action = 0; 2767 tx_buf->xdpf = NULL; 2768 j++; 2769 continue; 2770 } 2771 2772 skb = tx_buf->skb; 2773 if (!skb) { 2774 j++; 2775 continue; 2776 } 2777 2778 tx_buf->skb = NULL; 2779 2780 if (tx_buf->is_push) { 2781 dev_kfree_skb(skb); 2782 j += 2; 2783 continue; 2784 } 2785 2786 dma_unmap_single(&pdev->dev, 2787 dma_unmap_addr(tx_buf, mapping), 2788 skb_headlen(skb), 2789 DMA_TO_DEVICE); 2790 2791 last = tx_buf->nr_frags; 2792 j += 2; 2793 for (k = 0; k < last; k++, j++) { 2794 int ring_idx = j & bp->tx_ring_mask; 2795 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2796 2797 tx_buf = &txr->tx_buf_ring[ring_idx]; 2798 dma_unmap_page( 2799 &pdev->dev, 2800 dma_unmap_addr(tx_buf, mapping), 2801 skb_frag_size(frag), DMA_TO_DEVICE); 2802 } 2803 dev_kfree_skb(skb); 2804 } 2805 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2806 } 2807 } 2808 2809 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) 2810 { 2811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 2812 struct pci_dev *pdev = bp->pdev; 2813 struct bnxt_tpa_idx_map *map; 2814 int i, max_idx, max_agg_idx; 2815 2816 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2817 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2818 if (!rxr->rx_tpa) 2819 goto skip_rx_tpa_free; 2820 2821 for (i = 0; i < bp->max_tpa; i++) { 2822 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 2823 u8 *data = tpa_info->data; 2824 2825 if (!data) 2826 continue; 2827 2828 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, 2829 bp->rx_buf_use_size, bp->rx_dir, 2830 DMA_ATTR_WEAK_ORDERING); 2831 2832 tpa_info->data = NULL; 2833 2834 skb_free_frag(data); 2835 } 2836 2837 skip_rx_tpa_free: 2838 if (!rxr->rx_buf_ring) 2839 goto skip_rx_buf_free; 2840 2841 for (i = 0; i < max_idx; i++) { 2842 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 2843 dma_addr_t mapping = rx_buf->mapping; 2844 void *data = rx_buf->data; 2845 2846 if (!data) 2847 continue; 2848 2849 rx_buf->data = NULL; 2850 if (BNXT_RX_PAGE_MODE(bp)) { 2851 mapping -= bp->rx_dma_offset; 2852 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, 2853 bp->rx_dir, 2854 DMA_ATTR_WEAK_ORDERING); 2855 page_pool_recycle_direct(rxr->page_pool, data); 2856 } else { 2857 dma_unmap_single_attrs(&pdev->dev, mapping, 2858 bp->rx_buf_use_size, bp->rx_dir, 2859 DMA_ATTR_WEAK_ORDERING); 2860 skb_free_frag(data); 2861 } 2862 } 2863 2864 skip_rx_buf_free: 2865 if (!rxr->rx_agg_ring) 2866 goto skip_rx_agg_free; 2867 2868 for (i = 0; i < max_agg_idx; i++) { 2869 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 2870 struct page *page = rx_agg_buf->page; 2871 2872 if (!page) 2873 continue; 2874 2875 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2876 BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, 2877 DMA_ATTR_WEAK_ORDERING); 2878 2879 rx_agg_buf->page = NULL; 2880 __clear_bit(i, rxr->rx_agg_bmap); 2881 2882 __free_page(page); 2883 } 2884 2885 skip_rx_agg_free: 2886 if (rxr->rx_page) { 2887 __free_page(rxr->rx_page); 2888 rxr->rx_page = NULL; 2889 } 2890 map = rxr->rx_tpa_idx_map; 2891 if (map) 2892 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2893 } 2894 2895 static void bnxt_free_rx_skbs(struct bnxt *bp) 2896 { 2897 int i; 2898 2899 if (!bp->rx_ring) 2900 return; 2901 2902 for (i = 0; i < bp->rx_nr_rings; i++) 2903 bnxt_free_one_rx_ring_skbs(bp, i); 2904 } 2905 2906 static void bnxt_free_skbs(struct bnxt *bp) 2907 { 2908 bnxt_free_tx_skbs(bp); 2909 bnxt_free_rx_skbs(bp); 2910 } 2911 2912 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len) 2913 { 2914 u8 init_val = mem_init->init_val; 2915 u16 offset = mem_init->offset; 2916 u8 *p2 = p; 2917 int i; 2918 2919 if (!init_val) 2920 return; 2921 if (offset == BNXT_MEM_INVALID_OFFSET) { 2922 memset(p, init_val, len); 2923 return; 2924 } 2925 for (i = 0; i < len; i += mem_init->size) 2926 *(p2 + i + offset) = init_val; 2927 } 2928 2929 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2930 { 2931 struct pci_dev *pdev = bp->pdev; 2932 int i; 2933 2934 if (!rmem->pg_arr) 2935 goto skip_pages; 2936 2937 for (i = 0; i < rmem->nr_pages; i++) { 2938 if (!rmem->pg_arr[i]) 2939 continue; 2940 2941 dma_free_coherent(&pdev->dev, rmem->page_size, 2942 rmem->pg_arr[i], rmem->dma_arr[i]); 2943 2944 rmem->pg_arr[i] = NULL; 2945 } 2946 skip_pages: 2947 if (rmem->pg_tbl) { 2948 size_t pg_tbl_size = rmem->nr_pages * 8; 2949 2950 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2951 pg_tbl_size = rmem->page_size; 2952 dma_free_coherent(&pdev->dev, pg_tbl_size, 2953 rmem->pg_tbl, rmem->pg_tbl_map); 2954 rmem->pg_tbl = NULL; 2955 } 2956 if (rmem->vmem_size && *rmem->vmem) { 2957 vfree(*rmem->vmem); 2958 *rmem->vmem = NULL; 2959 } 2960 } 2961 2962 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2963 { 2964 struct pci_dev *pdev = bp->pdev; 2965 u64 valid_bit = 0; 2966 int i; 2967 2968 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2969 valid_bit = PTU_PTE_VALID; 2970 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2971 size_t pg_tbl_size = rmem->nr_pages * 8; 2972 2973 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2974 pg_tbl_size = rmem->page_size; 2975 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2976 &rmem->pg_tbl_map, 2977 GFP_KERNEL); 2978 if (!rmem->pg_tbl) 2979 return -ENOMEM; 2980 } 2981 2982 for (i = 0; i < rmem->nr_pages; i++) { 2983 u64 extra_bits = valid_bit; 2984 2985 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2986 rmem->page_size, 2987 &rmem->dma_arr[i], 2988 GFP_KERNEL); 2989 if (!rmem->pg_arr[i]) 2990 return -ENOMEM; 2991 2992 if (rmem->mem_init) 2993 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i], 2994 rmem->page_size); 2995 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2996 if (i == rmem->nr_pages - 2 && 2997 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2998 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2999 else if (i == rmem->nr_pages - 1 && 3000 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3001 extra_bits |= PTU_PTE_LAST; 3002 rmem->pg_tbl[i] = 3003 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 3004 } 3005 } 3006 3007 if (rmem->vmem_size) { 3008 *rmem->vmem = vzalloc(rmem->vmem_size); 3009 if (!(*rmem->vmem)) 3010 return -ENOMEM; 3011 } 3012 return 0; 3013 } 3014 3015 static void bnxt_free_tpa_info(struct bnxt *bp) 3016 { 3017 int i; 3018 3019 for (i = 0; i < bp->rx_nr_rings; i++) { 3020 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3021 3022 kfree(rxr->rx_tpa_idx_map); 3023 rxr->rx_tpa_idx_map = NULL; 3024 if (rxr->rx_tpa) { 3025 kfree(rxr->rx_tpa[0].agg_arr); 3026 rxr->rx_tpa[0].agg_arr = NULL; 3027 } 3028 kfree(rxr->rx_tpa); 3029 rxr->rx_tpa = NULL; 3030 } 3031 } 3032 3033 static int bnxt_alloc_tpa_info(struct bnxt *bp) 3034 { 3035 int i, j, total_aggs = 0; 3036 3037 bp->max_tpa = MAX_TPA; 3038 if (bp->flags & BNXT_FLAG_CHIP_P5) { 3039 if (!bp->max_tpa_v2) 3040 return 0; 3041 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3042 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 3043 } 3044 3045 for (i = 0; i < bp->rx_nr_rings; i++) { 3046 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3047 struct rx_agg_cmp *agg; 3048 3049 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3050 GFP_KERNEL); 3051 if (!rxr->rx_tpa) 3052 return -ENOMEM; 3053 3054 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3055 continue; 3056 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 3057 rxr->rx_tpa[0].agg_arr = agg; 3058 if (!agg) 3059 return -ENOMEM; 3060 for (j = 1; j < bp->max_tpa; j++) 3061 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 3062 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3063 GFP_KERNEL); 3064 if (!rxr->rx_tpa_idx_map) 3065 return -ENOMEM; 3066 } 3067 return 0; 3068 } 3069 3070 static void bnxt_free_rx_rings(struct bnxt *bp) 3071 { 3072 int i; 3073 3074 if (!bp->rx_ring) 3075 return; 3076 3077 bnxt_free_tpa_info(bp); 3078 for (i = 0; i < bp->rx_nr_rings; i++) { 3079 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3080 struct bnxt_ring_struct *ring; 3081 3082 if (rxr->xdp_prog) 3083 bpf_prog_put(rxr->xdp_prog); 3084 3085 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3086 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3087 3088 page_pool_destroy(rxr->page_pool); 3089 rxr->page_pool = NULL; 3090 3091 kfree(rxr->rx_agg_bmap); 3092 rxr->rx_agg_bmap = NULL; 3093 3094 ring = &rxr->rx_ring_struct; 3095 bnxt_free_ring(bp, &ring->ring_mem); 3096 3097 ring = &rxr->rx_agg_ring_struct; 3098 bnxt_free_ring(bp, &ring->ring_mem); 3099 } 3100 } 3101 3102 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3103 struct bnxt_rx_ring_info *rxr) 3104 { 3105 struct page_pool_params pp = { 0 }; 3106 3107 pp.pool_size = bp->rx_ring_size; 3108 pp.nid = dev_to_node(&bp->pdev->dev); 3109 pp.dev = &bp->pdev->dev; 3110 pp.dma_dir = DMA_BIDIRECTIONAL; 3111 3112 rxr->page_pool = page_pool_create(&pp); 3113 if (IS_ERR(rxr->page_pool)) { 3114 int err = PTR_ERR(rxr->page_pool); 3115 3116 rxr->page_pool = NULL; 3117 return err; 3118 } 3119 return 0; 3120 } 3121 3122 static int bnxt_alloc_rx_rings(struct bnxt *bp) 3123 { 3124 int i, rc = 0, agg_rings = 0; 3125 3126 if (!bp->rx_ring) 3127 return -ENOMEM; 3128 3129 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3130 agg_rings = 1; 3131 3132 for (i = 0; i < bp->rx_nr_rings; i++) { 3133 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3134 struct bnxt_ring_struct *ring; 3135 3136 ring = &rxr->rx_ring_struct; 3137 3138 rc = bnxt_alloc_rx_page_pool(bp, rxr); 3139 if (rc) 3140 return rc; 3141 3142 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3143 if (rc < 0) 3144 return rc; 3145 3146 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3147 MEM_TYPE_PAGE_POOL, 3148 rxr->page_pool); 3149 if (rc) { 3150 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3151 return rc; 3152 } 3153 3154 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3155 if (rc) 3156 return rc; 3157 3158 ring->grp_idx = i; 3159 if (agg_rings) { 3160 u16 mem_size; 3161 3162 ring = &rxr->rx_agg_ring_struct; 3163 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3164 if (rc) 3165 return rc; 3166 3167 ring->grp_idx = i; 3168 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3169 mem_size = rxr->rx_agg_bmap_size / 8; 3170 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3171 if (!rxr->rx_agg_bmap) 3172 return -ENOMEM; 3173 } 3174 } 3175 if (bp->flags & BNXT_FLAG_TPA) 3176 rc = bnxt_alloc_tpa_info(bp); 3177 return rc; 3178 } 3179 3180 static void bnxt_free_tx_rings(struct bnxt *bp) 3181 { 3182 int i; 3183 struct pci_dev *pdev = bp->pdev; 3184 3185 if (!bp->tx_ring) 3186 return; 3187 3188 for (i = 0; i < bp->tx_nr_rings; i++) { 3189 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3190 struct bnxt_ring_struct *ring; 3191 3192 if (txr->tx_push) { 3193 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3194 txr->tx_push, txr->tx_push_mapping); 3195 txr->tx_push = NULL; 3196 } 3197 3198 ring = &txr->tx_ring_struct; 3199 3200 bnxt_free_ring(bp, &ring->ring_mem); 3201 } 3202 } 3203 3204 static int bnxt_alloc_tx_rings(struct bnxt *bp) 3205 { 3206 int i, j, rc; 3207 struct pci_dev *pdev = bp->pdev; 3208 3209 bp->tx_push_size = 0; 3210 if (bp->tx_push_thresh) { 3211 int push_size; 3212 3213 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3214 bp->tx_push_thresh); 3215 3216 if (push_size > 256) { 3217 push_size = 0; 3218 bp->tx_push_thresh = 0; 3219 } 3220 3221 bp->tx_push_size = push_size; 3222 } 3223 3224 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 3225 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3226 struct bnxt_ring_struct *ring; 3227 u8 qidx; 3228 3229 ring = &txr->tx_ring_struct; 3230 3231 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3232 if (rc) 3233 return rc; 3234 3235 ring->grp_idx = txr->bnapi->index; 3236 if (bp->tx_push_size) { 3237 dma_addr_t mapping; 3238 3239 /* One pre-allocated DMA buffer to backup 3240 * TX push operation 3241 */ 3242 txr->tx_push = dma_alloc_coherent(&pdev->dev, 3243 bp->tx_push_size, 3244 &txr->tx_push_mapping, 3245 GFP_KERNEL); 3246 3247 if (!txr->tx_push) 3248 return -ENOMEM; 3249 3250 mapping = txr->tx_push_mapping + 3251 sizeof(struct tx_push_bd); 3252 txr->data_mapping = cpu_to_le64(mapping); 3253 } 3254 qidx = bp->tc_to_qidx[j]; 3255 ring->queue_id = bp->q_info[qidx].queue_id; 3256 if (i < bp->tx_nr_rings_xdp) 3257 continue; 3258 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 3259 j++; 3260 } 3261 return 0; 3262 } 3263 3264 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 3265 { 3266 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3267 3268 kfree(cpr->cp_desc_ring); 3269 cpr->cp_desc_ring = NULL; 3270 ring->ring_mem.pg_arr = NULL; 3271 kfree(cpr->cp_desc_mapping); 3272 cpr->cp_desc_mapping = NULL; 3273 ring->ring_mem.dma_arr = NULL; 3274 } 3275 3276 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 3277 { 3278 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 3279 if (!cpr->cp_desc_ring) 3280 return -ENOMEM; 3281 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 3282 GFP_KERNEL); 3283 if (!cpr->cp_desc_mapping) 3284 return -ENOMEM; 3285 return 0; 3286 } 3287 3288 static void bnxt_free_all_cp_arrays(struct bnxt *bp) 3289 { 3290 int i; 3291 3292 if (!bp->bnapi) 3293 return; 3294 for (i = 0; i < bp->cp_nr_rings; i++) { 3295 struct bnxt_napi *bnapi = bp->bnapi[i]; 3296 3297 if (!bnapi) 3298 continue; 3299 bnxt_free_cp_arrays(&bnapi->cp_ring); 3300 } 3301 } 3302 3303 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 3304 { 3305 int i, n = bp->cp_nr_pages; 3306 3307 for (i = 0; i < bp->cp_nr_rings; i++) { 3308 struct bnxt_napi *bnapi = bp->bnapi[i]; 3309 int rc; 3310 3311 if (!bnapi) 3312 continue; 3313 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 3314 if (rc) 3315 return rc; 3316 } 3317 return 0; 3318 } 3319 3320 static void bnxt_free_cp_rings(struct bnxt *bp) 3321 { 3322 int i; 3323 3324 if (!bp->bnapi) 3325 return; 3326 3327 for (i = 0; i < bp->cp_nr_rings; i++) { 3328 struct bnxt_napi *bnapi = bp->bnapi[i]; 3329 struct bnxt_cp_ring_info *cpr; 3330 struct bnxt_ring_struct *ring; 3331 int j; 3332 3333 if (!bnapi) 3334 continue; 3335 3336 cpr = &bnapi->cp_ring; 3337 ring = &cpr->cp_ring_struct; 3338 3339 bnxt_free_ring(bp, &ring->ring_mem); 3340 3341 for (j = 0; j < 2; j++) { 3342 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3343 3344 if (cpr2) { 3345 ring = &cpr2->cp_ring_struct; 3346 bnxt_free_ring(bp, &ring->ring_mem); 3347 bnxt_free_cp_arrays(cpr2); 3348 kfree(cpr2); 3349 cpr->cp_ring_arr[j] = NULL; 3350 } 3351 } 3352 } 3353 } 3354 3355 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 3356 { 3357 struct bnxt_ring_mem_info *rmem; 3358 struct bnxt_ring_struct *ring; 3359 struct bnxt_cp_ring_info *cpr; 3360 int rc; 3361 3362 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3363 if (!cpr) 3364 return NULL; 3365 3366 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 3367 if (rc) { 3368 bnxt_free_cp_arrays(cpr); 3369 kfree(cpr); 3370 return NULL; 3371 } 3372 ring = &cpr->cp_ring_struct; 3373 rmem = &ring->ring_mem; 3374 rmem->nr_pages = bp->cp_nr_pages; 3375 rmem->page_size = HW_CMPD_RING_SIZE; 3376 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3377 rmem->dma_arr = cpr->cp_desc_mapping; 3378 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3379 rc = bnxt_alloc_ring(bp, rmem); 3380 if (rc) { 3381 bnxt_free_ring(bp, rmem); 3382 bnxt_free_cp_arrays(cpr); 3383 kfree(cpr); 3384 cpr = NULL; 3385 } 3386 return cpr; 3387 } 3388 3389 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3390 { 3391 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3392 int i, rc, ulp_base_vec, ulp_msix; 3393 3394 ulp_msix = bnxt_get_ulp_msix_num(bp); 3395 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3396 for (i = 0; i < bp->cp_nr_rings; i++) { 3397 struct bnxt_napi *bnapi = bp->bnapi[i]; 3398 struct bnxt_cp_ring_info *cpr; 3399 struct bnxt_ring_struct *ring; 3400 3401 if (!bnapi) 3402 continue; 3403 3404 cpr = &bnapi->cp_ring; 3405 cpr->bnapi = bnapi; 3406 ring = &cpr->cp_ring_struct; 3407 3408 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3409 if (rc) 3410 return rc; 3411 3412 if (ulp_msix && i >= ulp_base_vec) 3413 ring->map_idx = i + ulp_msix; 3414 else 3415 ring->map_idx = i; 3416 3417 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3418 continue; 3419 3420 if (i < bp->rx_nr_rings) { 3421 struct bnxt_cp_ring_info *cpr2 = 3422 bnxt_alloc_cp_sub_ring(bp); 3423 3424 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3425 if (!cpr2) 3426 return -ENOMEM; 3427 cpr2->bnapi = bnapi; 3428 } 3429 if ((sh && i < bp->tx_nr_rings) || 3430 (!sh && i >= bp->rx_nr_rings)) { 3431 struct bnxt_cp_ring_info *cpr2 = 3432 bnxt_alloc_cp_sub_ring(bp); 3433 3434 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3435 if (!cpr2) 3436 return -ENOMEM; 3437 cpr2->bnapi = bnapi; 3438 } 3439 } 3440 return 0; 3441 } 3442 3443 static void bnxt_init_ring_struct(struct bnxt *bp) 3444 { 3445 int i; 3446 3447 for (i = 0; i < bp->cp_nr_rings; i++) { 3448 struct bnxt_napi *bnapi = bp->bnapi[i]; 3449 struct bnxt_ring_mem_info *rmem; 3450 struct bnxt_cp_ring_info *cpr; 3451 struct bnxt_rx_ring_info *rxr; 3452 struct bnxt_tx_ring_info *txr; 3453 struct bnxt_ring_struct *ring; 3454 3455 if (!bnapi) 3456 continue; 3457 3458 cpr = &bnapi->cp_ring; 3459 ring = &cpr->cp_ring_struct; 3460 rmem = &ring->ring_mem; 3461 rmem->nr_pages = bp->cp_nr_pages; 3462 rmem->page_size = HW_CMPD_RING_SIZE; 3463 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3464 rmem->dma_arr = cpr->cp_desc_mapping; 3465 rmem->vmem_size = 0; 3466 3467 rxr = bnapi->rx_ring; 3468 if (!rxr) 3469 goto skip_rx; 3470 3471 ring = &rxr->rx_ring_struct; 3472 rmem = &ring->ring_mem; 3473 rmem->nr_pages = bp->rx_nr_pages; 3474 rmem->page_size = HW_RXBD_RING_SIZE; 3475 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3476 rmem->dma_arr = rxr->rx_desc_mapping; 3477 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3478 rmem->vmem = (void **)&rxr->rx_buf_ring; 3479 3480 ring = &rxr->rx_agg_ring_struct; 3481 rmem = &ring->ring_mem; 3482 rmem->nr_pages = bp->rx_agg_nr_pages; 3483 rmem->page_size = HW_RXBD_RING_SIZE; 3484 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3485 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3486 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3487 rmem->vmem = (void **)&rxr->rx_agg_ring; 3488 3489 skip_rx: 3490 txr = bnapi->tx_ring; 3491 if (!txr) 3492 continue; 3493 3494 ring = &txr->tx_ring_struct; 3495 rmem = &ring->ring_mem; 3496 rmem->nr_pages = bp->tx_nr_pages; 3497 rmem->page_size = HW_RXBD_RING_SIZE; 3498 rmem->pg_arr = (void **)txr->tx_desc_ring; 3499 rmem->dma_arr = txr->tx_desc_mapping; 3500 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3501 rmem->vmem = (void **)&txr->tx_buf_ring; 3502 } 3503 } 3504 3505 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3506 { 3507 int i; 3508 u32 prod; 3509 struct rx_bd **rx_buf_ring; 3510 3511 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3512 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3513 int j; 3514 struct rx_bd *rxbd; 3515 3516 rxbd = rx_buf_ring[i]; 3517 if (!rxbd) 3518 continue; 3519 3520 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3521 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3522 rxbd->rx_bd_opaque = prod; 3523 } 3524 } 3525 } 3526 3527 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 3528 { 3529 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 3530 struct net_device *dev = bp->dev; 3531 u32 prod; 3532 int i; 3533 3534 prod = rxr->rx_prod; 3535 for (i = 0; i < bp->rx_ring_size; i++) { 3536 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 3537 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3538 ring_nr, i, bp->rx_ring_size); 3539 break; 3540 } 3541 prod = NEXT_RX(prod); 3542 } 3543 rxr->rx_prod = prod; 3544 3545 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3546 return 0; 3547 3548 prod = rxr->rx_agg_prod; 3549 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3550 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { 3551 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3552 ring_nr, i, bp->rx_ring_size); 3553 break; 3554 } 3555 prod = NEXT_RX_AGG(prod); 3556 } 3557 rxr->rx_agg_prod = prod; 3558 3559 if (rxr->rx_tpa) { 3560 dma_addr_t mapping; 3561 u8 *data; 3562 3563 for (i = 0; i < bp->max_tpa; i++) { 3564 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); 3565 if (!data) 3566 return -ENOMEM; 3567 3568 rxr->rx_tpa[i].data = data; 3569 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3570 rxr->rx_tpa[i].mapping = mapping; 3571 } 3572 } 3573 return 0; 3574 } 3575 3576 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3577 { 3578 struct bnxt_rx_ring_info *rxr; 3579 struct bnxt_ring_struct *ring; 3580 u32 type; 3581 3582 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3583 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3584 3585 if (NET_IP_ALIGN == 2) 3586 type |= RX_BD_FLAGS_SOP; 3587 3588 rxr = &bp->rx_ring[ring_nr]; 3589 ring = &rxr->rx_ring_struct; 3590 bnxt_init_rxbd_pages(ring, type); 3591 3592 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3593 bpf_prog_add(bp->xdp_prog, 1); 3594 rxr->xdp_prog = bp->xdp_prog; 3595 } 3596 ring->fw_ring_id = INVALID_HW_RING_ID; 3597 3598 ring = &rxr->rx_agg_ring_struct; 3599 ring->fw_ring_id = INVALID_HW_RING_ID; 3600 3601 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 3602 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3603 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3604 3605 bnxt_init_rxbd_pages(ring, type); 3606 } 3607 3608 return bnxt_alloc_one_rx_ring(bp, ring_nr); 3609 } 3610 3611 static void bnxt_init_cp_rings(struct bnxt *bp) 3612 { 3613 int i, j; 3614 3615 for (i = 0; i < bp->cp_nr_rings; i++) { 3616 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3617 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3618 3619 ring->fw_ring_id = INVALID_HW_RING_ID; 3620 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3621 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3622 for (j = 0; j < 2; j++) { 3623 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3624 3625 if (!cpr2) 3626 continue; 3627 3628 ring = &cpr2->cp_ring_struct; 3629 ring->fw_ring_id = INVALID_HW_RING_ID; 3630 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3631 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3632 } 3633 } 3634 } 3635 3636 static int bnxt_init_rx_rings(struct bnxt *bp) 3637 { 3638 int i, rc = 0; 3639 3640 if (BNXT_RX_PAGE_MODE(bp)) { 3641 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3642 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3643 } else { 3644 bp->rx_offset = BNXT_RX_OFFSET; 3645 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3646 } 3647 3648 for (i = 0; i < bp->rx_nr_rings; i++) { 3649 rc = bnxt_init_one_rx_ring(bp, i); 3650 if (rc) 3651 break; 3652 } 3653 3654 return rc; 3655 } 3656 3657 static int bnxt_init_tx_rings(struct bnxt *bp) 3658 { 3659 u16 i; 3660 3661 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3662 BNXT_MIN_TX_DESC_CNT); 3663 3664 for (i = 0; i < bp->tx_nr_rings; i++) { 3665 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3666 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3667 3668 ring->fw_ring_id = INVALID_HW_RING_ID; 3669 } 3670 3671 return 0; 3672 } 3673 3674 static void bnxt_free_ring_grps(struct bnxt *bp) 3675 { 3676 kfree(bp->grp_info); 3677 bp->grp_info = NULL; 3678 } 3679 3680 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3681 { 3682 int i; 3683 3684 if (irq_re_init) { 3685 bp->grp_info = kcalloc(bp->cp_nr_rings, 3686 sizeof(struct bnxt_ring_grp_info), 3687 GFP_KERNEL); 3688 if (!bp->grp_info) 3689 return -ENOMEM; 3690 } 3691 for (i = 0; i < bp->cp_nr_rings; i++) { 3692 if (irq_re_init) 3693 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3694 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3695 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3696 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3697 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3698 } 3699 return 0; 3700 } 3701 3702 static void bnxt_free_vnics(struct bnxt *bp) 3703 { 3704 kfree(bp->vnic_info); 3705 bp->vnic_info = NULL; 3706 bp->nr_vnics = 0; 3707 } 3708 3709 static int bnxt_alloc_vnics(struct bnxt *bp) 3710 { 3711 int num_vnics = 1; 3712 3713 #ifdef CONFIG_RFS_ACCEL 3714 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3715 num_vnics += bp->rx_nr_rings; 3716 #endif 3717 3718 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3719 num_vnics++; 3720 3721 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3722 GFP_KERNEL); 3723 if (!bp->vnic_info) 3724 return -ENOMEM; 3725 3726 bp->nr_vnics = num_vnics; 3727 return 0; 3728 } 3729 3730 static void bnxt_init_vnics(struct bnxt *bp) 3731 { 3732 int i; 3733 3734 for (i = 0; i < bp->nr_vnics; i++) { 3735 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3736 int j; 3737 3738 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3739 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3740 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3741 3742 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3743 3744 if (bp->vnic_info[i].rss_hash_key) { 3745 if (i == 0) 3746 prandom_bytes(vnic->rss_hash_key, 3747 HW_HASH_KEY_SIZE); 3748 else 3749 memcpy(vnic->rss_hash_key, 3750 bp->vnic_info[0].rss_hash_key, 3751 HW_HASH_KEY_SIZE); 3752 } 3753 } 3754 } 3755 3756 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3757 { 3758 int pages; 3759 3760 pages = ring_size / desc_per_pg; 3761 3762 if (!pages) 3763 return 1; 3764 3765 pages++; 3766 3767 while (pages & (pages - 1)) 3768 pages++; 3769 3770 return pages; 3771 } 3772 3773 void bnxt_set_tpa_flags(struct bnxt *bp) 3774 { 3775 bp->flags &= ~BNXT_FLAG_TPA; 3776 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3777 return; 3778 if (bp->dev->features & NETIF_F_LRO) 3779 bp->flags |= BNXT_FLAG_LRO; 3780 else if (bp->dev->features & NETIF_F_GRO_HW) 3781 bp->flags |= BNXT_FLAG_GRO; 3782 } 3783 3784 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3785 * be set on entry. 3786 */ 3787 void bnxt_set_ring_params(struct bnxt *bp) 3788 { 3789 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3790 u32 agg_factor = 0, agg_ring_size = 0; 3791 3792 /* 8 for CRC and VLAN */ 3793 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3794 3795 rx_space = rx_size + NET_SKB_PAD + 3796 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3797 3798 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3799 ring_size = bp->rx_ring_size; 3800 bp->rx_agg_ring_size = 0; 3801 bp->rx_agg_nr_pages = 0; 3802 3803 if (bp->flags & BNXT_FLAG_TPA) 3804 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3805 3806 bp->flags &= ~BNXT_FLAG_JUMBO; 3807 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3808 u32 jumbo_factor; 3809 3810 bp->flags |= BNXT_FLAG_JUMBO; 3811 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3812 if (jumbo_factor > agg_factor) 3813 agg_factor = jumbo_factor; 3814 } 3815 if (agg_factor) { 3816 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 3817 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 3818 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 3819 bp->rx_ring_size, ring_size); 3820 bp->rx_ring_size = ring_size; 3821 } 3822 agg_ring_size = ring_size * agg_factor; 3823 3824 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3825 RX_DESC_CNT); 3826 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3827 u32 tmp = agg_ring_size; 3828 3829 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3830 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3831 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3832 tmp, agg_ring_size); 3833 } 3834 bp->rx_agg_ring_size = agg_ring_size; 3835 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3836 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3837 rx_space = rx_size + NET_SKB_PAD + 3838 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3839 } 3840 3841 bp->rx_buf_use_size = rx_size; 3842 bp->rx_buf_size = rx_space; 3843 3844 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3845 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3846 3847 ring_size = bp->tx_ring_size; 3848 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3849 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3850 3851 max_rx_cmpl = bp->rx_ring_size; 3852 /* MAX TPA needs to be added because TPA_START completions are 3853 * immediately recycled, so the TPA completions are not bound by 3854 * the RX ring size. 3855 */ 3856 if (bp->flags & BNXT_FLAG_TPA) 3857 max_rx_cmpl += bp->max_tpa; 3858 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3859 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3860 bp->cp_ring_size = ring_size; 3861 3862 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3863 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3864 bp->cp_nr_pages = MAX_CP_PAGES; 3865 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3866 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3867 ring_size, bp->cp_ring_size); 3868 } 3869 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3870 bp->cp_ring_mask = bp->cp_bit - 1; 3871 } 3872 3873 /* Changing allocation mode of RX rings. 3874 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3875 */ 3876 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3877 { 3878 if (page_mode) { 3879 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3880 return -EOPNOTSUPP; 3881 bp->dev->max_mtu = 3882 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3883 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3884 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3885 bp->rx_dir = DMA_BIDIRECTIONAL; 3886 bp->rx_skb_func = bnxt_rx_page_skb; 3887 /* Disable LRO or GRO_HW */ 3888 netdev_update_features(bp->dev); 3889 } else { 3890 bp->dev->max_mtu = bp->max_mtu; 3891 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3892 bp->rx_dir = DMA_FROM_DEVICE; 3893 bp->rx_skb_func = bnxt_rx_skb; 3894 } 3895 return 0; 3896 } 3897 3898 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3899 { 3900 int i; 3901 struct bnxt_vnic_info *vnic; 3902 struct pci_dev *pdev = bp->pdev; 3903 3904 if (!bp->vnic_info) 3905 return; 3906 3907 for (i = 0; i < bp->nr_vnics; i++) { 3908 vnic = &bp->vnic_info[i]; 3909 3910 kfree(vnic->fw_grp_ids); 3911 vnic->fw_grp_ids = NULL; 3912 3913 kfree(vnic->uc_list); 3914 vnic->uc_list = NULL; 3915 3916 if (vnic->mc_list) { 3917 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3918 vnic->mc_list, vnic->mc_list_mapping); 3919 vnic->mc_list = NULL; 3920 } 3921 3922 if (vnic->rss_table) { 3923 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3924 vnic->rss_table, 3925 vnic->rss_table_dma_addr); 3926 vnic->rss_table = NULL; 3927 } 3928 3929 vnic->rss_hash_key = NULL; 3930 vnic->flags = 0; 3931 } 3932 } 3933 3934 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3935 { 3936 int i, rc = 0, size; 3937 struct bnxt_vnic_info *vnic; 3938 struct pci_dev *pdev = bp->pdev; 3939 int max_rings; 3940 3941 for (i = 0; i < bp->nr_vnics; i++) { 3942 vnic = &bp->vnic_info[i]; 3943 3944 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3945 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3946 3947 if (mem_size > 0) { 3948 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3949 if (!vnic->uc_list) { 3950 rc = -ENOMEM; 3951 goto out; 3952 } 3953 } 3954 } 3955 3956 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3957 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3958 vnic->mc_list = 3959 dma_alloc_coherent(&pdev->dev, 3960 vnic->mc_list_size, 3961 &vnic->mc_list_mapping, 3962 GFP_KERNEL); 3963 if (!vnic->mc_list) { 3964 rc = -ENOMEM; 3965 goto out; 3966 } 3967 } 3968 3969 if (bp->flags & BNXT_FLAG_CHIP_P5) 3970 goto vnic_skip_grps; 3971 3972 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3973 max_rings = bp->rx_nr_rings; 3974 else 3975 max_rings = 1; 3976 3977 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3978 if (!vnic->fw_grp_ids) { 3979 rc = -ENOMEM; 3980 goto out; 3981 } 3982 vnic_skip_grps: 3983 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3984 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3985 continue; 3986 3987 /* Allocate rss table and hash key */ 3988 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3989 if (bp->flags & BNXT_FLAG_CHIP_P5) 3990 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3991 3992 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3993 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3994 vnic->rss_table_size, 3995 &vnic->rss_table_dma_addr, 3996 GFP_KERNEL); 3997 if (!vnic->rss_table) { 3998 rc = -ENOMEM; 3999 goto out; 4000 } 4001 4002 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 4003 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 4004 } 4005 return 0; 4006 4007 out: 4008 return rc; 4009 } 4010 4011 static void bnxt_free_hwrm_resources(struct bnxt *bp) 4012 { 4013 struct bnxt_hwrm_wait_token *token; 4014 4015 dma_pool_destroy(bp->hwrm_dma_pool); 4016 bp->hwrm_dma_pool = NULL; 4017 4018 rcu_read_lock(); 4019 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 4020 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 4021 rcu_read_unlock(); 4022 } 4023 4024 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 4025 { 4026 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 4027 BNXT_HWRM_DMA_SIZE, 4028 BNXT_HWRM_DMA_ALIGN, 0); 4029 if (!bp->hwrm_dma_pool) 4030 return -ENOMEM; 4031 4032 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 4033 4034 return 0; 4035 } 4036 4037 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 4038 { 4039 kfree(stats->hw_masks); 4040 stats->hw_masks = NULL; 4041 kfree(stats->sw_stats); 4042 stats->sw_stats = NULL; 4043 if (stats->hw_stats) { 4044 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 4045 stats->hw_stats_map); 4046 stats->hw_stats = NULL; 4047 } 4048 } 4049 4050 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 4051 bool alloc_masks) 4052 { 4053 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 4054 &stats->hw_stats_map, GFP_KERNEL); 4055 if (!stats->hw_stats) 4056 return -ENOMEM; 4057 4058 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 4059 if (!stats->sw_stats) 4060 goto stats_mem_err; 4061 4062 if (alloc_masks) { 4063 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 4064 if (!stats->hw_masks) 4065 goto stats_mem_err; 4066 } 4067 return 0; 4068 4069 stats_mem_err: 4070 bnxt_free_stats_mem(bp, stats); 4071 return -ENOMEM; 4072 } 4073 4074 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 4075 { 4076 int i; 4077 4078 for (i = 0; i < count; i++) 4079 mask_arr[i] = mask; 4080 } 4081 4082 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 4083 { 4084 int i; 4085 4086 for (i = 0; i < count; i++) 4087 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 4088 } 4089 4090 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 4091 struct bnxt_stats_mem *stats) 4092 { 4093 struct hwrm_func_qstats_ext_output *resp; 4094 struct hwrm_func_qstats_ext_input *req; 4095 __le64 *hw_masks; 4096 int rc; 4097 4098 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 4099 !(bp->flags & BNXT_FLAG_CHIP_P5)) 4100 return -EOPNOTSUPP; 4101 4102 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 4103 if (rc) 4104 return rc; 4105 4106 req->fid = cpu_to_le16(0xffff); 4107 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4108 4109 resp = hwrm_req_hold(bp, req); 4110 rc = hwrm_req_send(bp, req); 4111 if (!rc) { 4112 hw_masks = &resp->rx_ucast_pkts; 4113 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 4114 } 4115 hwrm_req_drop(bp, req); 4116 return rc; 4117 } 4118 4119 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 4120 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 4121 4122 static void bnxt_init_stats(struct bnxt *bp) 4123 { 4124 struct bnxt_napi *bnapi = bp->bnapi[0]; 4125 struct bnxt_cp_ring_info *cpr; 4126 struct bnxt_stats_mem *stats; 4127 __le64 *rx_stats, *tx_stats; 4128 int rc, rx_count, tx_count; 4129 u64 *rx_masks, *tx_masks; 4130 u64 mask; 4131 u8 flags; 4132 4133 cpr = &bnapi->cp_ring; 4134 stats = &cpr->stats; 4135 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 4136 if (rc) { 4137 if (bp->flags & BNXT_FLAG_CHIP_P5) 4138 mask = (1ULL << 48) - 1; 4139 else 4140 mask = -1ULL; 4141 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 4142 } 4143 if (bp->flags & BNXT_FLAG_PORT_STATS) { 4144 stats = &bp->port_stats; 4145 rx_stats = stats->hw_stats; 4146 rx_masks = stats->hw_masks; 4147 rx_count = sizeof(struct rx_port_stats) / 8; 4148 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4149 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 4150 tx_count = sizeof(struct tx_port_stats) / 8; 4151 4152 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 4153 rc = bnxt_hwrm_port_qstats(bp, flags); 4154 if (rc) { 4155 mask = (1ULL << 40) - 1; 4156 4157 bnxt_fill_masks(rx_masks, mask, rx_count); 4158 bnxt_fill_masks(tx_masks, mask, tx_count); 4159 } else { 4160 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4161 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 4162 bnxt_hwrm_port_qstats(bp, 0); 4163 } 4164 } 4165 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 4166 stats = &bp->rx_port_stats_ext; 4167 rx_stats = stats->hw_stats; 4168 rx_masks = stats->hw_masks; 4169 rx_count = sizeof(struct rx_port_stats_ext) / 8; 4170 stats = &bp->tx_port_stats_ext; 4171 tx_stats = stats->hw_stats; 4172 tx_masks = stats->hw_masks; 4173 tx_count = sizeof(struct tx_port_stats_ext) / 8; 4174 4175 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 4176 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 4177 if (rc) { 4178 mask = (1ULL << 40) - 1; 4179 4180 bnxt_fill_masks(rx_masks, mask, rx_count); 4181 if (tx_stats) 4182 bnxt_fill_masks(tx_masks, mask, tx_count); 4183 } else { 4184 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 4185 if (tx_stats) 4186 bnxt_copy_hw_masks(tx_masks, tx_stats, 4187 tx_count); 4188 bnxt_hwrm_port_qstats_ext(bp, 0); 4189 } 4190 } 4191 } 4192 4193 static void bnxt_free_port_stats(struct bnxt *bp) 4194 { 4195 bp->flags &= ~BNXT_FLAG_PORT_STATS; 4196 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 4197 4198 bnxt_free_stats_mem(bp, &bp->port_stats); 4199 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 4200 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 4201 } 4202 4203 static void bnxt_free_ring_stats(struct bnxt *bp) 4204 { 4205 int i; 4206 4207 if (!bp->bnapi) 4208 return; 4209 4210 for (i = 0; i < bp->cp_nr_rings; i++) { 4211 struct bnxt_napi *bnapi = bp->bnapi[i]; 4212 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4213 4214 bnxt_free_stats_mem(bp, &cpr->stats); 4215 } 4216 } 4217 4218 static int bnxt_alloc_stats(struct bnxt *bp) 4219 { 4220 u32 size, i; 4221 int rc; 4222 4223 size = bp->hw_ring_stats_size; 4224 4225 for (i = 0; i < bp->cp_nr_rings; i++) { 4226 struct bnxt_napi *bnapi = bp->bnapi[i]; 4227 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4228 4229 cpr->stats.len = size; 4230 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 4231 if (rc) 4232 return rc; 4233 4234 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4235 } 4236 4237 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 4238 return 0; 4239 4240 if (bp->port_stats.hw_stats) 4241 goto alloc_ext_stats; 4242 4243 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 4244 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 4245 if (rc) 4246 return rc; 4247 4248 bp->flags |= BNXT_FLAG_PORT_STATS; 4249 4250 alloc_ext_stats: 4251 /* Display extended statistics only if FW supports it */ 4252 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 4253 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 4254 return 0; 4255 4256 if (bp->rx_port_stats_ext.hw_stats) 4257 goto alloc_tx_ext_stats; 4258 4259 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 4260 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 4261 /* Extended stats are optional */ 4262 if (rc) 4263 return 0; 4264 4265 alloc_tx_ext_stats: 4266 if (bp->tx_port_stats_ext.hw_stats) 4267 return 0; 4268 4269 if (bp->hwrm_spec_code >= 0x10902 || 4270 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 4271 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 4272 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 4273 /* Extended stats are optional */ 4274 if (rc) 4275 return 0; 4276 } 4277 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 4278 return 0; 4279 } 4280 4281 static void bnxt_clear_ring_indices(struct bnxt *bp) 4282 { 4283 int i; 4284 4285 if (!bp->bnapi) 4286 return; 4287 4288 for (i = 0; i < bp->cp_nr_rings; i++) { 4289 struct bnxt_napi *bnapi = bp->bnapi[i]; 4290 struct bnxt_cp_ring_info *cpr; 4291 struct bnxt_rx_ring_info *rxr; 4292 struct bnxt_tx_ring_info *txr; 4293 4294 if (!bnapi) 4295 continue; 4296 4297 cpr = &bnapi->cp_ring; 4298 cpr->cp_raw_cons = 0; 4299 4300 txr = bnapi->tx_ring; 4301 if (txr) { 4302 txr->tx_prod = 0; 4303 txr->tx_cons = 0; 4304 } 4305 4306 rxr = bnapi->rx_ring; 4307 if (rxr) { 4308 rxr->rx_prod = 0; 4309 rxr->rx_agg_prod = 0; 4310 rxr->rx_sw_agg_prod = 0; 4311 rxr->rx_next_cons = 0; 4312 } 4313 } 4314 } 4315 4316 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 4317 { 4318 #ifdef CONFIG_RFS_ACCEL 4319 int i; 4320 4321 /* Under rtnl_lock and all our NAPIs have been disabled. It's 4322 * safe to delete the hash table. 4323 */ 4324 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 4325 struct hlist_head *head; 4326 struct hlist_node *tmp; 4327 struct bnxt_ntuple_filter *fltr; 4328 4329 head = &bp->ntp_fltr_hash_tbl[i]; 4330 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 4331 hlist_del(&fltr->hash); 4332 kfree(fltr); 4333 } 4334 } 4335 if (irq_reinit) { 4336 kfree(bp->ntp_fltr_bmap); 4337 bp->ntp_fltr_bmap = NULL; 4338 } 4339 bp->ntp_fltr_count = 0; 4340 #endif 4341 } 4342 4343 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 4344 { 4345 #ifdef CONFIG_RFS_ACCEL 4346 int i, rc = 0; 4347 4348 if (!(bp->flags & BNXT_FLAG_RFS)) 4349 return 0; 4350 4351 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 4352 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 4353 4354 bp->ntp_fltr_count = 0; 4355 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 4356 sizeof(long), 4357 GFP_KERNEL); 4358 4359 if (!bp->ntp_fltr_bmap) 4360 rc = -ENOMEM; 4361 4362 return rc; 4363 #else 4364 return 0; 4365 #endif 4366 } 4367 4368 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 4369 { 4370 bnxt_free_vnic_attributes(bp); 4371 bnxt_free_tx_rings(bp); 4372 bnxt_free_rx_rings(bp); 4373 bnxt_free_cp_rings(bp); 4374 bnxt_free_all_cp_arrays(bp); 4375 bnxt_free_ntp_fltrs(bp, irq_re_init); 4376 if (irq_re_init) { 4377 bnxt_free_ring_stats(bp); 4378 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 4379 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 4380 bnxt_free_port_stats(bp); 4381 bnxt_free_ring_grps(bp); 4382 bnxt_free_vnics(bp); 4383 kfree(bp->tx_ring_map); 4384 bp->tx_ring_map = NULL; 4385 kfree(bp->tx_ring); 4386 bp->tx_ring = NULL; 4387 kfree(bp->rx_ring); 4388 bp->rx_ring = NULL; 4389 kfree(bp->bnapi); 4390 bp->bnapi = NULL; 4391 } else { 4392 bnxt_clear_ring_indices(bp); 4393 } 4394 } 4395 4396 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 4397 { 4398 int i, j, rc, size, arr_size; 4399 void *bnapi; 4400 4401 if (irq_re_init) { 4402 /* Allocate bnapi mem pointer array and mem block for 4403 * all queues 4404 */ 4405 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 4406 bp->cp_nr_rings); 4407 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 4408 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 4409 if (!bnapi) 4410 return -ENOMEM; 4411 4412 bp->bnapi = bnapi; 4413 bnapi += arr_size; 4414 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 4415 bp->bnapi[i] = bnapi; 4416 bp->bnapi[i]->index = i; 4417 bp->bnapi[i]->bp = bp; 4418 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4419 struct bnxt_cp_ring_info *cpr = 4420 &bp->bnapi[i]->cp_ring; 4421 4422 cpr->cp_ring_struct.ring_mem.flags = 4423 BNXT_RMEM_RING_PTE_FLAG; 4424 } 4425 } 4426 4427 bp->rx_ring = kcalloc(bp->rx_nr_rings, 4428 sizeof(struct bnxt_rx_ring_info), 4429 GFP_KERNEL); 4430 if (!bp->rx_ring) 4431 return -ENOMEM; 4432 4433 for (i = 0; i < bp->rx_nr_rings; i++) { 4434 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4435 4436 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4437 rxr->rx_ring_struct.ring_mem.flags = 4438 BNXT_RMEM_RING_PTE_FLAG; 4439 rxr->rx_agg_ring_struct.ring_mem.flags = 4440 BNXT_RMEM_RING_PTE_FLAG; 4441 } 4442 rxr->bnapi = bp->bnapi[i]; 4443 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4444 } 4445 4446 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4447 sizeof(struct bnxt_tx_ring_info), 4448 GFP_KERNEL); 4449 if (!bp->tx_ring) 4450 return -ENOMEM; 4451 4452 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4453 GFP_KERNEL); 4454 4455 if (!bp->tx_ring_map) 4456 return -ENOMEM; 4457 4458 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4459 j = 0; 4460 else 4461 j = bp->rx_nr_rings; 4462 4463 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4464 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4465 4466 if (bp->flags & BNXT_FLAG_CHIP_P5) 4467 txr->tx_ring_struct.ring_mem.flags = 4468 BNXT_RMEM_RING_PTE_FLAG; 4469 txr->bnapi = bp->bnapi[j]; 4470 bp->bnapi[j]->tx_ring = txr; 4471 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4472 if (i >= bp->tx_nr_rings_xdp) { 4473 txr->txq_index = i - bp->tx_nr_rings_xdp; 4474 bp->bnapi[j]->tx_int = bnxt_tx_int; 4475 } else { 4476 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4477 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4478 } 4479 } 4480 4481 rc = bnxt_alloc_stats(bp); 4482 if (rc) 4483 goto alloc_mem_err; 4484 bnxt_init_stats(bp); 4485 4486 rc = bnxt_alloc_ntp_fltrs(bp); 4487 if (rc) 4488 goto alloc_mem_err; 4489 4490 rc = bnxt_alloc_vnics(bp); 4491 if (rc) 4492 goto alloc_mem_err; 4493 } 4494 4495 rc = bnxt_alloc_all_cp_arrays(bp); 4496 if (rc) 4497 goto alloc_mem_err; 4498 4499 bnxt_init_ring_struct(bp); 4500 4501 rc = bnxt_alloc_rx_rings(bp); 4502 if (rc) 4503 goto alloc_mem_err; 4504 4505 rc = bnxt_alloc_tx_rings(bp); 4506 if (rc) 4507 goto alloc_mem_err; 4508 4509 rc = bnxt_alloc_cp_rings(bp); 4510 if (rc) 4511 goto alloc_mem_err; 4512 4513 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4514 BNXT_VNIC_UCAST_FLAG; 4515 rc = bnxt_alloc_vnic_attributes(bp); 4516 if (rc) 4517 goto alloc_mem_err; 4518 return 0; 4519 4520 alloc_mem_err: 4521 bnxt_free_mem(bp, true); 4522 return rc; 4523 } 4524 4525 static void bnxt_disable_int(struct bnxt *bp) 4526 { 4527 int i; 4528 4529 if (!bp->bnapi) 4530 return; 4531 4532 for (i = 0; i < bp->cp_nr_rings; i++) { 4533 struct bnxt_napi *bnapi = bp->bnapi[i]; 4534 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4535 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4536 4537 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4538 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4539 } 4540 } 4541 4542 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4543 { 4544 struct bnxt_napi *bnapi = bp->bnapi[n]; 4545 struct bnxt_cp_ring_info *cpr; 4546 4547 cpr = &bnapi->cp_ring; 4548 return cpr->cp_ring_struct.map_idx; 4549 } 4550 4551 static void bnxt_disable_int_sync(struct bnxt *bp) 4552 { 4553 int i; 4554 4555 if (!bp->irq_tbl) 4556 return; 4557 4558 atomic_inc(&bp->intr_sem); 4559 4560 bnxt_disable_int(bp); 4561 for (i = 0; i < bp->cp_nr_rings; i++) { 4562 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4563 4564 synchronize_irq(bp->irq_tbl[map_idx].vector); 4565 } 4566 } 4567 4568 static void bnxt_enable_int(struct bnxt *bp) 4569 { 4570 int i; 4571 4572 atomic_set(&bp->intr_sem, 0); 4573 for (i = 0; i < bp->cp_nr_rings; i++) { 4574 struct bnxt_napi *bnapi = bp->bnapi[i]; 4575 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4576 4577 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4578 } 4579 } 4580 4581 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4582 bool async_only) 4583 { 4584 DECLARE_BITMAP(async_events_bmap, 256); 4585 u32 *events = (u32 *)async_events_bmap; 4586 struct hwrm_func_drv_rgtr_output *resp; 4587 struct hwrm_func_drv_rgtr_input *req; 4588 u32 flags; 4589 int rc, i; 4590 4591 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 4592 if (rc) 4593 return rc; 4594 4595 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4596 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4597 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4598 4599 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4600 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4601 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4602 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4603 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4604 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4605 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4606 req->flags = cpu_to_le32(flags); 4607 req->ver_maj_8b = DRV_VER_MAJ; 4608 req->ver_min_8b = DRV_VER_MIN; 4609 req->ver_upd_8b = DRV_VER_UPD; 4610 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 4611 req->ver_min = cpu_to_le16(DRV_VER_MIN); 4612 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 4613 4614 if (BNXT_PF(bp)) { 4615 u32 data[8]; 4616 int i; 4617 4618 memset(data, 0, sizeof(data)); 4619 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4620 u16 cmd = bnxt_vf_req_snif[i]; 4621 unsigned int bit, idx; 4622 4623 idx = cmd / 32; 4624 bit = cmd % 32; 4625 data[idx] |= 1 << bit; 4626 } 4627 4628 for (i = 0; i < 8; i++) 4629 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 4630 4631 req->enables |= 4632 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4633 } 4634 4635 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4636 req->flags |= cpu_to_le32( 4637 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4638 4639 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4640 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4641 u16 event_id = bnxt_async_events_arr[i]; 4642 4643 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4644 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4645 continue; 4646 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4647 } 4648 if (bmap && bmap_size) { 4649 for (i = 0; i < bmap_size; i++) { 4650 if (test_bit(i, bmap)) 4651 __set_bit(i, async_events_bmap); 4652 } 4653 } 4654 for (i = 0; i < 8; i++) 4655 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 4656 4657 if (async_only) 4658 req->enables = 4659 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4660 4661 resp = hwrm_req_hold(bp, req); 4662 rc = hwrm_req_send(bp, req); 4663 if (!rc) { 4664 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4665 if (resp->flags & 4666 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4667 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4668 } 4669 hwrm_req_drop(bp, req); 4670 return rc; 4671 } 4672 4673 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4674 { 4675 struct hwrm_func_drv_unrgtr_input *req; 4676 int rc; 4677 4678 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4679 return 0; 4680 4681 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 4682 if (rc) 4683 return rc; 4684 return hwrm_req_send(bp, req); 4685 } 4686 4687 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4688 { 4689 struct hwrm_tunnel_dst_port_free_input *req; 4690 int rc; 4691 4692 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 4693 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 4694 return 0; 4695 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 4696 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 4697 return 0; 4698 4699 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 4700 if (rc) 4701 return rc; 4702 4703 req->tunnel_type = tunnel_type; 4704 4705 switch (tunnel_type) { 4706 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4707 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4708 bp->vxlan_port = 0; 4709 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4710 break; 4711 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4712 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4713 bp->nge_port = 0; 4714 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4715 break; 4716 default: 4717 break; 4718 } 4719 4720 rc = hwrm_req_send(bp, req); 4721 if (rc) 4722 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4723 rc); 4724 return rc; 4725 } 4726 4727 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4728 u8 tunnel_type) 4729 { 4730 struct hwrm_tunnel_dst_port_alloc_output *resp; 4731 struct hwrm_tunnel_dst_port_alloc_input *req; 4732 int rc; 4733 4734 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 4735 if (rc) 4736 return rc; 4737 4738 req->tunnel_type = tunnel_type; 4739 req->tunnel_dst_port_val = port; 4740 4741 resp = hwrm_req_hold(bp, req); 4742 rc = hwrm_req_send(bp, req); 4743 if (rc) { 4744 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4745 rc); 4746 goto err_out; 4747 } 4748 4749 switch (tunnel_type) { 4750 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4751 bp->vxlan_port = port; 4752 bp->vxlan_fw_dst_port_id = 4753 le16_to_cpu(resp->tunnel_dst_port_id); 4754 break; 4755 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4756 bp->nge_port = port; 4757 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4758 break; 4759 default: 4760 break; 4761 } 4762 4763 err_out: 4764 hwrm_req_drop(bp, req); 4765 return rc; 4766 } 4767 4768 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4769 { 4770 struct hwrm_cfa_l2_set_rx_mask_input *req; 4771 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4772 int rc; 4773 4774 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 4775 if (rc) 4776 return rc; 4777 4778 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4779 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 4780 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4781 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4782 } 4783 req->mask = cpu_to_le32(vnic->rx_mask); 4784 return hwrm_req_send_silent(bp, req); 4785 } 4786 4787 #ifdef CONFIG_RFS_ACCEL 4788 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4789 struct bnxt_ntuple_filter *fltr) 4790 { 4791 struct hwrm_cfa_ntuple_filter_free_input *req; 4792 int rc; 4793 4794 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 4795 if (rc) 4796 return rc; 4797 4798 req->ntuple_filter_id = fltr->filter_id; 4799 return hwrm_req_send(bp, req); 4800 } 4801 4802 #define BNXT_NTP_FLTR_FLAGS \ 4803 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4804 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4805 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4806 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4807 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4808 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4809 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4810 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4811 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4812 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4813 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4814 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4815 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4816 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4817 4818 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4819 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4820 4821 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4822 struct bnxt_ntuple_filter *fltr) 4823 { 4824 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4825 struct hwrm_cfa_ntuple_filter_alloc_input *req; 4826 struct flow_keys *keys = &fltr->fkeys; 4827 struct bnxt_vnic_info *vnic; 4828 u32 flags = 0; 4829 int rc; 4830 4831 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 4832 if (rc) 4833 return rc; 4834 4835 req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4836 4837 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4838 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4839 req->dst_id = cpu_to_le16(fltr->rxq); 4840 } else { 4841 vnic = &bp->vnic_info[fltr->rxq + 1]; 4842 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 4843 } 4844 req->flags = cpu_to_le32(flags); 4845 req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4846 4847 req->ethertype = htons(ETH_P_IP); 4848 memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4849 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4850 req->ip_protocol = keys->basic.ip_proto; 4851 4852 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4853 int i; 4854 4855 req->ethertype = htons(ETH_P_IPV6); 4856 req->ip_addr_type = 4857 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4858 *(struct in6_addr *)&req->src_ipaddr[0] = 4859 keys->addrs.v6addrs.src; 4860 *(struct in6_addr *)&req->dst_ipaddr[0] = 4861 keys->addrs.v6addrs.dst; 4862 for (i = 0; i < 4; i++) { 4863 req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4864 req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4865 } 4866 } else { 4867 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 4868 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4869 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4870 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4871 } 4872 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4873 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4874 req->tunnel_type = 4875 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4876 } 4877 4878 req->src_port = keys->ports.src; 4879 req->src_port_mask = cpu_to_be16(0xffff); 4880 req->dst_port = keys->ports.dst; 4881 req->dst_port_mask = cpu_to_be16(0xffff); 4882 4883 resp = hwrm_req_hold(bp, req); 4884 rc = hwrm_req_send(bp, req); 4885 if (!rc) 4886 fltr->filter_id = resp->ntuple_filter_id; 4887 hwrm_req_drop(bp, req); 4888 return rc; 4889 } 4890 #endif 4891 4892 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4893 const u8 *mac_addr) 4894 { 4895 struct hwrm_cfa_l2_filter_alloc_output *resp; 4896 struct hwrm_cfa_l2_filter_alloc_input *req; 4897 int rc; 4898 4899 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 4900 if (rc) 4901 return rc; 4902 4903 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4904 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4905 req->flags |= 4906 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4907 req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4908 req->enables = 4909 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4910 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4911 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4912 memcpy(req->l2_addr, mac_addr, ETH_ALEN); 4913 req->l2_addr_mask[0] = 0xff; 4914 req->l2_addr_mask[1] = 0xff; 4915 req->l2_addr_mask[2] = 0xff; 4916 req->l2_addr_mask[3] = 0xff; 4917 req->l2_addr_mask[4] = 0xff; 4918 req->l2_addr_mask[5] = 0xff; 4919 4920 resp = hwrm_req_hold(bp, req); 4921 rc = hwrm_req_send(bp, req); 4922 if (!rc) 4923 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4924 resp->l2_filter_id; 4925 hwrm_req_drop(bp, req); 4926 return rc; 4927 } 4928 4929 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4930 { 4931 struct hwrm_cfa_l2_filter_free_input *req; 4932 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4933 int rc; 4934 4935 /* Any associated ntuple filters will also be cleared by firmware. */ 4936 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 4937 if (rc) 4938 return rc; 4939 hwrm_req_hold(bp, req); 4940 for (i = 0; i < num_of_vnics; i++) { 4941 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4942 4943 for (j = 0; j < vnic->uc_filter_count; j++) { 4944 req->l2_filter_id = vnic->fw_l2_filter_id[j]; 4945 4946 rc = hwrm_req_send(bp, req); 4947 } 4948 vnic->uc_filter_count = 0; 4949 } 4950 hwrm_req_drop(bp, req); 4951 return rc; 4952 } 4953 4954 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4955 { 4956 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4957 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4958 struct hwrm_vnic_tpa_cfg_input *req; 4959 int rc; 4960 4961 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4962 return 0; 4963 4964 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 4965 if (rc) 4966 return rc; 4967 4968 if (tpa_flags) { 4969 u16 mss = bp->dev->mtu - 40; 4970 u32 nsegs, n, segs = 0, flags; 4971 4972 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4973 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4974 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4975 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4976 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4977 if (tpa_flags & BNXT_FLAG_GRO) 4978 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4979 4980 req->flags = cpu_to_le32(flags); 4981 4982 req->enables = 4983 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4984 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4985 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4986 4987 /* Number of segs are log2 units, and first packet is not 4988 * included as part of this units. 4989 */ 4990 if (mss <= BNXT_RX_PAGE_SIZE) { 4991 n = BNXT_RX_PAGE_SIZE / mss; 4992 nsegs = (MAX_SKB_FRAGS - 1) * n; 4993 } else { 4994 n = mss / BNXT_RX_PAGE_SIZE; 4995 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4996 n++; 4997 nsegs = (MAX_SKB_FRAGS - n) / n; 4998 } 4999 5000 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5001 segs = MAX_TPA_SEGS_P5; 5002 max_aggs = bp->max_tpa; 5003 } else { 5004 segs = ilog2(nsegs); 5005 } 5006 req->max_agg_segs = cpu_to_le16(segs); 5007 req->max_aggs = cpu_to_le16(max_aggs); 5008 5009 req->min_agg_len = cpu_to_le32(512); 5010 } 5011 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5012 5013 return hwrm_req_send(bp, req); 5014 } 5015 5016 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 5017 { 5018 struct bnxt_ring_grp_info *grp_info; 5019 5020 grp_info = &bp->grp_info[ring->grp_idx]; 5021 return grp_info->cp_fw_ring_id; 5022 } 5023 5024 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 5025 { 5026 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5027 struct bnxt_napi *bnapi = rxr->bnapi; 5028 struct bnxt_cp_ring_info *cpr; 5029 5030 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 5031 return cpr->cp_ring_struct.fw_ring_id; 5032 } else { 5033 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 5034 } 5035 } 5036 5037 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 5038 { 5039 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5040 struct bnxt_napi *bnapi = txr->bnapi; 5041 struct bnxt_cp_ring_info *cpr; 5042 5043 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 5044 return cpr->cp_ring_struct.fw_ring_id; 5045 } else { 5046 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 5047 } 5048 } 5049 5050 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 5051 { 5052 int entries; 5053 5054 if (bp->flags & BNXT_FLAG_CHIP_P5) 5055 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 5056 else 5057 entries = HW_HASH_INDEX_SIZE; 5058 5059 bp->rss_indir_tbl_entries = entries; 5060 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 5061 GFP_KERNEL); 5062 if (!bp->rss_indir_tbl) 5063 return -ENOMEM; 5064 return 0; 5065 } 5066 5067 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 5068 { 5069 u16 max_rings, max_entries, pad, i; 5070 5071 if (!bp->rx_nr_rings) 5072 return; 5073 5074 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5075 max_rings = bp->rx_nr_rings - 1; 5076 else 5077 max_rings = bp->rx_nr_rings; 5078 5079 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 5080 5081 for (i = 0; i < max_entries; i++) 5082 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 5083 5084 pad = bp->rss_indir_tbl_entries - max_entries; 5085 if (pad) 5086 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 5087 } 5088 5089 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 5090 { 5091 u16 i, tbl_size, max_ring = 0; 5092 5093 if (!bp->rss_indir_tbl) 5094 return 0; 5095 5096 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5097 for (i = 0; i < tbl_size; i++) 5098 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 5099 return max_ring; 5100 } 5101 5102 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 5103 { 5104 if (bp->flags & BNXT_FLAG_CHIP_P5) 5105 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 5106 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 5107 return 2; 5108 return 1; 5109 } 5110 5111 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5112 { 5113 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 5114 u16 i, j; 5115 5116 /* Fill the RSS indirection table with ring group ids */ 5117 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 5118 if (!no_rss) 5119 j = bp->rss_indir_tbl[i]; 5120 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 5121 } 5122 } 5123 5124 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 5125 struct bnxt_vnic_info *vnic) 5126 { 5127 __le16 *ring_tbl = vnic->rss_table; 5128 struct bnxt_rx_ring_info *rxr; 5129 u16 tbl_size, i; 5130 5131 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 5132 5133 for (i = 0; i < tbl_size; i++) { 5134 u16 ring_id, j; 5135 5136 j = bp->rss_indir_tbl[i]; 5137 rxr = &bp->rx_ring[j]; 5138 5139 ring_id = rxr->rx_ring_struct.fw_ring_id; 5140 *ring_tbl++ = cpu_to_le16(ring_id); 5141 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5142 *ring_tbl++ = cpu_to_le16(ring_id); 5143 } 5144 } 5145 5146 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 5147 { 5148 if (bp->flags & BNXT_FLAG_CHIP_P5) 5149 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 5150 else 5151 __bnxt_fill_hw_rss_tbl(bp, vnic); 5152 } 5153 5154 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 5155 { 5156 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5157 struct hwrm_vnic_rss_cfg_input *req; 5158 int rc; 5159 5160 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 5161 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 5162 return 0; 5163 5164 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5165 if (rc) 5166 return rc; 5167 5168 if (set_rss) { 5169 bnxt_fill_hw_rss_tbl(bp, vnic); 5170 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5171 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5172 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 5173 req->hash_key_tbl_addr = 5174 cpu_to_le64(vnic->rss_hash_key_dma_addr); 5175 } 5176 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5177 return hwrm_req_send(bp, req); 5178 } 5179 5180 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 5181 { 5182 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5183 struct hwrm_vnic_rss_cfg_input *req; 5184 dma_addr_t ring_tbl_map; 5185 u32 i, nr_ctxs; 5186 int rc; 5187 5188 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 5189 if (rc) 5190 return rc; 5191 5192 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5193 if (!set_rss) 5194 return hwrm_req_send(bp, req); 5195 5196 bnxt_fill_hw_rss_tbl(bp, vnic); 5197 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 5198 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 5199 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 5200 ring_tbl_map = vnic->rss_table_dma_addr; 5201 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 5202 5203 hwrm_req_hold(bp, req); 5204 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 5205 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 5206 req->ring_table_pair_index = i; 5207 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 5208 rc = hwrm_req_send(bp, req); 5209 if (rc) 5210 goto exit; 5211 } 5212 5213 exit: 5214 hwrm_req_drop(bp, req); 5215 return rc; 5216 } 5217 5218 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5219 { 5220 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5221 struct hwrm_vnic_plcmodes_cfg_input *req; 5222 int rc; 5223 5224 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 5225 if (rc) 5226 return rc; 5227 5228 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5229 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5230 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5231 req->enables = 5232 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5233 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5234 /* thresholds not implemented in firmware yet */ 5235 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5236 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5237 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5238 return hwrm_req_send(bp, req); 5239 } 5240 5241 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5242 u16 ctx_idx) 5243 { 5244 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 5245 5246 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 5247 return; 5248 5249 req->rss_cos_lb_ctx_id = 5250 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5251 5252 hwrm_req_send(bp, req); 5253 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5254 } 5255 5256 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5257 { 5258 int i, j; 5259 5260 for (i = 0; i < bp->nr_vnics; i++) { 5261 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5262 5263 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5264 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5265 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5266 } 5267 } 5268 bp->rsscos_nr_ctxs = 0; 5269 } 5270 5271 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5272 { 5273 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 5274 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 5275 int rc; 5276 5277 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 5278 if (rc) 5279 return rc; 5280 5281 resp = hwrm_req_hold(bp, req); 5282 rc = hwrm_req_send(bp, req); 5283 if (!rc) 5284 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5285 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5286 hwrm_req_drop(bp, req); 5287 5288 return rc; 5289 } 5290 5291 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5292 { 5293 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5294 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5295 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5296 } 5297 5298 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5299 { 5300 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5301 struct hwrm_vnic_cfg_input *req; 5302 unsigned int ring = 0, grp_idx; 5303 u16 def_vlan = 0; 5304 int rc; 5305 5306 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 5307 if (rc) 5308 return rc; 5309 5310 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5311 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5312 5313 req->default_rx_ring_id = 5314 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5315 req->default_cmpl_ring_id = 5316 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5317 req->enables = 5318 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5319 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5320 goto vnic_mru; 5321 } 5322 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5323 /* Only RSS support for now TBD: COS & LB */ 5324 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5325 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5326 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5327 VNIC_CFG_REQ_ENABLES_MRU); 5328 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5329 req->rss_rule = 5330 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5331 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5332 VNIC_CFG_REQ_ENABLES_MRU); 5333 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5334 } else { 5335 req->rss_rule = cpu_to_le16(0xffff); 5336 } 5337 5338 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5339 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5340 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5341 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5342 } else { 5343 req->cos_rule = cpu_to_le16(0xffff); 5344 } 5345 5346 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5347 ring = 0; 5348 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5349 ring = vnic_id - 1; 5350 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5351 ring = bp->rx_nr_rings - 1; 5352 5353 grp_idx = bp->rx_ring[ring].bnapi->index; 5354 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5355 req->lb_rule = cpu_to_le16(0xffff); 5356 vnic_mru: 5357 req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5358 5359 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5360 #ifdef CONFIG_BNXT_SRIOV 5361 if (BNXT_VF(bp)) 5362 def_vlan = bp->vf.vlan; 5363 #endif 5364 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5365 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5366 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5367 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5368 5369 return hwrm_req_send(bp, req); 5370 } 5371 5372 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5373 { 5374 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5375 struct hwrm_vnic_free_input *req; 5376 5377 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 5378 return; 5379 5380 req->vnic_id = 5381 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5382 5383 hwrm_req_send(bp, req); 5384 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5385 } 5386 } 5387 5388 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5389 { 5390 u16 i; 5391 5392 for (i = 0; i < bp->nr_vnics; i++) 5393 bnxt_hwrm_vnic_free_one(bp, i); 5394 } 5395 5396 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5397 unsigned int start_rx_ring_idx, 5398 unsigned int nr_rings) 5399 { 5400 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5401 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5402 struct hwrm_vnic_alloc_output *resp; 5403 struct hwrm_vnic_alloc_input *req; 5404 int rc; 5405 5406 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 5407 if (rc) 5408 return rc; 5409 5410 if (bp->flags & BNXT_FLAG_CHIP_P5) 5411 goto vnic_no_ring_grps; 5412 5413 /* map ring groups to this vnic */ 5414 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5415 grp_idx = bp->rx_ring[i].bnapi->index; 5416 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5417 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5418 j, nr_rings); 5419 break; 5420 } 5421 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5422 } 5423 5424 vnic_no_ring_grps: 5425 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5426 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5427 if (vnic_id == 0) 5428 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5429 5430 resp = hwrm_req_hold(bp, req); 5431 rc = hwrm_req_send(bp, req); 5432 if (!rc) 5433 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5434 hwrm_req_drop(bp, req); 5435 return rc; 5436 } 5437 5438 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5439 { 5440 struct hwrm_vnic_qcaps_output *resp; 5441 struct hwrm_vnic_qcaps_input *req; 5442 int rc; 5443 5444 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5445 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5446 if (bp->hwrm_spec_code < 0x10600) 5447 return 0; 5448 5449 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 5450 if (rc) 5451 return rc; 5452 5453 resp = hwrm_req_hold(bp, req); 5454 rc = hwrm_req_send(bp, req); 5455 if (!rc) { 5456 u32 flags = le32_to_cpu(resp->flags); 5457 5458 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5459 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5460 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5461 if (flags & 5462 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5463 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5464 5465 /* Older P5 fw before EXT_HW_STATS support did not set 5466 * VLAN_STRIP_CAP properly. 5467 */ 5468 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5469 (BNXT_CHIP_P5_THOR(bp) && 5470 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5471 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5472 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5473 if (bp->max_tpa_v2) { 5474 if (BNXT_CHIP_P5_THOR(bp)) 5475 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 5476 else 5477 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2; 5478 } 5479 } 5480 hwrm_req_drop(bp, req); 5481 return rc; 5482 } 5483 5484 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5485 { 5486 struct hwrm_ring_grp_alloc_output *resp; 5487 struct hwrm_ring_grp_alloc_input *req; 5488 int rc; 5489 u16 i; 5490 5491 if (bp->flags & BNXT_FLAG_CHIP_P5) 5492 return 0; 5493 5494 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 5495 if (rc) 5496 return rc; 5497 5498 resp = hwrm_req_hold(bp, req); 5499 for (i = 0; i < bp->rx_nr_rings; i++) { 5500 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5501 5502 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5503 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5504 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5505 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5506 5507 rc = hwrm_req_send(bp, req); 5508 5509 if (rc) 5510 break; 5511 5512 bp->grp_info[grp_idx].fw_grp_id = 5513 le32_to_cpu(resp->ring_group_id); 5514 } 5515 hwrm_req_drop(bp, req); 5516 return rc; 5517 } 5518 5519 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5520 { 5521 struct hwrm_ring_grp_free_input *req; 5522 u16 i; 5523 5524 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5525 return; 5526 5527 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 5528 return; 5529 5530 hwrm_req_hold(bp, req); 5531 for (i = 0; i < bp->cp_nr_rings; i++) { 5532 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5533 continue; 5534 req->ring_group_id = 5535 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5536 5537 hwrm_req_send(bp, req); 5538 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5539 } 5540 hwrm_req_drop(bp, req); 5541 } 5542 5543 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5544 struct bnxt_ring_struct *ring, 5545 u32 ring_type, u32 map_index) 5546 { 5547 struct hwrm_ring_alloc_output *resp; 5548 struct hwrm_ring_alloc_input *req; 5549 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5550 struct bnxt_ring_grp_info *grp_info; 5551 int rc, err = 0; 5552 u16 ring_id; 5553 5554 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 5555 if (rc) 5556 goto exit; 5557 5558 req->enables = 0; 5559 if (rmem->nr_pages > 1) { 5560 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5561 /* Page size is in log2 units */ 5562 req->page_size = BNXT_PAGE_SHIFT; 5563 req->page_tbl_depth = 1; 5564 } else { 5565 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5566 } 5567 req->fbo = 0; 5568 /* Association of ring index with doorbell index and MSIX number */ 5569 req->logical_id = cpu_to_le16(map_index); 5570 5571 switch (ring_type) { 5572 case HWRM_RING_ALLOC_TX: { 5573 struct bnxt_tx_ring_info *txr; 5574 5575 txr = container_of(ring, struct bnxt_tx_ring_info, 5576 tx_ring_struct); 5577 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5578 /* Association of transmit ring with completion ring */ 5579 grp_info = &bp->grp_info[ring->grp_idx]; 5580 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5581 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 5582 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5583 req->queue_id = cpu_to_le16(ring->queue_id); 5584 break; 5585 } 5586 case HWRM_RING_ALLOC_RX: 5587 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5588 req->length = cpu_to_le32(bp->rx_ring_mask + 1); 5589 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5590 u16 flags = 0; 5591 5592 /* Association of rx ring with stats context */ 5593 grp_info = &bp->grp_info[ring->grp_idx]; 5594 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5595 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5596 req->enables |= cpu_to_le32( 5597 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5598 if (NET_IP_ALIGN == 2) 5599 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5600 req->flags = cpu_to_le16(flags); 5601 } 5602 break; 5603 case HWRM_RING_ALLOC_AGG: 5604 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5605 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5606 /* Association of agg ring with rx ring */ 5607 grp_info = &bp->grp_info[ring->grp_idx]; 5608 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5609 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5610 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5611 req->enables |= cpu_to_le32( 5612 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5613 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5614 } else { 5615 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5616 } 5617 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5618 break; 5619 case HWRM_RING_ALLOC_CMPL: 5620 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5621 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5622 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5623 /* Association of cp ring with nq */ 5624 grp_info = &bp->grp_info[map_index]; 5625 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5626 req->cq_handle = cpu_to_le64(ring->handle); 5627 req->enables |= cpu_to_le32( 5628 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5629 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5630 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5631 } 5632 break; 5633 case HWRM_RING_ALLOC_NQ: 5634 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5635 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 5636 if (bp->flags & BNXT_FLAG_USING_MSIX) 5637 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5638 break; 5639 default: 5640 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5641 ring_type); 5642 return -1; 5643 } 5644 5645 resp = hwrm_req_hold(bp, req); 5646 rc = hwrm_req_send(bp, req); 5647 err = le16_to_cpu(resp->error_code); 5648 ring_id = le16_to_cpu(resp->ring_id); 5649 hwrm_req_drop(bp, req); 5650 5651 exit: 5652 if (rc || err) { 5653 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5654 ring_type, rc, err); 5655 return -EIO; 5656 } 5657 ring->fw_ring_id = ring_id; 5658 return rc; 5659 } 5660 5661 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5662 { 5663 int rc; 5664 5665 if (BNXT_PF(bp)) { 5666 struct hwrm_func_cfg_input *req; 5667 5668 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 5669 if (rc) 5670 return rc; 5671 5672 req->fid = cpu_to_le16(0xffff); 5673 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5674 req->async_event_cr = cpu_to_le16(idx); 5675 return hwrm_req_send(bp, req); 5676 } else { 5677 struct hwrm_func_vf_cfg_input *req; 5678 5679 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 5680 if (rc) 5681 return rc; 5682 5683 req->enables = 5684 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5685 req->async_event_cr = cpu_to_le16(idx); 5686 return hwrm_req_send(bp, req); 5687 } 5688 } 5689 5690 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5691 u32 map_idx, u32 xid) 5692 { 5693 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5694 if (BNXT_PF(bp)) 5695 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5696 else 5697 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5698 switch (ring_type) { 5699 case HWRM_RING_ALLOC_TX: 5700 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5701 break; 5702 case HWRM_RING_ALLOC_RX: 5703 case HWRM_RING_ALLOC_AGG: 5704 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5705 break; 5706 case HWRM_RING_ALLOC_CMPL: 5707 db->db_key64 = DBR_PATH_L2; 5708 break; 5709 case HWRM_RING_ALLOC_NQ: 5710 db->db_key64 = DBR_PATH_L2; 5711 break; 5712 } 5713 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5714 } else { 5715 db->doorbell = bp->bar1 + map_idx * 0x80; 5716 switch (ring_type) { 5717 case HWRM_RING_ALLOC_TX: 5718 db->db_key32 = DB_KEY_TX; 5719 break; 5720 case HWRM_RING_ALLOC_RX: 5721 case HWRM_RING_ALLOC_AGG: 5722 db->db_key32 = DB_KEY_RX; 5723 break; 5724 case HWRM_RING_ALLOC_CMPL: 5725 db->db_key32 = DB_KEY_CP; 5726 break; 5727 } 5728 } 5729 } 5730 5731 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5732 { 5733 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5734 int i, rc = 0; 5735 u32 type; 5736 5737 if (bp->flags & BNXT_FLAG_CHIP_P5) 5738 type = HWRM_RING_ALLOC_NQ; 5739 else 5740 type = HWRM_RING_ALLOC_CMPL; 5741 for (i = 0; i < bp->cp_nr_rings; i++) { 5742 struct bnxt_napi *bnapi = bp->bnapi[i]; 5743 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5744 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5745 u32 map_idx = ring->map_idx; 5746 unsigned int vector; 5747 5748 vector = bp->irq_tbl[map_idx].vector; 5749 disable_irq_nosync(vector); 5750 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5751 if (rc) { 5752 enable_irq(vector); 5753 goto err_out; 5754 } 5755 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5756 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5757 enable_irq(vector); 5758 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5759 5760 if (!i) { 5761 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5762 if (rc) 5763 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5764 } 5765 } 5766 5767 type = HWRM_RING_ALLOC_TX; 5768 for (i = 0; i < bp->tx_nr_rings; i++) { 5769 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5770 struct bnxt_ring_struct *ring; 5771 u32 map_idx; 5772 5773 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5774 struct bnxt_napi *bnapi = txr->bnapi; 5775 struct bnxt_cp_ring_info *cpr, *cpr2; 5776 u32 type2 = HWRM_RING_ALLOC_CMPL; 5777 5778 cpr = &bnapi->cp_ring; 5779 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5780 ring = &cpr2->cp_ring_struct; 5781 ring->handle = BNXT_TX_HDL; 5782 map_idx = bnapi->index; 5783 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5784 if (rc) 5785 goto err_out; 5786 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5787 ring->fw_ring_id); 5788 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5789 } 5790 ring = &txr->tx_ring_struct; 5791 map_idx = i; 5792 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5793 if (rc) 5794 goto err_out; 5795 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5796 } 5797 5798 type = HWRM_RING_ALLOC_RX; 5799 for (i = 0; i < bp->rx_nr_rings; i++) { 5800 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5801 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5802 struct bnxt_napi *bnapi = rxr->bnapi; 5803 u32 map_idx = bnapi->index; 5804 5805 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5806 if (rc) 5807 goto err_out; 5808 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5809 /* If we have agg rings, post agg buffers first. */ 5810 if (!agg_rings) 5811 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5812 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5813 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5814 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5815 u32 type2 = HWRM_RING_ALLOC_CMPL; 5816 struct bnxt_cp_ring_info *cpr2; 5817 5818 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5819 ring = &cpr2->cp_ring_struct; 5820 ring->handle = BNXT_RX_HDL; 5821 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5822 if (rc) 5823 goto err_out; 5824 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5825 ring->fw_ring_id); 5826 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5827 } 5828 } 5829 5830 if (agg_rings) { 5831 type = HWRM_RING_ALLOC_AGG; 5832 for (i = 0; i < bp->rx_nr_rings; i++) { 5833 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5834 struct bnxt_ring_struct *ring = 5835 &rxr->rx_agg_ring_struct; 5836 u32 grp_idx = ring->grp_idx; 5837 u32 map_idx = grp_idx + bp->rx_nr_rings; 5838 5839 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5840 if (rc) 5841 goto err_out; 5842 5843 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5844 ring->fw_ring_id); 5845 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5846 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5847 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5848 } 5849 } 5850 err_out: 5851 return rc; 5852 } 5853 5854 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5855 struct bnxt_ring_struct *ring, 5856 u32 ring_type, int cmpl_ring_id) 5857 { 5858 struct hwrm_ring_free_output *resp; 5859 struct hwrm_ring_free_input *req; 5860 u16 error_code = 0; 5861 int rc; 5862 5863 if (BNXT_NO_FW_ACCESS(bp)) 5864 return 0; 5865 5866 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 5867 if (rc) 5868 goto exit; 5869 5870 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 5871 req->ring_type = ring_type; 5872 req->ring_id = cpu_to_le16(ring->fw_ring_id); 5873 5874 resp = hwrm_req_hold(bp, req); 5875 rc = hwrm_req_send(bp, req); 5876 error_code = le16_to_cpu(resp->error_code); 5877 hwrm_req_drop(bp, req); 5878 exit: 5879 if (rc || error_code) { 5880 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5881 ring_type, rc, error_code); 5882 return -EIO; 5883 } 5884 return 0; 5885 } 5886 5887 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5888 { 5889 u32 type; 5890 int i; 5891 5892 if (!bp->bnapi) 5893 return; 5894 5895 for (i = 0; i < bp->tx_nr_rings; i++) { 5896 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5897 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5898 5899 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5900 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5901 5902 hwrm_ring_free_send_msg(bp, ring, 5903 RING_FREE_REQ_RING_TYPE_TX, 5904 close_path ? cmpl_ring_id : 5905 INVALID_HW_RING_ID); 5906 ring->fw_ring_id = INVALID_HW_RING_ID; 5907 } 5908 } 5909 5910 for (i = 0; i < bp->rx_nr_rings; i++) { 5911 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5912 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5913 u32 grp_idx = rxr->bnapi->index; 5914 5915 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5916 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5917 5918 hwrm_ring_free_send_msg(bp, ring, 5919 RING_FREE_REQ_RING_TYPE_RX, 5920 close_path ? cmpl_ring_id : 5921 INVALID_HW_RING_ID); 5922 ring->fw_ring_id = INVALID_HW_RING_ID; 5923 bp->grp_info[grp_idx].rx_fw_ring_id = 5924 INVALID_HW_RING_ID; 5925 } 5926 } 5927 5928 if (bp->flags & BNXT_FLAG_CHIP_P5) 5929 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5930 else 5931 type = RING_FREE_REQ_RING_TYPE_RX; 5932 for (i = 0; i < bp->rx_nr_rings; i++) { 5933 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5934 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5935 u32 grp_idx = rxr->bnapi->index; 5936 5937 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5938 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5939 5940 hwrm_ring_free_send_msg(bp, ring, type, 5941 close_path ? cmpl_ring_id : 5942 INVALID_HW_RING_ID); 5943 ring->fw_ring_id = INVALID_HW_RING_ID; 5944 bp->grp_info[grp_idx].agg_fw_ring_id = 5945 INVALID_HW_RING_ID; 5946 } 5947 } 5948 5949 /* The completion rings are about to be freed. After that the 5950 * IRQ doorbell will not work anymore. So we need to disable 5951 * IRQ here. 5952 */ 5953 bnxt_disable_int_sync(bp); 5954 5955 if (bp->flags & BNXT_FLAG_CHIP_P5) 5956 type = RING_FREE_REQ_RING_TYPE_NQ; 5957 else 5958 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5959 for (i = 0; i < bp->cp_nr_rings; i++) { 5960 struct bnxt_napi *bnapi = bp->bnapi[i]; 5961 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5962 struct bnxt_ring_struct *ring; 5963 int j; 5964 5965 for (j = 0; j < 2; j++) { 5966 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5967 5968 if (cpr2) { 5969 ring = &cpr2->cp_ring_struct; 5970 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5971 continue; 5972 hwrm_ring_free_send_msg(bp, ring, 5973 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5974 INVALID_HW_RING_ID); 5975 ring->fw_ring_id = INVALID_HW_RING_ID; 5976 } 5977 } 5978 ring = &cpr->cp_ring_struct; 5979 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5980 hwrm_ring_free_send_msg(bp, ring, type, 5981 INVALID_HW_RING_ID); 5982 ring->fw_ring_id = INVALID_HW_RING_ID; 5983 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5984 } 5985 } 5986 } 5987 5988 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5989 bool shared); 5990 5991 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5992 { 5993 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5994 struct hwrm_func_qcfg_output *resp; 5995 struct hwrm_func_qcfg_input *req; 5996 int rc; 5997 5998 if (bp->hwrm_spec_code < 0x10601) 5999 return 0; 6000 6001 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6002 if (rc) 6003 return rc; 6004 6005 req->fid = cpu_to_le16(0xffff); 6006 resp = hwrm_req_hold(bp, req); 6007 rc = hwrm_req_send(bp, req); 6008 if (rc) { 6009 hwrm_req_drop(bp, req); 6010 return rc; 6011 } 6012 6013 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6014 if (BNXT_NEW_RM(bp)) { 6015 u16 cp, stats; 6016 6017 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 6018 hw_resc->resv_hw_ring_grps = 6019 le32_to_cpu(resp->alloc_hw_ring_grps); 6020 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 6021 cp = le16_to_cpu(resp->alloc_cmpl_rings); 6022 stats = le16_to_cpu(resp->alloc_stat_ctx); 6023 hw_resc->resv_irqs = cp; 6024 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6025 int rx = hw_resc->resv_rx_rings; 6026 int tx = hw_resc->resv_tx_rings; 6027 6028 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6029 rx >>= 1; 6030 if (cp < (rx + tx)) { 6031 bnxt_trim_rings(bp, &rx, &tx, cp, false); 6032 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6033 rx <<= 1; 6034 hw_resc->resv_rx_rings = rx; 6035 hw_resc->resv_tx_rings = tx; 6036 } 6037 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 6038 hw_resc->resv_hw_ring_grps = rx; 6039 } 6040 hw_resc->resv_cp_rings = cp; 6041 hw_resc->resv_stat_ctxs = stats; 6042 } 6043 hwrm_req_drop(bp, req); 6044 return 0; 6045 } 6046 6047 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 6048 { 6049 struct hwrm_func_qcfg_output *resp; 6050 struct hwrm_func_qcfg_input *req; 6051 int rc; 6052 6053 if (bp->hwrm_spec_code < 0x10601) 6054 return 0; 6055 6056 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6057 if (rc) 6058 return rc; 6059 6060 req->fid = cpu_to_le16(fid); 6061 resp = hwrm_req_hold(bp, req); 6062 rc = hwrm_req_send(bp, req); 6063 if (!rc) 6064 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 6065 6066 hwrm_req_drop(bp, req); 6067 return rc; 6068 } 6069 6070 static bool bnxt_rfs_supported(struct bnxt *bp); 6071 6072 static struct hwrm_func_cfg_input * 6073 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6074 int ring_grps, int cp_rings, int stats, int vnics) 6075 { 6076 struct hwrm_func_cfg_input *req; 6077 u32 enables = 0; 6078 6079 if (hwrm_req_init(bp, req, HWRM_FUNC_CFG)) 6080 return NULL; 6081 6082 req->fid = cpu_to_le16(0xffff); 6083 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6084 req->num_tx_rings = cpu_to_le16(tx_rings); 6085 if (BNXT_NEW_RM(bp)) { 6086 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 6087 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6088 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6089 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 6090 enables |= tx_rings + ring_grps ? 6091 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6092 enables |= rx_rings ? 6093 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6094 } else { 6095 enables |= cp_rings ? 6096 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6097 enables |= ring_grps ? 6098 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 6099 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6100 } 6101 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 6102 6103 req->num_rx_rings = cpu_to_le16(rx_rings); 6104 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6105 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6106 req->num_msix = cpu_to_le16(cp_rings); 6107 req->num_rsscos_ctxs = 6108 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6109 } else { 6110 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6111 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6112 req->num_rsscos_ctxs = cpu_to_le16(1); 6113 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 6114 bnxt_rfs_supported(bp)) 6115 req->num_rsscos_ctxs = 6116 cpu_to_le16(ring_grps + 1); 6117 } 6118 req->num_stat_ctxs = cpu_to_le16(stats); 6119 req->num_vnics = cpu_to_le16(vnics); 6120 } 6121 req->enables = cpu_to_le32(enables); 6122 return req; 6123 } 6124 6125 static struct hwrm_func_vf_cfg_input * 6126 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6127 int ring_grps, int cp_rings, int stats, int vnics) 6128 { 6129 struct hwrm_func_vf_cfg_input *req; 6130 u32 enables = 0; 6131 6132 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 6133 return NULL; 6134 6135 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 6136 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 6137 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 6138 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 6139 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6140 enables |= tx_rings + ring_grps ? 6141 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6142 } else { 6143 enables |= cp_rings ? 6144 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 6145 enables |= ring_grps ? 6146 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 6147 } 6148 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 6149 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 6150 6151 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 6152 req->num_tx_rings = cpu_to_le16(tx_rings); 6153 req->num_rx_rings = cpu_to_le16(rx_rings); 6154 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6155 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 6156 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 6157 } else { 6158 req->num_cmpl_rings = cpu_to_le16(cp_rings); 6159 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 6160 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 6161 } 6162 req->num_stat_ctxs = cpu_to_le16(stats); 6163 req->num_vnics = cpu_to_le16(vnics); 6164 6165 req->enables = cpu_to_le32(enables); 6166 return req; 6167 } 6168 6169 static int 6170 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6171 int ring_grps, int cp_rings, int stats, int vnics) 6172 { 6173 struct hwrm_func_cfg_input *req; 6174 int rc; 6175 6176 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6177 cp_rings, stats, vnics); 6178 if (!req) 6179 return -ENOMEM; 6180 6181 if (!req->enables) { 6182 hwrm_req_drop(bp, req); 6183 return 0; 6184 } 6185 6186 rc = hwrm_req_send(bp, req); 6187 if (rc) 6188 return rc; 6189 6190 if (bp->hwrm_spec_code < 0x10601) 6191 bp->hw_resc.resv_tx_rings = tx_rings; 6192 6193 return bnxt_hwrm_get_rings(bp); 6194 } 6195 6196 static int 6197 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6198 int ring_grps, int cp_rings, int stats, int vnics) 6199 { 6200 struct hwrm_func_vf_cfg_input *req; 6201 int rc; 6202 6203 if (!BNXT_NEW_RM(bp)) { 6204 bp->hw_resc.resv_tx_rings = tx_rings; 6205 return 0; 6206 } 6207 6208 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6209 cp_rings, stats, vnics); 6210 if (!req) 6211 return -ENOMEM; 6212 6213 rc = hwrm_req_send(bp, req); 6214 if (rc) 6215 return rc; 6216 6217 return bnxt_hwrm_get_rings(bp); 6218 } 6219 6220 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 6221 int cp, int stat, int vnic) 6222 { 6223 if (BNXT_PF(bp)) 6224 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 6225 vnic); 6226 else 6227 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 6228 vnic); 6229 } 6230 6231 int bnxt_nq_rings_in_use(struct bnxt *bp) 6232 { 6233 int cp = bp->cp_nr_rings; 6234 int ulp_msix, ulp_base; 6235 6236 ulp_msix = bnxt_get_ulp_msix_num(bp); 6237 if (ulp_msix) { 6238 ulp_base = bnxt_get_ulp_msix_base(bp); 6239 cp += ulp_msix; 6240 if ((ulp_base + ulp_msix) > cp) 6241 cp = ulp_base + ulp_msix; 6242 } 6243 return cp; 6244 } 6245 6246 static int bnxt_cp_rings_in_use(struct bnxt *bp) 6247 { 6248 int cp; 6249 6250 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6251 return bnxt_nq_rings_in_use(bp); 6252 6253 cp = bp->tx_nr_rings + bp->rx_nr_rings; 6254 return cp; 6255 } 6256 6257 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 6258 { 6259 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 6260 int cp = bp->cp_nr_rings; 6261 6262 if (!ulp_stat) 6263 return cp; 6264 6265 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 6266 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 6267 6268 return cp + ulp_stat; 6269 } 6270 6271 /* Check if a default RSS map needs to be setup. This function is only 6272 * used on older firmware that does not require reserving RX rings. 6273 */ 6274 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 6275 { 6276 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6277 6278 /* The RSS map is valid for RX rings set to resv_rx_rings */ 6279 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 6280 hw_resc->resv_rx_rings = bp->rx_nr_rings; 6281 if (!netif_is_rxfh_configured(bp->dev)) 6282 bnxt_set_dflt_rss_indir_tbl(bp); 6283 } 6284 } 6285 6286 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6287 { 6288 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6289 int cp = bnxt_cp_rings_in_use(bp); 6290 int nq = bnxt_nq_rings_in_use(bp); 6291 int rx = bp->rx_nr_rings, stat; 6292 int vnic = 1, grp = rx; 6293 6294 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 6295 bp->hwrm_spec_code >= 0x10601) 6296 return true; 6297 6298 /* Old firmware does not need RX ring reservations but we still 6299 * need to setup a default RSS map when needed. With new firmware 6300 * we go through RX ring reservations first and then set up the 6301 * RSS map for the successfully reserved RX rings when needed. 6302 */ 6303 if (!BNXT_NEW_RM(bp)) { 6304 bnxt_check_rss_tbl_no_rmgr(bp); 6305 return false; 6306 } 6307 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6308 vnic = rx + 1; 6309 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6310 rx <<= 1; 6311 stat = bnxt_get_func_stat_ctxs(bp); 6312 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6313 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6314 (hw_resc->resv_hw_ring_grps != grp && 6315 !(bp->flags & BNXT_FLAG_CHIP_P5))) 6316 return true; 6317 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6318 hw_resc->resv_irqs != nq) 6319 return true; 6320 return false; 6321 } 6322 6323 static int __bnxt_reserve_rings(struct bnxt *bp) 6324 { 6325 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6326 int cp = bnxt_nq_rings_in_use(bp); 6327 int tx = bp->tx_nr_rings; 6328 int rx = bp->rx_nr_rings; 6329 int grp, rx_rings, rc; 6330 int vnic = 1, stat; 6331 bool sh = false; 6332 6333 if (!bnxt_need_reserve_rings(bp)) 6334 return 0; 6335 6336 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6337 sh = true; 6338 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6339 vnic = rx + 1; 6340 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6341 rx <<= 1; 6342 grp = bp->rx_nr_rings; 6343 stat = bnxt_get_func_stat_ctxs(bp); 6344 6345 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6346 if (rc) 6347 return rc; 6348 6349 tx = hw_resc->resv_tx_rings; 6350 if (BNXT_NEW_RM(bp)) { 6351 rx = hw_resc->resv_rx_rings; 6352 cp = hw_resc->resv_irqs; 6353 grp = hw_resc->resv_hw_ring_grps; 6354 vnic = hw_resc->resv_vnics; 6355 stat = hw_resc->resv_stat_ctxs; 6356 } 6357 6358 rx_rings = rx; 6359 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6360 if (rx >= 2) { 6361 rx_rings = rx >> 1; 6362 } else { 6363 if (netif_running(bp->dev)) 6364 return -ENOMEM; 6365 6366 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6367 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6368 bp->dev->hw_features &= ~NETIF_F_LRO; 6369 bp->dev->features &= ~NETIF_F_LRO; 6370 bnxt_set_ring_params(bp); 6371 } 6372 } 6373 rx_rings = min_t(int, rx_rings, grp); 6374 cp = min_t(int, cp, bp->cp_nr_rings); 6375 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6376 stat -= bnxt_get_ulp_stat_ctxs(bp); 6377 cp = min_t(int, cp, stat); 6378 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6379 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6380 rx = rx_rings << 1; 6381 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6382 bp->tx_nr_rings = tx; 6383 6384 /* If we cannot reserve all the RX rings, reset the RSS map only 6385 * if absolutely necessary 6386 */ 6387 if (rx_rings != bp->rx_nr_rings) { 6388 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6389 rx_rings, bp->rx_nr_rings); 6390 if (netif_is_rxfh_configured(bp->dev) && 6391 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6392 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6393 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6394 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6395 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6396 } 6397 } 6398 bp->rx_nr_rings = rx_rings; 6399 bp->cp_nr_rings = cp; 6400 6401 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6402 return -ENOMEM; 6403 6404 if (!netif_is_rxfh_configured(bp->dev)) 6405 bnxt_set_dflt_rss_indir_tbl(bp); 6406 6407 return rc; 6408 } 6409 6410 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6411 int ring_grps, int cp_rings, int stats, 6412 int vnics) 6413 { 6414 struct hwrm_func_vf_cfg_input *req; 6415 u32 flags; 6416 6417 if (!BNXT_NEW_RM(bp)) 6418 return 0; 6419 6420 req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6421 cp_rings, stats, vnics); 6422 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6423 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6424 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6425 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6426 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6427 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6428 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6429 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6430 6431 req->flags = cpu_to_le32(flags); 6432 return hwrm_req_send_silent(bp, req); 6433 } 6434 6435 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6436 int ring_grps, int cp_rings, int stats, 6437 int vnics) 6438 { 6439 struct hwrm_func_cfg_input *req; 6440 u32 flags; 6441 6442 req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps, 6443 cp_rings, stats, vnics); 6444 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6445 if (BNXT_NEW_RM(bp)) { 6446 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6447 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6448 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6449 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6450 if (bp->flags & BNXT_FLAG_CHIP_P5) 6451 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6452 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6453 else 6454 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6455 } 6456 6457 req->flags = cpu_to_le32(flags); 6458 return hwrm_req_send_silent(bp, req); 6459 } 6460 6461 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6462 int ring_grps, int cp_rings, int stats, 6463 int vnics) 6464 { 6465 if (bp->hwrm_spec_code < 0x10801) 6466 return 0; 6467 6468 if (BNXT_PF(bp)) 6469 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6470 ring_grps, cp_rings, stats, 6471 vnics); 6472 6473 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6474 cp_rings, stats, vnics); 6475 } 6476 6477 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6478 { 6479 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6480 struct hwrm_ring_aggint_qcaps_output *resp; 6481 struct hwrm_ring_aggint_qcaps_input *req; 6482 int rc; 6483 6484 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6485 coal_cap->num_cmpl_dma_aggr_max = 63; 6486 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6487 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6488 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6489 coal_cap->int_lat_tmr_min_max = 65535; 6490 coal_cap->int_lat_tmr_max_max = 65535; 6491 coal_cap->num_cmpl_aggr_int_max = 65535; 6492 coal_cap->timer_units = 80; 6493 6494 if (bp->hwrm_spec_code < 0x10902) 6495 return; 6496 6497 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 6498 return; 6499 6500 resp = hwrm_req_hold(bp, req); 6501 rc = hwrm_req_send_silent(bp, req); 6502 if (!rc) { 6503 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6504 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6505 coal_cap->num_cmpl_dma_aggr_max = 6506 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6507 coal_cap->num_cmpl_dma_aggr_during_int_max = 6508 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6509 coal_cap->cmpl_aggr_dma_tmr_max = 6510 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6511 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6512 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6513 coal_cap->int_lat_tmr_min_max = 6514 le16_to_cpu(resp->int_lat_tmr_min_max); 6515 coal_cap->int_lat_tmr_max_max = 6516 le16_to_cpu(resp->int_lat_tmr_max_max); 6517 coal_cap->num_cmpl_aggr_int_max = 6518 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6519 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6520 } 6521 hwrm_req_drop(bp, req); 6522 } 6523 6524 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6525 { 6526 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6527 6528 return usec * 1000 / coal_cap->timer_units; 6529 } 6530 6531 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6532 struct bnxt_coal *hw_coal, 6533 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6534 { 6535 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6536 u16 val, tmr, max, flags = hw_coal->flags; 6537 u32 cmpl_params = coal_cap->cmpl_params; 6538 6539 max = hw_coal->bufs_per_record * 128; 6540 if (hw_coal->budget) 6541 max = hw_coal->bufs_per_record * hw_coal->budget; 6542 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6543 6544 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6545 req->num_cmpl_aggr_int = cpu_to_le16(val); 6546 6547 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6548 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6549 6550 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6551 coal_cap->num_cmpl_dma_aggr_during_int_max); 6552 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6553 6554 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6555 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6556 req->int_lat_tmr_max = cpu_to_le16(tmr); 6557 6558 /* min timer set to 1/2 of interrupt timer */ 6559 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6560 val = tmr / 2; 6561 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6562 req->int_lat_tmr_min = cpu_to_le16(val); 6563 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6564 } 6565 6566 /* buf timer set to 1/4 of interrupt timer */ 6567 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6568 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6569 6570 if (cmpl_params & 6571 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6572 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6573 val = clamp_t(u16, tmr, 1, 6574 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6575 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6576 req->enables |= 6577 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6578 } 6579 6580 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6581 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6582 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6583 req->flags = cpu_to_le16(flags); 6584 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6585 } 6586 6587 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6588 struct bnxt_coal *hw_coal) 6589 { 6590 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 6591 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6592 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6593 u32 nq_params = coal_cap->nq_params; 6594 u16 tmr; 6595 int rc; 6596 6597 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6598 return 0; 6599 6600 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6601 if (rc) 6602 return rc; 6603 6604 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6605 req->flags = 6606 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6607 6608 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6609 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6610 req->int_lat_tmr_min = cpu_to_le16(tmr); 6611 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6612 return hwrm_req_send(bp, req); 6613 } 6614 6615 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6616 { 6617 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 6618 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6619 struct bnxt_coal coal; 6620 int rc; 6621 6622 /* Tick values in micro seconds. 6623 * 1 coal_buf x bufs_per_record = 1 completion record. 6624 */ 6625 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6626 6627 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6628 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6629 6630 if (!bnapi->rx_ring) 6631 return -ENODEV; 6632 6633 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6634 if (rc) 6635 return rc; 6636 6637 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 6638 6639 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6640 6641 return hwrm_req_send(bp, req_rx); 6642 } 6643 6644 int bnxt_hwrm_set_coal(struct bnxt *bp) 6645 { 6646 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx, 6647 *req; 6648 int i, rc; 6649 6650 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6651 if (rc) 6652 return rc; 6653 6654 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 6655 if (rc) { 6656 hwrm_req_drop(bp, req_rx); 6657 return rc; 6658 } 6659 6660 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 6661 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 6662 6663 hwrm_req_hold(bp, req_rx); 6664 hwrm_req_hold(bp, req_tx); 6665 for (i = 0; i < bp->cp_nr_rings; i++) { 6666 struct bnxt_napi *bnapi = bp->bnapi[i]; 6667 struct bnxt_coal *hw_coal; 6668 u16 ring_id; 6669 6670 req = req_rx; 6671 if (!bnapi->rx_ring) { 6672 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6673 req = req_tx; 6674 } else { 6675 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6676 } 6677 req->ring_id = cpu_to_le16(ring_id); 6678 6679 rc = hwrm_req_send(bp, req); 6680 if (rc) 6681 break; 6682 6683 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6684 continue; 6685 6686 if (bnapi->rx_ring && bnapi->tx_ring) { 6687 req = req_tx; 6688 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6689 req->ring_id = cpu_to_le16(ring_id); 6690 rc = hwrm_req_send(bp, req); 6691 if (rc) 6692 break; 6693 } 6694 if (bnapi->rx_ring) 6695 hw_coal = &bp->rx_coal; 6696 else 6697 hw_coal = &bp->tx_coal; 6698 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6699 } 6700 hwrm_req_drop(bp, req_rx); 6701 hwrm_req_drop(bp, req_tx); 6702 return rc; 6703 } 6704 6705 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6706 { 6707 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 6708 struct hwrm_stat_ctx_free_input *req; 6709 int i; 6710 6711 if (!bp->bnapi) 6712 return; 6713 6714 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6715 return; 6716 6717 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 6718 return; 6719 if (BNXT_FW_MAJ(bp) <= 20) { 6720 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 6721 hwrm_req_drop(bp, req); 6722 return; 6723 } 6724 hwrm_req_hold(bp, req0); 6725 } 6726 hwrm_req_hold(bp, req); 6727 for (i = 0; i < bp->cp_nr_rings; i++) { 6728 struct bnxt_napi *bnapi = bp->bnapi[i]; 6729 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6730 6731 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6732 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6733 if (req0) { 6734 req0->stat_ctx_id = req->stat_ctx_id; 6735 hwrm_req_send(bp, req0); 6736 } 6737 hwrm_req_send(bp, req); 6738 6739 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6740 } 6741 } 6742 hwrm_req_drop(bp, req); 6743 if (req0) 6744 hwrm_req_drop(bp, req0); 6745 } 6746 6747 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6748 { 6749 struct hwrm_stat_ctx_alloc_output *resp; 6750 struct hwrm_stat_ctx_alloc_input *req; 6751 int rc, i; 6752 6753 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6754 return 0; 6755 6756 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 6757 if (rc) 6758 return rc; 6759 6760 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6761 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6762 6763 resp = hwrm_req_hold(bp, req); 6764 for (i = 0; i < bp->cp_nr_rings; i++) { 6765 struct bnxt_napi *bnapi = bp->bnapi[i]; 6766 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6767 6768 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 6769 6770 rc = hwrm_req_send(bp, req); 6771 if (rc) 6772 break; 6773 6774 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6775 6776 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6777 } 6778 hwrm_req_drop(bp, req); 6779 return rc; 6780 } 6781 6782 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6783 { 6784 struct hwrm_func_qcfg_output *resp; 6785 struct hwrm_func_qcfg_input *req; 6786 u32 min_db_offset = 0; 6787 u16 flags; 6788 int rc; 6789 6790 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 6791 if (rc) 6792 return rc; 6793 6794 req->fid = cpu_to_le16(0xffff); 6795 resp = hwrm_req_hold(bp, req); 6796 rc = hwrm_req_send(bp, req); 6797 if (rc) 6798 goto func_qcfg_exit; 6799 6800 #ifdef CONFIG_BNXT_SRIOV 6801 if (BNXT_VF(bp)) { 6802 struct bnxt_vf_info *vf = &bp->vf; 6803 6804 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6805 } else { 6806 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6807 } 6808 #endif 6809 flags = le16_to_cpu(resp->flags); 6810 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6811 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6812 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6813 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6814 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6815 } 6816 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6817 bp->flags |= BNXT_FLAG_MULTI_HOST; 6818 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 6819 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 6820 6821 switch (resp->port_partition_type) { 6822 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6823 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6824 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6825 bp->port_partition_type = resp->port_partition_type; 6826 break; 6827 } 6828 if (bp->hwrm_spec_code < 0x10707 || 6829 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6830 bp->br_mode = BRIDGE_MODE_VEB; 6831 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6832 bp->br_mode = BRIDGE_MODE_VEPA; 6833 else 6834 bp->br_mode = BRIDGE_MODE_UNDEF; 6835 6836 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6837 if (!bp->max_mtu) 6838 bp->max_mtu = BNXT_MAX_MTU; 6839 6840 if (bp->db_size) 6841 goto func_qcfg_exit; 6842 6843 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6844 if (BNXT_PF(bp)) 6845 min_db_offset = DB_PF_OFFSET_P5; 6846 else 6847 min_db_offset = DB_VF_OFFSET_P5; 6848 } 6849 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6850 1024); 6851 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6852 bp->db_size <= min_db_offset) 6853 bp->db_size = pci_resource_len(bp->pdev, 2); 6854 6855 func_qcfg_exit: 6856 hwrm_req_drop(bp, req); 6857 return rc; 6858 } 6859 6860 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx, 6861 struct hwrm_func_backing_store_qcaps_output *resp) 6862 { 6863 struct bnxt_mem_init *mem_init; 6864 u16 init_mask; 6865 u8 init_val; 6866 u8 *offset; 6867 int i; 6868 6869 init_val = resp->ctx_kind_initializer; 6870 init_mask = le16_to_cpu(resp->ctx_init_mask); 6871 offset = &resp->qp_init_offset; 6872 mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 6873 for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) { 6874 mem_init->init_val = init_val; 6875 mem_init->offset = BNXT_MEM_INVALID_OFFSET; 6876 if (!init_mask) 6877 continue; 6878 if (i == BNXT_CTX_MEM_INIT_STAT) 6879 offset = &resp->stat_init_offset; 6880 if (init_mask & (1 << i)) 6881 mem_init->offset = *offset * 4; 6882 else 6883 mem_init->init_val = 0; 6884 } 6885 ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size; 6886 ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size; 6887 ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size; 6888 ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size; 6889 ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size; 6890 ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size; 6891 } 6892 6893 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6894 { 6895 struct hwrm_func_backing_store_qcaps_output *resp; 6896 struct hwrm_func_backing_store_qcaps_input *req; 6897 int rc; 6898 6899 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6900 return 0; 6901 6902 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 6903 if (rc) 6904 return rc; 6905 6906 resp = hwrm_req_hold(bp, req); 6907 rc = hwrm_req_send_silent(bp, req); 6908 if (!rc) { 6909 struct bnxt_ctx_pg_info *ctx_pg; 6910 struct bnxt_ctx_mem_info *ctx; 6911 int i, tqm_rings; 6912 6913 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6914 if (!ctx) { 6915 rc = -ENOMEM; 6916 goto ctx_err; 6917 } 6918 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6919 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6920 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6921 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6922 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6923 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6924 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6925 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6926 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6927 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6928 ctx->vnic_max_vnic_entries = 6929 le16_to_cpu(resp->vnic_max_vnic_entries); 6930 ctx->vnic_max_ring_table_entries = 6931 le16_to_cpu(resp->vnic_max_ring_table_entries); 6932 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6933 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6934 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6935 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6936 ctx->tqm_min_entries_per_ring = 6937 le32_to_cpu(resp->tqm_min_entries_per_ring); 6938 ctx->tqm_max_entries_per_ring = 6939 le32_to_cpu(resp->tqm_max_entries_per_ring); 6940 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6941 if (!ctx->tqm_entries_multiple) 6942 ctx->tqm_entries_multiple = 1; 6943 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6944 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6945 ctx->mrav_num_entries_units = 6946 le16_to_cpu(resp->mrav_num_entries_units); 6947 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6948 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6949 6950 bnxt_init_ctx_initializer(ctx, resp); 6951 6952 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6953 if (!ctx->tqm_fp_rings_count) 6954 ctx->tqm_fp_rings_count = bp->max_q; 6955 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 6956 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 6957 6958 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS; 6959 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6960 if (!ctx_pg) { 6961 kfree(ctx); 6962 rc = -ENOMEM; 6963 goto ctx_err; 6964 } 6965 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6966 ctx->tqm_mem[i] = ctx_pg; 6967 bp->ctx = ctx; 6968 } else { 6969 rc = 0; 6970 } 6971 ctx_err: 6972 hwrm_req_drop(bp, req); 6973 return rc; 6974 } 6975 6976 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6977 __le64 *pg_dir) 6978 { 6979 if (!rmem->nr_pages) 6980 return; 6981 6982 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 6983 if (rmem->depth >= 1) { 6984 if (rmem->depth == 2) 6985 *pg_attr |= 2; 6986 else 6987 *pg_attr |= 1; 6988 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6989 } else { 6990 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6991 } 6992 } 6993 6994 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6995 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6996 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6997 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6998 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6999 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 7000 7001 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 7002 { 7003 struct hwrm_func_backing_store_cfg_input *req; 7004 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7005 struct bnxt_ctx_pg_info *ctx_pg; 7006 void **__req = (void **)&req; 7007 u32 req_len = sizeof(*req); 7008 __le32 *num_entries; 7009 __le64 *pg_dir; 7010 u32 flags = 0; 7011 u8 *pg_attr; 7012 u32 ena; 7013 int rc; 7014 int i; 7015 7016 if (!ctx) 7017 return 0; 7018 7019 if (req_len > bp->hwrm_max_ext_req_len) 7020 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 7021 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 7022 if (rc) 7023 return rc; 7024 7025 req->enables = cpu_to_le32(enables); 7026 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 7027 ctx_pg = &ctx->qp_mem; 7028 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 7029 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 7030 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 7031 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 7032 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7033 &req->qpc_pg_size_qpc_lvl, 7034 &req->qpc_page_dir); 7035 } 7036 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 7037 ctx_pg = &ctx->srq_mem; 7038 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 7039 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 7040 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 7041 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7042 &req->srq_pg_size_srq_lvl, 7043 &req->srq_page_dir); 7044 } 7045 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 7046 ctx_pg = &ctx->cq_mem; 7047 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 7048 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 7049 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 7050 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7051 &req->cq_pg_size_cq_lvl, 7052 &req->cq_page_dir); 7053 } 7054 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 7055 ctx_pg = &ctx->vnic_mem; 7056 req->vnic_num_vnic_entries = 7057 cpu_to_le16(ctx->vnic_max_vnic_entries); 7058 req->vnic_num_ring_table_entries = 7059 cpu_to_le16(ctx->vnic_max_ring_table_entries); 7060 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 7061 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7062 &req->vnic_pg_size_vnic_lvl, 7063 &req->vnic_page_dir); 7064 } 7065 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 7066 ctx_pg = &ctx->stat_mem; 7067 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 7068 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 7069 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7070 &req->stat_pg_size_stat_lvl, 7071 &req->stat_page_dir); 7072 } 7073 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 7074 ctx_pg = &ctx->mrav_mem; 7075 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 7076 if (ctx->mrav_num_entries_units) 7077 flags |= 7078 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 7079 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 7080 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7081 &req->mrav_pg_size_mrav_lvl, 7082 &req->mrav_page_dir); 7083 } 7084 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 7085 ctx_pg = &ctx->tim_mem; 7086 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 7087 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 7088 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 7089 &req->tim_pg_size_tim_lvl, 7090 &req->tim_page_dir); 7091 } 7092 for (i = 0, num_entries = &req->tqm_sp_num_entries, 7093 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 7094 pg_dir = &req->tqm_sp_page_dir, 7095 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 7096 i < BNXT_MAX_TQM_RINGS; 7097 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 7098 if (!(enables & ena)) 7099 continue; 7100 7101 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 7102 ctx_pg = ctx->tqm_mem[i]; 7103 *num_entries = cpu_to_le32(ctx_pg->entries); 7104 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 7105 } 7106 req->flags = cpu_to_le32(flags); 7107 return hwrm_req_send(bp, req); 7108 } 7109 7110 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 7111 struct bnxt_ctx_pg_info *ctx_pg) 7112 { 7113 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7114 7115 rmem->page_size = BNXT_PAGE_SIZE; 7116 rmem->pg_arr = ctx_pg->ctx_pg_arr; 7117 rmem->dma_arr = ctx_pg->ctx_dma_arr; 7118 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 7119 if (rmem->depth >= 1) 7120 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 7121 return bnxt_alloc_ring(bp, rmem); 7122 } 7123 7124 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 7125 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 7126 u8 depth, struct bnxt_mem_init *mem_init) 7127 { 7128 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7129 int rc; 7130 7131 if (!mem_size) 7132 return -EINVAL; 7133 7134 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7135 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 7136 ctx_pg->nr_pages = 0; 7137 return -EINVAL; 7138 } 7139 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 7140 int nr_tbls, i; 7141 7142 rmem->depth = 2; 7143 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 7144 GFP_KERNEL); 7145 if (!ctx_pg->ctx_pg_tbl) 7146 return -ENOMEM; 7147 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 7148 rmem->nr_pages = nr_tbls; 7149 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7150 if (rc) 7151 return rc; 7152 for (i = 0; i < nr_tbls; i++) { 7153 struct bnxt_ctx_pg_info *pg_tbl; 7154 7155 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 7156 if (!pg_tbl) 7157 return -ENOMEM; 7158 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 7159 rmem = &pg_tbl->ring_mem; 7160 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 7161 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 7162 rmem->depth = 1; 7163 rmem->nr_pages = MAX_CTX_PAGES; 7164 rmem->mem_init = mem_init; 7165 if (i == (nr_tbls - 1)) { 7166 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 7167 7168 if (rem) 7169 rmem->nr_pages = rem; 7170 } 7171 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 7172 if (rc) 7173 break; 7174 } 7175 } else { 7176 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 7177 if (rmem->nr_pages > 1 || depth) 7178 rmem->depth = 1; 7179 rmem->mem_init = mem_init; 7180 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 7181 } 7182 return rc; 7183 } 7184 7185 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 7186 struct bnxt_ctx_pg_info *ctx_pg) 7187 { 7188 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 7189 7190 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 7191 ctx_pg->ctx_pg_tbl) { 7192 int i, nr_tbls = rmem->nr_pages; 7193 7194 for (i = 0; i < nr_tbls; i++) { 7195 struct bnxt_ctx_pg_info *pg_tbl; 7196 struct bnxt_ring_mem_info *rmem2; 7197 7198 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 7199 if (!pg_tbl) 7200 continue; 7201 rmem2 = &pg_tbl->ring_mem; 7202 bnxt_free_ring(bp, rmem2); 7203 ctx_pg->ctx_pg_arr[i] = NULL; 7204 kfree(pg_tbl); 7205 ctx_pg->ctx_pg_tbl[i] = NULL; 7206 } 7207 kfree(ctx_pg->ctx_pg_tbl); 7208 ctx_pg->ctx_pg_tbl = NULL; 7209 } 7210 bnxt_free_ring(bp, rmem); 7211 ctx_pg->nr_pages = 0; 7212 } 7213 7214 void bnxt_free_ctx_mem(struct bnxt *bp) 7215 { 7216 struct bnxt_ctx_mem_info *ctx = bp->ctx; 7217 int i; 7218 7219 if (!ctx) 7220 return; 7221 7222 if (ctx->tqm_mem[0]) { 7223 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 7224 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 7225 kfree(ctx->tqm_mem[0]); 7226 ctx->tqm_mem[0] = NULL; 7227 } 7228 7229 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 7230 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 7231 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 7232 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 7233 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 7234 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 7235 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 7236 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 7237 } 7238 7239 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 7240 { 7241 struct bnxt_ctx_pg_info *ctx_pg; 7242 struct bnxt_ctx_mem_info *ctx; 7243 struct bnxt_mem_init *init; 7244 u32 mem_size, ena, entries; 7245 u32 entries_sp, min; 7246 u32 num_mr, num_ah; 7247 u32 extra_srqs = 0; 7248 u32 extra_qps = 0; 7249 u8 pg_lvl = 1; 7250 int i, rc; 7251 7252 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 7253 if (rc) { 7254 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 7255 rc); 7256 return rc; 7257 } 7258 ctx = bp->ctx; 7259 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 7260 return 0; 7261 7262 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 7263 pg_lvl = 2; 7264 extra_qps = 65536; 7265 extra_srqs = 8192; 7266 } 7267 7268 ctx_pg = &ctx->qp_mem; 7269 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 7270 extra_qps; 7271 if (ctx->qp_entry_size) { 7272 mem_size = ctx->qp_entry_size * ctx_pg->entries; 7273 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP]; 7274 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7275 if (rc) 7276 return rc; 7277 } 7278 7279 ctx_pg = &ctx->srq_mem; 7280 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 7281 if (ctx->srq_entry_size) { 7282 mem_size = ctx->srq_entry_size * ctx_pg->entries; 7283 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ]; 7284 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7285 if (rc) 7286 return rc; 7287 } 7288 7289 ctx_pg = &ctx->cq_mem; 7290 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 7291 if (ctx->cq_entry_size) { 7292 mem_size = ctx->cq_entry_size * ctx_pg->entries; 7293 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ]; 7294 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init); 7295 if (rc) 7296 return rc; 7297 } 7298 7299 ctx_pg = &ctx->vnic_mem; 7300 ctx_pg->entries = ctx->vnic_max_vnic_entries + 7301 ctx->vnic_max_ring_table_entries; 7302 if (ctx->vnic_entry_size) { 7303 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 7304 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC]; 7305 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7306 if (rc) 7307 return rc; 7308 } 7309 7310 ctx_pg = &ctx->stat_mem; 7311 ctx_pg->entries = ctx->stat_max_entries; 7312 if (ctx->stat_entry_size) { 7313 mem_size = ctx->stat_entry_size * ctx_pg->entries; 7314 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT]; 7315 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init); 7316 if (rc) 7317 return rc; 7318 } 7319 7320 ena = 0; 7321 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 7322 goto skip_rdma; 7323 7324 ctx_pg = &ctx->mrav_mem; 7325 /* 128K extra is needed to accommodate static AH context 7326 * allocation by f/w. 7327 */ 7328 num_mr = 1024 * 256; 7329 num_ah = 1024 * 128; 7330 ctx_pg->entries = num_mr + num_ah; 7331 if (ctx->mrav_entry_size) { 7332 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 7333 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV]; 7334 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init); 7335 if (rc) 7336 return rc; 7337 } 7338 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 7339 if (ctx->mrav_num_entries_units) 7340 ctx_pg->entries = 7341 ((num_mr / ctx->mrav_num_entries_units) << 16) | 7342 (num_ah / ctx->mrav_num_entries_units); 7343 7344 ctx_pg = &ctx->tim_mem; 7345 ctx_pg->entries = ctx->qp_mem.entries; 7346 if (ctx->tim_entry_size) { 7347 mem_size = ctx->tim_entry_size * ctx_pg->entries; 7348 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL); 7349 if (rc) 7350 return rc; 7351 } 7352 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 7353 7354 skip_rdma: 7355 min = ctx->tqm_min_entries_per_ring; 7356 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 7357 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 7358 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 7359 entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries); 7360 entries = roundup(entries, ctx->tqm_entries_multiple); 7361 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 7362 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 7363 ctx_pg = ctx->tqm_mem[i]; 7364 ctx_pg->entries = i ? entries : entries_sp; 7365 if (ctx->tqm_entry_size) { 7366 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 7367 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, 7368 NULL); 7369 if (rc) 7370 return rc; 7371 } 7372 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 7373 } 7374 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 7375 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 7376 if (rc) { 7377 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7378 rc); 7379 return rc; 7380 } 7381 ctx->flags |= BNXT_CTX_FLAG_INITED; 7382 return 0; 7383 } 7384 7385 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7386 { 7387 struct hwrm_func_resource_qcaps_output *resp; 7388 struct hwrm_func_resource_qcaps_input *req; 7389 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7390 int rc; 7391 7392 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 7393 if (rc) 7394 return rc; 7395 7396 req->fid = cpu_to_le16(0xffff); 7397 resp = hwrm_req_hold(bp, req); 7398 rc = hwrm_req_send_silent(bp, req); 7399 if (rc) 7400 goto hwrm_func_resc_qcaps_exit; 7401 7402 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7403 if (!all) 7404 goto hwrm_func_resc_qcaps_exit; 7405 7406 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7407 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7408 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7409 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7410 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7411 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7412 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7413 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7414 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7415 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7416 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7417 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7418 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7419 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7420 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7421 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7422 7423 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7424 u16 max_msix = le16_to_cpu(resp->max_msix); 7425 7426 hw_resc->max_nqs = max_msix; 7427 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7428 } 7429 7430 if (BNXT_PF(bp)) { 7431 struct bnxt_pf_info *pf = &bp->pf; 7432 7433 pf->vf_resv_strategy = 7434 le16_to_cpu(resp->vf_reservation_strategy); 7435 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7436 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7437 } 7438 hwrm_func_resc_qcaps_exit: 7439 hwrm_req_drop(bp, req); 7440 return rc; 7441 } 7442 7443 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 7444 { 7445 struct hwrm_port_mac_ptp_qcfg_output *resp; 7446 struct hwrm_port_mac_ptp_qcfg_input *req; 7447 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 7448 bool phc_cfg; 7449 u8 flags; 7450 int rc; 7451 7452 if (bp->hwrm_spec_code < 0x10801) { 7453 rc = -ENODEV; 7454 goto no_ptp; 7455 } 7456 7457 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 7458 if (rc) 7459 goto no_ptp; 7460 7461 req->port_id = cpu_to_le16(bp->pf.port_id); 7462 resp = hwrm_req_hold(bp, req); 7463 rc = hwrm_req_send(bp, req); 7464 if (rc) 7465 goto exit; 7466 7467 flags = resp->flags; 7468 if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 7469 rc = -ENODEV; 7470 goto exit; 7471 } 7472 if (!ptp) { 7473 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 7474 if (!ptp) { 7475 rc = -ENOMEM; 7476 goto exit; 7477 } 7478 ptp->bp = bp; 7479 bp->ptp_cfg = ptp; 7480 } 7481 if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { 7482 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 7483 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 7484 } else if (bp->flags & BNXT_FLAG_CHIP_P5) { 7485 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 7486 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 7487 } else { 7488 rc = -ENODEV; 7489 goto exit; 7490 } 7491 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; 7492 rc = bnxt_ptp_init(bp, phc_cfg); 7493 if (rc) 7494 netdev_warn(bp->dev, "PTP initialization failed.\n"); 7495 exit: 7496 hwrm_req_drop(bp, req); 7497 if (!rc) 7498 return 0; 7499 7500 no_ptp: 7501 bnxt_ptp_clear(bp); 7502 kfree(ptp); 7503 bp->ptp_cfg = NULL; 7504 return rc; 7505 } 7506 7507 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7508 { 7509 struct hwrm_func_qcaps_output *resp; 7510 struct hwrm_func_qcaps_input *req; 7511 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7512 u32 flags, flags_ext; 7513 int rc; 7514 7515 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 7516 if (rc) 7517 return rc; 7518 7519 req->fid = cpu_to_le16(0xffff); 7520 resp = hwrm_req_hold(bp, req); 7521 rc = hwrm_req_send(bp, req); 7522 if (rc) 7523 goto hwrm_func_qcaps_exit; 7524 7525 flags = le32_to_cpu(resp->flags); 7526 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7527 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7528 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7529 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7530 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7531 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7532 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7533 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7534 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7535 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7536 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7537 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7538 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7539 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7540 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7541 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7542 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 7543 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 7544 7545 flags_ext = le32_to_cpu(resp->flags_ext); 7546 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7547 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7548 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 7549 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 7550 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) 7551 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; 7552 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 7553 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 7554 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 7555 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 7556 7557 bp->tx_push_thresh = 0; 7558 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7559 BNXT_FW_MAJ(bp) > 217) 7560 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7561 7562 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7563 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7564 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7565 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7566 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7567 if (!hw_resc->max_hw_ring_grps) 7568 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7569 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7570 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7571 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7572 7573 if (BNXT_PF(bp)) { 7574 struct bnxt_pf_info *pf = &bp->pf; 7575 7576 pf->fw_fid = le16_to_cpu(resp->fid); 7577 pf->port_id = le16_to_cpu(resp->port_id); 7578 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7579 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7580 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7581 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7582 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7583 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7584 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7585 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7586 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7587 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7588 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7589 bp->flags |= BNXT_FLAG_WOL_CAP; 7590 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 7591 __bnxt_hwrm_ptp_qcfg(bp); 7592 } else { 7593 bnxt_ptp_clear(bp); 7594 kfree(bp->ptp_cfg); 7595 bp->ptp_cfg = NULL; 7596 } 7597 } else { 7598 #ifdef CONFIG_BNXT_SRIOV 7599 struct bnxt_vf_info *vf = &bp->vf; 7600 7601 vf->fw_fid = le16_to_cpu(resp->fid); 7602 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7603 #endif 7604 } 7605 7606 hwrm_func_qcaps_exit: 7607 hwrm_req_drop(bp, req); 7608 return rc; 7609 } 7610 7611 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 7612 { 7613 struct hwrm_dbg_qcaps_output *resp; 7614 struct hwrm_dbg_qcaps_input *req; 7615 int rc; 7616 7617 bp->fw_dbg_cap = 0; 7618 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 7619 return; 7620 7621 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 7622 if (rc) 7623 return; 7624 7625 req->fid = cpu_to_le16(0xffff); 7626 resp = hwrm_req_hold(bp, req); 7627 rc = hwrm_req_send(bp, req); 7628 if (rc) 7629 goto hwrm_dbg_qcaps_exit; 7630 7631 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 7632 7633 hwrm_dbg_qcaps_exit: 7634 hwrm_req_drop(bp, req); 7635 } 7636 7637 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7638 7639 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7640 { 7641 int rc; 7642 7643 rc = __bnxt_hwrm_func_qcaps(bp); 7644 if (rc) 7645 return rc; 7646 7647 bnxt_hwrm_dbg_qcaps(bp); 7648 7649 rc = bnxt_hwrm_queue_qportcfg(bp); 7650 if (rc) { 7651 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7652 return rc; 7653 } 7654 if (bp->hwrm_spec_code >= 0x10803) { 7655 rc = bnxt_alloc_ctx_mem(bp); 7656 if (rc) 7657 return rc; 7658 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7659 if (!rc) 7660 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7661 } 7662 return 0; 7663 } 7664 7665 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7666 { 7667 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7668 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 7669 u32 flags; 7670 int rc; 7671 7672 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7673 return 0; 7674 7675 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 7676 if (rc) 7677 return rc; 7678 7679 resp = hwrm_req_hold(bp, req); 7680 rc = hwrm_req_send(bp, req); 7681 if (rc) 7682 goto hwrm_cfa_adv_qcaps_exit; 7683 7684 flags = le32_to_cpu(resp->flags); 7685 if (flags & 7686 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7687 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7688 7689 hwrm_cfa_adv_qcaps_exit: 7690 hwrm_req_drop(bp, req); 7691 return rc; 7692 } 7693 7694 static int __bnxt_alloc_fw_health(struct bnxt *bp) 7695 { 7696 if (bp->fw_health) 7697 return 0; 7698 7699 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 7700 if (!bp->fw_health) 7701 return -ENOMEM; 7702 7703 mutex_init(&bp->fw_health->lock); 7704 return 0; 7705 } 7706 7707 static int bnxt_alloc_fw_health(struct bnxt *bp) 7708 { 7709 int rc; 7710 7711 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 7712 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7713 return 0; 7714 7715 rc = __bnxt_alloc_fw_health(bp); 7716 if (rc) { 7717 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 7718 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7719 return rc; 7720 } 7721 7722 return 0; 7723 } 7724 7725 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 7726 { 7727 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 7728 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7729 BNXT_FW_HEALTH_WIN_MAP_OFF); 7730 } 7731 7732 static void bnxt_inv_fw_health_reg(struct bnxt *bp) 7733 { 7734 struct bnxt_fw_health *fw_health = bp->fw_health; 7735 u32 reg_type; 7736 7737 if (!fw_health) 7738 return; 7739 7740 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 7741 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7742 fw_health->status_reliable = false; 7743 7744 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 7745 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 7746 fw_health->resets_reliable = false; 7747 } 7748 7749 static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 7750 { 7751 void __iomem *hs; 7752 u32 status_loc; 7753 u32 reg_type; 7754 u32 sig; 7755 7756 if (bp->fw_health) 7757 bp->fw_health->status_reliable = false; 7758 7759 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 7760 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 7761 7762 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 7763 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 7764 if (!bp->chip_num) { 7765 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 7766 bp->chip_num = readl(bp->bar0 + 7767 BNXT_FW_HEALTH_WIN_BASE + 7768 BNXT_GRC_REG_CHIP_NUM); 7769 } 7770 if (!BNXT_CHIP_P5(bp)) 7771 return; 7772 7773 status_loc = BNXT_GRC_REG_STATUS_P5 | 7774 BNXT_FW_HEALTH_REG_TYPE_BAR0; 7775 } else { 7776 status_loc = readl(hs + offsetof(struct hcomm_status, 7777 fw_status_loc)); 7778 } 7779 7780 if (__bnxt_alloc_fw_health(bp)) { 7781 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 7782 return; 7783 } 7784 7785 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 7786 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 7787 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 7788 __bnxt_map_fw_health_reg(bp, status_loc); 7789 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 7790 BNXT_FW_HEALTH_WIN_OFF(status_loc); 7791 } 7792 7793 bp->fw_health->status_reliable = true; 7794 } 7795 7796 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7797 { 7798 struct bnxt_fw_health *fw_health = bp->fw_health; 7799 u32 reg_base = 0xffffffff; 7800 int i; 7801 7802 bp->fw_health->status_reliable = false; 7803 bp->fw_health->resets_reliable = false; 7804 /* Only pre-map the monitoring GRC registers using window 3 */ 7805 for (i = 0; i < 4; i++) { 7806 u32 reg = fw_health->regs[i]; 7807 7808 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7809 continue; 7810 if (reg_base == 0xffffffff) 7811 reg_base = reg & BNXT_GRC_BASE_MASK; 7812 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7813 return -ERANGE; 7814 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 7815 } 7816 bp->fw_health->status_reliable = true; 7817 bp->fw_health->resets_reliable = true; 7818 if (reg_base == 0xffffffff) 7819 return 0; 7820 7821 __bnxt_map_fw_health_reg(bp, reg_base); 7822 return 0; 7823 } 7824 7825 static void bnxt_remap_fw_health_regs(struct bnxt *bp) 7826 { 7827 if (!bp->fw_health) 7828 return; 7829 7830 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 7831 bp->fw_health->status_reliable = true; 7832 bp->fw_health->resets_reliable = true; 7833 } else { 7834 bnxt_try_map_fw_health_reg(bp); 7835 } 7836 } 7837 7838 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7839 { 7840 struct bnxt_fw_health *fw_health = bp->fw_health; 7841 struct hwrm_error_recovery_qcfg_output *resp; 7842 struct hwrm_error_recovery_qcfg_input *req; 7843 int rc, i; 7844 7845 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7846 return 0; 7847 7848 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 7849 if (rc) 7850 return rc; 7851 7852 resp = hwrm_req_hold(bp, req); 7853 rc = hwrm_req_send(bp, req); 7854 if (rc) 7855 goto err_recovery_out; 7856 fw_health->flags = le32_to_cpu(resp->flags); 7857 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7858 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7859 rc = -EINVAL; 7860 goto err_recovery_out; 7861 } 7862 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7863 fw_health->master_func_wait_dsecs = 7864 le32_to_cpu(resp->master_func_wait_period); 7865 fw_health->normal_func_wait_dsecs = 7866 le32_to_cpu(resp->normal_func_wait_period); 7867 fw_health->post_reset_wait_dsecs = 7868 le32_to_cpu(resp->master_func_wait_period_after_reset); 7869 fw_health->post_reset_max_wait_dsecs = 7870 le32_to_cpu(resp->max_bailout_time_after_reset); 7871 fw_health->regs[BNXT_FW_HEALTH_REG] = 7872 le32_to_cpu(resp->fw_health_status_reg); 7873 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7874 le32_to_cpu(resp->fw_heartbeat_reg); 7875 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7876 le32_to_cpu(resp->fw_reset_cnt_reg); 7877 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7878 le32_to_cpu(resp->reset_inprogress_reg); 7879 fw_health->fw_reset_inprog_reg_mask = 7880 le32_to_cpu(resp->reset_inprogress_reg_mask); 7881 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7882 if (fw_health->fw_reset_seq_cnt >= 16) { 7883 rc = -EINVAL; 7884 goto err_recovery_out; 7885 } 7886 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7887 fw_health->fw_reset_seq_regs[i] = 7888 le32_to_cpu(resp->reset_reg[i]); 7889 fw_health->fw_reset_seq_vals[i] = 7890 le32_to_cpu(resp->reset_reg_val[i]); 7891 fw_health->fw_reset_seq_delay_msec[i] = 7892 resp->delay_after_reset[i]; 7893 } 7894 err_recovery_out: 7895 hwrm_req_drop(bp, req); 7896 if (!rc) 7897 rc = bnxt_map_fw_health_regs(bp); 7898 if (rc) 7899 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7900 return rc; 7901 } 7902 7903 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7904 { 7905 struct hwrm_func_reset_input *req; 7906 int rc; 7907 7908 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 7909 if (rc) 7910 return rc; 7911 7912 req->enables = 0; 7913 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 7914 return hwrm_req_send(bp, req); 7915 } 7916 7917 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 7918 { 7919 struct hwrm_nvm_get_dev_info_output nvm_info; 7920 7921 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 7922 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 7923 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 7924 nvm_info.nvm_cfg_ver_upd); 7925 } 7926 7927 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7928 { 7929 struct hwrm_queue_qportcfg_output *resp; 7930 struct hwrm_queue_qportcfg_input *req; 7931 u8 i, j, *qptr; 7932 bool no_rdma; 7933 int rc = 0; 7934 7935 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 7936 if (rc) 7937 return rc; 7938 7939 resp = hwrm_req_hold(bp, req); 7940 rc = hwrm_req_send(bp, req); 7941 if (rc) 7942 goto qportcfg_exit; 7943 7944 if (!resp->max_configurable_queues) { 7945 rc = -EINVAL; 7946 goto qportcfg_exit; 7947 } 7948 bp->max_tc = resp->max_configurable_queues; 7949 bp->max_lltc = resp->max_configurable_lossless_queues; 7950 if (bp->max_tc > BNXT_MAX_QUEUE) 7951 bp->max_tc = BNXT_MAX_QUEUE; 7952 7953 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7954 qptr = &resp->queue_id0; 7955 for (i = 0, j = 0; i < bp->max_tc; i++) { 7956 bp->q_info[j].queue_id = *qptr; 7957 bp->q_ids[i] = *qptr++; 7958 bp->q_info[j].queue_profile = *qptr++; 7959 bp->tc_to_qidx[j] = j; 7960 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7961 (no_rdma && BNXT_PF(bp))) 7962 j++; 7963 } 7964 bp->max_q = bp->max_tc; 7965 bp->max_tc = max_t(u8, j, 1); 7966 7967 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7968 bp->max_tc = 1; 7969 7970 if (bp->max_lltc > bp->max_tc) 7971 bp->max_lltc = bp->max_tc; 7972 7973 qportcfg_exit: 7974 hwrm_req_drop(bp, req); 7975 return rc; 7976 } 7977 7978 static int bnxt_hwrm_poll(struct bnxt *bp) 7979 { 7980 struct hwrm_ver_get_input *req; 7981 int rc; 7982 7983 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 7984 if (rc) 7985 return rc; 7986 7987 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 7988 req->hwrm_intf_min = HWRM_VERSION_MINOR; 7989 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 7990 7991 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 7992 rc = hwrm_req_send(bp, req); 7993 return rc; 7994 } 7995 7996 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7997 { 7998 struct hwrm_ver_get_output *resp; 7999 struct hwrm_ver_get_input *req; 8000 u16 fw_maj, fw_min, fw_bld, fw_rsv; 8001 u32 dev_caps_cfg, hwrm_ver; 8002 int rc, len; 8003 8004 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 8005 if (rc) 8006 return rc; 8007 8008 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 8009 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 8010 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 8011 req->hwrm_intf_min = HWRM_VERSION_MINOR; 8012 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 8013 8014 resp = hwrm_req_hold(bp, req); 8015 rc = hwrm_req_send(bp, req); 8016 if (rc) 8017 goto hwrm_ver_get_exit; 8018 8019 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 8020 8021 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 8022 resp->hwrm_intf_min_8b << 8 | 8023 resp->hwrm_intf_upd_8b; 8024 if (resp->hwrm_intf_maj_8b < 1) { 8025 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 8026 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8027 resp->hwrm_intf_upd_8b); 8028 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 8029 } 8030 8031 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 8032 HWRM_VERSION_UPDATE; 8033 8034 if (bp->hwrm_spec_code > hwrm_ver) 8035 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8036 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 8037 HWRM_VERSION_UPDATE); 8038 else 8039 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 8040 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 8041 resp->hwrm_intf_upd_8b); 8042 8043 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 8044 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 8045 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 8046 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 8047 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 8048 len = FW_VER_STR_LEN; 8049 } else { 8050 fw_maj = resp->hwrm_fw_maj_8b; 8051 fw_min = resp->hwrm_fw_min_8b; 8052 fw_bld = resp->hwrm_fw_bld_8b; 8053 fw_rsv = resp->hwrm_fw_rsvd_8b; 8054 len = BC_HWRM_STR_LEN; 8055 } 8056 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 8057 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 8058 fw_rsv); 8059 8060 if (strlen(resp->active_pkg_name)) { 8061 int fw_ver_len = strlen(bp->fw_ver_str); 8062 8063 snprintf(bp->fw_ver_str + fw_ver_len, 8064 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 8065 resp->active_pkg_name); 8066 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 8067 } 8068 8069 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 8070 if (!bp->hwrm_cmd_timeout) 8071 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 8072 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 8073 if (!bp->hwrm_cmd_max_timeout) 8074 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 8075 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT) 8076 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n", 8077 bp->hwrm_cmd_max_timeout / 1000); 8078 8079 if (resp->hwrm_intf_maj_8b >= 1) { 8080 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 8081 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 8082 } 8083 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 8084 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 8085 8086 bp->chip_num = le16_to_cpu(resp->chip_num); 8087 bp->chip_rev = resp->chip_rev; 8088 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 8089 !resp->chip_metal) 8090 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 8091 8092 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 8093 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 8094 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 8095 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 8096 8097 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 8098 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 8099 8100 if (dev_caps_cfg & 8101 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 8102 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 8103 8104 if (dev_caps_cfg & 8105 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 8106 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 8107 8108 if (dev_caps_cfg & 8109 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 8110 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 8111 8112 hwrm_ver_get_exit: 8113 hwrm_req_drop(bp, req); 8114 return rc; 8115 } 8116 8117 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 8118 { 8119 struct hwrm_fw_set_time_input *req; 8120 struct tm tm; 8121 time64_t now = ktime_get_real_seconds(); 8122 int rc; 8123 8124 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 8125 bp->hwrm_spec_code < 0x10400) 8126 return -EOPNOTSUPP; 8127 8128 time64_to_tm(now, 0, &tm); 8129 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 8130 if (rc) 8131 return rc; 8132 8133 req->year = cpu_to_le16(1900 + tm.tm_year); 8134 req->month = 1 + tm.tm_mon; 8135 req->day = tm.tm_mday; 8136 req->hour = tm.tm_hour; 8137 req->minute = tm.tm_min; 8138 req->second = tm.tm_sec; 8139 return hwrm_req_send(bp, req); 8140 } 8141 8142 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 8143 { 8144 u64 sw_tmp; 8145 8146 hw &= mask; 8147 sw_tmp = (*sw & ~mask) | hw; 8148 if (hw < (*sw & mask)) 8149 sw_tmp += mask + 1; 8150 WRITE_ONCE(*sw, sw_tmp); 8151 } 8152 8153 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 8154 int count, bool ignore_zero) 8155 { 8156 int i; 8157 8158 for (i = 0; i < count; i++) { 8159 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 8160 8161 if (ignore_zero && !hw) 8162 continue; 8163 8164 if (masks[i] == -1ULL) 8165 sw_stats[i] = hw; 8166 else 8167 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 8168 } 8169 } 8170 8171 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 8172 { 8173 if (!stats->hw_stats) 8174 return; 8175 8176 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8177 stats->hw_masks, stats->len / 8, false); 8178 } 8179 8180 static void bnxt_accumulate_all_stats(struct bnxt *bp) 8181 { 8182 struct bnxt_stats_mem *ring0_stats; 8183 bool ignore_zero = false; 8184 int i; 8185 8186 /* Chip bug. Counter intermittently becomes 0. */ 8187 if (bp->flags & BNXT_FLAG_CHIP_P5) 8188 ignore_zero = true; 8189 8190 for (i = 0; i < bp->cp_nr_rings; i++) { 8191 struct bnxt_napi *bnapi = bp->bnapi[i]; 8192 struct bnxt_cp_ring_info *cpr; 8193 struct bnxt_stats_mem *stats; 8194 8195 cpr = &bnapi->cp_ring; 8196 stats = &cpr->stats; 8197 if (!i) 8198 ring0_stats = stats; 8199 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 8200 ring0_stats->hw_masks, 8201 ring0_stats->len / 8, ignore_zero); 8202 } 8203 if (bp->flags & BNXT_FLAG_PORT_STATS) { 8204 struct bnxt_stats_mem *stats = &bp->port_stats; 8205 __le64 *hw_stats = stats->hw_stats; 8206 u64 *sw_stats = stats->sw_stats; 8207 u64 *masks = stats->hw_masks; 8208 int cnt; 8209 8210 cnt = sizeof(struct rx_port_stats) / 8; 8211 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8212 8213 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8214 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8215 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 8216 cnt = sizeof(struct tx_port_stats) / 8; 8217 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 8218 } 8219 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 8220 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 8221 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 8222 } 8223 } 8224 8225 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 8226 { 8227 struct hwrm_port_qstats_input *req; 8228 struct bnxt_pf_info *pf = &bp->pf; 8229 int rc; 8230 8231 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 8232 return 0; 8233 8234 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8235 return -EOPNOTSUPP; 8236 8237 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 8238 if (rc) 8239 return rc; 8240 8241 req->flags = flags; 8242 req->port_id = cpu_to_le16(pf->port_id); 8243 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 8244 BNXT_TX_PORT_STATS_BYTE_OFFSET); 8245 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 8246 return hwrm_req_send(bp, req); 8247 } 8248 8249 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 8250 { 8251 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 8252 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 8253 struct hwrm_port_qstats_ext_output *resp_qs; 8254 struct hwrm_port_qstats_ext_input *req_qs; 8255 struct bnxt_pf_info *pf = &bp->pf; 8256 u32 tx_stat_size; 8257 int rc; 8258 8259 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 8260 return 0; 8261 8262 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 8263 return -EOPNOTSUPP; 8264 8265 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 8266 if (rc) 8267 return rc; 8268 8269 req_qs->flags = flags; 8270 req_qs->port_id = cpu_to_le16(pf->port_id); 8271 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 8272 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 8273 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 8274 sizeof(struct tx_port_stats_ext) : 0; 8275 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 8276 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 8277 resp_qs = hwrm_req_hold(bp, req_qs); 8278 rc = hwrm_req_send(bp, req_qs); 8279 if (!rc) { 8280 bp->fw_rx_stats_ext_size = 8281 le16_to_cpu(resp_qs->rx_stat_size) / 8; 8282 if (BNXT_FW_MAJ(bp) < 220 && 8283 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 8284 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 8285 8286 bp->fw_tx_stats_ext_size = tx_stat_size ? 8287 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 8288 } else { 8289 bp->fw_rx_stats_ext_size = 0; 8290 bp->fw_tx_stats_ext_size = 0; 8291 } 8292 hwrm_req_drop(bp, req_qs); 8293 8294 if (flags) 8295 return rc; 8296 8297 if (bp->fw_tx_stats_ext_size <= 8298 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 8299 bp->pri2cos_valid = 0; 8300 return rc; 8301 } 8302 8303 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 8304 if (rc) 8305 return rc; 8306 8307 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 8308 8309 resp_qc = hwrm_req_hold(bp, req_qc); 8310 rc = hwrm_req_send(bp, req_qc); 8311 if (!rc) { 8312 u8 *pri2cos; 8313 int i, j; 8314 8315 pri2cos = &resp_qc->pri0_cos_queue_id; 8316 for (i = 0; i < 8; i++) { 8317 u8 queue_id = pri2cos[i]; 8318 u8 queue_idx; 8319 8320 /* Per port queue IDs start from 0, 10, 20, etc */ 8321 queue_idx = queue_id % 10; 8322 if (queue_idx > BNXT_MAX_QUEUE) { 8323 bp->pri2cos_valid = false; 8324 hwrm_req_drop(bp, req_qc); 8325 return rc; 8326 } 8327 for (j = 0; j < bp->max_q; j++) { 8328 if (bp->q_ids[j] == queue_id) 8329 bp->pri2cos_idx[i] = queue_idx; 8330 } 8331 } 8332 bp->pri2cos_valid = true; 8333 } 8334 hwrm_req_drop(bp, req_qc); 8335 8336 return rc; 8337 } 8338 8339 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 8340 { 8341 bnxt_hwrm_tunnel_dst_port_free(bp, 8342 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 8343 bnxt_hwrm_tunnel_dst_port_free(bp, 8344 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 8345 } 8346 8347 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 8348 { 8349 int rc, i; 8350 u32 tpa_flags = 0; 8351 8352 if (set_tpa) 8353 tpa_flags = bp->flags & BNXT_FLAG_TPA; 8354 else if (BNXT_NO_FW_ACCESS(bp)) 8355 return 0; 8356 for (i = 0; i < bp->nr_vnics; i++) { 8357 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 8358 if (rc) { 8359 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 8360 i, rc); 8361 return rc; 8362 } 8363 } 8364 return 0; 8365 } 8366 8367 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 8368 { 8369 int i; 8370 8371 for (i = 0; i < bp->nr_vnics; i++) 8372 bnxt_hwrm_vnic_set_rss(bp, i, false); 8373 } 8374 8375 static void bnxt_clear_vnic(struct bnxt *bp) 8376 { 8377 if (!bp->vnic_info) 8378 return; 8379 8380 bnxt_hwrm_clear_vnic_filter(bp); 8381 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 8382 /* clear all RSS setting before free vnic ctx */ 8383 bnxt_hwrm_clear_vnic_rss(bp); 8384 bnxt_hwrm_vnic_ctx_free(bp); 8385 } 8386 /* before free the vnic, undo the vnic tpa settings */ 8387 if (bp->flags & BNXT_FLAG_TPA) 8388 bnxt_set_tpa(bp, false); 8389 bnxt_hwrm_vnic_free(bp); 8390 if (bp->flags & BNXT_FLAG_CHIP_P5) 8391 bnxt_hwrm_vnic_ctx_free(bp); 8392 } 8393 8394 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 8395 bool irq_re_init) 8396 { 8397 bnxt_clear_vnic(bp); 8398 bnxt_hwrm_ring_free(bp, close_path); 8399 bnxt_hwrm_ring_grp_free(bp); 8400 if (irq_re_init) { 8401 bnxt_hwrm_stat_ctx_free(bp); 8402 bnxt_hwrm_free_tunnel_ports(bp); 8403 } 8404 } 8405 8406 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 8407 { 8408 struct hwrm_func_cfg_input *req; 8409 u8 evb_mode; 8410 int rc; 8411 8412 if (br_mode == BRIDGE_MODE_VEB) 8413 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 8414 else if (br_mode == BRIDGE_MODE_VEPA) 8415 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 8416 else 8417 return -EINVAL; 8418 8419 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8420 if (rc) 8421 return rc; 8422 8423 req->fid = cpu_to_le16(0xffff); 8424 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 8425 req->evb_mode = evb_mode; 8426 return hwrm_req_send(bp, req); 8427 } 8428 8429 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 8430 { 8431 struct hwrm_func_cfg_input *req; 8432 int rc; 8433 8434 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 8435 return 0; 8436 8437 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG); 8438 if (rc) 8439 return rc; 8440 8441 req->fid = cpu_to_le16(0xffff); 8442 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 8443 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 8444 if (size == 128) 8445 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 8446 8447 return hwrm_req_send(bp, req); 8448 } 8449 8450 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8451 { 8452 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 8453 int rc; 8454 8455 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 8456 goto skip_rss_ctx; 8457 8458 /* allocate context for vnic */ 8459 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 8460 if (rc) { 8461 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8462 vnic_id, rc); 8463 goto vnic_setup_err; 8464 } 8465 bp->rsscos_nr_ctxs++; 8466 8467 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8468 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 8469 if (rc) { 8470 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 8471 vnic_id, rc); 8472 goto vnic_setup_err; 8473 } 8474 bp->rsscos_nr_ctxs++; 8475 } 8476 8477 skip_rss_ctx: 8478 /* configure default vnic, ring grp */ 8479 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8480 if (rc) { 8481 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8482 vnic_id, rc); 8483 goto vnic_setup_err; 8484 } 8485 8486 /* Enable RSS hashing on vnic */ 8487 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 8488 if (rc) { 8489 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 8490 vnic_id, rc); 8491 goto vnic_setup_err; 8492 } 8493 8494 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8495 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8496 if (rc) { 8497 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8498 vnic_id, rc); 8499 } 8500 } 8501 8502 vnic_setup_err: 8503 return rc; 8504 } 8505 8506 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 8507 { 8508 int rc, i, nr_ctxs; 8509 8510 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 8511 for (i = 0; i < nr_ctxs; i++) { 8512 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 8513 if (rc) { 8514 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 8515 vnic_id, i, rc); 8516 break; 8517 } 8518 bp->rsscos_nr_ctxs++; 8519 } 8520 if (i < nr_ctxs) 8521 return -ENOMEM; 8522 8523 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 8524 if (rc) { 8525 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 8526 vnic_id, rc); 8527 return rc; 8528 } 8529 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 8530 if (rc) { 8531 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 8532 vnic_id, rc); 8533 return rc; 8534 } 8535 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8536 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 8537 if (rc) { 8538 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 8539 vnic_id, rc); 8540 } 8541 } 8542 return rc; 8543 } 8544 8545 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 8546 { 8547 if (bp->flags & BNXT_FLAG_CHIP_P5) 8548 return __bnxt_setup_vnic_p5(bp, vnic_id); 8549 else 8550 return __bnxt_setup_vnic(bp, vnic_id); 8551 } 8552 8553 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 8554 { 8555 #ifdef CONFIG_RFS_ACCEL 8556 int i, rc = 0; 8557 8558 if (bp->flags & BNXT_FLAG_CHIP_P5) 8559 return 0; 8560 8561 for (i = 0; i < bp->rx_nr_rings; i++) { 8562 struct bnxt_vnic_info *vnic; 8563 u16 vnic_id = i + 1; 8564 u16 ring_id = i; 8565 8566 if (vnic_id >= bp->nr_vnics) 8567 break; 8568 8569 vnic = &bp->vnic_info[vnic_id]; 8570 vnic->flags |= BNXT_VNIC_RFS_FLAG; 8571 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 8572 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 8573 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 8574 if (rc) { 8575 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 8576 vnic_id, rc); 8577 break; 8578 } 8579 rc = bnxt_setup_vnic(bp, vnic_id); 8580 if (rc) 8581 break; 8582 } 8583 return rc; 8584 #else 8585 return 0; 8586 #endif 8587 } 8588 8589 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 8590 static bool bnxt_promisc_ok(struct bnxt *bp) 8591 { 8592 #ifdef CONFIG_BNXT_SRIOV 8593 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 8594 return false; 8595 #endif 8596 return true; 8597 } 8598 8599 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 8600 { 8601 unsigned int rc = 0; 8602 8603 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 8604 if (rc) { 8605 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8606 rc); 8607 return rc; 8608 } 8609 8610 rc = bnxt_hwrm_vnic_cfg(bp, 1); 8611 if (rc) { 8612 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 8613 rc); 8614 return rc; 8615 } 8616 return rc; 8617 } 8618 8619 static int bnxt_cfg_rx_mode(struct bnxt *); 8620 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 8621 8622 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 8623 { 8624 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 8625 int rc = 0; 8626 unsigned int rx_nr_rings = bp->rx_nr_rings; 8627 8628 if (irq_re_init) { 8629 rc = bnxt_hwrm_stat_ctx_alloc(bp); 8630 if (rc) { 8631 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 8632 rc); 8633 goto err_out; 8634 } 8635 } 8636 8637 rc = bnxt_hwrm_ring_alloc(bp); 8638 if (rc) { 8639 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 8640 goto err_out; 8641 } 8642 8643 rc = bnxt_hwrm_ring_grp_alloc(bp); 8644 if (rc) { 8645 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 8646 goto err_out; 8647 } 8648 8649 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8650 rx_nr_rings--; 8651 8652 /* default vnic 0 */ 8653 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 8654 if (rc) { 8655 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 8656 goto err_out; 8657 } 8658 8659 rc = bnxt_setup_vnic(bp, 0); 8660 if (rc) 8661 goto err_out; 8662 8663 if (bp->flags & BNXT_FLAG_RFS) { 8664 rc = bnxt_alloc_rfs_vnics(bp); 8665 if (rc) 8666 goto err_out; 8667 } 8668 8669 if (bp->flags & BNXT_FLAG_TPA) { 8670 rc = bnxt_set_tpa(bp, true); 8671 if (rc) 8672 goto err_out; 8673 } 8674 8675 if (BNXT_VF(bp)) 8676 bnxt_update_vf_mac(bp); 8677 8678 /* Filter for default vnic 0 */ 8679 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 8680 if (rc) { 8681 if (BNXT_VF(bp) && rc == -ENODEV) 8682 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 8683 else 8684 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 8685 goto err_out; 8686 } 8687 vnic->uc_filter_count = 1; 8688 8689 vnic->rx_mask = 0; 8690 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 8691 goto skip_rx_mask; 8692 8693 if (bp->dev->flags & IFF_BROADCAST) 8694 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 8695 8696 if (bp->dev->flags & IFF_PROMISC) 8697 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 8698 8699 if (bp->dev->flags & IFF_ALLMULTI) { 8700 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 8701 vnic->mc_list_count = 0; 8702 } else if (bp->dev->flags & IFF_MULTICAST) { 8703 u32 mask = 0; 8704 8705 bnxt_mc_list_updated(bp, &mask); 8706 vnic->rx_mask |= mask; 8707 } 8708 8709 rc = bnxt_cfg_rx_mode(bp); 8710 if (rc) 8711 goto err_out; 8712 8713 skip_rx_mask: 8714 rc = bnxt_hwrm_set_coal(bp); 8715 if (rc) 8716 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 8717 rc); 8718 8719 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8720 rc = bnxt_setup_nitroa0_vnic(bp); 8721 if (rc) 8722 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 8723 rc); 8724 } 8725 8726 if (BNXT_VF(bp)) { 8727 bnxt_hwrm_func_qcfg(bp); 8728 netdev_update_features(bp->dev); 8729 } 8730 8731 return 0; 8732 8733 err_out: 8734 bnxt_hwrm_resource_free(bp, 0, true); 8735 8736 return rc; 8737 } 8738 8739 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 8740 { 8741 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 8742 return 0; 8743 } 8744 8745 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 8746 { 8747 bnxt_init_cp_rings(bp); 8748 bnxt_init_rx_rings(bp); 8749 bnxt_init_tx_rings(bp); 8750 bnxt_init_ring_grps(bp, irq_re_init); 8751 bnxt_init_vnics(bp); 8752 8753 return bnxt_init_chip(bp, irq_re_init); 8754 } 8755 8756 static int bnxt_set_real_num_queues(struct bnxt *bp) 8757 { 8758 int rc; 8759 struct net_device *dev = bp->dev; 8760 8761 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 8762 bp->tx_nr_rings_xdp); 8763 if (rc) 8764 return rc; 8765 8766 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8767 if (rc) 8768 return rc; 8769 8770 #ifdef CONFIG_RFS_ACCEL 8771 if (bp->flags & BNXT_FLAG_RFS) 8772 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8773 #endif 8774 8775 return rc; 8776 } 8777 8778 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8779 bool shared) 8780 { 8781 int _rx = *rx, _tx = *tx; 8782 8783 if (shared) { 8784 *rx = min_t(int, _rx, max); 8785 *tx = min_t(int, _tx, max); 8786 } else { 8787 if (max < 2) 8788 return -ENOMEM; 8789 8790 while (_rx + _tx > max) { 8791 if (_rx > _tx && _rx > 1) 8792 _rx--; 8793 else if (_tx > 1) 8794 _tx--; 8795 } 8796 *rx = _rx; 8797 *tx = _tx; 8798 } 8799 return 0; 8800 } 8801 8802 static void bnxt_setup_msix(struct bnxt *bp) 8803 { 8804 const int len = sizeof(bp->irq_tbl[0].name); 8805 struct net_device *dev = bp->dev; 8806 int tcs, i; 8807 8808 tcs = netdev_get_num_tc(dev); 8809 if (tcs) { 8810 int i, off, count; 8811 8812 for (i = 0; i < tcs; i++) { 8813 count = bp->tx_nr_rings_per_tc; 8814 off = i * count; 8815 netdev_set_tc_queue(dev, i, count, off); 8816 } 8817 } 8818 8819 for (i = 0; i < bp->cp_nr_rings; i++) { 8820 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8821 char *attr; 8822 8823 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8824 attr = "TxRx"; 8825 else if (i < bp->rx_nr_rings) 8826 attr = "rx"; 8827 else 8828 attr = "tx"; 8829 8830 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8831 attr, i); 8832 bp->irq_tbl[map_idx].handler = bnxt_msix; 8833 } 8834 } 8835 8836 static void bnxt_setup_inta(struct bnxt *bp) 8837 { 8838 const int len = sizeof(bp->irq_tbl[0].name); 8839 8840 if (netdev_get_num_tc(bp->dev)) 8841 netdev_reset_tc(bp->dev); 8842 8843 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8844 0); 8845 bp->irq_tbl[0].handler = bnxt_inta; 8846 } 8847 8848 static int bnxt_init_int_mode(struct bnxt *bp); 8849 8850 static int bnxt_setup_int_mode(struct bnxt *bp) 8851 { 8852 int rc; 8853 8854 if (!bp->irq_tbl) { 8855 rc = bnxt_init_int_mode(bp); 8856 if (rc || !bp->irq_tbl) 8857 return rc ?: -ENODEV; 8858 } 8859 8860 if (bp->flags & BNXT_FLAG_USING_MSIX) 8861 bnxt_setup_msix(bp); 8862 else 8863 bnxt_setup_inta(bp); 8864 8865 rc = bnxt_set_real_num_queues(bp); 8866 return rc; 8867 } 8868 8869 #ifdef CONFIG_RFS_ACCEL 8870 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8871 { 8872 return bp->hw_resc.max_rsscos_ctxs; 8873 } 8874 8875 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8876 { 8877 return bp->hw_resc.max_vnics; 8878 } 8879 #endif 8880 8881 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8882 { 8883 return bp->hw_resc.max_stat_ctxs; 8884 } 8885 8886 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8887 { 8888 return bp->hw_resc.max_cp_rings; 8889 } 8890 8891 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8892 { 8893 unsigned int cp = bp->hw_resc.max_cp_rings; 8894 8895 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8896 cp -= bnxt_get_ulp_msix_num(bp); 8897 8898 return cp; 8899 } 8900 8901 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8902 { 8903 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8904 8905 if (bp->flags & BNXT_FLAG_CHIP_P5) 8906 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8907 8908 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8909 } 8910 8911 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8912 { 8913 bp->hw_resc.max_irqs = max_irqs; 8914 } 8915 8916 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8917 { 8918 unsigned int cp; 8919 8920 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8921 if (bp->flags & BNXT_FLAG_CHIP_P5) 8922 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8923 else 8924 return cp - bp->cp_nr_rings; 8925 } 8926 8927 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8928 { 8929 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8930 } 8931 8932 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8933 { 8934 int max_cp = bnxt_get_max_func_cp_rings(bp); 8935 int max_irq = bnxt_get_max_func_irqs(bp); 8936 int total_req = bp->cp_nr_rings + num; 8937 int max_idx, avail_msix; 8938 8939 max_idx = bp->total_irqs; 8940 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8941 max_idx = min_t(int, bp->total_irqs, max_cp); 8942 avail_msix = max_idx - bp->cp_nr_rings; 8943 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8944 return avail_msix; 8945 8946 if (max_irq < total_req) { 8947 num = max_irq - bp->cp_nr_rings; 8948 if (num <= 0) 8949 return 0; 8950 } 8951 return num; 8952 } 8953 8954 static int bnxt_get_num_msix(struct bnxt *bp) 8955 { 8956 if (!BNXT_NEW_RM(bp)) 8957 return bnxt_get_max_func_irqs(bp); 8958 8959 return bnxt_nq_rings_in_use(bp); 8960 } 8961 8962 static int bnxt_init_msix(struct bnxt *bp) 8963 { 8964 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8965 struct msix_entry *msix_ent; 8966 8967 total_vecs = bnxt_get_num_msix(bp); 8968 max = bnxt_get_max_func_irqs(bp); 8969 if (total_vecs > max) 8970 total_vecs = max; 8971 8972 if (!total_vecs) 8973 return 0; 8974 8975 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8976 if (!msix_ent) 8977 return -ENOMEM; 8978 8979 for (i = 0; i < total_vecs; i++) { 8980 msix_ent[i].entry = i; 8981 msix_ent[i].vector = 0; 8982 } 8983 8984 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8985 min = 2; 8986 8987 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8988 ulp_msix = bnxt_get_ulp_msix_num(bp); 8989 if (total_vecs < 0 || total_vecs < ulp_msix) { 8990 rc = -ENODEV; 8991 goto msix_setup_exit; 8992 } 8993 8994 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8995 if (bp->irq_tbl) { 8996 for (i = 0; i < total_vecs; i++) 8997 bp->irq_tbl[i].vector = msix_ent[i].vector; 8998 8999 bp->total_irqs = total_vecs; 9000 /* Trim rings based upon num of vectors allocated */ 9001 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 9002 total_vecs - ulp_msix, min == 1); 9003 if (rc) 9004 goto msix_setup_exit; 9005 9006 bp->cp_nr_rings = (min == 1) ? 9007 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 9008 bp->tx_nr_rings + bp->rx_nr_rings; 9009 9010 } else { 9011 rc = -ENOMEM; 9012 goto msix_setup_exit; 9013 } 9014 bp->flags |= BNXT_FLAG_USING_MSIX; 9015 kfree(msix_ent); 9016 return 0; 9017 9018 msix_setup_exit: 9019 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 9020 kfree(bp->irq_tbl); 9021 bp->irq_tbl = NULL; 9022 pci_disable_msix(bp->pdev); 9023 kfree(msix_ent); 9024 return rc; 9025 } 9026 9027 static int bnxt_init_inta(struct bnxt *bp) 9028 { 9029 bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL); 9030 if (!bp->irq_tbl) 9031 return -ENOMEM; 9032 9033 bp->total_irqs = 1; 9034 bp->rx_nr_rings = 1; 9035 bp->tx_nr_rings = 1; 9036 bp->cp_nr_rings = 1; 9037 bp->flags |= BNXT_FLAG_SHARED_RINGS; 9038 bp->irq_tbl[0].vector = bp->pdev->irq; 9039 return 0; 9040 } 9041 9042 static int bnxt_init_int_mode(struct bnxt *bp) 9043 { 9044 int rc = -ENODEV; 9045 9046 if (bp->flags & BNXT_FLAG_MSIX_CAP) 9047 rc = bnxt_init_msix(bp); 9048 9049 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 9050 /* fallback to INTA */ 9051 rc = bnxt_init_inta(bp); 9052 } 9053 return rc; 9054 } 9055 9056 static void bnxt_clear_int_mode(struct bnxt *bp) 9057 { 9058 if (bp->flags & BNXT_FLAG_USING_MSIX) 9059 pci_disable_msix(bp->pdev); 9060 9061 kfree(bp->irq_tbl); 9062 bp->irq_tbl = NULL; 9063 bp->flags &= ~BNXT_FLAG_USING_MSIX; 9064 } 9065 9066 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 9067 { 9068 int tcs = netdev_get_num_tc(bp->dev); 9069 bool irq_cleared = false; 9070 int rc; 9071 9072 if (!bnxt_need_reserve_rings(bp)) 9073 return 0; 9074 9075 if (irq_re_init && BNXT_NEW_RM(bp) && 9076 bnxt_get_num_msix(bp) != bp->total_irqs) { 9077 bnxt_ulp_irq_stop(bp); 9078 bnxt_clear_int_mode(bp); 9079 irq_cleared = true; 9080 } 9081 rc = __bnxt_reserve_rings(bp); 9082 if (irq_cleared) { 9083 if (!rc) 9084 rc = bnxt_init_int_mode(bp); 9085 bnxt_ulp_irq_restart(bp, rc); 9086 } 9087 if (rc) { 9088 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 9089 return rc; 9090 } 9091 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 9092 netdev_err(bp->dev, "tx ring reservation failure\n"); 9093 netdev_reset_tc(bp->dev); 9094 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 9095 return -ENOMEM; 9096 } 9097 return 0; 9098 } 9099 9100 static void bnxt_free_irq(struct bnxt *bp) 9101 { 9102 struct bnxt_irq *irq; 9103 int i; 9104 9105 #ifdef CONFIG_RFS_ACCEL 9106 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 9107 bp->dev->rx_cpu_rmap = NULL; 9108 #endif 9109 if (!bp->irq_tbl || !bp->bnapi) 9110 return; 9111 9112 for (i = 0; i < bp->cp_nr_rings; i++) { 9113 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9114 9115 irq = &bp->irq_tbl[map_idx]; 9116 if (irq->requested) { 9117 if (irq->have_cpumask) { 9118 irq_set_affinity_hint(irq->vector, NULL); 9119 free_cpumask_var(irq->cpu_mask); 9120 irq->have_cpumask = 0; 9121 } 9122 free_irq(irq->vector, bp->bnapi[i]); 9123 } 9124 9125 irq->requested = 0; 9126 } 9127 } 9128 9129 static int bnxt_request_irq(struct bnxt *bp) 9130 { 9131 int i, j, rc = 0; 9132 unsigned long flags = 0; 9133 #ifdef CONFIG_RFS_ACCEL 9134 struct cpu_rmap *rmap; 9135 #endif 9136 9137 rc = bnxt_setup_int_mode(bp); 9138 if (rc) { 9139 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 9140 rc); 9141 return rc; 9142 } 9143 #ifdef CONFIG_RFS_ACCEL 9144 rmap = bp->dev->rx_cpu_rmap; 9145 #endif 9146 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 9147 flags = IRQF_SHARED; 9148 9149 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 9150 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 9151 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 9152 9153 #ifdef CONFIG_RFS_ACCEL 9154 if (rmap && bp->bnapi[i]->rx_ring) { 9155 rc = irq_cpu_rmap_add(rmap, irq->vector); 9156 if (rc) 9157 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 9158 j); 9159 j++; 9160 } 9161 #endif 9162 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 9163 bp->bnapi[i]); 9164 if (rc) 9165 break; 9166 9167 irq->requested = 1; 9168 9169 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 9170 int numa_node = dev_to_node(&bp->pdev->dev); 9171 9172 irq->have_cpumask = 1; 9173 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 9174 irq->cpu_mask); 9175 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 9176 if (rc) { 9177 netdev_warn(bp->dev, 9178 "Set affinity failed, IRQ = %d\n", 9179 irq->vector); 9180 break; 9181 } 9182 } 9183 } 9184 return rc; 9185 } 9186 9187 static void bnxt_del_napi(struct bnxt *bp) 9188 { 9189 int i; 9190 9191 if (!bp->bnapi) 9192 return; 9193 9194 for (i = 0; i < bp->cp_nr_rings; i++) { 9195 struct bnxt_napi *bnapi = bp->bnapi[i]; 9196 9197 __netif_napi_del(&bnapi->napi); 9198 } 9199 /* We called __netif_napi_del(), we need 9200 * to respect an RCU grace period before freeing napi structures. 9201 */ 9202 synchronize_net(); 9203 } 9204 9205 static void bnxt_init_napi(struct bnxt *bp) 9206 { 9207 int i; 9208 unsigned int cp_nr_rings = bp->cp_nr_rings; 9209 struct bnxt_napi *bnapi; 9210 9211 if (bp->flags & BNXT_FLAG_USING_MSIX) { 9212 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 9213 9214 if (bp->flags & BNXT_FLAG_CHIP_P5) 9215 poll_fn = bnxt_poll_p5; 9216 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 9217 cp_nr_rings--; 9218 for (i = 0; i < cp_nr_rings; i++) { 9219 bnapi = bp->bnapi[i]; 9220 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 9221 } 9222 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 9223 bnapi = bp->bnapi[cp_nr_rings]; 9224 netif_napi_add(bp->dev, &bnapi->napi, 9225 bnxt_poll_nitroa0, 64); 9226 } 9227 } else { 9228 bnapi = bp->bnapi[0]; 9229 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 9230 } 9231 } 9232 9233 static void bnxt_disable_napi(struct bnxt *bp) 9234 { 9235 int i; 9236 9237 if (!bp->bnapi || 9238 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 9239 return; 9240 9241 for (i = 0; i < bp->cp_nr_rings; i++) { 9242 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 9243 9244 napi_disable(&bp->bnapi[i]->napi); 9245 if (bp->bnapi[i]->rx_ring) 9246 cancel_work_sync(&cpr->dim.work); 9247 } 9248 } 9249 9250 static void bnxt_enable_napi(struct bnxt *bp) 9251 { 9252 int i; 9253 9254 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 9255 for (i = 0; i < bp->cp_nr_rings; i++) { 9256 struct bnxt_napi *bnapi = bp->bnapi[i]; 9257 struct bnxt_cp_ring_info *cpr; 9258 9259 cpr = &bnapi->cp_ring; 9260 if (bnapi->in_reset) 9261 cpr->sw_stats.rx.rx_resets++; 9262 bnapi->in_reset = false; 9263 9264 if (bnapi->rx_ring) { 9265 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 9266 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 9267 } 9268 napi_enable(&bnapi->napi); 9269 } 9270 } 9271 9272 void bnxt_tx_disable(struct bnxt *bp) 9273 { 9274 int i; 9275 struct bnxt_tx_ring_info *txr; 9276 9277 if (bp->tx_ring) { 9278 for (i = 0; i < bp->tx_nr_rings; i++) { 9279 txr = &bp->tx_ring[i]; 9280 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 9281 } 9282 } 9283 /* Make sure napi polls see @dev_state change */ 9284 synchronize_net(); 9285 /* Drop carrier first to prevent TX timeout */ 9286 netif_carrier_off(bp->dev); 9287 /* Stop all TX queues */ 9288 netif_tx_disable(bp->dev); 9289 } 9290 9291 void bnxt_tx_enable(struct bnxt *bp) 9292 { 9293 int i; 9294 struct bnxt_tx_ring_info *txr; 9295 9296 for (i = 0; i < bp->tx_nr_rings; i++) { 9297 txr = &bp->tx_ring[i]; 9298 WRITE_ONCE(txr->dev_state, 0); 9299 } 9300 /* Make sure napi polls see @dev_state change */ 9301 synchronize_net(); 9302 netif_tx_wake_all_queues(bp->dev); 9303 if (bp->link_info.link_up) 9304 netif_carrier_on(bp->dev); 9305 } 9306 9307 static char *bnxt_report_fec(struct bnxt_link_info *link_info) 9308 { 9309 u8 active_fec = link_info->active_fec_sig_mode & 9310 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 9311 9312 switch (active_fec) { 9313 default: 9314 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 9315 return "None"; 9316 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 9317 return "Clause 74 BaseR"; 9318 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 9319 return "Clause 91 RS(528,514)"; 9320 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 9321 return "Clause 91 RS544_1XN"; 9322 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 9323 return "Clause 91 RS(544,514)"; 9324 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 9325 return "Clause 91 RS272_1XN"; 9326 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 9327 return "Clause 91 RS(272,257)"; 9328 } 9329 } 9330 9331 void bnxt_report_link(struct bnxt *bp) 9332 { 9333 if (bp->link_info.link_up) { 9334 const char *signal = ""; 9335 const char *flow_ctrl; 9336 const char *duplex; 9337 u32 speed; 9338 u16 fec; 9339 9340 netif_carrier_on(bp->dev); 9341 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 9342 if (speed == SPEED_UNKNOWN) { 9343 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 9344 return; 9345 } 9346 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 9347 duplex = "full"; 9348 else 9349 duplex = "half"; 9350 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 9351 flow_ctrl = "ON - receive & transmit"; 9352 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 9353 flow_ctrl = "ON - transmit"; 9354 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 9355 flow_ctrl = "ON - receive"; 9356 else 9357 flow_ctrl = "none"; 9358 if (bp->link_info.phy_qcfg_resp.option_flags & 9359 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 9360 u8 sig_mode = bp->link_info.active_fec_sig_mode & 9361 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 9362 switch (sig_mode) { 9363 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 9364 signal = "(NRZ) "; 9365 break; 9366 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 9367 signal = "(PAM4) "; 9368 break; 9369 default: 9370 break; 9371 } 9372 } 9373 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 9374 speed, signal, duplex, flow_ctrl); 9375 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 9376 netdev_info(bp->dev, "EEE is %s\n", 9377 bp->eee.eee_active ? "active" : 9378 "not active"); 9379 fec = bp->link_info.fec_cfg; 9380 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 9381 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 9382 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 9383 bnxt_report_fec(&bp->link_info)); 9384 } else { 9385 netif_carrier_off(bp->dev); 9386 netdev_err(bp->dev, "NIC Link is Down\n"); 9387 } 9388 } 9389 9390 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 9391 { 9392 if (!resp->supported_speeds_auto_mode && 9393 !resp->supported_speeds_force_mode && 9394 !resp->supported_pam4_speeds_auto_mode && 9395 !resp->supported_pam4_speeds_force_mode) 9396 return true; 9397 return false; 9398 } 9399 9400 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 9401 { 9402 struct bnxt_link_info *link_info = &bp->link_info; 9403 struct hwrm_port_phy_qcaps_output *resp; 9404 struct hwrm_port_phy_qcaps_input *req; 9405 int rc = 0; 9406 9407 if (bp->hwrm_spec_code < 0x10201) 9408 return 0; 9409 9410 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 9411 if (rc) 9412 return rc; 9413 9414 resp = hwrm_req_hold(bp, req); 9415 rc = hwrm_req_send(bp, req); 9416 if (rc) 9417 goto hwrm_phy_qcaps_exit; 9418 9419 bp->phy_flags = resp->flags; 9420 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 9421 struct ethtool_eee *eee = &bp->eee; 9422 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 9423 9424 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9425 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 9426 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 9427 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 9428 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 9429 } 9430 9431 if (bp->hwrm_spec_code >= 0x10a01) { 9432 if (bnxt_phy_qcaps_no_speed(resp)) { 9433 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 9434 netdev_warn(bp->dev, "Ethernet link disabled\n"); 9435 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 9436 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 9437 netdev_info(bp->dev, "Ethernet link enabled\n"); 9438 /* Phy re-enabled, reprobe the speeds */ 9439 link_info->support_auto_speeds = 0; 9440 link_info->support_pam4_auto_speeds = 0; 9441 } 9442 } 9443 if (resp->supported_speeds_auto_mode) 9444 link_info->support_auto_speeds = 9445 le16_to_cpu(resp->supported_speeds_auto_mode); 9446 if (resp->supported_pam4_speeds_auto_mode) 9447 link_info->support_pam4_auto_speeds = 9448 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 9449 9450 bp->port_count = resp->port_cnt; 9451 9452 hwrm_phy_qcaps_exit: 9453 hwrm_req_drop(bp, req); 9454 return rc; 9455 } 9456 9457 static bool bnxt_support_dropped(u16 advertising, u16 supported) 9458 { 9459 u16 diff = advertising ^ supported; 9460 9461 return ((supported | diff) != supported); 9462 } 9463 9464 int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 9465 { 9466 struct bnxt_link_info *link_info = &bp->link_info; 9467 struct hwrm_port_phy_qcfg_output *resp; 9468 struct hwrm_port_phy_qcfg_input *req; 9469 u8 link_up = link_info->link_up; 9470 bool support_changed = false; 9471 int rc; 9472 9473 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 9474 if (rc) 9475 return rc; 9476 9477 resp = hwrm_req_hold(bp, req); 9478 rc = hwrm_req_send(bp, req); 9479 if (rc) { 9480 hwrm_req_drop(bp, req); 9481 if (BNXT_VF(bp) && rc == -ENODEV) { 9482 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 9483 rc = 0; 9484 } 9485 return rc; 9486 } 9487 9488 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 9489 link_info->phy_link_status = resp->link; 9490 link_info->duplex = resp->duplex_cfg; 9491 if (bp->hwrm_spec_code >= 0x10800) 9492 link_info->duplex = resp->duplex_state; 9493 link_info->pause = resp->pause; 9494 link_info->auto_mode = resp->auto_mode; 9495 link_info->auto_pause_setting = resp->auto_pause; 9496 link_info->lp_pause = resp->link_partner_adv_pause; 9497 link_info->force_pause_setting = resp->force_pause; 9498 link_info->duplex_setting = resp->duplex_cfg; 9499 if (link_info->phy_link_status == BNXT_LINK_LINK) 9500 link_info->link_speed = le16_to_cpu(resp->link_speed); 9501 else 9502 link_info->link_speed = 0; 9503 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 9504 link_info->force_pam4_link_speed = 9505 le16_to_cpu(resp->force_pam4_link_speed); 9506 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 9507 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 9508 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 9509 link_info->auto_pam4_link_speeds = 9510 le16_to_cpu(resp->auto_pam4_link_speed_mask); 9511 link_info->lp_auto_link_speeds = 9512 le16_to_cpu(resp->link_partner_adv_speeds); 9513 link_info->lp_auto_pam4_link_speeds = 9514 resp->link_partner_pam4_adv_speeds; 9515 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 9516 link_info->phy_ver[0] = resp->phy_maj; 9517 link_info->phy_ver[1] = resp->phy_min; 9518 link_info->phy_ver[2] = resp->phy_bld; 9519 link_info->media_type = resp->media_type; 9520 link_info->phy_type = resp->phy_type; 9521 link_info->transceiver = resp->xcvr_pkg_type; 9522 link_info->phy_addr = resp->eee_config_phy_addr & 9523 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 9524 link_info->module_status = resp->module_status; 9525 9526 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 9527 struct ethtool_eee *eee = &bp->eee; 9528 u16 fw_speeds; 9529 9530 eee->eee_active = 0; 9531 if (resp->eee_config_phy_addr & 9532 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 9533 eee->eee_active = 1; 9534 fw_speeds = le16_to_cpu( 9535 resp->link_partner_adv_eee_link_speed_mask); 9536 eee->lp_advertised = 9537 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9538 } 9539 9540 /* Pull initial EEE config */ 9541 if (!chng_link_state) { 9542 if (resp->eee_config_phy_addr & 9543 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 9544 eee->eee_enabled = 1; 9545 9546 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 9547 eee->advertised = 9548 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 9549 9550 if (resp->eee_config_phy_addr & 9551 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 9552 __le32 tmr; 9553 9554 eee->tx_lpi_enabled = 1; 9555 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 9556 eee->tx_lpi_timer = le32_to_cpu(tmr) & 9557 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 9558 } 9559 } 9560 } 9561 9562 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 9563 if (bp->hwrm_spec_code >= 0x10504) { 9564 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 9565 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 9566 } 9567 /* TODO: need to add more logic to report VF link */ 9568 if (chng_link_state) { 9569 if (link_info->phy_link_status == BNXT_LINK_LINK) 9570 link_info->link_up = 1; 9571 else 9572 link_info->link_up = 0; 9573 if (link_up != link_info->link_up) 9574 bnxt_report_link(bp); 9575 } else { 9576 /* alwasy link down if not require to update link state */ 9577 link_info->link_up = 0; 9578 } 9579 hwrm_req_drop(bp, req); 9580 9581 if (!BNXT_PHY_CFG_ABLE(bp)) 9582 return 0; 9583 9584 /* Check if any advertised speeds are no longer supported. The caller 9585 * holds the link_lock mutex, so we can modify link_info settings. 9586 */ 9587 if (bnxt_support_dropped(link_info->advertising, 9588 link_info->support_auto_speeds)) { 9589 link_info->advertising = link_info->support_auto_speeds; 9590 support_changed = true; 9591 } 9592 if (bnxt_support_dropped(link_info->advertising_pam4, 9593 link_info->support_pam4_auto_speeds)) { 9594 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 9595 support_changed = true; 9596 } 9597 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 9598 bnxt_hwrm_set_link_setting(bp, true, false); 9599 return 0; 9600 } 9601 9602 static void bnxt_get_port_module_status(struct bnxt *bp) 9603 { 9604 struct bnxt_link_info *link_info = &bp->link_info; 9605 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 9606 u8 module_status; 9607 9608 if (bnxt_update_link(bp, true)) 9609 return; 9610 9611 module_status = link_info->module_status; 9612 switch (module_status) { 9613 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 9614 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 9615 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 9616 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 9617 bp->pf.port_id); 9618 if (bp->hwrm_spec_code >= 0x10201) { 9619 netdev_warn(bp->dev, "Module part number %s\n", 9620 resp->phy_vendor_partnumber); 9621 } 9622 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 9623 netdev_warn(bp->dev, "TX is disabled\n"); 9624 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 9625 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 9626 } 9627 } 9628 9629 static void 9630 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9631 { 9632 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 9633 if (bp->hwrm_spec_code >= 0x10201) 9634 req->auto_pause = 9635 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 9636 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9637 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 9638 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9639 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 9640 req->enables |= 9641 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9642 } else { 9643 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 9644 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 9645 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 9646 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 9647 req->enables |= 9648 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 9649 if (bp->hwrm_spec_code >= 0x10201) { 9650 req->auto_pause = req->force_pause; 9651 req->enables |= cpu_to_le32( 9652 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 9653 } 9654 } 9655 } 9656 9657 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 9658 { 9659 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 9660 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 9661 if (bp->link_info.advertising) { 9662 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 9663 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 9664 } 9665 if (bp->link_info.advertising_pam4) { 9666 req->enables |= 9667 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 9668 req->auto_link_pam4_speed_mask = 9669 cpu_to_le16(bp->link_info.advertising_pam4); 9670 } 9671 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 9672 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 9673 } else { 9674 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 9675 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 9676 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9677 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 9678 } else { 9679 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 9680 } 9681 } 9682 9683 /* tell chimp that the setting takes effect immediately */ 9684 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 9685 } 9686 9687 int bnxt_hwrm_set_pause(struct bnxt *bp) 9688 { 9689 struct hwrm_port_phy_cfg_input *req; 9690 int rc; 9691 9692 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9693 if (rc) 9694 return rc; 9695 9696 bnxt_hwrm_set_pause_common(bp, req); 9697 9698 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 9699 bp->link_info.force_link_chng) 9700 bnxt_hwrm_set_link_common(bp, req); 9701 9702 rc = hwrm_req_send(bp, req); 9703 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 9704 /* since changing of pause setting doesn't trigger any link 9705 * change event, the driver needs to update the current pause 9706 * result upon successfully return of the phy_cfg command 9707 */ 9708 bp->link_info.pause = 9709 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 9710 bp->link_info.auto_pause_setting = 0; 9711 if (!bp->link_info.force_link_chng) 9712 bnxt_report_link(bp); 9713 } 9714 bp->link_info.force_link_chng = false; 9715 return rc; 9716 } 9717 9718 static void bnxt_hwrm_set_eee(struct bnxt *bp, 9719 struct hwrm_port_phy_cfg_input *req) 9720 { 9721 struct ethtool_eee *eee = &bp->eee; 9722 9723 if (eee->eee_enabled) { 9724 u16 eee_speeds; 9725 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 9726 9727 if (eee->tx_lpi_enabled) 9728 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 9729 else 9730 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 9731 9732 req->flags |= cpu_to_le32(flags); 9733 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 9734 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 9735 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 9736 } else { 9737 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 9738 } 9739 } 9740 9741 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 9742 { 9743 struct hwrm_port_phy_cfg_input *req; 9744 int rc; 9745 9746 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9747 if (rc) 9748 return rc; 9749 9750 if (set_pause) 9751 bnxt_hwrm_set_pause_common(bp, req); 9752 9753 bnxt_hwrm_set_link_common(bp, req); 9754 9755 if (set_eee) 9756 bnxt_hwrm_set_eee(bp, req); 9757 return hwrm_req_send(bp, req); 9758 } 9759 9760 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 9761 { 9762 struct hwrm_port_phy_cfg_input *req; 9763 int rc; 9764 9765 if (!BNXT_SINGLE_PF(bp)) 9766 return 0; 9767 9768 if (pci_num_vf(bp->pdev) && 9769 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 9770 return 0; 9771 9772 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 9773 if (rc) 9774 return rc; 9775 9776 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 9777 return hwrm_req_send(bp, req); 9778 } 9779 9780 static int bnxt_fw_reset_via_optee(struct bnxt *bp) 9781 { 9782 #ifdef CONFIG_TEE_BNXT_FW 9783 int rc = tee_bnxt_fw_load(); 9784 9785 if (rc) 9786 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 9787 9788 return rc; 9789 #else 9790 netdev_err(bp->dev, "OP-TEE not supported\n"); 9791 return -ENODEV; 9792 #endif 9793 } 9794 9795 static int bnxt_try_recover_fw(struct bnxt *bp) 9796 { 9797 if (bp->fw_health && bp->fw_health->status_reliable) { 9798 int retry = 0, rc; 9799 u32 sts; 9800 9801 do { 9802 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 9803 rc = bnxt_hwrm_poll(bp); 9804 if (!BNXT_FW_IS_BOOTING(sts) && 9805 !BNXT_FW_IS_RECOVERING(sts)) 9806 break; 9807 retry++; 9808 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 9809 9810 if (!BNXT_FW_IS_HEALTHY(sts)) { 9811 netdev_err(bp->dev, 9812 "Firmware not responding, status: 0x%x\n", 9813 sts); 9814 rc = -ENODEV; 9815 } 9816 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 9817 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 9818 return bnxt_fw_reset_via_optee(bp); 9819 } 9820 return rc; 9821 } 9822 9823 return -ENODEV; 9824 } 9825 9826 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 9827 { 9828 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9829 int rc; 9830 9831 if (!BNXT_NEW_RM(bp)) 9832 return 0; /* no resource reservations required */ 9833 9834 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9835 if (rc) 9836 netdev_err(bp->dev, "resc_qcaps failed\n"); 9837 9838 hw_resc->resv_cp_rings = 0; 9839 hw_resc->resv_stat_ctxs = 0; 9840 hw_resc->resv_irqs = 0; 9841 hw_resc->resv_tx_rings = 0; 9842 hw_resc->resv_rx_rings = 0; 9843 hw_resc->resv_hw_ring_grps = 0; 9844 hw_resc->resv_vnics = 0; 9845 if (!fw_reset) { 9846 bp->tx_nr_rings = 0; 9847 bp->rx_nr_rings = 0; 9848 } 9849 9850 return rc; 9851 } 9852 9853 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 9854 { 9855 struct hwrm_func_drv_if_change_output *resp; 9856 struct hwrm_func_drv_if_change_input *req; 9857 bool fw_reset = !bp->irq_tbl; 9858 bool resc_reinit = false; 9859 int rc, retry = 0; 9860 u32 flags = 0; 9861 9862 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 9863 return 0; 9864 9865 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 9866 if (rc) 9867 return rc; 9868 9869 if (up) 9870 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 9871 resp = hwrm_req_hold(bp, req); 9872 9873 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 9874 while (retry < BNXT_FW_IF_RETRY) { 9875 rc = hwrm_req_send(bp, req); 9876 if (rc != -EAGAIN) 9877 break; 9878 9879 msleep(50); 9880 retry++; 9881 } 9882 9883 if (rc == -EAGAIN) { 9884 hwrm_req_drop(bp, req); 9885 return rc; 9886 } else if (!rc) { 9887 flags = le32_to_cpu(resp->flags); 9888 } else if (up) { 9889 rc = bnxt_try_recover_fw(bp); 9890 fw_reset = true; 9891 } 9892 hwrm_req_drop(bp, req); 9893 if (rc) 9894 return rc; 9895 9896 if (!up) { 9897 bnxt_inv_fw_health_reg(bp); 9898 return 0; 9899 } 9900 9901 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 9902 resc_reinit = true; 9903 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 9904 fw_reset = true; 9905 else 9906 bnxt_remap_fw_health_regs(bp); 9907 9908 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 9909 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 9910 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9911 return -ENODEV; 9912 } 9913 if (resc_reinit || fw_reset) { 9914 if (fw_reset) { 9915 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9916 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 9917 bnxt_ulp_stop(bp); 9918 bnxt_free_ctx_mem(bp); 9919 kfree(bp->ctx); 9920 bp->ctx = NULL; 9921 bnxt_dcb_free(bp); 9922 rc = bnxt_fw_init_one(bp); 9923 if (rc) { 9924 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9925 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9926 return rc; 9927 } 9928 bnxt_clear_int_mode(bp); 9929 rc = bnxt_init_int_mode(bp); 9930 if (rc) { 9931 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 9932 netdev_err(bp->dev, "init int mode failed\n"); 9933 return rc; 9934 } 9935 } 9936 rc = bnxt_cancel_reservations(bp, fw_reset); 9937 } 9938 return rc; 9939 } 9940 9941 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 9942 { 9943 struct hwrm_port_led_qcaps_output *resp; 9944 struct hwrm_port_led_qcaps_input *req; 9945 struct bnxt_pf_info *pf = &bp->pf; 9946 int rc; 9947 9948 bp->num_leds = 0; 9949 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 9950 return 0; 9951 9952 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 9953 if (rc) 9954 return rc; 9955 9956 req->port_id = cpu_to_le16(pf->port_id); 9957 resp = hwrm_req_hold(bp, req); 9958 rc = hwrm_req_send(bp, req); 9959 if (rc) { 9960 hwrm_req_drop(bp, req); 9961 return rc; 9962 } 9963 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9964 int i; 9965 9966 bp->num_leds = resp->num_leds; 9967 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9968 bp->num_leds); 9969 for (i = 0; i < bp->num_leds; i++) { 9970 struct bnxt_led_info *led = &bp->leds[i]; 9971 __le16 caps = led->led_state_caps; 9972 9973 if (!led->led_group_id || 9974 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9975 bp->num_leds = 0; 9976 break; 9977 } 9978 } 9979 } 9980 hwrm_req_drop(bp, req); 9981 return 0; 9982 } 9983 9984 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9985 { 9986 struct hwrm_wol_filter_alloc_output *resp; 9987 struct hwrm_wol_filter_alloc_input *req; 9988 int rc; 9989 9990 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 9991 if (rc) 9992 return rc; 9993 9994 req->port_id = cpu_to_le16(bp->pf.port_id); 9995 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9996 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9997 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 9998 9999 resp = hwrm_req_hold(bp, req); 10000 rc = hwrm_req_send(bp, req); 10001 if (!rc) 10002 bp->wol_filter_id = resp->wol_filter_id; 10003 hwrm_req_drop(bp, req); 10004 return rc; 10005 } 10006 10007 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 10008 { 10009 struct hwrm_wol_filter_free_input *req; 10010 int rc; 10011 10012 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 10013 if (rc) 10014 return rc; 10015 10016 req->port_id = cpu_to_le16(bp->pf.port_id); 10017 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 10018 req->wol_filter_id = bp->wol_filter_id; 10019 10020 return hwrm_req_send(bp, req); 10021 } 10022 10023 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 10024 { 10025 struct hwrm_wol_filter_qcfg_output *resp; 10026 struct hwrm_wol_filter_qcfg_input *req; 10027 u16 next_handle = 0; 10028 int rc; 10029 10030 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 10031 if (rc) 10032 return rc; 10033 10034 req->port_id = cpu_to_le16(bp->pf.port_id); 10035 req->handle = cpu_to_le16(handle); 10036 resp = hwrm_req_hold(bp, req); 10037 rc = hwrm_req_send(bp, req); 10038 if (!rc) { 10039 next_handle = le16_to_cpu(resp->next_handle); 10040 if (next_handle != 0) { 10041 if (resp->wol_type == 10042 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 10043 bp->wol = 1; 10044 bp->wol_filter_id = resp->wol_filter_id; 10045 } 10046 } 10047 } 10048 hwrm_req_drop(bp, req); 10049 return next_handle; 10050 } 10051 10052 static void bnxt_get_wol_settings(struct bnxt *bp) 10053 { 10054 u16 handle = 0; 10055 10056 bp->wol = 0; 10057 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 10058 return; 10059 10060 do { 10061 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 10062 } while (handle && handle != 0xffff); 10063 } 10064 10065 #ifdef CONFIG_BNXT_HWMON 10066 static ssize_t bnxt_show_temp(struct device *dev, 10067 struct device_attribute *devattr, char *buf) 10068 { 10069 struct hwrm_temp_monitor_query_output *resp; 10070 struct hwrm_temp_monitor_query_input *req; 10071 struct bnxt *bp = dev_get_drvdata(dev); 10072 u32 len = 0; 10073 int rc; 10074 10075 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10076 if (rc) 10077 return rc; 10078 resp = hwrm_req_hold(bp, req); 10079 rc = hwrm_req_send(bp, req); 10080 if (!rc) 10081 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */ 10082 hwrm_req_drop(bp, req); 10083 if (rc) 10084 return rc; 10085 return len; 10086 } 10087 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 10088 10089 static struct attribute *bnxt_attrs[] = { 10090 &sensor_dev_attr_temp1_input.dev_attr.attr, 10091 NULL 10092 }; 10093 ATTRIBUTE_GROUPS(bnxt); 10094 10095 static void bnxt_hwmon_close(struct bnxt *bp) 10096 { 10097 if (bp->hwmon_dev) { 10098 hwmon_device_unregister(bp->hwmon_dev); 10099 bp->hwmon_dev = NULL; 10100 } 10101 } 10102 10103 static void bnxt_hwmon_open(struct bnxt *bp) 10104 { 10105 struct hwrm_temp_monitor_query_input *req; 10106 struct pci_dev *pdev = bp->pdev; 10107 int rc; 10108 10109 rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY); 10110 if (!rc) 10111 rc = hwrm_req_send_silent(bp, req); 10112 if (rc == -EACCES || rc == -EOPNOTSUPP) { 10113 bnxt_hwmon_close(bp); 10114 return; 10115 } 10116 10117 if (bp->hwmon_dev) 10118 return; 10119 10120 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 10121 DRV_MODULE_NAME, bp, 10122 bnxt_groups); 10123 if (IS_ERR(bp->hwmon_dev)) { 10124 bp->hwmon_dev = NULL; 10125 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 10126 } 10127 } 10128 #else 10129 static void bnxt_hwmon_close(struct bnxt *bp) 10130 { 10131 } 10132 10133 static void bnxt_hwmon_open(struct bnxt *bp) 10134 { 10135 } 10136 #endif 10137 10138 static bool bnxt_eee_config_ok(struct bnxt *bp) 10139 { 10140 struct ethtool_eee *eee = &bp->eee; 10141 struct bnxt_link_info *link_info = &bp->link_info; 10142 10143 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 10144 return true; 10145 10146 if (eee->eee_enabled) { 10147 u32 advertising = 10148 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 10149 10150 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10151 eee->eee_enabled = 0; 10152 return false; 10153 } 10154 if (eee->advertised & ~advertising) { 10155 eee->advertised = advertising & eee->supported; 10156 return false; 10157 } 10158 } 10159 return true; 10160 } 10161 10162 static int bnxt_update_phy_setting(struct bnxt *bp) 10163 { 10164 int rc; 10165 bool update_link = false; 10166 bool update_pause = false; 10167 bool update_eee = false; 10168 struct bnxt_link_info *link_info = &bp->link_info; 10169 10170 rc = bnxt_update_link(bp, true); 10171 if (rc) { 10172 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 10173 rc); 10174 return rc; 10175 } 10176 if (!BNXT_SINGLE_PF(bp)) 10177 return 0; 10178 10179 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10180 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 10181 link_info->req_flow_ctrl) 10182 update_pause = true; 10183 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 10184 link_info->force_pause_setting != link_info->req_flow_ctrl) 10185 update_pause = true; 10186 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 10187 if (BNXT_AUTO_MODE(link_info->auto_mode)) 10188 update_link = true; 10189 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 10190 link_info->req_link_speed != link_info->force_link_speed) 10191 update_link = true; 10192 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 10193 link_info->req_link_speed != link_info->force_pam4_link_speed) 10194 update_link = true; 10195 if (link_info->req_duplex != link_info->duplex_setting) 10196 update_link = true; 10197 } else { 10198 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 10199 update_link = true; 10200 if (link_info->advertising != link_info->auto_link_speeds || 10201 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 10202 update_link = true; 10203 } 10204 10205 /* The last close may have shutdown the link, so need to call 10206 * PHY_CFG to bring it back up. 10207 */ 10208 if (!bp->link_info.link_up) 10209 update_link = true; 10210 10211 if (!bnxt_eee_config_ok(bp)) 10212 update_eee = true; 10213 10214 if (update_link) 10215 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 10216 else if (update_pause) 10217 rc = bnxt_hwrm_set_pause(bp); 10218 if (rc) { 10219 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 10220 rc); 10221 return rc; 10222 } 10223 10224 return rc; 10225 } 10226 10227 /* Common routine to pre-map certain register block to different GRC window. 10228 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 10229 * in PF and 3 windows in VF that can be customized to map in different 10230 * register blocks. 10231 */ 10232 static void bnxt_preset_reg_win(struct bnxt *bp) 10233 { 10234 if (BNXT_PF(bp)) { 10235 /* CAG registers map to GRC window #4 */ 10236 writel(BNXT_CAG_REG_BASE, 10237 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 10238 } 10239 } 10240 10241 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 10242 10243 static int bnxt_reinit_after_abort(struct bnxt *bp) 10244 { 10245 int rc; 10246 10247 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10248 return -EBUSY; 10249 10250 if (bp->dev->reg_state == NETREG_UNREGISTERED) 10251 return -ENODEV; 10252 10253 rc = bnxt_fw_init_one(bp); 10254 if (!rc) { 10255 bnxt_clear_int_mode(bp); 10256 rc = bnxt_init_int_mode(bp); 10257 if (!rc) { 10258 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10259 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 10260 } 10261 } 10262 return rc; 10263 } 10264 10265 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10266 { 10267 int rc = 0; 10268 10269 bnxt_preset_reg_win(bp); 10270 netif_carrier_off(bp->dev); 10271 if (irq_re_init) { 10272 /* Reserve rings now if none were reserved at driver probe. */ 10273 rc = bnxt_init_dflt_ring_mode(bp); 10274 if (rc) { 10275 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 10276 return rc; 10277 } 10278 } 10279 rc = bnxt_reserve_rings(bp, irq_re_init); 10280 if (rc) 10281 return rc; 10282 if ((bp->flags & BNXT_FLAG_RFS) && 10283 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 10284 /* disable RFS if falling back to INTA */ 10285 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 10286 bp->flags &= ~BNXT_FLAG_RFS; 10287 } 10288 10289 rc = bnxt_alloc_mem(bp, irq_re_init); 10290 if (rc) { 10291 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10292 goto open_err_free_mem; 10293 } 10294 10295 if (irq_re_init) { 10296 bnxt_init_napi(bp); 10297 rc = bnxt_request_irq(bp); 10298 if (rc) { 10299 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 10300 goto open_err_irq; 10301 } 10302 } 10303 10304 rc = bnxt_init_nic(bp, irq_re_init); 10305 if (rc) { 10306 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10307 goto open_err_irq; 10308 } 10309 10310 bnxt_enable_napi(bp); 10311 bnxt_debug_dev_init(bp); 10312 10313 if (link_re_init) { 10314 mutex_lock(&bp->link_lock); 10315 rc = bnxt_update_phy_setting(bp); 10316 mutex_unlock(&bp->link_lock); 10317 if (rc) { 10318 netdev_warn(bp->dev, "failed to update phy settings\n"); 10319 if (BNXT_SINGLE_PF(bp)) { 10320 bp->link_info.phy_retry = true; 10321 bp->link_info.phy_retry_expires = 10322 jiffies + 5 * HZ; 10323 } 10324 } 10325 } 10326 10327 if (irq_re_init) 10328 udp_tunnel_nic_reset_ntf(bp->dev); 10329 10330 set_bit(BNXT_STATE_OPEN, &bp->state); 10331 bnxt_enable_int(bp); 10332 /* Enable TX queues */ 10333 bnxt_tx_enable(bp); 10334 mod_timer(&bp->timer, jiffies + bp->current_interval); 10335 /* Poll link status and check for SFP+ module status */ 10336 mutex_lock(&bp->link_lock); 10337 bnxt_get_port_module_status(bp); 10338 mutex_unlock(&bp->link_lock); 10339 10340 /* VF-reps may need to be re-opened after the PF is re-opened */ 10341 if (BNXT_PF(bp)) 10342 bnxt_vf_reps_open(bp); 10343 bnxt_ptp_init_rtc(bp, true); 10344 return 0; 10345 10346 open_err_irq: 10347 bnxt_del_napi(bp); 10348 10349 open_err_free_mem: 10350 bnxt_free_skbs(bp); 10351 bnxt_free_irq(bp); 10352 bnxt_free_mem(bp, true); 10353 return rc; 10354 } 10355 10356 /* rtnl_lock held */ 10357 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10358 { 10359 int rc = 0; 10360 10361 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 10362 rc = -EIO; 10363 if (!rc) 10364 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 10365 if (rc) { 10366 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 10367 dev_close(bp->dev); 10368 } 10369 return rc; 10370 } 10371 10372 /* rtnl_lock held, open the NIC half way by allocating all resources, but 10373 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 10374 * self tests. 10375 */ 10376 int bnxt_half_open_nic(struct bnxt *bp) 10377 { 10378 int rc = 0; 10379 10380 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10381 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 10382 rc = -ENODEV; 10383 goto half_open_err; 10384 } 10385 10386 rc = bnxt_alloc_mem(bp, true); 10387 if (rc) { 10388 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 10389 goto half_open_err; 10390 } 10391 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10392 rc = bnxt_init_nic(bp, true); 10393 if (rc) { 10394 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10395 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 10396 goto half_open_err; 10397 } 10398 return 0; 10399 10400 half_open_err: 10401 bnxt_free_skbs(bp); 10402 bnxt_free_mem(bp, true); 10403 dev_close(bp->dev); 10404 return rc; 10405 } 10406 10407 /* rtnl_lock held, this call can only be made after a previous successful 10408 * call to bnxt_half_open_nic(). 10409 */ 10410 void bnxt_half_close_nic(struct bnxt *bp) 10411 { 10412 bnxt_hwrm_resource_free(bp, false, true); 10413 bnxt_free_skbs(bp); 10414 bnxt_free_mem(bp, true); 10415 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 10416 } 10417 10418 void bnxt_reenable_sriov(struct bnxt *bp) 10419 { 10420 if (BNXT_PF(bp)) { 10421 struct bnxt_pf_info *pf = &bp->pf; 10422 int n = pf->active_vfs; 10423 10424 if (n) 10425 bnxt_cfg_hw_sriov(bp, &n, true); 10426 } 10427 } 10428 10429 static int bnxt_open(struct net_device *dev) 10430 { 10431 struct bnxt *bp = netdev_priv(dev); 10432 int rc; 10433 10434 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 10435 rc = bnxt_reinit_after_abort(bp); 10436 if (rc) { 10437 if (rc == -EBUSY) 10438 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 10439 else 10440 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 10441 return -ENODEV; 10442 } 10443 } 10444 10445 rc = bnxt_hwrm_if_change(bp, true); 10446 if (rc) 10447 return rc; 10448 10449 rc = __bnxt_open_nic(bp, true, true); 10450 if (rc) { 10451 bnxt_hwrm_if_change(bp, false); 10452 } else { 10453 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 10454 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10455 bnxt_ulp_start(bp, 0); 10456 bnxt_reenable_sriov(bp); 10457 } 10458 } 10459 bnxt_hwmon_open(bp); 10460 } 10461 10462 return rc; 10463 } 10464 10465 static bool bnxt_drv_busy(struct bnxt *bp) 10466 { 10467 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 10468 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 10469 } 10470 10471 static void bnxt_get_ring_stats(struct bnxt *bp, 10472 struct rtnl_link_stats64 *stats); 10473 10474 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 10475 bool link_re_init) 10476 { 10477 /* Close the VF-reps before closing PF */ 10478 if (BNXT_PF(bp)) 10479 bnxt_vf_reps_close(bp); 10480 10481 /* Change device state to avoid TX queue wake up's */ 10482 bnxt_tx_disable(bp); 10483 10484 clear_bit(BNXT_STATE_OPEN, &bp->state); 10485 smp_mb__after_atomic(); 10486 while (bnxt_drv_busy(bp)) 10487 msleep(20); 10488 10489 /* Flush rings and and disable interrupts */ 10490 bnxt_shutdown_nic(bp, irq_re_init); 10491 10492 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 10493 10494 bnxt_debug_dev_exit(bp); 10495 bnxt_disable_napi(bp); 10496 del_timer_sync(&bp->timer); 10497 bnxt_free_skbs(bp); 10498 10499 /* Save ring stats before shutdown */ 10500 if (bp->bnapi && irq_re_init) 10501 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 10502 if (irq_re_init) { 10503 bnxt_free_irq(bp); 10504 bnxt_del_napi(bp); 10505 } 10506 bnxt_free_mem(bp, irq_re_init); 10507 } 10508 10509 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 10510 { 10511 int rc = 0; 10512 10513 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10514 /* If we get here, it means firmware reset is in progress 10515 * while we are trying to close. We can safely proceed with 10516 * the close because we are holding rtnl_lock(). Some firmware 10517 * messages may fail as we proceed to close. We set the 10518 * ABORT_ERR flag here so that the FW reset thread will later 10519 * abort when it gets the rtnl_lock() and sees the flag. 10520 */ 10521 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 10522 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 10523 } 10524 10525 #ifdef CONFIG_BNXT_SRIOV 10526 if (bp->sriov_cfg) { 10527 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 10528 !bp->sriov_cfg, 10529 BNXT_SRIOV_CFG_WAIT_TMO); 10530 if (rc) 10531 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 10532 } 10533 #endif 10534 __bnxt_close_nic(bp, irq_re_init, link_re_init); 10535 return rc; 10536 } 10537 10538 static int bnxt_close(struct net_device *dev) 10539 { 10540 struct bnxt *bp = netdev_priv(dev); 10541 10542 bnxt_hwmon_close(bp); 10543 bnxt_close_nic(bp, true, true); 10544 bnxt_hwrm_shutdown_link(bp); 10545 bnxt_hwrm_if_change(bp, false); 10546 return 0; 10547 } 10548 10549 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 10550 u16 *val) 10551 { 10552 struct hwrm_port_phy_mdio_read_output *resp; 10553 struct hwrm_port_phy_mdio_read_input *req; 10554 int rc; 10555 10556 if (bp->hwrm_spec_code < 0x10a00) 10557 return -EOPNOTSUPP; 10558 10559 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 10560 if (rc) 10561 return rc; 10562 10563 req->port_id = cpu_to_le16(bp->pf.port_id); 10564 req->phy_addr = phy_addr; 10565 req->reg_addr = cpu_to_le16(reg & 0x1f); 10566 if (mdio_phy_id_is_c45(phy_addr)) { 10567 req->cl45_mdio = 1; 10568 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10569 req->dev_addr = mdio_phy_id_devad(phy_addr); 10570 req->reg_addr = cpu_to_le16(reg); 10571 } 10572 10573 resp = hwrm_req_hold(bp, req); 10574 rc = hwrm_req_send(bp, req); 10575 if (!rc) 10576 *val = le16_to_cpu(resp->reg_data); 10577 hwrm_req_drop(bp, req); 10578 return rc; 10579 } 10580 10581 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 10582 u16 val) 10583 { 10584 struct hwrm_port_phy_mdio_write_input *req; 10585 int rc; 10586 10587 if (bp->hwrm_spec_code < 0x10a00) 10588 return -EOPNOTSUPP; 10589 10590 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 10591 if (rc) 10592 return rc; 10593 10594 req->port_id = cpu_to_le16(bp->pf.port_id); 10595 req->phy_addr = phy_addr; 10596 req->reg_addr = cpu_to_le16(reg & 0x1f); 10597 if (mdio_phy_id_is_c45(phy_addr)) { 10598 req->cl45_mdio = 1; 10599 req->phy_addr = mdio_phy_id_prtad(phy_addr); 10600 req->dev_addr = mdio_phy_id_devad(phy_addr); 10601 req->reg_addr = cpu_to_le16(reg); 10602 } 10603 req->reg_data = cpu_to_le16(val); 10604 10605 return hwrm_req_send(bp, req); 10606 } 10607 10608 /* rtnl_lock held */ 10609 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 10610 { 10611 struct mii_ioctl_data *mdio = if_mii(ifr); 10612 struct bnxt *bp = netdev_priv(dev); 10613 int rc; 10614 10615 switch (cmd) { 10616 case SIOCGMIIPHY: 10617 mdio->phy_id = bp->link_info.phy_addr; 10618 10619 fallthrough; 10620 case SIOCGMIIREG: { 10621 u16 mii_regval = 0; 10622 10623 if (!netif_running(dev)) 10624 return -EAGAIN; 10625 10626 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 10627 &mii_regval); 10628 mdio->val_out = mii_regval; 10629 return rc; 10630 } 10631 10632 case SIOCSMIIREG: 10633 if (!netif_running(dev)) 10634 return -EAGAIN; 10635 10636 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 10637 mdio->val_in); 10638 10639 case SIOCSHWTSTAMP: 10640 return bnxt_hwtstamp_set(dev, ifr); 10641 10642 case SIOCGHWTSTAMP: 10643 return bnxt_hwtstamp_get(dev, ifr); 10644 10645 default: 10646 /* do nothing */ 10647 break; 10648 } 10649 return -EOPNOTSUPP; 10650 } 10651 10652 static void bnxt_get_ring_stats(struct bnxt *bp, 10653 struct rtnl_link_stats64 *stats) 10654 { 10655 int i; 10656 10657 for (i = 0; i < bp->cp_nr_rings; i++) { 10658 struct bnxt_napi *bnapi = bp->bnapi[i]; 10659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10660 u64 *sw = cpr->stats.sw_stats; 10661 10662 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 10663 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10664 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 10665 10666 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 10667 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 10668 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 10669 10670 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 10671 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 10672 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 10673 10674 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 10675 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 10676 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 10677 10678 stats->rx_missed_errors += 10679 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 10680 10681 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 10682 10683 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 10684 10685 stats->rx_dropped += 10686 cpr->sw_stats.rx.rx_netpoll_discards + 10687 cpr->sw_stats.rx.rx_oom_discards; 10688 } 10689 } 10690 10691 static void bnxt_add_prev_stats(struct bnxt *bp, 10692 struct rtnl_link_stats64 *stats) 10693 { 10694 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 10695 10696 stats->rx_packets += prev_stats->rx_packets; 10697 stats->tx_packets += prev_stats->tx_packets; 10698 stats->rx_bytes += prev_stats->rx_bytes; 10699 stats->tx_bytes += prev_stats->tx_bytes; 10700 stats->rx_missed_errors += prev_stats->rx_missed_errors; 10701 stats->multicast += prev_stats->multicast; 10702 stats->rx_dropped += prev_stats->rx_dropped; 10703 stats->tx_dropped += prev_stats->tx_dropped; 10704 } 10705 10706 static void 10707 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 10708 { 10709 struct bnxt *bp = netdev_priv(dev); 10710 10711 set_bit(BNXT_STATE_READ_STATS, &bp->state); 10712 /* Make sure bnxt_close_nic() sees that we are reading stats before 10713 * we check the BNXT_STATE_OPEN flag. 10714 */ 10715 smp_mb__after_atomic(); 10716 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10717 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10718 *stats = bp->net_stats_prev; 10719 return; 10720 } 10721 10722 bnxt_get_ring_stats(bp, stats); 10723 bnxt_add_prev_stats(bp, stats); 10724 10725 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10726 u64 *rx = bp->port_stats.sw_stats; 10727 u64 *tx = bp->port_stats.sw_stats + 10728 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10729 10730 stats->rx_crc_errors = 10731 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 10732 stats->rx_frame_errors = 10733 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 10734 stats->rx_length_errors = 10735 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 10736 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 10737 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 10738 stats->rx_errors = 10739 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 10740 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 10741 stats->collisions = 10742 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 10743 stats->tx_fifo_errors = 10744 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 10745 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 10746 } 10747 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 10748 } 10749 10750 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 10751 { 10752 struct net_device *dev = bp->dev; 10753 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10754 struct netdev_hw_addr *ha; 10755 u8 *haddr; 10756 int mc_count = 0; 10757 bool update = false; 10758 int off = 0; 10759 10760 netdev_for_each_mc_addr(ha, dev) { 10761 if (mc_count >= BNXT_MAX_MC_ADDRS) { 10762 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10763 vnic->mc_list_count = 0; 10764 return false; 10765 } 10766 haddr = ha->addr; 10767 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 10768 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 10769 update = true; 10770 } 10771 off += ETH_ALEN; 10772 mc_count++; 10773 } 10774 if (mc_count) 10775 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10776 10777 if (mc_count != vnic->mc_list_count) { 10778 vnic->mc_list_count = mc_count; 10779 update = true; 10780 } 10781 return update; 10782 } 10783 10784 static bool bnxt_uc_list_updated(struct bnxt *bp) 10785 { 10786 struct net_device *dev = bp->dev; 10787 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10788 struct netdev_hw_addr *ha; 10789 int off = 0; 10790 10791 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 10792 return true; 10793 10794 netdev_for_each_uc_addr(ha, dev) { 10795 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 10796 return true; 10797 10798 off += ETH_ALEN; 10799 } 10800 return false; 10801 } 10802 10803 static void bnxt_set_rx_mode(struct net_device *dev) 10804 { 10805 struct bnxt *bp = netdev_priv(dev); 10806 struct bnxt_vnic_info *vnic; 10807 bool mc_update = false; 10808 bool uc_update; 10809 u32 mask; 10810 10811 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 10812 return; 10813 10814 vnic = &bp->vnic_info[0]; 10815 mask = vnic->rx_mask; 10816 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 10817 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 10818 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 10819 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 10820 10821 if (dev->flags & IFF_PROMISC) 10822 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10823 10824 uc_update = bnxt_uc_list_updated(bp); 10825 10826 if (dev->flags & IFF_BROADCAST) 10827 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 10828 if (dev->flags & IFF_ALLMULTI) { 10829 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10830 vnic->mc_list_count = 0; 10831 } else if (dev->flags & IFF_MULTICAST) { 10832 mc_update = bnxt_mc_list_updated(bp, &mask); 10833 } 10834 10835 if (mask != vnic->rx_mask || uc_update || mc_update) { 10836 vnic->rx_mask = mask; 10837 10838 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 10839 bnxt_queue_sp_work(bp); 10840 } 10841 } 10842 10843 static int bnxt_cfg_rx_mode(struct bnxt *bp) 10844 { 10845 struct net_device *dev = bp->dev; 10846 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 10847 struct hwrm_cfa_l2_filter_free_input *req; 10848 struct netdev_hw_addr *ha; 10849 int i, off = 0, rc; 10850 bool uc_update; 10851 10852 netif_addr_lock_bh(dev); 10853 uc_update = bnxt_uc_list_updated(bp); 10854 netif_addr_unlock_bh(dev); 10855 10856 if (!uc_update) 10857 goto skip_uc; 10858 10859 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 10860 if (rc) 10861 return rc; 10862 hwrm_req_hold(bp, req); 10863 for (i = 1; i < vnic->uc_filter_count; i++) { 10864 req->l2_filter_id = vnic->fw_l2_filter_id[i]; 10865 10866 rc = hwrm_req_send(bp, req); 10867 } 10868 hwrm_req_drop(bp, req); 10869 10870 vnic->uc_filter_count = 1; 10871 10872 netif_addr_lock_bh(dev); 10873 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 10874 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10875 } else { 10876 netdev_for_each_uc_addr(ha, dev) { 10877 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 10878 off += ETH_ALEN; 10879 vnic->uc_filter_count++; 10880 } 10881 } 10882 netif_addr_unlock_bh(dev); 10883 10884 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 10885 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 10886 if (rc) { 10887 if (BNXT_VF(bp) && rc == -ENODEV) { 10888 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 10889 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 10890 else 10891 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 10892 rc = 0; 10893 } else { 10894 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 10895 } 10896 vnic->uc_filter_count = i; 10897 return rc; 10898 } 10899 } 10900 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 10901 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 10902 10903 skip_uc: 10904 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 10905 !bnxt_promisc_ok(bp)) 10906 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 10907 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10908 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 10909 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 10910 rc); 10911 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 10912 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 10913 vnic->mc_list_count = 0; 10914 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 10915 } 10916 if (rc) 10917 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 10918 rc); 10919 10920 return rc; 10921 } 10922 10923 static bool bnxt_can_reserve_rings(struct bnxt *bp) 10924 { 10925 #ifdef CONFIG_BNXT_SRIOV 10926 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 10927 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 10928 10929 /* No minimum rings were provisioned by the PF. Don't 10930 * reserve rings by default when device is down. 10931 */ 10932 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 10933 return true; 10934 10935 if (!netif_running(bp->dev)) 10936 return false; 10937 } 10938 #endif 10939 return true; 10940 } 10941 10942 /* If the chip and firmware supports RFS */ 10943 static bool bnxt_rfs_supported(struct bnxt *bp) 10944 { 10945 if (bp->flags & BNXT_FLAG_CHIP_P5) { 10946 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 10947 return true; 10948 return false; 10949 } 10950 /* 212 firmware is broken for aRFS */ 10951 if (BNXT_FW_MAJ(bp) == 212) 10952 return false; 10953 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 10954 return true; 10955 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10956 return true; 10957 return false; 10958 } 10959 10960 /* If runtime conditions support RFS */ 10961 static bool bnxt_rfs_capable(struct bnxt *bp) 10962 { 10963 #ifdef CONFIG_RFS_ACCEL 10964 int vnics, max_vnics, max_rss_ctxs; 10965 10966 if (bp->flags & BNXT_FLAG_CHIP_P5) 10967 return bnxt_rfs_supported(bp); 10968 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 10969 return false; 10970 10971 vnics = 1 + bp->rx_nr_rings; 10972 max_vnics = bnxt_get_max_func_vnics(bp); 10973 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 10974 10975 /* RSS contexts not a limiting factor */ 10976 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 10977 max_rss_ctxs = max_vnics; 10978 if (vnics > max_vnics || vnics > max_rss_ctxs) { 10979 if (bp->rx_nr_rings > 1) 10980 netdev_warn(bp->dev, 10981 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 10982 min(max_rss_ctxs - 1, max_vnics - 1)); 10983 return false; 10984 } 10985 10986 if (!BNXT_NEW_RM(bp)) 10987 return true; 10988 10989 if (vnics == bp->hw_resc.resv_vnics) 10990 return true; 10991 10992 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 10993 if (vnics <= bp->hw_resc.resv_vnics) 10994 return true; 10995 10996 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 10997 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 10998 return false; 10999 #else 11000 return false; 11001 #endif 11002 } 11003 11004 static netdev_features_t bnxt_fix_features(struct net_device *dev, 11005 netdev_features_t features) 11006 { 11007 struct bnxt *bp = netdev_priv(dev); 11008 netdev_features_t vlan_features; 11009 11010 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 11011 features &= ~NETIF_F_NTUPLE; 11012 11013 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 11014 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 11015 11016 if (!(features & NETIF_F_GRO)) 11017 features &= ~NETIF_F_GRO_HW; 11018 11019 if (features & NETIF_F_GRO_HW) 11020 features &= ~NETIF_F_LRO; 11021 11022 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 11023 * turned on or off together. 11024 */ 11025 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 11026 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 11027 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11028 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11029 else if (vlan_features) 11030 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 11031 } 11032 #ifdef CONFIG_BNXT_SRIOV 11033 if (BNXT_VF(bp) && bp->vf.vlan) 11034 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 11035 #endif 11036 return features; 11037 } 11038 11039 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 11040 { 11041 struct bnxt *bp = netdev_priv(dev); 11042 u32 flags = bp->flags; 11043 u32 changes; 11044 int rc = 0; 11045 bool re_init = false; 11046 bool update_tpa = false; 11047 11048 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 11049 if (features & NETIF_F_GRO_HW) 11050 flags |= BNXT_FLAG_GRO; 11051 else if (features & NETIF_F_LRO) 11052 flags |= BNXT_FLAG_LRO; 11053 11054 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 11055 flags &= ~BNXT_FLAG_TPA; 11056 11057 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 11058 flags |= BNXT_FLAG_STRIP_VLAN; 11059 11060 if (features & NETIF_F_NTUPLE) 11061 flags |= BNXT_FLAG_RFS; 11062 11063 changes = flags ^ bp->flags; 11064 if (changes & BNXT_FLAG_TPA) { 11065 update_tpa = true; 11066 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 11067 (flags & BNXT_FLAG_TPA) == 0 || 11068 (bp->flags & BNXT_FLAG_CHIP_P5)) 11069 re_init = true; 11070 } 11071 11072 if (changes & ~BNXT_FLAG_TPA) 11073 re_init = true; 11074 11075 if (flags != bp->flags) { 11076 u32 old_flags = bp->flags; 11077 11078 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11079 bp->flags = flags; 11080 if (update_tpa) 11081 bnxt_set_ring_params(bp); 11082 return rc; 11083 } 11084 11085 if (re_init) { 11086 bnxt_close_nic(bp, false, false); 11087 bp->flags = flags; 11088 if (update_tpa) 11089 bnxt_set_ring_params(bp); 11090 11091 return bnxt_open_nic(bp, false, false); 11092 } 11093 if (update_tpa) { 11094 bp->flags = flags; 11095 rc = bnxt_set_tpa(bp, 11096 (flags & BNXT_FLAG_TPA) ? 11097 true : false); 11098 if (rc) 11099 bp->flags = old_flags; 11100 } 11101 } 11102 return rc; 11103 } 11104 11105 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 11106 u8 **nextp) 11107 { 11108 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 11109 int hdr_count = 0; 11110 u8 *nexthdr; 11111 int start; 11112 11113 /* Check that there are at most 2 IPv6 extension headers, no 11114 * fragment header, and each is <= 64 bytes. 11115 */ 11116 start = nw_off + sizeof(*ip6h); 11117 nexthdr = &ip6h->nexthdr; 11118 while (ipv6_ext_hdr(*nexthdr)) { 11119 struct ipv6_opt_hdr *hp; 11120 int hdrlen; 11121 11122 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 11123 *nexthdr == NEXTHDR_FRAGMENT) 11124 return false; 11125 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 11126 skb_headlen(skb), NULL); 11127 if (!hp) 11128 return false; 11129 if (*nexthdr == NEXTHDR_AUTH) 11130 hdrlen = ipv6_authlen(hp); 11131 else 11132 hdrlen = ipv6_optlen(hp); 11133 11134 if (hdrlen > 64) 11135 return false; 11136 nexthdr = &hp->nexthdr; 11137 start += hdrlen; 11138 hdr_count++; 11139 } 11140 if (nextp) { 11141 /* Caller will check inner protocol */ 11142 if (skb->encapsulation) { 11143 *nextp = nexthdr; 11144 return true; 11145 } 11146 *nextp = NULL; 11147 } 11148 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 11149 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 11150 } 11151 11152 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 11153 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 11154 { 11155 struct udphdr *uh = udp_hdr(skb); 11156 __be16 udp_port = uh->dest; 11157 11158 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port) 11159 return false; 11160 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) { 11161 struct ethhdr *eh = inner_eth_hdr(skb); 11162 11163 switch (eh->h_proto) { 11164 case htons(ETH_P_IP): 11165 return true; 11166 case htons(ETH_P_IPV6): 11167 return bnxt_exthdr_check(bp, skb, 11168 skb_inner_network_offset(skb), 11169 NULL); 11170 } 11171 } 11172 return false; 11173 } 11174 11175 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 11176 { 11177 switch (l4_proto) { 11178 case IPPROTO_UDP: 11179 return bnxt_udp_tunl_check(bp, skb); 11180 case IPPROTO_IPIP: 11181 return true; 11182 case IPPROTO_GRE: { 11183 switch (skb->inner_protocol) { 11184 default: 11185 return false; 11186 case htons(ETH_P_IP): 11187 return true; 11188 case htons(ETH_P_IPV6): 11189 fallthrough; 11190 } 11191 } 11192 case IPPROTO_IPV6: 11193 /* Check ext headers of inner ipv6 */ 11194 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 11195 NULL); 11196 } 11197 return false; 11198 } 11199 11200 static netdev_features_t bnxt_features_check(struct sk_buff *skb, 11201 struct net_device *dev, 11202 netdev_features_t features) 11203 { 11204 struct bnxt *bp = netdev_priv(dev); 11205 u8 *l4_proto; 11206 11207 features = vlan_features_check(skb, features); 11208 switch (vlan_get_protocol(skb)) { 11209 case htons(ETH_P_IP): 11210 if (!skb->encapsulation) 11211 return features; 11212 l4_proto = &ip_hdr(skb)->protocol; 11213 if (bnxt_tunl_check(bp, skb, *l4_proto)) 11214 return features; 11215 break; 11216 case htons(ETH_P_IPV6): 11217 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 11218 &l4_proto)) 11219 break; 11220 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 11221 return features; 11222 break; 11223 } 11224 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11225 } 11226 11227 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 11228 u32 *reg_buf) 11229 { 11230 struct hwrm_dbg_read_direct_output *resp; 11231 struct hwrm_dbg_read_direct_input *req; 11232 __le32 *dbg_reg_buf; 11233 dma_addr_t mapping; 11234 int rc, i; 11235 11236 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 11237 if (rc) 11238 return rc; 11239 11240 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 11241 &mapping); 11242 if (!dbg_reg_buf) { 11243 rc = -ENOMEM; 11244 goto dbg_rd_reg_exit; 11245 } 11246 11247 req->host_dest_addr = cpu_to_le64(mapping); 11248 11249 resp = hwrm_req_hold(bp, req); 11250 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 11251 req->read_len32 = cpu_to_le32(num_words); 11252 11253 rc = hwrm_req_send(bp, req); 11254 if (rc || resp->error_code) { 11255 rc = -EIO; 11256 goto dbg_rd_reg_exit; 11257 } 11258 for (i = 0; i < num_words; i++) 11259 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 11260 11261 dbg_rd_reg_exit: 11262 hwrm_req_drop(bp, req); 11263 return rc; 11264 } 11265 11266 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 11267 u32 ring_id, u32 *prod, u32 *cons) 11268 { 11269 struct hwrm_dbg_ring_info_get_output *resp; 11270 struct hwrm_dbg_ring_info_get_input *req; 11271 int rc; 11272 11273 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 11274 if (rc) 11275 return rc; 11276 11277 req->ring_type = ring_type; 11278 req->fw_ring_id = cpu_to_le32(ring_id); 11279 resp = hwrm_req_hold(bp, req); 11280 rc = hwrm_req_send(bp, req); 11281 if (!rc) { 11282 *prod = le32_to_cpu(resp->producer_index); 11283 *cons = le32_to_cpu(resp->consumer_index); 11284 } 11285 hwrm_req_drop(bp, req); 11286 return rc; 11287 } 11288 11289 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 11290 { 11291 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 11292 int i = bnapi->index; 11293 11294 if (!txr) 11295 return; 11296 11297 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 11298 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 11299 txr->tx_cons); 11300 } 11301 11302 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 11303 { 11304 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 11305 int i = bnapi->index; 11306 11307 if (!rxr) 11308 return; 11309 11310 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 11311 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 11312 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 11313 rxr->rx_sw_agg_prod); 11314 } 11315 11316 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 11317 { 11318 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 11319 int i = bnapi->index; 11320 11321 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 11322 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 11323 } 11324 11325 static void bnxt_dbg_dump_states(struct bnxt *bp) 11326 { 11327 int i; 11328 struct bnxt_napi *bnapi; 11329 11330 for (i = 0; i < bp->cp_nr_rings; i++) { 11331 bnapi = bp->bnapi[i]; 11332 if (netif_msg_drv(bp)) { 11333 bnxt_dump_tx_sw_state(bnapi); 11334 bnxt_dump_rx_sw_state(bnapi); 11335 bnxt_dump_cp_sw_state(bnapi); 11336 } 11337 } 11338 } 11339 11340 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 11341 { 11342 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 11343 struct hwrm_ring_reset_input *req; 11344 struct bnxt_napi *bnapi = rxr->bnapi; 11345 struct bnxt_cp_ring_info *cpr; 11346 u16 cp_ring_id; 11347 int rc; 11348 11349 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 11350 if (rc) 11351 return rc; 11352 11353 cpr = &bnapi->cp_ring; 11354 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 11355 req->cmpl_ring = cpu_to_le16(cp_ring_id); 11356 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 11357 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 11358 return hwrm_req_send_silent(bp, req); 11359 } 11360 11361 static void bnxt_reset_task(struct bnxt *bp, bool silent) 11362 { 11363 if (!silent) 11364 bnxt_dbg_dump_states(bp); 11365 if (netif_running(bp->dev)) { 11366 int rc; 11367 11368 if (silent) { 11369 bnxt_close_nic(bp, false, false); 11370 bnxt_open_nic(bp, false, false); 11371 } else { 11372 bnxt_ulp_stop(bp); 11373 bnxt_close_nic(bp, true, false); 11374 rc = bnxt_open_nic(bp, true, false); 11375 bnxt_ulp_start(bp, rc); 11376 } 11377 } 11378 } 11379 11380 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 11381 { 11382 struct bnxt *bp = netdev_priv(dev); 11383 11384 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 11385 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 11386 bnxt_queue_sp_work(bp); 11387 } 11388 11389 static void bnxt_fw_health_check(struct bnxt *bp) 11390 { 11391 struct bnxt_fw_health *fw_health = bp->fw_health; 11392 u32 val; 11393 11394 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11395 return; 11396 11397 /* Make sure it is enabled before checking the tmr_counter. */ 11398 smp_rmb(); 11399 if (fw_health->tmr_counter) { 11400 fw_health->tmr_counter--; 11401 return; 11402 } 11403 11404 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11405 if (val == fw_health->last_fw_heartbeat) { 11406 fw_health->arrests++; 11407 goto fw_reset; 11408 } 11409 11410 fw_health->last_fw_heartbeat = val; 11411 11412 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11413 if (val != fw_health->last_fw_reset_cnt) { 11414 fw_health->discoveries++; 11415 goto fw_reset; 11416 } 11417 11418 fw_health->tmr_counter = fw_health->tmr_multiplier; 11419 return; 11420 11421 fw_reset: 11422 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 11423 bnxt_queue_sp_work(bp); 11424 } 11425 11426 static void bnxt_timer(struct timer_list *t) 11427 { 11428 struct bnxt *bp = from_timer(bp, t, timer); 11429 struct net_device *dev = bp->dev; 11430 11431 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 11432 return; 11433 11434 if (atomic_read(&bp->intr_sem) != 0) 11435 goto bnxt_restart_timer; 11436 11437 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 11438 bnxt_fw_health_check(bp); 11439 11440 if (bp->link_info.link_up && bp->stats_coal_ticks) { 11441 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 11442 bnxt_queue_sp_work(bp); 11443 } 11444 11445 if (bnxt_tc_flower_enabled(bp)) { 11446 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 11447 bnxt_queue_sp_work(bp); 11448 } 11449 11450 #ifdef CONFIG_RFS_ACCEL 11451 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 11452 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 11453 bnxt_queue_sp_work(bp); 11454 } 11455 #endif /*CONFIG_RFS_ACCEL*/ 11456 11457 if (bp->link_info.phy_retry) { 11458 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 11459 bp->link_info.phy_retry = false; 11460 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 11461 } else { 11462 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 11463 bnxt_queue_sp_work(bp); 11464 } 11465 } 11466 11467 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) { 11468 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 11469 bnxt_queue_sp_work(bp); 11470 } 11471 11472 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 11473 netif_carrier_ok(dev)) { 11474 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 11475 bnxt_queue_sp_work(bp); 11476 } 11477 bnxt_restart_timer: 11478 mod_timer(&bp->timer, jiffies + bp->current_interval); 11479 } 11480 11481 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 11482 { 11483 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 11484 * set. If the device is being closed, bnxt_close() may be holding 11485 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 11486 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 11487 */ 11488 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11489 rtnl_lock(); 11490 } 11491 11492 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 11493 { 11494 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11495 rtnl_unlock(); 11496 } 11497 11498 /* Only called from bnxt_sp_task() */ 11499 static void bnxt_reset(struct bnxt *bp, bool silent) 11500 { 11501 bnxt_rtnl_lock_sp(bp); 11502 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 11503 bnxt_reset_task(bp, silent); 11504 bnxt_rtnl_unlock_sp(bp); 11505 } 11506 11507 /* Only called from bnxt_sp_task() */ 11508 static void bnxt_rx_ring_reset(struct bnxt *bp) 11509 { 11510 int i; 11511 11512 bnxt_rtnl_lock_sp(bp); 11513 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11514 bnxt_rtnl_unlock_sp(bp); 11515 return; 11516 } 11517 /* Disable and flush TPA before resetting the RX ring */ 11518 if (bp->flags & BNXT_FLAG_TPA) 11519 bnxt_set_tpa(bp, false); 11520 for (i = 0; i < bp->rx_nr_rings; i++) { 11521 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 11522 struct bnxt_cp_ring_info *cpr; 11523 int rc; 11524 11525 if (!rxr->bnapi->in_reset) 11526 continue; 11527 11528 rc = bnxt_hwrm_rx_ring_reset(bp, i); 11529 if (rc) { 11530 if (rc == -EINVAL || rc == -EOPNOTSUPP) 11531 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 11532 else 11533 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 11534 rc); 11535 bnxt_reset_task(bp, true); 11536 break; 11537 } 11538 bnxt_free_one_rx_ring_skbs(bp, i); 11539 rxr->rx_prod = 0; 11540 rxr->rx_agg_prod = 0; 11541 rxr->rx_sw_agg_prod = 0; 11542 rxr->rx_next_cons = 0; 11543 rxr->bnapi->in_reset = false; 11544 bnxt_alloc_one_rx_ring(bp, i); 11545 cpr = &rxr->bnapi->cp_ring; 11546 cpr->sw_stats.rx.rx_resets++; 11547 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11548 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 11549 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 11550 } 11551 if (bp->flags & BNXT_FLAG_TPA) 11552 bnxt_set_tpa(bp, true); 11553 bnxt_rtnl_unlock_sp(bp); 11554 } 11555 11556 static void bnxt_fw_reset_close(struct bnxt *bp) 11557 { 11558 bnxt_ulp_stop(bp); 11559 /* When firmware is in fatal state, quiesce device and disable 11560 * bus master to prevent any potential bad DMAs before freeing 11561 * kernel memory. 11562 */ 11563 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 11564 u16 val = 0; 11565 11566 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 11567 if (val == 0xffff) 11568 bp->fw_reset_min_dsecs = 0; 11569 bnxt_tx_disable(bp); 11570 bnxt_disable_napi(bp); 11571 bnxt_disable_int_sync(bp); 11572 bnxt_free_irq(bp); 11573 bnxt_clear_int_mode(bp); 11574 pci_disable_device(bp->pdev); 11575 } 11576 __bnxt_close_nic(bp, true, false); 11577 bnxt_vf_reps_free(bp); 11578 bnxt_clear_int_mode(bp); 11579 bnxt_hwrm_func_drv_unrgtr(bp); 11580 if (pci_is_enabled(bp->pdev)) 11581 pci_disable_device(bp->pdev); 11582 bnxt_free_ctx_mem(bp); 11583 kfree(bp->ctx); 11584 bp->ctx = NULL; 11585 } 11586 11587 static bool is_bnxt_fw_ok(struct bnxt *bp) 11588 { 11589 struct bnxt_fw_health *fw_health = bp->fw_health; 11590 bool no_heartbeat = false, has_reset = false; 11591 u32 val; 11592 11593 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 11594 if (val == fw_health->last_fw_heartbeat) 11595 no_heartbeat = true; 11596 11597 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 11598 if (val != fw_health->last_fw_reset_cnt) 11599 has_reset = true; 11600 11601 if (!no_heartbeat && has_reset) 11602 return true; 11603 11604 return false; 11605 } 11606 11607 /* rtnl_lock is acquired before calling this function */ 11608 static void bnxt_force_fw_reset(struct bnxt *bp) 11609 { 11610 struct bnxt_fw_health *fw_health = bp->fw_health; 11611 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11612 u32 wait_dsecs; 11613 11614 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 11615 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 11616 return; 11617 11618 if (ptp) { 11619 spin_lock_bh(&ptp->ptp_lock); 11620 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11621 spin_unlock_bh(&ptp->ptp_lock); 11622 } else { 11623 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11624 } 11625 bnxt_fw_reset_close(bp); 11626 wait_dsecs = fw_health->master_func_wait_dsecs; 11627 if (fw_health->primary) { 11628 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 11629 wait_dsecs = 0; 11630 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 11631 } else { 11632 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 11633 wait_dsecs = fw_health->normal_func_wait_dsecs; 11634 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11635 } 11636 11637 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 11638 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 11639 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 11640 } 11641 11642 void bnxt_fw_exception(struct bnxt *bp) 11643 { 11644 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 11645 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 11646 bnxt_rtnl_lock_sp(bp); 11647 bnxt_force_fw_reset(bp); 11648 bnxt_rtnl_unlock_sp(bp); 11649 } 11650 11651 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 11652 * < 0 on error. 11653 */ 11654 static int bnxt_get_registered_vfs(struct bnxt *bp) 11655 { 11656 #ifdef CONFIG_BNXT_SRIOV 11657 int rc; 11658 11659 if (!BNXT_PF(bp)) 11660 return 0; 11661 11662 rc = bnxt_hwrm_func_qcfg(bp); 11663 if (rc) { 11664 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 11665 return rc; 11666 } 11667 if (bp->pf.registered_vfs) 11668 return bp->pf.registered_vfs; 11669 if (bp->sriov_cfg) 11670 return 1; 11671 #endif 11672 return 0; 11673 } 11674 11675 void bnxt_fw_reset(struct bnxt *bp) 11676 { 11677 bnxt_rtnl_lock_sp(bp); 11678 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 11679 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 11680 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 11681 int n = 0, tmo; 11682 11683 if (ptp) { 11684 spin_lock_bh(&ptp->ptp_lock); 11685 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11686 spin_unlock_bh(&ptp->ptp_lock); 11687 } else { 11688 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11689 } 11690 if (bp->pf.active_vfs && 11691 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 11692 n = bnxt_get_registered_vfs(bp); 11693 if (n < 0) { 11694 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 11695 n); 11696 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 11697 dev_close(bp->dev); 11698 goto fw_reset_exit; 11699 } else if (n > 0) { 11700 u16 vf_tmo_dsecs = n * 10; 11701 11702 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 11703 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 11704 bp->fw_reset_state = 11705 BNXT_FW_RESET_STATE_POLL_VF; 11706 bnxt_queue_fw_reset_work(bp, HZ / 10); 11707 goto fw_reset_exit; 11708 } 11709 bnxt_fw_reset_close(bp); 11710 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 11711 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 11712 tmo = HZ / 10; 11713 } else { 11714 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 11715 tmo = bp->fw_reset_min_dsecs * HZ / 10; 11716 } 11717 bnxt_queue_fw_reset_work(bp, tmo); 11718 } 11719 fw_reset_exit: 11720 bnxt_rtnl_unlock_sp(bp); 11721 } 11722 11723 static void bnxt_chk_missed_irq(struct bnxt *bp) 11724 { 11725 int i; 11726 11727 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11728 return; 11729 11730 for (i = 0; i < bp->cp_nr_rings; i++) { 11731 struct bnxt_napi *bnapi = bp->bnapi[i]; 11732 struct bnxt_cp_ring_info *cpr; 11733 u32 fw_ring_id; 11734 int j; 11735 11736 if (!bnapi) 11737 continue; 11738 11739 cpr = &bnapi->cp_ring; 11740 for (j = 0; j < 2; j++) { 11741 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 11742 u32 val[2]; 11743 11744 if (!cpr2 || cpr2->has_more_work || 11745 !bnxt_has_work(bp, cpr2)) 11746 continue; 11747 11748 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 11749 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 11750 continue; 11751 } 11752 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 11753 bnxt_dbg_hwrm_ring_info_get(bp, 11754 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 11755 fw_ring_id, &val[0], &val[1]); 11756 cpr->sw_stats.cmn.missed_irqs++; 11757 } 11758 } 11759 } 11760 11761 static void bnxt_cfg_ntp_filters(struct bnxt *); 11762 11763 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 11764 { 11765 struct bnxt_link_info *link_info = &bp->link_info; 11766 11767 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 11768 link_info->autoneg = BNXT_AUTONEG_SPEED; 11769 if (bp->hwrm_spec_code >= 0x10201) { 11770 if (link_info->auto_pause_setting & 11771 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 11772 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11773 } else { 11774 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 11775 } 11776 link_info->advertising = link_info->auto_link_speeds; 11777 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 11778 } else { 11779 link_info->req_link_speed = link_info->force_link_speed; 11780 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 11781 if (link_info->force_pam4_link_speed) { 11782 link_info->req_link_speed = 11783 link_info->force_pam4_link_speed; 11784 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 11785 } 11786 link_info->req_duplex = link_info->duplex_setting; 11787 } 11788 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 11789 link_info->req_flow_ctrl = 11790 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 11791 else 11792 link_info->req_flow_ctrl = link_info->force_pause_setting; 11793 } 11794 11795 static void bnxt_fw_echo_reply(struct bnxt *bp) 11796 { 11797 struct bnxt_fw_health *fw_health = bp->fw_health; 11798 struct hwrm_func_echo_response_input *req; 11799 int rc; 11800 11801 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 11802 if (rc) 11803 return; 11804 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 11805 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 11806 hwrm_req_send(bp, req); 11807 } 11808 11809 static void bnxt_sp_task(struct work_struct *work) 11810 { 11811 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 11812 11813 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11814 smp_mb__after_atomic(); 11815 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 11816 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11817 return; 11818 } 11819 11820 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 11821 bnxt_cfg_rx_mode(bp); 11822 11823 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 11824 bnxt_cfg_ntp_filters(bp); 11825 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 11826 bnxt_hwrm_exec_fwd_req(bp); 11827 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 11828 bnxt_hwrm_port_qstats(bp, 0); 11829 bnxt_hwrm_port_qstats_ext(bp, 0); 11830 bnxt_accumulate_all_stats(bp); 11831 } 11832 11833 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 11834 int rc; 11835 11836 mutex_lock(&bp->link_lock); 11837 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 11838 &bp->sp_event)) 11839 bnxt_hwrm_phy_qcaps(bp); 11840 11841 rc = bnxt_update_link(bp, true); 11842 if (rc) 11843 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 11844 rc); 11845 11846 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 11847 &bp->sp_event)) 11848 bnxt_init_ethtool_link_settings(bp); 11849 mutex_unlock(&bp->link_lock); 11850 } 11851 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 11852 int rc; 11853 11854 mutex_lock(&bp->link_lock); 11855 rc = bnxt_update_phy_setting(bp); 11856 mutex_unlock(&bp->link_lock); 11857 if (rc) { 11858 netdev_warn(bp->dev, "update phy settings retry failed\n"); 11859 } else { 11860 bp->link_info.phy_retry = false; 11861 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 11862 } 11863 } 11864 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 11865 mutex_lock(&bp->link_lock); 11866 bnxt_get_port_module_status(bp); 11867 mutex_unlock(&bp->link_lock); 11868 } 11869 11870 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 11871 bnxt_tc_flow_stats_work(bp); 11872 11873 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 11874 bnxt_chk_missed_irq(bp); 11875 11876 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 11877 bnxt_fw_echo_reply(bp); 11878 11879 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 11880 * must be the last functions to be called before exiting. 11881 */ 11882 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 11883 bnxt_reset(bp, false); 11884 11885 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 11886 bnxt_reset(bp, true); 11887 11888 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 11889 bnxt_rx_ring_reset(bp); 11890 11891 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 11892 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 11893 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 11894 bnxt_devlink_health_fw_report(bp); 11895 else 11896 bnxt_fw_reset(bp); 11897 } 11898 11899 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 11900 if (!is_bnxt_fw_ok(bp)) 11901 bnxt_devlink_health_fw_report(bp); 11902 } 11903 11904 smp_mb__before_atomic(); 11905 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 11906 } 11907 11908 /* Under rtnl_lock */ 11909 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 11910 int tx_xdp) 11911 { 11912 int max_rx, max_tx, tx_sets = 1; 11913 int tx_rings_needed, stats; 11914 int rx_rings = rx; 11915 int cp, vnics, rc; 11916 11917 if (tcs) 11918 tx_sets = tcs; 11919 11920 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 11921 if (rc) 11922 return rc; 11923 11924 if (max_rx < rx) 11925 return -ENOMEM; 11926 11927 tx_rings_needed = tx * tx_sets + tx_xdp; 11928 if (max_tx < tx_rings_needed) 11929 return -ENOMEM; 11930 11931 vnics = 1; 11932 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 11933 vnics += rx_rings; 11934 11935 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11936 rx_rings <<= 1; 11937 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 11938 stats = cp; 11939 if (BNXT_NEW_RM(bp)) { 11940 cp += bnxt_get_ulp_msix_num(bp); 11941 stats += bnxt_get_ulp_stat_ctxs(bp); 11942 } 11943 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 11944 stats, vnics); 11945 } 11946 11947 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 11948 { 11949 if (bp->bar2) { 11950 pci_iounmap(pdev, bp->bar2); 11951 bp->bar2 = NULL; 11952 } 11953 11954 if (bp->bar1) { 11955 pci_iounmap(pdev, bp->bar1); 11956 bp->bar1 = NULL; 11957 } 11958 11959 if (bp->bar0) { 11960 pci_iounmap(pdev, bp->bar0); 11961 bp->bar0 = NULL; 11962 } 11963 } 11964 11965 static void bnxt_cleanup_pci(struct bnxt *bp) 11966 { 11967 bnxt_unmap_bars(bp, bp->pdev); 11968 pci_release_regions(bp->pdev); 11969 if (pci_is_enabled(bp->pdev)) 11970 pci_disable_device(bp->pdev); 11971 } 11972 11973 static void bnxt_init_dflt_coal(struct bnxt *bp) 11974 { 11975 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 11976 struct bnxt_coal *coal; 11977 u16 flags = 0; 11978 11979 if (coal_cap->cmpl_params & 11980 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 11981 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 11982 11983 /* Tick values in micro seconds. 11984 * 1 coal_buf x bufs_per_record = 1 completion record. 11985 */ 11986 coal = &bp->rx_coal; 11987 coal->coal_ticks = 10; 11988 coal->coal_bufs = 30; 11989 coal->coal_ticks_irq = 1; 11990 coal->coal_bufs_irq = 2; 11991 coal->idle_thresh = 50; 11992 coal->bufs_per_record = 2; 11993 coal->budget = 64; /* NAPI budget */ 11994 coal->flags = flags; 11995 11996 coal = &bp->tx_coal; 11997 coal->coal_ticks = 28; 11998 coal->coal_bufs = 30; 11999 coal->coal_ticks_irq = 2; 12000 coal->coal_bufs_irq = 2; 12001 coal->bufs_per_record = 1; 12002 coal->flags = flags; 12003 12004 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 12005 } 12006 12007 static int bnxt_fw_init_one_p1(struct bnxt *bp) 12008 { 12009 int rc; 12010 12011 bp->fw_cap = 0; 12012 rc = bnxt_hwrm_ver_get(bp); 12013 bnxt_try_map_fw_health_reg(bp); 12014 if (rc) { 12015 rc = bnxt_try_recover_fw(bp); 12016 if (rc) 12017 return rc; 12018 rc = bnxt_hwrm_ver_get(bp); 12019 if (rc) 12020 return rc; 12021 } 12022 12023 bnxt_nvm_cfg_ver_get(bp); 12024 12025 rc = bnxt_hwrm_func_reset(bp); 12026 if (rc) 12027 return -ENODEV; 12028 12029 bnxt_hwrm_fw_set_time(bp); 12030 return 0; 12031 } 12032 12033 static int bnxt_fw_init_one_p2(struct bnxt *bp) 12034 { 12035 int rc; 12036 12037 /* Get the MAX capabilities for this function */ 12038 rc = bnxt_hwrm_func_qcaps(bp); 12039 if (rc) { 12040 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 12041 rc); 12042 return -ENODEV; 12043 } 12044 12045 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 12046 if (rc) 12047 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 12048 rc); 12049 12050 if (bnxt_alloc_fw_health(bp)) { 12051 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 12052 } else { 12053 rc = bnxt_hwrm_error_recovery_qcfg(bp); 12054 if (rc) 12055 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 12056 rc); 12057 } 12058 12059 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 12060 if (rc) 12061 return -ENODEV; 12062 12063 bnxt_hwrm_func_qcfg(bp); 12064 bnxt_hwrm_vnic_qcaps(bp); 12065 bnxt_hwrm_port_led_qcaps(bp); 12066 bnxt_ethtool_init(bp); 12067 bnxt_dcb_init(bp); 12068 return 0; 12069 } 12070 12071 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 12072 { 12073 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 12074 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 12075 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 12076 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 12077 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 12078 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 12079 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 12080 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 12081 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 12082 } 12083 } 12084 12085 static void bnxt_set_dflt_rfs(struct bnxt *bp) 12086 { 12087 struct net_device *dev = bp->dev; 12088 12089 dev->hw_features &= ~NETIF_F_NTUPLE; 12090 dev->features &= ~NETIF_F_NTUPLE; 12091 bp->flags &= ~BNXT_FLAG_RFS; 12092 if (bnxt_rfs_supported(bp)) { 12093 dev->hw_features |= NETIF_F_NTUPLE; 12094 if (bnxt_rfs_capable(bp)) { 12095 bp->flags |= BNXT_FLAG_RFS; 12096 dev->features |= NETIF_F_NTUPLE; 12097 } 12098 } 12099 } 12100 12101 static void bnxt_fw_init_one_p3(struct bnxt *bp) 12102 { 12103 struct pci_dev *pdev = bp->pdev; 12104 12105 bnxt_set_dflt_rss_hash_type(bp); 12106 bnxt_set_dflt_rfs(bp); 12107 12108 bnxt_get_wol_settings(bp); 12109 if (bp->flags & BNXT_FLAG_WOL_CAP) 12110 device_set_wakeup_enable(&pdev->dev, bp->wol); 12111 else 12112 device_set_wakeup_capable(&pdev->dev, false); 12113 12114 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 12115 bnxt_hwrm_coal_params_qcaps(bp); 12116 } 12117 12118 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 12119 12120 int bnxt_fw_init_one(struct bnxt *bp) 12121 { 12122 int rc; 12123 12124 rc = bnxt_fw_init_one_p1(bp); 12125 if (rc) { 12126 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 12127 return rc; 12128 } 12129 rc = bnxt_fw_init_one_p2(bp); 12130 if (rc) { 12131 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 12132 return rc; 12133 } 12134 rc = bnxt_probe_phy(bp, false); 12135 if (rc) 12136 return rc; 12137 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 12138 if (rc) 12139 return rc; 12140 12141 /* In case fw capabilities have changed, destroy the unneeded 12142 * reporters and create newly capable ones. 12143 */ 12144 bnxt_dl_fw_reporters_destroy(bp, false); 12145 bnxt_dl_fw_reporters_create(bp); 12146 bnxt_fw_init_one_p3(bp); 12147 return 0; 12148 } 12149 12150 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 12151 { 12152 struct bnxt_fw_health *fw_health = bp->fw_health; 12153 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 12154 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 12155 u32 reg_type, reg_off, delay_msecs; 12156 12157 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 12158 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 12159 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 12160 switch (reg_type) { 12161 case BNXT_FW_HEALTH_REG_TYPE_CFG: 12162 pci_write_config_dword(bp->pdev, reg_off, val); 12163 break; 12164 case BNXT_FW_HEALTH_REG_TYPE_GRC: 12165 writel(reg_off & BNXT_GRC_BASE_MASK, 12166 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 12167 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 12168 fallthrough; 12169 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 12170 writel(val, bp->bar0 + reg_off); 12171 break; 12172 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 12173 writel(val, bp->bar1 + reg_off); 12174 break; 12175 } 12176 if (delay_msecs) { 12177 pci_read_config_dword(bp->pdev, 0, &val); 12178 msleep(delay_msecs); 12179 } 12180 } 12181 12182 bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 12183 { 12184 struct hwrm_func_qcfg_output *resp; 12185 struct hwrm_func_qcfg_input *req; 12186 bool result = true; /* firmware will enforce if unknown */ 12187 12188 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 12189 return result; 12190 12191 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 12192 return result; 12193 12194 req->fid = cpu_to_le16(0xffff); 12195 resp = hwrm_req_hold(bp, req); 12196 if (!hwrm_req_send(bp, req)) 12197 result = !!(le16_to_cpu(resp->flags) & 12198 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 12199 hwrm_req_drop(bp, req); 12200 return result; 12201 } 12202 12203 static void bnxt_reset_all(struct bnxt *bp) 12204 { 12205 struct bnxt_fw_health *fw_health = bp->fw_health; 12206 int i, rc; 12207 12208 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12209 bnxt_fw_reset_via_optee(bp); 12210 bp->fw_reset_timestamp = jiffies; 12211 return; 12212 } 12213 12214 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 12215 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 12216 bnxt_fw_reset_writel(bp, i); 12217 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 12218 struct hwrm_fw_reset_input *req; 12219 12220 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 12221 if (!rc) { 12222 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 12223 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 12224 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 12225 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 12226 rc = hwrm_req_send(bp, req); 12227 } 12228 if (rc != -ENODEV) 12229 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 12230 } 12231 bp->fw_reset_timestamp = jiffies; 12232 } 12233 12234 static bool bnxt_fw_reset_timeout(struct bnxt *bp) 12235 { 12236 return time_after(jiffies, bp->fw_reset_timestamp + 12237 (bp->fw_reset_max_dsecs * HZ / 10)); 12238 } 12239 12240 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 12241 { 12242 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12243 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { 12244 bnxt_ulp_start(bp, rc); 12245 bnxt_dl_health_fw_status_update(bp, false); 12246 } 12247 bp->fw_reset_state = 0; 12248 dev_close(bp->dev); 12249 } 12250 12251 static void bnxt_fw_reset_task(struct work_struct *work) 12252 { 12253 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 12254 int rc = 0; 12255 12256 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 12257 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 12258 return; 12259 } 12260 12261 switch (bp->fw_reset_state) { 12262 case BNXT_FW_RESET_STATE_POLL_VF: { 12263 int n = bnxt_get_registered_vfs(bp); 12264 int tmo; 12265 12266 if (n < 0) { 12267 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 12268 n, jiffies_to_msecs(jiffies - 12269 bp->fw_reset_timestamp)); 12270 goto fw_reset_abort; 12271 } else if (n > 0) { 12272 if (bnxt_fw_reset_timeout(bp)) { 12273 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12274 bp->fw_reset_state = 0; 12275 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 12276 n); 12277 return; 12278 } 12279 bnxt_queue_fw_reset_work(bp, HZ / 10); 12280 return; 12281 } 12282 bp->fw_reset_timestamp = jiffies; 12283 rtnl_lock(); 12284 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 12285 bnxt_fw_reset_abort(bp, rc); 12286 rtnl_unlock(); 12287 return; 12288 } 12289 bnxt_fw_reset_close(bp); 12290 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 12291 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 12292 tmo = HZ / 10; 12293 } else { 12294 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12295 tmo = bp->fw_reset_min_dsecs * HZ / 10; 12296 } 12297 rtnl_unlock(); 12298 bnxt_queue_fw_reset_work(bp, tmo); 12299 return; 12300 } 12301 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 12302 u32 val; 12303 12304 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12305 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 12306 !bnxt_fw_reset_timeout(bp)) { 12307 bnxt_queue_fw_reset_work(bp, HZ / 5); 12308 return; 12309 } 12310 12311 if (!bp->fw_health->primary) { 12312 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 12313 12314 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12315 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 12316 return; 12317 } 12318 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 12319 } 12320 fallthrough; 12321 case BNXT_FW_RESET_STATE_RESET_FW: 12322 bnxt_reset_all(bp); 12323 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 12324 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 12325 return; 12326 case BNXT_FW_RESET_STATE_ENABLE_DEV: 12327 bnxt_inv_fw_health_reg(bp); 12328 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 12329 !bp->fw_reset_min_dsecs) { 12330 u16 val; 12331 12332 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 12333 if (val == 0xffff) { 12334 if (bnxt_fw_reset_timeout(bp)) { 12335 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 12336 rc = -ETIMEDOUT; 12337 goto fw_reset_abort; 12338 } 12339 bnxt_queue_fw_reset_work(bp, HZ / 1000); 12340 return; 12341 } 12342 } 12343 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 12344 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 12345 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 12346 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 12347 bnxt_dl_remote_reload(bp); 12348 if (pci_enable_device(bp->pdev)) { 12349 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 12350 rc = -ENODEV; 12351 goto fw_reset_abort; 12352 } 12353 pci_set_master(bp->pdev); 12354 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 12355 fallthrough; 12356 case BNXT_FW_RESET_STATE_POLL_FW: 12357 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 12358 rc = bnxt_hwrm_poll(bp); 12359 if (rc) { 12360 if (bnxt_fw_reset_timeout(bp)) { 12361 netdev_err(bp->dev, "Firmware reset aborted\n"); 12362 goto fw_reset_abort_status; 12363 } 12364 bnxt_queue_fw_reset_work(bp, HZ / 5); 12365 return; 12366 } 12367 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 12368 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 12369 fallthrough; 12370 case BNXT_FW_RESET_STATE_OPENING: 12371 while (!rtnl_trylock()) { 12372 bnxt_queue_fw_reset_work(bp, HZ / 10); 12373 return; 12374 } 12375 rc = bnxt_open(bp->dev); 12376 if (rc) { 12377 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 12378 bnxt_fw_reset_abort(bp, rc); 12379 rtnl_unlock(); 12380 return; 12381 } 12382 12383 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 12384 bp->fw_health->enabled) { 12385 bp->fw_health->last_fw_reset_cnt = 12386 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 12387 } 12388 bp->fw_reset_state = 0; 12389 /* Make sure fw_reset_state is 0 before clearing the flag */ 12390 smp_mb__before_atomic(); 12391 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12392 bnxt_ulp_start(bp, 0); 12393 bnxt_reenable_sriov(bp); 12394 bnxt_vf_reps_alloc(bp); 12395 bnxt_vf_reps_open(bp); 12396 bnxt_ptp_reapply_pps(bp); 12397 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 12398 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 12399 bnxt_dl_health_fw_recovery_done(bp); 12400 bnxt_dl_health_fw_status_update(bp, true); 12401 } 12402 rtnl_unlock(); 12403 break; 12404 } 12405 return; 12406 12407 fw_reset_abort_status: 12408 if (bp->fw_health->status_reliable || 12409 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 12410 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12411 12412 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 12413 } 12414 fw_reset_abort: 12415 rtnl_lock(); 12416 bnxt_fw_reset_abort(bp, rc); 12417 rtnl_unlock(); 12418 } 12419 12420 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 12421 { 12422 int rc; 12423 struct bnxt *bp = netdev_priv(dev); 12424 12425 SET_NETDEV_DEV(dev, &pdev->dev); 12426 12427 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 12428 rc = pci_enable_device(pdev); 12429 if (rc) { 12430 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 12431 goto init_err; 12432 } 12433 12434 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 12435 dev_err(&pdev->dev, 12436 "Cannot find PCI device base address, aborting\n"); 12437 rc = -ENODEV; 12438 goto init_err_disable; 12439 } 12440 12441 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 12442 if (rc) { 12443 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 12444 goto init_err_disable; 12445 } 12446 12447 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 12448 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 12449 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 12450 rc = -EIO; 12451 goto init_err_release; 12452 } 12453 12454 pci_set_master(pdev); 12455 12456 bp->dev = dev; 12457 bp->pdev = pdev; 12458 12459 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 12460 * determines the BAR size. 12461 */ 12462 bp->bar0 = pci_ioremap_bar(pdev, 0); 12463 if (!bp->bar0) { 12464 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 12465 rc = -ENOMEM; 12466 goto init_err_release; 12467 } 12468 12469 bp->bar2 = pci_ioremap_bar(pdev, 4); 12470 if (!bp->bar2) { 12471 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 12472 rc = -ENOMEM; 12473 goto init_err_release; 12474 } 12475 12476 pci_enable_pcie_error_reporting(pdev); 12477 12478 INIT_WORK(&bp->sp_task, bnxt_sp_task); 12479 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 12480 12481 spin_lock_init(&bp->ntp_fltr_lock); 12482 #if BITS_PER_LONG == 32 12483 spin_lock_init(&bp->db_lock); 12484 #endif 12485 12486 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 12487 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 12488 12489 timer_setup(&bp->timer, bnxt_timer, 0); 12490 bp->current_interval = BNXT_TIMER_INTERVAL; 12491 12492 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 12493 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 12494 12495 clear_bit(BNXT_STATE_OPEN, &bp->state); 12496 return 0; 12497 12498 init_err_release: 12499 bnxt_unmap_bars(bp, pdev); 12500 pci_release_regions(pdev); 12501 12502 init_err_disable: 12503 pci_disable_device(pdev); 12504 12505 init_err: 12506 return rc; 12507 } 12508 12509 /* rtnl_lock held */ 12510 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 12511 { 12512 struct sockaddr *addr = p; 12513 struct bnxt *bp = netdev_priv(dev); 12514 int rc = 0; 12515 12516 if (!is_valid_ether_addr(addr->sa_data)) 12517 return -EADDRNOTAVAIL; 12518 12519 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 12520 return 0; 12521 12522 rc = bnxt_approve_mac(bp, addr->sa_data, true); 12523 if (rc) 12524 return rc; 12525 12526 eth_hw_addr_set(dev, addr->sa_data); 12527 if (netif_running(dev)) { 12528 bnxt_close_nic(bp, false, false); 12529 rc = bnxt_open_nic(bp, false, false); 12530 } 12531 12532 return rc; 12533 } 12534 12535 /* rtnl_lock held */ 12536 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 12537 { 12538 struct bnxt *bp = netdev_priv(dev); 12539 12540 if (netif_running(dev)) 12541 bnxt_close_nic(bp, true, false); 12542 12543 dev->mtu = new_mtu; 12544 bnxt_set_ring_params(bp); 12545 12546 if (netif_running(dev)) 12547 return bnxt_open_nic(bp, true, false); 12548 12549 return 0; 12550 } 12551 12552 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 12553 { 12554 struct bnxt *bp = netdev_priv(dev); 12555 bool sh = false; 12556 int rc; 12557 12558 if (tc > bp->max_tc) { 12559 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 12560 tc, bp->max_tc); 12561 return -EINVAL; 12562 } 12563 12564 if (netdev_get_num_tc(dev) == tc) 12565 return 0; 12566 12567 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 12568 sh = true; 12569 12570 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 12571 sh, tc, bp->tx_nr_rings_xdp); 12572 if (rc) 12573 return rc; 12574 12575 /* Needs to close the device and do hw resource re-allocations */ 12576 if (netif_running(bp->dev)) 12577 bnxt_close_nic(bp, true, false); 12578 12579 if (tc) { 12580 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 12581 netdev_set_num_tc(dev, tc); 12582 } else { 12583 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 12584 netdev_reset_tc(dev); 12585 } 12586 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 12587 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 12588 bp->tx_nr_rings + bp->rx_nr_rings; 12589 12590 if (netif_running(bp->dev)) 12591 return bnxt_open_nic(bp, true, false); 12592 12593 return 0; 12594 } 12595 12596 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 12597 void *cb_priv) 12598 { 12599 struct bnxt *bp = cb_priv; 12600 12601 if (!bnxt_tc_flower_enabled(bp) || 12602 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 12603 return -EOPNOTSUPP; 12604 12605 switch (type) { 12606 case TC_SETUP_CLSFLOWER: 12607 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 12608 default: 12609 return -EOPNOTSUPP; 12610 } 12611 } 12612 12613 LIST_HEAD(bnxt_block_cb_list); 12614 12615 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 12616 void *type_data) 12617 { 12618 struct bnxt *bp = netdev_priv(dev); 12619 12620 switch (type) { 12621 case TC_SETUP_BLOCK: 12622 return flow_block_cb_setup_simple(type_data, 12623 &bnxt_block_cb_list, 12624 bnxt_setup_tc_block_cb, 12625 bp, bp, true); 12626 case TC_SETUP_QDISC_MQPRIO: { 12627 struct tc_mqprio_qopt *mqprio = type_data; 12628 12629 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 12630 12631 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 12632 } 12633 default: 12634 return -EOPNOTSUPP; 12635 } 12636 } 12637 12638 #ifdef CONFIG_RFS_ACCEL 12639 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 12640 struct bnxt_ntuple_filter *f2) 12641 { 12642 struct flow_keys *keys1 = &f1->fkeys; 12643 struct flow_keys *keys2 = &f2->fkeys; 12644 12645 if (keys1->basic.n_proto != keys2->basic.n_proto || 12646 keys1->basic.ip_proto != keys2->basic.ip_proto) 12647 return false; 12648 12649 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 12650 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 12651 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 12652 return false; 12653 } else { 12654 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 12655 sizeof(keys1->addrs.v6addrs.src)) || 12656 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 12657 sizeof(keys1->addrs.v6addrs.dst))) 12658 return false; 12659 } 12660 12661 if (keys1->ports.ports == keys2->ports.ports && 12662 keys1->control.flags == keys2->control.flags && 12663 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 12664 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 12665 return true; 12666 12667 return false; 12668 } 12669 12670 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 12671 u16 rxq_index, u32 flow_id) 12672 { 12673 struct bnxt *bp = netdev_priv(dev); 12674 struct bnxt_ntuple_filter *fltr, *new_fltr; 12675 struct flow_keys *fkeys; 12676 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 12677 int rc = 0, idx, bit_id, l2_idx = 0; 12678 struct hlist_head *head; 12679 u32 flags; 12680 12681 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 12682 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 12683 int off = 0, j; 12684 12685 netif_addr_lock_bh(dev); 12686 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 12687 if (ether_addr_equal(eth->h_dest, 12688 vnic->uc_list + off)) { 12689 l2_idx = j + 1; 12690 break; 12691 } 12692 } 12693 netif_addr_unlock_bh(dev); 12694 if (!l2_idx) 12695 return -EINVAL; 12696 } 12697 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 12698 if (!new_fltr) 12699 return -ENOMEM; 12700 12701 fkeys = &new_fltr->fkeys; 12702 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 12703 rc = -EPROTONOSUPPORT; 12704 goto err_free; 12705 } 12706 12707 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 12708 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 12709 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 12710 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 12711 rc = -EPROTONOSUPPORT; 12712 goto err_free; 12713 } 12714 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 12715 bp->hwrm_spec_code < 0x10601) { 12716 rc = -EPROTONOSUPPORT; 12717 goto err_free; 12718 } 12719 flags = fkeys->control.flags; 12720 if (((flags & FLOW_DIS_ENCAPSULATION) && 12721 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 12722 rc = -EPROTONOSUPPORT; 12723 goto err_free; 12724 } 12725 12726 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 12727 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 12728 12729 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 12730 head = &bp->ntp_fltr_hash_tbl[idx]; 12731 rcu_read_lock(); 12732 hlist_for_each_entry_rcu(fltr, head, hash) { 12733 if (bnxt_fltr_match(fltr, new_fltr)) { 12734 rcu_read_unlock(); 12735 rc = 0; 12736 goto err_free; 12737 } 12738 } 12739 rcu_read_unlock(); 12740 12741 spin_lock_bh(&bp->ntp_fltr_lock); 12742 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 12743 BNXT_NTP_FLTR_MAX_FLTR, 0); 12744 if (bit_id < 0) { 12745 spin_unlock_bh(&bp->ntp_fltr_lock); 12746 rc = -ENOMEM; 12747 goto err_free; 12748 } 12749 12750 new_fltr->sw_id = (u16)bit_id; 12751 new_fltr->flow_id = flow_id; 12752 new_fltr->l2_fltr_idx = l2_idx; 12753 new_fltr->rxq = rxq_index; 12754 hlist_add_head_rcu(&new_fltr->hash, head); 12755 bp->ntp_fltr_count++; 12756 spin_unlock_bh(&bp->ntp_fltr_lock); 12757 12758 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 12759 bnxt_queue_sp_work(bp); 12760 12761 return new_fltr->sw_id; 12762 12763 err_free: 12764 kfree(new_fltr); 12765 return rc; 12766 } 12767 12768 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12769 { 12770 int i; 12771 12772 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 12773 struct hlist_head *head; 12774 struct hlist_node *tmp; 12775 struct bnxt_ntuple_filter *fltr; 12776 int rc; 12777 12778 head = &bp->ntp_fltr_hash_tbl[i]; 12779 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 12780 bool del = false; 12781 12782 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 12783 if (rps_may_expire_flow(bp->dev, fltr->rxq, 12784 fltr->flow_id, 12785 fltr->sw_id)) { 12786 bnxt_hwrm_cfa_ntuple_filter_free(bp, 12787 fltr); 12788 del = true; 12789 } 12790 } else { 12791 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 12792 fltr); 12793 if (rc) 12794 del = true; 12795 else 12796 set_bit(BNXT_FLTR_VALID, &fltr->state); 12797 } 12798 12799 if (del) { 12800 spin_lock_bh(&bp->ntp_fltr_lock); 12801 hlist_del_rcu(&fltr->hash); 12802 bp->ntp_fltr_count--; 12803 spin_unlock_bh(&bp->ntp_fltr_lock); 12804 synchronize_rcu(); 12805 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 12806 kfree(fltr); 12807 } 12808 } 12809 } 12810 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 12811 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 12812 } 12813 12814 #else 12815 12816 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 12817 { 12818 } 12819 12820 #endif /* CONFIG_RFS_ACCEL */ 12821 12822 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 12823 { 12824 struct bnxt *bp = netdev_priv(netdev); 12825 struct udp_tunnel_info ti; 12826 unsigned int cmd; 12827 12828 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 12829 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 12830 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 12831 else 12832 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 12833 12834 if (ti.port) 12835 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 12836 12837 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 12838 } 12839 12840 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 12841 .sync_table = bnxt_udp_tunnel_sync, 12842 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 12843 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 12844 .tables = { 12845 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 12846 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 12847 }, 12848 }; 12849 12850 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12851 struct net_device *dev, u32 filter_mask, 12852 int nlflags) 12853 { 12854 struct bnxt *bp = netdev_priv(dev); 12855 12856 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 12857 nlflags, filter_mask, NULL); 12858 } 12859 12860 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 12861 u16 flags, struct netlink_ext_ack *extack) 12862 { 12863 struct bnxt *bp = netdev_priv(dev); 12864 struct nlattr *attr, *br_spec; 12865 int rem, rc = 0; 12866 12867 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 12868 return -EOPNOTSUPP; 12869 12870 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12871 if (!br_spec) 12872 return -EINVAL; 12873 12874 nla_for_each_nested(attr, br_spec, rem) { 12875 u16 mode; 12876 12877 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12878 continue; 12879 12880 if (nla_len(attr) < sizeof(mode)) 12881 return -EINVAL; 12882 12883 mode = nla_get_u16(attr); 12884 if (mode == bp->br_mode) 12885 break; 12886 12887 rc = bnxt_hwrm_set_br_mode(bp, mode); 12888 if (!rc) 12889 bp->br_mode = mode; 12890 break; 12891 } 12892 return rc; 12893 } 12894 12895 int bnxt_get_port_parent_id(struct net_device *dev, 12896 struct netdev_phys_item_id *ppid) 12897 { 12898 struct bnxt *bp = netdev_priv(dev); 12899 12900 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 12901 return -EOPNOTSUPP; 12902 12903 /* The PF and it's VF-reps only support the switchdev framework */ 12904 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 12905 return -EOPNOTSUPP; 12906 12907 ppid->id_len = sizeof(bp->dsn); 12908 memcpy(ppid->id, bp->dsn, ppid->id_len); 12909 12910 return 0; 12911 } 12912 12913 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 12914 { 12915 struct bnxt *bp = netdev_priv(dev); 12916 12917 return &bp->dl_port; 12918 } 12919 12920 static const struct net_device_ops bnxt_netdev_ops = { 12921 .ndo_open = bnxt_open, 12922 .ndo_start_xmit = bnxt_start_xmit, 12923 .ndo_stop = bnxt_close, 12924 .ndo_get_stats64 = bnxt_get_stats64, 12925 .ndo_set_rx_mode = bnxt_set_rx_mode, 12926 .ndo_eth_ioctl = bnxt_ioctl, 12927 .ndo_validate_addr = eth_validate_addr, 12928 .ndo_set_mac_address = bnxt_change_mac_addr, 12929 .ndo_change_mtu = bnxt_change_mtu, 12930 .ndo_fix_features = bnxt_fix_features, 12931 .ndo_set_features = bnxt_set_features, 12932 .ndo_features_check = bnxt_features_check, 12933 .ndo_tx_timeout = bnxt_tx_timeout, 12934 #ifdef CONFIG_BNXT_SRIOV 12935 .ndo_get_vf_config = bnxt_get_vf_config, 12936 .ndo_set_vf_mac = bnxt_set_vf_mac, 12937 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 12938 .ndo_set_vf_rate = bnxt_set_vf_bw, 12939 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 12940 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 12941 .ndo_set_vf_trust = bnxt_set_vf_trust, 12942 #endif 12943 .ndo_setup_tc = bnxt_setup_tc, 12944 #ifdef CONFIG_RFS_ACCEL 12945 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 12946 #endif 12947 .ndo_bpf = bnxt_xdp, 12948 .ndo_xdp_xmit = bnxt_xdp_xmit, 12949 .ndo_bridge_getlink = bnxt_bridge_getlink, 12950 .ndo_bridge_setlink = bnxt_bridge_setlink, 12951 .ndo_get_devlink_port = bnxt_get_devlink_port, 12952 }; 12953 12954 static void bnxt_remove_one(struct pci_dev *pdev) 12955 { 12956 struct net_device *dev = pci_get_drvdata(pdev); 12957 struct bnxt *bp = netdev_priv(dev); 12958 12959 if (BNXT_PF(bp)) 12960 bnxt_sriov_disable(bp); 12961 12962 if (BNXT_PF(bp)) 12963 devlink_port_type_clear(&bp->dl_port); 12964 12965 bnxt_ptp_clear(bp); 12966 pci_disable_pcie_error_reporting(pdev); 12967 unregister_netdev(dev); 12968 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 12969 /* Flush any pending tasks */ 12970 cancel_work_sync(&bp->sp_task); 12971 cancel_delayed_work_sync(&bp->fw_reset_task); 12972 bp->sp_event = 0; 12973 12974 bnxt_dl_fw_reporters_destroy(bp, true); 12975 bnxt_dl_unregister(bp); 12976 bnxt_shutdown_tc(bp); 12977 12978 bnxt_clear_int_mode(bp); 12979 bnxt_hwrm_func_drv_unrgtr(bp); 12980 bnxt_free_hwrm_resources(bp); 12981 bnxt_ethtool_free(bp); 12982 bnxt_dcb_free(bp); 12983 kfree(bp->edev); 12984 bp->edev = NULL; 12985 kfree(bp->ptp_cfg); 12986 bp->ptp_cfg = NULL; 12987 kfree(bp->fw_health); 12988 bp->fw_health = NULL; 12989 bnxt_cleanup_pci(bp); 12990 bnxt_free_ctx_mem(bp); 12991 kfree(bp->ctx); 12992 bp->ctx = NULL; 12993 kfree(bp->rss_indir_tbl); 12994 bp->rss_indir_tbl = NULL; 12995 bnxt_free_port_stats(bp); 12996 free_netdev(dev); 12997 } 12998 12999 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 13000 { 13001 int rc = 0; 13002 struct bnxt_link_info *link_info = &bp->link_info; 13003 13004 bp->phy_flags = 0; 13005 rc = bnxt_hwrm_phy_qcaps(bp); 13006 if (rc) { 13007 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 13008 rc); 13009 return rc; 13010 } 13011 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 13012 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 13013 else 13014 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 13015 if (!fw_dflt) 13016 return 0; 13017 13018 mutex_lock(&bp->link_lock); 13019 rc = bnxt_update_link(bp, false); 13020 if (rc) { 13021 mutex_unlock(&bp->link_lock); 13022 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 13023 rc); 13024 return rc; 13025 } 13026 13027 /* Older firmware does not have supported_auto_speeds, so assume 13028 * that all supported speeds can be autonegotiated. 13029 */ 13030 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 13031 link_info->support_auto_speeds = link_info->support_speeds; 13032 13033 bnxt_init_ethtool_link_settings(bp); 13034 mutex_unlock(&bp->link_lock); 13035 return 0; 13036 } 13037 13038 static int bnxt_get_max_irq(struct pci_dev *pdev) 13039 { 13040 u16 ctrl; 13041 13042 if (!pdev->msix_cap) 13043 return 1; 13044 13045 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 13046 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 13047 } 13048 13049 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13050 int *max_cp) 13051 { 13052 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 13053 int max_ring_grps = 0, max_irq; 13054 13055 *max_tx = hw_resc->max_tx_rings; 13056 *max_rx = hw_resc->max_rx_rings; 13057 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 13058 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 13059 bnxt_get_ulp_msix_num(bp), 13060 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 13061 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 13062 *max_cp = min_t(int, *max_cp, max_irq); 13063 max_ring_grps = hw_resc->max_hw_ring_grps; 13064 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 13065 *max_cp -= 1; 13066 *max_rx -= 2; 13067 } 13068 if (bp->flags & BNXT_FLAG_AGG_RINGS) 13069 *max_rx >>= 1; 13070 if (bp->flags & BNXT_FLAG_CHIP_P5) { 13071 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 13072 /* On P5 chips, max_cp output param should be available NQs */ 13073 *max_cp = max_irq; 13074 } 13075 *max_rx = min_t(int, *max_rx, max_ring_grps); 13076 } 13077 13078 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 13079 { 13080 int rx, tx, cp; 13081 13082 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 13083 *max_rx = rx; 13084 *max_tx = tx; 13085 if (!rx || !tx || !cp) 13086 return -ENOMEM; 13087 13088 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 13089 } 13090 13091 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 13092 bool shared) 13093 { 13094 int rc; 13095 13096 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13097 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 13098 /* Not enough rings, try disabling agg rings. */ 13099 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 13100 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 13101 if (rc) { 13102 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 13103 bp->flags |= BNXT_FLAG_AGG_RINGS; 13104 return rc; 13105 } 13106 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 13107 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13108 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13109 bnxt_set_ring_params(bp); 13110 } 13111 13112 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 13113 int max_cp, max_stat, max_irq; 13114 13115 /* Reserve minimum resources for RoCE */ 13116 max_cp = bnxt_get_max_func_cp_rings(bp); 13117 max_stat = bnxt_get_max_func_stat_ctxs(bp); 13118 max_irq = bnxt_get_max_func_irqs(bp); 13119 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 13120 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 13121 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 13122 return 0; 13123 13124 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 13125 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 13126 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 13127 max_cp = min_t(int, max_cp, max_irq); 13128 max_cp = min_t(int, max_cp, max_stat); 13129 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 13130 if (rc) 13131 rc = 0; 13132 } 13133 return rc; 13134 } 13135 13136 /* In initial default shared ring setting, each shared ring must have a 13137 * RX/TX ring pair. 13138 */ 13139 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 13140 { 13141 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 13142 bp->rx_nr_rings = bp->cp_nr_rings; 13143 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 13144 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13145 } 13146 13147 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 13148 { 13149 int dflt_rings, max_rx_rings, max_tx_rings, rc; 13150 13151 if (!bnxt_can_reserve_rings(bp)) 13152 return 0; 13153 13154 if (sh) 13155 bp->flags |= BNXT_FLAG_SHARED_RINGS; 13156 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 13157 /* Reduce default rings on multi-port cards so that total default 13158 * rings do not exceed CPU count. 13159 */ 13160 if (bp->port_count > 1) { 13161 int max_rings = 13162 max_t(int, num_online_cpus() / bp->port_count, 1); 13163 13164 dflt_rings = min_t(int, dflt_rings, max_rings); 13165 } 13166 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 13167 if (rc) 13168 return rc; 13169 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 13170 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 13171 if (sh) 13172 bnxt_trim_dflt_sh_rings(bp); 13173 else 13174 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 13175 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 13176 13177 rc = __bnxt_reserve_rings(bp); 13178 if (rc && rc != -ENODEV) 13179 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 13180 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13181 if (sh) 13182 bnxt_trim_dflt_sh_rings(bp); 13183 13184 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 13185 if (bnxt_need_reserve_rings(bp)) { 13186 rc = __bnxt_reserve_rings(bp); 13187 if (rc && rc != -ENODEV) 13188 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 13189 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13190 } 13191 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 13192 bp->rx_nr_rings++; 13193 bp->cp_nr_rings++; 13194 } 13195 if (rc) { 13196 bp->tx_nr_rings = 0; 13197 bp->rx_nr_rings = 0; 13198 } 13199 return rc; 13200 } 13201 13202 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 13203 { 13204 int rc; 13205 13206 if (bp->tx_nr_rings) 13207 return 0; 13208 13209 bnxt_ulp_irq_stop(bp); 13210 bnxt_clear_int_mode(bp); 13211 rc = bnxt_set_dflt_rings(bp, true); 13212 if (rc) { 13213 if (BNXT_VF(bp) && rc == -ENODEV) 13214 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13215 else 13216 netdev_err(bp->dev, "Not enough rings available.\n"); 13217 goto init_dflt_ring_err; 13218 } 13219 rc = bnxt_init_int_mode(bp); 13220 if (rc) 13221 goto init_dflt_ring_err; 13222 13223 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13224 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 13225 bp->flags |= BNXT_FLAG_RFS; 13226 bp->dev->features |= NETIF_F_NTUPLE; 13227 } 13228 init_dflt_ring_err: 13229 bnxt_ulp_irq_restart(bp, rc); 13230 return rc; 13231 } 13232 13233 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 13234 { 13235 int rc; 13236 13237 ASSERT_RTNL(); 13238 bnxt_hwrm_func_qcaps(bp); 13239 13240 if (netif_running(bp->dev)) 13241 __bnxt_close_nic(bp, true, false); 13242 13243 bnxt_ulp_irq_stop(bp); 13244 bnxt_clear_int_mode(bp); 13245 rc = bnxt_init_int_mode(bp); 13246 bnxt_ulp_irq_restart(bp, rc); 13247 13248 if (netif_running(bp->dev)) { 13249 if (rc) 13250 dev_close(bp->dev); 13251 else 13252 rc = bnxt_open_nic(bp, true, false); 13253 } 13254 13255 return rc; 13256 } 13257 13258 static int bnxt_init_mac_addr(struct bnxt *bp) 13259 { 13260 int rc = 0; 13261 13262 if (BNXT_PF(bp)) { 13263 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 13264 } else { 13265 #ifdef CONFIG_BNXT_SRIOV 13266 struct bnxt_vf_info *vf = &bp->vf; 13267 bool strict_approval = true; 13268 13269 if (is_valid_ether_addr(vf->mac_addr)) { 13270 /* overwrite netdev dev_addr with admin VF MAC */ 13271 eth_hw_addr_set(bp->dev, vf->mac_addr); 13272 /* Older PF driver or firmware may not approve this 13273 * correctly. 13274 */ 13275 strict_approval = false; 13276 } else { 13277 eth_hw_addr_random(bp->dev); 13278 } 13279 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 13280 #endif 13281 } 13282 return rc; 13283 } 13284 13285 static void bnxt_vpd_read_info(struct bnxt *bp) 13286 { 13287 struct pci_dev *pdev = bp->pdev; 13288 unsigned int vpd_size, kw_len; 13289 int pos, size; 13290 u8 *vpd_data; 13291 13292 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 13293 if (IS_ERR(vpd_data)) { 13294 pci_warn(pdev, "Unable to read VPD\n"); 13295 return; 13296 } 13297 13298 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13299 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 13300 if (pos < 0) 13301 goto read_sn; 13302 13303 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13304 memcpy(bp->board_partno, &vpd_data[pos], size); 13305 13306 read_sn: 13307 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 13308 PCI_VPD_RO_KEYWORD_SERIALNO, 13309 &kw_len); 13310 if (pos < 0) 13311 goto exit; 13312 13313 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 13314 memcpy(bp->board_serialno, &vpd_data[pos], size); 13315 exit: 13316 kfree(vpd_data); 13317 } 13318 13319 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 13320 { 13321 struct pci_dev *pdev = bp->pdev; 13322 u64 qword; 13323 13324 qword = pci_get_dsn(pdev); 13325 if (!qword) { 13326 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 13327 return -EOPNOTSUPP; 13328 } 13329 13330 put_unaligned_le64(qword, dsn); 13331 13332 bp->flags |= BNXT_FLAG_DSN_VALID; 13333 return 0; 13334 } 13335 13336 static int bnxt_map_db_bar(struct bnxt *bp) 13337 { 13338 if (!bp->db_size) 13339 return -ENODEV; 13340 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 13341 if (!bp->bar1) 13342 return -ENOMEM; 13343 return 0; 13344 } 13345 13346 void bnxt_print_device_info(struct bnxt *bp) 13347 { 13348 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 13349 board_info[bp->board_idx].name, 13350 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 13351 13352 pcie_print_link_status(bp->pdev); 13353 } 13354 13355 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 13356 { 13357 struct net_device *dev; 13358 struct bnxt *bp; 13359 int rc, max_irqs; 13360 13361 if (pci_is_bridge(pdev)) 13362 return -ENODEV; 13363 13364 /* Clear any pending DMA transactions from crash kernel 13365 * while loading driver in capture kernel. 13366 */ 13367 if (is_kdump_kernel()) { 13368 pci_clear_master(pdev); 13369 pcie_flr(pdev); 13370 } 13371 13372 max_irqs = bnxt_get_max_irq(pdev); 13373 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 13374 if (!dev) 13375 return -ENOMEM; 13376 13377 bp = netdev_priv(dev); 13378 bp->board_idx = ent->driver_data; 13379 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 13380 bnxt_set_max_func_irqs(bp, max_irqs); 13381 13382 if (bnxt_vf_pciid(bp->board_idx)) 13383 bp->flags |= BNXT_FLAG_VF; 13384 13385 if (pdev->msix_cap) 13386 bp->flags |= BNXT_FLAG_MSIX_CAP; 13387 13388 rc = bnxt_init_board(pdev, dev); 13389 if (rc < 0) 13390 goto init_err_free; 13391 13392 dev->netdev_ops = &bnxt_netdev_ops; 13393 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 13394 dev->ethtool_ops = &bnxt_ethtool_ops; 13395 pci_set_drvdata(pdev, dev); 13396 13397 rc = bnxt_alloc_hwrm_resources(bp); 13398 if (rc) 13399 goto init_err_pci_clean; 13400 13401 mutex_init(&bp->hwrm_cmd_lock); 13402 mutex_init(&bp->link_lock); 13403 13404 rc = bnxt_fw_init_one_p1(bp); 13405 if (rc) 13406 goto init_err_pci_clean; 13407 13408 if (BNXT_PF(bp)) 13409 bnxt_vpd_read_info(bp); 13410 13411 if (BNXT_CHIP_P5(bp)) { 13412 bp->flags |= BNXT_FLAG_CHIP_P5; 13413 if (BNXT_CHIP_SR2(bp)) 13414 bp->flags |= BNXT_FLAG_CHIP_SR2; 13415 } 13416 13417 rc = bnxt_alloc_rss_indir_tbl(bp); 13418 if (rc) 13419 goto init_err_pci_clean; 13420 13421 rc = bnxt_fw_init_one_p2(bp); 13422 if (rc) 13423 goto init_err_pci_clean; 13424 13425 rc = bnxt_map_db_bar(bp); 13426 if (rc) { 13427 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 13428 rc); 13429 goto init_err_pci_clean; 13430 } 13431 13432 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13433 NETIF_F_TSO | NETIF_F_TSO6 | 13434 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13435 NETIF_F_GSO_IPXIP4 | 13436 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13437 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 13438 NETIF_F_RXCSUM | NETIF_F_GRO; 13439 13440 if (BNXT_SUPPORTS_TPA(bp)) 13441 dev->hw_features |= NETIF_F_LRO; 13442 13443 dev->hw_enc_features = 13444 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 13445 NETIF_F_TSO | NETIF_F_TSO6 | 13446 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 13447 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 13448 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 13449 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 13450 13451 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 13452 NETIF_F_GSO_GRE_CSUM; 13453 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 13454 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 13455 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 13456 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 13457 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 13458 if (BNXT_SUPPORTS_TPA(bp)) 13459 dev->hw_features |= NETIF_F_GRO_HW; 13460 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 13461 if (dev->features & NETIF_F_GRO_HW) 13462 dev->features &= ~NETIF_F_LRO; 13463 dev->priv_flags |= IFF_UNICAST_FLT; 13464 13465 #ifdef CONFIG_BNXT_SRIOV 13466 init_waitqueue_head(&bp->sriov_cfg_wait); 13467 mutex_init(&bp->sriov_lock); 13468 #endif 13469 if (BNXT_SUPPORTS_TPA(bp)) { 13470 bp->gro_func = bnxt_gro_func_5730x; 13471 if (BNXT_CHIP_P4(bp)) 13472 bp->gro_func = bnxt_gro_func_5731x; 13473 else if (BNXT_CHIP_P5(bp)) 13474 bp->gro_func = bnxt_gro_func_5750x; 13475 } 13476 if (!BNXT_CHIP_P4_PLUS(bp)) 13477 bp->flags |= BNXT_FLAG_DOUBLE_DB; 13478 13479 rc = bnxt_init_mac_addr(bp); 13480 if (rc) { 13481 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 13482 rc = -EADDRNOTAVAIL; 13483 goto init_err_pci_clean; 13484 } 13485 13486 if (BNXT_PF(bp)) { 13487 /* Read the adapter's DSN to use as the eswitch switch_id */ 13488 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 13489 } 13490 13491 /* MTU range: 60 - FW defined max */ 13492 dev->min_mtu = ETH_ZLEN; 13493 dev->max_mtu = bp->max_mtu; 13494 13495 rc = bnxt_probe_phy(bp, true); 13496 if (rc) 13497 goto init_err_pci_clean; 13498 13499 bnxt_set_rx_skb_mode(bp, false); 13500 bnxt_set_tpa_flags(bp); 13501 bnxt_set_ring_params(bp); 13502 rc = bnxt_set_dflt_rings(bp, true); 13503 if (rc) { 13504 if (BNXT_VF(bp) && rc == -ENODEV) { 13505 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 13506 } else { 13507 netdev_err(bp->dev, "Not enough rings available.\n"); 13508 rc = -ENOMEM; 13509 } 13510 goto init_err_pci_clean; 13511 } 13512 13513 bnxt_fw_init_one_p3(bp); 13514 13515 bnxt_init_dflt_coal(bp); 13516 13517 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13518 bp->flags |= BNXT_FLAG_STRIP_VLAN; 13519 13520 rc = bnxt_init_int_mode(bp); 13521 if (rc) 13522 goto init_err_pci_clean; 13523 13524 /* No TC has been set yet and rings may have been trimmed due to 13525 * limited MSIX, so we re-initialize the TX rings per TC. 13526 */ 13527 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 13528 13529 if (BNXT_PF(bp)) { 13530 if (!bnxt_pf_wq) { 13531 bnxt_pf_wq = 13532 create_singlethread_workqueue("bnxt_pf_wq"); 13533 if (!bnxt_pf_wq) { 13534 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 13535 rc = -ENOMEM; 13536 goto init_err_pci_clean; 13537 } 13538 } 13539 rc = bnxt_init_tc(bp); 13540 if (rc) 13541 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 13542 rc); 13543 } 13544 13545 bnxt_inv_fw_health_reg(bp); 13546 rc = bnxt_dl_register(bp); 13547 if (rc) 13548 goto init_err_dl; 13549 13550 rc = register_netdev(dev); 13551 if (rc) 13552 goto init_err_cleanup; 13553 13554 if (BNXT_PF(bp)) 13555 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 13556 bnxt_dl_fw_reporters_create(bp); 13557 13558 bnxt_print_device_info(bp); 13559 13560 pci_save_state(pdev); 13561 return 0; 13562 13563 init_err_cleanup: 13564 bnxt_dl_unregister(bp); 13565 init_err_dl: 13566 bnxt_shutdown_tc(bp); 13567 bnxt_clear_int_mode(bp); 13568 13569 init_err_pci_clean: 13570 bnxt_hwrm_func_drv_unrgtr(bp); 13571 bnxt_free_hwrm_resources(bp); 13572 bnxt_ethtool_free(bp); 13573 bnxt_ptp_clear(bp); 13574 kfree(bp->ptp_cfg); 13575 bp->ptp_cfg = NULL; 13576 kfree(bp->fw_health); 13577 bp->fw_health = NULL; 13578 bnxt_cleanup_pci(bp); 13579 bnxt_free_ctx_mem(bp); 13580 kfree(bp->ctx); 13581 bp->ctx = NULL; 13582 kfree(bp->rss_indir_tbl); 13583 bp->rss_indir_tbl = NULL; 13584 13585 init_err_free: 13586 free_netdev(dev); 13587 return rc; 13588 } 13589 13590 static void bnxt_shutdown(struct pci_dev *pdev) 13591 { 13592 struct net_device *dev = pci_get_drvdata(pdev); 13593 struct bnxt *bp; 13594 13595 if (!dev) 13596 return; 13597 13598 rtnl_lock(); 13599 bp = netdev_priv(dev); 13600 if (!bp) 13601 goto shutdown_exit; 13602 13603 if (netif_running(dev)) 13604 dev_close(dev); 13605 13606 bnxt_ulp_shutdown(bp); 13607 bnxt_clear_int_mode(bp); 13608 pci_disable_device(pdev); 13609 13610 if (system_state == SYSTEM_POWER_OFF) { 13611 pci_wake_from_d3(pdev, bp->wol); 13612 pci_set_power_state(pdev, PCI_D3hot); 13613 } 13614 13615 shutdown_exit: 13616 rtnl_unlock(); 13617 } 13618 13619 #ifdef CONFIG_PM_SLEEP 13620 static int bnxt_suspend(struct device *device) 13621 { 13622 struct net_device *dev = dev_get_drvdata(device); 13623 struct bnxt *bp = netdev_priv(dev); 13624 int rc = 0; 13625 13626 rtnl_lock(); 13627 bnxt_ulp_stop(bp); 13628 if (netif_running(dev)) { 13629 netif_device_detach(dev); 13630 rc = bnxt_close(dev); 13631 } 13632 bnxt_hwrm_func_drv_unrgtr(bp); 13633 pci_disable_device(bp->pdev); 13634 bnxt_free_ctx_mem(bp); 13635 kfree(bp->ctx); 13636 bp->ctx = NULL; 13637 rtnl_unlock(); 13638 return rc; 13639 } 13640 13641 static int bnxt_resume(struct device *device) 13642 { 13643 struct net_device *dev = dev_get_drvdata(device); 13644 struct bnxt *bp = netdev_priv(dev); 13645 int rc = 0; 13646 13647 rtnl_lock(); 13648 rc = pci_enable_device(bp->pdev); 13649 if (rc) { 13650 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 13651 rc); 13652 goto resume_exit; 13653 } 13654 pci_set_master(bp->pdev); 13655 if (bnxt_hwrm_ver_get(bp)) { 13656 rc = -ENODEV; 13657 goto resume_exit; 13658 } 13659 rc = bnxt_hwrm_func_reset(bp); 13660 if (rc) { 13661 rc = -EBUSY; 13662 goto resume_exit; 13663 } 13664 13665 rc = bnxt_hwrm_func_qcaps(bp); 13666 if (rc) 13667 goto resume_exit; 13668 13669 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 13670 rc = -ENODEV; 13671 goto resume_exit; 13672 } 13673 13674 bnxt_get_wol_settings(bp); 13675 if (netif_running(dev)) { 13676 rc = bnxt_open(dev); 13677 if (!rc) 13678 netif_device_attach(dev); 13679 } 13680 13681 resume_exit: 13682 bnxt_ulp_start(bp, rc); 13683 if (!rc) 13684 bnxt_reenable_sriov(bp); 13685 rtnl_unlock(); 13686 return rc; 13687 } 13688 13689 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 13690 #define BNXT_PM_OPS (&bnxt_pm_ops) 13691 13692 #else 13693 13694 #define BNXT_PM_OPS NULL 13695 13696 #endif /* CONFIG_PM_SLEEP */ 13697 13698 /** 13699 * bnxt_io_error_detected - called when PCI error is detected 13700 * @pdev: Pointer to PCI device 13701 * @state: The current pci connection state 13702 * 13703 * This function is called after a PCI bus error affecting 13704 * this device has been detected. 13705 */ 13706 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 13707 pci_channel_state_t state) 13708 { 13709 struct net_device *netdev = pci_get_drvdata(pdev); 13710 struct bnxt *bp = netdev_priv(netdev); 13711 13712 netdev_info(netdev, "PCI I/O error detected\n"); 13713 13714 rtnl_lock(); 13715 netif_device_detach(netdev); 13716 13717 bnxt_ulp_stop(bp); 13718 13719 if (state == pci_channel_io_perm_failure) { 13720 rtnl_unlock(); 13721 return PCI_ERS_RESULT_DISCONNECT; 13722 } 13723 13724 if (state == pci_channel_io_frozen) 13725 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 13726 13727 if (netif_running(netdev)) 13728 bnxt_close(netdev); 13729 13730 if (pci_is_enabled(pdev)) 13731 pci_disable_device(pdev); 13732 bnxt_free_ctx_mem(bp); 13733 kfree(bp->ctx); 13734 bp->ctx = NULL; 13735 rtnl_unlock(); 13736 13737 /* Request a slot slot reset. */ 13738 return PCI_ERS_RESULT_NEED_RESET; 13739 } 13740 13741 /** 13742 * bnxt_io_slot_reset - called after the pci bus has been reset. 13743 * @pdev: Pointer to PCI device 13744 * 13745 * Restart the card from scratch, as if from a cold-boot. 13746 * At this point, the card has exprienced a hard reset, 13747 * followed by fixups by BIOS, and has its config space 13748 * set up identically to what it was at cold boot. 13749 */ 13750 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 13751 { 13752 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 13753 struct net_device *netdev = pci_get_drvdata(pdev); 13754 struct bnxt *bp = netdev_priv(netdev); 13755 int err = 0, off; 13756 13757 netdev_info(bp->dev, "PCI Slot Reset\n"); 13758 13759 rtnl_lock(); 13760 13761 if (pci_enable_device(pdev)) { 13762 dev_err(&pdev->dev, 13763 "Cannot re-enable PCI device after reset.\n"); 13764 } else { 13765 pci_set_master(pdev); 13766 /* Upon fatal error, our device internal logic that latches to 13767 * BAR value is getting reset and will restore only upon 13768 * rewritting the BARs. 13769 * 13770 * As pci_restore_state() does not re-write the BARs if the 13771 * value is same as saved value earlier, driver needs to 13772 * write the BARs to 0 to force restore, in case of fatal error. 13773 */ 13774 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 13775 &bp->state)) { 13776 for (off = PCI_BASE_ADDRESS_0; 13777 off <= PCI_BASE_ADDRESS_5; off += 4) 13778 pci_write_config_dword(bp->pdev, off, 0); 13779 } 13780 pci_restore_state(pdev); 13781 pci_save_state(pdev); 13782 13783 err = bnxt_hwrm_func_reset(bp); 13784 if (!err) 13785 result = PCI_ERS_RESULT_RECOVERED; 13786 } 13787 13788 rtnl_unlock(); 13789 13790 return result; 13791 } 13792 13793 /** 13794 * bnxt_io_resume - called when traffic can start flowing again. 13795 * @pdev: Pointer to PCI device 13796 * 13797 * This callback is called when the error recovery driver tells 13798 * us that its OK to resume normal operation. 13799 */ 13800 static void bnxt_io_resume(struct pci_dev *pdev) 13801 { 13802 struct net_device *netdev = pci_get_drvdata(pdev); 13803 struct bnxt *bp = netdev_priv(netdev); 13804 int err; 13805 13806 netdev_info(bp->dev, "PCI Slot Resume\n"); 13807 rtnl_lock(); 13808 13809 err = bnxt_hwrm_func_qcaps(bp); 13810 if (!err && netif_running(netdev)) 13811 err = bnxt_open(netdev); 13812 13813 bnxt_ulp_start(bp, err); 13814 if (!err) { 13815 bnxt_reenable_sriov(bp); 13816 netif_device_attach(netdev); 13817 } 13818 13819 rtnl_unlock(); 13820 } 13821 13822 static const struct pci_error_handlers bnxt_err_handler = { 13823 .error_detected = bnxt_io_error_detected, 13824 .slot_reset = bnxt_io_slot_reset, 13825 .resume = bnxt_io_resume 13826 }; 13827 13828 static struct pci_driver bnxt_pci_driver = { 13829 .name = DRV_MODULE_NAME, 13830 .id_table = bnxt_pci_tbl, 13831 .probe = bnxt_init_one, 13832 .remove = bnxt_remove_one, 13833 .shutdown = bnxt_shutdown, 13834 .driver.pm = BNXT_PM_OPS, 13835 .err_handler = &bnxt_err_handler, 13836 #if defined(CONFIG_BNXT_SRIOV) 13837 .sriov_configure = bnxt_sriov_configure, 13838 #endif 13839 }; 13840 13841 static int __init bnxt_init(void) 13842 { 13843 bnxt_debug_init(); 13844 return pci_register_driver(&bnxt_pci_driver); 13845 } 13846 13847 static void __exit bnxt_exit(void) 13848 { 13849 pci_unregister_driver(&bnxt_pci_driver); 13850 if (bnxt_pf_wq) 13851 destroy_workqueue(bnxt_pf_wq); 13852 bnxt_debug_exit(); 13853 } 13854 13855 module_init(bnxt_init); 13856 module_exit(bnxt_exit); 13857