1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 13 #include <linux/stringify.h> 14 #include <linux/kernel.h> 15 #include <linux/timer.h> 16 #include <linux/errno.h> 17 #include <linux/ioport.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 #include <linux/interrupt.h> 21 #include <linux/pci.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/skbuff.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/bitops.h> 27 #include <linux/io.h> 28 #include <linux/irq.h> 29 #include <linux/delay.h> 30 #include <asm/byteorder.h> 31 #include <asm/page.h> 32 #include <linux/time.h> 33 #include <linux/mii.h> 34 #include <linux/mdio.h> 35 #include <linux/if.h> 36 #include <linux/if_vlan.h> 37 #include <linux/if_bridge.h> 38 #include <linux/rtc.h> 39 #include <linux/bpf.h> 40 #include <net/ip.h> 41 #include <net/tcp.h> 42 #include <net/udp.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 #include <net/udp_tunnel.h> 46 #include <linux/workqueue.h> 47 #include <linux/prefetch.h> 48 #include <linux/cache.h> 49 #include <linux/log2.h> 50 #include <linux/aer.h> 51 #include <linux/bitmap.h> 52 #include <linux/cpu_rmap.h> 53 #include <linux/cpumask.h> 54 #include <net/pkt_cls.h> 55 #include <linux/hwmon.h> 56 #include <linux/hwmon-sysfs.h> 57 #include <net/page_pool.h> 58 59 #include "bnxt_hsi.h" 60 #include "bnxt.h" 61 #include "bnxt_ulp.h" 62 #include "bnxt_sriov.h" 63 #include "bnxt_ethtool.h" 64 #include "bnxt_dcb.h" 65 #include "bnxt_xdp.h" 66 #include "bnxt_vfr.h" 67 #include "bnxt_tc.h" 68 #include "bnxt_devlink.h" 69 #include "bnxt_debugfs.h" 70 71 #define BNXT_TX_TIMEOUT (5 * HZ) 72 73 MODULE_LICENSE("GPL"); 74 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 75 76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 78 #define BNXT_RX_COPY_THRESH 256 79 80 #define BNXT_TX_PUSH_THRESH 164 81 82 enum board_idx { 83 BCM57301, 84 BCM57302, 85 BCM57304, 86 BCM57417_NPAR, 87 BCM58700, 88 BCM57311, 89 BCM57312, 90 BCM57402, 91 BCM57404, 92 BCM57406, 93 BCM57402_NPAR, 94 BCM57407, 95 BCM57412, 96 BCM57414, 97 BCM57416, 98 BCM57417, 99 BCM57412_NPAR, 100 BCM57314, 101 BCM57417_SFP, 102 BCM57416_SFP, 103 BCM57404_NPAR, 104 BCM57406_NPAR, 105 BCM57407_SFP, 106 BCM57407_NPAR, 107 BCM57414_NPAR, 108 BCM57416_NPAR, 109 BCM57452, 110 BCM57454, 111 BCM5745x_NPAR, 112 BCM57508, 113 BCM57504, 114 BCM57502, 115 BCM57508_NPAR, 116 BCM57504_NPAR, 117 BCM57502_NPAR, 118 BCM58802, 119 BCM58804, 120 BCM58808, 121 NETXTREME_E_VF, 122 NETXTREME_C_VF, 123 NETXTREME_S_VF, 124 NETXTREME_E_P5_VF, 125 }; 126 127 /* indexed by enum above */ 128 static const struct { 129 char *name; 130 } board_info[] = { 131 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 132 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 133 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 134 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 135 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 136 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 137 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 138 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 139 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 140 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 141 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 142 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 143 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 144 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 145 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 146 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 147 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 148 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 149 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 150 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 151 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 152 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 153 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 154 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 155 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 156 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 157 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 158 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 159 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 160 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 161 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 162 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 163 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 164 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 165 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 166 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 167 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 168 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 169 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 170 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 171 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 172 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 173 }; 174 175 static const struct pci_device_id bnxt_pci_tbl[] = { 176 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 177 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 179 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 181 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 182 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 183 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 184 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 185 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 186 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 187 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 188 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 189 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 190 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 191 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 192 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 193 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 194 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 195 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 196 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 197 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 198 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 199 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 200 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 201 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 202 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 203 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 204 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 205 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 206 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 207 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 208 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 209 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 210 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 211 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 212 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 213 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 214 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR }, 215 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 216 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR }, 217 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR }, 218 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 219 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR }, 220 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 221 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 222 #ifdef CONFIG_BNXT_SRIOV 223 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 224 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 225 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 226 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 227 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 228 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 229 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 230 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 231 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 232 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 233 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 234 #endif 235 { 0 } 236 }; 237 238 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 239 240 static const u16 bnxt_vf_req_snif[] = { 241 HWRM_FUNC_CFG, 242 HWRM_FUNC_VF_CFG, 243 HWRM_PORT_PHY_QCFG, 244 HWRM_CFA_L2_FILTER_ALLOC, 245 }; 246 247 static const u16 bnxt_async_events_arr[] = { 248 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 249 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 250 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 251 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 252 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 253 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 254 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 255 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 256 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 257 }; 258 259 static struct workqueue_struct *bnxt_pf_wq; 260 261 static bool bnxt_vf_pciid(enum board_idx idx) 262 { 263 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 264 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF); 265 } 266 267 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 268 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 269 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 270 271 #define BNXT_CP_DB_IRQ_DIS(db) \ 272 writel(DB_CP_IRQ_DIS_FLAGS, db) 273 274 #define BNXT_DB_CQ(db, idx) \ 275 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell) 276 277 #define BNXT_DB_NQ_P5(db, idx) \ 278 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell) 279 280 #define BNXT_DB_CQ_ARM(db, idx) \ 281 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell) 282 283 #define BNXT_DB_NQ_ARM_P5(db, idx) \ 284 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell) 285 286 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 287 { 288 if (bp->flags & BNXT_FLAG_CHIP_P5) 289 BNXT_DB_NQ_P5(db, idx); 290 else 291 BNXT_DB_CQ(db, idx); 292 } 293 294 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 295 { 296 if (bp->flags & BNXT_FLAG_CHIP_P5) 297 BNXT_DB_NQ_ARM_P5(db, idx); 298 else 299 BNXT_DB_CQ_ARM(db, idx); 300 } 301 302 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 303 { 304 if (bp->flags & BNXT_FLAG_CHIP_P5) 305 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx), 306 db->doorbell); 307 else 308 BNXT_DB_CQ(db, idx); 309 } 310 311 const u16 bnxt_lhint_arr[] = { 312 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 313 TX_BD_FLAGS_LHINT_512_TO_1023, 314 TX_BD_FLAGS_LHINT_1024_TO_2047, 315 TX_BD_FLAGS_LHINT_1024_TO_2047, 316 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 317 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 318 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 319 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 320 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 321 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 322 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 323 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 324 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 325 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 326 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 327 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 328 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 329 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 330 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 331 }; 332 333 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 334 { 335 struct metadata_dst *md_dst = skb_metadata_dst(skb); 336 337 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 338 return 0; 339 340 return md_dst->u.port_info.port_id; 341 } 342 343 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 344 { 345 struct bnxt *bp = netdev_priv(dev); 346 struct tx_bd *txbd; 347 struct tx_bd_ext *txbd1; 348 struct netdev_queue *txq; 349 int i; 350 dma_addr_t mapping; 351 unsigned int length, pad = 0; 352 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 353 u16 prod, last_frag; 354 struct pci_dev *pdev = bp->pdev; 355 struct bnxt_tx_ring_info *txr; 356 struct bnxt_sw_tx_bd *tx_buf; 357 358 i = skb_get_queue_mapping(skb); 359 if (unlikely(i >= bp->tx_nr_rings)) { 360 dev_kfree_skb_any(skb); 361 return NETDEV_TX_OK; 362 } 363 364 txq = netdev_get_tx_queue(dev, i); 365 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 366 prod = txr->tx_prod; 367 368 free_size = bnxt_tx_avail(bp, txr); 369 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 370 netif_tx_stop_queue(txq); 371 return NETDEV_TX_BUSY; 372 } 373 374 length = skb->len; 375 len = skb_headlen(skb); 376 last_frag = skb_shinfo(skb)->nr_frags; 377 378 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 379 380 txbd->tx_bd_opaque = prod; 381 382 tx_buf = &txr->tx_buf_ring[prod]; 383 tx_buf->skb = skb; 384 tx_buf->nr_frags = last_frag; 385 386 vlan_tag_flags = 0; 387 cfa_action = bnxt_xmit_get_cfa_action(skb); 388 if (skb_vlan_tag_present(skb)) { 389 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 390 skb_vlan_tag_get(skb); 391 /* Currently supports 8021Q, 8021AD vlan offloads 392 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 393 */ 394 if (skb->vlan_proto == htons(ETH_P_8021Q)) 395 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 396 } 397 398 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 399 struct tx_push_buffer *tx_push_buf = txr->tx_push; 400 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 401 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 402 void __iomem *db = txr->tx_db.doorbell; 403 void *pdata = tx_push_buf->data; 404 u64 *end; 405 int j, push_len; 406 407 /* Set COAL_NOW to be ready quickly for the next push */ 408 tx_push->tx_bd_len_flags_type = 409 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 410 TX_BD_TYPE_LONG_TX_BD | 411 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 412 TX_BD_FLAGS_COAL_NOW | 413 TX_BD_FLAGS_PACKET_END | 414 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 415 416 if (skb->ip_summed == CHECKSUM_PARTIAL) 417 tx_push1->tx_bd_hsize_lflags = 418 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 419 else 420 tx_push1->tx_bd_hsize_lflags = 0; 421 422 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 423 tx_push1->tx_bd_cfa_action = 424 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 425 426 end = pdata + length; 427 end = PTR_ALIGN(end, 8) - 1; 428 *end = 0; 429 430 skb_copy_from_linear_data(skb, pdata, len); 431 pdata += len; 432 for (j = 0; j < last_frag; j++) { 433 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 434 void *fptr; 435 436 fptr = skb_frag_address_safe(frag); 437 if (!fptr) 438 goto normal_tx; 439 440 memcpy(pdata, fptr, skb_frag_size(frag)); 441 pdata += skb_frag_size(frag); 442 } 443 444 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 445 txbd->tx_bd_haddr = txr->data_mapping; 446 prod = NEXT_TX(prod); 447 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 448 memcpy(txbd, tx_push1, sizeof(*txbd)); 449 prod = NEXT_TX(prod); 450 tx_push->doorbell = 451 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 452 txr->tx_prod = prod; 453 454 tx_buf->is_push = 1; 455 netdev_tx_sent_queue(txq, skb->len); 456 wmb(); /* Sync is_push and byte queue before pushing data */ 457 458 push_len = (length + sizeof(*tx_push) + 7) / 8; 459 if (push_len > 16) { 460 __iowrite64_copy(db, tx_push_buf, 16); 461 __iowrite32_copy(db + 4, tx_push_buf + 1, 462 (push_len - 16) << 1); 463 } else { 464 __iowrite64_copy(db, tx_push_buf, push_len); 465 } 466 467 goto tx_done; 468 } 469 470 normal_tx: 471 if (length < BNXT_MIN_PKT_SIZE) { 472 pad = BNXT_MIN_PKT_SIZE - length; 473 if (skb_pad(skb, pad)) { 474 /* SKB already freed. */ 475 tx_buf->skb = NULL; 476 return NETDEV_TX_OK; 477 } 478 length = BNXT_MIN_PKT_SIZE; 479 } 480 481 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 482 483 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 484 dev_kfree_skb_any(skb); 485 tx_buf->skb = NULL; 486 return NETDEV_TX_OK; 487 } 488 489 dma_unmap_addr_set(tx_buf, mapping, mapping); 490 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 491 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 492 493 txbd->tx_bd_haddr = cpu_to_le64(mapping); 494 495 prod = NEXT_TX(prod); 496 txbd1 = (struct tx_bd_ext *) 497 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 498 499 txbd1->tx_bd_hsize_lflags = 0; 500 if (skb_is_gso(skb)) { 501 u32 hdr_len; 502 503 if (skb->encapsulation) 504 hdr_len = skb_inner_network_offset(skb) + 505 skb_inner_network_header_len(skb) + 506 inner_tcp_hdrlen(skb); 507 else 508 hdr_len = skb_transport_offset(skb) + 509 tcp_hdrlen(skb); 510 511 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 512 TX_BD_FLAGS_T_IPID | 513 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 514 length = skb_shinfo(skb)->gso_size; 515 txbd1->tx_bd_mss = cpu_to_le32(length); 516 length += hdr_len; 517 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 518 txbd1->tx_bd_hsize_lflags = 519 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 520 txbd1->tx_bd_mss = 0; 521 } 522 523 length >>= 9; 524 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 525 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 526 skb->len); 527 i = 0; 528 goto tx_dma_error; 529 } 530 flags |= bnxt_lhint_arr[length]; 531 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 532 533 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 534 txbd1->tx_bd_cfa_action = 535 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 536 for (i = 0; i < last_frag; i++) { 537 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 538 539 prod = NEXT_TX(prod); 540 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 541 542 len = skb_frag_size(frag); 543 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 544 DMA_TO_DEVICE); 545 546 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 547 goto tx_dma_error; 548 549 tx_buf = &txr->tx_buf_ring[prod]; 550 dma_unmap_addr_set(tx_buf, mapping, mapping); 551 552 txbd->tx_bd_haddr = cpu_to_le64(mapping); 553 554 flags = len << TX_BD_LEN_SHIFT; 555 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 556 } 557 558 flags &= ~TX_BD_LEN; 559 txbd->tx_bd_len_flags_type = 560 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 561 TX_BD_FLAGS_PACKET_END); 562 563 netdev_tx_sent_queue(txq, skb->len); 564 565 /* Sync BD data before updating doorbell */ 566 wmb(); 567 568 prod = NEXT_TX(prod); 569 txr->tx_prod = prod; 570 571 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 572 bnxt_db_write(bp, &txr->tx_db, prod); 573 574 tx_done: 575 576 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 577 if (netdev_xmit_more() && !tx_buf->is_push) 578 bnxt_db_write(bp, &txr->tx_db, prod); 579 580 netif_tx_stop_queue(txq); 581 582 /* netif_tx_stop_queue() must be done before checking 583 * tx index in bnxt_tx_avail() below, because in 584 * bnxt_tx_int(), we update tx index before checking for 585 * netif_tx_queue_stopped(). 586 */ 587 smp_mb(); 588 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 589 netif_tx_wake_queue(txq); 590 } 591 return NETDEV_TX_OK; 592 593 tx_dma_error: 594 last_frag = i; 595 596 /* start back at beginning and unmap skb */ 597 prod = txr->tx_prod; 598 tx_buf = &txr->tx_buf_ring[prod]; 599 tx_buf->skb = NULL; 600 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 601 skb_headlen(skb), PCI_DMA_TODEVICE); 602 prod = NEXT_TX(prod); 603 604 /* unmap remaining mapped pages */ 605 for (i = 0; i < last_frag; i++) { 606 prod = NEXT_TX(prod); 607 tx_buf = &txr->tx_buf_ring[prod]; 608 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 609 skb_frag_size(&skb_shinfo(skb)->frags[i]), 610 PCI_DMA_TODEVICE); 611 } 612 613 dev_kfree_skb_any(skb); 614 return NETDEV_TX_OK; 615 } 616 617 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 618 { 619 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 620 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 621 u16 cons = txr->tx_cons; 622 struct pci_dev *pdev = bp->pdev; 623 int i; 624 unsigned int tx_bytes = 0; 625 626 for (i = 0; i < nr_pkts; i++) { 627 struct bnxt_sw_tx_bd *tx_buf; 628 struct sk_buff *skb; 629 int j, last; 630 631 tx_buf = &txr->tx_buf_ring[cons]; 632 cons = NEXT_TX(cons); 633 skb = tx_buf->skb; 634 tx_buf->skb = NULL; 635 636 if (tx_buf->is_push) { 637 tx_buf->is_push = 0; 638 goto next_tx_int; 639 } 640 641 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 642 skb_headlen(skb), PCI_DMA_TODEVICE); 643 last = tx_buf->nr_frags; 644 645 for (j = 0; j < last; j++) { 646 cons = NEXT_TX(cons); 647 tx_buf = &txr->tx_buf_ring[cons]; 648 dma_unmap_page( 649 &pdev->dev, 650 dma_unmap_addr(tx_buf, mapping), 651 skb_frag_size(&skb_shinfo(skb)->frags[j]), 652 PCI_DMA_TODEVICE); 653 } 654 655 next_tx_int: 656 cons = NEXT_TX(cons); 657 658 tx_bytes += skb->len; 659 dev_kfree_skb_any(skb); 660 } 661 662 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 663 txr->tx_cons = cons; 664 665 /* Need to make the tx_cons update visible to bnxt_start_xmit() 666 * before checking for netif_tx_queue_stopped(). Without the 667 * memory barrier, there is a small possibility that bnxt_start_xmit() 668 * will miss it and cause the queue to be stopped forever. 669 */ 670 smp_mb(); 671 672 if (unlikely(netif_tx_queue_stopped(txq)) && 673 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 674 __netif_tx_lock(txq, smp_processor_id()); 675 if (netif_tx_queue_stopped(txq) && 676 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 677 txr->dev_state != BNXT_DEV_STATE_CLOSING) 678 netif_tx_wake_queue(txq); 679 __netif_tx_unlock(txq); 680 } 681 } 682 683 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 684 struct bnxt_rx_ring_info *rxr, 685 gfp_t gfp) 686 { 687 struct device *dev = &bp->pdev->dev; 688 struct page *page; 689 690 page = page_pool_dev_alloc_pages(rxr->page_pool); 691 if (!page) 692 return NULL; 693 694 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, 695 DMA_ATTR_WEAK_ORDERING); 696 if (dma_mapping_error(dev, *mapping)) { 697 page_pool_recycle_direct(rxr->page_pool, page); 698 return NULL; 699 } 700 *mapping += bp->rx_dma_offset; 701 return page; 702 } 703 704 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 705 gfp_t gfp) 706 { 707 u8 *data; 708 struct pci_dev *pdev = bp->pdev; 709 710 data = kmalloc(bp->rx_buf_size, gfp); 711 if (!data) 712 return NULL; 713 714 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, 715 bp->rx_buf_use_size, bp->rx_dir, 716 DMA_ATTR_WEAK_ORDERING); 717 718 if (dma_mapping_error(&pdev->dev, *mapping)) { 719 kfree(data); 720 data = NULL; 721 } 722 return data; 723 } 724 725 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 726 u16 prod, gfp_t gfp) 727 { 728 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 729 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 730 dma_addr_t mapping; 731 732 if (BNXT_RX_PAGE_MODE(bp)) { 733 struct page *page = 734 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); 735 736 if (!page) 737 return -ENOMEM; 738 739 rx_buf->data = page; 740 rx_buf->data_ptr = page_address(page) + bp->rx_offset; 741 } else { 742 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 743 744 if (!data) 745 return -ENOMEM; 746 747 rx_buf->data = data; 748 rx_buf->data_ptr = data + bp->rx_offset; 749 } 750 rx_buf->mapping = mapping; 751 752 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 753 return 0; 754 } 755 756 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 757 { 758 u16 prod = rxr->rx_prod; 759 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 760 struct rx_bd *cons_bd, *prod_bd; 761 762 prod_rx_buf = &rxr->rx_buf_ring[prod]; 763 cons_rx_buf = &rxr->rx_buf_ring[cons]; 764 765 prod_rx_buf->data = data; 766 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 767 768 prod_rx_buf->mapping = cons_rx_buf->mapping; 769 770 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 771 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 772 773 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 774 } 775 776 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 777 { 778 u16 next, max = rxr->rx_agg_bmap_size; 779 780 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 781 if (next >= max) 782 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 783 return next; 784 } 785 786 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 787 struct bnxt_rx_ring_info *rxr, 788 u16 prod, gfp_t gfp) 789 { 790 struct rx_bd *rxbd = 791 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 792 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 793 struct pci_dev *pdev = bp->pdev; 794 struct page *page; 795 dma_addr_t mapping; 796 u16 sw_prod = rxr->rx_sw_agg_prod; 797 unsigned int offset = 0; 798 799 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 800 page = rxr->rx_page; 801 if (!page) { 802 page = alloc_page(gfp); 803 if (!page) 804 return -ENOMEM; 805 rxr->rx_page = page; 806 rxr->rx_page_offset = 0; 807 } 808 offset = rxr->rx_page_offset; 809 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 810 if (rxr->rx_page_offset == PAGE_SIZE) 811 rxr->rx_page = NULL; 812 else 813 get_page(page); 814 } else { 815 page = alloc_page(gfp); 816 if (!page) 817 return -ENOMEM; 818 } 819 820 mapping = dma_map_page_attrs(&pdev->dev, page, offset, 821 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE, 822 DMA_ATTR_WEAK_ORDERING); 823 if (dma_mapping_error(&pdev->dev, mapping)) { 824 __free_page(page); 825 return -EIO; 826 } 827 828 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 829 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 830 831 __set_bit(sw_prod, rxr->rx_agg_bmap); 832 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 833 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 834 835 rx_agg_buf->page = page; 836 rx_agg_buf->offset = offset; 837 rx_agg_buf->mapping = mapping; 838 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 839 rxbd->rx_bd_opaque = sw_prod; 840 return 0; 841 } 842 843 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 844 struct bnxt_cp_ring_info *cpr, 845 u16 cp_cons, u16 curr) 846 { 847 struct rx_agg_cmp *agg; 848 849 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 850 agg = (struct rx_agg_cmp *) 851 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 852 return agg; 853 } 854 855 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 856 struct bnxt_rx_ring_info *rxr, 857 u16 agg_id, u16 curr) 858 { 859 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 860 861 return &tpa_info->agg_arr[curr]; 862 } 863 864 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 865 u16 start, u32 agg_bufs, bool tpa) 866 { 867 struct bnxt_napi *bnapi = cpr->bnapi; 868 struct bnxt *bp = bnapi->bp; 869 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 870 u16 prod = rxr->rx_agg_prod; 871 u16 sw_prod = rxr->rx_sw_agg_prod; 872 bool p5_tpa = false; 873 u32 i; 874 875 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 876 p5_tpa = true; 877 878 for (i = 0; i < agg_bufs; i++) { 879 u16 cons; 880 struct rx_agg_cmp *agg; 881 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 882 struct rx_bd *prod_bd; 883 struct page *page; 884 885 if (p5_tpa) 886 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 887 else 888 agg = bnxt_get_agg(bp, cpr, idx, start + i); 889 cons = agg->rx_agg_cmp_opaque; 890 __clear_bit(cons, rxr->rx_agg_bmap); 891 892 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 893 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 894 895 __set_bit(sw_prod, rxr->rx_agg_bmap); 896 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 897 cons_rx_buf = &rxr->rx_agg_ring[cons]; 898 899 /* It is possible for sw_prod to be equal to cons, so 900 * set cons_rx_buf->page to NULL first. 901 */ 902 page = cons_rx_buf->page; 903 cons_rx_buf->page = NULL; 904 prod_rx_buf->page = page; 905 prod_rx_buf->offset = cons_rx_buf->offset; 906 907 prod_rx_buf->mapping = cons_rx_buf->mapping; 908 909 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 910 911 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 912 prod_bd->rx_bd_opaque = sw_prod; 913 914 prod = NEXT_RX_AGG(prod); 915 sw_prod = NEXT_RX_AGG(sw_prod); 916 } 917 rxr->rx_agg_prod = prod; 918 rxr->rx_sw_agg_prod = sw_prod; 919 } 920 921 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 922 struct bnxt_rx_ring_info *rxr, 923 u16 cons, void *data, u8 *data_ptr, 924 dma_addr_t dma_addr, 925 unsigned int offset_and_len) 926 { 927 unsigned int payload = offset_and_len >> 16; 928 unsigned int len = offset_and_len & 0xffff; 929 skb_frag_t *frag; 930 struct page *page = data; 931 u16 prod = rxr->rx_prod; 932 struct sk_buff *skb; 933 int off, err; 934 935 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 936 if (unlikely(err)) { 937 bnxt_reuse_rx_data(rxr, cons, data); 938 return NULL; 939 } 940 dma_addr -= bp->rx_dma_offset; 941 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, 942 DMA_ATTR_WEAK_ORDERING); 943 page_pool_release_page(rxr->page_pool, page); 944 945 if (unlikely(!payload)) 946 payload = eth_get_headlen(bp->dev, data_ptr, len); 947 948 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 949 if (!skb) { 950 __free_page(page); 951 return NULL; 952 } 953 954 off = (void *)data_ptr - page_address(page); 955 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); 956 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 957 payload + NET_IP_ALIGN); 958 959 frag = &skb_shinfo(skb)->frags[0]; 960 skb_frag_size_sub(frag, payload); 961 skb_frag_off_add(frag, payload); 962 skb->data_len -= payload; 963 skb->tail += payload; 964 965 return skb; 966 } 967 968 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 969 struct bnxt_rx_ring_info *rxr, u16 cons, 970 void *data, u8 *data_ptr, 971 dma_addr_t dma_addr, 972 unsigned int offset_and_len) 973 { 974 u16 prod = rxr->rx_prod; 975 struct sk_buff *skb; 976 int err; 977 978 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 979 if (unlikely(err)) { 980 bnxt_reuse_rx_data(rxr, cons, data); 981 return NULL; 982 } 983 984 skb = build_skb(data, 0); 985 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 986 bp->rx_dir, DMA_ATTR_WEAK_ORDERING); 987 if (!skb) { 988 kfree(data); 989 return NULL; 990 } 991 992 skb_reserve(skb, bp->rx_offset); 993 skb_put(skb, offset_and_len & 0xffff); 994 return skb; 995 } 996 997 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, 998 struct bnxt_cp_ring_info *cpr, 999 struct sk_buff *skb, u16 idx, 1000 u32 agg_bufs, bool tpa) 1001 { 1002 struct bnxt_napi *bnapi = cpr->bnapi; 1003 struct pci_dev *pdev = bp->pdev; 1004 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1005 u16 prod = rxr->rx_agg_prod; 1006 bool p5_tpa = false; 1007 u32 i; 1008 1009 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) 1010 p5_tpa = true; 1011 1012 for (i = 0; i < agg_bufs; i++) { 1013 u16 cons, frag_len; 1014 struct rx_agg_cmp *agg; 1015 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1016 struct page *page; 1017 dma_addr_t mapping; 1018 1019 if (p5_tpa) 1020 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1021 else 1022 agg = bnxt_get_agg(bp, cpr, idx, i); 1023 cons = agg->rx_agg_cmp_opaque; 1024 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1025 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1026 1027 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1028 skb_fill_page_desc(skb, i, cons_rx_buf->page, 1029 cons_rx_buf->offset, frag_len); 1030 __clear_bit(cons, rxr->rx_agg_bmap); 1031 1032 /* It is possible for bnxt_alloc_rx_page() to allocate 1033 * a sw_prod index that equals the cons index, so we 1034 * need to clear the cons entry now. 1035 */ 1036 mapping = cons_rx_buf->mapping; 1037 page = cons_rx_buf->page; 1038 cons_rx_buf->page = NULL; 1039 1040 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 1041 struct skb_shared_info *shinfo; 1042 unsigned int nr_frags; 1043 1044 shinfo = skb_shinfo(skb); 1045 nr_frags = --shinfo->nr_frags; 1046 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 1047 1048 dev_kfree_skb(skb); 1049 1050 cons_rx_buf->page = page; 1051 1052 /* Update prod since possibly some pages have been 1053 * allocated already. 1054 */ 1055 rxr->rx_agg_prod = prod; 1056 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1057 return NULL; 1058 } 1059 1060 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 1061 PCI_DMA_FROMDEVICE, 1062 DMA_ATTR_WEAK_ORDERING); 1063 1064 skb->data_len += frag_len; 1065 skb->len += frag_len; 1066 skb->truesize += PAGE_SIZE; 1067 1068 prod = NEXT_RX_AGG(prod); 1069 } 1070 rxr->rx_agg_prod = prod; 1071 return skb; 1072 } 1073 1074 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1075 u8 agg_bufs, u32 *raw_cons) 1076 { 1077 u16 last; 1078 struct rx_agg_cmp *agg; 1079 1080 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1081 last = RING_CMP(*raw_cons); 1082 agg = (struct rx_agg_cmp *) 1083 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1084 return RX_AGG_CMP_VALID(agg, *raw_cons); 1085 } 1086 1087 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1088 unsigned int len, 1089 dma_addr_t mapping) 1090 { 1091 struct bnxt *bp = bnapi->bp; 1092 struct pci_dev *pdev = bp->pdev; 1093 struct sk_buff *skb; 1094 1095 skb = napi_alloc_skb(&bnapi->napi, len); 1096 if (!skb) 1097 return NULL; 1098 1099 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh, 1100 bp->rx_dir); 1101 1102 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1103 len + NET_IP_ALIGN); 1104 1105 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh, 1106 bp->rx_dir); 1107 1108 skb_put(skb, len); 1109 return skb; 1110 } 1111 1112 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1113 u32 *raw_cons, void *cmp) 1114 { 1115 struct rx_cmp *rxcmp = cmp; 1116 u32 tmp_raw_cons = *raw_cons; 1117 u8 cmp_type, agg_bufs = 0; 1118 1119 cmp_type = RX_CMP_TYPE(rxcmp); 1120 1121 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1122 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1123 RX_CMP_AGG_BUFS) >> 1124 RX_CMP_AGG_BUFS_SHIFT; 1125 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1126 struct rx_tpa_end_cmp *tpa_end = cmp; 1127 1128 if (bp->flags & BNXT_FLAG_CHIP_P5) 1129 return 0; 1130 1131 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1132 } 1133 1134 if (agg_bufs) { 1135 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1136 return -EBUSY; 1137 } 1138 *raw_cons = tmp_raw_cons; 1139 return 0; 1140 } 1141 1142 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 1143 { 1144 if (BNXT_PF(bp)) 1145 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 1146 else 1147 schedule_delayed_work(&bp->fw_reset_task, delay); 1148 } 1149 1150 static void bnxt_queue_sp_work(struct bnxt *bp) 1151 { 1152 if (BNXT_PF(bp)) 1153 queue_work(bnxt_pf_wq, &bp->sp_task); 1154 else 1155 schedule_work(&bp->sp_task); 1156 } 1157 1158 static void bnxt_cancel_sp_work(struct bnxt *bp) 1159 { 1160 if (BNXT_PF(bp)) 1161 flush_workqueue(bnxt_pf_wq); 1162 else 1163 cancel_work_sync(&bp->sp_task); 1164 } 1165 1166 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 1167 { 1168 if (!rxr->bnapi->in_reset) { 1169 rxr->bnapi->in_reset = true; 1170 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 1171 bnxt_queue_sp_work(bp); 1172 } 1173 rxr->rx_next_cons = 0xffff; 1174 } 1175 1176 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1177 { 1178 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1179 u16 idx = agg_id & MAX_TPA_P5_MASK; 1180 1181 if (test_bit(idx, map->agg_idx_bmap)) 1182 idx = find_first_zero_bit(map->agg_idx_bmap, 1183 BNXT_AGG_IDX_BMAP_SIZE); 1184 __set_bit(idx, map->agg_idx_bmap); 1185 map->agg_id_tbl[agg_id] = idx; 1186 return idx; 1187 } 1188 1189 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1190 { 1191 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1192 1193 __clear_bit(idx, map->agg_idx_bmap); 1194 } 1195 1196 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1197 { 1198 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1199 1200 return map->agg_id_tbl[agg_id]; 1201 } 1202 1203 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1204 struct rx_tpa_start_cmp *tpa_start, 1205 struct rx_tpa_start_cmp_ext *tpa_start1) 1206 { 1207 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1208 struct bnxt_tpa_info *tpa_info; 1209 u16 cons, prod, agg_id; 1210 struct rx_bd *prod_bd; 1211 dma_addr_t mapping; 1212 1213 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1214 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1215 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1216 } else { 1217 agg_id = TPA_START_AGG_ID(tpa_start); 1218 } 1219 cons = tpa_start->rx_tpa_start_cmp_opaque; 1220 prod = rxr->rx_prod; 1221 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1222 prod_rx_buf = &rxr->rx_buf_ring[prod]; 1223 tpa_info = &rxr->rx_tpa[agg_id]; 1224 1225 if (unlikely(cons != rxr->rx_next_cons || 1226 TPA_START_ERROR(tpa_start))) { 1227 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1228 cons, rxr->rx_next_cons, 1229 TPA_START_ERROR_CODE(tpa_start1)); 1230 bnxt_sched_reset(bp, rxr); 1231 return; 1232 } 1233 /* Store cfa_code in tpa_info to use in tpa_end 1234 * completion processing. 1235 */ 1236 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1237 prod_rx_buf->data = tpa_info->data; 1238 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1239 1240 mapping = tpa_info->mapping; 1241 prod_rx_buf->mapping = mapping; 1242 1243 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 1244 1245 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1246 1247 tpa_info->data = cons_rx_buf->data; 1248 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1249 cons_rx_buf->data = NULL; 1250 tpa_info->mapping = cons_rx_buf->mapping; 1251 1252 tpa_info->len = 1253 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1254 RX_TPA_START_CMP_LEN_SHIFT; 1255 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1256 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 1257 1258 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1259 tpa_info->gso_type = SKB_GSO_TCPV4; 1260 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1261 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1)) 1262 tpa_info->gso_type = SKB_GSO_TCPV6; 1263 tpa_info->rss_hash = 1264 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1265 } else { 1266 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1267 tpa_info->gso_type = 0; 1268 if (netif_msg_rx_err(bp)) 1269 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 1270 } 1271 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1272 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1273 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1274 tpa_info->agg_count = 0; 1275 1276 rxr->rx_prod = NEXT_RX(prod); 1277 cons = NEXT_RX(cons); 1278 rxr->rx_next_cons = NEXT_RX(cons); 1279 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1280 1281 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1282 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1283 cons_rx_buf->data = NULL; 1284 } 1285 1286 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1287 { 1288 if (agg_bufs) 1289 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1290 } 1291 1292 #ifdef CONFIG_INET 1293 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1294 { 1295 struct udphdr *uh = NULL; 1296 1297 if (ip_proto == htons(ETH_P_IP)) { 1298 struct iphdr *iph = (struct iphdr *)skb->data; 1299 1300 if (iph->protocol == IPPROTO_UDP) 1301 uh = (struct udphdr *)(iph + 1); 1302 } else { 1303 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1304 1305 if (iph->nexthdr == IPPROTO_UDP) 1306 uh = (struct udphdr *)(iph + 1); 1307 } 1308 if (uh) { 1309 if (uh->check) 1310 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1311 else 1312 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1313 } 1314 } 1315 #endif 1316 1317 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1318 int payload_off, int tcp_ts, 1319 struct sk_buff *skb) 1320 { 1321 #ifdef CONFIG_INET 1322 struct tcphdr *th; 1323 int len, nw_off; 1324 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1325 u32 hdr_info = tpa_info->hdr_info; 1326 bool loopback = false; 1327 1328 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1329 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1330 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1331 1332 /* If the packet is an internal loopback packet, the offsets will 1333 * have an extra 4 bytes. 1334 */ 1335 if (inner_mac_off == 4) { 1336 loopback = true; 1337 } else if (inner_mac_off > 4) { 1338 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1339 ETH_HLEN - 2)); 1340 1341 /* We only support inner iPv4/ipv6. If we don't see the 1342 * correct protocol ID, it must be a loopback packet where 1343 * the offsets are off by 4. 1344 */ 1345 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1346 loopback = true; 1347 } 1348 if (loopback) { 1349 /* internal loopback packet, subtract all offsets by 4 */ 1350 inner_ip_off -= 4; 1351 inner_mac_off -= 4; 1352 outer_ip_off -= 4; 1353 } 1354 1355 nw_off = inner_ip_off - ETH_HLEN; 1356 skb_set_network_header(skb, nw_off); 1357 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1358 struct ipv6hdr *iph = ipv6_hdr(skb); 1359 1360 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1361 len = skb->len - skb_transport_offset(skb); 1362 th = tcp_hdr(skb); 1363 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1364 } else { 1365 struct iphdr *iph = ip_hdr(skb); 1366 1367 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1368 len = skb->len - skb_transport_offset(skb); 1369 th = tcp_hdr(skb); 1370 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1371 } 1372 1373 if (inner_mac_off) { /* tunnel */ 1374 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1375 ETH_HLEN - 2)); 1376 1377 bnxt_gro_tunnel(skb, proto); 1378 } 1379 #endif 1380 return skb; 1381 } 1382 1383 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1384 int payload_off, int tcp_ts, 1385 struct sk_buff *skb) 1386 { 1387 #ifdef CONFIG_INET 1388 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1389 u32 hdr_info = tpa_info->hdr_info; 1390 int iphdr_len, nw_off; 1391 1392 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1393 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1394 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1395 1396 nw_off = inner_ip_off - ETH_HLEN; 1397 skb_set_network_header(skb, nw_off); 1398 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1399 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1400 skb_set_transport_header(skb, nw_off + iphdr_len); 1401 1402 if (inner_mac_off) { /* tunnel */ 1403 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1404 ETH_HLEN - 2)); 1405 1406 bnxt_gro_tunnel(skb, proto); 1407 } 1408 #endif 1409 return skb; 1410 } 1411 1412 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1413 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1414 1415 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1416 int payload_off, int tcp_ts, 1417 struct sk_buff *skb) 1418 { 1419 #ifdef CONFIG_INET 1420 struct tcphdr *th; 1421 int len, nw_off, tcp_opt_len = 0; 1422 1423 if (tcp_ts) 1424 tcp_opt_len = 12; 1425 1426 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1427 struct iphdr *iph; 1428 1429 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1430 ETH_HLEN; 1431 skb_set_network_header(skb, nw_off); 1432 iph = ip_hdr(skb); 1433 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1434 len = skb->len - skb_transport_offset(skb); 1435 th = tcp_hdr(skb); 1436 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1437 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1438 struct ipv6hdr *iph; 1439 1440 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1441 ETH_HLEN; 1442 skb_set_network_header(skb, nw_off); 1443 iph = ipv6_hdr(skb); 1444 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1445 len = skb->len - skb_transport_offset(skb); 1446 th = tcp_hdr(skb); 1447 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1448 } else { 1449 dev_kfree_skb_any(skb); 1450 return NULL; 1451 } 1452 1453 if (nw_off) /* tunnel */ 1454 bnxt_gro_tunnel(skb, skb->protocol); 1455 #endif 1456 return skb; 1457 } 1458 1459 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1460 struct bnxt_tpa_info *tpa_info, 1461 struct rx_tpa_end_cmp *tpa_end, 1462 struct rx_tpa_end_cmp_ext *tpa_end1, 1463 struct sk_buff *skb) 1464 { 1465 #ifdef CONFIG_INET 1466 int payload_off; 1467 u16 segs; 1468 1469 segs = TPA_END_TPA_SEGS(tpa_end); 1470 if (segs == 1) 1471 return skb; 1472 1473 NAPI_GRO_CB(skb)->count = segs; 1474 skb_shinfo(skb)->gso_size = 1475 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1476 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1477 if (bp->flags & BNXT_FLAG_CHIP_P5) 1478 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1479 else 1480 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1481 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1482 if (likely(skb)) 1483 tcp_gro_complete(skb); 1484 #endif 1485 return skb; 1486 } 1487 1488 /* Given the cfa_code of a received packet determine which 1489 * netdev (vf-rep or PF) the packet is destined to. 1490 */ 1491 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1492 { 1493 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1494 1495 /* if vf-rep dev is NULL, the must belongs to the PF */ 1496 return dev ? dev : bp->dev; 1497 } 1498 1499 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1500 struct bnxt_cp_ring_info *cpr, 1501 u32 *raw_cons, 1502 struct rx_tpa_end_cmp *tpa_end, 1503 struct rx_tpa_end_cmp_ext *tpa_end1, 1504 u8 *event) 1505 { 1506 struct bnxt_napi *bnapi = cpr->bnapi; 1507 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1508 u8 *data_ptr, agg_bufs; 1509 unsigned int len; 1510 struct bnxt_tpa_info *tpa_info; 1511 dma_addr_t mapping; 1512 struct sk_buff *skb; 1513 u16 idx = 0, agg_id; 1514 void *data; 1515 bool gro; 1516 1517 if (unlikely(bnapi->in_reset)) { 1518 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1519 1520 if (rc < 0) 1521 return ERR_PTR(-EBUSY); 1522 return NULL; 1523 } 1524 1525 if (bp->flags & BNXT_FLAG_CHIP_P5) { 1526 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1527 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1528 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1529 tpa_info = &rxr->rx_tpa[agg_id]; 1530 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1531 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1532 agg_bufs, tpa_info->agg_count); 1533 agg_bufs = tpa_info->agg_count; 1534 } 1535 tpa_info->agg_count = 0; 1536 *event |= BNXT_AGG_EVENT; 1537 bnxt_free_agg_idx(rxr, agg_id); 1538 idx = agg_id; 1539 gro = !!(bp->flags & BNXT_FLAG_GRO); 1540 } else { 1541 agg_id = TPA_END_AGG_ID(tpa_end); 1542 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1543 tpa_info = &rxr->rx_tpa[agg_id]; 1544 idx = RING_CMP(*raw_cons); 1545 if (agg_bufs) { 1546 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1547 return ERR_PTR(-EBUSY); 1548 1549 *event |= BNXT_AGG_EVENT; 1550 idx = NEXT_CMP(idx); 1551 } 1552 gro = !!TPA_END_GRO(tpa_end); 1553 } 1554 data = tpa_info->data; 1555 data_ptr = tpa_info->data_ptr; 1556 prefetch(data_ptr); 1557 len = tpa_info->len; 1558 mapping = tpa_info->mapping; 1559 1560 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1561 bnxt_abort_tpa(cpr, idx, agg_bufs); 1562 if (agg_bufs > MAX_SKB_FRAGS) 1563 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1564 agg_bufs, (int)MAX_SKB_FRAGS); 1565 return NULL; 1566 } 1567 1568 if (len <= bp->rx_copy_thresh) { 1569 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1570 if (!skb) { 1571 bnxt_abort_tpa(cpr, idx, agg_bufs); 1572 return NULL; 1573 } 1574 } else { 1575 u8 *new_data; 1576 dma_addr_t new_mapping; 1577 1578 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1579 if (!new_data) { 1580 bnxt_abort_tpa(cpr, idx, agg_bufs); 1581 return NULL; 1582 } 1583 1584 tpa_info->data = new_data; 1585 tpa_info->data_ptr = new_data + bp->rx_offset; 1586 tpa_info->mapping = new_mapping; 1587 1588 skb = build_skb(data, 0); 1589 dma_unmap_single_attrs(&bp->pdev->dev, mapping, 1590 bp->rx_buf_use_size, bp->rx_dir, 1591 DMA_ATTR_WEAK_ORDERING); 1592 1593 if (!skb) { 1594 kfree(data); 1595 bnxt_abort_tpa(cpr, idx, agg_bufs); 1596 return NULL; 1597 } 1598 skb_reserve(skb, bp->rx_offset); 1599 skb_put(skb, len); 1600 } 1601 1602 if (agg_bufs) { 1603 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); 1604 if (!skb) { 1605 /* Page reuse already handled by bnxt_rx_pages(). */ 1606 return NULL; 1607 } 1608 } 1609 1610 skb->protocol = 1611 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code)); 1612 1613 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1614 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1615 1616 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1617 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1618 u16 vlan_proto = tpa_info->metadata >> 1619 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1620 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1621 1622 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1623 } 1624 1625 skb_checksum_none_assert(skb); 1626 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1627 skb->ip_summed = CHECKSUM_UNNECESSARY; 1628 skb->csum_level = 1629 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1630 } 1631 1632 if (gro) 1633 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1634 1635 return skb; 1636 } 1637 1638 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1639 struct rx_agg_cmp *rx_agg) 1640 { 1641 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1642 struct bnxt_tpa_info *tpa_info; 1643 1644 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1645 tpa_info = &rxr->rx_tpa[agg_id]; 1646 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 1647 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 1648 } 1649 1650 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 1651 struct sk_buff *skb) 1652 { 1653 if (skb->dev != bp->dev) { 1654 /* this packet belongs to a vf-rep */ 1655 bnxt_vf_rep_rx(bp, skb); 1656 return; 1657 } 1658 skb_record_rx_queue(skb, bnapi->index); 1659 napi_gro_receive(&bnapi->napi, skb); 1660 } 1661 1662 /* returns the following: 1663 * 1 - 1 packet successfully received 1664 * 0 - successful TPA_START, packet not completed yet 1665 * -EBUSY - completion ring does not have all the agg buffers yet 1666 * -ENOMEM - packet aborted due to out of memory 1667 * -EIO - packet aborted due to hw error indicated in BD 1668 */ 1669 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1670 u32 *raw_cons, u8 *event) 1671 { 1672 struct bnxt_napi *bnapi = cpr->bnapi; 1673 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1674 struct net_device *dev = bp->dev; 1675 struct rx_cmp *rxcmp; 1676 struct rx_cmp_ext *rxcmp1; 1677 u32 tmp_raw_cons = *raw_cons; 1678 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1679 struct bnxt_sw_rx_bd *rx_buf; 1680 unsigned int len; 1681 u8 *data_ptr, agg_bufs, cmp_type; 1682 dma_addr_t dma_addr; 1683 struct sk_buff *skb; 1684 void *data; 1685 int rc = 0; 1686 u32 misc; 1687 1688 rxcmp = (struct rx_cmp *) 1689 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1690 1691 cmp_type = RX_CMP_TYPE(rxcmp); 1692 1693 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 1694 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 1695 goto next_rx_no_prod_no_len; 1696 } 1697 1698 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1699 cp_cons = RING_CMP(tmp_raw_cons); 1700 rxcmp1 = (struct rx_cmp_ext *) 1701 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1702 1703 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1704 return -EBUSY; 1705 1706 prod = rxr->rx_prod; 1707 1708 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1709 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1710 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1711 1712 *event |= BNXT_RX_EVENT; 1713 goto next_rx_no_prod_no_len; 1714 1715 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1716 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 1717 (struct rx_tpa_end_cmp *)rxcmp, 1718 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 1719 1720 if (IS_ERR(skb)) 1721 return -EBUSY; 1722 1723 rc = -ENOMEM; 1724 if (likely(skb)) { 1725 bnxt_deliver_skb(bp, bnapi, skb); 1726 rc = 1; 1727 } 1728 *event |= BNXT_RX_EVENT; 1729 goto next_rx_no_prod_no_len; 1730 } 1731 1732 cons = rxcmp->rx_cmp_opaque; 1733 if (unlikely(cons != rxr->rx_next_cons)) { 1734 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp); 1735 1736 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 1737 cons, rxr->rx_next_cons); 1738 bnxt_sched_reset(bp, rxr); 1739 return rc1; 1740 } 1741 rx_buf = &rxr->rx_buf_ring[cons]; 1742 data = rx_buf->data; 1743 data_ptr = rx_buf->data_ptr; 1744 prefetch(data_ptr); 1745 1746 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 1747 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 1748 1749 if (agg_bufs) { 1750 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1751 return -EBUSY; 1752 1753 cp_cons = NEXT_CMP(cp_cons); 1754 *event |= BNXT_AGG_EVENT; 1755 } 1756 *event |= BNXT_RX_EVENT; 1757 1758 rx_buf->data = NULL; 1759 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1760 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 1761 1762 bnxt_reuse_rx_data(rxr, cons, data); 1763 if (agg_bufs) 1764 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 1765 false); 1766 1767 rc = -EIO; 1768 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 1769 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++; 1770 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 1771 netdev_warn(bp->dev, "RX buffer error %x\n", 1772 rx_err); 1773 bnxt_sched_reset(bp, rxr); 1774 } 1775 } 1776 goto next_rx_no_len; 1777 } 1778 1779 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1780 dma_addr = rx_buf->mapping; 1781 1782 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { 1783 rc = 1; 1784 goto next_rx; 1785 } 1786 1787 if (len <= bp->rx_copy_thresh) { 1788 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 1789 bnxt_reuse_rx_data(rxr, cons, data); 1790 if (!skb) { 1791 if (agg_bufs) 1792 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 1793 agg_bufs, false); 1794 rc = -ENOMEM; 1795 goto next_rx; 1796 } 1797 } else { 1798 u32 payload; 1799 1800 if (rx_buf->data_ptr == data_ptr) 1801 payload = misc & RX_CMP_PAYLOAD_OFFSET; 1802 else 1803 payload = 0; 1804 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 1805 payload | len); 1806 if (!skb) { 1807 rc = -ENOMEM; 1808 goto next_rx; 1809 } 1810 } 1811 1812 if (agg_bufs) { 1813 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); 1814 if (!skb) { 1815 rc = -ENOMEM; 1816 goto next_rx; 1817 } 1818 } 1819 1820 if (RX_CMP_HASH_VALID(rxcmp)) { 1821 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1822 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1823 1824 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1825 if (hash_type != 1 && hash_type != 3) 1826 type = PKT_HASH_TYPE_L3; 1827 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1828 } 1829 1830 cfa_code = RX_CMP_CFA_CODE(rxcmp1); 1831 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code)); 1832 1833 if ((rxcmp1->rx_cmp_flags2 & 1834 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1835 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1836 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1837 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1838 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1839 1840 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1841 } 1842 1843 skb_checksum_none_assert(skb); 1844 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1845 if (dev->features & NETIF_F_RXCSUM) { 1846 skb->ip_summed = CHECKSUM_UNNECESSARY; 1847 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1848 } 1849 } else { 1850 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1851 if (dev->features & NETIF_F_RXCSUM) 1852 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++; 1853 } 1854 } 1855 1856 bnxt_deliver_skb(bp, bnapi, skb); 1857 rc = 1; 1858 1859 next_rx: 1860 cpr->rx_packets += 1; 1861 cpr->rx_bytes += len; 1862 1863 next_rx_no_len: 1864 rxr->rx_prod = NEXT_RX(prod); 1865 rxr->rx_next_cons = NEXT_RX(cons); 1866 1867 next_rx_no_prod_no_len: 1868 *raw_cons = tmp_raw_cons; 1869 1870 return rc; 1871 } 1872 1873 /* In netpoll mode, if we are using a combined completion ring, we need to 1874 * discard the rx packets and recycle the buffers. 1875 */ 1876 static int bnxt_force_rx_discard(struct bnxt *bp, 1877 struct bnxt_cp_ring_info *cpr, 1878 u32 *raw_cons, u8 *event) 1879 { 1880 u32 tmp_raw_cons = *raw_cons; 1881 struct rx_cmp_ext *rxcmp1; 1882 struct rx_cmp *rxcmp; 1883 u16 cp_cons; 1884 u8 cmp_type; 1885 1886 cp_cons = RING_CMP(tmp_raw_cons); 1887 rxcmp = (struct rx_cmp *) 1888 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1889 1890 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1891 cp_cons = RING_CMP(tmp_raw_cons); 1892 rxcmp1 = (struct rx_cmp_ext *) 1893 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1894 1895 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1896 return -EBUSY; 1897 1898 cmp_type = RX_CMP_TYPE(rxcmp); 1899 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1900 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1901 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1902 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1903 struct rx_tpa_end_cmp_ext *tpa_end1; 1904 1905 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 1906 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 1907 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 1908 } 1909 return bnxt_rx_pkt(bp, cpr, raw_cons, event); 1910 } 1911 1912 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 1913 { 1914 struct bnxt_fw_health *fw_health = bp->fw_health; 1915 u32 reg = fw_health->regs[reg_idx]; 1916 u32 reg_type, reg_off, val = 0; 1917 1918 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 1919 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 1920 switch (reg_type) { 1921 case BNXT_FW_HEALTH_REG_TYPE_CFG: 1922 pci_read_config_dword(bp->pdev, reg_off, &val); 1923 break; 1924 case BNXT_FW_HEALTH_REG_TYPE_GRC: 1925 reg_off = fw_health->mapped_regs[reg_idx]; 1926 /* fall through */ 1927 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 1928 val = readl(bp->bar0 + reg_off); 1929 break; 1930 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 1931 val = readl(bp->bar1 + reg_off); 1932 break; 1933 } 1934 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 1935 val &= fw_health->fw_reset_inprog_reg_mask; 1936 return val; 1937 } 1938 1939 #define BNXT_GET_EVENT_PORT(data) \ 1940 ((data) & \ 1941 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1942 1943 static int bnxt_async_event_process(struct bnxt *bp, 1944 struct hwrm_async_event_cmpl *cmpl) 1945 { 1946 u16 event_id = le16_to_cpu(cmpl->event_id); 1947 1948 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1949 switch (event_id) { 1950 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1951 u32 data1 = le32_to_cpu(cmpl->event_data1); 1952 struct bnxt_link_info *link_info = &bp->link_info; 1953 1954 if (BNXT_VF(bp)) 1955 goto async_event_process_exit; 1956 1957 /* print unsupported speed warning in forced speed mode only */ 1958 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 1959 (data1 & 0x20000)) { 1960 u16 fw_speed = link_info->force_link_speed; 1961 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1962 1963 if (speed != SPEED_UNKNOWN) 1964 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1965 speed); 1966 } 1967 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 1968 } 1969 /* fall through */ 1970 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 1971 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 1972 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 1973 /* fall through */ 1974 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1975 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1976 break; 1977 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1978 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1979 break; 1980 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1981 u32 data1 = le32_to_cpu(cmpl->event_data1); 1982 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1983 1984 if (BNXT_VF(bp)) 1985 break; 1986 1987 if (bp->pf.port_id != port_id) 1988 break; 1989 1990 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1991 break; 1992 } 1993 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1994 if (BNXT_PF(bp)) 1995 goto async_event_process_exit; 1996 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1997 break; 1998 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 1999 u32 data1 = le32_to_cpu(cmpl->event_data1); 2000 2001 if (!bp->fw_health) 2002 goto async_event_process_exit; 2003 2004 bp->fw_reset_timestamp = jiffies; 2005 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2006 if (!bp->fw_reset_min_dsecs) 2007 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2008 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2009 if (!bp->fw_reset_max_dsecs) 2010 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2011 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2012 netdev_warn(bp->dev, "Firmware fatal reset event received\n"); 2013 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2014 } else { 2015 netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n", 2016 bp->fw_reset_max_dsecs * 100); 2017 } 2018 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2019 break; 2020 } 2021 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2022 struct bnxt_fw_health *fw_health = bp->fw_health; 2023 u32 data1 = le32_to_cpu(cmpl->event_data1); 2024 2025 if (!fw_health) 2026 goto async_event_process_exit; 2027 2028 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1); 2029 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2030 if (!fw_health->enabled) 2031 break; 2032 2033 if (netif_msg_drv(bp)) 2034 netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n", 2035 fw_health->enabled, fw_health->master, 2036 bnxt_fw_health_readl(bp, 2037 BNXT_FW_RESET_CNT_REG), 2038 bnxt_fw_health_readl(bp, 2039 BNXT_FW_HEALTH_REG)); 2040 fw_health->tmr_multiplier = 2041 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2042 bp->current_interval * 10); 2043 fw_health->tmr_counter = fw_health->tmr_multiplier; 2044 fw_health->last_fw_heartbeat = 2045 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2046 fw_health->last_fw_reset_cnt = 2047 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2048 goto async_event_process_exit; 2049 } 2050 default: 2051 goto async_event_process_exit; 2052 } 2053 bnxt_queue_sp_work(bp); 2054 async_event_process_exit: 2055 bnxt_ulp_async_events(bp, cmpl); 2056 return 0; 2057 } 2058 2059 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2060 { 2061 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2062 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2063 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2064 (struct hwrm_fwd_req_cmpl *)txcmp; 2065 2066 switch (cmpl_type) { 2067 case CMPL_BASE_TYPE_HWRM_DONE: 2068 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2069 if (seq_id == bp->hwrm_intr_seq_id) 2070 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id; 2071 else 2072 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 2073 break; 2074 2075 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2076 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2077 2078 if ((vf_id < bp->pf.first_vf_id) || 2079 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2080 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2081 vf_id); 2082 return -EINVAL; 2083 } 2084 2085 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2086 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 2087 bnxt_queue_sp_work(bp); 2088 break; 2089 2090 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2091 bnxt_async_event_process(bp, 2092 (struct hwrm_async_event_cmpl *)txcmp); 2093 2094 default: 2095 break; 2096 } 2097 2098 return 0; 2099 } 2100 2101 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2102 { 2103 struct bnxt_napi *bnapi = dev_instance; 2104 struct bnxt *bp = bnapi->bp; 2105 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2106 u32 cons = RING_CMP(cpr->cp_raw_cons); 2107 2108 cpr->event_ctr++; 2109 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2110 napi_schedule(&bnapi->napi); 2111 return IRQ_HANDLED; 2112 } 2113 2114 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 2115 { 2116 u32 raw_cons = cpr->cp_raw_cons; 2117 u16 cons = RING_CMP(raw_cons); 2118 struct tx_cmp *txcmp; 2119 2120 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2121 2122 return TX_CMP_VALID(txcmp, raw_cons); 2123 } 2124 2125 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 2126 { 2127 struct bnxt_napi *bnapi = dev_instance; 2128 struct bnxt *bp = bnapi->bp; 2129 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2130 u32 cons = RING_CMP(cpr->cp_raw_cons); 2131 u32 int_status; 2132 2133 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2134 2135 if (!bnxt_has_work(bp, cpr)) { 2136 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 2137 /* return if erroneous interrupt */ 2138 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 2139 return IRQ_NONE; 2140 } 2141 2142 /* disable ring IRQ */ 2143 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell); 2144 2145 /* Return here if interrupt is shared and is disabled. */ 2146 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 2147 return IRQ_HANDLED; 2148 2149 napi_schedule(&bnapi->napi); 2150 return IRQ_HANDLED; 2151 } 2152 2153 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2154 int budget) 2155 { 2156 struct bnxt_napi *bnapi = cpr->bnapi; 2157 u32 raw_cons = cpr->cp_raw_cons; 2158 u32 cons; 2159 int tx_pkts = 0; 2160 int rx_pkts = 0; 2161 u8 event = 0; 2162 struct tx_cmp *txcmp; 2163 2164 cpr->has_more_work = 0; 2165 cpr->had_work_done = 1; 2166 while (1) { 2167 int rc; 2168 2169 cons = RING_CMP(raw_cons); 2170 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2171 2172 if (!TX_CMP_VALID(txcmp, raw_cons)) 2173 break; 2174 2175 /* The valid test of the entry must be done first before 2176 * reading any further. 2177 */ 2178 dma_rmb(); 2179 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 2180 tx_pkts++; 2181 /* return full budget so NAPI will complete. */ 2182 if (unlikely(tx_pkts > bp->tx_wake_thresh)) { 2183 rx_pkts = budget; 2184 raw_cons = NEXT_RAW_CMP(raw_cons); 2185 if (budget) 2186 cpr->has_more_work = 1; 2187 break; 2188 } 2189 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2190 if (likely(budget)) 2191 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2192 else 2193 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 2194 &event); 2195 if (likely(rc >= 0)) 2196 rx_pkts += rc; 2197 /* Increment rx_pkts when rc is -ENOMEM to count towards 2198 * the NAPI budget. Otherwise, we may potentially loop 2199 * here forever if we consistently cannot allocate 2200 * buffers. 2201 */ 2202 else if (rc == -ENOMEM && budget) 2203 rx_pkts++; 2204 else if (rc == -EBUSY) /* partial completion */ 2205 break; 2206 } else if (unlikely((TX_CMP_TYPE(txcmp) == 2207 CMPL_BASE_TYPE_HWRM_DONE) || 2208 (TX_CMP_TYPE(txcmp) == 2209 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 2210 (TX_CMP_TYPE(txcmp) == 2211 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 2212 bnxt_hwrm_handler(bp, txcmp); 2213 } 2214 raw_cons = NEXT_RAW_CMP(raw_cons); 2215 2216 if (rx_pkts && rx_pkts == budget) { 2217 cpr->has_more_work = 1; 2218 break; 2219 } 2220 } 2221 2222 if (event & BNXT_REDIRECT_EVENT) 2223 xdp_do_flush_map(); 2224 2225 if (event & BNXT_TX_EVENT) { 2226 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 2227 u16 prod = txr->tx_prod; 2228 2229 /* Sync BD data before updating doorbell */ 2230 wmb(); 2231 2232 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 2233 } 2234 2235 cpr->cp_raw_cons = raw_cons; 2236 bnapi->tx_pkts += tx_pkts; 2237 bnapi->events |= event; 2238 return rx_pkts; 2239 } 2240 2241 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) 2242 { 2243 if (bnapi->tx_pkts) { 2244 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts); 2245 bnapi->tx_pkts = 0; 2246 } 2247 2248 if (bnapi->events & BNXT_RX_EVENT) { 2249 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2250 2251 if (bnapi->events & BNXT_AGG_EVENT) 2252 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2253 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2254 } 2255 bnapi->events = 0; 2256 } 2257 2258 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2259 int budget) 2260 { 2261 struct bnxt_napi *bnapi = cpr->bnapi; 2262 int rx_pkts; 2263 2264 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 2265 2266 /* ACK completion ring before freeing tx ring and producing new 2267 * buffers in rx/agg rings to prevent overflowing the completion 2268 * ring. 2269 */ 2270 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 2271 2272 __bnxt_poll_work_done(bp, bnapi); 2273 return rx_pkts; 2274 } 2275 2276 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 2277 { 2278 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2279 struct bnxt *bp = bnapi->bp; 2280 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2281 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2282 struct tx_cmp *txcmp; 2283 struct rx_cmp_ext *rxcmp1; 2284 u32 cp_cons, tmp_raw_cons; 2285 u32 raw_cons = cpr->cp_raw_cons; 2286 u32 rx_pkts = 0; 2287 u8 event = 0; 2288 2289 while (1) { 2290 int rc; 2291 2292 cp_cons = RING_CMP(raw_cons); 2293 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2294 2295 if (!TX_CMP_VALID(txcmp, raw_cons)) 2296 break; 2297 2298 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 2299 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 2300 cp_cons = RING_CMP(tmp_raw_cons); 2301 rxcmp1 = (struct rx_cmp_ext *) 2302 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2303 2304 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2305 break; 2306 2307 /* force an error to recycle the buffer */ 2308 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2309 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2310 2311 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 2312 if (likely(rc == -EIO) && budget) 2313 rx_pkts++; 2314 else if (rc == -EBUSY) /* partial completion */ 2315 break; 2316 } else if (unlikely(TX_CMP_TYPE(txcmp) == 2317 CMPL_BASE_TYPE_HWRM_DONE)) { 2318 bnxt_hwrm_handler(bp, txcmp); 2319 } else { 2320 netdev_err(bp->dev, 2321 "Invalid completion received on special ring\n"); 2322 } 2323 raw_cons = NEXT_RAW_CMP(raw_cons); 2324 2325 if (rx_pkts == budget) 2326 break; 2327 } 2328 2329 cpr->cp_raw_cons = raw_cons; 2330 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 2331 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 2332 2333 if (event & BNXT_AGG_EVENT) 2334 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 2335 2336 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 2337 napi_complete_done(napi, rx_pkts); 2338 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2339 } 2340 return rx_pkts; 2341 } 2342 2343 static int bnxt_poll(struct napi_struct *napi, int budget) 2344 { 2345 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2346 struct bnxt *bp = bnapi->bp; 2347 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2348 int work_done = 0; 2349 2350 while (1) { 2351 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 2352 2353 if (work_done >= budget) { 2354 if (!budget) 2355 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2356 break; 2357 } 2358 2359 if (!bnxt_has_work(bp, cpr)) { 2360 if (napi_complete_done(napi, work_done)) 2361 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 2362 break; 2363 } 2364 } 2365 if (bp->flags & BNXT_FLAG_DIM) { 2366 struct dim_sample dim_sample = {}; 2367 2368 dim_update_sample(cpr->event_ctr, 2369 cpr->rx_packets, 2370 cpr->rx_bytes, 2371 &dim_sample); 2372 net_dim(&cpr->dim, dim_sample); 2373 } 2374 return work_done; 2375 } 2376 2377 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 2378 { 2379 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2380 int i, work_done = 0; 2381 2382 for (i = 0; i < 2; i++) { 2383 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2384 2385 if (cpr2) { 2386 work_done += __bnxt_poll_work(bp, cpr2, 2387 budget - work_done); 2388 cpr->has_more_work |= cpr2->has_more_work; 2389 } 2390 } 2391 return work_done; 2392 } 2393 2394 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 2395 u64 dbr_type) 2396 { 2397 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2398 int i; 2399 2400 for (i = 0; i < 2; i++) { 2401 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i]; 2402 struct bnxt_db_info *db; 2403 2404 if (cpr2 && cpr2->had_work_done) { 2405 db = &cpr2->cp_db; 2406 writeq(db->db_key64 | dbr_type | 2407 RING_CMP(cpr2->cp_raw_cons), db->doorbell); 2408 cpr2->had_work_done = 0; 2409 } 2410 } 2411 __bnxt_poll_work_done(bp, bnapi); 2412 } 2413 2414 static int bnxt_poll_p5(struct napi_struct *napi, int budget) 2415 { 2416 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 2417 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2418 u32 raw_cons = cpr->cp_raw_cons; 2419 struct bnxt *bp = bnapi->bp; 2420 struct nqe_cn *nqcmp; 2421 int work_done = 0; 2422 u32 cons; 2423 2424 if (cpr->has_more_work) { 2425 cpr->has_more_work = 0; 2426 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 2427 } 2428 while (1) { 2429 cons = RING_CMP(raw_cons); 2430 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 2431 2432 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 2433 if (cpr->has_more_work) 2434 break; 2435 2436 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL); 2437 cpr->cp_raw_cons = raw_cons; 2438 if (napi_complete_done(napi, work_done)) 2439 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 2440 cpr->cp_raw_cons); 2441 return work_done; 2442 } 2443 2444 /* The valid test of the entry must be done first before 2445 * reading any further. 2446 */ 2447 dma_rmb(); 2448 2449 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) { 2450 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 2451 struct bnxt_cp_ring_info *cpr2; 2452 2453 cpr2 = cpr->cp_ring_arr[idx]; 2454 work_done += __bnxt_poll_work(bp, cpr2, 2455 budget - work_done); 2456 cpr->has_more_work |= cpr2->has_more_work; 2457 } else { 2458 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 2459 } 2460 raw_cons = NEXT_RAW_CMP(raw_cons); 2461 } 2462 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ); 2463 if (raw_cons != cpr->cp_raw_cons) { 2464 cpr->cp_raw_cons = raw_cons; 2465 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 2466 } 2467 return work_done; 2468 } 2469 2470 static void bnxt_free_tx_skbs(struct bnxt *bp) 2471 { 2472 int i, max_idx; 2473 struct pci_dev *pdev = bp->pdev; 2474 2475 if (!bp->tx_ring) 2476 return; 2477 2478 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 2479 for (i = 0; i < bp->tx_nr_rings; i++) { 2480 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2481 int j; 2482 2483 for (j = 0; j < max_idx;) { 2484 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 2485 struct sk_buff *skb; 2486 int k, last; 2487 2488 if (i < bp->tx_nr_rings_xdp && 2489 tx_buf->action == XDP_REDIRECT) { 2490 dma_unmap_single(&pdev->dev, 2491 dma_unmap_addr(tx_buf, mapping), 2492 dma_unmap_len(tx_buf, len), 2493 PCI_DMA_TODEVICE); 2494 xdp_return_frame(tx_buf->xdpf); 2495 tx_buf->action = 0; 2496 tx_buf->xdpf = NULL; 2497 j++; 2498 continue; 2499 } 2500 2501 skb = tx_buf->skb; 2502 if (!skb) { 2503 j++; 2504 continue; 2505 } 2506 2507 tx_buf->skb = NULL; 2508 2509 if (tx_buf->is_push) { 2510 dev_kfree_skb(skb); 2511 j += 2; 2512 continue; 2513 } 2514 2515 dma_unmap_single(&pdev->dev, 2516 dma_unmap_addr(tx_buf, mapping), 2517 skb_headlen(skb), 2518 PCI_DMA_TODEVICE); 2519 2520 last = tx_buf->nr_frags; 2521 j += 2; 2522 for (k = 0; k < last; k++, j++) { 2523 int ring_idx = j & bp->tx_ring_mask; 2524 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2525 2526 tx_buf = &txr->tx_buf_ring[ring_idx]; 2527 dma_unmap_page( 2528 &pdev->dev, 2529 dma_unmap_addr(tx_buf, mapping), 2530 skb_frag_size(frag), PCI_DMA_TODEVICE); 2531 } 2532 dev_kfree_skb(skb); 2533 } 2534 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 2535 } 2536 } 2537 2538 static void bnxt_free_rx_skbs(struct bnxt *bp) 2539 { 2540 int i, max_idx, max_agg_idx; 2541 struct pci_dev *pdev = bp->pdev; 2542 2543 if (!bp->rx_ring) 2544 return; 2545 2546 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 2547 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 2548 for (i = 0; i < bp->rx_nr_rings; i++) { 2549 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2550 struct bnxt_tpa_idx_map *map; 2551 int j; 2552 2553 if (rxr->rx_tpa) { 2554 for (j = 0; j < bp->max_tpa; j++) { 2555 struct bnxt_tpa_info *tpa_info = 2556 &rxr->rx_tpa[j]; 2557 u8 *data = tpa_info->data; 2558 2559 if (!data) 2560 continue; 2561 2562 dma_unmap_single_attrs(&pdev->dev, 2563 tpa_info->mapping, 2564 bp->rx_buf_use_size, 2565 bp->rx_dir, 2566 DMA_ATTR_WEAK_ORDERING); 2567 2568 tpa_info->data = NULL; 2569 2570 kfree(data); 2571 } 2572 } 2573 2574 for (j = 0; j < max_idx; j++) { 2575 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 2576 dma_addr_t mapping = rx_buf->mapping; 2577 void *data = rx_buf->data; 2578 2579 if (!data) 2580 continue; 2581 2582 rx_buf->data = NULL; 2583 2584 if (BNXT_RX_PAGE_MODE(bp)) { 2585 mapping -= bp->rx_dma_offset; 2586 dma_unmap_page_attrs(&pdev->dev, mapping, 2587 PAGE_SIZE, bp->rx_dir, 2588 DMA_ATTR_WEAK_ORDERING); 2589 page_pool_recycle_direct(rxr->page_pool, data); 2590 } else { 2591 dma_unmap_single_attrs(&pdev->dev, mapping, 2592 bp->rx_buf_use_size, 2593 bp->rx_dir, 2594 DMA_ATTR_WEAK_ORDERING); 2595 kfree(data); 2596 } 2597 } 2598 2599 for (j = 0; j < max_agg_idx; j++) { 2600 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 2601 &rxr->rx_agg_ring[j]; 2602 struct page *page = rx_agg_buf->page; 2603 2604 if (!page) 2605 continue; 2606 2607 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, 2608 BNXT_RX_PAGE_SIZE, 2609 PCI_DMA_FROMDEVICE, 2610 DMA_ATTR_WEAK_ORDERING); 2611 2612 rx_agg_buf->page = NULL; 2613 __clear_bit(j, rxr->rx_agg_bmap); 2614 2615 __free_page(page); 2616 } 2617 if (rxr->rx_page) { 2618 __free_page(rxr->rx_page); 2619 rxr->rx_page = NULL; 2620 } 2621 map = rxr->rx_tpa_idx_map; 2622 if (map) 2623 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 2624 } 2625 } 2626 2627 static void bnxt_free_skbs(struct bnxt *bp) 2628 { 2629 bnxt_free_tx_skbs(bp); 2630 bnxt_free_rx_skbs(bp); 2631 } 2632 2633 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2634 { 2635 struct pci_dev *pdev = bp->pdev; 2636 int i; 2637 2638 for (i = 0; i < rmem->nr_pages; i++) { 2639 if (!rmem->pg_arr[i]) 2640 continue; 2641 2642 dma_free_coherent(&pdev->dev, rmem->page_size, 2643 rmem->pg_arr[i], rmem->dma_arr[i]); 2644 2645 rmem->pg_arr[i] = NULL; 2646 } 2647 if (rmem->pg_tbl) { 2648 size_t pg_tbl_size = rmem->nr_pages * 8; 2649 2650 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2651 pg_tbl_size = rmem->page_size; 2652 dma_free_coherent(&pdev->dev, pg_tbl_size, 2653 rmem->pg_tbl, rmem->pg_tbl_map); 2654 rmem->pg_tbl = NULL; 2655 } 2656 if (rmem->vmem_size && *rmem->vmem) { 2657 vfree(*rmem->vmem); 2658 *rmem->vmem = NULL; 2659 } 2660 } 2661 2662 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 2663 { 2664 struct pci_dev *pdev = bp->pdev; 2665 u64 valid_bit = 0; 2666 int i; 2667 2668 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 2669 valid_bit = PTU_PTE_VALID; 2670 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 2671 size_t pg_tbl_size = rmem->nr_pages * 8; 2672 2673 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 2674 pg_tbl_size = rmem->page_size; 2675 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 2676 &rmem->pg_tbl_map, 2677 GFP_KERNEL); 2678 if (!rmem->pg_tbl) 2679 return -ENOMEM; 2680 } 2681 2682 for (i = 0; i < rmem->nr_pages; i++) { 2683 u64 extra_bits = valid_bit; 2684 2685 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 2686 rmem->page_size, 2687 &rmem->dma_arr[i], 2688 GFP_KERNEL); 2689 if (!rmem->pg_arr[i]) 2690 return -ENOMEM; 2691 2692 if (rmem->init_val) 2693 memset(rmem->pg_arr[i], rmem->init_val, 2694 rmem->page_size); 2695 if (rmem->nr_pages > 1 || rmem->depth > 0) { 2696 if (i == rmem->nr_pages - 2 && 2697 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2698 extra_bits |= PTU_PTE_NEXT_TO_LAST; 2699 else if (i == rmem->nr_pages - 1 && 2700 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 2701 extra_bits |= PTU_PTE_LAST; 2702 rmem->pg_tbl[i] = 2703 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 2704 } 2705 } 2706 2707 if (rmem->vmem_size) { 2708 *rmem->vmem = vzalloc(rmem->vmem_size); 2709 if (!(*rmem->vmem)) 2710 return -ENOMEM; 2711 } 2712 return 0; 2713 } 2714 2715 static void bnxt_free_tpa_info(struct bnxt *bp) 2716 { 2717 int i; 2718 2719 for (i = 0; i < bp->rx_nr_rings; i++) { 2720 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2721 2722 kfree(rxr->rx_tpa_idx_map); 2723 rxr->rx_tpa_idx_map = NULL; 2724 if (rxr->rx_tpa) { 2725 kfree(rxr->rx_tpa[0].agg_arr); 2726 rxr->rx_tpa[0].agg_arr = NULL; 2727 } 2728 kfree(rxr->rx_tpa); 2729 rxr->rx_tpa = NULL; 2730 } 2731 } 2732 2733 static int bnxt_alloc_tpa_info(struct bnxt *bp) 2734 { 2735 int i, j, total_aggs = 0; 2736 2737 bp->max_tpa = MAX_TPA; 2738 if (bp->flags & BNXT_FLAG_CHIP_P5) { 2739 if (!bp->max_tpa_v2) 2740 return 0; 2741 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 2742 total_aggs = bp->max_tpa * MAX_SKB_FRAGS; 2743 } 2744 2745 for (i = 0; i < bp->rx_nr_rings; i++) { 2746 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2747 struct rx_agg_cmp *agg; 2748 2749 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 2750 GFP_KERNEL); 2751 if (!rxr->rx_tpa) 2752 return -ENOMEM; 2753 2754 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 2755 continue; 2756 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL); 2757 rxr->rx_tpa[0].agg_arr = agg; 2758 if (!agg) 2759 return -ENOMEM; 2760 for (j = 1; j < bp->max_tpa; j++) 2761 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS; 2762 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 2763 GFP_KERNEL); 2764 if (!rxr->rx_tpa_idx_map) 2765 return -ENOMEM; 2766 } 2767 return 0; 2768 } 2769 2770 static void bnxt_free_rx_rings(struct bnxt *bp) 2771 { 2772 int i; 2773 2774 if (!bp->rx_ring) 2775 return; 2776 2777 bnxt_free_tpa_info(bp); 2778 for (i = 0; i < bp->rx_nr_rings; i++) { 2779 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2780 struct bnxt_ring_struct *ring; 2781 2782 if (rxr->xdp_prog) 2783 bpf_prog_put(rxr->xdp_prog); 2784 2785 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 2786 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2787 2788 page_pool_destroy(rxr->page_pool); 2789 rxr->page_pool = NULL; 2790 2791 kfree(rxr->rx_agg_bmap); 2792 rxr->rx_agg_bmap = NULL; 2793 2794 ring = &rxr->rx_ring_struct; 2795 bnxt_free_ring(bp, &ring->ring_mem); 2796 2797 ring = &rxr->rx_agg_ring_struct; 2798 bnxt_free_ring(bp, &ring->ring_mem); 2799 } 2800 } 2801 2802 static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 2803 struct bnxt_rx_ring_info *rxr) 2804 { 2805 struct page_pool_params pp = { 0 }; 2806 2807 pp.pool_size = bp->rx_ring_size; 2808 pp.nid = dev_to_node(&bp->pdev->dev); 2809 pp.dev = &bp->pdev->dev; 2810 pp.dma_dir = DMA_BIDIRECTIONAL; 2811 2812 rxr->page_pool = page_pool_create(&pp); 2813 if (IS_ERR(rxr->page_pool)) { 2814 int err = PTR_ERR(rxr->page_pool); 2815 2816 rxr->page_pool = NULL; 2817 return err; 2818 } 2819 return 0; 2820 } 2821 2822 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2823 { 2824 int i, rc = 0, agg_rings = 0; 2825 2826 if (!bp->rx_ring) 2827 return -ENOMEM; 2828 2829 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2830 agg_rings = 1; 2831 2832 for (i = 0; i < bp->rx_nr_rings; i++) { 2833 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2834 struct bnxt_ring_struct *ring; 2835 2836 ring = &rxr->rx_ring_struct; 2837 2838 rc = bnxt_alloc_rx_page_pool(bp, rxr); 2839 if (rc) 2840 return rc; 2841 2842 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); 2843 if (rc < 0) 2844 return rc; 2845 2846 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 2847 MEM_TYPE_PAGE_POOL, 2848 rxr->page_pool); 2849 if (rc) { 2850 xdp_rxq_info_unreg(&rxr->xdp_rxq); 2851 return rc; 2852 } 2853 2854 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2855 if (rc) 2856 return rc; 2857 2858 ring->grp_idx = i; 2859 if (agg_rings) { 2860 u16 mem_size; 2861 2862 ring = &rxr->rx_agg_ring_struct; 2863 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2864 if (rc) 2865 return rc; 2866 2867 ring->grp_idx = i; 2868 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2869 mem_size = rxr->rx_agg_bmap_size / 8; 2870 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2871 if (!rxr->rx_agg_bmap) 2872 return -ENOMEM; 2873 } 2874 } 2875 if (bp->flags & BNXT_FLAG_TPA) 2876 rc = bnxt_alloc_tpa_info(bp); 2877 return rc; 2878 } 2879 2880 static void bnxt_free_tx_rings(struct bnxt *bp) 2881 { 2882 int i; 2883 struct pci_dev *pdev = bp->pdev; 2884 2885 if (!bp->tx_ring) 2886 return; 2887 2888 for (i = 0; i < bp->tx_nr_rings; i++) { 2889 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2890 struct bnxt_ring_struct *ring; 2891 2892 if (txr->tx_push) { 2893 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2894 txr->tx_push, txr->tx_push_mapping); 2895 txr->tx_push = NULL; 2896 } 2897 2898 ring = &txr->tx_ring_struct; 2899 2900 bnxt_free_ring(bp, &ring->ring_mem); 2901 } 2902 } 2903 2904 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2905 { 2906 int i, j, rc; 2907 struct pci_dev *pdev = bp->pdev; 2908 2909 bp->tx_push_size = 0; 2910 if (bp->tx_push_thresh) { 2911 int push_size; 2912 2913 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2914 bp->tx_push_thresh); 2915 2916 if (push_size > 256) { 2917 push_size = 0; 2918 bp->tx_push_thresh = 0; 2919 } 2920 2921 bp->tx_push_size = push_size; 2922 } 2923 2924 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2925 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2926 struct bnxt_ring_struct *ring; 2927 u8 qidx; 2928 2929 ring = &txr->tx_ring_struct; 2930 2931 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 2932 if (rc) 2933 return rc; 2934 2935 ring->grp_idx = txr->bnapi->index; 2936 if (bp->tx_push_size) { 2937 dma_addr_t mapping; 2938 2939 /* One pre-allocated DMA buffer to backup 2940 * TX push operation 2941 */ 2942 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2943 bp->tx_push_size, 2944 &txr->tx_push_mapping, 2945 GFP_KERNEL); 2946 2947 if (!txr->tx_push) 2948 return -ENOMEM; 2949 2950 mapping = txr->tx_push_mapping + 2951 sizeof(struct tx_push_bd); 2952 txr->data_mapping = cpu_to_le64(mapping); 2953 } 2954 qidx = bp->tc_to_qidx[j]; 2955 ring->queue_id = bp->q_info[qidx].queue_id; 2956 if (i < bp->tx_nr_rings_xdp) 2957 continue; 2958 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2959 j++; 2960 } 2961 return 0; 2962 } 2963 2964 static void bnxt_free_cp_rings(struct bnxt *bp) 2965 { 2966 int i; 2967 2968 if (!bp->bnapi) 2969 return; 2970 2971 for (i = 0; i < bp->cp_nr_rings; i++) { 2972 struct bnxt_napi *bnapi = bp->bnapi[i]; 2973 struct bnxt_cp_ring_info *cpr; 2974 struct bnxt_ring_struct *ring; 2975 int j; 2976 2977 if (!bnapi) 2978 continue; 2979 2980 cpr = &bnapi->cp_ring; 2981 ring = &cpr->cp_ring_struct; 2982 2983 bnxt_free_ring(bp, &ring->ring_mem); 2984 2985 for (j = 0; j < 2; j++) { 2986 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 2987 2988 if (cpr2) { 2989 ring = &cpr2->cp_ring_struct; 2990 bnxt_free_ring(bp, &ring->ring_mem); 2991 kfree(cpr2); 2992 cpr->cp_ring_arr[j] = NULL; 2993 } 2994 } 2995 } 2996 } 2997 2998 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp) 2999 { 3000 struct bnxt_ring_mem_info *rmem; 3001 struct bnxt_ring_struct *ring; 3002 struct bnxt_cp_ring_info *cpr; 3003 int rc; 3004 3005 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL); 3006 if (!cpr) 3007 return NULL; 3008 3009 ring = &cpr->cp_ring_struct; 3010 rmem = &ring->ring_mem; 3011 rmem->nr_pages = bp->cp_nr_pages; 3012 rmem->page_size = HW_CMPD_RING_SIZE; 3013 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3014 rmem->dma_arr = cpr->cp_desc_mapping; 3015 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 3016 rc = bnxt_alloc_ring(bp, rmem); 3017 if (rc) { 3018 bnxt_free_ring(bp, rmem); 3019 kfree(cpr); 3020 cpr = NULL; 3021 } 3022 return cpr; 3023 } 3024 3025 static int bnxt_alloc_cp_rings(struct bnxt *bp) 3026 { 3027 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 3028 int i, rc, ulp_base_vec, ulp_msix; 3029 3030 ulp_msix = bnxt_get_ulp_msix_num(bp); 3031 ulp_base_vec = bnxt_get_ulp_msix_base(bp); 3032 for (i = 0; i < bp->cp_nr_rings; i++) { 3033 struct bnxt_napi *bnapi = bp->bnapi[i]; 3034 struct bnxt_cp_ring_info *cpr; 3035 struct bnxt_ring_struct *ring; 3036 3037 if (!bnapi) 3038 continue; 3039 3040 cpr = &bnapi->cp_ring; 3041 cpr->bnapi = bnapi; 3042 ring = &cpr->cp_ring_struct; 3043 3044 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3045 if (rc) 3046 return rc; 3047 3048 if (ulp_msix && i >= ulp_base_vec) 3049 ring->map_idx = i + ulp_msix; 3050 else 3051 ring->map_idx = i; 3052 3053 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 3054 continue; 3055 3056 if (i < bp->rx_nr_rings) { 3057 struct bnxt_cp_ring_info *cpr2 = 3058 bnxt_alloc_cp_sub_ring(bp); 3059 3060 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2; 3061 if (!cpr2) 3062 return -ENOMEM; 3063 cpr2->bnapi = bnapi; 3064 } 3065 if ((sh && i < bp->tx_nr_rings) || 3066 (!sh && i >= bp->rx_nr_rings)) { 3067 struct bnxt_cp_ring_info *cpr2 = 3068 bnxt_alloc_cp_sub_ring(bp); 3069 3070 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2; 3071 if (!cpr2) 3072 return -ENOMEM; 3073 cpr2->bnapi = bnapi; 3074 } 3075 } 3076 return 0; 3077 } 3078 3079 static void bnxt_init_ring_struct(struct bnxt *bp) 3080 { 3081 int i; 3082 3083 for (i = 0; i < bp->cp_nr_rings; i++) { 3084 struct bnxt_napi *bnapi = bp->bnapi[i]; 3085 struct bnxt_ring_mem_info *rmem; 3086 struct bnxt_cp_ring_info *cpr; 3087 struct bnxt_rx_ring_info *rxr; 3088 struct bnxt_tx_ring_info *txr; 3089 struct bnxt_ring_struct *ring; 3090 3091 if (!bnapi) 3092 continue; 3093 3094 cpr = &bnapi->cp_ring; 3095 ring = &cpr->cp_ring_struct; 3096 rmem = &ring->ring_mem; 3097 rmem->nr_pages = bp->cp_nr_pages; 3098 rmem->page_size = HW_CMPD_RING_SIZE; 3099 rmem->pg_arr = (void **)cpr->cp_desc_ring; 3100 rmem->dma_arr = cpr->cp_desc_mapping; 3101 rmem->vmem_size = 0; 3102 3103 rxr = bnapi->rx_ring; 3104 if (!rxr) 3105 goto skip_rx; 3106 3107 ring = &rxr->rx_ring_struct; 3108 rmem = &ring->ring_mem; 3109 rmem->nr_pages = bp->rx_nr_pages; 3110 rmem->page_size = HW_RXBD_RING_SIZE; 3111 rmem->pg_arr = (void **)rxr->rx_desc_ring; 3112 rmem->dma_arr = rxr->rx_desc_mapping; 3113 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 3114 rmem->vmem = (void **)&rxr->rx_buf_ring; 3115 3116 ring = &rxr->rx_agg_ring_struct; 3117 rmem = &ring->ring_mem; 3118 rmem->nr_pages = bp->rx_agg_nr_pages; 3119 rmem->page_size = HW_RXBD_RING_SIZE; 3120 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 3121 rmem->dma_arr = rxr->rx_agg_desc_mapping; 3122 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 3123 rmem->vmem = (void **)&rxr->rx_agg_ring; 3124 3125 skip_rx: 3126 txr = bnapi->tx_ring; 3127 if (!txr) 3128 continue; 3129 3130 ring = &txr->tx_ring_struct; 3131 rmem = &ring->ring_mem; 3132 rmem->nr_pages = bp->tx_nr_pages; 3133 rmem->page_size = HW_RXBD_RING_SIZE; 3134 rmem->pg_arr = (void **)txr->tx_desc_ring; 3135 rmem->dma_arr = txr->tx_desc_mapping; 3136 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 3137 rmem->vmem = (void **)&txr->tx_buf_ring; 3138 } 3139 } 3140 3141 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 3142 { 3143 int i; 3144 u32 prod; 3145 struct rx_bd **rx_buf_ring; 3146 3147 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 3148 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 3149 int j; 3150 struct rx_bd *rxbd; 3151 3152 rxbd = rx_buf_ring[i]; 3153 if (!rxbd) 3154 continue; 3155 3156 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 3157 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 3158 rxbd->rx_bd_opaque = prod; 3159 } 3160 } 3161 } 3162 3163 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 3164 { 3165 struct net_device *dev = bp->dev; 3166 struct bnxt_rx_ring_info *rxr; 3167 struct bnxt_ring_struct *ring; 3168 u32 prod, type; 3169 int i; 3170 3171 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 3172 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 3173 3174 if (NET_IP_ALIGN == 2) 3175 type |= RX_BD_FLAGS_SOP; 3176 3177 rxr = &bp->rx_ring[ring_nr]; 3178 ring = &rxr->rx_ring_struct; 3179 bnxt_init_rxbd_pages(ring, type); 3180 3181 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 3182 bpf_prog_add(bp->xdp_prog, 1); 3183 rxr->xdp_prog = bp->xdp_prog; 3184 } 3185 prod = rxr->rx_prod; 3186 for (i = 0; i < bp->rx_ring_size; i++) { 3187 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 3188 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 3189 ring_nr, i, bp->rx_ring_size); 3190 break; 3191 } 3192 prod = NEXT_RX(prod); 3193 } 3194 rxr->rx_prod = prod; 3195 ring->fw_ring_id = INVALID_HW_RING_ID; 3196 3197 ring = &rxr->rx_agg_ring_struct; 3198 ring->fw_ring_id = INVALID_HW_RING_ID; 3199 3200 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 3201 return 0; 3202 3203 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 3204 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 3205 3206 bnxt_init_rxbd_pages(ring, type); 3207 3208 prod = rxr->rx_agg_prod; 3209 for (i = 0; i < bp->rx_agg_ring_size; i++) { 3210 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 3211 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 3212 ring_nr, i, bp->rx_ring_size); 3213 break; 3214 } 3215 prod = NEXT_RX_AGG(prod); 3216 } 3217 rxr->rx_agg_prod = prod; 3218 3219 if (bp->flags & BNXT_FLAG_TPA) { 3220 if (rxr->rx_tpa) { 3221 u8 *data; 3222 dma_addr_t mapping; 3223 3224 for (i = 0; i < bp->max_tpa; i++) { 3225 data = __bnxt_alloc_rx_data(bp, &mapping, 3226 GFP_KERNEL); 3227 if (!data) 3228 return -ENOMEM; 3229 3230 rxr->rx_tpa[i].data = data; 3231 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 3232 rxr->rx_tpa[i].mapping = mapping; 3233 } 3234 } else { 3235 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 3236 return -ENOMEM; 3237 } 3238 } 3239 3240 return 0; 3241 } 3242 3243 static void bnxt_init_cp_rings(struct bnxt *bp) 3244 { 3245 int i, j; 3246 3247 for (i = 0; i < bp->cp_nr_rings; i++) { 3248 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 3249 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3250 3251 ring->fw_ring_id = INVALID_HW_RING_ID; 3252 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3253 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3254 for (j = 0; j < 2; j++) { 3255 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 3256 3257 if (!cpr2) 3258 continue; 3259 3260 ring = &cpr2->cp_ring_struct; 3261 ring->fw_ring_id = INVALID_HW_RING_ID; 3262 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 3263 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 3264 } 3265 } 3266 } 3267 3268 static int bnxt_init_rx_rings(struct bnxt *bp) 3269 { 3270 int i, rc = 0; 3271 3272 if (BNXT_RX_PAGE_MODE(bp)) { 3273 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 3274 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 3275 } else { 3276 bp->rx_offset = BNXT_RX_OFFSET; 3277 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 3278 } 3279 3280 for (i = 0; i < bp->rx_nr_rings; i++) { 3281 rc = bnxt_init_one_rx_ring(bp, i); 3282 if (rc) 3283 break; 3284 } 3285 3286 return rc; 3287 } 3288 3289 static int bnxt_init_tx_rings(struct bnxt *bp) 3290 { 3291 u16 i; 3292 3293 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 3294 MAX_SKB_FRAGS + 1); 3295 3296 for (i = 0; i < bp->tx_nr_rings; i++) { 3297 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3298 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3299 3300 ring->fw_ring_id = INVALID_HW_RING_ID; 3301 } 3302 3303 return 0; 3304 } 3305 3306 static void bnxt_free_ring_grps(struct bnxt *bp) 3307 { 3308 kfree(bp->grp_info); 3309 bp->grp_info = NULL; 3310 } 3311 3312 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 3313 { 3314 int i; 3315 3316 if (irq_re_init) { 3317 bp->grp_info = kcalloc(bp->cp_nr_rings, 3318 sizeof(struct bnxt_ring_grp_info), 3319 GFP_KERNEL); 3320 if (!bp->grp_info) 3321 return -ENOMEM; 3322 } 3323 for (i = 0; i < bp->cp_nr_rings; i++) { 3324 if (irq_re_init) 3325 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 3326 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3327 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 3328 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 3329 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3330 } 3331 return 0; 3332 } 3333 3334 static void bnxt_free_vnics(struct bnxt *bp) 3335 { 3336 kfree(bp->vnic_info); 3337 bp->vnic_info = NULL; 3338 bp->nr_vnics = 0; 3339 } 3340 3341 static int bnxt_alloc_vnics(struct bnxt *bp) 3342 { 3343 int num_vnics = 1; 3344 3345 #ifdef CONFIG_RFS_ACCEL 3346 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 3347 num_vnics += bp->rx_nr_rings; 3348 #endif 3349 3350 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3351 num_vnics++; 3352 3353 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 3354 GFP_KERNEL); 3355 if (!bp->vnic_info) 3356 return -ENOMEM; 3357 3358 bp->nr_vnics = num_vnics; 3359 return 0; 3360 } 3361 3362 static void bnxt_init_vnics(struct bnxt *bp) 3363 { 3364 int i; 3365 3366 for (i = 0; i < bp->nr_vnics; i++) { 3367 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3368 int j; 3369 3370 vnic->fw_vnic_id = INVALID_HW_RING_ID; 3371 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 3372 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 3373 3374 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 3375 3376 if (bp->vnic_info[i].rss_hash_key) { 3377 if (i == 0) 3378 prandom_bytes(vnic->rss_hash_key, 3379 HW_HASH_KEY_SIZE); 3380 else 3381 memcpy(vnic->rss_hash_key, 3382 bp->vnic_info[0].rss_hash_key, 3383 HW_HASH_KEY_SIZE); 3384 } 3385 } 3386 } 3387 3388 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 3389 { 3390 int pages; 3391 3392 pages = ring_size / desc_per_pg; 3393 3394 if (!pages) 3395 return 1; 3396 3397 pages++; 3398 3399 while (pages & (pages - 1)) 3400 pages++; 3401 3402 return pages; 3403 } 3404 3405 void bnxt_set_tpa_flags(struct bnxt *bp) 3406 { 3407 bp->flags &= ~BNXT_FLAG_TPA; 3408 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 3409 return; 3410 if (bp->dev->features & NETIF_F_LRO) 3411 bp->flags |= BNXT_FLAG_LRO; 3412 else if (bp->dev->features & NETIF_F_GRO_HW) 3413 bp->flags |= BNXT_FLAG_GRO; 3414 } 3415 3416 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 3417 * be set on entry. 3418 */ 3419 void bnxt_set_ring_params(struct bnxt *bp) 3420 { 3421 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3422 u32 agg_factor = 0, agg_ring_size = 0; 3423 3424 /* 8 for CRC and VLAN */ 3425 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3426 3427 rx_space = rx_size + NET_SKB_PAD + 3428 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3429 3430 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 3431 ring_size = bp->rx_ring_size; 3432 bp->rx_agg_ring_size = 0; 3433 bp->rx_agg_nr_pages = 0; 3434 3435 if (bp->flags & BNXT_FLAG_TPA) 3436 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 3437 3438 bp->flags &= ~BNXT_FLAG_JUMBO; 3439 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 3440 u32 jumbo_factor; 3441 3442 bp->flags |= BNXT_FLAG_JUMBO; 3443 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 3444 if (jumbo_factor > agg_factor) 3445 agg_factor = jumbo_factor; 3446 } 3447 agg_ring_size = ring_size * agg_factor; 3448 3449 if (agg_ring_size) { 3450 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 3451 RX_DESC_CNT); 3452 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3453 u32 tmp = agg_ring_size; 3454 3455 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3456 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3457 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 3458 tmp, agg_ring_size); 3459 } 3460 bp->rx_agg_ring_size = agg_ring_size; 3461 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3462 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 3463 rx_space = rx_size + NET_SKB_PAD + 3464 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3465 } 3466 3467 bp->rx_buf_use_size = rx_size; 3468 bp->rx_buf_size = rx_space; 3469 3470 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 3471 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 3472 3473 ring_size = bp->tx_ring_size; 3474 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 3475 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 3476 3477 max_rx_cmpl = bp->rx_ring_size; 3478 /* MAX TPA needs to be added because TPA_START completions are 3479 * immediately recycled, so the TPA completions are not bound by 3480 * the RX ring size. 3481 */ 3482 if (bp->flags & BNXT_FLAG_TPA) 3483 max_rx_cmpl += bp->max_tpa; 3484 /* RX and TPA completions are 32-byte, all others are 16-byte */ 3485 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 3486 bp->cp_ring_size = ring_size; 3487 3488 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 3489 if (bp->cp_nr_pages > MAX_CP_PAGES) { 3490 bp->cp_nr_pages = MAX_CP_PAGES; 3491 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3492 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 3493 ring_size, bp->cp_ring_size); 3494 } 3495 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 3496 bp->cp_ring_mask = bp->cp_bit - 1; 3497 } 3498 3499 /* Changing allocation mode of RX rings. 3500 * TODO: Update when extending xdp_rxq_info to support allocation modes. 3501 */ 3502 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 3503 { 3504 if (page_mode) { 3505 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) 3506 return -EOPNOTSUPP; 3507 bp->dev->max_mtu = 3508 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 3509 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 3510 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; 3511 bp->rx_dir = DMA_BIDIRECTIONAL; 3512 bp->rx_skb_func = bnxt_rx_page_skb; 3513 /* Disable LRO or GRO_HW */ 3514 netdev_update_features(bp->dev); 3515 } else { 3516 bp->dev->max_mtu = bp->max_mtu; 3517 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 3518 bp->rx_dir = DMA_FROM_DEVICE; 3519 bp->rx_skb_func = bnxt_rx_skb; 3520 } 3521 return 0; 3522 } 3523 3524 static void bnxt_free_vnic_attributes(struct bnxt *bp) 3525 { 3526 int i; 3527 struct bnxt_vnic_info *vnic; 3528 struct pci_dev *pdev = bp->pdev; 3529 3530 if (!bp->vnic_info) 3531 return; 3532 3533 for (i = 0; i < bp->nr_vnics; i++) { 3534 vnic = &bp->vnic_info[i]; 3535 3536 kfree(vnic->fw_grp_ids); 3537 vnic->fw_grp_ids = NULL; 3538 3539 kfree(vnic->uc_list); 3540 vnic->uc_list = NULL; 3541 3542 if (vnic->mc_list) { 3543 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 3544 vnic->mc_list, vnic->mc_list_mapping); 3545 vnic->mc_list = NULL; 3546 } 3547 3548 if (vnic->rss_table) { 3549 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 3550 vnic->rss_table, 3551 vnic->rss_table_dma_addr); 3552 vnic->rss_table = NULL; 3553 } 3554 3555 vnic->rss_hash_key = NULL; 3556 vnic->flags = 0; 3557 } 3558 } 3559 3560 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 3561 { 3562 int i, rc = 0, size; 3563 struct bnxt_vnic_info *vnic; 3564 struct pci_dev *pdev = bp->pdev; 3565 int max_rings; 3566 3567 for (i = 0; i < bp->nr_vnics; i++) { 3568 vnic = &bp->vnic_info[i]; 3569 3570 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 3571 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 3572 3573 if (mem_size > 0) { 3574 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 3575 if (!vnic->uc_list) { 3576 rc = -ENOMEM; 3577 goto out; 3578 } 3579 } 3580 } 3581 3582 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 3583 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 3584 vnic->mc_list = 3585 dma_alloc_coherent(&pdev->dev, 3586 vnic->mc_list_size, 3587 &vnic->mc_list_mapping, 3588 GFP_KERNEL); 3589 if (!vnic->mc_list) { 3590 rc = -ENOMEM; 3591 goto out; 3592 } 3593 } 3594 3595 if (bp->flags & BNXT_FLAG_CHIP_P5) 3596 goto vnic_skip_grps; 3597 3598 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3599 max_rings = bp->rx_nr_rings; 3600 else 3601 max_rings = 1; 3602 3603 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 3604 if (!vnic->fw_grp_ids) { 3605 rc = -ENOMEM; 3606 goto out; 3607 } 3608 vnic_skip_grps: 3609 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 3610 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 3611 continue; 3612 3613 /* Allocate rss table and hash key */ 3614 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 3615 if (bp->flags & BNXT_FLAG_CHIP_P5) 3616 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 3617 3618 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 3619 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 3620 vnic->rss_table_size, 3621 &vnic->rss_table_dma_addr, 3622 GFP_KERNEL); 3623 if (!vnic->rss_table) { 3624 rc = -ENOMEM; 3625 goto out; 3626 } 3627 3628 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 3629 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 3630 } 3631 return 0; 3632 3633 out: 3634 return rc; 3635 } 3636 3637 static void bnxt_free_hwrm_resources(struct bnxt *bp) 3638 { 3639 struct pci_dev *pdev = bp->pdev; 3640 3641 if (bp->hwrm_cmd_resp_addr) { 3642 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 3643 bp->hwrm_cmd_resp_dma_addr); 3644 bp->hwrm_cmd_resp_addr = NULL; 3645 } 3646 3647 if (bp->hwrm_cmd_kong_resp_addr) { 3648 dma_free_coherent(&pdev->dev, PAGE_SIZE, 3649 bp->hwrm_cmd_kong_resp_addr, 3650 bp->hwrm_cmd_kong_resp_dma_addr); 3651 bp->hwrm_cmd_kong_resp_addr = NULL; 3652 } 3653 } 3654 3655 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp) 3656 { 3657 struct pci_dev *pdev = bp->pdev; 3658 3659 if (bp->hwrm_cmd_kong_resp_addr) 3660 return 0; 3661 3662 bp->hwrm_cmd_kong_resp_addr = 3663 dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3664 &bp->hwrm_cmd_kong_resp_dma_addr, 3665 GFP_KERNEL); 3666 if (!bp->hwrm_cmd_kong_resp_addr) 3667 return -ENOMEM; 3668 3669 return 0; 3670 } 3671 3672 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 3673 { 3674 struct pci_dev *pdev = bp->pdev; 3675 3676 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 3677 &bp->hwrm_cmd_resp_dma_addr, 3678 GFP_KERNEL); 3679 if (!bp->hwrm_cmd_resp_addr) 3680 return -ENOMEM; 3681 3682 return 0; 3683 } 3684 3685 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp) 3686 { 3687 if (bp->hwrm_short_cmd_req_addr) { 3688 struct pci_dev *pdev = bp->pdev; 3689 3690 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3691 bp->hwrm_short_cmd_req_addr, 3692 bp->hwrm_short_cmd_req_dma_addr); 3693 bp->hwrm_short_cmd_req_addr = NULL; 3694 } 3695 } 3696 3697 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp) 3698 { 3699 struct pci_dev *pdev = bp->pdev; 3700 3701 if (bp->hwrm_short_cmd_req_addr) 3702 return 0; 3703 3704 bp->hwrm_short_cmd_req_addr = 3705 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len, 3706 &bp->hwrm_short_cmd_req_dma_addr, 3707 GFP_KERNEL); 3708 if (!bp->hwrm_short_cmd_req_addr) 3709 return -ENOMEM; 3710 3711 return 0; 3712 } 3713 3714 static void bnxt_free_port_stats(struct bnxt *bp) 3715 { 3716 struct pci_dev *pdev = bp->pdev; 3717 3718 bp->flags &= ~BNXT_FLAG_PORT_STATS; 3719 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 3720 3721 if (bp->hw_rx_port_stats) { 3722 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 3723 bp->hw_rx_port_stats, 3724 bp->hw_rx_port_stats_map); 3725 bp->hw_rx_port_stats = NULL; 3726 } 3727 3728 if (bp->hw_tx_port_stats_ext) { 3729 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext), 3730 bp->hw_tx_port_stats_ext, 3731 bp->hw_tx_port_stats_ext_map); 3732 bp->hw_tx_port_stats_ext = NULL; 3733 } 3734 3735 if (bp->hw_rx_port_stats_ext) { 3736 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3737 bp->hw_rx_port_stats_ext, 3738 bp->hw_rx_port_stats_ext_map); 3739 bp->hw_rx_port_stats_ext = NULL; 3740 } 3741 3742 if (bp->hw_pcie_stats) { 3743 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats), 3744 bp->hw_pcie_stats, bp->hw_pcie_stats_map); 3745 bp->hw_pcie_stats = NULL; 3746 } 3747 } 3748 3749 static void bnxt_free_ring_stats(struct bnxt *bp) 3750 { 3751 struct pci_dev *pdev = bp->pdev; 3752 int size, i; 3753 3754 if (!bp->bnapi) 3755 return; 3756 3757 size = bp->hw_ring_stats_size; 3758 3759 for (i = 0; i < bp->cp_nr_rings; i++) { 3760 struct bnxt_napi *bnapi = bp->bnapi[i]; 3761 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3762 3763 if (cpr->hw_stats) { 3764 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 3765 cpr->hw_stats_map); 3766 cpr->hw_stats = NULL; 3767 } 3768 } 3769 } 3770 3771 static int bnxt_alloc_stats(struct bnxt *bp) 3772 { 3773 u32 size, i; 3774 struct pci_dev *pdev = bp->pdev; 3775 3776 size = bp->hw_ring_stats_size; 3777 3778 for (i = 0; i < bp->cp_nr_rings; i++) { 3779 struct bnxt_napi *bnapi = bp->bnapi[i]; 3780 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3781 3782 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 3783 &cpr->hw_stats_map, 3784 GFP_KERNEL); 3785 if (!cpr->hw_stats) 3786 return -ENOMEM; 3787 3788 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 3789 } 3790 3791 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 3792 return 0; 3793 3794 if (bp->hw_rx_port_stats) 3795 goto alloc_ext_stats; 3796 3797 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 3798 sizeof(struct tx_port_stats) + 1024; 3799 3800 bp->hw_rx_port_stats = 3801 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 3802 &bp->hw_rx_port_stats_map, 3803 GFP_KERNEL); 3804 if (!bp->hw_rx_port_stats) 3805 return -ENOMEM; 3806 3807 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512; 3808 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 3809 sizeof(struct rx_port_stats) + 512; 3810 bp->flags |= BNXT_FLAG_PORT_STATS; 3811 3812 alloc_ext_stats: 3813 /* Display extended statistics only if FW supports it */ 3814 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 3815 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 3816 return 0; 3817 3818 if (bp->hw_rx_port_stats_ext) 3819 goto alloc_tx_ext_stats; 3820 3821 bp->hw_rx_port_stats_ext = 3822 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext), 3823 &bp->hw_rx_port_stats_ext_map, GFP_KERNEL); 3824 if (!bp->hw_rx_port_stats_ext) 3825 return 0; 3826 3827 alloc_tx_ext_stats: 3828 if (bp->hw_tx_port_stats_ext) 3829 goto alloc_pcie_stats; 3830 3831 if (bp->hwrm_spec_code >= 0x10902 || 3832 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 3833 bp->hw_tx_port_stats_ext = 3834 dma_alloc_coherent(&pdev->dev, 3835 sizeof(struct tx_port_stats_ext), 3836 &bp->hw_tx_port_stats_ext_map, 3837 GFP_KERNEL); 3838 } 3839 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 3840 3841 alloc_pcie_stats: 3842 if (bp->hw_pcie_stats || 3843 !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED)) 3844 return 0; 3845 3846 bp->hw_pcie_stats = 3847 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats), 3848 &bp->hw_pcie_stats_map, GFP_KERNEL); 3849 if (!bp->hw_pcie_stats) 3850 return 0; 3851 3852 bp->flags |= BNXT_FLAG_PCIE_STATS; 3853 return 0; 3854 } 3855 3856 static void bnxt_clear_ring_indices(struct bnxt *bp) 3857 { 3858 int i; 3859 3860 if (!bp->bnapi) 3861 return; 3862 3863 for (i = 0; i < bp->cp_nr_rings; i++) { 3864 struct bnxt_napi *bnapi = bp->bnapi[i]; 3865 struct bnxt_cp_ring_info *cpr; 3866 struct bnxt_rx_ring_info *rxr; 3867 struct bnxt_tx_ring_info *txr; 3868 3869 if (!bnapi) 3870 continue; 3871 3872 cpr = &bnapi->cp_ring; 3873 cpr->cp_raw_cons = 0; 3874 3875 txr = bnapi->tx_ring; 3876 if (txr) { 3877 txr->tx_prod = 0; 3878 txr->tx_cons = 0; 3879 } 3880 3881 rxr = bnapi->rx_ring; 3882 if (rxr) { 3883 rxr->rx_prod = 0; 3884 rxr->rx_agg_prod = 0; 3885 rxr->rx_sw_agg_prod = 0; 3886 rxr->rx_next_cons = 0; 3887 } 3888 } 3889 } 3890 3891 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 3892 { 3893 #ifdef CONFIG_RFS_ACCEL 3894 int i; 3895 3896 /* Under rtnl_lock and all our NAPIs have been disabled. It's 3897 * safe to delete the hash table. 3898 */ 3899 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 3900 struct hlist_head *head; 3901 struct hlist_node *tmp; 3902 struct bnxt_ntuple_filter *fltr; 3903 3904 head = &bp->ntp_fltr_hash_tbl[i]; 3905 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 3906 hlist_del(&fltr->hash); 3907 kfree(fltr); 3908 } 3909 } 3910 if (irq_reinit) { 3911 kfree(bp->ntp_fltr_bmap); 3912 bp->ntp_fltr_bmap = NULL; 3913 } 3914 bp->ntp_fltr_count = 0; 3915 #endif 3916 } 3917 3918 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 3919 { 3920 #ifdef CONFIG_RFS_ACCEL 3921 int i, rc = 0; 3922 3923 if (!(bp->flags & BNXT_FLAG_RFS)) 3924 return 0; 3925 3926 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 3927 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 3928 3929 bp->ntp_fltr_count = 0; 3930 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 3931 sizeof(long), 3932 GFP_KERNEL); 3933 3934 if (!bp->ntp_fltr_bmap) 3935 rc = -ENOMEM; 3936 3937 return rc; 3938 #else 3939 return 0; 3940 #endif 3941 } 3942 3943 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 3944 { 3945 bnxt_free_vnic_attributes(bp); 3946 bnxt_free_tx_rings(bp); 3947 bnxt_free_rx_rings(bp); 3948 bnxt_free_cp_rings(bp); 3949 bnxt_free_ntp_fltrs(bp, irq_re_init); 3950 if (irq_re_init) { 3951 bnxt_free_ring_stats(bp); 3952 bnxt_free_ring_grps(bp); 3953 bnxt_free_vnics(bp); 3954 kfree(bp->tx_ring_map); 3955 bp->tx_ring_map = NULL; 3956 kfree(bp->tx_ring); 3957 bp->tx_ring = NULL; 3958 kfree(bp->rx_ring); 3959 bp->rx_ring = NULL; 3960 kfree(bp->bnapi); 3961 bp->bnapi = NULL; 3962 } else { 3963 bnxt_clear_ring_indices(bp); 3964 } 3965 } 3966 3967 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 3968 { 3969 int i, j, rc, size, arr_size; 3970 void *bnapi; 3971 3972 if (irq_re_init) { 3973 /* Allocate bnapi mem pointer array and mem block for 3974 * all queues 3975 */ 3976 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 3977 bp->cp_nr_rings); 3978 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 3979 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 3980 if (!bnapi) 3981 return -ENOMEM; 3982 3983 bp->bnapi = bnapi; 3984 bnapi += arr_size; 3985 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 3986 bp->bnapi[i] = bnapi; 3987 bp->bnapi[i]->index = i; 3988 bp->bnapi[i]->bp = bp; 3989 if (bp->flags & BNXT_FLAG_CHIP_P5) { 3990 struct bnxt_cp_ring_info *cpr = 3991 &bp->bnapi[i]->cp_ring; 3992 3993 cpr->cp_ring_struct.ring_mem.flags = 3994 BNXT_RMEM_RING_PTE_FLAG; 3995 } 3996 } 3997 3998 bp->rx_ring = kcalloc(bp->rx_nr_rings, 3999 sizeof(struct bnxt_rx_ring_info), 4000 GFP_KERNEL); 4001 if (!bp->rx_ring) 4002 return -ENOMEM; 4003 4004 for (i = 0; i < bp->rx_nr_rings; i++) { 4005 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 4006 4007 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4008 rxr->rx_ring_struct.ring_mem.flags = 4009 BNXT_RMEM_RING_PTE_FLAG; 4010 rxr->rx_agg_ring_struct.ring_mem.flags = 4011 BNXT_RMEM_RING_PTE_FLAG; 4012 } 4013 rxr->bnapi = bp->bnapi[i]; 4014 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 4015 } 4016 4017 bp->tx_ring = kcalloc(bp->tx_nr_rings, 4018 sizeof(struct bnxt_tx_ring_info), 4019 GFP_KERNEL); 4020 if (!bp->tx_ring) 4021 return -ENOMEM; 4022 4023 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 4024 GFP_KERNEL); 4025 4026 if (!bp->tx_ring_map) 4027 return -ENOMEM; 4028 4029 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4030 j = 0; 4031 else 4032 j = bp->rx_nr_rings; 4033 4034 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 4035 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4036 4037 if (bp->flags & BNXT_FLAG_CHIP_P5) 4038 txr->tx_ring_struct.ring_mem.flags = 4039 BNXT_RMEM_RING_PTE_FLAG; 4040 txr->bnapi = bp->bnapi[j]; 4041 bp->bnapi[j]->tx_ring = txr; 4042 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 4043 if (i >= bp->tx_nr_rings_xdp) { 4044 txr->txq_index = i - bp->tx_nr_rings_xdp; 4045 bp->bnapi[j]->tx_int = bnxt_tx_int; 4046 } else { 4047 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP; 4048 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp; 4049 } 4050 } 4051 4052 rc = bnxt_alloc_stats(bp); 4053 if (rc) 4054 goto alloc_mem_err; 4055 4056 rc = bnxt_alloc_ntp_fltrs(bp); 4057 if (rc) 4058 goto alloc_mem_err; 4059 4060 rc = bnxt_alloc_vnics(bp); 4061 if (rc) 4062 goto alloc_mem_err; 4063 } 4064 4065 bnxt_init_ring_struct(bp); 4066 4067 rc = bnxt_alloc_rx_rings(bp); 4068 if (rc) 4069 goto alloc_mem_err; 4070 4071 rc = bnxt_alloc_tx_rings(bp); 4072 if (rc) 4073 goto alloc_mem_err; 4074 4075 rc = bnxt_alloc_cp_rings(bp); 4076 if (rc) 4077 goto alloc_mem_err; 4078 4079 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 4080 BNXT_VNIC_UCAST_FLAG; 4081 rc = bnxt_alloc_vnic_attributes(bp); 4082 if (rc) 4083 goto alloc_mem_err; 4084 return 0; 4085 4086 alloc_mem_err: 4087 bnxt_free_mem(bp, true); 4088 return rc; 4089 } 4090 4091 static void bnxt_disable_int(struct bnxt *bp) 4092 { 4093 int i; 4094 4095 if (!bp->bnapi) 4096 return; 4097 4098 for (i = 0; i < bp->cp_nr_rings; i++) { 4099 struct bnxt_napi *bnapi = bp->bnapi[i]; 4100 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4101 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4102 4103 if (ring->fw_ring_id != INVALID_HW_RING_ID) 4104 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 4105 } 4106 } 4107 4108 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 4109 { 4110 struct bnxt_napi *bnapi = bp->bnapi[n]; 4111 struct bnxt_cp_ring_info *cpr; 4112 4113 cpr = &bnapi->cp_ring; 4114 return cpr->cp_ring_struct.map_idx; 4115 } 4116 4117 static void bnxt_disable_int_sync(struct bnxt *bp) 4118 { 4119 int i; 4120 4121 atomic_inc(&bp->intr_sem); 4122 4123 bnxt_disable_int(bp); 4124 for (i = 0; i < bp->cp_nr_rings; i++) { 4125 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 4126 4127 synchronize_irq(bp->irq_tbl[map_idx].vector); 4128 } 4129 } 4130 4131 static void bnxt_enable_int(struct bnxt *bp) 4132 { 4133 int i; 4134 4135 atomic_set(&bp->intr_sem, 0); 4136 for (i = 0; i < bp->cp_nr_rings; i++) { 4137 struct bnxt_napi *bnapi = bp->bnapi[i]; 4138 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4139 4140 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 4141 } 4142 } 4143 4144 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 4145 u16 cmpl_ring, u16 target_id) 4146 { 4147 struct input *req = request; 4148 4149 req->req_type = cpu_to_le16(req_type); 4150 req->cmpl_ring = cpu_to_le16(cmpl_ring); 4151 req->target_id = cpu_to_le16(target_id); 4152 if (bnxt_kong_hwrm_message(bp, req)) 4153 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 4154 else 4155 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 4156 } 4157 4158 static int bnxt_hwrm_to_stderr(u32 hwrm_err) 4159 { 4160 switch (hwrm_err) { 4161 case HWRM_ERR_CODE_SUCCESS: 4162 return 0; 4163 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: 4164 return -EACCES; 4165 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: 4166 return -ENOSPC; 4167 case HWRM_ERR_CODE_INVALID_PARAMS: 4168 case HWRM_ERR_CODE_INVALID_FLAGS: 4169 case HWRM_ERR_CODE_INVALID_ENABLES: 4170 case HWRM_ERR_CODE_UNSUPPORTED_TLV: 4171 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR: 4172 return -EINVAL; 4173 case HWRM_ERR_CODE_NO_BUFFER: 4174 return -ENOMEM; 4175 case HWRM_ERR_CODE_HOT_RESET_PROGRESS: 4176 case HWRM_ERR_CODE_BUSY: 4177 return -EAGAIN; 4178 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: 4179 return -EOPNOTSUPP; 4180 default: 4181 return -EIO; 4182 } 4183 } 4184 4185 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 4186 int timeout, bool silent) 4187 { 4188 int i, intr_process, rc, tmo_count; 4189 struct input *req = msg; 4190 u32 *data = msg; 4191 u8 *valid; 4192 u16 cp_ring_id, len = 0; 4193 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 4194 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN; 4195 struct hwrm_short_input short_input = {0}; 4196 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER; 4197 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM; 4198 u16 dst = BNXT_HWRM_CHNL_CHIMP; 4199 4200 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4201 return -EBUSY; 4202 4203 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4204 if (msg_len > bp->hwrm_max_ext_req_len || 4205 !bp->hwrm_short_cmd_req_addr) 4206 return -EINVAL; 4207 } 4208 4209 if (bnxt_hwrm_kong_chnl(bp, req)) { 4210 dst = BNXT_HWRM_CHNL_KONG; 4211 bar_offset = BNXT_GRCPF_REG_KONG_COMM; 4212 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER; 4213 resp = bp->hwrm_cmd_kong_resp_addr; 4214 } 4215 4216 memset(resp, 0, PAGE_SIZE); 4217 cp_ring_id = le16_to_cpu(req->cmpl_ring); 4218 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 4219 4220 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst)); 4221 /* currently supports only one outstanding message */ 4222 if (intr_process) 4223 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 4224 4225 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 4226 msg_len > BNXT_HWRM_MAX_REQ_LEN) { 4227 void *short_cmd_req = bp->hwrm_short_cmd_req_addr; 4228 u16 max_msg_len; 4229 4230 /* Set boundary for maximum extended request length for short 4231 * cmd format. If passed up from device use the max supported 4232 * internal req length. 4233 */ 4234 max_msg_len = bp->hwrm_max_ext_req_len; 4235 4236 memcpy(short_cmd_req, req, msg_len); 4237 if (msg_len < max_msg_len) 4238 memset(short_cmd_req + msg_len, 0, 4239 max_msg_len - msg_len); 4240 4241 short_input.req_type = req->req_type; 4242 short_input.signature = 4243 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD); 4244 short_input.size = cpu_to_le16(msg_len); 4245 short_input.req_addr = 4246 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr); 4247 4248 data = (u32 *)&short_input; 4249 msg_len = sizeof(short_input); 4250 4251 /* Sync memory write before updating doorbell */ 4252 wmb(); 4253 4254 max_req_len = BNXT_HWRM_SHORT_REQ_LEN; 4255 } 4256 4257 /* Write request msg to hwrm channel */ 4258 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4); 4259 4260 for (i = msg_len; i < max_req_len; i += 4) 4261 writel(0, bp->bar0 + bar_offset + i); 4262 4263 /* Ring channel doorbell */ 4264 writel(1, bp->bar0 + doorbell_offset); 4265 4266 if (!pci_is_enabled(bp->pdev)) 4267 return 0; 4268 4269 if (!timeout) 4270 timeout = DFLT_HWRM_CMD_TIMEOUT; 4271 /* convert timeout to usec */ 4272 timeout *= 1000; 4273 4274 i = 0; 4275 /* Short timeout for the first few iterations: 4276 * number of loops = number of loops for short timeout + 4277 * number of loops for standard timeout. 4278 */ 4279 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER; 4280 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER; 4281 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT); 4282 4283 if (intr_process) { 4284 u16 seq_id = bp->hwrm_intr_seq_id; 4285 4286 /* Wait until hwrm response cmpl interrupt is processed */ 4287 while (bp->hwrm_intr_seq_id != (u16)~seq_id && 4288 i++ < tmo_count) { 4289 /* Abort the wait for completion if the FW health 4290 * check has failed. 4291 */ 4292 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4293 return -EBUSY; 4294 /* on first few passes, just barely sleep */ 4295 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4296 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4297 HWRM_SHORT_MAX_TIMEOUT); 4298 else 4299 usleep_range(HWRM_MIN_TIMEOUT, 4300 HWRM_MAX_TIMEOUT); 4301 } 4302 4303 if (bp->hwrm_intr_seq_id != (u16)~seq_id) { 4304 if (!silent) 4305 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 4306 le16_to_cpu(req->req_type)); 4307 return -EBUSY; 4308 } 4309 len = le16_to_cpu(resp->resp_len); 4310 valid = ((u8 *)resp) + len - 1; 4311 } else { 4312 int j; 4313 4314 /* Check if response len is updated */ 4315 for (i = 0; i < tmo_count; i++) { 4316 /* Abort the wait for completion if the FW health 4317 * check has failed. 4318 */ 4319 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 4320 return -EBUSY; 4321 len = le16_to_cpu(resp->resp_len); 4322 if (len) 4323 break; 4324 /* on first few passes, just barely sleep */ 4325 if (i < HWRM_SHORT_TIMEOUT_COUNTER) 4326 usleep_range(HWRM_SHORT_MIN_TIMEOUT, 4327 HWRM_SHORT_MAX_TIMEOUT); 4328 else 4329 usleep_range(HWRM_MIN_TIMEOUT, 4330 HWRM_MAX_TIMEOUT); 4331 } 4332 4333 if (i >= tmo_count) { 4334 if (!silent) 4335 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 4336 HWRM_TOTAL_TIMEOUT(i), 4337 le16_to_cpu(req->req_type), 4338 le16_to_cpu(req->seq_id), len); 4339 return -EBUSY; 4340 } 4341 4342 /* Last byte of resp contains valid bit */ 4343 valid = ((u8 *)resp) + len - 1; 4344 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) { 4345 /* make sure we read from updated DMA memory */ 4346 dma_rmb(); 4347 if (*valid) 4348 break; 4349 usleep_range(1, 5); 4350 } 4351 4352 if (j >= HWRM_VALID_BIT_DELAY_USEC) { 4353 if (!silent) 4354 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 4355 HWRM_TOTAL_TIMEOUT(i), 4356 le16_to_cpu(req->req_type), 4357 le16_to_cpu(req->seq_id), len, 4358 *valid); 4359 return -EBUSY; 4360 } 4361 } 4362 4363 /* Zero valid bit for compatibility. Valid bit in an older spec 4364 * may become a new field in a newer spec. We must make sure that 4365 * a new field not implemented by old spec will read zero. 4366 */ 4367 *valid = 0; 4368 rc = le16_to_cpu(resp->error_code); 4369 if (rc && !silent) 4370 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 4371 le16_to_cpu(resp->req_type), 4372 le16_to_cpu(resp->seq_id), rc); 4373 return bnxt_hwrm_to_stderr(rc); 4374 } 4375 4376 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4377 { 4378 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 4379 } 4380 4381 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4382 int timeout) 4383 { 4384 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4385 } 4386 4387 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 4388 { 4389 int rc; 4390 4391 mutex_lock(&bp->hwrm_cmd_lock); 4392 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 4393 mutex_unlock(&bp->hwrm_cmd_lock); 4394 return rc; 4395 } 4396 4397 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 4398 int timeout) 4399 { 4400 int rc; 4401 4402 mutex_lock(&bp->hwrm_cmd_lock); 4403 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 4404 mutex_unlock(&bp->hwrm_cmd_lock); 4405 return rc; 4406 } 4407 4408 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 4409 bool async_only) 4410 { 4411 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; 4412 struct hwrm_func_drv_rgtr_input req = {0}; 4413 DECLARE_BITMAP(async_events_bmap, 256); 4414 u32 *events = (u32 *)async_events_bmap; 4415 u32 flags; 4416 int rc, i; 4417 4418 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 4419 4420 req.enables = 4421 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 4422 FUNC_DRV_RGTR_REQ_ENABLES_VER | 4423 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4424 4425 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 4426 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 4427 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 4428 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 4429 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 4430 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 4431 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 4432 req.flags = cpu_to_le32(flags); 4433 req.ver_maj_8b = DRV_VER_MAJ; 4434 req.ver_min_8b = DRV_VER_MIN; 4435 req.ver_upd_8b = DRV_VER_UPD; 4436 req.ver_maj = cpu_to_le16(DRV_VER_MAJ); 4437 req.ver_min = cpu_to_le16(DRV_VER_MIN); 4438 req.ver_upd = cpu_to_le16(DRV_VER_UPD); 4439 4440 if (BNXT_PF(bp)) { 4441 u32 data[8]; 4442 int i; 4443 4444 memset(data, 0, sizeof(data)); 4445 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 4446 u16 cmd = bnxt_vf_req_snif[i]; 4447 unsigned int bit, idx; 4448 4449 idx = cmd / 32; 4450 bit = cmd % 32; 4451 data[idx] |= 1 << bit; 4452 } 4453 4454 for (i = 0; i < 8; i++) 4455 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 4456 4457 req.enables |= 4458 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 4459 } 4460 4461 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 4462 req.flags |= cpu_to_le32( 4463 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 4464 4465 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 4466 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 4467 u16 event_id = bnxt_async_events_arr[i]; 4468 4469 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 4470 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 4471 continue; 4472 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 4473 } 4474 if (bmap && bmap_size) { 4475 for (i = 0; i < bmap_size; i++) { 4476 if (test_bit(i, bmap)) 4477 __set_bit(i, async_events_bmap); 4478 } 4479 } 4480 for (i = 0; i < 8; i++) 4481 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 4482 4483 if (async_only) 4484 req.enables = 4485 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 4486 4487 mutex_lock(&bp->hwrm_cmd_lock); 4488 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4489 if (!rc) { 4490 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 4491 if (resp->flags & 4492 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 4493 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 4494 } 4495 mutex_unlock(&bp->hwrm_cmd_lock); 4496 return rc; 4497 } 4498 4499 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 4500 { 4501 struct hwrm_func_drv_unrgtr_input req = {0}; 4502 4503 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 4504 return 0; 4505 4506 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 4507 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4508 } 4509 4510 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 4511 { 4512 u32 rc = 0; 4513 struct hwrm_tunnel_dst_port_free_input req = {0}; 4514 4515 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 4516 req.tunnel_type = tunnel_type; 4517 4518 switch (tunnel_type) { 4519 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 4520 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 4521 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 4522 break; 4523 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 4524 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 4525 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 4526 break; 4527 default: 4528 break; 4529 } 4530 4531 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4532 if (rc) 4533 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 4534 rc); 4535 return rc; 4536 } 4537 4538 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 4539 u8 tunnel_type) 4540 { 4541 u32 rc = 0; 4542 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 4543 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4544 4545 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 4546 4547 req.tunnel_type = tunnel_type; 4548 req.tunnel_dst_port_val = port; 4549 4550 mutex_lock(&bp->hwrm_cmd_lock); 4551 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4552 if (rc) { 4553 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 4554 rc); 4555 goto err_out; 4556 } 4557 4558 switch (tunnel_type) { 4559 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 4560 bp->vxlan_fw_dst_port_id = 4561 le16_to_cpu(resp->tunnel_dst_port_id); 4562 break; 4563 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 4564 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 4565 break; 4566 default: 4567 break; 4568 } 4569 4570 err_out: 4571 mutex_unlock(&bp->hwrm_cmd_lock); 4572 return rc; 4573 } 4574 4575 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 4576 { 4577 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 4578 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4579 4580 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 4581 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 4582 4583 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 4584 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 4585 req.mask = cpu_to_le32(vnic->rx_mask); 4586 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4587 } 4588 4589 #ifdef CONFIG_RFS_ACCEL 4590 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 4591 struct bnxt_ntuple_filter *fltr) 4592 { 4593 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 4594 4595 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 4596 req.ntuple_filter_id = fltr->filter_id; 4597 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4598 } 4599 4600 #define BNXT_NTP_FLTR_FLAGS \ 4601 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 4602 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 4603 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 4604 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 4605 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 4606 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 4607 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 4608 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 4609 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 4610 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 4611 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 4612 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 4613 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 4614 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 4615 4616 #define BNXT_NTP_TUNNEL_FLTR_FLAG \ 4617 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 4618 4619 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 4620 struct bnxt_ntuple_filter *fltr) 4621 { 4622 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 4623 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 4624 struct flow_keys *keys = &fltr->fkeys; 4625 struct bnxt_vnic_info *vnic; 4626 u32 flags = 0; 4627 int rc = 0; 4628 4629 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 4630 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 4631 4632 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 4633 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 4634 req.dst_id = cpu_to_le16(fltr->rxq); 4635 } else { 4636 vnic = &bp->vnic_info[fltr->rxq + 1]; 4637 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 4638 } 4639 req.flags = cpu_to_le32(flags); 4640 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 4641 4642 req.ethertype = htons(ETH_P_IP); 4643 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 4644 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 4645 req.ip_protocol = keys->basic.ip_proto; 4646 4647 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 4648 int i; 4649 4650 req.ethertype = htons(ETH_P_IPV6); 4651 req.ip_addr_type = 4652 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 4653 *(struct in6_addr *)&req.src_ipaddr[0] = 4654 keys->addrs.v6addrs.src; 4655 *(struct in6_addr *)&req.dst_ipaddr[0] = 4656 keys->addrs.v6addrs.dst; 4657 for (i = 0; i < 4; i++) { 4658 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4659 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff); 4660 } 4661 } else { 4662 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 4663 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4664 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 4665 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 4666 } 4667 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 4668 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 4669 req.tunnel_type = 4670 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 4671 } 4672 4673 req.src_port = keys->ports.src; 4674 req.src_port_mask = cpu_to_be16(0xffff); 4675 req.dst_port = keys->ports.dst; 4676 req.dst_port_mask = cpu_to_be16(0xffff); 4677 4678 mutex_lock(&bp->hwrm_cmd_lock); 4679 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4680 if (!rc) { 4681 resp = bnxt_get_hwrm_resp_addr(bp, &req); 4682 fltr->filter_id = resp->ntuple_filter_id; 4683 } 4684 mutex_unlock(&bp->hwrm_cmd_lock); 4685 return rc; 4686 } 4687 #endif 4688 4689 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 4690 u8 *mac_addr) 4691 { 4692 u32 rc = 0; 4693 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 4694 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4695 4696 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 4697 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 4698 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 4699 req.flags |= 4700 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 4701 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 4702 req.enables = 4703 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 4704 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 4705 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 4706 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 4707 req.l2_addr_mask[0] = 0xff; 4708 req.l2_addr_mask[1] = 0xff; 4709 req.l2_addr_mask[2] = 0xff; 4710 req.l2_addr_mask[3] = 0xff; 4711 req.l2_addr_mask[4] = 0xff; 4712 req.l2_addr_mask[5] = 0xff; 4713 4714 mutex_lock(&bp->hwrm_cmd_lock); 4715 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4716 if (!rc) 4717 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 4718 resp->l2_filter_id; 4719 mutex_unlock(&bp->hwrm_cmd_lock); 4720 return rc; 4721 } 4722 4723 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 4724 { 4725 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 4726 int rc = 0; 4727 4728 /* Any associated ntuple filters will also be cleared by firmware. */ 4729 mutex_lock(&bp->hwrm_cmd_lock); 4730 for (i = 0; i < num_of_vnics; i++) { 4731 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4732 4733 for (j = 0; j < vnic->uc_filter_count; j++) { 4734 struct hwrm_cfa_l2_filter_free_input req = {0}; 4735 4736 bnxt_hwrm_cmd_hdr_init(bp, &req, 4737 HWRM_CFA_L2_FILTER_FREE, -1, -1); 4738 4739 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 4740 4741 rc = _hwrm_send_message(bp, &req, sizeof(req), 4742 HWRM_CMD_TIMEOUT); 4743 } 4744 vnic->uc_filter_count = 0; 4745 } 4746 mutex_unlock(&bp->hwrm_cmd_lock); 4747 4748 return rc; 4749 } 4750 4751 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 4752 { 4753 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4754 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 4755 struct hwrm_vnic_tpa_cfg_input req = {0}; 4756 4757 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 4758 return 0; 4759 4760 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 4761 4762 if (tpa_flags) { 4763 u16 mss = bp->dev->mtu - 40; 4764 u32 nsegs, n, segs = 0, flags; 4765 4766 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 4767 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 4768 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 4769 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 4770 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 4771 if (tpa_flags & BNXT_FLAG_GRO) 4772 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 4773 4774 req.flags = cpu_to_le32(flags); 4775 4776 req.enables = 4777 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 4778 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 4779 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 4780 4781 /* Number of segs are log2 units, and first packet is not 4782 * included as part of this units. 4783 */ 4784 if (mss <= BNXT_RX_PAGE_SIZE) { 4785 n = BNXT_RX_PAGE_SIZE / mss; 4786 nsegs = (MAX_SKB_FRAGS - 1) * n; 4787 } else { 4788 n = mss / BNXT_RX_PAGE_SIZE; 4789 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 4790 n++; 4791 nsegs = (MAX_SKB_FRAGS - n) / n; 4792 } 4793 4794 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4795 segs = MAX_TPA_SEGS_P5; 4796 max_aggs = bp->max_tpa; 4797 } else { 4798 segs = ilog2(nsegs); 4799 } 4800 req.max_agg_segs = cpu_to_le16(segs); 4801 req.max_aggs = cpu_to_le16(max_aggs); 4802 4803 req.min_agg_len = cpu_to_le32(512); 4804 } 4805 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4806 4807 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4808 } 4809 4810 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 4811 { 4812 struct bnxt_ring_grp_info *grp_info; 4813 4814 grp_info = &bp->grp_info[ring->grp_idx]; 4815 return grp_info->cp_fw_ring_id; 4816 } 4817 4818 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 4819 { 4820 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4821 struct bnxt_napi *bnapi = rxr->bnapi; 4822 struct bnxt_cp_ring_info *cpr; 4823 4824 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL]; 4825 return cpr->cp_ring_struct.fw_ring_id; 4826 } else { 4827 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 4828 } 4829 } 4830 4831 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 4832 { 4833 if (bp->flags & BNXT_FLAG_CHIP_P5) { 4834 struct bnxt_napi *bnapi = txr->bnapi; 4835 struct bnxt_cp_ring_info *cpr; 4836 4837 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL]; 4838 return cpr->cp_ring_struct.fw_ring_id; 4839 } else { 4840 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 4841 } 4842 } 4843 4844 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 4845 { 4846 int entries; 4847 4848 if (bp->flags & BNXT_FLAG_CHIP_P5) 4849 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 4850 else 4851 entries = HW_HASH_INDEX_SIZE; 4852 4853 bp->rss_indir_tbl_entries = entries; 4854 bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), 4855 GFP_KERNEL); 4856 if (!bp->rss_indir_tbl) 4857 return -ENOMEM; 4858 return 0; 4859 } 4860 4861 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp) 4862 { 4863 u16 max_rings, max_entries, pad, i; 4864 4865 if (!bp->rx_nr_rings) 4866 return; 4867 4868 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4869 max_rings = bp->rx_nr_rings - 1; 4870 else 4871 max_rings = bp->rx_nr_rings; 4872 4873 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 4874 4875 for (i = 0; i < max_entries; i++) 4876 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 4877 4878 pad = bp->rss_indir_tbl_entries - max_entries; 4879 if (pad) 4880 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); 4881 } 4882 4883 static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 4884 { 4885 u16 i, tbl_size, max_ring = 0; 4886 4887 if (!bp->rss_indir_tbl) 4888 return 0; 4889 4890 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 4891 for (i = 0; i < tbl_size; i++) 4892 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 4893 return max_ring; 4894 } 4895 4896 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 4897 { 4898 if (bp->flags & BNXT_FLAG_CHIP_P5) 4899 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5); 4900 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4901 return 2; 4902 return 1; 4903 } 4904 4905 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 4906 { 4907 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 4908 u16 i, j; 4909 4910 /* Fill the RSS indirection table with ring group ids */ 4911 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 4912 if (!no_rss) 4913 j = bp->rss_indir_tbl[i]; 4914 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 4915 } 4916 } 4917 4918 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 4919 struct bnxt_vnic_info *vnic) 4920 { 4921 __le16 *ring_tbl = vnic->rss_table; 4922 struct bnxt_rx_ring_info *rxr; 4923 u16 tbl_size, i; 4924 4925 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 4926 4927 for (i = 0; i < tbl_size; i++) { 4928 u16 ring_id, j; 4929 4930 j = bp->rss_indir_tbl[i]; 4931 rxr = &bp->rx_ring[j]; 4932 4933 ring_id = rxr->rx_ring_struct.fw_ring_id; 4934 *ring_tbl++ = cpu_to_le16(ring_id); 4935 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 4936 *ring_tbl++ = cpu_to_le16(ring_id); 4937 } 4938 } 4939 4940 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 4941 { 4942 if (bp->flags & BNXT_FLAG_CHIP_P5) 4943 __bnxt_fill_hw_rss_tbl_p5(bp, vnic); 4944 else 4945 __bnxt_fill_hw_rss_tbl(bp, vnic); 4946 } 4947 4948 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 4949 { 4950 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4951 struct hwrm_vnic_rss_cfg_input req = {0}; 4952 4953 if ((bp->flags & BNXT_FLAG_CHIP_P5) || 4954 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 4955 return 0; 4956 4957 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 4958 if (set_rss) { 4959 bnxt_fill_hw_rss_tbl(bp, vnic); 4960 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 4961 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 4962 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 4963 req.hash_key_tbl_addr = 4964 cpu_to_le64(vnic->rss_hash_key_dma_addr); 4965 } 4966 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 4967 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4968 } 4969 4970 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss) 4971 { 4972 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 4973 struct hwrm_vnic_rss_cfg_input req = {0}; 4974 dma_addr_t ring_tbl_map; 4975 u32 i, nr_ctxs; 4976 4977 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 4978 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 4979 if (!set_rss) { 4980 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4981 return 0; 4982 } 4983 bnxt_fill_hw_rss_tbl(bp, vnic); 4984 req.hash_type = cpu_to_le32(bp->rss_hash_cfg); 4985 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 4986 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 4987 ring_tbl_map = vnic->rss_table_dma_addr; 4988 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 4989 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 4990 int rc; 4991 4992 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 4993 req.ring_table_pair_index = i; 4994 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 4995 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4996 if (rc) 4997 return rc; 4998 } 4999 return 0; 5000 } 5001 5002 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 5003 { 5004 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5005 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 5006 5007 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 5008 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 5009 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 5010 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 5011 req.enables = 5012 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 5013 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 5014 /* thresholds not implemented in firmware yet */ 5015 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 5016 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 5017 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5018 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5019 } 5020 5021 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 5022 u16 ctx_idx) 5023 { 5024 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 5025 5026 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 5027 req.rss_cos_lb_ctx_id = 5028 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 5029 5030 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5031 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 5032 } 5033 5034 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 5035 { 5036 int i, j; 5037 5038 for (i = 0; i < bp->nr_vnics; i++) { 5039 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 5040 5041 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 5042 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 5043 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 5044 } 5045 } 5046 bp->rsscos_nr_ctxs = 0; 5047 } 5048 5049 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 5050 { 5051 int rc; 5052 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 5053 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 5054 bp->hwrm_cmd_resp_addr; 5055 5056 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 5057 -1); 5058 5059 mutex_lock(&bp->hwrm_cmd_lock); 5060 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5061 if (!rc) 5062 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 5063 le16_to_cpu(resp->rss_cos_lb_ctx_id); 5064 mutex_unlock(&bp->hwrm_cmd_lock); 5065 5066 return rc; 5067 } 5068 5069 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 5070 { 5071 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 5072 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 5073 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 5074 } 5075 5076 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 5077 { 5078 unsigned int ring = 0, grp_idx; 5079 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5080 struct hwrm_vnic_cfg_input req = {0}; 5081 u16 def_vlan = 0; 5082 5083 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 5084 5085 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5086 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 5087 5088 req.default_rx_ring_id = 5089 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 5090 req.default_cmpl_ring_id = 5091 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 5092 req.enables = 5093 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 5094 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 5095 goto vnic_mru; 5096 } 5097 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 5098 /* Only RSS support for now TBD: COS & LB */ 5099 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 5100 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 5101 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5102 VNIC_CFG_REQ_ENABLES_MRU); 5103 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 5104 req.rss_rule = 5105 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]); 5106 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 5107 VNIC_CFG_REQ_ENABLES_MRU); 5108 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 5109 } else { 5110 req.rss_rule = cpu_to_le16(0xffff); 5111 } 5112 5113 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 5114 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 5115 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 5116 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 5117 } else { 5118 req.cos_rule = cpu_to_le16(0xffff); 5119 } 5120 5121 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 5122 ring = 0; 5123 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 5124 ring = vnic_id - 1; 5125 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 5126 ring = bp->rx_nr_rings - 1; 5127 5128 grp_idx = bp->rx_ring[ring].bnapi->index; 5129 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 5130 req.lb_rule = cpu_to_le16(0xffff); 5131 vnic_mru: 5132 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN); 5133 5134 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 5135 #ifdef CONFIG_BNXT_SRIOV 5136 if (BNXT_VF(bp)) 5137 def_vlan = bp->vf.vlan; 5138 #endif 5139 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 5140 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 5141 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) 5142 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 5143 5144 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5145 } 5146 5147 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 5148 { 5149 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 5150 struct hwrm_vnic_free_input req = {0}; 5151 5152 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 5153 req.vnic_id = 5154 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 5155 5156 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5157 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 5158 } 5159 } 5160 5161 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 5162 { 5163 u16 i; 5164 5165 for (i = 0; i < bp->nr_vnics; i++) 5166 bnxt_hwrm_vnic_free_one(bp, i); 5167 } 5168 5169 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 5170 unsigned int start_rx_ring_idx, 5171 unsigned int nr_rings) 5172 { 5173 int rc = 0; 5174 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 5175 struct hwrm_vnic_alloc_input req = {0}; 5176 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5177 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5178 5179 if (bp->flags & BNXT_FLAG_CHIP_P5) 5180 goto vnic_no_ring_grps; 5181 5182 /* map ring groups to this vnic */ 5183 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 5184 grp_idx = bp->rx_ring[i].bnapi->index; 5185 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 5186 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 5187 j, nr_rings); 5188 break; 5189 } 5190 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 5191 } 5192 5193 vnic_no_ring_grps: 5194 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 5195 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 5196 if (vnic_id == 0) 5197 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 5198 5199 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 5200 5201 mutex_lock(&bp->hwrm_cmd_lock); 5202 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5203 if (!rc) 5204 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 5205 mutex_unlock(&bp->hwrm_cmd_lock); 5206 return rc; 5207 } 5208 5209 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 5210 { 5211 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5212 struct hwrm_vnic_qcaps_input req = {0}; 5213 int rc; 5214 5215 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 5216 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP); 5217 if (bp->hwrm_spec_code < 0x10600) 5218 return 0; 5219 5220 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1); 5221 mutex_lock(&bp->hwrm_cmd_lock); 5222 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5223 if (!rc) { 5224 u32 flags = le32_to_cpu(resp->flags); 5225 5226 if (!(bp->flags & BNXT_FLAG_CHIP_P5) && 5227 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 5228 bp->flags |= BNXT_FLAG_NEW_RSS_CAP; 5229 if (flags & 5230 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 5231 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 5232 5233 /* Older P5 fw before EXT_HW_STATS support did not set 5234 * VLAN_STRIP_CAP properly. 5235 */ 5236 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 5237 ((bp->flags & BNXT_FLAG_CHIP_P5) && 5238 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 5239 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 5240 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 5241 if (bp->max_tpa_v2) 5242 bp->hw_ring_stats_size = 5243 sizeof(struct ctx_hw_stats_ext); 5244 } 5245 mutex_unlock(&bp->hwrm_cmd_lock); 5246 return rc; 5247 } 5248 5249 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 5250 { 5251 u16 i; 5252 u32 rc = 0; 5253 5254 if (bp->flags & BNXT_FLAG_CHIP_P5) 5255 return 0; 5256 5257 mutex_lock(&bp->hwrm_cmd_lock); 5258 for (i = 0; i < bp->rx_nr_rings; i++) { 5259 struct hwrm_ring_grp_alloc_input req = {0}; 5260 struct hwrm_ring_grp_alloc_output *resp = 5261 bp->hwrm_cmd_resp_addr; 5262 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 5263 5264 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 5265 5266 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 5267 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 5268 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 5269 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 5270 5271 rc = _hwrm_send_message(bp, &req, sizeof(req), 5272 HWRM_CMD_TIMEOUT); 5273 if (rc) 5274 break; 5275 5276 bp->grp_info[grp_idx].fw_grp_id = 5277 le32_to_cpu(resp->ring_group_id); 5278 } 5279 mutex_unlock(&bp->hwrm_cmd_lock); 5280 return rc; 5281 } 5282 5283 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 5284 { 5285 u16 i; 5286 struct hwrm_ring_grp_free_input req = {0}; 5287 5288 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5)) 5289 return; 5290 5291 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 5292 5293 mutex_lock(&bp->hwrm_cmd_lock); 5294 for (i = 0; i < bp->cp_nr_rings; i++) { 5295 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 5296 continue; 5297 req.ring_group_id = 5298 cpu_to_le32(bp->grp_info[i].fw_grp_id); 5299 5300 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5301 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 5302 } 5303 mutex_unlock(&bp->hwrm_cmd_lock); 5304 } 5305 5306 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 5307 struct bnxt_ring_struct *ring, 5308 u32 ring_type, u32 map_index) 5309 { 5310 int rc = 0, err = 0; 5311 struct hwrm_ring_alloc_input req = {0}; 5312 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 5313 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 5314 struct bnxt_ring_grp_info *grp_info; 5315 u16 ring_id; 5316 5317 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 5318 5319 req.enables = 0; 5320 if (rmem->nr_pages > 1) { 5321 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 5322 /* Page size is in log2 units */ 5323 req.page_size = BNXT_PAGE_SHIFT; 5324 req.page_tbl_depth = 1; 5325 } else { 5326 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 5327 } 5328 req.fbo = 0; 5329 /* Association of ring index with doorbell index and MSIX number */ 5330 req.logical_id = cpu_to_le16(map_index); 5331 5332 switch (ring_type) { 5333 case HWRM_RING_ALLOC_TX: { 5334 struct bnxt_tx_ring_info *txr; 5335 5336 txr = container_of(ring, struct bnxt_tx_ring_info, 5337 tx_ring_struct); 5338 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 5339 /* Association of transmit ring with completion ring */ 5340 grp_info = &bp->grp_info[ring->grp_idx]; 5341 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 5342 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 5343 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5344 req.queue_id = cpu_to_le16(ring->queue_id); 5345 break; 5346 } 5347 case HWRM_RING_ALLOC_RX: 5348 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5349 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 5350 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5351 u16 flags = 0; 5352 5353 /* Association of rx ring with stats context */ 5354 grp_info = &bp->grp_info[ring->grp_idx]; 5355 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 5356 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5357 req.enables |= cpu_to_le32( 5358 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5359 if (NET_IP_ALIGN == 2) 5360 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD; 5361 req.flags = cpu_to_le16(flags); 5362 } 5363 break; 5364 case HWRM_RING_ALLOC_AGG: 5365 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5366 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 5367 /* Association of agg ring with rx ring */ 5368 grp_info = &bp->grp_info[ring->grp_idx]; 5369 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 5370 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 5371 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 5372 req.enables |= cpu_to_le32( 5373 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID | 5374 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID); 5375 } else { 5376 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 5377 } 5378 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 5379 break; 5380 case HWRM_RING_ALLOC_CMPL: 5381 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 5382 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5383 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5384 /* Association of cp ring with nq */ 5385 grp_info = &bp->grp_info[map_index]; 5386 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 5387 req.cq_handle = cpu_to_le64(ring->handle); 5388 req.enables |= cpu_to_le32( 5389 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 5390 } else if (bp->flags & BNXT_FLAG_USING_MSIX) { 5391 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5392 } 5393 break; 5394 case HWRM_RING_ALLOC_NQ: 5395 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 5396 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 5397 if (bp->flags & BNXT_FLAG_USING_MSIX) 5398 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 5399 break; 5400 default: 5401 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 5402 ring_type); 5403 return -1; 5404 } 5405 5406 mutex_lock(&bp->hwrm_cmd_lock); 5407 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5408 err = le16_to_cpu(resp->error_code); 5409 ring_id = le16_to_cpu(resp->ring_id); 5410 mutex_unlock(&bp->hwrm_cmd_lock); 5411 5412 if (rc || err) { 5413 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 5414 ring_type, rc, err); 5415 return -EIO; 5416 } 5417 ring->fw_ring_id = ring_id; 5418 return rc; 5419 } 5420 5421 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 5422 { 5423 int rc; 5424 5425 if (BNXT_PF(bp)) { 5426 struct hwrm_func_cfg_input req = {0}; 5427 5428 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 5429 req.fid = cpu_to_le16(0xffff); 5430 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5431 req.async_event_cr = cpu_to_le16(idx); 5432 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5433 } else { 5434 struct hwrm_func_vf_cfg_input req = {0}; 5435 5436 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1); 5437 req.enables = 5438 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 5439 req.async_event_cr = cpu_to_le16(idx); 5440 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5441 } 5442 return rc; 5443 } 5444 5445 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 5446 u32 map_idx, u32 xid) 5447 { 5448 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5449 if (BNXT_PF(bp)) 5450 db->doorbell = bp->bar1 + DB_PF_OFFSET_P5; 5451 else 5452 db->doorbell = bp->bar1 + DB_VF_OFFSET_P5; 5453 switch (ring_type) { 5454 case HWRM_RING_ALLOC_TX: 5455 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 5456 break; 5457 case HWRM_RING_ALLOC_RX: 5458 case HWRM_RING_ALLOC_AGG: 5459 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 5460 break; 5461 case HWRM_RING_ALLOC_CMPL: 5462 db->db_key64 = DBR_PATH_L2; 5463 break; 5464 case HWRM_RING_ALLOC_NQ: 5465 db->db_key64 = DBR_PATH_L2; 5466 break; 5467 } 5468 db->db_key64 |= (u64)xid << DBR_XID_SFT; 5469 } else { 5470 db->doorbell = bp->bar1 + map_idx * 0x80; 5471 switch (ring_type) { 5472 case HWRM_RING_ALLOC_TX: 5473 db->db_key32 = DB_KEY_TX; 5474 break; 5475 case HWRM_RING_ALLOC_RX: 5476 case HWRM_RING_ALLOC_AGG: 5477 db->db_key32 = DB_KEY_RX; 5478 break; 5479 case HWRM_RING_ALLOC_CMPL: 5480 db->db_key32 = DB_KEY_CP; 5481 break; 5482 } 5483 } 5484 } 5485 5486 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 5487 { 5488 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 5489 int i, rc = 0; 5490 u32 type; 5491 5492 if (bp->flags & BNXT_FLAG_CHIP_P5) 5493 type = HWRM_RING_ALLOC_NQ; 5494 else 5495 type = HWRM_RING_ALLOC_CMPL; 5496 for (i = 0; i < bp->cp_nr_rings; i++) { 5497 struct bnxt_napi *bnapi = bp->bnapi[i]; 5498 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5499 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5500 u32 map_idx = ring->map_idx; 5501 unsigned int vector; 5502 5503 vector = bp->irq_tbl[map_idx].vector; 5504 disable_irq_nosync(vector); 5505 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5506 if (rc) { 5507 enable_irq(vector); 5508 goto err_out; 5509 } 5510 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 5511 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5512 enable_irq(vector); 5513 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 5514 5515 if (!i) { 5516 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 5517 if (rc) 5518 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 5519 } 5520 } 5521 5522 type = HWRM_RING_ALLOC_TX; 5523 for (i = 0; i < bp->tx_nr_rings; i++) { 5524 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5525 struct bnxt_ring_struct *ring; 5526 u32 map_idx; 5527 5528 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5529 struct bnxt_napi *bnapi = txr->bnapi; 5530 struct bnxt_cp_ring_info *cpr, *cpr2; 5531 u32 type2 = HWRM_RING_ALLOC_CMPL; 5532 5533 cpr = &bnapi->cp_ring; 5534 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL]; 5535 ring = &cpr2->cp_ring_struct; 5536 ring->handle = BNXT_TX_HDL; 5537 map_idx = bnapi->index; 5538 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5539 if (rc) 5540 goto err_out; 5541 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5542 ring->fw_ring_id); 5543 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5544 } 5545 ring = &txr->tx_ring_struct; 5546 map_idx = i; 5547 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5548 if (rc) 5549 goto err_out; 5550 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id); 5551 } 5552 5553 type = HWRM_RING_ALLOC_RX; 5554 for (i = 0; i < bp->rx_nr_rings; i++) { 5555 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5556 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5557 struct bnxt_napi *bnapi = rxr->bnapi; 5558 u32 map_idx = bnapi->index; 5559 5560 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5561 if (rc) 5562 goto err_out; 5563 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 5564 /* If we have agg rings, post agg buffers first. */ 5565 if (!agg_rings) 5566 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5567 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 5568 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5569 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5570 u32 type2 = HWRM_RING_ALLOC_CMPL; 5571 struct bnxt_cp_ring_info *cpr2; 5572 5573 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL]; 5574 ring = &cpr2->cp_ring_struct; 5575 ring->handle = BNXT_RX_HDL; 5576 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx); 5577 if (rc) 5578 goto err_out; 5579 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx, 5580 ring->fw_ring_id); 5581 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons); 5582 } 5583 } 5584 5585 if (agg_rings) { 5586 type = HWRM_RING_ALLOC_AGG; 5587 for (i = 0; i < bp->rx_nr_rings; i++) { 5588 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5589 struct bnxt_ring_struct *ring = 5590 &rxr->rx_agg_ring_struct; 5591 u32 grp_idx = ring->grp_idx; 5592 u32 map_idx = grp_idx + bp->rx_nr_rings; 5593 5594 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 5595 if (rc) 5596 goto err_out; 5597 5598 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 5599 ring->fw_ring_id); 5600 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 5601 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 5602 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 5603 } 5604 } 5605 err_out: 5606 return rc; 5607 } 5608 5609 static int hwrm_ring_free_send_msg(struct bnxt *bp, 5610 struct bnxt_ring_struct *ring, 5611 u32 ring_type, int cmpl_ring_id) 5612 { 5613 int rc; 5614 struct hwrm_ring_free_input req = {0}; 5615 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 5616 u16 error_code; 5617 5618 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 5619 return 0; 5620 5621 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 5622 req.ring_type = ring_type; 5623 req.ring_id = cpu_to_le16(ring->fw_ring_id); 5624 5625 mutex_lock(&bp->hwrm_cmd_lock); 5626 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5627 error_code = le16_to_cpu(resp->error_code); 5628 mutex_unlock(&bp->hwrm_cmd_lock); 5629 5630 if (rc || error_code) { 5631 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 5632 ring_type, rc, error_code); 5633 return -EIO; 5634 } 5635 return 0; 5636 } 5637 5638 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 5639 { 5640 u32 type; 5641 int i; 5642 5643 if (!bp->bnapi) 5644 return; 5645 5646 for (i = 0; i < bp->tx_nr_rings; i++) { 5647 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5648 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 5649 5650 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5651 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr); 5652 5653 hwrm_ring_free_send_msg(bp, ring, 5654 RING_FREE_REQ_RING_TYPE_TX, 5655 close_path ? cmpl_ring_id : 5656 INVALID_HW_RING_ID); 5657 ring->fw_ring_id = INVALID_HW_RING_ID; 5658 } 5659 } 5660 5661 for (i = 0; i < bp->rx_nr_rings; i++) { 5662 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5663 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 5664 u32 grp_idx = rxr->bnapi->index; 5665 5666 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5667 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5668 5669 hwrm_ring_free_send_msg(bp, ring, 5670 RING_FREE_REQ_RING_TYPE_RX, 5671 close_path ? cmpl_ring_id : 5672 INVALID_HW_RING_ID); 5673 ring->fw_ring_id = INVALID_HW_RING_ID; 5674 bp->grp_info[grp_idx].rx_fw_ring_id = 5675 INVALID_HW_RING_ID; 5676 } 5677 } 5678 5679 if (bp->flags & BNXT_FLAG_CHIP_P5) 5680 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 5681 else 5682 type = RING_FREE_REQ_RING_TYPE_RX; 5683 for (i = 0; i < bp->rx_nr_rings; i++) { 5684 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5685 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 5686 u32 grp_idx = rxr->bnapi->index; 5687 5688 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5689 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 5690 5691 hwrm_ring_free_send_msg(bp, ring, type, 5692 close_path ? cmpl_ring_id : 5693 INVALID_HW_RING_ID); 5694 ring->fw_ring_id = INVALID_HW_RING_ID; 5695 bp->grp_info[grp_idx].agg_fw_ring_id = 5696 INVALID_HW_RING_ID; 5697 } 5698 } 5699 5700 /* The completion rings are about to be freed. After that the 5701 * IRQ doorbell will not work anymore. So we need to disable 5702 * IRQ here. 5703 */ 5704 bnxt_disable_int_sync(bp); 5705 5706 if (bp->flags & BNXT_FLAG_CHIP_P5) 5707 type = RING_FREE_REQ_RING_TYPE_NQ; 5708 else 5709 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 5710 for (i = 0; i < bp->cp_nr_rings; i++) { 5711 struct bnxt_napi *bnapi = bp->bnapi[i]; 5712 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5713 struct bnxt_ring_struct *ring; 5714 int j; 5715 5716 for (j = 0; j < 2; j++) { 5717 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 5718 5719 if (cpr2) { 5720 ring = &cpr2->cp_ring_struct; 5721 if (ring->fw_ring_id == INVALID_HW_RING_ID) 5722 continue; 5723 hwrm_ring_free_send_msg(bp, ring, 5724 RING_FREE_REQ_RING_TYPE_L2_CMPL, 5725 INVALID_HW_RING_ID); 5726 ring->fw_ring_id = INVALID_HW_RING_ID; 5727 } 5728 } 5729 ring = &cpr->cp_ring_struct; 5730 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 5731 hwrm_ring_free_send_msg(bp, ring, type, 5732 INVALID_HW_RING_ID); 5733 ring->fw_ring_id = INVALID_HW_RING_ID; 5734 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 5735 } 5736 } 5737 } 5738 5739 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 5740 bool shared); 5741 5742 static int bnxt_hwrm_get_rings(struct bnxt *bp) 5743 { 5744 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5745 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 5746 struct hwrm_func_qcfg_input req = {0}; 5747 int rc; 5748 5749 if (bp->hwrm_spec_code < 0x10601) 5750 return 0; 5751 5752 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5753 req.fid = cpu_to_le16(0xffff); 5754 mutex_lock(&bp->hwrm_cmd_lock); 5755 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5756 if (rc) { 5757 mutex_unlock(&bp->hwrm_cmd_lock); 5758 return rc; 5759 } 5760 5761 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5762 if (BNXT_NEW_RM(bp)) { 5763 u16 cp, stats; 5764 5765 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 5766 hw_resc->resv_hw_ring_grps = 5767 le32_to_cpu(resp->alloc_hw_ring_grps); 5768 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 5769 cp = le16_to_cpu(resp->alloc_cmpl_rings); 5770 stats = le16_to_cpu(resp->alloc_stat_ctx); 5771 hw_resc->resv_irqs = cp; 5772 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5773 int rx = hw_resc->resv_rx_rings; 5774 int tx = hw_resc->resv_tx_rings; 5775 5776 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5777 rx >>= 1; 5778 if (cp < (rx + tx)) { 5779 bnxt_trim_rings(bp, &rx, &tx, cp, false); 5780 if (bp->flags & BNXT_FLAG_AGG_RINGS) 5781 rx <<= 1; 5782 hw_resc->resv_rx_rings = rx; 5783 hw_resc->resv_tx_rings = tx; 5784 } 5785 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 5786 hw_resc->resv_hw_ring_grps = rx; 5787 } 5788 hw_resc->resv_cp_rings = cp; 5789 hw_resc->resv_stat_ctxs = stats; 5790 } 5791 mutex_unlock(&bp->hwrm_cmd_lock); 5792 return 0; 5793 } 5794 5795 /* Caller must hold bp->hwrm_cmd_lock */ 5796 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 5797 { 5798 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5799 struct hwrm_func_qcfg_input req = {0}; 5800 int rc; 5801 5802 if (bp->hwrm_spec_code < 0x10601) 5803 return 0; 5804 5805 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 5806 req.fid = cpu_to_le16(fid); 5807 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5808 if (!rc) 5809 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 5810 5811 return rc; 5812 } 5813 5814 static bool bnxt_rfs_supported(struct bnxt *bp); 5815 5816 static void 5817 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req, 5818 int tx_rings, int rx_rings, int ring_grps, 5819 int cp_rings, int stats, int vnics) 5820 { 5821 u32 enables = 0; 5822 5823 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1); 5824 req->fid = cpu_to_le16(0xffff); 5825 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 5826 req->num_tx_rings = cpu_to_le16(tx_rings); 5827 if (BNXT_NEW_RM(bp)) { 5828 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 5829 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 5830 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5831 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 5832 enables |= tx_rings + ring_grps ? 5833 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 5834 enables |= rx_rings ? 5835 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 5836 } else { 5837 enables |= cp_rings ? 5838 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 5839 enables |= ring_grps ? 5840 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS | 5841 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 5842 } 5843 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 5844 5845 req->num_rx_rings = cpu_to_le16(rx_rings); 5846 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5847 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 5848 req->num_msix = cpu_to_le16(cp_rings); 5849 req->num_rsscos_ctxs = 5850 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 5851 } else { 5852 req->num_cmpl_rings = cpu_to_le16(cp_rings); 5853 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 5854 req->num_rsscos_ctxs = cpu_to_le16(1); 5855 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) && 5856 bnxt_rfs_supported(bp)) 5857 req->num_rsscos_ctxs = 5858 cpu_to_le16(ring_grps + 1); 5859 } 5860 req->num_stat_ctxs = cpu_to_le16(stats); 5861 req->num_vnics = cpu_to_le16(vnics); 5862 } 5863 req->enables = cpu_to_le32(enables); 5864 } 5865 5866 static void 5867 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, 5868 struct hwrm_func_vf_cfg_input *req, int tx_rings, 5869 int rx_rings, int ring_grps, int cp_rings, 5870 int stats, int vnics) 5871 { 5872 u32 enables = 0; 5873 5874 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1); 5875 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 5876 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 5877 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 5878 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 5879 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5880 enables |= tx_rings + ring_grps ? 5881 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 5882 } else { 5883 enables |= cp_rings ? 5884 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 5885 enables |= ring_grps ? 5886 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 5887 } 5888 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 5889 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 5890 5891 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 5892 req->num_tx_rings = cpu_to_le16(tx_rings); 5893 req->num_rx_rings = cpu_to_le16(rx_rings); 5894 if (bp->flags & BNXT_FLAG_CHIP_P5) { 5895 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps); 5896 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64)); 5897 } else { 5898 req->num_cmpl_rings = cpu_to_le16(cp_rings); 5899 req->num_hw_ring_grps = cpu_to_le16(ring_grps); 5900 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX); 5901 } 5902 req->num_stat_ctxs = cpu_to_le16(stats); 5903 req->num_vnics = cpu_to_le16(vnics); 5904 5905 req->enables = cpu_to_le32(enables); 5906 } 5907 5908 static int 5909 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 5910 int ring_grps, int cp_rings, int stats, int vnics) 5911 { 5912 struct hwrm_func_cfg_input req = {0}; 5913 int rc; 5914 5915 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 5916 cp_rings, stats, vnics); 5917 if (!req.enables) 5918 return 0; 5919 5920 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5921 if (rc) 5922 return rc; 5923 5924 if (bp->hwrm_spec_code < 0x10601) 5925 bp->hw_resc.resv_tx_rings = tx_rings; 5926 5927 return bnxt_hwrm_get_rings(bp); 5928 } 5929 5930 static int 5931 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 5932 int ring_grps, int cp_rings, int stats, int vnics) 5933 { 5934 struct hwrm_func_vf_cfg_input req = {0}; 5935 int rc; 5936 5937 if (!BNXT_NEW_RM(bp)) { 5938 bp->hw_resc.resv_tx_rings = tx_rings; 5939 return 0; 5940 } 5941 5942 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 5943 cp_rings, stats, vnics); 5944 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5945 if (rc) 5946 return rc; 5947 5948 return bnxt_hwrm_get_rings(bp); 5949 } 5950 5951 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp, 5952 int cp, int stat, int vnic) 5953 { 5954 if (BNXT_PF(bp)) 5955 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat, 5956 vnic); 5957 else 5958 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat, 5959 vnic); 5960 } 5961 5962 int bnxt_nq_rings_in_use(struct bnxt *bp) 5963 { 5964 int cp = bp->cp_nr_rings; 5965 int ulp_msix, ulp_base; 5966 5967 ulp_msix = bnxt_get_ulp_msix_num(bp); 5968 if (ulp_msix) { 5969 ulp_base = bnxt_get_ulp_msix_base(bp); 5970 cp += ulp_msix; 5971 if ((ulp_base + ulp_msix) > cp) 5972 cp = ulp_base + ulp_msix; 5973 } 5974 return cp; 5975 } 5976 5977 static int bnxt_cp_rings_in_use(struct bnxt *bp) 5978 { 5979 int cp; 5980 5981 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 5982 return bnxt_nq_rings_in_use(bp); 5983 5984 cp = bp->tx_nr_rings + bp->rx_nr_rings; 5985 return cp; 5986 } 5987 5988 static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 5989 { 5990 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); 5991 int cp = bp->cp_nr_rings; 5992 5993 if (!ulp_stat) 5994 return cp; 5995 5996 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) 5997 return bnxt_get_ulp_msix_base(bp) + ulp_stat; 5998 5999 return cp + ulp_stat; 6000 } 6001 6002 static bool bnxt_need_reserve_rings(struct bnxt *bp) 6003 { 6004 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6005 int cp = bnxt_cp_rings_in_use(bp); 6006 int nq = bnxt_nq_rings_in_use(bp); 6007 int rx = bp->rx_nr_rings, stat; 6008 int vnic = 1, grp = rx; 6009 6010 if (bp->hwrm_spec_code < 0x10601) 6011 return false; 6012 6013 if (hw_resc->resv_tx_rings != bp->tx_nr_rings) 6014 return true; 6015 6016 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6017 vnic = rx + 1; 6018 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6019 rx <<= 1; 6020 stat = bnxt_get_func_stat_ctxs(bp); 6021 if (BNXT_NEW_RM(bp) && 6022 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 6023 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 6024 (hw_resc->resv_hw_ring_grps != grp && 6025 !(bp->flags & BNXT_FLAG_CHIP_P5)))) 6026 return true; 6027 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) && 6028 hw_resc->resv_irqs != nq) 6029 return true; 6030 return false; 6031 } 6032 6033 static int __bnxt_reserve_rings(struct bnxt *bp) 6034 { 6035 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 6036 int cp = bnxt_nq_rings_in_use(bp); 6037 int tx = bp->tx_nr_rings; 6038 int rx = bp->rx_nr_rings; 6039 int grp, rx_rings, rc; 6040 int vnic = 1, stat; 6041 bool sh = false; 6042 6043 if (!bnxt_need_reserve_rings(bp)) 6044 return 0; 6045 6046 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6047 sh = true; 6048 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5)) 6049 vnic = rx + 1; 6050 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6051 rx <<= 1; 6052 grp = bp->rx_nr_rings; 6053 stat = bnxt_get_func_stat_ctxs(bp); 6054 6055 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic); 6056 if (rc) 6057 return rc; 6058 6059 tx = hw_resc->resv_tx_rings; 6060 if (BNXT_NEW_RM(bp)) { 6061 rx = hw_resc->resv_rx_rings; 6062 cp = hw_resc->resv_irqs; 6063 grp = hw_resc->resv_hw_ring_grps; 6064 vnic = hw_resc->resv_vnics; 6065 stat = hw_resc->resv_stat_ctxs; 6066 } 6067 6068 rx_rings = rx; 6069 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 6070 if (rx >= 2) { 6071 rx_rings = rx >> 1; 6072 } else { 6073 if (netif_running(bp->dev)) 6074 return -ENOMEM; 6075 6076 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 6077 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 6078 bp->dev->hw_features &= ~NETIF_F_LRO; 6079 bp->dev->features &= ~NETIF_F_LRO; 6080 bnxt_set_ring_params(bp); 6081 } 6082 } 6083 rx_rings = min_t(int, rx_rings, grp); 6084 cp = min_t(int, cp, bp->cp_nr_rings); 6085 if (stat > bnxt_get_ulp_stat_ctxs(bp)) 6086 stat -= bnxt_get_ulp_stat_ctxs(bp); 6087 cp = min_t(int, cp, stat); 6088 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh); 6089 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6090 rx = rx_rings << 1; 6091 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings; 6092 bp->tx_nr_rings = tx; 6093 6094 /* If we cannot reserve all the RX rings, reset the RSS map only 6095 * if absolutely necessary 6096 */ 6097 if (rx_rings != bp->rx_nr_rings) { 6098 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 6099 rx_rings, bp->rx_nr_rings); 6100 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) && 6101 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 6102 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 6103 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 6104 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 6105 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6106 } 6107 } 6108 bp->rx_nr_rings = rx_rings; 6109 bp->cp_nr_rings = cp; 6110 6111 if (!tx || !rx || !cp || !grp || !vnic || !stat) 6112 return -ENOMEM; 6113 6114 return rc; 6115 } 6116 6117 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6118 int ring_grps, int cp_rings, int stats, 6119 int vnics) 6120 { 6121 struct hwrm_func_vf_cfg_input req = {0}; 6122 u32 flags; 6123 6124 if (!BNXT_NEW_RM(bp)) 6125 return 0; 6126 6127 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6128 cp_rings, stats, vnics); 6129 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 6130 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6131 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6132 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6133 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 6134 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 6135 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6136 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6137 6138 req.flags = cpu_to_le32(flags); 6139 return hwrm_send_message_silent(bp, &req, sizeof(req), 6140 HWRM_CMD_TIMEOUT); 6141 } 6142 6143 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6144 int ring_grps, int cp_rings, int stats, 6145 int vnics) 6146 { 6147 struct hwrm_func_cfg_input req = {0}; 6148 u32 flags; 6149 6150 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps, 6151 cp_rings, stats, vnics); 6152 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 6153 if (BNXT_NEW_RM(bp)) { 6154 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 6155 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 6156 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 6157 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 6158 if (bp->flags & BNXT_FLAG_CHIP_P5) 6159 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 6160 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 6161 else 6162 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 6163 } 6164 6165 req.flags = cpu_to_le32(flags); 6166 return hwrm_send_message_silent(bp, &req, sizeof(req), 6167 HWRM_CMD_TIMEOUT); 6168 } 6169 6170 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings, 6171 int ring_grps, int cp_rings, int stats, 6172 int vnics) 6173 { 6174 if (bp->hwrm_spec_code < 0x10801) 6175 return 0; 6176 6177 if (BNXT_PF(bp)) 6178 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings, 6179 ring_grps, cp_rings, stats, 6180 vnics); 6181 6182 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps, 6183 cp_rings, stats, vnics); 6184 } 6185 6186 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 6187 { 6188 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 6189 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6190 struct hwrm_ring_aggint_qcaps_input req = {0}; 6191 int rc; 6192 6193 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 6194 coal_cap->num_cmpl_dma_aggr_max = 63; 6195 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 6196 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 6197 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 6198 coal_cap->int_lat_tmr_min_max = 65535; 6199 coal_cap->int_lat_tmr_max_max = 65535; 6200 coal_cap->num_cmpl_aggr_int_max = 65535; 6201 coal_cap->timer_units = 80; 6202 6203 if (bp->hwrm_spec_code < 0x10902) 6204 return; 6205 6206 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1); 6207 mutex_lock(&bp->hwrm_cmd_lock); 6208 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6209 if (!rc) { 6210 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 6211 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 6212 coal_cap->num_cmpl_dma_aggr_max = 6213 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 6214 coal_cap->num_cmpl_dma_aggr_during_int_max = 6215 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 6216 coal_cap->cmpl_aggr_dma_tmr_max = 6217 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 6218 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 6219 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 6220 coal_cap->int_lat_tmr_min_max = 6221 le16_to_cpu(resp->int_lat_tmr_min_max); 6222 coal_cap->int_lat_tmr_max_max = 6223 le16_to_cpu(resp->int_lat_tmr_max_max); 6224 coal_cap->num_cmpl_aggr_int_max = 6225 le16_to_cpu(resp->num_cmpl_aggr_int_max); 6226 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 6227 } 6228 mutex_unlock(&bp->hwrm_cmd_lock); 6229 } 6230 6231 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 6232 { 6233 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6234 6235 return usec * 1000 / coal_cap->timer_units; 6236 } 6237 6238 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 6239 struct bnxt_coal *hw_coal, 6240 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 6241 { 6242 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6243 u32 cmpl_params = coal_cap->cmpl_params; 6244 u16 val, tmr, max, flags = 0; 6245 6246 max = hw_coal->bufs_per_record * 128; 6247 if (hw_coal->budget) 6248 max = hw_coal->bufs_per_record * hw_coal->budget; 6249 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 6250 6251 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 6252 req->num_cmpl_aggr_int = cpu_to_le16(val); 6253 6254 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 6255 req->num_cmpl_dma_aggr = cpu_to_le16(val); 6256 6257 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 6258 coal_cap->num_cmpl_dma_aggr_during_int_max); 6259 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 6260 6261 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 6262 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 6263 req->int_lat_tmr_max = cpu_to_le16(tmr); 6264 6265 /* min timer set to 1/2 of interrupt timer */ 6266 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 6267 val = tmr / 2; 6268 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 6269 req->int_lat_tmr_min = cpu_to_le16(val); 6270 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6271 } 6272 6273 /* buf timer set to 1/4 of interrupt timer */ 6274 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 6275 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 6276 6277 if (cmpl_params & 6278 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 6279 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 6280 val = clamp_t(u16, tmr, 1, 6281 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 6282 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 6283 req->enables |= 6284 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 6285 } 6286 6287 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 6288 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 6289 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 6290 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 6291 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 6292 req->flags = cpu_to_le16(flags); 6293 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 6294 } 6295 6296 /* Caller holds bp->hwrm_cmd_lock */ 6297 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 6298 struct bnxt_coal *hw_coal) 6299 { 6300 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0}; 6301 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6302 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 6303 u32 nq_params = coal_cap->nq_params; 6304 u16 tmr; 6305 6306 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 6307 return 0; 6308 6309 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, 6310 -1, -1); 6311 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 6312 req.flags = 6313 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 6314 6315 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 6316 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 6317 req.int_lat_tmr_min = cpu_to_le16(tmr); 6318 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 6319 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6320 } 6321 6322 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 6323 { 6324 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; 6325 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6326 struct bnxt_coal coal; 6327 6328 /* Tick values in micro seconds. 6329 * 1 coal_buf x bufs_per_record = 1 completion record. 6330 */ 6331 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 6332 6333 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 6334 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 6335 6336 if (!bnapi->rx_ring) 6337 return -ENODEV; 6338 6339 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6340 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6341 6342 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx); 6343 6344 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 6345 6346 return hwrm_send_message(bp, &req_rx, sizeof(req_rx), 6347 HWRM_CMD_TIMEOUT); 6348 } 6349 6350 int bnxt_hwrm_set_coal(struct bnxt *bp) 6351 { 6352 int i, rc = 0; 6353 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 6354 req_tx = {0}, *req; 6355 6356 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 6357 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6358 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 6359 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 6360 6361 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx); 6362 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx); 6363 6364 mutex_lock(&bp->hwrm_cmd_lock); 6365 for (i = 0; i < bp->cp_nr_rings; i++) { 6366 struct bnxt_napi *bnapi = bp->bnapi[i]; 6367 struct bnxt_coal *hw_coal; 6368 u16 ring_id; 6369 6370 req = &req_rx; 6371 if (!bnapi->rx_ring) { 6372 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6373 req = &req_tx; 6374 } else { 6375 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 6376 } 6377 req->ring_id = cpu_to_le16(ring_id); 6378 6379 rc = _hwrm_send_message(bp, req, sizeof(*req), 6380 HWRM_CMD_TIMEOUT); 6381 if (rc) 6382 break; 6383 6384 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 6385 continue; 6386 6387 if (bnapi->rx_ring && bnapi->tx_ring) { 6388 req = &req_tx; 6389 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring); 6390 req->ring_id = cpu_to_le16(ring_id); 6391 rc = _hwrm_send_message(bp, req, sizeof(*req), 6392 HWRM_CMD_TIMEOUT); 6393 if (rc) 6394 break; 6395 } 6396 if (bnapi->rx_ring) 6397 hw_coal = &bp->rx_coal; 6398 else 6399 hw_coal = &bp->tx_coal; 6400 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 6401 } 6402 mutex_unlock(&bp->hwrm_cmd_lock); 6403 return rc; 6404 } 6405 6406 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 6407 { 6408 struct hwrm_stat_ctx_clr_stats_input req0 = {0}; 6409 struct hwrm_stat_ctx_free_input req = {0}; 6410 int i; 6411 6412 if (!bp->bnapi) 6413 return; 6414 6415 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6416 return; 6417 6418 bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1); 6419 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 6420 6421 mutex_lock(&bp->hwrm_cmd_lock); 6422 for (i = 0; i < bp->cp_nr_rings; i++) { 6423 struct bnxt_napi *bnapi = bp->bnapi[i]; 6424 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6425 6426 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 6427 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 6428 if (BNXT_FW_MAJ(bp) <= 20) { 6429 req0.stat_ctx_id = req.stat_ctx_id; 6430 _hwrm_send_message(bp, &req0, sizeof(req0), 6431 HWRM_CMD_TIMEOUT); 6432 } 6433 _hwrm_send_message(bp, &req, sizeof(req), 6434 HWRM_CMD_TIMEOUT); 6435 6436 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 6437 } 6438 } 6439 mutex_unlock(&bp->hwrm_cmd_lock); 6440 } 6441 6442 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 6443 { 6444 int rc = 0, i; 6445 struct hwrm_stat_ctx_alloc_input req = {0}; 6446 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 6447 6448 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6449 return 0; 6450 6451 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 6452 6453 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 6454 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 6455 6456 mutex_lock(&bp->hwrm_cmd_lock); 6457 for (i = 0; i < bp->cp_nr_rings; i++) { 6458 struct bnxt_napi *bnapi = bp->bnapi[i]; 6459 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 6460 6461 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 6462 6463 rc = _hwrm_send_message(bp, &req, sizeof(req), 6464 HWRM_CMD_TIMEOUT); 6465 if (rc) 6466 break; 6467 6468 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 6469 6470 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 6471 } 6472 mutex_unlock(&bp->hwrm_cmd_lock); 6473 return rc; 6474 } 6475 6476 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 6477 { 6478 struct hwrm_func_qcfg_input req = {0}; 6479 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 6480 u32 min_db_offset = 0; 6481 u16 flags; 6482 int rc; 6483 6484 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 6485 req.fid = cpu_to_le16(0xffff); 6486 mutex_lock(&bp->hwrm_cmd_lock); 6487 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6488 if (rc) 6489 goto func_qcfg_exit; 6490 6491 #ifdef CONFIG_BNXT_SRIOV 6492 if (BNXT_VF(bp)) { 6493 struct bnxt_vf_info *vf = &bp->vf; 6494 6495 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 6496 } else { 6497 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 6498 } 6499 #endif 6500 flags = le16_to_cpu(resp->flags); 6501 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 6502 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 6503 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 6504 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 6505 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 6506 } 6507 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 6508 bp->flags |= BNXT_FLAG_MULTI_HOST; 6509 6510 switch (resp->port_partition_type) { 6511 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 6512 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 6513 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 6514 bp->port_partition_type = resp->port_partition_type; 6515 break; 6516 } 6517 if (bp->hwrm_spec_code < 0x10707 || 6518 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 6519 bp->br_mode = BRIDGE_MODE_VEB; 6520 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 6521 bp->br_mode = BRIDGE_MODE_VEPA; 6522 else 6523 bp->br_mode = BRIDGE_MODE_UNDEF; 6524 6525 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 6526 if (!bp->max_mtu) 6527 bp->max_mtu = BNXT_MAX_MTU; 6528 6529 if (bp->db_size) 6530 goto func_qcfg_exit; 6531 6532 if (bp->flags & BNXT_FLAG_CHIP_P5) { 6533 if (BNXT_PF(bp)) 6534 min_db_offset = DB_PF_OFFSET_P5; 6535 else 6536 min_db_offset = DB_VF_OFFSET_P5; 6537 } 6538 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 6539 1024); 6540 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 6541 bp->db_size <= min_db_offset) 6542 bp->db_size = pci_resource_len(bp->pdev, 2); 6543 6544 func_qcfg_exit: 6545 mutex_unlock(&bp->hwrm_cmd_lock); 6546 return rc; 6547 } 6548 6549 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 6550 { 6551 struct hwrm_func_backing_store_qcaps_input req = {0}; 6552 struct hwrm_func_backing_store_qcaps_output *resp = 6553 bp->hwrm_cmd_resp_addr; 6554 int rc; 6555 6556 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) 6557 return 0; 6558 6559 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1); 6560 mutex_lock(&bp->hwrm_cmd_lock); 6561 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6562 if (!rc) { 6563 struct bnxt_ctx_pg_info *ctx_pg; 6564 struct bnxt_ctx_mem_info *ctx; 6565 int i, tqm_rings; 6566 6567 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 6568 if (!ctx) { 6569 rc = -ENOMEM; 6570 goto ctx_err; 6571 } 6572 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries); 6573 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 6574 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 6575 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size); 6576 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 6577 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries); 6578 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size); 6579 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 6580 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries); 6581 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size); 6582 ctx->vnic_max_vnic_entries = 6583 le16_to_cpu(resp->vnic_max_vnic_entries); 6584 ctx->vnic_max_ring_table_entries = 6585 le16_to_cpu(resp->vnic_max_ring_table_entries); 6586 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size); 6587 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries); 6588 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size); 6589 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size); 6590 ctx->tqm_min_entries_per_ring = 6591 le32_to_cpu(resp->tqm_min_entries_per_ring); 6592 ctx->tqm_max_entries_per_ring = 6593 le32_to_cpu(resp->tqm_max_entries_per_ring); 6594 ctx->tqm_entries_multiple = resp->tqm_entries_multiple; 6595 if (!ctx->tqm_entries_multiple) 6596 ctx->tqm_entries_multiple = 1; 6597 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries); 6598 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size); 6599 ctx->mrav_num_entries_units = 6600 le16_to_cpu(resp->mrav_num_entries_units); 6601 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size); 6602 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries); 6603 ctx->ctx_kind_initializer = resp->ctx_kind_initializer; 6604 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 6605 if (!ctx->tqm_fp_rings_count) 6606 ctx->tqm_fp_rings_count = bp->max_q; 6607 6608 tqm_rings = ctx->tqm_fp_rings_count + 1; 6609 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL); 6610 if (!ctx_pg) { 6611 kfree(ctx); 6612 rc = -ENOMEM; 6613 goto ctx_err; 6614 } 6615 for (i = 0; i < tqm_rings; i++, ctx_pg++) 6616 ctx->tqm_mem[i] = ctx_pg; 6617 bp->ctx = ctx; 6618 } else { 6619 rc = 0; 6620 } 6621 ctx_err: 6622 mutex_unlock(&bp->hwrm_cmd_lock); 6623 return rc; 6624 } 6625 6626 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 6627 __le64 *pg_dir) 6628 { 6629 u8 pg_size = 0; 6630 6631 if (BNXT_PAGE_SHIFT == 13) 6632 pg_size = 1 << 4; 6633 else if (BNXT_PAGE_SIZE == 16) 6634 pg_size = 2 << 4; 6635 6636 *pg_attr = pg_size; 6637 if (rmem->depth >= 1) { 6638 if (rmem->depth == 2) 6639 *pg_attr |= 2; 6640 else 6641 *pg_attr |= 1; 6642 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 6643 } else { 6644 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 6645 } 6646 } 6647 6648 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 6649 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 6650 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 6651 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 6652 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 6653 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 6654 6655 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 6656 { 6657 struct hwrm_func_backing_store_cfg_input req = {0}; 6658 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6659 struct bnxt_ctx_pg_info *ctx_pg; 6660 __le32 *num_entries; 6661 __le64 *pg_dir; 6662 u32 flags = 0; 6663 u8 *pg_attr; 6664 u32 ena; 6665 int i; 6666 6667 if (!ctx) 6668 return 0; 6669 6670 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1); 6671 req.enables = cpu_to_le32(enables); 6672 6673 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 6674 ctx_pg = &ctx->qp_mem; 6675 req.qp_num_entries = cpu_to_le32(ctx_pg->entries); 6676 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries); 6677 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries); 6678 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size); 6679 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6680 &req.qpc_pg_size_qpc_lvl, 6681 &req.qpc_page_dir); 6682 } 6683 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 6684 ctx_pg = &ctx->srq_mem; 6685 req.srq_num_entries = cpu_to_le32(ctx_pg->entries); 6686 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries); 6687 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size); 6688 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6689 &req.srq_pg_size_srq_lvl, 6690 &req.srq_page_dir); 6691 } 6692 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 6693 ctx_pg = &ctx->cq_mem; 6694 req.cq_num_entries = cpu_to_le32(ctx_pg->entries); 6695 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries); 6696 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size); 6697 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl, 6698 &req.cq_page_dir); 6699 } 6700 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 6701 ctx_pg = &ctx->vnic_mem; 6702 req.vnic_num_vnic_entries = 6703 cpu_to_le16(ctx->vnic_max_vnic_entries); 6704 req.vnic_num_ring_table_entries = 6705 cpu_to_le16(ctx->vnic_max_ring_table_entries); 6706 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size); 6707 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6708 &req.vnic_pg_size_vnic_lvl, 6709 &req.vnic_page_dir); 6710 } 6711 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 6712 ctx_pg = &ctx->stat_mem; 6713 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries); 6714 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size); 6715 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6716 &req.stat_pg_size_stat_lvl, 6717 &req.stat_page_dir); 6718 } 6719 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 6720 ctx_pg = &ctx->mrav_mem; 6721 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries); 6722 if (ctx->mrav_num_entries_units) 6723 flags |= 6724 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 6725 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size); 6726 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6727 &req.mrav_pg_size_mrav_lvl, 6728 &req.mrav_page_dir); 6729 } 6730 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 6731 ctx_pg = &ctx->tim_mem; 6732 req.tim_num_entries = cpu_to_le32(ctx_pg->entries); 6733 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size); 6734 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 6735 &req.tim_pg_size_tim_lvl, 6736 &req.tim_page_dir); 6737 } 6738 for (i = 0, num_entries = &req.tqm_sp_num_entries, 6739 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl, 6740 pg_dir = &req.tqm_sp_page_dir, 6741 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP; 6742 i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 6743 if (!(enables & ena)) 6744 continue; 6745 6746 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size); 6747 ctx_pg = ctx->tqm_mem[i]; 6748 *num_entries = cpu_to_le32(ctx_pg->entries); 6749 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 6750 } 6751 req.flags = cpu_to_le32(flags); 6752 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 6753 } 6754 6755 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 6756 struct bnxt_ctx_pg_info *ctx_pg) 6757 { 6758 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 6759 6760 rmem->page_size = BNXT_PAGE_SIZE; 6761 rmem->pg_arr = ctx_pg->ctx_pg_arr; 6762 rmem->dma_arr = ctx_pg->ctx_dma_arr; 6763 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 6764 if (rmem->depth >= 1) 6765 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 6766 return bnxt_alloc_ring(bp, rmem); 6767 } 6768 6769 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 6770 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 6771 u8 depth, bool use_init_val) 6772 { 6773 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 6774 int rc; 6775 6776 if (!mem_size) 6777 return -EINVAL; 6778 6779 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 6780 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 6781 ctx_pg->nr_pages = 0; 6782 return -EINVAL; 6783 } 6784 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 6785 int nr_tbls, i; 6786 6787 rmem->depth = 2; 6788 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 6789 GFP_KERNEL); 6790 if (!ctx_pg->ctx_pg_tbl) 6791 return -ENOMEM; 6792 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 6793 rmem->nr_pages = nr_tbls; 6794 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 6795 if (rc) 6796 return rc; 6797 for (i = 0; i < nr_tbls; i++) { 6798 struct bnxt_ctx_pg_info *pg_tbl; 6799 6800 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 6801 if (!pg_tbl) 6802 return -ENOMEM; 6803 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 6804 rmem = &pg_tbl->ring_mem; 6805 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 6806 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 6807 rmem->depth = 1; 6808 rmem->nr_pages = MAX_CTX_PAGES; 6809 if (use_init_val) 6810 rmem->init_val = bp->ctx->ctx_kind_initializer; 6811 if (i == (nr_tbls - 1)) { 6812 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 6813 6814 if (rem) 6815 rmem->nr_pages = rem; 6816 } 6817 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 6818 if (rc) 6819 break; 6820 } 6821 } else { 6822 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 6823 if (rmem->nr_pages > 1 || depth) 6824 rmem->depth = 1; 6825 if (use_init_val) 6826 rmem->init_val = bp->ctx->ctx_kind_initializer; 6827 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 6828 } 6829 return rc; 6830 } 6831 6832 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 6833 struct bnxt_ctx_pg_info *ctx_pg) 6834 { 6835 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 6836 6837 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 6838 ctx_pg->ctx_pg_tbl) { 6839 int i, nr_tbls = rmem->nr_pages; 6840 6841 for (i = 0; i < nr_tbls; i++) { 6842 struct bnxt_ctx_pg_info *pg_tbl; 6843 struct bnxt_ring_mem_info *rmem2; 6844 6845 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 6846 if (!pg_tbl) 6847 continue; 6848 rmem2 = &pg_tbl->ring_mem; 6849 bnxt_free_ring(bp, rmem2); 6850 ctx_pg->ctx_pg_arr[i] = NULL; 6851 kfree(pg_tbl); 6852 ctx_pg->ctx_pg_tbl[i] = NULL; 6853 } 6854 kfree(ctx_pg->ctx_pg_tbl); 6855 ctx_pg->ctx_pg_tbl = NULL; 6856 } 6857 bnxt_free_ring(bp, rmem); 6858 ctx_pg->nr_pages = 0; 6859 } 6860 6861 static void bnxt_free_ctx_mem(struct bnxt *bp) 6862 { 6863 struct bnxt_ctx_mem_info *ctx = bp->ctx; 6864 int i; 6865 6866 if (!ctx) 6867 return; 6868 6869 if (ctx->tqm_mem[0]) { 6870 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 6871 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]); 6872 kfree(ctx->tqm_mem[0]); 6873 ctx->tqm_mem[0] = NULL; 6874 } 6875 6876 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem); 6877 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem); 6878 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem); 6879 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem); 6880 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem); 6881 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem); 6882 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem); 6883 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 6884 } 6885 6886 static int bnxt_alloc_ctx_mem(struct bnxt *bp) 6887 { 6888 struct bnxt_ctx_pg_info *ctx_pg; 6889 struct bnxt_ctx_mem_info *ctx; 6890 u32 mem_size, ena, entries; 6891 u32 entries_sp, min; 6892 u32 num_mr, num_ah; 6893 u32 extra_srqs = 0; 6894 u32 extra_qps = 0; 6895 u8 pg_lvl = 1; 6896 int i, rc; 6897 6898 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 6899 if (rc) { 6900 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 6901 rc); 6902 return rc; 6903 } 6904 ctx = bp->ctx; 6905 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 6906 return 0; 6907 6908 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 6909 pg_lvl = 2; 6910 extra_qps = 65536; 6911 extra_srqs = 8192; 6912 } 6913 6914 ctx_pg = &ctx->qp_mem; 6915 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries + 6916 extra_qps; 6917 mem_size = ctx->qp_entry_size * ctx_pg->entries; 6918 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true); 6919 if (rc) 6920 return rc; 6921 6922 ctx_pg = &ctx->srq_mem; 6923 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs; 6924 mem_size = ctx->srq_entry_size * ctx_pg->entries; 6925 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true); 6926 if (rc) 6927 return rc; 6928 6929 ctx_pg = &ctx->cq_mem; 6930 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2; 6931 mem_size = ctx->cq_entry_size * ctx_pg->entries; 6932 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true); 6933 if (rc) 6934 return rc; 6935 6936 ctx_pg = &ctx->vnic_mem; 6937 ctx_pg->entries = ctx->vnic_max_vnic_entries + 6938 ctx->vnic_max_ring_table_entries; 6939 mem_size = ctx->vnic_entry_size * ctx_pg->entries; 6940 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true); 6941 if (rc) 6942 return rc; 6943 6944 ctx_pg = &ctx->stat_mem; 6945 ctx_pg->entries = ctx->stat_max_entries; 6946 mem_size = ctx->stat_entry_size * ctx_pg->entries; 6947 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true); 6948 if (rc) 6949 return rc; 6950 6951 ena = 0; 6952 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 6953 goto skip_rdma; 6954 6955 ctx_pg = &ctx->mrav_mem; 6956 /* 128K extra is needed to accommodate static AH context 6957 * allocation by f/w. 6958 */ 6959 num_mr = 1024 * 256; 6960 num_ah = 1024 * 128; 6961 ctx_pg->entries = num_mr + num_ah; 6962 mem_size = ctx->mrav_entry_size * ctx_pg->entries; 6963 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true); 6964 if (rc) 6965 return rc; 6966 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 6967 if (ctx->mrav_num_entries_units) 6968 ctx_pg->entries = 6969 ((num_mr / ctx->mrav_num_entries_units) << 16) | 6970 (num_ah / ctx->mrav_num_entries_units); 6971 6972 ctx_pg = &ctx->tim_mem; 6973 ctx_pg->entries = ctx->qp_mem.entries; 6974 mem_size = ctx->tim_entry_size * ctx_pg->entries; 6975 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false); 6976 if (rc) 6977 return rc; 6978 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 6979 6980 skip_rdma: 6981 min = ctx->tqm_min_entries_per_ring; 6982 entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 6983 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; 6984 entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); 6985 entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries; 6986 entries = roundup(entries, ctx->tqm_entries_multiple); 6987 entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); 6988 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { 6989 ctx_pg = ctx->tqm_mem[i]; 6990 ctx_pg->entries = i ? entries : entries_sp; 6991 mem_size = ctx->tqm_entry_size * ctx_pg->entries; 6992 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false); 6993 if (rc) 6994 return rc; 6995 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 6996 } 6997 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 6998 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 6999 if (rc) { 7000 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 7001 rc); 7002 return rc; 7003 } 7004 ctx->flags |= BNXT_CTX_FLAG_INITED; 7005 return 0; 7006 } 7007 7008 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 7009 { 7010 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7011 struct hwrm_func_resource_qcaps_input req = {0}; 7012 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7013 int rc; 7014 7015 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1); 7016 req.fid = cpu_to_le16(0xffff); 7017 7018 mutex_lock(&bp->hwrm_cmd_lock); 7019 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), 7020 HWRM_CMD_TIMEOUT); 7021 if (rc) 7022 goto hwrm_func_resc_qcaps_exit; 7023 7024 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 7025 if (!all) 7026 goto hwrm_func_resc_qcaps_exit; 7027 7028 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 7029 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7030 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 7031 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7032 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 7033 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7034 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 7035 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7036 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 7037 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 7038 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 7039 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7040 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 7041 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7042 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 7043 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7044 7045 if (bp->flags & BNXT_FLAG_CHIP_P5) { 7046 u16 max_msix = le16_to_cpu(resp->max_msix); 7047 7048 hw_resc->max_nqs = max_msix; 7049 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 7050 } 7051 7052 if (BNXT_PF(bp)) { 7053 struct bnxt_pf_info *pf = &bp->pf; 7054 7055 pf->vf_resv_strategy = 7056 le16_to_cpu(resp->vf_reservation_strategy); 7057 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 7058 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 7059 } 7060 hwrm_func_resc_qcaps_exit: 7061 mutex_unlock(&bp->hwrm_cmd_lock); 7062 return rc; 7063 } 7064 7065 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 7066 { 7067 int rc = 0; 7068 struct hwrm_func_qcaps_input req = {0}; 7069 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 7070 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7071 u32 flags, flags_ext; 7072 7073 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 7074 req.fid = cpu_to_le16(0xffff); 7075 7076 mutex_lock(&bp->hwrm_cmd_lock); 7077 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7078 if (rc) 7079 goto hwrm_func_qcaps_exit; 7080 7081 flags = le32_to_cpu(resp->flags); 7082 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 7083 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 7084 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 7085 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 7086 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 7087 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 7088 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 7089 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 7090 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 7091 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 7092 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 7093 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 7094 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 7095 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 7096 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 7097 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 7098 7099 flags_ext = le32_to_cpu(resp->flags_ext); 7100 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 7101 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 7102 7103 bp->tx_push_thresh = 0; 7104 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 7105 BNXT_FW_MAJ(bp) > 217) 7106 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 7107 7108 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 7109 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 7110 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 7111 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 7112 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 7113 if (!hw_resc->max_hw_ring_grps) 7114 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 7115 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 7116 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 7117 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 7118 7119 if (BNXT_PF(bp)) { 7120 struct bnxt_pf_info *pf = &bp->pf; 7121 7122 pf->fw_fid = le16_to_cpu(resp->fid); 7123 pf->port_id = le16_to_cpu(resp->port_id); 7124 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 7125 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 7126 pf->max_vfs = le16_to_cpu(resp->max_vfs); 7127 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 7128 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 7129 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 7130 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 7131 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 7132 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 7133 bp->flags &= ~BNXT_FLAG_WOL_CAP; 7134 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 7135 bp->flags |= BNXT_FLAG_WOL_CAP; 7136 } else { 7137 #ifdef CONFIG_BNXT_SRIOV 7138 struct bnxt_vf_info *vf = &bp->vf; 7139 7140 vf->fw_fid = le16_to_cpu(resp->fid); 7141 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 7142 #endif 7143 } 7144 7145 hwrm_func_qcaps_exit: 7146 mutex_unlock(&bp->hwrm_cmd_lock); 7147 return rc; 7148 } 7149 7150 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 7151 7152 static int bnxt_hwrm_func_qcaps(struct bnxt *bp) 7153 { 7154 int rc; 7155 7156 rc = __bnxt_hwrm_func_qcaps(bp); 7157 if (rc) 7158 return rc; 7159 rc = bnxt_hwrm_queue_qportcfg(bp); 7160 if (rc) { 7161 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 7162 return rc; 7163 } 7164 if (bp->hwrm_spec_code >= 0x10803) { 7165 rc = bnxt_alloc_ctx_mem(bp); 7166 if (rc) 7167 return rc; 7168 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 7169 if (!rc) 7170 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 7171 } 7172 return 0; 7173 } 7174 7175 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 7176 { 7177 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0}; 7178 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 7179 int rc = 0; 7180 u32 flags; 7181 7182 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 7183 return 0; 7184 7185 resp = bp->hwrm_cmd_resp_addr; 7186 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1); 7187 7188 mutex_lock(&bp->hwrm_cmd_lock); 7189 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7190 if (rc) 7191 goto hwrm_cfa_adv_qcaps_exit; 7192 7193 flags = le32_to_cpu(resp->flags); 7194 if (flags & 7195 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 7196 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 7197 7198 hwrm_cfa_adv_qcaps_exit: 7199 mutex_unlock(&bp->hwrm_cmd_lock); 7200 return rc; 7201 } 7202 7203 static int bnxt_map_fw_health_regs(struct bnxt *bp) 7204 { 7205 struct bnxt_fw_health *fw_health = bp->fw_health; 7206 u32 reg_base = 0xffffffff; 7207 int i; 7208 7209 /* Only pre-map the monitoring GRC registers using window 3 */ 7210 for (i = 0; i < 4; i++) { 7211 u32 reg = fw_health->regs[i]; 7212 7213 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 7214 continue; 7215 if (reg_base == 0xffffffff) 7216 reg_base = reg & BNXT_GRC_BASE_MASK; 7217 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 7218 return -ERANGE; 7219 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE + 7220 (reg & BNXT_GRC_OFFSET_MASK); 7221 } 7222 if (reg_base == 0xffffffff) 7223 return 0; 7224 7225 writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 7226 BNXT_FW_HEALTH_WIN_MAP_OFF); 7227 return 0; 7228 } 7229 7230 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 7231 { 7232 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 7233 struct bnxt_fw_health *fw_health = bp->fw_health; 7234 struct hwrm_error_recovery_qcfg_input req = {0}; 7235 int rc, i; 7236 7237 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 7238 return 0; 7239 7240 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1); 7241 mutex_lock(&bp->hwrm_cmd_lock); 7242 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7243 if (rc) 7244 goto err_recovery_out; 7245 fw_health->flags = le32_to_cpu(resp->flags); 7246 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 7247 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 7248 rc = -EINVAL; 7249 goto err_recovery_out; 7250 } 7251 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 7252 fw_health->master_func_wait_dsecs = 7253 le32_to_cpu(resp->master_func_wait_period); 7254 fw_health->normal_func_wait_dsecs = 7255 le32_to_cpu(resp->normal_func_wait_period); 7256 fw_health->post_reset_wait_dsecs = 7257 le32_to_cpu(resp->master_func_wait_period_after_reset); 7258 fw_health->post_reset_max_wait_dsecs = 7259 le32_to_cpu(resp->max_bailout_time_after_reset); 7260 fw_health->regs[BNXT_FW_HEALTH_REG] = 7261 le32_to_cpu(resp->fw_health_status_reg); 7262 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 7263 le32_to_cpu(resp->fw_heartbeat_reg); 7264 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 7265 le32_to_cpu(resp->fw_reset_cnt_reg); 7266 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 7267 le32_to_cpu(resp->reset_inprogress_reg); 7268 fw_health->fw_reset_inprog_reg_mask = 7269 le32_to_cpu(resp->reset_inprogress_reg_mask); 7270 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 7271 if (fw_health->fw_reset_seq_cnt >= 16) { 7272 rc = -EINVAL; 7273 goto err_recovery_out; 7274 } 7275 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 7276 fw_health->fw_reset_seq_regs[i] = 7277 le32_to_cpu(resp->reset_reg[i]); 7278 fw_health->fw_reset_seq_vals[i] = 7279 le32_to_cpu(resp->reset_reg_val[i]); 7280 fw_health->fw_reset_seq_delay_msec[i] = 7281 resp->delay_after_reset[i]; 7282 } 7283 err_recovery_out: 7284 mutex_unlock(&bp->hwrm_cmd_lock); 7285 if (!rc) 7286 rc = bnxt_map_fw_health_regs(bp); 7287 if (rc) 7288 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 7289 return rc; 7290 } 7291 7292 static int bnxt_hwrm_func_reset(struct bnxt *bp) 7293 { 7294 struct hwrm_func_reset_input req = {0}; 7295 7296 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 7297 req.enables = 0; 7298 7299 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 7300 } 7301 7302 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 7303 { 7304 int rc = 0; 7305 struct hwrm_queue_qportcfg_input req = {0}; 7306 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 7307 u8 i, j, *qptr; 7308 bool no_rdma; 7309 7310 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 7311 7312 mutex_lock(&bp->hwrm_cmd_lock); 7313 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7314 if (rc) 7315 goto qportcfg_exit; 7316 7317 if (!resp->max_configurable_queues) { 7318 rc = -EINVAL; 7319 goto qportcfg_exit; 7320 } 7321 bp->max_tc = resp->max_configurable_queues; 7322 bp->max_lltc = resp->max_configurable_lossless_queues; 7323 if (bp->max_tc > BNXT_MAX_QUEUE) 7324 bp->max_tc = BNXT_MAX_QUEUE; 7325 7326 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 7327 qptr = &resp->queue_id0; 7328 for (i = 0, j = 0; i < bp->max_tc; i++) { 7329 bp->q_info[j].queue_id = *qptr; 7330 bp->q_ids[i] = *qptr++; 7331 bp->q_info[j].queue_profile = *qptr++; 7332 bp->tc_to_qidx[j] = j; 7333 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 7334 (no_rdma && BNXT_PF(bp))) 7335 j++; 7336 } 7337 bp->max_q = bp->max_tc; 7338 bp->max_tc = max_t(u8, j, 1); 7339 7340 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 7341 bp->max_tc = 1; 7342 7343 if (bp->max_lltc > bp->max_tc) 7344 bp->max_lltc = bp->max_tc; 7345 7346 qportcfg_exit: 7347 mutex_unlock(&bp->hwrm_cmd_lock); 7348 return rc; 7349 } 7350 7351 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent) 7352 { 7353 struct hwrm_ver_get_input req = {0}; 7354 int rc; 7355 7356 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 7357 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 7358 req.hwrm_intf_min = HWRM_VERSION_MINOR; 7359 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 7360 7361 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT, 7362 silent); 7363 return rc; 7364 } 7365 7366 static int bnxt_hwrm_ver_get(struct bnxt *bp) 7367 { 7368 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 7369 u16 fw_maj, fw_min, fw_bld, fw_rsv; 7370 u32 dev_caps_cfg, hwrm_ver; 7371 int rc, len; 7372 7373 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 7374 mutex_lock(&bp->hwrm_cmd_lock); 7375 rc = __bnxt_hwrm_ver_get(bp, false); 7376 if (rc) 7377 goto hwrm_ver_get_exit; 7378 7379 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 7380 7381 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 7382 resp->hwrm_intf_min_8b << 8 | 7383 resp->hwrm_intf_upd_8b; 7384 if (resp->hwrm_intf_maj_8b < 1) { 7385 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 7386 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7387 resp->hwrm_intf_upd_8b); 7388 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 7389 } 7390 7391 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 7392 HWRM_VERSION_UPDATE; 7393 7394 if (bp->hwrm_spec_code > hwrm_ver) 7395 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7396 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 7397 HWRM_VERSION_UPDATE); 7398 else 7399 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 7400 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 7401 resp->hwrm_intf_upd_8b); 7402 7403 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 7404 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 7405 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 7406 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 7407 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 7408 len = FW_VER_STR_LEN; 7409 } else { 7410 fw_maj = resp->hwrm_fw_maj_8b; 7411 fw_min = resp->hwrm_fw_min_8b; 7412 fw_bld = resp->hwrm_fw_bld_8b; 7413 fw_rsv = resp->hwrm_fw_rsvd_8b; 7414 len = BC_HWRM_STR_LEN; 7415 } 7416 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 7417 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 7418 fw_rsv); 7419 7420 if (strlen(resp->active_pkg_name)) { 7421 int fw_ver_len = strlen(bp->fw_ver_str); 7422 7423 snprintf(bp->fw_ver_str + fw_ver_len, 7424 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 7425 resp->active_pkg_name); 7426 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 7427 } 7428 7429 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 7430 if (!bp->hwrm_cmd_timeout) 7431 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 7432 7433 if (resp->hwrm_intf_maj_8b >= 1) { 7434 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 7435 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 7436 } 7437 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 7438 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 7439 7440 bp->chip_num = le16_to_cpu(resp->chip_num); 7441 bp->chip_rev = resp->chip_rev; 7442 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 7443 !resp->chip_metal) 7444 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 7445 7446 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 7447 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 7448 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 7449 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 7450 7451 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 7452 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 7453 7454 if (dev_caps_cfg & 7455 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 7456 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 7457 7458 if (dev_caps_cfg & 7459 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 7460 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 7461 7462 if (dev_caps_cfg & 7463 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 7464 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 7465 7466 hwrm_ver_get_exit: 7467 mutex_unlock(&bp->hwrm_cmd_lock); 7468 return rc; 7469 } 7470 7471 int bnxt_hwrm_fw_set_time(struct bnxt *bp) 7472 { 7473 struct hwrm_fw_set_time_input req = {0}; 7474 struct tm tm; 7475 time64_t now = ktime_get_real_seconds(); 7476 7477 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 7478 bp->hwrm_spec_code < 0x10400) 7479 return -EOPNOTSUPP; 7480 7481 time64_to_tm(now, 0, &tm); 7482 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1); 7483 req.year = cpu_to_le16(1900 + tm.tm_year); 7484 req.month = 1 + tm.tm_mon; 7485 req.day = tm.tm_mday; 7486 req.hour = tm.tm_hour; 7487 req.minute = tm.tm_min; 7488 req.second = tm.tm_sec; 7489 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7490 } 7491 7492 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 7493 { 7494 struct bnxt_pf_info *pf = &bp->pf; 7495 struct hwrm_port_qstats_input req = {0}; 7496 7497 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 7498 return 0; 7499 7500 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 7501 req.port_id = cpu_to_le16(pf->port_id); 7502 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 7503 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 7504 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7505 } 7506 7507 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp) 7508 { 7509 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr; 7510 struct hwrm_queue_pri2cos_qcfg_input req2 = {0}; 7511 struct hwrm_port_qstats_ext_input req = {0}; 7512 struct bnxt_pf_info *pf = &bp->pf; 7513 u32 tx_stat_size; 7514 int rc; 7515 7516 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 7517 return 0; 7518 7519 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1); 7520 req.port_id = cpu_to_le16(pf->port_id); 7521 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 7522 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map); 7523 tx_stat_size = bp->hw_tx_port_stats_ext ? 7524 sizeof(*bp->hw_tx_port_stats_ext) : 0; 7525 req.tx_stat_size = cpu_to_le16(tx_stat_size); 7526 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map); 7527 mutex_lock(&bp->hwrm_cmd_lock); 7528 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7529 if (!rc) { 7530 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8; 7531 bp->fw_tx_stats_ext_size = tx_stat_size ? 7532 le16_to_cpu(resp->tx_stat_size) / 8 : 0; 7533 } else { 7534 bp->fw_rx_stats_ext_size = 0; 7535 bp->fw_tx_stats_ext_size = 0; 7536 } 7537 if (bp->fw_tx_stats_ext_size <= 7538 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 7539 mutex_unlock(&bp->hwrm_cmd_lock); 7540 bp->pri2cos_valid = 0; 7541 return rc; 7542 } 7543 7544 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1); 7545 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 7546 7547 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT); 7548 if (!rc) { 7549 struct hwrm_queue_pri2cos_qcfg_output *resp2; 7550 u8 *pri2cos; 7551 int i, j; 7552 7553 resp2 = bp->hwrm_cmd_resp_addr; 7554 pri2cos = &resp2->pri0_cos_queue_id; 7555 for (i = 0; i < 8; i++) { 7556 u8 queue_id = pri2cos[i]; 7557 u8 queue_idx; 7558 7559 /* Per port queue IDs start from 0, 10, 20, etc */ 7560 queue_idx = queue_id % 10; 7561 if (queue_idx > BNXT_MAX_QUEUE) { 7562 bp->pri2cos_valid = false; 7563 goto qstats_done; 7564 } 7565 for (j = 0; j < bp->max_q; j++) { 7566 if (bp->q_ids[j] == queue_id) 7567 bp->pri2cos_idx[i] = queue_idx; 7568 } 7569 } 7570 bp->pri2cos_valid = 1; 7571 } 7572 qstats_done: 7573 mutex_unlock(&bp->hwrm_cmd_lock); 7574 return rc; 7575 } 7576 7577 static int bnxt_hwrm_pcie_qstats(struct bnxt *bp) 7578 { 7579 struct hwrm_pcie_qstats_input req = {0}; 7580 7581 if (!(bp->flags & BNXT_FLAG_PCIE_STATS)) 7582 return 0; 7583 7584 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1); 7585 req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats)); 7586 req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map); 7587 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7588 } 7589 7590 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 7591 { 7592 if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID) 7593 bnxt_hwrm_tunnel_dst_port_free( 7594 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 7595 if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID) 7596 bnxt_hwrm_tunnel_dst_port_free( 7597 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 7598 } 7599 7600 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 7601 { 7602 int rc, i; 7603 u32 tpa_flags = 0; 7604 7605 if (set_tpa) 7606 tpa_flags = bp->flags & BNXT_FLAG_TPA; 7607 else if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 7608 return 0; 7609 for (i = 0; i < bp->nr_vnics; i++) { 7610 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 7611 if (rc) { 7612 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 7613 i, rc); 7614 return rc; 7615 } 7616 } 7617 return 0; 7618 } 7619 7620 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 7621 { 7622 int i; 7623 7624 for (i = 0; i < bp->nr_vnics; i++) 7625 bnxt_hwrm_vnic_set_rss(bp, i, false); 7626 } 7627 7628 static void bnxt_clear_vnic(struct bnxt *bp) 7629 { 7630 if (!bp->vnic_info) 7631 return; 7632 7633 bnxt_hwrm_clear_vnic_filter(bp); 7634 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) { 7635 /* clear all RSS setting before free vnic ctx */ 7636 bnxt_hwrm_clear_vnic_rss(bp); 7637 bnxt_hwrm_vnic_ctx_free(bp); 7638 } 7639 /* before free the vnic, undo the vnic tpa settings */ 7640 if (bp->flags & BNXT_FLAG_TPA) 7641 bnxt_set_tpa(bp, false); 7642 bnxt_hwrm_vnic_free(bp); 7643 if (bp->flags & BNXT_FLAG_CHIP_P5) 7644 bnxt_hwrm_vnic_ctx_free(bp); 7645 } 7646 7647 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 7648 bool irq_re_init) 7649 { 7650 bnxt_clear_vnic(bp); 7651 bnxt_hwrm_ring_free(bp, close_path); 7652 bnxt_hwrm_ring_grp_free(bp); 7653 if (irq_re_init) { 7654 bnxt_hwrm_stat_ctx_free(bp); 7655 bnxt_hwrm_free_tunnel_ports(bp); 7656 } 7657 } 7658 7659 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 7660 { 7661 struct hwrm_func_cfg_input req = {0}; 7662 7663 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 7664 req.fid = cpu_to_le16(0xffff); 7665 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 7666 if (br_mode == BRIDGE_MODE_VEB) 7667 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 7668 else if (br_mode == BRIDGE_MODE_VEPA) 7669 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 7670 else 7671 return -EINVAL; 7672 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7673 } 7674 7675 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 7676 { 7677 struct hwrm_func_cfg_input req = {0}; 7678 7679 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 7680 return 0; 7681 7682 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1); 7683 req.fid = cpu_to_le16(0xffff); 7684 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 7685 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 7686 if (size == 128) 7687 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 7688 7689 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 7690 } 7691 7692 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 7693 { 7694 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 7695 int rc; 7696 7697 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 7698 goto skip_rss_ctx; 7699 7700 /* allocate context for vnic */ 7701 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 7702 if (rc) { 7703 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 7704 vnic_id, rc); 7705 goto vnic_setup_err; 7706 } 7707 bp->rsscos_nr_ctxs++; 7708 7709 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 7710 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 7711 if (rc) { 7712 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 7713 vnic_id, rc); 7714 goto vnic_setup_err; 7715 } 7716 bp->rsscos_nr_ctxs++; 7717 } 7718 7719 skip_rss_ctx: 7720 /* configure default vnic, ring grp */ 7721 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 7722 if (rc) { 7723 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 7724 vnic_id, rc); 7725 goto vnic_setup_err; 7726 } 7727 7728 /* Enable RSS hashing on vnic */ 7729 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 7730 if (rc) { 7731 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 7732 vnic_id, rc); 7733 goto vnic_setup_err; 7734 } 7735 7736 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 7737 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 7738 if (rc) { 7739 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 7740 vnic_id, rc); 7741 } 7742 } 7743 7744 vnic_setup_err: 7745 return rc; 7746 } 7747 7748 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id) 7749 { 7750 int rc, i, nr_ctxs; 7751 7752 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 7753 for (i = 0; i < nr_ctxs; i++) { 7754 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i); 7755 if (rc) { 7756 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 7757 vnic_id, i, rc); 7758 break; 7759 } 7760 bp->rsscos_nr_ctxs++; 7761 } 7762 if (i < nr_ctxs) 7763 return -ENOMEM; 7764 7765 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true); 7766 if (rc) { 7767 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 7768 vnic_id, rc); 7769 return rc; 7770 } 7771 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 7772 if (rc) { 7773 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 7774 vnic_id, rc); 7775 return rc; 7776 } 7777 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 7778 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 7779 if (rc) { 7780 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 7781 vnic_id, rc); 7782 } 7783 } 7784 return rc; 7785 } 7786 7787 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 7788 { 7789 if (bp->flags & BNXT_FLAG_CHIP_P5) 7790 return __bnxt_setup_vnic_p5(bp, vnic_id); 7791 else 7792 return __bnxt_setup_vnic(bp, vnic_id); 7793 } 7794 7795 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 7796 { 7797 #ifdef CONFIG_RFS_ACCEL 7798 int i, rc = 0; 7799 7800 if (bp->flags & BNXT_FLAG_CHIP_P5) 7801 return 0; 7802 7803 for (i = 0; i < bp->rx_nr_rings; i++) { 7804 struct bnxt_vnic_info *vnic; 7805 u16 vnic_id = i + 1; 7806 u16 ring_id = i; 7807 7808 if (vnic_id >= bp->nr_vnics) 7809 break; 7810 7811 vnic = &bp->vnic_info[vnic_id]; 7812 vnic->flags |= BNXT_VNIC_RFS_FLAG; 7813 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 7814 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 7815 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 7816 if (rc) { 7817 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 7818 vnic_id, rc); 7819 break; 7820 } 7821 rc = bnxt_setup_vnic(bp, vnic_id); 7822 if (rc) 7823 break; 7824 } 7825 return rc; 7826 #else 7827 return 0; 7828 #endif 7829 } 7830 7831 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 7832 static bool bnxt_promisc_ok(struct bnxt *bp) 7833 { 7834 #ifdef CONFIG_BNXT_SRIOV 7835 if (BNXT_VF(bp) && !bp->vf.vlan) 7836 return false; 7837 #endif 7838 return true; 7839 } 7840 7841 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 7842 { 7843 unsigned int rc = 0; 7844 7845 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 7846 if (rc) { 7847 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 7848 rc); 7849 return rc; 7850 } 7851 7852 rc = bnxt_hwrm_vnic_cfg(bp, 1); 7853 if (rc) { 7854 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 7855 rc); 7856 return rc; 7857 } 7858 return rc; 7859 } 7860 7861 static int bnxt_cfg_rx_mode(struct bnxt *); 7862 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 7863 7864 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 7865 { 7866 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 7867 int rc = 0; 7868 unsigned int rx_nr_rings = bp->rx_nr_rings; 7869 7870 if (irq_re_init) { 7871 rc = bnxt_hwrm_stat_ctx_alloc(bp); 7872 if (rc) { 7873 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 7874 rc); 7875 goto err_out; 7876 } 7877 } 7878 7879 rc = bnxt_hwrm_ring_alloc(bp); 7880 if (rc) { 7881 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 7882 goto err_out; 7883 } 7884 7885 rc = bnxt_hwrm_ring_grp_alloc(bp); 7886 if (rc) { 7887 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 7888 goto err_out; 7889 } 7890 7891 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 7892 rx_nr_rings--; 7893 7894 /* default vnic 0 */ 7895 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 7896 if (rc) { 7897 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 7898 goto err_out; 7899 } 7900 7901 rc = bnxt_setup_vnic(bp, 0); 7902 if (rc) 7903 goto err_out; 7904 7905 if (bp->flags & BNXT_FLAG_RFS) { 7906 rc = bnxt_alloc_rfs_vnics(bp); 7907 if (rc) 7908 goto err_out; 7909 } 7910 7911 if (bp->flags & BNXT_FLAG_TPA) { 7912 rc = bnxt_set_tpa(bp, true); 7913 if (rc) 7914 goto err_out; 7915 } 7916 7917 if (BNXT_VF(bp)) 7918 bnxt_update_vf_mac(bp); 7919 7920 /* Filter for default vnic 0 */ 7921 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 7922 if (rc) { 7923 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 7924 goto err_out; 7925 } 7926 vnic->uc_filter_count = 1; 7927 7928 vnic->rx_mask = 0; 7929 if (bp->dev->flags & IFF_BROADCAST) 7930 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 7931 7932 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 7933 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 7934 7935 if (bp->dev->flags & IFF_ALLMULTI) { 7936 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 7937 vnic->mc_list_count = 0; 7938 } else { 7939 u32 mask = 0; 7940 7941 bnxt_mc_list_updated(bp, &mask); 7942 vnic->rx_mask |= mask; 7943 } 7944 7945 rc = bnxt_cfg_rx_mode(bp); 7946 if (rc) 7947 goto err_out; 7948 7949 rc = bnxt_hwrm_set_coal(bp); 7950 if (rc) 7951 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 7952 rc); 7953 7954 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 7955 rc = bnxt_setup_nitroa0_vnic(bp); 7956 if (rc) 7957 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 7958 rc); 7959 } 7960 7961 if (BNXT_VF(bp)) { 7962 bnxt_hwrm_func_qcfg(bp); 7963 netdev_update_features(bp->dev); 7964 } 7965 7966 return 0; 7967 7968 err_out: 7969 bnxt_hwrm_resource_free(bp, 0, true); 7970 7971 return rc; 7972 } 7973 7974 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 7975 { 7976 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 7977 return 0; 7978 } 7979 7980 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 7981 { 7982 bnxt_init_cp_rings(bp); 7983 bnxt_init_rx_rings(bp); 7984 bnxt_init_tx_rings(bp); 7985 bnxt_init_ring_grps(bp, irq_re_init); 7986 bnxt_init_vnics(bp); 7987 7988 return bnxt_init_chip(bp, irq_re_init); 7989 } 7990 7991 static int bnxt_set_real_num_queues(struct bnxt *bp) 7992 { 7993 int rc; 7994 struct net_device *dev = bp->dev; 7995 7996 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 7997 bp->tx_nr_rings_xdp); 7998 if (rc) 7999 return rc; 8000 8001 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 8002 if (rc) 8003 return rc; 8004 8005 #ifdef CONFIG_RFS_ACCEL 8006 if (bp->flags & BNXT_FLAG_RFS) 8007 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 8008 #endif 8009 8010 return rc; 8011 } 8012 8013 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 8014 bool shared) 8015 { 8016 int _rx = *rx, _tx = *tx; 8017 8018 if (shared) { 8019 *rx = min_t(int, _rx, max); 8020 *tx = min_t(int, _tx, max); 8021 } else { 8022 if (max < 2) 8023 return -ENOMEM; 8024 8025 while (_rx + _tx > max) { 8026 if (_rx > _tx && _rx > 1) 8027 _rx--; 8028 else if (_tx > 1) 8029 _tx--; 8030 } 8031 *rx = _rx; 8032 *tx = _tx; 8033 } 8034 return 0; 8035 } 8036 8037 static void bnxt_setup_msix(struct bnxt *bp) 8038 { 8039 const int len = sizeof(bp->irq_tbl[0].name); 8040 struct net_device *dev = bp->dev; 8041 int tcs, i; 8042 8043 tcs = netdev_get_num_tc(dev); 8044 if (tcs) { 8045 int i, off, count; 8046 8047 for (i = 0; i < tcs; i++) { 8048 count = bp->tx_nr_rings_per_tc; 8049 off = i * count; 8050 netdev_set_tc_queue(dev, i, count, off); 8051 } 8052 } 8053 8054 for (i = 0; i < bp->cp_nr_rings; i++) { 8055 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8056 char *attr; 8057 8058 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8059 attr = "TxRx"; 8060 else if (i < bp->rx_nr_rings) 8061 attr = "rx"; 8062 else 8063 attr = "tx"; 8064 8065 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 8066 attr, i); 8067 bp->irq_tbl[map_idx].handler = bnxt_msix; 8068 } 8069 } 8070 8071 static void bnxt_setup_inta(struct bnxt *bp) 8072 { 8073 const int len = sizeof(bp->irq_tbl[0].name); 8074 8075 if (netdev_get_num_tc(bp->dev)) 8076 netdev_reset_tc(bp->dev); 8077 8078 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx", 8079 0); 8080 bp->irq_tbl[0].handler = bnxt_inta; 8081 } 8082 8083 static int bnxt_setup_int_mode(struct bnxt *bp) 8084 { 8085 int rc; 8086 8087 if (bp->flags & BNXT_FLAG_USING_MSIX) 8088 bnxt_setup_msix(bp); 8089 else 8090 bnxt_setup_inta(bp); 8091 8092 rc = bnxt_set_real_num_queues(bp); 8093 return rc; 8094 } 8095 8096 #ifdef CONFIG_RFS_ACCEL 8097 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 8098 { 8099 return bp->hw_resc.max_rsscos_ctxs; 8100 } 8101 8102 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 8103 { 8104 return bp->hw_resc.max_vnics; 8105 } 8106 #endif 8107 8108 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 8109 { 8110 return bp->hw_resc.max_stat_ctxs; 8111 } 8112 8113 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 8114 { 8115 return bp->hw_resc.max_cp_rings; 8116 } 8117 8118 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 8119 { 8120 unsigned int cp = bp->hw_resc.max_cp_rings; 8121 8122 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8123 cp -= bnxt_get_ulp_msix_num(bp); 8124 8125 return cp; 8126 } 8127 8128 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 8129 { 8130 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8131 8132 if (bp->flags & BNXT_FLAG_CHIP_P5) 8133 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 8134 8135 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 8136 } 8137 8138 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 8139 { 8140 bp->hw_resc.max_irqs = max_irqs; 8141 } 8142 8143 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 8144 { 8145 unsigned int cp; 8146 8147 cp = bnxt_get_max_func_cp_rings_for_en(bp); 8148 if (bp->flags & BNXT_FLAG_CHIP_P5) 8149 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 8150 else 8151 return cp - bp->cp_nr_rings; 8152 } 8153 8154 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 8155 { 8156 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 8157 } 8158 8159 int bnxt_get_avail_msix(struct bnxt *bp, int num) 8160 { 8161 int max_cp = bnxt_get_max_func_cp_rings(bp); 8162 int max_irq = bnxt_get_max_func_irqs(bp); 8163 int total_req = bp->cp_nr_rings + num; 8164 int max_idx, avail_msix; 8165 8166 max_idx = bp->total_irqs; 8167 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 8168 max_idx = min_t(int, bp->total_irqs, max_cp); 8169 avail_msix = max_idx - bp->cp_nr_rings; 8170 if (!BNXT_NEW_RM(bp) || avail_msix >= num) 8171 return avail_msix; 8172 8173 if (max_irq < total_req) { 8174 num = max_irq - bp->cp_nr_rings; 8175 if (num <= 0) 8176 return 0; 8177 } 8178 return num; 8179 } 8180 8181 static int bnxt_get_num_msix(struct bnxt *bp) 8182 { 8183 if (!BNXT_NEW_RM(bp)) 8184 return bnxt_get_max_func_irqs(bp); 8185 8186 return bnxt_nq_rings_in_use(bp); 8187 } 8188 8189 static int bnxt_init_msix(struct bnxt *bp) 8190 { 8191 int i, total_vecs, max, rc = 0, min = 1, ulp_msix; 8192 struct msix_entry *msix_ent; 8193 8194 total_vecs = bnxt_get_num_msix(bp); 8195 max = bnxt_get_max_func_irqs(bp); 8196 if (total_vecs > max) 8197 total_vecs = max; 8198 8199 if (!total_vecs) 8200 return 0; 8201 8202 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 8203 if (!msix_ent) 8204 return -ENOMEM; 8205 8206 for (i = 0; i < total_vecs; i++) { 8207 msix_ent[i].entry = i; 8208 msix_ent[i].vector = 0; 8209 } 8210 8211 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 8212 min = 2; 8213 8214 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 8215 ulp_msix = bnxt_get_ulp_msix_num(bp); 8216 if (total_vecs < 0 || total_vecs < ulp_msix) { 8217 rc = -ENODEV; 8218 goto msix_setup_exit; 8219 } 8220 8221 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 8222 if (bp->irq_tbl) { 8223 for (i = 0; i < total_vecs; i++) 8224 bp->irq_tbl[i].vector = msix_ent[i].vector; 8225 8226 bp->total_irqs = total_vecs; 8227 /* Trim rings based upon num of vectors allocated */ 8228 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 8229 total_vecs - ulp_msix, min == 1); 8230 if (rc) 8231 goto msix_setup_exit; 8232 8233 bp->cp_nr_rings = (min == 1) ? 8234 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 8235 bp->tx_nr_rings + bp->rx_nr_rings; 8236 8237 } else { 8238 rc = -ENOMEM; 8239 goto msix_setup_exit; 8240 } 8241 bp->flags |= BNXT_FLAG_USING_MSIX; 8242 kfree(msix_ent); 8243 return 0; 8244 8245 msix_setup_exit: 8246 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc); 8247 kfree(bp->irq_tbl); 8248 bp->irq_tbl = NULL; 8249 pci_disable_msix(bp->pdev); 8250 kfree(msix_ent); 8251 return rc; 8252 } 8253 8254 static int bnxt_init_inta(struct bnxt *bp) 8255 { 8256 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 8257 if (!bp->irq_tbl) 8258 return -ENOMEM; 8259 8260 bp->total_irqs = 1; 8261 bp->rx_nr_rings = 1; 8262 bp->tx_nr_rings = 1; 8263 bp->cp_nr_rings = 1; 8264 bp->flags |= BNXT_FLAG_SHARED_RINGS; 8265 bp->irq_tbl[0].vector = bp->pdev->irq; 8266 return 0; 8267 } 8268 8269 static int bnxt_init_int_mode(struct bnxt *bp) 8270 { 8271 int rc = 0; 8272 8273 if (bp->flags & BNXT_FLAG_MSIX_CAP) 8274 rc = bnxt_init_msix(bp); 8275 8276 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 8277 /* fallback to INTA */ 8278 rc = bnxt_init_inta(bp); 8279 } 8280 return rc; 8281 } 8282 8283 static void bnxt_clear_int_mode(struct bnxt *bp) 8284 { 8285 if (bp->flags & BNXT_FLAG_USING_MSIX) 8286 pci_disable_msix(bp->pdev); 8287 8288 kfree(bp->irq_tbl); 8289 bp->irq_tbl = NULL; 8290 bp->flags &= ~BNXT_FLAG_USING_MSIX; 8291 } 8292 8293 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 8294 { 8295 int tcs = netdev_get_num_tc(bp->dev); 8296 bool irq_cleared = false; 8297 int rc; 8298 8299 if (!bnxt_need_reserve_rings(bp)) 8300 return 0; 8301 8302 if (irq_re_init && BNXT_NEW_RM(bp) && 8303 bnxt_get_num_msix(bp) != bp->total_irqs) { 8304 bnxt_ulp_irq_stop(bp); 8305 bnxt_clear_int_mode(bp); 8306 irq_cleared = true; 8307 } 8308 rc = __bnxt_reserve_rings(bp); 8309 if (irq_cleared) { 8310 if (!rc) 8311 rc = bnxt_init_int_mode(bp); 8312 bnxt_ulp_irq_restart(bp, rc); 8313 } 8314 if (!netif_is_rxfh_configured(bp->dev)) 8315 bnxt_set_dflt_rss_indir_tbl(bp); 8316 8317 if (rc) { 8318 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 8319 return rc; 8320 } 8321 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) { 8322 netdev_err(bp->dev, "tx ring reservation failure\n"); 8323 netdev_reset_tc(bp->dev); 8324 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 8325 return -ENOMEM; 8326 } 8327 return 0; 8328 } 8329 8330 static void bnxt_free_irq(struct bnxt *bp) 8331 { 8332 struct bnxt_irq *irq; 8333 int i; 8334 8335 #ifdef CONFIG_RFS_ACCEL 8336 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 8337 bp->dev->rx_cpu_rmap = NULL; 8338 #endif 8339 if (!bp->irq_tbl || !bp->bnapi) 8340 return; 8341 8342 for (i = 0; i < bp->cp_nr_rings; i++) { 8343 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8344 8345 irq = &bp->irq_tbl[map_idx]; 8346 if (irq->requested) { 8347 if (irq->have_cpumask) { 8348 irq_set_affinity_hint(irq->vector, NULL); 8349 free_cpumask_var(irq->cpu_mask); 8350 irq->have_cpumask = 0; 8351 } 8352 free_irq(irq->vector, bp->bnapi[i]); 8353 } 8354 8355 irq->requested = 0; 8356 } 8357 } 8358 8359 static int bnxt_request_irq(struct bnxt *bp) 8360 { 8361 int i, j, rc = 0; 8362 unsigned long flags = 0; 8363 #ifdef CONFIG_RFS_ACCEL 8364 struct cpu_rmap *rmap; 8365 #endif 8366 8367 rc = bnxt_setup_int_mode(bp); 8368 if (rc) { 8369 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 8370 rc); 8371 return rc; 8372 } 8373 #ifdef CONFIG_RFS_ACCEL 8374 rmap = bp->dev->rx_cpu_rmap; 8375 #endif 8376 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 8377 flags = IRQF_SHARED; 8378 8379 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 8380 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 8381 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 8382 8383 #ifdef CONFIG_RFS_ACCEL 8384 if (rmap && bp->bnapi[i]->rx_ring) { 8385 rc = irq_cpu_rmap_add(rmap, irq->vector); 8386 if (rc) 8387 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 8388 j); 8389 j++; 8390 } 8391 #endif 8392 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 8393 bp->bnapi[i]); 8394 if (rc) 8395 break; 8396 8397 irq->requested = 1; 8398 8399 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 8400 int numa_node = dev_to_node(&bp->pdev->dev); 8401 8402 irq->have_cpumask = 1; 8403 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 8404 irq->cpu_mask); 8405 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 8406 if (rc) { 8407 netdev_warn(bp->dev, 8408 "Set affinity failed, IRQ = %d\n", 8409 irq->vector); 8410 break; 8411 } 8412 } 8413 } 8414 return rc; 8415 } 8416 8417 static void bnxt_del_napi(struct bnxt *bp) 8418 { 8419 int i; 8420 8421 if (!bp->bnapi) 8422 return; 8423 8424 for (i = 0; i < bp->cp_nr_rings; i++) { 8425 struct bnxt_napi *bnapi = bp->bnapi[i]; 8426 8427 napi_hash_del(&bnapi->napi); 8428 netif_napi_del(&bnapi->napi); 8429 } 8430 /* We called napi_hash_del() before netif_napi_del(), we need 8431 * to respect an RCU grace period before freeing napi structures. 8432 */ 8433 synchronize_net(); 8434 } 8435 8436 static void bnxt_init_napi(struct bnxt *bp) 8437 { 8438 int i; 8439 unsigned int cp_nr_rings = bp->cp_nr_rings; 8440 struct bnxt_napi *bnapi; 8441 8442 if (bp->flags & BNXT_FLAG_USING_MSIX) { 8443 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 8444 8445 if (bp->flags & BNXT_FLAG_CHIP_P5) 8446 poll_fn = bnxt_poll_p5; 8447 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8448 cp_nr_rings--; 8449 for (i = 0; i < cp_nr_rings; i++) { 8450 bnapi = bp->bnapi[i]; 8451 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); 8452 } 8453 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 8454 bnapi = bp->bnapi[cp_nr_rings]; 8455 netif_napi_add(bp->dev, &bnapi->napi, 8456 bnxt_poll_nitroa0, 64); 8457 } 8458 } else { 8459 bnapi = bp->bnapi[0]; 8460 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 8461 } 8462 } 8463 8464 static void bnxt_disable_napi(struct bnxt *bp) 8465 { 8466 int i; 8467 8468 if (!bp->bnapi) 8469 return; 8470 8471 for (i = 0; i < bp->cp_nr_rings; i++) { 8472 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 8473 8474 if (bp->bnapi[i]->rx_ring) 8475 cancel_work_sync(&cpr->dim.work); 8476 8477 napi_disable(&bp->bnapi[i]->napi); 8478 } 8479 } 8480 8481 static void bnxt_enable_napi(struct bnxt *bp) 8482 { 8483 int i; 8484 8485 for (i = 0; i < bp->cp_nr_rings; i++) { 8486 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 8487 bp->bnapi[i]->in_reset = false; 8488 8489 if (bp->bnapi[i]->rx_ring) { 8490 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 8491 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 8492 } 8493 napi_enable(&bp->bnapi[i]->napi); 8494 } 8495 } 8496 8497 void bnxt_tx_disable(struct bnxt *bp) 8498 { 8499 int i; 8500 struct bnxt_tx_ring_info *txr; 8501 8502 if (bp->tx_ring) { 8503 for (i = 0; i < bp->tx_nr_rings; i++) { 8504 txr = &bp->tx_ring[i]; 8505 txr->dev_state = BNXT_DEV_STATE_CLOSING; 8506 } 8507 } 8508 /* Stop all TX queues */ 8509 netif_tx_disable(bp->dev); 8510 netif_carrier_off(bp->dev); 8511 } 8512 8513 void bnxt_tx_enable(struct bnxt *bp) 8514 { 8515 int i; 8516 struct bnxt_tx_ring_info *txr; 8517 8518 for (i = 0; i < bp->tx_nr_rings; i++) { 8519 txr = &bp->tx_ring[i]; 8520 txr->dev_state = 0; 8521 } 8522 netif_tx_wake_all_queues(bp->dev); 8523 if (bp->link_info.link_up) 8524 netif_carrier_on(bp->dev); 8525 } 8526 8527 static void bnxt_report_link(struct bnxt *bp) 8528 { 8529 if (bp->link_info.link_up) { 8530 const char *duplex; 8531 const char *flow_ctrl; 8532 u32 speed; 8533 u16 fec; 8534 8535 netif_carrier_on(bp->dev); 8536 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 8537 duplex = "full"; 8538 else 8539 duplex = "half"; 8540 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 8541 flow_ctrl = "ON - receive & transmit"; 8542 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 8543 flow_ctrl = "ON - transmit"; 8544 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 8545 flow_ctrl = "ON - receive"; 8546 else 8547 flow_ctrl = "none"; 8548 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 8549 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n", 8550 speed, duplex, flow_ctrl); 8551 if (bp->flags & BNXT_FLAG_EEE_CAP) 8552 netdev_info(bp->dev, "EEE is %s\n", 8553 bp->eee.eee_active ? "active" : 8554 "not active"); 8555 fec = bp->link_info.fec_cfg; 8556 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 8557 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n", 8558 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 8559 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" : 8560 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None"); 8561 } else { 8562 netif_carrier_off(bp->dev); 8563 netdev_err(bp->dev, "NIC Link is Down\n"); 8564 } 8565 } 8566 8567 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 8568 { 8569 int rc = 0; 8570 struct hwrm_port_phy_qcaps_input req = {0}; 8571 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 8572 struct bnxt_link_info *link_info = &bp->link_info; 8573 8574 bp->flags &= ~BNXT_FLAG_EEE_CAP; 8575 if (bp->test_info) 8576 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK | 8577 BNXT_TEST_FL_AN_PHY_LPBK); 8578 if (bp->hwrm_spec_code < 0x10201) 8579 return 0; 8580 8581 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 8582 8583 mutex_lock(&bp->hwrm_cmd_lock); 8584 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8585 if (rc) 8586 goto hwrm_phy_qcaps_exit; 8587 8588 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 8589 struct ethtool_eee *eee = &bp->eee; 8590 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 8591 8592 bp->flags |= BNXT_FLAG_EEE_CAP; 8593 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 8594 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 8595 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 8596 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 8597 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 8598 } 8599 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) { 8600 if (bp->test_info) 8601 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK; 8602 } 8603 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) { 8604 if (bp->test_info) 8605 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK; 8606 } 8607 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) { 8608 if (BNXT_PF(bp)) 8609 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG; 8610 } 8611 if (resp->supported_speeds_auto_mode) 8612 link_info->support_auto_speeds = 8613 le16_to_cpu(resp->supported_speeds_auto_mode); 8614 8615 bp->port_count = resp->port_cnt; 8616 8617 hwrm_phy_qcaps_exit: 8618 mutex_unlock(&bp->hwrm_cmd_lock); 8619 return rc; 8620 } 8621 8622 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 8623 { 8624 int rc = 0; 8625 struct bnxt_link_info *link_info = &bp->link_info; 8626 struct hwrm_port_phy_qcfg_input req = {0}; 8627 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 8628 u8 link_up = link_info->link_up; 8629 u16 diff; 8630 8631 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 8632 8633 mutex_lock(&bp->hwrm_cmd_lock); 8634 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8635 if (rc) { 8636 mutex_unlock(&bp->hwrm_cmd_lock); 8637 return rc; 8638 } 8639 8640 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 8641 link_info->phy_link_status = resp->link; 8642 link_info->duplex = resp->duplex_cfg; 8643 if (bp->hwrm_spec_code >= 0x10800) 8644 link_info->duplex = resp->duplex_state; 8645 link_info->pause = resp->pause; 8646 link_info->auto_mode = resp->auto_mode; 8647 link_info->auto_pause_setting = resp->auto_pause; 8648 link_info->lp_pause = resp->link_partner_adv_pause; 8649 link_info->force_pause_setting = resp->force_pause; 8650 link_info->duplex_setting = resp->duplex_cfg; 8651 if (link_info->phy_link_status == BNXT_LINK_LINK) 8652 link_info->link_speed = le16_to_cpu(resp->link_speed); 8653 else 8654 link_info->link_speed = 0; 8655 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 8656 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 8657 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 8658 link_info->lp_auto_link_speeds = 8659 le16_to_cpu(resp->link_partner_adv_speeds); 8660 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 8661 link_info->phy_ver[0] = resp->phy_maj; 8662 link_info->phy_ver[1] = resp->phy_min; 8663 link_info->phy_ver[2] = resp->phy_bld; 8664 link_info->media_type = resp->media_type; 8665 link_info->phy_type = resp->phy_type; 8666 link_info->transceiver = resp->xcvr_pkg_type; 8667 link_info->phy_addr = resp->eee_config_phy_addr & 8668 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 8669 link_info->module_status = resp->module_status; 8670 8671 if (bp->flags & BNXT_FLAG_EEE_CAP) { 8672 struct ethtool_eee *eee = &bp->eee; 8673 u16 fw_speeds; 8674 8675 eee->eee_active = 0; 8676 if (resp->eee_config_phy_addr & 8677 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 8678 eee->eee_active = 1; 8679 fw_speeds = le16_to_cpu( 8680 resp->link_partner_adv_eee_link_speed_mask); 8681 eee->lp_advertised = 8682 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 8683 } 8684 8685 /* Pull initial EEE config */ 8686 if (!chng_link_state) { 8687 if (resp->eee_config_phy_addr & 8688 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 8689 eee->eee_enabled = 1; 8690 8691 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 8692 eee->advertised = 8693 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 8694 8695 if (resp->eee_config_phy_addr & 8696 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 8697 __le32 tmr; 8698 8699 eee->tx_lpi_enabled = 1; 8700 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 8701 eee->tx_lpi_timer = le32_to_cpu(tmr) & 8702 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 8703 } 8704 } 8705 } 8706 8707 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 8708 if (bp->hwrm_spec_code >= 0x10504) 8709 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 8710 8711 /* TODO: need to add more logic to report VF link */ 8712 if (chng_link_state) { 8713 if (link_info->phy_link_status == BNXT_LINK_LINK) 8714 link_info->link_up = 1; 8715 else 8716 link_info->link_up = 0; 8717 if (link_up != link_info->link_up) 8718 bnxt_report_link(bp); 8719 } else { 8720 /* alwasy link down if not require to update link state */ 8721 link_info->link_up = 0; 8722 } 8723 mutex_unlock(&bp->hwrm_cmd_lock); 8724 8725 if (!BNXT_PHY_CFG_ABLE(bp)) 8726 return 0; 8727 8728 diff = link_info->support_auto_speeds ^ link_info->advertising; 8729 if ((link_info->support_auto_speeds | diff) != 8730 link_info->support_auto_speeds) { 8731 /* An advertised speed is no longer supported, so we need to 8732 * update the advertisement settings. Caller holds RTNL 8733 * so we can modify link settings. 8734 */ 8735 link_info->advertising = link_info->support_auto_speeds; 8736 if (link_info->autoneg & BNXT_AUTONEG_SPEED) 8737 bnxt_hwrm_set_link_setting(bp, true, false); 8738 } 8739 return 0; 8740 } 8741 8742 static void bnxt_get_port_module_status(struct bnxt *bp) 8743 { 8744 struct bnxt_link_info *link_info = &bp->link_info; 8745 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 8746 u8 module_status; 8747 8748 if (bnxt_update_link(bp, true)) 8749 return; 8750 8751 module_status = link_info->module_status; 8752 switch (module_status) { 8753 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 8754 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 8755 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 8756 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 8757 bp->pf.port_id); 8758 if (bp->hwrm_spec_code >= 0x10201) { 8759 netdev_warn(bp->dev, "Module part number %s\n", 8760 resp->phy_vendor_partnumber); 8761 } 8762 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 8763 netdev_warn(bp->dev, "TX is disabled\n"); 8764 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 8765 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 8766 } 8767 } 8768 8769 static void 8770 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 8771 { 8772 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 8773 if (bp->hwrm_spec_code >= 0x10201) 8774 req->auto_pause = 8775 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 8776 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 8777 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 8778 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 8779 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 8780 req->enables |= 8781 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 8782 } else { 8783 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 8784 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 8785 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 8786 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 8787 req->enables |= 8788 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 8789 if (bp->hwrm_spec_code >= 0x10201) { 8790 req->auto_pause = req->force_pause; 8791 req->enables |= cpu_to_le32( 8792 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 8793 } 8794 } 8795 } 8796 8797 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 8798 struct hwrm_port_phy_cfg_input *req) 8799 { 8800 u8 autoneg = bp->link_info.autoneg; 8801 u16 fw_link_speed = bp->link_info.req_link_speed; 8802 u16 advertising = bp->link_info.advertising; 8803 8804 if (autoneg & BNXT_AUTONEG_SPEED) { 8805 req->auto_mode |= 8806 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 8807 8808 req->enables |= cpu_to_le32( 8809 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 8810 req->auto_link_speed_mask = cpu_to_le16(advertising); 8811 8812 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 8813 req->flags |= 8814 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 8815 } else { 8816 req->force_link_speed = cpu_to_le16(fw_link_speed); 8817 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 8818 } 8819 8820 /* tell chimp that the setting takes effect immediately */ 8821 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 8822 } 8823 8824 int bnxt_hwrm_set_pause(struct bnxt *bp) 8825 { 8826 struct hwrm_port_phy_cfg_input req = {0}; 8827 int rc; 8828 8829 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 8830 bnxt_hwrm_set_pause_common(bp, &req); 8831 8832 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 8833 bp->link_info.force_link_chng) 8834 bnxt_hwrm_set_link_common(bp, &req); 8835 8836 mutex_lock(&bp->hwrm_cmd_lock); 8837 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8838 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 8839 /* since changing of pause setting doesn't trigger any link 8840 * change event, the driver needs to update the current pause 8841 * result upon successfully return of the phy_cfg command 8842 */ 8843 bp->link_info.pause = 8844 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 8845 bp->link_info.auto_pause_setting = 0; 8846 if (!bp->link_info.force_link_chng) 8847 bnxt_report_link(bp); 8848 } 8849 bp->link_info.force_link_chng = false; 8850 mutex_unlock(&bp->hwrm_cmd_lock); 8851 return rc; 8852 } 8853 8854 static void bnxt_hwrm_set_eee(struct bnxt *bp, 8855 struct hwrm_port_phy_cfg_input *req) 8856 { 8857 struct ethtool_eee *eee = &bp->eee; 8858 8859 if (eee->eee_enabled) { 8860 u16 eee_speeds; 8861 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 8862 8863 if (eee->tx_lpi_enabled) 8864 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 8865 else 8866 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 8867 8868 req->flags |= cpu_to_le32(flags); 8869 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 8870 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 8871 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 8872 } else { 8873 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 8874 } 8875 } 8876 8877 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 8878 { 8879 struct hwrm_port_phy_cfg_input req = {0}; 8880 8881 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 8882 if (set_pause) 8883 bnxt_hwrm_set_pause_common(bp, &req); 8884 8885 bnxt_hwrm_set_link_common(bp, &req); 8886 8887 if (set_eee) 8888 bnxt_hwrm_set_eee(bp, &req); 8889 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8890 } 8891 8892 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 8893 { 8894 struct hwrm_port_phy_cfg_input req = {0}; 8895 8896 if (!BNXT_SINGLE_PF(bp)) 8897 return 0; 8898 8899 if (pci_num_vf(bp->pdev)) 8900 return 0; 8901 8902 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 8903 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 8904 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8905 } 8906 8907 static int bnxt_fw_init_one(struct bnxt *bp); 8908 8909 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 8910 { 8911 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr; 8912 struct hwrm_func_drv_if_change_input req = {0}; 8913 bool resc_reinit = false, fw_reset = false; 8914 u32 flags = 0; 8915 int rc; 8916 8917 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 8918 return 0; 8919 8920 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1); 8921 if (up) 8922 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 8923 mutex_lock(&bp->hwrm_cmd_lock); 8924 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8925 if (!rc) 8926 flags = le32_to_cpu(resp->flags); 8927 mutex_unlock(&bp->hwrm_cmd_lock); 8928 if (rc) 8929 return rc; 8930 8931 if (!up) 8932 return 0; 8933 8934 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 8935 resc_reinit = true; 8936 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE) 8937 fw_reset = true; 8938 8939 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 8940 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 8941 return -ENODEV; 8942 } 8943 if (resc_reinit || fw_reset) { 8944 if (fw_reset) { 8945 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 8946 bnxt_ulp_stop(bp); 8947 bnxt_free_ctx_mem(bp); 8948 kfree(bp->ctx); 8949 bp->ctx = NULL; 8950 bnxt_dcb_free(bp); 8951 rc = bnxt_fw_init_one(bp); 8952 if (rc) { 8953 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 8954 return rc; 8955 } 8956 bnxt_clear_int_mode(bp); 8957 rc = bnxt_init_int_mode(bp); 8958 if (rc) { 8959 netdev_err(bp->dev, "init int mode failed\n"); 8960 return rc; 8961 } 8962 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 8963 } 8964 if (BNXT_NEW_RM(bp)) { 8965 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 8966 8967 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 8968 hw_resc->resv_cp_rings = 0; 8969 hw_resc->resv_stat_ctxs = 0; 8970 hw_resc->resv_irqs = 0; 8971 hw_resc->resv_tx_rings = 0; 8972 hw_resc->resv_rx_rings = 0; 8973 hw_resc->resv_hw_ring_grps = 0; 8974 hw_resc->resv_vnics = 0; 8975 if (!fw_reset) { 8976 bp->tx_nr_rings = 0; 8977 bp->rx_nr_rings = 0; 8978 } 8979 } 8980 } 8981 return 0; 8982 } 8983 8984 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 8985 { 8986 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 8987 struct hwrm_port_led_qcaps_input req = {0}; 8988 struct bnxt_pf_info *pf = &bp->pf; 8989 int rc; 8990 8991 bp->num_leds = 0; 8992 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 8993 return 0; 8994 8995 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1); 8996 req.port_id = cpu_to_le16(pf->port_id); 8997 mutex_lock(&bp->hwrm_cmd_lock); 8998 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 8999 if (rc) { 9000 mutex_unlock(&bp->hwrm_cmd_lock); 9001 return rc; 9002 } 9003 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 9004 int i; 9005 9006 bp->num_leds = resp->num_leds; 9007 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 9008 bp->num_leds); 9009 for (i = 0; i < bp->num_leds; i++) { 9010 struct bnxt_led_info *led = &bp->leds[i]; 9011 __le16 caps = led->led_state_caps; 9012 9013 if (!led->led_group_id || 9014 !BNXT_LED_ALT_BLINK_CAP(caps)) { 9015 bp->num_leds = 0; 9016 break; 9017 } 9018 } 9019 } 9020 mutex_unlock(&bp->hwrm_cmd_lock); 9021 return 0; 9022 } 9023 9024 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 9025 { 9026 struct hwrm_wol_filter_alloc_input req = {0}; 9027 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 9028 int rc; 9029 9030 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1); 9031 req.port_id = cpu_to_le16(bp->pf.port_id); 9032 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 9033 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 9034 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN); 9035 mutex_lock(&bp->hwrm_cmd_lock); 9036 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9037 if (!rc) 9038 bp->wol_filter_id = resp->wol_filter_id; 9039 mutex_unlock(&bp->hwrm_cmd_lock); 9040 return rc; 9041 } 9042 9043 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 9044 { 9045 struct hwrm_wol_filter_free_input req = {0}; 9046 9047 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1); 9048 req.port_id = cpu_to_le16(bp->pf.port_id); 9049 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 9050 req.wol_filter_id = bp->wol_filter_id; 9051 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9052 } 9053 9054 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 9055 { 9056 struct hwrm_wol_filter_qcfg_input req = {0}; 9057 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 9058 u16 next_handle = 0; 9059 int rc; 9060 9061 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1); 9062 req.port_id = cpu_to_le16(bp->pf.port_id); 9063 req.handle = cpu_to_le16(handle); 9064 mutex_lock(&bp->hwrm_cmd_lock); 9065 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9066 if (!rc) { 9067 next_handle = le16_to_cpu(resp->next_handle); 9068 if (next_handle != 0) { 9069 if (resp->wol_type == 9070 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 9071 bp->wol = 1; 9072 bp->wol_filter_id = resp->wol_filter_id; 9073 } 9074 } 9075 } 9076 mutex_unlock(&bp->hwrm_cmd_lock); 9077 return next_handle; 9078 } 9079 9080 static void bnxt_get_wol_settings(struct bnxt *bp) 9081 { 9082 u16 handle = 0; 9083 9084 bp->wol = 0; 9085 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 9086 return; 9087 9088 do { 9089 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 9090 } while (handle && handle != 0xffff); 9091 } 9092 9093 #ifdef CONFIG_BNXT_HWMON 9094 static ssize_t bnxt_show_temp(struct device *dev, 9095 struct device_attribute *devattr, char *buf) 9096 { 9097 struct hwrm_temp_monitor_query_input req = {0}; 9098 struct hwrm_temp_monitor_query_output *resp; 9099 struct bnxt *bp = dev_get_drvdata(dev); 9100 u32 temp = 0; 9101 9102 resp = bp->hwrm_cmd_resp_addr; 9103 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1); 9104 mutex_lock(&bp->hwrm_cmd_lock); 9105 if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT)) 9106 temp = resp->temp * 1000; /* display millidegree */ 9107 mutex_unlock(&bp->hwrm_cmd_lock); 9108 9109 return sprintf(buf, "%u\n", temp); 9110 } 9111 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0); 9112 9113 static struct attribute *bnxt_attrs[] = { 9114 &sensor_dev_attr_temp1_input.dev_attr.attr, 9115 NULL 9116 }; 9117 ATTRIBUTE_GROUPS(bnxt); 9118 9119 static void bnxt_hwmon_close(struct bnxt *bp) 9120 { 9121 if (bp->hwmon_dev) { 9122 hwmon_device_unregister(bp->hwmon_dev); 9123 bp->hwmon_dev = NULL; 9124 } 9125 } 9126 9127 static void bnxt_hwmon_open(struct bnxt *bp) 9128 { 9129 struct pci_dev *pdev = bp->pdev; 9130 9131 if (bp->hwmon_dev) 9132 return; 9133 9134 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, 9135 DRV_MODULE_NAME, bp, 9136 bnxt_groups); 9137 if (IS_ERR(bp->hwmon_dev)) { 9138 bp->hwmon_dev = NULL; 9139 dev_warn(&pdev->dev, "Cannot register hwmon device\n"); 9140 } 9141 } 9142 #else 9143 static void bnxt_hwmon_close(struct bnxt *bp) 9144 { 9145 } 9146 9147 static void bnxt_hwmon_open(struct bnxt *bp) 9148 { 9149 } 9150 #endif 9151 9152 static bool bnxt_eee_config_ok(struct bnxt *bp) 9153 { 9154 struct ethtool_eee *eee = &bp->eee; 9155 struct bnxt_link_info *link_info = &bp->link_info; 9156 9157 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 9158 return true; 9159 9160 if (eee->eee_enabled) { 9161 u32 advertising = 9162 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 9163 9164 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9165 eee->eee_enabled = 0; 9166 return false; 9167 } 9168 if (eee->advertised & ~advertising) { 9169 eee->advertised = advertising & eee->supported; 9170 return false; 9171 } 9172 } 9173 return true; 9174 } 9175 9176 static int bnxt_update_phy_setting(struct bnxt *bp) 9177 { 9178 int rc; 9179 bool update_link = false; 9180 bool update_pause = false; 9181 bool update_eee = false; 9182 struct bnxt_link_info *link_info = &bp->link_info; 9183 9184 rc = bnxt_update_link(bp, true); 9185 if (rc) { 9186 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 9187 rc); 9188 return rc; 9189 } 9190 if (!BNXT_SINGLE_PF(bp)) 9191 return 0; 9192 9193 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9194 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 9195 link_info->req_flow_ctrl) 9196 update_pause = true; 9197 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 9198 link_info->force_pause_setting != link_info->req_flow_ctrl) 9199 update_pause = true; 9200 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 9201 if (BNXT_AUTO_MODE(link_info->auto_mode)) 9202 update_link = true; 9203 if (link_info->req_link_speed != link_info->force_link_speed) 9204 update_link = true; 9205 if (link_info->req_duplex != link_info->duplex_setting) 9206 update_link = true; 9207 } else { 9208 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 9209 update_link = true; 9210 if (link_info->advertising != link_info->auto_link_speeds) 9211 update_link = true; 9212 } 9213 9214 /* The last close may have shutdown the link, so need to call 9215 * PHY_CFG to bring it back up. 9216 */ 9217 if (!bp->link_info.link_up) 9218 update_link = true; 9219 9220 if (!bnxt_eee_config_ok(bp)) 9221 update_eee = true; 9222 9223 if (update_link) 9224 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 9225 else if (update_pause) 9226 rc = bnxt_hwrm_set_pause(bp); 9227 if (rc) { 9228 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 9229 rc); 9230 return rc; 9231 } 9232 9233 return rc; 9234 } 9235 9236 /* Common routine to pre-map certain register block to different GRC window. 9237 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 9238 * in PF and 3 windows in VF that can be customized to map in different 9239 * register blocks. 9240 */ 9241 static void bnxt_preset_reg_win(struct bnxt *bp) 9242 { 9243 if (BNXT_PF(bp)) { 9244 /* CAG registers map to GRC window #4 */ 9245 writel(BNXT_CAG_REG_BASE, 9246 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 9247 } 9248 } 9249 9250 static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 9251 9252 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9253 { 9254 int rc = 0; 9255 9256 bnxt_preset_reg_win(bp); 9257 netif_carrier_off(bp->dev); 9258 if (irq_re_init) { 9259 /* Reserve rings now if none were reserved at driver probe. */ 9260 rc = bnxt_init_dflt_ring_mode(bp); 9261 if (rc) { 9262 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 9263 return rc; 9264 } 9265 } 9266 rc = bnxt_reserve_rings(bp, irq_re_init); 9267 if (rc) 9268 return rc; 9269 if ((bp->flags & BNXT_FLAG_RFS) && 9270 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 9271 /* disable RFS if falling back to INTA */ 9272 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 9273 bp->flags &= ~BNXT_FLAG_RFS; 9274 } 9275 9276 rc = bnxt_alloc_mem(bp, irq_re_init); 9277 if (rc) { 9278 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 9279 goto open_err_free_mem; 9280 } 9281 9282 if (irq_re_init) { 9283 bnxt_init_napi(bp); 9284 rc = bnxt_request_irq(bp); 9285 if (rc) { 9286 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 9287 goto open_err_irq; 9288 } 9289 } 9290 9291 bnxt_enable_napi(bp); 9292 bnxt_debug_dev_init(bp); 9293 9294 rc = bnxt_init_nic(bp, irq_re_init); 9295 if (rc) { 9296 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 9297 goto open_err; 9298 } 9299 9300 if (link_re_init) { 9301 mutex_lock(&bp->link_lock); 9302 rc = bnxt_update_phy_setting(bp); 9303 mutex_unlock(&bp->link_lock); 9304 if (rc) { 9305 netdev_warn(bp->dev, "failed to update phy settings\n"); 9306 if (BNXT_SINGLE_PF(bp)) { 9307 bp->link_info.phy_retry = true; 9308 bp->link_info.phy_retry_expires = 9309 jiffies + 5 * HZ; 9310 } 9311 } 9312 } 9313 9314 if (irq_re_init) 9315 udp_tunnel_nic_reset_ntf(bp->dev); 9316 9317 set_bit(BNXT_STATE_OPEN, &bp->state); 9318 bnxt_enable_int(bp); 9319 /* Enable TX queues */ 9320 bnxt_tx_enable(bp); 9321 mod_timer(&bp->timer, jiffies + bp->current_interval); 9322 /* Poll link status and check for SFP+ module status */ 9323 bnxt_get_port_module_status(bp); 9324 9325 /* VF-reps may need to be re-opened after the PF is re-opened */ 9326 if (BNXT_PF(bp)) 9327 bnxt_vf_reps_open(bp); 9328 return 0; 9329 9330 open_err: 9331 bnxt_debug_dev_exit(bp); 9332 bnxt_disable_napi(bp); 9333 9334 open_err_irq: 9335 bnxt_del_napi(bp); 9336 9337 open_err_free_mem: 9338 bnxt_free_skbs(bp); 9339 bnxt_free_irq(bp); 9340 bnxt_free_mem(bp, true); 9341 return rc; 9342 } 9343 9344 /* rtnl_lock held */ 9345 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9346 { 9347 int rc = 0; 9348 9349 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 9350 if (rc) { 9351 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 9352 dev_close(bp->dev); 9353 } 9354 return rc; 9355 } 9356 9357 /* rtnl_lock held, open the NIC half way by allocating all resources, but 9358 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline 9359 * self tests. 9360 */ 9361 int bnxt_half_open_nic(struct bnxt *bp) 9362 { 9363 int rc = 0; 9364 9365 rc = bnxt_alloc_mem(bp, false); 9366 if (rc) { 9367 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 9368 goto half_open_err; 9369 } 9370 rc = bnxt_init_nic(bp, false); 9371 if (rc) { 9372 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 9373 goto half_open_err; 9374 } 9375 return 0; 9376 9377 half_open_err: 9378 bnxt_free_skbs(bp); 9379 bnxt_free_mem(bp, false); 9380 dev_close(bp->dev); 9381 return rc; 9382 } 9383 9384 /* rtnl_lock held, this call can only be made after a previous successful 9385 * call to bnxt_half_open_nic(). 9386 */ 9387 void bnxt_half_close_nic(struct bnxt *bp) 9388 { 9389 bnxt_hwrm_resource_free(bp, false, false); 9390 bnxt_free_skbs(bp); 9391 bnxt_free_mem(bp, false); 9392 } 9393 9394 static void bnxt_reenable_sriov(struct bnxt *bp) 9395 { 9396 if (BNXT_PF(bp)) { 9397 struct bnxt_pf_info *pf = &bp->pf; 9398 int n = pf->active_vfs; 9399 9400 if (n) 9401 bnxt_cfg_hw_sriov(bp, &n, true); 9402 } 9403 } 9404 9405 static int bnxt_open(struct net_device *dev) 9406 { 9407 struct bnxt *bp = netdev_priv(dev); 9408 int rc; 9409 9410 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 9411 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n"); 9412 return -ENODEV; 9413 } 9414 9415 rc = bnxt_hwrm_if_change(bp, true); 9416 if (rc) 9417 return rc; 9418 rc = __bnxt_open_nic(bp, true, true); 9419 if (rc) { 9420 bnxt_hwrm_if_change(bp, false); 9421 } else { 9422 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 9423 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 9424 bnxt_ulp_start(bp, 0); 9425 bnxt_reenable_sriov(bp); 9426 } 9427 } 9428 bnxt_hwmon_open(bp); 9429 } 9430 9431 return rc; 9432 } 9433 9434 static bool bnxt_drv_busy(struct bnxt *bp) 9435 { 9436 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 9437 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 9438 } 9439 9440 static void bnxt_get_ring_stats(struct bnxt *bp, 9441 struct rtnl_link_stats64 *stats); 9442 9443 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 9444 bool link_re_init) 9445 { 9446 /* Close the VF-reps before closing PF */ 9447 if (BNXT_PF(bp)) 9448 bnxt_vf_reps_close(bp); 9449 9450 /* Change device state to avoid TX queue wake up's */ 9451 bnxt_tx_disable(bp); 9452 9453 clear_bit(BNXT_STATE_OPEN, &bp->state); 9454 smp_mb__after_atomic(); 9455 while (bnxt_drv_busy(bp)) 9456 msleep(20); 9457 9458 /* Flush rings and and disable interrupts */ 9459 bnxt_shutdown_nic(bp, irq_re_init); 9460 9461 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 9462 9463 bnxt_debug_dev_exit(bp); 9464 bnxt_disable_napi(bp); 9465 del_timer_sync(&bp->timer); 9466 bnxt_free_skbs(bp); 9467 9468 /* Save ring stats before shutdown */ 9469 if (bp->bnapi && irq_re_init) 9470 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 9471 if (irq_re_init) { 9472 bnxt_free_irq(bp); 9473 bnxt_del_napi(bp); 9474 } 9475 bnxt_free_mem(bp, irq_re_init); 9476 } 9477 9478 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 9479 { 9480 int rc = 0; 9481 9482 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 9483 /* If we get here, it means firmware reset is in progress 9484 * while we are trying to close. We can safely proceed with 9485 * the close because we are holding rtnl_lock(). Some firmware 9486 * messages may fail as we proceed to close. We set the 9487 * ABORT_ERR flag here so that the FW reset thread will later 9488 * abort when it gets the rtnl_lock() and sees the flag. 9489 */ 9490 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 9491 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 9492 } 9493 9494 #ifdef CONFIG_BNXT_SRIOV 9495 if (bp->sriov_cfg) { 9496 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 9497 !bp->sriov_cfg, 9498 BNXT_SRIOV_CFG_WAIT_TMO); 9499 if (rc) 9500 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 9501 } 9502 #endif 9503 __bnxt_close_nic(bp, irq_re_init, link_re_init); 9504 return rc; 9505 } 9506 9507 static int bnxt_close(struct net_device *dev) 9508 { 9509 struct bnxt *bp = netdev_priv(dev); 9510 9511 bnxt_hwmon_close(bp); 9512 bnxt_close_nic(bp, true, true); 9513 bnxt_hwrm_shutdown_link(bp); 9514 bnxt_hwrm_if_change(bp, false); 9515 return 0; 9516 } 9517 9518 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 9519 u16 *val) 9520 { 9521 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr; 9522 struct hwrm_port_phy_mdio_read_input req = {0}; 9523 int rc; 9524 9525 if (bp->hwrm_spec_code < 0x10a00) 9526 return -EOPNOTSUPP; 9527 9528 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1); 9529 req.port_id = cpu_to_le16(bp->pf.port_id); 9530 req.phy_addr = phy_addr; 9531 req.reg_addr = cpu_to_le16(reg & 0x1f); 9532 if (mdio_phy_id_is_c45(phy_addr)) { 9533 req.cl45_mdio = 1; 9534 req.phy_addr = mdio_phy_id_prtad(phy_addr); 9535 req.dev_addr = mdio_phy_id_devad(phy_addr); 9536 req.reg_addr = cpu_to_le16(reg); 9537 } 9538 9539 mutex_lock(&bp->hwrm_cmd_lock); 9540 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9541 if (!rc) 9542 *val = le16_to_cpu(resp->reg_data); 9543 mutex_unlock(&bp->hwrm_cmd_lock); 9544 return rc; 9545 } 9546 9547 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 9548 u16 val) 9549 { 9550 struct hwrm_port_phy_mdio_write_input req = {0}; 9551 9552 if (bp->hwrm_spec_code < 0x10a00) 9553 return -EOPNOTSUPP; 9554 9555 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1); 9556 req.port_id = cpu_to_le16(bp->pf.port_id); 9557 req.phy_addr = phy_addr; 9558 req.reg_addr = cpu_to_le16(reg & 0x1f); 9559 if (mdio_phy_id_is_c45(phy_addr)) { 9560 req.cl45_mdio = 1; 9561 req.phy_addr = mdio_phy_id_prtad(phy_addr); 9562 req.dev_addr = mdio_phy_id_devad(phy_addr); 9563 req.reg_addr = cpu_to_le16(reg); 9564 } 9565 req.reg_data = cpu_to_le16(val); 9566 9567 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 9568 } 9569 9570 /* rtnl_lock held */ 9571 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 9572 { 9573 struct mii_ioctl_data *mdio = if_mii(ifr); 9574 struct bnxt *bp = netdev_priv(dev); 9575 int rc; 9576 9577 switch (cmd) { 9578 case SIOCGMIIPHY: 9579 mdio->phy_id = bp->link_info.phy_addr; 9580 9581 /* fallthru */ 9582 case SIOCGMIIREG: { 9583 u16 mii_regval = 0; 9584 9585 if (!netif_running(dev)) 9586 return -EAGAIN; 9587 9588 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 9589 &mii_regval); 9590 mdio->val_out = mii_regval; 9591 return rc; 9592 } 9593 9594 case SIOCSMIIREG: 9595 if (!netif_running(dev)) 9596 return -EAGAIN; 9597 9598 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 9599 mdio->val_in); 9600 9601 default: 9602 /* do nothing */ 9603 break; 9604 } 9605 return -EOPNOTSUPP; 9606 } 9607 9608 static void bnxt_get_ring_stats(struct bnxt *bp, 9609 struct rtnl_link_stats64 *stats) 9610 { 9611 int i; 9612 9613 9614 for (i = 0; i < bp->cp_nr_rings; i++) { 9615 struct bnxt_napi *bnapi = bp->bnapi[i]; 9616 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 9617 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 9618 9619 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 9620 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 9621 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 9622 9623 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 9624 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 9625 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 9626 9627 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 9628 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 9629 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 9630 9631 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 9632 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 9633 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 9634 9635 stats->rx_missed_errors += 9636 le64_to_cpu(hw_stats->rx_discard_pkts); 9637 9638 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 9639 9640 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 9641 } 9642 } 9643 9644 static void bnxt_add_prev_stats(struct bnxt *bp, 9645 struct rtnl_link_stats64 *stats) 9646 { 9647 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 9648 9649 stats->rx_packets += prev_stats->rx_packets; 9650 stats->tx_packets += prev_stats->tx_packets; 9651 stats->rx_bytes += prev_stats->rx_bytes; 9652 stats->tx_bytes += prev_stats->tx_bytes; 9653 stats->rx_missed_errors += prev_stats->rx_missed_errors; 9654 stats->multicast += prev_stats->multicast; 9655 stats->tx_dropped += prev_stats->tx_dropped; 9656 } 9657 9658 static void 9659 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 9660 { 9661 struct bnxt *bp = netdev_priv(dev); 9662 9663 set_bit(BNXT_STATE_READ_STATS, &bp->state); 9664 /* Make sure bnxt_close_nic() sees that we are reading stats before 9665 * we check the BNXT_STATE_OPEN flag. 9666 */ 9667 smp_mb__after_atomic(); 9668 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 9669 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 9670 *stats = bp->net_stats_prev; 9671 return; 9672 } 9673 9674 bnxt_get_ring_stats(bp, stats); 9675 bnxt_add_prev_stats(bp, stats); 9676 9677 if (bp->flags & BNXT_FLAG_PORT_STATS) { 9678 struct rx_port_stats *rx = bp->hw_rx_port_stats; 9679 struct tx_port_stats *tx = bp->hw_tx_port_stats; 9680 9681 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 9682 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 9683 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 9684 le64_to_cpu(rx->rx_ovrsz_frames) + 9685 le64_to_cpu(rx->rx_runt_frames); 9686 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 9687 le64_to_cpu(rx->rx_jbr_frames); 9688 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 9689 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 9690 stats->tx_errors = le64_to_cpu(tx->tx_err); 9691 } 9692 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 9693 } 9694 9695 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 9696 { 9697 struct net_device *dev = bp->dev; 9698 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 9699 struct netdev_hw_addr *ha; 9700 u8 *haddr; 9701 int mc_count = 0; 9702 bool update = false; 9703 int off = 0; 9704 9705 netdev_for_each_mc_addr(ha, dev) { 9706 if (mc_count >= BNXT_MAX_MC_ADDRS) { 9707 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 9708 vnic->mc_list_count = 0; 9709 return false; 9710 } 9711 haddr = ha->addr; 9712 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 9713 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 9714 update = true; 9715 } 9716 off += ETH_ALEN; 9717 mc_count++; 9718 } 9719 if (mc_count) 9720 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 9721 9722 if (mc_count != vnic->mc_list_count) { 9723 vnic->mc_list_count = mc_count; 9724 update = true; 9725 } 9726 return update; 9727 } 9728 9729 static bool bnxt_uc_list_updated(struct bnxt *bp) 9730 { 9731 struct net_device *dev = bp->dev; 9732 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 9733 struct netdev_hw_addr *ha; 9734 int off = 0; 9735 9736 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 9737 return true; 9738 9739 netdev_for_each_uc_addr(ha, dev) { 9740 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 9741 return true; 9742 9743 off += ETH_ALEN; 9744 } 9745 return false; 9746 } 9747 9748 static void bnxt_set_rx_mode(struct net_device *dev) 9749 { 9750 struct bnxt *bp = netdev_priv(dev); 9751 struct bnxt_vnic_info *vnic; 9752 bool mc_update = false; 9753 bool uc_update; 9754 u32 mask; 9755 9756 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 9757 return; 9758 9759 vnic = &bp->vnic_info[0]; 9760 mask = vnic->rx_mask; 9761 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 9762 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 9763 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 9764 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 9765 9766 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 9767 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 9768 9769 uc_update = bnxt_uc_list_updated(bp); 9770 9771 if (dev->flags & IFF_BROADCAST) 9772 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 9773 if (dev->flags & IFF_ALLMULTI) { 9774 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 9775 vnic->mc_list_count = 0; 9776 } else { 9777 mc_update = bnxt_mc_list_updated(bp, &mask); 9778 } 9779 9780 if (mask != vnic->rx_mask || uc_update || mc_update) { 9781 vnic->rx_mask = mask; 9782 9783 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 9784 bnxt_queue_sp_work(bp); 9785 } 9786 } 9787 9788 static int bnxt_cfg_rx_mode(struct bnxt *bp) 9789 { 9790 struct net_device *dev = bp->dev; 9791 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 9792 struct netdev_hw_addr *ha; 9793 int i, off = 0, rc; 9794 bool uc_update; 9795 9796 netif_addr_lock_bh(dev); 9797 uc_update = bnxt_uc_list_updated(bp); 9798 netif_addr_unlock_bh(dev); 9799 9800 if (!uc_update) 9801 goto skip_uc; 9802 9803 mutex_lock(&bp->hwrm_cmd_lock); 9804 for (i = 1; i < vnic->uc_filter_count; i++) { 9805 struct hwrm_cfa_l2_filter_free_input req = {0}; 9806 9807 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 9808 -1); 9809 9810 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 9811 9812 rc = _hwrm_send_message(bp, &req, sizeof(req), 9813 HWRM_CMD_TIMEOUT); 9814 } 9815 mutex_unlock(&bp->hwrm_cmd_lock); 9816 9817 vnic->uc_filter_count = 1; 9818 9819 netif_addr_lock_bh(dev); 9820 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 9821 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 9822 } else { 9823 netdev_for_each_uc_addr(ha, dev) { 9824 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 9825 off += ETH_ALEN; 9826 vnic->uc_filter_count++; 9827 } 9828 } 9829 netif_addr_unlock_bh(dev); 9830 9831 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 9832 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 9833 if (rc) { 9834 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 9835 rc); 9836 vnic->uc_filter_count = i; 9837 return rc; 9838 } 9839 } 9840 9841 skip_uc: 9842 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 9843 if (rc && vnic->mc_list_count) { 9844 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 9845 rc); 9846 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 9847 vnic->mc_list_count = 0; 9848 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 9849 } 9850 if (rc) 9851 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 9852 rc); 9853 9854 return rc; 9855 } 9856 9857 static bool bnxt_can_reserve_rings(struct bnxt *bp) 9858 { 9859 #ifdef CONFIG_BNXT_SRIOV 9860 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 9861 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9862 9863 /* No minimum rings were provisioned by the PF. Don't 9864 * reserve rings by default when device is down. 9865 */ 9866 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 9867 return true; 9868 9869 if (!netif_running(bp->dev)) 9870 return false; 9871 } 9872 #endif 9873 return true; 9874 } 9875 9876 /* If the chip and firmware supports RFS */ 9877 static bool bnxt_rfs_supported(struct bnxt *bp) 9878 { 9879 if (bp->flags & BNXT_FLAG_CHIP_P5) { 9880 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 9881 return true; 9882 return false; 9883 } 9884 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 9885 return true; 9886 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 9887 return true; 9888 return false; 9889 } 9890 9891 /* If runtime conditions support RFS */ 9892 static bool bnxt_rfs_capable(struct bnxt *bp) 9893 { 9894 #ifdef CONFIG_RFS_ACCEL 9895 int vnics, max_vnics, max_rss_ctxs; 9896 9897 if (bp->flags & BNXT_FLAG_CHIP_P5) 9898 return bnxt_rfs_supported(bp); 9899 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp)) 9900 return false; 9901 9902 vnics = 1 + bp->rx_nr_rings; 9903 max_vnics = bnxt_get_max_func_vnics(bp); 9904 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 9905 9906 /* RSS contexts not a limiting factor */ 9907 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP) 9908 max_rss_ctxs = max_vnics; 9909 if (vnics > max_vnics || vnics > max_rss_ctxs) { 9910 if (bp->rx_nr_rings > 1) 9911 netdev_warn(bp->dev, 9912 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 9913 min(max_rss_ctxs - 1, max_vnics - 1)); 9914 return false; 9915 } 9916 9917 if (!BNXT_NEW_RM(bp)) 9918 return true; 9919 9920 if (vnics == bp->hw_resc.resv_vnics) 9921 return true; 9922 9923 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics); 9924 if (vnics <= bp->hw_resc.resv_vnics) 9925 return true; 9926 9927 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 9928 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1); 9929 return false; 9930 #else 9931 return false; 9932 #endif 9933 } 9934 9935 static netdev_features_t bnxt_fix_features(struct net_device *dev, 9936 netdev_features_t features) 9937 { 9938 struct bnxt *bp = netdev_priv(dev); 9939 netdev_features_t vlan_features; 9940 9941 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 9942 features &= ~NETIF_F_NTUPLE; 9943 9944 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 9945 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 9946 9947 if (!(features & NETIF_F_GRO)) 9948 features &= ~NETIF_F_GRO_HW; 9949 9950 if (features & NETIF_F_GRO_HW) 9951 features &= ~NETIF_F_LRO; 9952 9953 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 9954 * turned on or off together. 9955 */ 9956 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 9957 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 9958 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 9959 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 9960 else if (vlan_features) 9961 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 9962 } 9963 #ifdef CONFIG_BNXT_SRIOV 9964 if (BNXT_VF(bp) && bp->vf.vlan) 9965 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 9966 #endif 9967 return features; 9968 } 9969 9970 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 9971 { 9972 struct bnxt *bp = netdev_priv(dev); 9973 u32 flags = bp->flags; 9974 u32 changes; 9975 int rc = 0; 9976 bool re_init = false; 9977 bool update_tpa = false; 9978 9979 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 9980 if (features & NETIF_F_GRO_HW) 9981 flags |= BNXT_FLAG_GRO; 9982 else if (features & NETIF_F_LRO) 9983 flags |= BNXT_FLAG_LRO; 9984 9985 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 9986 flags &= ~BNXT_FLAG_TPA; 9987 9988 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 9989 flags |= BNXT_FLAG_STRIP_VLAN; 9990 9991 if (features & NETIF_F_NTUPLE) 9992 flags |= BNXT_FLAG_RFS; 9993 9994 changes = flags ^ bp->flags; 9995 if (changes & BNXT_FLAG_TPA) { 9996 update_tpa = true; 9997 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 9998 (flags & BNXT_FLAG_TPA) == 0 || 9999 (bp->flags & BNXT_FLAG_CHIP_P5)) 10000 re_init = true; 10001 } 10002 10003 if (changes & ~BNXT_FLAG_TPA) 10004 re_init = true; 10005 10006 if (flags != bp->flags) { 10007 u32 old_flags = bp->flags; 10008 10009 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10010 bp->flags = flags; 10011 if (update_tpa) 10012 bnxt_set_ring_params(bp); 10013 return rc; 10014 } 10015 10016 if (re_init) { 10017 bnxt_close_nic(bp, false, false); 10018 bp->flags = flags; 10019 if (update_tpa) 10020 bnxt_set_ring_params(bp); 10021 10022 return bnxt_open_nic(bp, false, false); 10023 } 10024 if (update_tpa) { 10025 bp->flags = flags; 10026 rc = bnxt_set_tpa(bp, 10027 (flags & BNXT_FLAG_TPA) ? 10028 true : false); 10029 if (rc) 10030 bp->flags = old_flags; 10031 } 10032 } 10033 return rc; 10034 } 10035 10036 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 10037 u32 ring_id, u32 *prod, u32 *cons) 10038 { 10039 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr; 10040 struct hwrm_dbg_ring_info_get_input req = {0}; 10041 int rc; 10042 10043 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1); 10044 req.ring_type = ring_type; 10045 req.fw_ring_id = cpu_to_le32(ring_id); 10046 mutex_lock(&bp->hwrm_cmd_lock); 10047 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10048 if (!rc) { 10049 *prod = le32_to_cpu(resp->producer_index); 10050 *cons = le32_to_cpu(resp->consumer_index); 10051 } 10052 mutex_unlock(&bp->hwrm_cmd_lock); 10053 return rc; 10054 } 10055 10056 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 10057 { 10058 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 10059 int i = bnapi->index; 10060 10061 if (!txr) 10062 return; 10063 10064 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 10065 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 10066 txr->tx_cons); 10067 } 10068 10069 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 10070 { 10071 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 10072 int i = bnapi->index; 10073 10074 if (!rxr) 10075 return; 10076 10077 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 10078 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 10079 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 10080 rxr->rx_sw_agg_prod); 10081 } 10082 10083 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 10084 { 10085 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 10086 int i = bnapi->index; 10087 10088 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 10089 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 10090 } 10091 10092 static void bnxt_dbg_dump_states(struct bnxt *bp) 10093 { 10094 int i; 10095 struct bnxt_napi *bnapi; 10096 10097 for (i = 0; i < bp->cp_nr_rings; i++) { 10098 bnapi = bp->bnapi[i]; 10099 if (netif_msg_drv(bp)) { 10100 bnxt_dump_tx_sw_state(bnapi); 10101 bnxt_dump_rx_sw_state(bnapi); 10102 bnxt_dump_cp_sw_state(bnapi); 10103 } 10104 } 10105 } 10106 10107 static void bnxt_reset_task(struct bnxt *bp, bool silent) 10108 { 10109 if (!silent) 10110 bnxt_dbg_dump_states(bp); 10111 if (netif_running(bp->dev)) { 10112 int rc; 10113 10114 if (silent) { 10115 bnxt_close_nic(bp, false, false); 10116 bnxt_open_nic(bp, false, false); 10117 } else { 10118 bnxt_ulp_stop(bp); 10119 bnxt_close_nic(bp, true, false); 10120 rc = bnxt_open_nic(bp, true, false); 10121 bnxt_ulp_start(bp, rc); 10122 } 10123 } 10124 } 10125 10126 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 10127 { 10128 struct bnxt *bp = netdev_priv(dev); 10129 10130 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 10131 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 10132 bnxt_queue_sp_work(bp); 10133 } 10134 10135 static void bnxt_fw_health_check(struct bnxt *bp) 10136 { 10137 struct bnxt_fw_health *fw_health = bp->fw_health; 10138 u32 val; 10139 10140 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10141 return; 10142 10143 if (fw_health->tmr_counter) { 10144 fw_health->tmr_counter--; 10145 return; 10146 } 10147 10148 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 10149 if (val == fw_health->last_fw_heartbeat) 10150 goto fw_reset; 10151 10152 fw_health->last_fw_heartbeat = val; 10153 10154 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 10155 if (val != fw_health->last_fw_reset_cnt) 10156 goto fw_reset; 10157 10158 fw_health->tmr_counter = fw_health->tmr_multiplier; 10159 return; 10160 10161 fw_reset: 10162 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event); 10163 bnxt_queue_sp_work(bp); 10164 } 10165 10166 static void bnxt_timer(struct timer_list *t) 10167 { 10168 struct bnxt *bp = from_timer(bp, t, timer); 10169 struct net_device *dev = bp->dev; 10170 10171 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 10172 return; 10173 10174 if (atomic_read(&bp->intr_sem) != 0) 10175 goto bnxt_restart_timer; 10176 10177 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 10178 bnxt_fw_health_check(bp); 10179 10180 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) && 10181 bp->stats_coal_ticks) { 10182 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 10183 bnxt_queue_sp_work(bp); 10184 } 10185 10186 if (bnxt_tc_flower_enabled(bp)) { 10187 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event); 10188 bnxt_queue_sp_work(bp); 10189 } 10190 10191 #ifdef CONFIG_RFS_ACCEL 10192 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) { 10193 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 10194 bnxt_queue_sp_work(bp); 10195 } 10196 #endif /*CONFIG_RFS_ACCEL*/ 10197 10198 if (bp->link_info.phy_retry) { 10199 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 10200 bp->link_info.phy_retry = false; 10201 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 10202 } else { 10203 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event); 10204 bnxt_queue_sp_work(bp); 10205 } 10206 } 10207 10208 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev && 10209 netif_carrier_ok(dev)) { 10210 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event); 10211 bnxt_queue_sp_work(bp); 10212 } 10213 bnxt_restart_timer: 10214 mod_timer(&bp->timer, jiffies + bp->current_interval); 10215 } 10216 10217 static void bnxt_rtnl_lock_sp(struct bnxt *bp) 10218 { 10219 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 10220 * set. If the device is being closed, bnxt_close() may be holding 10221 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 10222 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 10223 */ 10224 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10225 rtnl_lock(); 10226 } 10227 10228 static void bnxt_rtnl_unlock_sp(struct bnxt *bp) 10229 { 10230 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10231 rtnl_unlock(); 10232 } 10233 10234 /* Only called from bnxt_sp_task() */ 10235 static void bnxt_reset(struct bnxt *bp, bool silent) 10236 { 10237 bnxt_rtnl_lock_sp(bp); 10238 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 10239 bnxt_reset_task(bp, silent); 10240 bnxt_rtnl_unlock_sp(bp); 10241 } 10242 10243 static void bnxt_fw_reset_close(struct bnxt *bp) 10244 { 10245 bnxt_ulp_stop(bp); 10246 /* When firmware is fatal state, disable PCI device to prevent 10247 * any potential bad DMAs before freeing kernel memory. 10248 */ 10249 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 10250 pci_disable_device(bp->pdev); 10251 __bnxt_close_nic(bp, true, false); 10252 bnxt_clear_int_mode(bp); 10253 bnxt_hwrm_func_drv_unrgtr(bp); 10254 if (pci_is_enabled(bp->pdev)) 10255 pci_disable_device(bp->pdev); 10256 bnxt_free_ctx_mem(bp); 10257 kfree(bp->ctx); 10258 bp->ctx = NULL; 10259 } 10260 10261 static bool is_bnxt_fw_ok(struct bnxt *bp) 10262 { 10263 struct bnxt_fw_health *fw_health = bp->fw_health; 10264 bool no_heartbeat = false, has_reset = false; 10265 u32 val; 10266 10267 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 10268 if (val == fw_health->last_fw_heartbeat) 10269 no_heartbeat = true; 10270 10271 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 10272 if (val != fw_health->last_fw_reset_cnt) 10273 has_reset = true; 10274 10275 if (!no_heartbeat && has_reset) 10276 return true; 10277 10278 return false; 10279 } 10280 10281 /* rtnl_lock is acquired before calling this function */ 10282 static void bnxt_force_fw_reset(struct bnxt *bp) 10283 { 10284 struct bnxt_fw_health *fw_health = bp->fw_health; 10285 u32 wait_dsecs; 10286 10287 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 10288 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 10289 return; 10290 10291 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10292 bnxt_fw_reset_close(bp); 10293 wait_dsecs = fw_health->master_func_wait_dsecs; 10294 if (fw_health->master) { 10295 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 10296 wait_dsecs = 0; 10297 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 10298 } else { 10299 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 10300 wait_dsecs = fw_health->normal_func_wait_dsecs; 10301 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 10302 } 10303 10304 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 10305 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 10306 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 10307 } 10308 10309 void bnxt_fw_exception(struct bnxt *bp) 10310 { 10311 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 10312 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 10313 bnxt_rtnl_lock_sp(bp); 10314 bnxt_force_fw_reset(bp); 10315 bnxt_rtnl_unlock_sp(bp); 10316 } 10317 10318 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or 10319 * < 0 on error. 10320 */ 10321 static int bnxt_get_registered_vfs(struct bnxt *bp) 10322 { 10323 #ifdef CONFIG_BNXT_SRIOV 10324 int rc; 10325 10326 if (!BNXT_PF(bp)) 10327 return 0; 10328 10329 rc = bnxt_hwrm_func_qcfg(bp); 10330 if (rc) { 10331 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 10332 return rc; 10333 } 10334 if (bp->pf.registered_vfs) 10335 return bp->pf.registered_vfs; 10336 if (bp->sriov_cfg) 10337 return 1; 10338 #endif 10339 return 0; 10340 } 10341 10342 void bnxt_fw_reset(struct bnxt *bp) 10343 { 10344 bnxt_rtnl_lock_sp(bp); 10345 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 10346 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10347 int n = 0, tmo; 10348 10349 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10350 if (bp->pf.active_vfs && 10351 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 10352 n = bnxt_get_registered_vfs(bp); 10353 if (n < 0) { 10354 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 10355 n); 10356 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10357 dev_close(bp->dev); 10358 goto fw_reset_exit; 10359 } else if (n > 0) { 10360 u16 vf_tmo_dsecs = n * 10; 10361 10362 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 10363 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 10364 bp->fw_reset_state = 10365 BNXT_FW_RESET_STATE_POLL_VF; 10366 bnxt_queue_fw_reset_work(bp, HZ / 10); 10367 goto fw_reset_exit; 10368 } 10369 bnxt_fw_reset_close(bp); 10370 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 10371 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 10372 tmo = HZ / 10; 10373 } else { 10374 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 10375 tmo = bp->fw_reset_min_dsecs * HZ / 10; 10376 } 10377 bnxt_queue_fw_reset_work(bp, tmo); 10378 } 10379 fw_reset_exit: 10380 bnxt_rtnl_unlock_sp(bp); 10381 } 10382 10383 static void bnxt_chk_missed_irq(struct bnxt *bp) 10384 { 10385 int i; 10386 10387 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 10388 return; 10389 10390 for (i = 0; i < bp->cp_nr_rings; i++) { 10391 struct bnxt_napi *bnapi = bp->bnapi[i]; 10392 struct bnxt_cp_ring_info *cpr; 10393 u32 fw_ring_id; 10394 int j; 10395 10396 if (!bnapi) 10397 continue; 10398 10399 cpr = &bnapi->cp_ring; 10400 for (j = 0; j < 2; j++) { 10401 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j]; 10402 u32 val[2]; 10403 10404 if (!cpr2 || cpr2->has_more_work || 10405 !bnxt_has_work(bp, cpr2)) 10406 continue; 10407 10408 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 10409 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 10410 continue; 10411 } 10412 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 10413 bnxt_dbg_hwrm_ring_info_get(bp, 10414 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 10415 fw_ring_id, &val[0], &val[1]); 10416 cpr->sw_stats.cmn.missed_irqs++; 10417 } 10418 } 10419 } 10420 10421 static void bnxt_cfg_ntp_filters(struct bnxt *); 10422 10423 static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 10424 { 10425 struct bnxt_link_info *link_info = &bp->link_info; 10426 10427 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 10428 link_info->autoneg = BNXT_AUTONEG_SPEED; 10429 if (bp->hwrm_spec_code >= 0x10201) { 10430 if (link_info->auto_pause_setting & 10431 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 10432 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 10433 } else { 10434 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 10435 } 10436 link_info->advertising = link_info->auto_link_speeds; 10437 } else { 10438 link_info->req_link_speed = link_info->force_link_speed; 10439 link_info->req_duplex = link_info->duplex_setting; 10440 } 10441 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 10442 link_info->req_flow_ctrl = 10443 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 10444 else 10445 link_info->req_flow_ctrl = link_info->force_pause_setting; 10446 } 10447 10448 static void bnxt_sp_task(struct work_struct *work) 10449 { 10450 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 10451 10452 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10453 smp_mb__after_atomic(); 10454 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 10455 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10456 return; 10457 } 10458 10459 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 10460 bnxt_cfg_rx_mode(bp); 10461 10462 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 10463 bnxt_cfg_ntp_filters(bp); 10464 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 10465 bnxt_hwrm_exec_fwd_req(bp); 10466 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 10467 bnxt_hwrm_port_qstats(bp); 10468 bnxt_hwrm_port_qstats_ext(bp); 10469 bnxt_hwrm_pcie_qstats(bp); 10470 } 10471 10472 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 10473 int rc; 10474 10475 mutex_lock(&bp->link_lock); 10476 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 10477 &bp->sp_event)) 10478 bnxt_hwrm_phy_qcaps(bp); 10479 10480 rc = bnxt_update_link(bp, true); 10481 if (rc) 10482 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 10483 rc); 10484 10485 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 10486 &bp->sp_event)) 10487 bnxt_init_ethtool_link_settings(bp); 10488 mutex_unlock(&bp->link_lock); 10489 } 10490 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 10491 int rc; 10492 10493 mutex_lock(&bp->link_lock); 10494 rc = bnxt_update_phy_setting(bp); 10495 mutex_unlock(&bp->link_lock); 10496 if (rc) { 10497 netdev_warn(bp->dev, "update phy settings retry failed\n"); 10498 } else { 10499 bp->link_info.phy_retry = false; 10500 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 10501 } 10502 } 10503 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 10504 mutex_lock(&bp->link_lock); 10505 bnxt_get_port_module_status(bp); 10506 mutex_unlock(&bp->link_lock); 10507 } 10508 10509 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 10510 bnxt_tc_flow_stats_work(bp); 10511 10512 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 10513 bnxt_chk_missed_irq(bp); 10514 10515 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 10516 * must be the last functions to be called before exiting. 10517 */ 10518 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 10519 bnxt_reset(bp, false); 10520 10521 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 10522 bnxt_reset(bp, true); 10523 10524 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) 10525 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT); 10526 10527 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 10528 if (!is_bnxt_fw_ok(bp)) 10529 bnxt_devlink_health_report(bp, 10530 BNXT_FW_EXCEPTION_SP_EVENT); 10531 } 10532 10533 smp_mb__before_atomic(); 10534 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 10535 } 10536 10537 /* Under rtnl_lock */ 10538 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 10539 int tx_xdp) 10540 { 10541 int max_rx, max_tx, tx_sets = 1; 10542 int tx_rings_needed, stats; 10543 int rx_rings = rx; 10544 int cp, vnics, rc; 10545 10546 if (tcs) 10547 tx_sets = tcs; 10548 10549 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh); 10550 if (rc) 10551 return rc; 10552 10553 if (max_rx < rx) 10554 return -ENOMEM; 10555 10556 tx_rings_needed = tx * tx_sets + tx_xdp; 10557 if (max_tx < tx_rings_needed) 10558 return -ENOMEM; 10559 10560 vnics = 1; 10561 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) 10562 vnics += rx_rings; 10563 10564 if (bp->flags & BNXT_FLAG_AGG_RINGS) 10565 rx_rings <<= 1; 10566 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx; 10567 stats = cp; 10568 if (BNXT_NEW_RM(bp)) { 10569 cp += bnxt_get_ulp_msix_num(bp); 10570 stats += bnxt_get_ulp_stat_ctxs(bp); 10571 } 10572 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp, 10573 stats, vnics); 10574 } 10575 10576 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 10577 { 10578 if (bp->bar2) { 10579 pci_iounmap(pdev, bp->bar2); 10580 bp->bar2 = NULL; 10581 } 10582 10583 if (bp->bar1) { 10584 pci_iounmap(pdev, bp->bar1); 10585 bp->bar1 = NULL; 10586 } 10587 10588 if (bp->bar0) { 10589 pci_iounmap(pdev, bp->bar0); 10590 bp->bar0 = NULL; 10591 } 10592 } 10593 10594 static void bnxt_cleanup_pci(struct bnxt *bp) 10595 { 10596 bnxt_unmap_bars(bp, bp->pdev); 10597 pci_release_regions(bp->pdev); 10598 if (pci_is_enabled(bp->pdev)) 10599 pci_disable_device(bp->pdev); 10600 } 10601 10602 static void bnxt_init_dflt_coal(struct bnxt *bp) 10603 { 10604 struct bnxt_coal *coal; 10605 10606 /* Tick values in micro seconds. 10607 * 1 coal_buf x bufs_per_record = 1 completion record. 10608 */ 10609 coal = &bp->rx_coal; 10610 coal->coal_ticks = 10; 10611 coal->coal_bufs = 30; 10612 coal->coal_ticks_irq = 1; 10613 coal->coal_bufs_irq = 2; 10614 coal->idle_thresh = 50; 10615 coal->bufs_per_record = 2; 10616 coal->budget = 64; /* NAPI budget */ 10617 10618 coal = &bp->tx_coal; 10619 coal->coal_ticks = 28; 10620 coal->coal_bufs = 30; 10621 coal->coal_ticks_irq = 2; 10622 coal->coal_bufs_irq = 2; 10623 coal->bufs_per_record = 1; 10624 10625 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 10626 } 10627 10628 static void bnxt_alloc_fw_health(struct bnxt *bp) 10629 { 10630 if (bp->fw_health) 10631 return; 10632 10633 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 10634 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 10635 return; 10636 10637 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 10638 if (!bp->fw_health) { 10639 netdev_warn(bp->dev, "Failed to allocate fw_health\n"); 10640 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 10641 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 10642 } 10643 } 10644 10645 static int bnxt_fw_init_one_p1(struct bnxt *bp) 10646 { 10647 int rc; 10648 10649 bp->fw_cap = 0; 10650 rc = bnxt_hwrm_ver_get(bp); 10651 if (rc) 10652 return rc; 10653 10654 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) { 10655 rc = bnxt_alloc_kong_hwrm_resources(bp); 10656 if (rc) 10657 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL; 10658 } 10659 10660 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) || 10661 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) { 10662 rc = bnxt_alloc_hwrm_short_cmd_req(bp); 10663 if (rc) 10664 return rc; 10665 } 10666 rc = bnxt_hwrm_func_reset(bp); 10667 if (rc) 10668 return -ENODEV; 10669 10670 bnxt_hwrm_fw_set_time(bp); 10671 return 0; 10672 } 10673 10674 static int bnxt_fw_init_one_p2(struct bnxt *bp) 10675 { 10676 int rc; 10677 10678 /* Get the MAX capabilities for this function */ 10679 rc = bnxt_hwrm_func_qcaps(bp); 10680 if (rc) { 10681 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 10682 rc); 10683 return -ENODEV; 10684 } 10685 10686 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 10687 if (rc) 10688 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 10689 rc); 10690 10691 bnxt_alloc_fw_health(bp); 10692 rc = bnxt_hwrm_error_recovery_qcfg(bp); 10693 if (rc) 10694 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 10695 rc); 10696 10697 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 10698 if (rc) 10699 return -ENODEV; 10700 10701 bnxt_hwrm_func_qcfg(bp); 10702 bnxt_hwrm_vnic_qcaps(bp); 10703 bnxt_hwrm_port_led_qcaps(bp); 10704 bnxt_ethtool_init(bp); 10705 bnxt_dcb_init(bp); 10706 return 0; 10707 } 10708 10709 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 10710 { 10711 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP; 10712 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 10713 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 10714 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 10715 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 10716 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 10717 bp->flags |= BNXT_FLAG_UDP_RSS_CAP; 10718 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 10719 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 10720 } 10721 } 10722 10723 static void bnxt_set_dflt_rfs(struct bnxt *bp) 10724 { 10725 struct net_device *dev = bp->dev; 10726 10727 dev->hw_features &= ~NETIF_F_NTUPLE; 10728 dev->features &= ~NETIF_F_NTUPLE; 10729 bp->flags &= ~BNXT_FLAG_RFS; 10730 if (bnxt_rfs_supported(bp)) { 10731 dev->hw_features |= NETIF_F_NTUPLE; 10732 if (bnxt_rfs_capable(bp)) { 10733 bp->flags |= BNXT_FLAG_RFS; 10734 dev->features |= NETIF_F_NTUPLE; 10735 } 10736 } 10737 } 10738 10739 static void bnxt_fw_init_one_p3(struct bnxt *bp) 10740 { 10741 struct pci_dev *pdev = bp->pdev; 10742 10743 bnxt_set_dflt_rss_hash_type(bp); 10744 bnxt_set_dflt_rfs(bp); 10745 10746 bnxt_get_wol_settings(bp); 10747 if (bp->flags & BNXT_FLAG_WOL_CAP) 10748 device_set_wakeup_enable(&pdev->dev, bp->wol); 10749 else 10750 device_set_wakeup_capable(&pdev->dev, false); 10751 10752 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 10753 bnxt_hwrm_coal_params_qcaps(bp); 10754 } 10755 10756 static int bnxt_fw_init_one(struct bnxt *bp) 10757 { 10758 int rc; 10759 10760 rc = bnxt_fw_init_one_p1(bp); 10761 if (rc) { 10762 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 10763 return rc; 10764 } 10765 rc = bnxt_fw_init_one_p2(bp); 10766 if (rc) { 10767 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 10768 return rc; 10769 } 10770 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 10771 if (rc) 10772 return rc; 10773 10774 /* In case fw capabilities have changed, destroy the unneeded 10775 * reporters and create newly capable ones. 10776 */ 10777 bnxt_dl_fw_reporters_destroy(bp, false); 10778 bnxt_dl_fw_reporters_create(bp); 10779 bnxt_fw_init_one_p3(bp); 10780 return 0; 10781 } 10782 10783 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 10784 { 10785 struct bnxt_fw_health *fw_health = bp->fw_health; 10786 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 10787 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 10788 u32 reg_type, reg_off, delay_msecs; 10789 10790 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 10791 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 10792 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 10793 switch (reg_type) { 10794 case BNXT_FW_HEALTH_REG_TYPE_CFG: 10795 pci_write_config_dword(bp->pdev, reg_off, val); 10796 break; 10797 case BNXT_FW_HEALTH_REG_TYPE_GRC: 10798 writel(reg_off & BNXT_GRC_BASE_MASK, 10799 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 10800 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 10801 /* fall through */ 10802 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 10803 writel(val, bp->bar0 + reg_off); 10804 break; 10805 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 10806 writel(val, bp->bar1 + reg_off); 10807 break; 10808 } 10809 if (delay_msecs) { 10810 pci_read_config_dword(bp->pdev, 0, &val); 10811 msleep(delay_msecs); 10812 } 10813 } 10814 10815 static void bnxt_reset_all(struct bnxt *bp) 10816 { 10817 struct bnxt_fw_health *fw_health = bp->fw_health; 10818 int i, rc; 10819 10820 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 10821 #ifdef CONFIG_TEE_BNXT_FW 10822 rc = tee_bnxt_fw_load(); 10823 if (rc) 10824 netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc); 10825 bp->fw_reset_timestamp = jiffies; 10826 #endif 10827 return; 10828 } 10829 10830 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 10831 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 10832 bnxt_fw_reset_writel(bp, i); 10833 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 10834 struct hwrm_fw_reset_input req = {0}; 10835 10836 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1); 10837 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr); 10838 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 10839 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 10840 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 10841 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 10842 if (rc) 10843 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 10844 } 10845 bp->fw_reset_timestamp = jiffies; 10846 } 10847 10848 static void bnxt_fw_reset_task(struct work_struct *work) 10849 { 10850 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 10851 int rc; 10852 10853 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 10854 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 10855 return; 10856 } 10857 10858 switch (bp->fw_reset_state) { 10859 case BNXT_FW_RESET_STATE_POLL_VF: { 10860 int n = bnxt_get_registered_vfs(bp); 10861 int tmo; 10862 10863 if (n < 0) { 10864 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 10865 n, jiffies_to_msecs(jiffies - 10866 bp->fw_reset_timestamp)); 10867 goto fw_reset_abort; 10868 } else if (n > 0) { 10869 if (time_after(jiffies, bp->fw_reset_timestamp + 10870 (bp->fw_reset_max_dsecs * HZ / 10))) { 10871 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10872 bp->fw_reset_state = 0; 10873 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 10874 n); 10875 return; 10876 } 10877 bnxt_queue_fw_reset_work(bp, HZ / 10); 10878 return; 10879 } 10880 bp->fw_reset_timestamp = jiffies; 10881 rtnl_lock(); 10882 bnxt_fw_reset_close(bp); 10883 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 10884 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 10885 tmo = HZ / 10; 10886 } else { 10887 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 10888 tmo = bp->fw_reset_min_dsecs * HZ / 10; 10889 } 10890 rtnl_unlock(); 10891 bnxt_queue_fw_reset_work(bp, tmo); 10892 return; 10893 } 10894 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 10895 u32 val; 10896 10897 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 10898 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 10899 !time_after(jiffies, bp->fw_reset_timestamp + 10900 (bp->fw_reset_max_dsecs * HZ / 10))) { 10901 bnxt_queue_fw_reset_work(bp, HZ / 5); 10902 return; 10903 } 10904 10905 if (!bp->fw_health->master) { 10906 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 10907 10908 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 10909 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 10910 return; 10911 } 10912 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 10913 } 10914 /* fall through */ 10915 case BNXT_FW_RESET_STATE_RESET_FW: 10916 bnxt_reset_all(bp); 10917 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 10918 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 10919 return; 10920 case BNXT_FW_RESET_STATE_ENABLE_DEV: 10921 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 10922 u32 val; 10923 10924 val = bnxt_fw_health_readl(bp, 10925 BNXT_FW_RESET_INPROG_REG); 10926 if (val) 10927 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n", 10928 val); 10929 } 10930 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 10931 if (pci_enable_device(bp->pdev)) { 10932 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 10933 goto fw_reset_abort; 10934 } 10935 pci_set_master(bp->pdev); 10936 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 10937 /* fall through */ 10938 case BNXT_FW_RESET_STATE_POLL_FW: 10939 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 10940 rc = __bnxt_hwrm_ver_get(bp, true); 10941 if (rc) { 10942 if (time_after(jiffies, bp->fw_reset_timestamp + 10943 (bp->fw_reset_max_dsecs * HZ / 10))) { 10944 netdev_err(bp->dev, "Firmware reset aborted\n"); 10945 goto fw_reset_abort; 10946 } 10947 bnxt_queue_fw_reset_work(bp, HZ / 5); 10948 return; 10949 } 10950 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 10951 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 10952 /* fall through */ 10953 case BNXT_FW_RESET_STATE_OPENING: 10954 while (!rtnl_trylock()) { 10955 bnxt_queue_fw_reset_work(bp, HZ / 10); 10956 return; 10957 } 10958 rc = bnxt_open(bp->dev); 10959 if (rc) { 10960 netdev_err(bp->dev, "bnxt_open_nic() failed\n"); 10961 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10962 dev_close(bp->dev); 10963 } 10964 10965 bp->fw_reset_state = 0; 10966 /* Make sure fw_reset_state is 0 before clearing the flag */ 10967 smp_mb__before_atomic(); 10968 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10969 bnxt_ulp_start(bp, rc); 10970 if (!rc) 10971 bnxt_reenable_sriov(bp); 10972 bnxt_dl_health_recovery_done(bp); 10973 bnxt_dl_health_status_update(bp, true); 10974 rtnl_unlock(); 10975 break; 10976 } 10977 return; 10978 10979 fw_reset_abort: 10980 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 10981 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) 10982 bnxt_dl_health_status_update(bp, false); 10983 bp->fw_reset_state = 0; 10984 rtnl_lock(); 10985 dev_close(bp->dev); 10986 rtnl_unlock(); 10987 } 10988 10989 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 10990 { 10991 int rc; 10992 struct bnxt *bp = netdev_priv(dev); 10993 10994 SET_NETDEV_DEV(dev, &pdev->dev); 10995 10996 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 10997 rc = pci_enable_device(pdev); 10998 if (rc) { 10999 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 11000 goto init_err; 11001 } 11002 11003 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 11004 dev_err(&pdev->dev, 11005 "Cannot find PCI device base address, aborting\n"); 11006 rc = -ENODEV; 11007 goto init_err_disable; 11008 } 11009 11010 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 11011 if (rc) { 11012 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 11013 goto init_err_disable; 11014 } 11015 11016 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 11017 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 11018 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 11019 goto init_err_disable; 11020 } 11021 11022 pci_set_master(pdev); 11023 11024 bp->dev = dev; 11025 bp->pdev = pdev; 11026 11027 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 11028 * determines the BAR size. 11029 */ 11030 bp->bar0 = pci_ioremap_bar(pdev, 0); 11031 if (!bp->bar0) { 11032 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 11033 rc = -ENOMEM; 11034 goto init_err_release; 11035 } 11036 11037 bp->bar2 = pci_ioremap_bar(pdev, 4); 11038 if (!bp->bar2) { 11039 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 11040 rc = -ENOMEM; 11041 goto init_err_release; 11042 } 11043 11044 pci_enable_pcie_error_reporting(pdev); 11045 11046 INIT_WORK(&bp->sp_task, bnxt_sp_task); 11047 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 11048 11049 spin_lock_init(&bp->ntp_fltr_lock); 11050 #if BITS_PER_LONG == 32 11051 spin_lock_init(&bp->db_lock); 11052 #endif 11053 11054 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 11055 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 11056 11057 bnxt_init_dflt_coal(bp); 11058 11059 timer_setup(&bp->timer, bnxt_timer, 0); 11060 bp->current_interval = BNXT_TIMER_INTERVAL; 11061 11062 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 11063 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 11064 11065 clear_bit(BNXT_STATE_OPEN, &bp->state); 11066 return 0; 11067 11068 init_err_release: 11069 bnxt_unmap_bars(bp, pdev); 11070 pci_release_regions(pdev); 11071 11072 init_err_disable: 11073 pci_disable_device(pdev); 11074 11075 init_err: 11076 return rc; 11077 } 11078 11079 /* rtnl_lock held */ 11080 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 11081 { 11082 struct sockaddr *addr = p; 11083 struct bnxt *bp = netdev_priv(dev); 11084 int rc = 0; 11085 11086 if (!is_valid_ether_addr(addr->sa_data)) 11087 return -EADDRNOTAVAIL; 11088 11089 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 11090 return 0; 11091 11092 rc = bnxt_approve_mac(bp, addr->sa_data, true); 11093 if (rc) 11094 return rc; 11095 11096 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 11097 if (netif_running(dev)) { 11098 bnxt_close_nic(bp, false, false); 11099 rc = bnxt_open_nic(bp, false, false); 11100 } 11101 11102 return rc; 11103 } 11104 11105 /* rtnl_lock held */ 11106 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 11107 { 11108 struct bnxt *bp = netdev_priv(dev); 11109 11110 if (netif_running(dev)) 11111 bnxt_close_nic(bp, true, false); 11112 11113 dev->mtu = new_mtu; 11114 bnxt_set_ring_params(bp); 11115 11116 if (netif_running(dev)) 11117 return bnxt_open_nic(bp, true, false); 11118 11119 return 0; 11120 } 11121 11122 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 11123 { 11124 struct bnxt *bp = netdev_priv(dev); 11125 bool sh = false; 11126 int rc; 11127 11128 if (tc > bp->max_tc) { 11129 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 11130 tc, bp->max_tc); 11131 return -EINVAL; 11132 } 11133 11134 if (netdev_get_num_tc(dev) == tc) 11135 return 0; 11136 11137 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 11138 sh = true; 11139 11140 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 11141 sh, tc, bp->tx_nr_rings_xdp); 11142 if (rc) 11143 return rc; 11144 11145 /* Needs to close the device and do hw resource re-allocations */ 11146 if (netif_running(bp->dev)) 11147 bnxt_close_nic(bp, true, false); 11148 11149 if (tc) { 11150 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 11151 netdev_set_num_tc(dev, tc); 11152 } else { 11153 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11154 netdev_reset_tc(dev); 11155 } 11156 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 11157 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 11158 bp->tx_nr_rings + bp->rx_nr_rings; 11159 11160 if (netif_running(bp->dev)) 11161 return bnxt_open_nic(bp, true, false); 11162 11163 return 0; 11164 } 11165 11166 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 11167 void *cb_priv) 11168 { 11169 struct bnxt *bp = cb_priv; 11170 11171 if (!bnxt_tc_flower_enabled(bp) || 11172 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 11173 return -EOPNOTSUPP; 11174 11175 switch (type) { 11176 case TC_SETUP_CLSFLOWER: 11177 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 11178 default: 11179 return -EOPNOTSUPP; 11180 } 11181 } 11182 11183 LIST_HEAD(bnxt_block_cb_list); 11184 11185 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 11186 void *type_data) 11187 { 11188 struct bnxt *bp = netdev_priv(dev); 11189 11190 switch (type) { 11191 case TC_SETUP_BLOCK: 11192 return flow_block_cb_setup_simple(type_data, 11193 &bnxt_block_cb_list, 11194 bnxt_setup_tc_block_cb, 11195 bp, bp, true); 11196 case TC_SETUP_QDISC_MQPRIO: { 11197 struct tc_mqprio_qopt *mqprio = type_data; 11198 11199 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 11200 11201 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 11202 } 11203 default: 11204 return -EOPNOTSUPP; 11205 } 11206 } 11207 11208 #ifdef CONFIG_RFS_ACCEL 11209 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 11210 struct bnxt_ntuple_filter *f2) 11211 { 11212 struct flow_keys *keys1 = &f1->fkeys; 11213 struct flow_keys *keys2 = &f2->fkeys; 11214 11215 if (keys1->basic.n_proto != keys2->basic.n_proto || 11216 keys1->basic.ip_proto != keys2->basic.ip_proto) 11217 return false; 11218 11219 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 11220 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 11221 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst) 11222 return false; 11223 } else { 11224 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src, 11225 sizeof(keys1->addrs.v6addrs.src)) || 11226 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst, 11227 sizeof(keys1->addrs.v6addrs.dst))) 11228 return false; 11229 } 11230 11231 if (keys1->ports.ports == keys2->ports.ports && 11232 keys1->control.flags == keys2->control.flags && 11233 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 11234 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 11235 return true; 11236 11237 return false; 11238 } 11239 11240 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 11241 u16 rxq_index, u32 flow_id) 11242 { 11243 struct bnxt *bp = netdev_priv(dev); 11244 struct bnxt_ntuple_filter *fltr, *new_fltr; 11245 struct flow_keys *fkeys; 11246 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 11247 int rc = 0, idx, bit_id, l2_idx = 0; 11248 struct hlist_head *head; 11249 u32 flags; 11250 11251 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 11252 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 11253 int off = 0, j; 11254 11255 netif_addr_lock_bh(dev); 11256 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 11257 if (ether_addr_equal(eth->h_dest, 11258 vnic->uc_list + off)) { 11259 l2_idx = j + 1; 11260 break; 11261 } 11262 } 11263 netif_addr_unlock_bh(dev); 11264 if (!l2_idx) 11265 return -EINVAL; 11266 } 11267 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 11268 if (!new_fltr) 11269 return -ENOMEM; 11270 11271 fkeys = &new_fltr->fkeys; 11272 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 11273 rc = -EPROTONOSUPPORT; 11274 goto err_free; 11275 } 11276 11277 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 11278 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 11279 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 11280 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 11281 rc = -EPROTONOSUPPORT; 11282 goto err_free; 11283 } 11284 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) && 11285 bp->hwrm_spec_code < 0x10601) { 11286 rc = -EPROTONOSUPPORT; 11287 goto err_free; 11288 } 11289 flags = fkeys->control.flags; 11290 if (((flags & FLOW_DIS_ENCAPSULATION) && 11291 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 11292 rc = -EPROTONOSUPPORT; 11293 goto err_free; 11294 } 11295 11296 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 11297 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 11298 11299 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 11300 head = &bp->ntp_fltr_hash_tbl[idx]; 11301 rcu_read_lock(); 11302 hlist_for_each_entry_rcu(fltr, head, hash) { 11303 if (bnxt_fltr_match(fltr, new_fltr)) { 11304 rcu_read_unlock(); 11305 rc = 0; 11306 goto err_free; 11307 } 11308 } 11309 rcu_read_unlock(); 11310 11311 spin_lock_bh(&bp->ntp_fltr_lock); 11312 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 11313 BNXT_NTP_FLTR_MAX_FLTR, 0); 11314 if (bit_id < 0) { 11315 spin_unlock_bh(&bp->ntp_fltr_lock); 11316 rc = -ENOMEM; 11317 goto err_free; 11318 } 11319 11320 new_fltr->sw_id = (u16)bit_id; 11321 new_fltr->flow_id = flow_id; 11322 new_fltr->l2_fltr_idx = l2_idx; 11323 new_fltr->rxq = rxq_index; 11324 hlist_add_head_rcu(&new_fltr->hash, head); 11325 bp->ntp_fltr_count++; 11326 spin_unlock_bh(&bp->ntp_fltr_lock); 11327 11328 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 11329 bnxt_queue_sp_work(bp); 11330 11331 return new_fltr->sw_id; 11332 11333 err_free: 11334 kfree(new_fltr); 11335 return rc; 11336 } 11337 11338 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 11339 { 11340 int i; 11341 11342 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 11343 struct hlist_head *head; 11344 struct hlist_node *tmp; 11345 struct bnxt_ntuple_filter *fltr; 11346 int rc; 11347 11348 head = &bp->ntp_fltr_hash_tbl[i]; 11349 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 11350 bool del = false; 11351 11352 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 11353 if (rps_may_expire_flow(bp->dev, fltr->rxq, 11354 fltr->flow_id, 11355 fltr->sw_id)) { 11356 bnxt_hwrm_cfa_ntuple_filter_free(bp, 11357 fltr); 11358 del = true; 11359 } 11360 } else { 11361 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 11362 fltr); 11363 if (rc) 11364 del = true; 11365 else 11366 set_bit(BNXT_FLTR_VALID, &fltr->state); 11367 } 11368 11369 if (del) { 11370 spin_lock_bh(&bp->ntp_fltr_lock); 11371 hlist_del_rcu(&fltr->hash); 11372 bp->ntp_fltr_count--; 11373 spin_unlock_bh(&bp->ntp_fltr_lock); 11374 synchronize_rcu(); 11375 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 11376 kfree(fltr); 11377 } 11378 } 11379 } 11380 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 11381 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 11382 } 11383 11384 #else 11385 11386 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 11387 { 11388 } 11389 11390 #endif /* CONFIG_RFS_ACCEL */ 11391 11392 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table) 11393 { 11394 struct bnxt *bp = netdev_priv(netdev); 11395 struct udp_tunnel_info ti; 11396 unsigned int cmd; 11397 11398 udp_tunnel_nic_get_port(netdev, table, 0, &ti); 11399 if (ti.type == UDP_TUNNEL_TYPE_VXLAN) 11400 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 11401 else 11402 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 11403 11404 if (ti.port) 11405 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd); 11406 11407 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 11408 } 11409 11410 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 11411 .sync_table = bnxt_udp_tunnel_sync, 11412 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 11413 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 11414 .tables = { 11415 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 11416 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 11417 }, 11418 }; 11419 11420 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 11421 struct net_device *dev, u32 filter_mask, 11422 int nlflags) 11423 { 11424 struct bnxt *bp = netdev_priv(dev); 11425 11426 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 11427 nlflags, filter_mask, NULL); 11428 } 11429 11430 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 11431 u16 flags, struct netlink_ext_ack *extack) 11432 { 11433 struct bnxt *bp = netdev_priv(dev); 11434 struct nlattr *attr, *br_spec; 11435 int rem, rc = 0; 11436 11437 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 11438 return -EOPNOTSUPP; 11439 11440 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 11441 if (!br_spec) 11442 return -EINVAL; 11443 11444 nla_for_each_nested(attr, br_spec, rem) { 11445 u16 mode; 11446 11447 if (nla_type(attr) != IFLA_BRIDGE_MODE) 11448 continue; 11449 11450 if (nla_len(attr) < sizeof(mode)) 11451 return -EINVAL; 11452 11453 mode = nla_get_u16(attr); 11454 if (mode == bp->br_mode) 11455 break; 11456 11457 rc = bnxt_hwrm_set_br_mode(bp, mode); 11458 if (!rc) 11459 bp->br_mode = mode; 11460 break; 11461 } 11462 return rc; 11463 } 11464 11465 int bnxt_get_port_parent_id(struct net_device *dev, 11466 struct netdev_phys_item_id *ppid) 11467 { 11468 struct bnxt *bp = netdev_priv(dev); 11469 11470 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 11471 return -EOPNOTSUPP; 11472 11473 /* The PF and it's VF-reps only support the switchdev framework */ 11474 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 11475 return -EOPNOTSUPP; 11476 11477 ppid->id_len = sizeof(bp->dsn); 11478 memcpy(ppid->id, bp->dsn, ppid->id_len); 11479 11480 return 0; 11481 } 11482 11483 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev) 11484 { 11485 struct bnxt *bp = netdev_priv(dev); 11486 11487 return &bp->dl_port; 11488 } 11489 11490 static const struct net_device_ops bnxt_netdev_ops = { 11491 .ndo_open = bnxt_open, 11492 .ndo_start_xmit = bnxt_start_xmit, 11493 .ndo_stop = bnxt_close, 11494 .ndo_get_stats64 = bnxt_get_stats64, 11495 .ndo_set_rx_mode = bnxt_set_rx_mode, 11496 .ndo_do_ioctl = bnxt_ioctl, 11497 .ndo_validate_addr = eth_validate_addr, 11498 .ndo_set_mac_address = bnxt_change_mac_addr, 11499 .ndo_change_mtu = bnxt_change_mtu, 11500 .ndo_fix_features = bnxt_fix_features, 11501 .ndo_set_features = bnxt_set_features, 11502 .ndo_tx_timeout = bnxt_tx_timeout, 11503 #ifdef CONFIG_BNXT_SRIOV 11504 .ndo_get_vf_config = bnxt_get_vf_config, 11505 .ndo_set_vf_mac = bnxt_set_vf_mac, 11506 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 11507 .ndo_set_vf_rate = bnxt_set_vf_bw, 11508 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 11509 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 11510 .ndo_set_vf_trust = bnxt_set_vf_trust, 11511 #endif 11512 .ndo_setup_tc = bnxt_setup_tc, 11513 #ifdef CONFIG_RFS_ACCEL 11514 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 11515 #endif 11516 .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, 11517 .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, 11518 .ndo_bpf = bnxt_xdp, 11519 .ndo_xdp_xmit = bnxt_xdp_xmit, 11520 .ndo_bridge_getlink = bnxt_bridge_getlink, 11521 .ndo_bridge_setlink = bnxt_bridge_setlink, 11522 .ndo_get_devlink_port = bnxt_get_devlink_port, 11523 }; 11524 11525 static void bnxt_remove_one(struct pci_dev *pdev) 11526 { 11527 struct net_device *dev = pci_get_drvdata(pdev); 11528 struct bnxt *bp = netdev_priv(dev); 11529 11530 if (BNXT_PF(bp)) 11531 bnxt_sriov_disable(bp); 11532 11533 bnxt_dl_fw_reporters_destroy(bp, true); 11534 if (BNXT_PF(bp)) 11535 devlink_port_type_clear(&bp->dl_port); 11536 pci_disable_pcie_error_reporting(pdev); 11537 unregister_netdev(dev); 11538 bnxt_dl_unregister(bp); 11539 bnxt_shutdown_tc(bp); 11540 bnxt_cancel_sp_work(bp); 11541 bp->sp_event = 0; 11542 11543 bnxt_clear_int_mode(bp); 11544 bnxt_hwrm_func_drv_unrgtr(bp); 11545 bnxt_free_hwrm_resources(bp); 11546 bnxt_free_hwrm_short_cmd_req(bp); 11547 bnxt_ethtool_free(bp); 11548 bnxt_dcb_free(bp); 11549 kfree(bp->edev); 11550 bp->edev = NULL; 11551 kfree(bp->fw_health); 11552 bp->fw_health = NULL; 11553 bnxt_cleanup_pci(bp); 11554 bnxt_free_ctx_mem(bp); 11555 kfree(bp->ctx); 11556 bp->ctx = NULL; 11557 kfree(bp->rss_indir_tbl); 11558 bp->rss_indir_tbl = NULL; 11559 bnxt_free_port_stats(bp); 11560 free_netdev(dev); 11561 } 11562 11563 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 11564 { 11565 int rc = 0; 11566 struct bnxt_link_info *link_info = &bp->link_info; 11567 11568 rc = bnxt_hwrm_phy_qcaps(bp); 11569 if (rc) { 11570 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 11571 rc); 11572 return rc; 11573 } 11574 if (!fw_dflt) 11575 return 0; 11576 11577 rc = bnxt_update_link(bp, false); 11578 if (rc) { 11579 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 11580 rc); 11581 return rc; 11582 } 11583 11584 /* Older firmware does not have supported_auto_speeds, so assume 11585 * that all supported speeds can be autonegotiated. 11586 */ 11587 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 11588 link_info->support_auto_speeds = link_info->support_speeds; 11589 11590 bnxt_init_ethtool_link_settings(bp); 11591 return 0; 11592 } 11593 11594 static int bnxt_get_max_irq(struct pci_dev *pdev) 11595 { 11596 u16 ctrl; 11597 11598 if (!pdev->msix_cap) 11599 return 1; 11600 11601 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 11602 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 11603 } 11604 11605 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 11606 int *max_cp) 11607 { 11608 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 11609 int max_ring_grps = 0, max_irq; 11610 11611 *max_tx = hw_resc->max_tx_rings; 11612 *max_rx = hw_resc->max_rx_rings; 11613 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 11614 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 11615 bnxt_get_ulp_msix_num(bp), 11616 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp)); 11617 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) 11618 *max_cp = min_t(int, *max_cp, max_irq); 11619 max_ring_grps = hw_resc->max_hw_ring_grps; 11620 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 11621 *max_cp -= 1; 11622 *max_rx -= 2; 11623 } 11624 if (bp->flags & BNXT_FLAG_AGG_RINGS) 11625 *max_rx >>= 1; 11626 if (bp->flags & BNXT_FLAG_CHIP_P5) { 11627 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 11628 /* On P5 chips, max_cp output param should be available NQs */ 11629 *max_cp = max_irq; 11630 } 11631 *max_rx = min_t(int, *max_rx, max_ring_grps); 11632 } 11633 11634 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 11635 { 11636 int rx, tx, cp; 11637 11638 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 11639 *max_rx = rx; 11640 *max_tx = tx; 11641 if (!rx || !tx || !cp) 11642 return -ENOMEM; 11643 11644 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 11645 } 11646 11647 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 11648 bool shared) 11649 { 11650 int rc; 11651 11652 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 11653 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 11654 /* Not enough rings, try disabling agg rings. */ 11655 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 11656 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 11657 if (rc) { 11658 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 11659 bp->flags |= BNXT_FLAG_AGG_RINGS; 11660 return rc; 11661 } 11662 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 11663 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 11664 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 11665 bnxt_set_ring_params(bp); 11666 } 11667 11668 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 11669 int max_cp, max_stat, max_irq; 11670 11671 /* Reserve minimum resources for RoCE */ 11672 max_cp = bnxt_get_max_func_cp_rings(bp); 11673 max_stat = bnxt_get_max_func_stat_ctxs(bp); 11674 max_irq = bnxt_get_max_func_irqs(bp); 11675 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 11676 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 11677 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 11678 return 0; 11679 11680 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 11681 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 11682 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 11683 max_cp = min_t(int, max_cp, max_irq); 11684 max_cp = min_t(int, max_cp, max_stat); 11685 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 11686 if (rc) 11687 rc = 0; 11688 } 11689 return rc; 11690 } 11691 11692 /* In initial default shared ring setting, each shared ring must have a 11693 * RX/TX ring pair. 11694 */ 11695 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 11696 { 11697 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 11698 bp->rx_nr_rings = bp->cp_nr_rings; 11699 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 11700 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11701 } 11702 11703 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 11704 { 11705 int dflt_rings, max_rx_rings, max_tx_rings, rc; 11706 11707 if (!bnxt_can_reserve_rings(bp)) 11708 return 0; 11709 11710 if (sh) 11711 bp->flags |= BNXT_FLAG_SHARED_RINGS; 11712 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 11713 /* Reduce default rings on multi-port cards so that total default 11714 * rings do not exceed CPU count. 11715 */ 11716 if (bp->port_count > 1) { 11717 int max_rings = 11718 max_t(int, num_online_cpus() / bp->port_count, 1); 11719 11720 dflt_rings = min_t(int, dflt_rings, max_rings); 11721 } 11722 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 11723 if (rc) 11724 return rc; 11725 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 11726 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 11727 if (sh) 11728 bnxt_trim_dflt_sh_rings(bp); 11729 else 11730 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 11731 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 11732 11733 rc = __bnxt_reserve_rings(bp); 11734 if (rc) 11735 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 11736 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 11737 if (sh) 11738 bnxt_trim_dflt_sh_rings(bp); 11739 11740 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 11741 if (bnxt_need_reserve_rings(bp)) { 11742 rc = __bnxt_reserve_rings(bp); 11743 if (rc) 11744 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 11745 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 11746 } 11747 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 11748 bp->rx_nr_rings++; 11749 bp->cp_nr_rings++; 11750 } 11751 if (rc) { 11752 bp->tx_nr_rings = 0; 11753 bp->rx_nr_rings = 0; 11754 } 11755 return rc; 11756 } 11757 11758 static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 11759 { 11760 int rc; 11761 11762 if (bp->tx_nr_rings) 11763 return 0; 11764 11765 bnxt_ulp_irq_stop(bp); 11766 bnxt_clear_int_mode(bp); 11767 rc = bnxt_set_dflt_rings(bp, true); 11768 if (rc) { 11769 netdev_err(bp->dev, "Not enough rings available.\n"); 11770 goto init_dflt_ring_err; 11771 } 11772 rc = bnxt_init_int_mode(bp); 11773 if (rc) 11774 goto init_dflt_ring_err; 11775 11776 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 11777 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) { 11778 bp->flags |= BNXT_FLAG_RFS; 11779 bp->dev->features |= NETIF_F_NTUPLE; 11780 } 11781 init_dflt_ring_err: 11782 bnxt_ulp_irq_restart(bp, rc); 11783 return rc; 11784 } 11785 11786 int bnxt_restore_pf_fw_resources(struct bnxt *bp) 11787 { 11788 int rc; 11789 11790 ASSERT_RTNL(); 11791 bnxt_hwrm_func_qcaps(bp); 11792 11793 if (netif_running(bp->dev)) 11794 __bnxt_close_nic(bp, true, false); 11795 11796 bnxt_ulp_irq_stop(bp); 11797 bnxt_clear_int_mode(bp); 11798 rc = bnxt_init_int_mode(bp); 11799 bnxt_ulp_irq_restart(bp, rc); 11800 11801 if (netif_running(bp->dev)) { 11802 if (rc) 11803 dev_close(bp->dev); 11804 else 11805 rc = bnxt_open_nic(bp, true, false); 11806 } 11807 11808 return rc; 11809 } 11810 11811 static int bnxt_init_mac_addr(struct bnxt *bp) 11812 { 11813 int rc = 0; 11814 11815 if (BNXT_PF(bp)) { 11816 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN); 11817 } else { 11818 #ifdef CONFIG_BNXT_SRIOV 11819 struct bnxt_vf_info *vf = &bp->vf; 11820 bool strict_approval = true; 11821 11822 if (is_valid_ether_addr(vf->mac_addr)) { 11823 /* overwrite netdev dev_addr with admin VF MAC */ 11824 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 11825 /* Older PF driver or firmware may not approve this 11826 * correctly. 11827 */ 11828 strict_approval = false; 11829 } else { 11830 eth_hw_addr_random(bp->dev); 11831 } 11832 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 11833 #endif 11834 } 11835 return rc; 11836 } 11837 11838 #define BNXT_VPD_LEN 512 11839 static void bnxt_vpd_read_info(struct bnxt *bp) 11840 { 11841 struct pci_dev *pdev = bp->pdev; 11842 int i, len, pos, ro_size; 11843 ssize_t vpd_size; 11844 u8 *vpd_data; 11845 11846 vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL); 11847 if (!vpd_data) 11848 return; 11849 11850 vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data); 11851 if (vpd_size <= 0) { 11852 netdev_err(bp->dev, "Unable to read VPD\n"); 11853 goto exit; 11854 } 11855 11856 i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 11857 if (i < 0) { 11858 netdev_err(bp->dev, "VPD READ-Only not found\n"); 11859 goto exit; 11860 } 11861 11862 ro_size = pci_vpd_lrdt_size(&vpd_data[i]); 11863 i += PCI_VPD_LRDT_TAG_SIZE; 11864 if (i + ro_size > vpd_size) 11865 goto exit; 11866 11867 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 11868 PCI_VPD_RO_KEYWORD_PARTNO); 11869 if (pos < 0) 11870 goto read_sn; 11871 11872 len = pci_vpd_info_field_size(&vpd_data[pos]); 11873 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 11874 if (len + pos > vpd_size) 11875 goto read_sn; 11876 11877 strlcpy(bp->board_partno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN)); 11878 11879 read_sn: 11880 pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size, 11881 PCI_VPD_RO_KEYWORD_SERIALNO); 11882 if (pos < 0) 11883 goto exit; 11884 11885 len = pci_vpd_info_field_size(&vpd_data[pos]); 11886 pos += PCI_VPD_INFO_FLD_HDR_SIZE; 11887 if (len + pos > vpd_size) 11888 goto exit; 11889 11890 strlcpy(bp->board_serialno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN)); 11891 exit: 11892 kfree(vpd_data); 11893 } 11894 11895 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 11896 { 11897 struct pci_dev *pdev = bp->pdev; 11898 u64 qword; 11899 11900 qword = pci_get_dsn(pdev); 11901 if (!qword) { 11902 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 11903 return -EOPNOTSUPP; 11904 } 11905 11906 put_unaligned_le64(qword, dsn); 11907 11908 bp->flags |= BNXT_FLAG_DSN_VALID; 11909 return 0; 11910 } 11911 11912 static int bnxt_map_db_bar(struct bnxt *bp) 11913 { 11914 if (!bp->db_size) 11915 return -ENODEV; 11916 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 11917 if (!bp->bar1) 11918 return -ENOMEM; 11919 return 0; 11920 } 11921 11922 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 11923 { 11924 struct net_device *dev; 11925 struct bnxt *bp; 11926 int rc, max_irqs; 11927 11928 if (pci_is_bridge(pdev)) 11929 return -ENODEV; 11930 11931 /* Clear any pending DMA transactions from crash kernel 11932 * while loading driver in capture kernel. 11933 */ 11934 if (is_kdump_kernel()) { 11935 pci_clear_master(pdev); 11936 pcie_flr(pdev); 11937 } 11938 11939 max_irqs = bnxt_get_max_irq(pdev); 11940 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 11941 if (!dev) 11942 return -ENOMEM; 11943 11944 bp = netdev_priv(dev); 11945 bnxt_set_max_func_irqs(bp, max_irqs); 11946 11947 if (bnxt_vf_pciid(ent->driver_data)) 11948 bp->flags |= BNXT_FLAG_VF; 11949 11950 if (pdev->msix_cap) 11951 bp->flags |= BNXT_FLAG_MSIX_CAP; 11952 11953 rc = bnxt_init_board(pdev, dev); 11954 if (rc < 0) 11955 goto init_err_free; 11956 11957 dev->netdev_ops = &bnxt_netdev_ops; 11958 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 11959 dev->ethtool_ops = &bnxt_ethtool_ops; 11960 pci_set_drvdata(pdev, dev); 11961 11962 if (BNXT_PF(bp)) 11963 bnxt_vpd_read_info(bp); 11964 11965 rc = bnxt_alloc_hwrm_resources(bp); 11966 if (rc) 11967 goto init_err_pci_clean; 11968 11969 mutex_init(&bp->hwrm_cmd_lock); 11970 mutex_init(&bp->link_lock); 11971 11972 rc = bnxt_fw_init_one_p1(bp); 11973 if (rc) 11974 goto init_err_pci_clean; 11975 11976 if (BNXT_CHIP_P5(bp)) 11977 bp->flags |= BNXT_FLAG_CHIP_P5; 11978 11979 rc = bnxt_fw_init_one_p2(bp); 11980 if (rc) 11981 goto init_err_pci_clean; 11982 11983 rc = bnxt_map_db_bar(bp); 11984 if (rc) { 11985 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 11986 rc); 11987 goto init_err_pci_clean; 11988 } 11989 11990 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 11991 NETIF_F_TSO | NETIF_F_TSO6 | 11992 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 11993 NETIF_F_GSO_IPXIP4 | 11994 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 11995 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 11996 NETIF_F_RXCSUM | NETIF_F_GRO; 11997 11998 if (BNXT_SUPPORTS_TPA(bp)) 11999 dev->hw_features |= NETIF_F_LRO; 12000 12001 dev->hw_enc_features = 12002 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 12003 NETIF_F_TSO | NETIF_F_TSO6 | 12004 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 12005 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 12006 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 12007 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 12008 12009 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 12010 NETIF_F_GSO_GRE_CSUM; 12011 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 12012 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 12013 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 12014 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 12015 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 12016 if (BNXT_SUPPORTS_TPA(bp)) 12017 dev->hw_features |= NETIF_F_GRO_HW; 12018 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 12019 if (dev->features & NETIF_F_GRO_HW) 12020 dev->features &= ~NETIF_F_LRO; 12021 dev->priv_flags |= IFF_UNICAST_FLT; 12022 12023 #ifdef CONFIG_BNXT_SRIOV 12024 init_waitqueue_head(&bp->sriov_cfg_wait); 12025 mutex_init(&bp->sriov_lock); 12026 #endif 12027 if (BNXT_SUPPORTS_TPA(bp)) { 12028 bp->gro_func = bnxt_gro_func_5730x; 12029 if (BNXT_CHIP_P4(bp)) 12030 bp->gro_func = bnxt_gro_func_5731x; 12031 else if (BNXT_CHIP_P5(bp)) 12032 bp->gro_func = bnxt_gro_func_5750x; 12033 } 12034 if (!BNXT_CHIP_P4_PLUS(bp)) 12035 bp->flags |= BNXT_FLAG_DOUBLE_DB; 12036 12037 bp->ulp_probe = bnxt_ulp_probe; 12038 12039 rc = bnxt_init_mac_addr(bp); 12040 if (rc) { 12041 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 12042 rc = -EADDRNOTAVAIL; 12043 goto init_err_pci_clean; 12044 } 12045 12046 if (BNXT_PF(bp)) { 12047 /* Read the adapter's DSN to use as the eswitch switch_id */ 12048 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 12049 } 12050 12051 /* MTU range: 60 - FW defined max */ 12052 dev->min_mtu = ETH_ZLEN; 12053 dev->max_mtu = bp->max_mtu; 12054 12055 rc = bnxt_probe_phy(bp, true); 12056 if (rc) 12057 goto init_err_pci_clean; 12058 12059 bnxt_set_rx_skb_mode(bp, false); 12060 bnxt_set_tpa_flags(bp); 12061 bnxt_set_ring_params(bp); 12062 rc = bnxt_set_dflt_rings(bp, true); 12063 if (rc) { 12064 netdev_err(bp->dev, "Not enough rings available.\n"); 12065 rc = -ENOMEM; 12066 goto init_err_pci_clean; 12067 } 12068 12069 bnxt_fw_init_one_p3(bp); 12070 12071 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 12072 bp->flags |= BNXT_FLAG_STRIP_VLAN; 12073 12074 rc = bnxt_init_int_mode(bp); 12075 if (rc) 12076 goto init_err_pci_clean; 12077 12078 /* No TC has been set yet and rings may have been trimmed due to 12079 * limited MSIX, so we re-initialize the TX rings per TC. 12080 */ 12081 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 12082 12083 rc = bnxt_alloc_rss_indir_tbl(bp); 12084 if (rc) 12085 goto init_err_pci_clean; 12086 bnxt_set_dflt_rss_indir_tbl(bp); 12087 12088 if (BNXT_PF(bp)) { 12089 if (!bnxt_pf_wq) { 12090 bnxt_pf_wq = 12091 create_singlethread_workqueue("bnxt_pf_wq"); 12092 if (!bnxt_pf_wq) { 12093 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 12094 goto init_err_pci_clean; 12095 } 12096 } 12097 rc = bnxt_init_tc(bp); 12098 if (rc) 12099 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 12100 rc); 12101 } 12102 12103 bnxt_dl_register(bp); 12104 12105 rc = register_netdev(dev); 12106 if (rc) 12107 goto init_err_cleanup; 12108 12109 if (BNXT_PF(bp)) 12110 devlink_port_type_eth_set(&bp->dl_port, bp->dev); 12111 bnxt_dl_fw_reporters_create(bp); 12112 12113 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 12114 board_info[ent->driver_data].name, 12115 (long)pci_resource_start(pdev, 0), dev->dev_addr); 12116 pcie_print_link_status(pdev); 12117 12118 return 0; 12119 12120 init_err_cleanup: 12121 bnxt_dl_unregister(bp); 12122 bnxt_shutdown_tc(bp); 12123 bnxt_clear_int_mode(bp); 12124 12125 init_err_pci_clean: 12126 bnxt_hwrm_func_drv_unrgtr(bp); 12127 bnxt_free_hwrm_short_cmd_req(bp); 12128 bnxt_free_hwrm_resources(bp); 12129 kfree(bp->fw_health); 12130 bp->fw_health = NULL; 12131 bnxt_cleanup_pci(bp); 12132 bnxt_free_ctx_mem(bp); 12133 kfree(bp->ctx); 12134 bp->ctx = NULL; 12135 kfree(bp->rss_indir_tbl); 12136 bp->rss_indir_tbl = NULL; 12137 12138 init_err_free: 12139 free_netdev(dev); 12140 return rc; 12141 } 12142 12143 static void bnxt_shutdown(struct pci_dev *pdev) 12144 { 12145 struct net_device *dev = pci_get_drvdata(pdev); 12146 struct bnxt *bp; 12147 12148 if (!dev) 12149 return; 12150 12151 rtnl_lock(); 12152 bp = netdev_priv(dev); 12153 if (!bp) 12154 goto shutdown_exit; 12155 12156 if (netif_running(dev)) 12157 dev_close(dev); 12158 12159 bnxt_ulp_shutdown(bp); 12160 bnxt_clear_int_mode(bp); 12161 pci_disable_device(pdev); 12162 12163 if (system_state == SYSTEM_POWER_OFF) { 12164 pci_wake_from_d3(pdev, bp->wol); 12165 pci_set_power_state(pdev, PCI_D3hot); 12166 } 12167 12168 shutdown_exit: 12169 rtnl_unlock(); 12170 } 12171 12172 #ifdef CONFIG_PM_SLEEP 12173 static int bnxt_suspend(struct device *device) 12174 { 12175 struct net_device *dev = dev_get_drvdata(device); 12176 struct bnxt *bp = netdev_priv(dev); 12177 int rc = 0; 12178 12179 rtnl_lock(); 12180 bnxt_ulp_stop(bp); 12181 if (netif_running(dev)) { 12182 netif_device_detach(dev); 12183 rc = bnxt_close(dev); 12184 } 12185 bnxt_hwrm_func_drv_unrgtr(bp); 12186 pci_disable_device(bp->pdev); 12187 bnxt_free_ctx_mem(bp); 12188 kfree(bp->ctx); 12189 bp->ctx = NULL; 12190 rtnl_unlock(); 12191 return rc; 12192 } 12193 12194 static int bnxt_resume(struct device *device) 12195 { 12196 struct net_device *dev = dev_get_drvdata(device); 12197 struct bnxt *bp = netdev_priv(dev); 12198 int rc = 0; 12199 12200 rtnl_lock(); 12201 rc = pci_enable_device(bp->pdev); 12202 if (rc) { 12203 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 12204 rc); 12205 goto resume_exit; 12206 } 12207 pci_set_master(bp->pdev); 12208 if (bnxt_hwrm_ver_get(bp)) { 12209 rc = -ENODEV; 12210 goto resume_exit; 12211 } 12212 rc = bnxt_hwrm_func_reset(bp); 12213 if (rc) { 12214 rc = -EBUSY; 12215 goto resume_exit; 12216 } 12217 12218 rc = bnxt_hwrm_func_qcaps(bp); 12219 if (rc) 12220 goto resume_exit; 12221 12222 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 12223 rc = -ENODEV; 12224 goto resume_exit; 12225 } 12226 12227 bnxt_get_wol_settings(bp); 12228 if (netif_running(dev)) { 12229 rc = bnxt_open(dev); 12230 if (!rc) 12231 netif_device_attach(dev); 12232 } 12233 12234 resume_exit: 12235 bnxt_ulp_start(bp, rc); 12236 if (!rc) 12237 bnxt_reenable_sriov(bp); 12238 rtnl_unlock(); 12239 return rc; 12240 } 12241 12242 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 12243 #define BNXT_PM_OPS (&bnxt_pm_ops) 12244 12245 #else 12246 12247 #define BNXT_PM_OPS NULL 12248 12249 #endif /* CONFIG_PM_SLEEP */ 12250 12251 /** 12252 * bnxt_io_error_detected - called when PCI error is detected 12253 * @pdev: Pointer to PCI device 12254 * @state: The current pci connection state 12255 * 12256 * This function is called after a PCI bus error affecting 12257 * this device has been detected. 12258 */ 12259 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 12260 pci_channel_state_t state) 12261 { 12262 struct net_device *netdev = pci_get_drvdata(pdev); 12263 struct bnxt *bp = netdev_priv(netdev); 12264 12265 netdev_info(netdev, "PCI I/O error detected\n"); 12266 12267 rtnl_lock(); 12268 netif_device_detach(netdev); 12269 12270 bnxt_ulp_stop(bp); 12271 12272 if (state == pci_channel_io_perm_failure) { 12273 rtnl_unlock(); 12274 return PCI_ERS_RESULT_DISCONNECT; 12275 } 12276 12277 if (netif_running(netdev)) 12278 bnxt_close(netdev); 12279 12280 pci_disable_device(pdev); 12281 bnxt_free_ctx_mem(bp); 12282 kfree(bp->ctx); 12283 bp->ctx = NULL; 12284 rtnl_unlock(); 12285 12286 /* Request a slot slot reset. */ 12287 return PCI_ERS_RESULT_NEED_RESET; 12288 } 12289 12290 /** 12291 * bnxt_io_slot_reset - called after the pci bus has been reset. 12292 * @pdev: Pointer to PCI device 12293 * 12294 * Restart the card from scratch, as if from a cold-boot. 12295 * At this point, the card has exprienced a hard reset, 12296 * followed by fixups by BIOS, and has its config space 12297 * set up identically to what it was at cold boot. 12298 */ 12299 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 12300 { 12301 struct net_device *netdev = pci_get_drvdata(pdev); 12302 struct bnxt *bp = netdev_priv(netdev); 12303 int err = 0; 12304 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 12305 12306 netdev_info(bp->dev, "PCI Slot Reset\n"); 12307 12308 rtnl_lock(); 12309 12310 if (pci_enable_device(pdev)) { 12311 dev_err(&pdev->dev, 12312 "Cannot re-enable PCI device after reset.\n"); 12313 } else { 12314 pci_set_master(pdev); 12315 12316 err = bnxt_hwrm_func_reset(bp); 12317 if (!err) { 12318 err = bnxt_hwrm_func_qcaps(bp); 12319 if (!err && netif_running(netdev)) 12320 err = bnxt_open(netdev); 12321 } 12322 bnxt_ulp_start(bp, err); 12323 if (!err) { 12324 bnxt_reenable_sriov(bp); 12325 result = PCI_ERS_RESULT_RECOVERED; 12326 } 12327 } 12328 12329 if (result != PCI_ERS_RESULT_RECOVERED) { 12330 if (netif_running(netdev)) 12331 dev_close(netdev); 12332 pci_disable_device(pdev); 12333 } 12334 12335 rtnl_unlock(); 12336 12337 return result; 12338 } 12339 12340 /** 12341 * bnxt_io_resume - called when traffic can start flowing again. 12342 * @pdev: Pointer to PCI device 12343 * 12344 * This callback is called when the error recovery driver tells 12345 * us that its OK to resume normal operation. 12346 */ 12347 static void bnxt_io_resume(struct pci_dev *pdev) 12348 { 12349 struct net_device *netdev = pci_get_drvdata(pdev); 12350 12351 rtnl_lock(); 12352 12353 netif_device_attach(netdev); 12354 12355 rtnl_unlock(); 12356 } 12357 12358 static const struct pci_error_handlers bnxt_err_handler = { 12359 .error_detected = bnxt_io_error_detected, 12360 .slot_reset = bnxt_io_slot_reset, 12361 .resume = bnxt_io_resume 12362 }; 12363 12364 static struct pci_driver bnxt_pci_driver = { 12365 .name = DRV_MODULE_NAME, 12366 .id_table = bnxt_pci_tbl, 12367 .probe = bnxt_init_one, 12368 .remove = bnxt_remove_one, 12369 .shutdown = bnxt_shutdown, 12370 .driver.pm = BNXT_PM_OPS, 12371 .err_handler = &bnxt_err_handler, 12372 #if defined(CONFIG_BNXT_SRIOV) 12373 .sriov_configure = bnxt_sriov_configure, 12374 #endif 12375 }; 12376 12377 static int __init bnxt_init(void) 12378 { 12379 bnxt_debug_init(); 12380 return pci_register_driver(&bnxt_pci_driver); 12381 } 12382 12383 static void __exit bnxt_exit(void) 12384 { 12385 pci_unregister_driver(&bnxt_pci_driver); 12386 if (bnxt_pf_wq) 12387 destroy_workqueue(bnxt_pf_wq); 12388 bnxt_debug_exit(); 12389 } 12390 12391 module_init(bnxt_init); 12392 module_exit(bnxt_exit); 12393