1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 10 #include <linux/module.h> 11 12 #include <linux/stringify.h> 13 #include <linux/kernel.h> 14 #include <linux/timer.h> 15 #include <linux/errno.h> 16 #include <linux/ioport.h> 17 #include <linux/slab.h> 18 #include <linux/vmalloc.h> 19 #include <linux/interrupt.h> 20 #include <linux/pci.h> 21 #include <linux/netdevice.h> 22 #include <linux/etherdevice.h> 23 #include <linux/skbuff.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/bitops.h> 26 #include <linux/io.h> 27 #include <linux/irq.h> 28 #include <linux/delay.h> 29 #include <asm/byteorder.h> 30 #include <asm/page.h> 31 #include <linux/time.h> 32 #include <linux/mii.h> 33 #include <linux/if.h> 34 #include <linux/if_vlan.h> 35 #include <net/ip.h> 36 #include <net/tcp.h> 37 #include <net/udp.h> 38 #include <net/checksum.h> 39 #include <net/ip6_checksum.h> 40 #include <net/udp_tunnel.h> 41 #ifdef CONFIG_NET_RX_BUSY_POLL 42 #include <net/busy_poll.h> 43 #endif 44 #include <linux/workqueue.h> 45 #include <linux/prefetch.h> 46 #include <linux/cache.h> 47 #include <linux/log2.h> 48 #include <linux/aer.h> 49 #include <linux/bitmap.h> 50 #include <linux/cpu_rmap.h> 51 52 #include "bnxt_hsi.h" 53 #include "bnxt.h" 54 #include "bnxt_sriov.h" 55 #include "bnxt_ethtool.h" 56 57 #define BNXT_TX_TIMEOUT (5 * HZ) 58 59 static const char version[] = 60 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; 61 62 MODULE_LICENSE("GPL"); 63 MODULE_DESCRIPTION("Broadcom BCM573xx network driver"); 64 MODULE_VERSION(DRV_MODULE_VERSION); 65 66 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 67 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD 68 #define BNXT_RX_COPY_THRESH 256 69 70 #define BNXT_TX_PUSH_THRESH 164 71 72 enum board_idx { 73 BCM57301, 74 BCM57302, 75 BCM57304, 76 BCM57417_NPAR, 77 BCM58700, 78 BCM57311, 79 BCM57312, 80 BCM57402, 81 BCM57404, 82 BCM57406, 83 BCM57402_NPAR, 84 BCM57407, 85 BCM57412, 86 BCM57414, 87 BCM57416, 88 BCM57417, 89 BCM57412_NPAR, 90 BCM57314, 91 BCM57417_SFP, 92 BCM57416_SFP, 93 BCM57404_NPAR, 94 BCM57406_NPAR, 95 BCM57407_SFP, 96 BCM57414_NPAR, 97 BCM57416_NPAR, 98 BCM57304_VF, 99 BCM57404_VF, 100 BCM57414_VF, 101 BCM57314_VF, 102 }; 103 104 /* indexed by enum above */ 105 static const struct { 106 char *name; 107 } board_info[] = { 108 { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" }, 109 { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" }, 110 { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" }, 111 { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 112 { "Broadcom BCM58700 Nitro 4-port 1Gb/2.5Gb/10Gb Ethernet" }, 113 { "Broadcom BCM57311 NetXtreme-C Single-port 10Gb Ethernet" }, 114 { "Broadcom BCM57312 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" }, 115 { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" }, 116 { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" }, 117 { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" }, 118 { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 119 { "Broadcom BCM57407 NetXtreme-E Dual-port 10GBase-T Ethernet" }, 120 { "Broadcom BCM57412 NetXtreme-E Dual-port 10Gb Ethernet" }, 121 { "Broadcom BCM57414 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" }, 122 { "Broadcom BCM57416 NetXtreme-E Dual-port 10GBase-T Ethernet" }, 123 { "Broadcom BCM57417 NetXtreme-E Dual-port 10GBase-T Ethernet" }, 124 { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 125 { "Broadcom BCM57314 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" }, 126 { "Broadcom BCM57417 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" }, 127 { "Broadcom BCM57416 NetXtreme-E Dual-port 10Gb Ethernet" }, 128 { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 129 { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 130 { "Broadcom BCM57407 NetXtreme-E Dual-port 25Gb Ethernet" }, 131 { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 132 { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 133 { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" }, 134 { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" }, 135 { "Broadcom BCM57414 NetXtreme-E Ethernet Virtual Function" }, 136 { "Broadcom BCM57314 NetXtreme-E Ethernet Virtual Function" }, 137 }; 138 139 static const struct pci_device_id bnxt_pci_tbl[] = { 140 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 141 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 142 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 143 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 144 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 145 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 146 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 147 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 148 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 149 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 150 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 151 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 152 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 153 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 154 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 155 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 156 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 157 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 158 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 159 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 160 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 161 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 162 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 163 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 165 #ifdef CONFIG_BNXT_SRIOV 166 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF }, 167 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF }, 168 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = BCM57414_VF }, 169 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = BCM57314_VF }, 170 #endif 171 { 0 } 172 }; 173 174 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 175 176 static const u16 bnxt_vf_req_snif[] = { 177 HWRM_FUNC_CFG, 178 HWRM_PORT_PHY_QCFG, 179 HWRM_CFA_L2_FILTER_ALLOC, 180 }; 181 182 static const u16 bnxt_async_events_arr[] = { 183 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 184 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 185 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 186 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 187 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 188 }; 189 190 static bool bnxt_vf_pciid(enum board_idx idx) 191 { 192 return (idx == BCM57304_VF || idx == BCM57404_VF || 193 idx == BCM57314_VF || idx == BCM57414_VF); 194 } 195 196 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 197 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 198 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS) 199 200 #define BNXT_CP_DB_REARM(db, raw_cons) \ 201 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db) 202 203 #define BNXT_CP_DB(db, raw_cons) \ 204 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db) 205 206 #define BNXT_CP_DB_IRQ_DIS(db) \ 207 writel(DB_CP_IRQ_DIS_FLAGS, db) 208 209 static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 210 { 211 /* Tell compiler to fetch tx indices from memory. */ 212 barrier(); 213 214 return bp->tx_ring_size - 215 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask); 216 } 217 218 static const u16 bnxt_lhint_arr[] = { 219 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 220 TX_BD_FLAGS_LHINT_512_TO_1023, 221 TX_BD_FLAGS_LHINT_1024_TO_2047, 222 TX_BD_FLAGS_LHINT_1024_TO_2047, 223 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 224 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 225 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 226 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 227 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 228 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 229 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 230 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 231 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 232 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 233 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 234 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 235 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 236 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 237 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 238 }; 239 240 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 241 { 242 struct bnxt *bp = netdev_priv(dev); 243 struct tx_bd *txbd; 244 struct tx_bd_ext *txbd1; 245 struct netdev_queue *txq; 246 int i; 247 dma_addr_t mapping; 248 unsigned int length, pad = 0; 249 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 250 u16 prod, last_frag; 251 struct pci_dev *pdev = bp->pdev; 252 struct bnxt_tx_ring_info *txr; 253 struct bnxt_sw_tx_bd *tx_buf; 254 255 i = skb_get_queue_mapping(skb); 256 if (unlikely(i >= bp->tx_nr_rings)) { 257 dev_kfree_skb_any(skb); 258 return NETDEV_TX_OK; 259 } 260 261 txr = &bp->tx_ring[i]; 262 txq = netdev_get_tx_queue(dev, i); 263 prod = txr->tx_prod; 264 265 free_size = bnxt_tx_avail(bp, txr); 266 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 267 netif_tx_stop_queue(txq); 268 return NETDEV_TX_BUSY; 269 } 270 271 length = skb->len; 272 len = skb_headlen(skb); 273 last_frag = skb_shinfo(skb)->nr_frags; 274 275 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 276 277 txbd->tx_bd_opaque = prod; 278 279 tx_buf = &txr->tx_buf_ring[prod]; 280 tx_buf->skb = skb; 281 tx_buf->nr_frags = last_frag; 282 283 vlan_tag_flags = 0; 284 cfa_action = 0; 285 if (skb_vlan_tag_present(skb)) { 286 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 287 skb_vlan_tag_get(skb); 288 /* Currently supports 8021Q, 8021AD vlan offloads 289 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 290 */ 291 if (skb->vlan_proto == htons(ETH_P_8021Q)) 292 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 293 } 294 295 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { 296 struct tx_push_buffer *tx_push_buf = txr->tx_push; 297 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 298 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 299 void *pdata = tx_push_buf->data; 300 u64 *end; 301 int j, push_len; 302 303 /* Set COAL_NOW to be ready quickly for the next push */ 304 tx_push->tx_bd_len_flags_type = 305 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 306 TX_BD_TYPE_LONG_TX_BD | 307 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 308 TX_BD_FLAGS_COAL_NOW | 309 TX_BD_FLAGS_PACKET_END | 310 (2 << TX_BD_FLAGS_BD_CNT_SHIFT)); 311 312 if (skb->ip_summed == CHECKSUM_PARTIAL) 313 tx_push1->tx_bd_hsize_lflags = 314 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 315 else 316 tx_push1->tx_bd_hsize_lflags = 0; 317 318 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 319 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action); 320 321 end = pdata + length; 322 end = PTR_ALIGN(end, 8) - 1; 323 *end = 0; 324 325 skb_copy_from_linear_data(skb, pdata, len); 326 pdata += len; 327 for (j = 0; j < last_frag; j++) { 328 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 329 void *fptr; 330 331 fptr = skb_frag_address_safe(frag); 332 if (!fptr) 333 goto normal_tx; 334 335 memcpy(pdata, fptr, skb_frag_size(frag)); 336 pdata += skb_frag_size(frag); 337 } 338 339 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 340 txbd->tx_bd_haddr = txr->data_mapping; 341 prod = NEXT_TX(prod); 342 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 343 memcpy(txbd, tx_push1, sizeof(*txbd)); 344 prod = NEXT_TX(prod); 345 tx_push->doorbell = 346 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod); 347 txr->tx_prod = prod; 348 349 tx_buf->is_push = 1; 350 netdev_tx_sent_queue(txq, skb->len); 351 wmb(); /* Sync is_push and byte queue before pushing data */ 352 353 push_len = (length + sizeof(*tx_push) + 7) / 8; 354 if (push_len > 16) { 355 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16); 356 __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1, 357 push_len - 16); 358 } else { 359 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 360 push_len); 361 } 362 363 goto tx_done; 364 } 365 366 normal_tx: 367 if (length < BNXT_MIN_PKT_SIZE) { 368 pad = BNXT_MIN_PKT_SIZE - length; 369 if (skb_pad(skb, pad)) { 370 /* SKB already freed. */ 371 tx_buf->skb = NULL; 372 return NETDEV_TX_OK; 373 } 374 length = BNXT_MIN_PKT_SIZE; 375 } 376 377 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 378 379 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) { 380 dev_kfree_skb_any(skb); 381 tx_buf->skb = NULL; 382 return NETDEV_TX_OK; 383 } 384 385 dma_unmap_addr_set(tx_buf, mapping, mapping); 386 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 387 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT); 388 389 txbd->tx_bd_haddr = cpu_to_le64(mapping); 390 391 prod = NEXT_TX(prod); 392 txbd1 = (struct tx_bd_ext *) 393 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 394 395 txbd1->tx_bd_hsize_lflags = 0; 396 if (skb_is_gso(skb)) { 397 u32 hdr_len; 398 399 if (skb->encapsulation) 400 hdr_len = skb_inner_network_offset(skb) + 401 skb_inner_network_header_len(skb) + 402 inner_tcp_hdrlen(skb); 403 else 404 hdr_len = skb_transport_offset(skb) + 405 tcp_hdrlen(skb); 406 407 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO | 408 TX_BD_FLAGS_T_IPID | 409 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 410 length = skb_shinfo(skb)->gso_size; 411 txbd1->tx_bd_mss = cpu_to_le32(length); 412 length += hdr_len; 413 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 414 txbd1->tx_bd_hsize_lflags = 415 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 416 txbd1->tx_bd_mss = 0; 417 } 418 419 length >>= 9; 420 flags |= bnxt_lhint_arr[length]; 421 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 422 423 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 424 txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action); 425 for (i = 0; i < last_frag; i++) { 426 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 427 428 prod = NEXT_TX(prod); 429 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; 430 431 len = skb_frag_size(frag); 432 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 433 DMA_TO_DEVICE); 434 435 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 436 goto tx_dma_error; 437 438 tx_buf = &txr->tx_buf_ring[prod]; 439 dma_unmap_addr_set(tx_buf, mapping, mapping); 440 441 txbd->tx_bd_haddr = cpu_to_le64(mapping); 442 443 flags = len << TX_BD_LEN_SHIFT; 444 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 445 } 446 447 flags &= ~TX_BD_LEN; 448 txbd->tx_bd_len_flags_type = 449 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 450 TX_BD_FLAGS_PACKET_END); 451 452 netdev_tx_sent_queue(txq, skb->len); 453 454 /* Sync BD data before updating doorbell */ 455 wmb(); 456 457 prod = NEXT_TX(prod); 458 txr->tx_prod = prod; 459 460 writel(DB_KEY_TX | prod, txr->tx_doorbell); 461 writel(DB_KEY_TX | prod, txr->tx_doorbell); 462 463 tx_done: 464 465 mmiowb(); 466 467 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 468 netif_tx_stop_queue(txq); 469 470 /* netif_tx_stop_queue() must be done before checking 471 * tx index in bnxt_tx_avail() below, because in 472 * bnxt_tx_int(), we update tx index before checking for 473 * netif_tx_queue_stopped(). 474 */ 475 smp_mb(); 476 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) 477 netif_tx_wake_queue(txq); 478 } 479 return NETDEV_TX_OK; 480 481 tx_dma_error: 482 last_frag = i; 483 484 /* start back at beginning and unmap skb */ 485 prod = txr->tx_prod; 486 tx_buf = &txr->tx_buf_ring[prod]; 487 tx_buf->skb = NULL; 488 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 489 skb_headlen(skb), PCI_DMA_TODEVICE); 490 prod = NEXT_TX(prod); 491 492 /* unmap remaining mapped pages */ 493 for (i = 0; i < last_frag; i++) { 494 prod = NEXT_TX(prod); 495 tx_buf = &txr->tx_buf_ring[prod]; 496 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 497 skb_frag_size(&skb_shinfo(skb)->frags[i]), 498 PCI_DMA_TODEVICE); 499 } 500 501 dev_kfree_skb_any(skb); 502 return NETDEV_TX_OK; 503 } 504 505 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) 506 { 507 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 508 int index = txr - &bp->tx_ring[0]; 509 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index); 510 u16 cons = txr->tx_cons; 511 struct pci_dev *pdev = bp->pdev; 512 int i; 513 unsigned int tx_bytes = 0; 514 515 for (i = 0; i < nr_pkts; i++) { 516 struct bnxt_sw_tx_bd *tx_buf; 517 struct sk_buff *skb; 518 int j, last; 519 520 tx_buf = &txr->tx_buf_ring[cons]; 521 cons = NEXT_TX(cons); 522 skb = tx_buf->skb; 523 tx_buf->skb = NULL; 524 525 if (tx_buf->is_push) { 526 tx_buf->is_push = 0; 527 goto next_tx_int; 528 } 529 530 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 531 skb_headlen(skb), PCI_DMA_TODEVICE); 532 last = tx_buf->nr_frags; 533 534 for (j = 0; j < last; j++) { 535 cons = NEXT_TX(cons); 536 tx_buf = &txr->tx_buf_ring[cons]; 537 dma_unmap_page( 538 &pdev->dev, 539 dma_unmap_addr(tx_buf, mapping), 540 skb_frag_size(&skb_shinfo(skb)->frags[j]), 541 PCI_DMA_TODEVICE); 542 } 543 544 next_tx_int: 545 cons = NEXT_TX(cons); 546 547 tx_bytes += skb->len; 548 dev_kfree_skb_any(skb); 549 } 550 551 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); 552 txr->tx_cons = cons; 553 554 /* Need to make the tx_cons update visible to bnxt_start_xmit() 555 * before checking for netif_tx_queue_stopped(). Without the 556 * memory barrier, there is a small possibility that bnxt_start_xmit() 557 * will miss it and cause the queue to be stopped forever. 558 */ 559 smp_mb(); 560 561 if (unlikely(netif_tx_queue_stopped(txq)) && 562 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) { 563 __netif_tx_lock(txq, smp_processor_id()); 564 if (netif_tx_queue_stopped(txq) && 565 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh && 566 txr->dev_state != BNXT_DEV_STATE_CLOSING) 567 netif_tx_wake_queue(txq); 568 __netif_tx_unlock(txq); 569 } 570 } 571 572 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping, 573 gfp_t gfp) 574 { 575 u8 *data; 576 struct pci_dev *pdev = bp->pdev; 577 578 data = kmalloc(bp->rx_buf_size, gfp); 579 if (!data) 580 return NULL; 581 582 *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET, 583 bp->rx_buf_use_size, PCI_DMA_FROMDEVICE); 584 585 if (dma_mapping_error(&pdev->dev, *mapping)) { 586 kfree(data); 587 data = NULL; 588 } 589 return data; 590 } 591 592 static inline int bnxt_alloc_rx_data(struct bnxt *bp, 593 struct bnxt_rx_ring_info *rxr, 594 u16 prod, gfp_t gfp) 595 { 596 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 597 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 598 u8 *data; 599 dma_addr_t mapping; 600 601 data = __bnxt_alloc_rx_data(bp, &mapping, gfp); 602 if (!data) 603 return -ENOMEM; 604 605 rx_buf->data = data; 606 dma_unmap_addr_set(rx_buf, mapping, mapping); 607 608 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 609 610 return 0; 611 } 612 613 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, 614 u8 *data) 615 { 616 u16 prod = rxr->rx_prod; 617 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 618 struct rx_bd *cons_bd, *prod_bd; 619 620 prod_rx_buf = &rxr->rx_buf_ring[prod]; 621 cons_rx_buf = &rxr->rx_buf_ring[cons]; 622 623 prod_rx_buf->data = data; 624 625 dma_unmap_addr_set(prod_rx_buf, mapping, 626 dma_unmap_addr(cons_rx_buf, mapping)); 627 628 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 629 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)]; 630 631 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 632 } 633 634 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 635 { 636 u16 next, max = rxr->rx_agg_bmap_size; 637 638 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 639 if (next >= max) 640 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 641 return next; 642 } 643 644 static inline int bnxt_alloc_rx_page(struct bnxt *bp, 645 struct bnxt_rx_ring_info *rxr, 646 u16 prod, gfp_t gfp) 647 { 648 struct rx_bd *rxbd = 649 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 650 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 651 struct pci_dev *pdev = bp->pdev; 652 struct page *page; 653 dma_addr_t mapping; 654 u16 sw_prod = rxr->rx_sw_agg_prod; 655 unsigned int offset = 0; 656 657 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 658 page = rxr->rx_page; 659 if (!page) { 660 page = alloc_page(gfp); 661 if (!page) 662 return -ENOMEM; 663 rxr->rx_page = page; 664 rxr->rx_page_offset = 0; 665 } 666 offset = rxr->rx_page_offset; 667 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; 668 if (rxr->rx_page_offset == PAGE_SIZE) 669 rxr->rx_page = NULL; 670 else 671 get_page(page); 672 } else { 673 page = alloc_page(gfp); 674 if (!page) 675 return -ENOMEM; 676 } 677 678 mapping = dma_map_page(&pdev->dev, page, offset, BNXT_RX_PAGE_SIZE, 679 PCI_DMA_FROMDEVICE); 680 if (dma_mapping_error(&pdev->dev, mapping)) { 681 __free_page(page); 682 return -EIO; 683 } 684 685 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 686 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 687 688 __set_bit(sw_prod, rxr->rx_agg_bmap); 689 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 690 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod); 691 692 rx_agg_buf->page = page; 693 rx_agg_buf->offset = offset; 694 rx_agg_buf->mapping = mapping; 695 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 696 rxbd->rx_bd_opaque = sw_prod; 697 return 0; 698 } 699 700 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons, 701 u32 agg_bufs) 702 { 703 struct bnxt *bp = bnapi->bp; 704 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 705 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 706 u16 prod = rxr->rx_agg_prod; 707 u16 sw_prod = rxr->rx_sw_agg_prod; 708 u32 i; 709 710 for (i = 0; i < agg_bufs; i++) { 711 u16 cons; 712 struct rx_agg_cmp *agg; 713 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 714 struct rx_bd *prod_bd; 715 struct page *page; 716 717 agg = (struct rx_agg_cmp *) 718 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 719 cons = agg->rx_agg_cmp_opaque; 720 __clear_bit(cons, rxr->rx_agg_bmap); 721 722 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 723 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 724 725 __set_bit(sw_prod, rxr->rx_agg_bmap); 726 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 727 cons_rx_buf = &rxr->rx_agg_ring[cons]; 728 729 /* It is possible for sw_prod to be equal to cons, so 730 * set cons_rx_buf->page to NULL first. 731 */ 732 page = cons_rx_buf->page; 733 cons_rx_buf->page = NULL; 734 prod_rx_buf->page = page; 735 prod_rx_buf->offset = cons_rx_buf->offset; 736 737 prod_rx_buf->mapping = cons_rx_buf->mapping; 738 739 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 740 741 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 742 prod_bd->rx_bd_opaque = sw_prod; 743 744 prod = NEXT_RX_AGG(prod); 745 sw_prod = NEXT_RX_AGG(sw_prod); 746 cp_cons = NEXT_CMP(cp_cons); 747 } 748 rxr->rx_agg_prod = prod; 749 rxr->rx_sw_agg_prod = sw_prod; 750 } 751 752 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 753 struct bnxt_rx_ring_info *rxr, u16 cons, 754 u16 prod, u8 *data, dma_addr_t dma_addr, 755 unsigned int len) 756 { 757 int err; 758 struct sk_buff *skb; 759 760 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 761 if (unlikely(err)) { 762 bnxt_reuse_rx_data(rxr, cons, data); 763 return NULL; 764 } 765 766 skb = build_skb(data, 0); 767 dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 768 PCI_DMA_FROMDEVICE); 769 if (!skb) { 770 kfree(data); 771 return NULL; 772 } 773 774 skb_reserve(skb, BNXT_RX_OFFSET); 775 skb_put(skb, len); 776 return skb; 777 } 778 779 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi, 780 struct sk_buff *skb, u16 cp_cons, 781 u32 agg_bufs) 782 { 783 struct pci_dev *pdev = bp->pdev; 784 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 785 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 786 u16 prod = rxr->rx_agg_prod; 787 u32 i; 788 789 for (i = 0; i < agg_bufs; i++) { 790 u16 cons, frag_len; 791 struct rx_agg_cmp *agg; 792 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 793 struct page *page; 794 dma_addr_t mapping; 795 796 agg = (struct rx_agg_cmp *) 797 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 798 cons = agg->rx_agg_cmp_opaque; 799 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 800 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 801 802 cons_rx_buf = &rxr->rx_agg_ring[cons]; 803 skb_fill_page_desc(skb, i, cons_rx_buf->page, 804 cons_rx_buf->offset, frag_len); 805 __clear_bit(cons, rxr->rx_agg_bmap); 806 807 /* It is possible for bnxt_alloc_rx_page() to allocate 808 * a sw_prod index that equals the cons index, so we 809 * need to clear the cons entry now. 810 */ 811 mapping = dma_unmap_addr(cons_rx_buf, mapping); 812 page = cons_rx_buf->page; 813 cons_rx_buf->page = NULL; 814 815 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { 816 struct skb_shared_info *shinfo; 817 unsigned int nr_frags; 818 819 shinfo = skb_shinfo(skb); 820 nr_frags = --shinfo->nr_frags; 821 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); 822 823 dev_kfree_skb(skb); 824 825 cons_rx_buf->page = page; 826 827 /* Update prod since possibly some pages have been 828 * allocated already. 829 */ 830 rxr->rx_agg_prod = prod; 831 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i); 832 return NULL; 833 } 834 835 dma_unmap_page(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, 836 PCI_DMA_FROMDEVICE); 837 838 skb->data_len += frag_len; 839 skb->len += frag_len; 840 skb->truesize += PAGE_SIZE; 841 842 prod = NEXT_RX_AGG(prod); 843 cp_cons = NEXT_CMP(cp_cons); 844 } 845 rxr->rx_agg_prod = prod; 846 return skb; 847 } 848 849 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 850 u8 agg_bufs, u32 *raw_cons) 851 { 852 u16 last; 853 struct rx_agg_cmp *agg; 854 855 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 856 last = RING_CMP(*raw_cons); 857 agg = (struct rx_agg_cmp *) 858 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 859 return RX_AGG_CMP_VALID(agg, *raw_cons); 860 } 861 862 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 863 unsigned int len, 864 dma_addr_t mapping) 865 { 866 struct bnxt *bp = bnapi->bp; 867 struct pci_dev *pdev = bp->pdev; 868 struct sk_buff *skb; 869 870 skb = napi_alloc_skb(&bnapi->napi, len); 871 if (!skb) 872 return NULL; 873 874 dma_sync_single_for_cpu(&pdev->dev, mapping, 875 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE); 876 877 memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET); 878 879 dma_sync_single_for_device(&pdev->dev, mapping, 880 bp->rx_copy_thresh, 881 PCI_DMA_FROMDEVICE); 882 883 skb_put(skb, len); 884 return skb; 885 } 886 887 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi, 888 u32 *raw_cons, void *cmp) 889 { 890 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 891 struct rx_cmp *rxcmp = cmp; 892 u32 tmp_raw_cons = *raw_cons; 893 u8 cmp_type, agg_bufs = 0; 894 895 cmp_type = RX_CMP_TYPE(rxcmp); 896 897 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 898 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 899 RX_CMP_AGG_BUFS) >> 900 RX_CMP_AGG_BUFS_SHIFT; 901 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 902 struct rx_tpa_end_cmp *tpa_end = cmp; 903 904 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 905 RX_TPA_END_CMP_AGG_BUFS) >> 906 RX_TPA_END_CMP_AGG_BUFS_SHIFT; 907 } 908 909 if (agg_bufs) { 910 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 911 return -EBUSY; 912 } 913 *raw_cons = tmp_raw_cons; 914 return 0; 915 } 916 917 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 918 { 919 if (!rxr->bnapi->in_reset) { 920 rxr->bnapi->in_reset = true; 921 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 922 schedule_work(&bp->sp_task); 923 } 924 rxr->rx_next_cons = 0xffff; 925 } 926 927 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 928 struct rx_tpa_start_cmp *tpa_start, 929 struct rx_tpa_start_cmp_ext *tpa_start1) 930 { 931 u8 agg_id = TPA_START_AGG_ID(tpa_start); 932 u16 cons, prod; 933 struct bnxt_tpa_info *tpa_info; 934 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 935 struct rx_bd *prod_bd; 936 dma_addr_t mapping; 937 938 cons = tpa_start->rx_tpa_start_cmp_opaque; 939 prod = rxr->rx_prod; 940 cons_rx_buf = &rxr->rx_buf_ring[cons]; 941 prod_rx_buf = &rxr->rx_buf_ring[prod]; 942 tpa_info = &rxr->rx_tpa[agg_id]; 943 944 if (unlikely(cons != rxr->rx_next_cons)) { 945 bnxt_sched_reset(bp, rxr); 946 return; 947 } 948 949 prod_rx_buf->data = tpa_info->data; 950 951 mapping = tpa_info->mapping; 952 dma_unmap_addr_set(prod_rx_buf, mapping, mapping); 953 954 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)]; 955 956 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 957 958 tpa_info->data = cons_rx_buf->data; 959 cons_rx_buf->data = NULL; 960 tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping); 961 962 tpa_info->len = 963 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 964 RX_TPA_START_CMP_LEN_SHIFT; 965 if (likely(TPA_START_HASH_VALID(tpa_start))) { 966 u32 hash_type = TPA_START_HASH_TYPE(tpa_start); 967 968 tpa_info->hash_type = PKT_HASH_TYPE_L4; 969 tpa_info->gso_type = SKB_GSO_TCPV4; 970 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 971 if (hash_type == 3) 972 tpa_info->gso_type = SKB_GSO_TCPV6; 973 tpa_info->rss_hash = 974 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 975 } else { 976 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 977 tpa_info->gso_type = 0; 978 if (netif_msg_rx_err(bp)) 979 netdev_warn(bp->dev, "TPA packet without valid hash\n"); 980 } 981 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 982 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 983 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 984 985 rxr->rx_prod = NEXT_RX(prod); 986 cons = NEXT_RX(cons); 987 rxr->rx_next_cons = NEXT_RX(cons); 988 cons_rx_buf = &rxr->rx_buf_ring[cons]; 989 990 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 991 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 992 cons_rx_buf->data = NULL; 993 } 994 995 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi, 996 u16 cp_cons, u32 agg_bufs) 997 { 998 if (agg_bufs) 999 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1000 } 1001 1002 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1003 int payload_off, int tcp_ts, 1004 struct sk_buff *skb) 1005 { 1006 #ifdef CONFIG_INET 1007 struct tcphdr *th; 1008 int len, nw_off; 1009 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1010 u32 hdr_info = tpa_info->hdr_info; 1011 bool loopback = false; 1012 1013 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1014 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1015 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1016 1017 /* If the packet is an internal loopback packet, the offsets will 1018 * have an extra 4 bytes. 1019 */ 1020 if (inner_mac_off == 4) { 1021 loopback = true; 1022 } else if (inner_mac_off > 4) { 1023 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1024 ETH_HLEN - 2)); 1025 1026 /* We only support inner iPv4/ipv6. If we don't see the 1027 * correct protocol ID, it must be a loopback packet where 1028 * the offsets are off by 4. 1029 */ 1030 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1031 loopback = true; 1032 } 1033 if (loopback) { 1034 /* internal loopback packet, subtract all offsets by 4 */ 1035 inner_ip_off -= 4; 1036 inner_mac_off -= 4; 1037 outer_ip_off -= 4; 1038 } 1039 1040 nw_off = inner_ip_off - ETH_HLEN; 1041 skb_set_network_header(skb, nw_off); 1042 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1043 struct ipv6hdr *iph = ipv6_hdr(skb); 1044 1045 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1046 len = skb->len - skb_transport_offset(skb); 1047 th = tcp_hdr(skb); 1048 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1049 } else { 1050 struct iphdr *iph = ip_hdr(skb); 1051 1052 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1053 len = skb->len - skb_transport_offset(skb); 1054 th = tcp_hdr(skb); 1055 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1056 } 1057 1058 if (inner_mac_off) { /* tunnel */ 1059 struct udphdr *uh = NULL; 1060 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1061 ETH_HLEN - 2)); 1062 1063 if (proto == htons(ETH_P_IP)) { 1064 struct iphdr *iph = (struct iphdr *)skb->data; 1065 1066 if (iph->protocol == IPPROTO_UDP) 1067 uh = (struct udphdr *)(iph + 1); 1068 } else { 1069 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1070 1071 if (iph->nexthdr == IPPROTO_UDP) 1072 uh = (struct udphdr *)(iph + 1); 1073 } 1074 if (uh) { 1075 if (uh->check) 1076 skb_shinfo(skb)->gso_type |= 1077 SKB_GSO_UDP_TUNNEL_CSUM; 1078 else 1079 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1080 } 1081 } 1082 #endif 1083 return skb; 1084 } 1085 1086 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1087 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1088 1089 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1090 int payload_off, int tcp_ts, 1091 struct sk_buff *skb) 1092 { 1093 #ifdef CONFIG_INET 1094 struct tcphdr *th; 1095 int len, nw_off, tcp_opt_len; 1096 1097 if (tcp_ts) 1098 tcp_opt_len = 12; 1099 1100 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1101 struct iphdr *iph; 1102 1103 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1104 ETH_HLEN; 1105 skb_set_network_header(skb, nw_off); 1106 iph = ip_hdr(skb); 1107 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1108 len = skb->len - skb_transport_offset(skb); 1109 th = tcp_hdr(skb); 1110 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1111 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1112 struct ipv6hdr *iph; 1113 1114 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1115 ETH_HLEN; 1116 skb_set_network_header(skb, nw_off); 1117 iph = ipv6_hdr(skb); 1118 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1119 len = skb->len - skb_transport_offset(skb); 1120 th = tcp_hdr(skb); 1121 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1122 } else { 1123 dev_kfree_skb_any(skb); 1124 return NULL; 1125 } 1126 tcp_gro_complete(skb); 1127 1128 if (nw_off) { /* tunnel */ 1129 struct udphdr *uh = NULL; 1130 1131 if (skb->protocol == htons(ETH_P_IP)) { 1132 struct iphdr *iph = (struct iphdr *)skb->data; 1133 1134 if (iph->protocol == IPPROTO_UDP) 1135 uh = (struct udphdr *)(iph + 1); 1136 } else { 1137 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1138 1139 if (iph->nexthdr == IPPROTO_UDP) 1140 uh = (struct udphdr *)(iph + 1); 1141 } 1142 if (uh) { 1143 if (uh->check) 1144 skb_shinfo(skb)->gso_type |= 1145 SKB_GSO_UDP_TUNNEL_CSUM; 1146 else 1147 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1148 } 1149 } 1150 #endif 1151 return skb; 1152 } 1153 1154 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1155 struct bnxt_tpa_info *tpa_info, 1156 struct rx_tpa_end_cmp *tpa_end, 1157 struct rx_tpa_end_cmp_ext *tpa_end1, 1158 struct sk_buff *skb) 1159 { 1160 #ifdef CONFIG_INET 1161 int payload_off; 1162 u16 segs; 1163 1164 segs = TPA_END_TPA_SEGS(tpa_end); 1165 if (segs == 1) 1166 return skb; 1167 1168 NAPI_GRO_CB(skb)->count = segs; 1169 skb_shinfo(skb)->gso_size = 1170 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1171 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1172 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1173 RX_TPA_END_CMP_PAYLOAD_OFFSET) >> 1174 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT; 1175 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1176 #endif 1177 return skb; 1178 } 1179 1180 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1181 struct bnxt_napi *bnapi, 1182 u32 *raw_cons, 1183 struct rx_tpa_end_cmp *tpa_end, 1184 struct rx_tpa_end_cmp_ext *tpa_end1, 1185 bool *agg_event) 1186 { 1187 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1188 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1189 u8 agg_id = TPA_END_AGG_ID(tpa_end); 1190 u8 *data, agg_bufs; 1191 u16 cp_cons = RING_CMP(*raw_cons); 1192 unsigned int len; 1193 struct bnxt_tpa_info *tpa_info; 1194 dma_addr_t mapping; 1195 struct sk_buff *skb; 1196 1197 if (unlikely(bnapi->in_reset)) { 1198 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end); 1199 1200 if (rc < 0) 1201 return ERR_PTR(-EBUSY); 1202 return NULL; 1203 } 1204 1205 tpa_info = &rxr->rx_tpa[agg_id]; 1206 data = tpa_info->data; 1207 prefetch(data); 1208 len = tpa_info->len; 1209 mapping = tpa_info->mapping; 1210 1211 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) & 1212 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT; 1213 1214 if (agg_bufs) { 1215 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1216 return ERR_PTR(-EBUSY); 1217 1218 *agg_event = true; 1219 cp_cons = NEXT_CMP(cp_cons); 1220 } 1221 1222 if (unlikely(agg_bufs > MAX_SKB_FRAGS)) { 1223 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1224 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1225 agg_bufs, (int)MAX_SKB_FRAGS); 1226 return NULL; 1227 } 1228 1229 if (len <= bp->rx_copy_thresh) { 1230 skb = bnxt_copy_skb(bnapi, data, len, mapping); 1231 if (!skb) { 1232 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1233 return NULL; 1234 } 1235 } else { 1236 u8 *new_data; 1237 dma_addr_t new_mapping; 1238 1239 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC); 1240 if (!new_data) { 1241 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1242 return NULL; 1243 } 1244 1245 tpa_info->data = new_data; 1246 tpa_info->mapping = new_mapping; 1247 1248 skb = build_skb(data, 0); 1249 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size, 1250 PCI_DMA_FROMDEVICE); 1251 1252 if (!skb) { 1253 kfree(data); 1254 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs); 1255 return NULL; 1256 } 1257 skb_reserve(skb, BNXT_RX_OFFSET); 1258 skb_put(skb, len); 1259 } 1260 1261 if (agg_bufs) { 1262 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1263 if (!skb) { 1264 /* Page reuse already handled by bnxt_rx_pages(). */ 1265 return NULL; 1266 } 1267 } 1268 skb->protocol = eth_type_trans(skb, bp->dev); 1269 1270 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1271 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1272 1273 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) && 1274 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1275 u16 vlan_proto = tpa_info->metadata >> 1276 RX_CMP_FLAGS2_METADATA_TPID_SFT; 1277 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK; 1278 1279 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1280 } 1281 1282 skb_checksum_none_assert(skb); 1283 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1284 skb->ip_summed = CHECKSUM_UNNECESSARY; 1285 skb->csum_level = 1286 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1287 } 1288 1289 if (TPA_END_GRO(tpa_end)) 1290 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1291 1292 return skb; 1293 } 1294 1295 /* returns the following: 1296 * 1 - 1 packet successfully received 1297 * 0 - successful TPA_START, packet not completed yet 1298 * -EBUSY - completion ring does not have all the agg buffers yet 1299 * -ENOMEM - packet aborted due to out of memory 1300 * -EIO - packet aborted due to hw error indicated in BD 1301 */ 1302 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons, 1303 bool *agg_event) 1304 { 1305 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1306 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1307 struct net_device *dev = bp->dev; 1308 struct rx_cmp *rxcmp; 1309 struct rx_cmp_ext *rxcmp1; 1310 u32 tmp_raw_cons = *raw_cons; 1311 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 1312 struct bnxt_sw_rx_bd *rx_buf; 1313 unsigned int len; 1314 u8 *data, agg_bufs, cmp_type; 1315 dma_addr_t dma_addr; 1316 struct sk_buff *skb; 1317 int rc = 0; 1318 1319 rxcmp = (struct rx_cmp *) 1320 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1321 1322 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 1323 cp_cons = RING_CMP(tmp_raw_cons); 1324 rxcmp1 = (struct rx_cmp_ext *) 1325 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1326 1327 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1328 return -EBUSY; 1329 1330 cmp_type = RX_CMP_TYPE(rxcmp); 1331 1332 prod = rxr->rx_prod; 1333 1334 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) { 1335 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp, 1336 (struct rx_tpa_start_cmp_ext *)rxcmp1); 1337 1338 goto next_rx_no_prod; 1339 1340 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1341 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons, 1342 (struct rx_tpa_end_cmp *)rxcmp, 1343 (struct rx_tpa_end_cmp_ext *)rxcmp1, 1344 agg_event); 1345 1346 if (unlikely(IS_ERR(skb))) 1347 return -EBUSY; 1348 1349 rc = -ENOMEM; 1350 if (likely(skb)) { 1351 skb_record_rx_queue(skb, bnapi->index); 1352 skb_mark_napi_id(skb, &bnapi->napi); 1353 if (bnxt_busy_polling(bnapi)) 1354 netif_receive_skb(skb); 1355 else 1356 napi_gro_receive(&bnapi->napi, skb); 1357 rc = 1; 1358 } 1359 goto next_rx_no_prod; 1360 } 1361 1362 cons = rxcmp->rx_cmp_opaque; 1363 rx_buf = &rxr->rx_buf_ring[cons]; 1364 data = rx_buf->data; 1365 if (unlikely(cons != rxr->rx_next_cons)) { 1366 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp); 1367 1368 bnxt_sched_reset(bp, rxr); 1369 return rc1; 1370 } 1371 prefetch(data); 1372 1373 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >> 1374 RX_CMP_AGG_BUFS_SHIFT; 1375 1376 if (agg_bufs) { 1377 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1378 return -EBUSY; 1379 1380 cp_cons = NEXT_CMP(cp_cons); 1381 *agg_event = true; 1382 } 1383 1384 rx_buf->data = NULL; 1385 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 1386 bnxt_reuse_rx_data(rxr, cons, data); 1387 if (agg_bufs) 1388 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs); 1389 1390 rc = -EIO; 1391 goto next_rx; 1392 } 1393 1394 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; 1395 dma_addr = dma_unmap_addr(rx_buf, mapping); 1396 1397 if (len <= bp->rx_copy_thresh) { 1398 skb = bnxt_copy_skb(bnapi, data, len, dma_addr); 1399 bnxt_reuse_rx_data(rxr, cons, data); 1400 if (!skb) { 1401 rc = -ENOMEM; 1402 goto next_rx; 1403 } 1404 } else { 1405 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len); 1406 if (!skb) { 1407 rc = -ENOMEM; 1408 goto next_rx; 1409 } 1410 } 1411 1412 if (agg_bufs) { 1413 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs); 1414 if (!skb) { 1415 rc = -ENOMEM; 1416 goto next_rx; 1417 } 1418 } 1419 1420 if (RX_CMP_HASH_VALID(rxcmp)) { 1421 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); 1422 enum pkt_hash_types type = PKT_HASH_TYPE_L4; 1423 1424 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1425 if (hash_type != 1 && hash_type != 3) 1426 type = PKT_HASH_TYPE_L3; 1427 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 1428 } 1429 1430 skb->protocol = eth_type_trans(skb, dev); 1431 1432 if ((rxcmp1->rx_cmp_flags2 & 1433 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) && 1434 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) { 1435 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 1436 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK; 1437 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT; 1438 1439 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag); 1440 } 1441 1442 skb_checksum_none_assert(skb); 1443 if (RX_CMP_L4_CS_OK(rxcmp1)) { 1444 if (dev->features & NETIF_F_RXCSUM) { 1445 skb->ip_summed = CHECKSUM_UNNECESSARY; 1446 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 1447 } 1448 } else { 1449 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 1450 if (dev->features & NETIF_F_RXCSUM) 1451 cpr->rx_l4_csum_errors++; 1452 } 1453 } 1454 1455 skb_record_rx_queue(skb, bnapi->index); 1456 skb_mark_napi_id(skb, &bnapi->napi); 1457 if (bnxt_busy_polling(bnapi)) 1458 netif_receive_skb(skb); 1459 else 1460 napi_gro_receive(&bnapi->napi, skb); 1461 rc = 1; 1462 1463 next_rx: 1464 rxr->rx_prod = NEXT_RX(prod); 1465 rxr->rx_next_cons = NEXT_RX(cons); 1466 1467 next_rx_no_prod: 1468 *raw_cons = tmp_raw_cons; 1469 1470 return rc; 1471 } 1472 1473 #define BNXT_GET_EVENT_PORT(data) \ 1474 ((data) & \ 1475 HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 1476 1477 static int bnxt_async_event_process(struct bnxt *bp, 1478 struct hwrm_async_event_cmpl *cmpl) 1479 { 1480 u16 event_id = le16_to_cpu(cmpl->event_id); 1481 1482 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 1483 switch (event_id) { 1484 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 1485 u32 data1 = le32_to_cpu(cmpl->event_data1); 1486 struct bnxt_link_info *link_info = &bp->link_info; 1487 1488 if (BNXT_VF(bp)) 1489 goto async_event_process_exit; 1490 if (data1 & 0x20000) { 1491 u16 fw_speed = link_info->force_link_speed; 1492 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 1493 1494 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 1495 speed); 1496 } 1497 /* fall thru */ 1498 } 1499 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 1500 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 1501 break; 1502 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 1503 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 1504 break; 1505 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 1506 u32 data1 = le32_to_cpu(cmpl->event_data1); 1507 u16 port_id = BNXT_GET_EVENT_PORT(data1); 1508 1509 if (BNXT_VF(bp)) 1510 break; 1511 1512 if (bp->pf.port_id != port_id) 1513 break; 1514 1515 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 1516 break; 1517 } 1518 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 1519 if (BNXT_PF(bp)) 1520 goto async_event_process_exit; 1521 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 1522 break; 1523 default: 1524 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n", 1525 event_id); 1526 goto async_event_process_exit; 1527 } 1528 schedule_work(&bp->sp_task); 1529 async_event_process_exit: 1530 return 0; 1531 } 1532 1533 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 1534 { 1535 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 1536 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 1537 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 1538 (struct hwrm_fwd_req_cmpl *)txcmp; 1539 1540 switch (cmpl_type) { 1541 case CMPL_BASE_TYPE_HWRM_DONE: 1542 seq_id = le16_to_cpu(h_cmpl->sequence_id); 1543 if (seq_id == bp->hwrm_intr_seq_id) 1544 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID; 1545 else 1546 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id); 1547 break; 1548 1549 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 1550 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 1551 1552 if ((vf_id < bp->pf.first_vf_id) || 1553 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 1554 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 1555 vf_id); 1556 return -EINVAL; 1557 } 1558 1559 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 1560 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event); 1561 schedule_work(&bp->sp_task); 1562 break; 1563 1564 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 1565 bnxt_async_event_process(bp, 1566 (struct hwrm_async_event_cmpl *)txcmp); 1567 1568 default: 1569 break; 1570 } 1571 1572 return 0; 1573 } 1574 1575 static irqreturn_t bnxt_msix(int irq, void *dev_instance) 1576 { 1577 struct bnxt_napi *bnapi = dev_instance; 1578 struct bnxt *bp = bnapi->bp; 1579 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1580 u32 cons = RING_CMP(cpr->cp_raw_cons); 1581 1582 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1583 napi_schedule(&bnapi->napi); 1584 return IRQ_HANDLED; 1585 } 1586 1587 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 1588 { 1589 u32 raw_cons = cpr->cp_raw_cons; 1590 u16 cons = RING_CMP(raw_cons); 1591 struct tx_cmp *txcmp; 1592 1593 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1594 1595 return TX_CMP_VALID(txcmp, raw_cons); 1596 } 1597 1598 static irqreturn_t bnxt_inta(int irq, void *dev_instance) 1599 { 1600 struct bnxt_napi *bnapi = dev_instance; 1601 struct bnxt *bp = bnapi->bp; 1602 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1603 u32 cons = RING_CMP(cpr->cp_raw_cons); 1604 u32 int_status; 1605 1606 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 1607 1608 if (!bnxt_has_work(bp, cpr)) { 1609 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS); 1610 /* return if erroneous interrupt */ 1611 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id))) 1612 return IRQ_NONE; 1613 } 1614 1615 /* disable ring IRQ */ 1616 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell); 1617 1618 /* Return here if interrupt is shared and is disabled. */ 1619 if (unlikely(atomic_read(&bp->intr_sem) != 0)) 1620 return IRQ_HANDLED; 1621 1622 napi_schedule(&bnapi->napi); 1623 return IRQ_HANDLED; 1624 } 1625 1626 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 1627 { 1628 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1629 u32 raw_cons = cpr->cp_raw_cons; 1630 u32 cons; 1631 int tx_pkts = 0; 1632 int rx_pkts = 0; 1633 bool rx_event = false; 1634 bool agg_event = false; 1635 struct tx_cmp *txcmp; 1636 1637 while (1) { 1638 int rc; 1639 1640 cons = RING_CMP(raw_cons); 1641 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 1642 1643 if (!TX_CMP_VALID(txcmp, raw_cons)) 1644 break; 1645 1646 /* The valid test of the entry must be done first before 1647 * reading any further. 1648 */ 1649 dma_rmb(); 1650 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) { 1651 tx_pkts++; 1652 /* return full budget so NAPI will complete. */ 1653 if (unlikely(tx_pkts > bp->tx_wake_thresh)) 1654 rx_pkts = budget; 1655 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1656 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event); 1657 if (likely(rc >= 0)) 1658 rx_pkts += rc; 1659 else if (rc == -EBUSY) /* partial completion */ 1660 break; 1661 rx_event = true; 1662 } else if (unlikely((TX_CMP_TYPE(txcmp) == 1663 CMPL_BASE_TYPE_HWRM_DONE) || 1664 (TX_CMP_TYPE(txcmp) == 1665 CMPL_BASE_TYPE_HWRM_FWD_REQ) || 1666 (TX_CMP_TYPE(txcmp) == 1667 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) { 1668 bnxt_hwrm_handler(bp, txcmp); 1669 } 1670 raw_cons = NEXT_RAW_CMP(raw_cons); 1671 1672 if (rx_pkts == budget) 1673 break; 1674 } 1675 1676 cpr->cp_raw_cons = raw_cons; 1677 /* ACK completion ring before freeing tx ring and producing new 1678 * buffers in rx/agg rings to prevent overflowing the completion 1679 * ring. 1680 */ 1681 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1682 1683 if (tx_pkts) 1684 bnxt_tx_int(bp, bnapi, tx_pkts); 1685 1686 if (rx_event) { 1687 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1688 1689 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 1690 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 1691 if (agg_event) { 1692 writel(DB_KEY_RX | rxr->rx_agg_prod, 1693 rxr->rx_agg_doorbell); 1694 writel(DB_KEY_RX | rxr->rx_agg_prod, 1695 rxr->rx_agg_doorbell); 1696 } 1697 } 1698 return rx_pkts; 1699 } 1700 1701 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 1702 { 1703 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1704 struct bnxt *bp = bnapi->bp; 1705 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1706 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1707 struct tx_cmp *txcmp; 1708 struct rx_cmp_ext *rxcmp1; 1709 u32 cp_cons, tmp_raw_cons; 1710 u32 raw_cons = cpr->cp_raw_cons; 1711 u32 rx_pkts = 0; 1712 bool agg_event = false; 1713 1714 while (1) { 1715 int rc; 1716 1717 cp_cons = RING_CMP(raw_cons); 1718 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1719 1720 if (!TX_CMP_VALID(txcmp, raw_cons)) 1721 break; 1722 1723 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 1724 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 1725 cp_cons = RING_CMP(tmp_raw_cons); 1726 rxcmp1 = (struct rx_cmp_ext *) 1727 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1728 1729 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 1730 break; 1731 1732 /* force an error to recycle the buffer */ 1733 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 1734 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 1735 1736 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event); 1737 if (likely(rc == -EIO)) 1738 rx_pkts++; 1739 else if (rc == -EBUSY) /* partial completion */ 1740 break; 1741 } else if (unlikely(TX_CMP_TYPE(txcmp) == 1742 CMPL_BASE_TYPE_HWRM_DONE)) { 1743 bnxt_hwrm_handler(bp, txcmp); 1744 } else { 1745 netdev_err(bp->dev, 1746 "Invalid completion received on special ring\n"); 1747 } 1748 raw_cons = NEXT_RAW_CMP(raw_cons); 1749 1750 if (rx_pkts == budget) 1751 break; 1752 } 1753 1754 cpr->cp_raw_cons = raw_cons; 1755 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 1756 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 1757 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 1758 1759 if (agg_event) { 1760 writel(DB_KEY_RX | rxr->rx_agg_prod, rxr->rx_agg_doorbell); 1761 writel(DB_KEY_RX | rxr->rx_agg_prod, rxr->rx_agg_doorbell); 1762 } 1763 1764 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 1765 napi_complete(napi); 1766 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 1767 } 1768 return rx_pkts; 1769 } 1770 1771 static int bnxt_poll(struct napi_struct *napi, int budget) 1772 { 1773 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1774 struct bnxt *bp = bnapi->bp; 1775 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1776 int work_done = 0; 1777 1778 if (!bnxt_lock_napi(bnapi)) 1779 return budget; 1780 1781 while (1) { 1782 work_done += bnxt_poll_work(bp, bnapi, budget - work_done); 1783 1784 if (work_done >= budget) 1785 break; 1786 1787 if (!bnxt_has_work(bp, cpr)) { 1788 napi_complete(napi); 1789 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 1790 break; 1791 } 1792 } 1793 mmiowb(); 1794 bnxt_unlock_napi(bnapi); 1795 return work_done; 1796 } 1797 1798 #ifdef CONFIG_NET_RX_BUSY_POLL 1799 static int bnxt_busy_poll(struct napi_struct *napi) 1800 { 1801 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 1802 struct bnxt *bp = bnapi->bp; 1803 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 1804 int rx_work, budget = 4; 1805 1806 if (atomic_read(&bp->intr_sem) != 0) 1807 return LL_FLUSH_FAILED; 1808 1809 if (!bnxt_lock_poll(bnapi)) 1810 return LL_FLUSH_BUSY; 1811 1812 rx_work = bnxt_poll_work(bp, bnapi, budget); 1813 1814 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 1815 1816 bnxt_unlock_poll(bnapi); 1817 return rx_work; 1818 } 1819 #endif 1820 1821 static void bnxt_free_tx_skbs(struct bnxt *bp) 1822 { 1823 int i, max_idx; 1824 struct pci_dev *pdev = bp->pdev; 1825 1826 if (!bp->tx_ring) 1827 return; 1828 1829 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 1830 for (i = 0; i < bp->tx_nr_rings; i++) { 1831 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 1832 int j; 1833 1834 for (j = 0; j < max_idx;) { 1835 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 1836 struct sk_buff *skb = tx_buf->skb; 1837 int k, last; 1838 1839 if (!skb) { 1840 j++; 1841 continue; 1842 } 1843 1844 tx_buf->skb = NULL; 1845 1846 if (tx_buf->is_push) { 1847 dev_kfree_skb(skb); 1848 j += 2; 1849 continue; 1850 } 1851 1852 dma_unmap_single(&pdev->dev, 1853 dma_unmap_addr(tx_buf, mapping), 1854 skb_headlen(skb), 1855 PCI_DMA_TODEVICE); 1856 1857 last = tx_buf->nr_frags; 1858 j += 2; 1859 for (k = 0; k < last; k++, j++) { 1860 int ring_idx = j & bp->tx_ring_mask; 1861 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 1862 1863 tx_buf = &txr->tx_buf_ring[ring_idx]; 1864 dma_unmap_page( 1865 &pdev->dev, 1866 dma_unmap_addr(tx_buf, mapping), 1867 skb_frag_size(frag), PCI_DMA_TODEVICE); 1868 } 1869 dev_kfree_skb(skb); 1870 } 1871 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i)); 1872 } 1873 } 1874 1875 static void bnxt_free_rx_skbs(struct bnxt *bp) 1876 { 1877 int i, max_idx, max_agg_idx; 1878 struct pci_dev *pdev = bp->pdev; 1879 1880 if (!bp->rx_ring) 1881 return; 1882 1883 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 1884 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 1885 for (i = 0; i < bp->rx_nr_rings; i++) { 1886 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 1887 int j; 1888 1889 if (rxr->rx_tpa) { 1890 for (j = 0; j < MAX_TPA; j++) { 1891 struct bnxt_tpa_info *tpa_info = 1892 &rxr->rx_tpa[j]; 1893 u8 *data = tpa_info->data; 1894 1895 if (!data) 1896 continue; 1897 1898 dma_unmap_single( 1899 &pdev->dev, 1900 dma_unmap_addr(tpa_info, mapping), 1901 bp->rx_buf_use_size, 1902 PCI_DMA_FROMDEVICE); 1903 1904 tpa_info->data = NULL; 1905 1906 kfree(data); 1907 } 1908 } 1909 1910 for (j = 0; j < max_idx; j++) { 1911 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j]; 1912 u8 *data = rx_buf->data; 1913 1914 if (!data) 1915 continue; 1916 1917 dma_unmap_single(&pdev->dev, 1918 dma_unmap_addr(rx_buf, mapping), 1919 bp->rx_buf_use_size, 1920 PCI_DMA_FROMDEVICE); 1921 1922 rx_buf->data = NULL; 1923 1924 kfree(data); 1925 } 1926 1927 for (j = 0; j < max_agg_idx; j++) { 1928 struct bnxt_sw_rx_agg_bd *rx_agg_buf = 1929 &rxr->rx_agg_ring[j]; 1930 struct page *page = rx_agg_buf->page; 1931 1932 if (!page) 1933 continue; 1934 1935 dma_unmap_page(&pdev->dev, 1936 dma_unmap_addr(rx_agg_buf, mapping), 1937 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE); 1938 1939 rx_agg_buf->page = NULL; 1940 __clear_bit(j, rxr->rx_agg_bmap); 1941 1942 __free_page(page); 1943 } 1944 if (rxr->rx_page) { 1945 __free_page(rxr->rx_page); 1946 rxr->rx_page = NULL; 1947 } 1948 } 1949 } 1950 1951 static void bnxt_free_skbs(struct bnxt *bp) 1952 { 1953 bnxt_free_tx_skbs(bp); 1954 bnxt_free_rx_skbs(bp); 1955 } 1956 1957 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 1958 { 1959 struct pci_dev *pdev = bp->pdev; 1960 int i; 1961 1962 for (i = 0; i < ring->nr_pages; i++) { 1963 if (!ring->pg_arr[i]) 1964 continue; 1965 1966 dma_free_coherent(&pdev->dev, ring->page_size, 1967 ring->pg_arr[i], ring->dma_arr[i]); 1968 1969 ring->pg_arr[i] = NULL; 1970 } 1971 if (ring->pg_tbl) { 1972 dma_free_coherent(&pdev->dev, ring->nr_pages * 8, 1973 ring->pg_tbl, ring->pg_tbl_map); 1974 ring->pg_tbl = NULL; 1975 } 1976 if (ring->vmem_size && *ring->vmem) { 1977 vfree(*ring->vmem); 1978 *ring->vmem = NULL; 1979 } 1980 } 1981 1982 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring) 1983 { 1984 int i; 1985 struct pci_dev *pdev = bp->pdev; 1986 1987 if (ring->nr_pages > 1) { 1988 ring->pg_tbl = dma_alloc_coherent(&pdev->dev, 1989 ring->nr_pages * 8, 1990 &ring->pg_tbl_map, 1991 GFP_KERNEL); 1992 if (!ring->pg_tbl) 1993 return -ENOMEM; 1994 } 1995 1996 for (i = 0; i < ring->nr_pages; i++) { 1997 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 1998 ring->page_size, 1999 &ring->dma_arr[i], 2000 GFP_KERNEL); 2001 if (!ring->pg_arr[i]) 2002 return -ENOMEM; 2003 2004 if (ring->nr_pages > 1) 2005 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]); 2006 } 2007 2008 if (ring->vmem_size) { 2009 *ring->vmem = vzalloc(ring->vmem_size); 2010 if (!(*ring->vmem)) 2011 return -ENOMEM; 2012 } 2013 return 0; 2014 } 2015 2016 static void bnxt_free_rx_rings(struct bnxt *bp) 2017 { 2018 int i; 2019 2020 if (!bp->rx_ring) 2021 return; 2022 2023 for (i = 0; i < bp->rx_nr_rings; i++) { 2024 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2025 struct bnxt_ring_struct *ring; 2026 2027 kfree(rxr->rx_tpa); 2028 rxr->rx_tpa = NULL; 2029 2030 kfree(rxr->rx_agg_bmap); 2031 rxr->rx_agg_bmap = NULL; 2032 2033 ring = &rxr->rx_ring_struct; 2034 bnxt_free_ring(bp, ring); 2035 2036 ring = &rxr->rx_agg_ring_struct; 2037 bnxt_free_ring(bp, ring); 2038 } 2039 } 2040 2041 static int bnxt_alloc_rx_rings(struct bnxt *bp) 2042 { 2043 int i, rc, agg_rings = 0, tpa_rings = 0; 2044 2045 if (!bp->rx_ring) 2046 return -ENOMEM; 2047 2048 if (bp->flags & BNXT_FLAG_AGG_RINGS) 2049 agg_rings = 1; 2050 2051 if (bp->flags & BNXT_FLAG_TPA) 2052 tpa_rings = 1; 2053 2054 for (i = 0; i < bp->rx_nr_rings; i++) { 2055 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 2056 struct bnxt_ring_struct *ring; 2057 2058 ring = &rxr->rx_ring_struct; 2059 2060 rc = bnxt_alloc_ring(bp, ring); 2061 if (rc) 2062 return rc; 2063 2064 if (agg_rings) { 2065 u16 mem_size; 2066 2067 ring = &rxr->rx_agg_ring_struct; 2068 rc = bnxt_alloc_ring(bp, ring); 2069 if (rc) 2070 return rc; 2071 2072 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 2073 mem_size = rxr->rx_agg_bmap_size / 8; 2074 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 2075 if (!rxr->rx_agg_bmap) 2076 return -ENOMEM; 2077 2078 if (tpa_rings) { 2079 rxr->rx_tpa = kcalloc(MAX_TPA, 2080 sizeof(struct bnxt_tpa_info), 2081 GFP_KERNEL); 2082 if (!rxr->rx_tpa) 2083 return -ENOMEM; 2084 } 2085 } 2086 } 2087 return 0; 2088 } 2089 2090 static void bnxt_free_tx_rings(struct bnxt *bp) 2091 { 2092 int i; 2093 struct pci_dev *pdev = bp->pdev; 2094 2095 if (!bp->tx_ring) 2096 return; 2097 2098 for (i = 0; i < bp->tx_nr_rings; i++) { 2099 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2100 struct bnxt_ring_struct *ring; 2101 2102 if (txr->tx_push) { 2103 dma_free_coherent(&pdev->dev, bp->tx_push_size, 2104 txr->tx_push, txr->tx_push_mapping); 2105 txr->tx_push = NULL; 2106 } 2107 2108 ring = &txr->tx_ring_struct; 2109 2110 bnxt_free_ring(bp, ring); 2111 } 2112 } 2113 2114 static int bnxt_alloc_tx_rings(struct bnxt *bp) 2115 { 2116 int i, j, rc; 2117 struct pci_dev *pdev = bp->pdev; 2118 2119 bp->tx_push_size = 0; 2120 if (bp->tx_push_thresh) { 2121 int push_size; 2122 2123 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 2124 bp->tx_push_thresh); 2125 2126 if (push_size > 256) { 2127 push_size = 0; 2128 bp->tx_push_thresh = 0; 2129 } 2130 2131 bp->tx_push_size = push_size; 2132 } 2133 2134 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 2135 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2136 struct bnxt_ring_struct *ring; 2137 2138 ring = &txr->tx_ring_struct; 2139 2140 rc = bnxt_alloc_ring(bp, ring); 2141 if (rc) 2142 return rc; 2143 2144 if (bp->tx_push_size) { 2145 dma_addr_t mapping; 2146 2147 /* One pre-allocated DMA buffer to backup 2148 * TX push operation 2149 */ 2150 txr->tx_push = dma_alloc_coherent(&pdev->dev, 2151 bp->tx_push_size, 2152 &txr->tx_push_mapping, 2153 GFP_KERNEL); 2154 2155 if (!txr->tx_push) 2156 return -ENOMEM; 2157 2158 mapping = txr->tx_push_mapping + 2159 sizeof(struct tx_push_bd); 2160 txr->data_mapping = cpu_to_le64(mapping); 2161 2162 memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); 2163 } 2164 ring->queue_id = bp->q_info[j].queue_id; 2165 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) 2166 j++; 2167 } 2168 return 0; 2169 } 2170 2171 static void bnxt_free_cp_rings(struct bnxt *bp) 2172 { 2173 int i; 2174 2175 if (!bp->bnapi) 2176 return; 2177 2178 for (i = 0; i < bp->cp_nr_rings; i++) { 2179 struct bnxt_napi *bnapi = bp->bnapi[i]; 2180 struct bnxt_cp_ring_info *cpr; 2181 struct bnxt_ring_struct *ring; 2182 2183 if (!bnapi) 2184 continue; 2185 2186 cpr = &bnapi->cp_ring; 2187 ring = &cpr->cp_ring_struct; 2188 2189 bnxt_free_ring(bp, ring); 2190 } 2191 } 2192 2193 static int bnxt_alloc_cp_rings(struct bnxt *bp) 2194 { 2195 int i, rc; 2196 2197 for (i = 0; i < bp->cp_nr_rings; i++) { 2198 struct bnxt_napi *bnapi = bp->bnapi[i]; 2199 struct bnxt_cp_ring_info *cpr; 2200 struct bnxt_ring_struct *ring; 2201 2202 if (!bnapi) 2203 continue; 2204 2205 cpr = &bnapi->cp_ring; 2206 ring = &cpr->cp_ring_struct; 2207 2208 rc = bnxt_alloc_ring(bp, ring); 2209 if (rc) 2210 return rc; 2211 } 2212 return 0; 2213 } 2214 2215 static void bnxt_init_ring_struct(struct bnxt *bp) 2216 { 2217 int i; 2218 2219 for (i = 0; i < bp->cp_nr_rings; i++) { 2220 struct bnxt_napi *bnapi = bp->bnapi[i]; 2221 struct bnxt_cp_ring_info *cpr; 2222 struct bnxt_rx_ring_info *rxr; 2223 struct bnxt_tx_ring_info *txr; 2224 struct bnxt_ring_struct *ring; 2225 2226 if (!bnapi) 2227 continue; 2228 2229 cpr = &bnapi->cp_ring; 2230 ring = &cpr->cp_ring_struct; 2231 ring->nr_pages = bp->cp_nr_pages; 2232 ring->page_size = HW_CMPD_RING_SIZE; 2233 ring->pg_arr = (void **)cpr->cp_desc_ring; 2234 ring->dma_arr = cpr->cp_desc_mapping; 2235 ring->vmem_size = 0; 2236 2237 rxr = bnapi->rx_ring; 2238 if (!rxr) 2239 goto skip_rx; 2240 2241 ring = &rxr->rx_ring_struct; 2242 ring->nr_pages = bp->rx_nr_pages; 2243 ring->page_size = HW_RXBD_RING_SIZE; 2244 ring->pg_arr = (void **)rxr->rx_desc_ring; 2245 ring->dma_arr = rxr->rx_desc_mapping; 2246 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 2247 ring->vmem = (void **)&rxr->rx_buf_ring; 2248 2249 ring = &rxr->rx_agg_ring_struct; 2250 ring->nr_pages = bp->rx_agg_nr_pages; 2251 ring->page_size = HW_RXBD_RING_SIZE; 2252 ring->pg_arr = (void **)rxr->rx_agg_desc_ring; 2253 ring->dma_arr = rxr->rx_agg_desc_mapping; 2254 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 2255 ring->vmem = (void **)&rxr->rx_agg_ring; 2256 2257 skip_rx: 2258 txr = bnapi->tx_ring; 2259 if (!txr) 2260 continue; 2261 2262 ring = &txr->tx_ring_struct; 2263 ring->nr_pages = bp->tx_nr_pages; 2264 ring->page_size = HW_RXBD_RING_SIZE; 2265 ring->pg_arr = (void **)txr->tx_desc_ring; 2266 ring->dma_arr = txr->tx_desc_mapping; 2267 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 2268 ring->vmem = (void **)&txr->tx_buf_ring; 2269 } 2270 } 2271 2272 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 2273 { 2274 int i; 2275 u32 prod; 2276 struct rx_bd **rx_buf_ring; 2277 2278 rx_buf_ring = (struct rx_bd **)ring->pg_arr; 2279 for (i = 0, prod = 0; i < ring->nr_pages; i++) { 2280 int j; 2281 struct rx_bd *rxbd; 2282 2283 rxbd = rx_buf_ring[i]; 2284 if (!rxbd) 2285 continue; 2286 2287 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 2288 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 2289 rxbd->rx_bd_opaque = prod; 2290 } 2291 } 2292 } 2293 2294 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 2295 { 2296 struct net_device *dev = bp->dev; 2297 struct bnxt_rx_ring_info *rxr; 2298 struct bnxt_ring_struct *ring; 2299 u32 prod, type; 2300 int i; 2301 2302 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 2303 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 2304 2305 if (NET_IP_ALIGN == 2) 2306 type |= RX_BD_FLAGS_SOP; 2307 2308 rxr = &bp->rx_ring[ring_nr]; 2309 ring = &rxr->rx_ring_struct; 2310 bnxt_init_rxbd_pages(ring, type); 2311 2312 prod = rxr->rx_prod; 2313 for (i = 0; i < bp->rx_ring_size; i++) { 2314 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) { 2315 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n", 2316 ring_nr, i, bp->rx_ring_size); 2317 break; 2318 } 2319 prod = NEXT_RX(prod); 2320 } 2321 rxr->rx_prod = prod; 2322 ring->fw_ring_id = INVALID_HW_RING_ID; 2323 2324 ring = &rxr->rx_agg_ring_struct; 2325 ring->fw_ring_id = INVALID_HW_RING_ID; 2326 2327 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 2328 return 0; 2329 2330 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 2331 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 2332 2333 bnxt_init_rxbd_pages(ring, type); 2334 2335 prod = rxr->rx_agg_prod; 2336 for (i = 0; i < bp->rx_agg_ring_size; i++) { 2337 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) { 2338 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n", 2339 ring_nr, i, bp->rx_ring_size); 2340 break; 2341 } 2342 prod = NEXT_RX_AGG(prod); 2343 } 2344 rxr->rx_agg_prod = prod; 2345 2346 if (bp->flags & BNXT_FLAG_TPA) { 2347 if (rxr->rx_tpa) { 2348 u8 *data; 2349 dma_addr_t mapping; 2350 2351 for (i = 0; i < MAX_TPA; i++) { 2352 data = __bnxt_alloc_rx_data(bp, &mapping, 2353 GFP_KERNEL); 2354 if (!data) 2355 return -ENOMEM; 2356 2357 rxr->rx_tpa[i].data = data; 2358 rxr->rx_tpa[i].mapping = mapping; 2359 } 2360 } else { 2361 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n"); 2362 return -ENOMEM; 2363 } 2364 } 2365 2366 return 0; 2367 } 2368 2369 static int bnxt_init_rx_rings(struct bnxt *bp) 2370 { 2371 int i, rc = 0; 2372 2373 for (i = 0; i < bp->rx_nr_rings; i++) { 2374 rc = bnxt_init_one_rx_ring(bp, i); 2375 if (rc) 2376 break; 2377 } 2378 2379 return rc; 2380 } 2381 2382 static int bnxt_init_tx_rings(struct bnxt *bp) 2383 { 2384 u16 i; 2385 2386 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 2387 MAX_SKB_FRAGS + 1); 2388 2389 for (i = 0; i < bp->tx_nr_rings; i++) { 2390 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 2391 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 2392 2393 ring->fw_ring_id = INVALID_HW_RING_ID; 2394 } 2395 2396 return 0; 2397 } 2398 2399 static void bnxt_free_ring_grps(struct bnxt *bp) 2400 { 2401 kfree(bp->grp_info); 2402 bp->grp_info = NULL; 2403 } 2404 2405 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 2406 { 2407 int i; 2408 2409 if (irq_re_init) { 2410 bp->grp_info = kcalloc(bp->cp_nr_rings, 2411 sizeof(struct bnxt_ring_grp_info), 2412 GFP_KERNEL); 2413 if (!bp->grp_info) 2414 return -ENOMEM; 2415 } 2416 for (i = 0; i < bp->cp_nr_rings; i++) { 2417 if (irq_re_init) 2418 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 2419 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 2420 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 2421 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 2422 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 2423 } 2424 return 0; 2425 } 2426 2427 static void bnxt_free_vnics(struct bnxt *bp) 2428 { 2429 kfree(bp->vnic_info); 2430 bp->vnic_info = NULL; 2431 bp->nr_vnics = 0; 2432 } 2433 2434 static int bnxt_alloc_vnics(struct bnxt *bp) 2435 { 2436 int num_vnics = 1; 2437 2438 #ifdef CONFIG_RFS_ACCEL 2439 if (bp->flags & BNXT_FLAG_RFS) 2440 num_vnics += bp->rx_nr_rings; 2441 #endif 2442 2443 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 2444 num_vnics++; 2445 2446 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 2447 GFP_KERNEL); 2448 if (!bp->vnic_info) 2449 return -ENOMEM; 2450 2451 bp->nr_vnics = num_vnics; 2452 return 0; 2453 } 2454 2455 static void bnxt_init_vnics(struct bnxt *bp) 2456 { 2457 int i; 2458 2459 for (i = 0; i < bp->nr_vnics; i++) { 2460 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 2461 2462 vnic->fw_vnic_id = INVALID_HW_RING_ID; 2463 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 2464 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 2465 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 2466 2467 if (bp->vnic_info[i].rss_hash_key) { 2468 if (i == 0) 2469 prandom_bytes(vnic->rss_hash_key, 2470 HW_HASH_KEY_SIZE); 2471 else 2472 memcpy(vnic->rss_hash_key, 2473 bp->vnic_info[0].rss_hash_key, 2474 HW_HASH_KEY_SIZE); 2475 } 2476 } 2477 } 2478 2479 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 2480 { 2481 int pages; 2482 2483 pages = ring_size / desc_per_pg; 2484 2485 if (!pages) 2486 return 1; 2487 2488 pages++; 2489 2490 while (pages & (pages - 1)) 2491 pages++; 2492 2493 return pages; 2494 } 2495 2496 static void bnxt_set_tpa_flags(struct bnxt *bp) 2497 { 2498 bp->flags &= ~BNXT_FLAG_TPA; 2499 if (bp->dev->features & NETIF_F_LRO) 2500 bp->flags |= BNXT_FLAG_LRO; 2501 if (bp->dev->features & NETIF_F_GRO) 2502 bp->flags |= BNXT_FLAG_GRO; 2503 } 2504 2505 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 2506 * be set on entry. 2507 */ 2508 void bnxt_set_ring_params(struct bnxt *bp) 2509 { 2510 u32 ring_size, rx_size, rx_space; 2511 u32 agg_factor = 0, agg_ring_size = 0; 2512 2513 /* 8 for CRC and VLAN */ 2514 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 2515 2516 rx_space = rx_size + NET_SKB_PAD + 2517 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2518 2519 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; 2520 ring_size = bp->rx_ring_size; 2521 bp->rx_agg_ring_size = 0; 2522 bp->rx_agg_nr_pages = 0; 2523 2524 if (bp->flags & BNXT_FLAG_TPA) 2525 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 2526 2527 bp->flags &= ~BNXT_FLAG_JUMBO; 2528 if (rx_space > PAGE_SIZE) { 2529 u32 jumbo_factor; 2530 2531 bp->flags |= BNXT_FLAG_JUMBO; 2532 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 2533 if (jumbo_factor > agg_factor) 2534 agg_factor = jumbo_factor; 2535 } 2536 agg_ring_size = ring_size * agg_factor; 2537 2538 if (agg_ring_size) { 2539 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 2540 RX_DESC_CNT); 2541 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 2542 u32 tmp = agg_ring_size; 2543 2544 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 2545 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 2546 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 2547 tmp, agg_ring_size); 2548 } 2549 bp->rx_agg_ring_size = agg_ring_size; 2550 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 2551 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); 2552 rx_space = rx_size + NET_SKB_PAD + 2553 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2554 } 2555 2556 bp->rx_buf_use_size = rx_size; 2557 bp->rx_buf_size = rx_space; 2558 2559 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 2560 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 2561 2562 ring_size = bp->tx_ring_size; 2563 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 2564 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 2565 2566 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size; 2567 bp->cp_ring_size = ring_size; 2568 2569 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 2570 if (bp->cp_nr_pages > MAX_CP_PAGES) { 2571 bp->cp_nr_pages = MAX_CP_PAGES; 2572 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 2573 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 2574 ring_size, bp->cp_ring_size); 2575 } 2576 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 2577 bp->cp_ring_mask = bp->cp_bit - 1; 2578 } 2579 2580 static void bnxt_free_vnic_attributes(struct bnxt *bp) 2581 { 2582 int i; 2583 struct bnxt_vnic_info *vnic; 2584 struct pci_dev *pdev = bp->pdev; 2585 2586 if (!bp->vnic_info) 2587 return; 2588 2589 for (i = 0; i < bp->nr_vnics; i++) { 2590 vnic = &bp->vnic_info[i]; 2591 2592 kfree(vnic->fw_grp_ids); 2593 vnic->fw_grp_ids = NULL; 2594 2595 kfree(vnic->uc_list); 2596 vnic->uc_list = NULL; 2597 2598 if (vnic->mc_list) { 2599 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 2600 vnic->mc_list, vnic->mc_list_mapping); 2601 vnic->mc_list = NULL; 2602 } 2603 2604 if (vnic->rss_table) { 2605 dma_free_coherent(&pdev->dev, PAGE_SIZE, 2606 vnic->rss_table, 2607 vnic->rss_table_dma_addr); 2608 vnic->rss_table = NULL; 2609 } 2610 2611 vnic->rss_hash_key = NULL; 2612 vnic->flags = 0; 2613 } 2614 } 2615 2616 static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 2617 { 2618 int i, rc = 0, size; 2619 struct bnxt_vnic_info *vnic; 2620 struct pci_dev *pdev = bp->pdev; 2621 int max_rings; 2622 2623 for (i = 0; i < bp->nr_vnics; i++) { 2624 vnic = &bp->vnic_info[i]; 2625 2626 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 2627 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 2628 2629 if (mem_size > 0) { 2630 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 2631 if (!vnic->uc_list) { 2632 rc = -ENOMEM; 2633 goto out; 2634 } 2635 } 2636 } 2637 2638 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 2639 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 2640 vnic->mc_list = 2641 dma_alloc_coherent(&pdev->dev, 2642 vnic->mc_list_size, 2643 &vnic->mc_list_mapping, 2644 GFP_KERNEL); 2645 if (!vnic->mc_list) { 2646 rc = -ENOMEM; 2647 goto out; 2648 } 2649 } 2650 2651 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 2652 max_rings = bp->rx_nr_rings; 2653 else 2654 max_rings = 1; 2655 2656 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 2657 if (!vnic->fw_grp_ids) { 2658 rc = -ENOMEM; 2659 goto out; 2660 } 2661 2662 /* Allocate rss table and hash key */ 2663 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2664 &vnic->rss_table_dma_addr, 2665 GFP_KERNEL); 2666 if (!vnic->rss_table) { 2667 rc = -ENOMEM; 2668 goto out; 2669 } 2670 2671 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 2672 2673 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 2674 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 2675 } 2676 return 0; 2677 2678 out: 2679 return rc; 2680 } 2681 2682 static void bnxt_free_hwrm_resources(struct bnxt *bp) 2683 { 2684 struct pci_dev *pdev = bp->pdev; 2685 2686 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr, 2687 bp->hwrm_cmd_resp_dma_addr); 2688 2689 bp->hwrm_cmd_resp_addr = NULL; 2690 if (bp->hwrm_dbg_resp_addr) { 2691 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE, 2692 bp->hwrm_dbg_resp_addr, 2693 bp->hwrm_dbg_resp_dma_addr); 2694 2695 bp->hwrm_dbg_resp_addr = NULL; 2696 } 2697 } 2698 2699 static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 2700 { 2701 struct pci_dev *pdev = bp->pdev; 2702 2703 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, 2704 &bp->hwrm_cmd_resp_dma_addr, 2705 GFP_KERNEL); 2706 if (!bp->hwrm_cmd_resp_addr) 2707 return -ENOMEM; 2708 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev, 2709 HWRM_DBG_REG_BUF_SIZE, 2710 &bp->hwrm_dbg_resp_dma_addr, 2711 GFP_KERNEL); 2712 if (!bp->hwrm_dbg_resp_addr) 2713 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n"); 2714 2715 return 0; 2716 } 2717 2718 static void bnxt_free_stats(struct bnxt *bp) 2719 { 2720 u32 size, i; 2721 struct pci_dev *pdev = bp->pdev; 2722 2723 if (bp->hw_rx_port_stats) { 2724 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size, 2725 bp->hw_rx_port_stats, 2726 bp->hw_rx_port_stats_map); 2727 bp->hw_rx_port_stats = NULL; 2728 bp->flags &= ~BNXT_FLAG_PORT_STATS; 2729 } 2730 2731 if (!bp->bnapi) 2732 return; 2733 2734 size = sizeof(struct ctx_hw_stats); 2735 2736 for (i = 0; i < bp->cp_nr_rings; i++) { 2737 struct bnxt_napi *bnapi = bp->bnapi[i]; 2738 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2739 2740 if (cpr->hw_stats) { 2741 dma_free_coherent(&pdev->dev, size, cpr->hw_stats, 2742 cpr->hw_stats_map); 2743 cpr->hw_stats = NULL; 2744 } 2745 } 2746 } 2747 2748 static int bnxt_alloc_stats(struct bnxt *bp) 2749 { 2750 u32 size, i; 2751 struct pci_dev *pdev = bp->pdev; 2752 2753 size = sizeof(struct ctx_hw_stats); 2754 2755 for (i = 0; i < bp->cp_nr_rings; i++) { 2756 struct bnxt_napi *bnapi = bp->bnapi[i]; 2757 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2758 2759 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size, 2760 &cpr->hw_stats_map, 2761 GFP_KERNEL); 2762 if (!cpr->hw_stats) 2763 return -ENOMEM; 2764 2765 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 2766 } 2767 2768 if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) { 2769 bp->hw_port_stats_size = sizeof(struct rx_port_stats) + 2770 sizeof(struct tx_port_stats) + 1024; 2771 2772 bp->hw_rx_port_stats = 2773 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size, 2774 &bp->hw_rx_port_stats_map, 2775 GFP_KERNEL); 2776 if (!bp->hw_rx_port_stats) 2777 return -ENOMEM; 2778 2779 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 2780 512; 2781 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map + 2782 sizeof(struct rx_port_stats) + 512; 2783 bp->flags |= BNXT_FLAG_PORT_STATS; 2784 } 2785 return 0; 2786 } 2787 2788 static void bnxt_clear_ring_indices(struct bnxt *bp) 2789 { 2790 int i; 2791 2792 if (!bp->bnapi) 2793 return; 2794 2795 for (i = 0; i < bp->cp_nr_rings; i++) { 2796 struct bnxt_napi *bnapi = bp->bnapi[i]; 2797 struct bnxt_cp_ring_info *cpr; 2798 struct bnxt_rx_ring_info *rxr; 2799 struct bnxt_tx_ring_info *txr; 2800 2801 if (!bnapi) 2802 continue; 2803 2804 cpr = &bnapi->cp_ring; 2805 cpr->cp_raw_cons = 0; 2806 2807 txr = bnapi->tx_ring; 2808 if (txr) { 2809 txr->tx_prod = 0; 2810 txr->tx_cons = 0; 2811 } 2812 2813 rxr = bnapi->rx_ring; 2814 if (rxr) { 2815 rxr->rx_prod = 0; 2816 rxr->rx_agg_prod = 0; 2817 rxr->rx_sw_agg_prod = 0; 2818 rxr->rx_next_cons = 0; 2819 } 2820 } 2821 } 2822 2823 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit) 2824 { 2825 #ifdef CONFIG_RFS_ACCEL 2826 int i; 2827 2828 /* Under rtnl_lock and all our NAPIs have been disabled. It's 2829 * safe to delete the hash table. 2830 */ 2831 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 2832 struct hlist_head *head; 2833 struct hlist_node *tmp; 2834 struct bnxt_ntuple_filter *fltr; 2835 2836 head = &bp->ntp_fltr_hash_tbl[i]; 2837 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 2838 hlist_del(&fltr->hash); 2839 kfree(fltr); 2840 } 2841 } 2842 if (irq_reinit) { 2843 kfree(bp->ntp_fltr_bmap); 2844 bp->ntp_fltr_bmap = NULL; 2845 } 2846 bp->ntp_fltr_count = 0; 2847 #endif 2848 } 2849 2850 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 2851 { 2852 #ifdef CONFIG_RFS_ACCEL 2853 int i, rc = 0; 2854 2855 if (!(bp->flags & BNXT_FLAG_RFS)) 2856 return 0; 2857 2858 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 2859 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 2860 2861 bp->ntp_fltr_count = 0; 2862 bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR), 2863 GFP_KERNEL); 2864 2865 if (!bp->ntp_fltr_bmap) 2866 rc = -ENOMEM; 2867 2868 return rc; 2869 #else 2870 return 0; 2871 #endif 2872 } 2873 2874 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 2875 { 2876 bnxt_free_vnic_attributes(bp); 2877 bnxt_free_tx_rings(bp); 2878 bnxt_free_rx_rings(bp); 2879 bnxt_free_cp_rings(bp); 2880 bnxt_free_ntp_fltrs(bp, irq_re_init); 2881 if (irq_re_init) { 2882 bnxt_free_stats(bp); 2883 bnxt_free_ring_grps(bp); 2884 bnxt_free_vnics(bp); 2885 kfree(bp->tx_ring); 2886 bp->tx_ring = NULL; 2887 kfree(bp->rx_ring); 2888 bp->rx_ring = NULL; 2889 kfree(bp->bnapi); 2890 bp->bnapi = NULL; 2891 } else { 2892 bnxt_clear_ring_indices(bp); 2893 } 2894 } 2895 2896 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 2897 { 2898 int i, j, rc, size, arr_size; 2899 void *bnapi; 2900 2901 if (irq_re_init) { 2902 /* Allocate bnapi mem pointer array and mem block for 2903 * all queues 2904 */ 2905 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 2906 bp->cp_nr_rings); 2907 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 2908 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 2909 if (!bnapi) 2910 return -ENOMEM; 2911 2912 bp->bnapi = bnapi; 2913 bnapi += arr_size; 2914 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 2915 bp->bnapi[i] = bnapi; 2916 bp->bnapi[i]->index = i; 2917 bp->bnapi[i]->bp = bp; 2918 } 2919 2920 bp->rx_ring = kcalloc(bp->rx_nr_rings, 2921 sizeof(struct bnxt_rx_ring_info), 2922 GFP_KERNEL); 2923 if (!bp->rx_ring) 2924 return -ENOMEM; 2925 2926 for (i = 0; i < bp->rx_nr_rings; i++) { 2927 bp->rx_ring[i].bnapi = bp->bnapi[i]; 2928 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 2929 } 2930 2931 bp->tx_ring = kcalloc(bp->tx_nr_rings, 2932 sizeof(struct bnxt_tx_ring_info), 2933 GFP_KERNEL); 2934 if (!bp->tx_ring) 2935 return -ENOMEM; 2936 2937 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 2938 j = 0; 2939 else 2940 j = bp->rx_nr_rings; 2941 2942 for (i = 0; i < bp->tx_nr_rings; i++, j++) { 2943 bp->tx_ring[i].bnapi = bp->bnapi[j]; 2944 bp->bnapi[j]->tx_ring = &bp->tx_ring[i]; 2945 } 2946 2947 rc = bnxt_alloc_stats(bp); 2948 if (rc) 2949 goto alloc_mem_err; 2950 2951 rc = bnxt_alloc_ntp_fltrs(bp); 2952 if (rc) 2953 goto alloc_mem_err; 2954 2955 rc = bnxt_alloc_vnics(bp); 2956 if (rc) 2957 goto alloc_mem_err; 2958 } 2959 2960 bnxt_init_ring_struct(bp); 2961 2962 rc = bnxt_alloc_rx_rings(bp); 2963 if (rc) 2964 goto alloc_mem_err; 2965 2966 rc = bnxt_alloc_tx_rings(bp); 2967 if (rc) 2968 goto alloc_mem_err; 2969 2970 rc = bnxt_alloc_cp_rings(bp); 2971 if (rc) 2972 goto alloc_mem_err; 2973 2974 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG | 2975 BNXT_VNIC_UCAST_FLAG; 2976 rc = bnxt_alloc_vnic_attributes(bp); 2977 if (rc) 2978 goto alloc_mem_err; 2979 return 0; 2980 2981 alloc_mem_err: 2982 bnxt_free_mem(bp, true); 2983 return rc; 2984 } 2985 2986 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type, 2987 u16 cmpl_ring, u16 target_id) 2988 { 2989 struct input *req = request; 2990 2991 req->req_type = cpu_to_le16(req_type); 2992 req->cmpl_ring = cpu_to_le16(cmpl_ring); 2993 req->target_id = cpu_to_le16(target_id); 2994 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr); 2995 } 2996 2997 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len, 2998 int timeout, bool silent) 2999 { 3000 int i, intr_process, rc, tmo_count; 3001 struct input *req = msg; 3002 u32 *data = msg; 3003 __le32 *resp_len, *valid; 3004 u16 cp_ring_id, len = 0; 3005 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr; 3006 3007 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++); 3008 memset(resp, 0, PAGE_SIZE); 3009 cp_ring_id = le16_to_cpu(req->cmpl_ring); 3010 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1; 3011 3012 /* Write request msg to hwrm channel */ 3013 __iowrite32_copy(bp->bar0, data, msg_len / 4); 3014 3015 for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4) 3016 writel(0, bp->bar0 + i); 3017 3018 /* currently supports only one outstanding message */ 3019 if (intr_process) 3020 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id); 3021 3022 /* Ring channel doorbell */ 3023 writel(1, bp->bar0 + 0x100); 3024 3025 if (!timeout) 3026 timeout = DFLT_HWRM_CMD_TIMEOUT; 3027 3028 i = 0; 3029 tmo_count = timeout * 40; 3030 if (intr_process) { 3031 /* Wait until hwrm response cmpl interrupt is processed */ 3032 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID && 3033 i++ < tmo_count) { 3034 usleep_range(25, 40); 3035 } 3036 3037 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) { 3038 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n", 3039 le16_to_cpu(req->req_type)); 3040 return -1; 3041 } 3042 } else { 3043 /* Check if response len is updated */ 3044 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET; 3045 for (i = 0; i < tmo_count; i++) { 3046 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >> 3047 HWRM_RESP_LEN_SFT; 3048 if (len) 3049 break; 3050 usleep_range(25, 40); 3051 } 3052 3053 if (i >= tmo_count) { 3054 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n", 3055 timeout, le16_to_cpu(req->req_type), 3056 le16_to_cpu(req->seq_id), len); 3057 return -1; 3058 } 3059 3060 /* Last word of resp contains valid bit */ 3061 valid = bp->hwrm_cmd_resp_addr + len - 4; 3062 for (i = 0; i < 5; i++) { 3063 if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK) 3064 break; 3065 udelay(1); 3066 } 3067 3068 if (i >= 5) { 3069 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n", 3070 timeout, le16_to_cpu(req->req_type), 3071 le16_to_cpu(req->seq_id), len, *valid); 3072 return -1; 3073 } 3074 } 3075 3076 rc = le16_to_cpu(resp->error_code); 3077 if (rc && !silent) 3078 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n", 3079 le16_to_cpu(resp->req_type), 3080 le16_to_cpu(resp->seq_id), rc); 3081 return rc; 3082 } 3083 3084 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3085 { 3086 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false); 3087 } 3088 3089 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout) 3090 { 3091 int rc; 3092 3093 mutex_lock(&bp->hwrm_cmd_lock); 3094 rc = _hwrm_send_message(bp, msg, msg_len, timeout); 3095 mutex_unlock(&bp->hwrm_cmd_lock); 3096 return rc; 3097 } 3098 3099 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len, 3100 int timeout) 3101 { 3102 int rc; 3103 3104 mutex_lock(&bp->hwrm_cmd_lock); 3105 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true); 3106 mutex_unlock(&bp->hwrm_cmd_lock); 3107 return rc; 3108 } 3109 3110 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp) 3111 { 3112 struct hwrm_func_drv_rgtr_input req = {0}; 3113 int i; 3114 DECLARE_BITMAP(async_events_bmap, 256); 3115 u32 *events = (u32 *)async_events_bmap; 3116 3117 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1); 3118 3119 req.enables = 3120 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 3121 FUNC_DRV_RGTR_REQ_ENABLES_VER | 3122 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 3123 3124 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 3125 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) 3126 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 3127 3128 for (i = 0; i < 8; i++) 3129 req.async_event_fwd[i] |= cpu_to_le32(events[i]); 3130 3131 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 3132 req.ver_maj = DRV_VER_MAJ; 3133 req.ver_min = DRV_VER_MIN; 3134 req.ver_upd = DRV_VER_UPD; 3135 3136 if (BNXT_PF(bp)) { 3137 DECLARE_BITMAP(vf_req_snif_bmap, 256); 3138 u32 *data = (u32 *)vf_req_snif_bmap; 3139 3140 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap)); 3141 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) 3142 __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap); 3143 3144 for (i = 0; i < 8; i++) 3145 req.vf_req_fwd[i] = cpu_to_le32(data[i]); 3146 3147 req.enables |= 3148 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 3149 } 3150 3151 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3152 } 3153 3154 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 3155 { 3156 struct hwrm_func_drv_unrgtr_input req = {0}; 3157 3158 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1); 3159 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3160 } 3161 3162 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 3163 { 3164 u32 rc = 0; 3165 struct hwrm_tunnel_dst_port_free_input req = {0}; 3166 3167 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1); 3168 req.tunnel_type = tunnel_type; 3169 3170 switch (tunnel_type) { 3171 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 3172 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id; 3173 break; 3174 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 3175 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id; 3176 break; 3177 default: 3178 break; 3179 } 3180 3181 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3182 if (rc) 3183 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 3184 rc); 3185 return rc; 3186 } 3187 3188 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 3189 u8 tunnel_type) 3190 { 3191 u32 rc = 0; 3192 struct hwrm_tunnel_dst_port_alloc_input req = {0}; 3193 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3194 3195 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1); 3196 3197 req.tunnel_type = tunnel_type; 3198 req.tunnel_dst_port_val = port; 3199 3200 mutex_lock(&bp->hwrm_cmd_lock); 3201 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3202 if (rc) { 3203 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 3204 rc); 3205 goto err_out; 3206 } 3207 3208 if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN) 3209 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; 3210 3211 else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE) 3212 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id; 3213 err_out: 3214 mutex_unlock(&bp->hwrm_cmd_lock); 3215 return rc; 3216 } 3217 3218 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 3219 { 3220 struct hwrm_cfa_l2_set_rx_mask_input req = {0}; 3221 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3222 3223 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1); 3224 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3225 3226 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count); 3227 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 3228 req.mask = cpu_to_le32(vnic->rx_mask); 3229 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3230 } 3231 3232 #ifdef CONFIG_RFS_ACCEL 3233 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 3234 struct bnxt_ntuple_filter *fltr) 3235 { 3236 struct hwrm_cfa_ntuple_filter_free_input req = {0}; 3237 3238 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1); 3239 req.ntuple_filter_id = fltr->filter_id; 3240 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3241 } 3242 3243 #define BNXT_NTP_FLTR_FLAGS \ 3244 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 3245 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 3246 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \ 3247 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 3248 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 3249 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 3250 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 3251 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 3252 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 3253 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 3254 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 3255 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 3256 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 3257 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 3258 3259 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 3260 struct bnxt_ntuple_filter *fltr) 3261 { 3262 int rc = 0; 3263 struct hwrm_cfa_ntuple_filter_alloc_input req = {0}; 3264 struct hwrm_cfa_ntuple_filter_alloc_output *resp = 3265 bp->hwrm_cmd_resp_addr; 3266 struct flow_keys *keys = &fltr->fkeys; 3267 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1]; 3268 3269 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1); 3270 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx]; 3271 3272 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 3273 3274 req.ethertype = htons(ETH_P_IP); 3275 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN); 3276 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 3277 req.ip_protocol = keys->basic.ip_proto; 3278 3279 req.src_ipaddr[0] = keys->addrs.v4addrs.src; 3280 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3281 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst; 3282 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff); 3283 3284 req.src_port = keys->ports.src; 3285 req.src_port_mask = cpu_to_be16(0xffff); 3286 req.dst_port = keys->ports.dst; 3287 req.dst_port_mask = cpu_to_be16(0xffff); 3288 3289 req.dst_id = cpu_to_le16(vnic->fw_vnic_id); 3290 mutex_lock(&bp->hwrm_cmd_lock); 3291 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3292 if (!rc) 3293 fltr->filter_id = resp->ntuple_filter_id; 3294 mutex_unlock(&bp->hwrm_cmd_lock); 3295 return rc; 3296 } 3297 #endif 3298 3299 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 3300 u8 *mac_addr) 3301 { 3302 u32 rc = 0; 3303 struct hwrm_cfa_l2_filter_alloc_input req = {0}; 3304 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3305 3306 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1); 3307 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 3308 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 3309 req.flags |= 3310 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 3311 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id); 3312 req.enables = 3313 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 3314 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 3315 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 3316 memcpy(req.l2_addr, mac_addr, ETH_ALEN); 3317 req.l2_addr_mask[0] = 0xff; 3318 req.l2_addr_mask[1] = 0xff; 3319 req.l2_addr_mask[2] = 0xff; 3320 req.l2_addr_mask[3] = 0xff; 3321 req.l2_addr_mask[4] = 0xff; 3322 req.l2_addr_mask[5] = 0xff; 3323 3324 mutex_lock(&bp->hwrm_cmd_lock); 3325 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3326 if (!rc) 3327 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] = 3328 resp->l2_filter_id; 3329 mutex_unlock(&bp->hwrm_cmd_lock); 3330 return rc; 3331 } 3332 3333 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 3334 { 3335 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 3336 int rc = 0; 3337 3338 /* Any associated ntuple filters will also be cleared by firmware. */ 3339 mutex_lock(&bp->hwrm_cmd_lock); 3340 for (i = 0; i < num_of_vnics; i++) { 3341 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3342 3343 for (j = 0; j < vnic->uc_filter_count; j++) { 3344 struct hwrm_cfa_l2_filter_free_input req = {0}; 3345 3346 bnxt_hwrm_cmd_hdr_init(bp, &req, 3347 HWRM_CFA_L2_FILTER_FREE, -1, -1); 3348 3349 req.l2_filter_id = vnic->fw_l2_filter_id[j]; 3350 3351 rc = _hwrm_send_message(bp, &req, sizeof(req), 3352 HWRM_CMD_TIMEOUT); 3353 } 3354 vnic->uc_filter_count = 0; 3355 } 3356 mutex_unlock(&bp->hwrm_cmd_lock); 3357 3358 return rc; 3359 } 3360 3361 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags) 3362 { 3363 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3364 struct hwrm_vnic_tpa_cfg_input req = {0}; 3365 3366 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1); 3367 3368 if (tpa_flags) { 3369 u16 mss = bp->dev->mtu - 40; 3370 u32 nsegs, n, segs = 0, flags; 3371 3372 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 3373 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 3374 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 3375 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 3376 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 3377 if (tpa_flags & BNXT_FLAG_GRO) 3378 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 3379 3380 req.flags = cpu_to_le32(flags); 3381 3382 req.enables = 3383 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 3384 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 3385 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 3386 3387 /* Number of segs are log2 units, and first packet is not 3388 * included as part of this units. 3389 */ 3390 if (mss <= BNXT_RX_PAGE_SIZE) { 3391 n = BNXT_RX_PAGE_SIZE / mss; 3392 nsegs = (MAX_SKB_FRAGS - 1) * n; 3393 } else { 3394 n = mss / BNXT_RX_PAGE_SIZE; 3395 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 3396 n++; 3397 nsegs = (MAX_SKB_FRAGS - n) / n; 3398 } 3399 3400 segs = ilog2(nsegs); 3401 req.max_agg_segs = cpu_to_le16(segs); 3402 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX); 3403 3404 req.min_agg_len = cpu_to_le32(512); 3405 } 3406 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3407 3408 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3409 } 3410 3411 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss) 3412 { 3413 u32 i, j, max_rings; 3414 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3415 struct hwrm_vnic_rss_cfg_input req = {0}; 3416 3417 if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 3418 return 0; 3419 3420 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1); 3421 if (set_rss) { 3422 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 | 3423 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 | 3424 BNXT_RSS_HASH_TYPE_FLAG_IPV6 | 3425 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6; 3426 3427 req.hash_type = cpu_to_le32(vnic->hash_type); 3428 3429 if (vnic->flags & BNXT_VNIC_RSS_FLAG) { 3430 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 3431 max_rings = bp->rx_nr_rings - 1; 3432 else 3433 max_rings = bp->rx_nr_rings; 3434 } else { 3435 max_rings = 1; 3436 } 3437 3438 /* Fill the RSS indirection table with ring group ids */ 3439 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) { 3440 if (j == max_rings) 3441 j = 0; 3442 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 3443 } 3444 3445 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 3446 req.hash_key_tbl_addr = 3447 cpu_to_le64(vnic->rss_hash_key_dma_addr); 3448 } 3449 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 3450 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3451 } 3452 3453 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) 3454 { 3455 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3456 struct hwrm_vnic_plcmodes_cfg_input req = {0}; 3457 3458 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1); 3459 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | 3460 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 3461 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 3462 req.enables = 3463 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | 3464 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 3465 /* thresholds not implemented in firmware yet */ 3466 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); 3467 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh); 3468 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id); 3469 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3470 } 3471 3472 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id, 3473 u16 ctx_idx) 3474 { 3475 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; 3476 3477 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1); 3478 req.rss_cos_lb_ctx_id = 3479 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]); 3480 3481 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3482 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 3483 } 3484 3485 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 3486 { 3487 int i, j; 3488 3489 for (i = 0; i < bp->nr_vnics; i++) { 3490 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 3491 3492 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 3493 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 3494 bnxt_hwrm_vnic_ctx_free_one(bp, i, j); 3495 } 3496 } 3497 bp->rsscos_nr_ctxs = 0; 3498 } 3499 3500 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx) 3501 { 3502 int rc; 3503 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; 3504 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = 3505 bp->hwrm_cmd_resp_addr; 3506 3507 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1, 3508 -1); 3509 3510 mutex_lock(&bp->hwrm_cmd_lock); 3511 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3512 if (!rc) 3513 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = 3514 le16_to_cpu(resp->rss_cos_lb_ctx_id); 3515 mutex_unlock(&bp->hwrm_cmd_lock); 3516 3517 return rc; 3518 } 3519 3520 static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id) 3521 { 3522 unsigned int ring = 0, grp_idx; 3523 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 3524 struct hwrm_vnic_cfg_input req = {0}; 3525 u16 def_vlan = 0; 3526 3527 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1); 3528 3529 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 3530 /* Only RSS support for now TBD: COS & LB */ 3531 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 3532 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 3533 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 3534 VNIC_CFG_REQ_ENABLES_MRU); 3535 } else { 3536 req.rss_rule = cpu_to_le16(0xffff); 3537 } 3538 3539 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 3540 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 3541 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 3542 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 3543 } else { 3544 req.cos_rule = cpu_to_le16(0xffff); 3545 } 3546 3547 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 3548 ring = 0; 3549 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 3550 ring = vnic_id - 1; 3551 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 3552 ring = bp->rx_nr_rings - 1; 3553 3554 grp_idx = bp->rx_ring[ring].bnapi->index; 3555 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id); 3556 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 3557 3558 req.lb_rule = cpu_to_le16(0xffff); 3559 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + 3560 VLAN_HLEN); 3561 3562 #ifdef CONFIG_BNXT_SRIOV 3563 if (BNXT_VF(bp)) 3564 def_vlan = bp->vf.vlan; 3565 #endif 3566 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 3567 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 3568 3569 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3570 } 3571 3572 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id) 3573 { 3574 u32 rc = 0; 3575 3576 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) { 3577 struct hwrm_vnic_free_input req = {0}; 3578 3579 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1); 3580 req.vnic_id = 3581 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id); 3582 3583 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3584 if (rc) 3585 return rc; 3586 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID; 3587 } 3588 return rc; 3589 } 3590 3591 static void bnxt_hwrm_vnic_free(struct bnxt *bp) 3592 { 3593 u16 i; 3594 3595 for (i = 0; i < bp->nr_vnics; i++) 3596 bnxt_hwrm_vnic_free_one(bp, i); 3597 } 3598 3599 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id, 3600 unsigned int start_rx_ring_idx, 3601 unsigned int nr_rings) 3602 { 3603 int rc = 0; 3604 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 3605 struct hwrm_vnic_alloc_input req = {0}; 3606 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3607 3608 /* map ring groups to this vnic */ 3609 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 3610 grp_idx = bp->rx_ring[i].bnapi->index; 3611 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 3612 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 3613 j, nr_rings); 3614 break; 3615 } 3616 bp->vnic_info[vnic_id].fw_grp_ids[j] = 3617 bp->grp_info[grp_idx].fw_grp_id; 3618 } 3619 3620 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID; 3621 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID; 3622 if (vnic_id == 0) 3623 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 3624 3625 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1); 3626 3627 mutex_lock(&bp->hwrm_cmd_lock); 3628 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3629 if (!rc) 3630 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id); 3631 mutex_unlock(&bp->hwrm_cmd_lock); 3632 return rc; 3633 } 3634 3635 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 3636 { 3637 u16 i; 3638 u32 rc = 0; 3639 3640 mutex_lock(&bp->hwrm_cmd_lock); 3641 for (i = 0; i < bp->rx_nr_rings; i++) { 3642 struct hwrm_ring_grp_alloc_input req = {0}; 3643 struct hwrm_ring_grp_alloc_output *resp = 3644 bp->hwrm_cmd_resp_addr; 3645 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 3646 3647 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1); 3648 3649 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 3650 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 3651 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 3652 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 3653 3654 rc = _hwrm_send_message(bp, &req, sizeof(req), 3655 HWRM_CMD_TIMEOUT); 3656 if (rc) 3657 break; 3658 3659 bp->grp_info[grp_idx].fw_grp_id = 3660 le32_to_cpu(resp->ring_group_id); 3661 } 3662 mutex_unlock(&bp->hwrm_cmd_lock); 3663 return rc; 3664 } 3665 3666 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp) 3667 { 3668 u16 i; 3669 u32 rc = 0; 3670 struct hwrm_ring_grp_free_input req = {0}; 3671 3672 if (!bp->grp_info) 3673 return 0; 3674 3675 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1); 3676 3677 mutex_lock(&bp->hwrm_cmd_lock); 3678 for (i = 0; i < bp->cp_nr_rings; i++) { 3679 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 3680 continue; 3681 req.ring_group_id = 3682 cpu_to_le32(bp->grp_info[i].fw_grp_id); 3683 3684 rc = _hwrm_send_message(bp, &req, sizeof(req), 3685 HWRM_CMD_TIMEOUT); 3686 if (rc) 3687 break; 3688 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 3689 } 3690 mutex_unlock(&bp->hwrm_cmd_lock); 3691 return rc; 3692 } 3693 3694 static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 3695 struct bnxt_ring_struct *ring, 3696 u32 ring_type, u32 map_index, 3697 u32 stats_ctx_id) 3698 { 3699 int rc = 0, err = 0; 3700 struct hwrm_ring_alloc_input req = {0}; 3701 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; 3702 u16 ring_id; 3703 3704 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1); 3705 3706 req.enables = 0; 3707 if (ring->nr_pages > 1) { 3708 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map); 3709 /* Page size is in log2 units */ 3710 req.page_size = BNXT_PAGE_SHIFT; 3711 req.page_tbl_depth = 1; 3712 } else { 3713 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]); 3714 } 3715 req.fbo = 0; 3716 /* Association of ring index with doorbell index and MSIX number */ 3717 req.logical_id = cpu_to_le16(map_index); 3718 3719 switch (ring_type) { 3720 case HWRM_RING_ALLOC_TX: 3721 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 3722 /* Association of transmit ring with completion ring */ 3723 req.cmpl_ring_id = 3724 cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id); 3725 req.length = cpu_to_le32(bp->tx_ring_mask + 1); 3726 req.stat_ctx_id = cpu_to_le32(stats_ctx_id); 3727 req.queue_id = cpu_to_le16(ring->queue_id); 3728 break; 3729 case HWRM_RING_ALLOC_RX: 3730 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 3731 req.length = cpu_to_le32(bp->rx_ring_mask + 1); 3732 break; 3733 case HWRM_RING_ALLOC_AGG: 3734 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 3735 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1); 3736 break; 3737 case HWRM_RING_ALLOC_CMPL: 3738 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL; 3739 req.length = cpu_to_le32(bp->cp_ring_mask + 1); 3740 if (bp->flags & BNXT_FLAG_USING_MSIX) 3741 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 3742 break; 3743 default: 3744 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 3745 ring_type); 3746 return -1; 3747 } 3748 3749 mutex_lock(&bp->hwrm_cmd_lock); 3750 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3751 err = le16_to_cpu(resp->error_code); 3752 ring_id = le16_to_cpu(resp->ring_id); 3753 mutex_unlock(&bp->hwrm_cmd_lock); 3754 3755 if (rc || err) { 3756 switch (ring_type) { 3757 case RING_FREE_REQ_RING_TYPE_CMPL: 3758 netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n", 3759 rc, err); 3760 return -1; 3761 3762 case RING_FREE_REQ_RING_TYPE_RX: 3763 netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n", 3764 rc, err); 3765 return -1; 3766 3767 case RING_FREE_REQ_RING_TYPE_TX: 3768 netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n", 3769 rc, err); 3770 return -1; 3771 3772 default: 3773 netdev_err(bp->dev, "Invalid ring\n"); 3774 return -1; 3775 } 3776 } 3777 ring->fw_ring_id = ring_id; 3778 return rc; 3779 } 3780 3781 static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 3782 { 3783 int i, rc = 0; 3784 3785 for (i = 0; i < bp->cp_nr_rings; i++) { 3786 struct bnxt_napi *bnapi = bp->bnapi[i]; 3787 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3788 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3789 3790 cpr->cp_doorbell = bp->bar1 + i * 0x80; 3791 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i, 3792 INVALID_STATS_CTX_ID); 3793 if (rc) 3794 goto err_out; 3795 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 3796 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 3797 } 3798 3799 for (i = 0; i < bp->tx_nr_rings; i++) { 3800 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3801 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3802 u32 map_idx = txr->bnapi->index; 3803 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx; 3804 3805 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX, 3806 map_idx, fw_stats_ctx); 3807 if (rc) 3808 goto err_out; 3809 txr->tx_doorbell = bp->bar1 + map_idx * 0x80; 3810 } 3811 3812 for (i = 0; i < bp->rx_nr_rings; i++) { 3813 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3814 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 3815 u32 map_idx = rxr->bnapi->index; 3816 3817 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX, 3818 map_idx, INVALID_STATS_CTX_ID); 3819 if (rc) 3820 goto err_out; 3821 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80; 3822 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell); 3823 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 3824 } 3825 3826 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 3827 for (i = 0; i < bp->rx_nr_rings; i++) { 3828 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3829 struct bnxt_ring_struct *ring = 3830 &rxr->rx_agg_ring_struct; 3831 u32 grp_idx = rxr->bnapi->index; 3832 u32 map_idx = grp_idx + bp->rx_nr_rings; 3833 3834 rc = hwrm_ring_alloc_send_msg(bp, ring, 3835 HWRM_RING_ALLOC_AGG, 3836 map_idx, 3837 INVALID_STATS_CTX_ID); 3838 if (rc) 3839 goto err_out; 3840 3841 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80; 3842 writel(DB_KEY_RX | rxr->rx_agg_prod, 3843 rxr->rx_agg_doorbell); 3844 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 3845 } 3846 } 3847 err_out: 3848 return rc; 3849 } 3850 3851 static int hwrm_ring_free_send_msg(struct bnxt *bp, 3852 struct bnxt_ring_struct *ring, 3853 u32 ring_type, int cmpl_ring_id) 3854 { 3855 int rc; 3856 struct hwrm_ring_free_input req = {0}; 3857 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; 3858 u16 error_code; 3859 3860 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1); 3861 req.ring_type = ring_type; 3862 req.ring_id = cpu_to_le16(ring->fw_ring_id); 3863 3864 mutex_lock(&bp->hwrm_cmd_lock); 3865 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 3866 error_code = le16_to_cpu(resp->error_code); 3867 mutex_unlock(&bp->hwrm_cmd_lock); 3868 3869 if (rc || error_code) { 3870 switch (ring_type) { 3871 case RING_FREE_REQ_RING_TYPE_CMPL: 3872 netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n", 3873 rc); 3874 return rc; 3875 case RING_FREE_REQ_RING_TYPE_RX: 3876 netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n", 3877 rc); 3878 return rc; 3879 case RING_FREE_REQ_RING_TYPE_TX: 3880 netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n", 3881 rc); 3882 return rc; 3883 default: 3884 netdev_err(bp->dev, "Invalid ring\n"); 3885 return -1; 3886 } 3887 } 3888 return 0; 3889 } 3890 3891 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 3892 { 3893 int i; 3894 3895 if (!bp->bnapi) 3896 return; 3897 3898 for (i = 0; i < bp->tx_nr_rings; i++) { 3899 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3900 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 3901 u32 grp_idx = txr->bnapi->index; 3902 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 3903 3904 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 3905 hwrm_ring_free_send_msg(bp, ring, 3906 RING_FREE_REQ_RING_TYPE_TX, 3907 close_path ? cmpl_ring_id : 3908 INVALID_HW_RING_ID); 3909 ring->fw_ring_id = INVALID_HW_RING_ID; 3910 } 3911 } 3912 3913 for (i = 0; i < bp->rx_nr_rings; i++) { 3914 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3915 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 3916 u32 grp_idx = rxr->bnapi->index; 3917 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 3918 3919 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 3920 hwrm_ring_free_send_msg(bp, ring, 3921 RING_FREE_REQ_RING_TYPE_RX, 3922 close_path ? cmpl_ring_id : 3923 INVALID_HW_RING_ID); 3924 ring->fw_ring_id = INVALID_HW_RING_ID; 3925 bp->grp_info[grp_idx].rx_fw_ring_id = 3926 INVALID_HW_RING_ID; 3927 } 3928 } 3929 3930 for (i = 0; i < bp->rx_nr_rings; i++) { 3931 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3932 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 3933 u32 grp_idx = rxr->bnapi->index; 3934 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id; 3935 3936 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 3937 hwrm_ring_free_send_msg(bp, ring, 3938 RING_FREE_REQ_RING_TYPE_RX, 3939 close_path ? cmpl_ring_id : 3940 INVALID_HW_RING_ID); 3941 ring->fw_ring_id = INVALID_HW_RING_ID; 3942 bp->grp_info[grp_idx].agg_fw_ring_id = 3943 INVALID_HW_RING_ID; 3944 } 3945 } 3946 3947 for (i = 0; i < bp->cp_nr_rings; i++) { 3948 struct bnxt_napi *bnapi = bp->bnapi[i]; 3949 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3950 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 3951 3952 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 3953 hwrm_ring_free_send_msg(bp, ring, 3954 RING_FREE_REQ_RING_TYPE_CMPL, 3955 INVALID_HW_RING_ID); 3956 ring->fw_ring_id = INVALID_HW_RING_ID; 3957 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 3958 } 3959 } 3960 } 3961 3962 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs, 3963 u32 buf_tmrs, u16 flags, 3964 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 3965 { 3966 req->flags = cpu_to_le16(flags); 3967 req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs); 3968 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16); 3969 req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs); 3970 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16); 3971 /* Minimum time between 2 interrupts set to buf_tmr x 2 */ 3972 req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2); 3973 req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4); 3974 req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4); 3975 } 3976 3977 int bnxt_hwrm_set_coal(struct bnxt *bp) 3978 { 3979 int i, rc = 0; 3980 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, 3981 req_tx = {0}, *req; 3982 u16 max_buf, max_buf_irq; 3983 u16 buf_tmr, buf_tmr_irq; 3984 u32 flags; 3985 3986 bnxt_hwrm_cmd_hdr_init(bp, &req_rx, 3987 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 3988 bnxt_hwrm_cmd_hdr_init(bp, &req_tx, 3989 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); 3990 3991 /* Each rx completion (2 records) should be DMAed immediately. 3992 * DMA 1/4 of the completion buffers at a time. 3993 */ 3994 max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2); 3995 /* max_buf must not be zero */ 3996 max_buf = clamp_t(u16, max_buf, 1, 63); 3997 max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63); 3998 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks); 3999 /* buf timer set to 1/4 of interrupt timer */ 4000 buf_tmr = max_t(u16, buf_tmr / 4, 1); 4001 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq); 4002 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1); 4003 4004 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4005 4006 /* RING_IDLE generates more IRQs for lower latency. Enable it only 4007 * if coal_ticks is less than 25 us. 4008 */ 4009 if (bp->rx_coal_ticks < 25) 4010 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 4011 4012 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf, 4013 buf_tmr_irq << 16 | buf_tmr, flags, &req_rx); 4014 4015 /* max_buf must not be zero */ 4016 max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63); 4017 max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63); 4018 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks); 4019 /* buf timer set to 1/4 of interrupt timer */ 4020 buf_tmr = max_t(u16, buf_tmr / 4, 1); 4021 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq); 4022 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1); 4023 4024 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 4025 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf, 4026 buf_tmr_irq << 16 | buf_tmr, flags, &req_tx); 4027 4028 mutex_lock(&bp->hwrm_cmd_lock); 4029 for (i = 0; i < bp->cp_nr_rings; i++) { 4030 struct bnxt_napi *bnapi = bp->bnapi[i]; 4031 4032 req = &req_rx; 4033 if (!bnapi->rx_ring) 4034 req = &req_tx; 4035 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id); 4036 4037 rc = _hwrm_send_message(bp, req, sizeof(*req), 4038 HWRM_CMD_TIMEOUT); 4039 if (rc) 4040 break; 4041 } 4042 mutex_unlock(&bp->hwrm_cmd_lock); 4043 return rc; 4044 } 4045 4046 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 4047 { 4048 int rc = 0, i; 4049 struct hwrm_stat_ctx_free_input req = {0}; 4050 4051 if (!bp->bnapi) 4052 return 0; 4053 4054 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4055 return 0; 4056 4057 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1); 4058 4059 mutex_lock(&bp->hwrm_cmd_lock); 4060 for (i = 0; i < bp->cp_nr_rings; i++) { 4061 struct bnxt_napi *bnapi = bp->bnapi[i]; 4062 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4063 4064 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 4065 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 4066 4067 rc = _hwrm_send_message(bp, &req, sizeof(req), 4068 HWRM_CMD_TIMEOUT); 4069 if (rc) 4070 break; 4071 4072 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 4073 } 4074 } 4075 mutex_unlock(&bp->hwrm_cmd_lock); 4076 return rc; 4077 } 4078 4079 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 4080 { 4081 int rc = 0, i; 4082 struct hwrm_stat_ctx_alloc_input req = {0}; 4083 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; 4084 4085 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4086 return 0; 4087 4088 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1); 4089 4090 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 4091 4092 mutex_lock(&bp->hwrm_cmd_lock); 4093 for (i = 0; i < bp->cp_nr_rings; i++) { 4094 struct bnxt_napi *bnapi = bp->bnapi[i]; 4095 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4096 4097 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map); 4098 4099 rc = _hwrm_send_message(bp, &req, sizeof(req), 4100 HWRM_CMD_TIMEOUT); 4101 if (rc) 4102 break; 4103 4104 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 4105 4106 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 4107 } 4108 mutex_unlock(&bp->hwrm_cmd_lock); 4109 return 0; 4110 } 4111 4112 static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 4113 { 4114 struct hwrm_func_qcfg_input req = {0}; 4115 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 4116 int rc; 4117 4118 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1); 4119 req.fid = cpu_to_le16(0xffff); 4120 mutex_lock(&bp->hwrm_cmd_lock); 4121 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4122 if (rc) 4123 goto func_qcfg_exit; 4124 4125 #ifdef CONFIG_BNXT_SRIOV 4126 if (BNXT_VF(bp)) { 4127 struct bnxt_vf_info *vf = &bp->vf; 4128 4129 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 4130 } 4131 #endif 4132 switch (resp->port_partition_type) { 4133 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 4134 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 4135 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 4136 bp->port_partition_type = resp->port_partition_type; 4137 break; 4138 } 4139 4140 func_qcfg_exit: 4141 mutex_unlock(&bp->hwrm_cmd_lock); 4142 return rc; 4143 } 4144 4145 int bnxt_hwrm_func_qcaps(struct bnxt *bp) 4146 { 4147 int rc = 0; 4148 struct hwrm_func_qcaps_input req = {0}; 4149 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 4150 4151 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1); 4152 req.fid = cpu_to_le16(0xffff); 4153 4154 mutex_lock(&bp->hwrm_cmd_lock); 4155 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4156 if (rc) 4157 goto hwrm_func_qcaps_exit; 4158 4159 if (BNXT_PF(bp)) { 4160 struct bnxt_pf_info *pf = &bp->pf; 4161 4162 pf->fw_fid = le16_to_cpu(resp->fid); 4163 pf->port_id = le16_to_cpu(resp->port_id); 4164 bp->dev->dev_port = pf->port_id; 4165 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 4166 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN); 4167 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 4168 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 4169 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 4170 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 4171 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 4172 if (!pf->max_hw_ring_grps) 4173 pf->max_hw_ring_grps = pf->max_tx_rings; 4174 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 4175 pf->max_vnics = le16_to_cpu(resp->max_vnics); 4176 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 4177 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 4178 pf->max_vfs = le16_to_cpu(resp->max_vfs); 4179 pf->max_encap_records = le32_to_cpu(resp->max_encap_records); 4180 pf->max_decap_records = le32_to_cpu(resp->max_decap_records); 4181 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 4182 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 4183 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 4184 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 4185 } else { 4186 #ifdef CONFIG_BNXT_SRIOV 4187 struct bnxt_vf_info *vf = &bp->vf; 4188 4189 vf->fw_fid = le16_to_cpu(resp->fid); 4190 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 4191 if (is_valid_ether_addr(vf->mac_addr)) 4192 /* overwrite netdev dev_adr with admin VF MAC */ 4193 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN); 4194 else 4195 random_ether_addr(bp->dev->dev_addr); 4196 4197 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 4198 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 4199 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 4200 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 4201 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 4202 if (!vf->max_hw_ring_grps) 4203 vf->max_hw_ring_grps = vf->max_tx_rings; 4204 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 4205 vf->max_vnics = le16_to_cpu(resp->max_vnics); 4206 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 4207 #endif 4208 } 4209 4210 bp->tx_push_thresh = 0; 4211 if (resp->flags & 4212 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)) 4213 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 4214 4215 hwrm_func_qcaps_exit: 4216 mutex_unlock(&bp->hwrm_cmd_lock); 4217 return rc; 4218 } 4219 4220 static int bnxt_hwrm_func_reset(struct bnxt *bp) 4221 { 4222 struct hwrm_func_reset_input req = {0}; 4223 4224 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1); 4225 req.enables = 0; 4226 4227 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT); 4228 } 4229 4230 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 4231 { 4232 int rc = 0; 4233 struct hwrm_queue_qportcfg_input req = {0}; 4234 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; 4235 u8 i, *qptr; 4236 4237 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1); 4238 4239 mutex_lock(&bp->hwrm_cmd_lock); 4240 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4241 if (rc) 4242 goto qportcfg_exit; 4243 4244 if (!resp->max_configurable_queues) { 4245 rc = -EINVAL; 4246 goto qportcfg_exit; 4247 } 4248 bp->max_tc = resp->max_configurable_queues; 4249 if (bp->max_tc > BNXT_MAX_QUEUE) 4250 bp->max_tc = BNXT_MAX_QUEUE; 4251 4252 qptr = &resp->queue_id0; 4253 for (i = 0; i < bp->max_tc; i++) { 4254 bp->q_info[i].queue_id = *qptr++; 4255 bp->q_info[i].queue_profile = *qptr++; 4256 } 4257 4258 qportcfg_exit: 4259 mutex_unlock(&bp->hwrm_cmd_lock); 4260 return rc; 4261 } 4262 4263 static int bnxt_hwrm_ver_get(struct bnxt *bp) 4264 { 4265 int rc; 4266 struct hwrm_ver_get_input req = {0}; 4267 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr; 4268 4269 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 4270 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1); 4271 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 4272 req.hwrm_intf_min = HWRM_VERSION_MINOR; 4273 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 4274 mutex_lock(&bp->hwrm_cmd_lock); 4275 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4276 if (rc) 4277 goto hwrm_ver_get_exit; 4278 4279 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 4280 4281 bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 | 4282 resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd; 4283 if (resp->hwrm_intf_maj < 1) { 4284 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 4285 resp->hwrm_intf_maj, resp->hwrm_intf_min, 4286 resp->hwrm_intf_upd); 4287 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 4288 } 4289 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d", 4290 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld, 4291 resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd); 4292 4293 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 4294 if (!bp->hwrm_cmd_timeout) 4295 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 4296 4297 if (resp->hwrm_intf_maj >= 1) 4298 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 4299 4300 bp->chip_num = le16_to_cpu(resp->chip_num); 4301 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 4302 !resp->chip_metal) 4303 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 4304 4305 hwrm_ver_get_exit: 4306 mutex_unlock(&bp->hwrm_cmd_lock); 4307 return rc; 4308 } 4309 4310 static int bnxt_hwrm_port_qstats(struct bnxt *bp) 4311 { 4312 int rc; 4313 struct bnxt_pf_info *pf = &bp->pf; 4314 struct hwrm_port_qstats_input req = {0}; 4315 4316 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 4317 return 0; 4318 4319 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1); 4320 req.port_id = cpu_to_le16(pf->port_id); 4321 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map); 4322 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map); 4323 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 4324 return rc; 4325 } 4326 4327 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 4328 { 4329 if (bp->vxlan_port_cnt) { 4330 bnxt_hwrm_tunnel_dst_port_free( 4331 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 4332 } 4333 bp->vxlan_port_cnt = 0; 4334 if (bp->nge_port_cnt) { 4335 bnxt_hwrm_tunnel_dst_port_free( 4336 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 4337 } 4338 bp->nge_port_cnt = 0; 4339 } 4340 4341 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 4342 { 4343 int rc, i; 4344 u32 tpa_flags = 0; 4345 4346 if (set_tpa) 4347 tpa_flags = bp->flags & BNXT_FLAG_TPA; 4348 for (i = 0; i < bp->nr_vnics; i++) { 4349 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags); 4350 if (rc) { 4351 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 4352 rc, i); 4353 return rc; 4354 } 4355 } 4356 return 0; 4357 } 4358 4359 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 4360 { 4361 int i; 4362 4363 for (i = 0; i < bp->nr_vnics; i++) 4364 bnxt_hwrm_vnic_set_rss(bp, i, false); 4365 } 4366 4367 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 4368 bool irq_re_init) 4369 { 4370 if (bp->vnic_info) { 4371 bnxt_hwrm_clear_vnic_filter(bp); 4372 /* clear all RSS setting before free vnic ctx */ 4373 bnxt_hwrm_clear_vnic_rss(bp); 4374 bnxt_hwrm_vnic_ctx_free(bp); 4375 /* before free the vnic, undo the vnic tpa settings */ 4376 if (bp->flags & BNXT_FLAG_TPA) 4377 bnxt_set_tpa(bp, false); 4378 bnxt_hwrm_vnic_free(bp); 4379 } 4380 bnxt_hwrm_ring_free(bp, close_path); 4381 bnxt_hwrm_ring_grp_free(bp); 4382 if (irq_re_init) { 4383 bnxt_hwrm_stat_ctx_free(bp); 4384 bnxt_hwrm_free_tunnel_ports(bp); 4385 } 4386 } 4387 4388 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id) 4389 { 4390 int rc; 4391 4392 /* allocate context for vnic */ 4393 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0); 4394 if (rc) { 4395 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 4396 vnic_id, rc); 4397 goto vnic_setup_err; 4398 } 4399 bp->rsscos_nr_ctxs++; 4400 4401 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 4402 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1); 4403 if (rc) { 4404 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 4405 vnic_id, rc); 4406 goto vnic_setup_err; 4407 } 4408 bp->rsscos_nr_ctxs++; 4409 } 4410 4411 /* configure default vnic, ring grp */ 4412 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id); 4413 if (rc) { 4414 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 4415 vnic_id, rc); 4416 goto vnic_setup_err; 4417 } 4418 4419 /* Enable RSS hashing on vnic */ 4420 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true); 4421 if (rc) { 4422 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 4423 vnic_id, rc); 4424 goto vnic_setup_err; 4425 } 4426 4427 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 4428 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id); 4429 if (rc) { 4430 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 4431 vnic_id, rc); 4432 } 4433 } 4434 4435 vnic_setup_err: 4436 return rc; 4437 } 4438 4439 static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 4440 { 4441 #ifdef CONFIG_RFS_ACCEL 4442 int i, rc = 0; 4443 4444 for (i = 0; i < bp->rx_nr_rings; i++) { 4445 u16 vnic_id = i + 1; 4446 u16 ring_id = i; 4447 4448 if (vnic_id >= bp->nr_vnics) 4449 break; 4450 4451 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG; 4452 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1); 4453 if (rc) { 4454 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 4455 vnic_id, rc); 4456 break; 4457 } 4458 rc = bnxt_setup_vnic(bp, vnic_id); 4459 if (rc) 4460 break; 4461 } 4462 return rc; 4463 #else 4464 return 0; 4465 #endif 4466 } 4467 4468 /* Allow PF and VF with default VLAN to be in promiscuous mode */ 4469 static bool bnxt_promisc_ok(struct bnxt *bp) 4470 { 4471 #ifdef CONFIG_BNXT_SRIOV 4472 if (BNXT_VF(bp) && !bp->vf.vlan) 4473 return false; 4474 #endif 4475 return true; 4476 } 4477 4478 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 4479 { 4480 unsigned int rc = 0; 4481 4482 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1); 4483 if (rc) { 4484 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 4485 rc); 4486 return rc; 4487 } 4488 4489 rc = bnxt_hwrm_vnic_cfg(bp, 1); 4490 if (rc) { 4491 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 4492 rc); 4493 return rc; 4494 } 4495 return rc; 4496 } 4497 4498 static int bnxt_cfg_rx_mode(struct bnxt *); 4499 static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 4500 4501 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 4502 { 4503 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 4504 int rc = 0; 4505 unsigned int rx_nr_rings = bp->rx_nr_rings; 4506 4507 if (irq_re_init) { 4508 rc = bnxt_hwrm_stat_ctx_alloc(bp); 4509 if (rc) { 4510 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 4511 rc); 4512 goto err_out; 4513 } 4514 } 4515 4516 rc = bnxt_hwrm_ring_alloc(bp); 4517 if (rc) { 4518 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 4519 goto err_out; 4520 } 4521 4522 rc = bnxt_hwrm_ring_grp_alloc(bp); 4523 if (rc) { 4524 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 4525 goto err_out; 4526 } 4527 4528 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4529 rx_nr_rings--; 4530 4531 /* default vnic 0 */ 4532 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings); 4533 if (rc) { 4534 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 4535 goto err_out; 4536 } 4537 4538 rc = bnxt_setup_vnic(bp, 0); 4539 if (rc) 4540 goto err_out; 4541 4542 if (bp->flags & BNXT_FLAG_RFS) { 4543 rc = bnxt_alloc_rfs_vnics(bp); 4544 if (rc) 4545 goto err_out; 4546 } 4547 4548 if (bp->flags & BNXT_FLAG_TPA) { 4549 rc = bnxt_set_tpa(bp, true); 4550 if (rc) 4551 goto err_out; 4552 } 4553 4554 if (BNXT_VF(bp)) 4555 bnxt_update_vf_mac(bp); 4556 4557 /* Filter for default vnic 0 */ 4558 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 4559 if (rc) { 4560 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 4561 goto err_out; 4562 } 4563 vnic->uc_filter_count = 1; 4564 4565 vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 4566 4567 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 4568 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 4569 4570 if (bp->dev->flags & IFF_ALLMULTI) { 4571 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 4572 vnic->mc_list_count = 0; 4573 } else { 4574 u32 mask = 0; 4575 4576 bnxt_mc_list_updated(bp, &mask); 4577 vnic->rx_mask |= mask; 4578 } 4579 4580 rc = bnxt_cfg_rx_mode(bp); 4581 if (rc) 4582 goto err_out; 4583 4584 rc = bnxt_hwrm_set_coal(bp); 4585 if (rc) 4586 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 4587 rc); 4588 4589 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 4590 rc = bnxt_setup_nitroa0_vnic(bp); 4591 if (rc) 4592 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 4593 rc); 4594 } 4595 4596 if (BNXT_VF(bp)) { 4597 bnxt_hwrm_func_qcfg(bp); 4598 netdev_update_features(bp->dev); 4599 } 4600 4601 return 0; 4602 4603 err_out: 4604 bnxt_hwrm_resource_free(bp, 0, true); 4605 4606 return rc; 4607 } 4608 4609 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 4610 { 4611 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 4612 return 0; 4613 } 4614 4615 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 4616 { 4617 bnxt_init_rx_rings(bp); 4618 bnxt_init_tx_rings(bp); 4619 bnxt_init_ring_grps(bp, irq_re_init); 4620 bnxt_init_vnics(bp); 4621 4622 return bnxt_init_chip(bp, irq_re_init); 4623 } 4624 4625 static void bnxt_disable_int(struct bnxt *bp) 4626 { 4627 int i; 4628 4629 if (!bp->bnapi) 4630 return; 4631 4632 for (i = 0; i < bp->cp_nr_rings; i++) { 4633 struct bnxt_napi *bnapi = bp->bnapi[i]; 4634 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4635 4636 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons); 4637 } 4638 } 4639 4640 static void bnxt_enable_int(struct bnxt *bp) 4641 { 4642 int i; 4643 4644 atomic_set(&bp->intr_sem, 0); 4645 for (i = 0; i < bp->cp_nr_rings; i++) { 4646 struct bnxt_napi *bnapi = bp->bnapi[i]; 4647 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 4648 4649 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons); 4650 } 4651 } 4652 4653 static int bnxt_set_real_num_queues(struct bnxt *bp) 4654 { 4655 int rc; 4656 struct net_device *dev = bp->dev; 4657 4658 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings); 4659 if (rc) 4660 return rc; 4661 4662 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 4663 if (rc) 4664 return rc; 4665 4666 #ifdef CONFIG_RFS_ACCEL 4667 if (bp->flags & BNXT_FLAG_RFS) 4668 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 4669 #endif 4670 4671 return rc; 4672 } 4673 4674 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 4675 bool shared) 4676 { 4677 int _rx = *rx, _tx = *tx; 4678 4679 if (shared) { 4680 *rx = min_t(int, _rx, max); 4681 *tx = min_t(int, _tx, max); 4682 } else { 4683 if (max < 2) 4684 return -ENOMEM; 4685 4686 while (_rx + _tx > max) { 4687 if (_rx > _tx && _rx > 1) 4688 _rx--; 4689 else if (_tx > 1) 4690 _tx--; 4691 } 4692 *rx = _rx; 4693 *tx = _tx; 4694 } 4695 return 0; 4696 } 4697 4698 static int bnxt_setup_msix(struct bnxt *bp) 4699 { 4700 struct msix_entry *msix_ent; 4701 struct net_device *dev = bp->dev; 4702 int i, total_vecs, rc = 0, min = 1; 4703 const int len = sizeof(bp->irq_tbl[0].name); 4704 4705 bp->flags &= ~BNXT_FLAG_USING_MSIX; 4706 total_vecs = bp->cp_nr_rings; 4707 4708 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL); 4709 if (!msix_ent) 4710 return -ENOMEM; 4711 4712 for (i = 0; i < total_vecs; i++) { 4713 msix_ent[i].entry = i; 4714 msix_ent[i].vector = 0; 4715 } 4716 4717 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 4718 min = 2; 4719 4720 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs); 4721 if (total_vecs < 0) { 4722 rc = -ENODEV; 4723 goto msix_setup_exit; 4724 } 4725 4726 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL); 4727 if (bp->irq_tbl) { 4728 int tcs; 4729 4730 /* Trim rings based upon num of vectors allocated */ 4731 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 4732 total_vecs, min == 1); 4733 if (rc) 4734 goto msix_setup_exit; 4735 4736 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 4737 tcs = netdev_get_num_tc(dev); 4738 if (tcs > 1) { 4739 bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs; 4740 if (bp->tx_nr_rings_per_tc == 0) { 4741 netdev_reset_tc(dev); 4742 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 4743 } else { 4744 int i, off, count; 4745 4746 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs; 4747 for (i = 0; i < tcs; i++) { 4748 count = bp->tx_nr_rings_per_tc; 4749 off = i * count; 4750 netdev_set_tc_queue(dev, i, count, off); 4751 } 4752 } 4753 } 4754 bp->cp_nr_rings = total_vecs; 4755 4756 for (i = 0; i < bp->cp_nr_rings; i++) { 4757 char *attr; 4758 4759 bp->irq_tbl[i].vector = msix_ent[i].vector; 4760 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 4761 attr = "TxRx"; 4762 else if (i < bp->rx_nr_rings) 4763 attr = "rx"; 4764 else 4765 attr = "tx"; 4766 4767 snprintf(bp->irq_tbl[i].name, len, 4768 "%s-%s-%d", dev->name, attr, i); 4769 bp->irq_tbl[i].handler = bnxt_msix; 4770 } 4771 rc = bnxt_set_real_num_queues(bp); 4772 if (rc) 4773 goto msix_setup_exit; 4774 } else { 4775 rc = -ENOMEM; 4776 goto msix_setup_exit; 4777 } 4778 bp->flags |= BNXT_FLAG_USING_MSIX; 4779 kfree(msix_ent); 4780 return 0; 4781 4782 msix_setup_exit: 4783 netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc); 4784 pci_disable_msix(bp->pdev); 4785 kfree(msix_ent); 4786 return rc; 4787 } 4788 4789 static int bnxt_setup_inta(struct bnxt *bp) 4790 { 4791 int rc; 4792 const int len = sizeof(bp->irq_tbl[0].name); 4793 4794 if (netdev_get_num_tc(bp->dev)) 4795 netdev_reset_tc(bp->dev); 4796 4797 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL); 4798 if (!bp->irq_tbl) { 4799 rc = -ENOMEM; 4800 return rc; 4801 } 4802 bp->rx_nr_rings = 1; 4803 bp->tx_nr_rings = 1; 4804 bp->cp_nr_rings = 1; 4805 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 4806 bp->flags |= BNXT_FLAG_SHARED_RINGS; 4807 bp->irq_tbl[0].vector = bp->pdev->irq; 4808 snprintf(bp->irq_tbl[0].name, len, 4809 "%s-%s-%d", bp->dev->name, "TxRx", 0); 4810 bp->irq_tbl[0].handler = bnxt_inta; 4811 rc = bnxt_set_real_num_queues(bp); 4812 return rc; 4813 } 4814 4815 static int bnxt_setup_int_mode(struct bnxt *bp) 4816 { 4817 int rc = 0; 4818 4819 if (bp->flags & BNXT_FLAG_MSIX_CAP) 4820 rc = bnxt_setup_msix(bp); 4821 4822 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) { 4823 /* fallback to INTA */ 4824 rc = bnxt_setup_inta(bp); 4825 } 4826 return rc; 4827 } 4828 4829 static void bnxt_free_irq(struct bnxt *bp) 4830 { 4831 struct bnxt_irq *irq; 4832 int i; 4833 4834 #ifdef CONFIG_RFS_ACCEL 4835 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 4836 bp->dev->rx_cpu_rmap = NULL; 4837 #endif 4838 if (!bp->irq_tbl) 4839 return; 4840 4841 for (i = 0; i < bp->cp_nr_rings; i++) { 4842 irq = &bp->irq_tbl[i]; 4843 if (irq->requested) 4844 free_irq(irq->vector, bp->bnapi[i]); 4845 irq->requested = 0; 4846 } 4847 if (bp->flags & BNXT_FLAG_USING_MSIX) 4848 pci_disable_msix(bp->pdev); 4849 kfree(bp->irq_tbl); 4850 bp->irq_tbl = NULL; 4851 } 4852 4853 static int bnxt_request_irq(struct bnxt *bp) 4854 { 4855 int i, j, rc = 0; 4856 unsigned long flags = 0; 4857 #ifdef CONFIG_RFS_ACCEL 4858 struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap; 4859 #endif 4860 4861 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) 4862 flags = IRQF_SHARED; 4863 4864 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 4865 struct bnxt_irq *irq = &bp->irq_tbl[i]; 4866 #ifdef CONFIG_RFS_ACCEL 4867 if (rmap && bp->bnapi[i]->rx_ring) { 4868 rc = irq_cpu_rmap_add(rmap, irq->vector); 4869 if (rc) 4870 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 4871 j); 4872 j++; 4873 } 4874 #endif 4875 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 4876 bp->bnapi[i]); 4877 if (rc) 4878 break; 4879 4880 irq->requested = 1; 4881 } 4882 return rc; 4883 } 4884 4885 static void bnxt_del_napi(struct bnxt *bp) 4886 { 4887 int i; 4888 4889 if (!bp->bnapi) 4890 return; 4891 4892 for (i = 0; i < bp->cp_nr_rings; i++) { 4893 struct bnxt_napi *bnapi = bp->bnapi[i]; 4894 4895 napi_hash_del(&bnapi->napi); 4896 netif_napi_del(&bnapi->napi); 4897 } 4898 } 4899 4900 static void bnxt_init_napi(struct bnxt *bp) 4901 { 4902 int i; 4903 unsigned int cp_nr_rings = bp->cp_nr_rings; 4904 struct bnxt_napi *bnapi; 4905 4906 if (bp->flags & BNXT_FLAG_USING_MSIX) { 4907 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4908 cp_nr_rings--; 4909 for (i = 0; i < cp_nr_rings; i++) { 4910 bnapi = bp->bnapi[i]; 4911 netif_napi_add(bp->dev, &bnapi->napi, 4912 bnxt_poll, 64); 4913 } 4914 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 4915 bnapi = bp->bnapi[cp_nr_rings]; 4916 netif_napi_add(bp->dev, &bnapi->napi, 4917 bnxt_poll_nitroa0, 64); 4918 napi_hash_add(&bnapi->napi); 4919 } 4920 } else { 4921 bnapi = bp->bnapi[0]; 4922 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); 4923 } 4924 } 4925 4926 static void bnxt_disable_napi(struct bnxt *bp) 4927 { 4928 int i; 4929 4930 if (!bp->bnapi) 4931 return; 4932 4933 for (i = 0; i < bp->cp_nr_rings; i++) { 4934 napi_disable(&bp->bnapi[i]->napi); 4935 bnxt_disable_poll(bp->bnapi[i]); 4936 } 4937 } 4938 4939 static void bnxt_enable_napi(struct bnxt *bp) 4940 { 4941 int i; 4942 4943 for (i = 0; i < bp->cp_nr_rings; i++) { 4944 bp->bnapi[i]->in_reset = false; 4945 bnxt_enable_poll(bp->bnapi[i]); 4946 napi_enable(&bp->bnapi[i]->napi); 4947 } 4948 } 4949 4950 static void bnxt_tx_disable(struct bnxt *bp) 4951 { 4952 int i; 4953 struct bnxt_tx_ring_info *txr; 4954 struct netdev_queue *txq; 4955 4956 if (bp->tx_ring) { 4957 for (i = 0; i < bp->tx_nr_rings; i++) { 4958 txr = &bp->tx_ring[i]; 4959 txq = netdev_get_tx_queue(bp->dev, i); 4960 txr->dev_state = BNXT_DEV_STATE_CLOSING; 4961 } 4962 } 4963 /* Stop all TX queues */ 4964 netif_tx_disable(bp->dev); 4965 netif_carrier_off(bp->dev); 4966 } 4967 4968 static void bnxt_tx_enable(struct bnxt *bp) 4969 { 4970 int i; 4971 struct bnxt_tx_ring_info *txr; 4972 struct netdev_queue *txq; 4973 4974 for (i = 0; i < bp->tx_nr_rings; i++) { 4975 txr = &bp->tx_ring[i]; 4976 txq = netdev_get_tx_queue(bp->dev, i); 4977 txr->dev_state = 0; 4978 } 4979 netif_tx_wake_all_queues(bp->dev); 4980 if (bp->link_info.link_up) 4981 netif_carrier_on(bp->dev); 4982 } 4983 4984 static void bnxt_report_link(struct bnxt *bp) 4985 { 4986 if (bp->link_info.link_up) { 4987 const char *duplex; 4988 const char *flow_ctrl; 4989 u16 speed; 4990 4991 netif_carrier_on(bp->dev); 4992 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 4993 duplex = "full"; 4994 else 4995 duplex = "half"; 4996 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 4997 flow_ctrl = "ON - receive & transmit"; 4998 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 4999 flow_ctrl = "ON - transmit"; 5000 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 5001 flow_ctrl = "ON - receive"; 5002 else 5003 flow_ctrl = "none"; 5004 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 5005 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n", 5006 speed, duplex, flow_ctrl); 5007 if (bp->flags & BNXT_FLAG_EEE_CAP) 5008 netdev_info(bp->dev, "EEE is %s\n", 5009 bp->eee.eee_active ? "active" : 5010 "not active"); 5011 } else { 5012 netif_carrier_off(bp->dev); 5013 netdev_err(bp->dev, "NIC Link is Down\n"); 5014 } 5015 } 5016 5017 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 5018 { 5019 int rc = 0; 5020 struct hwrm_port_phy_qcaps_input req = {0}; 5021 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr; 5022 struct bnxt_link_info *link_info = &bp->link_info; 5023 5024 if (bp->hwrm_spec_code < 0x10201) 5025 return 0; 5026 5027 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1); 5028 5029 mutex_lock(&bp->hwrm_cmd_lock); 5030 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5031 if (rc) 5032 goto hwrm_phy_qcaps_exit; 5033 5034 if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) { 5035 struct ethtool_eee *eee = &bp->eee; 5036 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 5037 5038 bp->flags |= BNXT_FLAG_EEE_CAP; 5039 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 5040 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 5041 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 5042 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 5043 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 5044 } 5045 link_info->support_auto_speeds = 5046 le16_to_cpu(resp->supported_speeds_auto_mode); 5047 5048 hwrm_phy_qcaps_exit: 5049 mutex_unlock(&bp->hwrm_cmd_lock); 5050 return rc; 5051 } 5052 5053 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 5054 { 5055 int rc = 0; 5056 struct bnxt_link_info *link_info = &bp->link_info; 5057 struct hwrm_port_phy_qcfg_input req = {0}; 5058 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; 5059 u8 link_up = link_info->link_up; 5060 5061 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1); 5062 5063 mutex_lock(&bp->hwrm_cmd_lock); 5064 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5065 if (rc) { 5066 mutex_unlock(&bp->hwrm_cmd_lock); 5067 return rc; 5068 } 5069 5070 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 5071 link_info->phy_link_status = resp->link; 5072 link_info->duplex = resp->duplex; 5073 link_info->pause = resp->pause; 5074 link_info->auto_mode = resp->auto_mode; 5075 link_info->auto_pause_setting = resp->auto_pause; 5076 link_info->lp_pause = resp->link_partner_adv_pause; 5077 link_info->force_pause_setting = resp->force_pause; 5078 link_info->duplex_setting = resp->duplex; 5079 if (link_info->phy_link_status == BNXT_LINK_LINK) 5080 link_info->link_speed = le16_to_cpu(resp->link_speed); 5081 else 5082 link_info->link_speed = 0; 5083 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 5084 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 5085 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 5086 link_info->lp_auto_link_speeds = 5087 le16_to_cpu(resp->link_partner_adv_speeds); 5088 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 5089 link_info->phy_ver[0] = resp->phy_maj; 5090 link_info->phy_ver[1] = resp->phy_min; 5091 link_info->phy_ver[2] = resp->phy_bld; 5092 link_info->media_type = resp->media_type; 5093 link_info->phy_type = resp->phy_type; 5094 link_info->transceiver = resp->xcvr_pkg_type; 5095 link_info->phy_addr = resp->eee_config_phy_addr & 5096 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 5097 link_info->module_status = resp->module_status; 5098 5099 if (bp->flags & BNXT_FLAG_EEE_CAP) { 5100 struct ethtool_eee *eee = &bp->eee; 5101 u16 fw_speeds; 5102 5103 eee->eee_active = 0; 5104 if (resp->eee_config_phy_addr & 5105 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 5106 eee->eee_active = 1; 5107 fw_speeds = le16_to_cpu( 5108 resp->link_partner_adv_eee_link_speed_mask); 5109 eee->lp_advertised = 5110 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 5111 } 5112 5113 /* Pull initial EEE config */ 5114 if (!chng_link_state) { 5115 if (resp->eee_config_phy_addr & 5116 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 5117 eee->eee_enabled = 1; 5118 5119 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 5120 eee->advertised = 5121 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0); 5122 5123 if (resp->eee_config_phy_addr & 5124 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 5125 __le32 tmr; 5126 5127 eee->tx_lpi_enabled = 1; 5128 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 5129 eee->tx_lpi_timer = le32_to_cpu(tmr) & 5130 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 5131 } 5132 } 5133 } 5134 /* TODO: need to add more logic to report VF link */ 5135 if (chng_link_state) { 5136 if (link_info->phy_link_status == BNXT_LINK_LINK) 5137 link_info->link_up = 1; 5138 else 5139 link_info->link_up = 0; 5140 if (link_up != link_info->link_up) 5141 bnxt_report_link(bp); 5142 } else { 5143 /* alwasy link down if not require to update link state */ 5144 link_info->link_up = 0; 5145 } 5146 mutex_unlock(&bp->hwrm_cmd_lock); 5147 return 0; 5148 } 5149 5150 static void bnxt_get_port_module_status(struct bnxt *bp) 5151 { 5152 struct bnxt_link_info *link_info = &bp->link_info; 5153 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 5154 u8 module_status; 5155 5156 if (bnxt_update_link(bp, true)) 5157 return; 5158 5159 module_status = link_info->module_status; 5160 switch (module_status) { 5161 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 5162 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 5163 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 5164 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 5165 bp->pf.port_id); 5166 if (bp->hwrm_spec_code >= 0x10201) { 5167 netdev_warn(bp->dev, "Module part number %s\n", 5168 resp->phy_vendor_partnumber); 5169 } 5170 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 5171 netdev_warn(bp->dev, "TX is disabled\n"); 5172 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 5173 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 5174 } 5175 } 5176 5177 static void 5178 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 5179 { 5180 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 5181 if (bp->hwrm_spec_code >= 0x10201) 5182 req->auto_pause = 5183 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 5184 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 5185 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 5186 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 5187 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 5188 req->enables |= 5189 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 5190 } else { 5191 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 5192 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 5193 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 5194 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 5195 req->enables |= 5196 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 5197 if (bp->hwrm_spec_code >= 0x10201) { 5198 req->auto_pause = req->force_pause; 5199 req->enables |= cpu_to_le32( 5200 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 5201 } 5202 } 5203 } 5204 5205 static void bnxt_hwrm_set_link_common(struct bnxt *bp, 5206 struct hwrm_port_phy_cfg_input *req) 5207 { 5208 u8 autoneg = bp->link_info.autoneg; 5209 u16 fw_link_speed = bp->link_info.req_link_speed; 5210 u32 advertising = bp->link_info.advertising; 5211 5212 if (autoneg & BNXT_AUTONEG_SPEED) { 5213 req->auto_mode |= 5214 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 5215 5216 req->enables |= cpu_to_le32( 5217 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 5218 req->auto_link_speed_mask = cpu_to_le16(advertising); 5219 5220 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 5221 req->flags |= 5222 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 5223 } else { 5224 req->force_link_speed = cpu_to_le16(fw_link_speed); 5225 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 5226 } 5227 5228 /* tell chimp that the setting takes effect immediately */ 5229 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 5230 } 5231 5232 int bnxt_hwrm_set_pause(struct bnxt *bp) 5233 { 5234 struct hwrm_port_phy_cfg_input req = {0}; 5235 int rc; 5236 5237 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 5238 bnxt_hwrm_set_pause_common(bp, &req); 5239 5240 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 5241 bp->link_info.force_link_chng) 5242 bnxt_hwrm_set_link_common(bp, &req); 5243 5244 mutex_lock(&bp->hwrm_cmd_lock); 5245 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5246 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 5247 /* since changing of pause setting doesn't trigger any link 5248 * change event, the driver needs to update the current pause 5249 * result upon successfully return of the phy_cfg command 5250 */ 5251 bp->link_info.pause = 5252 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 5253 bp->link_info.auto_pause_setting = 0; 5254 if (!bp->link_info.force_link_chng) 5255 bnxt_report_link(bp); 5256 } 5257 bp->link_info.force_link_chng = false; 5258 mutex_unlock(&bp->hwrm_cmd_lock); 5259 return rc; 5260 } 5261 5262 static void bnxt_hwrm_set_eee(struct bnxt *bp, 5263 struct hwrm_port_phy_cfg_input *req) 5264 { 5265 struct ethtool_eee *eee = &bp->eee; 5266 5267 if (eee->eee_enabled) { 5268 u16 eee_speeds; 5269 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 5270 5271 if (eee->tx_lpi_enabled) 5272 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 5273 else 5274 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 5275 5276 req->flags |= cpu_to_le32(flags); 5277 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 5278 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 5279 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 5280 } else { 5281 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 5282 } 5283 } 5284 5285 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 5286 { 5287 struct hwrm_port_phy_cfg_input req = {0}; 5288 5289 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 5290 if (set_pause) 5291 bnxt_hwrm_set_pause_common(bp, &req); 5292 5293 bnxt_hwrm_set_link_common(bp, &req); 5294 5295 if (set_eee) 5296 bnxt_hwrm_set_eee(bp, &req); 5297 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5298 } 5299 5300 static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 5301 { 5302 struct hwrm_port_phy_cfg_input req = {0}; 5303 5304 if (!BNXT_SINGLE_PF(bp)) 5305 return 0; 5306 5307 if (pci_num_vf(bp->pdev)) 5308 return 0; 5309 5310 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1); 5311 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DOWN); 5312 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); 5313 } 5314 5315 static bool bnxt_eee_config_ok(struct bnxt *bp) 5316 { 5317 struct ethtool_eee *eee = &bp->eee; 5318 struct bnxt_link_info *link_info = &bp->link_info; 5319 5320 if (!(bp->flags & BNXT_FLAG_EEE_CAP)) 5321 return true; 5322 5323 if (eee->eee_enabled) { 5324 u32 advertising = 5325 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0); 5326 5327 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 5328 eee->eee_enabled = 0; 5329 return false; 5330 } 5331 if (eee->advertised & ~advertising) { 5332 eee->advertised = advertising & eee->supported; 5333 return false; 5334 } 5335 } 5336 return true; 5337 } 5338 5339 static int bnxt_update_phy_setting(struct bnxt *bp) 5340 { 5341 int rc; 5342 bool update_link = false; 5343 bool update_pause = false; 5344 bool update_eee = false; 5345 struct bnxt_link_info *link_info = &bp->link_info; 5346 5347 rc = bnxt_update_link(bp, true); 5348 if (rc) { 5349 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 5350 rc); 5351 return rc; 5352 } 5353 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 5354 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 5355 link_info->req_flow_ctrl) 5356 update_pause = true; 5357 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 5358 link_info->force_pause_setting != link_info->req_flow_ctrl) 5359 update_pause = true; 5360 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 5361 if (BNXT_AUTO_MODE(link_info->auto_mode)) 5362 update_link = true; 5363 if (link_info->req_link_speed != link_info->force_link_speed) 5364 update_link = true; 5365 if (link_info->req_duplex != link_info->duplex_setting) 5366 update_link = true; 5367 } else { 5368 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 5369 update_link = true; 5370 if (link_info->advertising != link_info->auto_link_speeds) 5371 update_link = true; 5372 } 5373 5374 if (!bnxt_eee_config_ok(bp)) 5375 update_eee = true; 5376 5377 if (update_link) 5378 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 5379 else if (update_pause) 5380 rc = bnxt_hwrm_set_pause(bp); 5381 if (rc) { 5382 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 5383 rc); 5384 return rc; 5385 } 5386 5387 return rc; 5388 } 5389 5390 /* Common routine to pre-map certain register block to different GRC window. 5391 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows 5392 * in PF and 3 windows in VF that can be customized to map in different 5393 * register blocks. 5394 */ 5395 static void bnxt_preset_reg_win(struct bnxt *bp) 5396 { 5397 if (BNXT_PF(bp)) { 5398 /* CAG registers map to GRC window #4 */ 5399 writel(BNXT_CAG_REG_BASE, 5400 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12); 5401 } 5402 } 5403 5404 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 5405 { 5406 int rc = 0; 5407 5408 bnxt_preset_reg_win(bp); 5409 netif_carrier_off(bp->dev); 5410 if (irq_re_init) { 5411 rc = bnxt_setup_int_mode(bp); 5412 if (rc) { 5413 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 5414 rc); 5415 return rc; 5416 } 5417 } 5418 if ((bp->flags & BNXT_FLAG_RFS) && 5419 !(bp->flags & BNXT_FLAG_USING_MSIX)) { 5420 /* disable RFS if falling back to INTA */ 5421 bp->dev->hw_features &= ~NETIF_F_NTUPLE; 5422 bp->flags &= ~BNXT_FLAG_RFS; 5423 } 5424 5425 rc = bnxt_alloc_mem(bp, irq_re_init); 5426 if (rc) { 5427 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 5428 goto open_err_free_mem; 5429 } 5430 5431 if (irq_re_init) { 5432 bnxt_init_napi(bp); 5433 rc = bnxt_request_irq(bp); 5434 if (rc) { 5435 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 5436 goto open_err; 5437 } 5438 } 5439 5440 bnxt_enable_napi(bp); 5441 5442 rc = bnxt_init_nic(bp, irq_re_init); 5443 if (rc) { 5444 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 5445 goto open_err; 5446 } 5447 5448 if (link_re_init) { 5449 rc = bnxt_update_phy_setting(bp); 5450 if (rc) 5451 netdev_warn(bp->dev, "failed to update phy settings\n"); 5452 } 5453 5454 if (irq_re_init) 5455 udp_tunnel_get_rx_info(bp->dev); 5456 5457 set_bit(BNXT_STATE_OPEN, &bp->state); 5458 bnxt_enable_int(bp); 5459 /* Enable TX queues */ 5460 bnxt_tx_enable(bp); 5461 mod_timer(&bp->timer, jiffies + bp->current_interval); 5462 /* Poll link status and check for SFP+ module status */ 5463 bnxt_get_port_module_status(bp); 5464 5465 return 0; 5466 5467 open_err: 5468 bnxt_disable_napi(bp); 5469 bnxt_del_napi(bp); 5470 5471 open_err_free_mem: 5472 bnxt_free_skbs(bp); 5473 bnxt_free_irq(bp); 5474 bnxt_free_mem(bp, true); 5475 return rc; 5476 } 5477 5478 /* rtnl_lock held */ 5479 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 5480 { 5481 int rc = 0; 5482 5483 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 5484 if (rc) { 5485 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 5486 dev_close(bp->dev); 5487 } 5488 return rc; 5489 } 5490 5491 static int bnxt_open(struct net_device *dev) 5492 { 5493 struct bnxt *bp = netdev_priv(dev); 5494 int rc = 0; 5495 5496 if (!test_bit(BNXT_STATE_FN_RST_DONE, &bp->state)) { 5497 rc = bnxt_hwrm_func_reset(bp); 5498 if (rc) { 5499 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n", 5500 rc); 5501 rc = -EBUSY; 5502 return rc; 5503 } 5504 /* Do func_reset during the 1st PF open only to prevent killing 5505 * the VFs when the PF is brought down and up. 5506 */ 5507 if (BNXT_PF(bp)) 5508 set_bit(BNXT_STATE_FN_RST_DONE, &bp->state); 5509 } 5510 return __bnxt_open_nic(bp, true, true); 5511 } 5512 5513 static void bnxt_disable_int_sync(struct bnxt *bp) 5514 { 5515 int i; 5516 5517 atomic_inc(&bp->intr_sem); 5518 if (!netif_running(bp->dev)) 5519 return; 5520 5521 bnxt_disable_int(bp); 5522 for (i = 0; i < bp->cp_nr_rings; i++) 5523 synchronize_irq(bp->irq_tbl[i].vector); 5524 } 5525 5526 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 5527 { 5528 int rc = 0; 5529 5530 #ifdef CONFIG_BNXT_SRIOV 5531 if (bp->sriov_cfg) { 5532 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 5533 !bp->sriov_cfg, 5534 BNXT_SRIOV_CFG_WAIT_TMO); 5535 if (rc) 5536 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n"); 5537 } 5538 #endif 5539 /* Change device state to avoid TX queue wake up's */ 5540 bnxt_tx_disable(bp); 5541 5542 clear_bit(BNXT_STATE_OPEN, &bp->state); 5543 smp_mb__after_atomic(); 5544 while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state)) 5545 msleep(20); 5546 5547 /* Flush rings before disabling interrupts */ 5548 bnxt_shutdown_nic(bp, irq_re_init); 5549 5550 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 5551 5552 bnxt_disable_napi(bp); 5553 bnxt_disable_int_sync(bp); 5554 del_timer_sync(&bp->timer); 5555 bnxt_free_skbs(bp); 5556 5557 if (irq_re_init) { 5558 bnxt_free_irq(bp); 5559 bnxt_del_napi(bp); 5560 } 5561 bnxt_free_mem(bp, irq_re_init); 5562 return rc; 5563 } 5564 5565 static int bnxt_close(struct net_device *dev) 5566 { 5567 struct bnxt *bp = netdev_priv(dev); 5568 5569 bnxt_close_nic(bp, true, true); 5570 bnxt_hwrm_shutdown_link(bp); 5571 return 0; 5572 } 5573 5574 /* rtnl_lock held */ 5575 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 5576 { 5577 switch (cmd) { 5578 case SIOCGMIIPHY: 5579 /* fallthru */ 5580 case SIOCGMIIREG: { 5581 if (!netif_running(dev)) 5582 return -EAGAIN; 5583 5584 return 0; 5585 } 5586 5587 case SIOCSMIIREG: 5588 if (!netif_running(dev)) 5589 return -EAGAIN; 5590 5591 return 0; 5592 5593 default: 5594 /* do nothing */ 5595 break; 5596 } 5597 return -EOPNOTSUPP; 5598 } 5599 5600 static struct rtnl_link_stats64 * 5601 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 5602 { 5603 u32 i; 5604 struct bnxt *bp = netdev_priv(dev); 5605 5606 memset(stats, 0, sizeof(struct rtnl_link_stats64)); 5607 5608 if (!bp->bnapi) 5609 return stats; 5610 5611 /* TODO check if we need to synchronize with bnxt_close path */ 5612 for (i = 0; i < bp->cp_nr_rings; i++) { 5613 struct bnxt_napi *bnapi = bp->bnapi[i]; 5614 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5615 struct ctx_hw_stats *hw_stats = cpr->hw_stats; 5616 5617 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts); 5618 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts); 5619 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts); 5620 5621 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts); 5622 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts); 5623 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts); 5624 5625 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes); 5626 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes); 5627 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes); 5628 5629 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes); 5630 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes); 5631 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes); 5632 5633 stats->rx_missed_errors += 5634 le64_to_cpu(hw_stats->rx_discard_pkts); 5635 5636 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts); 5637 5638 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts); 5639 } 5640 5641 if (bp->flags & BNXT_FLAG_PORT_STATS) { 5642 struct rx_port_stats *rx = bp->hw_rx_port_stats; 5643 struct tx_port_stats *tx = bp->hw_tx_port_stats; 5644 5645 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames); 5646 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames); 5647 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) + 5648 le64_to_cpu(rx->rx_ovrsz_frames) + 5649 le64_to_cpu(rx->rx_runt_frames); 5650 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) + 5651 le64_to_cpu(rx->rx_jbr_frames); 5652 stats->collisions = le64_to_cpu(tx->tx_total_collisions); 5653 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns); 5654 stats->tx_errors = le64_to_cpu(tx->tx_err); 5655 } 5656 5657 return stats; 5658 } 5659 5660 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 5661 { 5662 struct net_device *dev = bp->dev; 5663 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5664 struct netdev_hw_addr *ha; 5665 u8 *haddr; 5666 int mc_count = 0; 5667 bool update = false; 5668 int off = 0; 5669 5670 netdev_for_each_mc_addr(ha, dev) { 5671 if (mc_count >= BNXT_MAX_MC_ADDRS) { 5672 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5673 vnic->mc_list_count = 0; 5674 return false; 5675 } 5676 haddr = ha->addr; 5677 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 5678 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 5679 update = true; 5680 } 5681 off += ETH_ALEN; 5682 mc_count++; 5683 } 5684 if (mc_count) 5685 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 5686 5687 if (mc_count != vnic->mc_list_count) { 5688 vnic->mc_list_count = mc_count; 5689 update = true; 5690 } 5691 return update; 5692 } 5693 5694 static bool bnxt_uc_list_updated(struct bnxt *bp) 5695 { 5696 struct net_device *dev = bp->dev; 5697 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5698 struct netdev_hw_addr *ha; 5699 int off = 0; 5700 5701 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 5702 return true; 5703 5704 netdev_for_each_uc_addr(ha, dev) { 5705 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 5706 return true; 5707 5708 off += ETH_ALEN; 5709 } 5710 return false; 5711 } 5712 5713 static void bnxt_set_rx_mode(struct net_device *dev) 5714 { 5715 struct bnxt *bp = netdev_priv(dev); 5716 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5717 u32 mask = vnic->rx_mask; 5718 bool mc_update = false; 5719 bool uc_update; 5720 5721 if (!netif_running(dev)) 5722 return; 5723 5724 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 5725 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 5726 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST); 5727 5728 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp)) 5729 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5730 5731 uc_update = bnxt_uc_list_updated(bp); 5732 5733 if (dev->flags & IFF_ALLMULTI) { 5734 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 5735 vnic->mc_list_count = 0; 5736 } else { 5737 mc_update = bnxt_mc_list_updated(bp, &mask); 5738 } 5739 5740 if (mask != vnic->rx_mask || uc_update || mc_update) { 5741 vnic->rx_mask = mask; 5742 5743 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event); 5744 schedule_work(&bp->sp_task); 5745 } 5746 } 5747 5748 static int bnxt_cfg_rx_mode(struct bnxt *bp) 5749 { 5750 struct net_device *dev = bp->dev; 5751 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 5752 struct netdev_hw_addr *ha; 5753 int i, off = 0, rc; 5754 bool uc_update; 5755 5756 netif_addr_lock_bh(dev); 5757 uc_update = bnxt_uc_list_updated(bp); 5758 netif_addr_unlock_bh(dev); 5759 5760 if (!uc_update) 5761 goto skip_uc; 5762 5763 mutex_lock(&bp->hwrm_cmd_lock); 5764 for (i = 1; i < vnic->uc_filter_count; i++) { 5765 struct hwrm_cfa_l2_filter_free_input req = {0}; 5766 5767 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1, 5768 -1); 5769 5770 req.l2_filter_id = vnic->fw_l2_filter_id[i]; 5771 5772 rc = _hwrm_send_message(bp, &req, sizeof(req), 5773 HWRM_CMD_TIMEOUT); 5774 } 5775 mutex_unlock(&bp->hwrm_cmd_lock); 5776 5777 vnic->uc_filter_count = 1; 5778 5779 netif_addr_lock_bh(dev); 5780 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 5781 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 5782 } else { 5783 netdev_for_each_uc_addr(ha, dev) { 5784 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 5785 off += ETH_ALEN; 5786 vnic->uc_filter_count++; 5787 } 5788 } 5789 netif_addr_unlock_bh(dev); 5790 5791 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 5792 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 5793 if (rc) { 5794 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", 5795 rc); 5796 vnic->uc_filter_count = i; 5797 return rc; 5798 } 5799 } 5800 5801 skip_uc: 5802 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 5803 if (rc) 5804 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n", 5805 rc); 5806 5807 return rc; 5808 } 5809 5810 static bool bnxt_rfs_capable(struct bnxt *bp) 5811 { 5812 #ifdef CONFIG_RFS_ACCEL 5813 struct bnxt_pf_info *pf = &bp->pf; 5814 int vnics; 5815 5816 if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP)) 5817 return false; 5818 5819 vnics = 1 + bp->rx_nr_rings; 5820 if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics) { 5821 netdev_warn(bp->dev, 5822 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 5823 min(pf->max_rsscos_ctxs - 1, pf->max_vnics - 1)); 5824 return false; 5825 } 5826 5827 return true; 5828 #else 5829 return false; 5830 #endif 5831 } 5832 5833 static netdev_features_t bnxt_fix_features(struct net_device *dev, 5834 netdev_features_t features) 5835 { 5836 struct bnxt *bp = netdev_priv(dev); 5837 5838 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) 5839 features &= ~NETIF_F_NTUPLE; 5840 5841 /* Both CTAG and STAG VLAN accelaration on the RX side have to be 5842 * turned on or off together. 5843 */ 5844 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) != 5845 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) { 5846 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 5847 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 5848 NETIF_F_HW_VLAN_STAG_RX); 5849 else 5850 features |= NETIF_F_HW_VLAN_CTAG_RX | 5851 NETIF_F_HW_VLAN_STAG_RX; 5852 } 5853 #ifdef CONFIG_BNXT_SRIOV 5854 if (BNXT_VF(bp)) { 5855 if (bp->vf.vlan) { 5856 features &= ~(NETIF_F_HW_VLAN_CTAG_RX | 5857 NETIF_F_HW_VLAN_STAG_RX); 5858 } 5859 } 5860 #endif 5861 return features; 5862 } 5863 5864 static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 5865 { 5866 struct bnxt *bp = netdev_priv(dev); 5867 u32 flags = bp->flags; 5868 u32 changes; 5869 int rc = 0; 5870 bool re_init = false; 5871 bool update_tpa = false; 5872 5873 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 5874 if ((features & NETIF_F_GRO) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 5875 flags |= BNXT_FLAG_GRO; 5876 if (features & NETIF_F_LRO) 5877 flags |= BNXT_FLAG_LRO; 5878 5879 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5880 flags |= BNXT_FLAG_STRIP_VLAN; 5881 5882 if (features & NETIF_F_NTUPLE) 5883 flags |= BNXT_FLAG_RFS; 5884 5885 changes = flags ^ bp->flags; 5886 if (changes & BNXT_FLAG_TPA) { 5887 update_tpa = true; 5888 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 5889 (flags & BNXT_FLAG_TPA) == 0) 5890 re_init = true; 5891 } 5892 5893 if (changes & ~BNXT_FLAG_TPA) 5894 re_init = true; 5895 5896 if (flags != bp->flags) { 5897 u32 old_flags = bp->flags; 5898 5899 bp->flags = flags; 5900 5901 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 5902 if (update_tpa) 5903 bnxt_set_ring_params(bp); 5904 return rc; 5905 } 5906 5907 if (re_init) { 5908 bnxt_close_nic(bp, false, false); 5909 if (update_tpa) 5910 bnxt_set_ring_params(bp); 5911 5912 return bnxt_open_nic(bp, false, false); 5913 } 5914 if (update_tpa) { 5915 rc = bnxt_set_tpa(bp, 5916 (flags & BNXT_FLAG_TPA) ? 5917 true : false); 5918 if (rc) 5919 bp->flags = old_flags; 5920 } 5921 } 5922 return rc; 5923 } 5924 5925 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 5926 { 5927 struct bnxt_tx_ring_info *txr = bnapi->tx_ring; 5928 int i = bnapi->index; 5929 5930 if (!txr) 5931 return; 5932 5933 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 5934 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 5935 txr->tx_cons); 5936 } 5937 5938 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 5939 { 5940 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 5941 int i = bnapi->index; 5942 5943 if (!rxr) 5944 return; 5945 5946 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 5947 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 5948 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 5949 rxr->rx_sw_agg_prod); 5950 } 5951 5952 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 5953 { 5954 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5955 int i = bnapi->index; 5956 5957 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 5958 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 5959 } 5960 5961 static void bnxt_dbg_dump_states(struct bnxt *bp) 5962 { 5963 int i; 5964 struct bnxt_napi *bnapi; 5965 5966 for (i = 0; i < bp->cp_nr_rings; i++) { 5967 bnapi = bp->bnapi[i]; 5968 if (netif_msg_drv(bp)) { 5969 bnxt_dump_tx_sw_state(bnapi); 5970 bnxt_dump_rx_sw_state(bnapi); 5971 bnxt_dump_cp_sw_state(bnapi); 5972 } 5973 } 5974 } 5975 5976 static void bnxt_reset_task(struct bnxt *bp, bool silent) 5977 { 5978 if (!silent) 5979 bnxt_dbg_dump_states(bp); 5980 if (netif_running(bp->dev)) { 5981 bnxt_close_nic(bp, false, false); 5982 bnxt_open_nic(bp, false, false); 5983 } 5984 } 5985 5986 static void bnxt_tx_timeout(struct net_device *dev) 5987 { 5988 struct bnxt *bp = netdev_priv(dev); 5989 5990 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 5991 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 5992 schedule_work(&bp->sp_task); 5993 } 5994 5995 #ifdef CONFIG_NET_POLL_CONTROLLER 5996 static void bnxt_poll_controller(struct net_device *dev) 5997 { 5998 struct bnxt *bp = netdev_priv(dev); 5999 int i; 6000 6001 for (i = 0; i < bp->cp_nr_rings; i++) { 6002 struct bnxt_irq *irq = &bp->irq_tbl[i]; 6003 6004 disable_irq(irq->vector); 6005 irq->handler(irq->vector, bp->bnapi[i]); 6006 enable_irq(irq->vector); 6007 } 6008 } 6009 #endif 6010 6011 static void bnxt_timer(unsigned long data) 6012 { 6013 struct bnxt *bp = (struct bnxt *)data; 6014 struct net_device *dev = bp->dev; 6015 6016 if (!netif_running(dev)) 6017 return; 6018 6019 if (atomic_read(&bp->intr_sem) != 0) 6020 goto bnxt_restart_timer; 6021 6022 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) { 6023 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event); 6024 schedule_work(&bp->sp_task); 6025 } 6026 bnxt_restart_timer: 6027 mod_timer(&bp->timer, jiffies + bp->current_interval); 6028 } 6029 6030 /* Only called from bnxt_sp_task() */ 6031 static void bnxt_reset(struct bnxt *bp, bool silent) 6032 { 6033 /* bnxt_reset_task() calls bnxt_close_nic() which waits 6034 * for BNXT_STATE_IN_SP_TASK to clear. 6035 * If there is a parallel dev_close(), bnxt_close() may be holding 6036 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we 6037 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). 6038 */ 6039 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 6040 rtnl_lock(); 6041 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 6042 bnxt_reset_task(bp, silent); 6043 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 6044 rtnl_unlock(); 6045 } 6046 6047 static void bnxt_cfg_ntp_filters(struct bnxt *); 6048 6049 static void bnxt_sp_task(struct work_struct *work) 6050 { 6051 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 6052 int rc; 6053 6054 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 6055 smp_mb__after_atomic(); 6056 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 6057 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 6058 return; 6059 } 6060 6061 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 6062 bnxt_cfg_rx_mode(bp); 6063 6064 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 6065 bnxt_cfg_ntp_filters(bp); 6066 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 6067 rc = bnxt_update_link(bp, true); 6068 if (rc) 6069 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 6070 rc); 6071 } 6072 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 6073 bnxt_hwrm_exec_fwd_req(bp); 6074 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) { 6075 bnxt_hwrm_tunnel_dst_port_alloc( 6076 bp, bp->vxlan_port, 6077 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 6078 } 6079 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) { 6080 bnxt_hwrm_tunnel_dst_port_free( 6081 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 6082 } 6083 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) { 6084 bnxt_hwrm_tunnel_dst_port_alloc( 6085 bp, bp->nge_port, 6086 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 6087 } 6088 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) { 6089 bnxt_hwrm_tunnel_dst_port_free( 6090 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 6091 } 6092 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 6093 bnxt_reset(bp, false); 6094 6095 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 6096 bnxt_reset(bp, true); 6097 6098 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) 6099 bnxt_get_port_module_status(bp); 6100 6101 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) 6102 bnxt_hwrm_port_qstats(bp); 6103 6104 smp_mb__before_atomic(); 6105 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 6106 } 6107 6108 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 6109 { 6110 int rc; 6111 struct bnxt *bp = netdev_priv(dev); 6112 6113 SET_NETDEV_DEV(dev, &pdev->dev); 6114 6115 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 6116 rc = pci_enable_device(pdev); 6117 if (rc) { 6118 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 6119 goto init_err; 6120 } 6121 6122 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 6123 dev_err(&pdev->dev, 6124 "Cannot find PCI device base address, aborting\n"); 6125 rc = -ENODEV; 6126 goto init_err_disable; 6127 } 6128 6129 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 6130 if (rc) { 6131 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 6132 goto init_err_disable; 6133 } 6134 6135 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 6136 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 6137 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 6138 goto init_err_disable; 6139 } 6140 6141 pci_set_master(pdev); 6142 6143 bp->dev = dev; 6144 bp->pdev = pdev; 6145 6146 bp->bar0 = pci_ioremap_bar(pdev, 0); 6147 if (!bp->bar0) { 6148 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 6149 rc = -ENOMEM; 6150 goto init_err_release; 6151 } 6152 6153 bp->bar1 = pci_ioremap_bar(pdev, 2); 6154 if (!bp->bar1) { 6155 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n"); 6156 rc = -ENOMEM; 6157 goto init_err_release; 6158 } 6159 6160 bp->bar2 = pci_ioremap_bar(pdev, 4); 6161 if (!bp->bar2) { 6162 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 6163 rc = -ENOMEM; 6164 goto init_err_release; 6165 } 6166 6167 pci_enable_pcie_error_reporting(pdev); 6168 6169 INIT_WORK(&bp->sp_task, bnxt_sp_task); 6170 6171 spin_lock_init(&bp->ntp_fltr_lock); 6172 6173 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 6174 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 6175 6176 /* tick values in micro seconds */ 6177 bp->rx_coal_ticks = 12; 6178 bp->rx_coal_bufs = 30; 6179 bp->rx_coal_ticks_irq = 1; 6180 bp->rx_coal_bufs_irq = 2; 6181 6182 bp->tx_coal_ticks = 25; 6183 bp->tx_coal_bufs = 30; 6184 bp->tx_coal_ticks_irq = 2; 6185 bp->tx_coal_bufs_irq = 2; 6186 6187 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 6188 6189 init_timer(&bp->timer); 6190 bp->timer.data = (unsigned long)bp; 6191 bp->timer.function = bnxt_timer; 6192 bp->current_interval = BNXT_TIMER_INTERVAL; 6193 6194 clear_bit(BNXT_STATE_OPEN, &bp->state); 6195 6196 return 0; 6197 6198 init_err_release: 6199 if (bp->bar2) { 6200 pci_iounmap(pdev, bp->bar2); 6201 bp->bar2 = NULL; 6202 } 6203 6204 if (bp->bar1) { 6205 pci_iounmap(pdev, bp->bar1); 6206 bp->bar1 = NULL; 6207 } 6208 6209 if (bp->bar0) { 6210 pci_iounmap(pdev, bp->bar0); 6211 bp->bar0 = NULL; 6212 } 6213 6214 pci_release_regions(pdev); 6215 6216 init_err_disable: 6217 pci_disable_device(pdev); 6218 6219 init_err: 6220 return rc; 6221 } 6222 6223 /* rtnl_lock held */ 6224 static int bnxt_change_mac_addr(struct net_device *dev, void *p) 6225 { 6226 struct sockaddr *addr = p; 6227 struct bnxt *bp = netdev_priv(dev); 6228 int rc = 0; 6229 6230 if (!is_valid_ether_addr(addr->sa_data)) 6231 return -EADDRNOTAVAIL; 6232 6233 rc = bnxt_approve_mac(bp, addr->sa_data); 6234 if (rc) 6235 return rc; 6236 6237 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 6238 return 0; 6239 6240 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 6241 if (netif_running(dev)) { 6242 bnxt_close_nic(bp, false, false); 6243 rc = bnxt_open_nic(bp, false, false); 6244 } 6245 6246 return rc; 6247 } 6248 6249 /* rtnl_lock held */ 6250 static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 6251 { 6252 struct bnxt *bp = netdev_priv(dev); 6253 6254 if (new_mtu < 60 || new_mtu > 9500) 6255 return -EINVAL; 6256 6257 if (netif_running(dev)) 6258 bnxt_close_nic(bp, false, false); 6259 6260 dev->mtu = new_mtu; 6261 bnxt_set_ring_params(bp); 6262 6263 if (netif_running(dev)) 6264 return bnxt_open_nic(bp, false, false); 6265 6266 return 0; 6267 } 6268 6269 static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto, 6270 struct tc_to_netdev *ntc) 6271 { 6272 struct bnxt *bp = netdev_priv(dev); 6273 u8 tc; 6274 6275 if (ntc->type != TC_SETUP_MQPRIO) 6276 return -EINVAL; 6277 6278 tc = ntc->tc; 6279 6280 if (tc > bp->max_tc) { 6281 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n", 6282 tc, bp->max_tc); 6283 return -EINVAL; 6284 } 6285 6286 if (netdev_get_num_tc(dev) == tc) 6287 return 0; 6288 6289 if (tc) { 6290 int max_rx_rings, max_tx_rings, rc; 6291 bool sh = false; 6292 6293 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 6294 sh = true; 6295 6296 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh); 6297 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings) 6298 return -ENOMEM; 6299 } 6300 6301 /* Needs to close the device and do hw resource re-allocations */ 6302 if (netif_running(bp->dev)) 6303 bnxt_close_nic(bp, true, false); 6304 6305 if (tc) { 6306 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 6307 netdev_set_num_tc(dev, tc); 6308 } else { 6309 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 6310 netdev_reset_tc(dev); 6311 } 6312 bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings); 6313 bp->num_stat_ctxs = bp->cp_nr_rings; 6314 6315 if (netif_running(bp->dev)) 6316 return bnxt_open_nic(bp, true, false); 6317 6318 return 0; 6319 } 6320 6321 #ifdef CONFIG_RFS_ACCEL 6322 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 6323 struct bnxt_ntuple_filter *f2) 6324 { 6325 struct flow_keys *keys1 = &f1->fkeys; 6326 struct flow_keys *keys2 = &f2->fkeys; 6327 6328 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src && 6329 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst && 6330 keys1->ports.ports == keys2->ports.ports && 6331 keys1->basic.ip_proto == keys2->basic.ip_proto && 6332 keys1->basic.n_proto == keys2->basic.n_proto && 6333 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) && 6334 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr)) 6335 return true; 6336 6337 return false; 6338 } 6339 6340 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 6341 u16 rxq_index, u32 flow_id) 6342 { 6343 struct bnxt *bp = netdev_priv(dev); 6344 struct bnxt_ntuple_filter *fltr, *new_fltr; 6345 struct flow_keys *fkeys; 6346 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 6347 int rc = 0, idx, bit_id, l2_idx = 0; 6348 struct hlist_head *head; 6349 6350 if (skb->encapsulation) 6351 return -EPROTONOSUPPORT; 6352 6353 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) { 6354 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 6355 int off = 0, j; 6356 6357 netif_addr_lock_bh(dev); 6358 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) { 6359 if (ether_addr_equal(eth->h_dest, 6360 vnic->uc_list + off)) { 6361 l2_idx = j + 1; 6362 break; 6363 } 6364 } 6365 netif_addr_unlock_bh(dev); 6366 if (!l2_idx) 6367 return -EINVAL; 6368 } 6369 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 6370 if (!new_fltr) 6371 return -ENOMEM; 6372 6373 fkeys = &new_fltr->fkeys; 6374 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 6375 rc = -EPROTONOSUPPORT; 6376 goto err_free; 6377 } 6378 6379 if ((fkeys->basic.n_proto != htons(ETH_P_IP)) || 6380 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 6381 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 6382 rc = -EPROTONOSUPPORT; 6383 goto err_free; 6384 } 6385 6386 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN); 6387 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN); 6388 6389 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 6390 head = &bp->ntp_fltr_hash_tbl[idx]; 6391 rcu_read_lock(); 6392 hlist_for_each_entry_rcu(fltr, head, hash) { 6393 if (bnxt_fltr_match(fltr, new_fltr)) { 6394 rcu_read_unlock(); 6395 rc = 0; 6396 goto err_free; 6397 } 6398 } 6399 rcu_read_unlock(); 6400 6401 spin_lock_bh(&bp->ntp_fltr_lock); 6402 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 6403 BNXT_NTP_FLTR_MAX_FLTR, 0); 6404 if (bit_id < 0) { 6405 spin_unlock_bh(&bp->ntp_fltr_lock); 6406 rc = -ENOMEM; 6407 goto err_free; 6408 } 6409 6410 new_fltr->sw_id = (u16)bit_id; 6411 new_fltr->flow_id = flow_id; 6412 new_fltr->l2_fltr_idx = l2_idx; 6413 new_fltr->rxq = rxq_index; 6414 hlist_add_head_rcu(&new_fltr->hash, head); 6415 bp->ntp_fltr_count++; 6416 spin_unlock_bh(&bp->ntp_fltr_lock); 6417 6418 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event); 6419 schedule_work(&bp->sp_task); 6420 6421 return new_fltr->sw_id; 6422 6423 err_free: 6424 kfree(new_fltr); 6425 return rc; 6426 } 6427 6428 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 6429 { 6430 int i; 6431 6432 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 6433 struct hlist_head *head; 6434 struct hlist_node *tmp; 6435 struct bnxt_ntuple_filter *fltr; 6436 int rc; 6437 6438 head = &bp->ntp_fltr_hash_tbl[i]; 6439 hlist_for_each_entry_safe(fltr, tmp, head, hash) { 6440 bool del = false; 6441 6442 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) { 6443 if (rps_may_expire_flow(bp->dev, fltr->rxq, 6444 fltr->flow_id, 6445 fltr->sw_id)) { 6446 bnxt_hwrm_cfa_ntuple_filter_free(bp, 6447 fltr); 6448 del = true; 6449 } 6450 } else { 6451 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 6452 fltr); 6453 if (rc) 6454 del = true; 6455 else 6456 set_bit(BNXT_FLTR_VALID, &fltr->state); 6457 } 6458 6459 if (del) { 6460 spin_lock_bh(&bp->ntp_fltr_lock); 6461 hlist_del_rcu(&fltr->hash); 6462 bp->ntp_fltr_count--; 6463 spin_unlock_bh(&bp->ntp_fltr_lock); 6464 synchronize_rcu(); 6465 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 6466 kfree(fltr); 6467 } 6468 } 6469 } 6470 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 6471 netdev_info(bp->dev, "Receive PF driver unload event!"); 6472 } 6473 6474 #else 6475 6476 static void bnxt_cfg_ntp_filters(struct bnxt *bp) 6477 { 6478 } 6479 6480 #endif /* CONFIG_RFS_ACCEL */ 6481 6482 static void bnxt_udp_tunnel_add(struct net_device *dev, 6483 struct udp_tunnel_info *ti) 6484 { 6485 struct bnxt *bp = netdev_priv(dev); 6486 6487 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 6488 return; 6489 6490 if (!netif_running(dev)) 6491 return; 6492 6493 switch (ti->type) { 6494 case UDP_TUNNEL_TYPE_VXLAN: 6495 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port) 6496 return; 6497 6498 bp->vxlan_port_cnt++; 6499 if (bp->vxlan_port_cnt == 1) { 6500 bp->vxlan_port = ti->port; 6501 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event); 6502 schedule_work(&bp->sp_task); 6503 } 6504 break; 6505 case UDP_TUNNEL_TYPE_GENEVE: 6506 if (bp->nge_port_cnt && bp->nge_port != ti->port) 6507 return; 6508 6509 bp->nge_port_cnt++; 6510 if (bp->nge_port_cnt == 1) { 6511 bp->nge_port = ti->port; 6512 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event); 6513 } 6514 break; 6515 default: 6516 return; 6517 } 6518 6519 schedule_work(&bp->sp_task); 6520 } 6521 6522 static void bnxt_udp_tunnel_del(struct net_device *dev, 6523 struct udp_tunnel_info *ti) 6524 { 6525 struct bnxt *bp = netdev_priv(dev); 6526 6527 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET) 6528 return; 6529 6530 if (!netif_running(dev)) 6531 return; 6532 6533 switch (ti->type) { 6534 case UDP_TUNNEL_TYPE_VXLAN: 6535 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port) 6536 return; 6537 bp->vxlan_port_cnt--; 6538 6539 if (bp->vxlan_port_cnt != 0) 6540 return; 6541 6542 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event); 6543 break; 6544 case UDP_TUNNEL_TYPE_GENEVE: 6545 if (!bp->nge_port_cnt || bp->nge_port != ti->port) 6546 return; 6547 bp->nge_port_cnt--; 6548 6549 if (bp->nge_port_cnt != 0) 6550 return; 6551 6552 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event); 6553 break; 6554 default: 6555 return; 6556 } 6557 6558 schedule_work(&bp->sp_task); 6559 } 6560 6561 static const struct net_device_ops bnxt_netdev_ops = { 6562 .ndo_open = bnxt_open, 6563 .ndo_start_xmit = bnxt_start_xmit, 6564 .ndo_stop = bnxt_close, 6565 .ndo_get_stats64 = bnxt_get_stats64, 6566 .ndo_set_rx_mode = bnxt_set_rx_mode, 6567 .ndo_do_ioctl = bnxt_ioctl, 6568 .ndo_validate_addr = eth_validate_addr, 6569 .ndo_set_mac_address = bnxt_change_mac_addr, 6570 .ndo_change_mtu = bnxt_change_mtu, 6571 .ndo_fix_features = bnxt_fix_features, 6572 .ndo_set_features = bnxt_set_features, 6573 .ndo_tx_timeout = bnxt_tx_timeout, 6574 #ifdef CONFIG_BNXT_SRIOV 6575 .ndo_get_vf_config = bnxt_get_vf_config, 6576 .ndo_set_vf_mac = bnxt_set_vf_mac, 6577 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 6578 .ndo_set_vf_rate = bnxt_set_vf_bw, 6579 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 6580 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 6581 #endif 6582 #ifdef CONFIG_NET_POLL_CONTROLLER 6583 .ndo_poll_controller = bnxt_poll_controller, 6584 #endif 6585 .ndo_setup_tc = bnxt_setup_tc, 6586 #ifdef CONFIG_RFS_ACCEL 6587 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 6588 #endif 6589 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, 6590 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, 6591 #ifdef CONFIG_NET_RX_BUSY_POLL 6592 .ndo_busy_poll = bnxt_busy_poll, 6593 #endif 6594 }; 6595 6596 static void bnxt_remove_one(struct pci_dev *pdev) 6597 { 6598 struct net_device *dev = pci_get_drvdata(pdev); 6599 struct bnxt *bp = netdev_priv(dev); 6600 6601 if (BNXT_PF(bp)) 6602 bnxt_sriov_disable(bp); 6603 6604 pci_disable_pcie_error_reporting(pdev); 6605 unregister_netdev(dev); 6606 cancel_work_sync(&bp->sp_task); 6607 bp->sp_event = 0; 6608 6609 bnxt_hwrm_func_drv_unrgtr(bp); 6610 bnxt_free_hwrm_resources(bp); 6611 pci_iounmap(pdev, bp->bar2); 6612 pci_iounmap(pdev, bp->bar1); 6613 pci_iounmap(pdev, bp->bar0); 6614 free_netdev(dev); 6615 6616 pci_release_regions(pdev); 6617 pci_disable_device(pdev); 6618 } 6619 6620 static int bnxt_probe_phy(struct bnxt *bp) 6621 { 6622 int rc = 0; 6623 struct bnxt_link_info *link_info = &bp->link_info; 6624 6625 rc = bnxt_hwrm_phy_qcaps(bp); 6626 if (rc) { 6627 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 6628 rc); 6629 return rc; 6630 } 6631 6632 rc = bnxt_update_link(bp, false); 6633 if (rc) { 6634 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 6635 rc); 6636 return rc; 6637 } 6638 6639 /* Older firmware does not have supported_auto_speeds, so assume 6640 * that all supported speeds can be autonegotiated. 6641 */ 6642 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 6643 link_info->support_auto_speeds = link_info->support_speeds; 6644 6645 /*initialize the ethool setting copy with NVM settings */ 6646 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 6647 link_info->autoneg = BNXT_AUTONEG_SPEED; 6648 if (bp->hwrm_spec_code >= 0x10201) { 6649 if (link_info->auto_pause_setting & 6650 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 6651 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 6652 } else { 6653 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 6654 } 6655 link_info->advertising = link_info->auto_link_speeds; 6656 } else { 6657 link_info->req_link_speed = link_info->force_link_speed; 6658 link_info->req_duplex = link_info->duplex_setting; 6659 } 6660 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 6661 link_info->req_flow_ctrl = 6662 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 6663 else 6664 link_info->req_flow_ctrl = link_info->force_pause_setting; 6665 return rc; 6666 } 6667 6668 static int bnxt_get_max_irq(struct pci_dev *pdev) 6669 { 6670 u16 ctrl; 6671 6672 if (!pdev->msix_cap) 6673 return 1; 6674 6675 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 6676 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 6677 } 6678 6679 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 6680 int *max_cp) 6681 { 6682 int max_ring_grps = 0; 6683 6684 #ifdef CONFIG_BNXT_SRIOV 6685 if (!BNXT_PF(bp)) { 6686 *max_tx = bp->vf.max_tx_rings; 6687 *max_rx = bp->vf.max_rx_rings; 6688 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings); 6689 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs); 6690 max_ring_grps = bp->vf.max_hw_ring_grps; 6691 } else 6692 #endif 6693 { 6694 *max_tx = bp->pf.max_tx_rings; 6695 *max_rx = bp->pf.max_rx_rings; 6696 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings); 6697 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs); 6698 max_ring_grps = bp->pf.max_hw_ring_grps; 6699 } 6700 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 6701 *max_cp -= 1; 6702 *max_rx -= 2; 6703 } 6704 if (bp->flags & BNXT_FLAG_AGG_RINGS) 6705 *max_rx >>= 1; 6706 *max_rx = min_t(int, *max_rx, max_ring_grps); 6707 } 6708 6709 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 6710 { 6711 int rx, tx, cp; 6712 6713 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 6714 if (!rx || !tx || !cp) 6715 return -ENOMEM; 6716 6717 *max_rx = rx; 6718 *max_tx = tx; 6719 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 6720 } 6721 6722 static int bnxt_set_dflt_rings(struct bnxt *bp) 6723 { 6724 int dflt_rings, max_rx_rings, max_tx_rings, rc; 6725 bool sh = true; 6726 6727 if (sh) 6728 bp->flags |= BNXT_FLAG_SHARED_RINGS; 6729 dflt_rings = netif_get_num_default_rss_queues(); 6730 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh); 6731 if (rc) 6732 return rc; 6733 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 6734 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 6735 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 6736 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) : 6737 bp->tx_nr_rings + bp->rx_nr_rings; 6738 bp->num_stat_ctxs = bp->cp_nr_rings; 6739 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6740 bp->rx_nr_rings++; 6741 bp->cp_nr_rings++; 6742 } 6743 return rc; 6744 } 6745 6746 static void bnxt_parse_log_pcie_link(struct bnxt *bp) 6747 { 6748 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN; 6749 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN; 6750 6751 if (pcie_get_minimum_link(bp->pdev, &speed, &width) || 6752 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) 6753 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n"); 6754 else 6755 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n", 6756 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : 6757 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : 6758 speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : 6759 "Unknown", width); 6760 } 6761 6762 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 6763 { 6764 static int version_printed; 6765 struct net_device *dev; 6766 struct bnxt *bp; 6767 int rc, max_irqs; 6768 6769 if (pdev->device == 0x16cd && pci_is_bridge(pdev)) 6770 return -ENODEV; 6771 6772 if (version_printed++ == 0) 6773 pr_info("%s", version); 6774 6775 max_irqs = bnxt_get_max_irq(pdev); 6776 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs); 6777 if (!dev) 6778 return -ENOMEM; 6779 6780 bp = netdev_priv(dev); 6781 6782 if (bnxt_vf_pciid(ent->driver_data)) 6783 bp->flags |= BNXT_FLAG_VF; 6784 6785 if (pdev->msix_cap) 6786 bp->flags |= BNXT_FLAG_MSIX_CAP; 6787 6788 rc = bnxt_init_board(pdev, dev); 6789 if (rc < 0) 6790 goto init_err_free; 6791 6792 dev->netdev_ops = &bnxt_netdev_ops; 6793 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 6794 dev->ethtool_ops = &bnxt_ethtool_ops; 6795 6796 pci_set_drvdata(pdev, dev); 6797 6798 rc = bnxt_alloc_hwrm_resources(bp); 6799 if (rc) 6800 goto init_err; 6801 6802 mutex_init(&bp->hwrm_cmd_lock); 6803 rc = bnxt_hwrm_ver_get(bp); 6804 if (rc) 6805 goto init_err; 6806 6807 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 6808 NETIF_F_TSO | NETIF_F_TSO6 | 6809 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 6810 NETIF_F_GSO_IPXIP4 | 6811 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 6812 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 6813 NETIF_F_RXCSUM | NETIF_F_GRO; 6814 6815 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 6816 dev->hw_features |= NETIF_F_LRO; 6817 6818 dev->hw_enc_features = 6819 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 6820 NETIF_F_TSO | NETIF_F_TSO6 | 6821 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 6822 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 6823 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 6824 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 6825 NETIF_F_GSO_GRE_CSUM; 6826 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 6827 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 6828 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; 6829 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 6830 dev->priv_flags |= IFF_UNICAST_FLT; 6831 6832 #ifdef CONFIG_BNXT_SRIOV 6833 init_waitqueue_head(&bp->sriov_cfg_wait); 6834 #endif 6835 bp->gro_func = bnxt_gro_func_5730x; 6836 if (BNXT_CHIP_NUM_57X1X(bp->chip_num)) 6837 bp->gro_func = bnxt_gro_func_5731x; 6838 6839 rc = bnxt_hwrm_func_drv_rgtr(bp); 6840 if (rc) 6841 goto init_err; 6842 6843 /* Get the MAX capabilities for this function */ 6844 rc = bnxt_hwrm_func_qcaps(bp); 6845 if (rc) { 6846 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 6847 rc); 6848 rc = -1; 6849 goto init_err; 6850 } 6851 6852 rc = bnxt_hwrm_queue_qportcfg(bp); 6853 if (rc) { 6854 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n", 6855 rc); 6856 rc = -1; 6857 goto init_err; 6858 } 6859 6860 bnxt_hwrm_func_qcfg(bp); 6861 6862 bnxt_set_tpa_flags(bp); 6863 bnxt_set_ring_params(bp); 6864 if (BNXT_PF(bp)) 6865 bp->pf.max_irqs = max_irqs; 6866 #if defined(CONFIG_BNXT_SRIOV) 6867 else 6868 bp->vf.max_irqs = max_irqs; 6869 #endif 6870 bnxt_set_dflt_rings(bp); 6871 6872 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) { 6873 dev->hw_features |= NETIF_F_NTUPLE; 6874 if (bnxt_rfs_capable(bp)) { 6875 bp->flags |= BNXT_FLAG_RFS; 6876 dev->features |= NETIF_F_NTUPLE; 6877 } 6878 } 6879 6880 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX) 6881 bp->flags |= BNXT_FLAG_STRIP_VLAN; 6882 6883 rc = bnxt_probe_phy(bp); 6884 if (rc) 6885 goto init_err; 6886 6887 rc = register_netdev(dev); 6888 if (rc) 6889 goto init_err; 6890 6891 netdev_info(dev, "%s found at mem %lx, node addr %pM\n", 6892 board_info[ent->driver_data].name, 6893 (long)pci_resource_start(pdev, 0), dev->dev_addr); 6894 6895 bnxt_parse_log_pcie_link(bp); 6896 6897 return 0; 6898 6899 init_err: 6900 pci_iounmap(pdev, bp->bar0); 6901 pci_release_regions(pdev); 6902 pci_disable_device(pdev); 6903 6904 init_err_free: 6905 free_netdev(dev); 6906 return rc; 6907 } 6908 6909 /** 6910 * bnxt_io_error_detected - called when PCI error is detected 6911 * @pdev: Pointer to PCI device 6912 * @state: The current pci connection state 6913 * 6914 * This function is called after a PCI bus error affecting 6915 * this device has been detected. 6916 */ 6917 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 6918 pci_channel_state_t state) 6919 { 6920 struct net_device *netdev = pci_get_drvdata(pdev); 6921 struct bnxt *bp = netdev_priv(netdev); 6922 6923 netdev_info(netdev, "PCI I/O error detected\n"); 6924 6925 rtnl_lock(); 6926 netif_device_detach(netdev); 6927 6928 if (state == pci_channel_io_perm_failure) { 6929 rtnl_unlock(); 6930 return PCI_ERS_RESULT_DISCONNECT; 6931 } 6932 6933 if (netif_running(netdev)) 6934 bnxt_close(netdev); 6935 6936 /* So that func_reset will be done during slot_reset */ 6937 clear_bit(BNXT_STATE_FN_RST_DONE, &bp->state); 6938 pci_disable_device(pdev); 6939 rtnl_unlock(); 6940 6941 /* Request a slot slot reset. */ 6942 return PCI_ERS_RESULT_NEED_RESET; 6943 } 6944 6945 /** 6946 * bnxt_io_slot_reset - called after the pci bus has been reset. 6947 * @pdev: Pointer to PCI device 6948 * 6949 * Restart the card from scratch, as if from a cold-boot. 6950 * At this point, the card has exprienced a hard reset, 6951 * followed by fixups by BIOS, and has its config space 6952 * set up identically to what it was at cold boot. 6953 */ 6954 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 6955 { 6956 struct net_device *netdev = pci_get_drvdata(pdev); 6957 struct bnxt *bp = netdev_priv(netdev); 6958 int err = 0; 6959 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 6960 6961 netdev_info(bp->dev, "PCI Slot Reset\n"); 6962 6963 rtnl_lock(); 6964 6965 if (pci_enable_device(pdev)) { 6966 dev_err(&pdev->dev, 6967 "Cannot re-enable PCI device after reset.\n"); 6968 } else { 6969 pci_set_master(pdev); 6970 6971 if (netif_running(netdev)) 6972 err = bnxt_open(netdev); 6973 6974 if (!err) 6975 result = PCI_ERS_RESULT_RECOVERED; 6976 } 6977 6978 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev)) 6979 dev_close(netdev); 6980 6981 rtnl_unlock(); 6982 6983 err = pci_cleanup_aer_uncorrect_error_status(pdev); 6984 if (err) { 6985 dev_err(&pdev->dev, 6986 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 6987 err); /* non-fatal, continue */ 6988 } 6989 6990 return PCI_ERS_RESULT_RECOVERED; 6991 } 6992 6993 /** 6994 * bnxt_io_resume - called when traffic can start flowing again. 6995 * @pdev: Pointer to PCI device 6996 * 6997 * This callback is called when the error recovery driver tells 6998 * us that its OK to resume normal operation. 6999 */ 7000 static void bnxt_io_resume(struct pci_dev *pdev) 7001 { 7002 struct net_device *netdev = pci_get_drvdata(pdev); 7003 7004 rtnl_lock(); 7005 7006 netif_device_attach(netdev); 7007 7008 rtnl_unlock(); 7009 } 7010 7011 static const struct pci_error_handlers bnxt_err_handler = { 7012 .error_detected = bnxt_io_error_detected, 7013 .slot_reset = bnxt_io_slot_reset, 7014 .resume = bnxt_io_resume 7015 }; 7016 7017 static struct pci_driver bnxt_pci_driver = { 7018 .name = DRV_MODULE_NAME, 7019 .id_table = bnxt_pci_tbl, 7020 .probe = bnxt_init_one, 7021 .remove = bnxt_remove_one, 7022 .err_handler = &bnxt_err_handler, 7023 #if defined(CONFIG_BNXT_SRIOV) 7024 .sriov_configure = bnxt_sriov_configure, 7025 #endif 7026 }; 7027 7028 module_pci_driver(bnxt_pci_driver); 7029