1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Linux network driver for QLogic BR-series Converged Network Adapter. 4 */ 5 /* 6 * Copyright (c) 2005-2014 Brocade Communications Systems, Inc. 7 * Copyright (c) 2014-2015 QLogic Corporation 8 * All rights reserved 9 * www.qlogic.com 10 */ 11 #include <linux/bitops.h> 12 #include <linux/netdevice.h> 13 #include <linux/skbuff.h> 14 #include <linux/etherdevice.h> 15 #include <linux/in.h> 16 #include <linux/ethtool.h> 17 #include <linux/if_vlan.h> 18 #include <linux/if_ether.h> 19 #include <linux/ip.h> 20 #include <linux/prefetch.h> 21 #include <linux/module.h> 22 #include <net/gro.h> 23 24 #include "bnad.h" 25 #include "bna.h" 26 #include "cna.h" 27 28 static DEFINE_MUTEX(bnad_fwimg_mutex); 29 30 /* 31 * Module params 32 */ 33 static uint bnad_msix_disable; 34 module_param(bnad_msix_disable, uint, 0444); 35 MODULE_PARM_DESC(bnad_msix_disable, "Disable MSIX mode"); 36 37 static uint bnad_ioc_auto_recover = 1; 38 module_param(bnad_ioc_auto_recover, uint, 0444); 39 MODULE_PARM_DESC(bnad_ioc_auto_recover, "Enable / Disable auto recovery"); 40 41 static uint bna_debugfs_enable = 1; 42 module_param(bna_debugfs_enable, uint, 0644); 43 MODULE_PARM_DESC(bna_debugfs_enable, "Enables debugfs feature, default=1," 44 " Range[false:0|true:1]"); 45 46 /* 47 * Global variables 48 */ 49 static u32 bnad_rxqs_per_cq = 2; 50 static atomic_t bna_id; 51 static const u8 bnad_bcast_addr[] __aligned(2) = 52 { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; 53 54 /* 55 * Local MACROS 56 */ 57 #define BNAD_GET_MBOX_IRQ(_bnad) \ 58 (((_bnad)->cfg_flags & BNAD_CF_MSIX) ? \ 59 ((_bnad)->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector) : \ 60 ((_bnad)->pcidev->irq)) 61 62 #define BNAD_FILL_UNMAPQ_MEM_REQ(_res_info, _num, _size) \ 63 do { \ 64 (_res_info)->res_type = BNA_RES_T_MEM; \ 65 (_res_info)->res_u.mem_info.mem_type = BNA_MEM_T_KVA; \ 66 (_res_info)->res_u.mem_info.num = (_num); \ 67 (_res_info)->res_u.mem_info.len = (_size); \ 68 } while (0) 69 70 /* 71 * Reinitialize completions in CQ, once Rx is taken down 72 */ 73 static void 74 bnad_cq_cleanup(struct bnad *bnad, struct bna_ccb *ccb) 75 { 76 struct bna_cq_entry *cmpl; 77 int i; 78 79 for (i = 0; i < ccb->q_depth; i++) { 80 cmpl = &((struct bna_cq_entry *)ccb->sw_q)[i]; 81 cmpl->valid = 0; 82 } 83 } 84 85 /* Tx Datapath functions */ 86 87 88 /* Caller should ensure that the entry at unmap_q[index] is valid */ 89 static u32 90 bnad_tx_buff_unmap(struct bnad *bnad, 91 struct bnad_tx_unmap *unmap_q, 92 u32 q_depth, u32 index) 93 { 94 struct bnad_tx_unmap *unmap; 95 struct sk_buff *skb; 96 int vector, nvecs; 97 98 unmap = &unmap_q[index]; 99 nvecs = unmap->nvecs; 100 101 skb = unmap->skb; 102 unmap->skb = NULL; 103 unmap->nvecs = 0; 104 dma_unmap_single(&bnad->pcidev->dev, 105 dma_unmap_addr(&unmap->vectors[0], dma_addr), 106 skb_headlen(skb), DMA_TO_DEVICE); 107 dma_unmap_addr_set(&unmap->vectors[0], dma_addr, 0); 108 nvecs--; 109 110 vector = 0; 111 while (nvecs) { 112 vector++; 113 if (vector == BFI_TX_MAX_VECTORS_PER_WI) { 114 vector = 0; 115 BNA_QE_INDX_INC(index, q_depth); 116 unmap = &unmap_q[index]; 117 } 118 119 dma_unmap_page(&bnad->pcidev->dev, 120 dma_unmap_addr(&unmap->vectors[vector], dma_addr), 121 dma_unmap_len(&unmap->vectors[vector], dma_len), 122 DMA_TO_DEVICE); 123 dma_unmap_addr_set(&unmap->vectors[vector], dma_addr, 0); 124 nvecs--; 125 } 126 127 BNA_QE_INDX_INC(index, q_depth); 128 129 return index; 130 } 131 132 /* 133 * Frees all pending Tx Bufs 134 * At this point no activity is expected on the Q, 135 * so DMA unmap & freeing is fine. 136 */ 137 static void 138 bnad_txq_cleanup(struct bnad *bnad, struct bna_tcb *tcb) 139 { 140 struct bnad_tx_unmap *unmap_q = tcb->unmap_q; 141 struct sk_buff *skb; 142 int i; 143 144 for (i = 0; i < tcb->q_depth; i++) { 145 skb = unmap_q[i].skb; 146 if (!skb) 147 continue; 148 bnad_tx_buff_unmap(bnad, unmap_q, tcb->q_depth, i); 149 150 dev_kfree_skb_any(skb); 151 } 152 } 153 154 /* 155 * bnad_txcmpl_process : Frees the Tx bufs on Tx completion 156 * Can be called in a) Interrupt context 157 * b) Sending context 158 */ 159 static u32 160 bnad_txcmpl_process(struct bnad *bnad, struct bna_tcb *tcb) 161 { 162 u32 sent_packets = 0, sent_bytes = 0; 163 u32 wis, unmap_wis, hw_cons, cons, q_depth; 164 struct bnad_tx_unmap *unmap_q = tcb->unmap_q; 165 struct bnad_tx_unmap *unmap; 166 struct sk_buff *skb; 167 168 /* Just return if TX is stopped */ 169 if (!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) 170 return 0; 171 172 hw_cons = *(tcb->hw_consumer_index); 173 rmb(); 174 cons = tcb->consumer_index; 175 q_depth = tcb->q_depth; 176 177 wis = BNA_Q_INDEX_CHANGE(cons, hw_cons, q_depth); 178 BUG_ON(!(wis <= BNA_QE_IN_USE_CNT(tcb, tcb->q_depth))); 179 180 while (wis) { 181 unmap = &unmap_q[cons]; 182 183 skb = unmap->skb; 184 185 sent_packets++; 186 sent_bytes += skb->len; 187 188 unmap_wis = BNA_TXQ_WI_NEEDED(unmap->nvecs); 189 wis -= unmap_wis; 190 191 cons = bnad_tx_buff_unmap(bnad, unmap_q, q_depth, cons); 192 dev_kfree_skb_any(skb); 193 } 194 195 /* Update consumer pointers. */ 196 tcb->consumer_index = hw_cons; 197 198 tcb->txq->tx_packets += sent_packets; 199 tcb->txq->tx_bytes += sent_bytes; 200 201 return sent_packets; 202 } 203 204 static u32 205 bnad_tx_complete(struct bnad *bnad, struct bna_tcb *tcb) 206 { 207 struct net_device *netdev = bnad->netdev; 208 u32 sent = 0; 209 210 if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) 211 return 0; 212 213 sent = bnad_txcmpl_process(bnad, tcb); 214 if (sent) { 215 if (netif_queue_stopped(netdev) && 216 netif_carrier_ok(netdev) && 217 BNA_QE_FREE_CNT(tcb, tcb->q_depth) >= 218 BNAD_NETIF_WAKE_THRESHOLD) { 219 if (test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) { 220 netif_wake_queue(netdev); 221 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 222 } 223 } 224 } 225 226 if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 227 bna_ib_ack(tcb->i_dbell, sent); 228 229 smp_mb__before_atomic(); 230 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 231 232 return sent; 233 } 234 235 /* MSIX Tx Completion Handler */ 236 static irqreturn_t 237 bnad_msix_tx(int irq, void *data) 238 { 239 struct bna_tcb *tcb = (struct bna_tcb *)data; 240 struct bnad *bnad = tcb->bnad; 241 242 bnad_tx_complete(bnad, tcb); 243 244 return IRQ_HANDLED; 245 } 246 247 static inline void 248 bnad_rxq_alloc_uninit(struct bnad *bnad, struct bna_rcb *rcb) 249 { 250 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 251 252 unmap_q->reuse_pi = -1; 253 unmap_q->alloc_order = -1; 254 unmap_q->map_size = 0; 255 unmap_q->type = BNAD_RXBUF_NONE; 256 } 257 258 /* Default is page-based allocation. Multi-buffer support - TBD */ 259 static int 260 bnad_rxq_alloc_init(struct bnad *bnad, struct bna_rcb *rcb) 261 { 262 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 263 int order; 264 265 bnad_rxq_alloc_uninit(bnad, rcb); 266 267 order = get_order(rcb->rxq->buffer_size); 268 269 unmap_q->type = BNAD_RXBUF_PAGE; 270 271 if (bna_is_small_rxq(rcb->id)) { 272 unmap_q->alloc_order = 0; 273 unmap_q->map_size = rcb->rxq->buffer_size; 274 } else { 275 if (rcb->rxq->multi_buffer) { 276 unmap_q->alloc_order = 0; 277 unmap_q->map_size = rcb->rxq->buffer_size; 278 unmap_q->type = BNAD_RXBUF_MULTI_BUFF; 279 } else { 280 unmap_q->alloc_order = order; 281 unmap_q->map_size = 282 (rcb->rxq->buffer_size > 2048) ? 283 PAGE_SIZE << order : 2048; 284 } 285 } 286 287 BUG_ON((PAGE_SIZE << order) % unmap_q->map_size); 288 289 return 0; 290 } 291 292 static inline void 293 bnad_rxq_cleanup_page(struct bnad *bnad, struct bnad_rx_unmap *unmap) 294 { 295 if (!unmap->page) 296 return; 297 298 dma_unmap_page(&bnad->pcidev->dev, 299 dma_unmap_addr(&unmap->vector, dma_addr), 300 unmap->vector.len, DMA_FROM_DEVICE); 301 put_page(unmap->page); 302 unmap->page = NULL; 303 dma_unmap_addr_set(&unmap->vector, dma_addr, 0); 304 unmap->vector.len = 0; 305 } 306 307 static inline void 308 bnad_rxq_cleanup_skb(struct bnad *bnad, struct bnad_rx_unmap *unmap) 309 { 310 if (!unmap->skb) 311 return; 312 313 dma_unmap_single(&bnad->pcidev->dev, 314 dma_unmap_addr(&unmap->vector, dma_addr), 315 unmap->vector.len, DMA_FROM_DEVICE); 316 dev_kfree_skb_any(unmap->skb); 317 unmap->skb = NULL; 318 dma_unmap_addr_set(&unmap->vector, dma_addr, 0); 319 unmap->vector.len = 0; 320 } 321 322 static void 323 bnad_rxq_cleanup(struct bnad *bnad, struct bna_rcb *rcb) 324 { 325 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 326 int i; 327 328 for (i = 0; i < rcb->q_depth; i++) { 329 struct bnad_rx_unmap *unmap = &unmap_q->unmap[i]; 330 331 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 332 bnad_rxq_cleanup_skb(bnad, unmap); 333 else 334 bnad_rxq_cleanup_page(bnad, unmap); 335 } 336 bnad_rxq_alloc_uninit(bnad, rcb); 337 } 338 339 static u32 340 bnad_rxq_refill_page(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) 341 { 342 u32 alloced, prod, q_depth; 343 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 344 struct bnad_rx_unmap *unmap, *prev; 345 struct bna_rxq_entry *rxent; 346 struct page *page; 347 u32 page_offset, alloc_size; 348 dma_addr_t dma_addr; 349 350 prod = rcb->producer_index; 351 q_depth = rcb->q_depth; 352 353 alloc_size = PAGE_SIZE << unmap_q->alloc_order; 354 alloced = 0; 355 356 while (nalloc--) { 357 unmap = &unmap_q->unmap[prod]; 358 359 if (unmap_q->reuse_pi < 0) { 360 page = alloc_pages(GFP_ATOMIC | __GFP_COMP, 361 unmap_q->alloc_order); 362 page_offset = 0; 363 } else { 364 prev = &unmap_q->unmap[unmap_q->reuse_pi]; 365 page = prev->page; 366 page_offset = prev->page_offset + unmap_q->map_size; 367 get_page(page); 368 } 369 370 if (unlikely(!page)) { 371 BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); 372 rcb->rxq->rxbuf_alloc_failed++; 373 goto finishing; 374 } 375 376 dma_addr = dma_map_page(&bnad->pcidev->dev, page, page_offset, 377 unmap_q->map_size, DMA_FROM_DEVICE); 378 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 379 put_page(page); 380 BNAD_UPDATE_CTR(bnad, rxbuf_map_failed); 381 rcb->rxq->rxbuf_map_failed++; 382 goto finishing; 383 } 384 385 unmap->page = page; 386 unmap->page_offset = page_offset; 387 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); 388 unmap->vector.len = unmap_q->map_size; 389 page_offset += unmap_q->map_size; 390 391 if (page_offset < alloc_size) 392 unmap_q->reuse_pi = prod; 393 else 394 unmap_q->reuse_pi = -1; 395 396 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; 397 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); 398 BNA_QE_INDX_INC(prod, q_depth); 399 alloced++; 400 } 401 402 finishing: 403 if (likely(alloced)) { 404 rcb->producer_index = prod; 405 smp_mb(); 406 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) 407 bna_rxq_prod_indx_doorbell(rcb); 408 } 409 410 return alloced; 411 } 412 413 static u32 414 bnad_rxq_refill_skb(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) 415 { 416 u32 alloced, prod, q_depth, buff_sz; 417 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 418 struct bnad_rx_unmap *unmap; 419 struct bna_rxq_entry *rxent; 420 struct sk_buff *skb; 421 dma_addr_t dma_addr; 422 423 buff_sz = rcb->rxq->buffer_size; 424 prod = rcb->producer_index; 425 q_depth = rcb->q_depth; 426 427 alloced = 0; 428 while (nalloc--) { 429 unmap = &unmap_q->unmap[prod]; 430 431 skb = netdev_alloc_skb_ip_align(bnad->netdev, buff_sz); 432 433 if (unlikely(!skb)) { 434 BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); 435 rcb->rxq->rxbuf_alloc_failed++; 436 goto finishing; 437 } 438 439 dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, 440 buff_sz, DMA_FROM_DEVICE); 441 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 442 dev_kfree_skb_any(skb); 443 BNAD_UPDATE_CTR(bnad, rxbuf_map_failed); 444 rcb->rxq->rxbuf_map_failed++; 445 goto finishing; 446 } 447 448 unmap->skb = skb; 449 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); 450 unmap->vector.len = buff_sz; 451 452 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; 453 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); 454 BNA_QE_INDX_INC(prod, q_depth); 455 alloced++; 456 } 457 458 finishing: 459 if (likely(alloced)) { 460 rcb->producer_index = prod; 461 smp_mb(); 462 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) 463 bna_rxq_prod_indx_doorbell(rcb); 464 } 465 466 return alloced; 467 } 468 469 static inline void 470 bnad_rxq_post(struct bnad *bnad, struct bna_rcb *rcb) 471 { 472 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 473 u32 to_alloc; 474 475 to_alloc = BNA_QE_FREE_CNT(rcb, rcb->q_depth); 476 if (!(to_alloc >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT)) 477 return; 478 479 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 480 bnad_rxq_refill_skb(bnad, rcb, to_alloc); 481 else 482 bnad_rxq_refill_page(bnad, rcb, to_alloc); 483 } 484 485 #define flags_cksum_prot_mask (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 486 BNA_CQ_EF_IPV6 | \ 487 BNA_CQ_EF_TCP | BNA_CQ_EF_UDP | \ 488 BNA_CQ_EF_L4_CKSUM_OK) 489 490 #define flags_tcp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 491 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) 492 #define flags_tcp6 (BNA_CQ_EF_IPV6 | \ 493 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) 494 #define flags_udp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 495 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) 496 #define flags_udp6 (BNA_CQ_EF_IPV6 | \ 497 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) 498 499 static void 500 bnad_cq_drop_packet(struct bnad *bnad, struct bna_rcb *rcb, 501 u32 sop_ci, u32 nvecs) 502 { 503 struct bnad_rx_unmap_q *unmap_q; 504 struct bnad_rx_unmap *unmap; 505 u32 ci, vec; 506 507 unmap_q = rcb->unmap_q; 508 for (vec = 0, ci = sop_ci; vec < nvecs; vec++) { 509 unmap = &unmap_q->unmap[ci]; 510 BNA_QE_INDX_INC(ci, rcb->q_depth); 511 512 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 513 bnad_rxq_cleanup_skb(bnad, unmap); 514 else 515 bnad_rxq_cleanup_page(bnad, unmap); 516 } 517 } 518 519 static void 520 bnad_cq_setup_skb_frags(struct bna_ccb *ccb, struct sk_buff *skb, u32 nvecs) 521 { 522 struct bna_rcb *rcb; 523 struct bnad *bnad; 524 struct bnad_rx_unmap_q *unmap_q; 525 struct bna_cq_entry *cq, *cmpl; 526 u32 ci, pi, totlen = 0; 527 528 cq = ccb->sw_q; 529 pi = ccb->producer_index; 530 cmpl = &cq[pi]; 531 532 rcb = bna_is_small_rxq(cmpl->rxq_id) ? ccb->rcb[1] : ccb->rcb[0]; 533 unmap_q = rcb->unmap_q; 534 bnad = rcb->bnad; 535 ci = rcb->consumer_index; 536 537 /* prefetch header */ 538 prefetch(page_address(unmap_q->unmap[ci].page) + 539 unmap_q->unmap[ci].page_offset); 540 541 while (nvecs--) { 542 struct bnad_rx_unmap *unmap; 543 u32 len; 544 545 unmap = &unmap_q->unmap[ci]; 546 BNA_QE_INDX_INC(ci, rcb->q_depth); 547 548 dma_unmap_page(&bnad->pcidev->dev, 549 dma_unmap_addr(&unmap->vector, dma_addr), 550 unmap->vector.len, DMA_FROM_DEVICE); 551 552 len = ntohs(cmpl->length); 553 skb->truesize += unmap->vector.len; 554 totlen += len; 555 556 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 557 unmap->page, unmap->page_offset, len); 558 559 unmap->page = NULL; 560 unmap->vector.len = 0; 561 562 BNA_QE_INDX_INC(pi, ccb->q_depth); 563 cmpl = &cq[pi]; 564 } 565 566 skb->len += totlen; 567 skb->data_len += totlen; 568 } 569 570 static inline void 571 bnad_cq_setup_skb(struct bnad *bnad, struct sk_buff *skb, 572 struct bnad_rx_unmap *unmap, u32 len) 573 { 574 prefetch(skb->data); 575 576 dma_unmap_single(&bnad->pcidev->dev, 577 dma_unmap_addr(&unmap->vector, dma_addr), 578 unmap->vector.len, DMA_FROM_DEVICE); 579 580 skb_put(skb, len); 581 skb->protocol = eth_type_trans(skb, bnad->netdev); 582 583 unmap->skb = NULL; 584 unmap->vector.len = 0; 585 } 586 587 static u32 588 bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget) 589 { 590 struct bna_cq_entry *cq, *cmpl, *next_cmpl; 591 struct bna_rcb *rcb = NULL; 592 struct bnad_rx_unmap_q *unmap_q; 593 struct bnad_rx_unmap *unmap = NULL; 594 struct sk_buff *skb = NULL; 595 struct bna_pkt_rate *pkt_rt = &ccb->pkt_rate; 596 struct bnad_rx_ctrl *rx_ctrl = ccb->ctrl; 597 u32 packets = 0, len = 0, totlen = 0; 598 u32 pi, vec, sop_ci = 0, nvecs = 0; 599 u32 flags, masked_flags; 600 601 prefetch(bnad->netdev); 602 603 cq = ccb->sw_q; 604 605 while (packets < budget) { 606 cmpl = &cq[ccb->producer_index]; 607 if (!cmpl->valid) 608 break; 609 /* The 'valid' field is set by the adapter, only after writing 610 * the other fields of completion entry. Hence, do not load 611 * other fields of completion entry *before* the 'valid' is 612 * loaded. Adding the rmb() here prevents the compiler and/or 613 * CPU from reordering the reads which would potentially result 614 * in reading stale values in completion entry. 615 */ 616 rmb(); 617 618 BNA_UPDATE_PKT_CNT(pkt_rt, ntohs(cmpl->length)); 619 620 if (bna_is_small_rxq(cmpl->rxq_id)) 621 rcb = ccb->rcb[1]; 622 else 623 rcb = ccb->rcb[0]; 624 625 unmap_q = rcb->unmap_q; 626 627 /* start of packet ci */ 628 sop_ci = rcb->consumer_index; 629 630 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) { 631 unmap = &unmap_q->unmap[sop_ci]; 632 skb = unmap->skb; 633 } else { 634 skb = napi_get_frags(&rx_ctrl->napi); 635 if (unlikely(!skb)) 636 break; 637 } 638 prefetch(skb); 639 640 flags = ntohl(cmpl->flags); 641 len = ntohs(cmpl->length); 642 totlen = len; 643 nvecs = 1; 644 645 /* Check all the completions for this frame. 646 * busy-wait doesn't help much, break here. 647 */ 648 if (BNAD_RXBUF_IS_MULTI_BUFF(unmap_q->type) && 649 (flags & BNA_CQ_EF_EOP) == 0) { 650 pi = ccb->producer_index; 651 do { 652 BNA_QE_INDX_INC(pi, ccb->q_depth); 653 next_cmpl = &cq[pi]; 654 655 if (!next_cmpl->valid) 656 break; 657 /* The 'valid' field is set by the adapter, only 658 * after writing the other fields of completion 659 * entry. Hence, do not load other fields of 660 * completion entry *before* the 'valid' is 661 * loaded. Adding the rmb() here prevents the 662 * compiler and/or CPU from reordering the reads 663 * which would potentially result in reading 664 * stale values in completion entry. 665 */ 666 rmb(); 667 668 len = ntohs(next_cmpl->length); 669 flags = ntohl(next_cmpl->flags); 670 671 nvecs++; 672 totlen += len; 673 } while ((flags & BNA_CQ_EF_EOP) == 0); 674 675 if (!next_cmpl->valid) 676 break; 677 } 678 packets++; 679 680 /* TODO: BNA_CQ_EF_LOCAL ? */ 681 if (unlikely(flags & (BNA_CQ_EF_MAC_ERROR | 682 BNA_CQ_EF_FCS_ERROR | 683 BNA_CQ_EF_TOO_LONG))) { 684 bnad_cq_drop_packet(bnad, rcb, sop_ci, nvecs); 685 rcb->rxq->rx_packets_with_error++; 686 687 goto next; 688 } 689 690 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 691 bnad_cq_setup_skb(bnad, skb, unmap, len); 692 else 693 bnad_cq_setup_skb_frags(ccb, skb, nvecs); 694 695 rcb->rxq->rx_packets++; 696 rcb->rxq->rx_bytes += totlen; 697 ccb->bytes_per_intr += totlen; 698 699 masked_flags = flags & flags_cksum_prot_mask; 700 701 if (likely 702 ((bnad->netdev->features & NETIF_F_RXCSUM) && 703 ((masked_flags == flags_tcp4) || 704 (masked_flags == flags_udp4) || 705 (masked_flags == flags_tcp6) || 706 (masked_flags == flags_udp6)))) 707 skb->ip_summed = CHECKSUM_UNNECESSARY; 708 else 709 skb_checksum_none_assert(skb); 710 711 if ((flags & BNA_CQ_EF_VLAN) && 712 (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 713 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cmpl->vlan_tag)); 714 715 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 716 netif_receive_skb(skb); 717 else 718 napi_gro_frags(&rx_ctrl->napi); 719 720 next: 721 BNA_QE_INDX_ADD(rcb->consumer_index, nvecs, rcb->q_depth); 722 for (vec = 0; vec < nvecs; vec++) { 723 cmpl = &cq[ccb->producer_index]; 724 cmpl->valid = 0; 725 BNA_QE_INDX_INC(ccb->producer_index, ccb->q_depth); 726 } 727 } 728 729 napi_gro_flush(&rx_ctrl->napi, false); 730 if (likely(test_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags))) 731 bna_ib_ack_disable_irq(ccb->i_dbell, packets); 732 733 bnad_rxq_post(bnad, ccb->rcb[0]); 734 if (ccb->rcb[1]) 735 bnad_rxq_post(bnad, ccb->rcb[1]); 736 737 return packets; 738 } 739 740 static void 741 bnad_netif_rx_schedule_poll(struct bnad *bnad, struct bna_ccb *ccb) 742 { 743 struct bnad_rx_ctrl *rx_ctrl = (struct bnad_rx_ctrl *)(ccb->ctrl); 744 struct napi_struct *napi = &rx_ctrl->napi; 745 746 if (likely(napi_schedule_prep(napi))) { 747 __napi_schedule(napi); 748 rx_ctrl->rx_schedule++; 749 } 750 } 751 752 /* MSIX Rx Path Handler */ 753 static irqreturn_t 754 bnad_msix_rx(int irq, void *data) 755 { 756 struct bna_ccb *ccb = (struct bna_ccb *)data; 757 758 if (ccb) { 759 ((struct bnad_rx_ctrl *)ccb->ctrl)->rx_intr_ctr++; 760 bnad_netif_rx_schedule_poll(ccb->bnad, ccb); 761 } 762 763 return IRQ_HANDLED; 764 } 765 766 /* Interrupt handlers */ 767 768 /* Mbox Interrupt Handlers */ 769 static irqreturn_t 770 bnad_msix_mbox_handler(int irq, void *data) 771 { 772 u32 intr_status; 773 unsigned long flags; 774 struct bnad *bnad = (struct bnad *)data; 775 776 spin_lock_irqsave(&bnad->bna_lock, flags); 777 if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) { 778 spin_unlock_irqrestore(&bnad->bna_lock, flags); 779 return IRQ_HANDLED; 780 } 781 782 bna_intr_status_get(&bnad->bna, intr_status); 783 784 if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status)) 785 bna_mbox_handler(&bnad->bna, intr_status); 786 787 spin_unlock_irqrestore(&bnad->bna_lock, flags); 788 789 return IRQ_HANDLED; 790 } 791 792 static irqreturn_t 793 bnad_isr(int irq, void *data) 794 { 795 int i, j; 796 u32 intr_status; 797 unsigned long flags; 798 struct bnad *bnad = (struct bnad *)data; 799 struct bnad_rx_info *rx_info; 800 struct bnad_rx_ctrl *rx_ctrl; 801 struct bna_tcb *tcb = NULL; 802 803 spin_lock_irqsave(&bnad->bna_lock, flags); 804 if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) { 805 spin_unlock_irqrestore(&bnad->bna_lock, flags); 806 return IRQ_NONE; 807 } 808 809 bna_intr_status_get(&bnad->bna, intr_status); 810 811 if (unlikely(!intr_status)) { 812 spin_unlock_irqrestore(&bnad->bna_lock, flags); 813 return IRQ_NONE; 814 } 815 816 if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status)) 817 bna_mbox_handler(&bnad->bna, intr_status); 818 819 spin_unlock_irqrestore(&bnad->bna_lock, flags); 820 821 if (!BNA_IS_INTX_DATA_INTR(intr_status)) 822 return IRQ_HANDLED; 823 824 /* Process data interrupts */ 825 /* Tx processing */ 826 for (i = 0; i < bnad->num_tx; i++) { 827 for (j = 0; j < bnad->num_txq_per_tx; j++) { 828 tcb = bnad->tx_info[i].tcb[j]; 829 if (tcb && test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) 830 bnad_tx_complete(bnad, bnad->tx_info[i].tcb[j]); 831 } 832 } 833 /* Rx processing */ 834 for (i = 0; i < bnad->num_rx; i++) { 835 rx_info = &bnad->rx_info[i]; 836 if (!rx_info->rx) 837 continue; 838 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 839 rx_ctrl = &rx_info->rx_ctrl[j]; 840 if (rx_ctrl->ccb) 841 bnad_netif_rx_schedule_poll(bnad, 842 rx_ctrl->ccb); 843 } 844 } 845 return IRQ_HANDLED; 846 } 847 848 /* 849 * Called in interrupt / callback context 850 * with bna_lock held, so cfg_flags access is OK 851 */ 852 static void 853 bnad_enable_mbox_irq(struct bnad *bnad) 854 { 855 clear_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 856 857 BNAD_UPDATE_CTR(bnad, mbox_intr_enabled); 858 } 859 860 /* 861 * Called with bnad->bna_lock held b'cos of 862 * bnad->cfg_flags access. 863 */ 864 static void 865 bnad_disable_mbox_irq(struct bnad *bnad) 866 { 867 set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 868 869 BNAD_UPDATE_CTR(bnad, mbox_intr_disabled); 870 } 871 872 static void 873 bnad_set_netdev_perm_addr(struct bnad *bnad) 874 { 875 struct net_device *netdev = bnad->netdev; 876 877 ether_addr_copy(netdev->perm_addr, bnad->perm_addr); 878 if (is_zero_ether_addr(netdev->dev_addr)) 879 eth_hw_addr_set(netdev, bnad->perm_addr); 880 } 881 882 /* Control Path Handlers */ 883 884 /* Callbacks */ 885 void 886 bnad_cb_mbox_intr_enable(struct bnad *bnad) 887 { 888 bnad_enable_mbox_irq(bnad); 889 } 890 891 void 892 bnad_cb_mbox_intr_disable(struct bnad *bnad) 893 { 894 bnad_disable_mbox_irq(bnad); 895 } 896 897 void 898 bnad_cb_ioceth_ready(struct bnad *bnad) 899 { 900 bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS; 901 complete(&bnad->bnad_completions.ioc_comp); 902 } 903 904 void 905 bnad_cb_ioceth_failed(struct bnad *bnad) 906 { 907 bnad->bnad_completions.ioc_comp_status = BNA_CB_FAIL; 908 complete(&bnad->bnad_completions.ioc_comp); 909 } 910 911 void 912 bnad_cb_ioceth_disabled(struct bnad *bnad) 913 { 914 bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS; 915 complete(&bnad->bnad_completions.ioc_comp); 916 } 917 918 static void 919 bnad_cb_enet_disabled(void *arg) 920 { 921 struct bnad *bnad = (struct bnad *)arg; 922 923 netif_carrier_off(bnad->netdev); 924 complete(&bnad->bnad_completions.enet_comp); 925 } 926 927 void 928 bnad_cb_ethport_link_status(struct bnad *bnad, 929 enum bna_link_status link_status) 930 { 931 bool link_up = false; 932 933 link_up = (link_status == BNA_LINK_UP) || (link_status == BNA_CEE_UP); 934 935 if (link_status == BNA_CEE_UP) { 936 if (!test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) 937 BNAD_UPDATE_CTR(bnad, cee_toggle); 938 set_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags); 939 } else { 940 if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) 941 BNAD_UPDATE_CTR(bnad, cee_toggle); 942 clear_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags); 943 } 944 945 if (link_up) { 946 if (!netif_carrier_ok(bnad->netdev)) { 947 uint tx_id, tcb_id; 948 netdev_info(bnad->netdev, "link up\n"); 949 netif_carrier_on(bnad->netdev); 950 BNAD_UPDATE_CTR(bnad, link_toggle); 951 for (tx_id = 0; tx_id < bnad->num_tx; tx_id++) { 952 for (tcb_id = 0; tcb_id < bnad->num_txq_per_tx; 953 tcb_id++) { 954 struct bna_tcb *tcb = 955 bnad->tx_info[tx_id].tcb[tcb_id]; 956 u32 txq_id; 957 if (!tcb) 958 continue; 959 960 txq_id = tcb->id; 961 962 if (test_bit(BNAD_TXQ_TX_STARTED, 963 &tcb->flags)) { 964 /* 965 * Force an immediate 966 * Transmit Schedule */ 967 netif_wake_subqueue( 968 bnad->netdev, 969 txq_id); 970 BNAD_UPDATE_CTR(bnad, 971 netif_queue_wakeup); 972 } else { 973 netif_stop_subqueue( 974 bnad->netdev, 975 txq_id); 976 BNAD_UPDATE_CTR(bnad, 977 netif_queue_stop); 978 } 979 } 980 } 981 } 982 } else { 983 if (netif_carrier_ok(bnad->netdev)) { 984 netdev_info(bnad->netdev, "link down\n"); 985 netif_carrier_off(bnad->netdev); 986 BNAD_UPDATE_CTR(bnad, link_toggle); 987 } 988 } 989 } 990 991 static void 992 bnad_cb_tx_disabled(void *arg, struct bna_tx *tx) 993 { 994 struct bnad *bnad = (struct bnad *)arg; 995 996 complete(&bnad->bnad_completions.tx_comp); 997 } 998 999 static void 1000 bnad_cb_tcb_setup(struct bnad *bnad, struct bna_tcb *tcb) 1001 { 1002 struct bnad_tx_info *tx_info = 1003 (struct bnad_tx_info *)tcb->txq->tx->priv; 1004 1005 tcb->priv = tcb; 1006 tx_info->tcb[tcb->id] = tcb; 1007 } 1008 1009 static void 1010 bnad_cb_tcb_destroy(struct bnad *bnad, struct bna_tcb *tcb) 1011 { 1012 struct bnad_tx_info *tx_info = 1013 (struct bnad_tx_info *)tcb->txq->tx->priv; 1014 1015 tx_info->tcb[tcb->id] = NULL; 1016 tcb->priv = NULL; 1017 } 1018 1019 static void 1020 bnad_cb_ccb_setup(struct bnad *bnad, struct bna_ccb *ccb) 1021 { 1022 struct bnad_rx_info *rx_info = 1023 (struct bnad_rx_info *)ccb->cq->rx->priv; 1024 1025 rx_info->rx_ctrl[ccb->id].ccb = ccb; 1026 ccb->ctrl = &rx_info->rx_ctrl[ccb->id]; 1027 } 1028 1029 static void 1030 bnad_cb_ccb_destroy(struct bnad *bnad, struct bna_ccb *ccb) 1031 { 1032 struct bnad_rx_info *rx_info = 1033 (struct bnad_rx_info *)ccb->cq->rx->priv; 1034 1035 rx_info->rx_ctrl[ccb->id].ccb = NULL; 1036 } 1037 1038 static void 1039 bnad_cb_tx_stall(struct bnad *bnad, struct bna_tx *tx) 1040 { 1041 struct bnad_tx_info *tx_info = tx->priv; 1042 struct bna_tcb *tcb; 1043 u32 txq_id; 1044 int i; 1045 1046 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1047 tcb = tx_info->tcb[i]; 1048 if (!tcb) 1049 continue; 1050 txq_id = tcb->id; 1051 clear_bit(BNAD_TXQ_TX_STARTED, &tcb->flags); 1052 netif_stop_subqueue(bnad->netdev, txq_id); 1053 } 1054 } 1055 1056 static void 1057 bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) 1058 { 1059 struct bnad_tx_info *tx_info = tx->priv; 1060 struct bna_tcb *tcb; 1061 u32 txq_id; 1062 int i; 1063 1064 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1065 tcb = tx_info->tcb[i]; 1066 if (!tcb) 1067 continue; 1068 txq_id = tcb->id; 1069 1070 BUG_ON(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)); 1071 set_bit(BNAD_TXQ_TX_STARTED, &tcb->flags); 1072 BUG_ON(*(tcb->hw_consumer_index) != 0); 1073 1074 if (netif_carrier_ok(bnad->netdev)) { 1075 netif_wake_subqueue(bnad->netdev, txq_id); 1076 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 1077 } 1078 } 1079 1080 /* 1081 * Workaround for first ioceth enable failure & we 1082 * get a 0 MAC address. We try to get the MAC address 1083 * again here. 1084 */ 1085 if (is_zero_ether_addr(bnad->perm_addr)) { 1086 bna_enet_perm_mac_get(&bnad->bna.enet, bnad->perm_addr); 1087 bnad_set_netdev_perm_addr(bnad); 1088 } 1089 } 1090 1091 /* 1092 * Free all TxQs buffers and then notify TX_E_CLEANUP_DONE to Tx fsm. 1093 */ 1094 static void 1095 bnad_tx_cleanup(struct work_struct *work) 1096 { 1097 struct bnad_tx_info *tx_info = 1098 container_of(work, struct bnad_tx_info, tx_cleanup_work.work); 1099 struct bnad *bnad = NULL; 1100 struct bna_tcb *tcb; 1101 unsigned long flags; 1102 u32 i, pending = 0; 1103 1104 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1105 tcb = tx_info->tcb[i]; 1106 if (!tcb) 1107 continue; 1108 1109 bnad = tcb->bnad; 1110 1111 if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { 1112 pending++; 1113 continue; 1114 } 1115 1116 bnad_txq_cleanup(bnad, tcb); 1117 1118 smp_mb__before_atomic(); 1119 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 1120 } 1121 1122 if (pending) { 1123 queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 1124 msecs_to_jiffies(1)); 1125 return; 1126 } 1127 1128 spin_lock_irqsave(&bnad->bna_lock, flags); 1129 bna_tx_cleanup_complete(tx_info->tx); 1130 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1131 } 1132 1133 static void 1134 bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) 1135 { 1136 struct bnad_tx_info *tx_info = tx->priv; 1137 struct bna_tcb *tcb; 1138 int i; 1139 1140 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1141 tcb = tx_info->tcb[i]; 1142 if (!tcb) 1143 continue; 1144 } 1145 1146 queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 0); 1147 } 1148 1149 static void 1150 bnad_cb_rx_stall(struct bnad *bnad, struct bna_rx *rx) 1151 { 1152 struct bnad_rx_info *rx_info = rx->priv; 1153 struct bna_ccb *ccb; 1154 struct bnad_rx_ctrl *rx_ctrl; 1155 int i; 1156 1157 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1158 rx_ctrl = &rx_info->rx_ctrl[i]; 1159 ccb = rx_ctrl->ccb; 1160 if (!ccb) 1161 continue; 1162 1163 clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[0]->flags); 1164 1165 if (ccb->rcb[1]) 1166 clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[1]->flags); 1167 } 1168 } 1169 1170 /* 1171 * Free all RxQs buffers and then notify RX_E_CLEANUP_DONE to Rx fsm. 1172 */ 1173 static void 1174 bnad_rx_cleanup(struct work_struct *work) 1175 { 1176 struct bnad_rx_info *rx_info = 1177 container_of(work, struct bnad_rx_info, rx_cleanup_work); 1178 struct bnad_rx_ctrl *rx_ctrl; 1179 struct bnad *bnad = NULL; 1180 unsigned long flags; 1181 u32 i; 1182 1183 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1184 rx_ctrl = &rx_info->rx_ctrl[i]; 1185 1186 if (!rx_ctrl->ccb) 1187 continue; 1188 1189 bnad = rx_ctrl->ccb->bnad; 1190 1191 /* 1192 * Wait till the poll handler has exited 1193 * and nothing can be scheduled anymore 1194 */ 1195 napi_disable(&rx_ctrl->napi); 1196 1197 bnad_cq_cleanup(bnad, rx_ctrl->ccb); 1198 bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[0]); 1199 if (rx_ctrl->ccb->rcb[1]) 1200 bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[1]); 1201 } 1202 1203 spin_lock_irqsave(&bnad->bna_lock, flags); 1204 bna_rx_cleanup_complete(rx_info->rx); 1205 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1206 } 1207 1208 static void 1209 bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) 1210 { 1211 struct bnad_rx_info *rx_info = rx->priv; 1212 struct bna_ccb *ccb; 1213 struct bnad_rx_ctrl *rx_ctrl; 1214 int i; 1215 1216 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1217 rx_ctrl = &rx_info->rx_ctrl[i]; 1218 ccb = rx_ctrl->ccb; 1219 if (!ccb) 1220 continue; 1221 1222 clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags); 1223 1224 if (ccb->rcb[1]) 1225 clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[1]->flags); 1226 } 1227 1228 queue_work(bnad->work_q, &rx_info->rx_cleanup_work); 1229 } 1230 1231 static void 1232 bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx) 1233 { 1234 struct bnad_rx_info *rx_info = rx->priv; 1235 struct bna_ccb *ccb; 1236 struct bna_rcb *rcb; 1237 struct bnad_rx_ctrl *rx_ctrl; 1238 int i, j; 1239 1240 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1241 rx_ctrl = &rx_info->rx_ctrl[i]; 1242 ccb = rx_ctrl->ccb; 1243 if (!ccb) 1244 continue; 1245 1246 napi_enable(&rx_ctrl->napi); 1247 1248 for (j = 0; j < BNAD_MAX_RXQ_PER_RXP; j++) { 1249 rcb = ccb->rcb[j]; 1250 if (!rcb) 1251 continue; 1252 1253 bnad_rxq_alloc_init(bnad, rcb); 1254 set_bit(BNAD_RXQ_STARTED, &rcb->flags); 1255 set_bit(BNAD_RXQ_POST_OK, &rcb->flags); 1256 bnad_rxq_post(bnad, rcb); 1257 } 1258 } 1259 } 1260 1261 static void 1262 bnad_cb_rx_disabled(void *arg, struct bna_rx *rx) 1263 { 1264 struct bnad *bnad = (struct bnad *)arg; 1265 1266 complete(&bnad->bnad_completions.rx_comp); 1267 } 1268 1269 static void 1270 bnad_cb_rx_mcast_add(struct bnad *bnad, struct bna_rx *rx) 1271 { 1272 bnad->bnad_completions.mcast_comp_status = BNA_CB_SUCCESS; 1273 complete(&bnad->bnad_completions.mcast_comp); 1274 } 1275 1276 void 1277 bnad_cb_stats_get(struct bnad *bnad, enum bna_cb_status status, 1278 struct bna_stats *stats) 1279 { 1280 if (status == BNA_CB_SUCCESS) 1281 BNAD_UPDATE_CTR(bnad, hw_stats_updates); 1282 1283 if (!netif_running(bnad->netdev) || 1284 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1285 return; 1286 1287 mod_timer(&bnad->stats_timer, 1288 jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ)); 1289 } 1290 1291 static void 1292 bnad_cb_enet_mtu_set(struct bnad *bnad) 1293 { 1294 bnad->bnad_completions.mtu_comp_status = BNA_CB_SUCCESS; 1295 complete(&bnad->bnad_completions.mtu_comp); 1296 } 1297 1298 void 1299 bnad_cb_completion(void *arg, enum bfa_status status) 1300 { 1301 struct bnad_iocmd_comp *iocmd_comp = 1302 (struct bnad_iocmd_comp *)arg; 1303 1304 iocmd_comp->comp_status = (u32) status; 1305 complete(&iocmd_comp->comp); 1306 } 1307 1308 /* Resource allocation, free functions */ 1309 1310 static void 1311 bnad_mem_free(struct bnad *bnad, 1312 struct bna_mem_info *mem_info) 1313 { 1314 int i; 1315 dma_addr_t dma_pa; 1316 1317 if (mem_info->mdl == NULL) 1318 return; 1319 1320 for (i = 0; i < mem_info->num; i++) { 1321 if (mem_info->mdl[i].kva != NULL) { 1322 if (mem_info->mem_type == BNA_MEM_T_DMA) { 1323 BNA_GET_DMA_ADDR(&(mem_info->mdl[i].dma), 1324 dma_pa); 1325 dma_free_coherent(&bnad->pcidev->dev, 1326 mem_info->mdl[i].len, 1327 mem_info->mdl[i].kva, dma_pa); 1328 } else 1329 kfree(mem_info->mdl[i].kva); 1330 } 1331 } 1332 kfree(mem_info->mdl); 1333 mem_info->mdl = NULL; 1334 } 1335 1336 static int 1337 bnad_mem_alloc(struct bnad *bnad, 1338 struct bna_mem_info *mem_info) 1339 { 1340 int i; 1341 dma_addr_t dma_pa; 1342 1343 if ((mem_info->num == 0) || (mem_info->len == 0)) { 1344 mem_info->mdl = NULL; 1345 return 0; 1346 } 1347 1348 mem_info->mdl = kzalloc_objs(struct bna_mem_descr, mem_info->num); 1349 if (mem_info->mdl == NULL) 1350 return -ENOMEM; 1351 1352 if (mem_info->mem_type == BNA_MEM_T_DMA) { 1353 for (i = 0; i < mem_info->num; i++) { 1354 mem_info->mdl[i].len = mem_info->len; 1355 mem_info->mdl[i].kva = 1356 dma_alloc_coherent(&bnad->pcidev->dev, 1357 mem_info->len, &dma_pa, 1358 GFP_KERNEL); 1359 if (mem_info->mdl[i].kva == NULL) 1360 goto err_return; 1361 1362 BNA_SET_DMA_ADDR(dma_pa, 1363 &(mem_info->mdl[i].dma)); 1364 } 1365 } else { 1366 for (i = 0; i < mem_info->num; i++) { 1367 mem_info->mdl[i].len = mem_info->len; 1368 mem_info->mdl[i].kva = kzalloc(mem_info->len, 1369 GFP_KERNEL); 1370 if (mem_info->mdl[i].kva == NULL) 1371 goto err_return; 1372 } 1373 } 1374 1375 return 0; 1376 1377 err_return: 1378 bnad_mem_free(bnad, mem_info); 1379 return -ENOMEM; 1380 } 1381 1382 /* Free IRQ for Mailbox */ 1383 static void 1384 bnad_mbox_irq_free(struct bnad *bnad) 1385 { 1386 int irq; 1387 unsigned long flags; 1388 1389 spin_lock_irqsave(&bnad->bna_lock, flags); 1390 bnad_disable_mbox_irq(bnad); 1391 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1392 1393 irq = BNAD_GET_MBOX_IRQ(bnad); 1394 free_irq(irq, bnad); 1395 } 1396 1397 /* 1398 * Allocates IRQ for Mailbox, but keep it disabled 1399 * This will be enabled once we get the mbox enable callback 1400 * from bna 1401 */ 1402 static int 1403 bnad_mbox_irq_alloc(struct bnad *bnad) 1404 { 1405 int err = 0; 1406 unsigned long irq_flags, flags; 1407 u32 irq; 1408 irq_handler_t irq_handler; 1409 1410 spin_lock_irqsave(&bnad->bna_lock, flags); 1411 if (bnad->cfg_flags & BNAD_CF_MSIX) { 1412 irq_handler = (irq_handler_t)bnad_msix_mbox_handler; 1413 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector; 1414 irq_flags = 0; 1415 } else { 1416 irq_handler = (irq_handler_t)bnad_isr; 1417 irq = bnad->pcidev->irq; 1418 irq_flags = IRQF_SHARED; 1419 } 1420 1421 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1422 sprintf(bnad->mbox_irq_name, "%s", BNAD_NAME); 1423 1424 /* 1425 * Set the Mbox IRQ disable flag, so that the IRQ handler 1426 * called from request_irq() for SHARED IRQs do not execute 1427 */ 1428 set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 1429 1430 BNAD_UPDATE_CTR(bnad, mbox_intr_disabled); 1431 1432 err = request_irq(irq, irq_handler, irq_flags, 1433 bnad->mbox_irq_name, bnad); 1434 1435 return err; 1436 } 1437 1438 static void 1439 bnad_txrx_irq_free(struct bnad *bnad, struct bna_intr_info *intr_info) 1440 { 1441 kfree(intr_info->idl); 1442 intr_info->idl = NULL; 1443 } 1444 1445 /* Allocates Interrupt Descriptor List for MSIX/INT-X vectors */ 1446 static int 1447 bnad_txrx_irq_alloc(struct bnad *bnad, enum bnad_intr_source src, 1448 u32 txrx_id, struct bna_intr_info *intr_info) 1449 { 1450 int i, vector_start = 0; 1451 u32 cfg_flags; 1452 unsigned long flags; 1453 1454 spin_lock_irqsave(&bnad->bna_lock, flags); 1455 cfg_flags = bnad->cfg_flags; 1456 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1457 1458 if (cfg_flags & BNAD_CF_MSIX) { 1459 intr_info->intr_type = BNA_INTR_T_MSIX; 1460 intr_info->idl = kzalloc_objs(struct bna_intr_descr, 1461 intr_info->num); 1462 if (!intr_info->idl) 1463 return -ENOMEM; 1464 1465 switch (src) { 1466 case BNAD_INTR_TX: 1467 vector_start = BNAD_MAILBOX_MSIX_VECTORS + txrx_id; 1468 break; 1469 1470 case BNAD_INTR_RX: 1471 vector_start = BNAD_MAILBOX_MSIX_VECTORS + 1472 (bnad->num_tx * bnad->num_txq_per_tx) + 1473 txrx_id; 1474 break; 1475 1476 default: 1477 BUG(); 1478 } 1479 1480 for (i = 0; i < intr_info->num; i++) 1481 intr_info->idl[i].vector = vector_start + i; 1482 } else { 1483 intr_info->intr_type = BNA_INTR_T_INTX; 1484 intr_info->num = 1; 1485 intr_info->idl = kzalloc_objs(struct bna_intr_descr, 1486 intr_info->num); 1487 if (!intr_info->idl) 1488 return -ENOMEM; 1489 1490 switch (src) { 1491 case BNAD_INTR_TX: 1492 intr_info->idl[0].vector = BNAD_INTX_TX_IB_BITMASK; 1493 break; 1494 1495 case BNAD_INTR_RX: 1496 intr_info->idl[0].vector = BNAD_INTX_RX_IB_BITMASK; 1497 break; 1498 } 1499 } 1500 return 0; 1501 } 1502 1503 /* NOTE: Should be called for MSIX only 1504 * Unregisters Tx MSIX vector(s) from the kernel 1505 */ 1506 static void 1507 bnad_tx_msix_unregister(struct bnad *bnad, struct bnad_tx_info *tx_info, 1508 int num_txqs) 1509 { 1510 int i; 1511 int vector_num; 1512 1513 for (i = 0; i < num_txqs; i++) { 1514 if (tx_info->tcb[i] == NULL) 1515 continue; 1516 1517 vector_num = tx_info->tcb[i]->intr_vector; 1518 free_irq(bnad->msix_table[vector_num].vector, tx_info->tcb[i]); 1519 } 1520 } 1521 1522 /* NOTE: Should be called for MSIX only 1523 * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel 1524 */ 1525 static int 1526 bnad_tx_msix_register(struct bnad *bnad, struct bnad_tx_info *tx_info, 1527 u32 tx_id, int num_txqs) 1528 { 1529 int i; 1530 int err; 1531 int vector_num; 1532 1533 for (i = 0; i < num_txqs; i++) { 1534 vector_num = tx_info->tcb[i]->intr_vector; 1535 snprintf(tx_info->tcb[i]->name, BNA_Q_NAME_SIZE, "%s TXQ %d", 1536 bnad->netdev->name, 1537 tx_id + tx_info->tcb[i]->id); 1538 err = request_irq(bnad->msix_table[vector_num].vector, 1539 (irq_handler_t)bnad_msix_tx, 0, 1540 tx_info->tcb[i]->name, 1541 tx_info->tcb[i]); 1542 if (err) 1543 goto err_return; 1544 } 1545 1546 return 0; 1547 1548 err_return: 1549 if (i > 0) 1550 bnad_tx_msix_unregister(bnad, tx_info, (i - 1)); 1551 return -1; 1552 } 1553 1554 /* NOTE: Should be called for MSIX only 1555 * Unregisters Rx MSIX vector(s) from the kernel 1556 */ 1557 static void 1558 bnad_rx_msix_unregister(struct bnad *bnad, struct bnad_rx_info *rx_info, 1559 int num_rxps) 1560 { 1561 int i; 1562 int vector_num; 1563 1564 for (i = 0; i < num_rxps; i++) { 1565 if (rx_info->rx_ctrl[i].ccb == NULL) 1566 continue; 1567 1568 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector; 1569 free_irq(bnad->msix_table[vector_num].vector, 1570 rx_info->rx_ctrl[i].ccb); 1571 } 1572 } 1573 1574 /* NOTE: Should be called for MSIX only 1575 * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel 1576 */ 1577 static int 1578 bnad_rx_msix_register(struct bnad *bnad, struct bnad_rx_info *rx_info, 1579 u32 rx_id, int num_rxps) 1580 { 1581 int i; 1582 int err; 1583 int vector_num; 1584 1585 for (i = 0; i < num_rxps; i++) { 1586 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector; 1587 snprintf(rx_info->rx_ctrl[i].ccb->name, BNA_Q_NAME_SIZE, 1588 "%s CQ %d", bnad->netdev->name, 1589 rx_id + rx_info->rx_ctrl[i].ccb->id); 1590 err = request_irq(bnad->msix_table[vector_num].vector, 1591 (irq_handler_t)bnad_msix_rx, 0, 1592 rx_info->rx_ctrl[i].ccb->name, 1593 rx_info->rx_ctrl[i].ccb); 1594 if (err) 1595 goto err_return; 1596 } 1597 1598 return 0; 1599 1600 err_return: 1601 if (i > 0) 1602 bnad_rx_msix_unregister(bnad, rx_info, (i - 1)); 1603 return -1; 1604 } 1605 1606 /* Free Tx object Resources */ 1607 static void 1608 bnad_tx_res_free(struct bnad *bnad, struct bna_res_info *res_info) 1609 { 1610 int i; 1611 1612 for (i = 0; i < BNA_TX_RES_T_MAX; i++) { 1613 if (res_info[i].res_type == BNA_RES_T_MEM) 1614 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 1615 else if (res_info[i].res_type == BNA_RES_T_INTR) 1616 bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info); 1617 } 1618 } 1619 1620 /* Allocates memory and interrupt resources for Tx object */ 1621 static int 1622 bnad_tx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 1623 u32 tx_id) 1624 { 1625 int i, err = 0; 1626 1627 for (i = 0; i < BNA_TX_RES_T_MAX; i++) { 1628 if (res_info[i].res_type == BNA_RES_T_MEM) 1629 err = bnad_mem_alloc(bnad, 1630 &res_info[i].res_u.mem_info); 1631 else if (res_info[i].res_type == BNA_RES_T_INTR) 1632 err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_TX, tx_id, 1633 &res_info[i].res_u.intr_info); 1634 if (err) 1635 goto err_return; 1636 } 1637 return 0; 1638 1639 err_return: 1640 bnad_tx_res_free(bnad, res_info); 1641 return err; 1642 } 1643 1644 /* Free Rx object Resources */ 1645 static void 1646 bnad_rx_res_free(struct bnad *bnad, struct bna_res_info *res_info) 1647 { 1648 int i; 1649 1650 for (i = 0; i < BNA_RX_RES_T_MAX; i++) { 1651 if (res_info[i].res_type == BNA_RES_T_MEM) 1652 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 1653 else if (res_info[i].res_type == BNA_RES_T_INTR) 1654 bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info); 1655 } 1656 } 1657 1658 /* Allocates memory and interrupt resources for Rx object */ 1659 static int 1660 bnad_rx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 1661 uint rx_id) 1662 { 1663 int i, err = 0; 1664 1665 /* All memory needs to be allocated before setup_ccbs */ 1666 for (i = 0; i < BNA_RX_RES_T_MAX; i++) { 1667 if (res_info[i].res_type == BNA_RES_T_MEM) 1668 err = bnad_mem_alloc(bnad, 1669 &res_info[i].res_u.mem_info); 1670 else if (res_info[i].res_type == BNA_RES_T_INTR) 1671 err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_RX, rx_id, 1672 &res_info[i].res_u.intr_info); 1673 if (err) 1674 goto err_return; 1675 } 1676 return 0; 1677 1678 err_return: 1679 bnad_rx_res_free(bnad, res_info); 1680 return err; 1681 } 1682 1683 /* Timer callbacks */ 1684 /* a) IOC timer */ 1685 static void 1686 bnad_ioc_timeout(struct timer_list *t) 1687 { 1688 struct bnad *bnad = timer_container_of(bnad, t, 1689 bna.ioceth.ioc.ioc_timer); 1690 unsigned long flags; 1691 1692 spin_lock_irqsave(&bnad->bna_lock, flags); 1693 bfa_nw_ioc_timeout(&bnad->bna.ioceth.ioc); 1694 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1695 } 1696 1697 static void 1698 bnad_ioc_hb_check(struct timer_list *t) 1699 { 1700 struct bnad *bnad = timer_container_of(bnad, t, 1701 bna.ioceth.ioc.hb_timer); 1702 unsigned long flags; 1703 1704 spin_lock_irqsave(&bnad->bna_lock, flags); 1705 bfa_nw_ioc_hb_check(&bnad->bna.ioceth.ioc); 1706 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1707 } 1708 1709 static void 1710 bnad_iocpf_timeout(struct timer_list *t) 1711 { 1712 struct bnad *bnad = timer_container_of(bnad, t, 1713 bna.ioceth.ioc.iocpf_timer); 1714 unsigned long flags; 1715 1716 spin_lock_irqsave(&bnad->bna_lock, flags); 1717 bfa_nw_iocpf_timeout(&bnad->bna.ioceth.ioc); 1718 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1719 } 1720 1721 static void 1722 bnad_iocpf_sem_timeout(struct timer_list *t) 1723 { 1724 struct bnad *bnad = timer_container_of(bnad, t, 1725 bna.ioceth.ioc.sem_timer); 1726 unsigned long flags; 1727 1728 spin_lock_irqsave(&bnad->bna_lock, flags); 1729 bfa_nw_iocpf_sem_timeout(&bnad->bna.ioceth.ioc); 1730 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1731 } 1732 1733 /* 1734 * All timer routines use bnad->bna_lock to protect against 1735 * the following race, which may occur in case of no locking: 1736 * Time CPU m CPU n 1737 * 0 1 = test_bit 1738 * 1 clear_bit 1739 * 2 timer_delete_sync 1740 * 3 mod_timer 1741 */ 1742 1743 /* b) Dynamic Interrupt Moderation Timer */ 1744 static void 1745 bnad_dim_timeout(struct timer_list *t) 1746 { 1747 struct bnad *bnad = timer_container_of(bnad, t, dim_timer); 1748 struct bnad_rx_info *rx_info; 1749 struct bnad_rx_ctrl *rx_ctrl; 1750 int i, j; 1751 unsigned long flags; 1752 1753 if (!netif_carrier_ok(bnad->netdev)) 1754 return; 1755 1756 spin_lock_irqsave(&bnad->bna_lock, flags); 1757 for (i = 0; i < bnad->num_rx; i++) { 1758 rx_info = &bnad->rx_info[i]; 1759 if (!rx_info->rx) 1760 continue; 1761 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 1762 rx_ctrl = &rx_info->rx_ctrl[j]; 1763 if (!rx_ctrl->ccb) 1764 continue; 1765 bna_rx_dim_update(rx_ctrl->ccb); 1766 } 1767 } 1768 1769 /* Check for BNAD_CF_DIM_ENABLED, does not eliminate a race */ 1770 if (test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) 1771 mod_timer(&bnad->dim_timer, 1772 jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ)); 1773 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1774 } 1775 1776 /* c) Statistics Timer */ 1777 static void 1778 bnad_stats_timeout(struct timer_list *t) 1779 { 1780 struct bnad *bnad = timer_container_of(bnad, t, stats_timer); 1781 unsigned long flags; 1782 1783 if (!netif_running(bnad->netdev) || 1784 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1785 return; 1786 1787 spin_lock_irqsave(&bnad->bna_lock, flags); 1788 bna_hw_stats_get(&bnad->bna); 1789 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1790 } 1791 1792 /* 1793 * Set up timer for DIM 1794 * Called with bnad->bna_lock held 1795 */ 1796 void 1797 bnad_dim_timer_start(struct bnad *bnad) 1798 { 1799 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED && 1800 !test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) { 1801 timer_setup(&bnad->dim_timer, bnad_dim_timeout, 0); 1802 set_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags); 1803 mod_timer(&bnad->dim_timer, 1804 jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ)); 1805 } 1806 } 1807 1808 /* 1809 * Set up timer for statistics 1810 * Called with mutex_lock(&bnad->conf_mutex) held 1811 */ 1812 static void 1813 bnad_stats_timer_start(struct bnad *bnad) 1814 { 1815 unsigned long flags; 1816 1817 spin_lock_irqsave(&bnad->bna_lock, flags); 1818 if (!test_and_set_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) { 1819 timer_setup(&bnad->stats_timer, bnad_stats_timeout, 0); 1820 mod_timer(&bnad->stats_timer, 1821 jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ)); 1822 } 1823 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1824 } 1825 1826 /* 1827 * Stops the stats timer 1828 * Called with mutex_lock(&bnad->conf_mutex) held 1829 */ 1830 static void 1831 bnad_stats_timer_stop(struct bnad *bnad) 1832 { 1833 int to_del = 0; 1834 unsigned long flags; 1835 1836 spin_lock_irqsave(&bnad->bna_lock, flags); 1837 if (test_and_clear_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1838 to_del = 1; 1839 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1840 if (to_del) 1841 timer_delete_sync(&bnad->stats_timer); 1842 } 1843 1844 /* Utilities */ 1845 1846 static void 1847 bnad_netdev_mc_list_get(struct net_device *netdev, u8 *mc_list) 1848 { 1849 int i = 1; /* Index 0 has broadcast address */ 1850 struct netdev_hw_addr *mc_addr; 1851 1852 netdev_for_each_mc_addr(mc_addr, netdev) { 1853 ether_addr_copy(&mc_list[i * ETH_ALEN], &mc_addr->addr[0]); 1854 i++; 1855 } 1856 } 1857 1858 static int 1859 bnad_napi_poll_rx(struct napi_struct *napi, int budget) 1860 { 1861 struct bnad_rx_ctrl *rx_ctrl = 1862 container_of(napi, struct bnad_rx_ctrl, napi); 1863 struct bnad *bnad = rx_ctrl->bnad; 1864 int rcvd = 0; 1865 1866 rx_ctrl->rx_poll_ctr++; 1867 1868 if (!netif_carrier_ok(bnad->netdev)) 1869 goto poll_exit; 1870 1871 rcvd = bnad_cq_process(bnad, rx_ctrl->ccb, budget); 1872 if (rcvd >= budget) 1873 return rcvd; 1874 1875 poll_exit: 1876 napi_complete_done(napi, rcvd); 1877 1878 rx_ctrl->rx_complete++; 1879 1880 if (rx_ctrl->ccb) 1881 bnad_enable_rx_irq_unsafe(rx_ctrl->ccb); 1882 1883 return rcvd; 1884 } 1885 1886 static void 1887 bnad_napi_add(struct bnad *bnad, u32 rx_id) 1888 { 1889 struct bnad_rx_ctrl *rx_ctrl; 1890 int i; 1891 1892 /* Initialize & enable NAPI */ 1893 for (i = 0; i < bnad->num_rxp_per_rx; i++) { 1894 rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i]; 1895 netif_napi_add(bnad->netdev, &rx_ctrl->napi, 1896 bnad_napi_poll_rx); 1897 } 1898 } 1899 1900 static void 1901 bnad_napi_delete(struct bnad *bnad, u32 rx_id) 1902 { 1903 int i; 1904 1905 /* First disable and then clean up */ 1906 for (i = 0; i < bnad->num_rxp_per_rx; i++) 1907 netif_napi_del(&bnad->rx_info[rx_id].rx_ctrl[i].napi); 1908 } 1909 1910 /* Should be held with conf_lock held */ 1911 void 1912 bnad_destroy_tx(struct bnad *bnad, u32 tx_id) 1913 { 1914 struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id]; 1915 struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0]; 1916 unsigned long flags; 1917 1918 if (!tx_info->tx) 1919 return; 1920 1921 init_completion(&bnad->bnad_completions.tx_comp); 1922 spin_lock_irqsave(&bnad->bna_lock, flags); 1923 bna_tx_disable(tx_info->tx, BNA_HARD_CLEANUP, bnad_cb_tx_disabled); 1924 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1925 wait_for_completion(&bnad->bnad_completions.tx_comp); 1926 1927 if (tx_info->tcb[0]->intr_type == BNA_INTR_T_MSIX) 1928 bnad_tx_msix_unregister(bnad, tx_info, 1929 bnad->num_txq_per_tx); 1930 1931 spin_lock_irqsave(&bnad->bna_lock, flags); 1932 bna_tx_destroy(tx_info->tx); 1933 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1934 1935 tx_info->tx = NULL; 1936 tx_info->tx_id = 0; 1937 1938 bnad_tx_res_free(bnad, res_info); 1939 } 1940 1941 /* Should be held with conf_lock held */ 1942 int 1943 bnad_setup_tx(struct bnad *bnad, u32 tx_id) 1944 { 1945 int err; 1946 struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id]; 1947 struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0]; 1948 struct bna_intr_info *intr_info = 1949 &res_info[BNA_TX_RES_INTR_T_TXCMPL].res_u.intr_info; 1950 struct bna_tx_config *tx_config = &bnad->tx_config[tx_id]; 1951 static const struct bna_tx_event_cbfn tx_cbfn = { 1952 .tcb_setup_cbfn = bnad_cb_tcb_setup, 1953 .tcb_destroy_cbfn = bnad_cb_tcb_destroy, 1954 .tx_stall_cbfn = bnad_cb_tx_stall, 1955 .tx_resume_cbfn = bnad_cb_tx_resume, 1956 .tx_cleanup_cbfn = bnad_cb_tx_cleanup, 1957 }; 1958 1959 struct bna_tx *tx; 1960 unsigned long flags; 1961 1962 tx_info->tx_id = tx_id; 1963 1964 /* Initialize the Tx object configuration */ 1965 tx_config->num_txq = bnad->num_txq_per_tx; 1966 tx_config->txq_depth = bnad->txq_depth; 1967 tx_config->tx_type = BNA_TX_T_REGULAR; 1968 tx_config->coalescing_timeo = bnad->tx_coalescing_timeo; 1969 1970 /* Get BNA's resource requirement for one tx object */ 1971 spin_lock_irqsave(&bnad->bna_lock, flags); 1972 bna_tx_res_req(bnad->num_txq_per_tx, 1973 bnad->txq_depth, res_info); 1974 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1975 1976 /* Fill Unmap Q memory requirements */ 1977 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_TX_RES_MEM_T_UNMAPQ], 1978 bnad->num_txq_per_tx, (sizeof(struct bnad_tx_unmap) * 1979 bnad->txq_depth)); 1980 1981 /* Allocate resources */ 1982 err = bnad_tx_res_alloc(bnad, res_info, tx_id); 1983 if (err) 1984 return err; 1985 1986 /* Ask BNA to create one Tx object, supplying required resources */ 1987 spin_lock_irqsave(&bnad->bna_lock, flags); 1988 tx = bna_tx_create(&bnad->bna, bnad, tx_config, &tx_cbfn, res_info, 1989 tx_info); 1990 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1991 if (!tx) { 1992 err = -ENOMEM; 1993 goto err_return; 1994 } 1995 tx_info->tx = tx; 1996 1997 INIT_DELAYED_WORK(&tx_info->tx_cleanup_work, bnad_tx_cleanup); 1998 1999 /* Register ISR for the Tx object */ 2000 if (intr_info->intr_type == BNA_INTR_T_MSIX) { 2001 err = bnad_tx_msix_register(bnad, tx_info, 2002 tx_id, bnad->num_txq_per_tx); 2003 if (err) 2004 goto cleanup_tx; 2005 } 2006 2007 spin_lock_irqsave(&bnad->bna_lock, flags); 2008 bna_tx_enable(tx); 2009 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2010 2011 return 0; 2012 2013 cleanup_tx: 2014 spin_lock_irqsave(&bnad->bna_lock, flags); 2015 bna_tx_destroy(tx_info->tx); 2016 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2017 tx_info->tx = NULL; 2018 tx_info->tx_id = 0; 2019 err_return: 2020 bnad_tx_res_free(bnad, res_info); 2021 return err; 2022 } 2023 2024 /* Setup the rx config for bna_rx_create */ 2025 /* bnad decides the configuration */ 2026 static void 2027 bnad_init_rx_config(struct bnad *bnad, struct bna_rx_config *rx_config) 2028 { 2029 memset(rx_config, 0, sizeof(*rx_config)); 2030 rx_config->rx_type = BNA_RX_T_REGULAR; 2031 rx_config->num_paths = bnad->num_rxp_per_rx; 2032 rx_config->coalescing_timeo = bnad->rx_coalescing_timeo; 2033 2034 if (bnad->num_rxp_per_rx > 1) { 2035 rx_config->rss_status = BNA_STATUS_T_ENABLED; 2036 rx_config->rss_config.hash_type = 2037 (BFI_ENET_RSS_IPV6 | 2038 BFI_ENET_RSS_IPV6_TCP | 2039 BFI_ENET_RSS_IPV4 | 2040 BFI_ENET_RSS_IPV4_TCP); 2041 rx_config->rss_config.hash_mask = 2042 bnad->num_rxp_per_rx - 1; 2043 netdev_rss_key_fill(rx_config->rss_config.toeplitz_hash_key, 2044 sizeof(rx_config->rss_config.toeplitz_hash_key)); 2045 } else { 2046 rx_config->rss_status = BNA_STATUS_T_DISABLED; 2047 memset(&rx_config->rss_config, 0, 2048 sizeof(rx_config->rss_config)); 2049 } 2050 2051 rx_config->frame_size = BNAD_FRAME_SIZE(bnad->netdev->mtu); 2052 rx_config->q0_multi_buf = BNA_STATUS_T_DISABLED; 2053 2054 /* BNA_RXP_SINGLE - one data-buffer queue 2055 * BNA_RXP_SLR - one small-buffer and one large-buffer queues 2056 * BNA_RXP_HDS - one header-buffer and one data-buffer queues 2057 */ 2058 /* TODO: configurable param for queue type */ 2059 rx_config->rxp_type = BNA_RXP_SLR; 2060 2061 if (BNAD_PCI_DEV_IS_CAT2(bnad) && 2062 rx_config->frame_size > 4096) { 2063 /* though size_routing_enable is set in SLR, 2064 * small packets may get routed to same rxq. 2065 * set buf_size to 2048 instead of PAGE_SIZE. 2066 */ 2067 rx_config->q0_buf_size = 2048; 2068 /* this should be in multiples of 2 */ 2069 rx_config->q0_num_vecs = 4; 2070 rx_config->q0_depth = bnad->rxq_depth * rx_config->q0_num_vecs; 2071 rx_config->q0_multi_buf = BNA_STATUS_T_ENABLED; 2072 } else { 2073 rx_config->q0_buf_size = rx_config->frame_size; 2074 rx_config->q0_num_vecs = 1; 2075 rx_config->q0_depth = bnad->rxq_depth; 2076 } 2077 2078 /* initialize for q1 for BNA_RXP_SLR/BNA_RXP_HDS */ 2079 if (rx_config->rxp_type == BNA_RXP_SLR) { 2080 rx_config->q1_depth = bnad->rxq_depth; 2081 rx_config->q1_buf_size = BFI_SMALL_RXBUF_SIZE; 2082 } 2083 2084 rx_config->vlan_strip_status = 2085 (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) ? 2086 BNA_STATUS_T_ENABLED : BNA_STATUS_T_DISABLED; 2087 } 2088 2089 static void 2090 bnad_rx_ctrl_init(struct bnad *bnad, u32 rx_id) 2091 { 2092 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2093 int i; 2094 2095 for (i = 0; i < bnad->num_rxp_per_rx; i++) 2096 rx_info->rx_ctrl[i].bnad = bnad; 2097 } 2098 2099 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2100 static u32 2101 bnad_reinit_rx(struct bnad *bnad) 2102 { 2103 struct net_device *netdev = bnad->netdev; 2104 u32 err = 0, current_err = 0; 2105 u32 rx_id = 0, count = 0; 2106 unsigned long flags; 2107 2108 /* destroy and create new rx objects */ 2109 for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { 2110 if (!bnad->rx_info[rx_id].rx) 2111 continue; 2112 bnad_destroy_rx(bnad, rx_id); 2113 } 2114 2115 spin_lock_irqsave(&bnad->bna_lock, flags); 2116 bna_enet_mtu_set(&bnad->bna.enet, 2117 BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); 2118 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2119 2120 for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { 2121 count++; 2122 current_err = bnad_setup_rx(bnad, rx_id); 2123 if (current_err && !err) { 2124 err = current_err; 2125 netdev_err(netdev, "RXQ:%u setup failed\n", rx_id); 2126 } 2127 } 2128 2129 /* restore rx configuration */ 2130 if (bnad->rx_info[0].rx && !err) { 2131 bnad_restore_vlans(bnad, 0); 2132 bnad_enable_default_bcast(bnad); 2133 spin_lock_irqsave(&bnad->bna_lock, flags); 2134 bnad_mac_addr_set_locked(bnad, netdev->dev_addr); 2135 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2136 bnad_set_rx_mode(netdev); 2137 } 2138 2139 return count; 2140 } 2141 2142 /* Called with bnad_conf_lock() held */ 2143 void 2144 bnad_destroy_rx(struct bnad *bnad, u32 rx_id) 2145 { 2146 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2147 struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; 2148 struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0]; 2149 unsigned long flags; 2150 int to_del = 0; 2151 2152 if (!rx_info->rx) 2153 return; 2154 2155 if (0 == rx_id) { 2156 spin_lock_irqsave(&bnad->bna_lock, flags); 2157 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED && 2158 test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) { 2159 clear_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags); 2160 to_del = 1; 2161 } 2162 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2163 if (to_del) 2164 timer_delete_sync(&bnad->dim_timer); 2165 } 2166 2167 init_completion(&bnad->bnad_completions.rx_comp); 2168 spin_lock_irqsave(&bnad->bna_lock, flags); 2169 bna_rx_disable(rx_info->rx, BNA_HARD_CLEANUP, bnad_cb_rx_disabled); 2170 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2171 wait_for_completion(&bnad->bnad_completions.rx_comp); 2172 2173 if (rx_info->rx_ctrl[0].ccb->intr_type == BNA_INTR_T_MSIX) 2174 bnad_rx_msix_unregister(bnad, rx_info, rx_config->num_paths); 2175 2176 bnad_napi_delete(bnad, rx_id); 2177 2178 spin_lock_irqsave(&bnad->bna_lock, flags); 2179 bna_rx_destroy(rx_info->rx); 2180 2181 rx_info->rx = NULL; 2182 rx_info->rx_id = 0; 2183 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2184 2185 bnad_rx_res_free(bnad, res_info); 2186 } 2187 2188 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2189 int 2190 bnad_setup_rx(struct bnad *bnad, u32 rx_id) 2191 { 2192 int err; 2193 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2194 struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0]; 2195 struct bna_intr_info *intr_info = 2196 &res_info[BNA_RX_RES_T_INTR].res_u.intr_info; 2197 struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; 2198 static const struct bna_rx_event_cbfn rx_cbfn = { 2199 .rcb_setup_cbfn = NULL, 2200 .rcb_destroy_cbfn = NULL, 2201 .ccb_setup_cbfn = bnad_cb_ccb_setup, 2202 .ccb_destroy_cbfn = bnad_cb_ccb_destroy, 2203 .rx_stall_cbfn = bnad_cb_rx_stall, 2204 .rx_cleanup_cbfn = bnad_cb_rx_cleanup, 2205 .rx_post_cbfn = bnad_cb_rx_post, 2206 }; 2207 struct bna_rx *rx; 2208 unsigned long flags; 2209 2210 rx_info->rx_id = rx_id; 2211 2212 /* Initialize the Rx object configuration */ 2213 bnad_init_rx_config(bnad, rx_config); 2214 2215 /* Get BNA's resource requirement for one Rx object */ 2216 spin_lock_irqsave(&bnad->bna_lock, flags); 2217 bna_rx_res_req(rx_config, res_info); 2218 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2219 2220 /* Fill Unmap Q memory requirements */ 2221 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPDQ], 2222 rx_config->num_paths, 2223 (rx_config->q0_depth * 2224 sizeof(struct bnad_rx_unmap)) + 2225 sizeof(struct bnad_rx_unmap_q)); 2226 2227 if (rx_config->rxp_type != BNA_RXP_SINGLE) { 2228 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPHQ], 2229 rx_config->num_paths, 2230 (rx_config->q1_depth * 2231 sizeof(struct bnad_rx_unmap) + 2232 sizeof(struct bnad_rx_unmap_q))); 2233 } 2234 /* Allocate resource */ 2235 err = bnad_rx_res_alloc(bnad, res_info, rx_id); 2236 if (err) 2237 return err; 2238 2239 bnad_rx_ctrl_init(bnad, rx_id); 2240 2241 /* Ask BNA to create one Rx object, supplying required resources */ 2242 spin_lock_irqsave(&bnad->bna_lock, flags); 2243 rx = bna_rx_create(&bnad->bna, bnad, rx_config, &rx_cbfn, res_info, 2244 rx_info); 2245 if (!rx) { 2246 err = -ENOMEM; 2247 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2248 goto err_return; 2249 } 2250 rx_info->rx = rx; 2251 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2252 2253 INIT_WORK(&rx_info->rx_cleanup_work, bnad_rx_cleanup); 2254 2255 /* 2256 * Init NAPI, so that state is set to NAPI_STATE_SCHED, 2257 * so that IRQ handler cannot schedule NAPI at this point. 2258 */ 2259 bnad_napi_add(bnad, rx_id); 2260 2261 /* Register ISR for the Rx object */ 2262 if (intr_info->intr_type == BNA_INTR_T_MSIX) { 2263 err = bnad_rx_msix_register(bnad, rx_info, rx_id, 2264 rx_config->num_paths); 2265 if (err) 2266 goto err_return; 2267 } 2268 2269 spin_lock_irqsave(&bnad->bna_lock, flags); 2270 if (0 == rx_id) { 2271 /* Set up Dynamic Interrupt Moderation Vector */ 2272 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED) 2273 bna_rx_dim_reconfig(&bnad->bna, bna_napi_dim_vector); 2274 2275 /* Enable VLAN filtering only on the default Rx */ 2276 bna_rx_vlanfilter_enable(rx); 2277 2278 /* Start the DIM timer */ 2279 bnad_dim_timer_start(bnad); 2280 } 2281 2282 bna_rx_enable(rx); 2283 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2284 2285 return 0; 2286 2287 err_return: 2288 bnad_destroy_rx(bnad, rx_id); 2289 return err; 2290 } 2291 2292 /* Called with conf_lock & bnad->bna_lock held */ 2293 void 2294 bnad_tx_coalescing_timeo_set(struct bnad *bnad) 2295 { 2296 struct bnad_tx_info *tx_info; 2297 2298 tx_info = &bnad->tx_info[0]; 2299 if (!tx_info->tx) 2300 return; 2301 2302 bna_tx_coalescing_timeo_set(tx_info->tx, bnad->tx_coalescing_timeo); 2303 } 2304 2305 /* Called with conf_lock & bnad->bna_lock held */ 2306 void 2307 bnad_rx_coalescing_timeo_set(struct bnad *bnad) 2308 { 2309 struct bnad_rx_info *rx_info; 2310 int i; 2311 2312 for (i = 0; i < bnad->num_rx; i++) { 2313 rx_info = &bnad->rx_info[i]; 2314 if (!rx_info->rx) 2315 continue; 2316 bna_rx_coalescing_timeo_set(rx_info->rx, 2317 bnad->rx_coalescing_timeo); 2318 } 2319 } 2320 2321 /* 2322 * Called with bnad->bna_lock held 2323 */ 2324 int 2325 bnad_mac_addr_set_locked(struct bnad *bnad, const u8 *mac_addr) 2326 { 2327 int ret; 2328 2329 if (!is_valid_ether_addr(mac_addr)) 2330 return -EADDRNOTAVAIL; 2331 2332 /* If datapath is down, pretend everything went through */ 2333 if (!bnad->rx_info[0].rx) 2334 return 0; 2335 2336 ret = bna_rx_ucast_set(bnad->rx_info[0].rx, mac_addr); 2337 if (ret != BNA_CB_SUCCESS) 2338 return -EADDRNOTAVAIL; 2339 2340 return 0; 2341 } 2342 2343 /* Should be called with conf_lock held */ 2344 int 2345 bnad_enable_default_bcast(struct bnad *bnad) 2346 { 2347 struct bnad_rx_info *rx_info = &bnad->rx_info[0]; 2348 int ret; 2349 unsigned long flags; 2350 2351 init_completion(&bnad->bnad_completions.mcast_comp); 2352 2353 spin_lock_irqsave(&bnad->bna_lock, flags); 2354 ret = bna_rx_mcast_add(rx_info->rx, bnad_bcast_addr, 2355 bnad_cb_rx_mcast_add); 2356 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2357 2358 if (ret == BNA_CB_SUCCESS) 2359 wait_for_completion(&bnad->bnad_completions.mcast_comp); 2360 else 2361 return -ENODEV; 2362 2363 if (bnad->bnad_completions.mcast_comp_status != BNA_CB_SUCCESS) 2364 return -ENODEV; 2365 2366 return 0; 2367 } 2368 2369 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2370 void 2371 bnad_restore_vlans(struct bnad *bnad, u32 rx_id) 2372 { 2373 u16 vid; 2374 unsigned long flags; 2375 2376 for_each_set_bit(vid, bnad->active_vlans, VLAN_N_VID) { 2377 spin_lock_irqsave(&bnad->bna_lock, flags); 2378 bna_rx_vlan_add(bnad->rx_info[rx_id].rx, vid); 2379 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2380 } 2381 } 2382 2383 /* Statistics utilities */ 2384 void 2385 bnad_netdev_qstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats) 2386 { 2387 int i, j; 2388 2389 for (i = 0; i < bnad->num_rx; i++) { 2390 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 2391 if (bnad->rx_info[i].rx_ctrl[j].ccb) { 2392 stats->rx_packets += bnad->rx_info[i]. 2393 rx_ctrl[j].ccb->rcb[0]->rxq->rx_packets; 2394 stats->rx_bytes += bnad->rx_info[i]. 2395 rx_ctrl[j].ccb->rcb[0]->rxq->rx_bytes; 2396 if (bnad->rx_info[i].rx_ctrl[j].ccb->rcb[1] && 2397 bnad->rx_info[i].rx_ctrl[j].ccb-> 2398 rcb[1]->rxq) { 2399 stats->rx_packets += 2400 bnad->rx_info[i].rx_ctrl[j]. 2401 ccb->rcb[1]->rxq->rx_packets; 2402 stats->rx_bytes += 2403 bnad->rx_info[i].rx_ctrl[j]. 2404 ccb->rcb[1]->rxq->rx_bytes; 2405 } 2406 } 2407 } 2408 } 2409 for (i = 0; i < bnad->num_tx; i++) { 2410 for (j = 0; j < bnad->num_txq_per_tx; j++) { 2411 if (bnad->tx_info[i].tcb[j]) { 2412 stats->tx_packets += 2413 bnad->tx_info[i].tcb[j]->txq->tx_packets; 2414 stats->tx_bytes += 2415 bnad->tx_info[i].tcb[j]->txq->tx_bytes; 2416 } 2417 } 2418 } 2419 } 2420 2421 /* 2422 * Must be called with the bna_lock held. 2423 */ 2424 void 2425 bnad_netdev_hwstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats) 2426 { 2427 struct bfi_enet_stats_mac *mac_stats; 2428 u32 bmap; 2429 int i; 2430 2431 mac_stats = &bnad->stats.bna_stats->hw_stats.mac_stats; 2432 stats->rx_errors = 2433 mac_stats->rx_fcs_error + mac_stats->rx_alignment_error + 2434 mac_stats->rx_frame_length_error + mac_stats->rx_code_error + 2435 mac_stats->rx_undersize; 2436 stats->tx_errors = mac_stats->tx_fcs_error + 2437 mac_stats->tx_undersize; 2438 stats->rx_dropped = mac_stats->rx_drop; 2439 stats->tx_dropped = mac_stats->tx_drop; 2440 stats->multicast = mac_stats->rx_multicast; 2441 stats->collisions = mac_stats->tx_total_collision; 2442 2443 stats->rx_length_errors = mac_stats->rx_frame_length_error; 2444 2445 /* receive ring buffer overflow ?? */ 2446 2447 stats->rx_crc_errors = mac_stats->rx_fcs_error; 2448 stats->rx_frame_errors = mac_stats->rx_alignment_error; 2449 /* recv'r fifo overrun */ 2450 bmap = bna_rx_rid_mask(&bnad->bna); 2451 for (i = 0; bmap; i++) { 2452 if (bmap & 1) { 2453 stats->rx_fifo_errors += 2454 bnad->stats.bna_stats-> 2455 hw_stats.rxf_stats[i].frame_drops; 2456 break; 2457 } 2458 bmap >>= 1; 2459 } 2460 } 2461 2462 static void 2463 bnad_mbox_irq_sync(struct bnad *bnad) 2464 { 2465 u32 irq; 2466 unsigned long flags; 2467 2468 spin_lock_irqsave(&bnad->bna_lock, flags); 2469 if (bnad->cfg_flags & BNAD_CF_MSIX) 2470 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector; 2471 else 2472 irq = bnad->pcidev->irq; 2473 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2474 2475 synchronize_irq(irq); 2476 } 2477 2478 /* Utility used by bnad_start_xmit, for doing TSO */ 2479 static int 2480 bnad_tso_prepare(struct bnad *bnad, struct sk_buff *skb) 2481 { 2482 int err; 2483 2484 err = skb_cow_head(skb, 0); 2485 if (err < 0) { 2486 BNAD_UPDATE_CTR(bnad, tso_err); 2487 return err; 2488 } 2489 2490 /* 2491 * For TSO, the TCP checksum field is seeded with pseudo-header sum 2492 * excluding the length field. 2493 */ 2494 if (vlan_get_protocol(skb) == htons(ETH_P_IP)) { 2495 struct iphdr *iph = ip_hdr(skb); 2496 2497 /* Do we really need these? */ 2498 iph->tot_len = 0; 2499 iph->check = 0; 2500 2501 tcp_hdr(skb)->check = 2502 ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 2503 IPPROTO_TCP, 0); 2504 BNAD_UPDATE_CTR(bnad, tso4); 2505 } else { 2506 tcp_v6_gso_csum_prep(skb); 2507 BNAD_UPDATE_CTR(bnad, tso6); 2508 } 2509 2510 return 0; 2511 } 2512 2513 /* 2514 * Initialize Q numbers depending on Rx Paths 2515 * Called with bnad->bna_lock held, because of cfg_flags 2516 * access. 2517 */ 2518 static void 2519 bnad_q_num_init(struct bnad *bnad) 2520 { 2521 int rxps; 2522 2523 rxps = min((uint)num_online_cpus(), 2524 (uint)(BNAD_MAX_RX * BNAD_MAX_RXP_PER_RX)); 2525 2526 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) 2527 rxps = 1; /* INTx */ 2528 2529 bnad->num_rx = 1; 2530 bnad->num_tx = 1; 2531 bnad->num_rxp_per_rx = rxps; 2532 bnad->num_txq_per_tx = BNAD_TXQ_NUM; 2533 } 2534 2535 /* 2536 * Adjusts the Q numbers, given a number of msix vectors 2537 * Give preference to RSS as opposed to Tx priority Queues, 2538 * in such a case, just use 1 Tx Q 2539 * Called with bnad->bna_lock held b'cos of cfg_flags access 2540 */ 2541 static void 2542 bnad_q_num_adjust(struct bnad *bnad, int msix_vectors, int temp) 2543 { 2544 bnad->num_txq_per_tx = 1; 2545 if ((msix_vectors >= (bnad->num_tx * bnad->num_txq_per_tx) + 2546 bnad_rxqs_per_cq + BNAD_MAILBOX_MSIX_VECTORS) && 2547 (bnad->cfg_flags & BNAD_CF_MSIX)) { 2548 bnad->num_rxp_per_rx = msix_vectors - 2549 (bnad->num_tx * bnad->num_txq_per_tx) - 2550 BNAD_MAILBOX_MSIX_VECTORS; 2551 } else 2552 bnad->num_rxp_per_rx = 1; 2553 } 2554 2555 /* Enable / disable ioceth */ 2556 static int 2557 bnad_ioceth_disable(struct bnad *bnad) 2558 { 2559 unsigned long flags; 2560 int err = 0; 2561 2562 spin_lock_irqsave(&bnad->bna_lock, flags); 2563 init_completion(&bnad->bnad_completions.ioc_comp); 2564 bna_ioceth_disable(&bnad->bna.ioceth, BNA_HARD_CLEANUP); 2565 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2566 2567 wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp, 2568 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT)); 2569 2570 err = bnad->bnad_completions.ioc_comp_status; 2571 return err; 2572 } 2573 2574 /* 2575 * The IOC timers rearm one another, so deleting one cannot stop a 2576 * sibling callback from arming it again. Shut them down so a later 2577 * mod_timer() is ignored. 2578 */ 2579 static void 2580 bnad_ioc_timers_shutdown(struct bnad *bnad) 2581 { 2582 struct bfa_ioc *ioc = &bnad->bna.ioceth.ioc; 2583 2584 timer_shutdown_sync(&ioc->ioc_timer); 2585 timer_shutdown_sync(&ioc->sem_timer); 2586 timer_shutdown_sync(&ioc->hb_timer); 2587 timer_shutdown_sync(&ioc->iocpf_timer); 2588 } 2589 2590 static int 2591 bnad_ioceth_enable(struct bnad *bnad) 2592 { 2593 int err = 0; 2594 unsigned long flags; 2595 2596 spin_lock_irqsave(&bnad->bna_lock, flags); 2597 init_completion(&bnad->bnad_completions.ioc_comp); 2598 bnad->bnad_completions.ioc_comp_status = BNA_CB_WAITING; 2599 bna_ioceth_enable(&bnad->bna.ioceth); 2600 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2601 2602 wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp, 2603 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT)); 2604 2605 err = bnad->bnad_completions.ioc_comp_status; 2606 2607 return err; 2608 } 2609 2610 /* Free BNA resources */ 2611 static void 2612 bnad_res_free(struct bnad *bnad, struct bna_res_info *res_info, 2613 u32 res_val_max) 2614 { 2615 int i; 2616 2617 for (i = 0; i < res_val_max; i++) 2618 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 2619 } 2620 2621 /* Allocates memory and interrupt resources for BNA */ 2622 static int 2623 bnad_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 2624 u32 res_val_max) 2625 { 2626 int i, err; 2627 2628 for (i = 0; i < res_val_max; i++) { 2629 err = bnad_mem_alloc(bnad, &res_info[i].res_u.mem_info); 2630 if (err) 2631 goto err_return; 2632 } 2633 return 0; 2634 2635 err_return: 2636 bnad_res_free(bnad, res_info, res_val_max); 2637 return err; 2638 } 2639 2640 /* Interrupt enable / disable */ 2641 static void 2642 bnad_enable_msix(struct bnad *bnad) 2643 { 2644 int i, ret; 2645 unsigned long flags; 2646 2647 spin_lock_irqsave(&bnad->bna_lock, flags); 2648 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) { 2649 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2650 return; 2651 } 2652 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2653 2654 if (bnad->msix_table) 2655 return; 2656 2657 bnad->msix_table = 2658 kzalloc_objs(struct msix_entry, bnad->msix_num); 2659 2660 if (!bnad->msix_table) 2661 goto intx_mode; 2662 2663 for (i = 0; i < bnad->msix_num; i++) 2664 bnad->msix_table[i].entry = i; 2665 2666 ret = pci_enable_msix_range(bnad->pcidev, bnad->msix_table, 2667 1, bnad->msix_num); 2668 if (ret < 0) { 2669 goto intx_mode; 2670 } else if (ret < bnad->msix_num) { 2671 dev_warn(&bnad->pcidev->dev, 2672 "%d MSI-X vectors allocated < %d requested\n", 2673 ret, bnad->msix_num); 2674 2675 spin_lock_irqsave(&bnad->bna_lock, flags); 2676 /* ret = #of vectors that we got */ 2677 bnad_q_num_adjust(bnad, (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2, 2678 (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2); 2679 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2680 2681 bnad->msix_num = BNAD_NUM_TXQ + BNAD_NUM_RXP + 2682 BNAD_MAILBOX_MSIX_VECTORS; 2683 2684 if (bnad->msix_num > ret) { 2685 pci_disable_msix(bnad->pcidev); 2686 goto intx_mode; 2687 } 2688 } 2689 2690 pci_intx(bnad->pcidev, 0); 2691 2692 return; 2693 2694 intx_mode: 2695 dev_warn(&bnad->pcidev->dev, 2696 "MSI-X enable failed - operating in INTx mode\n"); 2697 2698 kfree(bnad->msix_table); 2699 bnad->msix_table = NULL; 2700 bnad->msix_num = 0; 2701 spin_lock_irqsave(&bnad->bna_lock, flags); 2702 bnad->cfg_flags &= ~BNAD_CF_MSIX; 2703 bnad_q_num_init(bnad); 2704 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2705 } 2706 2707 static void 2708 bnad_disable_msix(struct bnad *bnad) 2709 { 2710 u32 cfg_flags; 2711 unsigned long flags; 2712 2713 spin_lock_irqsave(&bnad->bna_lock, flags); 2714 cfg_flags = bnad->cfg_flags; 2715 if (bnad->cfg_flags & BNAD_CF_MSIX) 2716 bnad->cfg_flags &= ~BNAD_CF_MSIX; 2717 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2718 2719 if (cfg_flags & BNAD_CF_MSIX) { 2720 pci_disable_msix(bnad->pcidev); 2721 kfree(bnad->msix_table); 2722 bnad->msix_table = NULL; 2723 } 2724 } 2725 2726 /* Netdev entry points */ 2727 static int 2728 bnad_open(struct net_device *netdev) 2729 { 2730 int err; 2731 struct bnad *bnad = netdev_priv(netdev); 2732 struct bna_pause_config pause_config; 2733 unsigned long flags; 2734 2735 mutex_lock(&bnad->conf_mutex); 2736 2737 /* Tx */ 2738 err = bnad_setup_tx(bnad, 0); 2739 if (err) 2740 goto err_return; 2741 2742 /* Rx */ 2743 err = bnad_setup_rx(bnad, 0); 2744 if (err) 2745 goto cleanup_tx; 2746 2747 /* Port */ 2748 pause_config.tx_pause = 0; 2749 pause_config.rx_pause = 0; 2750 2751 spin_lock_irqsave(&bnad->bna_lock, flags); 2752 bna_enet_mtu_set(&bnad->bna.enet, 2753 BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); 2754 bna_enet_pause_config(&bnad->bna.enet, &pause_config); 2755 bna_enet_enable(&bnad->bna.enet); 2756 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2757 2758 /* Enable broadcast */ 2759 bnad_enable_default_bcast(bnad); 2760 2761 /* Restore VLANs, if any */ 2762 bnad_restore_vlans(bnad, 0); 2763 2764 /* Set the UCAST address */ 2765 spin_lock_irqsave(&bnad->bna_lock, flags); 2766 bnad_mac_addr_set_locked(bnad, netdev->dev_addr); 2767 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2768 2769 /* Start the stats timer */ 2770 bnad_stats_timer_start(bnad); 2771 2772 mutex_unlock(&bnad->conf_mutex); 2773 2774 return 0; 2775 2776 cleanup_tx: 2777 bnad_destroy_tx(bnad, 0); 2778 2779 err_return: 2780 mutex_unlock(&bnad->conf_mutex); 2781 return err; 2782 } 2783 2784 static int 2785 bnad_stop(struct net_device *netdev) 2786 { 2787 struct bnad *bnad = netdev_priv(netdev); 2788 unsigned long flags; 2789 2790 mutex_lock(&bnad->conf_mutex); 2791 2792 /* Stop the stats timer */ 2793 bnad_stats_timer_stop(bnad); 2794 2795 init_completion(&bnad->bnad_completions.enet_comp); 2796 2797 spin_lock_irqsave(&bnad->bna_lock, flags); 2798 bna_enet_disable(&bnad->bna.enet, BNA_HARD_CLEANUP, 2799 bnad_cb_enet_disabled); 2800 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2801 2802 wait_for_completion(&bnad->bnad_completions.enet_comp); 2803 2804 bnad_destroy_tx(bnad, 0); 2805 bnad_destroy_rx(bnad, 0); 2806 2807 /* Synchronize mailbox IRQ */ 2808 bnad_mbox_irq_sync(bnad); 2809 2810 mutex_unlock(&bnad->conf_mutex); 2811 2812 return 0; 2813 } 2814 2815 /* TX */ 2816 /* Returns 0 for success */ 2817 static int 2818 bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb, 2819 struct sk_buff *skb, struct bna_txq_entry *txqent) 2820 { 2821 u16 flags = 0; 2822 u32 gso_size; 2823 u16 vlan_tag = 0; 2824 2825 if (skb_vlan_tag_present(skb)) { 2826 vlan_tag = (u16)skb_vlan_tag_get(skb); 2827 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); 2828 } 2829 if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) { 2830 vlan_tag = ((tcb->priority & 0x7) << VLAN_PRIO_SHIFT) 2831 | (vlan_tag & 0x1fff); 2832 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); 2833 } 2834 txqent->hdr.wi.vlan_tag = htons(vlan_tag); 2835 2836 if (skb_is_gso(skb)) { 2837 gso_size = skb_shinfo(skb)->gso_size; 2838 if (unlikely(gso_size > bnad->netdev->mtu)) { 2839 BNAD_UPDATE_CTR(bnad, tx_skb_mss_too_long); 2840 return -EINVAL; 2841 } 2842 if (unlikely((gso_size + skb_tcp_all_headers(skb)) >= skb->len)) { 2843 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); 2844 txqent->hdr.wi.lso_mss = 0; 2845 BNAD_UPDATE_CTR(bnad, tx_skb_tso_too_short); 2846 } else { 2847 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND_LSO); 2848 txqent->hdr.wi.lso_mss = htons(gso_size); 2849 } 2850 2851 if (bnad_tso_prepare(bnad, skb)) { 2852 BNAD_UPDATE_CTR(bnad, tx_skb_tso_prepare); 2853 return -EINVAL; 2854 } 2855 2856 flags |= (BNA_TXQ_WI_CF_IP_CKSUM | BNA_TXQ_WI_CF_TCP_CKSUM); 2857 txqent->hdr.wi.l4_hdr_size_n_offset = 2858 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET( 2859 tcp_hdrlen(skb) >> 2, skb_transport_offset(skb))); 2860 } else { 2861 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); 2862 txqent->hdr.wi.lso_mss = 0; 2863 2864 if (unlikely(skb->len > (bnad->netdev->mtu + VLAN_ETH_HLEN))) { 2865 BNAD_UPDATE_CTR(bnad, tx_skb_non_tso_too_long); 2866 return -EINVAL; 2867 } 2868 2869 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2870 __be16 net_proto = vlan_get_protocol(skb); 2871 u8 proto = 0; 2872 2873 if (net_proto == htons(ETH_P_IP)) 2874 proto = ip_hdr(skb)->protocol; 2875 #ifdef NETIF_F_IPV6_CSUM 2876 else if (net_proto == htons(ETH_P_IPV6)) { 2877 /* nexthdr may not be TCP immediately. */ 2878 proto = ipv6_hdr(skb)->nexthdr; 2879 } 2880 #endif 2881 if (proto == IPPROTO_TCP) { 2882 flags |= BNA_TXQ_WI_CF_TCP_CKSUM; 2883 txqent->hdr.wi.l4_hdr_size_n_offset = 2884 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET 2885 (0, skb_transport_offset(skb))); 2886 2887 BNAD_UPDATE_CTR(bnad, tcpcsum_offload); 2888 2889 if (unlikely(skb_headlen(skb) < 2890 skb_tcp_all_headers(skb))) { 2891 BNAD_UPDATE_CTR(bnad, tx_skb_tcp_hdr); 2892 return -EINVAL; 2893 } 2894 } else if (proto == IPPROTO_UDP) { 2895 flags |= BNA_TXQ_WI_CF_UDP_CKSUM; 2896 txqent->hdr.wi.l4_hdr_size_n_offset = 2897 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET 2898 (0, skb_transport_offset(skb))); 2899 2900 BNAD_UPDATE_CTR(bnad, udpcsum_offload); 2901 if (unlikely(skb_headlen(skb) < 2902 skb_transport_offset(skb) + 2903 sizeof(struct udphdr))) { 2904 BNAD_UPDATE_CTR(bnad, tx_skb_udp_hdr); 2905 return -EINVAL; 2906 } 2907 } else { 2908 2909 BNAD_UPDATE_CTR(bnad, tx_skb_csum_err); 2910 return -EINVAL; 2911 } 2912 } else 2913 txqent->hdr.wi.l4_hdr_size_n_offset = 0; 2914 } 2915 2916 txqent->hdr.wi.flags = htons(flags); 2917 txqent->hdr.wi.frame_length = htonl(skb->len); 2918 2919 return 0; 2920 } 2921 2922 /* 2923 * bnad_start_xmit : Netdev entry point for Transmit 2924 * Called under lock held by net_device 2925 */ 2926 static netdev_tx_t 2927 bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) 2928 { 2929 struct bnad *bnad = netdev_priv(netdev); 2930 u32 txq_id = 0; 2931 struct bna_tcb *tcb = NULL; 2932 struct bnad_tx_unmap *unmap_q, *unmap, *head_unmap; 2933 u32 prod, q_depth, vect_id; 2934 u32 wis, vectors, len; 2935 int i; 2936 dma_addr_t dma_addr; 2937 struct bna_txq_entry *txqent; 2938 2939 len = skb_headlen(skb); 2940 2941 /* Sanity checks for the skb */ 2942 2943 if (unlikely(skb->len <= ETH_HLEN)) { 2944 dev_kfree_skb_any(skb); 2945 BNAD_UPDATE_CTR(bnad, tx_skb_too_short); 2946 return NETDEV_TX_OK; 2947 } 2948 if (unlikely(len > BFI_TX_MAX_DATA_PER_VECTOR)) { 2949 dev_kfree_skb_any(skb); 2950 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); 2951 return NETDEV_TX_OK; 2952 } 2953 if (unlikely(len == 0)) { 2954 dev_kfree_skb_any(skb); 2955 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); 2956 return NETDEV_TX_OK; 2957 } 2958 2959 tcb = bnad->tx_info[0].tcb[txq_id]; 2960 2961 /* 2962 * Takes care of the Tx that is scheduled between clearing the flag 2963 * and the netif_tx_stop_all_queues() call. 2964 */ 2965 if (unlikely(!tcb || !test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) { 2966 dev_kfree_skb_any(skb); 2967 BNAD_UPDATE_CTR(bnad, tx_skb_stopping); 2968 return NETDEV_TX_OK; 2969 } 2970 2971 q_depth = tcb->q_depth; 2972 prod = tcb->producer_index; 2973 unmap_q = tcb->unmap_q; 2974 2975 vectors = 1 + skb_shinfo(skb)->nr_frags; 2976 wis = BNA_TXQ_WI_NEEDED(vectors); /* 4 vectors per work item */ 2977 2978 if (unlikely(vectors > BFI_TX_MAX_VECTORS_PER_PKT)) { 2979 dev_kfree_skb_any(skb); 2980 BNAD_UPDATE_CTR(bnad, tx_skb_max_vectors); 2981 return NETDEV_TX_OK; 2982 } 2983 2984 /* Check for available TxQ resources */ 2985 if (unlikely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { 2986 if ((*tcb->hw_consumer_index != tcb->consumer_index) && 2987 !test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { 2988 u32 sent; 2989 sent = bnad_txcmpl_process(bnad, tcb); 2990 if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 2991 bna_ib_ack(tcb->i_dbell, sent); 2992 smp_mb__before_atomic(); 2993 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 2994 } else { 2995 netif_stop_queue(netdev); 2996 BNAD_UPDATE_CTR(bnad, netif_queue_stop); 2997 } 2998 2999 smp_mb(); 3000 /* 3001 * Check again to deal with race condition between 3002 * netif_stop_queue here, and netif_wake_queue in 3003 * interrupt handler which is not inside netif tx lock. 3004 */ 3005 if (likely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { 3006 BNAD_UPDATE_CTR(bnad, netif_queue_stop); 3007 return NETDEV_TX_BUSY; 3008 } else { 3009 netif_wake_queue(netdev); 3010 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 3011 } 3012 } 3013 3014 txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; 3015 head_unmap = &unmap_q[prod]; 3016 3017 /* Program the opcode, flags, frame_len, num_vectors in WI */ 3018 if (bnad_txq_wi_prepare(bnad, tcb, skb, txqent)) { 3019 dev_kfree_skb_any(skb); 3020 return NETDEV_TX_OK; 3021 } 3022 txqent->hdr.wi.reserved = 0; 3023 txqent->hdr.wi.num_vectors = vectors; 3024 3025 head_unmap->nvecs = 0; 3026 3027 /* Program the vectors */ 3028 unmap = head_unmap; 3029 dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, 3030 len, DMA_TO_DEVICE); 3031 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 3032 dev_kfree_skb_any(skb); 3033 BNAD_UPDATE_CTR(bnad, tx_skb_map_failed); 3034 return NETDEV_TX_OK; 3035 } 3036 head_unmap->skb = skb; 3037 BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr); 3038 txqent->vector[0].length = htons(len); 3039 dma_unmap_addr_set(&unmap->vectors[0], dma_addr, dma_addr); 3040 head_unmap->nvecs++; 3041 3042 for (i = 0, vect_id = 0; i < vectors - 1; i++) { 3043 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3044 u32 size = skb_frag_size(frag); 3045 3046 if (unlikely(size == 0)) { 3047 /* Undo the changes starting at tcb->producer_index */ 3048 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, 3049 tcb->producer_index); 3050 dev_kfree_skb_any(skb); 3051 BNAD_UPDATE_CTR(bnad, tx_skb_frag_zero); 3052 return NETDEV_TX_OK; 3053 } 3054 3055 len += size; 3056 3057 vect_id++; 3058 if (vect_id == BFI_TX_MAX_VECTORS_PER_WI) { 3059 vect_id = 0; 3060 BNA_QE_INDX_INC(prod, q_depth); 3061 txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; 3062 txqent->hdr.wi_ext.opcode = htons(BNA_TXQ_WI_EXTENSION); 3063 unmap = &unmap_q[prod]; 3064 } 3065 3066 dma_addr = skb_frag_dma_map(&bnad->pcidev->dev, frag, 3067 0, size, DMA_TO_DEVICE); 3068 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 3069 /* Undo the changes starting at tcb->producer_index */ 3070 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, 3071 tcb->producer_index); 3072 dev_kfree_skb_any(skb); 3073 BNAD_UPDATE_CTR(bnad, tx_skb_map_failed); 3074 return NETDEV_TX_OK; 3075 } 3076 3077 dma_unmap_len_set(&unmap->vectors[vect_id], dma_len, size); 3078 BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[vect_id].host_addr); 3079 txqent->vector[vect_id].length = htons(size); 3080 dma_unmap_addr_set(&unmap->vectors[vect_id], dma_addr, 3081 dma_addr); 3082 head_unmap->nvecs++; 3083 } 3084 3085 if (unlikely(len != skb->len)) { 3086 /* Undo the changes starting at tcb->producer_index */ 3087 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, tcb->producer_index); 3088 dev_kfree_skb_any(skb); 3089 BNAD_UPDATE_CTR(bnad, tx_skb_len_mismatch); 3090 return NETDEV_TX_OK; 3091 } 3092 3093 BNA_QE_INDX_INC(prod, q_depth); 3094 tcb->producer_index = prod; 3095 3096 wmb(); 3097 3098 if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 3099 return NETDEV_TX_OK; 3100 3101 skb_tx_timestamp(skb); 3102 3103 bna_txq_prod_indx_doorbell(tcb); 3104 3105 return NETDEV_TX_OK; 3106 } 3107 3108 /* 3109 * Used spin_lock to synchronize reading of stats structures, which 3110 * is written by BNA under the same lock. 3111 */ 3112 static void 3113 bnad_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 3114 { 3115 struct bnad *bnad = netdev_priv(netdev); 3116 unsigned long flags; 3117 3118 spin_lock_irqsave(&bnad->bna_lock, flags); 3119 3120 bnad_netdev_qstats_fill(bnad, stats); 3121 bnad_netdev_hwstats_fill(bnad, stats); 3122 3123 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3124 } 3125 3126 static void 3127 bnad_set_rx_ucast_fltr(struct bnad *bnad) 3128 { 3129 struct net_device *netdev = bnad->netdev; 3130 int uc_count = netdev_uc_count(netdev); 3131 enum bna_cb_status ret; 3132 u8 *mac_list; 3133 struct netdev_hw_addr *ha; 3134 int entry; 3135 3136 if (netdev_uc_empty(bnad->netdev)) { 3137 bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL); 3138 return; 3139 } 3140 3141 if (uc_count > bna_attr(&bnad->bna)->num_ucmac) 3142 goto mode_default; 3143 3144 mac_list = kcalloc(ETH_ALEN, uc_count, GFP_ATOMIC); 3145 if (mac_list == NULL) 3146 goto mode_default; 3147 3148 entry = 0; 3149 netdev_for_each_uc_addr(ha, netdev) { 3150 ether_addr_copy(&mac_list[entry * ETH_ALEN], &ha->addr[0]); 3151 entry++; 3152 } 3153 3154 ret = bna_rx_ucast_listset(bnad->rx_info[0].rx, entry, mac_list); 3155 kfree(mac_list); 3156 3157 if (ret != BNA_CB_SUCCESS) 3158 goto mode_default; 3159 3160 return; 3161 3162 /* ucast packets not in UCAM are routed to default function */ 3163 mode_default: 3164 bnad->cfg_flags |= BNAD_CF_DEFAULT; 3165 bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL); 3166 } 3167 3168 static void 3169 bnad_set_rx_mcast_fltr(struct bnad *bnad) 3170 { 3171 struct net_device *netdev = bnad->netdev; 3172 int mc_count = netdev_mc_count(netdev); 3173 enum bna_cb_status ret; 3174 u8 *mac_list; 3175 3176 if (netdev->flags & IFF_ALLMULTI) 3177 goto mode_allmulti; 3178 3179 if (netdev_mc_empty(netdev)) 3180 return; 3181 3182 if (mc_count > bna_attr(&bnad->bna)->num_mcmac) 3183 goto mode_allmulti; 3184 3185 mac_list = kcalloc(mc_count + 1, ETH_ALEN, GFP_ATOMIC); 3186 3187 if (mac_list == NULL) 3188 goto mode_allmulti; 3189 3190 ether_addr_copy(&mac_list[0], &bnad_bcast_addr[0]); 3191 3192 /* copy rest of the MCAST addresses */ 3193 bnad_netdev_mc_list_get(netdev, mac_list); 3194 ret = bna_rx_mcast_listset(bnad->rx_info[0].rx, mc_count + 1, mac_list); 3195 kfree(mac_list); 3196 3197 if (ret != BNA_CB_SUCCESS) 3198 goto mode_allmulti; 3199 3200 return; 3201 3202 mode_allmulti: 3203 bnad->cfg_flags |= BNAD_CF_ALLMULTI; 3204 bna_rx_mcast_delall(bnad->rx_info[0].rx); 3205 } 3206 3207 void 3208 bnad_set_rx_mode(struct net_device *netdev) 3209 { 3210 struct bnad *bnad = netdev_priv(netdev); 3211 enum bna_rxmode new_mode, mode_mask; 3212 unsigned long flags; 3213 3214 spin_lock_irqsave(&bnad->bna_lock, flags); 3215 3216 if (bnad->rx_info[0].rx == NULL) { 3217 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3218 return; 3219 } 3220 3221 /* clear bnad flags to update it with new settings */ 3222 bnad->cfg_flags &= ~(BNAD_CF_PROMISC | BNAD_CF_DEFAULT | 3223 BNAD_CF_ALLMULTI); 3224 3225 new_mode = 0; 3226 if (netdev->flags & IFF_PROMISC) { 3227 new_mode |= BNAD_RXMODE_PROMISC_DEFAULT; 3228 bnad->cfg_flags |= BNAD_CF_PROMISC; 3229 } else { 3230 bnad_set_rx_mcast_fltr(bnad); 3231 3232 if (bnad->cfg_flags & BNAD_CF_ALLMULTI) 3233 new_mode |= BNA_RXMODE_ALLMULTI; 3234 3235 bnad_set_rx_ucast_fltr(bnad); 3236 3237 if (bnad->cfg_flags & BNAD_CF_DEFAULT) 3238 new_mode |= BNA_RXMODE_DEFAULT; 3239 } 3240 3241 mode_mask = BNA_RXMODE_PROMISC | BNA_RXMODE_DEFAULT | 3242 BNA_RXMODE_ALLMULTI; 3243 bna_rx_mode_set(bnad->rx_info[0].rx, new_mode, mode_mask); 3244 3245 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3246 } 3247 3248 /* 3249 * bna_lock is used to sync writes to netdev->addr 3250 * conf_lock cannot be used since this call may be made 3251 * in a non-blocking context. 3252 */ 3253 static int 3254 bnad_set_mac_address(struct net_device *netdev, void *addr) 3255 { 3256 int err; 3257 struct bnad *bnad = netdev_priv(netdev); 3258 struct sockaddr *sa = (struct sockaddr *)addr; 3259 unsigned long flags; 3260 3261 spin_lock_irqsave(&bnad->bna_lock, flags); 3262 3263 err = bnad_mac_addr_set_locked(bnad, sa->sa_data); 3264 if (!err) 3265 eth_hw_addr_set(netdev, sa->sa_data); 3266 3267 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3268 3269 return err; 3270 } 3271 3272 static int 3273 bnad_mtu_set(struct bnad *bnad, int frame_size) 3274 { 3275 unsigned long flags; 3276 3277 init_completion(&bnad->bnad_completions.mtu_comp); 3278 3279 spin_lock_irqsave(&bnad->bna_lock, flags); 3280 bna_enet_mtu_set(&bnad->bna.enet, frame_size, bnad_cb_enet_mtu_set); 3281 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3282 3283 wait_for_completion(&bnad->bnad_completions.mtu_comp); 3284 3285 return bnad->bnad_completions.mtu_comp_status; 3286 } 3287 3288 static int 3289 bnad_change_mtu(struct net_device *netdev, int new_mtu) 3290 { 3291 int err, mtu; 3292 struct bnad *bnad = netdev_priv(netdev); 3293 u32 frame, new_frame; 3294 3295 mutex_lock(&bnad->conf_mutex); 3296 3297 mtu = netdev->mtu; 3298 WRITE_ONCE(netdev->mtu, new_mtu); 3299 3300 frame = BNAD_FRAME_SIZE(mtu); 3301 new_frame = BNAD_FRAME_SIZE(new_mtu); 3302 3303 /* check if multi-buffer needs to be enabled */ 3304 if (BNAD_PCI_DEV_IS_CAT2(bnad) && 3305 netif_running(bnad->netdev)) { 3306 /* only when transition is over 4K */ 3307 if ((frame <= 4096 && new_frame > 4096) || 3308 (frame > 4096 && new_frame <= 4096)) 3309 bnad_reinit_rx(bnad); 3310 } 3311 3312 err = bnad_mtu_set(bnad, new_frame); 3313 if (err) 3314 err = -EBUSY; 3315 3316 mutex_unlock(&bnad->conf_mutex); 3317 return err; 3318 } 3319 3320 static int 3321 bnad_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 3322 { 3323 struct bnad *bnad = netdev_priv(netdev); 3324 unsigned long flags; 3325 3326 if (!bnad->rx_info[0].rx) 3327 return 0; 3328 3329 mutex_lock(&bnad->conf_mutex); 3330 3331 spin_lock_irqsave(&bnad->bna_lock, flags); 3332 bna_rx_vlan_add(bnad->rx_info[0].rx, vid); 3333 set_bit(vid, bnad->active_vlans); 3334 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3335 3336 mutex_unlock(&bnad->conf_mutex); 3337 3338 return 0; 3339 } 3340 3341 static int 3342 bnad_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 3343 { 3344 struct bnad *bnad = netdev_priv(netdev); 3345 unsigned long flags; 3346 3347 if (!bnad->rx_info[0].rx) 3348 return 0; 3349 3350 mutex_lock(&bnad->conf_mutex); 3351 3352 spin_lock_irqsave(&bnad->bna_lock, flags); 3353 clear_bit(vid, bnad->active_vlans); 3354 bna_rx_vlan_del(bnad->rx_info[0].rx, vid); 3355 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3356 3357 mutex_unlock(&bnad->conf_mutex); 3358 3359 return 0; 3360 } 3361 3362 static int bnad_set_features(struct net_device *dev, netdev_features_t features) 3363 { 3364 struct bnad *bnad = netdev_priv(dev); 3365 netdev_features_t changed = features ^ dev->features; 3366 3367 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(dev)) { 3368 unsigned long flags; 3369 3370 spin_lock_irqsave(&bnad->bna_lock, flags); 3371 3372 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3373 bna_rx_vlan_strip_enable(bnad->rx_info[0].rx); 3374 else 3375 bna_rx_vlan_strip_disable(bnad->rx_info[0].rx); 3376 3377 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3378 } 3379 3380 return 0; 3381 } 3382 3383 #ifdef CONFIG_NET_POLL_CONTROLLER 3384 static void 3385 bnad_netpoll(struct net_device *netdev) 3386 { 3387 struct bnad *bnad = netdev_priv(netdev); 3388 struct bnad_rx_info *rx_info; 3389 struct bnad_rx_ctrl *rx_ctrl; 3390 u32 curr_mask; 3391 int i, j; 3392 3393 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) { 3394 bna_intx_disable(&bnad->bna, curr_mask); 3395 bnad_isr(bnad->pcidev->irq, netdev); 3396 bna_intx_enable(&bnad->bna, curr_mask); 3397 } else { 3398 /* 3399 * Tx processing may happen in sending context, so no need 3400 * to explicitly process completions here 3401 */ 3402 3403 /* Rx processing */ 3404 for (i = 0; i < bnad->num_rx; i++) { 3405 rx_info = &bnad->rx_info[i]; 3406 if (!rx_info->rx) 3407 continue; 3408 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 3409 rx_ctrl = &rx_info->rx_ctrl[j]; 3410 if (rx_ctrl->ccb) 3411 bnad_netif_rx_schedule_poll(bnad, 3412 rx_ctrl->ccb); 3413 } 3414 } 3415 } 3416 } 3417 #endif 3418 3419 static const struct net_device_ops bnad_netdev_ops = { 3420 .ndo_open = bnad_open, 3421 .ndo_stop = bnad_stop, 3422 .ndo_start_xmit = bnad_start_xmit, 3423 .ndo_get_stats64 = bnad_get_stats64, 3424 .ndo_set_rx_mode = bnad_set_rx_mode, 3425 .ndo_validate_addr = eth_validate_addr, 3426 .ndo_set_mac_address = bnad_set_mac_address, 3427 .ndo_change_mtu = bnad_change_mtu, 3428 .ndo_vlan_rx_add_vid = bnad_vlan_rx_add_vid, 3429 .ndo_vlan_rx_kill_vid = bnad_vlan_rx_kill_vid, 3430 .ndo_set_features = bnad_set_features, 3431 #ifdef CONFIG_NET_POLL_CONTROLLER 3432 .ndo_poll_controller = bnad_netpoll 3433 #endif 3434 }; 3435 3436 static void 3437 bnad_netdev_init(struct bnad *bnad) 3438 { 3439 struct net_device *netdev = bnad->netdev; 3440 3441 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3442 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3443 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_TX | 3444 NETIF_F_HW_VLAN_CTAG_RX; 3445 3446 netdev->vlan_features = NETIF_F_SG | NETIF_F_HIGHDMA | 3447 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3448 NETIF_F_TSO | NETIF_F_TSO6; 3449 3450 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER | 3451 NETIF_F_HIGHDMA; 3452 3453 netdev->mem_start = bnad->mmio_start; 3454 netdev->mem_end = bnad->mmio_start + bnad->mmio_len - 1; 3455 3456 /* MTU range: 46 - 9000 */ 3457 netdev->min_mtu = ETH_ZLEN - ETH_HLEN; 3458 netdev->max_mtu = BNAD_JUMBO_MTU; 3459 3460 netdev->netdev_ops = &bnad_netdev_ops; 3461 bnad_set_ethtool_ops(netdev); 3462 } 3463 3464 /* 3465 * 1. Initialize the bnad structure 3466 * 2. Setup netdev pointer in pci_dev 3467 * 3. Initialize no. of TxQ & CQs & MSIX vectors 3468 * 4. Initialize work queue. 3469 */ 3470 static int 3471 bnad_init(struct bnad *bnad, 3472 struct pci_dev *pdev, struct net_device *netdev) 3473 { 3474 unsigned long flags; 3475 3476 SET_NETDEV_DEV(netdev, &pdev->dev); 3477 pci_set_drvdata(pdev, netdev); 3478 3479 bnad->netdev = netdev; 3480 bnad->pcidev = pdev; 3481 bnad->mmio_start = pci_resource_start(pdev, 0); 3482 bnad->mmio_len = pci_resource_len(pdev, 0); 3483 bnad->bar0 = ioremap(bnad->mmio_start, bnad->mmio_len); 3484 if (!bnad->bar0) { 3485 dev_err(&pdev->dev, "ioremap for bar0 failed\n"); 3486 return -ENOMEM; 3487 } 3488 dev_info(&pdev->dev, "bar0 mapped to %p, len %llu\n", bnad->bar0, 3489 (unsigned long long) bnad->mmio_len); 3490 3491 spin_lock_irqsave(&bnad->bna_lock, flags); 3492 if (!bnad_msix_disable) 3493 bnad->cfg_flags = BNAD_CF_MSIX; 3494 3495 bnad->cfg_flags |= BNAD_CF_DIM_ENABLED; 3496 3497 bnad_q_num_init(bnad); 3498 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3499 3500 bnad->msix_num = (bnad->num_tx * bnad->num_txq_per_tx) + 3501 (bnad->num_rx * bnad->num_rxp_per_rx) + 3502 BNAD_MAILBOX_MSIX_VECTORS; 3503 3504 bnad->txq_depth = BNAD_TXQ_DEPTH; 3505 bnad->rxq_depth = BNAD_RXQ_DEPTH; 3506 3507 bnad->tx_coalescing_timeo = BFI_TX_COALESCING_TIMEO; 3508 bnad->rx_coalescing_timeo = BFI_RX_COALESCING_TIMEO; 3509 3510 sprintf(bnad->wq_name, "%s_wq_%d", BNAD_NAME, bnad->id); 3511 bnad->work_q = create_singlethread_workqueue(bnad->wq_name); 3512 if (!bnad->work_q) { 3513 iounmap(bnad->bar0); 3514 return -ENOMEM; 3515 } 3516 3517 return 0; 3518 } 3519 3520 /* 3521 * Must be called after bnad_pci_uninit() 3522 * so that iounmap() and pci_set_drvdata(NULL) 3523 * happens only after PCI uninitialization. 3524 */ 3525 static void 3526 bnad_uninit(struct bnad *bnad) 3527 { 3528 if (bnad->work_q) { 3529 destroy_workqueue(bnad->work_q); 3530 bnad->work_q = NULL; 3531 } 3532 3533 if (bnad->bar0) 3534 iounmap(bnad->bar0); 3535 } 3536 3537 /* 3538 * Initialize locks 3539 a) Per ioceth mutes used for serializing configuration 3540 changes from OS interface 3541 b) spin lock used to protect bna state machine 3542 */ 3543 static void 3544 bnad_lock_init(struct bnad *bnad) 3545 { 3546 spin_lock_init(&bnad->bna_lock); 3547 mutex_init(&bnad->conf_mutex); 3548 } 3549 3550 static void 3551 bnad_lock_uninit(struct bnad *bnad) 3552 { 3553 mutex_destroy(&bnad->conf_mutex); 3554 } 3555 3556 /* PCI Initialization */ 3557 static int 3558 bnad_pci_init(struct bnad *bnad, struct pci_dev *pdev) 3559 { 3560 int err; 3561 3562 err = pci_enable_device(pdev); 3563 if (err) 3564 return err; 3565 err = pci_request_regions(pdev, BNAD_NAME); 3566 if (err) 3567 goto disable_device; 3568 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3569 if (err) 3570 goto release_regions; 3571 pci_set_master(pdev); 3572 return 0; 3573 3574 release_regions: 3575 pci_release_regions(pdev); 3576 disable_device: 3577 pci_disable_device(pdev); 3578 3579 return err; 3580 } 3581 3582 static void 3583 bnad_pci_uninit(struct pci_dev *pdev) 3584 { 3585 pci_release_regions(pdev); 3586 pci_disable_device(pdev); 3587 } 3588 3589 static int 3590 bnad_pci_probe(struct pci_dev *pdev, 3591 const struct pci_device_id *pcidev_id) 3592 { 3593 int err; 3594 struct bnad *bnad; 3595 struct bna *bna; 3596 struct net_device *netdev; 3597 struct bfa_pcidev pcidev_info; 3598 unsigned long flags; 3599 3600 mutex_lock(&bnad_fwimg_mutex); 3601 if (!cna_get_firmware_buf(pdev)) { 3602 mutex_unlock(&bnad_fwimg_mutex); 3603 dev_err(&pdev->dev, "failed to load firmware image!\n"); 3604 return -ENODEV; 3605 } 3606 mutex_unlock(&bnad_fwimg_mutex); 3607 3608 /* 3609 * Allocates sizeof(struct net_device + struct bnad) 3610 * bnad = netdev->priv 3611 */ 3612 netdev = alloc_etherdev(sizeof(struct bnad)); 3613 if (!netdev) { 3614 err = -ENOMEM; 3615 return err; 3616 } 3617 bnad = netdev_priv(netdev); 3618 bnad_lock_init(bnad); 3619 bnad->id = atomic_inc_return(&bna_id) - 1; 3620 3621 mutex_lock(&bnad->conf_mutex); 3622 /* PCI initialization */ 3623 err = bnad_pci_init(bnad, pdev); 3624 if (err) 3625 goto unlock_mutex; 3626 3627 /* 3628 * Initialize bnad structure 3629 * Setup relation between pci_dev & netdev 3630 */ 3631 err = bnad_init(bnad, pdev, netdev); 3632 if (err) 3633 goto pci_uninit; 3634 3635 /* Initialize netdev structure, set up ethtool ops */ 3636 bnad_netdev_init(bnad); 3637 3638 /* Set link to down state */ 3639 netif_carrier_off(netdev); 3640 3641 /* Setup the debugfs node for this bfad */ 3642 if (bna_debugfs_enable) 3643 bnad_debugfs_init(bnad); 3644 3645 /* Get resource requirement form bna */ 3646 spin_lock_irqsave(&bnad->bna_lock, flags); 3647 bna_res_req(&bnad->res_info[0]); 3648 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3649 3650 /* Allocate resources from bna */ 3651 err = bnad_res_alloc(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3652 if (err) 3653 goto drv_uninit; 3654 3655 bna = &bnad->bna; 3656 3657 /* Setup pcidev_info for bna_init() */ 3658 pcidev_info.pci_slot = PCI_SLOT(bnad->pcidev->devfn); 3659 pcidev_info.pci_func = PCI_FUNC(bnad->pcidev->devfn); 3660 pcidev_info.device_id = bnad->pcidev->device; 3661 pcidev_info.pci_bar_kva = bnad->bar0; 3662 3663 spin_lock_irqsave(&bnad->bna_lock, flags); 3664 bna_init(bna, bnad, &pcidev_info, &bnad->res_info[0]); 3665 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3666 3667 bnad->stats.bna_stats = &bna->stats; 3668 3669 bnad_enable_msix(bnad); 3670 err = bnad_mbox_irq_alloc(bnad); 3671 if (err) 3672 goto res_free; 3673 3674 /* Set up timers */ 3675 timer_setup(&bnad->bna.ioceth.ioc.ioc_timer, bnad_ioc_timeout, 0); 3676 timer_setup(&bnad->bna.ioceth.ioc.hb_timer, bnad_ioc_hb_check, 0); 3677 timer_setup(&bnad->bna.ioceth.ioc.iocpf_timer, bnad_iocpf_timeout, 0); 3678 timer_setup(&bnad->bna.ioceth.ioc.sem_timer, bnad_iocpf_sem_timeout, 3679 0); 3680 3681 /* 3682 * Start the chip 3683 * If the call back comes with error, we bail out. 3684 * This is a catastrophic error. 3685 */ 3686 err = bnad_ioceth_enable(bnad); 3687 if (err) { 3688 dev_err(&pdev->dev, "initialization failed err=%d\n", err); 3689 goto probe_success; 3690 } 3691 3692 spin_lock_irqsave(&bnad->bna_lock, flags); 3693 if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) || 3694 bna_num_rxp_set(bna, BNAD_NUM_RXP + 1)) { 3695 bnad_q_num_adjust(bnad, bna_attr(bna)->num_txq - 1, 3696 bna_attr(bna)->num_rxp - 1); 3697 if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) || 3698 bna_num_rxp_set(bna, BNAD_NUM_RXP + 1)) 3699 err = -EIO; 3700 } 3701 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3702 if (err) 3703 goto disable_ioceth; 3704 3705 spin_lock_irqsave(&bnad->bna_lock, flags); 3706 bna_mod_res_req(&bnad->bna, &bnad->mod_res_info[0]); 3707 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3708 3709 err = bnad_res_alloc(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3710 if (err) { 3711 err = -EIO; 3712 goto disable_ioceth; 3713 } 3714 3715 spin_lock_irqsave(&bnad->bna_lock, flags); 3716 bna_mod_init(&bnad->bna, &bnad->mod_res_info[0]); 3717 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3718 3719 /* Get the burnt-in mac */ 3720 spin_lock_irqsave(&bnad->bna_lock, flags); 3721 bna_enet_perm_mac_get(&bna->enet, bnad->perm_addr); 3722 bnad_set_netdev_perm_addr(bnad); 3723 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3724 3725 mutex_unlock(&bnad->conf_mutex); 3726 3727 /* Finally, reguister with net_device layer */ 3728 err = register_netdev(netdev); 3729 if (err) { 3730 dev_err(&pdev->dev, "registering net device failed\n"); 3731 goto probe_uninit; 3732 } 3733 set_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags); 3734 3735 return 0; 3736 3737 probe_success: 3738 mutex_unlock(&bnad->conf_mutex); 3739 return 0; 3740 3741 probe_uninit: 3742 mutex_lock(&bnad->conf_mutex); 3743 bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3744 disable_ioceth: 3745 bnad_ioceth_disable(bnad); 3746 bnad_ioc_timers_shutdown(bnad); 3747 spin_lock_irqsave(&bnad->bna_lock, flags); 3748 bna_uninit(bna); 3749 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3750 bnad_mbox_irq_free(bnad); 3751 bnad_disable_msix(bnad); 3752 res_free: 3753 bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3754 drv_uninit: 3755 /* Remove the debugfs node for this bnad */ 3756 kfree(bnad->regdata); 3757 bnad_debugfs_uninit(bnad); 3758 bnad_uninit(bnad); 3759 pci_uninit: 3760 bnad_pci_uninit(pdev); 3761 unlock_mutex: 3762 mutex_unlock(&bnad->conf_mutex); 3763 bnad_lock_uninit(bnad); 3764 free_netdev(netdev); 3765 return err; 3766 } 3767 3768 static void 3769 bnad_pci_remove(struct pci_dev *pdev) 3770 { 3771 struct net_device *netdev = pci_get_drvdata(pdev); 3772 struct bnad *bnad; 3773 struct bna *bna; 3774 unsigned long flags; 3775 3776 if (!netdev) 3777 return; 3778 3779 bnad = netdev_priv(netdev); 3780 bna = &bnad->bna; 3781 3782 if (test_and_clear_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags)) 3783 unregister_netdev(netdev); 3784 3785 mutex_lock(&bnad->conf_mutex); 3786 bnad_ioceth_disable(bnad); 3787 bnad_ioc_timers_shutdown(bnad); 3788 spin_lock_irqsave(&bnad->bna_lock, flags); 3789 bna_uninit(bna); 3790 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3791 3792 bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3793 bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3794 bnad_mbox_irq_free(bnad); 3795 bnad_disable_msix(bnad); 3796 bnad_pci_uninit(pdev); 3797 mutex_unlock(&bnad->conf_mutex); 3798 bnad_lock_uninit(bnad); 3799 /* Remove the debugfs node for this bnad */ 3800 kfree(bnad->regdata); 3801 bnad_debugfs_uninit(bnad); 3802 bnad_uninit(bnad); 3803 free_netdev(netdev); 3804 } 3805 3806 static const struct pci_device_id bnad_pci_id_table[] = { 3807 { 3808 PCI_DEVICE(PCI_VENDOR_ID_BROCADE, 3809 PCI_DEVICE_ID_BROCADE_CT), 3810 .class = PCI_CLASS_NETWORK_ETHERNET << 8, 3811 .class_mask = 0xffff00 3812 }, 3813 { 3814 PCI_DEVICE(PCI_VENDOR_ID_BROCADE, 3815 BFA_PCI_DEVICE_ID_CT2), 3816 .class = PCI_CLASS_NETWORK_ETHERNET << 8, 3817 .class_mask = 0xffff00 3818 }, 3819 {0, }, 3820 }; 3821 3822 MODULE_DEVICE_TABLE(pci, bnad_pci_id_table); 3823 3824 static struct pci_driver bnad_pci_driver = { 3825 .name = BNAD_NAME, 3826 .id_table = bnad_pci_id_table, 3827 .probe = bnad_pci_probe, 3828 .remove = bnad_pci_remove, 3829 }; 3830 3831 static int __init 3832 bnad_module_init(void) 3833 { 3834 int err; 3835 3836 bfa_nw_ioc_auto_recover(bnad_ioc_auto_recover); 3837 3838 err = pci_register_driver(&bnad_pci_driver); 3839 if (err < 0) { 3840 pr_err("bna: PCI driver registration failed err=%d\n", err); 3841 return err; 3842 } 3843 3844 return 0; 3845 } 3846 3847 static void __exit 3848 bnad_module_exit(void) 3849 { 3850 pci_unregister_driver(&bnad_pci_driver); 3851 release_firmware(bfi_fw); 3852 } 3853 3854 module_init(bnad_module_init); 3855 module_exit(bnad_module_exit); 3856 3857 MODULE_AUTHOR("Brocade"); 3858 MODULE_LICENSE("GPL"); 3859 MODULE_DESCRIPTION("QLogic BR-series 10G PCIe Ethernet driver"); 3860 MODULE_FIRMWARE(CNA_FW_FILE_CT); 3861 MODULE_FIRMWARE(CNA_FW_FILE_CT2); 3862