1 /* 2 * Linux network driver for QLogic BR-series Converged Network Adapter. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License (GPL) Version 2 as 6 * published by the Free Software Foundation 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 */ 13 /* 14 * Copyright (c) 2005-2014 Brocade Communications Systems, Inc. 15 * Copyright (c) 2014-2015 QLogic Corporation 16 * All rights reserved 17 * www.qlogic.com 18 */ 19 #include <linux/bitops.h> 20 #include <linux/netdevice.h> 21 #include <linux/skbuff.h> 22 #include <linux/etherdevice.h> 23 #include <linux/in.h> 24 #include <linux/ethtool.h> 25 #include <linux/if_vlan.h> 26 #include <linux/if_ether.h> 27 #include <linux/ip.h> 28 #include <linux/prefetch.h> 29 #include <linux/module.h> 30 31 #include "bnad.h" 32 #include "bna.h" 33 #include "cna.h" 34 35 static DEFINE_MUTEX(bnad_fwimg_mutex); 36 37 /* 38 * Module params 39 */ 40 static uint bnad_msix_disable; 41 module_param(bnad_msix_disable, uint, 0444); 42 MODULE_PARM_DESC(bnad_msix_disable, "Disable MSIX mode"); 43 44 static uint bnad_ioc_auto_recover = 1; 45 module_param(bnad_ioc_auto_recover, uint, 0444); 46 MODULE_PARM_DESC(bnad_ioc_auto_recover, "Enable / Disable auto recovery"); 47 48 static uint bna_debugfs_enable = 1; 49 module_param(bna_debugfs_enable, uint, S_IRUGO | S_IWUSR); 50 MODULE_PARM_DESC(bna_debugfs_enable, "Enables debugfs feature, default=1," 51 " Range[false:0|true:1]"); 52 53 /* 54 * Global variables 55 */ 56 static u32 bnad_rxqs_per_cq = 2; 57 static atomic_t bna_id; 58 static const u8 bnad_bcast_addr[] __aligned(2) = 59 { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; 60 61 /* 62 * Local MACROS 63 */ 64 #define BNAD_GET_MBOX_IRQ(_bnad) \ 65 (((_bnad)->cfg_flags & BNAD_CF_MSIX) ? \ 66 ((_bnad)->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector) : \ 67 ((_bnad)->pcidev->irq)) 68 69 #define BNAD_FILL_UNMAPQ_MEM_REQ(_res_info, _num, _size) \ 70 do { \ 71 (_res_info)->res_type = BNA_RES_T_MEM; \ 72 (_res_info)->res_u.mem_info.mem_type = BNA_MEM_T_KVA; \ 73 (_res_info)->res_u.mem_info.num = (_num); \ 74 (_res_info)->res_u.mem_info.len = (_size); \ 75 } while (0) 76 77 /* 78 * Reinitialize completions in CQ, once Rx is taken down 79 */ 80 static void 81 bnad_cq_cleanup(struct bnad *bnad, struct bna_ccb *ccb) 82 { 83 struct bna_cq_entry *cmpl; 84 int i; 85 86 for (i = 0; i < ccb->q_depth; i++) { 87 cmpl = &((struct bna_cq_entry *)ccb->sw_q)[i]; 88 cmpl->valid = 0; 89 } 90 } 91 92 /* Tx Datapath functions */ 93 94 95 /* Caller should ensure that the entry at unmap_q[index] is valid */ 96 static u32 97 bnad_tx_buff_unmap(struct bnad *bnad, 98 struct bnad_tx_unmap *unmap_q, 99 u32 q_depth, u32 index) 100 { 101 struct bnad_tx_unmap *unmap; 102 struct sk_buff *skb; 103 int vector, nvecs; 104 105 unmap = &unmap_q[index]; 106 nvecs = unmap->nvecs; 107 108 skb = unmap->skb; 109 unmap->skb = NULL; 110 unmap->nvecs = 0; 111 dma_unmap_single(&bnad->pcidev->dev, 112 dma_unmap_addr(&unmap->vectors[0], dma_addr), 113 skb_headlen(skb), DMA_TO_DEVICE); 114 dma_unmap_addr_set(&unmap->vectors[0], dma_addr, 0); 115 nvecs--; 116 117 vector = 0; 118 while (nvecs) { 119 vector++; 120 if (vector == BFI_TX_MAX_VECTORS_PER_WI) { 121 vector = 0; 122 BNA_QE_INDX_INC(index, q_depth); 123 unmap = &unmap_q[index]; 124 } 125 126 dma_unmap_page(&bnad->pcidev->dev, 127 dma_unmap_addr(&unmap->vectors[vector], dma_addr), 128 dma_unmap_len(&unmap->vectors[vector], dma_len), 129 DMA_TO_DEVICE); 130 dma_unmap_addr_set(&unmap->vectors[vector], dma_addr, 0); 131 nvecs--; 132 } 133 134 BNA_QE_INDX_INC(index, q_depth); 135 136 return index; 137 } 138 139 /* 140 * Frees all pending Tx Bufs 141 * At this point no activity is expected on the Q, 142 * so DMA unmap & freeing is fine. 143 */ 144 static void 145 bnad_txq_cleanup(struct bnad *bnad, struct bna_tcb *tcb) 146 { 147 struct bnad_tx_unmap *unmap_q = tcb->unmap_q; 148 struct sk_buff *skb; 149 int i; 150 151 for (i = 0; i < tcb->q_depth; i++) { 152 skb = unmap_q[i].skb; 153 if (!skb) 154 continue; 155 bnad_tx_buff_unmap(bnad, unmap_q, tcb->q_depth, i); 156 157 dev_kfree_skb_any(skb); 158 } 159 } 160 161 /* 162 * bnad_txcmpl_process : Frees the Tx bufs on Tx completion 163 * Can be called in a) Interrupt context 164 * b) Sending context 165 */ 166 static u32 167 bnad_txcmpl_process(struct bnad *bnad, struct bna_tcb *tcb) 168 { 169 u32 sent_packets = 0, sent_bytes = 0; 170 u32 wis, unmap_wis, hw_cons, cons, q_depth; 171 struct bnad_tx_unmap *unmap_q = tcb->unmap_q; 172 struct bnad_tx_unmap *unmap; 173 struct sk_buff *skb; 174 175 /* Just return if TX is stopped */ 176 if (!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) 177 return 0; 178 179 hw_cons = *(tcb->hw_consumer_index); 180 rmb(); 181 cons = tcb->consumer_index; 182 q_depth = tcb->q_depth; 183 184 wis = BNA_Q_INDEX_CHANGE(cons, hw_cons, q_depth); 185 BUG_ON(!(wis <= BNA_QE_IN_USE_CNT(tcb, tcb->q_depth))); 186 187 while (wis) { 188 unmap = &unmap_q[cons]; 189 190 skb = unmap->skb; 191 192 sent_packets++; 193 sent_bytes += skb->len; 194 195 unmap_wis = BNA_TXQ_WI_NEEDED(unmap->nvecs); 196 wis -= unmap_wis; 197 198 cons = bnad_tx_buff_unmap(bnad, unmap_q, q_depth, cons); 199 dev_kfree_skb_any(skb); 200 } 201 202 /* Update consumer pointers. */ 203 tcb->consumer_index = hw_cons; 204 205 tcb->txq->tx_packets += sent_packets; 206 tcb->txq->tx_bytes += sent_bytes; 207 208 return sent_packets; 209 } 210 211 static u32 212 bnad_tx_complete(struct bnad *bnad, struct bna_tcb *tcb) 213 { 214 struct net_device *netdev = bnad->netdev; 215 u32 sent = 0; 216 217 if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) 218 return 0; 219 220 sent = bnad_txcmpl_process(bnad, tcb); 221 if (sent) { 222 if (netif_queue_stopped(netdev) && 223 netif_carrier_ok(netdev) && 224 BNA_QE_FREE_CNT(tcb, tcb->q_depth) >= 225 BNAD_NETIF_WAKE_THRESHOLD) { 226 if (test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) { 227 netif_wake_queue(netdev); 228 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 229 } 230 } 231 } 232 233 if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 234 bna_ib_ack(tcb->i_dbell, sent); 235 236 smp_mb__before_atomic(); 237 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 238 239 return sent; 240 } 241 242 /* MSIX Tx Completion Handler */ 243 static irqreturn_t 244 bnad_msix_tx(int irq, void *data) 245 { 246 struct bna_tcb *tcb = (struct bna_tcb *)data; 247 struct bnad *bnad = tcb->bnad; 248 249 bnad_tx_complete(bnad, tcb); 250 251 return IRQ_HANDLED; 252 } 253 254 static inline void 255 bnad_rxq_alloc_uninit(struct bnad *bnad, struct bna_rcb *rcb) 256 { 257 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 258 259 unmap_q->reuse_pi = -1; 260 unmap_q->alloc_order = -1; 261 unmap_q->map_size = 0; 262 unmap_q->type = BNAD_RXBUF_NONE; 263 } 264 265 /* Default is page-based allocation. Multi-buffer support - TBD */ 266 static int 267 bnad_rxq_alloc_init(struct bnad *bnad, struct bna_rcb *rcb) 268 { 269 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 270 int order; 271 272 bnad_rxq_alloc_uninit(bnad, rcb); 273 274 order = get_order(rcb->rxq->buffer_size); 275 276 unmap_q->type = BNAD_RXBUF_PAGE; 277 278 if (bna_is_small_rxq(rcb->id)) { 279 unmap_q->alloc_order = 0; 280 unmap_q->map_size = rcb->rxq->buffer_size; 281 } else { 282 if (rcb->rxq->multi_buffer) { 283 unmap_q->alloc_order = 0; 284 unmap_q->map_size = rcb->rxq->buffer_size; 285 unmap_q->type = BNAD_RXBUF_MULTI_BUFF; 286 } else { 287 unmap_q->alloc_order = order; 288 unmap_q->map_size = 289 (rcb->rxq->buffer_size > 2048) ? 290 PAGE_SIZE << order : 2048; 291 } 292 } 293 294 BUG_ON((PAGE_SIZE << order) % unmap_q->map_size); 295 296 return 0; 297 } 298 299 static inline void 300 bnad_rxq_cleanup_page(struct bnad *bnad, struct bnad_rx_unmap *unmap) 301 { 302 if (!unmap->page) 303 return; 304 305 dma_unmap_page(&bnad->pcidev->dev, 306 dma_unmap_addr(&unmap->vector, dma_addr), 307 unmap->vector.len, DMA_FROM_DEVICE); 308 put_page(unmap->page); 309 unmap->page = NULL; 310 dma_unmap_addr_set(&unmap->vector, dma_addr, 0); 311 unmap->vector.len = 0; 312 } 313 314 static inline void 315 bnad_rxq_cleanup_skb(struct bnad *bnad, struct bnad_rx_unmap *unmap) 316 { 317 if (!unmap->skb) 318 return; 319 320 dma_unmap_single(&bnad->pcidev->dev, 321 dma_unmap_addr(&unmap->vector, dma_addr), 322 unmap->vector.len, DMA_FROM_DEVICE); 323 dev_kfree_skb_any(unmap->skb); 324 unmap->skb = NULL; 325 dma_unmap_addr_set(&unmap->vector, dma_addr, 0); 326 unmap->vector.len = 0; 327 } 328 329 static void 330 bnad_rxq_cleanup(struct bnad *bnad, struct bna_rcb *rcb) 331 { 332 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 333 int i; 334 335 for (i = 0; i < rcb->q_depth; i++) { 336 struct bnad_rx_unmap *unmap = &unmap_q->unmap[i]; 337 338 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 339 bnad_rxq_cleanup_skb(bnad, unmap); 340 else 341 bnad_rxq_cleanup_page(bnad, unmap); 342 } 343 bnad_rxq_alloc_uninit(bnad, rcb); 344 } 345 346 static u32 347 bnad_rxq_refill_page(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) 348 { 349 u32 alloced, prod, q_depth; 350 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 351 struct bnad_rx_unmap *unmap, *prev; 352 struct bna_rxq_entry *rxent; 353 struct page *page; 354 u32 page_offset, alloc_size; 355 dma_addr_t dma_addr; 356 357 prod = rcb->producer_index; 358 q_depth = rcb->q_depth; 359 360 alloc_size = PAGE_SIZE << unmap_q->alloc_order; 361 alloced = 0; 362 363 while (nalloc--) { 364 unmap = &unmap_q->unmap[prod]; 365 366 if (unmap_q->reuse_pi < 0) { 367 page = alloc_pages(GFP_ATOMIC | __GFP_COMP, 368 unmap_q->alloc_order); 369 page_offset = 0; 370 } else { 371 prev = &unmap_q->unmap[unmap_q->reuse_pi]; 372 page = prev->page; 373 page_offset = prev->page_offset + unmap_q->map_size; 374 get_page(page); 375 } 376 377 if (unlikely(!page)) { 378 BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); 379 rcb->rxq->rxbuf_alloc_failed++; 380 goto finishing; 381 } 382 383 dma_addr = dma_map_page(&bnad->pcidev->dev, page, page_offset, 384 unmap_q->map_size, DMA_FROM_DEVICE); 385 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 386 put_page(page); 387 BNAD_UPDATE_CTR(bnad, rxbuf_map_failed); 388 rcb->rxq->rxbuf_map_failed++; 389 goto finishing; 390 } 391 392 unmap->page = page; 393 unmap->page_offset = page_offset; 394 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); 395 unmap->vector.len = unmap_q->map_size; 396 page_offset += unmap_q->map_size; 397 398 if (page_offset < alloc_size) 399 unmap_q->reuse_pi = prod; 400 else 401 unmap_q->reuse_pi = -1; 402 403 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; 404 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); 405 BNA_QE_INDX_INC(prod, q_depth); 406 alloced++; 407 } 408 409 finishing: 410 if (likely(alloced)) { 411 rcb->producer_index = prod; 412 smp_mb(); 413 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) 414 bna_rxq_prod_indx_doorbell(rcb); 415 } 416 417 return alloced; 418 } 419 420 static u32 421 bnad_rxq_refill_skb(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) 422 { 423 u32 alloced, prod, q_depth, buff_sz; 424 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 425 struct bnad_rx_unmap *unmap; 426 struct bna_rxq_entry *rxent; 427 struct sk_buff *skb; 428 dma_addr_t dma_addr; 429 430 buff_sz = rcb->rxq->buffer_size; 431 prod = rcb->producer_index; 432 q_depth = rcb->q_depth; 433 434 alloced = 0; 435 while (nalloc--) { 436 unmap = &unmap_q->unmap[prod]; 437 438 skb = netdev_alloc_skb_ip_align(bnad->netdev, buff_sz); 439 440 if (unlikely(!skb)) { 441 BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); 442 rcb->rxq->rxbuf_alloc_failed++; 443 goto finishing; 444 } 445 446 dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, 447 buff_sz, DMA_FROM_DEVICE); 448 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 449 dev_kfree_skb_any(skb); 450 BNAD_UPDATE_CTR(bnad, rxbuf_map_failed); 451 rcb->rxq->rxbuf_map_failed++; 452 goto finishing; 453 } 454 455 unmap->skb = skb; 456 dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); 457 unmap->vector.len = buff_sz; 458 459 rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; 460 BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); 461 BNA_QE_INDX_INC(prod, q_depth); 462 alloced++; 463 } 464 465 finishing: 466 if (likely(alloced)) { 467 rcb->producer_index = prod; 468 smp_mb(); 469 if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) 470 bna_rxq_prod_indx_doorbell(rcb); 471 } 472 473 return alloced; 474 } 475 476 static inline void 477 bnad_rxq_post(struct bnad *bnad, struct bna_rcb *rcb) 478 { 479 struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; 480 u32 to_alloc; 481 482 to_alloc = BNA_QE_FREE_CNT(rcb, rcb->q_depth); 483 if (!(to_alloc >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT)) 484 return; 485 486 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 487 bnad_rxq_refill_skb(bnad, rcb, to_alloc); 488 else 489 bnad_rxq_refill_page(bnad, rcb, to_alloc); 490 } 491 492 #define flags_cksum_prot_mask (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 493 BNA_CQ_EF_IPV6 | \ 494 BNA_CQ_EF_TCP | BNA_CQ_EF_UDP | \ 495 BNA_CQ_EF_L4_CKSUM_OK) 496 497 #define flags_tcp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 498 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) 499 #define flags_tcp6 (BNA_CQ_EF_IPV6 | \ 500 BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) 501 #define flags_udp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ 502 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) 503 #define flags_udp6 (BNA_CQ_EF_IPV6 | \ 504 BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) 505 506 static void 507 bnad_cq_drop_packet(struct bnad *bnad, struct bna_rcb *rcb, 508 u32 sop_ci, u32 nvecs) 509 { 510 struct bnad_rx_unmap_q *unmap_q; 511 struct bnad_rx_unmap *unmap; 512 u32 ci, vec; 513 514 unmap_q = rcb->unmap_q; 515 for (vec = 0, ci = sop_ci; vec < nvecs; vec++) { 516 unmap = &unmap_q->unmap[ci]; 517 BNA_QE_INDX_INC(ci, rcb->q_depth); 518 519 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 520 bnad_rxq_cleanup_skb(bnad, unmap); 521 else 522 bnad_rxq_cleanup_page(bnad, unmap); 523 } 524 } 525 526 static void 527 bnad_cq_setup_skb_frags(struct bna_ccb *ccb, struct sk_buff *skb, u32 nvecs) 528 { 529 struct bna_rcb *rcb; 530 struct bnad *bnad; 531 struct bnad_rx_unmap_q *unmap_q; 532 struct bna_cq_entry *cq, *cmpl; 533 u32 ci, pi, totlen = 0; 534 535 cq = ccb->sw_q; 536 pi = ccb->producer_index; 537 cmpl = &cq[pi]; 538 539 rcb = bna_is_small_rxq(cmpl->rxq_id) ? ccb->rcb[1] : ccb->rcb[0]; 540 unmap_q = rcb->unmap_q; 541 bnad = rcb->bnad; 542 ci = rcb->consumer_index; 543 544 /* prefetch header */ 545 prefetch(page_address(unmap_q->unmap[ci].page) + 546 unmap_q->unmap[ci].page_offset); 547 548 while (nvecs--) { 549 struct bnad_rx_unmap *unmap; 550 u32 len; 551 552 unmap = &unmap_q->unmap[ci]; 553 BNA_QE_INDX_INC(ci, rcb->q_depth); 554 555 dma_unmap_page(&bnad->pcidev->dev, 556 dma_unmap_addr(&unmap->vector, dma_addr), 557 unmap->vector.len, DMA_FROM_DEVICE); 558 559 len = ntohs(cmpl->length); 560 skb->truesize += unmap->vector.len; 561 totlen += len; 562 563 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 564 unmap->page, unmap->page_offset, len); 565 566 unmap->page = NULL; 567 unmap->vector.len = 0; 568 569 BNA_QE_INDX_INC(pi, ccb->q_depth); 570 cmpl = &cq[pi]; 571 } 572 573 skb->len += totlen; 574 skb->data_len += totlen; 575 } 576 577 static inline void 578 bnad_cq_setup_skb(struct bnad *bnad, struct sk_buff *skb, 579 struct bnad_rx_unmap *unmap, u32 len) 580 { 581 prefetch(skb->data); 582 583 dma_unmap_single(&bnad->pcidev->dev, 584 dma_unmap_addr(&unmap->vector, dma_addr), 585 unmap->vector.len, DMA_FROM_DEVICE); 586 587 skb_put(skb, len); 588 skb->protocol = eth_type_trans(skb, bnad->netdev); 589 590 unmap->skb = NULL; 591 unmap->vector.len = 0; 592 } 593 594 static u32 595 bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget) 596 { 597 struct bna_cq_entry *cq, *cmpl, *next_cmpl; 598 struct bna_rcb *rcb = NULL; 599 struct bnad_rx_unmap_q *unmap_q; 600 struct bnad_rx_unmap *unmap = NULL; 601 struct sk_buff *skb = NULL; 602 struct bna_pkt_rate *pkt_rt = &ccb->pkt_rate; 603 struct bnad_rx_ctrl *rx_ctrl = ccb->ctrl; 604 u32 packets = 0, len = 0, totlen = 0; 605 u32 pi, vec, sop_ci = 0, nvecs = 0; 606 u32 flags, masked_flags; 607 608 prefetch(bnad->netdev); 609 610 cq = ccb->sw_q; 611 612 while (packets < budget) { 613 cmpl = &cq[ccb->producer_index]; 614 if (!cmpl->valid) 615 break; 616 /* The 'valid' field is set by the adapter, only after writing 617 * the other fields of completion entry. Hence, do not load 618 * other fields of completion entry *before* the 'valid' is 619 * loaded. Adding the rmb() here prevents the compiler and/or 620 * CPU from reordering the reads which would potentially result 621 * in reading stale values in completion entry. 622 */ 623 rmb(); 624 625 BNA_UPDATE_PKT_CNT(pkt_rt, ntohs(cmpl->length)); 626 627 if (bna_is_small_rxq(cmpl->rxq_id)) 628 rcb = ccb->rcb[1]; 629 else 630 rcb = ccb->rcb[0]; 631 632 unmap_q = rcb->unmap_q; 633 634 /* start of packet ci */ 635 sop_ci = rcb->consumer_index; 636 637 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) { 638 unmap = &unmap_q->unmap[sop_ci]; 639 skb = unmap->skb; 640 } else { 641 skb = napi_get_frags(&rx_ctrl->napi); 642 if (unlikely(!skb)) 643 break; 644 } 645 prefetch(skb); 646 647 flags = ntohl(cmpl->flags); 648 len = ntohs(cmpl->length); 649 totlen = len; 650 nvecs = 1; 651 652 /* Check all the completions for this frame. 653 * busy-wait doesn't help much, break here. 654 */ 655 if (BNAD_RXBUF_IS_MULTI_BUFF(unmap_q->type) && 656 (flags & BNA_CQ_EF_EOP) == 0) { 657 pi = ccb->producer_index; 658 do { 659 BNA_QE_INDX_INC(pi, ccb->q_depth); 660 next_cmpl = &cq[pi]; 661 662 if (!next_cmpl->valid) 663 break; 664 /* The 'valid' field is set by the adapter, only 665 * after writing the other fields of completion 666 * entry. Hence, do not load other fields of 667 * completion entry *before* the 'valid' is 668 * loaded. Adding the rmb() here prevents the 669 * compiler and/or CPU from reordering the reads 670 * which would potentially result in reading 671 * stale values in completion entry. 672 */ 673 rmb(); 674 675 len = ntohs(next_cmpl->length); 676 flags = ntohl(next_cmpl->flags); 677 678 nvecs++; 679 totlen += len; 680 } while ((flags & BNA_CQ_EF_EOP) == 0); 681 682 if (!next_cmpl->valid) 683 break; 684 } 685 packets++; 686 687 /* TODO: BNA_CQ_EF_LOCAL ? */ 688 if (unlikely(flags & (BNA_CQ_EF_MAC_ERROR | 689 BNA_CQ_EF_FCS_ERROR | 690 BNA_CQ_EF_TOO_LONG))) { 691 bnad_cq_drop_packet(bnad, rcb, sop_ci, nvecs); 692 rcb->rxq->rx_packets_with_error++; 693 694 goto next; 695 } 696 697 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 698 bnad_cq_setup_skb(bnad, skb, unmap, len); 699 else 700 bnad_cq_setup_skb_frags(ccb, skb, nvecs); 701 702 rcb->rxq->rx_packets++; 703 rcb->rxq->rx_bytes += totlen; 704 ccb->bytes_per_intr += totlen; 705 706 masked_flags = flags & flags_cksum_prot_mask; 707 708 if (likely 709 ((bnad->netdev->features & NETIF_F_RXCSUM) && 710 ((masked_flags == flags_tcp4) || 711 (masked_flags == flags_udp4) || 712 (masked_flags == flags_tcp6) || 713 (masked_flags == flags_udp6)))) 714 skb->ip_summed = CHECKSUM_UNNECESSARY; 715 else 716 skb_checksum_none_assert(skb); 717 718 if ((flags & BNA_CQ_EF_VLAN) && 719 (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 720 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cmpl->vlan_tag)); 721 722 if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) 723 netif_receive_skb(skb); 724 else 725 napi_gro_frags(&rx_ctrl->napi); 726 727 next: 728 BNA_QE_INDX_ADD(rcb->consumer_index, nvecs, rcb->q_depth); 729 for (vec = 0; vec < nvecs; vec++) { 730 cmpl = &cq[ccb->producer_index]; 731 cmpl->valid = 0; 732 BNA_QE_INDX_INC(ccb->producer_index, ccb->q_depth); 733 } 734 } 735 736 napi_gro_flush(&rx_ctrl->napi, false); 737 if (likely(test_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags))) 738 bna_ib_ack_disable_irq(ccb->i_dbell, packets); 739 740 bnad_rxq_post(bnad, ccb->rcb[0]); 741 if (ccb->rcb[1]) 742 bnad_rxq_post(bnad, ccb->rcb[1]); 743 744 return packets; 745 } 746 747 static void 748 bnad_netif_rx_schedule_poll(struct bnad *bnad, struct bna_ccb *ccb) 749 { 750 struct bnad_rx_ctrl *rx_ctrl = (struct bnad_rx_ctrl *)(ccb->ctrl); 751 struct napi_struct *napi = &rx_ctrl->napi; 752 753 if (likely(napi_schedule_prep(napi))) { 754 __napi_schedule(napi); 755 rx_ctrl->rx_schedule++; 756 } 757 } 758 759 /* MSIX Rx Path Handler */ 760 static irqreturn_t 761 bnad_msix_rx(int irq, void *data) 762 { 763 struct bna_ccb *ccb = (struct bna_ccb *)data; 764 765 if (ccb) { 766 ((struct bnad_rx_ctrl *)ccb->ctrl)->rx_intr_ctr++; 767 bnad_netif_rx_schedule_poll(ccb->bnad, ccb); 768 } 769 770 return IRQ_HANDLED; 771 } 772 773 /* Interrupt handlers */ 774 775 /* Mbox Interrupt Handlers */ 776 static irqreturn_t 777 bnad_msix_mbox_handler(int irq, void *data) 778 { 779 u32 intr_status; 780 unsigned long flags; 781 struct bnad *bnad = (struct bnad *)data; 782 783 spin_lock_irqsave(&bnad->bna_lock, flags); 784 if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) { 785 spin_unlock_irqrestore(&bnad->bna_lock, flags); 786 return IRQ_HANDLED; 787 } 788 789 bna_intr_status_get(&bnad->bna, intr_status); 790 791 if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status)) 792 bna_mbox_handler(&bnad->bna, intr_status); 793 794 spin_unlock_irqrestore(&bnad->bna_lock, flags); 795 796 return IRQ_HANDLED; 797 } 798 799 static irqreturn_t 800 bnad_isr(int irq, void *data) 801 { 802 int i, j; 803 u32 intr_status; 804 unsigned long flags; 805 struct bnad *bnad = (struct bnad *)data; 806 struct bnad_rx_info *rx_info; 807 struct bnad_rx_ctrl *rx_ctrl; 808 struct bna_tcb *tcb = NULL; 809 810 spin_lock_irqsave(&bnad->bna_lock, flags); 811 if (unlikely(test_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags))) { 812 spin_unlock_irqrestore(&bnad->bna_lock, flags); 813 return IRQ_NONE; 814 } 815 816 bna_intr_status_get(&bnad->bna, intr_status); 817 818 if (unlikely(!intr_status)) { 819 spin_unlock_irqrestore(&bnad->bna_lock, flags); 820 return IRQ_NONE; 821 } 822 823 if (BNA_IS_MBOX_ERR_INTR(&bnad->bna, intr_status)) 824 bna_mbox_handler(&bnad->bna, intr_status); 825 826 spin_unlock_irqrestore(&bnad->bna_lock, flags); 827 828 if (!BNA_IS_INTX_DATA_INTR(intr_status)) 829 return IRQ_HANDLED; 830 831 /* Process data interrupts */ 832 /* Tx processing */ 833 for (i = 0; i < bnad->num_tx; i++) { 834 for (j = 0; j < bnad->num_txq_per_tx; j++) { 835 tcb = bnad->tx_info[i].tcb[j]; 836 if (tcb && test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) 837 bnad_tx_complete(bnad, bnad->tx_info[i].tcb[j]); 838 } 839 } 840 /* Rx processing */ 841 for (i = 0; i < bnad->num_rx; i++) { 842 rx_info = &bnad->rx_info[i]; 843 if (!rx_info->rx) 844 continue; 845 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 846 rx_ctrl = &rx_info->rx_ctrl[j]; 847 if (rx_ctrl->ccb) 848 bnad_netif_rx_schedule_poll(bnad, 849 rx_ctrl->ccb); 850 } 851 } 852 return IRQ_HANDLED; 853 } 854 855 /* 856 * Called in interrupt / callback context 857 * with bna_lock held, so cfg_flags access is OK 858 */ 859 static void 860 bnad_enable_mbox_irq(struct bnad *bnad) 861 { 862 clear_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 863 864 BNAD_UPDATE_CTR(bnad, mbox_intr_enabled); 865 } 866 867 /* 868 * Called with bnad->bna_lock held b'cos of 869 * bnad->cfg_flags access. 870 */ 871 static void 872 bnad_disable_mbox_irq(struct bnad *bnad) 873 { 874 set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 875 876 BNAD_UPDATE_CTR(bnad, mbox_intr_disabled); 877 } 878 879 static void 880 bnad_set_netdev_perm_addr(struct bnad *bnad) 881 { 882 struct net_device *netdev = bnad->netdev; 883 884 ether_addr_copy(netdev->perm_addr, bnad->perm_addr); 885 if (is_zero_ether_addr(netdev->dev_addr)) 886 ether_addr_copy(netdev->dev_addr, bnad->perm_addr); 887 } 888 889 /* Control Path Handlers */ 890 891 /* Callbacks */ 892 void 893 bnad_cb_mbox_intr_enable(struct bnad *bnad) 894 { 895 bnad_enable_mbox_irq(bnad); 896 } 897 898 void 899 bnad_cb_mbox_intr_disable(struct bnad *bnad) 900 { 901 bnad_disable_mbox_irq(bnad); 902 } 903 904 void 905 bnad_cb_ioceth_ready(struct bnad *bnad) 906 { 907 bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS; 908 complete(&bnad->bnad_completions.ioc_comp); 909 } 910 911 void 912 bnad_cb_ioceth_failed(struct bnad *bnad) 913 { 914 bnad->bnad_completions.ioc_comp_status = BNA_CB_FAIL; 915 complete(&bnad->bnad_completions.ioc_comp); 916 } 917 918 void 919 bnad_cb_ioceth_disabled(struct bnad *bnad) 920 { 921 bnad->bnad_completions.ioc_comp_status = BNA_CB_SUCCESS; 922 complete(&bnad->bnad_completions.ioc_comp); 923 } 924 925 static void 926 bnad_cb_enet_disabled(void *arg) 927 { 928 struct bnad *bnad = (struct bnad *)arg; 929 930 netif_carrier_off(bnad->netdev); 931 complete(&bnad->bnad_completions.enet_comp); 932 } 933 934 void 935 bnad_cb_ethport_link_status(struct bnad *bnad, 936 enum bna_link_status link_status) 937 { 938 bool link_up = false; 939 940 link_up = (link_status == BNA_LINK_UP) || (link_status == BNA_CEE_UP); 941 942 if (link_status == BNA_CEE_UP) { 943 if (!test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) 944 BNAD_UPDATE_CTR(bnad, cee_toggle); 945 set_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags); 946 } else { 947 if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) 948 BNAD_UPDATE_CTR(bnad, cee_toggle); 949 clear_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags); 950 } 951 952 if (link_up) { 953 if (!netif_carrier_ok(bnad->netdev)) { 954 uint tx_id, tcb_id; 955 netdev_info(bnad->netdev, "link up\n"); 956 netif_carrier_on(bnad->netdev); 957 BNAD_UPDATE_CTR(bnad, link_toggle); 958 for (tx_id = 0; tx_id < bnad->num_tx; tx_id++) { 959 for (tcb_id = 0; tcb_id < bnad->num_txq_per_tx; 960 tcb_id++) { 961 struct bna_tcb *tcb = 962 bnad->tx_info[tx_id].tcb[tcb_id]; 963 u32 txq_id; 964 if (!tcb) 965 continue; 966 967 txq_id = tcb->id; 968 969 if (test_bit(BNAD_TXQ_TX_STARTED, 970 &tcb->flags)) { 971 /* 972 * Force an immediate 973 * Transmit Schedule */ 974 netif_wake_subqueue( 975 bnad->netdev, 976 txq_id); 977 BNAD_UPDATE_CTR(bnad, 978 netif_queue_wakeup); 979 } else { 980 netif_stop_subqueue( 981 bnad->netdev, 982 txq_id); 983 BNAD_UPDATE_CTR(bnad, 984 netif_queue_stop); 985 } 986 } 987 } 988 } 989 } else { 990 if (netif_carrier_ok(bnad->netdev)) { 991 netdev_info(bnad->netdev, "link down\n"); 992 netif_carrier_off(bnad->netdev); 993 BNAD_UPDATE_CTR(bnad, link_toggle); 994 } 995 } 996 } 997 998 static void 999 bnad_cb_tx_disabled(void *arg, struct bna_tx *tx) 1000 { 1001 struct bnad *bnad = (struct bnad *)arg; 1002 1003 complete(&bnad->bnad_completions.tx_comp); 1004 } 1005 1006 static void 1007 bnad_cb_tcb_setup(struct bnad *bnad, struct bna_tcb *tcb) 1008 { 1009 struct bnad_tx_info *tx_info = 1010 (struct bnad_tx_info *)tcb->txq->tx->priv; 1011 1012 tcb->priv = tcb; 1013 tx_info->tcb[tcb->id] = tcb; 1014 } 1015 1016 static void 1017 bnad_cb_tcb_destroy(struct bnad *bnad, struct bna_tcb *tcb) 1018 { 1019 struct bnad_tx_info *tx_info = 1020 (struct bnad_tx_info *)tcb->txq->tx->priv; 1021 1022 tx_info->tcb[tcb->id] = NULL; 1023 tcb->priv = NULL; 1024 } 1025 1026 static void 1027 bnad_cb_ccb_setup(struct bnad *bnad, struct bna_ccb *ccb) 1028 { 1029 struct bnad_rx_info *rx_info = 1030 (struct bnad_rx_info *)ccb->cq->rx->priv; 1031 1032 rx_info->rx_ctrl[ccb->id].ccb = ccb; 1033 ccb->ctrl = &rx_info->rx_ctrl[ccb->id]; 1034 } 1035 1036 static void 1037 bnad_cb_ccb_destroy(struct bnad *bnad, struct bna_ccb *ccb) 1038 { 1039 struct bnad_rx_info *rx_info = 1040 (struct bnad_rx_info *)ccb->cq->rx->priv; 1041 1042 rx_info->rx_ctrl[ccb->id].ccb = NULL; 1043 } 1044 1045 static void 1046 bnad_cb_tx_stall(struct bnad *bnad, struct bna_tx *tx) 1047 { 1048 struct bnad_tx_info *tx_info = 1049 (struct bnad_tx_info *)tx->priv; 1050 struct bna_tcb *tcb; 1051 u32 txq_id; 1052 int i; 1053 1054 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1055 tcb = tx_info->tcb[i]; 1056 if (!tcb) 1057 continue; 1058 txq_id = tcb->id; 1059 clear_bit(BNAD_TXQ_TX_STARTED, &tcb->flags); 1060 netif_stop_subqueue(bnad->netdev, txq_id); 1061 } 1062 } 1063 1064 static void 1065 bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) 1066 { 1067 struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv; 1068 struct bna_tcb *tcb; 1069 u32 txq_id; 1070 int i; 1071 1072 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1073 tcb = tx_info->tcb[i]; 1074 if (!tcb) 1075 continue; 1076 txq_id = tcb->id; 1077 1078 BUG_ON(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)); 1079 set_bit(BNAD_TXQ_TX_STARTED, &tcb->flags); 1080 BUG_ON(*(tcb->hw_consumer_index) != 0); 1081 1082 if (netif_carrier_ok(bnad->netdev)) { 1083 netif_wake_subqueue(bnad->netdev, txq_id); 1084 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 1085 } 1086 } 1087 1088 /* 1089 * Workaround for first ioceth enable failure & we 1090 * get a 0 MAC address. We try to get the MAC address 1091 * again here. 1092 */ 1093 if (is_zero_ether_addr(bnad->perm_addr)) { 1094 bna_enet_perm_mac_get(&bnad->bna.enet, bnad->perm_addr); 1095 bnad_set_netdev_perm_addr(bnad); 1096 } 1097 } 1098 1099 /* 1100 * Free all TxQs buffers and then notify TX_E_CLEANUP_DONE to Tx fsm. 1101 */ 1102 static void 1103 bnad_tx_cleanup(struct delayed_work *work) 1104 { 1105 struct bnad_tx_info *tx_info = 1106 container_of(work, struct bnad_tx_info, tx_cleanup_work); 1107 struct bnad *bnad = NULL; 1108 struct bna_tcb *tcb; 1109 unsigned long flags; 1110 u32 i, pending = 0; 1111 1112 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1113 tcb = tx_info->tcb[i]; 1114 if (!tcb) 1115 continue; 1116 1117 bnad = tcb->bnad; 1118 1119 if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { 1120 pending++; 1121 continue; 1122 } 1123 1124 bnad_txq_cleanup(bnad, tcb); 1125 1126 smp_mb__before_atomic(); 1127 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 1128 } 1129 1130 if (pending) { 1131 queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 1132 msecs_to_jiffies(1)); 1133 return; 1134 } 1135 1136 spin_lock_irqsave(&bnad->bna_lock, flags); 1137 bna_tx_cleanup_complete(tx_info->tx); 1138 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1139 } 1140 1141 static void 1142 bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) 1143 { 1144 struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv; 1145 struct bna_tcb *tcb; 1146 int i; 1147 1148 for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { 1149 tcb = tx_info->tcb[i]; 1150 if (!tcb) 1151 continue; 1152 } 1153 1154 queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 0); 1155 } 1156 1157 static void 1158 bnad_cb_rx_stall(struct bnad *bnad, struct bna_rx *rx) 1159 { 1160 struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; 1161 struct bna_ccb *ccb; 1162 struct bnad_rx_ctrl *rx_ctrl; 1163 int i; 1164 1165 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1166 rx_ctrl = &rx_info->rx_ctrl[i]; 1167 ccb = rx_ctrl->ccb; 1168 if (!ccb) 1169 continue; 1170 1171 clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[0]->flags); 1172 1173 if (ccb->rcb[1]) 1174 clear_bit(BNAD_RXQ_POST_OK, &ccb->rcb[1]->flags); 1175 } 1176 } 1177 1178 /* 1179 * Free all RxQs buffers and then notify RX_E_CLEANUP_DONE to Rx fsm. 1180 */ 1181 static void 1182 bnad_rx_cleanup(void *work) 1183 { 1184 struct bnad_rx_info *rx_info = 1185 container_of(work, struct bnad_rx_info, rx_cleanup_work); 1186 struct bnad_rx_ctrl *rx_ctrl; 1187 struct bnad *bnad = NULL; 1188 unsigned long flags; 1189 u32 i; 1190 1191 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1192 rx_ctrl = &rx_info->rx_ctrl[i]; 1193 1194 if (!rx_ctrl->ccb) 1195 continue; 1196 1197 bnad = rx_ctrl->ccb->bnad; 1198 1199 /* 1200 * Wait till the poll handler has exited 1201 * and nothing can be scheduled anymore 1202 */ 1203 napi_disable(&rx_ctrl->napi); 1204 1205 bnad_cq_cleanup(bnad, rx_ctrl->ccb); 1206 bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[0]); 1207 if (rx_ctrl->ccb->rcb[1]) 1208 bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[1]); 1209 } 1210 1211 spin_lock_irqsave(&bnad->bna_lock, flags); 1212 bna_rx_cleanup_complete(rx_info->rx); 1213 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1214 } 1215 1216 static void 1217 bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) 1218 { 1219 struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; 1220 struct bna_ccb *ccb; 1221 struct bnad_rx_ctrl *rx_ctrl; 1222 int i; 1223 1224 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1225 rx_ctrl = &rx_info->rx_ctrl[i]; 1226 ccb = rx_ctrl->ccb; 1227 if (!ccb) 1228 continue; 1229 1230 clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags); 1231 1232 if (ccb->rcb[1]) 1233 clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[1]->flags); 1234 } 1235 1236 queue_work(bnad->work_q, &rx_info->rx_cleanup_work); 1237 } 1238 1239 static void 1240 bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx) 1241 { 1242 struct bnad_rx_info *rx_info = (struct bnad_rx_info *)rx->priv; 1243 struct bna_ccb *ccb; 1244 struct bna_rcb *rcb; 1245 struct bnad_rx_ctrl *rx_ctrl; 1246 int i, j; 1247 1248 for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { 1249 rx_ctrl = &rx_info->rx_ctrl[i]; 1250 ccb = rx_ctrl->ccb; 1251 if (!ccb) 1252 continue; 1253 1254 napi_enable(&rx_ctrl->napi); 1255 1256 for (j = 0; j < BNAD_MAX_RXQ_PER_RXP; j++) { 1257 rcb = ccb->rcb[j]; 1258 if (!rcb) 1259 continue; 1260 1261 bnad_rxq_alloc_init(bnad, rcb); 1262 set_bit(BNAD_RXQ_STARTED, &rcb->flags); 1263 set_bit(BNAD_RXQ_POST_OK, &rcb->flags); 1264 bnad_rxq_post(bnad, rcb); 1265 } 1266 } 1267 } 1268 1269 static void 1270 bnad_cb_rx_disabled(void *arg, struct bna_rx *rx) 1271 { 1272 struct bnad *bnad = (struct bnad *)arg; 1273 1274 complete(&bnad->bnad_completions.rx_comp); 1275 } 1276 1277 static void 1278 bnad_cb_rx_mcast_add(struct bnad *bnad, struct bna_rx *rx) 1279 { 1280 bnad->bnad_completions.mcast_comp_status = BNA_CB_SUCCESS; 1281 complete(&bnad->bnad_completions.mcast_comp); 1282 } 1283 1284 void 1285 bnad_cb_stats_get(struct bnad *bnad, enum bna_cb_status status, 1286 struct bna_stats *stats) 1287 { 1288 if (status == BNA_CB_SUCCESS) 1289 BNAD_UPDATE_CTR(bnad, hw_stats_updates); 1290 1291 if (!netif_running(bnad->netdev) || 1292 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1293 return; 1294 1295 mod_timer(&bnad->stats_timer, 1296 jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ)); 1297 } 1298 1299 static void 1300 bnad_cb_enet_mtu_set(struct bnad *bnad) 1301 { 1302 bnad->bnad_completions.mtu_comp_status = BNA_CB_SUCCESS; 1303 complete(&bnad->bnad_completions.mtu_comp); 1304 } 1305 1306 void 1307 bnad_cb_completion(void *arg, enum bfa_status status) 1308 { 1309 struct bnad_iocmd_comp *iocmd_comp = 1310 (struct bnad_iocmd_comp *)arg; 1311 1312 iocmd_comp->comp_status = (u32) status; 1313 complete(&iocmd_comp->comp); 1314 } 1315 1316 /* Resource allocation, free functions */ 1317 1318 static void 1319 bnad_mem_free(struct bnad *bnad, 1320 struct bna_mem_info *mem_info) 1321 { 1322 int i; 1323 dma_addr_t dma_pa; 1324 1325 if (mem_info->mdl == NULL) 1326 return; 1327 1328 for (i = 0; i < mem_info->num; i++) { 1329 if (mem_info->mdl[i].kva != NULL) { 1330 if (mem_info->mem_type == BNA_MEM_T_DMA) { 1331 BNA_GET_DMA_ADDR(&(mem_info->mdl[i].dma), 1332 dma_pa); 1333 dma_free_coherent(&bnad->pcidev->dev, 1334 mem_info->mdl[i].len, 1335 mem_info->mdl[i].kva, dma_pa); 1336 } else 1337 kfree(mem_info->mdl[i].kva); 1338 } 1339 } 1340 kfree(mem_info->mdl); 1341 mem_info->mdl = NULL; 1342 } 1343 1344 static int 1345 bnad_mem_alloc(struct bnad *bnad, 1346 struct bna_mem_info *mem_info) 1347 { 1348 int i; 1349 dma_addr_t dma_pa; 1350 1351 if ((mem_info->num == 0) || (mem_info->len == 0)) { 1352 mem_info->mdl = NULL; 1353 return 0; 1354 } 1355 1356 mem_info->mdl = kcalloc(mem_info->num, sizeof(struct bna_mem_descr), 1357 GFP_KERNEL); 1358 if (mem_info->mdl == NULL) 1359 return -ENOMEM; 1360 1361 if (mem_info->mem_type == BNA_MEM_T_DMA) { 1362 for (i = 0; i < mem_info->num; i++) { 1363 mem_info->mdl[i].len = mem_info->len; 1364 mem_info->mdl[i].kva = 1365 dma_alloc_coherent(&bnad->pcidev->dev, 1366 mem_info->len, &dma_pa, 1367 GFP_KERNEL); 1368 if (mem_info->mdl[i].kva == NULL) 1369 goto err_return; 1370 1371 BNA_SET_DMA_ADDR(dma_pa, 1372 &(mem_info->mdl[i].dma)); 1373 } 1374 } else { 1375 for (i = 0; i < mem_info->num; i++) { 1376 mem_info->mdl[i].len = mem_info->len; 1377 mem_info->mdl[i].kva = kzalloc(mem_info->len, 1378 GFP_KERNEL); 1379 if (mem_info->mdl[i].kva == NULL) 1380 goto err_return; 1381 } 1382 } 1383 1384 return 0; 1385 1386 err_return: 1387 bnad_mem_free(bnad, mem_info); 1388 return -ENOMEM; 1389 } 1390 1391 /* Free IRQ for Mailbox */ 1392 static void 1393 bnad_mbox_irq_free(struct bnad *bnad) 1394 { 1395 int irq; 1396 unsigned long flags; 1397 1398 spin_lock_irqsave(&bnad->bna_lock, flags); 1399 bnad_disable_mbox_irq(bnad); 1400 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1401 1402 irq = BNAD_GET_MBOX_IRQ(bnad); 1403 free_irq(irq, bnad); 1404 } 1405 1406 /* 1407 * Allocates IRQ for Mailbox, but keep it disabled 1408 * This will be enabled once we get the mbox enable callback 1409 * from bna 1410 */ 1411 static int 1412 bnad_mbox_irq_alloc(struct bnad *bnad) 1413 { 1414 int err = 0; 1415 unsigned long irq_flags, flags; 1416 u32 irq; 1417 irq_handler_t irq_handler; 1418 1419 spin_lock_irqsave(&bnad->bna_lock, flags); 1420 if (bnad->cfg_flags & BNAD_CF_MSIX) { 1421 irq_handler = (irq_handler_t)bnad_msix_mbox_handler; 1422 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector; 1423 irq_flags = 0; 1424 } else { 1425 irq_handler = (irq_handler_t)bnad_isr; 1426 irq = bnad->pcidev->irq; 1427 irq_flags = IRQF_SHARED; 1428 } 1429 1430 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1431 sprintf(bnad->mbox_irq_name, "%s", BNAD_NAME); 1432 1433 /* 1434 * Set the Mbox IRQ disable flag, so that the IRQ handler 1435 * called from request_irq() for SHARED IRQs do not execute 1436 */ 1437 set_bit(BNAD_RF_MBOX_IRQ_DISABLED, &bnad->run_flags); 1438 1439 BNAD_UPDATE_CTR(bnad, mbox_intr_disabled); 1440 1441 err = request_irq(irq, irq_handler, irq_flags, 1442 bnad->mbox_irq_name, bnad); 1443 1444 return err; 1445 } 1446 1447 static void 1448 bnad_txrx_irq_free(struct bnad *bnad, struct bna_intr_info *intr_info) 1449 { 1450 kfree(intr_info->idl); 1451 intr_info->idl = NULL; 1452 } 1453 1454 /* Allocates Interrupt Descriptor List for MSIX/INT-X vectors */ 1455 static int 1456 bnad_txrx_irq_alloc(struct bnad *bnad, enum bnad_intr_source src, 1457 u32 txrx_id, struct bna_intr_info *intr_info) 1458 { 1459 int i, vector_start = 0; 1460 u32 cfg_flags; 1461 unsigned long flags; 1462 1463 spin_lock_irqsave(&bnad->bna_lock, flags); 1464 cfg_flags = bnad->cfg_flags; 1465 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1466 1467 if (cfg_flags & BNAD_CF_MSIX) { 1468 intr_info->intr_type = BNA_INTR_T_MSIX; 1469 intr_info->idl = kcalloc(intr_info->num, 1470 sizeof(struct bna_intr_descr), 1471 GFP_KERNEL); 1472 if (!intr_info->idl) 1473 return -ENOMEM; 1474 1475 switch (src) { 1476 case BNAD_INTR_TX: 1477 vector_start = BNAD_MAILBOX_MSIX_VECTORS + txrx_id; 1478 break; 1479 1480 case BNAD_INTR_RX: 1481 vector_start = BNAD_MAILBOX_MSIX_VECTORS + 1482 (bnad->num_tx * bnad->num_txq_per_tx) + 1483 txrx_id; 1484 break; 1485 1486 default: 1487 BUG(); 1488 } 1489 1490 for (i = 0; i < intr_info->num; i++) 1491 intr_info->idl[i].vector = vector_start + i; 1492 } else { 1493 intr_info->intr_type = BNA_INTR_T_INTX; 1494 intr_info->num = 1; 1495 intr_info->idl = kcalloc(intr_info->num, 1496 sizeof(struct bna_intr_descr), 1497 GFP_KERNEL); 1498 if (!intr_info->idl) 1499 return -ENOMEM; 1500 1501 switch (src) { 1502 case BNAD_INTR_TX: 1503 intr_info->idl[0].vector = BNAD_INTX_TX_IB_BITMASK; 1504 break; 1505 1506 case BNAD_INTR_RX: 1507 intr_info->idl[0].vector = BNAD_INTX_RX_IB_BITMASK; 1508 break; 1509 } 1510 } 1511 return 0; 1512 } 1513 1514 /* NOTE: Should be called for MSIX only 1515 * Unregisters Tx MSIX vector(s) from the kernel 1516 */ 1517 static void 1518 bnad_tx_msix_unregister(struct bnad *bnad, struct bnad_tx_info *tx_info, 1519 int num_txqs) 1520 { 1521 int i; 1522 int vector_num; 1523 1524 for (i = 0; i < num_txqs; i++) { 1525 if (tx_info->tcb[i] == NULL) 1526 continue; 1527 1528 vector_num = tx_info->tcb[i]->intr_vector; 1529 free_irq(bnad->msix_table[vector_num].vector, tx_info->tcb[i]); 1530 } 1531 } 1532 1533 /* NOTE: Should be called for MSIX only 1534 * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel 1535 */ 1536 static int 1537 bnad_tx_msix_register(struct bnad *bnad, struct bnad_tx_info *tx_info, 1538 u32 tx_id, int num_txqs) 1539 { 1540 int i; 1541 int err; 1542 int vector_num; 1543 1544 for (i = 0; i < num_txqs; i++) { 1545 vector_num = tx_info->tcb[i]->intr_vector; 1546 sprintf(tx_info->tcb[i]->name, "%s TXQ %d", bnad->netdev->name, 1547 tx_id + tx_info->tcb[i]->id); 1548 err = request_irq(bnad->msix_table[vector_num].vector, 1549 (irq_handler_t)bnad_msix_tx, 0, 1550 tx_info->tcb[i]->name, 1551 tx_info->tcb[i]); 1552 if (err) 1553 goto err_return; 1554 } 1555 1556 return 0; 1557 1558 err_return: 1559 if (i > 0) 1560 bnad_tx_msix_unregister(bnad, tx_info, (i - 1)); 1561 return -1; 1562 } 1563 1564 /* NOTE: Should be called for MSIX only 1565 * Unregisters Rx MSIX vector(s) from the kernel 1566 */ 1567 static void 1568 bnad_rx_msix_unregister(struct bnad *bnad, struct bnad_rx_info *rx_info, 1569 int num_rxps) 1570 { 1571 int i; 1572 int vector_num; 1573 1574 for (i = 0; i < num_rxps; i++) { 1575 if (rx_info->rx_ctrl[i].ccb == NULL) 1576 continue; 1577 1578 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector; 1579 free_irq(bnad->msix_table[vector_num].vector, 1580 rx_info->rx_ctrl[i].ccb); 1581 } 1582 } 1583 1584 /* NOTE: Should be called for MSIX only 1585 * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel 1586 */ 1587 static int 1588 bnad_rx_msix_register(struct bnad *bnad, struct bnad_rx_info *rx_info, 1589 u32 rx_id, int num_rxps) 1590 { 1591 int i; 1592 int err; 1593 int vector_num; 1594 1595 for (i = 0; i < num_rxps; i++) { 1596 vector_num = rx_info->rx_ctrl[i].ccb->intr_vector; 1597 sprintf(rx_info->rx_ctrl[i].ccb->name, "%s CQ %d", 1598 bnad->netdev->name, 1599 rx_id + rx_info->rx_ctrl[i].ccb->id); 1600 err = request_irq(bnad->msix_table[vector_num].vector, 1601 (irq_handler_t)bnad_msix_rx, 0, 1602 rx_info->rx_ctrl[i].ccb->name, 1603 rx_info->rx_ctrl[i].ccb); 1604 if (err) 1605 goto err_return; 1606 } 1607 1608 return 0; 1609 1610 err_return: 1611 if (i > 0) 1612 bnad_rx_msix_unregister(bnad, rx_info, (i - 1)); 1613 return -1; 1614 } 1615 1616 /* Free Tx object Resources */ 1617 static void 1618 bnad_tx_res_free(struct bnad *bnad, struct bna_res_info *res_info) 1619 { 1620 int i; 1621 1622 for (i = 0; i < BNA_TX_RES_T_MAX; i++) { 1623 if (res_info[i].res_type == BNA_RES_T_MEM) 1624 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 1625 else if (res_info[i].res_type == BNA_RES_T_INTR) 1626 bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info); 1627 } 1628 } 1629 1630 /* Allocates memory and interrupt resources for Tx object */ 1631 static int 1632 bnad_tx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 1633 u32 tx_id) 1634 { 1635 int i, err = 0; 1636 1637 for (i = 0; i < BNA_TX_RES_T_MAX; i++) { 1638 if (res_info[i].res_type == BNA_RES_T_MEM) 1639 err = bnad_mem_alloc(bnad, 1640 &res_info[i].res_u.mem_info); 1641 else if (res_info[i].res_type == BNA_RES_T_INTR) 1642 err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_TX, tx_id, 1643 &res_info[i].res_u.intr_info); 1644 if (err) 1645 goto err_return; 1646 } 1647 return 0; 1648 1649 err_return: 1650 bnad_tx_res_free(bnad, res_info); 1651 return err; 1652 } 1653 1654 /* Free Rx object Resources */ 1655 static void 1656 bnad_rx_res_free(struct bnad *bnad, struct bna_res_info *res_info) 1657 { 1658 int i; 1659 1660 for (i = 0; i < BNA_RX_RES_T_MAX; i++) { 1661 if (res_info[i].res_type == BNA_RES_T_MEM) 1662 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 1663 else if (res_info[i].res_type == BNA_RES_T_INTR) 1664 bnad_txrx_irq_free(bnad, &res_info[i].res_u.intr_info); 1665 } 1666 } 1667 1668 /* Allocates memory and interrupt resources for Rx object */ 1669 static int 1670 bnad_rx_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 1671 uint rx_id) 1672 { 1673 int i, err = 0; 1674 1675 /* All memory needs to be allocated before setup_ccbs */ 1676 for (i = 0; i < BNA_RX_RES_T_MAX; i++) { 1677 if (res_info[i].res_type == BNA_RES_T_MEM) 1678 err = bnad_mem_alloc(bnad, 1679 &res_info[i].res_u.mem_info); 1680 else if (res_info[i].res_type == BNA_RES_T_INTR) 1681 err = bnad_txrx_irq_alloc(bnad, BNAD_INTR_RX, rx_id, 1682 &res_info[i].res_u.intr_info); 1683 if (err) 1684 goto err_return; 1685 } 1686 return 0; 1687 1688 err_return: 1689 bnad_rx_res_free(bnad, res_info); 1690 return err; 1691 } 1692 1693 /* Timer callbacks */ 1694 /* a) IOC timer */ 1695 static void 1696 bnad_ioc_timeout(struct timer_list *t) 1697 { 1698 struct bnad *bnad = from_timer(bnad, t, bna.ioceth.ioc.ioc_timer); 1699 unsigned long flags; 1700 1701 spin_lock_irqsave(&bnad->bna_lock, flags); 1702 bfa_nw_ioc_timeout(&bnad->bna.ioceth.ioc); 1703 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1704 } 1705 1706 static void 1707 bnad_ioc_hb_check(struct timer_list *t) 1708 { 1709 struct bnad *bnad = from_timer(bnad, t, bna.ioceth.ioc.hb_timer); 1710 unsigned long flags; 1711 1712 spin_lock_irqsave(&bnad->bna_lock, flags); 1713 bfa_nw_ioc_hb_check(&bnad->bna.ioceth.ioc); 1714 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1715 } 1716 1717 static void 1718 bnad_iocpf_timeout(struct timer_list *t) 1719 { 1720 struct bnad *bnad = from_timer(bnad, t, bna.ioceth.ioc.iocpf_timer); 1721 unsigned long flags; 1722 1723 spin_lock_irqsave(&bnad->bna_lock, flags); 1724 bfa_nw_iocpf_timeout(&bnad->bna.ioceth.ioc); 1725 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1726 } 1727 1728 static void 1729 bnad_iocpf_sem_timeout(struct timer_list *t) 1730 { 1731 struct bnad *bnad = from_timer(bnad, t, bna.ioceth.ioc.sem_timer); 1732 unsigned long flags; 1733 1734 spin_lock_irqsave(&bnad->bna_lock, flags); 1735 bfa_nw_iocpf_sem_timeout(&bnad->bna.ioceth.ioc); 1736 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1737 } 1738 1739 /* 1740 * All timer routines use bnad->bna_lock to protect against 1741 * the following race, which may occur in case of no locking: 1742 * Time CPU m CPU n 1743 * 0 1 = test_bit 1744 * 1 clear_bit 1745 * 2 del_timer_sync 1746 * 3 mod_timer 1747 */ 1748 1749 /* b) Dynamic Interrupt Moderation Timer */ 1750 static void 1751 bnad_dim_timeout(struct timer_list *t) 1752 { 1753 struct bnad *bnad = from_timer(bnad, t, dim_timer); 1754 struct bnad_rx_info *rx_info; 1755 struct bnad_rx_ctrl *rx_ctrl; 1756 int i, j; 1757 unsigned long flags; 1758 1759 if (!netif_carrier_ok(bnad->netdev)) 1760 return; 1761 1762 spin_lock_irqsave(&bnad->bna_lock, flags); 1763 for (i = 0; i < bnad->num_rx; i++) { 1764 rx_info = &bnad->rx_info[i]; 1765 if (!rx_info->rx) 1766 continue; 1767 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 1768 rx_ctrl = &rx_info->rx_ctrl[j]; 1769 if (!rx_ctrl->ccb) 1770 continue; 1771 bna_rx_dim_update(rx_ctrl->ccb); 1772 } 1773 } 1774 1775 /* Check for BNAD_CF_DIM_ENABLED, does not eleminate a race */ 1776 if (test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) 1777 mod_timer(&bnad->dim_timer, 1778 jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ)); 1779 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1780 } 1781 1782 /* c) Statistics Timer */ 1783 static void 1784 bnad_stats_timeout(struct timer_list *t) 1785 { 1786 struct bnad *bnad = from_timer(bnad, t, stats_timer); 1787 unsigned long flags; 1788 1789 if (!netif_running(bnad->netdev) || 1790 !test_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1791 return; 1792 1793 spin_lock_irqsave(&bnad->bna_lock, flags); 1794 bna_hw_stats_get(&bnad->bna); 1795 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1796 } 1797 1798 /* 1799 * Set up timer for DIM 1800 * Called with bnad->bna_lock held 1801 */ 1802 void 1803 bnad_dim_timer_start(struct bnad *bnad) 1804 { 1805 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED && 1806 !test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) { 1807 timer_setup(&bnad->dim_timer, bnad_dim_timeout, 0); 1808 set_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags); 1809 mod_timer(&bnad->dim_timer, 1810 jiffies + msecs_to_jiffies(BNAD_DIM_TIMER_FREQ)); 1811 } 1812 } 1813 1814 /* 1815 * Set up timer for statistics 1816 * Called with mutex_lock(&bnad->conf_mutex) held 1817 */ 1818 static void 1819 bnad_stats_timer_start(struct bnad *bnad) 1820 { 1821 unsigned long flags; 1822 1823 spin_lock_irqsave(&bnad->bna_lock, flags); 1824 if (!test_and_set_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) { 1825 timer_setup(&bnad->stats_timer, bnad_stats_timeout, 0); 1826 mod_timer(&bnad->stats_timer, 1827 jiffies + msecs_to_jiffies(BNAD_STATS_TIMER_FREQ)); 1828 } 1829 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1830 } 1831 1832 /* 1833 * Stops the stats timer 1834 * Called with mutex_lock(&bnad->conf_mutex) held 1835 */ 1836 static void 1837 bnad_stats_timer_stop(struct bnad *bnad) 1838 { 1839 int to_del = 0; 1840 unsigned long flags; 1841 1842 spin_lock_irqsave(&bnad->bna_lock, flags); 1843 if (test_and_clear_bit(BNAD_RF_STATS_TIMER_RUNNING, &bnad->run_flags)) 1844 to_del = 1; 1845 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1846 if (to_del) 1847 del_timer_sync(&bnad->stats_timer); 1848 } 1849 1850 /* Utilities */ 1851 1852 static void 1853 bnad_netdev_mc_list_get(struct net_device *netdev, u8 *mc_list) 1854 { 1855 int i = 1; /* Index 0 has broadcast address */ 1856 struct netdev_hw_addr *mc_addr; 1857 1858 netdev_for_each_mc_addr(mc_addr, netdev) { 1859 ether_addr_copy(&mc_list[i * ETH_ALEN], &mc_addr->addr[0]); 1860 i++; 1861 } 1862 } 1863 1864 static int 1865 bnad_napi_poll_rx(struct napi_struct *napi, int budget) 1866 { 1867 struct bnad_rx_ctrl *rx_ctrl = 1868 container_of(napi, struct bnad_rx_ctrl, napi); 1869 struct bnad *bnad = rx_ctrl->bnad; 1870 int rcvd = 0; 1871 1872 rx_ctrl->rx_poll_ctr++; 1873 1874 if (!netif_carrier_ok(bnad->netdev)) 1875 goto poll_exit; 1876 1877 rcvd = bnad_cq_process(bnad, rx_ctrl->ccb, budget); 1878 if (rcvd >= budget) 1879 return rcvd; 1880 1881 poll_exit: 1882 napi_complete_done(napi, rcvd); 1883 1884 rx_ctrl->rx_complete++; 1885 1886 if (rx_ctrl->ccb) 1887 bnad_enable_rx_irq_unsafe(rx_ctrl->ccb); 1888 1889 return rcvd; 1890 } 1891 1892 #define BNAD_NAPI_POLL_QUOTA 64 1893 static void 1894 bnad_napi_add(struct bnad *bnad, u32 rx_id) 1895 { 1896 struct bnad_rx_ctrl *rx_ctrl; 1897 int i; 1898 1899 /* Initialize & enable NAPI */ 1900 for (i = 0; i < bnad->num_rxp_per_rx; i++) { 1901 rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i]; 1902 netif_napi_add(bnad->netdev, &rx_ctrl->napi, 1903 bnad_napi_poll_rx, BNAD_NAPI_POLL_QUOTA); 1904 } 1905 } 1906 1907 static void 1908 bnad_napi_delete(struct bnad *bnad, u32 rx_id) 1909 { 1910 int i; 1911 1912 /* First disable and then clean up */ 1913 for (i = 0; i < bnad->num_rxp_per_rx; i++) 1914 netif_napi_del(&bnad->rx_info[rx_id].rx_ctrl[i].napi); 1915 } 1916 1917 /* Should be held with conf_lock held */ 1918 void 1919 bnad_destroy_tx(struct bnad *bnad, u32 tx_id) 1920 { 1921 struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id]; 1922 struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0]; 1923 unsigned long flags; 1924 1925 if (!tx_info->tx) 1926 return; 1927 1928 init_completion(&bnad->bnad_completions.tx_comp); 1929 spin_lock_irqsave(&bnad->bna_lock, flags); 1930 bna_tx_disable(tx_info->tx, BNA_HARD_CLEANUP, bnad_cb_tx_disabled); 1931 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1932 wait_for_completion(&bnad->bnad_completions.tx_comp); 1933 1934 if (tx_info->tcb[0]->intr_type == BNA_INTR_T_MSIX) 1935 bnad_tx_msix_unregister(bnad, tx_info, 1936 bnad->num_txq_per_tx); 1937 1938 spin_lock_irqsave(&bnad->bna_lock, flags); 1939 bna_tx_destroy(tx_info->tx); 1940 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1941 1942 tx_info->tx = NULL; 1943 tx_info->tx_id = 0; 1944 1945 bnad_tx_res_free(bnad, res_info); 1946 } 1947 1948 /* Should be held with conf_lock held */ 1949 int 1950 bnad_setup_tx(struct bnad *bnad, u32 tx_id) 1951 { 1952 int err; 1953 struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id]; 1954 struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0]; 1955 struct bna_intr_info *intr_info = 1956 &res_info[BNA_TX_RES_INTR_T_TXCMPL].res_u.intr_info; 1957 struct bna_tx_config *tx_config = &bnad->tx_config[tx_id]; 1958 static const struct bna_tx_event_cbfn tx_cbfn = { 1959 .tcb_setup_cbfn = bnad_cb_tcb_setup, 1960 .tcb_destroy_cbfn = bnad_cb_tcb_destroy, 1961 .tx_stall_cbfn = bnad_cb_tx_stall, 1962 .tx_resume_cbfn = bnad_cb_tx_resume, 1963 .tx_cleanup_cbfn = bnad_cb_tx_cleanup, 1964 }; 1965 1966 struct bna_tx *tx; 1967 unsigned long flags; 1968 1969 tx_info->tx_id = tx_id; 1970 1971 /* Initialize the Tx object configuration */ 1972 tx_config->num_txq = bnad->num_txq_per_tx; 1973 tx_config->txq_depth = bnad->txq_depth; 1974 tx_config->tx_type = BNA_TX_T_REGULAR; 1975 tx_config->coalescing_timeo = bnad->tx_coalescing_timeo; 1976 1977 /* Get BNA's resource requirement for one tx object */ 1978 spin_lock_irqsave(&bnad->bna_lock, flags); 1979 bna_tx_res_req(bnad->num_txq_per_tx, 1980 bnad->txq_depth, res_info); 1981 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1982 1983 /* Fill Unmap Q memory requirements */ 1984 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_TX_RES_MEM_T_UNMAPQ], 1985 bnad->num_txq_per_tx, (sizeof(struct bnad_tx_unmap) * 1986 bnad->txq_depth)); 1987 1988 /* Allocate resources */ 1989 err = bnad_tx_res_alloc(bnad, res_info, tx_id); 1990 if (err) 1991 return err; 1992 1993 /* Ask BNA to create one Tx object, supplying required resources */ 1994 spin_lock_irqsave(&bnad->bna_lock, flags); 1995 tx = bna_tx_create(&bnad->bna, bnad, tx_config, &tx_cbfn, res_info, 1996 tx_info); 1997 spin_unlock_irqrestore(&bnad->bna_lock, flags); 1998 if (!tx) { 1999 err = -ENOMEM; 2000 goto err_return; 2001 } 2002 tx_info->tx = tx; 2003 2004 INIT_DELAYED_WORK(&tx_info->tx_cleanup_work, 2005 (work_func_t)bnad_tx_cleanup); 2006 2007 /* Register ISR for the Tx object */ 2008 if (intr_info->intr_type == BNA_INTR_T_MSIX) { 2009 err = bnad_tx_msix_register(bnad, tx_info, 2010 tx_id, bnad->num_txq_per_tx); 2011 if (err) 2012 goto cleanup_tx; 2013 } 2014 2015 spin_lock_irqsave(&bnad->bna_lock, flags); 2016 bna_tx_enable(tx); 2017 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2018 2019 return 0; 2020 2021 cleanup_tx: 2022 spin_lock_irqsave(&bnad->bna_lock, flags); 2023 bna_tx_destroy(tx_info->tx); 2024 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2025 tx_info->tx = NULL; 2026 tx_info->tx_id = 0; 2027 err_return: 2028 bnad_tx_res_free(bnad, res_info); 2029 return err; 2030 } 2031 2032 /* Setup the rx config for bna_rx_create */ 2033 /* bnad decides the configuration */ 2034 static void 2035 bnad_init_rx_config(struct bnad *bnad, struct bna_rx_config *rx_config) 2036 { 2037 memset(rx_config, 0, sizeof(*rx_config)); 2038 rx_config->rx_type = BNA_RX_T_REGULAR; 2039 rx_config->num_paths = bnad->num_rxp_per_rx; 2040 rx_config->coalescing_timeo = bnad->rx_coalescing_timeo; 2041 2042 if (bnad->num_rxp_per_rx > 1) { 2043 rx_config->rss_status = BNA_STATUS_T_ENABLED; 2044 rx_config->rss_config.hash_type = 2045 (BFI_ENET_RSS_IPV6 | 2046 BFI_ENET_RSS_IPV6_TCP | 2047 BFI_ENET_RSS_IPV4 | 2048 BFI_ENET_RSS_IPV4_TCP); 2049 rx_config->rss_config.hash_mask = 2050 bnad->num_rxp_per_rx - 1; 2051 netdev_rss_key_fill(rx_config->rss_config.toeplitz_hash_key, 2052 sizeof(rx_config->rss_config.toeplitz_hash_key)); 2053 } else { 2054 rx_config->rss_status = BNA_STATUS_T_DISABLED; 2055 memset(&rx_config->rss_config, 0, 2056 sizeof(rx_config->rss_config)); 2057 } 2058 2059 rx_config->frame_size = BNAD_FRAME_SIZE(bnad->netdev->mtu); 2060 rx_config->q0_multi_buf = BNA_STATUS_T_DISABLED; 2061 2062 /* BNA_RXP_SINGLE - one data-buffer queue 2063 * BNA_RXP_SLR - one small-buffer and one large-buffer queues 2064 * BNA_RXP_HDS - one header-buffer and one data-buffer queues 2065 */ 2066 /* TODO: configurable param for queue type */ 2067 rx_config->rxp_type = BNA_RXP_SLR; 2068 2069 if (BNAD_PCI_DEV_IS_CAT2(bnad) && 2070 rx_config->frame_size > 4096) { 2071 /* though size_routing_enable is set in SLR, 2072 * small packets may get routed to same rxq. 2073 * set buf_size to 2048 instead of PAGE_SIZE. 2074 */ 2075 rx_config->q0_buf_size = 2048; 2076 /* this should be in multiples of 2 */ 2077 rx_config->q0_num_vecs = 4; 2078 rx_config->q0_depth = bnad->rxq_depth * rx_config->q0_num_vecs; 2079 rx_config->q0_multi_buf = BNA_STATUS_T_ENABLED; 2080 } else { 2081 rx_config->q0_buf_size = rx_config->frame_size; 2082 rx_config->q0_num_vecs = 1; 2083 rx_config->q0_depth = bnad->rxq_depth; 2084 } 2085 2086 /* initialize for q1 for BNA_RXP_SLR/BNA_RXP_HDS */ 2087 if (rx_config->rxp_type == BNA_RXP_SLR) { 2088 rx_config->q1_depth = bnad->rxq_depth; 2089 rx_config->q1_buf_size = BFI_SMALL_RXBUF_SIZE; 2090 } 2091 2092 rx_config->vlan_strip_status = 2093 (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) ? 2094 BNA_STATUS_T_ENABLED : BNA_STATUS_T_DISABLED; 2095 } 2096 2097 static void 2098 bnad_rx_ctrl_init(struct bnad *bnad, u32 rx_id) 2099 { 2100 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2101 int i; 2102 2103 for (i = 0; i < bnad->num_rxp_per_rx; i++) 2104 rx_info->rx_ctrl[i].bnad = bnad; 2105 } 2106 2107 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2108 static u32 2109 bnad_reinit_rx(struct bnad *bnad) 2110 { 2111 struct net_device *netdev = bnad->netdev; 2112 u32 err = 0, current_err = 0; 2113 u32 rx_id = 0, count = 0; 2114 unsigned long flags; 2115 2116 /* destroy and create new rx objects */ 2117 for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { 2118 if (!bnad->rx_info[rx_id].rx) 2119 continue; 2120 bnad_destroy_rx(bnad, rx_id); 2121 } 2122 2123 spin_lock_irqsave(&bnad->bna_lock, flags); 2124 bna_enet_mtu_set(&bnad->bna.enet, 2125 BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); 2126 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2127 2128 for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { 2129 count++; 2130 current_err = bnad_setup_rx(bnad, rx_id); 2131 if (current_err && !err) { 2132 err = current_err; 2133 netdev_err(netdev, "RXQ:%u setup failed\n", rx_id); 2134 } 2135 } 2136 2137 /* restore rx configuration */ 2138 if (bnad->rx_info[0].rx && !err) { 2139 bnad_restore_vlans(bnad, 0); 2140 bnad_enable_default_bcast(bnad); 2141 spin_lock_irqsave(&bnad->bna_lock, flags); 2142 bnad_mac_addr_set_locked(bnad, netdev->dev_addr); 2143 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2144 bnad_set_rx_mode(netdev); 2145 } 2146 2147 return count; 2148 } 2149 2150 /* Called with bnad_conf_lock() held */ 2151 void 2152 bnad_destroy_rx(struct bnad *bnad, u32 rx_id) 2153 { 2154 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2155 struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; 2156 struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0]; 2157 unsigned long flags; 2158 int to_del = 0; 2159 2160 if (!rx_info->rx) 2161 return; 2162 2163 if (0 == rx_id) { 2164 spin_lock_irqsave(&bnad->bna_lock, flags); 2165 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED && 2166 test_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags)) { 2167 clear_bit(BNAD_RF_DIM_TIMER_RUNNING, &bnad->run_flags); 2168 to_del = 1; 2169 } 2170 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2171 if (to_del) 2172 del_timer_sync(&bnad->dim_timer); 2173 } 2174 2175 init_completion(&bnad->bnad_completions.rx_comp); 2176 spin_lock_irqsave(&bnad->bna_lock, flags); 2177 bna_rx_disable(rx_info->rx, BNA_HARD_CLEANUP, bnad_cb_rx_disabled); 2178 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2179 wait_for_completion(&bnad->bnad_completions.rx_comp); 2180 2181 if (rx_info->rx_ctrl[0].ccb->intr_type == BNA_INTR_T_MSIX) 2182 bnad_rx_msix_unregister(bnad, rx_info, rx_config->num_paths); 2183 2184 bnad_napi_delete(bnad, rx_id); 2185 2186 spin_lock_irqsave(&bnad->bna_lock, flags); 2187 bna_rx_destroy(rx_info->rx); 2188 2189 rx_info->rx = NULL; 2190 rx_info->rx_id = 0; 2191 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2192 2193 bnad_rx_res_free(bnad, res_info); 2194 } 2195 2196 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2197 int 2198 bnad_setup_rx(struct bnad *bnad, u32 rx_id) 2199 { 2200 int err; 2201 struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; 2202 struct bna_res_info *res_info = &bnad->rx_res_info[rx_id].res_info[0]; 2203 struct bna_intr_info *intr_info = 2204 &res_info[BNA_RX_RES_T_INTR].res_u.intr_info; 2205 struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; 2206 static const struct bna_rx_event_cbfn rx_cbfn = { 2207 .rcb_setup_cbfn = NULL, 2208 .rcb_destroy_cbfn = NULL, 2209 .ccb_setup_cbfn = bnad_cb_ccb_setup, 2210 .ccb_destroy_cbfn = bnad_cb_ccb_destroy, 2211 .rx_stall_cbfn = bnad_cb_rx_stall, 2212 .rx_cleanup_cbfn = bnad_cb_rx_cleanup, 2213 .rx_post_cbfn = bnad_cb_rx_post, 2214 }; 2215 struct bna_rx *rx; 2216 unsigned long flags; 2217 2218 rx_info->rx_id = rx_id; 2219 2220 /* Initialize the Rx object configuration */ 2221 bnad_init_rx_config(bnad, rx_config); 2222 2223 /* Get BNA's resource requirement for one Rx object */ 2224 spin_lock_irqsave(&bnad->bna_lock, flags); 2225 bna_rx_res_req(rx_config, res_info); 2226 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2227 2228 /* Fill Unmap Q memory requirements */ 2229 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPDQ], 2230 rx_config->num_paths, 2231 (rx_config->q0_depth * 2232 sizeof(struct bnad_rx_unmap)) + 2233 sizeof(struct bnad_rx_unmap_q)); 2234 2235 if (rx_config->rxp_type != BNA_RXP_SINGLE) { 2236 BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPHQ], 2237 rx_config->num_paths, 2238 (rx_config->q1_depth * 2239 sizeof(struct bnad_rx_unmap) + 2240 sizeof(struct bnad_rx_unmap_q))); 2241 } 2242 /* Allocate resource */ 2243 err = bnad_rx_res_alloc(bnad, res_info, rx_id); 2244 if (err) 2245 return err; 2246 2247 bnad_rx_ctrl_init(bnad, rx_id); 2248 2249 /* Ask BNA to create one Rx object, supplying required resources */ 2250 spin_lock_irqsave(&bnad->bna_lock, flags); 2251 rx = bna_rx_create(&bnad->bna, bnad, rx_config, &rx_cbfn, res_info, 2252 rx_info); 2253 if (!rx) { 2254 err = -ENOMEM; 2255 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2256 goto err_return; 2257 } 2258 rx_info->rx = rx; 2259 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2260 2261 INIT_WORK(&rx_info->rx_cleanup_work, 2262 (work_func_t)(bnad_rx_cleanup)); 2263 2264 /* 2265 * Init NAPI, so that state is set to NAPI_STATE_SCHED, 2266 * so that IRQ handler cannot schedule NAPI at this point. 2267 */ 2268 bnad_napi_add(bnad, rx_id); 2269 2270 /* Register ISR for the Rx object */ 2271 if (intr_info->intr_type == BNA_INTR_T_MSIX) { 2272 err = bnad_rx_msix_register(bnad, rx_info, rx_id, 2273 rx_config->num_paths); 2274 if (err) 2275 goto err_return; 2276 } 2277 2278 spin_lock_irqsave(&bnad->bna_lock, flags); 2279 if (0 == rx_id) { 2280 /* Set up Dynamic Interrupt Moderation Vector */ 2281 if (bnad->cfg_flags & BNAD_CF_DIM_ENABLED) 2282 bna_rx_dim_reconfig(&bnad->bna, bna_napi_dim_vector); 2283 2284 /* Enable VLAN filtering only on the default Rx */ 2285 bna_rx_vlanfilter_enable(rx); 2286 2287 /* Start the DIM timer */ 2288 bnad_dim_timer_start(bnad); 2289 } 2290 2291 bna_rx_enable(rx); 2292 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2293 2294 return 0; 2295 2296 err_return: 2297 bnad_destroy_rx(bnad, rx_id); 2298 return err; 2299 } 2300 2301 /* Called with conf_lock & bnad->bna_lock held */ 2302 void 2303 bnad_tx_coalescing_timeo_set(struct bnad *bnad) 2304 { 2305 struct bnad_tx_info *tx_info; 2306 2307 tx_info = &bnad->tx_info[0]; 2308 if (!tx_info->tx) 2309 return; 2310 2311 bna_tx_coalescing_timeo_set(tx_info->tx, bnad->tx_coalescing_timeo); 2312 } 2313 2314 /* Called with conf_lock & bnad->bna_lock held */ 2315 void 2316 bnad_rx_coalescing_timeo_set(struct bnad *bnad) 2317 { 2318 struct bnad_rx_info *rx_info; 2319 int i; 2320 2321 for (i = 0; i < bnad->num_rx; i++) { 2322 rx_info = &bnad->rx_info[i]; 2323 if (!rx_info->rx) 2324 continue; 2325 bna_rx_coalescing_timeo_set(rx_info->rx, 2326 bnad->rx_coalescing_timeo); 2327 } 2328 } 2329 2330 /* 2331 * Called with bnad->bna_lock held 2332 */ 2333 int 2334 bnad_mac_addr_set_locked(struct bnad *bnad, const u8 *mac_addr) 2335 { 2336 int ret; 2337 2338 if (!is_valid_ether_addr(mac_addr)) 2339 return -EADDRNOTAVAIL; 2340 2341 /* If datapath is down, pretend everything went through */ 2342 if (!bnad->rx_info[0].rx) 2343 return 0; 2344 2345 ret = bna_rx_ucast_set(bnad->rx_info[0].rx, mac_addr); 2346 if (ret != BNA_CB_SUCCESS) 2347 return -EADDRNOTAVAIL; 2348 2349 return 0; 2350 } 2351 2352 /* Should be called with conf_lock held */ 2353 int 2354 bnad_enable_default_bcast(struct bnad *bnad) 2355 { 2356 struct bnad_rx_info *rx_info = &bnad->rx_info[0]; 2357 int ret; 2358 unsigned long flags; 2359 2360 init_completion(&bnad->bnad_completions.mcast_comp); 2361 2362 spin_lock_irqsave(&bnad->bna_lock, flags); 2363 ret = bna_rx_mcast_add(rx_info->rx, bnad_bcast_addr, 2364 bnad_cb_rx_mcast_add); 2365 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2366 2367 if (ret == BNA_CB_SUCCESS) 2368 wait_for_completion(&bnad->bnad_completions.mcast_comp); 2369 else 2370 return -ENODEV; 2371 2372 if (bnad->bnad_completions.mcast_comp_status != BNA_CB_SUCCESS) 2373 return -ENODEV; 2374 2375 return 0; 2376 } 2377 2378 /* Called with mutex_lock(&bnad->conf_mutex) held */ 2379 void 2380 bnad_restore_vlans(struct bnad *bnad, u32 rx_id) 2381 { 2382 u16 vid; 2383 unsigned long flags; 2384 2385 for_each_set_bit(vid, bnad->active_vlans, VLAN_N_VID) { 2386 spin_lock_irqsave(&bnad->bna_lock, flags); 2387 bna_rx_vlan_add(bnad->rx_info[rx_id].rx, vid); 2388 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2389 } 2390 } 2391 2392 /* Statistics utilities */ 2393 void 2394 bnad_netdev_qstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats) 2395 { 2396 int i, j; 2397 2398 for (i = 0; i < bnad->num_rx; i++) { 2399 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 2400 if (bnad->rx_info[i].rx_ctrl[j].ccb) { 2401 stats->rx_packets += bnad->rx_info[i]. 2402 rx_ctrl[j].ccb->rcb[0]->rxq->rx_packets; 2403 stats->rx_bytes += bnad->rx_info[i]. 2404 rx_ctrl[j].ccb->rcb[0]->rxq->rx_bytes; 2405 if (bnad->rx_info[i].rx_ctrl[j].ccb->rcb[1] && 2406 bnad->rx_info[i].rx_ctrl[j].ccb-> 2407 rcb[1]->rxq) { 2408 stats->rx_packets += 2409 bnad->rx_info[i].rx_ctrl[j]. 2410 ccb->rcb[1]->rxq->rx_packets; 2411 stats->rx_bytes += 2412 bnad->rx_info[i].rx_ctrl[j]. 2413 ccb->rcb[1]->rxq->rx_bytes; 2414 } 2415 } 2416 } 2417 } 2418 for (i = 0; i < bnad->num_tx; i++) { 2419 for (j = 0; j < bnad->num_txq_per_tx; j++) { 2420 if (bnad->tx_info[i].tcb[j]) { 2421 stats->tx_packets += 2422 bnad->tx_info[i].tcb[j]->txq->tx_packets; 2423 stats->tx_bytes += 2424 bnad->tx_info[i].tcb[j]->txq->tx_bytes; 2425 } 2426 } 2427 } 2428 } 2429 2430 /* 2431 * Must be called with the bna_lock held. 2432 */ 2433 void 2434 bnad_netdev_hwstats_fill(struct bnad *bnad, struct rtnl_link_stats64 *stats) 2435 { 2436 struct bfi_enet_stats_mac *mac_stats; 2437 u32 bmap; 2438 int i; 2439 2440 mac_stats = &bnad->stats.bna_stats->hw_stats.mac_stats; 2441 stats->rx_errors = 2442 mac_stats->rx_fcs_error + mac_stats->rx_alignment_error + 2443 mac_stats->rx_frame_length_error + mac_stats->rx_code_error + 2444 mac_stats->rx_undersize; 2445 stats->tx_errors = mac_stats->tx_fcs_error + 2446 mac_stats->tx_undersize; 2447 stats->rx_dropped = mac_stats->rx_drop; 2448 stats->tx_dropped = mac_stats->tx_drop; 2449 stats->multicast = mac_stats->rx_multicast; 2450 stats->collisions = mac_stats->tx_total_collision; 2451 2452 stats->rx_length_errors = mac_stats->rx_frame_length_error; 2453 2454 /* receive ring buffer overflow ?? */ 2455 2456 stats->rx_crc_errors = mac_stats->rx_fcs_error; 2457 stats->rx_frame_errors = mac_stats->rx_alignment_error; 2458 /* recv'r fifo overrun */ 2459 bmap = bna_rx_rid_mask(&bnad->bna); 2460 for (i = 0; bmap; i++) { 2461 if (bmap & 1) { 2462 stats->rx_fifo_errors += 2463 bnad->stats.bna_stats-> 2464 hw_stats.rxf_stats[i].frame_drops; 2465 break; 2466 } 2467 bmap >>= 1; 2468 } 2469 } 2470 2471 static void 2472 bnad_mbox_irq_sync(struct bnad *bnad) 2473 { 2474 u32 irq; 2475 unsigned long flags; 2476 2477 spin_lock_irqsave(&bnad->bna_lock, flags); 2478 if (bnad->cfg_flags & BNAD_CF_MSIX) 2479 irq = bnad->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector; 2480 else 2481 irq = bnad->pcidev->irq; 2482 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2483 2484 synchronize_irq(irq); 2485 } 2486 2487 /* Utility used by bnad_start_xmit, for doing TSO */ 2488 static int 2489 bnad_tso_prepare(struct bnad *bnad, struct sk_buff *skb) 2490 { 2491 int err; 2492 2493 err = skb_cow_head(skb, 0); 2494 if (err < 0) { 2495 BNAD_UPDATE_CTR(bnad, tso_err); 2496 return err; 2497 } 2498 2499 /* 2500 * For TSO, the TCP checksum field is seeded with pseudo-header sum 2501 * excluding the length field. 2502 */ 2503 if (vlan_get_protocol(skb) == htons(ETH_P_IP)) { 2504 struct iphdr *iph = ip_hdr(skb); 2505 2506 /* Do we really need these? */ 2507 iph->tot_len = 0; 2508 iph->check = 0; 2509 2510 tcp_hdr(skb)->check = 2511 ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 2512 IPPROTO_TCP, 0); 2513 BNAD_UPDATE_CTR(bnad, tso4); 2514 } else { 2515 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 2516 2517 ipv6h->payload_len = 0; 2518 tcp_hdr(skb)->check = 2519 ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr, 0, 2520 IPPROTO_TCP, 0); 2521 BNAD_UPDATE_CTR(bnad, tso6); 2522 } 2523 2524 return 0; 2525 } 2526 2527 /* 2528 * Initialize Q numbers depending on Rx Paths 2529 * Called with bnad->bna_lock held, because of cfg_flags 2530 * access. 2531 */ 2532 static void 2533 bnad_q_num_init(struct bnad *bnad) 2534 { 2535 int rxps; 2536 2537 rxps = min((uint)num_online_cpus(), 2538 (uint)(BNAD_MAX_RX * BNAD_MAX_RXP_PER_RX)); 2539 2540 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) 2541 rxps = 1; /* INTx */ 2542 2543 bnad->num_rx = 1; 2544 bnad->num_tx = 1; 2545 bnad->num_rxp_per_rx = rxps; 2546 bnad->num_txq_per_tx = BNAD_TXQ_NUM; 2547 } 2548 2549 /* 2550 * Adjusts the Q numbers, given a number of msix vectors 2551 * Give preference to RSS as opposed to Tx priority Queues, 2552 * in such a case, just use 1 Tx Q 2553 * Called with bnad->bna_lock held b'cos of cfg_flags access 2554 */ 2555 static void 2556 bnad_q_num_adjust(struct bnad *bnad, int msix_vectors, int temp) 2557 { 2558 bnad->num_txq_per_tx = 1; 2559 if ((msix_vectors >= (bnad->num_tx * bnad->num_txq_per_tx) + 2560 bnad_rxqs_per_cq + BNAD_MAILBOX_MSIX_VECTORS) && 2561 (bnad->cfg_flags & BNAD_CF_MSIX)) { 2562 bnad->num_rxp_per_rx = msix_vectors - 2563 (bnad->num_tx * bnad->num_txq_per_tx) - 2564 BNAD_MAILBOX_MSIX_VECTORS; 2565 } else 2566 bnad->num_rxp_per_rx = 1; 2567 } 2568 2569 /* Enable / disable ioceth */ 2570 static int 2571 bnad_ioceth_disable(struct bnad *bnad) 2572 { 2573 unsigned long flags; 2574 int err = 0; 2575 2576 spin_lock_irqsave(&bnad->bna_lock, flags); 2577 init_completion(&bnad->bnad_completions.ioc_comp); 2578 bna_ioceth_disable(&bnad->bna.ioceth, BNA_HARD_CLEANUP); 2579 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2580 2581 wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp, 2582 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT)); 2583 2584 err = bnad->bnad_completions.ioc_comp_status; 2585 return err; 2586 } 2587 2588 static int 2589 bnad_ioceth_enable(struct bnad *bnad) 2590 { 2591 int err = 0; 2592 unsigned long flags; 2593 2594 spin_lock_irqsave(&bnad->bna_lock, flags); 2595 init_completion(&bnad->bnad_completions.ioc_comp); 2596 bnad->bnad_completions.ioc_comp_status = BNA_CB_WAITING; 2597 bna_ioceth_enable(&bnad->bna.ioceth); 2598 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2599 2600 wait_for_completion_timeout(&bnad->bnad_completions.ioc_comp, 2601 msecs_to_jiffies(BNAD_IOCETH_TIMEOUT)); 2602 2603 err = bnad->bnad_completions.ioc_comp_status; 2604 2605 return err; 2606 } 2607 2608 /* Free BNA resources */ 2609 static void 2610 bnad_res_free(struct bnad *bnad, struct bna_res_info *res_info, 2611 u32 res_val_max) 2612 { 2613 int i; 2614 2615 for (i = 0; i < res_val_max; i++) 2616 bnad_mem_free(bnad, &res_info[i].res_u.mem_info); 2617 } 2618 2619 /* Allocates memory and interrupt resources for BNA */ 2620 static int 2621 bnad_res_alloc(struct bnad *bnad, struct bna_res_info *res_info, 2622 u32 res_val_max) 2623 { 2624 int i, err; 2625 2626 for (i = 0; i < res_val_max; i++) { 2627 err = bnad_mem_alloc(bnad, &res_info[i].res_u.mem_info); 2628 if (err) 2629 goto err_return; 2630 } 2631 return 0; 2632 2633 err_return: 2634 bnad_res_free(bnad, res_info, res_val_max); 2635 return err; 2636 } 2637 2638 /* Interrupt enable / disable */ 2639 static void 2640 bnad_enable_msix(struct bnad *bnad) 2641 { 2642 int i, ret; 2643 unsigned long flags; 2644 2645 spin_lock_irqsave(&bnad->bna_lock, flags); 2646 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) { 2647 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2648 return; 2649 } 2650 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2651 2652 if (bnad->msix_table) 2653 return; 2654 2655 bnad->msix_table = 2656 kcalloc(bnad->msix_num, sizeof(struct msix_entry), GFP_KERNEL); 2657 2658 if (!bnad->msix_table) 2659 goto intx_mode; 2660 2661 for (i = 0; i < bnad->msix_num; i++) 2662 bnad->msix_table[i].entry = i; 2663 2664 ret = pci_enable_msix_range(bnad->pcidev, bnad->msix_table, 2665 1, bnad->msix_num); 2666 if (ret < 0) { 2667 goto intx_mode; 2668 } else if (ret < bnad->msix_num) { 2669 dev_warn(&bnad->pcidev->dev, 2670 "%d MSI-X vectors allocated < %d requested\n", 2671 ret, bnad->msix_num); 2672 2673 spin_lock_irqsave(&bnad->bna_lock, flags); 2674 /* ret = #of vectors that we got */ 2675 bnad_q_num_adjust(bnad, (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2, 2676 (ret - BNAD_MAILBOX_MSIX_VECTORS) / 2); 2677 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2678 2679 bnad->msix_num = BNAD_NUM_TXQ + BNAD_NUM_RXP + 2680 BNAD_MAILBOX_MSIX_VECTORS; 2681 2682 if (bnad->msix_num > ret) { 2683 pci_disable_msix(bnad->pcidev); 2684 goto intx_mode; 2685 } 2686 } 2687 2688 pci_intx(bnad->pcidev, 0); 2689 2690 return; 2691 2692 intx_mode: 2693 dev_warn(&bnad->pcidev->dev, 2694 "MSI-X enable failed - operating in INTx mode\n"); 2695 2696 kfree(bnad->msix_table); 2697 bnad->msix_table = NULL; 2698 bnad->msix_num = 0; 2699 spin_lock_irqsave(&bnad->bna_lock, flags); 2700 bnad->cfg_flags &= ~BNAD_CF_MSIX; 2701 bnad_q_num_init(bnad); 2702 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2703 } 2704 2705 static void 2706 bnad_disable_msix(struct bnad *bnad) 2707 { 2708 u32 cfg_flags; 2709 unsigned long flags; 2710 2711 spin_lock_irqsave(&bnad->bna_lock, flags); 2712 cfg_flags = bnad->cfg_flags; 2713 if (bnad->cfg_flags & BNAD_CF_MSIX) 2714 bnad->cfg_flags &= ~BNAD_CF_MSIX; 2715 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2716 2717 if (cfg_flags & BNAD_CF_MSIX) { 2718 pci_disable_msix(bnad->pcidev); 2719 kfree(bnad->msix_table); 2720 bnad->msix_table = NULL; 2721 } 2722 } 2723 2724 /* Netdev entry points */ 2725 static int 2726 bnad_open(struct net_device *netdev) 2727 { 2728 int err; 2729 struct bnad *bnad = netdev_priv(netdev); 2730 struct bna_pause_config pause_config; 2731 unsigned long flags; 2732 2733 mutex_lock(&bnad->conf_mutex); 2734 2735 /* Tx */ 2736 err = bnad_setup_tx(bnad, 0); 2737 if (err) 2738 goto err_return; 2739 2740 /* Rx */ 2741 err = bnad_setup_rx(bnad, 0); 2742 if (err) 2743 goto cleanup_tx; 2744 2745 /* Port */ 2746 pause_config.tx_pause = 0; 2747 pause_config.rx_pause = 0; 2748 2749 spin_lock_irqsave(&bnad->bna_lock, flags); 2750 bna_enet_mtu_set(&bnad->bna.enet, 2751 BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); 2752 bna_enet_pause_config(&bnad->bna.enet, &pause_config); 2753 bna_enet_enable(&bnad->bna.enet); 2754 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2755 2756 /* Enable broadcast */ 2757 bnad_enable_default_bcast(bnad); 2758 2759 /* Restore VLANs, if any */ 2760 bnad_restore_vlans(bnad, 0); 2761 2762 /* Set the UCAST address */ 2763 spin_lock_irqsave(&bnad->bna_lock, flags); 2764 bnad_mac_addr_set_locked(bnad, netdev->dev_addr); 2765 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2766 2767 /* Start the stats timer */ 2768 bnad_stats_timer_start(bnad); 2769 2770 mutex_unlock(&bnad->conf_mutex); 2771 2772 return 0; 2773 2774 cleanup_tx: 2775 bnad_destroy_tx(bnad, 0); 2776 2777 err_return: 2778 mutex_unlock(&bnad->conf_mutex); 2779 return err; 2780 } 2781 2782 static int 2783 bnad_stop(struct net_device *netdev) 2784 { 2785 struct bnad *bnad = netdev_priv(netdev); 2786 unsigned long flags; 2787 2788 mutex_lock(&bnad->conf_mutex); 2789 2790 /* Stop the stats timer */ 2791 bnad_stats_timer_stop(bnad); 2792 2793 init_completion(&bnad->bnad_completions.enet_comp); 2794 2795 spin_lock_irqsave(&bnad->bna_lock, flags); 2796 bna_enet_disable(&bnad->bna.enet, BNA_HARD_CLEANUP, 2797 bnad_cb_enet_disabled); 2798 spin_unlock_irqrestore(&bnad->bna_lock, flags); 2799 2800 wait_for_completion(&bnad->bnad_completions.enet_comp); 2801 2802 bnad_destroy_tx(bnad, 0); 2803 bnad_destroy_rx(bnad, 0); 2804 2805 /* Synchronize mailbox IRQ */ 2806 bnad_mbox_irq_sync(bnad); 2807 2808 mutex_unlock(&bnad->conf_mutex); 2809 2810 return 0; 2811 } 2812 2813 /* TX */ 2814 /* Returns 0 for success */ 2815 static int 2816 bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb, 2817 struct sk_buff *skb, struct bna_txq_entry *txqent) 2818 { 2819 u16 flags = 0; 2820 u32 gso_size; 2821 u16 vlan_tag = 0; 2822 2823 if (skb_vlan_tag_present(skb)) { 2824 vlan_tag = (u16)skb_vlan_tag_get(skb); 2825 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); 2826 } 2827 if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) { 2828 vlan_tag = ((tcb->priority & 0x7) << VLAN_PRIO_SHIFT) 2829 | (vlan_tag & 0x1fff); 2830 flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); 2831 } 2832 txqent->hdr.wi.vlan_tag = htons(vlan_tag); 2833 2834 if (skb_is_gso(skb)) { 2835 gso_size = skb_shinfo(skb)->gso_size; 2836 if (unlikely(gso_size > bnad->netdev->mtu)) { 2837 BNAD_UPDATE_CTR(bnad, tx_skb_mss_too_long); 2838 return -EINVAL; 2839 } 2840 if (unlikely((gso_size + skb_transport_offset(skb) + 2841 tcp_hdrlen(skb)) >= skb->len)) { 2842 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); 2843 txqent->hdr.wi.lso_mss = 0; 2844 BNAD_UPDATE_CTR(bnad, tx_skb_tso_too_short); 2845 } else { 2846 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND_LSO); 2847 txqent->hdr.wi.lso_mss = htons(gso_size); 2848 } 2849 2850 if (bnad_tso_prepare(bnad, skb)) { 2851 BNAD_UPDATE_CTR(bnad, tx_skb_tso_prepare); 2852 return -EINVAL; 2853 } 2854 2855 flags |= (BNA_TXQ_WI_CF_IP_CKSUM | BNA_TXQ_WI_CF_TCP_CKSUM); 2856 txqent->hdr.wi.l4_hdr_size_n_offset = 2857 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET( 2858 tcp_hdrlen(skb) >> 2, skb_transport_offset(skb))); 2859 } else { 2860 txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); 2861 txqent->hdr.wi.lso_mss = 0; 2862 2863 if (unlikely(skb->len > (bnad->netdev->mtu + VLAN_ETH_HLEN))) { 2864 BNAD_UPDATE_CTR(bnad, tx_skb_non_tso_too_long); 2865 return -EINVAL; 2866 } 2867 2868 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2869 __be16 net_proto = vlan_get_protocol(skb); 2870 u8 proto = 0; 2871 2872 if (net_proto == htons(ETH_P_IP)) 2873 proto = ip_hdr(skb)->protocol; 2874 #ifdef NETIF_F_IPV6_CSUM 2875 else if (net_proto == htons(ETH_P_IPV6)) { 2876 /* nexthdr may not be TCP immediately. */ 2877 proto = ipv6_hdr(skb)->nexthdr; 2878 } 2879 #endif 2880 if (proto == IPPROTO_TCP) { 2881 flags |= BNA_TXQ_WI_CF_TCP_CKSUM; 2882 txqent->hdr.wi.l4_hdr_size_n_offset = 2883 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET 2884 (0, skb_transport_offset(skb))); 2885 2886 BNAD_UPDATE_CTR(bnad, tcpcsum_offload); 2887 2888 if (unlikely(skb_headlen(skb) < 2889 skb_transport_offset(skb) + 2890 tcp_hdrlen(skb))) { 2891 BNAD_UPDATE_CTR(bnad, tx_skb_tcp_hdr); 2892 return -EINVAL; 2893 } 2894 } else if (proto == IPPROTO_UDP) { 2895 flags |= BNA_TXQ_WI_CF_UDP_CKSUM; 2896 txqent->hdr.wi.l4_hdr_size_n_offset = 2897 htons(BNA_TXQ_WI_L4_HDR_N_OFFSET 2898 (0, skb_transport_offset(skb))); 2899 2900 BNAD_UPDATE_CTR(bnad, udpcsum_offload); 2901 if (unlikely(skb_headlen(skb) < 2902 skb_transport_offset(skb) + 2903 sizeof(struct udphdr))) { 2904 BNAD_UPDATE_CTR(bnad, tx_skb_udp_hdr); 2905 return -EINVAL; 2906 } 2907 } else { 2908 2909 BNAD_UPDATE_CTR(bnad, tx_skb_csum_err); 2910 return -EINVAL; 2911 } 2912 } else 2913 txqent->hdr.wi.l4_hdr_size_n_offset = 0; 2914 } 2915 2916 txqent->hdr.wi.flags = htons(flags); 2917 txqent->hdr.wi.frame_length = htonl(skb->len); 2918 2919 return 0; 2920 } 2921 2922 /* 2923 * bnad_start_xmit : Netdev entry point for Transmit 2924 * Called under lock held by net_device 2925 */ 2926 static netdev_tx_t 2927 bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) 2928 { 2929 struct bnad *bnad = netdev_priv(netdev); 2930 u32 txq_id = 0; 2931 struct bna_tcb *tcb = NULL; 2932 struct bnad_tx_unmap *unmap_q, *unmap, *head_unmap; 2933 u32 prod, q_depth, vect_id; 2934 u32 wis, vectors, len; 2935 int i; 2936 dma_addr_t dma_addr; 2937 struct bna_txq_entry *txqent; 2938 2939 len = skb_headlen(skb); 2940 2941 /* Sanity checks for the skb */ 2942 2943 if (unlikely(skb->len <= ETH_HLEN)) { 2944 dev_kfree_skb_any(skb); 2945 BNAD_UPDATE_CTR(bnad, tx_skb_too_short); 2946 return NETDEV_TX_OK; 2947 } 2948 if (unlikely(len > BFI_TX_MAX_DATA_PER_VECTOR)) { 2949 dev_kfree_skb_any(skb); 2950 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); 2951 return NETDEV_TX_OK; 2952 } 2953 if (unlikely(len == 0)) { 2954 dev_kfree_skb_any(skb); 2955 BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); 2956 return NETDEV_TX_OK; 2957 } 2958 2959 tcb = bnad->tx_info[0].tcb[txq_id]; 2960 2961 /* 2962 * Takes care of the Tx that is scheduled between clearing the flag 2963 * and the netif_tx_stop_all_queues() call. 2964 */ 2965 if (unlikely(!tcb || !test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) { 2966 dev_kfree_skb_any(skb); 2967 BNAD_UPDATE_CTR(bnad, tx_skb_stopping); 2968 return NETDEV_TX_OK; 2969 } 2970 2971 q_depth = tcb->q_depth; 2972 prod = tcb->producer_index; 2973 unmap_q = tcb->unmap_q; 2974 2975 vectors = 1 + skb_shinfo(skb)->nr_frags; 2976 wis = BNA_TXQ_WI_NEEDED(vectors); /* 4 vectors per work item */ 2977 2978 if (unlikely(vectors > BFI_TX_MAX_VECTORS_PER_PKT)) { 2979 dev_kfree_skb_any(skb); 2980 BNAD_UPDATE_CTR(bnad, tx_skb_max_vectors); 2981 return NETDEV_TX_OK; 2982 } 2983 2984 /* Check for available TxQ resources */ 2985 if (unlikely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { 2986 if ((*tcb->hw_consumer_index != tcb->consumer_index) && 2987 !test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { 2988 u32 sent; 2989 sent = bnad_txcmpl_process(bnad, tcb); 2990 if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 2991 bna_ib_ack(tcb->i_dbell, sent); 2992 smp_mb__before_atomic(); 2993 clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); 2994 } else { 2995 netif_stop_queue(netdev); 2996 BNAD_UPDATE_CTR(bnad, netif_queue_stop); 2997 } 2998 2999 smp_mb(); 3000 /* 3001 * Check again to deal with race condition between 3002 * netif_stop_queue here, and netif_wake_queue in 3003 * interrupt handler which is not inside netif tx lock. 3004 */ 3005 if (likely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { 3006 BNAD_UPDATE_CTR(bnad, netif_queue_stop); 3007 return NETDEV_TX_BUSY; 3008 } else { 3009 netif_wake_queue(netdev); 3010 BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); 3011 } 3012 } 3013 3014 txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; 3015 head_unmap = &unmap_q[prod]; 3016 3017 /* Program the opcode, flags, frame_len, num_vectors in WI */ 3018 if (bnad_txq_wi_prepare(bnad, tcb, skb, txqent)) { 3019 dev_kfree_skb_any(skb); 3020 return NETDEV_TX_OK; 3021 } 3022 txqent->hdr.wi.reserved = 0; 3023 txqent->hdr.wi.num_vectors = vectors; 3024 3025 head_unmap->skb = skb; 3026 head_unmap->nvecs = 0; 3027 3028 /* Program the vectors */ 3029 unmap = head_unmap; 3030 dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, 3031 len, DMA_TO_DEVICE); 3032 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 3033 dev_kfree_skb_any(skb); 3034 BNAD_UPDATE_CTR(bnad, tx_skb_map_failed); 3035 return NETDEV_TX_OK; 3036 } 3037 BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr); 3038 txqent->vector[0].length = htons(len); 3039 dma_unmap_addr_set(&unmap->vectors[0], dma_addr, dma_addr); 3040 head_unmap->nvecs++; 3041 3042 for (i = 0, vect_id = 0; i < vectors - 1; i++) { 3043 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 3044 u32 size = skb_frag_size(frag); 3045 3046 if (unlikely(size == 0)) { 3047 /* Undo the changes starting at tcb->producer_index */ 3048 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, 3049 tcb->producer_index); 3050 dev_kfree_skb_any(skb); 3051 BNAD_UPDATE_CTR(bnad, tx_skb_frag_zero); 3052 return NETDEV_TX_OK; 3053 } 3054 3055 len += size; 3056 3057 vect_id++; 3058 if (vect_id == BFI_TX_MAX_VECTORS_PER_WI) { 3059 vect_id = 0; 3060 BNA_QE_INDX_INC(prod, q_depth); 3061 txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; 3062 txqent->hdr.wi_ext.opcode = htons(BNA_TXQ_WI_EXTENSION); 3063 unmap = &unmap_q[prod]; 3064 } 3065 3066 dma_addr = skb_frag_dma_map(&bnad->pcidev->dev, frag, 3067 0, size, DMA_TO_DEVICE); 3068 if (dma_mapping_error(&bnad->pcidev->dev, dma_addr)) { 3069 /* Undo the changes starting at tcb->producer_index */ 3070 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, 3071 tcb->producer_index); 3072 dev_kfree_skb_any(skb); 3073 BNAD_UPDATE_CTR(bnad, tx_skb_map_failed); 3074 return NETDEV_TX_OK; 3075 } 3076 3077 dma_unmap_len_set(&unmap->vectors[vect_id], dma_len, size); 3078 BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[vect_id].host_addr); 3079 txqent->vector[vect_id].length = htons(size); 3080 dma_unmap_addr_set(&unmap->vectors[vect_id], dma_addr, 3081 dma_addr); 3082 head_unmap->nvecs++; 3083 } 3084 3085 if (unlikely(len != skb->len)) { 3086 /* Undo the changes starting at tcb->producer_index */ 3087 bnad_tx_buff_unmap(bnad, unmap_q, q_depth, tcb->producer_index); 3088 dev_kfree_skb_any(skb); 3089 BNAD_UPDATE_CTR(bnad, tx_skb_len_mismatch); 3090 return NETDEV_TX_OK; 3091 } 3092 3093 BNA_QE_INDX_INC(prod, q_depth); 3094 tcb->producer_index = prod; 3095 3096 wmb(); 3097 3098 if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) 3099 return NETDEV_TX_OK; 3100 3101 skb_tx_timestamp(skb); 3102 3103 bna_txq_prod_indx_doorbell(tcb); 3104 3105 return NETDEV_TX_OK; 3106 } 3107 3108 /* 3109 * Used spin_lock to synchronize reading of stats structures, which 3110 * is written by BNA under the same lock. 3111 */ 3112 static void 3113 bnad_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 3114 { 3115 struct bnad *bnad = netdev_priv(netdev); 3116 unsigned long flags; 3117 3118 spin_lock_irqsave(&bnad->bna_lock, flags); 3119 3120 bnad_netdev_qstats_fill(bnad, stats); 3121 bnad_netdev_hwstats_fill(bnad, stats); 3122 3123 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3124 } 3125 3126 static void 3127 bnad_set_rx_ucast_fltr(struct bnad *bnad) 3128 { 3129 struct net_device *netdev = bnad->netdev; 3130 int uc_count = netdev_uc_count(netdev); 3131 enum bna_cb_status ret; 3132 u8 *mac_list; 3133 struct netdev_hw_addr *ha; 3134 int entry; 3135 3136 if (netdev_uc_empty(bnad->netdev)) { 3137 bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL); 3138 return; 3139 } 3140 3141 if (uc_count > bna_attr(&bnad->bna)->num_ucmac) 3142 goto mode_default; 3143 3144 mac_list = kzalloc(uc_count * ETH_ALEN, GFP_ATOMIC); 3145 if (mac_list == NULL) 3146 goto mode_default; 3147 3148 entry = 0; 3149 netdev_for_each_uc_addr(ha, netdev) { 3150 ether_addr_copy(&mac_list[entry * ETH_ALEN], &ha->addr[0]); 3151 entry++; 3152 } 3153 3154 ret = bna_rx_ucast_listset(bnad->rx_info[0].rx, entry, mac_list); 3155 kfree(mac_list); 3156 3157 if (ret != BNA_CB_SUCCESS) 3158 goto mode_default; 3159 3160 return; 3161 3162 /* ucast packets not in UCAM are routed to default function */ 3163 mode_default: 3164 bnad->cfg_flags |= BNAD_CF_DEFAULT; 3165 bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL); 3166 } 3167 3168 static void 3169 bnad_set_rx_mcast_fltr(struct bnad *bnad) 3170 { 3171 struct net_device *netdev = bnad->netdev; 3172 int mc_count = netdev_mc_count(netdev); 3173 enum bna_cb_status ret; 3174 u8 *mac_list; 3175 3176 if (netdev->flags & IFF_ALLMULTI) 3177 goto mode_allmulti; 3178 3179 if (netdev_mc_empty(netdev)) 3180 return; 3181 3182 if (mc_count > bna_attr(&bnad->bna)->num_mcmac) 3183 goto mode_allmulti; 3184 3185 mac_list = kzalloc((mc_count + 1) * ETH_ALEN, GFP_ATOMIC); 3186 3187 if (mac_list == NULL) 3188 goto mode_allmulti; 3189 3190 ether_addr_copy(&mac_list[0], &bnad_bcast_addr[0]); 3191 3192 /* copy rest of the MCAST addresses */ 3193 bnad_netdev_mc_list_get(netdev, mac_list); 3194 ret = bna_rx_mcast_listset(bnad->rx_info[0].rx, mc_count + 1, mac_list); 3195 kfree(mac_list); 3196 3197 if (ret != BNA_CB_SUCCESS) 3198 goto mode_allmulti; 3199 3200 return; 3201 3202 mode_allmulti: 3203 bnad->cfg_flags |= BNAD_CF_ALLMULTI; 3204 bna_rx_mcast_delall(bnad->rx_info[0].rx); 3205 } 3206 3207 void 3208 bnad_set_rx_mode(struct net_device *netdev) 3209 { 3210 struct bnad *bnad = netdev_priv(netdev); 3211 enum bna_rxmode new_mode, mode_mask; 3212 unsigned long flags; 3213 3214 spin_lock_irqsave(&bnad->bna_lock, flags); 3215 3216 if (bnad->rx_info[0].rx == NULL) { 3217 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3218 return; 3219 } 3220 3221 /* clear bnad flags to update it with new settings */ 3222 bnad->cfg_flags &= ~(BNAD_CF_PROMISC | BNAD_CF_DEFAULT | 3223 BNAD_CF_ALLMULTI); 3224 3225 new_mode = 0; 3226 if (netdev->flags & IFF_PROMISC) { 3227 new_mode |= BNAD_RXMODE_PROMISC_DEFAULT; 3228 bnad->cfg_flags |= BNAD_CF_PROMISC; 3229 } else { 3230 bnad_set_rx_mcast_fltr(bnad); 3231 3232 if (bnad->cfg_flags & BNAD_CF_ALLMULTI) 3233 new_mode |= BNA_RXMODE_ALLMULTI; 3234 3235 bnad_set_rx_ucast_fltr(bnad); 3236 3237 if (bnad->cfg_flags & BNAD_CF_DEFAULT) 3238 new_mode |= BNA_RXMODE_DEFAULT; 3239 } 3240 3241 mode_mask = BNA_RXMODE_PROMISC | BNA_RXMODE_DEFAULT | 3242 BNA_RXMODE_ALLMULTI; 3243 bna_rx_mode_set(bnad->rx_info[0].rx, new_mode, mode_mask); 3244 3245 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3246 } 3247 3248 /* 3249 * bna_lock is used to sync writes to netdev->addr 3250 * conf_lock cannot be used since this call may be made 3251 * in a non-blocking context. 3252 */ 3253 static int 3254 bnad_set_mac_address(struct net_device *netdev, void *addr) 3255 { 3256 int err; 3257 struct bnad *bnad = netdev_priv(netdev); 3258 struct sockaddr *sa = (struct sockaddr *)addr; 3259 unsigned long flags; 3260 3261 spin_lock_irqsave(&bnad->bna_lock, flags); 3262 3263 err = bnad_mac_addr_set_locked(bnad, sa->sa_data); 3264 if (!err) 3265 ether_addr_copy(netdev->dev_addr, sa->sa_data); 3266 3267 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3268 3269 return err; 3270 } 3271 3272 static int 3273 bnad_mtu_set(struct bnad *bnad, int frame_size) 3274 { 3275 unsigned long flags; 3276 3277 init_completion(&bnad->bnad_completions.mtu_comp); 3278 3279 spin_lock_irqsave(&bnad->bna_lock, flags); 3280 bna_enet_mtu_set(&bnad->bna.enet, frame_size, bnad_cb_enet_mtu_set); 3281 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3282 3283 wait_for_completion(&bnad->bnad_completions.mtu_comp); 3284 3285 return bnad->bnad_completions.mtu_comp_status; 3286 } 3287 3288 static int 3289 bnad_change_mtu(struct net_device *netdev, int new_mtu) 3290 { 3291 int err, mtu; 3292 struct bnad *bnad = netdev_priv(netdev); 3293 u32 rx_count = 0, frame, new_frame; 3294 3295 mutex_lock(&bnad->conf_mutex); 3296 3297 mtu = netdev->mtu; 3298 netdev->mtu = new_mtu; 3299 3300 frame = BNAD_FRAME_SIZE(mtu); 3301 new_frame = BNAD_FRAME_SIZE(new_mtu); 3302 3303 /* check if multi-buffer needs to be enabled */ 3304 if (BNAD_PCI_DEV_IS_CAT2(bnad) && 3305 netif_running(bnad->netdev)) { 3306 /* only when transition is over 4K */ 3307 if ((frame <= 4096 && new_frame > 4096) || 3308 (frame > 4096 && new_frame <= 4096)) 3309 rx_count = bnad_reinit_rx(bnad); 3310 } 3311 3312 /* rx_count > 0 - new rx created 3313 * - Linux set err = 0 and return 3314 */ 3315 err = bnad_mtu_set(bnad, new_frame); 3316 if (err) 3317 err = -EBUSY; 3318 3319 mutex_unlock(&bnad->conf_mutex); 3320 return err; 3321 } 3322 3323 static int 3324 bnad_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 3325 { 3326 struct bnad *bnad = netdev_priv(netdev); 3327 unsigned long flags; 3328 3329 if (!bnad->rx_info[0].rx) 3330 return 0; 3331 3332 mutex_lock(&bnad->conf_mutex); 3333 3334 spin_lock_irqsave(&bnad->bna_lock, flags); 3335 bna_rx_vlan_add(bnad->rx_info[0].rx, vid); 3336 set_bit(vid, bnad->active_vlans); 3337 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3338 3339 mutex_unlock(&bnad->conf_mutex); 3340 3341 return 0; 3342 } 3343 3344 static int 3345 bnad_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 3346 { 3347 struct bnad *bnad = netdev_priv(netdev); 3348 unsigned long flags; 3349 3350 if (!bnad->rx_info[0].rx) 3351 return 0; 3352 3353 mutex_lock(&bnad->conf_mutex); 3354 3355 spin_lock_irqsave(&bnad->bna_lock, flags); 3356 clear_bit(vid, bnad->active_vlans); 3357 bna_rx_vlan_del(bnad->rx_info[0].rx, vid); 3358 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3359 3360 mutex_unlock(&bnad->conf_mutex); 3361 3362 return 0; 3363 } 3364 3365 static int bnad_set_features(struct net_device *dev, netdev_features_t features) 3366 { 3367 struct bnad *bnad = netdev_priv(dev); 3368 netdev_features_t changed = features ^ dev->features; 3369 3370 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(dev)) { 3371 unsigned long flags; 3372 3373 spin_lock_irqsave(&bnad->bna_lock, flags); 3374 3375 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3376 bna_rx_vlan_strip_enable(bnad->rx_info[0].rx); 3377 else 3378 bna_rx_vlan_strip_disable(bnad->rx_info[0].rx); 3379 3380 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3381 } 3382 3383 return 0; 3384 } 3385 3386 #ifdef CONFIG_NET_POLL_CONTROLLER 3387 static void 3388 bnad_netpoll(struct net_device *netdev) 3389 { 3390 struct bnad *bnad = netdev_priv(netdev); 3391 struct bnad_rx_info *rx_info; 3392 struct bnad_rx_ctrl *rx_ctrl; 3393 u32 curr_mask; 3394 int i, j; 3395 3396 if (!(bnad->cfg_flags & BNAD_CF_MSIX)) { 3397 bna_intx_disable(&bnad->bna, curr_mask); 3398 bnad_isr(bnad->pcidev->irq, netdev); 3399 bna_intx_enable(&bnad->bna, curr_mask); 3400 } else { 3401 /* 3402 * Tx processing may happen in sending context, so no need 3403 * to explicitly process completions here 3404 */ 3405 3406 /* Rx processing */ 3407 for (i = 0; i < bnad->num_rx; i++) { 3408 rx_info = &bnad->rx_info[i]; 3409 if (!rx_info->rx) 3410 continue; 3411 for (j = 0; j < bnad->num_rxp_per_rx; j++) { 3412 rx_ctrl = &rx_info->rx_ctrl[j]; 3413 if (rx_ctrl->ccb) 3414 bnad_netif_rx_schedule_poll(bnad, 3415 rx_ctrl->ccb); 3416 } 3417 } 3418 } 3419 } 3420 #endif 3421 3422 static const struct net_device_ops bnad_netdev_ops = { 3423 .ndo_open = bnad_open, 3424 .ndo_stop = bnad_stop, 3425 .ndo_start_xmit = bnad_start_xmit, 3426 .ndo_get_stats64 = bnad_get_stats64, 3427 .ndo_set_rx_mode = bnad_set_rx_mode, 3428 .ndo_validate_addr = eth_validate_addr, 3429 .ndo_set_mac_address = bnad_set_mac_address, 3430 .ndo_change_mtu = bnad_change_mtu, 3431 .ndo_vlan_rx_add_vid = bnad_vlan_rx_add_vid, 3432 .ndo_vlan_rx_kill_vid = bnad_vlan_rx_kill_vid, 3433 .ndo_set_features = bnad_set_features, 3434 #ifdef CONFIG_NET_POLL_CONTROLLER 3435 .ndo_poll_controller = bnad_netpoll 3436 #endif 3437 }; 3438 3439 static void 3440 bnad_netdev_init(struct bnad *bnad, bool using_dac) 3441 { 3442 struct net_device *netdev = bnad->netdev; 3443 3444 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3445 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3446 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_TX | 3447 NETIF_F_HW_VLAN_CTAG_RX; 3448 3449 netdev->vlan_features = NETIF_F_SG | NETIF_F_HIGHDMA | 3450 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3451 NETIF_F_TSO | NETIF_F_TSO6; 3452 3453 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3454 3455 if (using_dac) 3456 netdev->features |= NETIF_F_HIGHDMA; 3457 3458 netdev->mem_start = bnad->mmio_start; 3459 netdev->mem_end = bnad->mmio_start + bnad->mmio_len - 1; 3460 3461 /* MTU range: 46 - 9000 */ 3462 netdev->min_mtu = ETH_ZLEN - ETH_HLEN; 3463 netdev->max_mtu = BNAD_JUMBO_MTU; 3464 3465 netdev->netdev_ops = &bnad_netdev_ops; 3466 bnad_set_ethtool_ops(netdev); 3467 } 3468 3469 /* 3470 * 1. Initialize the bnad structure 3471 * 2. Setup netdev pointer in pci_dev 3472 * 3. Initialize no. of TxQ & CQs & MSIX vectors 3473 * 4. Initialize work queue. 3474 */ 3475 static int 3476 bnad_init(struct bnad *bnad, 3477 struct pci_dev *pdev, struct net_device *netdev) 3478 { 3479 unsigned long flags; 3480 3481 SET_NETDEV_DEV(netdev, &pdev->dev); 3482 pci_set_drvdata(pdev, netdev); 3483 3484 bnad->netdev = netdev; 3485 bnad->pcidev = pdev; 3486 bnad->mmio_start = pci_resource_start(pdev, 0); 3487 bnad->mmio_len = pci_resource_len(pdev, 0); 3488 bnad->bar0 = ioremap_nocache(bnad->mmio_start, bnad->mmio_len); 3489 if (!bnad->bar0) { 3490 dev_err(&pdev->dev, "ioremap for bar0 failed\n"); 3491 return -ENOMEM; 3492 } 3493 dev_info(&pdev->dev, "bar0 mapped to %p, len %llu\n", bnad->bar0, 3494 (unsigned long long) bnad->mmio_len); 3495 3496 spin_lock_irqsave(&bnad->bna_lock, flags); 3497 if (!bnad_msix_disable) 3498 bnad->cfg_flags = BNAD_CF_MSIX; 3499 3500 bnad->cfg_flags |= BNAD_CF_DIM_ENABLED; 3501 3502 bnad_q_num_init(bnad); 3503 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3504 3505 bnad->msix_num = (bnad->num_tx * bnad->num_txq_per_tx) + 3506 (bnad->num_rx * bnad->num_rxp_per_rx) + 3507 BNAD_MAILBOX_MSIX_VECTORS; 3508 3509 bnad->txq_depth = BNAD_TXQ_DEPTH; 3510 bnad->rxq_depth = BNAD_RXQ_DEPTH; 3511 3512 bnad->tx_coalescing_timeo = BFI_TX_COALESCING_TIMEO; 3513 bnad->rx_coalescing_timeo = BFI_RX_COALESCING_TIMEO; 3514 3515 sprintf(bnad->wq_name, "%s_wq_%d", BNAD_NAME, bnad->id); 3516 bnad->work_q = create_singlethread_workqueue(bnad->wq_name); 3517 if (!bnad->work_q) { 3518 iounmap(bnad->bar0); 3519 return -ENOMEM; 3520 } 3521 3522 return 0; 3523 } 3524 3525 /* 3526 * Must be called after bnad_pci_uninit() 3527 * so that iounmap() and pci_set_drvdata(NULL) 3528 * happens only after PCI uninitialization. 3529 */ 3530 static void 3531 bnad_uninit(struct bnad *bnad) 3532 { 3533 if (bnad->work_q) { 3534 flush_workqueue(bnad->work_q); 3535 destroy_workqueue(bnad->work_q); 3536 bnad->work_q = NULL; 3537 } 3538 3539 if (bnad->bar0) 3540 iounmap(bnad->bar0); 3541 } 3542 3543 /* 3544 * Initialize locks 3545 a) Per ioceth mutes used for serializing configuration 3546 changes from OS interface 3547 b) spin lock used to protect bna state machine 3548 */ 3549 static void 3550 bnad_lock_init(struct bnad *bnad) 3551 { 3552 spin_lock_init(&bnad->bna_lock); 3553 mutex_init(&bnad->conf_mutex); 3554 } 3555 3556 static void 3557 bnad_lock_uninit(struct bnad *bnad) 3558 { 3559 mutex_destroy(&bnad->conf_mutex); 3560 } 3561 3562 /* PCI Initialization */ 3563 static int 3564 bnad_pci_init(struct bnad *bnad, 3565 struct pci_dev *pdev, bool *using_dac) 3566 { 3567 int err; 3568 3569 err = pci_enable_device(pdev); 3570 if (err) 3571 return err; 3572 err = pci_request_regions(pdev, BNAD_NAME); 3573 if (err) 3574 goto disable_device; 3575 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3576 *using_dac = true; 3577 } else { 3578 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3579 if (err) 3580 goto release_regions; 3581 *using_dac = false; 3582 } 3583 pci_set_master(pdev); 3584 return 0; 3585 3586 release_regions: 3587 pci_release_regions(pdev); 3588 disable_device: 3589 pci_disable_device(pdev); 3590 3591 return err; 3592 } 3593 3594 static void 3595 bnad_pci_uninit(struct pci_dev *pdev) 3596 { 3597 pci_release_regions(pdev); 3598 pci_disable_device(pdev); 3599 } 3600 3601 static int 3602 bnad_pci_probe(struct pci_dev *pdev, 3603 const struct pci_device_id *pcidev_id) 3604 { 3605 bool using_dac; 3606 int err; 3607 struct bnad *bnad; 3608 struct bna *bna; 3609 struct net_device *netdev; 3610 struct bfa_pcidev pcidev_info; 3611 unsigned long flags; 3612 3613 mutex_lock(&bnad_fwimg_mutex); 3614 if (!cna_get_firmware_buf(pdev)) { 3615 mutex_unlock(&bnad_fwimg_mutex); 3616 dev_err(&pdev->dev, "failed to load firmware image!\n"); 3617 return -ENODEV; 3618 } 3619 mutex_unlock(&bnad_fwimg_mutex); 3620 3621 /* 3622 * Allocates sizeof(struct net_device + struct bnad) 3623 * bnad = netdev->priv 3624 */ 3625 netdev = alloc_etherdev(sizeof(struct bnad)); 3626 if (!netdev) { 3627 err = -ENOMEM; 3628 return err; 3629 } 3630 bnad = netdev_priv(netdev); 3631 bnad_lock_init(bnad); 3632 bnad->id = atomic_inc_return(&bna_id) - 1; 3633 3634 mutex_lock(&bnad->conf_mutex); 3635 /* 3636 * PCI initialization 3637 * Output : using_dac = 1 for 64 bit DMA 3638 * = 0 for 32 bit DMA 3639 */ 3640 using_dac = false; 3641 err = bnad_pci_init(bnad, pdev, &using_dac); 3642 if (err) 3643 goto unlock_mutex; 3644 3645 /* 3646 * Initialize bnad structure 3647 * Setup relation between pci_dev & netdev 3648 */ 3649 err = bnad_init(bnad, pdev, netdev); 3650 if (err) 3651 goto pci_uninit; 3652 3653 /* Initialize netdev structure, set up ethtool ops */ 3654 bnad_netdev_init(bnad, using_dac); 3655 3656 /* Set link to down state */ 3657 netif_carrier_off(netdev); 3658 3659 /* Setup the debugfs node for this bfad */ 3660 if (bna_debugfs_enable) 3661 bnad_debugfs_init(bnad); 3662 3663 /* Get resource requirement form bna */ 3664 spin_lock_irqsave(&bnad->bna_lock, flags); 3665 bna_res_req(&bnad->res_info[0]); 3666 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3667 3668 /* Allocate resources from bna */ 3669 err = bnad_res_alloc(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3670 if (err) 3671 goto drv_uninit; 3672 3673 bna = &bnad->bna; 3674 3675 /* Setup pcidev_info for bna_init() */ 3676 pcidev_info.pci_slot = PCI_SLOT(bnad->pcidev->devfn); 3677 pcidev_info.pci_func = PCI_FUNC(bnad->pcidev->devfn); 3678 pcidev_info.device_id = bnad->pcidev->device; 3679 pcidev_info.pci_bar_kva = bnad->bar0; 3680 3681 spin_lock_irqsave(&bnad->bna_lock, flags); 3682 bna_init(bna, bnad, &pcidev_info, &bnad->res_info[0]); 3683 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3684 3685 bnad->stats.bna_stats = &bna->stats; 3686 3687 bnad_enable_msix(bnad); 3688 err = bnad_mbox_irq_alloc(bnad); 3689 if (err) 3690 goto res_free; 3691 3692 /* Set up timers */ 3693 timer_setup(&bnad->bna.ioceth.ioc.ioc_timer, bnad_ioc_timeout, 0); 3694 timer_setup(&bnad->bna.ioceth.ioc.hb_timer, bnad_ioc_hb_check, 0); 3695 timer_setup(&bnad->bna.ioceth.ioc.iocpf_timer, bnad_iocpf_timeout, 0); 3696 timer_setup(&bnad->bna.ioceth.ioc.sem_timer, bnad_iocpf_sem_timeout, 3697 0); 3698 3699 /* 3700 * Start the chip 3701 * If the call back comes with error, we bail out. 3702 * This is a catastrophic error. 3703 */ 3704 err = bnad_ioceth_enable(bnad); 3705 if (err) { 3706 dev_err(&pdev->dev, "initialization failed err=%d\n", err); 3707 goto probe_success; 3708 } 3709 3710 spin_lock_irqsave(&bnad->bna_lock, flags); 3711 if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) || 3712 bna_num_rxp_set(bna, BNAD_NUM_RXP + 1)) { 3713 bnad_q_num_adjust(bnad, bna_attr(bna)->num_txq - 1, 3714 bna_attr(bna)->num_rxp - 1); 3715 if (bna_num_txq_set(bna, BNAD_NUM_TXQ + 1) || 3716 bna_num_rxp_set(bna, BNAD_NUM_RXP + 1)) 3717 err = -EIO; 3718 } 3719 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3720 if (err) 3721 goto disable_ioceth; 3722 3723 spin_lock_irqsave(&bnad->bna_lock, flags); 3724 bna_mod_res_req(&bnad->bna, &bnad->mod_res_info[0]); 3725 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3726 3727 err = bnad_res_alloc(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3728 if (err) { 3729 err = -EIO; 3730 goto disable_ioceth; 3731 } 3732 3733 spin_lock_irqsave(&bnad->bna_lock, flags); 3734 bna_mod_init(&bnad->bna, &bnad->mod_res_info[0]); 3735 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3736 3737 /* Get the burnt-in mac */ 3738 spin_lock_irqsave(&bnad->bna_lock, flags); 3739 bna_enet_perm_mac_get(&bna->enet, bnad->perm_addr); 3740 bnad_set_netdev_perm_addr(bnad); 3741 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3742 3743 mutex_unlock(&bnad->conf_mutex); 3744 3745 /* Finally, reguister with net_device layer */ 3746 err = register_netdev(netdev); 3747 if (err) { 3748 dev_err(&pdev->dev, "registering net device failed\n"); 3749 goto probe_uninit; 3750 } 3751 set_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags); 3752 3753 return 0; 3754 3755 probe_success: 3756 mutex_unlock(&bnad->conf_mutex); 3757 return 0; 3758 3759 probe_uninit: 3760 mutex_lock(&bnad->conf_mutex); 3761 bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3762 disable_ioceth: 3763 bnad_ioceth_disable(bnad); 3764 del_timer_sync(&bnad->bna.ioceth.ioc.ioc_timer); 3765 del_timer_sync(&bnad->bna.ioceth.ioc.sem_timer); 3766 del_timer_sync(&bnad->bna.ioceth.ioc.hb_timer); 3767 spin_lock_irqsave(&bnad->bna_lock, flags); 3768 bna_uninit(bna); 3769 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3770 bnad_mbox_irq_free(bnad); 3771 bnad_disable_msix(bnad); 3772 res_free: 3773 bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3774 drv_uninit: 3775 /* Remove the debugfs node for this bnad */ 3776 kfree(bnad->regdata); 3777 bnad_debugfs_uninit(bnad); 3778 bnad_uninit(bnad); 3779 pci_uninit: 3780 bnad_pci_uninit(pdev); 3781 unlock_mutex: 3782 mutex_unlock(&bnad->conf_mutex); 3783 bnad_lock_uninit(bnad); 3784 free_netdev(netdev); 3785 return err; 3786 } 3787 3788 static void 3789 bnad_pci_remove(struct pci_dev *pdev) 3790 { 3791 struct net_device *netdev = pci_get_drvdata(pdev); 3792 struct bnad *bnad; 3793 struct bna *bna; 3794 unsigned long flags; 3795 3796 if (!netdev) 3797 return; 3798 3799 bnad = netdev_priv(netdev); 3800 bna = &bnad->bna; 3801 3802 if (test_and_clear_bit(BNAD_RF_NETDEV_REGISTERED, &bnad->run_flags)) 3803 unregister_netdev(netdev); 3804 3805 mutex_lock(&bnad->conf_mutex); 3806 bnad_ioceth_disable(bnad); 3807 del_timer_sync(&bnad->bna.ioceth.ioc.ioc_timer); 3808 del_timer_sync(&bnad->bna.ioceth.ioc.sem_timer); 3809 del_timer_sync(&bnad->bna.ioceth.ioc.hb_timer); 3810 spin_lock_irqsave(&bnad->bna_lock, flags); 3811 bna_uninit(bna); 3812 spin_unlock_irqrestore(&bnad->bna_lock, flags); 3813 3814 bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); 3815 bnad_res_free(bnad, &bnad->res_info[0], BNA_RES_T_MAX); 3816 bnad_mbox_irq_free(bnad); 3817 bnad_disable_msix(bnad); 3818 bnad_pci_uninit(pdev); 3819 mutex_unlock(&bnad->conf_mutex); 3820 bnad_lock_uninit(bnad); 3821 /* Remove the debugfs node for this bnad */ 3822 kfree(bnad->regdata); 3823 bnad_debugfs_uninit(bnad); 3824 bnad_uninit(bnad); 3825 free_netdev(netdev); 3826 } 3827 3828 static const struct pci_device_id bnad_pci_id_table[] = { 3829 { 3830 PCI_DEVICE(PCI_VENDOR_ID_BROCADE, 3831 PCI_DEVICE_ID_BROCADE_CT), 3832 .class = PCI_CLASS_NETWORK_ETHERNET << 8, 3833 .class_mask = 0xffff00 3834 }, 3835 { 3836 PCI_DEVICE(PCI_VENDOR_ID_BROCADE, 3837 BFA_PCI_DEVICE_ID_CT2), 3838 .class = PCI_CLASS_NETWORK_ETHERNET << 8, 3839 .class_mask = 0xffff00 3840 }, 3841 {0, }, 3842 }; 3843 3844 MODULE_DEVICE_TABLE(pci, bnad_pci_id_table); 3845 3846 static struct pci_driver bnad_pci_driver = { 3847 .name = BNAD_NAME, 3848 .id_table = bnad_pci_id_table, 3849 .probe = bnad_pci_probe, 3850 .remove = bnad_pci_remove, 3851 }; 3852 3853 static int __init 3854 bnad_module_init(void) 3855 { 3856 int err; 3857 3858 pr_info("bna: QLogic BR-series 10G Ethernet driver - version: %s\n", 3859 BNAD_VERSION); 3860 3861 bfa_nw_ioc_auto_recover(bnad_ioc_auto_recover); 3862 3863 err = pci_register_driver(&bnad_pci_driver); 3864 if (err < 0) { 3865 pr_err("bna: PCI driver registration failed err=%d\n", err); 3866 return err; 3867 } 3868 3869 return 0; 3870 } 3871 3872 static void __exit 3873 bnad_module_exit(void) 3874 { 3875 pci_unregister_driver(&bnad_pci_driver); 3876 release_firmware(bfi_fw); 3877 } 3878 3879 module_init(bnad_module_init); 3880 module_exit(bnad_module_exit); 3881 3882 MODULE_AUTHOR("Brocade"); 3883 MODULE_LICENSE("GPL"); 3884 MODULE_DESCRIPTION("QLogic BR-series 10G PCIe Ethernet driver"); 3885 MODULE_VERSION(BNAD_VERSION); 3886 MODULE_FIRMWARE(CNA_FW_FILE_CT); 3887 MODULE_FIRMWARE(CNA_FW_FILE_CT2); 3888