1 /* bnx2x_cmn.c: QLogic Everest network driver. 2 * 3 * Copyright (c) 2007-2013 Broadcom Corporation 4 * Copyright (c) 2014 QLogic Corporation 5 * All rights reserved 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation. 10 * 11 * Maintained by: Ariel Elior <ariel.elior@qlogic.com> 12 * Written by: Eliezer Tamir 13 * Based on code from Michael Chan's bnx2 driver 14 * UDP CSUM errata workaround by Arik Gendelman 15 * Slowpath and fastpath rework by Vladislav Zolotarov 16 * Statistics and Link management by Yitchak Gertner 17 * 18 */ 19 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/etherdevice.h> 23 #include <linux/if_vlan.h> 24 #include <linux/interrupt.h> 25 #include <linux/ip.h> 26 #include <linux/crash_dump.h> 27 #include <net/tcp.h> 28 #include <net/ipv6.h> 29 #include <net/ip6_checksum.h> 30 #include <net/busy_poll.h> 31 #include <linux/prefetch.h> 32 #include "bnx2x_cmn.h" 33 #include "bnx2x_init.h" 34 #include "bnx2x_sp.h" 35 36 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp); 37 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp); 38 static int bnx2x_alloc_fp_mem(struct bnx2x *bp); 39 static int bnx2x_poll(struct napi_struct *napi, int budget); 40 41 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp) 42 { 43 int i; 44 45 /* Add NAPI objects */ 46 for_each_rx_queue_cnic(bp, i) { 47 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), 48 bnx2x_poll, NAPI_POLL_WEIGHT); 49 } 50 } 51 52 static void bnx2x_add_all_napi(struct bnx2x *bp) 53 { 54 int i; 55 56 /* Add NAPI objects */ 57 for_each_eth_queue(bp, i) { 58 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), 59 bnx2x_poll, NAPI_POLL_WEIGHT); 60 } 61 } 62 63 static int bnx2x_calc_num_queues(struct bnx2x *bp) 64 { 65 int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues(); 66 67 /* Reduce memory usage in kdump environment by using only one queue */ 68 if (is_kdump_kernel()) 69 nq = 1; 70 71 nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp)); 72 return nq; 73 } 74 75 /** 76 * bnx2x_move_fp - move content of the fastpath structure. 77 * 78 * @bp: driver handle 79 * @from: source FP index 80 * @to: destination FP index 81 * 82 * Makes sure the contents of the bp->fp[to].napi is kept 83 * intact. This is done by first copying the napi struct from 84 * the target to the source, and then mem copying the entire 85 * source onto the target. Update txdata pointers and related 86 * content. 87 */ 88 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to) 89 { 90 struct bnx2x_fastpath *from_fp = &bp->fp[from]; 91 struct bnx2x_fastpath *to_fp = &bp->fp[to]; 92 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from]; 93 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to]; 94 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from]; 95 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to]; 96 int old_max_eth_txqs, new_max_eth_txqs; 97 int old_txdata_index = 0, new_txdata_index = 0; 98 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info; 99 100 /* Copy the NAPI object as it has been already initialized */ 101 from_fp->napi = to_fp->napi; 102 103 /* Move bnx2x_fastpath contents */ 104 memcpy(to_fp, from_fp, sizeof(*to_fp)); 105 to_fp->index = to; 106 107 /* Retain the tpa_info of the original `to' version as we don't want 108 * 2 FPs to contain the same tpa_info pointer. 109 */ 110 to_fp->tpa_info = old_tpa_info; 111 112 /* move sp_objs contents as well, as their indices match fp ones */ 113 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs)); 114 115 /* move fp_stats contents as well, as their indices match fp ones */ 116 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats)); 117 118 /* Update txdata pointers in fp and move txdata content accordingly: 119 * Each fp consumes 'max_cos' txdata structures, so the index should be 120 * decremented by max_cos x delta. 121 */ 122 123 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos; 124 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) * 125 (bp)->max_cos; 126 if (from == FCOE_IDX(bp)) { 127 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 128 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET; 129 } 130 131 memcpy(&bp->bnx2x_txq[new_txdata_index], 132 &bp->bnx2x_txq[old_txdata_index], 133 sizeof(struct bnx2x_fp_txdata)); 134 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index]; 135 } 136 137 /** 138 * bnx2x_fill_fw_str - Fill buffer with FW version string. 139 * 140 * @bp: driver handle 141 * @buf: character buffer to fill with the fw name 142 * @buf_len: length of the above buffer 143 * 144 */ 145 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len) 146 { 147 if (IS_PF(bp)) { 148 u8 phy_fw_ver[PHY_FW_VER_LEN]; 149 150 phy_fw_ver[0] = '\0'; 151 bnx2x_get_ext_phy_fw_version(&bp->link_params, 152 phy_fw_ver, PHY_FW_VER_LEN); 153 strlcpy(buf, bp->fw_ver, buf_len); 154 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver), 155 "bc %d.%d.%d%s%s", 156 (bp->common.bc_ver & 0xff0000) >> 16, 157 (bp->common.bc_ver & 0xff00) >> 8, 158 (bp->common.bc_ver & 0xff), 159 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver); 160 } else { 161 bnx2x_vf_fill_fw_str(bp, buf, buf_len); 162 } 163 } 164 165 /** 166 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact 167 * 168 * @bp: driver handle 169 * @delta: number of eth queues which were not allocated 170 */ 171 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta) 172 { 173 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp); 174 175 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer 176 * backward along the array could cause memory to be overridden 177 */ 178 for (cos = 1; cos < bp->max_cos; cos++) { 179 for (i = 0; i < old_eth_num - delta; i++) { 180 struct bnx2x_fastpath *fp = &bp->fp[i]; 181 int new_idx = cos * (old_eth_num - delta) + i; 182 183 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos], 184 sizeof(struct bnx2x_fp_txdata)); 185 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx]; 186 } 187 } 188 } 189 190 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */ 191 192 /* free skb in the packet ring at pos idx 193 * return idx of last bd freed 194 */ 195 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata, 196 u16 idx, unsigned int *pkts_compl, 197 unsigned int *bytes_compl) 198 { 199 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx]; 200 struct eth_tx_start_bd *tx_start_bd; 201 struct eth_tx_bd *tx_data_bd; 202 struct sk_buff *skb = tx_buf->skb; 203 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons; 204 int nbd; 205 u16 split_bd_len = 0; 206 207 /* prefetch skb end pointer to speedup dev_kfree_skb() */ 208 prefetch(&skb->end); 209 210 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n", 211 txdata->txq_index, idx, tx_buf, skb); 212 213 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd; 214 215 nbd = le16_to_cpu(tx_start_bd->nbd) - 1; 216 #ifdef BNX2X_STOP_ON_ERROR 217 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) { 218 BNX2X_ERR("BAD nbd!\n"); 219 bnx2x_panic(); 220 } 221 #endif 222 new_cons = nbd + tx_buf->first_bd; 223 224 /* Get the next bd */ 225 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 226 227 /* Skip a parse bd... */ 228 --nbd; 229 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 230 231 if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) { 232 /* Skip second parse bd... */ 233 --nbd; 234 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 235 } 236 237 /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */ 238 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) { 239 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd; 240 split_bd_len = BD_UNMAP_LEN(tx_data_bd); 241 --nbd; 242 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 243 } 244 245 /* unmap first bd */ 246 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd), 247 BD_UNMAP_LEN(tx_start_bd) + split_bd_len, 248 DMA_TO_DEVICE); 249 250 /* now free frags */ 251 while (nbd > 0) { 252 253 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd; 254 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd), 255 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE); 256 if (--nbd) 257 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 258 } 259 260 /* release skb */ 261 WARN_ON(!skb); 262 if (likely(skb)) { 263 (*pkts_compl)++; 264 (*bytes_compl) += skb->len; 265 dev_kfree_skb_any(skb); 266 } 267 268 tx_buf->first_bd = 0; 269 tx_buf->skb = NULL; 270 271 return new_cons; 272 } 273 274 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata) 275 { 276 struct netdev_queue *txq; 277 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons; 278 unsigned int pkts_compl = 0, bytes_compl = 0; 279 280 #ifdef BNX2X_STOP_ON_ERROR 281 if (unlikely(bp->panic)) 282 return -1; 283 #endif 284 285 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index); 286 hw_cons = le16_to_cpu(*txdata->tx_cons_sb); 287 sw_cons = txdata->tx_pkt_cons; 288 289 while (sw_cons != hw_cons) { 290 u16 pkt_cons; 291 292 pkt_cons = TX_BD(sw_cons); 293 294 DP(NETIF_MSG_TX_DONE, 295 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n", 296 txdata->txq_index, hw_cons, sw_cons, pkt_cons); 297 298 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons, 299 &pkts_compl, &bytes_compl); 300 301 sw_cons++; 302 } 303 304 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl); 305 306 txdata->tx_pkt_cons = sw_cons; 307 txdata->tx_bd_cons = bd_cons; 308 309 /* Need to make the tx_bd_cons update visible to start_xmit() 310 * before checking for netif_tx_queue_stopped(). Without the 311 * memory barrier, there is a small possibility that 312 * start_xmit() will miss it and cause the queue to be stopped 313 * forever. 314 * On the other hand we need an rmb() here to ensure the proper 315 * ordering of bit testing in the following 316 * netif_tx_queue_stopped(txq) call. 317 */ 318 smp_mb(); 319 320 if (unlikely(netif_tx_queue_stopped(txq))) { 321 /* Taking tx_lock() is needed to prevent re-enabling the queue 322 * while it's empty. This could have happen if rx_action() gets 323 * suspended in bnx2x_tx_int() after the condition before 324 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()): 325 * 326 * stops the queue->sees fresh tx_bd_cons->releases the queue-> 327 * sends some packets consuming the whole queue again-> 328 * stops the queue 329 */ 330 331 __netif_tx_lock(txq, smp_processor_id()); 332 333 if ((netif_tx_queue_stopped(txq)) && 334 (bp->state == BNX2X_STATE_OPEN) && 335 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)) 336 netif_tx_wake_queue(txq); 337 338 __netif_tx_unlock(txq); 339 } 340 return 0; 341 } 342 343 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp, 344 u16 idx) 345 { 346 u16 last_max = fp->last_max_sge; 347 348 if (SUB_S16(idx, last_max) > 0) 349 fp->last_max_sge = idx; 350 } 351 352 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp, 353 u16 sge_len, 354 struct eth_end_agg_rx_cqe *cqe) 355 { 356 struct bnx2x *bp = fp->bp; 357 u16 last_max, last_elem, first_elem; 358 u16 delta = 0; 359 u16 i; 360 361 if (!sge_len) 362 return; 363 364 /* First mark all used pages */ 365 for (i = 0; i < sge_len; i++) 366 BIT_VEC64_CLEAR_BIT(fp->sge_mask, 367 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i]))); 368 369 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n", 370 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 371 372 /* Here we assume that the last SGE index is the biggest */ 373 prefetch((void *)(fp->sge_mask)); 374 bnx2x_update_last_max_sge(fp, 375 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1])); 376 377 last_max = RX_SGE(fp->last_max_sge); 378 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT; 379 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT; 380 381 /* If ring is not full */ 382 if (last_elem + 1 != first_elem) 383 last_elem++; 384 385 /* Now update the prod */ 386 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) { 387 if (likely(fp->sge_mask[i])) 388 break; 389 390 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK; 391 delta += BIT_VEC64_ELEM_SZ; 392 } 393 394 if (delta > 0) { 395 fp->rx_sge_prod += delta; 396 /* clear page-end entries */ 397 bnx2x_clear_sge_mask_next_elems(fp); 398 } 399 400 DP(NETIF_MSG_RX_STATUS, 401 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n", 402 fp->last_max_sge, fp->rx_sge_prod); 403 } 404 405 /* Get Toeplitz hash value in the skb using the value from the 406 * CQE (calculated by HW). 407 */ 408 static u32 bnx2x_get_rxhash(const struct bnx2x *bp, 409 const struct eth_fast_path_rx_cqe *cqe, 410 enum pkt_hash_types *rxhash_type) 411 { 412 /* Get Toeplitz hash from CQE */ 413 if ((bp->dev->features & NETIF_F_RXHASH) && 414 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) { 415 enum eth_rss_hash_type htype; 416 417 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE; 418 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) || 419 (htype == TCP_IPV6_HASH_TYPE)) ? 420 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3; 421 422 return le32_to_cpu(cqe->rss_hash_result); 423 } 424 *rxhash_type = PKT_HASH_TYPE_NONE; 425 return 0; 426 } 427 428 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue, 429 u16 cons, u16 prod, 430 struct eth_fast_path_rx_cqe *cqe) 431 { 432 struct bnx2x *bp = fp->bp; 433 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons]; 434 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod]; 435 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod]; 436 dma_addr_t mapping; 437 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue]; 438 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 439 440 /* print error if current state != stop */ 441 if (tpa_info->tpa_state != BNX2X_TPA_STOP) 442 BNX2X_ERR("start of bin not in stop [%d]\n", queue); 443 444 /* Try to map an empty data buffer from the aggregation info */ 445 mapping = dma_map_single(&bp->pdev->dev, 446 first_buf->data + NET_SKB_PAD, 447 fp->rx_buf_size, DMA_FROM_DEVICE); 448 /* 449 * ...if it fails - move the skb from the consumer to the producer 450 * and set the current aggregation state as ERROR to drop it 451 * when TPA_STOP arrives. 452 */ 453 454 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 455 /* Move the BD from the consumer to the producer */ 456 bnx2x_reuse_rx_data(fp, cons, prod); 457 tpa_info->tpa_state = BNX2X_TPA_ERROR; 458 return; 459 } 460 461 /* move empty data from pool to prod */ 462 prod_rx_buf->data = first_buf->data; 463 dma_unmap_addr_set(prod_rx_buf, mapping, mapping); 464 /* point prod_bd to new data */ 465 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 466 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 467 468 /* move partial skb from cons to pool (don't unmap yet) */ 469 *first_buf = *cons_rx_buf; 470 471 /* mark bin state as START */ 472 tpa_info->parsing_flags = 473 le16_to_cpu(cqe->pars_flags.flags); 474 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag); 475 tpa_info->tpa_state = BNX2X_TPA_START; 476 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd); 477 tpa_info->placement_offset = cqe->placement_offset; 478 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type); 479 if (fp->mode == TPA_MODE_GRO) { 480 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len); 481 tpa_info->full_page = SGE_PAGES / gro_size * gro_size; 482 tpa_info->gro_size = gro_size; 483 } 484 485 #ifdef BNX2X_STOP_ON_ERROR 486 fp->tpa_queue_used |= (1 << queue); 487 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n", 488 fp->tpa_queue_used); 489 #endif 490 } 491 492 /* Timestamp option length allowed for TPA aggregation: 493 * 494 * nop nop kind length echo val 495 */ 496 #define TPA_TSTAMP_OPT_LEN 12 497 /** 498 * bnx2x_set_gro_params - compute GRO values 499 * 500 * @skb: packet skb 501 * @parsing_flags: parsing flags from the START CQE 502 * @len_on_bd: total length of the first packet for the 503 * aggregation. 504 * @pkt_len: length of all segments 505 * 506 * Approximate value of the MSS for this aggregation calculated using 507 * the first packet of it. 508 * Compute number of aggregated segments, and gso_type. 509 */ 510 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags, 511 u16 len_on_bd, unsigned int pkt_len, 512 u16 num_of_coalesced_segs) 513 { 514 /* TPA aggregation won't have either IP options or TCP options 515 * other than timestamp or IPv6 extension headers. 516 */ 517 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr); 518 519 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) == 520 PRS_FLAG_OVERETH_IPV6) { 521 hdrs_len += sizeof(struct ipv6hdr); 522 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 523 } else { 524 hdrs_len += sizeof(struct iphdr); 525 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 526 } 527 528 /* Check if there was a TCP timestamp, if there is it's will 529 * always be 12 bytes length: nop nop kind length echo val. 530 * 531 * Otherwise FW would close the aggregation. 532 */ 533 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG) 534 hdrs_len += TPA_TSTAMP_OPT_LEN; 535 536 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len; 537 538 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count 539 * to skb_shinfo(skb)->gso_segs 540 */ 541 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs; 542 } 543 544 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp, 545 u16 index, gfp_t gfp_mask) 546 { 547 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index]; 548 struct eth_rx_sge *sge = &fp->rx_sge_ring[index]; 549 struct bnx2x_alloc_pool *pool = &fp->page_pool; 550 dma_addr_t mapping; 551 552 if (!pool->page || (PAGE_SIZE - pool->offset) < SGE_PAGE_SIZE) { 553 554 /* put page reference used by the memory pool, since we 555 * won't be using this page as the mempool anymore. 556 */ 557 if (pool->page) 558 put_page(pool->page); 559 560 pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT); 561 if (unlikely(!pool->page)) 562 return -ENOMEM; 563 564 pool->offset = 0; 565 } 566 567 mapping = dma_map_page(&bp->pdev->dev, pool->page, 568 pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE); 569 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 570 BNX2X_ERR("Can't map sge\n"); 571 return -ENOMEM; 572 } 573 574 get_page(pool->page); 575 sw_buf->page = pool->page; 576 sw_buf->offset = pool->offset; 577 578 dma_unmap_addr_set(sw_buf, mapping, mapping); 579 580 sge->addr_hi = cpu_to_le32(U64_HI(mapping)); 581 sge->addr_lo = cpu_to_le32(U64_LO(mapping)); 582 583 pool->offset += SGE_PAGE_SIZE; 584 585 return 0; 586 } 587 588 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp, 589 struct bnx2x_agg_info *tpa_info, 590 u16 pages, 591 struct sk_buff *skb, 592 struct eth_end_agg_rx_cqe *cqe, 593 u16 cqe_idx) 594 { 595 struct sw_rx_page *rx_pg, old_rx_pg; 596 u32 i, frag_len, frag_size; 597 int err, j, frag_id = 0; 598 u16 len_on_bd = tpa_info->len_on_bd; 599 u16 full_page = 0, gro_size = 0; 600 601 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd; 602 603 if (fp->mode == TPA_MODE_GRO) { 604 gro_size = tpa_info->gro_size; 605 full_page = tpa_info->full_page; 606 } 607 608 /* This is needed in order to enable forwarding support */ 609 if (frag_size) 610 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd, 611 le16_to_cpu(cqe->pkt_len), 612 le16_to_cpu(cqe->num_of_coalesced_segs)); 613 614 #ifdef BNX2X_STOP_ON_ERROR 615 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) { 616 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n", 617 pages, cqe_idx); 618 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len); 619 bnx2x_panic(); 620 return -EINVAL; 621 } 622 #endif 623 624 /* Run through the SGL and compose the fragmented skb */ 625 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) { 626 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j])); 627 628 /* FW gives the indices of the SGE as if the ring is an array 629 (meaning that "next" element will consume 2 indices) */ 630 if (fp->mode == TPA_MODE_GRO) 631 frag_len = min_t(u32, frag_size, (u32)full_page); 632 else /* LRO */ 633 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES); 634 635 rx_pg = &fp->rx_page_ring[sge_idx]; 636 old_rx_pg = *rx_pg; 637 638 /* If we fail to allocate a substitute page, we simply stop 639 where we are and drop the whole packet */ 640 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC); 641 if (unlikely(err)) { 642 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 643 return err; 644 } 645 646 dma_unmap_page(&bp->pdev->dev, 647 dma_unmap_addr(&old_rx_pg, mapping), 648 SGE_PAGE_SIZE, DMA_FROM_DEVICE); 649 /* Add one frag and update the appropriate fields in the skb */ 650 if (fp->mode == TPA_MODE_LRO) 651 skb_fill_page_desc(skb, j, old_rx_pg.page, 652 old_rx_pg.offset, frag_len); 653 else { /* GRO */ 654 int rem; 655 int offset = 0; 656 for (rem = frag_len; rem > 0; rem -= gro_size) { 657 int len = rem > gro_size ? gro_size : rem; 658 skb_fill_page_desc(skb, frag_id++, 659 old_rx_pg.page, 660 old_rx_pg.offset + offset, 661 len); 662 if (offset) 663 get_page(old_rx_pg.page); 664 offset += len; 665 } 666 } 667 668 skb->data_len += frag_len; 669 skb->truesize += SGE_PAGES; 670 skb->len += frag_len; 671 672 frag_size -= frag_len; 673 } 674 675 return 0; 676 } 677 678 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data) 679 { 680 if (fp->rx_frag_size) 681 skb_free_frag(data); 682 else 683 kfree(data); 684 } 685 686 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask) 687 { 688 if (fp->rx_frag_size) { 689 /* GFP_KERNEL allocations are used only during initialization */ 690 if (unlikely(gfpflags_allow_blocking(gfp_mask))) 691 return (void *)__get_free_page(gfp_mask); 692 693 return netdev_alloc_frag(fp->rx_frag_size); 694 } 695 696 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask); 697 } 698 699 #ifdef CONFIG_INET 700 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb) 701 { 702 const struct iphdr *iph = ip_hdr(skb); 703 struct tcphdr *th; 704 705 skb_set_transport_header(skb, sizeof(struct iphdr)); 706 th = tcp_hdr(skb); 707 708 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb), 709 iph->saddr, iph->daddr, 0); 710 } 711 712 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb) 713 { 714 struct ipv6hdr *iph = ipv6_hdr(skb); 715 struct tcphdr *th; 716 717 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 718 th = tcp_hdr(skb); 719 720 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb), 721 &iph->saddr, &iph->daddr, 0); 722 } 723 724 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb, 725 void (*gro_func)(struct bnx2x*, struct sk_buff*)) 726 { 727 skb_set_network_header(skb, 0); 728 gro_func(bp, skb); 729 tcp_gro_complete(skb); 730 } 731 #endif 732 733 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp, 734 struct sk_buff *skb) 735 { 736 #ifdef CONFIG_INET 737 if (skb_shinfo(skb)->gso_size) { 738 switch (be16_to_cpu(skb->protocol)) { 739 case ETH_P_IP: 740 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum); 741 break; 742 case ETH_P_IPV6: 743 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum); 744 break; 745 default: 746 WARN_ONCE(1, "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n", 747 be16_to_cpu(skb->protocol)); 748 } 749 } 750 #endif 751 skb_record_rx_queue(skb, fp->rx_queue); 752 napi_gro_receive(&fp->napi, skb); 753 } 754 755 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, 756 struct bnx2x_agg_info *tpa_info, 757 u16 pages, 758 struct eth_end_agg_rx_cqe *cqe, 759 u16 cqe_idx) 760 { 761 struct sw_rx_bd *rx_buf = &tpa_info->first_buf; 762 u8 pad = tpa_info->placement_offset; 763 u16 len = tpa_info->len_on_bd; 764 struct sk_buff *skb = NULL; 765 u8 *new_data, *data = rx_buf->data; 766 u8 old_tpa_state = tpa_info->tpa_state; 767 768 tpa_info->tpa_state = BNX2X_TPA_STOP; 769 770 /* If we there was an error during the handling of the TPA_START - 771 * drop this aggregation. 772 */ 773 if (old_tpa_state == BNX2X_TPA_ERROR) 774 goto drop; 775 776 /* Try to allocate the new data */ 777 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC); 778 /* Unmap skb in the pool anyway, as we are going to change 779 pool entry status to BNX2X_TPA_STOP even if new skb allocation 780 fails. */ 781 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), 782 fp->rx_buf_size, DMA_FROM_DEVICE); 783 if (likely(new_data)) 784 skb = build_skb(data, fp->rx_frag_size); 785 786 if (likely(skb)) { 787 #ifdef BNX2X_STOP_ON_ERROR 788 if (pad + len > fp->rx_buf_size) { 789 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n", 790 pad, len, fp->rx_buf_size); 791 bnx2x_panic(); 792 return; 793 } 794 #endif 795 796 skb_reserve(skb, pad + NET_SKB_PAD); 797 skb_put(skb, len); 798 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type); 799 800 skb->protocol = eth_type_trans(skb, bp->dev); 801 skb->ip_summed = CHECKSUM_UNNECESSARY; 802 803 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages, 804 skb, cqe, cqe_idx)) { 805 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN) 806 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag); 807 bnx2x_gro_receive(bp, fp, skb); 808 } else { 809 DP(NETIF_MSG_RX_STATUS, 810 "Failed to allocate new pages - dropping packet!\n"); 811 dev_kfree_skb_any(skb); 812 } 813 814 /* put new data in bin */ 815 rx_buf->data = new_data; 816 817 return; 818 } 819 if (new_data) 820 bnx2x_frag_free(fp, new_data); 821 drop: 822 /* drop the packet and keep the buffer in the bin */ 823 DP(NETIF_MSG_RX_STATUS, 824 "Failed to allocate or map a new skb - dropping packet!\n"); 825 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++; 826 } 827 828 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp, 829 u16 index, gfp_t gfp_mask) 830 { 831 u8 *data; 832 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index]; 833 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index]; 834 dma_addr_t mapping; 835 836 data = bnx2x_frag_alloc(fp, gfp_mask); 837 if (unlikely(data == NULL)) 838 return -ENOMEM; 839 840 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD, 841 fp->rx_buf_size, 842 DMA_FROM_DEVICE); 843 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 844 bnx2x_frag_free(fp, data); 845 BNX2X_ERR("Can't map rx data\n"); 846 return -ENOMEM; 847 } 848 849 rx_buf->data = data; 850 dma_unmap_addr_set(rx_buf, mapping, mapping); 851 852 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 853 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 854 855 return 0; 856 } 857 858 static 859 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe, 860 struct bnx2x_fastpath *fp, 861 struct bnx2x_eth_q_stats *qstats) 862 { 863 /* Do nothing if no L4 csum validation was done. 864 * We do not check whether IP csum was validated. For IPv4 we assume 865 * that if the card got as far as validating the L4 csum, it also 866 * validated the IP csum. IPv6 has no IP csum. 867 */ 868 if (cqe->fast_path_cqe.status_flags & 869 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG) 870 return; 871 872 /* If L4 validation was done, check if an error was found. */ 873 874 if (cqe->fast_path_cqe.type_error_flags & 875 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG | 876 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG)) 877 qstats->hw_csum_err++; 878 else 879 skb->ip_summed = CHECKSUM_UNNECESSARY; 880 } 881 882 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) 883 { 884 struct bnx2x *bp = fp->bp; 885 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons; 886 u16 sw_comp_cons, sw_comp_prod; 887 int rx_pkt = 0; 888 union eth_rx_cqe *cqe; 889 struct eth_fast_path_rx_cqe *cqe_fp; 890 891 #ifdef BNX2X_STOP_ON_ERROR 892 if (unlikely(bp->panic)) 893 return 0; 894 #endif 895 if (budget <= 0) 896 return rx_pkt; 897 898 bd_cons = fp->rx_bd_cons; 899 bd_prod = fp->rx_bd_prod; 900 bd_prod_fw = bd_prod; 901 sw_comp_cons = fp->rx_comp_cons; 902 sw_comp_prod = fp->rx_comp_prod; 903 904 comp_ring_cons = RCQ_BD(sw_comp_cons); 905 cqe = &fp->rx_comp_ring[comp_ring_cons]; 906 cqe_fp = &cqe->fast_path_cqe; 907 908 DP(NETIF_MSG_RX_STATUS, 909 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons); 910 911 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) { 912 struct sw_rx_bd *rx_buf = NULL; 913 struct sk_buff *skb; 914 u8 cqe_fp_flags; 915 enum eth_rx_cqe_type cqe_fp_type; 916 u16 len, pad, queue; 917 u8 *data; 918 u32 rxhash; 919 enum pkt_hash_types rxhash_type; 920 921 #ifdef BNX2X_STOP_ON_ERROR 922 if (unlikely(bp->panic)) 923 return 0; 924 #endif 925 926 bd_prod = RX_BD(bd_prod); 927 bd_cons = RX_BD(bd_cons); 928 929 /* A rmb() is required to ensure that the CQE is not read 930 * before it is written by the adapter DMA. PCI ordering 931 * rules will make sure the other fields are written before 932 * the marker at the end of struct eth_fast_path_rx_cqe 933 * but without rmb() a weakly ordered processor can process 934 * stale data. Without the barrier TPA state-machine might 935 * enter inconsistent state and kernel stack might be 936 * provided with incorrect packet description - these lead 937 * to various kernel crashed. 938 */ 939 rmb(); 940 941 cqe_fp_flags = cqe_fp->type_error_flags; 942 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE; 943 944 DP(NETIF_MSG_RX_STATUS, 945 "CQE type %x err %x status %x queue %x vlan %x len %u\n", 946 CQE_TYPE(cqe_fp_flags), 947 cqe_fp_flags, cqe_fp->status_flags, 948 le32_to_cpu(cqe_fp->rss_hash_result), 949 le16_to_cpu(cqe_fp->vlan_tag), 950 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len)); 951 952 /* is this a slowpath msg? */ 953 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) { 954 bnx2x_sp_event(fp, cqe); 955 goto next_cqe; 956 } 957 958 rx_buf = &fp->rx_buf_ring[bd_cons]; 959 data = rx_buf->data; 960 961 if (!CQE_TYPE_FAST(cqe_fp_type)) { 962 struct bnx2x_agg_info *tpa_info; 963 u16 frag_size, pages; 964 #ifdef BNX2X_STOP_ON_ERROR 965 /* sanity check */ 966 if (fp->mode == TPA_MODE_DISABLED && 967 (CQE_TYPE_START(cqe_fp_type) || 968 CQE_TYPE_STOP(cqe_fp_type))) 969 BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n", 970 CQE_TYPE(cqe_fp_type)); 971 #endif 972 973 if (CQE_TYPE_START(cqe_fp_type)) { 974 u16 queue = cqe_fp->queue_index; 975 DP(NETIF_MSG_RX_STATUS, 976 "calling tpa_start on queue %d\n", 977 queue); 978 979 bnx2x_tpa_start(fp, queue, 980 bd_cons, bd_prod, 981 cqe_fp); 982 983 goto next_rx; 984 } 985 queue = cqe->end_agg_cqe.queue_index; 986 tpa_info = &fp->tpa_info[queue]; 987 DP(NETIF_MSG_RX_STATUS, 988 "calling tpa_stop on queue %d\n", 989 queue); 990 991 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) - 992 tpa_info->len_on_bd; 993 994 if (fp->mode == TPA_MODE_GRO) 995 pages = (frag_size + tpa_info->full_page - 1) / 996 tpa_info->full_page; 997 else 998 pages = SGE_PAGE_ALIGN(frag_size) >> 999 SGE_PAGE_SHIFT; 1000 1001 bnx2x_tpa_stop(bp, fp, tpa_info, pages, 1002 &cqe->end_agg_cqe, comp_ring_cons); 1003 #ifdef BNX2X_STOP_ON_ERROR 1004 if (bp->panic) 1005 return 0; 1006 #endif 1007 1008 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe); 1009 goto next_cqe; 1010 } 1011 /* non TPA */ 1012 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len); 1013 pad = cqe_fp->placement_offset; 1014 dma_sync_single_for_cpu(&bp->pdev->dev, 1015 dma_unmap_addr(rx_buf, mapping), 1016 pad + RX_COPY_THRESH, 1017 DMA_FROM_DEVICE); 1018 pad += NET_SKB_PAD; 1019 prefetch(data + pad); /* speedup eth_type_trans() */ 1020 /* is this an error packet? */ 1021 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) { 1022 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 1023 "ERROR flags %x rx packet %u\n", 1024 cqe_fp_flags, sw_comp_cons); 1025 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++; 1026 goto reuse_rx; 1027 } 1028 1029 /* Since we don't have a jumbo ring 1030 * copy small packets if mtu > 1500 1031 */ 1032 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) && 1033 (len <= RX_COPY_THRESH)) { 1034 skb = napi_alloc_skb(&fp->napi, len); 1035 if (skb == NULL) { 1036 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 1037 "ERROR packet dropped because of alloc failure\n"); 1038 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 1039 goto reuse_rx; 1040 } 1041 memcpy(skb->data, data + pad, len); 1042 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 1043 } else { 1044 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod, 1045 GFP_ATOMIC) == 0)) { 1046 dma_unmap_single(&bp->pdev->dev, 1047 dma_unmap_addr(rx_buf, mapping), 1048 fp->rx_buf_size, 1049 DMA_FROM_DEVICE); 1050 skb = build_skb(data, fp->rx_frag_size); 1051 if (unlikely(!skb)) { 1052 bnx2x_frag_free(fp, data); 1053 bnx2x_fp_qstats(bp, fp)-> 1054 rx_skb_alloc_failed++; 1055 goto next_rx; 1056 } 1057 skb_reserve(skb, pad); 1058 } else { 1059 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS, 1060 "ERROR packet dropped because of alloc failure\n"); 1061 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++; 1062 reuse_rx: 1063 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod); 1064 goto next_rx; 1065 } 1066 } 1067 1068 skb_put(skb, len); 1069 skb->protocol = eth_type_trans(skb, bp->dev); 1070 1071 /* Set Toeplitz hash for a none-LRO skb */ 1072 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type); 1073 skb_set_hash(skb, rxhash, rxhash_type); 1074 1075 skb_checksum_none_assert(skb); 1076 1077 if (bp->dev->features & NETIF_F_RXCSUM) 1078 bnx2x_csum_validate(skb, cqe, fp, 1079 bnx2x_fp_qstats(bp, fp)); 1080 1081 skb_record_rx_queue(skb, fp->rx_queue); 1082 1083 /* Check if this packet was timestamped */ 1084 if (unlikely(cqe->fast_path_cqe.type_error_flags & 1085 (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT))) 1086 bnx2x_set_rx_ts(bp, skb); 1087 1088 if (le16_to_cpu(cqe_fp->pars_flags.flags) & 1089 PARSING_FLAGS_VLAN) 1090 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 1091 le16_to_cpu(cqe_fp->vlan_tag)); 1092 1093 napi_gro_receive(&fp->napi, skb); 1094 next_rx: 1095 rx_buf->data = NULL; 1096 1097 bd_cons = NEXT_RX_IDX(bd_cons); 1098 bd_prod = NEXT_RX_IDX(bd_prod); 1099 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw); 1100 rx_pkt++; 1101 next_cqe: 1102 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod); 1103 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons); 1104 1105 /* mark CQE as free */ 1106 BNX2X_SEED_CQE(cqe_fp); 1107 1108 if (rx_pkt == budget) 1109 break; 1110 1111 comp_ring_cons = RCQ_BD(sw_comp_cons); 1112 cqe = &fp->rx_comp_ring[comp_ring_cons]; 1113 cqe_fp = &cqe->fast_path_cqe; 1114 } /* while */ 1115 1116 fp->rx_bd_cons = bd_cons; 1117 fp->rx_bd_prod = bd_prod_fw; 1118 fp->rx_comp_cons = sw_comp_cons; 1119 fp->rx_comp_prod = sw_comp_prod; 1120 1121 /* Update producers */ 1122 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod, 1123 fp->rx_sge_prod); 1124 1125 return rx_pkt; 1126 } 1127 1128 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie) 1129 { 1130 struct bnx2x_fastpath *fp = fp_cookie; 1131 struct bnx2x *bp = fp->bp; 1132 u8 cos; 1133 1134 DP(NETIF_MSG_INTR, 1135 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n", 1136 fp->index, fp->fw_sb_id, fp->igu_sb_id); 1137 1138 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0); 1139 1140 #ifdef BNX2X_STOP_ON_ERROR 1141 if (unlikely(bp->panic)) 1142 return IRQ_HANDLED; 1143 #endif 1144 1145 /* Handle Rx and Tx according to MSI-X vector */ 1146 for_each_cos_in_tx_queue(fp, cos) 1147 prefetch(fp->txdata_ptr[cos]->tx_cons_sb); 1148 1149 prefetch(&fp->sb_running_index[SM_RX_ID]); 1150 napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi)); 1151 1152 return IRQ_HANDLED; 1153 } 1154 1155 /* HW Lock for shared dual port PHYs */ 1156 void bnx2x_acquire_phy_lock(struct bnx2x *bp) 1157 { 1158 mutex_lock(&bp->port.phy_mutex); 1159 1160 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1161 } 1162 1163 void bnx2x_release_phy_lock(struct bnx2x *bp) 1164 { 1165 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 1166 1167 mutex_unlock(&bp->port.phy_mutex); 1168 } 1169 1170 /* calculates MF speed according to current linespeed and MF configuration */ 1171 u16 bnx2x_get_mf_speed(struct bnx2x *bp) 1172 { 1173 u16 line_speed = bp->link_vars.line_speed; 1174 if (IS_MF(bp)) { 1175 u16 maxCfg = bnx2x_extract_max_cfg(bp, 1176 bp->mf_config[BP_VN(bp)]); 1177 1178 /* Calculate the current MAX line speed limit for the MF 1179 * devices 1180 */ 1181 if (IS_MF_PERCENT_BW(bp)) 1182 line_speed = (line_speed * maxCfg) / 100; 1183 else { /* SD mode */ 1184 u16 vn_max_rate = maxCfg * 100; 1185 1186 if (vn_max_rate < line_speed) 1187 line_speed = vn_max_rate; 1188 } 1189 } 1190 1191 return line_speed; 1192 } 1193 1194 /** 1195 * bnx2x_fill_report_data - fill link report data to report 1196 * 1197 * @bp: driver handle 1198 * @data: link state to update 1199 * 1200 * It uses a none-atomic bit operations because is called under the mutex. 1201 */ 1202 static void bnx2x_fill_report_data(struct bnx2x *bp, 1203 struct bnx2x_link_report_data *data) 1204 { 1205 memset(data, 0, sizeof(*data)); 1206 1207 if (IS_PF(bp)) { 1208 /* Fill the report data: effective line speed */ 1209 data->line_speed = bnx2x_get_mf_speed(bp); 1210 1211 /* Link is down */ 1212 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS)) 1213 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1214 &data->link_report_flags); 1215 1216 if (!BNX2X_NUM_ETH_QUEUES(bp)) 1217 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1218 &data->link_report_flags); 1219 1220 /* Full DUPLEX */ 1221 if (bp->link_vars.duplex == DUPLEX_FULL) 1222 __set_bit(BNX2X_LINK_REPORT_FD, 1223 &data->link_report_flags); 1224 1225 /* Rx Flow Control is ON */ 1226 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) 1227 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, 1228 &data->link_report_flags); 1229 1230 /* Tx Flow Control is ON */ 1231 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX) 1232 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, 1233 &data->link_report_flags); 1234 } else { /* VF */ 1235 *data = bp->vf_link_vars; 1236 } 1237 } 1238 1239 /** 1240 * bnx2x_link_report - report link status to OS. 1241 * 1242 * @bp: driver handle 1243 * 1244 * Calls the __bnx2x_link_report() under the same locking scheme 1245 * as a link/PHY state managing code to ensure a consistent link 1246 * reporting. 1247 */ 1248 1249 void bnx2x_link_report(struct bnx2x *bp) 1250 { 1251 bnx2x_acquire_phy_lock(bp); 1252 __bnx2x_link_report(bp); 1253 bnx2x_release_phy_lock(bp); 1254 } 1255 1256 /** 1257 * __bnx2x_link_report - report link status to OS. 1258 * 1259 * @bp: driver handle 1260 * 1261 * None atomic implementation. 1262 * Should be called under the phy_lock. 1263 */ 1264 void __bnx2x_link_report(struct bnx2x *bp) 1265 { 1266 struct bnx2x_link_report_data cur_data; 1267 1268 /* reread mf_cfg */ 1269 if (IS_PF(bp) && !CHIP_IS_E1(bp)) 1270 bnx2x_read_mf_cfg(bp); 1271 1272 /* Read the current link report info */ 1273 bnx2x_fill_report_data(bp, &cur_data); 1274 1275 /* Don't report link down or exactly the same link status twice */ 1276 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) || 1277 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1278 &bp->last_reported_link.link_report_flags) && 1279 test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1280 &cur_data.link_report_flags))) 1281 return; 1282 1283 bp->link_cnt++; 1284 1285 /* We are going to report a new link parameters now - 1286 * remember the current data for the next time. 1287 */ 1288 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data)); 1289 1290 /* propagate status to VFs */ 1291 if (IS_PF(bp)) 1292 bnx2x_iov_link_update(bp); 1293 1294 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN, 1295 &cur_data.link_report_flags)) { 1296 netif_carrier_off(bp->dev); 1297 netdev_err(bp->dev, "NIC Link is Down\n"); 1298 return; 1299 } else { 1300 const char *duplex; 1301 const char *flow; 1302 1303 netif_carrier_on(bp->dev); 1304 1305 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD, 1306 &cur_data.link_report_flags)) 1307 duplex = "full"; 1308 else 1309 duplex = "half"; 1310 1311 /* Handle the FC at the end so that only these flags would be 1312 * possibly set. This way we may easily check if there is no FC 1313 * enabled. 1314 */ 1315 if (cur_data.link_report_flags) { 1316 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON, 1317 &cur_data.link_report_flags)) { 1318 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON, 1319 &cur_data.link_report_flags)) 1320 flow = "ON - receive & transmit"; 1321 else 1322 flow = "ON - receive"; 1323 } else { 1324 flow = "ON - transmit"; 1325 } 1326 } else { 1327 flow = "none"; 1328 } 1329 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n", 1330 cur_data.line_speed, duplex, flow); 1331 } 1332 } 1333 1334 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp) 1335 { 1336 int i; 1337 1338 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) { 1339 struct eth_rx_sge *sge; 1340 1341 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2]; 1342 sge->addr_hi = 1343 cpu_to_le32(U64_HI(fp->rx_sge_mapping + 1344 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1345 1346 sge->addr_lo = 1347 cpu_to_le32(U64_LO(fp->rx_sge_mapping + 1348 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 1349 } 1350 } 1351 1352 static void bnx2x_free_tpa_pool(struct bnx2x *bp, 1353 struct bnx2x_fastpath *fp, int last) 1354 { 1355 int i; 1356 1357 for (i = 0; i < last; i++) { 1358 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i]; 1359 struct sw_rx_bd *first_buf = &tpa_info->first_buf; 1360 u8 *data = first_buf->data; 1361 1362 if (data == NULL) { 1363 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i); 1364 continue; 1365 } 1366 if (tpa_info->tpa_state == BNX2X_TPA_START) 1367 dma_unmap_single(&bp->pdev->dev, 1368 dma_unmap_addr(first_buf, mapping), 1369 fp->rx_buf_size, DMA_FROM_DEVICE); 1370 bnx2x_frag_free(fp, data); 1371 first_buf->data = NULL; 1372 } 1373 } 1374 1375 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp) 1376 { 1377 int j; 1378 1379 for_each_rx_queue_cnic(bp, j) { 1380 struct bnx2x_fastpath *fp = &bp->fp[j]; 1381 1382 fp->rx_bd_cons = 0; 1383 1384 /* Activate BD ring */ 1385 /* Warning! 1386 * this will generate an interrupt (to the TSTORM) 1387 * must only be done after chip is initialized 1388 */ 1389 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1390 fp->rx_sge_prod); 1391 } 1392 } 1393 1394 void bnx2x_init_rx_rings(struct bnx2x *bp) 1395 { 1396 int func = BP_FUNC(bp); 1397 u16 ring_prod; 1398 int i, j; 1399 1400 /* Allocate TPA resources */ 1401 for_each_eth_queue(bp, j) { 1402 struct bnx2x_fastpath *fp = &bp->fp[j]; 1403 1404 DP(NETIF_MSG_IFUP, 1405 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size); 1406 1407 if (fp->mode != TPA_MODE_DISABLED) { 1408 /* Fill the per-aggregation pool */ 1409 for (i = 0; i < MAX_AGG_QS(bp); i++) { 1410 struct bnx2x_agg_info *tpa_info = 1411 &fp->tpa_info[i]; 1412 struct sw_rx_bd *first_buf = 1413 &tpa_info->first_buf; 1414 1415 first_buf->data = 1416 bnx2x_frag_alloc(fp, GFP_KERNEL); 1417 if (!first_buf->data) { 1418 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n", 1419 j); 1420 bnx2x_free_tpa_pool(bp, fp, i); 1421 fp->mode = TPA_MODE_DISABLED; 1422 break; 1423 } 1424 dma_unmap_addr_set(first_buf, mapping, 0); 1425 tpa_info->tpa_state = BNX2X_TPA_STOP; 1426 } 1427 1428 /* "next page" elements initialization */ 1429 bnx2x_set_next_page_sgl(fp); 1430 1431 /* set SGEs bit mask */ 1432 bnx2x_init_sge_ring_bit_mask(fp); 1433 1434 /* Allocate SGEs and initialize the ring elements */ 1435 for (i = 0, ring_prod = 0; 1436 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) { 1437 1438 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod, 1439 GFP_KERNEL) < 0) { 1440 BNX2X_ERR("was only able to allocate %d rx sges\n", 1441 i); 1442 BNX2X_ERR("disabling TPA for queue[%d]\n", 1443 j); 1444 /* Cleanup already allocated elements */ 1445 bnx2x_free_rx_sge_range(bp, fp, 1446 ring_prod); 1447 bnx2x_free_tpa_pool(bp, fp, 1448 MAX_AGG_QS(bp)); 1449 fp->mode = TPA_MODE_DISABLED; 1450 ring_prod = 0; 1451 break; 1452 } 1453 ring_prod = NEXT_SGE_IDX(ring_prod); 1454 } 1455 1456 fp->rx_sge_prod = ring_prod; 1457 } 1458 } 1459 1460 for_each_eth_queue(bp, j) { 1461 struct bnx2x_fastpath *fp = &bp->fp[j]; 1462 1463 fp->rx_bd_cons = 0; 1464 1465 /* Activate BD ring */ 1466 /* Warning! 1467 * this will generate an interrupt (to the TSTORM) 1468 * must only be done after chip is initialized 1469 */ 1470 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod, 1471 fp->rx_sge_prod); 1472 1473 if (j != 0) 1474 continue; 1475 1476 if (CHIP_IS_E1(bp)) { 1477 REG_WR(bp, BAR_USTRORM_INTMEM + 1478 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func), 1479 U64_LO(fp->rx_comp_mapping)); 1480 REG_WR(bp, BAR_USTRORM_INTMEM + 1481 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4, 1482 U64_HI(fp->rx_comp_mapping)); 1483 } 1484 } 1485 } 1486 1487 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp) 1488 { 1489 u8 cos; 1490 struct bnx2x *bp = fp->bp; 1491 1492 for_each_cos_in_tx_queue(fp, cos) { 1493 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 1494 unsigned pkts_compl = 0, bytes_compl = 0; 1495 1496 u16 sw_prod = txdata->tx_pkt_prod; 1497 u16 sw_cons = txdata->tx_pkt_cons; 1498 1499 while (sw_cons != sw_prod) { 1500 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons), 1501 &pkts_compl, &bytes_compl); 1502 sw_cons++; 1503 } 1504 1505 netdev_tx_reset_queue( 1506 netdev_get_tx_queue(bp->dev, 1507 txdata->txq_index)); 1508 } 1509 } 1510 1511 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp) 1512 { 1513 int i; 1514 1515 for_each_tx_queue_cnic(bp, i) { 1516 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1517 } 1518 } 1519 1520 static void bnx2x_free_tx_skbs(struct bnx2x *bp) 1521 { 1522 int i; 1523 1524 for_each_eth_queue(bp, i) { 1525 bnx2x_free_tx_skbs_queue(&bp->fp[i]); 1526 } 1527 } 1528 1529 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp) 1530 { 1531 struct bnx2x *bp = fp->bp; 1532 int i; 1533 1534 /* ring wasn't allocated */ 1535 if (fp->rx_buf_ring == NULL) 1536 return; 1537 1538 for (i = 0; i < NUM_RX_BD; i++) { 1539 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i]; 1540 u8 *data = rx_buf->data; 1541 1542 if (data == NULL) 1543 continue; 1544 dma_unmap_single(&bp->pdev->dev, 1545 dma_unmap_addr(rx_buf, mapping), 1546 fp->rx_buf_size, DMA_FROM_DEVICE); 1547 1548 rx_buf->data = NULL; 1549 bnx2x_frag_free(fp, data); 1550 } 1551 } 1552 1553 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp) 1554 { 1555 int j; 1556 1557 for_each_rx_queue_cnic(bp, j) { 1558 bnx2x_free_rx_bds(&bp->fp[j]); 1559 } 1560 } 1561 1562 static void bnx2x_free_rx_skbs(struct bnx2x *bp) 1563 { 1564 int j; 1565 1566 for_each_eth_queue(bp, j) { 1567 struct bnx2x_fastpath *fp = &bp->fp[j]; 1568 1569 bnx2x_free_rx_bds(fp); 1570 1571 if (fp->mode != TPA_MODE_DISABLED) 1572 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp)); 1573 } 1574 } 1575 1576 static void bnx2x_free_skbs_cnic(struct bnx2x *bp) 1577 { 1578 bnx2x_free_tx_skbs_cnic(bp); 1579 bnx2x_free_rx_skbs_cnic(bp); 1580 } 1581 1582 void bnx2x_free_skbs(struct bnx2x *bp) 1583 { 1584 bnx2x_free_tx_skbs(bp); 1585 bnx2x_free_rx_skbs(bp); 1586 } 1587 1588 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value) 1589 { 1590 /* load old values */ 1591 u32 mf_cfg = bp->mf_config[BP_VN(bp)]; 1592 1593 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) { 1594 /* leave all but MAX value */ 1595 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK; 1596 1597 /* set new MAX value */ 1598 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT) 1599 & FUNC_MF_CFG_MAX_BW_MASK; 1600 1601 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg); 1602 } 1603 } 1604 1605 /** 1606 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors 1607 * 1608 * @bp: driver handle 1609 * @nvecs: number of vectors to be released 1610 */ 1611 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs) 1612 { 1613 int i, offset = 0; 1614 1615 if (nvecs == offset) 1616 return; 1617 1618 /* VFs don't have a default SB */ 1619 if (IS_PF(bp)) { 1620 free_irq(bp->msix_table[offset].vector, bp->dev); 1621 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n", 1622 bp->msix_table[offset].vector); 1623 offset++; 1624 } 1625 1626 if (CNIC_SUPPORT(bp)) { 1627 if (nvecs == offset) 1628 return; 1629 offset++; 1630 } 1631 1632 for_each_eth_queue(bp, i) { 1633 if (nvecs == offset) 1634 return; 1635 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n", 1636 i, bp->msix_table[offset].vector); 1637 1638 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]); 1639 } 1640 } 1641 1642 void bnx2x_free_irq(struct bnx2x *bp) 1643 { 1644 if (bp->flags & USING_MSIX_FLAG && 1645 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1646 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp); 1647 1648 /* vfs don't have a default status block */ 1649 if (IS_PF(bp)) 1650 nvecs++; 1651 1652 bnx2x_free_msix_irqs(bp, nvecs); 1653 } else { 1654 free_irq(bp->dev->irq, bp->dev); 1655 } 1656 } 1657 1658 int bnx2x_enable_msix(struct bnx2x *bp) 1659 { 1660 int msix_vec = 0, i, rc; 1661 1662 /* VFs don't have a default status block */ 1663 if (IS_PF(bp)) { 1664 bp->msix_table[msix_vec].entry = msix_vec; 1665 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n", 1666 bp->msix_table[0].entry); 1667 msix_vec++; 1668 } 1669 1670 /* Cnic requires an msix vector for itself */ 1671 if (CNIC_SUPPORT(bp)) { 1672 bp->msix_table[msix_vec].entry = msix_vec; 1673 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n", 1674 msix_vec, bp->msix_table[msix_vec].entry); 1675 msix_vec++; 1676 } 1677 1678 /* We need separate vectors for ETH queues only (not FCoE) */ 1679 for_each_eth_queue(bp, i) { 1680 bp->msix_table[msix_vec].entry = msix_vec; 1681 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n", 1682 msix_vec, msix_vec, i); 1683 msix_vec++; 1684 } 1685 1686 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n", 1687 msix_vec); 1688 1689 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1690 BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec); 1691 /* 1692 * reconfigure number of tx/rx queues according to available 1693 * MSI-X vectors 1694 */ 1695 if (rc == -ENOSPC) { 1696 /* Get by with single vector */ 1697 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1); 1698 if (rc < 0) { 1699 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n", 1700 rc); 1701 goto no_msix; 1702 } 1703 1704 BNX2X_DEV_INFO("Using single MSI-X vector\n"); 1705 bp->flags |= USING_SINGLE_MSIX_FLAG; 1706 1707 BNX2X_DEV_INFO("set number of queues to 1\n"); 1708 bp->num_ethernet_queues = 1; 1709 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1710 } else if (rc < 0) { 1711 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc); 1712 goto no_msix; 1713 } else if (rc < msix_vec) { 1714 /* how less vectors we will have? */ 1715 int diff = msix_vec - rc; 1716 1717 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc); 1718 1719 /* 1720 * decrease number of queues by number of unallocated entries 1721 */ 1722 bp->num_ethernet_queues -= diff; 1723 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1724 1725 BNX2X_DEV_INFO("New queue configuration set: %d\n", 1726 bp->num_queues); 1727 } 1728 1729 bp->flags |= USING_MSIX_FLAG; 1730 1731 return 0; 1732 1733 no_msix: 1734 /* fall to INTx if not enough memory */ 1735 if (rc == -ENOMEM) 1736 bp->flags |= DISABLE_MSI_FLAG; 1737 1738 return rc; 1739 } 1740 1741 static int bnx2x_req_msix_irqs(struct bnx2x *bp) 1742 { 1743 int i, rc, offset = 0; 1744 1745 /* no default status block for vf */ 1746 if (IS_PF(bp)) { 1747 rc = request_irq(bp->msix_table[offset++].vector, 1748 bnx2x_msix_sp_int, 0, 1749 bp->dev->name, bp->dev); 1750 if (rc) { 1751 BNX2X_ERR("request sp irq failed\n"); 1752 return -EBUSY; 1753 } 1754 } 1755 1756 if (CNIC_SUPPORT(bp)) 1757 offset++; 1758 1759 for_each_eth_queue(bp, i) { 1760 struct bnx2x_fastpath *fp = &bp->fp[i]; 1761 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", 1762 bp->dev->name, i); 1763 1764 rc = request_irq(bp->msix_table[offset].vector, 1765 bnx2x_msix_fp_int, 0, fp->name, fp); 1766 if (rc) { 1767 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i, 1768 bp->msix_table[offset].vector, rc); 1769 bnx2x_free_msix_irqs(bp, offset); 1770 return -EBUSY; 1771 } 1772 1773 offset++; 1774 } 1775 1776 i = BNX2X_NUM_ETH_QUEUES(bp); 1777 if (IS_PF(bp)) { 1778 offset = 1 + CNIC_SUPPORT(bp); 1779 netdev_info(bp->dev, 1780 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n", 1781 bp->msix_table[0].vector, 1782 0, bp->msix_table[offset].vector, 1783 i - 1, bp->msix_table[offset + i - 1].vector); 1784 } else { 1785 offset = CNIC_SUPPORT(bp); 1786 netdev_info(bp->dev, 1787 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n", 1788 0, bp->msix_table[offset].vector, 1789 i - 1, bp->msix_table[offset + i - 1].vector); 1790 } 1791 return 0; 1792 } 1793 1794 int bnx2x_enable_msi(struct bnx2x *bp) 1795 { 1796 int rc; 1797 1798 rc = pci_enable_msi(bp->pdev); 1799 if (rc) { 1800 BNX2X_DEV_INFO("MSI is not attainable\n"); 1801 return -1; 1802 } 1803 bp->flags |= USING_MSI_FLAG; 1804 1805 return 0; 1806 } 1807 1808 static int bnx2x_req_irq(struct bnx2x *bp) 1809 { 1810 unsigned long flags; 1811 unsigned int irq; 1812 1813 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG)) 1814 flags = 0; 1815 else 1816 flags = IRQF_SHARED; 1817 1818 if (bp->flags & USING_MSIX_FLAG) 1819 irq = bp->msix_table[0].vector; 1820 else 1821 irq = bp->pdev->irq; 1822 1823 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev); 1824 } 1825 1826 static int bnx2x_setup_irqs(struct bnx2x *bp) 1827 { 1828 int rc = 0; 1829 if (bp->flags & USING_MSIX_FLAG && 1830 !(bp->flags & USING_SINGLE_MSIX_FLAG)) { 1831 rc = bnx2x_req_msix_irqs(bp); 1832 if (rc) 1833 return rc; 1834 } else { 1835 rc = bnx2x_req_irq(bp); 1836 if (rc) { 1837 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc); 1838 return rc; 1839 } 1840 if (bp->flags & USING_MSI_FLAG) { 1841 bp->dev->irq = bp->pdev->irq; 1842 netdev_info(bp->dev, "using MSI IRQ %d\n", 1843 bp->dev->irq); 1844 } 1845 if (bp->flags & USING_MSIX_FLAG) { 1846 bp->dev->irq = bp->msix_table[0].vector; 1847 netdev_info(bp->dev, "using MSIX IRQ %d\n", 1848 bp->dev->irq); 1849 } 1850 } 1851 1852 return 0; 1853 } 1854 1855 static void bnx2x_napi_enable_cnic(struct bnx2x *bp) 1856 { 1857 int i; 1858 1859 for_each_rx_queue_cnic(bp, i) { 1860 napi_enable(&bnx2x_fp(bp, i, napi)); 1861 } 1862 } 1863 1864 static void bnx2x_napi_enable(struct bnx2x *bp) 1865 { 1866 int i; 1867 1868 for_each_eth_queue(bp, i) { 1869 napi_enable(&bnx2x_fp(bp, i, napi)); 1870 } 1871 } 1872 1873 static void bnx2x_napi_disable_cnic(struct bnx2x *bp) 1874 { 1875 int i; 1876 1877 for_each_rx_queue_cnic(bp, i) { 1878 napi_disable(&bnx2x_fp(bp, i, napi)); 1879 } 1880 } 1881 1882 static void bnx2x_napi_disable(struct bnx2x *bp) 1883 { 1884 int i; 1885 1886 for_each_eth_queue(bp, i) { 1887 napi_disable(&bnx2x_fp(bp, i, napi)); 1888 } 1889 } 1890 1891 void bnx2x_netif_start(struct bnx2x *bp) 1892 { 1893 if (netif_running(bp->dev)) { 1894 bnx2x_napi_enable(bp); 1895 if (CNIC_LOADED(bp)) 1896 bnx2x_napi_enable_cnic(bp); 1897 bnx2x_int_enable(bp); 1898 if (bp->state == BNX2X_STATE_OPEN) 1899 netif_tx_wake_all_queues(bp->dev); 1900 } 1901 } 1902 1903 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw) 1904 { 1905 bnx2x_int_disable_sync(bp, disable_hw); 1906 bnx2x_napi_disable(bp); 1907 if (CNIC_LOADED(bp)) 1908 bnx2x_napi_disable_cnic(bp); 1909 } 1910 1911 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb, 1912 void *accel_priv, select_queue_fallback_t fallback) 1913 { 1914 struct bnx2x *bp = netdev_priv(dev); 1915 1916 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) { 1917 struct ethhdr *hdr = (struct ethhdr *)skb->data; 1918 u16 ether_type = ntohs(hdr->h_proto); 1919 1920 /* Skip VLAN tag if present */ 1921 if (ether_type == ETH_P_8021Q) { 1922 struct vlan_ethhdr *vhdr = 1923 (struct vlan_ethhdr *)skb->data; 1924 1925 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto); 1926 } 1927 1928 /* If ethertype is FCoE or FIP - use FCoE ring */ 1929 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP)) 1930 return bnx2x_fcoe_tx(bp, txq_index); 1931 } 1932 1933 /* select a non-FCoE queue */ 1934 return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp); 1935 } 1936 1937 void bnx2x_set_num_queues(struct bnx2x *bp) 1938 { 1939 /* RSS queues */ 1940 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp); 1941 1942 /* override in STORAGE SD modes */ 1943 if (IS_MF_STORAGE_ONLY(bp)) 1944 bp->num_ethernet_queues = 1; 1945 1946 /* Add special queues */ 1947 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */ 1948 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues; 1949 1950 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues); 1951 } 1952 1953 /** 1954 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues 1955 * 1956 * @bp: Driver handle 1957 * 1958 * We currently support for at most 16 Tx queues for each CoS thus we will 1959 * allocate a multiple of 16 for ETH L2 rings according to the value of the 1960 * bp->max_cos. 1961 * 1962 * If there is an FCoE L2 queue the appropriate Tx queue will have the next 1963 * index after all ETH L2 indices. 1964 * 1965 * If the actual number of Tx queues (for each CoS) is less than 16 then there 1966 * will be the holes at the end of each group of 16 ETh L2 indices (0..15, 1967 * 16..31,...) with indices that are not coupled with any real Tx queue. 1968 * 1969 * The proper configuration of skb->queue_mapping is handled by 1970 * bnx2x_select_queue() and __skb_tx_hash(). 1971 * 1972 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash() 1973 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0). 1974 */ 1975 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic) 1976 { 1977 int rc, tx, rx; 1978 1979 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos; 1980 rx = BNX2X_NUM_ETH_QUEUES(bp); 1981 1982 /* account for fcoe queue */ 1983 if (include_cnic && !NO_FCOE(bp)) { 1984 rx++; 1985 tx++; 1986 } 1987 1988 rc = netif_set_real_num_tx_queues(bp->dev, tx); 1989 if (rc) { 1990 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc); 1991 return rc; 1992 } 1993 rc = netif_set_real_num_rx_queues(bp->dev, rx); 1994 if (rc) { 1995 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc); 1996 return rc; 1997 } 1998 1999 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n", 2000 tx, rx); 2001 2002 return rc; 2003 } 2004 2005 static void bnx2x_set_rx_buf_size(struct bnx2x *bp) 2006 { 2007 int i; 2008 2009 for_each_queue(bp, i) { 2010 struct bnx2x_fastpath *fp = &bp->fp[i]; 2011 u32 mtu; 2012 2013 /* Always use a mini-jumbo MTU for the FCoE L2 ring */ 2014 if (IS_FCOE_IDX(i)) 2015 /* 2016 * Although there are no IP frames expected to arrive to 2017 * this ring we still want to add an 2018 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer 2019 * overrun attack. 2020 */ 2021 mtu = BNX2X_FCOE_MINI_JUMBO_MTU; 2022 else 2023 mtu = bp->dev->mtu; 2024 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START + 2025 IP_HEADER_ALIGNMENT_PADDING + 2026 ETH_OVREHEAD + 2027 mtu + 2028 BNX2X_FW_RX_ALIGN_END; 2029 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */ 2030 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE) 2031 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD; 2032 else 2033 fp->rx_frag_size = 0; 2034 } 2035 } 2036 2037 static int bnx2x_init_rss(struct bnx2x *bp) 2038 { 2039 int i; 2040 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp); 2041 2042 /* Prepare the initial contents for the indirection table if RSS is 2043 * enabled 2044 */ 2045 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++) 2046 bp->rss_conf_obj.ind_table[i] = 2047 bp->fp->cl_id + 2048 ethtool_rxfh_indir_default(i, num_eth_queues); 2049 2050 /* 2051 * For 57710 and 57711 SEARCHER configuration (rss_keys) is 2052 * per-port, so if explicit configuration is needed , do it only 2053 * for a PMF. 2054 * 2055 * For 57712 and newer on the other hand it's a per-function 2056 * configuration. 2057 */ 2058 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp)); 2059 } 2060 2061 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj, 2062 bool config_hash, bool enable) 2063 { 2064 struct bnx2x_config_rss_params params = {NULL}; 2065 2066 /* Although RSS is meaningless when there is a single HW queue we 2067 * still need it enabled in order to have HW Rx hash generated. 2068 * 2069 * if (!is_eth_multi(bp)) 2070 * bp->multi_mode = ETH_RSS_MODE_DISABLED; 2071 */ 2072 2073 params.rss_obj = rss_obj; 2074 2075 __set_bit(RAMROD_COMP_WAIT, ¶ms.ramrod_flags); 2076 2077 if (enable) { 2078 __set_bit(BNX2X_RSS_MODE_REGULAR, ¶ms.rss_flags); 2079 2080 /* RSS configuration */ 2081 __set_bit(BNX2X_RSS_IPV4, ¶ms.rss_flags); 2082 __set_bit(BNX2X_RSS_IPV4_TCP, ¶ms.rss_flags); 2083 __set_bit(BNX2X_RSS_IPV6, ¶ms.rss_flags); 2084 __set_bit(BNX2X_RSS_IPV6_TCP, ¶ms.rss_flags); 2085 if (rss_obj->udp_rss_v4) 2086 __set_bit(BNX2X_RSS_IPV4_UDP, ¶ms.rss_flags); 2087 if (rss_obj->udp_rss_v6) 2088 __set_bit(BNX2X_RSS_IPV6_UDP, ¶ms.rss_flags); 2089 2090 if (!CHIP_IS_E1x(bp)) { 2091 /* valid only for TUNN_MODE_VXLAN tunnel mode */ 2092 __set_bit(BNX2X_RSS_IPV4_VXLAN, ¶ms.rss_flags); 2093 __set_bit(BNX2X_RSS_IPV6_VXLAN, ¶ms.rss_flags); 2094 2095 /* valid only for TUNN_MODE_GRE tunnel mode */ 2096 __set_bit(BNX2X_RSS_TUNN_INNER_HDRS, ¶ms.rss_flags); 2097 } 2098 } else { 2099 __set_bit(BNX2X_RSS_MODE_DISABLED, ¶ms.rss_flags); 2100 } 2101 2102 /* Hash bits */ 2103 params.rss_result_mask = MULTI_MASK; 2104 2105 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table)); 2106 2107 if (config_hash) { 2108 /* RSS keys */ 2109 netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4); 2110 __set_bit(BNX2X_RSS_SET_SRCH, ¶ms.rss_flags); 2111 } 2112 2113 if (IS_PF(bp)) 2114 return bnx2x_config_rss(bp, ¶ms); 2115 else 2116 return bnx2x_vfpf_config_rss(bp, ¶ms); 2117 } 2118 2119 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code) 2120 { 2121 struct bnx2x_func_state_params func_params = {NULL}; 2122 2123 /* Prepare parameters for function state transitions */ 2124 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags); 2125 2126 func_params.f_obj = &bp->func_obj; 2127 func_params.cmd = BNX2X_F_CMD_HW_INIT; 2128 2129 func_params.params.hw_init.load_phase = load_code; 2130 2131 return bnx2x_func_state_change(bp, &func_params); 2132 } 2133 2134 /* 2135 * Cleans the object that have internal lists without sending 2136 * ramrods. Should be run when interrupts are disabled. 2137 */ 2138 void bnx2x_squeeze_objects(struct bnx2x *bp) 2139 { 2140 int rc; 2141 unsigned long ramrod_flags = 0, vlan_mac_flags = 0; 2142 struct bnx2x_mcast_ramrod_params rparam = {NULL}; 2143 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj; 2144 2145 /***************** Cleanup MACs' object first *************************/ 2146 2147 /* Wait for completion of requested */ 2148 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags); 2149 /* Perform a dry cleanup */ 2150 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags); 2151 2152 /* Clean ETH primary MAC */ 2153 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags); 2154 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags, 2155 &ramrod_flags); 2156 if (rc != 0) 2157 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc); 2158 2159 /* Cleanup UC list */ 2160 vlan_mac_flags = 0; 2161 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags); 2162 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags, 2163 &ramrod_flags); 2164 if (rc != 0) 2165 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc); 2166 2167 /***************** Now clean mcast object *****************************/ 2168 rparam.mcast_obj = &bp->mcast_obj; 2169 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags); 2170 2171 /* Add a DEL command... - Since we're doing a driver cleanup only, 2172 * we take a lock surrounding both the initial send and the CONTs, 2173 * as we don't want a true completion to disrupt us in the middle. 2174 */ 2175 netif_addr_lock_bh(bp->dev); 2176 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL); 2177 if (rc < 0) 2178 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n", 2179 rc); 2180 2181 /* ...and wait until all pending commands are cleared */ 2182 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2183 while (rc != 0) { 2184 if (rc < 0) { 2185 BNX2X_ERR("Failed to clean multi-cast object: %d\n", 2186 rc); 2187 netif_addr_unlock_bh(bp->dev); 2188 return; 2189 } 2190 2191 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT); 2192 } 2193 netif_addr_unlock_bh(bp->dev); 2194 } 2195 2196 #ifndef BNX2X_STOP_ON_ERROR 2197 #define LOAD_ERROR_EXIT(bp, label) \ 2198 do { \ 2199 (bp)->state = BNX2X_STATE_ERROR; \ 2200 goto label; \ 2201 } while (0) 2202 2203 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2204 do { \ 2205 bp->cnic_loaded = false; \ 2206 goto label; \ 2207 } while (0) 2208 #else /*BNX2X_STOP_ON_ERROR*/ 2209 #define LOAD_ERROR_EXIT(bp, label) \ 2210 do { \ 2211 (bp)->state = BNX2X_STATE_ERROR; \ 2212 (bp)->panic = 1; \ 2213 return -EBUSY; \ 2214 } while (0) 2215 #define LOAD_ERROR_EXIT_CNIC(bp, label) \ 2216 do { \ 2217 bp->cnic_loaded = false; \ 2218 (bp)->panic = 1; \ 2219 return -EBUSY; \ 2220 } while (0) 2221 #endif /*BNX2X_STOP_ON_ERROR*/ 2222 2223 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp) 2224 { 2225 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping, 2226 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2227 return; 2228 } 2229 2230 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp) 2231 { 2232 int num_groups, vf_headroom = 0; 2233 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1; 2234 2235 /* number of queues for statistics is number of eth queues + FCoE */ 2236 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats; 2237 2238 /* Total number of FW statistics requests = 2239 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper 2240 * and fcoe l2 queue) stats + num of queues (which includes another 1 2241 * for fcoe l2 queue if applicable) 2242 */ 2243 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats; 2244 2245 /* vf stats appear in the request list, but their data is allocated by 2246 * the VFs themselves. We don't include them in the bp->fw_stats_num as 2247 * it is used to determine where to place the vf stats queries in the 2248 * request struct 2249 */ 2250 if (IS_SRIOV(bp)) 2251 vf_headroom = bnx2x_vf_headroom(bp); 2252 2253 /* Request is built from stats_query_header and an array of 2254 * stats_query_cmd_group each of which contains 2255 * STATS_QUERY_CMD_COUNT rules. The real number or requests is 2256 * configured in the stats_query_header. 2257 */ 2258 num_groups = 2259 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) + 2260 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ? 2261 1 : 0)); 2262 2263 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n", 2264 bp->fw_stats_num, vf_headroom, num_groups); 2265 bp->fw_stats_req_sz = sizeof(struct stats_query_header) + 2266 num_groups * sizeof(struct stats_query_cmd_group); 2267 2268 /* Data for statistics requests + stats_counter 2269 * stats_counter holds per-STORM counters that are incremented 2270 * when STORM has finished with the current request. 2271 * memory for FCoE offloaded statistics are counted anyway, 2272 * even if they will not be sent. 2273 * VF stats are not accounted for here as the data of VF stats is stored 2274 * in memory allocated by the VF, not here. 2275 */ 2276 bp->fw_stats_data_sz = sizeof(struct per_port_stats) + 2277 sizeof(struct per_pf_stats) + 2278 sizeof(struct fcoe_statistics_params) + 2279 sizeof(struct per_queue_stats) * num_queue_stats + 2280 sizeof(struct stats_counter); 2281 2282 bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping, 2283 bp->fw_stats_data_sz + bp->fw_stats_req_sz); 2284 if (!bp->fw_stats) 2285 goto alloc_mem_err; 2286 2287 /* Set shortcuts */ 2288 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats; 2289 bp->fw_stats_req_mapping = bp->fw_stats_mapping; 2290 bp->fw_stats_data = (struct bnx2x_fw_stats_data *) 2291 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz); 2292 bp->fw_stats_data_mapping = bp->fw_stats_mapping + 2293 bp->fw_stats_req_sz; 2294 2295 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n", 2296 U64_HI(bp->fw_stats_req_mapping), 2297 U64_LO(bp->fw_stats_req_mapping)); 2298 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n", 2299 U64_HI(bp->fw_stats_data_mapping), 2300 U64_LO(bp->fw_stats_data_mapping)); 2301 return 0; 2302 2303 alloc_mem_err: 2304 bnx2x_free_fw_stats_mem(bp); 2305 BNX2X_ERR("Can't allocate FW stats memory\n"); 2306 return -ENOMEM; 2307 } 2308 2309 /* send load request to mcp and analyze response */ 2310 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code) 2311 { 2312 u32 param; 2313 2314 /* init fw_seq */ 2315 bp->fw_seq = 2316 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) & 2317 DRV_MSG_SEQ_NUMBER_MASK); 2318 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq); 2319 2320 /* Get current FW pulse sequence */ 2321 bp->fw_drv_pulse_wr_seq = 2322 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) & 2323 DRV_PULSE_SEQ_MASK); 2324 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq); 2325 2326 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA; 2327 2328 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp)) 2329 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA; 2330 2331 /* load request */ 2332 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param); 2333 2334 /* if mcp fails to respond we must abort */ 2335 if (!(*load_code)) { 2336 BNX2X_ERR("MCP response failure, aborting\n"); 2337 return -EBUSY; 2338 } 2339 2340 /* If mcp refused (e.g. other port is in diagnostic mode) we 2341 * must abort 2342 */ 2343 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) { 2344 BNX2X_ERR("MCP refused load request, aborting\n"); 2345 return -EBUSY; 2346 } 2347 return 0; 2348 } 2349 2350 /* check whether another PF has already loaded FW to chip. In 2351 * virtualized environments a pf from another VM may have already 2352 * initialized the device including loading FW 2353 */ 2354 int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err) 2355 { 2356 /* is another pf loaded on this engine? */ 2357 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP && 2358 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) { 2359 /* build my FW version dword */ 2360 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) + 2361 (BCM_5710_FW_MINOR_VERSION << 8) + 2362 (BCM_5710_FW_REVISION_VERSION << 16) + 2363 (BCM_5710_FW_ENGINEERING_VERSION << 24); 2364 2365 /* read loaded FW from chip */ 2366 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM); 2367 2368 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n", 2369 loaded_fw, my_fw); 2370 2371 /* abort nic load if version mismatch */ 2372 if (my_fw != loaded_fw) { 2373 if (print_err) 2374 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n", 2375 loaded_fw, my_fw); 2376 else 2377 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n", 2378 loaded_fw, my_fw); 2379 return -EBUSY; 2380 } 2381 } 2382 return 0; 2383 } 2384 2385 /* returns the "mcp load_code" according to global load_count array */ 2386 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port) 2387 { 2388 int path = BP_PATH(bp); 2389 2390 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n", 2391 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1], 2392 bnx2x_load_count[path][2]); 2393 bnx2x_load_count[path][0]++; 2394 bnx2x_load_count[path][1 + port]++; 2395 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n", 2396 path, bnx2x_load_count[path][0], bnx2x_load_count[path][1], 2397 bnx2x_load_count[path][2]); 2398 if (bnx2x_load_count[path][0] == 1) 2399 return FW_MSG_CODE_DRV_LOAD_COMMON; 2400 else if (bnx2x_load_count[path][1 + port] == 1) 2401 return FW_MSG_CODE_DRV_LOAD_PORT; 2402 else 2403 return FW_MSG_CODE_DRV_LOAD_FUNCTION; 2404 } 2405 2406 /* mark PMF if applicable */ 2407 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code) 2408 { 2409 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2410 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) || 2411 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) { 2412 bp->port.pmf = 1; 2413 /* We need the barrier to ensure the ordering between the 2414 * writing to bp->port.pmf here and reading it from the 2415 * bnx2x_periodic_task(). 2416 */ 2417 smp_mb(); 2418 } else { 2419 bp->port.pmf = 0; 2420 } 2421 2422 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf); 2423 } 2424 2425 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code) 2426 { 2427 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 2428 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) && 2429 (bp->common.shmem2_base)) { 2430 if (SHMEM2_HAS(bp, dcc_support)) 2431 SHMEM2_WR(bp, dcc_support, 2432 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV | 2433 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV)); 2434 if (SHMEM2_HAS(bp, afex_driver_support)) 2435 SHMEM2_WR(bp, afex_driver_support, 2436 SHMEM_AFEX_SUPPORTED_VERSION_ONE); 2437 } 2438 2439 /* Set AFEX default VLAN tag to an invalid value */ 2440 bp->afex_def_vlan_tag = -1; 2441 } 2442 2443 /** 2444 * bnx2x_bz_fp - zero content of the fastpath structure. 2445 * 2446 * @bp: driver handle 2447 * @index: fastpath index to be zeroed 2448 * 2449 * Makes sure the contents of the bp->fp[index].napi is kept 2450 * intact. 2451 */ 2452 static void bnx2x_bz_fp(struct bnx2x *bp, int index) 2453 { 2454 struct bnx2x_fastpath *fp = &bp->fp[index]; 2455 int cos; 2456 struct napi_struct orig_napi = fp->napi; 2457 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info; 2458 2459 /* bzero bnx2x_fastpath contents */ 2460 if (fp->tpa_info) 2461 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 * 2462 sizeof(struct bnx2x_agg_info)); 2463 memset(fp, 0, sizeof(*fp)); 2464 2465 /* Restore the NAPI object as it has been already initialized */ 2466 fp->napi = orig_napi; 2467 fp->tpa_info = orig_tpa_info; 2468 fp->bp = bp; 2469 fp->index = index; 2470 if (IS_ETH_FP(fp)) 2471 fp->max_cos = bp->max_cos; 2472 else 2473 /* Special queues support only one CoS */ 2474 fp->max_cos = 1; 2475 2476 /* Init txdata pointers */ 2477 if (IS_FCOE_FP(fp)) 2478 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)]; 2479 if (IS_ETH_FP(fp)) 2480 for_each_cos_in_tx_queue(fp, cos) 2481 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos * 2482 BNX2X_NUM_ETH_QUEUES(bp) + index]; 2483 2484 /* set the tpa flag for each queue. The tpa flag determines the queue 2485 * minimal size so it must be set prior to queue memory allocation 2486 */ 2487 if (bp->dev->features & NETIF_F_LRO) 2488 fp->mode = TPA_MODE_LRO; 2489 else if (bp->dev->features & NETIF_F_GRO && 2490 bnx2x_mtu_allows_gro(bp->dev->mtu)) 2491 fp->mode = TPA_MODE_GRO; 2492 else 2493 fp->mode = TPA_MODE_DISABLED; 2494 2495 /* We don't want TPA if it's disabled in bp 2496 * or if this is an FCoE L2 ring. 2497 */ 2498 if (bp->disable_tpa || IS_FCOE_FP(fp)) 2499 fp->mode = TPA_MODE_DISABLED; 2500 } 2501 2502 void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state) 2503 { 2504 u32 cur; 2505 2506 if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp)) 2507 return; 2508 2509 cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]); 2510 DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n", 2511 cur, state); 2512 2513 SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state); 2514 } 2515 2516 int bnx2x_load_cnic(struct bnx2x *bp) 2517 { 2518 int i, rc, port = BP_PORT(bp); 2519 2520 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n"); 2521 2522 mutex_init(&bp->cnic_mutex); 2523 2524 if (IS_PF(bp)) { 2525 rc = bnx2x_alloc_mem_cnic(bp); 2526 if (rc) { 2527 BNX2X_ERR("Unable to allocate bp memory for cnic\n"); 2528 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2529 } 2530 } 2531 2532 rc = bnx2x_alloc_fp_mem_cnic(bp); 2533 if (rc) { 2534 BNX2X_ERR("Unable to allocate memory for cnic fps\n"); 2535 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2536 } 2537 2538 /* Update the number of queues with the cnic queues */ 2539 rc = bnx2x_set_real_num_queues(bp, 1); 2540 if (rc) { 2541 BNX2X_ERR("Unable to set real_num_queues including cnic\n"); 2542 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0); 2543 } 2544 2545 /* Add all CNIC NAPI objects */ 2546 bnx2x_add_all_napi_cnic(bp); 2547 DP(NETIF_MSG_IFUP, "cnic napi added\n"); 2548 bnx2x_napi_enable_cnic(bp); 2549 2550 rc = bnx2x_init_hw_func_cnic(bp); 2551 if (rc) 2552 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1); 2553 2554 bnx2x_nic_init_cnic(bp); 2555 2556 if (IS_PF(bp)) { 2557 /* Enable Timer scan */ 2558 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1); 2559 2560 /* setup cnic queues */ 2561 for_each_cnic_queue(bp, i) { 2562 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0); 2563 if (rc) { 2564 BNX2X_ERR("Queue setup failed\n"); 2565 LOAD_ERROR_EXIT(bp, load_error_cnic2); 2566 } 2567 } 2568 } 2569 2570 /* Initialize Rx filter. */ 2571 bnx2x_set_rx_mode_inner(bp); 2572 2573 /* re-read iscsi info */ 2574 bnx2x_get_iscsi_info(bp); 2575 bnx2x_setup_cnic_irq_info(bp); 2576 bnx2x_setup_cnic_info(bp); 2577 bp->cnic_loaded = true; 2578 if (bp->state == BNX2X_STATE_OPEN) 2579 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD); 2580 2581 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n"); 2582 2583 return 0; 2584 2585 #ifndef BNX2X_STOP_ON_ERROR 2586 load_error_cnic2: 2587 /* Disable Timer scan */ 2588 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0); 2589 2590 load_error_cnic1: 2591 bnx2x_napi_disable_cnic(bp); 2592 /* Update the number of queues without the cnic queues */ 2593 if (bnx2x_set_real_num_queues(bp, 0)) 2594 BNX2X_ERR("Unable to set real_num_queues not including cnic\n"); 2595 load_error_cnic0: 2596 BNX2X_ERR("CNIC-related load failed\n"); 2597 bnx2x_free_fp_mem_cnic(bp); 2598 bnx2x_free_mem_cnic(bp); 2599 return rc; 2600 #endif /* ! BNX2X_STOP_ON_ERROR */ 2601 } 2602 2603 /* must be called with rtnl_lock */ 2604 int bnx2x_nic_load(struct bnx2x *bp, int load_mode) 2605 { 2606 int port = BP_PORT(bp); 2607 int i, rc = 0, load_code = 0; 2608 2609 DP(NETIF_MSG_IFUP, "Starting NIC load\n"); 2610 DP(NETIF_MSG_IFUP, 2611 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled"); 2612 2613 #ifdef BNX2X_STOP_ON_ERROR 2614 if (unlikely(bp->panic)) { 2615 BNX2X_ERR("Can't load NIC when there is panic\n"); 2616 return -EPERM; 2617 } 2618 #endif 2619 2620 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD; 2621 2622 /* zero the structure w/o any lock, before SP handler is initialized */ 2623 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link)); 2624 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN, 2625 &bp->last_reported_link.link_report_flags); 2626 2627 if (IS_PF(bp)) 2628 /* must be called before memory allocation and HW init */ 2629 bnx2x_ilt_set_info(bp); 2630 2631 /* 2632 * Zero fastpath structures preserving invariants like napi, which are 2633 * allocated only once, fp index, max_cos, bp pointer. 2634 * Also set fp->mode and txdata_ptr. 2635 */ 2636 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues); 2637 for_each_queue(bp, i) 2638 bnx2x_bz_fp(bp, i); 2639 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + 2640 bp->num_cnic_queues) * 2641 sizeof(struct bnx2x_fp_txdata)); 2642 2643 bp->fcoe_init = false; 2644 2645 /* Set the receive queues buffer size */ 2646 bnx2x_set_rx_buf_size(bp); 2647 2648 if (IS_PF(bp)) { 2649 rc = bnx2x_alloc_mem(bp); 2650 if (rc) { 2651 BNX2X_ERR("Unable to allocate bp memory\n"); 2652 return rc; 2653 } 2654 } 2655 2656 /* need to be done after alloc mem, since it's self adjusting to amount 2657 * of memory available for RSS queues 2658 */ 2659 rc = bnx2x_alloc_fp_mem(bp); 2660 if (rc) { 2661 BNX2X_ERR("Unable to allocate memory for fps\n"); 2662 LOAD_ERROR_EXIT(bp, load_error0); 2663 } 2664 2665 /* Allocated memory for FW statistics */ 2666 if (bnx2x_alloc_fw_stats_mem(bp)) 2667 LOAD_ERROR_EXIT(bp, load_error0); 2668 2669 /* request pf to initialize status blocks */ 2670 if (IS_VF(bp)) { 2671 rc = bnx2x_vfpf_init(bp); 2672 if (rc) 2673 LOAD_ERROR_EXIT(bp, load_error0); 2674 } 2675 2676 /* As long as bnx2x_alloc_mem() may possibly update 2677 * bp->num_queues, bnx2x_set_real_num_queues() should always 2678 * come after it. At this stage cnic queues are not counted. 2679 */ 2680 rc = bnx2x_set_real_num_queues(bp, 0); 2681 if (rc) { 2682 BNX2X_ERR("Unable to set real_num_queues\n"); 2683 LOAD_ERROR_EXIT(bp, load_error0); 2684 } 2685 2686 /* configure multi cos mappings in kernel. 2687 * this configuration may be overridden by a multi class queue 2688 * discipline or by a dcbx negotiation result. 2689 */ 2690 bnx2x_setup_tc(bp->dev, bp->max_cos); 2691 2692 /* Add all NAPI objects */ 2693 bnx2x_add_all_napi(bp); 2694 DP(NETIF_MSG_IFUP, "napi added\n"); 2695 bnx2x_napi_enable(bp); 2696 2697 if (IS_PF(bp)) { 2698 /* set pf load just before approaching the MCP */ 2699 bnx2x_set_pf_load(bp); 2700 2701 /* if mcp exists send load request and analyze response */ 2702 if (!BP_NOMCP(bp)) { 2703 /* attempt to load pf */ 2704 rc = bnx2x_nic_load_request(bp, &load_code); 2705 if (rc) 2706 LOAD_ERROR_EXIT(bp, load_error1); 2707 2708 /* what did mcp say? */ 2709 rc = bnx2x_compare_fw_ver(bp, load_code, true); 2710 if (rc) { 2711 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2712 LOAD_ERROR_EXIT(bp, load_error2); 2713 } 2714 } else { 2715 load_code = bnx2x_nic_load_no_mcp(bp, port); 2716 } 2717 2718 /* mark pmf if applicable */ 2719 bnx2x_nic_load_pmf(bp, load_code); 2720 2721 /* Init Function state controlling object */ 2722 bnx2x__init_func_obj(bp); 2723 2724 /* Initialize HW */ 2725 rc = bnx2x_init_hw(bp, load_code); 2726 if (rc) { 2727 BNX2X_ERR("HW init failed, aborting\n"); 2728 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2729 LOAD_ERROR_EXIT(bp, load_error2); 2730 } 2731 } 2732 2733 bnx2x_pre_irq_nic_init(bp); 2734 2735 /* Connect to IRQs */ 2736 rc = bnx2x_setup_irqs(bp); 2737 if (rc) { 2738 BNX2X_ERR("setup irqs failed\n"); 2739 if (IS_PF(bp)) 2740 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2741 LOAD_ERROR_EXIT(bp, load_error2); 2742 } 2743 2744 /* Init per-function objects */ 2745 if (IS_PF(bp)) { 2746 /* Setup NIC internals and enable interrupts */ 2747 bnx2x_post_irq_nic_init(bp, load_code); 2748 2749 bnx2x_init_bp_objs(bp); 2750 bnx2x_iov_nic_init(bp); 2751 2752 /* Set AFEX default VLAN tag to an invalid value */ 2753 bp->afex_def_vlan_tag = -1; 2754 bnx2x_nic_load_afex_dcc(bp, load_code); 2755 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT; 2756 rc = bnx2x_func_start(bp); 2757 if (rc) { 2758 BNX2X_ERR("Function start failed!\n"); 2759 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0); 2760 2761 LOAD_ERROR_EXIT(bp, load_error3); 2762 } 2763 2764 /* Send LOAD_DONE command to MCP */ 2765 if (!BP_NOMCP(bp)) { 2766 load_code = bnx2x_fw_command(bp, 2767 DRV_MSG_CODE_LOAD_DONE, 0); 2768 if (!load_code) { 2769 BNX2X_ERR("MCP response failure, aborting\n"); 2770 rc = -EBUSY; 2771 LOAD_ERROR_EXIT(bp, load_error3); 2772 } 2773 } 2774 2775 /* initialize FW coalescing state machines in RAM */ 2776 bnx2x_update_coalesce(bp); 2777 } 2778 2779 /* setup the leading queue */ 2780 rc = bnx2x_setup_leading(bp); 2781 if (rc) { 2782 BNX2X_ERR("Setup leading failed!\n"); 2783 LOAD_ERROR_EXIT(bp, load_error3); 2784 } 2785 2786 /* set up the rest of the queues */ 2787 for_each_nondefault_eth_queue(bp, i) { 2788 if (IS_PF(bp)) 2789 rc = bnx2x_setup_queue(bp, &bp->fp[i], false); 2790 else /* VF */ 2791 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false); 2792 if (rc) { 2793 BNX2X_ERR("Queue %d setup failed\n", i); 2794 LOAD_ERROR_EXIT(bp, load_error3); 2795 } 2796 } 2797 2798 /* setup rss */ 2799 rc = bnx2x_init_rss(bp); 2800 if (rc) { 2801 BNX2X_ERR("PF RSS init failed\n"); 2802 LOAD_ERROR_EXIT(bp, load_error3); 2803 } 2804 2805 /* Now when Clients are configured we are ready to work */ 2806 bp->state = BNX2X_STATE_OPEN; 2807 2808 /* Configure a ucast MAC */ 2809 if (IS_PF(bp)) 2810 rc = bnx2x_set_eth_mac(bp, true); 2811 else /* vf */ 2812 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index, 2813 true); 2814 if (rc) { 2815 BNX2X_ERR("Setting Ethernet MAC failed\n"); 2816 LOAD_ERROR_EXIT(bp, load_error3); 2817 } 2818 2819 if (IS_PF(bp) && bp->pending_max) { 2820 bnx2x_update_max_mf_config(bp, bp->pending_max); 2821 bp->pending_max = 0; 2822 } 2823 2824 if (bp->port.pmf) { 2825 rc = bnx2x_initial_phy_init(bp, load_mode); 2826 if (rc) 2827 LOAD_ERROR_EXIT(bp, load_error3); 2828 } 2829 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN; 2830 2831 /* Start fast path */ 2832 2833 /* Re-configure vlan filters */ 2834 rc = bnx2x_vlan_reconfigure_vid(bp); 2835 if (rc) 2836 LOAD_ERROR_EXIT(bp, load_error3); 2837 2838 /* Initialize Rx filter. */ 2839 bnx2x_set_rx_mode_inner(bp); 2840 2841 if (bp->flags & PTP_SUPPORTED) { 2842 bnx2x_init_ptp(bp); 2843 bnx2x_configure_ptp_filters(bp); 2844 } 2845 /* Start Tx */ 2846 switch (load_mode) { 2847 case LOAD_NORMAL: 2848 /* Tx queue should be only re-enabled */ 2849 netif_tx_wake_all_queues(bp->dev); 2850 break; 2851 2852 case LOAD_OPEN: 2853 netif_tx_start_all_queues(bp->dev); 2854 smp_mb__after_atomic(); 2855 break; 2856 2857 case LOAD_DIAG: 2858 case LOAD_LOOPBACK_EXT: 2859 bp->state = BNX2X_STATE_DIAG; 2860 break; 2861 2862 default: 2863 break; 2864 } 2865 2866 if (bp->port.pmf) 2867 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0); 2868 else 2869 bnx2x__link_status_update(bp); 2870 2871 /* start the timer */ 2872 mod_timer(&bp->timer, jiffies + bp->current_interval); 2873 2874 if (CNIC_ENABLED(bp)) 2875 bnx2x_load_cnic(bp); 2876 2877 if (IS_PF(bp)) 2878 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0); 2879 2880 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2881 /* mark driver is loaded in shmem2 */ 2882 u32 val; 2883 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2884 val &= ~DRV_FLAGS_MTU_MASK; 2885 val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT); 2886 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2887 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED | 2888 DRV_FLAGS_CAPABILITIES_LOADED_L2); 2889 } 2890 2891 /* Wait for all pending SP commands to complete */ 2892 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) { 2893 BNX2X_ERR("Timeout waiting for SP elements to complete\n"); 2894 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 2895 return -EBUSY; 2896 } 2897 2898 /* Update driver data for On-Chip MFW dump. */ 2899 if (IS_PF(bp)) 2900 bnx2x_update_mfw_dump(bp); 2901 2902 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */ 2903 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG)) 2904 bnx2x_dcbx_init(bp, false); 2905 2906 if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp)) 2907 bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE); 2908 2909 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n"); 2910 2911 return 0; 2912 2913 #ifndef BNX2X_STOP_ON_ERROR 2914 load_error3: 2915 if (IS_PF(bp)) { 2916 bnx2x_int_disable_sync(bp, 1); 2917 2918 /* Clean queueable objects */ 2919 bnx2x_squeeze_objects(bp); 2920 } 2921 2922 /* Free SKBs, SGEs, TPA pool and driver internals */ 2923 bnx2x_free_skbs(bp); 2924 for_each_rx_queue(bp, i) 2925 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 2926 2927 /* Release IRQs */ 2928 bnx2x_free_irq(bp); 2929 load_error2: 2930 if (IS_PF(bp) && !BP_NOMCP(bp)) { 2931 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0); 2932 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0); 2933 } 2934 2935 bp->port.pmf = 0; 2936 load_error1: 2937 bnx2x_napi_disable(bp); 2938 bnx2x_del_all_napi(bp); 2939 2940 /* clear pf_load status, as it was already set */ 2941 if (IS_PF(bp)) 2942 bnx2x_clear_pf_load(bp); 2943 load_error0: 2944 bnx2x_free_fw_stats_mem(bp); 2945 bnx2x_free_fp_mem(bp); 2946 bnx2x_free_mem(bp); 2947 2948 return rc; 2949 #endif /* ! BNX2X_STOP_ON_ERROR */ 2950 } 2951 2952 int bnx2x_drain_tx_queues(struct bnx2x *bp) 2953 { 2954 u8 rc = 0, cos, i; 2955 2956 /* Wait until tx fastpath tasks complete */ 2957 for_each_tx_queue(bp, i) { 2958 struct bnx2x_fastpath *fp = &bp->fp[i]; 2959 2960 for_each_cos_in_tx_queue(fp, cos) 2961 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]); 2962 if (rc) 2963 return rc; 2964 } 2965 return 0; 2966 } 2967 2968 /* must be called with rtnl_lock */ 2969 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link) 2970 { 2971 int i; 2972 bool global = false; 2973 2974 DP(NETIF_MSG_IFUP, "Starting NIC unload\n"); 2975 2976 if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp)) 2977 bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED); 2978 2979 /* mark driver is unloaded in shmem2 */ 2980 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) { 2981 u32 val; 2982 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]); 2983 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)], 2984 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2); 2985 } 2986 2987 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE && 2988 (bp->state == BNX2X_STATE_CLOSED || 2989 bp->state == BNX2X_STATE_ERROR)) { 2990 /* We can get here if the driver has been unloaded 2991 * during parity error recovery and is either waiting for a 2992 * leader to complete or for other functions to unload and 2993 * then ifdown has been issued. In this case we want to 2994 * unload and let other functions to complete a recovery 2995 * process. 2996 */ 2997 bp->recovery_state = BNX2X_RECOVERY_DONE; 2998 bp->is_leader = 0; 2999 bnx2x_release_leader_lock(bp); 3000 smp_mb(); 3001 3002 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n"); 3003 BNX2X_ERR("Can't unload in closed or error state\n"); 3004 return -EINVAL; 3005 } 3006 3007 /* Nothing to do during unload if previous bnx2x_nic_load() 3008 * have not completed successfully - all resources are released. 3009 * 3010 * we can get here only after unsuccessful ndo_* callback, during which 3011 * dev->IFF_UP flag is still on. 3012 */ 3013 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR) 3014 return 0; 3015 3016 /* It's important to set the bp->state to the value different from 3017 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int() 3018 * may restart the Tx from the NAPI context (see bnx2x_tx_int()). 3019 */ 3020 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT; 3021 smp_mb(); 3022 3023 /* indicate to VFs that the PF is going down */ 3024 bnx2x_iov_channel_down(bp); 3025 3026 if (CNIC_LOADED(bp)) 3027 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD); 3028 3029 /* Stop Tx */ 3030 bnx2x_tx_disable(bp); 3031 netdev_reset_tc(bp->dev); 3032 3033 bp->rx_mode = BNX2X_RX_MODE_NONE; 3034 3035 del_timer_sync(&bp->timer); 3036 3037 if (IS_PF(bp)) { 3038 /* Set ALWAYS_ALIVE bit in shmem */ 3039 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE; 3040 bnx2x_drv_pulse(bp); 3041 bnx2x_stats_handle(bp, STATS_EVENT_STOP); 3042 bnx2x_save_statistics(bp); 3043 } 3044 3045 /* wait till consumers catch up with producers in all queues. 3046 * If we're recovering, FW can't write to host so no reason 3047 * to wait for the queues to complete all Tx. 3048 */ 3049 if (unload_mode != UNLOAD_RECOVERY) 3050 bnx2x_drain_tx_queues(bp); 3051 3052 /* if VF indicate to PF this function is going down (PF will delete sp 3053 * elements and clear initializations 3054 */ 3055 if (IS_VF(bp)) 3056 bnx2x_vfpf_close_vf(bp); 3057 else if (unload_mode != UNLOAD_RECOVERY) 3058 /* if this is a normal/close unload need to clean up chip*/ 3059 bnx2x_chip_cleanup(bp, unload_mode, keep_link); 3060 else { 3061 /* Send the UNLOAD_REQUEST to the MCP */ 3062 bnx2x_send_unload_req(bp, unload_mode); 3063 3064 /* Prevent transactions to host from the functions on the 3065 * engine that doesn't reset global blocks in case of global 3066 * attention once global blocks are reset and gates are opened 3067 * (the engine which leader will perform the recovery 3068 * last). 3069 */ 3070 if (!CHIP_IS_E1x(bp)) 3071 bnx2x_pf_disable(bp); 3072 3073 /* Disable HW interrupts, NAPI */ 3074 bnx2x_netif_stop(bp, 1); 3075 /* Delete all NAPI objects */ 3076 bnx2x_del_all_napi(bp); 3077 if (CNIC_LOADED(bp)) 3078 bnx2x_del_all_napi_cnic(bp); 3079 /* Release IRQs */ 3080 bnx2x_free_irq(bp); 3081 3082 /* Report UNLOAD_DONE to MCP */ 3083 bnx2x_send_unload_done(bp, false); 3084 } 3085 3086 /* 3087 * At this stage no more interrupts will arrive so we may safely clean 3088 * the queueable objects here in case they failed to get cleaned so far. 3089 */ 3090 if (IS_PF(bp)) 3091 bnx2x_squeeze_objects(bp); 3092 3093 /* There should be no more pending SP commands at this stage */ 3094 bp->sp_state = 0; 3095 3096 bp->port.pmf = 0; 3097 3098 /* clear pending work in rtnl task */ 3099 bp->sp_rtnl_state = 0; 3100 smp_mb(); 3101 3102 /* Free SKBs, SGEs, TPA pool and driver internals */ 3103 bnx2x_free_skbs(bp); 3104 if (CNIC_LOADED(bp)) 3105 bnx2x_free_skbs_cnic(bp); 3106 for_each_rx_queue(bp, i) 3107 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 3108 3109 bnx2x_free_fp_mem(bp); 3110 if (CNIC_LOADED(bp)) 3111 bnx2x_free_fp_mem_cnic(bp); 3112 3113 if (IS_PF(bp)) { 3114 if (CNIC_LOADED(bp)) 3115 bnx2x_free_mem_cnic(bp); 3116 } 3117 bnx2x_free_mem(bp); 3118 3119 bp->state = BNX2X_STATE_CLOSED; 3120 bp->cnic_loaded = false; 3121 3122 /* Clear driver version indication in shmem */ 3123 if (IS_PF(bp)) 3124 bnx2x_update_mng_version(bp); 3125 3126 /* Check if there are pending parity attentions. If there are - set 3127 * RECOVERY_IN_PROGRESS. 3128 */ 3129 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) { 3130 bnx2x_set_reset_in_progress(bp); 3131 3132 /* Set RESET_IS_GLOBAL if needed */ 3133 if (global) 3134 bnx2x_set_reset_global(bp); 3135 } 3136 3137 /* The last driver must disable a "close the gate" if there is no 3138 * parity attention or "process kill" pending. 3139 */ 3140 if (IS_PF(bp) && 3141 !bnx2x_clear_pf_load(bp) && 3142 bnx2x_reset_is_done(bp, BP_PATH(bp))) 3143 bnx2x_disable_close_the_gate(bp); 3144 3145 DP(NETIF_MSG_IFUP, "Ending NIC unload\n"); 3146 3147 return 0; 3148 } 3149 3150 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state) 3151 { 3152 u16 pmcsr; 3153 3154 /* If there is no power capability, silently succeed */ 3155 if (!bp->pdev->pm_cap) { 3156 BNX2X_DEV_INFO("No power capability. Breaking.\n"); 3157 return 0; 3158 } 3159 3160 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 3161 3162 switch (state) { 3163 case PCI_D0: 3164 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3165 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) | 3166 PCI_PM_CTRL_PME_STATUS)); 3167 3168 if (pmcsr & PCI_PM_CTRL_STATE_MASK) 3169 /* delay required during transition out of D3hot */ 3170 msleep(20); 3171 break; 3172 3173 case PCI_D3hot: 3174 /* If there are other clients above don't 3175 shut down the power */ 3176 if (atomic_read(&bp->pdev->enable_cnt) != 1) 3177 return 0; 3178 /* Don't shut down the power for emulation and FPGA */ 3179 if (CHIP_REV_IS_SLOW(bp)) 3180 return 0; 3181 3182 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 3183 pmcsr |= 3; 3184 3185 if (bp->wol) 3186 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 3187 3188 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, 3189 pmcsr); 3190 3191 /* No more memory access after this point until 3192 * device is brought back to D0. 3193 */ 3194 break; 3195 3196 default: 3197 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state); 3198 return -EINVAL; 3199 } 3200 return 0; 3201 } 3202 3203 /* 3204 * net_device service functions 3205 */ 3206 static int bnx2x_poll(struct napi_struct *napi, int budget) 3207 { 3208 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 3209 napi); 3210 struct bnx2x *bp = fp->bp; 3211 int rx_work_done; 3212 u8 cos; 3213 3214 #ifdef BNX2X_STOP_ON_ERROR 3215 if (unlikely(bp->panic)) { 3216 napi_complete(napi); 3217 return 0; 3218 } 3219 #endif 3220 for_each_cos_in_tx_queue(fp, cos) 3221 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos])) 3222 bnx2x_tx_int(bp, fp->txdata_ptr[cos]); 3223 3224 rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0; 3225 3226 if (rx_work_done < budget) { 3227 /* No need to update SB for FCoE L2 ring as long as 3228 * it's connected to the default SB and the SB 3229 * has been updated when NAPI was scheduled. 3230 */ 3231 if (IS_FCOE_FP(fp)) { 3232 napi_complete(napi); 3233 } else { 3234 bnx2x_update_fpsb_idx(fp); 3235 /* bnx2x_has_rx_work() reads the status block, 3236 * thus we need to ensure that status block indices 3237 * have been actually read (bnx2x_update_fpsb_idx) 3238 * prior to this check (bnx2x_has_rx_work) so that 3239 * we won't write the "newer" value of the status block 3240 * to IGU (if there was a DMA right after 3241 * bnx2x_has_rx_work and if there is no rmb, the memory 3242 * reading (bnx2x_update_fpsb_idx) may be postponed 3243 * to right before bnx2x_ack_sb). In this case there 3244 * will never be another interrupt until there is 3245 * another update of the status block, while there 3246 * is still unhandled work. 3247 */ 3248 rmb(); 3249 3250 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 3251 napi_complete(napi); 3252 /* Re-enable interrupts */ 3253 DP(NETIF_MSG_RX_STATUS, 3254 "Update index to %d\n", fp->fp_hc_idx); 3255 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 3256 le16_to_cpu(fp->fp_hc_idx), 3257 IGU_INT_ENABLE, 1); 3258 } else { 3259 rx_work_done = budget; 3260 } 3261 } 3262 } 3263 3264 return rx_work_done; 3265 } 3266 3267 /* we split the first BD into headers and data BDs 3268 * to ease the pain of our fellow microcode engineers 3269 * we use one mapping for both BDs 3270 */ 3271 static u16 bnx2x_tx_split(struct bnx2x *bp, 3272 struct bnx2x_fp_txdata *txdata, 3273 struct sw_tx_bd *tx_buf, 3274 struct eth_tx_start_bd **tx_bd, u16 hlen, 3275 u16 bd_prod) 3276 { 3277 struct eth_tx_start_bd *h_tx_bd = *tx_bd; 3278 struct eth_tx_bd *d_tx_bd; 3279 dma_addr_t mapping; 3280 int old_len = le16_to_cpu(h_tx_bd->nbytes); 3281 3282 /* first fix first BD */ 3283 h_tx_bd->nbytes = cpu_to_le16(hlen); 3284 3285 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n", 3286 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo); 3287 3288 /* now get a new data BD 3289 * (after the pbd) and fill it */ 3290 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3291 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 3292 3293 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi), 3294 le32_to_cpu(h_tx_bd->addr_lo)) + hlen; 3295 3296 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 3297 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 3298 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen); 3299 3300 /* this marks the BD as one that has no individual mapping */ 3301 tx_buf->flags |= BNX2X_TSO_SPLIT_BD; 3302 3303 DP(NETIF_MSG_TX_QUEUED, 3304 "TSO split data size is %d (%x:%x)\n", 3305 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo); 3306 3307 /* update tx_bd */ 3308 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd; 3309 3310 return bd_prod; 3311 } 3312 3313 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32))) 3314 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16))) 3315 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix) 3316 { 3317 __sum16 tsum = (__force __sum16) csum; 3318 3319 if (fix > 0) 3320 tsum = ~csum_fold(csum_sub((__force __wsum) csum, 3321 csum_partial(t_header - fix, fix, 0))); 3322 3323 else if (fix < 0) 3324 tsum = ~csum_fold(csum_add((__force __wsum) csum, 3325 csum_partial(t_header, -fix, 0))); 3326 3327 return bswab16(tsum); 3328 } 3329 3330 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb) 3331 { 3332 u32 rc; 3333 __u8 prot = 0; 3334 __be16 protocol; 3335 3336 if (skb->ip_summed != CHECKSUM_PARTIAL) 3337 return XMIT_PLAIN; 3338 3339 protocol = vlan_get_protocol(skb); 3340 if (protocol == htons(ETH_P_IPV6)) { 3341 rc = XMIT_CSUM_V6; 3342 prot = ipv6_hdr(skb)->nexthdr; 3343 } else { 3344 rc = XMIT_CSUM_V4; 3345 prot = ip_hdr(skb)->protocol; 3346 } 3347 3348 if (!CHIP_IS_E1x(bp) && skb->encapsulation) { 3349 if (inner_ip_hdr(skb)->version == 6) { 3350 rc |= XMIT_CSUM_ENC_V6; 3351 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 3352 rc |= XMIT_CSUM_TCP; 3353 } else { 3354 rc |= XMIT_CSUM_ENC_V4; 3355 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP) 3356 rc |= XMIT_CSUM_TCP; 3357 } 3358 } 3359 if (prot == IPPROTO_TCP) 3360 rc |= XMIT_CSUM_TCP; 3361 3362 if (skb_is_gso(skb)) { 3363 if (skb_is_gso_v6(skb)) { 3364 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP); 3365 if (rc & XMIT_CSUM_ENC) 3366 rc |= XMIT_GSO_ENC_V6; 3367 } else { 3368 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP); 3369 if (rc & XMIT_CSUM_ENC) 3370 rc |= XMIT_GSO_ENC_V4; 3371 } 3372 } 3373 3374 return rc; 3375 } 3376 3377 /* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */ 3378 #define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS 4 3379 3380 /* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */ 3381 #define BNX2X_NUM_TSO_WIN_SUB_BDS 3 3382 3383 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT) 3384 /* check if packet requires linearization (packet is too fragmented) 3385 no need to check fragmentation if page size > 8K (there will be no 3386 violation to FW restrictions) */ 3387 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb, 3388 u32 xmit_type) 3389 { 3390 int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS; 3391 int to_copy = 0, hlen = 0; 3392 3393 if (xmit_type & XMIT_GSO_ENC) 3394 num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS; 3395 3396 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) { 3397 if (xmit_type & XMIT_GSO) { 3398 unsigned short lso_mss = skb_shinfo(skb)->gso_size; 3399 int wnd_size = MAX_FETCH_BD - num_tso_win_sub; 3400 /* Number of windows to check */ 3401 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size; 3402 int wnd_idx = 0; 3403 int frag_idx = 0; 3404 u32 wnd_sum = 0; 3405 3406 /* Headers length */ 3407 if (xmit_type & XMIT_GSO_ENC) 3408 hlen = (int)(skb_inner_transport_header(skb) - 3409 skb->data) + 3410 inner_tcp_hdrlen(skb); 3411 else 3412 hlen = (int)(skb_transport_header(skb) - 3413 skb->data) + tcp_hdrlen(skb); 3414 3415 /* Amount of data (w/o headers) on linear part of SKB*/ 3416 first_bd_sz = skb_headlen(skb) - hlen; 3417 3418 wnd_sum = first_bd_sz; 3419 3420 /* Calculate the first sum - it's special */ 3421 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++) 3422 wnd_sum += 3423 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]); 3424 3425 /* If there was data on linear skb data - check it */ 3426 if (first_bd_sz > 0) { 3427 if (unlikely(wnd_sum < lso_mss)) { 3428 to_copy = 1; 3429 goto exit_lbl; 3430 } 3431 3432 wnd_sum -= first_bd_sz; 3433 } 3434 3435 /* Others are easier: run through the frag list and 3436 check all windows */ 3437 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) { 3438 wnd_sum += 3439 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]); 3440 3441 if (unlikely(wnd_sum < lso_mss)) { 3442 to_copy = 1; 3443 break; 3444 } 3445 wnd_sum -= 3446 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]); 3447 } 3448 } else { 3449 /* in non-LSO too fragmented packet should always 3450 be linearized */ 3451 to_copy = 1; 3452 } 3453 } 3454 3455 exit_lbl: 3456 if (unlikely(to_copy)) 3457 DP(NETIF_MSG_TX_QUEUED, 3458 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n", 3459 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO", 3460 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz); 3461 3462 return to_copy; 3463 } 3464 #endif 3465 3466 /** 3467 * bnx2x_set_pbd_gso - update PBD in GSO case. 3468 * 3469 * @skb: packet skb 3470 * @pbd: parse BD 3471 * @xmit_type: xmit flags 3472 */ 3473 static void bnx2x_set_pbd_gso(struct sk_buff *skb, 3474 struct eth_tx_parse_bd_e1x *pbd, 3475 u32 xmit_type) 3476 { 3477 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 3478 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq); 3479 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb)); 3480 3481 if (xmit_type & XMIT_GSO_V4) { 3482 pbd->ip_id = bswab16(ip_hdr(skb)->id); 3483 pbd->tcp_pseudo_csum = 3484 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr, 3485 ip_hdr(skb)->daddr, 3486 0, IPPROTO_TCP, 0)); 3487 } else { 3488 pbd->tcp_pseudo_csum = 3489 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3490 &ipv6_hdr(skb)->daddr, 3491 0, IPPROTO_TCP, 0)); 3492 } 3493 3494 pbd->global_data |= 3495 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN); 3496 } 3497 3498 /** 3499 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length 3500 * 3501 * @bp: driver handle 3502 * @skb: packet skb 3503 * @parsing_data: data to be updated 3504 * @xmit_type: xmit flags 3505 * 3506 * 57712/578xx related, when skb has encapsulation 3507 */ 3508 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb, 3509 u32 *parsing_data, u32 xmit_type) 3510 { 3511 *parsing_data |= 3512 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) << 3513 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3514 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3515 3516 if (xmit_type & XMIT_CSUM_TCP) { 3517 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) << 3518 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3519 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3520 3521 return skb_inner_transport_header(skb) + 3522 inner_tcp_hdrlen(skb) - skb->data; 3523 } 3524 3525 /* We support checksum offload for TCP and UDP only. 3526 * No need to pass the UDP header length - it's a constant. 3527 */ 3528 return skb_inner_transport_header(skb) + 3529 sizeof(struct udphdr) - skb->data; 3530 } 3531 3532 /** 3533 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length 3534 * 3535 * @bp: driver handle 3536 * @skb: packet skb 3537 * @parsing_data: data to be updated 3538 * @xmit_type: xmit flags 3539 * 3540 * 57712/578xx related 3541 */ 3542 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb, 3543 u32 *parsing_data, u32 xmit_type) 3544 { 3545 *parsing_data |= 3546 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) << 3547 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) & 3548 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W; 3549 3550 if (xmit_type & XMIT_CSUM_TCP) { 3551 *parsing_data |= ((tcp_hdrlen(skb) / 4) << 3552 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) & 3553 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW; 3554 3555 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data; 3556 } 3557 /* We support checksum offload for TCP and UDP only. 3558 * No need to pass the UDP header length - it's a constant. 3559 */ 3560 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data; 3561 } 3562 3563 /* set FW indication according to inner or outer protocols if tunneled */ 3564 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3565 struct eth_tx_start_bd *tx_start_bd, 3566 u32 xmit_type) 3567 { 3568 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM; 3569 3570 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6)) 3571 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6; 3572 3573 if (!(xmit_type & XMIT_CSUM_TCP)) 3574 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP; 3575 } 3576 3577 /** 3578 * bnx2x_set_pbd_csum - update PBD with checksum and return header length 3579 * 3580 * @bp: driver handle 3581 * @skb: packet skb 3582 * @pbd: parse BD to be updated 3583 * @xmit_type: xmit flags 3584 */ 3585 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb, 3586 struct eth_tx_parse_bd_e1x *pbd, 3587 u32 xmit_type) 3588 { 3589 u8 hlen = (skb_network_header(skb) - skb->data) >> 1; 3590 3591 /* for now NS flag is not used in Linux */ 3592 pbd->global_data = 3593 cpu_to_le16(hlen | 3594 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3595 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT)); 3596 3597 pbd->ip_hlen_w = (skb_transport_header(skb) - 3598 skb_network_header(skb)) >> 1; 3599 3600 hlen += pbd->ip_hlen_w; 3601 3602 /* We support checksum offload for TCP and UDP only */ 3603 if (xmit_type & XMIT_CSUM_TCP) 3604 hlen += tcp_hdrlen(skb) / 2; 3605 else 3606 hlen += sizeof(struct udphdr) / 2; 3607 3608 pbd->total_hlen_w = cpu_to_le16(hlen); 3609 hlen = hlen*2; 3610 3611 if (xmit_type & XMIT_CSUM_TCP) { 3612 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check); 3613 3614 } else { 3615 s8 fix = SKB_CS_OFF(skb); /* signed! */ 3616 3617 DP(NETIF_MSG_TX_QUEUED, 3618 "hlen %d fix %d csum before fix %x\n", 3619 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb)); 3620 3621 /* HW bug: fixup the CSUM */ 3622 pbd->tcp_pseudo_csum = 3623 bnx2x_csum_fix(skb_transport_header(skb), 3624 SKB_CS(skb), fix); 3625 3626 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n", 3627 pbd->tcp_pseudo_csum); 3628 } 3629 3630 return hlen; 3631 } 3632 3633 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb, 3634 struct eth_tx_parse_bd_e2 *pbd_e2, 3635 struct eth_tx_parse_2nd_bd *pbd2, 3636 u16 *global_data, 3637 u32 xmit_type) 3638 { 3639 u16 hlen_w = 0; 3640 u8 outerip_off, outerip_len = 0; 3641 3642 /* from outer IP to transport */ 3643 hlen_w = (skb_inner_transport_header(skb) - 3644 skb_network_header(skb)) >> 1; 3645 3646 /* transport len */ 3647 hlen_w += inner_tcp_hdrlen(skb) >> 1; 3648 3649 pbd2->fw_ip_hdr_to_payload_w = hlen_w; 3650 3651 /* outer IP header info */ 3652 if (xmit_type & XMIT_CSUM_V4) { 3653 struct iphdr *iph = ip_hdr(skb); 3654 u32 csum = (__force u32)(~iph->check) - 3655 (__force u32)iph->tot_len - 3656 (__force u32)iph->frag_off; 3657 3658 outerip_len = iph->ihl << 1; 3659 3660 pbd2->fw_ip_csum_wo_len_flags_frag = 3661 bswab16(csum_fold((__force __wsum)csum)); 3662 } else { 3663 pbd2->fw_ip_hdr_to_payload_w = 3664 hlen_w - ((sizeof(struct ipv6hdr)) >> 1); 3665 pbd_e2->data.tunnel_data.flags |= 3666 ETH_TUNNEL_DATA_IPV6_OUTER; 3667 } 3668 3669 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq); 3670 3671 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb)); 3672 3673 /* inner IP header info */ 3674 if (xmit_type & XMIT_CSUM_ENC_V4) { 3675 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id); 3676 3677 pbd_e2->data.tunnel_data.pseudo_csum = 3678 bswab16(~csum_tcpudp_magic( 3679 inner_ip_hdr(skb)->saddr, 3680 inner_ip_hdr(skb)->daddr, 3681 0, IPPROTO_TCP, 0)); 3682 } else { 3683 pbd_e2->data.tunnel_data.pseudo_csum = 3684 bswab16(~csum_ipv6_magic( 3685 &inner_ipv6_hdr(skb)->saddr, 3686 &inner_ipv6_hdr(skb)->daddr, 3687 0, IPPROTO_TCP, 0)); 3688 } 3689 3690 outerip_off = (skb_network_header(skb) - skb->data) >> 1; 3691 3692 *global_data |= 3693 outerip_off | 3694 (outerip_len << 3695 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) | 3696 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 3697 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT); 3698 3699 if (ip_hdr(skb)->protocol == IPPROTO_UDP) { 3700 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1); 3701 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1; 3702 } 3703 } 3704 3705 static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data, 3706 u32 xmit_type) 3707 { 3708 struct ipv6hdr *ipv6; 3709 3710 if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6))) 3711 return; 3712 3713 if (xmit_type & XMIT_GSO_ENC_V6) 3714 ipv6 = inner_ipv6_hdr(skb); 3715 else /* XMIT_GSO_V6 */ 3716 ipv6 = ipv6_hdr(skb); 3717 3718 if (ipv6->nexthdr == NEXTHDR_IPV6) 3719 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR; 3720 } 3721 3722 /* called with netif_tx_lock 3723 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call 3724 * netif_wake_queue() 3725 */ 3726 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev) 3727 { 3728 struct bnx2x *bp = netdev_priv(dev); 3729 3730 struct netdev_queue *txq; 3731 struct bnx2x_fp_txdata *txdata; 3732 struct sw_tx_bd *tx_buf; 3733 struct eth_tx_start_bd *tx_start_bd, *first_bd; 3734 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL; 3735 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL; 3736 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL; 3737 struct eth_tx_parse_2nd_bd *pbd2 = NULL; 3738 u32 pbd_e2_parsing_data = 0; 3739 u16 pkt_prod, bd_prod; 3740 int nbd, txq_index; 3741 dma_addr_t mapping; 3742 u32 xmit_type = bnx2x_xmit_type(bp, skb); 3743 int i; 3744 u8 hlen = 0; 3745 __le16 pkt_size = 0; 3746 struct ethhdr *eth; 3747 u8 mac_type = UNICAST_ADDRESS; 3748 3749 #ifdef BNX2X_STOP_ON_ERROR 3750 if (unlikely(bp->panic)) 3751 return NETDEV_TX_BUSY; 3752 #endif 3753 3754 txq_index = skb_get_queue_mapping(skb); 3755 txq = netdev_get_tx_queue(dev, txq_index); 3756 3757 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0)); 3758 3759 txdata = &bp->bnx2x_txq[txq_index]; 3760 3761 /* enable this debug print to view the transmission queue being used 3762 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n", 3763 txq_index, fp_index, txdata_index); */ 3764 3765 /* enable this debug print to view the transmission details 3766 DP(NETIF_MSG_TX_QUEUED, 3767 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n", 3768 txdata->cid, fp_index, txdata_index, txdata, fp); */ 3769 3770 if (unlikely(bnx2x_tx_avail(bp, txdata) < 3771 skb_shinfo(skb)->nr_frags + 3772 BDS_PER_TX_PKT + 3773 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) { 3774 /* Handle special storage cases separately */ 3775 if (txdata->tx_ring_size == 0) { 3776 struct bnx2x_eth_q_stats *q_stats = 3777 bnx2x_fp_qstats(bp, txdata->parent_fp); 3778 q_stats->driver_filtered_tx_pkt++; 3779 dev_kfree_skb(skb); 3780 return NETDEV_TX_OK; 3781 } 3782 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 3783 netif_tx_stop_queue(txq); 3784 BNX2X_ERR("BUG! Tx ring full when queue awake!\n"); 3785 3786 return NETDEV_TX_BUSY; 3787 } 3788 3789 DP(NETIF_MSG_TX_QUEUED, 3790 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n", 3791 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr, 3792 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type, 3793 skb->len); 3794 3795 eth = (struct ethhdr *)skb->data; 3796 3797 /* set flag according to packet type (UNICAST_ADDRESS is default)*/ 3798 if (unlikely(is_multicast_ether_addr(eth->h_dest))) { 3799 if (is_broadcast_ether_addr(eth->h_dest)) 3800 mac_type = BROADCAST_ADDRESS; 3801 else 3802 mac_type = MULTICAST_ADDRESS; 3803 } 3804 3805 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT) 3806 /* First, check if we need to linearize the skb (due to FW 3807 restrictions). No need to check fragmentation if page size > 8K 3808 (there will be no violation to FW restrictions) */ 3809 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) { 3810 /* Statistics of linearization */ 3811 bp->lin_cnt++; 3812 if (skb_linearize(skb) != 0) { 3813 DP(NETIF_MSG_TX_QUEUED, 3814 "SKB linearization failed - silently dropping this SKB\n"); 3815 dev_kfree_skb_any(skb); 3816 return NETDEV_TX_OK; 3817 } 3818 } 3819 #endif 3820 /* Map skb linear data for DMA */ 3821 mapping = dma_map_single(&bp->pdev->dev, skb->data, 3822 skb_headlen(skb), DMA_TO_DEVICE); 3823 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 3824 DP(NETIF_MSG_TX_QUEUED, 3825 "SKB mapping failed - silently dropping this SKB\n"); 3826 dev_kfree_skb_any(skb); 3827 return NETDEV_TX_OK; 3828 } 3829 /* 3830 Please read carefully. First we use one BD which we mark as start, 3831 then we have a parsing info BD (used for TSO or xsum), 3832 and only then we have the rest of the TSO BDs. 3833 (don't forget to mark the last one as last, 3834 and to unmap only AFTER you write to the BD ...) 3835 And above all, all pdb sizes are in words - NOT DWORDS! 3836 */ 3837 3838 /* get current pkt produced now - advance it just before sending packet 3839 * since mapping of pages may fail and cause packet to be dropped 3840 */ 3841 pkt_prod = txdata->tx_pkt_prod; 3842 bd_prod = TX_BD(txdata->tx_bd_prod); 3843 3844 /* get a tx_buf and first BD 3845 * tx_start_bd may be changed during SPLIT, 3846 * but first_bd will always stay first 3847 */ 3848 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)]; 3849 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd; 3850 first_bd = tx_start_bd; 3851 3852 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 3853 3854 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 3855 if (!(bp->flags & TX_TIMESTAMPING_EN)) { 3856 BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n"); 3857 } else if (bp->ptp_tx_skb) { 3858 BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n"); 3859 } else { 3860 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3861 /* schedule check for Tx timestamp */ 3862 bp->ptp_tx_skb = skb_get(skb); 3863 bp->ptp_tx_start = jiffies; 3864 schedule_work(&bp->ptp_task); 3865 } 3866 } 3867 3868 /* header nbd: indirectly zero other flags! */ 3869 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT; 3870 3871 /* remember the first BD of the packet */ 3872 tx_buf->first_bd = txdata->tx_bd_prod; 3873 tx_buf->skb = skb; 3874 tx_buf->flags = 0; 3875 3876 DP(NETIF_MSG_TX_QUEUED, 3877 "sending pkt %u @%p next_idx %u bd %u @%p\n", 3878 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd); 3879 3880 if (skb_vlan_tag_present(skb)) { 3881 tx_start_bd->vlan_or_ethertype = 3882 cpu_to_le16(skb_vlan_tag_get(skb)); 3883 tx_start_bd->bd_flags.as_bitfield |= 3884 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT); 3885 } else { 3886 /* when transmitting in a vf, start bd must hold the ethertype 3887 * for fw to enforce it 3888 */ 3889 #ifndef BNX2X_STOP_ON_ERROR 3890 if (IS_VF(bp)) 3891 #endif 3892 tx_start_bd->vlan_or_ethertype = 3893 cpu_to_le16(ntohs(eth->h_proto)); 3894 #ifndef BNX2X_STOP_ON_ERROR 3895 else 3896 /* used by FW for packet accounting */ 3897 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod); 3898 #endif 3899 } 3900 3901 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */ 3902 3903 /* turn on parsing and get a BD */ 3904 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3905 3906 if (xmit_type & XMIT_CSUM) 3907 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type); 3908 3909 if (!CHIP_IS_E1x(bp)) { 3910 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2; 3911 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2)); 3912 3913 if (xmit_type & XMIT_CSUM_ENC) { 3914 u16 global_data = 0; 3915 3916 /* Set PBD in enc checksum offload case */ 3917 hlen = bnx2x_set_pbd_csum_enc(bp, skb, 3918 &pbd_e2_parsing_data, 3919 xmit_type); 3920 3921 /* turn on 2nd parsing and get a BD */ 3922 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 3923 3924 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd; 3925 3926 memset(pbd2, 0, sizeof(*pbd2)); 3927 3928 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w = 3929 (skb_inner_network_header(skb) - 3930 skb->data) >> 1; 3931 3932 if (xmit_type & XMIT_GSO_ENC) 3933 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2, 3934 &global_data, 3935 xmit_type); 3936 3937 pbd2->global_data = cpu_to_le16(global_data); 3938 3939 /* add addition parse BD indication to start BD */ 3940 SET_FLAG(tx_start_bd->general_data, 3941 ETH_TX_START_BD_PARSE_NBDS, 1); 3942 /* set encapsulation flag in start BD */ 3943 SET_FLAG(tx_start_bd->general_data, 3944 ETH_TX_START_BD_TUNNEL_EXIST, 1); 3945 3946 tx_buf->flags |= BNX2X_HAS_SECOND_PBD; 3947 3948 nbd++; 3949 } else if (xmit_type & XMIT_CSUM) { 3950 /* Set PBD in checksum offload case w/o encapsulation */ 3951 hlen = bnx2x_set_pbd_csum_e2(bp, skb, 3952 &pbd_e2_parsing_data, 3953 xmit_type); 3954 } 3955 3956 bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type); 3957 /* Add the macs to the parsing BD if this is a vf or if 3958 * Tx Switching is enabled. 3959 */ 3960 if (IS_VF(bp)) { 3961 /* override GRE parameters in BD */ 3962 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi, 3963 &pbd_e2->data.mac_addr.src_mid, 3964 &pbd_e2->data.mac_addr.src_lo, 3965 eth->h_source); 3966 3967 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi, 3968 &pbd_e2->data.mac_addr.dst_mid, 3969 &pbd_e2->data.mac_addr.dst_lo, 3970 eth->h_dest); 3971 } else { 3972 if (bp->flags & TX_SWITCHING) 3973 bnx2x_set_fw_mac_addr( 3974 &pbd_e2->data.mac_addr.dst_hi, 3975 &pbd_e2->data.mac_addr.dst_mid, 3976 &pbd_e2->data.mac_addr.dst_lo, 3977 eth->h_dest); 3978 #ifdef BNX2X_STOP_ON_ERROR 3979 /* Enforce security is always set in Stop on Error - 3980 * source mac should be present in the parsing BD 3981 */ 3982 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi, 3983 &pbd_e2->data.mac_addr.src_mid, 3984 &pbd_e2->data.mac_addr.src_lo, 3985 eth->h_source); 3986 #endif 3987 } 3988 3989 SET_FLAG(pbd_e2_parsing_data, 3990 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type); 3991 } else { 3992 u16 global_data = 0; 3993 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x; 3994 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x)); 3995 /* Set PBD in checksum offload case */ 3996 if (xmit_type & XMIT_CSUM) 3997 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type); 3998 3999 SET_FLAG(global_data, 4000 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type); 4001 pbd_e1x->global_data |= cpu_to_le16(global_data); 4002 } 4003 4004 /* Setup the data pointer of the first BD of the packet */ 4005 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 4006 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 4007 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb)); 4008 pkt_size = tx_start_bd->nbytes; 4009 4010 DP(NETIF_MSG_TX_QUEUED, 4011 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n", 4012 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo, 4013 le16_to_cpu(tx_start_bd->nbytes), 4014 tx_start_bd->bd_flags.as_bitfield, 4015 le16_to_cpu(tx_start_bd->vlan_or_ethertype)); 4016 4017 if (xmit_type & XMIT_GSO) { 4018 4019 DP(NETIF_MSG_TX_QUEUED, 4020 "TSO packet len %d hlen %d total len %d tso size %d\n", 4021 skb->len, hlen, skb_headlen(skb), 4022 skb_shinfo(skb)->gso_size); 4023 4024 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO; 4025 4026 if (unlikely(skb_headlen(skb) > hlen)) { 4027 nbd++; 4028 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf, 4029 &tx_start_bd, hlen, 4030 bd_prod); 4031 } 4032 if (!CHIP_IS_E1x(bp)) 4033 pbd_e2_parsing_data |= 4034 (skb_shinfo(skb)->gso_size << 4035 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) & 4036 ETH_TX_PARSE_BD_E2_LSO_MSS; 4037 else 4038 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type); 4039 } 4040 4041 /* Set the PBD's parsing_data field if not zero 4042 * (for the chips newer than 57711). 4043 */ 4044 if (pbd_e2_parsing_data) 4045 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data); 4046 4047 tx_data_bd = (struct eth_tx_bd *)tx_start_bd; 4048 4049 /* Handle fragmented skb */ 4050 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4051 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4052 4053 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0, 4054 skb_frag_size(frag), DMA_TO_DEVICE); 4055 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { 4056 unsigned int pkts_compl = 0, bytes_compl = 0; 4057 4058 DP(NETIF_MSG_TX_QUEUED, 4059 "Unable to map page - dropping packet...\n"); 4060 4061 /* we need unmap all buffers already mapped 4062 * for this SKB; 4063 * first_bd->nbd need to be properly updated 4064 * before call to bnx2x_free_tx_pkt 4065 */ 4066 first_bd->nbd = cpu_to_le16(nbd); 4067 bnx2x_free_tx_pkt(bp, txdata, 4068 TX_BD(txdata->tx_pkt_prod), 4069 &pkts_compl, &bytes_compl); 4070 return NETDEV_TX_OK; 4071 } 4072 4073 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 4074 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 4075 if (total_pkt_bd == NULL) 4076 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd; 4077 4078 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 4079 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 4080 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag)); 4081 le16_add_cpu(&pkt_size, skb_frag_size(frag)); 4082 nbd++; 4083 4084 DP(NETIF_MSG_TX_QUEUED, 4085 "frag %d bd @%p addr (%x:%x) nbytes %d\n", 4086 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo, 4087 le16_to_cpu(tx_data_bd->nbytes)); 4088 } 4089 4090 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd); 4091 4092 /* update with actual num BDs */ 4093 first_bd->nbd = cpu_to_le16(nbd); 4094 4095 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 4096 4097 /* now send a tx doorbell, counting the next BD 4098 * if the packet contains or ends with it 4099 */ 4100 if (TX_BD_POFF(bd_prod) < nbd) 4101 nbd++; 4102 4103 /* total_pkt_bytes should be set on the first data BD if 4104 * it's not an LSO packet and there is more than one 4105 * data BD. In this case pkt_size is limited by an MTU value. 4106 * However we prefer to set it for an LSO packet (while we don't 4107 * have to) in order to save some CPU cycles in a none-LSO 4108 * case, when we much more care about them. 4109 */ 4110 if (total_pkt_bd != NULL) 4111 total_pkt_bd->total_pkt_bytes = pkt_size; 4112 4113 if (pbd_e1x) 4114 DP(NETIF_MSG_TX_QUEUED, 4115 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n", 4116 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w, 4117 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags, 4118 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq, 4119 le16_to_cpu(pbd_e1x->total_hlen_w)); 4120 if (pbd_e2) 4121 DP(NETIF_MSG_TX_QUEUED, 4122 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n", 4123 pbd_e2, 4124 pbd_e2->data.mac_addr.dst_hi, 4125 pbd_e2->data.mac_addr.dst_mid, 4126 pbd_e2->data.mac_addr.dst_lo, 4127 pbd_e2->data.mac_addr.src_hi, 4128 pbd_e2->data.mac_addr.src_mid, 4129 pbd_e2->data.mac_addr.src_lo, 4130 pbd_e2->parsing_data); 4131 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod); 4132 4133 netdev_tx_sent_queue(txq, skb->len); 4134 4135 skb_tx_timestamp(skb); 4136 4137 txdata->tx_pkt_prod++; 4138 /* 4139 * Make sure that the BD data is updated before updating the producer 4140 * since FW might read the BD right after the producer is updated. 4141 * This is only applicable for weak-ordered memory model archs such 4142 * as IA-64. The following barrier is also mandatory since FW will 4143 * assumes packets must have BDs. 4144 */ 4145 wmb(); 4146 4147 txdata->tx_db.data.prod += nbd; 4148 barrier(); 4149 4150 DOORBELL(bp, txdata->cid, txdata->tx_db.raw); 4151 4152 mmiowb(); 4153 4154 txdata->tx_bd_prod += nbd; 4155 4156 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) { 4157 netif_tx_stop_queue(txq); 4158 4159 /* paired memory barrier is in bnx2x_tx_int(), we have to keep 4160 * ordering of set_bit() in netif_tx_stop_queue() and read of 4161 * fp->bd_tx_cons */ 4162 smp_mb(); 4163 4164 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++; 4165 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT) 4166 netif_tx_wake_queue(txq); 4167 } 4168 txdata->tx_pkt++; 4169 4170 return NETDEV_TX_OK; 4171 } 4172 4173 void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default) 4174 { 4175 int mfw_vn = BP_FW_MB_IDX(bp); 4176 u32 tmp; 4177 4178 /* If the shmem shouldn't affect configuration, reflect */ 4179 if (!IS_MF_BD(bp)) { 4180 int i; 4181 4182 for (i = 0; i < BNX2X_MAX_PRIORITY; i++) 4183 c2s_map[i] = i; 4184 *c2s_default = 0; 4185 4186 return; 4187 } 4188 4189 tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]); 4190 tmp = (__force u32)be32_to_cpu((__force __be32)tmp); 4191 c2s_map[0] = tmp & 0xff; 4192 c2s_map[1] = (tmp >> 8) & 0xff; 4193 c2s_map[2] = (tmp >> 16) & 0xff; 4194 c2s_map[3] = (tmp >> 24) & 0xff; 4195 4196 tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]); 4197 tmp = (__force u32)be32_to_cpu((__force __be32)tmp); 4198 c2s_map[4] = tmp & 0xff; 4199 c2s_map[5] = (tmp >> 8) & 0xff; 4200 c2s_map[6] = (tmp >> 16) & 0xff; 4201 c2s_map[7] = (tmp >> 24) & 0xff; 4202 4203 tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]); 4204 tmp = (__force u32)be32_to_cpu((__force __be32)tmp); 4205 *c2s_default = (tmp >> (8 * mfw_vn)) & 0xff; 4206 } 4207 4208 /** 4209 * bnx2x_setup_tc - routine to configure net_device for multi tc 4210 * 4211 * @netdev: net device to configure 4212 * @tc: number of traffic classes to enable 4213 * 4214 * callback connected to the ndo_setup_tc function pointer 4215 */ 4216 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc) 4217 { 4218 struct bnx2x *bp = netdev_priv(dev); 4219 u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def; 4220 int cos, prio, count, offset; 4221 4222 /* setup tc must be called under rtnl lock */ 4223 ASSERT_RTNL(); 4224 4225 /* no traffic classes requested. Aborting */ 4226 if (!num_tc) { 4227 netdev_reset_tc(dev); 4228 return 0; 4229 } 4230 4231 /* requested to support too many traffic classes */ 4232 if (num_tc > bp->max_cos) { 4233 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n", 4234 num_tc, bp->max_cos); 4235 return -EINVAL; 4236 } 4237 4238 /* declare amount of supported traffic classes */ 4239 if (netdev_set_num_tc(dev, num_tc)) { 4240 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc); 4241 return -EINVAL; 4242 } 4243 4244 bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def); 4245 4246 /* configure priority to traffic class mapping */ 4247 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) { 4248 int outer_prio = c2s_map[prio]; 4249 4250 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]); 4251 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4252 "mapping priority %d to tc %d\n", 4253 outer_prio, bp->prio_to_cos[outer_prio]); 4254 } 4255 4256 /* Use this configuration to differentiate tc0 from other COSes 4257 This can be used for ets or pfc, and save the effort of setting 4258 up a multio class queue disc or negotiating DCBX with a switch 4259 netdev_set_prio_tc_map(dev, 0, 0); 4260 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0); 4261 for (prio = 1; prio < 16; prio++) { 4262 netdev_set_prio_tc_map(dev, prio, 1); 4263 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1); 4264 } */ 4265 4266 /* configure traffic class to transmission queue mapping */ 4267 for (cos = 0; cos < bp->max_cos; cos++) { 4268 count = BNX2X_NUM_ETH_QUEUES(bp); 4269 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp); 4270 netdev_set_tc_queue(dev, cos, count, offset); 4271 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP, 4272 "mapping tc %d to offset %d count %d\n", 4273 cos, offset, count); 4274 } 4275 4276 return 0; 4277 } 4278 4279 int __bnx2x_setup_tc(struct net_device *dev, u32 handle, __be16 proto, 4280 struct tc_to_netdev *tc) 4281 { 4282 if (tc->type != TC_SETUP_MQPRIO) 4283 return -EINVAL; 4284 return bnx2x_setup_tc(dev, tc->tc); 4285 } 4286 4287 /* called with rtnl_lock */ 4288 int bnx2x_change_mac_addr(struct net_device *dev, void *p) 4289 { 4290 struct sockaddr *addr = p; 4291 struct bnx2x *bp = netdev_priv(dev); 4292 int rc = 0; 4293 4294 if (!is_valid_ether_addr(addr->sa_data)) { 4295 BNX2X_ERR("Requested MAC address is not valid\n"); 4296 return -EINVAL; 4297 } 4298 4299 if (IS_MF_STORAGE_ONLY(bp)) { 4300 BNX2X_ERR("Can't change address on STORAGE ONLY function\n"); 4301 return -EINVAL; 4302 } 4303 4304 if (netif_running(dev)) { 4305 rc = bnx2x_set_eth_mac(bp, false); 4306 if (rc) 4307 return rc; 4308 } 4309 4310 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 4311 4312 if (netif_running(dev)) 4313 rc = bnx2x_set_eth_mac(bp, true); 4314 4315 if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg)) 4316 SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS); 4317 4318 return rc; 4319 } 4320 4321 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index) 4322 { 4323 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk); 4324 struct bnx2x_fastpath *fp = &bp->fp[fp_index]; 4325 u8 cos; 4326 4327 /* Common */ 4328 4329 if (IS_FCOE_IDX(fp_index)) { 4330 memset(sb, 0, sizeof(union host_hc_status_block)); 4331 fp->status_blk_mapping = 0; 4332 } else { 4333 /* status blocks */ 4334 if (!CHIP_IS_E1x(bp)) 4335 BNX2X_PCI_FREE(sb->e2_sb, 4336 bnx2x_fp(bp, fp_index, 4337 status_blk_mapping), 4338 sizeof(struct host_hc_status_block_e2)); 4339 else 4340 BNX2X_PCI_FREE(sb->e1x_sb, 4341 bnx2x_fp(bp, fp_index, 4342 status_blk_mapping), 4343 sizeof(struct host_hc_status_block_e1x)); 4344 } 4345 4346 /* Rx */ 4347 if (!skip_rx_queue(bp, fp_index)) { 4348 bnx2x_free_rx_bds(fp); 4349 4350 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4351 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring)); 4352 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring), 4353 bnx2x_fp(bp, fp_index, rx_desc_mapping), 4354 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4355 4356 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring), 4357 bnx2x_fp(bp, fp_index, rx_comp_mapping), 4358 sizeof(struct eth_fast_path_rx_cqe) * 4359 NUM_RCQ_BD); 4360 4361 /* SGE ring */ 4362 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring)); 4363 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring), 4364 bnx2x_fp(bp, fp_index, rx_sge_mapping), 4365 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4366 } 4367 4368 /* Tx */ 4369 if (!skip_tx_queue(bp, fp_index)) { 4370 /* fastpath tx rings: tx_buf tx_desc */ 4371 for_each_cos_in_tx_queue(fp, cos) { 4372 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4373 4374 DP(NETIF_MSG_IFDOWN, 4375 "freeing tx memory of fp %d cos %d cid %d\n", 4376 fp_index, cos, txdata->cid); 4377 4378 BNX2X_FREE(txdata->tx_buf_ring); 4379 BNX2X_PCI_FREE(txdata->tx_desc_ring, 4380 txdata->tx_desc_mapping, 4381 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4382 } 4383 } 4384 /* end of fastpath */ 4385 } 4386 4387 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp) 4388 { 4389 int i; 4390 for_each_cnic_queue(bp, i) 4391 bnx2x_free_fp_mem_at(bp, i); 4392 } 4393 4394 void bnx2x_free_fp_mem(struct bnx2x *bp) 4395 { 4396 int i; 4397 for_each_eth_queue(bp, i) 4398 bnx2x_free_fp_mem_at(bp, i); 4399 } 4400 4401 static void set_sb_shortcuts(struct bnx2x *bp, int index) 4402 { 4403 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk); 4404 if (!CHIP_IS_E1x(bp)) { 4405 bnx2x_fp(bp, index, sb_index_values) = 4406 (__le16 *)status_blk.e2_sb->sb.index_values; 4407 bnx2x_fp(bp, index, sb_running_index) = 4408 (__le16 *)status_blk.e2_sb->sb.running_index; 4409 } else { 4410 bnx2x_fp(bp, index, sb_index_values) = 4411 (__le16 *)status_blk.e1x_sb->sb.index_values; 4412 bnx2x_fp(bp, index, sb_running_index) = 4413 (__le16 *)status_blk.e1x_sb->sb.running_index; 4414 } 4415 } 4416 4417 /* Returns the number of actually allocated BDs */ 4418 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp, 4419 int rx_ring_size) 4420 { 4421 struct bnx2x *bp = fp->bp; 4422 u16 ring_prod, cqe_ring_prod; 4423 int i, failure_cnt = 0; 4424 4425 fp->rx_comp_cons = 0; 4426 cqe_ring_prod = ring_prod = 0; 4427 4428 /* This routine is called only during fo init so 4429 * fp->eth_q_stats.rx_skb_alloc_failed = 0 4430 */ 4431 for (i = 0; i < rx_ring_size; i++) { 4432 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) { 4433 failure_cnt++; 4434 continue; 4435 } 4436 ring_prod = NEXT_RX_IDX(ring_prod); 4437 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod); 4438 WARN_ON(ring_prod <= (i - failure_cnt)); 4439 } 4440 4441 if (failure_cnt) 4442 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n", 4443 i - failure_cnt, fp->index); 4444 4445 fp->rx_bd_prod = ring_prod; 4446 /* Limit the CQE producer by the CQE ring size */ 4447 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT, 4448 cqe_ring_prod); 4449 4450 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt; 4451 4452 return i - failure_cnt; 4453 } 4454 4455 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp) 4456 { 4457 int i; 4458 4459 for (i = 1; i <= NUM_RCQ_RINGS; i++) { 4460 struct eth_rx_cqe_next_page *nextpg; 4461 4462 nextpg = (struct eth_rx_cqe_next_page *) 4463 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1]; 4464 nextpg->addr_hi = 4465 cpu_to_le32(U64_HI(fp->rx_comp_mapping + 4466 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4467 nextpg->addr_lo = 4468 cpu_to_le32(U64_LO(fp->rx_comp_mapping + 4469 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 4470 } 4471 } 4472 4473 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index) 4474 { 4475 union host_hc_status_block *sb; 4476 struct bnx2x_fastpath *fp = &bp->fp[index]; 4477 int ring_size = 0; 4478 u8 cos; 4479 int rx_ring_size = 0; 4480 4481 if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) { 4482 rx_ring_size = MIN_RX_SIZE_NONTPA; 4483 bp->rx_ring_size = rx_ring_size; 4484 } else if (!bp->rx_ring_size) { 4485 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp); 4486 4487 if (CHIP_IS_E3(bp)) { 4488 u32 cfg = SHMEM_RD(bp, 4489 dev_info.port_hw_config[BP_PORT(bp)]. 4490 default_cfg); 4491 4492 /* Decrease ring size for 1G functions */ 4493 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) == 4494 PORT_HW_CFG_NET_SERDES_IF_SGMII) 4495 rx_ring_size /= 10; 4496 } 4497 4498 /* allocate at least number of buffers required by FW */ 4499 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA : 4500 MIN_RX_SIZE_TPA, rx_ring_size); 4501 4502 bp->rx_ring_size = rx_ring_size; 4503 } else /* if rx_ring_size specified - use it */ 4504 rx_ring_size = bp->rx_ring_size; 4505 4506 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size); 4507 4508 /* Common */ 4509 sb = &bnx2x_fp(bp, index, status_blk); 4510 4511 if (!IS_FCOE_IDX(index)) { 4512 /* status blocks */ 4513 if (!CHIP_IS_E1x(bp)) { 4514 sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping), 4515 sizeof(struct host_hc_status_block_e2)); 4516 if (!sb->e2_sb) 4517 goto alloc_mem_err; 4518 } else { 4519 sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping), 4520 sizeof(struct host_hc_status_block_e1x)); 4521 if (!sb->e1x_sb) 4522 goto alloc_mem_err; 4523 } 4524 } 4525 4526 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to 4527 * set shortcuts for it. 4528 */ 4529 if (!IS_FCOE_IDX(index)) 4530 set_sb_shortcuts(bp, index); 4531 4532 /* Tx */ 4533 if (!skip_tx_queue(bp, index)) { 4534 /* fastpath tx rings: tx_buf tx_desc */ 4535 for_each_cos_in_tx_queue(fp, cos) { 4536 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos]; 4537 4538 DP(NETIF_MSG_IFUP, 4539 "allocating tx memory of fp %d cos %d\n", 4540 index, cos); 4541 4542 txdata->tx_buf_ring = kcalloc(NUM_TX_BD, 4543 sizeof(struct sw_tx_bd), 4544 GFP_KERNEL); 4545 if (!txdata->tx_buf_ring) 4546 goto alloc_mem_err; 4547 txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping, 4548 sizeof(union eth_tx_bd_types) * NUM_TX_BD); 4549 if (!txdata->tx_desc_ring) 4550 goto alloc_mem_err; 4551 } 4552 } 4553 4554 /* Rx */ 4555 if (!skip_rx_queue(bp, index)) { 4556 /* fastpath rx rings: rx_buf rx_desc rx_comp */ 4557 bnx2x_fp(bp, index, rx_buf_ring) = 4558 kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL); 4559 if (!bnx2x_fp(bp, index, rx_buf_ring)) 4560 goto alloc_mem_err; 4561 bnx2x_fp(bp, index, rx_desc_ring) = 4562 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping), 4563 sizeof(struct eth_rx_bd) * NUM_RX_BD); 4564 if (!bnx2x_fp(bp, index, rx_desc_ring)) 4565 goto alloc_mem_err; 4566 4567 /* Seed all CQEs by 1s */ 4568 bnx2x_fp(bp, index, rx_comp_ring) = 4569 BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping), 4570 sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD); 4571 if (!bnx2x_fp(bp, index, rx_comp_ring)) 4572 goto alloc_mem_err; 4573 4574 /* SGE ring */ 4575 bnx2x_fp(bp, index, rx_page_ring) = 4576 kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page), 4577 GFP_KERNEL); 4578 if (!bnx2x_fp(bp, index, rx_page_ring)) 4579 goto alloc_mem_err; 4580 bnx2x_fp(bp, index, rx_sge_ring) = 4581 BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping), 4582 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES); 4583 if (!bnx2x_fp(bp, index, rx_sge_ring)) 4584 goto alloc_mem_err; 4585 /* RX BD ring */ 4586 bnx2x_set_next_page_rx_bd(fp); 4587 4588 /* CQ ring */ 4589 bnx2x_set_next_page_rx_cq(fp); 4590 4591 /* BDs */ 4592 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size); 4593 if (ring_size < rx_ring_size) 4594 goto alloc_mem_err; 4595 } 4596 4597 return 0; 4598 4599 /* handles low memory cases */ 4600 alloc_mem_err: 4601 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n", 4602 index, ring_size); 4603 /* FW will drop all packets if queue is not big enough, 4604 * In these cases we disable the queue 4605 * Min size is different for OOO, TPA and non-TPA queues 4606 */ 4607 if (ring_size < (fp->mode == TPA_MODE_DISABLED ? 4608 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) { 4609 /* release memory allocated for this queue */ 4610 bnx2x_free_fp_mem_at(bp, index); 4611 return -ENOMEM; 4612 } 4613 return 0; 4614 } 4615 4616 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp) 4617 { 4618 if (!NO_FCOE(bp)) 4619 /* FCoE */ 4620 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp))) 4621 /* we will fail load process instead of mark 4622 * NO_FCOE_FLAG 4623 */ 4624 return -ENOMEM; 4625 4626 return 0; 4627 } 4628 4629 static int bnx2x_alloc_fp_mem(struct bnx2x *bp) 4630 { 4631 int i; 4632 4633 /* 1. Allocate FP for leading - fatal if error 4634 * 2. Allocate RSS - fix number of queues if error 4635 */ 4636 4637 /* leading */ 4638 if (bnx2x_alloc_fp_mem_at(bp, 0)) 4639 return -ENOMEM; 4640 4641 /* RSS */ 4642 for_each_nondefault_eth_queue(bp, i) 4643 if (bnx2x_alloc_fp_mem_at(bp, i)) 4644 break; 4645 4646 /* handle memory failures */ 4647 if (i != BNX2X_NUM_ETH_QUEUES(bp)) { 4648 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i; 4649 4650 WARN_ON(delta < 0); 4651 bnx2x_shrink_eth_fp(bp, delta); 4652 if (CNIC_SUPPORT(bp)) 4653 /* move non eth FPs next to last eth FP 4654 * must be done in that order 4655 * FCOE_IDX < FWD_IDX < OOO_IDX 4656 */ 4657 4658 /* move FCoE fp even NO_FCOE_FLAG is on */ 4659 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta); 4660 bp->num_ethernet_queues -= delta; 4661 bp->num_queues = bp->num_ethernet_queues + 4662 bp->num_cnic_queues; 4663 BNX2X_ERR("Adjusted num of queues from %d to %d\n", 4664 bp->num_queues + delta, bp->num_queues); 4665 } 4666 4667 return 0; 4668 } 4669 4670 void bnx2x_free_mem_bp(struct bnx2x *bp) 4671 { 4672 int i; 4673 4674 for (i = 0; i < bp->fp_array_size; i++) 4675 kfree(bp->fp[i].tpa_info); 4676 kfree(bp->fp); 4677 kfree(bp->sp_objs); 4678 kfree(bp->fp_stats); 4679 kfree(bp->bnx2x_txq); 4680 kfree(bp->msix_table); 4681 kfree(bp->ilt); 4682 } 4683 4684 int bnx2x_alloc_mem_bp(struct bnx2x *bp) 4685 { 4686 struct bnx2x_fastpath *fp; 4687 struct msix_entry *tbl; 4688 struct bnx2x_ilt *ilt; 4689 int msix_table_size = 0; 4690 int fp_array_size, txq_array_size; 4691 int i; 4692 4693 /* 4694 * The biggest MSI-X table we might need is as a maximum number of fast 4695 * path IGU SBs plus default SB (for PF only). 4696 */ 4697 msix_table_size = bp->igu_sb_cnt; 4698 if (IS_PF(bp)) 4699 msix_table_size++; 4700 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size); 4701 4702 /* fp array: RSS plus CNIC related L2 queues */ 4703 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp); 4704 bp->fp_array_size = fp_array_size; 4705 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size); 4706 4707 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL); 4708 if (!fp) 4709 goto alloc_err; 4710 for (i = 0; i < bp->fp_array_size; i++) { 4711 fp[i].tpa_info = 4712 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2, 4713 sizeof(struct bnx2x_agg_info), GFP_KERNEL); 4714 if (!(fp[i].tpa_info)) 4715 goto alloc_err; 4716 } 4717 4718 bp->fp = fp; 4719 4720 /* allocate sp objs */ 4721 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs), 4722 GFP_KERNEL); 4723 if (!bp->sp_objs) 4724 goto alloc_err; 4725 4726 /* allocate fp_stats */ 4727 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats), 4728 GFP_KERNEL); 4729 if (!bp->fp_stats) 4730 goto alloc_err; 4731 4732 /* Allocate memory for the transmission queues array */ 4733 txq_array_size = 4734 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp); 4735 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size); 4736 4737 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata), 4738 GFP_KERNEL); 4739 if (!bp->bnx2x_txq) 4740 goto alloc_err; 4741 4742 /* msix table */ 4743 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL); 4744 if (!tbl) 4745 goto alloc_err; 4746 bp->msix_table = tbl; 4747 4748 /* ilt */ 4749 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL); 4750 if (!ilt) 4751 goto alloc_err; 4752 bp->ilt = ilt; 4753 4754 return 0; 4755 alloc_err: 4756 bnx2x_free_mem_bp(bp); 4757 return -ENOMEM; 4758 } 4759 4760 int bnx2x_reload_if_running(struct net_device *dev) 4761 { 4762 struct bnx2x *bp = netdev_priv(dev); 4763 4764 if (unlikely(!netif_running(dev))) 4765 return 0; 4766 4767 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true); 4768 return bnx2x_nic_load(bp, LOAD_NORMAL); 4769 } 4770 4771 int bnx2x_get_cur_phy_idx(struct bnx2x *bp) 4772 { 4773 u32 sel_phy_idx = 0; 4774 if (bp->link_params.num_phys <= 1) 4775 return INT_PHY; 4776 4777 if (bp->link_vars.link_up) { 4778 sel_phy_idx = EXT_PHY1; 4779 /* In case link is SERDES, check if the EXT_PHY2 is the one */ 4780 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) && 4781 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE)) 4782 sel_phy_idx = EXT_PHY2; 4783 } else { 4784 4785 switch (bnx2x_phy_selection(&bp->link_params)) { 4786 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT: 4787 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY: 4788 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY: 4789 sel_phy_idx = EXT_PHY1; 4790 break; 4791 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY: 4792 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY: 4793 sel_phy_idx = EXT_PHY2; 4794 break; 4795 } 4796 } 4797 4798 return sel_phy_idx; 4799 } 4800 int bnx2x_get_link_cfg_idx(struct bnx2x *bp) 4801 { 4802 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp); 4803 /* 4804 * The selected activated PHY is always after swapping (in case PHY 4805 * swapping is enabled). So when swapping is enabled, we need to reverse 4806 * the configuration 4807 */ 4808 4809 if (bp->link_params.multi_phy_config & 4810 PORT_HW_CFG_PHY_SWAPPED_ENABLED) { 4811 if (sel_phy_idx == EXT_PHY1) 4812 sel_phy_idx = EXT_PHY2; 4813 else if (sel_phy_idx == EXT_PHY2) 4814 sel_phy_idx = EXT_PHY1; 4815 } 4816 return LINK_CONFIG_IDX(sel_phy_idx); 4817 } 4818 4819 #ifdef NETDEV_FCOE_WWNN 4820 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type) 4821 { 4822 struct bnx2x *bp = netdev_priv(dev); 4823 struct cnic_eth_dev *cp = &bp->cnic_eth_dev; 4824 4825 switch (type) { 4826 case NETDEV_FCOE_WWNN: 4827 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi, 4828 cp->fcoe_wwn_node_name_lo); 4829 break; 4830 case NETDEV_FCOE_WWPN: 4831 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi, 4832 cp->fcoe_wwn_port_name_lo); 4833 break; 4834 default: 4835 BNX2X_ERR("Wrong WWN type requested - %d\n", type); 4836 return -EINVAL; 4837 } 4838 4839 return 0; 4840 } 4841 #endif 4842 4843 /* called with rtnl_lock */ 4844 int bnx2x_change_mtu(struct net_device *dev, int new_mtu) 4845 { 4846 struct bnx2x *bp = netdev_priv(dev); 4847 4848 if (pci_num_vf(bp->pdev)) { 4849 DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n"); 4850 return -EPERM; 4851 } 4852 4853 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 4854 BNX2X_ERR("Can't perform change MTU during parity recovery\n"); 4855 return -EAGAIN; 4856 } 4857 4858 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) || 4859 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) { 4860 BNX2X_ERR("Can't support requested MTU size\n"); 4861 return -EINVAL; 4862 } 4863 4864 /* This does not race with packet allocation 4865 * because the actual alloc size is 4866 * only updated as part of load 4867 */ 4868 dev->mtu = new_mtu; 4869 4870 if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg)) 4871 SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS); 4872 4873 return bnx2x_reload_if_running(dev); 4874 } 4875 4876 netdev_features_t bnx2x_fix_features(struct net_device *dev, 4877 netdev_features_t features) 4878 { 4879 struct bnx2x *bp = netdev_priv(dev); 4880 4881 if (pci_num_vf(bp->pdev)) { 4882 netdev_features_t changed = dev->features ^ features; 4883 4884 /* Revert the requested changes in features if they 4885 * would require internal reload of PF in bnx2x_set_features(). 4886 */ 4887 if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) { 4888 features &= ~NETIF_F_RXCSUM; 4889 features |= dev->features & NETIF_F_RXCSUM; 4890 } 4891 4892 if (changed & NETIF_F_LOOPBACK) { 4893 features &= ~NETIF_F_LOOPBACK; 4894 features |= dev->features & NETIF_F_LOOPBACK; 4895 } 4896 } 4897 4898 /* TPA requires Rx CSUM offloading */ 4899 if (!(features & NETIF_F_RXCSUM)) { 4900 features &= ~NETIF_F_LRO; 4901 features &= ~NETIF_F_GRO; 4902 } 4903 4904 return features; 4905 } 4906 4907 int bnx2x_set_features(struct net_device *dev, netdev_features_t features) 4908 { 4909 struct bnx2x *bp = netdev_priv(dev); 4910 netdev_features_t changes = features ^ dev->features; 4911 bool bnx2x_reload = false; 4912 int rc; 4913 4914 /* VFs or non SRIOV PFs should be able to change loopback feature */ 4915 if (!pci_num_vf(bp->pdev)) { 4916 if (features & NETIF_F_LOOPBACK) { 4917 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) { 4918 bp->link_params.loopback_mode = LOOPBACK_BMAC; 4919 bnx2x_reload = true; 4920 } 4921 } else { 4922 if (bp->link_params.loopback_mode != LOOPBACK_NONE) { 4923 bp->link_params.loopback_mode = LOOPBACK_NONE; 4924 bnx2x_reload = true; 4925 } 4926 } 4927 } 4928 4929 /* if GRO is changed while LRO is enabled, don't force a reload */ 4930 if ((changes & NETIF_F_GRO) && (features & NETIF_F_LRO)) 4931 changes &= ~NETIF_F_GRO; 4932 4933 /* if GRO is changed while HW TPA is off, don't force a reload */ 4934 if ((changes & NETIF_F_GRO) && bp->disable_tpa) 4935 changes &= ~NETIF_F_GRO; 4936 4937 if (changes) 4938 bnx2x_reload = true; 4939 4940 if (bnx2x_reload) { 4941 if (bp->recovery_state == BNX2X_RECOVERY_DONE) { 4942 dev->features = features; 4943 rc = bnx2x_reload_if_running(dev); 4944 return rc ? rc : 1; 4945 } 4946 /* else: bnx2x_nic_load() will be called at end of recovery */ 4947 } 4948 4949 return 0; 4950 } 4951 4952 void bnx2x_tx_timeout(struct net_device *dev) 4953 { 4954 struct bnx2x *bp = netdev_priv(dev); 4955 4956 #ifdef BNX2X_STOP_ON_ERROR 4957 if (!bp->panic) 4958 bnx2x_panic(); 4959 #endif 4960 4961 /* This allows the netif to be shutdown gracefully before resetting */ 4962 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0); 4963 } 4964 4965 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state) 4966 { 4967 struct net_device *dev = pci_get_drvdata(pdev); 4968 struct bnx2x *bp; 4969 4970 if (!dev) { 4971 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 4972 return -ENODEV; 4973 } 4974 bp = netdev_priv(dev); 4975 4976 rtnl_lock(); 4977 4978 pci_save_state(pdev); 4979 4980 if (!netif_running(dev)) { 4981 rtnl_unlock(); 4982 return 0; 4983 } 4984 4985 netif_device_detach(dev); 4986 4987 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false); 4988 4989 bnx2x_set_power_state(bp, pci_choose_state(pdev, state)); 4990 4991 rtnl_unlock(); 4992 4993 return 0; 4994 } 4995 4996 int bnx2x_resume(struct pci_dev *pdev) 4997 { 4998 struct net_device *dev = pci_get_drvdata(pdev); 4999 struct bnx2x *bp; 5000 int rc; 5001 5002 if (!dev) { 5003 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 5004 return -ENODEV; 5005 } 5006 bp = netdev_priv(dev); 5007 5008 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 5009 BNX2X_ERR("Handling parity error recovery. Try again later\n"); 5010 return -EAGAIN; 5011 } 5012 5013 rtnl_lock(); 5014 5015 pci_restore_state(pdev); 5016 5017 if (!netif_running(dev)) { 5018 rtnl_unlock(); 5019 return 0; 5020 } 5021 5022 bnx2x_set_power_state(bp, PCI_D0); 5023 netif_device_attach(dev); 5024 5025 rc = bnx2x_nic_load(bp, LOAD_OPEN); 5026 5027 rtnl_unlock(); 5028 5029 return rc; 5030 } 5031 5032 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt, 5033 u32 cid) 5034 { 5035 if (!cxt) { 5036 BNX2X_ERR("bad context pointer %p\n", cxt); 5037 return; 5038 } 5039 5040 /* ustorm cxt validation */ 5041 cxt->ustorm_ag_context.cdu_usage = 5042 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 5043 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE); 5044 /* xcontext validation */ 5045 cxt->xstorm_ag_context.cdu_reserved = 5046 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid), 5047 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE); 5048 } 5049 5050 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port, 5051 u8 fw_sb_id, u8 sb_index, 5052 u8 ticks) 5053 { 5054 u32 addr = BAR_CSTRORM_INTMEM + 5055 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index); 5056 REG_WR8(bp, addr, ticks); 5057 DP(NETIF_MSG_IFUP, 5058 "port %x fw_sb_id %d sb_index %d ticks %d\n", 5059 port, fw_sb_id, sb_index, ticks); 5060 } 5061 5062 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port, 5063 u16 fw_sb_id, u8 sb_index, 5064 u8 disable) 5065 { 5066 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT); 5067 u32 addr = BAR_CSTRORM_INTMEM + 5068 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index); 5069 u8 flags = REG_RD8(bp, addr); 5070 /* clear and set */ 5071 flags &= ~HC_INDEX_DATA_HC_ENABLED; 5072 flags |= enable_flag; 5073 REG_WR8(bp, addr, flags); 5074 DP(NETIF_MSG_IFUP, 5075 "port %x fw_sb_id %d sb_index %d disable %d\n", 5076 port, fw_sb_id, sb_index, disable); 5077 } 5078 5079 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id, 5080 u8 sb_index, u8 disable, u16 usec) 5081 { 5082 int port = BP_PORT(bp); 5083 u8 ticks = usec / BNX2X_BTR; 5084 5085 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks); 5086 5087 disable = disable ? 1 : (usec ? 0 : 1); 5088 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable); 5089 } 5090 5091 void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag, 5092 u32 verbose) 5093 { 5094 smp_mb__before_atomic(); 5095 set_bit(flag, &bp->sp_rtnl_state); 5096 smp_mb__after_atomic(); 5097 DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n", 5098 flag); 5099 schedule_delayed_work(&bp->sp_rtnl_task, 0); 5100 } 5101