1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2020 Intel Corporation. 4 * 5 */ 6 7 /* 8 * This file contains HFI1 support for IPOIB SDMA functionality 9 */ 10 11 #include <linux/log2.h> 12 #include <linux/circ_buf.h> 13 14 #include "sdma.h" 15 #include "verbs.h" 16 #include "trace_ibhdrs.h" 17 #include "ipoib.h" 18 19 /* Add a convenience helper */ 20 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 21 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 22 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 23 24 /** 25 * struct ipoib_txreq - IPOIB transmit descriptor 26 * @txreq: sdma transmit request 27 * @sdma_hdr: 9b ib headers 28 * @sdma_status: status returned by sdma engine 29 * @priv: ipoib netdev private data 30 * @txq: txq on which skb was output 31 * @skb: skb to send 32 */ 33 struct ipoib_txreq { 34 struct sdma_txreq txreq; 35 struct hfi1_sdma_header sdma_hdr; 36 int sdma_status; 37 struct hfi1_ipoib_dev_priv *priv; 38 struct hfi1_ipoib_txq *txq; 39 struct sk_buff *skb; 40 }; 41 42 struct ipoib_txparms { 43 struct hfi1_devdata *dd; 44 struct rdma_ah_attr *ah_attr; 45 struct hfi1_ibport *ibp; 46 struct hfi1_ipoib_txq *txq; 47 union hfi1_ipoib_flow flow; 48 u32 dqpn; 49 u8 hdr_dwords; 50 u8 entropy; 51 }; 52 53 static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed) 54 { 55 return sent - completed; 56 } 57 58 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq) 59 { 60 return hfi1_ipoib_txreqs(txq->sent_txreqs, 61 atomic64_read(&txq->complete_txreqs)); 62 } 63 64 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq) 65 { 66 if (atomic_inc_return(&txq->stops) == 1) 67 netif_stop_subqueue(txq->priv->netdev, txq->q_idx); 68 } 69 70 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq) 71 { 72 if (atomic_dec_and_test(&txq->stops)) 73 netif_wake_subqueue(txq->priv->netdev, txq->q_idx); 74 } 75 76 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq) 77 { 78 return min_t(uint, txq->priv->netdev->tx_queue_len, 79 txq->tx_ring.max_items - 1); 80 } 81 82 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq) 83 { 84 return min_t(uint, txq->priv->netdev->tx_queue_len, 85 txq->tx_ring.max_items) >> 1; 86 } 87 88 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) 89 { 90 ++txq->sent_txreqs; 91 if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) && 92 !atomic_xchg(&txq->ring_full, 1)) 93 hfi1_ipoib_stop_txq(txq); 94 } 95 96 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) 97 { 98 struct net_device *dev = txq->priv->netdev; 99 100 /* If shutting down just return as queue state is irrelevant */ 101 if (unlikely(dev->reg_state != NETREG_REGISTERED)) 102 return; 103 104 /* 105 * When the queue has been drained to less than half full it will be 106 * restarted. 107 * The size of the txreq ring is fixed at initialization. 108 * The tx queue len can be adjusted upward while the interface is 109 * running. 110 * The tx queue len can be large enough to overflow the txreq_ring. 111 * Use the minimum of the current tx_queue_len or the rings max txreqs 112 * to protect against ring overflow. 113 */ 114 if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) && 115 atomic_xchg(&txq->ring_full, 0)) 116 hfi1_ipoib_wake_txq(txq); 117 } 118 119 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget) 120 { 121 struct hfi1_ipoib_dev_priv *priv = tx->priv; 122 123 if (likely(!tx->sdma_status)) { 124 hfi1_ipoib_update_tx_netstats(priv, 1, tx->skb->len); 125 } else { 126 ++priv->netdev->stats.tx_errors; 127 dd_dev_warn(priv->dd, 128 "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n", 129 __func__, tx->sdma_status, 130 le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx, 131 tx->txq->sde->this_idx); 132 } 133 134 napi_consume_skb(tx->skb, budget); 135 sdma_txclean(priv->dd, &tx->txreq); 136 kmem_cache_free(priv->txreq_cache, tx); 137 } 138 139 static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget) 140 { 141 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 142 unsigned long head; 143 unsigned long tail; 144 unsigned int max_tx; 145 int work_done; 146 int tx_count; 147 148 spin_lock_bh(&tx_ring->consumer_lock); 149 150 /* Read index before reading contents at that index. */ 151 head = smp_load_acquire(&tx_ring->head); 152 tail = tx_ring->tail; 153 max_tx = tx_ring->max_items; 154 155 work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget); 156 157 for (tx_count = work_done; tx_count; tx_count--) { 158 hfi1_ipoib_free_tx(tx_ring->items[tail], budget); 159 tail = CIRC_NEXT(tail, max_tx); 160 } 161 162 atomic64_add(work_done, &txq->complete_txreqs); 163 164 /* Finished freeing tx items so store the tail value. */ 165 smp_store_release(&tx_ring->tail, tail); 166 167 spin_unlock_bh(&tx_ring->consumer_lock); 168 169 hfi1_ipoib_check_queue_stopped(txq); 170 171 return work_done; 172 } 173 174 static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget) 175 { 176 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev); 177 struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis]; 178 179 int work_done = hfi1_ipoib_drain_tx_ring(txq, budget); 180 181 if (work_done < budget) 182 napi_complete_done(napi, work_done); 183 184 return work_done; 185 } 186 187 static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx) 188 { 189 struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring; 190 unsigned long head; 191 unsigned long tail; 192 size_t max_tx; 193 194 spin_lock(&tx_ring->producer_lock); 195 196 head = tx_ring->head; 197 tail = READ_ONCE(tx_ring->tail); 198 max_tx = tx_ring->max_items; 199 200 if (likely(CIRC_SPACE(head, tail, max_tx))) { 201 tx_ring->items[head] = tx; 202 203 /* Finish storing txreq before incrementing head. */ 204 smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx)); 205 napi_schedule(tx->txq->napi); 206 } else { 207 struct hfi1_ipoib_txq *txq = tx->txq; 208 struct hfi1_ipoib_dev_priv *priv = tx->priv; 209 210 /* Ring was full */ 211 hfi1_ipoib_free_tx(tx, 0); 212 atomic64_inc(&txq->complete_txreqs); 213 dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx); 214 } 215 216 spin_unlock(&tx_ring->producer_lock); 217 } 218 219 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status) 220 { 221 struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq); 222 223 tx->sdma_status = status; 224 225 hfi1_ipoib_add_tx(tx); 226 } 227 228 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx, 229 struct ipoib_txparms *txp) 230 { 231 struct hfi1_devdata *dd = txp->dd; 232 struct sdma_txreq *txreq = &tx->txreq; 233 struct sk_buff *skb = tx->skb; 234 int ret = 0; 235 int i; 236 237 if (skb_headlen(skb)) { 238 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb)); 239 if (unlikely(ret)) 240 return ret; 241 } 242 243 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 244 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 245 246 ret = sdma_txadd_page(dd, 247 txreq, 248 skb_frag_page(frag), 249 frag->bv_offset, 250 skb_frag_size(frag)); 251 if (unlikely(ret)) 252 break; 253 } 254 255 return ret; 256 } 257 258 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx, 259 struct ipoib_txparms *txp) 260 { 261 struct hfi1_devdata *dd = txp->dd; 262 struct sdma_txreq *txreq = &tx->txreq; 263 struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr; 264 u16 pkt_bytes = 265 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len; 266 int ret; 267 268 ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete); 269 if (unlikely(ret)) 270 return ret; 271 272 /* add pbc + headers */ 273 ret = sdma_txadd_kvaddr(dd, 274 txreq, 275 sdma_hdr, 276 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2)); 277 if (unlikely(ret)) 278 return ret; 279 280 /* add the ulp payload */ 281 return hfi1_ipoib_build_ulp_payload(tx, txp); 282 } 283 284 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx, 285 struct ipoib_txparms *txp) 286 { 287 struct hfi1_ipoib_dev_priv *priv = tx->priv; 288 struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr; 289 struct sk_buff *skb = tx->skb; 290 struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp); 291 struct rdma_ah_attr *ah_attr = txp->ah_attr; 292 struct ib_other_headers *ohdr; 293 struct ib_grh *grh; 294 u16 dwords; 295 u16 slid; 296 u16 dlid; 297 u16 lrh0; 298 u32 bth0; 299 u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 | 300 priv->netdev->dev_addr[2] << 8 | 301 priv->netdev->dev_addr[3]); 302 u16 payload_dwords; 303 u8 pad_cnt; 304 305 pad_cnt = -skb->len & 3; 306 307 /* Includes ICRC */ 308 payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC; 309 310 /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ 311 txp->hdr_dwords = 7; 312 313 if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { 314 grh = &sdma_hdr->hdr.ibh.u.l.grh; 315 txp->hdr_dwords += 316 hfi1_make_grh(txp->ibp, 317 grh, 318 rdma_ah_read_grh(ah_attr), 319 txp->hdr_dwords - LRH_9B_DWORDS, 320 payload_dwords); 321 lrh0 = HFI1_LRH_GRH; 322 ohdr = &sdma_hdr->hdr.ibh.u.l.oth; 323 } else { 324 lrh0 = HFI1_LRH_BTH; 325 ohdr = &sdma_hdr->hdr.ibh.u.oth; 326 } 327 328 lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4; 329 lrh0 |= (txp->flow.sc5 & 0xf) << 12; 330 331 dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B); 332 if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) { 333 slid = be16_to_cpu(IB_LID_PERMISSIVE); 334 } else { 335 u16 lid = (u16)ppd->lid; 336 337 if (lid) { 338 lid |= rdma_ah_get_path_bits(ah_attr) & 339 ((1 << ppd->lmc) - 1); 340 slid = lid; 341 } else { 342 slid = be16_to_cpu(IB_LID_PERMISSIVE); 343 } 344 } 345 346 /* Includes ICRC */ 347 dwords = txp->hdr_dwords + payload_dwords; 348 349 /* Build the lrh */ 350 sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B; 351 hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid); 352 353 /* Build the bth */ 354 bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey; 355 356 ohdr->bth[0] = cpu_to_be32(bth0); 357 ohdr->bth[1] = cpu_to_be32(txp->dqpn); 358 ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs)); 359 360 /* Build the deth */ 361 ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey); 362 ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy << 363 HFI1_IPOIB_ENTROPY_SHIFT) | sqpn); 364 365 /* Construct the pbc. */ 366 sdma_hdr->pbc = 367 cpu_to_le64(create_pbc(ppd, 368 ib_is_sc5(txp->flow.sc5) << 369 PBC_DC_INFO_SHIFT, 370 0, 371 sc_to_vlt(priv->dd, txp->flow.sc5), 372 dwords - SIZE_OF_CRC + 373 (sizeof(sdma_hdr->pbc) >> 2))); 374 } 375 376 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev, 377 struct sk_buff *skb, 378 struct ipoib_txparms *txp) 379 { 380 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 381 struct ipoib_txreq *tx; 382 int ret; 383 384 tx = kmem_cache_alloc_node(priv->txreq_cache, 385 GFP_ATOMIC, 386 priv->dd->node); 387 if (unlikely(!tx)) 388 return ERR_PTR(-ENOMEM); 389 390 /* so that we can test if the sdma descriptors are there */ 391 tx->txreq.num_desc = 0; 392 tx->priv = priv; 393 tx->txq = txp->txq; 394 tx->skb = skb; 395 INIT_LIST_HEAD(&tx->txreq.list); 396 397 hfi1_ipoib_build_ib_tx_headers(tx, txp); 398 399 ret = hfi1_ipoib_build_tx_desc(tx, txp); 400 if (likely(!ret)) { 401 if (txp->txq->flow.as_int != txp->flow.as_int) { 402 txp->txq->flow.tx_queue = txp->flow.tx_queue; 403 txp->txq->flow.sc5 = txp->flow.sc5; 404 txp->txq->sde = 405 sdma_select_engine_sc(priv->dd, 406 txp->flow.tx_queue, 407 txp->flow.sc5); 408 } 409 410 return tx; 411 } 412 413 sdma_txclean(priv->dd, &tx->txreq); 414 kmem_cache_free(priv->txreq_cache, tx); 415 416 return ERR_PTR(ret); 417 } 418 419 static int hfi1_ipoib_submit_tx_list(struct net_device *dev, 420 struct hfi1_ipoib_txq *txq) 421 { 422 int ret; 423 u16 count_out; 424 425 ret = sdma_send_txlist(txq->sde, 426 iowait_get_ib_work(&txq->wait), 427 &txq->tx_list, 428 &count_out); 429 if (likely(!ret) || ret == -EBUSY || ret == -ECOMM) 430 return ret; 431 432 dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret); 433 434 return ret; 435 } 436 437 static int hfi1_ipoib_flush_tx_list(struct net_device *dev, 438 struct hfi1_ipoib_txq *txq) 439 { 440 int ret = 0; 441 442 if (!list_empty(&txq->tx_list)) { 443 /* Flush the current list */ 444 ret = hfi1_ipoib_submit_tx_list(dev, txq); 445 446 if (unlikely(ret)) 447 if (ret != -EBUSY) 448 ++dev->stats.tx_carrier_errors; 449 } 450 451 return ret; 452 } 453 454 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq, 455 struct ipoib_txreq *tx) 456 { 457 int ret; 458 459 ret = sdma_send_txreq(txq->sde, 460 iowait_get_ib_work(&txq->wait), 461 &tx->txreq, 462 txq->pkts_sent); 463 if (likely(!ret)) { 464 txq->pkts_sent = true; 465 iowait_starve_clear(txq->pkts_sent, &txq->wait); 466 } 467 468 return ret; 469 } 470 471 static int hfi1_ipoib_send_dma_single(struct net_device *dev, 472 struct sk_buff *skb, 473 struct ipoib_txparms *txp) 474 { 475 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 476 struct hfi1_ipoib_txq *txq = txp->txq; 477 struct ipoib_txreq *tx; 478 int ret; 479 480 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 481 if (IS_ERR(tx)) { 482 int ret = PTR_ERR(tx); 483 484 dev_kfree_skb_any(skb); 485 486 if (ret == -ENOMEM) 487 ++dev->stats.tx_errors; 488 else 489 ++dev->stats.tx_carrier_errors; 490 491 return NETDEV_TX_OK; 492 } 493 494 ret = hfi1_ipoib_submit_tx(txq, tx); 495 if (likely(!ret)) { 496 tx_ok: 497 trace_sdma_output_ibhdr(tx->priv->dd, 498 &tx->sdma_hdr.hdr, 499 ib_is_sc5(txp->flow.sc5)); 500 hfi1_ipoib_check_queue_depth(txq); 501 return NETDEV_TX_OK; 502 } 503 504 txq->pkts_sent = false; 505 506 if (ret == -EBUSY || ret == -ECOMM) 507 goto tx_ok; 508 509 sdma_txclean(priv->dd, &tx->txreq); 510 dev_kfree_skb_any(skb); 511 kmem_cache_free(priv->txreq_cache, tx); 512 ++dev->stats.tx_carrier_errors; 513 514 return NETDEV_TX_OK; 515 } 516 517 static int hfi1_ipoib_send_dma_list(struct net_device *dev, 518 struct sk_buff *skb, 519 struct ipoib_txparms *txp) 520 { 521 struct hfi1_ipoib_txq *txq = txp->txq; 522 struct ipoib_txreq *tx; 523 524 /* Has the flow change ? */ 525 if (txq->flow.as_int != txp->flow.as_int) { 526 int ret; 527 528 ret = hfi1_ipoib_flush_tx_list(dev, txq); 529 if (unlikely(ret)) { 530 if (ret == -EBUSY) 531 ++dev->stats.tx_dropped; 532 dev_kfree_skb_any(skb); 533 return NETDEV_TX_OK; 534 } 535 } 536 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 537 if (IS_ERR(tx)) { 538 int ret = PTR_ERR(tx); 539 540 dev_kfree_skb_any(skb); 541 542 if (ret == -ENOMEM) 543 ++dev->stats.tx_errors; 544 else 545 ++dev->stats.tx_carrier_errors; 546 547 return NETDEV_TX_OK; 548 } 549 550 list_add_tail(&tx->txreq.list, &txq->tx_list); 551 552 hfi1_ipoib_check_queue_depth(txq); 553 554 trace_sdma_output_ibhdr(tx->priv->dd, 555 &tx->sdma_hdr.hdr, 556 ib_is_sc5(txp->flow.sc5)); 557 558 if (!netdev_xmit_more()) 559 (void)hfi1_ipoib_flush_tx_list(dev, txq); 560 561 return NETDEV_TX_OK; 562 } 563 564 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb) 565 { 566 if (skb_transport_header_was_set(skb)) { 567 u8 *hdr = (u8 *)skb_transport_header(skb); 568 569 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]); 570 } 571 572 return (u8)skb_get_queue_mapping(skb); 573 } 574 575 int hfi1_ipoib_send_dma(struct net_device *dev, 576 struct sk_buff *skb, 577 struct ib_ah *address, 578 u32 dqpn) 579 { 580 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 581 struct ipoib_txparms txp; 582 struct rdma_netdev *rn = netdev_priv(dev); 583 584 if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) { 585 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n", 586 skb->len, 587 rn->mtu + HFI1_IPOIB_ENCAP_LEN); 588 ++dev->stats.tx_dropped; 589 ++dev->stats.tx_errors; 590 dev_kfree_skb_any(skb); 591 return NETDEV_TX_OK; 592 } 593 594 txp.dd = priv->dd; 595 txp.ah_attr = &ibah_to_rvtah(address)->attr; 596 txp.ibp = to_iport(priv->device, priv->port_num); 597 txp.txq = &priv->txqs[skb_get_queue_mapping(skb)]; 598 txp.dqpn = dqpn; 599 txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)]; 600 txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb); 601 txp.entropy = hfi1_ipoib_calc_entropy(skb); 602 603 if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list)) 604 return hfi1_ipoib_send_dma_list(dev, skb, &txp); 605 606 return hfi1_ipoib_send_dma_single(dev, skb, &txp); 607 } 608 609 /* 610 * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function 611 * 612 * This function gets called from sdma_send_txreq() when there are not enough 613 * sdma descriptors available to send the packet. It adds Tx queue's wait 614 * structure to sdma engine's dmawait list to be woken up when descriptors 615 * become available. 616 */ 617 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde, 618 struct iowait_work *wait, 619 struct sdma_txreq *txreq, 620 uint seq, 621 bool pkts_sent) 622 { 623 struct hfi1_ipoib_txq *txq = 624 container_of(wait->iow, struct hfi1_ipoib_txq, wait); 625 626 write_seqlock(&sde->waitlock); 627 628 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) { 629 if (sdma_progress(sde, seq, txreq)) { 630 write_sequnlock(&sde->waitlock); 631 return -EAGAIN; 632 } 633 634 if (list_empty(&txreq->list)) 635 /* came from non-list submit */ 636 list_add_tail(&txreq->list, &txq->tx_list); 637 if (list_empty(&txq->wait.list)) { 638 if (!atomic_xchg(&txq->no_desc, 1)) 639 hfi1_ipoib_stop_txq(txq); 640 iowait_queue(pkts_sent, wait->iow, &sde->dmawait); 641 } 642 643 write_sequnlock(&sde->waitlock); 644 return -EBUSY; 645 } 646 647 write_sequnlock(&sde->waitlock); 648 return -EINVAL; 649 } 650 651 /* 652 * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function 653 * 654 * This function gets called when SDMA descriptors becomes available and Tx 655 * queue's wait structure was previously added to sdma engine's dmawait list. 656 */ 657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason) 658 { 659 struct hfi1_ipoib_txq *txq = 660 container_of(wait, struct hfi1_ipoib_txq, wait); 661 662 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) 663 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND); 664 } 665 666 static void hfi1_ipoib_flush_txq(struct work_struct *work) 667 { 668 struct iowait_work *ioww = 669 container_of(work, struct iowait_work, iowork); 670 struct iowait *wait = iowait_ioww_to_iow(ioww); 671 struct hfi1_ipoib_txq *txq = 672 container_of(wait, struct hfi1_ipoib_txq, wait); 673 struct net_device *dev = txq->priv->netdev; 674 675 if (likely(dev->reg_state == NETREG_REGISTERED) && 676 likely(!hfi1_ipoib_flush_tx_list(dev, txq))) 677 if (atomic_xchg(&txq->no_desc, 0)) 678 hfi1_ipoib_wake_txq(txq); 679 } 680 681 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) 682 { 683 struct net_device *dev = priv->netdev; 684 char buf[HFI1_IPOIB_TXREQ_NAME_LEN]; 685 unsigned long tx_ring_size; 686 int i; 687 688 /* 689 * Ring holds 1 less than tx_ring_size 690 * Round up to next power of 2 in order to hold at least tx_queue_len 691 */ 692 tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1); 693 694 snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit); 695 priv->txreq_cache = kmem_cache_create(buf, 696 sizeof(struct ipoib_txreq), 697 0, 698 0, 699 NULL); 700 if (!priv->txreq_cache) 701 return -ENOMEM; 702 703 priv->tx_napis = kcalloc_node(dev->num_tx_queues, 704 sizeof(struct napi_struct), 705 GFP_ATOMIC, 706 priv->dd->node); 707 if (!priv->tx_napis) 708 goto free_txreq_cache; 709 710 priv->txqs = kcalloc_node(dev->num_tx_queues, 711 sizeof(struct hfi1_ipoib_txq), 712 GFP_ATOMIC, 713 priv->dd->node); 714 if (!priv->txqs) 715 goto free_tx_napis; 716 717 for (i = 0; i < dev->num_tx_queues; i++) { 718 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 719 720 iowait_init(&txq->wait, 721 0, 722 hfi1_ipoib_flush_txq, 723 NULL, 724 hfi1_ipoib_sdma_sleep, 725 hfi1_ipoib_sdma_wakeup, 726 NULL, 727 NULL); 728 txq->priv = priv; 729 txq->sde = NULL; 730 INIT_LIST_HEAD(&txq->tx_list); 731 atomic64_set(&txq->complete_txreqs, 0); 732 atomic_set(&txq->stops, 0); 733 atomic_set(&txq->ring_full, 0); 734 atomic_set(&txq->no_desc, 0); 735 txq->q_idx = i; 736 txq->flow.tx_queue = 0xff; 737 txq->flow.sc5 = 0xff; 738 txq->pkts_sent = false; 739 740 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 741 priv->dd->node); 742 743 txq->tx_ring.items = 744 vzalloc_node(array_size(tx_ring_size, 745 sizeof(struct ipoib_txreq)), 746 priv->dd->node); 747 if (!txq->tx_ring.items) 748 goto free_txqs; 749 750 spin_lock_init(&txq->tx_ring.producer_lock); 751 spin_lock_init(&txq->tx_ring.consumer_lock); 752 txq->tx_ring.max_items = tx_ring_size; 753 754 txq->napi = &priv->tx_napis[i]; 755 netif_tx_napi_add(dev, txq->napi, 756 hfi1_ipoib_process_tx_ring, 757 NAPI_POLL_WEIGHT); 758 } 759 760 return 0; 761 762 free_txqs: 763 for (i--; i >= 0; i--) { 764 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 765 766 netif_napi_del(txq->napi); 767 vfree(txq->tx_ring.items); 768 } 769 770 kfree(priv->txqs); 771 priv->txqs = NULL; 772 773 free_tx_napis: 774 kfree(priv->tx_napis); 775 priv->tx_napis = NULL; 776 777 free_txreq_cache: 778 kmem_cache_destroy(priv->txreq_cache); 779 priv->txreq_cache = NULL; 780 return -ENOMEM; 781 } 782 783 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq) 784 { 785 struct sdma_txreq *txreq; 786 struct sdma_txreq *txreq_tmp; 787 atomic64_t *complete_txreqs = &txq->complete_txreqs; 788 789 list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) { 790 struct ipoib_txreq *tx = 791 container_of(txreq, struct ipoib_txreq, txreq); 792 793 list_del(&txreq->list); 794 sdma_txclean(txq->priv->dd, &tx->txreq); 795 dev_kfree_skb_any(tx->skb); 796 kmem_cache_free(txq->priv->txreq_cache, tx); 797 atomic64_inc(complete_txreqs); 798 } 799 800 if (hfi1_ipoib_used(txq)) 801 dd_dev_warn(txq->priv->dd, 802 "txq %d not empty found %llu requests\n", 803 txq->q_idx, 804 hfi1_ipoib_txreqs(txq->sent_txreqs, 805 atomic64_read(complete_txreqs))); 806 } 807 808 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv) 809 { 810 int i; 811 812 for (i = 0; i < priv->netdev->num_tx_queues; i++) { 813 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 814 815 iowait_cancel_work(&txq->wait); 816 iowait_sdma_drain(&txq->wait); 817 hfi1_ipoib_drain_tx_list(txq); 818 netif_napi_del(txq->napi); 819 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); 820 vfree(txq->tx_ring.items); 821 } 822 823 kfree(priv->txqs); 824 priv->txqs = NULL; 825 826 kfree(priv->tx_napis); 827 priv->tx_napis = NULL; 828 829 kmem_cache_destroy(priv->txreq_cache); 830 priv->txreq_cache = NULL; 831 } 832 833 void hfi1_ipoib_napi_tx_enable(struct net_device *dev) 834 { 835 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 836 int i; 837 838 for (i = 0; i < dev->num_tx_queues; i++) { 839 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 840 841 napi_enable(txq->napi); 842 } 843 } 844 845 void hfi1_ipoib_napi_tx_disable(struct net_device *dev) 846 { 847 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 848 int i; 849 850 for (i = 0; i < dev->num_tx_queues; i++) { 851 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 852 853 napi_disable(txq->napi); 854 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items); 855 } 856 } 857