1cdedef59SAnirudh Venkataramanan // SPDX-License-Identifier: GPL-2.0 2cdedef59SAnirudh Venkataramanan /* Copyright (c) 2018, Intel Corporation. */ 3cdedef59SAnirudh Venkataramanan 4cdedef59SAnirudh Venkataramanan /* The driver transmit and receive code */ 5cdedef59SAnirudh Venkataramanan 6cdedef59SAnirudh Venkataramanan #include <linux/prefetch.h> 7cdedef59SAnirudh Venkataramanan #include <linux/mm.h> 8cdedef59SAnirudh Venkataramanan #include "ice.h" 9cdedef59SAnirudh Venkataramanan 10*2b245cb2SAnirudh Venkataramanan #define ICE_RX_HDR_SIZE 256 11*2b245cb2SAnirudh Venkataramanan 12cdedef59SAnirudh Venkataramanan /** 13cdedef59SAnirudh Venkataramanan * ice_unmap_and_free_tx_buf - Release a Tx buffer 14cdedef59SAnirudh Venkataramanan * @ring: the ring that owns the buffer 15cdedef59SAnirudh Venkataramanan * @tx_buf: the buffer to free 16cdedef59SAnirudh Venkataramanan */ 17cdedef59SAnirudh Venkataramanan static void 18cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(struct ice_ring *ring, struct ice_tx_buf *tx_buf) 19cdedef59SAnirudh Venkataramanan { 20cdedef59SAnirudh Venkataramanan if (tx_buf->skb) { 21cdedef59SAnirudh Venkataramanan dev_kfree_skb_any(tx_buf->skb); 22cdedef59SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) 23cdedef59SAnirudh Venkataramanan dma_unmap_single(ring->dev, 24cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 25cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 26cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 27cdedef59SAnirudh Venkataramanan } else if (dma_unmap_len(tx_buf, len)) { 28cdedef59SAnirudh Venkataramanan dma_unmap_page(ring->dev, 29cdedef59SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 30cdedef59SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 31cdedef59SAnirudh Venkataramanan DMA_TO_DEVICE); 32cdedef59SAnirudh Venkataramanan } 33cdedef59SAnirudh Venkataramanan 34cdedef59SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 35cdedef59SAnirudh Venkataramanan tx_buf->skb = NULL; 36cdedef59SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 37cdedef59SAnirudh Venkataramanan /* tx_buf must be completely set up in the transmit path */ 38cdedef59SAnirudh Venkataramanan } 39cdedef59SAnirudh Venkataramanan 40cdedef59SAnirudh Venkataramanan static struct netdev_queue *txring_txq(const struct ice_ring *ring) 41cdedef59SAnirudh Venkataramanan { 42cdedef59SAnirudh Venkataramanan return netdev_get_tx_queue(ring->netdev, ring->q_index); 43cdedef59SAnirudh Venkataramanan } 44cdedef59SAnirudh Venkataramanan 45cdedef59SAnirudh Venkataramanan /** 46cdedef59SAnirudh Venkataramanan * ice_clean_tx_ring - Free any empty Tx buffers 47cdedef59SAnirudh Venkataramanan * @tx_ring: ring to be cleaned 48cdedef59SAnirudh Venkataramanan */ 49cdedef59SAnirudh Venkataramanan void ice_clean_tx_ring(struct ice_ring *tx_ring) 50cdedef59SAnirudh Venkataramanan { 51cdedef59SAnirudh Venkataramanan unsigned long size; 52cdedef59SAnirudh Venkataramanan u16 i; 53cdedef59SAnirudh Venkataramanan 54cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 55cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 56cdedef59SAnirudh Venkataramanan return; 57cdedef59SAnirudh Venkataramanan 58cdedef59SAnirudh Venkataramanan /* Free all the Tx ring sk_bufss */ 59cdedef59SAnirudh Venkataramanan for (i = 0; i < tx_ring->count; i++) 60cdedef59SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]); 61cdedef59SAnirudh Venkataramanan 62cdedef59SAnirudh Venkataramanan size = sizeof(struct ice_tx_buf) * tx_ring->count; 63cdedef59SAnirudh Venkataramanan memset(tx_ring->tx_buf, 0, size); 64cdedef59SAnirudh Venkataramanan 65cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 66cdedef59SAnirudh Venkataramanan memset(tx_ring->desc, 0, tx_ring->size); 67cdedef59SAnirudh Venkataramanan 68cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 69cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 70cdedef59SAnirudh Venkataramanan 71cdedef59SAnirudh Venkataramanan if (!tx_ring->netdev) 72cdedef59SAnirudh Venkataramanan return; 73cdedef59SAnirudh Venkataramanan 74cdedef59SAnirudh Venkataramanan /* cleanup Tx queue statistics */ 75cdedef59SAnirudh Venkataramanan netdev_tx_reset_queue(txring_txq(tx_ring)); 76cdedef59SAnirudh Venkataramanan } 77cdedef59SAnirudh Venkataramanan 78cdedef59SAnirudh Venkataramanan /** 79cdedef59SAnirudh Venkataramanan * ice_free_tx_ring - Free Tx resources per queue 80cdedef59SAnirudh Venkataramanan * @tx_ring: Tx descriptor ring for a specific queue 81cdedef59SAnirudh Venkataramanan * 82cdedef59SAnirudh Venkataramanan * Free all transmit software resources 83cdedef59SAnirudh Venkataramanan */ 84cdedef59SAnirudh Venkataramanan void ice_free_tx_ring(struct ice_ring *tx_ring) 85cdedef59SAnirudh Venkataramanan { 86cdedef59SAnirudh Venkataramanan ice_clean_tx_ring(tx_ring); 87cdedef59SAnirudh Venkataramanan devm_kfree(tx_ring->dev, tx_ring->tx_buf); 88cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 89cdedef59SAnirudh Venkataramanan 90cdedef59SAnirudh Venkataramanan if (tx_ring->desc) { 91cdedef59SAnirudh Venkataramanan dmam_free_coherent(tx_ring->dev, tx_ring->size, 92cdedef59SAnirudh Venkataramanan tx_ring->desc, tx_ring->dma); 93cdedef59SAnirudh Venkataramanan tx_ring->desc = NULL; 94cdedef59SAnirudh Venkataramanan } 95cdedef59SAnirudh Venkataramanan } 96cdedef59SAnirudh Venkataramanan 97cdedef59SAnirudh Venkataramanan /** 98*2b245cb2SAnirudh Venkataramanan * ice_clean_tx_irq - Reclaim resources after transmit completes 99*2b245cb2SAnirudh Venkataramanan * @vsi: the VSI we care about 100*2b245cb2SAnirudh Venkataramanan * @tx_ring: Tx ring to clean 101*2b245cb2SAnirudh Venkataramanan * @napi_budget: Used to determine if we are in netpoll 102*2b245cb2SAnirudh Venkataramanan * 103*2b245cb2SAnirudh Venkataramanan * Returns true if there's any budget left (e.g. the clean is finished) 104*2b245cb2SAnirudh Venkataramanan */ 105*2b245cb2SAnirudh Venkataramanan static bool ice_clean_tx_irq(struct ice_vsi *vsi, struct ice_ring *tx_ring, 106*2b245cb2SAnirudh Venkataramanan int napi_budget) 107*2b245cb2SAnirudh Venkataramanan { 108*2b245cb2SAnirudh Venkataramanan unsigned int total_bytes = 0, total_pkts = 0; 109*2b245cb2SAnirudh Venkataramanan unsigned int budget = vsi->work_lmt; 110*2b245cb2SAnirudh Venkataramanan s16 i = tx_ring->next_to_clean; 111*2b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 112*2b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 113*2b245cb2SAnirudh Venkataramanan 114*2b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 115*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 116*2b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 117*2b245cb2SAnirudh Venkataramanan 118*2b245cb2SAnirudh Venkataramanan do { 119*2b245cb2SAnirudh Venkataramanan struct ice_tx_desc *eop_desc = tx_buf->next_to_watch; 120*2b245cb2SAnirudh Venkataramanan 121*2b245cb2SAnirudh Venkataramanan /* if next_to_watch is not set then there is no work pending */ 122*2b245cb2SAnirudh Venkataramanan if (!eop_desc) 123*2b245cb2SAnirudh Venkataramanan break; 124*2b245cb2SAnirudh Venkataramanan 125*2b245cb2SAnirudh Venkataramanan smp_rmb(); /* prevent any other reads prior to eop_desc */ 126*2b245cb2SAnirudh Venkataramanan 127*2b245cb2SAnirudh Venkataramanan /* if the descriptor isn't done, no work yet to do */ 128*2b245cb2SAnirudh Venkataramanan if (!(eop_desc->cmd_type_offset_bsz & 129*2b245cb2SAnirudh Venkataramanan cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) 130*2b245cb2SAnirudh Venkataramanan break; 131*2b245cb2SAnirudh Venkataramanan 132*2b245cb2SAnirudh Venkataramanan /* clear next_to_watch to prevent false hangs */ 133*2b245cb2SAnirudh Venkataramanan tx_buf->next_to_watch = NULL; 134*2b245cb2SAnirudh Venkataramanan 135*2b245cb2SAnirudh Venkataramanan /* update the statistics for this packet */ 136*2b245cb2SAnirudh Venkataramanan total_bytes += tx_buf->bytecount; 137*2b245cb2SAnirudh Venkataramanan total_pkts += tx_buf->gso_segs; 138*2b245cb2SAnirudh Venkataramanan 139*2b245cb2SAnirudh Venkataramanan /* free the skb */ 140*2b245cb2SAnirudh Venkataramanan napi_consume_skb(tx_buf->skb, napi_budget); 141*2b245cb2SAnirudh Venkataramanan 142*2b245cb2SAnirudh Venkataramanan /* unmap skb header data */ 143*2b245cb2SAnirudh Venkataramanan dma_unmap_single(tx_ring->dev, 144*2b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 145*2b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 146*2b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 147*2b245cb2SAnirudh Venkataramanan 148*2b245cb2SAnirudh Venkataramanan /* clear tx_buf data */ 149*2b245cb2SAnirudh Venkataramanan tx_buf->skb = NULL; 150*2b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 151*2b245cb2SAnirudh Venkataramanan 152*2b245cb2SAnirudh Venkataramanan /* unmap remaining buffers */ 153*2b245cb2SAnirudh Venkataramanan while (tx_desc != eop_desc) { 154*2b245cb2SAnirudh Venkataramanan tx_buf++; 155*2b245cb2SAnirudh Venkataramanan tx_desc++; 156*2b245cb2SAnirudh Venkataramanan i++; 157*2b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 158*2b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 159*2b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 160*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 161*2b245cb2SAnirudh Venkataramanan } 162*2b245cb2SAnirudh Venkataramanan 163*2b245cb2SAnirudh Venkataramanan /* unmap any remaining paged data */ 164*2b245cb2SAnirudh Venkataramanan if (dma_unmap_len(tx_buf, len)) { 165*2b245cb2SAnirudh Venkataramanan dma_unmap_page(tx_ring->dev, 166*2b245cb2SAnirudh Venkataramanan dma_unmap_addr(tx_buf, dma), 167*2b245cb2SAnirudh Venkataramanan dma_unmap_len(tx_buf, len), 168*2b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 169*2b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, 0); 170*2b245cb2SAnirudh Venkataramanan } 171*2b245cb2SAnirudh Venkataramanan } 172*2b245cb2SAnirudh Venkataramanan 173*2b245cb2SAnirudh Venkataramanan /* move us one more past the eop_desc for start of next pkt */ 174*2b245cb2SAnirudh Venkataramanan tx_buf++; 175*2b245cb2SAnirudh Venkataramanan tx_desc++; 176*2b245cb2SAnirudh Venkataramanan i++; 177*2b245cb2SAnirudh Venkataramanan if (unlikely(!i)) { 178*2b245cb2SAnirudh Venkataramanan i -= tx_ring->count; 179*2b245cb2SAnirudh Venkataramanan tx_buf = tx_ring->tx_buf; 180*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 181*2b245cb2SAnirudh Venkataramanan } 182*2b245cb2SAnirudh Venkataramanan 183*2b245cb2SAnirudh Venkataramanan prefetch(tx_desc); 184*2b245cb2SAnirudh Venkataramanan 185*2b245cb2SAnirudh Venkataramanan /* update budget accounting */ 186*2b245cb2SAnirudh Venkataramanan budget--; 187*2b245cb2SAnirudh Venkataramanan } while (likely(budget)); 188*2b245cb2SAnirudh Venkataramanan 189*2b245cb2SAnirudh Venkataramanan i += tx_ring->count; 190*2b245cb2SAnirudh Venkataramanan tx_ring->next_to_clean = i; 191*2b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&tx_ring->syncp); 192*2b245cb2SAnirudh Venkataramanan tx_ring->stats.bytes += total_bytes; 193*2b245cb2SAnirudh Venkataramanan tx_ring->stats.pkts += total_pkts; 194*2b245cb2SAnirudh Venkataramanan u64_stats_update_end(&tx_ring->syncp); 195*2b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_bytes += total_bytes; 196*2b245cb2SAnirudh Venkataramanan tx_ring->q_vector->tx.total_pkts += total_pkts; 197*2b245cb2SAnirudh Venkataramanan 198*2b245cb2SAnirudh Venkataramanan netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, 199*2b245cb2SAnirudh Venkataramanan total_bytes); 200*2b245cb2SAnirudh Venkataramanan 201*2b245cb2SAnirudh Venkataramanan #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 202*2b245cb2SAnirudh Venkataramanan if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) && 203*2b245cb2SAnirudh Venkataramanan (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 204*2b245cb2SAnirudh Venkataramanan /* Make sure that anybody stopping the queue after this 205*2b245cb2SAnirudh Venkataramanan * sees the new next_to_clean. 206*2b245cb2SAnirudh Venkataramanan */ 207*2b245cb2SAnirudh Venkataramanan smp_mb(); 208*2b245cb2SAnirudh Venkataramanan if (__netif_subqueue_stopped(tx_ring->netdev, 209*2b245cb2SAnirudh Venkataramanan tx_ring->q_index) && 210*2b245cb2SAnirudh Venkataramanan !test_bit(__ICE_DOWN, vsi->state)) { 211*2b245cb2SAnirudh Venkataramanan netif_wake_subqueue(tx_ring->netdev, 212*2b245cb2SAnirudh Venkataramanan tx_ring->q_index); 213*2b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 214*2b245cb2SAnirudh Venkataramanan } 215*2b245cb2SAnirudh Venkataramanan } 216*2b245cb2SAnirudh Venkataramanan 217*2b245cb2SAnirudh Venkataramanan return !!budget; 218*2b245cb2SAnirudh Venkataramanan } 219*2b245cb2SAnirudh Venkataramanan 220*2b245cb2SAnirudh Venkataramanan /** 221cdedef59SAnirudh Venkataramanan * ice_setup_tx_ring - Allocate the Tx descriptors 222cdedef59SAnirudh Venkataramanan * @tx_ring: the tx ring to set up 223cdedef59SAnirudh Venkataramanan * 224cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 225cdedef59SAnirudh Venkataramanan */ 226cdedef59SAnirudh Venkataramanan int ice_setup_tx_ring(struct ice_ring *tx_ring) 227cdedef59SAnirudh Venkataramanan { 228cdedef59SAnirudh Venkataramanan struct device *dev = tx_ring->dev; 229cdedef59SAnirudh Venkataramanan int bi_size; 230cdedef59SAnirudh Venkataramanan 231cdedef59SAnirudh Venkataramanan if (!dev) 232cdedef59SAnirudh Venkataramanan return -ENOMEM; 233cdedef59SAnirudh Venkataramanan 234cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 235cdedef59SAnirudh Venkataramanan WARN_ON(tx_ring->tx_buf); 236cdedef59SAnirudh Venkataramanan bi_size = sizeof(struct ice_tx_buf) * tx_ring->count; 237cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = devm_kzalloc(dev, bi_size, GFP_KERNEL); 238cdedef59SAnirudh Venkataramanan if (!tx_ring->tx_buf) 239cdedef59SAnirudh Venkataramanan return -ENOMEM; 240cdedef59SAnirudh Venkataramanan 241cdedef59SAnirudh Venkataramanan /* round up to nearest 4K */ 242cdedef59SAnirudh Venkataramanan tx_ring->size = tx_ring->count * sizeof(struct ice_tx_desc); 243cdedef59SAnirudh Venkataramanan tx_ring->size = ALIGN(tx_ring->size, 4096); 244cdedef59SAnirudh Venkataramanan tx_ring->desc = dmam_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, 245cdedef59SAnirudh Venkataramanan GFP_KERNEL); 246cdedef59SAnirudh Venkataramanan if (!tx_ring->desc) { 247cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 248cdedef59SAnirudh Venkataramanan tx_ring->size); 249cdedef59SAnirudh Venkataramanan goto err; 250cdedef59SAnirudh Venkataramanan } 251cdedef59SAnirudh Venkataramanan 252cdedef59SAnirudh Venkataramanan tx_ring->next_to_use = 0; 253cdedef59SAnirudh Venkataramanan tx_ring->next_to_clean = 0; 254cdedef59SAnirudh Venkataramanan return 0; 255cdedef59SAnirudh Venkataramanan 256cdedef59SAnirudh Venkataramanan err: 257cdedef59SAnirudh Venkataramanan devm_kfree(dev, tx_ring->tx_buf); 258cdedef59SAnirudh Venkataramanan tx_ring->tx_buf = NULL; 259cdedef59SAnirudh Venkataramanan return -ENOMEM; 260cdedef59SAnirudh Venkataramanan } 261cdedef59SAnirudh Venkataramanan 262cdedef59SAnirudh Venkataramanan /** 263cdedef59SAnirudh Venkataramanan * ice_clean_rx_ring - Free Rx buffers 264cdedef59SAnirudh Venkataramanan * @rx_ring: ring to be cleaned 265cdedef59SAnirudh Venkataramanan */ 266cdedef59SAnirudh Venkataramanan void ice_clean_rx_ring(struct ice_ring *rx_ring) 267cdedef59SAnirudh Venkataramanan { 268cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 269cdedef59SAnirudh Venkataramanan unsigned long size; 270cdedef59SAnirudh Venkataramanan u16 i; 271cdedef59SAnirudh Venkataramanan 272cdedef59SAnirudh Venkataramanan /* ring already cleared, nothing to do */ 273cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 274cdedef59SAnirudh Venkataramanan return; 275cdedef59SAnirudh Venkataramanan 276cdedef59SAnirudh Venkataramanan /* Free all the Rx ring sk_buffs */ 277cdedef59SAnirudh Venkataramanan for (i = 0; i < rx_ring->count; i++) { 278cdedef59SAnirudh Venkataramanan struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i]; 279cdedef59SAnirudh Venkataramanan 280cdedef59SAnirudh Venkataramanan if (rx_buf->skb) { 281cdedef59SAnirudh Venkataramanan dev_kfree_skb(rx_buf->skb); 282cdedef59SAnirudh Venkataramanan rx_buf->skb = NULL; 283cdedef59SAnirudh Venkataramanan } 284cdedef59SAnirudh Venkataramanan if (!rx_buf->page) 285cdedef59SAnirudh Venkataramanan continue; 286cdedef59SAnirudh Venkataramanan 287cdedef59SAnirudh Venkataramanan dma_unmap_page(dev, rx_buf->dma, PAGE_SIZE, DMA_FROM_DEVICE); 288cdedef59SAnirudh Venkataramanan __free_pages(rx_buf->page, 0); 289cdedef59SAnirudh Venkataramanan 290cdedef59SAnirudh Venkataramanan rx_buf->page = NULL; 291cdedef59SAnirudh Venkataramanan rx_buf->page_offset = 0; 292cdedef59SAnirudh Venkataramanan } 293cdedef59SAnirudh Venkataramanan 294cdedef59SAnirudh Venkataramanan size = sizeof(struct ice_rx_buf) * rx_ring->count; 295cdedef59SAnirudh Venkataramanan memset(rx_ring->rx_buf, 0, size); 296cdedef59SAnirudh Venkataramanan 297cdedef59SAnirudh Venkataramanan /* Zero out the descriptor ring */ 298cdedef59SAnirudh Venkataramanan memset(rx_ring->desc, 0, rx_ring->size); 299cdedef59SAnirudh Venkataramanan 300cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = 0; 301cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 302cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 303cdedef59SAnirudh Venkataramanan } 304cdedef59SAnirudh Venkataramanan 305cdedef59SAnirudh Venkataramanan /** 306cdedef59SAnirudh Venkataramanan * ice_free_rx_ring - Free Rx resources 307cdedef59SAnirudh Venkataramanan * @rx_ring: ring to clean the resources from 308cdedef59SAnirudh Venkataramanan * 309cdedef59SAnirudh Venkataramanan * Free all receive software resources 310cdedef59SAnirudh Venkataramanan */ 311cdedef59SAnirudh Venkataramanan void ice_free_rx_ring(struct ice_ring *rx_ring) 312cdedef59SAnirudh Venkataramanan { 313cdedef59SAnirudh Venkataramanan ice_clean_rx_ring(rx_ring); 314cdedef59SAnirudh Venkataramanan devm_kfree(rx_ring->dev, rx_ring->rx_buf); 315cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 316cdedef59SAnirudh Venkataramanan 317cdedef59SAnirudh Venkataramanan if (rx_ring->desc) { 318cdedef59SAnirudh Venkataramanan dmam_free_coherent(rx_ring->dev, rx_ring->size, 319cdedef59SAnirudh Venkataramanan rx_ring->desc, rx_ring->dma); 320cdedef59SAnirudh Venkataramanan rx_ring->desc = NULL; 321cdedef59SAnirudh Venkataramanan } 322cdedef59SAnirudh Venkataramanan } 323cdedef59SAnirudh Venkataramanan 324cdedef59SAnirudh Venkataramanan /** 325cdedef59SAnirudh Venkataramanan * ice_setup_rx_ring - Allocate the Rx descriptors 326cdedef59SAnirudh Venkataramanan * @rx_ring: the rx ring to set up 327cdedef59SAnirudh Venkataramanan * 328cdedef59SAnirudh Venkataramanan * Return 0 on success, negative on error 329cdedef59SAnirudh Venkataramanan */ 330cdedef59SAnirudh Venkataramanan int ice_setup_rx_ring(struct ice_ring *rx_ring) 331cdedef59SAnirudh Venkataramanan { 332cdedef59SAnirudh Venkataramanan struct device *dev = rx_ring->dev; 333cdedef59SAnirudh Venkataramanan int bi_size; 334cdedef59SAnirudh Venkataramanan 335cdedef59SAnirudh Venkataramanan if (!dev) 336cdedef59SAnirudh Venkataramanan return -ENOMEM; 337cdedef59SAnirudh Venkataramanan 338cdedef59SAnirudh Venkataramanan /* warn if we are about to overwrite the pointer */ 339cdedef59SAnirudh Venkataramanan WARN_ON(rx_ring->rx_buf); 340cdedef59SAnirudh Venkataramanan bi_size = sizeof(struct ice_rx_buf) * rx_ring->count; 341cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = devm_kzalloc(dev, bi_size, GFP_KERNEL); 342cdedef59SAnirudh Venkataramanan if (!rx_ring->rx_buf) 343cdedef59SAnirudh Venkataramanan return -ENOMEM; 344cdedef59SAnirudh Venkataramanan 345cdedef59SAnirudh Venkataramanan /* round up to nearest 4K */ 346cdedef59SAnirudh Venkataramanan rx_ring->size = rx_ring->count * sizeof(union ice_32byte_rx_desc); 347cdedef59SAnirudh Venkataramanan rx_ring->size = ALIGN(rx_ring->size, 4096); 348cdedef59SAnirudh Venkataramanan rx_ring->desc = dmam_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, 349cdedef59SAnirudh Venkataramanan GFP_KERNEL); 350cdedef59SAnirudh Venkataramanan if (!rx_ring->desc) { 351cdedef59SAnirudh Venkataramanan dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 352cdedef59SAnirudh Venkataramanan rx_ring->size); 353cdedef59SAnirudh Venkataramanan goto err; 354cdedef59SAnirudh Venkataramanan } 355cdedef59SAnirudh Venkataramanan 356cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = 0; 357cdedef59SAnirudh Venkataramanan rx_ring->next_to_clean = 0; 358cdedef59SAnirudh Venkataramanan return 0; 359cdedef59SAnirudh Venkataramanan 360cdedef59SAnirudh Venkataramanan err: 361cdedef59SAnirudh Venkataramanan devm_kfree(dev, rx_ring->rx_buf); 362cdedef59SAnirudh Venkataramanan rx_ring->rx_buf = NULL; 363cdedef59SAnirudh Venkataramanan return -ENOMEM; 364cdedef59SAnirudh Venkataramanan } 365cdedef59SAnirudh Venkataramanan 366cdedef59SAnirudh Venkataramanan /** 367cdedef59SAnirudh Venkataramanan * ice_release_rx_desc - Store the new tail and head values 368cdedef59SAnirudh Venkataramanan * @rx_ring: ring to bump 369cdedef59SAnirudh Venkataramanan * @val: new head index 370cdedef59SAnirudh Venkataramanan */ 371cdedef59SAnirudh Venkataramanan static void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val) 372cdedef59SAnirudh Venkataramanan { 373cdedef59SAnirudh Venkataramanan rx_ring->next_to_use = val; 374cdedef59SAnirudh Venkataramanan 375cdedef59SAnirudh Venkataramanan /* update next to alloc since we have filled the ring */ 376cdedef59SAnirudh Venkataramanan rx_ring->next_to_alloc = val; 377cdedef59SAnirudh Venkataramanan 378cdedef59SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w 379cdedef59SAnirudh Venkataramanan * know there are new descriptors to fetch. (Only 380cdedef59SAnirudh Venkataramanan * applicable for weak-ordered memory model archs, 381cdedef59SAnirudh Venkataramanan * such as IA-64). 382cdedef59SAnirudh Venkataramanan */ 383cdedef59SAnirudh Venkataramanan wmb(); 384cdedef59SAnirudh Venkataramanan writel(val, rx_ring->tail); 385cdedef59SAnirudh Venkataramanan } 386cdedef59SAnirudh Venkataramanan 387cdedef59SAnirudh Venkataramanan /** 388cdedef59SAnirudh Venkataramanan * ice_alloc_mapped_page - recycle or make a new page 389cdedef59SAnirudh Venkataramanan * @rx_ring: ring to use 390cdedef59SAnirudh Venkataramanan * @bi: rx_buf struct to modify 391cdedef59SAnirudh Venkataramanan * 392cdedef59SAnirudh Venkataramanan * Returns true if the page was successfully allocated or 393cdedef59SAnirudh Venkataramanan * reused. 394cdedef59SAnirudh Venkataramanan */ 395cdedef59SAnirudh Venkataramanan static bool ice_alloc_mapped_page(struct ice_ring *rx_ring, 396cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi) 397cdedef59SAnirudh Venkataramanan { 398cdedef59SAnirudh Venkataramanan struct page *page = bi->page; 399cdedef59SAnirudh Venkataramanan dma_addr_t dma; 400cdedef59SAnirudh Venkataramanan 401cdedef59SAnirudh Venkataramanan /* since we are recycling buffers we should seldom need to alloc */ 402*2b245cb2SAnirudh Venkataramanan if (likely(page)) { 403*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 404cdedef59SAnirudh Venkataramanan return true; 405*2b245cb2SAnirudh Venkataramanan } 406cdedef59SAnirudh Venkataramanan 407cdedef59SAnirudh Venkataramanan /* alloc new page for storage */ 408cdedef59SAnirudh Venkataramanan page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); 409*2b245cb2SAnirudh Venkataramanan if (unlikely(!page)) { 410*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 411cdedef59SAnirudh Venkataramanan return false; 412*2b245cb2SAnirudh Venkataramanan } 413cdedef59SAnirudh Venkataramanan 414cdedef59SAnirudh Venkataramanan /* map page for use */ 415cdedef59SAnirudh Venkataramanan dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); 416cdedef59SAnirudh Venkataramanan 417cdedef59SAnirudh Venkataramanan /* if mapping failed free memory back to system since 418cdedef59SAnirudh Venkataramanan * there isn't much point in holding memory we can't use 419cdedef59SAnirudh Venkataramanan */ 420cdedef59SAnirudh Venkataramanan if (dma_mapping_error(rx_ring->dev, dma)) { 421cdedef59SAnirudh Venkataramanan __free_pages(page, 0); 422*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_page_failed++; 423cdedef59SAnirudh Venkataramanan return false; 424cdedef59SAnirudh Venkataramanan } 425cdedef59SAnirudh Venkataramanan 426cdedef59SAnirudh Venkataramanan bi->dma = dma; 427cdedef59SAnirudh Venkataramanan bi->page = page; 428cdedef59SAnirudh Venkataramanan bi->page_offset = 0; 429cdedef59SAnirudh Venkataramanan 430cdedef59SAnirudh Venkataramanan return true; 431cdedef59SAnirudh Venkataramanan } 432cdedef59SAnirudh Venkataramanan 433cdedef59SAnirudh Venkataramanan /** 434cdedef59SAnirudh Venkataramanan * ice_alloc_rx_bufs - Replace used receive buffers 435cdedef59SAnirudh Venkataramanan * @rx_ring: ring to place buffers on 436cdedef59SAnirudh Venkataramanan * @cleaned_count: number of buffers to replace 437cdedef59SAnirudh Venkataramanan * 438cdedef59SAnirudh Venkataramanan * Returns false if all allocations were successful, true if any fail 439cdedef59SAnirudh Venkataramanan */ 440cdedef59SAnirudh Venkataramanan bool ice_alloc_rx_bufs(struct ice_ring *rx_ring, u16 cleaned_count) 441cdedef59SAnirudh Venkataramanan { 442cdedef59SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 443cdedef59SAnirudh Venkataramanan u16 ntu = rx_ring->next_to_use; 444cdedef59SAnirudh Venkataramanan struct ice_rx_buf *bi; 445cdedef59SAnirudh Venkataramanan 446cdedef59SAnirudh Venkataramanan /* do nothing if no valid netdev defined */ 447cdedef59SAnirudh Venkataramanan if (!rx_ring->netdev || !cleaned_count) 448cdedef59SAnirudh Venkataramanan return false; 449cdedef59SAnirudh Venkataramanan 450cdedef59SAnirudh Venkataramanan /* get the RX descriptor and buffer based on next_to_use */ 451cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, ntu); 452cdedef59SAnirudh Venkataramanan bi = &rx_ring->rx_buf[ntu]; 453cdedef59SAnirudh Venkataramanan 454cdedef59SAnirudh Venkataramanan do { 455cdedef59SAnirudh Venkataramanan if (!ice_alloc_mapped_page(rx_ring, bi)) 456cdedef59SAnirudh Venkataramanan goto no_bufs; 457cdedef59SAnirudh Venkataramanan 458cdedef59SAnirudh Venkataramanan /* Refresh the desc even if buffer_addrs didn't change 459cdedef59SAnirudh Venkataramanan * because each write-back erases this info. 460cdedef59SAnirudh Venkataramanan */ 461cdedef59SAnirudh Venkataramanan rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 462cdedef59SAnirudh Venkataramanan 463cdedef59SAnirudh Venkataramanan rx_desc++; 464cdedef59SAnirudh Venkataramanan bi++; 465cdedef59SAnirudh Venkataramanan ntu++; 466cdedef59SAnirudh Venkataramanan if (unlikely(ntu == rx_ring->count)) { 467cdedef59SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, 0); 468cdedef59SAnirudh Venkataramanan bi = rx_ring->rx_buf; 469cdedef59SAnirudh Venkataramanan ntu = 0; 470cdedef59SAnirudh Venkataramanan } 471cdedef59SAnirudh Venkataramanan 472cdedef59SAnirudh Venkataramanan /* clear the status bits for the next_to_use descriptor */ 473cdedef59SAnirudh Venkataramanan rx_desc->wb.status_error0 = 0; 474cdedef59SAnirudh Venkataramanan 475cdedef59SAnirudh Venkataramanan cleaned_count--; 476cdedef59SAnirudh Venkataramanan } while (cleaned_count); 477cdedef59SAnirudh Venkataramanan 478cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 479cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 480cdedef59SAnirudh Venkataramanan 481cdedef59SAnirudh Venkataramanan return false; 482cdedef59SAnirudh Venkataramanan 483cdedef59SAnirudh Venkataramanan no_bufs: 484cdedef59SAnirudh Venkataramanan if (rx_ring->next_to_use != ntu) 485cdedef59SAnirudh Venkataramanan ice_release_rx_desc(rx_ring, ntu); 486cdedef59SAnirudh Venkataramanan 487cdedef59SAnirudh Venkataramanan /* make sure to come back via polling to try again after 488cdedef59SAnirudh Venkataramanan * allocation failure 489cdedef59SAnirudh Venkataramanan */ 490cdedef59SAnirudh Venkataramanan return true; 491cdedef59SAnirudh Venkataramanan } 492*2b245cb2SAnirudh Venkataramanan 493*2b245cb2SAnirudh Venkataramanan /** 494*2b245cb2SAnirudh Venkataramanan * ice_page_is_reserved - check if reuse is possible 495*2b245cb2SAnirudh Venkataramanan * @page: page struct to check 496*2b245cb2SAnirudh Venkataramanan */ 497*2b245cb2SAnirudh Venkataramanan static bool ice_page_is_reserved(struct page *page) 498*2b245cb2SAnirudh Venkataramanan { 499*2b245cb2SAnirudh Venkataramanan return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 500*2b245cb2SAnirudh Venkataramanan } 501*2b245cb2SAnirudh Venkataramanan 502*2b245cb2SAnirudh Venkataramanan /** 503*2b245cb2SAnirudh Venkataramanan * ice_add_rx_frag - Add contents of Rx buffer to sk_buff 504*2b245cb2SAnirudh Venkataramanan * @rx_buf: buffer containing page to add 505*2b245cb2SAnirudh Venkataramanan * @rx_desc: descriptor containing length of buffer written by hardware 506*2b245cb2SAnirudh Venkataramanan * @skb: sk_buf to place the data into 507*2b245cb2SAnirudh Venkataramanan * 508*2b245cb2SAnirudh Venkataramanan * This function will add the data contained in rx_buf->page to the skb. 509*2b245cb2SAnirudh Venkataramanan * This is done either through a direct copy if the data in the buffer is 510*2b245cb2SAnirudh Venkataramanan * less than the skb header size, otherwise it will just attach the page as 511*2b245cb2SAnirudh Venkataramanan * a frag to the skb. 512*2b245cb2SAnirudh Venkataramanan * 513*2b245cb2SAnirudh Venkataramanan * The function will then update the page offset if necessary and return 514*2b245cb2SAnirudh Venkataramanan * true if the buffer can be reused by the adapter. 515*2b245cb2SAnirudh Venkataramanan */ 516*2b245cb2SAnirudh Venkataramanan static bool ice_add_rx_frag(struct ice_rx_buf *rx_buf, 517*2b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, 518*2b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 519*2b245cb2SAnirudh Venkataramanan { 520*2b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 521*2b245cb2SAnirudh Venkataramanan unsigned int truesize = ICE_RXBUF_2048; 522*2b245cb2SAnirudh Venkataramanan #else 523*2b245cb2SAnirudh Venkataramanan unsigned int last_offset = PAGE_SIZE - ICE_RXBUF_2048; 524*2b245cb2SAnirudh Venkataramanan unsigned int truesize; 525*2b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE < 8192) */ 526*2b245cb2SAnirudh Venkataramanan 527*2b245cb2SAnirudh Venkataramanan struct page *page; 528*2b245cb2SAnirudh Venkataramanan unsigned int size; 529*2b245cb2SAnirudh Venkataramanan 530*2b245cb2SAnirudh Venkataramanan size = le16_to_cpu(rx_desc->wb.pkt_len) & 531*2b245cb2SAnirudh Venkataramanan ICE_RX_FLX_DESC_PKT_LEN_M; 532*2b245cb2SAnirudh Venkataramanan 533*2b245cb2SAnirudh Venkataramanan page = rx_buf->page; 534*2b245cb2SAnirudh Venkataramanan 535*2b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE >= 8192) 536*2b245cb2SAnirudh Venkataramanan truesize = ALIGN(size, L1_CACHE_BYTES); 537*2b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE >= 8192) */ 538*2b245cb2SAnirudh Venkataramanan 539*2b245cb2SAnirudh Venkataramanan /* will the data fit in the skb we allocated? if so, just 540*2b245cb2SAnirudh Venkataramanan * copy it as it is pretty small anyway 541*2b245cb2SAnirudh Venkataramanan */ 542*2b245cb2SAnirudh Venkataramanan if (size <= ICE_RX_HDR_SIZE && !skb_is_nonlinear(skb)) { 543*2b245cb2SAnirudh Venkataramanan unsigned char *va = page_address(page) + rx_buf->page_offset; 544*2b245cb2SAnirudh Venkataramanan 545*2b245cb2SAnirudh Venkataramanan memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 546*2b245cb2SAnirudh Venkataramanan 547*2b245cb2SAnirudh Venkataramanan /* page is not reserved, we can reuse buffer as-is */ 548*2b245cb2SAnirudh Venkataramanan if (likely(!ice_page_is_reserved(page))) 549*2b245cb2SAnirudh Venkataramanan return true; 550*2b245cb2SAnirudh Venkataramanan 551*2b245cb2SAnirudh Venkataramanan /* this page cannot be reused so discard it */ 552*2b245cb2SAnirudh Venkataramanan __free_pages(page, 0); 553*2b245cb2SAnirudh Venkataramanan return false; 554*2b245cb2SAnirudh Venkataramanan } 555*2b245cb2SAnirudh Venkataramanan 556*2b245cb2SAnirudh Venkataramanan skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 557*2b245cb2SAnirudh Venkataramanan rx_buf->page_offset, size, truesize); 558*2b245cb2SAnirudh Venkataramanan 559*2b245cb2SAnirudh Venkataramanan /* avoid re-using remote pages */ 560*2b245cb2SAnirudh Venkataramanan if (unlikely(ice_page_is_reserved(page))) 561*2b245cb2SAnirudh Venkataramanan return false; 562*2b245cb2SAnirudh Venkataramanan 563*2b245cb2SAnirudh Venkataramanan #if (PAGE_SIZE < 8192) 564*2b245cb2SAnirudh Venkataramanan /* if we are only owner of page we can reuse it */ 565*2b245cb2SAnirudh Venkataramanan if (unlikely(page_count(page) != 1)) 566*2b245cb2SAnirudh Venkataramanan return false; 567*2b245cb2SAnirudh Venkataramanan 568*2b245cb2SAnirudh Venkataramanan /* flip page offset to other buffer */ 569*2b245cb2SAnirudh Venkataramanan rx_buf->page_offset ^= truesize; 570*2b245cb2SAnirudh Venkataramanan #else 571*2b245cb2SAnirudh Venkataramanan /* move offset up to the next cache line */ 572*2b245cb2SAnirudh Venkataramanan rx_buf->page_offset += truesize; 573*2b245cb2SAnirudh Venkataramanan 574*2b245cb2SAnirudh Venkataramanan if (rx_buf->page_offset > last_offset) 575*2b245cb2SAnirudh Venkataramanan return false; 576*2b245cb2SAnirudh Venkataramanan #endif /* PAGE_SIZE < 8192) */ 577*2b245cb2SAnirudh Venkataramanan 578*2b245cb2SAnirudh Venkataramanan /* Even if we own the page, we are not allowed to use atomic_set() 579*2b245cb2SAnirudh Venkataramanan * This would break get_page_unless_zero() users. 580*2b245cb2SAnirudh Venkataramanan */ 581*2b245cb2SAnirudh Venkataramanan get_page(rx_buf->page); 582*2b245cb2SAnirudh Venkataramanan 583*2b245cb2SAnirudh Venkataramanan return true; 584*2b245cb2SAnirudh Venkataramanan } 585*2b245cb2SAnirudh Venkataramanan 586*2b245cb2SAnirudh Venkataramanan /** 587*2b245cb2SAnirudh Venkataramanan * ice_reuse_rx_page - page flip buffer and store it back on the ring 588*2b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to store buffers on 589*2b245cb2SAnirudh Venkataramanan * @old_buf: donor buffer to have page reused 590*2b245cb2SAnirudh Venkataramanan * 591*2b245cb2SAnirudh Venkataramanan * Synchronizes page for reuse by the adapter 592*2b245cb2SAnirudh Venkataramanan */ 593*2b245cb2SAnirudh Venkataramanan static void ice_reuse_rx_page(struct ice_ring *rx_ring, 594*2b245cb2SAnirudh Venkataramanan struct ice_rx_buf *old_buf) 595*2b245cb2SAnirudh Venkataramanan { 596*2b245cb2SAnirudh Venkataramanan u16 nta = rx_ring->next_to_alloc; 597*2b245cb2SAnirudh Venkataramanan struct ice_rx_buf *new_buf; 598*2b245cb2SAnirudh Venkataramanan 599*2b245cb2SAnirudh Venkataramanan new_buf = &rx_ring->rx_buf[nta]; 600*2b245cb2SAnirudh Venkataramanan 601*2b245cb2SAnirudh Venkataramanan /* update, and store next to alloc */ 602*2b245cb2SAnirudh Venkataramanan nta++; 603*2b245cb2SAnirudh Venkataramanan rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 604*2b245cb2SAnirudh Venkataramanan 605*2b245cb2SAnirudh Venkataramanan /* transfer page from old buffer to new buffer */ 606*2b245cb2SAnirudh Venkataramanan *new_buf = *old_buf; 607*2b245cb2SAnirudh Venkataramanan } 608*2b245cb2SAnirudh Venkataramanan 609*2b245cb2SAnirudh Venkataramanan /** 610*2b245cb2SAnirudh Venkataramanan * ice_fetch_rx_buf - Allocate skb and populate it 611*2b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to transact packets on 612*2b245cb2SAnirudh Venkataramanan * @rx_desc: descriptor containing info written by hardware 613*2b245cb2SAnirudh Venkataramanan * 614*2b245cb2SAnirudh Venkataramanan * This function allocates an skb on the fly, and populates it with the page 615*2b245cb2SAnirudh Venkataramanan * data from the current receive descriptor, taking care to set up the skb 616*2b245cb2SAnirudh Venkataramanan * correctly, as well as handling calling the page recycle function if 617*2b245cb2SAnirudh Venkataramanan * necessary. 618*2b245cb2SAnirudh Venkataramanan */ 619*2b245cb2SAnirudh Venkataramanan static struct sk_buff *ice_fetch_rx_buf(struct ice_ring *rx_ring, 620*2b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc) 621*2b245cb2SAnirudh Venkataramanan { 622*2b245cb2SAnirudh Venkataramanan struct ice_rx_buf *rx_buf; 623*2b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 624*2b245cb2SAnirudh Venkataramanan struct page *page; 625*2b245cb2SAnirudh Venkataramanan 626*2b245cb2SAnirudh Venkataramanan rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean]; 627*2b245cb2SAnirudh Venkataramanan page = rx_buf->page; 628*2b245cb2SAnirudh Venkataramanan prefetchw(page); 629*2b245cb2SAnirudh Venkataramanan 630*2b245cb2SAnirudh Venkataramanan skb = rx_buf->skb; 631*2b245cb2SAnirudh Venkataramanan 632*2b245cb2SAnirudh Venkataramanan if (likely(!skb)) { 633*2b245cb2SAnirudh Venkataramanan u8 *page_addr = page_address(page) + rx_buf->page_offset; 634*2b245cb2SAnirudh Venkataramanan 635*2b245cb2SAnirudh Venkataramanan /* prefetch first cache line of first page */ 636*2b245cb2SAnirudh Venkataramanan prefetch(page_addr); 637*2b245cb2SAnirudh Venkataramanan #if L1_CACHE_BYTES < 128 638*2b245cb2SAnirudh Venkataramanan prefetch((void *)(page_addr + L1_CACHE_BYTES)); 639*2b245cb2SAnirudh Venkataramanan #endif /* L1_CACHE_BYTES */ 640*2b245cb2SAnirudh Venkataramanan 641*2b245cb2SAnirudh Venkataramanan /* allocate a skb to store the frags */ 642*2b245cb2SAnirudh Venkataramanan skb = __napi_alloc_skb(&rx_ring->q_vector->napi, 643*2b245cb2SAnirudh Venkataramanan ICE_RX_HDR_SIZE, 644*2b245cb2SAnirudh Venkataramanan GFP_ATOMIC | __GFP_NOWARN); 645*2b245cb2SAnirudh Venkataramanan if (unlikely(!skb)) { 646*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.alloc_buf_failed++; 647*2b245cb2SAnirudh Venkataramanan return NULL; 648*2b245cb2SAnirudh Venkataramanan } 649*2b245cb2SAnirudh Venkataramanan 650*2b245cb2SAnirudh Venkataramanan /* we will be copying header into skb->data in 651*2b245cb2SAnirudh Venkataramanan * pskb_may_pull so it is in our interest to prefetch 652*2b245cb2SAnirudh Venkataramanan * it now to avoid a possible cache miss 653*2b245cb2SAnirudh Venkataramanan */ 654*2b245cb2SAnirudh Venkataramanan prefetchw(skb->data); 655*2b245cb2SAnirudh Venkataramanan 656*2b245cb2SAnirudh Venkataramanan skb_record_rx_queue(skb, rx_ring->q_index); 657*2b245cb2SAnirudh Venkataramanan } else { 658*2b245cb2SAnirudh Venkataramanan /* we are reusing so sync this buffer for CPU use */ 659*2b245cb2SAnirudh Venkataramanan dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 660*2b245cb2SAnirudh Venkataramanan rx_buf->page_offset, 661*2b245cb2SAnirudh Venkataramanan ICE_RXBUF_2048, 662*2b245cb2SAnirudh Venkataramanan DMA_FROM_DEVICE); 663*2b245cb2SAnirudh Venkataramanan 664*2b245cb2SAnirudh Venkataramanan rx_buf->skb = NULL; 665*2b245cb2SAnirudh Venkataramanan } 666*2b245cb2SAnirudh Venkataramanan 667*2b245cb2SAnirudh Venkataramanan /* pull page into skb */ 668*2b245cb2SAnirudh Venkataramanan if (ice_add_rx_frag(rx_buf, rx_desc, skb)) { 669*2b245cb2SAnirudh Venkataramanan /* hand second half of page back to the ring */ 670*2b245cb2SAnirudh Venkataramanan ice_reuse_rx_page(rx_ring, rx_buf); 671*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.page_reuse_count++; 672*2b245cb2SAnirudh Venkataramanan } else { 673*2b245cb2SAnirudh Venkataramanan /* we are not reusing the buffer so unmap it */ 674*2b245cb2SAnirudh Venkataramanan dma_unmap_page(rx_ring->dev, rx_buf->dma, PAGE_SIZE, 675*2b245cb2SAnirudh Venkataramanan DMA_FROM_DEVICE); 676*2b245cb2SAnirudh Venkataramanan } 677*2b245cb2SAnirudh Venkataramanan 678*2b245cb2SAnirudh Venkataramanan /* clear contents of buffer_info */ 679*2b245cb2SAnirudh Venkataramanan rx_buf->page = NULL; 680*2b245cb2SAnirudh Venkataramanan 681*2b245cb2SAnirudh Venkataramanan return skb; 682*2b245cb2SAnirudh Venkataramanan } 683*2b245cb2SAnirudh Venkataramanan 684*2b245cb2SAnirudh Venkataramanan /** 685*2b245cb2SAnirudh Venkataramanan * ice_pull_tail - ice specific version of skb_pull_tail 686*2b245cb2SAnirudh Venkataramanan * @skb: pointer to current skb being adjusted 687*2b245cb2SAnirudh Venkataramanan * 688*2b245cb2SAnirudh Venkataramanan * This function is an ice specific version of __pskb_pull_tail. The 689*2b245cb2SAnirudh Venkataramanan * main difference between this version and the original function is that 690*2b245cb2SAnirudh Venkataramanan * this function can make several assumptions about the state of things 691*2b245cb2SAnirudh Venkataramanan * that allow for significant optimizations versus the standard function. 692*2b245cb2SAnirudh Venkataramanan * As a result we can do things like drop a frag and maintain an accurate 693*2b245cb2SAnirudh Venkataramanan * truesize for the skb. 694*2b245cb2SAnirudh Venkataramanan */ 695*2b245cb2SAnirudh Venkataramanan static void ice_pull_tail(struct sk_buff *skb) 696*2b245cb2SAnirudh Venkataramanan { 697*2b245cb2SAnirudh Venkataramanan struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 698*2b245cb2SAnirudh Venkataramanan unsigned int pull_len; 699*2b245cb2SAnirudh Venkataramanan unsigned char *va; 700*2b245cb2SAnirudh Venkataramanan 701*2b245cb2SAnirudh Venkataramanan /* it is valid to use page_address instead of kmap since we are 702*2b245cb2SAnirudh Venkataramanan * working with pages allocated out of the lomem pool per 703*2b245cb2SAnirudh Venkataramanan * alloc_page(GFP_ATOMIC) 704*2b245cb2SAnirudh Venkataramanan */ 705*2b245cb2SAnirudh Venkataramanan va = skb_frag_address(frag); 706*2b245cb2SAnirudh Venkataramanan 707*2b245cb2SAnirudh Venkataramanan /* we need the header to contain the greater of either ETH_HLEN or 708*2b245cb2SAnirudh Venkataramanan * 60 bytes if the skb->len is less than 60 for skb_pad. 709*2b245cb2SAnirudh Venkataramanan */ 710*2b245cb2SAnirudh Venkataramanan pull_len = eth_get_headlen(va, ICE_RX_HDR_SIZE); 711*2b245cb2SAnirudh Venkataramanan 712*2b245cb2SAnirudh Venkataramanan /* align pull length to size of long to optimize memcpy performance */ 713*2b245cb2SAnirudh Venkataramanan skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 714*2b245cb2SAnirudh Venkataramanan 715*2b245cb2SAnirudh Venkataramanan /* update all of the pointers */ 716*2b245cb2SAnirudh Venkataramanan skb_frag_size_sub(frag, pull_len); 717*2b245cb2SAnirudh Venkataramanan frag->page_offset += pull_len; 718*2b245cb2SAnirudh Venkataramanan skb->data_len -= pull_len; 719*2b245cb2SAnirudh Venkataramanan skb->tail += pull_len; 720*2b245cb2SAnirudh Venkataramanan } 721*2b245cb2SAnirudh Venkataramanan 722*2b245cb2SAnirudh Venkataramanan /** 723*2b245cb2SAnirudh Venkataramanan * ice_cleanup_headers - Correct empty headers 724*2b245cb2SAnirudh Venkataramanan * @skb: pointer to current skb being fixed 725*2b245cb2SAnirudh Venkataramanan * 726*2b245cb2SAnirudh Venkataramanan * Also address the case where we are pulling data in on pages only 727*2b245cb2SAnirudh Venkataramanan * and as such no data is present in the skb header. 728*2b245cb2SAnirudh Venkataramanan * 729*2b245cb2SAnirudh Venkataramanan * In addition if skb is not at least 60 bytes we need to pad it so that 730*2b245cb2SAnirudh Venkataramanan * it is large enough to qualify as a valid Ethernet frame. 731*2b245cb2SAnirudh Venkataramanan * 732*2b245cb2SAnirudh Venkataramanan * Returns true if an error was encountered and skb was freed. 733*2b245cb2SAnirudh Venkataramanan */ 734*2b245cb2SAnirudh Venkataramanan static bool ice_cleanup_headers(struct sk_buff *skb) 735*2b245cb2SAnirudh Venkataramanan { 736*2b245cb2SAnirudh Venkataramanan /* place header in linear portion of buffer */ 737*2b245cb2SAnirudh Venkataramanan if (skb_is_nonlinear(skb)) 738*2b245cb2SAnirudh Venkataramanan ice_pull_tail(skb); 739*2b245cb2SAnirudh Venkataramanan 740*2b245cb2SAnirudh Venkataramanan /* if eth_skb_pad returns an error the skb was freed */ 741*2b245cb2SAnirudh Venkataramanan if (eth_skb_pad(skb)) 742*2b245cb2SAnirudh Venkataramanan return true; 743*2b245cb2SAnirudh Venkataramanan 744*2b245cb2SAnirudh Venkataramanan return false; 745*2b245cb2SAnirudh Venkataramanan } 746*2b245cb2SAnirudh Venkataramanan 747*2b245cb2SAnirudh Venkataramanan /** 748*2b245cb2SAnirudh Venkataramanan * ice_test_staterr - tests bits in Rx descriptor status and error fields 749*2b245cb2SAnirudh Venkataramanan * @rx_desc: pointer to receive descriptor (in le64 format) 750*2b245cb2SAnirudh Venkataramanan * @stat_err_bits: value to mask 751*2b245cb2SAnirudh Venkataramanan * 752*2b245cb2SAnirudh Venkataramanan * This function does some fast chicanery in order to return the 753*2b245cb2SAnirudh Venkataramanan * value of the mask which is really only used for boolean tests. 754*2b245cb2SAnirudh Venkataramanan * The status_error_len doesn't need to be shifted because it begins 755*2b245cb2SAnirudh Venkataramanan * at offset zero. 756*2b245cb2SAnirudh Venkataramanan */ 757*2b245cb2SAnirudh Venkataramanan static bool ice_test_staterr(union ice_32b_rx_flex_desc *rx_desc, 758*2b245cb2SAnirudh Venkataramanan const u16 stat_err_bits) 759*2b245cb2SAnirudh Venkataramanan { 760*2b245cb2SAnirudh Venkataramanan return !!(rx_desc->wb.status_error0 & 761*2b245cb2SAnirudh Venkataramanan cpu_to_le16(stat_err_bits)); 762*2b245cb2SAnirudh Venkataramanan } 763*2b245cb2SAnirudh Venkataramanan 764*2b245cb2SAnirudh Venkataramanan /** 765*2b245cb2SAnirudh Venkataramanan * ice_is_non_eop - process handling of non-EOP buffers 766*2b245cb2SAnirudh Venkataramanan * @rx_ring: Rx ring being processed 767*2b245cb2SAnirudh Venkataramanan * @rx_desc: Rx descriptor for current buffer 768*2b245cb2SAnirudh Venkataramanan * @skb: Current socket buffer containing buffer in progress 769*2b245cb2SAnirudh Venkataramanan * 770*2b245cb2SAnirudh Venkataramanan * This function updates next to clean. If the buffer is an EOP buffer 771*2b245cb2SAnirudh Venkataramanan * this function exits returning false, otherwise it will place the 772*2b245cb2SAnirudh Venkataramanan * sk_buff in the next buffer to be chained and return true indicating 773*2b245cb2SAnirudh Venkataramanan * that this is in fact a non-EOP buffer. 774*2b245cb2SAnirudh Venkataramanan */ 775*2b245cb2SAnirudh Venkataramanan static bool ice_is_non_eop(struct ice_ring *rx_ring, 776*2b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc, 777*2b245cb2SAnirudh Venkataramanan struct sk_buff *skb) 778*2b245cb2SAnirudh Venkataramanan { 779*2b245cb2SAnirudh Venkataramanan u32 ntc = rx_ring->next_to_clean + 1; 780*2b245cb2SAnirudh Venkataramanan 781*2b245cb2SAnirudh Venkataramanan /* fetch, update, and store next to clean */ 782*2b245cb2SAnirudh Venkataramanan ntc = (ntc < rx_ring->count) ? ntc : 0; 783*2b245cb2SAnirudh Venkataramanan rx_ring->next_to_clean = ntc; 784*2b245cb2SAnirudh Venkataramanan 785*2b245cb2SAnirudh Venkataramanan prefetch(ICE_RX_DESC(rx_ring, ntc)); 786*2b245cb2SAnirudh Venkataramanan 787*2b245cb2SAnirudh Venkataramanan /* if we are the last buffer then there is nothing else to do */ 788*2b245cb2SAnirudh Venkataramanan #define ICE_RXD_EOF BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S) 789*2b245cb2SAnirudh Venkataramanan if (likely(ice_test_staterr(rx_desc, ICE_RXD_EOF))) 790*2b245cb2SAnirudh Venkataramanan return false; 791*2b245cb2SAnirudh Venkataramanan 792*2b245cb2SAnirudh Venkataramanan /* place skb in next buffer to be received */ 793*2b245cb2SAnirudh Venkataramanan rx_ring->rx_buf[ntc].skb = skb; 794*2b245cb2SAnirudh Venkataramanan rx_ring->rx_stats.non_eop_descs++; 795*2b245cb2SAnirudh Venkataramanan 796*2b245cb2SAnirudh Venkataramanan return true; 797*2b245cb2SAnirudh Venkataramanan } 798*2b245cb2SAnirudh Venkataramanan 799*2b245cb2SAnirudh Venkataramanan /** 800*2b245cb2SAnirudh Venkataramanan * ice_receive_skb - Send a completed packet up the stack 801*2b245cb2SAnirudh Venkataramanan * @rx_ring: rx ring in play 802*2b245cb2SAnirudh Venkataramanan * @skb: packet to send up 803*2b245cb2SAnirudh Venkataramanan * @vlan_tag: vlan tag for packet 804*2b245cb2SAnirudh Venkataramanan * 805*2b245cb2SAnirudh Venkataramanan * This function sends the completed packet (via. skb) up the stack using 806*2b245cb2SAnirudh Venkataramanan * gro receive functions (with/without vlan tag) 807*2b245cb2SAnirudh Venkataramanan */ 808*2b245cb2SAnirudh Venkataramanan static void ice_receive_skb(struct ice_ring *rx_ring, struct sk_buff *skb, 809*2b245cb2SAnirudh Venkataramanan u16 vlan_tag) 810*2b245cb2SAnirudh Venkataramanan { 811*2b245cb2SAnirudh Venkataramanan if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 812*2b245cb2SAnirudh Venkataramanan (vlan_tag & VLAN_VID_MASK)) { 813*2b245cb2SAnirudh Venkataramanan __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); 814*2b245cb2SAnirudh Venkataramanan } 815*2b245cb2SAnirudh Venkataramanan napi_gro_receive(&rx_ring->q_vector->napi, skb); 816*2b245cb2SAnirudh Venkataramanan } 817*2b245cb2SAnirudh Venkataramanan 818*2b245cb2SAnirudh Venkataramanan /** 819*2b245cb2SAnirudh Venkataramanan * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 820*2b245cb2SAnirudh Venkataramanan * @rx_ring: rx descriptor ring to transact packets on 821*2b245cb2SAnirudh Venkataramanan * @budget: Total limit on number of packets to process 822*2b245cb2SAnirudh Venkataramanan * 823*2b245cb2SAnirudh Venkataramanan * This function provides a "bounce buffer" approach to Rx interrupt 824*2b245cb2SAnirudh Venkataramanan * processing. The advantage to this is that on systems that have 825*2b245cb2SAnirudh Venkataramanan * expensive overhead for IOMMU access this provides a means of avoiding 826*2b245cb2SAnirudh Venkataramanan * it by maintaining the mapping of the page to the system. 827*2b245cb2SAnirudh Venkataramanan * 828*2b245cb2SAnirudh Venkataramanan * Returns amount of work completed 829*2b245cb2SAnirudh Venkataramanan */ 830*2b245cb2SAnirudh Venkataramanan static int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) 831*2b245cb2SAnirudh Venkataramanan { 832*2b245cb2SAnirudh Venkataramanan unsigned int total_rx_bytes = 0, total_rx_pkts = 0; 833*2b245cb2SAnirudh Venkataramanan u16 cleaned_count = ICE_DESC_UNUSED(rx_ring); 834*2b245cb2SAnirudh Venkataramanan bool failure = false; 835*2b245cb2SAnirudh Venkataramanan 836*2b245cb2SAnirudh Venkataramanan /* start the loop to process RX packets bounded by 'budget' */ 837*2b245cb2SAnirudh Venkataramanan while (likely(total_rx_pkts < (unsigned int)budget)) { 838*2b245cb2SAnirudh Venkataramanan union ice_32b_rx_flex_desc *rx_desc; 839*2b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 840*2b245cb2SAnirudh Venkataramanan u16 stat_err_bits; 841*2b245cb2SAnirudh Venkataramanan u16 vlan_tag = 0; 842*2b245cb2SAnirudh Venkataramanan 843*2b245cb2SAnirudh Venkataramanan /* return some buffers to hardware, one at a time is too slow */ 844*2b245cb2SAnirudh Venkataramanan if (cleaned_count >= ICE_RX_BUF_WRITE) { 845*2b245cb2SAnirudh Venkataramanan failure = failure || 846*2b245cb2SAnirudh Venkataramanan ice_alloc_rx_bufs(rx_ring, cleaned_count); 847*2b245cb2SAnirudh Venkataramanan cleaned_count = 0; 848*2b245cb2SAnirudh Venkataramanan } 849*2b245cb2SAnirudh Venkataramanan 850*2b245cb2SAnirudh Venkataramanan /* get the RX desc from RX ring based on 'next_to_clean' */ 851*2b245cb2SAnirudh Venkataramanan rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); 852*2b245cb2SAnirudh Venkataramanan 853*2b245cb2SAnirudh Venkataramanan /* status_error_len will always be zero for unused descriptors 854*2b245cb2SAnirudh Venkataramanan * because it's cleared in cleanup, and overlaps with hdr_addr 855*2b245cb2SAnirudh Venkataramanan * which is always zero because packet split isn't used, if the 856*2b245cb2SAnirudh Venkataramanan * hardware wrote DD then it will be non-zero 857*2b245cb2SAnirudh Venkataramanan */ 858*2b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); 859*2b245cb2SAnirudh Venkataramanan if (!ice_test_staterr(rx_desc, stat_err_bits)) 860*2b245cb2SAnirudh Venkataramanan break; 861*2b245cb2SAnirudh Venkataramanan 862*2b245cb2SAnirudh Venkataramanan /* This memory barrier is needed to keep us from reading 863*2b245cb2SAnirudh Venkataramanan * any other fields out of the rx_desc until we know the 864*2b245cb2SAnirudh Venkataramanan * DD bit is set. 865*2b245cb2SAnirudh Venkataramanan */ 866*2b245cb2SAnirudh Venkataramanan dma_rmb(); 867*2b245cb2SAnirudh Venkataramanan 868*2b245cb2SAnirudh Venkataramanan /* allocate (if needed) and populate skb */ 869*2b245cb2SAnirudh Venkataramanan skb = ice_fetch_rx_buf(rx_ring, rx_desc); 870*2b245cb2SAnirudh Venkataramanan if (!skb) 871*2b245cb2SAnirudh Venkataramanan break; 872*2b245cb2SAnirudh Venkataramanan 873*2b245cb2SAnirudh Venkataramanan cleaned_count++; 874*2b245cb2SAnirudh Venkataramanan 875*2b245cb2SAnirudh Venkataramanan /* skip if it is NOP desc */ 876*2b245cb2SAnirudh Venkataramanan if (ice_is_non_eop(rx_ring, rx_desc, skb)) 877*2b245cb2SAnirudh Venkataramanan continue; 878*2b245cb2SAnirudh Venkataramanan 879*2b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S); 880*2b245cb2SAnirudh Venkataramanan if (unlikely(ice_test_staterr(rx_desc, stat_err_bits))) { 881*2b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 882*2b245cb2SAnirudh Venkataramanan continue; 883*2b245cb2SAnirudh Venkataramanan } 884*2b245cb2SAnirudh Venkataramanan 885*2b245cb2SAnirudh Venkataramanan stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S); 886*2b245cb2SAnirudh Venkataramanan if (ice_test_staterr(rx_desc, stat_err_bits)) 887*2b245cb2SAnirudh Venkataramanan vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1); 888*2b245cb2SAnirudh Venkataramanan 889*2b245cb2SAnirudh Venkataramanan /* correct empty headers and pad skb if needed (to make valid 890*2b245cb2SAnirudh Venkataramanan * ethernet frame 891*2b245cb2SAnirudh Venkataramanan */ 892*2b245cb2SAnirudh Venkataramanan if (ice_cleanup_headers(skb)) { 893*2b245cb2SAnirudh Venkataramanan skb = NULL; 894*2b245cb2SAnirudh Venkataramanan continue; 895*2b245cb2SAnirudh Venkataramanan } 896*2b245cb2SAnirudh Venkataramanan 897*2b245cb2SAnirudh Venkataramanan /* probably a little skewed due to removing CRC */ 898*2b245cb2SAnirudh Venkataramanan total_rx_bytes += skb->len; 899*2b245cb2SAnirudh Venkataramanan 900*2b245cb2SAnirudh Venkataramanan /* send completed skb up the stack */ 901*2b245cb2SAnirudh Venkataramanan ice_receive_skb(rx_ring, skb, vlan_tag); 902*2b245cb2SAnirudh Venkataramanan 903*2b245cb2SAnirudh Venkataramanan /* update budget accounting */ 904*2b245cb2SAnirudh Venkataramanan total_rx_pkts++; 905*2b245cb2SAnirudh Venkataramanan } 906*2b245cb2SAnirudh Venkataramanan 907*2b245cb2SAnirudh Venkataramanan /* update queue and vector specific stats */ 908*2b245cb2SAnirudh Venkataramanan u64_stats_update_begin(&rx_ring->syncp); 909*2b245cb2SAnirudh Venkataramanan rx_ring->stats.pkts += total_rx_pkts; 910*2b245cb2SAnirudh Venkataramanan rx_ring->stats.bytes += total_rx_bytes; 911*2b245cb2SAnirudh Venkataramanan u64_stats_update_end(&rx_ring->syncp); 912*2b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_pkts += total_rx_pkts; 913*2b245cb2SAnirudh Venkataramanan rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 914*2b245cb2SAnirudh Venkataramanan 915*2b245cb2SAnirudh Venkataramanan /* guarantee a trip back through this routine if there was a failure */ 916*2b245cb2SAnirudh Venkataramanan return failure ? budget : (int)total_rx_pkts; 917*2b245cb2SAnirudh Venkataramanan } 918*2b245cb2SAnirudh Venkataramanan 919*2b245cb2SAnirudh Venkataramanan /** 920*2b245cb2SAnirudh Venkataramanan * ice_napi_poll - NAPI polling Rx/Tx cleanup routine 921*2b245cb2SAnirudh Venkataramanan * @napi: napi struct with our devices info in it 922*2b245cb2SAnirudh Venkataramanan * @budget: amount of work driver is allowed to do this pass, in packets 923*2b245cb2SAnirudh Venkataramanan * 924*2b245cb2SAnirudh Venkataramanan * This function will clean all queues associated with a q_vector. 925*2b245cb2SAnirudh Venkataramanan * 926*2b245cb2SAnirudh Venkataramanan * Returns the amount of work done 927*2b245cb2SAnirudh Venkataramanan */ 928*2b245cb2SAnirudh Venkataramanan int ice_napi_poll(struct napi_struct *napi, int budget) 929*2b245cb2SAnirudh Venkataramanan { 930*2b245cb2SAnirudh Venkataramanan struct ice_q_vector *q_vector = 931*2b245cb2SAnirudh Venkataramanan container_of(napi, struct ice_q_vector, napi); 932*2b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = q_vector->vsi; 933*2b245cb2SAnirudh Venkataramanan struct ice_pf *pf = vsi->back; 934*2b245cb2SAnirudh Venkataramanan bool clean_complete = true; 935*2b245cb2SAnirudh Venkataramanan int budget_per_ring = 0; 936*2b245cb2SAnirudh Venkataramanan struct ice_ring *ring; 937*2b245cb2SAnirudh Venkataramanan int work_done = 0; 938*2b245cb2SAnirudh Venkataramanan 939*2b245cb2SAnirudh Venkataramanan /* Since the actual Tx work is minimal, we can give the Tx a larger 940*2b245cb2SAnirudh Venkataramanan * budget and be more aggressive about cleaning up the Tx descriptors. 941*2b245cb2SAnirudh Venkataramanan */ 942*2b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->tx) 943*2b245cb2SAnirudh Venkataramanan if (!ice_clean_tx_irq(vsi, ring, budget)) 944*2b245cb2SAnirudh Venkataramanan clean_complete = false; 945*2b245cb2SAnirudh Venkataramanan 946*2b245cb2SAnirudh Venkataramanan /* Handle case where we are called by netpoll with a budget of 0 */ 947*2b245cb2SAnirudh Venkataramanan if (budget <= 0) 948*2b245cb2SAnirudh Venkataramanan return budget; 949*2b245cb2SAnirudh Venkataramanan 950*2b245cb2SAnirudh Venkataramanan /* We attempt to distribute budget to each Rx queue fairly, but don't 951*2b245cb2SAnirudh Venkataramanan * allow the budget to go below 1 because that would exit polling early. 952*2b245cb2SAnirudh Venkataramanan */ 953*2b245cb2SAnirudh Venkataramanan if (q_vector->num_ring_rx) 954*2b245cb2SAnirudh Venkataramanan budget_per_ring = max(budget / q_vector->num_ring_rx, 1); 955*2b245cb2SAnirudh Venkataramanan 956*2b245cb2SAnirudh Venkataramanan ice_for_each_ring(ring, q_vector->rx) { 957*2b245cb2SAnirudh Venkataramanan int cleaned; 958*2b245cb2SAnirudh Venkataramanan 959*2b245cb2SAnirudh Venkataramanan cleaned = ice_clean_rx_irq(ring, budget_per_ring); 960*2b245cb2SAnirudh Venkataramanan work_done += cleaned; 961*2b245cb2SAnirudh Venkataramanan /* if we clean as many as budgeted, we must not be done */ 962*2b245cb2SAnirudh Venkataramanan if (cleaned >= budget_per_ring) 963*2b245cb2SAnirudh Venkataramanan clean_complete = false; 964*2b245cb2SAnirudh Venkataramanan } 965*2b245cb2SAnirudh Venkataramanan 966*2b245cb2SAnirudh Venkataramanan /* If work not completed, return budget and polling will return */ 967*2b245cb2SAnirudh Venkataramanan if (!clean_complete) 968*2b245cb2SAnirudh Venkataramanan return budget; 969*2b245cb2SAnirudh Venkataramanan 970*2b245cb2SAnirudh Venkataramanan /* Work is done so exit the polling mode and re-enable the interrupt */ 971*2b245cb2SAnirudh Venkataramanan napi_complete_done(napi, work_done); 972*2b245cb2SAnirudh Venkataramanan if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 973*2b245cb2SAnirudh Venkataramanan ice_irq_dynamic_ena(&vsi->back->hw, vsi, q_vector); 974*2b245cb2SAnirudh Venkataramanan return 0; 975*2b245cb2SAnirudh Venkataramanan } 976*2b245cb2SAnirudh Venkataramanan 977*2b245cb2SAnirudh Venkataramanan /* helper function for building cmd/type/offset */ 978*2b245cb2SAnirudh Venkataramanan static __le64 979*2b245cb2SAnirudh Venkataramanan build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) 980*2b245cb2SAnirudh Venkataramanan { 981*2b245cb2SAnirudh Venkataramanan return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA | 982*2b245cb2SAnirudh Venkataramanan (td_cmd << ICE_TXD_QW1_CMD_S) | 983*2b245cb2SAnirudh Venkataramanan (td_offset << ICE_TXD_QW1_OFFSET_S) | 984*2b245cb2SAnirudh Venkataramanan ((u64)size << ICE_TXD_QW1_TX_BUF_SZ_S) | 985*2b245cb2SAnirudh Venkataramanan (td_tag << ICE_TXD_QW1_L2TAG1_S)); 986*2b245cb2SAnirudh Venkataramanan } 987*2b245cb2SAnirudh Venkataramanan 988*2b245cb2SAnirudh Venkataramanan /** 989*2b245cb2SAnirudh Venkataramanan * __ice_maybe_stop_tx - 2nd level check for tx stop conditions 990*2b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 991*2b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 992*2b245cb2SAnirudh Venkataramanan * 993*2b245cb2SAnirudh Venkataramanan * Returns -EBUSY if a stop is needed, else 0 994*2b245cb2SAnirudh Venkataramanan */ 995*2b245cb2SAnirudh Venkataramanan static int __ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 996*2b245cb2SAnirudh Venkataramanan { 997*2b245cb2SAnirudh Venkataramanan netif_stop_subqueue(tx_ring->netdev, tx_ring->q_index); 998*2b245cb2SAnirudh Venkataramanan /* Memory barrier before checking head and tail */ 999*2b245cb2SAnirudh Venkataramanan smp_mb(); 1000*2b245cb2SAnirudh Venkataramanan 1001*2b245cb2SAnirudh Venkataramanan /* Check again in a case another CPU has just made room available. */ 1002*2b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) < size)) 1003*2b245cb2SAnirudh Venkataramanan return -EBUSY; 1004*2b245cb2SAnirudh Venkataramanan 1005*2b245cb2SAnirudh Venkataramanan /* A reprieve! - use start_subqueue because it doesn't call schedule */ 1006*2b245cb2SAnirudh Venkataramanan netif_start_subqueue(tx_ring->netdev, tx_ring->q_index); 1007*2b245cb2SAnirudh Venkataramanan ++tx_ring->tx_stats.restart_q; 1008*2b245cb2SAnirudh Venkataramanan return 0; 1009*2b245cb2SAnirudh Venkataramanan } 1010*2b245cb2SAnirudh Venkataramanan 1011*2b245cb2SAnirudh Venkataramanan /** 1012*2b245cb2SAnirudh Venkataramanan * ice_maybe_stop_tx - 1st level check for tx stop conditions 1013*2b245cb2SAnirudh Venkataramanan * @tx_ring: the ring to be checked 1014*2b245cb2SAnirudh Venkataramanan * @size: the size buffer we want to assure is available 1015*2b245cb2SAnirudh Venkataramanan * 1016*2b245cb2SAnirudh Venkataramanan * Returns 0 if stop is not needed 1017*2b245cb2SAnirudh Venkataramanan */ 1018*2b245cb2SAnirudh Venkataramanan static int ice_maybe_stop_tx(struct ice_ring *tx_ring, unsigned int size) 1019*2b245cb2SAnirudh Venkataramanan { 1020*2b245cb2SAnirudh Venkataramanan if (likely(ICE_DESC_UNUSED(tx_ring) >= size)) 1021*2b245cb2SAnirudh Venkataramanan return 0; 1022*2b245cb2SAnirudh Venkataramanan return __ice_maybe_stop_tx(tx_ring, size); 1023*2b245cb2SAnirudh Venkataramanan } 1024*2b245cb2SAnirudh Venkataramanan 1025*2b245cb2SAnirudh Venkataramanan /** 1026*2b245cb2SAnirudh Venkataramanan * ice_tx_map - Build the Tx descriptor 1027*2b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 1028*2b245cb2SAnirudh Venkataramanan * @first: first buffer info buffer to use 1029*2b245cb2SAnirudh Venkataramanan * 1030*2b245cb2SAnirudh Venkataramanan * This function loops over the skb data pointed to by *first 1031*2b245cb2SAnirudh Venkataramanan * and gets a physical address for each memory location and programs 1032*2b245cb2SAnirudh Venkataramanan * it and the length into the transmit descriptor. 1033*2b245cb2SAnirudh Venkataramanan */ 1034*2b245cb2SAnirudh Venkataramanan static void ice_tx_map(struct ice_ring *tx_ring, struct ice_tx_buf *first) 1035*2b245cb2SAnirudh Venkataramanan { 1036*2b245cb2SAnirudh Venkataramanan u64 td_offset = 0, td_tag = 0, td_cmd = 0; 1037*2b245cb2SAnirudh Venkataramanan u16 i = tx_ring->next_to_use; 1038*2b245cb2SAnirudh Venkataramanan struct skb_frag_struct *frag; 1039*2b245cb2SAnirudh Venkataramanan unsigned int data_len, size; 1040*2b245cb2SAnirudh Venkataramanan struct ice_tx_desc *tx_desc; 1041*2b245cb2SAnirudh Venkataramanan struct ice_tx_buf *tx_buf; 1042*2b245cb2SAnirudh Venkataramanan struct sk_buff *skb; 1043*2b245cb2SAnirudh Venkataramanan dma_addr_t dma; 1044*2b245cb2SAnirudh Venkataramanan 1045*2b245cb2SAnirudh Venkataramanan skb = first->skb; 1046*2b245cb2SAnirudh Venkataramanan 1047*2b245cb2SAnirudh Venkataramanan data_len = skb->data_len; 1048*2b245cb2SAnirudh Venkataramanan size = skb_headlen(skb); 1049*2b245cb2SAnirudh Venkataramanan 1050*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, i); 1051*2b245cb2SAnirudh Venkataramanan 1052*2b245cb2SAnirudh Venkataramanan dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1053*2b245cb2SAnirudh Venkataramanan 1054*2b245cb2SAnirudh Venkataramanan tx_buf = first; 1055*2b245cb2SAnirudh Venkataramanan 1056*2b245cb2SAnirudh Venkataramanan for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1057*2b245cb2SAnirudh Venkataramanan unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 1058*2b245cb2SAnirudh Venkataramanan 1059*2b245cb2SAnirudh Venkataramanan if (dma_mapping_error(tx_ring->dev, dma)) 1060*2b245cb2SAnirudh Venkataramanan goto dma_error; 1061*2b245cb2SAnirudh Venkataramanan 1062*2b245cb2SAnirudh Venkataramanan /* record length, and DMA address */ 1063*2b245cb2SAnirudh Venkataramanan dma_unmap_len_set(tx_buf, len, size); 1064*2b245cb2SAnirudh Venkataramanan dma_unmap_addr_set(tx_buf, dma, dma); 1065*2b245cb2SAnirudh Venkataramanan 1066*2b245cb2SAnirudh Venkataramanan /* align size to end of page */ 1067*2b245cb2SAnirudh Venkataramanan max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1); 1068*2b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 1069*2b245cb2SAnirudh Venkataramanan 1070*2b245cb2SAnirudh Venkataramanan /* account for data chunks larger than the hardware 1071*2b245cb2SAnirudh Venkataramanan * can handle 1072*2b245cb2SAnirudh Venkataramanan */ 1073*2b245cb2SAnirudh Venkataramanan while (unlikely(size > ICE_MAX_DATA_PER_TXD)) { 1074*2b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 1075*2b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, max_data, td_tag); 1076*2b245cb2SAnirudh Venkataramanan 1077*2b245cb2SAnirudh Venkataramanan tx_desc++; 1078*2b245cb2SAnirudh Venkataramanan i++; 1079*2b245cb2SAnirudh Venkataramanan 1080*2b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 1081*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1082*2b245cb2SAnirudh Venkataramanan i = 0; 1083*2b245cb2SAnirudh Venkataramanan } 1084*2b245cb2SAnirudh Venkataramanan 1085*2b245cb2SAnirudh Venkataramanan dma += max_data; 1086*2b245cb2SAnirudh Venkataramanan size -= max_data; 1087*2b245cb2SAnirudh Venkataramanan 1088*2b245cb2SAnirudh Venkataramanan max_data = ICE_MAX_DATA_PER_TXD_ALIGNED; 1089*2b245cb2SAnirudh Venkataramanan tx_desc->buf_addr = cpu_to_le64(dma); 1090*2b245cb2SAnirudh Venkataramanan } 1091*2b245cb2SAnirudh Venkataramanan 1092*2b245cb2SAnirudh Venkataramanan if (likely(!data_len)) 1093*2b245cb2SAnirudh Venkataramanan break; 1094*2b245cb2SAnirudh Venkataramanan 1095*2b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 1096*2b245cb2SAnirudh Venkataramanan size, td_tag); 1097*2b245cb2SAnirudh Venkataramanan 1098*2b245cb2SAnirudh Venkataramanan tx_desc++; 1099*2b245cb2SAnirudh Venkataramanan i++; 1100*2b245cb2SAnirudh Venkataramanan 1101*2b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) { 1102*2b245cb2SAnirudh Venkataramanan tx_desc = ICE_TX_DESC(tx_ring, 0); 1103*2b245cb2SAnirudh Venkataramanan i = 0; 1104*2b245cb2SAnirudh Venkataramanan } 1105*2b245cb2SAnirudh Venkataramanan 1106*2b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag); 1107*2b245cb2SAnirudh Venkataramanan data_len -= size; 1108*2b245cb2SAnirudh Venkataramanan 1109*2b245cb2SAnirudh Venkataramanan dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 1110*2b245cb2SAnirudh Venkataramanan DMA_TO_DEVICE); 1111*2b245cb2SAnirudh Venkataramanan 1112*2b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 1113*2b245cb2SAnirudh Venkataramanan } 1114*2b245cb2SAnirudh Venkataramanan 1115*2b245cb2SAnirudh Venkataramanan /* record bytecount for BQL */ 1116*2b245cb2SAnirudh Venkataramanan netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1117*2b245cb2SAnirudh Venkataramanan 1118*2b245cb2SAnirudh Venkataramanan /* record SW timestamp if HW timestamp is not available */ 1119*2b245cb2SAnirudh Venkataramanan skb_tx_timestamp(first->skb); 1120*2b245cb2SAnirudh Venkataramanan 1121*2b245cb2SAnirudh Venkataramanan i++; 1122*2b245cb2SAnirudh Venkataramanan if (i == tx_ring->count) 1123*2b245cb2SAnirudh Venkataramanan i = 0; 1124*2b245cb2SAnirudh Venkataramanan 1125*2b245cb2SAnirudh Venkataramanan /* write last descriptor with RS and EOP bits */ 1126*2b245cb2SAnirudh Venkataramanan td_cmd |= (u64)(ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS); 1127*2b245cb2SAnirudh Venkataramanan tx_desc->cmd_type_offset_bsz = 1128*2b245cb2SAnirudh Venkataramanan build_ctob(td_cmd, td_offset, size, td_tag); 1129*2b245cb2SAnirudh Venkataramanan 1130*2b245cb2SAnirudh Venkataramanan /* Force memory writes to complete before letting h/w know there 1131*2b245cb2SAnirudh Venkataramanan * are new descriptors to fetch. 1132*2b245cb2SAnirudh Venkataramanan * 1133*2b245cb2SAnirudh Venkataramanan * We also use this memory barrier to make certain all of the 1134*2b245cb2SAnirudh Venkataramanan * status bits have been updated before next_to_watch is written. 1135*2b245cb2SAnirudh Venkataramanan */ 1136*2b245cb2SAnirudh Venkataramanan wmb(); 1137*2b245cb2SAnirudh Venkataramanan 1138*2b245cb2SAnirudh Venkataramanan /* set next_to_watch value indicating a packet is present */ 1139*2b245cb2SAnirudh Venkataramanan first->next_to_watch = tx_desc; 1140*2b245cb2SAnirudh Venkataramanan 1141*2b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 1142*2b245cb2SAnirudh Venkataramanan 1143*2b245cb2SAnirudh Venkataramanan ice_maybe_stop_tx(tx_ring, DESC_NEEDED); 1144*2b245cb2SAnirudh Venkataramanan 1145*2b245cb2SAnirudh Venkataramanan /* notify HW of packet */ 1146*2b245cb2SAnirudh Venkataramanan if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 1147*2b245cb2SAnirudh Venkataramanan writel(i, tx_ring->tail); 1148*2b245cb2SAnirudh Venkataramanan 1149*2b245cb2SAnirudh Venkataramanan /* we need this if more than one processor can write to our tail 1150*2b245cb2SAnirudh Venkataramanan * at a time, it synchronizes IO on IA64/Altix systems 1151*2b245cb2SAnirudh Venkataramanan */ 1152*2b245cb2SAnirudh Venkataramanan mmiowb(); 1153*2b245cb2SAnirudh Venkataramanan } 1154*2b245cb2SAnirudh Venkataramanan 1155*2b245cb2SAnirudh Venkataramanan return; 1156*2b245cb2SAnirudh Venkataramanan 1157*2b245cb2SAnirudh Venkataramanan dma_error: 1158*2b245cb2SAnirudh Venkataramanan /* clear dma mappings for failed tx_buf map */ 1159*2b245cb2SAnirudh Venkataramanan for (;;) { 1160*2b245cb2SAnirudh Venkataramanan tx_buf = &tx_ring->tx_buf[i]; 1161*2b245cb2SAnirudh Venkataramanan ice_unmap_and_free_tx_buf(tx_ring, tx_buf); 1162*2b245cb2SAnirudh Venkataramanan if (tx_buf == first) 1163*2b245cb2SAnirudh Venkataramanan break; 1164*2b245cb2SAnirudh Venkataramanan if (i == 0) 1165*2b245cb2SAnirudh Venkataramanan i = tx_ring->count; 1166*2b245cb2SAnirudh Venkataramanan i--; 1167*2b245cb2SAnirudh Venkataramanan } 1168*2b245cb2SAnirudh Venkataramanan 1169*2b245cb2SAnirudh Venkataramanan tx_ring->next_to_use = i; 1170*2b245cb2SAnirudh Venkataramanan } 1171*2b245cb2SAnirudh Venkataramanan 1172*2b245cb2SAnirudh Venkataramanan /** 1173*2b245cb2SAnirudh Venkataramanan * ice_txd_use_count - estimate the number of descriptors needed for Tx 1174*2b245cb2SAnirudh Venkataramanan * @size: transmit request size in bytes 1175*2b245cb2SAnirudh Venkataramanan * 1176*2b245cb2SAnirudh Venkataramanan * Due to hardware alignment restrictions (4K alignment), we need to 1177*2b245cb2SAnirudh Venkataramanan * assume that we can have no more than 12K of data per descriptor, even 1178*2b245cb2SAnirudh Venkataramanan * though each descriptor can take up to 16K - 1 bytes of aligned memory. 1179*2b245cb2SAnirudh Venkataramanan * Thus, we need to divide by 12K. But division is slow! Instead, 1180*2b245cb2SAnirudh Venkataramanan * we decompose the operation into shifts and one relatively cheap 1181*2b245cb2SAnirudh Venkataramanan * multiply operation. 1182*2b245cb2SAnirudh Venkataramanan * 1183*2b245cb2SAnirudh Venkataramanan * To divide by 12K, we first divide by 4K, then divide by 3: 1184*2b245cb2SAnirudh Venkataramanan * To divide by 4K, shift right by 12 bits 1185*2b245cb2SAnirudh Venkataramanan * To divide by 3, multiply by 85, then divide by 256 1186*2b245cb2SAnirudh Venkataramanan * (Divide by 256 is done by shifting right by 8 bits) 1187*2b245cb2SAnirudh Venkataramanan * Finally, we add one to round up. Because 256 isn't an exact multiple of 1188*2b245cb2SAnirudh Venkataramanan * 3, we'll underestimate near each multiple of 12K. This is actually more 1189*2b245cb2SAnirudh Venkataramanan * accurate as we have 4K - 1 of wiggle room that we can fit into the last 1190*2b245cb2SAnirudh Venkataramanan * segment. For our purposes this is accurate out to 1M which is orders of 1191*2b245cb2SAnirudh Venkataramanan * magnitude greater than our largest possible GSO size. 1192*2b245cb2SAnirudh Venkataramanan * 1193*2b245cb2SAnirudh Venkataramanan * This would then be implemented as: 1194*2b245cb2SAnirudh Venkataramanan * return (((size >> 12) * 85) >> 8) + 1; 1195*2b245cb2SAnirudh Venkataramanan * 1196*2b245cb2SAnirudh Venkataramanan * Since multiplication and division are commutative, we can reorder 1197*2b245cb2SAnirudh Venkataramanan * operations into: 1198*2b245cb2SAnirudh Venkataramanan * return ((size * 85) >> 20) + 1; 1199*2b245cb2SAnirudh Venkataramanan */ 1200*2b245cb2SAnirudh Venkataramanan static unsigned int ice_txd_use_count(unsigned int size) 1201*2b245cb2SAnirudh Venkataramanan { 1202*2b245cb2SAnirudh Venkataramanan return ((size * 85) >> 20) + 1; 1203*2b245cb2SAnirudh Venkataramanan } 1204*2b245cb2SAnirudh Venkataramanan 1205*2b245cb2SAnirudh Venkataramanan /** 1206*2b245cb2SAnirudh Venkataramanan * ice_xmit_desc_count - calculate number of tx descriptors needed 1207*2b245cb2SAnirudh Venkataramanan * @skb: send buffer 1208*2b245cb2SAnirudh Venkataramanan * 1209*2b245cb2SAnirudh Venkataramanan * Returns number of data descriptors needed for this skb. 1210*2b245cb2SAnirudh Venkataramanan */ 1211*2b245cb2SAnirudh Venkataramanan static unsigned int ice_xmit_desc_count(struct sk_buff *skb) 1212*2b245cb2SAnirudh Venkataramanan { 1213*2b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1214*2b245cb2SAnirudh Venkataramanan unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 1215*2b245cb2SAnirudh Venkataramanan unsigned int count = 0, size = skb_headlen(skb); 1216*2b245cb2SAnirudh Venkataramanan 1217*2b245cb2SAnirudh Venkataramanan for (;;) { 1218*2b245cb2SAnirudh Venkataramanan count += ice_txd_use_count(size); 1219*2b245cb2SAnirudh Venkataramanan 1220*2b245cb2SAnirudh Venkataramanan if (!nr_frags--) 1221*2b245cb2SAnirudh Venkataramanan break; 1222*2b245cb2SAnirudh Venkataramanan 1223*2b245cb2SAnirudh Venkataramanan size = skb_frag_size(frag++); 1224*2b245cb2SAnirudh Venkataramanan } 1225*2b245cb2SAnirudh Venkataramanan 1226*2b245cb2SAnirudh Venkataramanan return count; 1227*2b245cb2SAnirudh Venkataramanan } 1228*2b245cb2SAnirudh Venkataramanan 1229*2b245cb2SAnirudh Venkataramanan /** 1230*2b245cb2SAnirudh Venkataramanan * __ice_chk_linearize - Check if there are more than 8 buffers per packet 1231*2b245cb2SAnirudh Venkataramanan * @skb: send buffer 1232*2b245cb2SAnirudh Venkataramanan * 1233*2b245cb2SAnirudh Venkataramanan * Note: This HW can't DMA more than 8 buffers to build a packet on the wire 1234*2b245cb2SAnirudh Venkataramanan * and so we need to figure out the cases where we need to linearize the skb. 1235*2b245cb2SAnirudh Venkataramanan * 1236*2b245cb2SAnirudh Venkataramanan * For TSO we need to count the TSO header and segment payload separately. 1237*2b245cb2SAnirudh Venkataramanan * As such we need to check cases where we have 7 fragments or more as we 1238*2b245cb2SAnirudh Venkataramanan * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 1239*2b245cb2SAnirudh Venkataramanan * the segment payload in the first descriptor, and another 7 for the 1240*2b245cb2SAnirudh Venkataramanan * fragments. 1241*2b245cb2SAnirudh Venkataramanan */ 1242*2b245cb2SAnirudh Venkataramanan static bool __ice_chk_linearize(struct sk_buff *skb) 1243*2b245cb2SAnirudh Venkataramanan { 1244*2b245cb2SAnirudh Venkataramanan const struct skb_frag_struct *frag, *stale; 1245*2b245cb2SAnirudh Venkataramanan int nr_frags, sum; 1246*2b245cb2SAnirudh Venkataramanan 1247*2b245cb2SAnirudh Venkataramanan /* no need to check if number of frags is less than 7 */ 1248*2b245cb2SAnirudh Venkataramanan nr_frags = skb_shinfo(skb)->nr_frags; 1249*2b245cb2SAnirudh Venkataramanan if (nr_frags < (ICE_MAX_BUF_TXD - 1)) 1250*2b245cb2SAnirudh Venkataramanan return false; 1251*2b245cb2SAnirudh Venkataramanan 1252*2b245cb2SAnirudh Venkataramanan /* We need to walk through the list and validate that each group 1253*2b245cb2SAnirudh Venkataramanan * of 6 fragments totals at least gso_size. 1254*2b245cb2SAnirudh Venkataramanan */ 1255*2b245cb2SAnirudh Venkataramanan nr_frags -= ICE_MAX_BUF_TXD - 2; 1256*2b245cb2SAnirudh Venkataramanan frag = &skb_shinfo(skb)->frags[0]; 1257*2b245cb2SAnirudh Venkataramanan 1258*2b245cb2SAnirudh Venkataramanan /* Initialize size to the negative value of gso_size minus 1. We 1259*2b245cb2SAnirudh Venkataramanan * use this as the worst case scenerio in which the frag ahead 1260*2b245cb2SAnirudh Venkataramanan * of us only provides one byte which is why we are limited to 6 1261*2b245cb2SAnirudh Venkataramanan * descriptors for a single transmit as the header and previous 1262*2b245cb2SAnirudh Venkataramanan * fragment are already consuming 2 descriptors. 1263*2b245cb2SAnirudh Venkataramanan */ 1264*2b245cb2SAnirudh Venkataramanan sum = 1 - skb_shinfo(skb)->gso_size; 1265*2b245cb2SAnirudh Venkataramanan 1266*2b245cb2SAnirudh Venkataramanan /* Add size of frags 0 through 4 to create our initial sum */ 1267*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1268*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1269*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1270*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1271*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1272*2b245cb2SAnirudh Venkataramanan 1273*2b245cb2SAnirudh Venkataramanan /* Walk through fragments adding latest fragment, testing it, and 1274*2b245cb2SAnirudh Venkataramanan * then removing stale fragments from the sum. 1275*2b245cb2SAnirudh Venkataramanan */ 1276*2b245cb2SAnirudh Venkataramanan stale = &skb_shinfo(skb)->frags[0]; 1277*2b245cb2SAnirudh Venkataramanan for (;;) { 1278*2b245cb2SAnirudh Venkataramanan sum += skb_frag_size(frag++); 1279*2b245cb2SAnirudh Venkataramanan 1280*2b245cb2SAnirudh Venkataramanan /* if sum is negative we failed to make sufficient progress */ 1281*2b245cb2SAnirudh Venkataramanan if (sum < 0) 1282*2b245cb2SAnirudh Venkataramanan return true; 1283*2b245cb2SAnirudh Venkataramanan 1284*2b245cb2SAnirudh Venkataramanan if (!nr_frags--) 1285*2b245cb2SAnirudh Venkataramanan break; 1286*2b245cb2SAnirudh Venkataramanan 1287*2b245cb2SAnirudh Venkataramanan sum -= skb_frag_size(stale++); 1288*2b245cb2SAnirudh Venkataramanan } 1289*2b245cb2SAnirudh Venkataramanan 1290*2b245cb2SAnirudh Venkataramanan return false; 1291*2b245cb2SAnirudh Venkataramanan } 1292*2b245cb2SAnirudh Venkataramanan 1293*2b245cb2SAnirudh Venkataramanan /** 1294*2b245cb2SAnirudh Venkataramanan * ice_chk_linearize - Check if there are more than 8 fragments per packet 1295*2b245cb2SAnirudh Venkataramanan * @skb: send buffer 1296*2b245cb2SAnirudh Venkataramanan * @count: number of buffers used 1297*2b245cb2SAnirudh Venkataramanan * 1298*2b245cb2SAnirudh Venkataramanan * Note: Our HW can't scatter-gather more than 8 fragments to build 1299*2b245cb2SAnirudh Venkataramanan * a packet on the wire and so we need to figure out the cases where we 1300*2b245cb2SAnirudh Venkataramanan * need to linearize the skb. 1301*2b245cb2SAnirudh Venkataramanan */ 1302*2b245cb2SAnirudh Venkataramanan static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count) 1303*2b245cb2SAnirudh Venkataramanan { 1304*2b245cb2SAnirudh Venkataramanan /* Both TSO and single send will work if count is less than 8 */ 1305*2b245cb2SAnirudh Venkataramanan if (likely(count < ICE_MAX_BUF_TXD)) 1306*2b245cb2SAnirudh Venkataramanan return false; 1307*2b245cb2SAnirudh Venkataramanan 1308*2b245cb2SAnirudh Venkataramanan if (skb_is_gso(skb)) 1309*2b245cb2SAnirudh Venkataramanan return __ice_chk_linearize(skb); 1310*2b245cb2SAnirudh Venkataramanan 1311*2b245cb2SAnirudh Venkataramanan /* we can support up to 8 data buffers for a single send */ 1312*2b245cb2SAnirudh Venkataramanan return count != ICE_MAX_BUF_TXD; 1313*2b245cb2SAnirudh Venkataramanan } 1314*2b245cb2SAnirudh Venkataramanan 1315*2b245cb2SAnirudh Venkataramanan /** 1316*2b245cb2SAnirudh Venkataramanan * ice_xmit_frame_ring - Sends buffer on Tx ring 1317*2b245cb2SAnirudh Venkataramanan * @skb: send buffer 1318*2b245cb2SAnirudh Venkataramanan * @tx_ring: ring to send buffer on 1319*2b245cb2SAnirudh Venkataramanan * 1320*2b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 1321*2b245cb2SAnirudh Venkataramanan */ 1322*2b245cb2SAnirudh Venkataramanan static netdev_tx_t 1323*2b245cb2SAnirudh Venkataramanan ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) 1324*2b245cb2SAnirudh Venkataramanan { 1325*2b245cb2SAnirudh Venkataramanan struct ice_tx_buf *first; 1326*2b245cb2SAnirudh Venkataramanan unsigned int count; 1327*2b245cb2SAnirudh Venkataramanan 1328*2b245cb2SAnirudh Venkataramanan count = ice_xmit_desc_count(skb); 1329*2b245cb2SAnirudh Venkataramanan if (ice_chk_linearize(skb, count)) { 1330*2b245cb2SAnirudh Venkataramanan if (__skb_linearize(skb)) 1331*2b245cb2SAnirudh Venkataramanan goto out_drop; 1332*2b245cb2SAnirudh Venkataramanan count = ice_txd_use_count(skb->len); 1333*2b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_linearize++; 1334*2b245cb2SAnirudh Venkataramanan } 1335*2b245cb2SAnirudh Venkataramanan 1336*2b245cb2SAnirudh Venkataramanan /* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD, 1337*2b245cb2SAnirudh Venkataramanan * + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD, 1338*2b245cb2SAnirudh Venkataramanan * + 4 desc gap to avoid the cache line where head is, 1339*2b245cb2SAnirudh Venkataramanan * + 1 desc for context descriptor, 1340*2b245cb2SAnirudh Venkataramanan * otherwise try next time 1341*2b245cb2SAnirudh Venkataramanan */ 1342*2b245cb2SAnirudh Venkataramanan if (ice_maybe_stop_tx(tx_ring, count + 4 + 1)) { 1343*2b245cb2SAnirudh Venkataramanan tx_ring->tx_stats.tx_busy++; 1344*2b245cb2SAnirudh Venkataramanan return NETDEV_TX_BUSY; 1345*2b245cb2SAnirudh Venkataramanan } 1346*2b245cb2SAnirudh Venkataramanan 1347*2b245cb2SAnirudh Venkataramanan /* record the location of the first descriptor for this packet */ 1348*2b245cb2SAnirudh Venkataramanan first = &tx_ring->tx_buf[tx_ring->next_to_use]; 1349*2b245cb2SAnirudh Venkataramanan first->skb = skb; 1350*2b245cb2SAnirudh Venkataramanan first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 1351*2b245cb2SAnirudh Venkataramanan first->gso_segs = 1; 1352*2b245cb2SAnirudh Venkataramanan 1353*2b245cb2SAnirudh Venkataramanan ice_tx_map(tx_ring, first); 1354*2b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 1355*2b245cb2SAnirudh Venkataramanan 1356*2b245cb2SAnirudh Venkataramanan out_drop: 1357*2b245cb2SAnirudh Venkataramanan dev_kfree_skb_any(skb); 1358*2b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 1359*2b245cb2SAnirudh Venkataramanan } 1360*2b245cb2SAnirudh Venkataramanan 1361*2b245cb2SAnirudh Venkataramanan /** 1362*2b245cb2SAnirudh Venkataramanan * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer 1363*2b245cb2SAnirudh Venkataramanan * @skb: send buffer 1364*2b245cb2SAnirudh Venkataramanan * @netdev: network interface device structure 1365*2b245cb2SAnirudh Venkataramanan * 1366*2b245cb2SAnirudh Venkataramanan * Returns NETDEV_TX_OK if sent, else an error code 1367*2b245cb2SAnirudh Venkataramanan */ 1368*2b245cb2SAnirudh Venkataramanan netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1369*2b245cb2SAnirudh Venkataramanan { 1370*2b245cb2SAnirudh Venkataramanan struct ice_netdev_priv *np = netdev_priv(netdev); 1371*2b245cb2SAnirudh Venkataramanan struct ice_vsi *vsi = np->vsi; 1372*2b245cb2SAnirudh Venkataramanan struct ice_ring *tx_ring; 1373*2b245cb2SAnirudh Venkataramanan 1374*2b245cb2SAnirudh Venkataramanan tx_ring = vsi->tx_rings[skb->queue_mapping]; 1375*2b245cb2SAnirudh Venkataramanan 1376*2b245cb2SAnirudh Venkataramanan /* hardware can't handle really short frames, hardware padding works 1377*2b245cb2SAnirudh Venkataramanan * beyond this point 1378*2b245cb2SAnirudh Venkataramanan */ 1379*2b245cb2SAnirudh Venkataramanan if (skb_put_padto(skb, ICE_MIN_TX_LEN)) 1380*2b245cb2SAnirudh Venkataramanan return NETDEV_TX_OK; 1381*2b245cb2SAnirudh Venkataramanan 1382*2b245cb2SAnirudh Venkataramanan return ice_xmit_frame_ring(skb, tx_ring); 1383*2b245cb2SAnirudh Venkataramanan } 1384