1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause 2 3 /* Packet transmit logic for Mellanox Gigabit Ethernet driver 4 * 5 * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES 6 */ 7 8 #include <linux/skbuff.h> 9 10 #include "mlxbf_gige.h" 11 #include "mlxbf_gige_regs.h" 12 13 /* Transmit Initialization 14 * 1) Allocates TX WQE array using coherent DMA mapping 15 * 2) Allocates TX completion counter using coherent DMA mapping 16 */ 17 int mlxbf_gige_tx_init(struct mlxbf_gige *priv) 18 { 19 size_t size; 20 21 size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries; 22 priv->tx_wqe_base = dma_alloc_coherent(priv->dev, size, 23 &priv->tx_wqe_base_dma, 24 GFP_KERNEL); 25 if (!priv->tx_wqe_base) 26 return -ENOMEM; 27 28 priv->tx_wqe_next = priv->tx_wqe_base; 29 30 /* Write TX WQE base address into MMIO reg */ 31 writeq(priv->tx_wqe_base_dma, priv->base + MLXBF_GIGE_TX_WQ_BASE); 32 33 /* Allocate address for TX completion count */ 34 priv->tx_cc = dma_alloc_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ, 35 &priv->tx_cc_dma, GFP_KERNEL); 36 if (!priv->tx_cc) { 37 dma_free_coherent(priv->dev, size, 38 priv->tx_wqe_base, priv->tx_wqe_base_dma); 39 return -ENOMEM; 40 } 41 42 /* Write TX CC base address into MMIO reg */ 43 writeq(priv->tx_cc_dma, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS); 44 45 writeq(ilog2(priv->tx_q_entries), 46 priv->base + MLXBF_GIGE_TX_WQ_SIZE_LOG2); 47 48 priv->prev_tx_ci = 0; 49 priv->tx_pi = 0; 50 51 return 0; 52 } 53 54 /* Transmit Deinitialization 55 * This routine will free allocations done by mlxbf_gige_tx_init(), 56 * namely the TX WQE array and the TX completion counter 57 */ 58 void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv) 59 { 60 u64 *tx_wqe_addr; 61 size_t size; 62 int i; 63 64 tx_wqe_addr = priv->tx_wqe_base; 65 66 for (i = 0; i < priv->tx_q_entries; i++) { 67 if (priv->tx_skb[i]) { 68 dma_unmap_single(priv->dev, *tx_wqe_addr, 69 priv->tx_skb[i]->len, DMA_TO_DEVICE); 70 dev_kfree_skb(priv->tx_skb[i]); 71 priv->tx_skb[i] = NULL; 72 } 73 tx_wqe_addr += 2; 74 } 75 76 size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries; 77 dma_free_coherent(priv->dev, size, 78 priv->tx_wqe_base, priv->tx_wqe_base_dma); 79 80 dma_free_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ, 81 priv->tx_cc, priv->tx_cc_dma); 82 83 priv->tx_wqe_base = NULL; 84 priv->tx_wqe_base_dma = 0; 85 priv->tx_cc = NULL; 86 priv->tx_cc_dma = 0; 87 priv->tx_wqe_next = NULL; 88 writeq(0, priv->base + MLXBF_GIGE_TX_WQ_BASE); 89 writeq(0, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS); 90 } 91 92 /* Function that returns status of TX ring: 93 * 0: TX ring is full, i.e. there are no 94 * available un-used entries in TX ring. 95 * non-null: TX ring is not full, i.e. there are 96 * some available entries in TX ring. 97 * The non-null value is a measure of 98 * how many TX entries are available, but 99 * it is not the exact number of available 100 * entries (see below). 101 * 102 * The algorithm makes the assumption that if 103 * (prev_tx_ci == tx_pi) then the TX ring is empty. 104 * An empty ring actually has (tx_q_entries-1) 105 * entries, which allows the algorithm to differentiate 106 * the case of an empty ring vs. a full ring. 107 */ 108 static u16 mlxbf_gige_tx_buffs_avail(struct mlxbf_gige *priv) 109 { 110 unsigned long flags; 111 u16 avail; 112 113 spin_lock_irqsave(&priv->lock, flags); 114 115 if (priv->prev_tx_ci == priv->tx_pi) 116 avail = priv->tx_q_entries - 1; 117 else 118 avail = ((priv->tx_q_entries + priv->prev_tx_ci - priv->tx_pi) 119 % priv->tx_q_entries) - 1; 120 121 spin_unlock_irqrestore(&priv->lock, flags); 122 123 return avail; 124 } 125 126 bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv) 127 { 128 struct net_device_stats *stats; 129 u16 tx_wqe_index; 130 u64 *tx_wqe_addr; 131 u64 tx_status; 132 u16 tx_ci; 133 134 tx_status = readq(priv->base + MLXBF_GIGE_TX_STATUS); 135 if (tx_status & MLXBF_GIGE_TX_STATUS_DATA_FIFO_FULL) 136 priv->stats.tx_fifo_full++; 137 tx_ci = readq(priv->base + MLXBF_GIGE_TX_CONSUMER_INDEX); 138 stats = &priv->netdev->stats; 139 140 /* Transmit completion logic needs to loop until the completion 141 * index (in SW) equals TX consumer index (from HW). These 142 * parameters are unsigned 16-bit values and the wrap case needs 143 * to be supported, that is TX consumer index wrapped from 0xFFFF 144 * to 0 while TX completion index is still < 0xFFFF. 145 */ 146 for (; priv->prev_tx_ci != tx_ci; priv->prev_tx_ci++) { 147 tx_wqe_index = priv->prev_tx_ci % priv->tx_q_entries; 148 /* Each TX WQE is 16 bytes. The 8 MSB store the 2KB TX 149 * buffer address and the 8 LSB contain information 150 * about the TX WQE. 151 */ 152 tx_wqe_addr = priv->tx_wqe_base + 153 (tx_wqe_index * MLXBF_GIGE_TX_WQE_SZ_QWORDS); 154 155 stats->tx_packets++; 156 stats->tx_bytes += MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr); 157 158 dma_unmap_single(priv->dev, *tx_wqe_addr, 159 priv->tx_skb[tx_wqe_index]->len, DMA_TO_DEVICE); 160 dev_consume_skb_any(priv->tx_skb[tx_wqe_index]); 161 priv->tx_skb[tx_wqe_index] = NULL; 162 163 /* Ensure completion of updates across all cores */ 164 mb(); 165 } 166 167 /* Since the TX ring was likely just drained, check if TX queue 168 * had previously been stopped and now that there are TX buffers 169 * available the TX queue can be awakened. 170 */ 171 if (netif_queue_stopped(priv->netdev) && 172 mlxbf_gige_tx_buffs_avail(priv)) 173 netif_wake_queue(priv->netdev); 174 175 return true; 176 } 177 178 /* Function to advance the tx_wqe_next pointer to next TX WQE */ 179 void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv) 180 { 181 /* Advance tx_wqe_next pointer */ 182 priv->tx_wqe_next += MLXBF_GIGE_TX_WQE_SZ_QWORDS; 183 184 /* Check if 'next' pointer is beyond end of TX ring */ 185 /* If so, set 'next' back to 'base' pointer of ring */ 186 if (priv->tx_wqe_next == (priv->tx_wqe_base + 187 (priv->tx_q_entries * MLXBF_GIGE_TX_WQE_SZ_QWORDS))) 188 priv->tx_wqe_next = priv->tx_wqe_base; 189 } 190 191 netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb, 192 struct net_device *netdev) 193 { 194 struct mlxbf_gige *priv = netdev_priv(netdev); 195 long buff_addr, start_dma_page, end_dma_page; 196 struct sk_buff *tx_skb; 197 dma_addr_t tx_buf_dma; 198 unsigned long flags; 199 u64 *tx_wqe_addr; 200 u64 word2; 201 202 /* If needed, linearize TX SKB as hardware DMA expects this */ 203 if (skb->len > MLXBF_GIGE_DEFAULT_BUF_SZ || skb_linearize(skb)) { 204 dev_kfree_skb(skb); 205 netdev->stats.tx_dropped++; 206 return NETDEV_TX_OK; 207 } 208 209 buff_addr = (long)skb->data; 210 start_dma_page = buff_addr >> MLXBF_GIGE_DMA_PAGE_SHIFT; 211 end_dma_page = (buff_addr + skb->len - 1) >> MLXBF_GIGE_DMA_PAGE_SHIFT; 212 213 /* Verify that payload pointer and data length of SKB to be 214 * transmitted does not violate the hardware DMA limitation. 215 */ 216 if (start_dma_page != end_dma_page) { 217 /* DMA operation would fail as-is, alloc new aligned SKB */ 218 tx_skb = mlxbf_gige_alloc_skb(priv, skb->len, 219 &tx_buf_dma, DMA_TO_DEVICE); 220 if (!tx_skb) { 221 /* Free original skb, could not alloc new aligned SKB */ 222 dev_kfree_skb(skb); 223 netdev->stats.tx_dropped++; 224 return NETDEV_TX_OK; 225 } 226 227 skb_put_data(tx_skb, skb->data, skb->len); 228 229 /* Free the original SKB */ 230 dev_kfree_skb(skb); 231 } else { 232 tx_skb = skb; 233 tx_buf_dma = dma_map_single(priv->dev, skb->data, 234 skb->len, DMA_TO_DEVICE); 235 if (dma_mapping_error(priv->dev, tx_buf_dma)) { 236 dev_kfree_skb(skb); 237 netdev->stats.tx_dropped++; 238 return NETDEV_TX_OK; 239 } 240 } 241 242 /* Get address of TX WQE */ 243 tx_wqe_addr = priv->tx_wqe_next; 244 245 mlxbf_gige_update_tx_wqe_next(priv); 246 247 /* Put PA of buffer address into first 64-bit word of TX WQE */ 248 *tx_wqe_addr = tx_buf_dma; 249 250 /* Set TX WQE pkt_len appropriately 251 * NOTE: GigE silicon will automatically pad up to 252 * minimum packet length if needed. 253 */ 254 word2 = tx_skb->len & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK; 255 256 /* Write entire 2nd word of TX WQE */ 257 *(tx_wqe_addr + 1) = word2; 258 259 spin_lock_irqsave(&priv->lock, flags); 260 priv->tx_skb[priv->tx_pi % priv->tx_q_entries] = tx_skb; 261 priv->tx_pi++; 262 spin_unlock_irqrestore(&priv->lock, flags); 263 264 if (!netdev_xmit_more()) { 265 /* Create memory barrier before write to TX PI */ 266 wmb(); 267 writeq(priv->tx_pi, priv->base + MLXBF_GIGE_TX_PRODUCER_INDEX); 268 } 269 270 /* Check if the last TX entry was just used */ 271 if (!mlxbf_gige_tx_buffs_avail(priv)) { 272 /* TX ring is full, inform stack */ 273 netif_stop_queue(netdev); 274 275 /* Since there is no separate "TX complete" interrupt, need 276 * to explicitly schedule NAPI poll. This will trigger logic 277 * which processes TX completions, and will hopefully drain 278 * the TX ring allowing the TX queue to be awakened. 279 */ 280 napi_schedule(&priv->napi); 281 } 282 283 return NETDEV_TX_OK; 284 } 285