1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell Octeon EP (EndPoint) Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/pci.h> 9 #include <linux/etherdevice.h> 10 #include <linux/vmalloc.h> 11 12 #include "octep_config.h" 13 #include "octep_main.h" 14 15 /* Reset various index of Tx queue data structure. */ 16 static void octep_iq_reset_indices(struct octep_iq *iq) 17 { 18 iq->fill_cnt = 0; 19 iq->host_write_index = 0; 20 iq->octep_read_index = 0; 21 iq->flush_index = 0; 22 iq->pkts_processed = 0; 23 iq->pkt_in_done = 0; 24 atomic_set(&iq->instr_pending, 0); 25 } 26 27 /** 28 * octep_iq_process_completions() - Process Tx queue completions. 29 * 30 * @iq: Octeon Tx queue data structure. 31 * @budget: max number of completions to be processed in one invocation. 32 */ 33 int octep_iq_process_completions(struct octep_iq *iq, u16 budget) 34 { 35 u32 compl_pkts, compl_bytes, compl_sg; 36 struct octep_device *oct = iq->octep_dev; 37 struct octep_tx_buffer *tx_buffer; 38 struct skb_shared_info *shinfo; 39 u32 fi = iq->flush_index; 40 struct sk_buff *skb; 41 u8 frags, i; 42 43 compl_pkts = 0; 44 compl_sg = 0; 45 compl_bytes = 0; 46 iq->octep_read_index = oct->hw_ops.update_iq_read_idx(iq); 47 48 while (likely(budget && (fi != iq->octep_read_index))) { 49 tx_buffer = iq->buff_info + fi; 50 skb = tx_buffer->skb; 51 52 fi++; 53 if (unlikely(fi == iq->max_count)) 54 fi = 0; 55 compl_bytes += skb->len; 56 compl_pkts++; 57 budget--; 58 59 if (!tx_buffer->gather) { 60 dma_unmap_single(iq->dev, tx_buffer->dma, 61 tx_buffer->skb->len, DMA_TO_DEVICE); 62 dev_kfree_skb_any(skb); 63 continue; 64 } 65 66 /* Scatter/Gather */ 67 shinfo = skb_shinfo(skb); 68 frags = shinfo->nr_frags; 69 compl_sg++; 70 71 dma_unmap_single(iq->dev, tx_buffer->sglist[0].dma_ptr[0], 72 tx_buffer->sglist[0].len[3], DMA_TO_DEVICE); 73 74 i = 1; /* entry 0 is main skb, unmapped above */ 75 while (frags--) { 76 dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 77 tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE); 78 i++; 79 } 80 81 dev_kfree_skb_any(skb); 82 } 83 84 iq->pkts_processed += compl_pkts; 85 atomic_sub(compl_pkts, &iq->instr_pending); 86 iq->stats.instr_completed += compl_pkts; 87 iq->stats.bytes_sent += compl_bytes; 88 iq->stats.sgentry_sent += compl_sg; 89 iq->flush_index = fi; 90 91 netdev_tx_completed_queue(iq->netdev_q, compl_pkts, compl_bytes); 92 93 if (unlikely(__netif_subqueue_stopped(iq->netdev, iq->q_no)) && 94 ((iq->max_count - atomic_read(&iq->instr_pending)) > 95 OCTEP_WAKE_QUEUE_THRESHOLD)) 96 netif_wake_subqueue(iq->netdev, iq->q_no); 97 return !budget; 98 } 99 100 /** 101 * octep_iq_free_pending() - Free Tx buffers for pending completions. 102 * 103 * @iq: Octeon Tx queue data structure. 104 */ 105 static void octep_iq_free_pending(struct octep_iq *iq) 106 { 107 struct octep_tx_buffer *tx_buffer; 108 struct skb_shared_info *shinfo; 109 u32 fi = iq->flush_index; 110 struct sk_buff *skb; 111 u8 frags, i; 112 113 while (fi != iq->host_write_index) { 114 tx_buffer = iq->buff_info + fi; 115 skb = tx_buffer->skb; 116 117 fi++; 118 if (unlikely(fi == iq->max_count)) 119 fi = 0; 120 121 if (!tx_buffer->gather) { 122 dma_unmap_single(iq->dev, tx_buffer->dma, 123 tx_buffer->skb->len, DMA_TO_DEVICE); 124 dev_kfree_skb_any(skb); 125 continue; 126 } 127 128 /* Scatter/Gather */ 129 shinfo = skb_shinfo(skb); 130 frags = shinfo->nr_frags; 131 132 dma_unmap_single(iq->dev, 133 tx_buffer->sglist[0].dma_ptr[0], 134 tx_buffer->sglist[0].len[3], 135 DMA_TO_DEVICE); 136 137 i = 1; /* entry 0 is main skb, unmapped above */ 138 while (frags--) { 139 dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], 140 tx_buffer->sglist[i >> 2].len[3 - (i & 3)], DMA_TO_DEVICE); 141 i++; 142 } 143 144 dev_kfree_skb_any(skb); 145 } 146 147 atomic_set(&iq->instr_pending, 0); 148 iq->flush_index = fi; 149 netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no)); 150 } 151 152 /** 153 * octep_clean_iqs() - Clean Tx queues to shutdown the device. 154 * 155 * @oct: Octeon device private data structure. 156 * 157 * Free the buffers in Tx queue descriptors pending completion and 158 * reset queue indices 159 */ 160 void octep_clean_iqs(struct octep_device *oct) 161 { 162 int i; 163 164 for (i = 0; i < oct->num_iqs; i++) { 165 octep_iq_free_pending(oct->iq[i]); 166 octep_iq_reset_indices(oct->iq[i]); 167 } 168 } 169 170 /** 171 * octep_setup_iq() - Setup a Tx queue. 172 * 173 * @oct: Octeon device private data structure. 174 * @q_no: Tx queue number to be setup. 175 * 176 * Allocate resources for a Tx queue. 177 */ 178 static int octep_setup_iq(struct octep_device *oct, int q_no) 179 { 180 u32 desc_ring_size, buff_info_size, sglist_size; 181 struct octep_iq *iq; 182 int i; 183 184 iq = vzalloc(sizeof(*iq)); 185 if (!iq) 186 goto iq_alloc_err; 187 oct->iq[q_no] = iq; 188 189 iq->octep_dev = oct; 190 iq->netdev = oct->netdev; 191 iq->dev = &oct->pdev->dev; 192 iq->q_no = q_no; 193 iq->max_count = CFG_GET_IQ_NUM_DESC(oct->conf); 194 iq->ring_size_mask = iq->max_count - 1; 195 iq->fill_threshold = CFG_GET_IQ_DB_MIN(oct->conf); 196 iq->netdev_q = netdev_get_tx_queue(iq->netdev, q_no); 197 198 /* Allocate memory for hardware queue descriptors */ 199 desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 200 iq->desc_ring = dma_alloc_coherent(iq->dev, desc_ring_size, 201 &iq->desc_ring_dma, GFP_KERNEL); 202 if (unlikely(!iq->desc_ring)) { 203 dev_err(iq->dev, 204 "Failed to allocate DMA memory for IQ-%d\n", q_no); 205 goto desc_dma_alloc_err; 206 } 207 208 /* Allocate memory for hardware SGLIST descriptors */ 209 sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * 210 CFG_GET_IQ_NUM_DESC(oct->conf); 211 iq->sglist = dma_alloc_coherent(iq->dev, sglist_size, 212 &iq->sglist_dma, GFP_KERNEL); 213 if (unlikely(!iq->sglist)) { 214 dev_err(iq->dev, 215 "Failed to allocate DMA memory for IQ-%d SGLIST\n", 216 q_no); 217 goto sglist_alloc_err; 218 } 219 220 /* allocate memory to manage Tx packets pending completion */ 221 buff_info_size = OCTEP_IQ_TXBUFF_INFO_SIZE * iq->max_count; 222 iq->buff_info = vzalloc(buff_info_size); 223 if (!iq->buff_info) { 224 dev_err(iq->dev, 225 "Failed to allocate buff info for IQ-%d\n", q_no); 226 goto buff_info_err; 227 } 228 229 /* Setup sglist addresses in tx_buffer entries */ 230 for (i = 0; i < CFG_GET_IQ_NUM_DESC(oct->conf); i++) { 231 struct octep_tx_buffer *tx_buffer; 232 233 tx_buffer = &iq->buff_info[i]; 234 tx_buffer->sglist = 235 &iq->sglist[i * OCTEP_SGLIST_ENTRIES_PER_PKT]; 236 tx_buffer->sglist_dma = 237 iq->sglist_dma + (i * OCTEP_SGLIST_SIZE_PER_PKT); 238 } 239 240 octep_iq_reset_indices(iq); 241 oct->hw_ops.setup_iq_regs(oct, q_no); 242 243 oct->num_iqs++; 244 return 0; 245 246 buff_info_err: 247 dma_free_coherent(iq->dev, sglist_size, iq->sglist, iq->sglist_dma); 248 sglist_alloc_err: 249 dma_free_coherent(iq->dev, desc_ring_size, 250 iq->desc_ring, iq->desc_ring_dma); 251 desc_dma_alloc_err: 252 vfree(iq); 253 oct->iq[q_no] = NULL; 254 iq_alloc_err: 255 return -1; 256 } 257 258 /** 259 * octep_free_iq() - Free Tx queue resources. 260 * 261 * @iq: Octeon Tx queue data structure. 262 * 263 * Free all the resources allocated for a Tx queue. 264 */ 265 static void octep_free_iq(struct octep_iq *iq) 266 { 267 struct octep_device *oct = iq->octep_dev; 268 u64 desc_ring_size, sglist_size; 269 int q_no = iq->q_no; 270 271 desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); 272 273 vfree(iq->buff_info); 274 275 if (iq->desc_ring) 276 dma_free_coherent(iq->dev, desc_ring_size, 277 iq->desc_ring, iq->desc_ring_dma); 278 279 sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * 280 CFG_GET_IQ_NUM_DESC(oct->conf); 281 if (iq->sglist) 282 dma_free_coherent(iq->dev, sglist_size, 283 iq->sglist, iq->sglist_dma); 284 285 vfree(iq); 286 oct->iq[q_no] = NULL; 287 oct->num_iqs--; 288 } 289 290 /** 291 * octep_setup_iqs() - setup resources for all Tx queues. 292 * 293 * @oct: Octeon device private data structure. 294 */ 295 int octep_setup_iqs(struct octep_device *oct) 296 { 297 int i; 298 299 oct->num_iqs = 0; 300 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 301 if (octep_setup_iq(oct, i)) { 302 dev_err(&oct->pdev->dev, 303 "Failed to setup IQ(TxQ)-%d.\n", i); 304 goto iq_setup_err; 305 } 306 dev_dbg(&oct->pdev->dev, "Successfully setup IQ(TxQ)-%d.\n", i); 307 } 308 309 return 0; 310 311 iq_setup_err: 312 while (i) { 313 i--; 314 octep_free_iq(oct->iq[i]); 315 } 316 return -1; 317 } 318 319 /** 320 * octep_free_iqs() - Free resources of all Tx queues. 321 * 322 * @oct: Octeon device private data structure. 323 */ 324 void octep_free_iqs(struct octep_device *oct) 325 { 326 int i; 327 328 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 329 octep_free_iq(oct->iq[i]); 330 dev_dbg(&oct->pdev->dev, 331 "Successfully destroyed IQ(TxQ)-%d.\n", i); 332 } 333 oct->num_iqs = 0; 334 } 335