1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Driver for Alibaba Elastic Ethernet Adapter. 4 * 5 * Copyright (C) 2025 Alibaba Inc. 6 */ 7 8 #include <net/netdev_queues.h> 9 10 #include "eea_net.h" 11 #include "eea_pci.h" 12 #include "eea_ring.h" 13 14 struct eea_sq_free_stats { 15 u64 packets; 16 u64 bytes; 17 }; 18 19 struct eea_tx_meta { 20 struct eea_tx_meta *next; 21 22 u32 id; 23 24 union { 25 struct sk_buff *skb; 26 void *data; 27 }; 28 29 u32 num; 30 31 dma_addr_t dma_addr; 32 struct eea_tx_desc *desc; 33 u32 dma_len; 34 bool unmap; 35 bool unmap_single; 36 }; 37 38 static struct eea_tx_meta *eea_tx_meta_get(struct eea_net_tx *tx) 39 { 40 struct eea_tx_meta *meta; 41 42 if (!tx->free) 43 return NULL; 44 45 meta = tx->free; 46 tx->free = meta->next; 47 48 return meta; 49 } 50 51 static void eea_tx_meta_put_and_unmap(struct eea_net_tx *tx, 52 struct eea_tx_meta *meta) 53 { 54 struct eea_tx_meta *head; 55 56 head = meta; 57 58 while (true) { 59 if (meta->unmap) { 60 if (meta->unmap_single) 61 dma_unmap_single(tx->dma_dev, meta->dma_addr, 62 meta->dma_len, DMA_TO_DEVICE); 63 else 64 dma_unmap_page(tx->dma_dev, meta->dma_addr, 65 meta->dma_len, DMA_TO_DEVICE); 66 } 67 68 if (meta->next) { 69 meta = meta->next; 70 continue; 71 } 72 73 break; 74 } 75 76 meta->next = tx->free; 77 tx->free = head; 78 } 79 80 static void eea_meta_free_xmit(struct eea_net_tx *tx, 81 struct eea_tx_meta *meta, 82 int budget, 83 struct eea_tx_cdesc *desc, 84 struct eea_sq_free_stats *stats) 85 { 86 struct sk_buff *skb = meta->skb; 87 88 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && desc)) { 89 struct skb_shared_hwtstamps ts = {}; 90 91 ts.hwtstamp = EEA_DESC_TS(desc) + tx->enet->hw_ts_offset; 92 skb_tstamp_tx(skb, &ts); 93 } 94 95 ++stats->packets; 96 stats->bytes += meta->skb->len; 97 napi_consume_skb(meta->skb, budget); 98 99 meta->data = NULL; 100 } 101 102 static int eea_clean_tx(struct eea_net_tx *tx, int budget) 103 { 104 struct eea_sq_free_stats stats = {0}; 105 struct eea_tx_cdesc *desc; 106 struct eea_tx_meta *meta; 107 int desc_n; 108 u16 id; 109 110 while (stats.packets < budget) { 111 desc = eea_ering_cq_get_desc(tx->ering); 112 if (!desc) 113 break; 114 115 id = le16_to_cpu(desc->id); 116 if (unlikely(id >= tx->ering->num)) { 117 if (net_ratelimit()) 118 netdev_err(tx->enet->netdev, "tx invalid id %d\n", 119 id); 120 eea_ering_cq_ack_desc(tx->ering, 1); 121 continue; 122 } 123 124 meta = &tx->meta[id]; 125 126 if (meta->data) { 127 eea_tx_meta_put_and_unmap(tx, meta); 128 eea_meta_free_xmit(tx, meta, budget, desc, &stats); 129 desc_n = meta->num; 130 } else { 131 if (net_ratelimit()) 132 netdev_err(tx->enet->netdev, 133 "tx meta->data is null. id %d num: %d\n", 134 meta->id, meta->num); 135 desc_n = 1; 136 } 137 138 eea_ering_cq_ack_desc(tx->ering, desc_n); 139 } 140 141 if (stats.packets) { 142 u64_stats_update_begin(&tx->stats.syncp); 143 u64_stats_add(&tx->stats.bytes, stats.bytes); 144 u64_stats_add(&tx->stats.packets, stats.packets); 145 u64_stats_update_end(&tx->stats.syncp); 146 } 147 148 return stats.packets; 149 } 150 151 int eea_poll_tx(struct eea_net_tx *tx, int budget) 152 { 153 struct eea_net *enet = tx->enet; 154 u32 index = tx - enet->tx; 155 struct netdev_queue *txq; 156 int num; 157 158 txq = netdev_get_tx_queue(enet->netdev, index); 159 160 __netif_tx_lock(txq, smp_processor_id()); 161 162 num = eea_clean_tx(tx, budget); 163 164 if (netif_tx_queue_stopped(txq) && 165 tx->ering->num_free >= MAX_SKB_FRAGS + 2) 166 netif_tx_wake_queue(txq); 167 168 __netif_tx_unlock(txq); 169 170 return num; 171 } 172 173 static int eea_fill_desc_from_skb(const struct sk_buff *skb, 174 struct eea_tx_desc *desc) 175 { 176 if (skb_is_gso(skb)) { 177 struct skb_shared_info *sinfo = skb_shinfo(skb); 178 179 desc->gso_size = cpu_to_le16(sinfo->gso_size); 180 if (sinfo->gso_type & SKB_GSO_TCPV4) 181 desc->gso_type = EEA_TX_GSO_TCPV4; 182 183 else if (sinfo->gso_type & SKB_GSO_TCPV6) 184 desc->gso_type = EEA_TX_GSO_TCPV6; 185 186 else if (sinfo->gso_type & SKB_GSO_UDP_L4) 187 desc->gso_type = EEA_TX_GSO_UDP_L4; 188 189 else 190 return -EINVAL; 191 192 if (sinfo->gso_type & SKB_GSO_TCP_ECN) 193 desc->gso_type |= EEA_TX_GSO_ECN; 194 } else { 195 desc->gso_type = EEA_TX_GSO_NONE; 196 } 197 198 if (skb->ip_summed == CHECKSUM_PARTIAL) { 199 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 200 desc->csum_offset = cpu_to_le16(skb->csum_offset); 201 } 202 203 return 0; 204 } 205 206 static struct eea_tx_meta *__eea_tx_desc_fill(struct eea_net_tx *tx, 207 struct eea_tx_meta *head_meta, 208 dma_addr_t addr, u32 data_len, 209 u32 dma_len, bool last, 210 void *data, u16 flags, 211 bool unmap) 212 { 213 struct eea_tx_meta *meta; 214 struct eea_tx_desc *desc; 215 216 meta = eea_tx_meta_get(tx); 217 218 desc = eea_ering_sq_alloc_desc(tx->ering, meta->id, last, flags); 219 desc->addr = cpu_to_le64(addr); 220 desc->len = cpu_to_le16(data_len); 221 222 meta->next = NULL; 223 meta->dma_len = dma_len; 224 meta->dma_addr = addr; 225 meta->data = data; 226 meta->num = 1; 227 meta->desc = desc; 228 meta->unmap = unmap; 229 meta->unmap_single = false; 230 231 if (head_meta) { 232 meta->next = head_meta->next; 233 head_meta->next = meta; 234 ++head_meta->num; 235 } 236 237 return meta; 238 } 239 240 static struct eea_tx_meta *eea_tx_desc_fill(struct eea_net_tx *tx, 241 struct eea_tx_meta *head_meta, 242 dma_addr_t addr, u32 length, 243 bool is_last, void *data, u16 flags) 244 { 245 struct eea_tx_meta *meta; 246 u16 len, last; 247 248 WARN_ON_ONCE(length >= 2 * USHRT_MAX); 249 250 /* Since eea does not support BIG TCP, the maximum GSO size is capped at 251 * 64KB. Consequently, a single skb buffer (head or fragment) will not 252 * require more than two descriptors 253 */ 254 if (length > USHRT_MAX) { 255 len = USHRT_MAX; 256 last = false; 257 } else { 258 len = length; 259 last = is_last; 260 } 261 262 meta = __eea_tx_desc_fill(tx, head_meta, addr, len, length, 263 last, data, flags, true); 264 265 if (length > USHRT_MAX) { 266 if (!head_meta) 267 head_meta = meta; 268 269 addr += USHRT_MAX; 270 len = length - USHRT_MAX; 271 272 __eea_tx_desc_fill(tx, head_meta, addr, len, 0, is_last, 273 NULL, 0, false); 274 } 275 276 return meta; 277 } 278 279 static int eea_tx_add_skb_frag(struct eea_net_tx *tx, 280 struct eea_tx_meta *head_meta, 281 const skb_frag_t *frag, bool is_last) 282 { 283 u32 len = skb_frag_size(frag); 284 dma_addr_t addr; 285 286 addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, DMA_TO_DEVICE); 287 if (unlikely(dma_mapping_error(tx->dma_dev, addr))) 288 return -ENOMEM; 289 290 eea_tx_desc_fill(tx, head_meta, addr, len, is_last, NULL, 0); 291 292 return 0; 293 } 294 295 static int eea_tx_post_skb(struct eea_net_tx *tx, struct sk_buff *skb) 296 { 297 const struct skb_shared_info *shinfo = skb_shinfo(skb); 298 u32 hlen = skb_headlen(skb); 299 struct eea_tx_meta *meta; 300 const skb_frag_t *frag; 301 dma_addr_t addr; 302 u32 len = hlen; 303 int i, err; 304 u16 flags; 305 bool last; 306 307 if (len) { 308 addr = dma_map_single(tx->dma_dev, skb->data, len, 309 DMA_TO_DEVICE); 310 if (unlikely(dma_mapping_error(tx->dma_dev, addr))) 311 return -ENOMEM; 312 313 last = !shinfo->nr_frags; 314 i = 0; 315 } else { 316 /* The net stack will never submit an skb with an skb->len of 317 * 0. If the head len is 0, the number of frags must be greater 318 * than 0. 319 */ 320 frag = &shinfo->frags[0]; 321 len = skb_frag_size(frag); 322 323 addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, 324 DMA_TO_DEVICE); 325 if (unlikely(dma_mapping_error(tx->dma_dev, addr))) 326 return -ENOMEM; 327 328 last = shinfo->nr_frags == 1; 329 i = 1; 330 } 331 332 flags = skb->ip_summed == CHECKSUM_PARTIAL ? EEA_DESC_F_DO_CSUM : 0; 333 334 meta = eea_tx_desc_fill(tx, NULL, addr, len, last, skb, flags); 335 meta->unmap_single = !!hlen; 336 337 err = eea_fill_desc_from_skb(skb, meta->desc); 338 if (err) 339 goto err_cancel; 340 341 for (; i < shinfo->nr_frags; i++) { 342 frag = &shinfo->frags[i]; 343 bool is_last = i == (shinfo->nr_frags - 1); 344 345 err = eea_tx_add_skb_frag(tx, meta, frag, is_last); 346 if (err) 347 goto err_cancel; 348 } 349 350 eea_ering_sq_commit_desc(tx->ering); 351 352 u64_stats_update_begin(&tx->stats.syncp); 353 u64_stats_add(&tx->stats.descs, meta->num); 354 u64_stats_update_end(&tx->stats.syncp); 355 356 return 0; 357 358 err_cancel: 359 eea_ering_sq_cancel(tx->ering); 360 eea_tx_meta_put_and_unmap(tx, meta); 361 meta->data = NULL; 362 return err; 363 } 364 365 static void eea_tx_kick(struct eea_net_tx *tx) 366 { 367 eea_ering_kick(tx->ering); 368 369 u64_stats_update_begin(&tx->stats.syncp); 370 u64_stats_inc(&tx->stats.kicks); 371 u64_stats_update_end(&tx->stats.syncp); 372 } 373 374 static int eea_tx_check_free_num(struct eea_net_tx *tx, 375 struct netdev_queue *txq) 376 { 377 int n; 378 379 /* MAX_SKB_FRAGS + 1: Covers the skb linear head and all paged fragments 380 * 1: Extra slot for a head or fragment that exceeds 64KB. 381 */ 382 n = MAX_SKB_FRAGS + 2; 383 return netif_txq_maybe_stop(txq, tx->ering->num_free, n, n); 384 } 385 386 netdev_tx_t eea_tx_xmit(struct sk_buff *skb, struct net_device *netdev) 387 { 388 struct eea_net *enet = netdev_priv(netdev); 389 int qnum = skb_get_queue_mapping(skb); 390 struct eea_net_tx *tx = &enet->tx[qnum]; 391 struct netdev_queue *txq; 392 int err, enable; 393 394 txq = netdev_get_tx_queue(netdev, qnum); 395 396 enable = eea_tx_check_free_num(tx, txq); 397 if (!enable) 398 return NETDEV_TX_BUSY; 399 400 err = eea_tx_post_skb(tx, skb); 401 if (unlikely(err)) { 402 u64_stats_update_begin(&tx->stats.syncp); 403 u64_stats_inc(&tx->stats.drops); 404 u64_stats_update_end(&tx->stats.syncp); 405 406 dev_kfree_skb_any(skb); 407 } else { 408 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 409 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 410 skb_tx_timestamp(skb); 411 } 412 413 /* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */ 414 eea_tx_check_free_num(tx, txq); 415 416 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 417 eea_tx_kick(tx); 418 419 return NETDEV_TX_OK; 420 } 421 422 static void eea_free_meta(struct eea_net_tx *tx, struct eea_net_cfg *cfg) 423 { 424 struct eea_sq_free_stats stats = {0}; 425 struct eea_tx_meta *meta; 426 int i; 427 428 while ((meta = eea_tx_meta_get(tx))) 429 meta->skb = NULL; 430 431 for (i = 0; i < cfg->tx_ring_depth; i++) { 432 meta = &tx->meta[i]; 433 434 if (!meta->skb) 435 continue; 436 437 eea_tx_meta_put_and_unmap(tx, meta); 438 439 eea_meta_free_xmit(tx, meta, 0, NULL, &stats); 440 } 441 442 kvfree(tx->meta); 443 tx->meta = NULL; 444 } 445 446 /* Maybe called before eea_bind_q_and_cfg. So the cfg must be passed. */ 447 void eea_free_tx(struct eea_net_tx *tx, struct eea_net_cfg *cfg) 448 { 449 if (!tx) 450 return; 451 452 if (tx->ering) { 453 eea_ering_free(tx->ering); 454 tx->ering = NULL; 455 } 456 457 if (tx->meta) 458 eea_free_meta(tx, cfg); 459 } 460 461 int eea_alloc_tx(struct eea_net_init_ctx *ctx, struct eea_net_tx *tx, u32 idx) 462 { 463 struct eea_tx_meta *meta; 464 struct eea_ring *ering; 465 u32 i; 466 467 u64_stats_init(&tx->stats.syncp); 468 469 snprintf(tx->name, sizeof(tx->name), "tx.%u", idx); 470 471 ering = eea_ering_alloc(idx * 2 + 1, ctx->cfg.tx_ring_depth, ctx->edev, 472 ctx->cfg.tx_sq_desc_size, 473 ctx->cfg.tx_cq_desc_size, 474 tx->name); 475 if (!ering) 476 goto err_free_tx; 477 478 tx->ering = ering; 479 tx->index = idx; 480 tx->dma_dev = ctx->edev->dma_dev; 481 482 /* meta */ 483 tx->meta = kvcalloc(ctx->cfg.tx_ring_depth, 484 sizeof(*tx->meta), GFP_KERNEL); 485 if (!tx->meta) 486 goto err_free_tx; 487 488 for (i = 0; i < ctx->cfg.tx_ring_depth; ++i) { 489 meta = &tx->meta[i]; 490 meta->id = i; 491 meta->next = tx->free; 492 tx->free = meta; 493 } 494 495 return 0; 496 497 err_free_tx: 498 eea_free_tx(tx, &ctx->cfg); 499 return -ENOMEM; 500 } 501