1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Driver for Alibaba Elastic Ethernet Adapter. 4 * 5 * Copyright (C) 2025 Alibaba Inc. 6 */ 7 8 #include <net/netdev_rx_queue.h> 9 #include <net/page_pool/helpers.h> 10 11 #include "eea_adminq.h" 12 #include "eea_net.h" 13 #include "eea_pci.h" 14 #include "eea_ring.h" 15 16 #define EEA_ENABLE_F_NAPI BIT(0) 17 18 #define EEA_PAGE_FRAGS_NUM 1024 19 20 #define EEA_RX_BUF_ALIGN 128 21 22 #define EEA_RX_BUF_MAX_LEN (10 * 1024) 23 24 struct eea_rx_ctx { 25 u32 len; 26 u32 hdr_len; 27 28 u16 flags; 29 bool more; 30 31 struct eea_rx_meta *meta; 32 33 struct eea_rx_ctx_stats stats; 34 }; 35 36 static struct eea_rx_meta *eea_rx_meta_get(struct eea_net_rx *rx) 37 { 38 struct eea_rx_meta *meta; 39 40 if (!rx->free) 41 return NULL; 42 43 meta = rx->free; 44 rx->free = meta->next; 45 46 return meta; 47 } 48 49 static void eea_rx_meta_put(struct eea_net_rx *rx, struct eea_rx_meta *meta) 50 { 51 meta->next = rx->free; 52 rx->free = meta; 53 } 54 55 static void eea_free_rx_buffer(struct eea_net_rx *rx, struct eea_rx_meta *meta, 56 bool allow_direct) 57 { 58 u32 drain_count; 59 60 drain_count = EEA_PAGE_FRAGS_NUM - meta->frags; 61 62 if (page_pool_unref_page(meta->page, drain_count) == 0) 63 page_pool_put_unrefed_page(rx->pp, meta->page, -1, 64 allow_direct); 65 66 meta->page = NULL; 67 } 68 69 static void eea_rx_meta_dma_sync_for_device(struct eea_net_rx *rx, 70 struct eea_rx_meta *meta) 71 { 72 u32 len; 73 74 if (meta->sync_for_cpu <= meta->offset + rx->headroom) 75 return; 76 77 len = meta->sync_for_cpu - meta->offset - rx->headroom; 78 79 dma_sync_single_for_device(rx->enet->edev->dma_dev, 80 meta->dma + meta->offset + rx->headroom, 81 len, DMA_FROM_DEVICE); 82 meta->sync_for_cpu = 0; 83 } 84 85 static void meta_align_offset(struct eea_net_rx *rx, struct eea_rx_meta *meta) 86 { 87 int h, b; 88 89 h = rx->headroom; 90 b = meta->offset + h; 91 92 /* For better performance, we align the buffer address to 93 * EEA_RX_BUF_ALIGN, as required by the device design. 94 */ 95 b = ALIGN(b, EEA_RX_BUF_ALIGN); 96 97 meta->offset = b - h; 98 } 99 100 static int eea_alloc_rx_buffer(struct eea_net_rx *rx, struct eea_rx_meta *meta) 101 { 102 struct page *page; 103 104 if (meta->page) { 105 eea_rx_meta_dma_sync_for_device(rx, meta); 106 return 0; 107 } 108 109 page = page_pool_dev_alloc_pages(rx->pp); 110 if (!page) 111 return -ENOMEM; 112 113 page_pool_fragment_page(page, EEA_PAGE_FRAGS_NUM); 114 115 meta->page = page; 116 meta->dma = page_pool_get_dma_addr(page); 117 meta->offset = 0; 118 meta->frags = 0; 119 meta->sync_for_cpu = 0; 120 121 meta_align_offset(rx, meta); 122 123 return 0; 124 } 125 126 static u32 eea_consume_rx_buffer(struct eea_net_rx *rx, 127 struct eea_rx_meta *meta, 128 u32 consumed) 129 { 130 u32 offset; 131 int min; 132 133 offset = meta->offset; 134 135 meta->offset += consumed; 136 ++meta->frags; 137 138 min = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 139 min += rx->headroom; 140 min += SKB_DATA_ALIGN(ETH_DATA_LEN); 141 142 meta_align_offset(rx, meta); 143 144 if (min + meta->offset > PAGE_SIZE) { 145 eea_free_rx_buffer(rx, meta, true); 146 return PAGE_SIZE - offset; 147 } 148 149 return meta->offset - offset; 150 } 151 152 static void eea_free_rx_hdr(struct eea_net_rx *rx, struct eea_net_cfg *cfg) 153 { 154 struct eea_rx_meta *meta; 155 int i; 156 157 for (i = 0; i < cfg->rx_ring_depth; ++i) { 158 meta = &rx->meta[i]; 159 meta->hdr_addr = NULL; 160 161 if (!meta->hdr_page) 162 continue; 163 164 dma_unmap_page(rx->dma_dev, meta->hdr_dma, PAGE_SIZE, 165 DMA_FROM_DEVICE); 166 put_page(meta->hdr_page); 167 168 meta->hdr_page = NULL; 169 } 170 } 171 172 static int eea_alloc_rx_hdr(struct eea_net_init_ctx *ctx, struct eea_net_rx *rx) 173 { 174 struct page *hdr_page = NULL; 175 struct eea_rx_meta *meta; 176 u32 offset = 0, hdrsize; 177 struct device *dmadev; 178 dma_addr_t dma; 179 int i; 180 181 dmadev = ctx->edev->dma_dev; 182 hdrsize = ctx->cfg.split_hdr; 183 184 for (i = 0; i < ctx->cfg.rx_ring_depth; ++i) { 185 meta = &rx->meta[i]; 186 meta->hdr_page = NULL; 187 188 if (!hdr_page || offset + hdrsize > PAGE_SIZE) { 189 hdr_page = alloc_page(GFP_KERNEL); 190 if (!hdr_page) 191 goto err; 192 193 dma = dma_map_page(dmadev, hdr_page, 0, PAGE_SIZE, 194 DMA_FROM_DEVICE); 195 196 if (unlikely(dma_mapping_error(dmadev, dma))) { 197 put_page(hdr_page); 198 goto err; 199 } 200 201 offset = 0; 202 meta->hdr_page = hdr_page; 203 } 204 205 meta->hdr_dma = dma + offset; 206 meta->hdr_addr = page_address(hdr_page) + offset; 207 offset += hdrsize; 208 } 209 210 return 0; 211 212 err: 213 eea_free_rx_hdr(rx, &ctx->cfg); 214 return -ENOMEM; 215 } 216 217 static void eea_rx_meta_dma_sync_for_cpu(struct eea_net_rx *rx, 218 struct eea_rx_meta *meta, u32 len) 219 { 220 dma_sync_single_for_cpu(rx->enet->edev->dma_dev, 221 meta->dma + meta->offset + meta->headroom, 222 len, DMA_FROM_DEVICE); 223 meta->sync_for_cpu = meta->offset + meta->headroom + len; 224 } 225 226 static int eea_harden_check_overflow(struct eea_rx_ctx *ctx, 227 struct eea_net *enet) 228 { 229 u32 max_len; 230 231 max_len = ctx->meta->truesize - ctx->meta->headroom - 232 ctx->meta->tailroom; 233 234 if (unlikely(ctx->len > max_len)) { 235 pr_debug("%s: rx error: len %u exceeds truesize %u\n", 236 enet->netdev->name, ctx->len, max_len); 237 ++ctx->stats.length_errors; 238 return -EINVAL; 239 } 240 241 return 0; 242 } 243 244 static int eea_harden_check_size(struct eea_rx_ctx *ctx, struct eea_net *enet) 245 { 246 int err; 247 248 err = eea_harden_check_overflow(ctx, enet); 249 if (err) 250 return err; 251 252 if (ctx->hdr_len) { 253 if (unlikely(ctx->hdr_len < ETH_HLEN)) { 254 pr_debug("%s: short hdr %u\n", enet->netdev->name, 255 ctx->hdr_len); 256 ++ctx->stats.length_errors; 257 return -EINVAL; 258 } 259 260 if (unlikely(ctx->hdr_len > enet->cfg.split_hdr)) { 261 pr_debug("%s: rx error: hdr len %u exceeds hdr buffer size %u\n", 262 enet->netdev->name, ctx->hdr_len, 263 enet->cfg.split_hdr); 264 ++ctx->stats.length_errors; 265 return -EINVAL; 266 } 267 268 return 0; 269 } 270 271 if (unlikely(ctx->len < ETH_HLEN)) { 272 pr_debug("%s: short packet %u\n", enet->netdev->name, ctx->len); 273 ++ctx->stats.length_errors; 274 return -EINVAL; 275 } 276 277 return 0; 278 } 279 280 static struct sk_buff *eea_build_skb(void *buf, u32 buflen, u32 headroom, 281 u32 len) 282 { 283 struct sk_buff *skb; 284 285 skb = build_skb(buf, buflen); 286 if (unlikely(!skb)) 287 return NULL; 288 289 skb_reserve(skb, headroom); 290 skb_put(skb, len); 291 292 return skb; 293 } 294 295 static struct sk_buff *eea_rx_build_split_hdr_skb(struct eea_net_rx *rx, 296 struct eea_rx_ctx *ctx) 297 { 298 struct eea_rx_meta *meta = ctx->meta; 299 u32 truesize, offset; 300 struct sk_buff *skb; 301 struct page *page; 302 303 dma_sync_single_for_cpu(rx->enet->edev->dma_dev, meta->hdr_dma, 304 ctx->hdr_len, DMA_FROM_DEVICE); 305 306 skb = napi_alloc_skb(rx->napi, ctx->hdr_len); 307 if (unlikely(!skb)) 308 return NULL; 309 310 skb_put_data(skb, ctx->meta->hdr_addr, ctx->hdr_len); 311 312 if (ctx->len) { 313 page = meta->page; 314 offset = meta->offset + meta->headroom; 315 316 truesize = eea_consume_rx_buffer(rx, meta, 317 meta->headroom + ctx->len); 318 319 skb_add_rx_frag(skb, 0, page, offset, ctx->len, truesize); 320 } 321 322 skb_mark_for_recycle(skb); 323 324 return skb; 325 } 326 327 static struct sk_buff *eea_rx_build_skb(struct eea_net_rx *rx, 328 struct eea_rx_ctx *ctx) 329 { 330 struct eea_rx_meta *meta = ctx->meta; 331 u32 shinfo_size, bufsize, truesize; 332 struct sk_buff *skb; 333 struct page *page; 334 void *buf; 335 336 page = meta->page; 337 338 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 339 340 buf = page_address(page) + meta->offset; 341 bufsize = meta->headroom + SKB_DATA_ALIGN(ctx->len) + shinfo_size; 342 343 skb = eea_build_skb(buf, bufsize, meta->headroom, ctx->len); 344 if (unlikely(!skb)) 345 return NULL; 346 347 truesize = eea_consume_rx_buffer(rx, meta, bufsize); 348 skb_mark_for_recycle(skb); 349 350 skb->truesize += truesize - bufsize; 351 352 return skb; 353 } 354 355 static void process_remain_buf(struct eea_net_rx *rx, struct eea_rx_ctx *ctx) 356 { 357 struct eea_net *enet = rx->enet; 358 struct sk_buff *head_skb; 359 u32 offset, truesize, nr_frags; 360 struct page *page; 361 362 if (eea_harden_check_overflow(ctx, enet)) 363 goto err; 364 365 head_skb = rx->pkt.head_skb; 366 367 nr_frags = skb_shinfo(head_skb)->nr_frags; 368 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) 369 goto err; 370 371 offset = ctx->meta->offset + ctx->meta->headroom; 372 page = ctx->meta->page; 373 truesize = eea_consume_rx_buffer(rx, ctx->meta, 374 ctx->meta->headroom + ctx->len); 375 376 skb_add_rx_frag(head_skb, nr_frags, page, offset, ctx->len, truesize); 377 378 return; 379 380 err: 381 dev_kfree_skb(rx->pkt.head_skb); 382 ++ctx->stats.drops; 383 rx->pkt.do_drop = true; 384 rx->pkt.head_skb = NULL; 385 } 386 387 static void process_first_buf(struct eea_net_rx *rx, struct eea_rx_ctx *ctx) 388 { 389 struct eea_net *enet = rx->enet; 390 struct sk_buff *skb = NULL; 391 392 if (eea_harden_check_size(ctx, enet)) 393 goto err; 394 395 rx->pkt.data_valid = ctx->flags & EEA_DESC_F_DATA_VALID; 396 397 if (ctx->hdr_len) 398 skb = eea_rx_build_split_hdr_skb(rx, ctx); 399 else 400 skb = eea_rx_build_skb(rx, ctx); 401 402 if (unlikely(!skb)) 403 goto err; 404 405 rx->pkt.head_skb = skb; 406 407 return; 408 409 err: 410 ++ctx->stats.drops; 411 rx->pkt.do_drop = true; 412 } 413 414 static void eea_submit_skb(struct eea_net_rx *rx, struct sk_buff *skb, 415 struct eea_rx_cdesc *desc) 416 { 417 struct eea_net *enet = rx->enet; 418 419 if (rx->pkt.data_valid) 420 skb->ip_summed = CHECKSUM_UNNECESSARY; 421 422 if (enet->cfg.ts_cfg.rx_filter == HWTSTAMP_FILTER_ALL) 423 skb_hwtstamps(skb)->hwtstamp = EEA_DESC_TS(desc) + 424 enet->hw_ts_offset; 425 426 skb_record_rx_queue(skb, rx->index); 427 skb->protocol = eth_type_trans(skb, enet->netdev); 428 429 napi_gro_receive(rx->napi, skb); 430 } 431 432 static int eea_rx_desc_to_ctx(struct eea_net_rx *rx, 433 struct eea_rx_ctx *ctx, 434 struct eea_rx_cdesc *desc) 435 { 436 u16 id; 437 438 ctx->meta = NULL; 439 440 id = le16_to_cpu(desc->id); 441 if (unlikely(id >= rx->ering->num)) { 442 if (net_ratelimit()) 443 netdev_err(rx->enet->netdev, "rx invalid id %d\n", id); 444 return -EINVAL; 445 } 446 447 ctx->meta = &rx->meta[id]; 448 if (!ctx->meta->in_use) { 449 if (net_ratelimit()) 450 netdev_err(rx->enet->netdev, "rx invalid id %d\n", id); 451 ctx->meta = NULL; 452 return -EINVAL; 453 } 454 455 ctx->meta->in_use = false; 456 457 ctx->len = le16_to_cpu(desc->len); 458 if (unlikely(ctx->len > ctx->meta->len)) { 459 if (net_ratelimit()) 460 netdev_err(rx->enet->netdev, "rx invalid len(%d) id:%d\n", 461 ctx->len, id); 462 return -EINVAL; 463 } 464 465 ctx->flags = le16_to_cpu(desc->flags); 466 467 ctx->hdr_len = 0; 468 if (ctx->flags & EEA_DESC_F_SPLIT_HDR) { 469 ctx->hdr_len = le16_to_cpu(desc->len_ex) & 470 EEA_RX_CDESC_HDR_LEN_MASK; 471 ctx->stats.split_hdr_bytes += ctx->hdr_len; 472 ++ctx->stats.split_hdr_packets; 473 } 474 475 ctx->more = ctx->flags & EEA_RING_DESC_F_MORE; 476 477 return 0; 478 } 479 480 static int eea_cleanrx(struct eea_net_rx *rx, int budget, 481 struct eea_rx_ctx *ctx) 482 { 483 struct eea_rx_cdesc *desc; 484 struct eea_rx_meta *meta; 485 int recv, err; 486 487 for (recv = 0; recv < budget; ) { 488 desc = eea_ering_cq_get_desc(rx->ering); 489 if (!desc) 490 break; 491 492 err = eea_rx_desc_to_ctx(rx, ctx, desc); 493 if (unlikely(err)) { 494 if (ctx->meta) 495 eea_rx_meta_put(rx, ctx->meta); 496 497 if (rx->pkt.head_skb) { 498 dev_kfree_skb(rx->pkt.head_skb); 499 ++ctx->stats.drops; 500 } 501 502 /* A hardware error occurred; we are attempting to 503 * mitigate the impact. Subsequent packets may be 504 * corrupted. 505 */ 506 ctx->more = false; 507 goto ack; 508 } 509 510 meta = ctx->meta; 511 512 if (unlikely(rx->pkt.do_drop)) 513 goto skip; 514 515 eea_rx_meta_dma_sync_for_cpu(rx, meta, ctx->len); 516 517 rx->pkt.recv_len += ctx->len; 518 rx->pkt.recv_len += ctx->hdr_len; 519 520 if (!rx->pkt.idx) 521 process_first_buf(rx, ctx); 522 else 523 process_remain_buf(rx, ctx); 524 525 ++rx->pkt.idx; 526 527 if (!ctx->more && rx->pkt.head_skb) { 528 eea_submit_skb(rx, rx->pkt.head_skb, desc); 529 ctx->stats.bytes += rx->pkt.recv_len; 530 ++ctx->stats.packets; 531 } 532 533 skip: 534 eea_rx_meta_put(rx, meta); 535 ack: 536 eea_ering_cq_ack_desc(rx->ering, 1); 537 ++ctx->stats.descs; 538 539 if (!ctx->more) { 540 memset(&rx->pkt, 0, sizeof(rx->pkt)); 541 ++recv; 542 } 543 } 544 545 return recv; 546 } 547 548 static void eea_rx_dma_sync_hdr(struct eea_net_rx *rx, dma_addr_t addr) 549 { 550 dma_sync_single_for_device(rx->dma_dev, addr, 551 rx->enet->cfg.split_hdr, 552 DMA_FROM_DEVICE); 553 } 554 555 /* Only be called from napi. */ 556 static void eea_rx_post(struct eea_net_rx *rx, struct eea_rx_ctx *ctx) 557 { 558 u32 tailroom, headroom, room, len; 559 struct eea_rx_meta *meta; 560 struct eea_rx_desc *desc; 561 int err = 0, num = 0; 562 dma_addr_t addr; 563 564 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 565 headroom = rx->headroom; 566 room = headroom + tailroom; 567 568 while (true) { 569 meta = eea_rx_meta_get(rx); 570 if (!meta) 571 break; 572 573 err = eea_alloc_rx_buffer(rx, meta); 574 if (err) { 575 eea_rx_meta_put(rx, meta); 576 break; 577 } 578 579 len = min_t(u32, PAGE_SIZE - meta->offset - room, 580 EEA_RX_BUF_MAX_LEN); 581 582 len = ALIGN_DOWN(len, SMP_CACHE_BYTES); 583 584 addr = meta->dma + meta->offset + headroom; 585 586 desc = eea_ering_sq_alloc_desc(rx->ering, meta->id, true, 0); 587 desc->addr = cpu_to_le64(addr); 588 desc->len = cpu_to_le16(len); 589 590 if (meta->hdr_addr) { 591 eea_rx_dma_sync_hdr(rx, meta->hdr_dma); 592 desc->hdr_addr = cpu_to_le64(meta->hdr_dma); 593 } 594 595 eea_ering_sq_commit_desc(rx->ering); 596 597 meta->truesize = len + room; 598 meta->headroom = headroom; 599 meta->tailroom = tailroom; 600 meta->len = len; 601 meta->in_use = true; 602 ++num; 603 } 604 605 if (num) { 606 eea_ering_kick(rx->ering); 607 ++ctx->stats.kicks; 608 } 609 } 610 611 static int eea_poll(struct napi_struct *napi, int budget) 612 { 613 struct eea_irq_blk *blk = container_of(napi, struct eea_irq_blk, napi); 614 struct eea_net_rx *rx = blk->rx; 615 struct eea_net_tx *tx = &rx->enet->tx[rx->index]; 616 struct eea_rx_ctx ctx = {}; 617 bool busy = false; 618 u32 received; 619 620 busy |= eea_poll_tx(tx, budget) >= budget; 621 622 received = eea_cleanrx(rx, budget, &ctx); 623 624 if (rx->ering->num_free > budget) { 625 /* Due to the hardware design, there is no notification when 626 * buffers are exhausted. Therefore, we should proactively 627 * pre-fill the buffers to avoid starvation. 628 */ 629 eea_rx_post(rx, &ctx); 630 631 if (rx->ering->num - rx->ering->num_free < budget) 632 busy = true; 633 } 634 635 eea_update_rx_stats(&rx->stats, &ctx.stats); 636 637 busy |= received >= budget; 638 639 if (busy) 640 return budget; 641 642 if (napi_complete_done(napi, received)) 643 eea_ering_irq_active(rx->ering, tx->ering); 644 645 return received; 646 } 647 648 static void eea_free_rx_buffers(struct eea_net_rx *rx, struct eea_net_cfg *cfg) 649 { 650 struct eea_rx_meta *meta; 651 u32 i; 652 653 if (rx->pkt.head_skb) { 654 dev_kfree_skb(rx->pkt.head_skb); 655 rx->pkt.head_skb = NULL; 656 } 657 658 for (i = 0; i < cfg->rx_ring_depth; ++i) { 659 meta = &rx->meta[i]; 660 if (!meta->page) 661 continue; 662 663 eea_free_rx_buffer(rx, meta, false); 664 } 665 } 666 667 static struct page_pool *eea_create_pp(struct eea_net_init_ctx *ctx, u32 idx) 668 { 669 struct page_pool_params pp_params = {0}; 670 671 pp_params.order = 0; 672 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 673 pp_params.pool_size = ctx->cfg.rx_ring_depth; 674 pp_params.nid = dev_to_node(ctx->edev->dma_dev); 675 pp_params.dev = ctx->edev->dma_dev; 676 pp_params.netdev = ctx->netdev; 677 pp_params.dma_dir = DMA_FROM_DEVICE; 678 pp_params.max_len = PAGE_SIZE; 679 pp_params.queue_idx = idx; 680 681 return page_pool_create(&pp_params); 682 } 683 684 static void eea_destroy_page_pool(struct eea_net_rx *rx) 685 { 686 if (rx->pp) 687 page_pool_destroy(rx->pp); 688 } 689 690 void enet_rx_stop(struct eea_net_rx *rx) 691 { 692 if (rx->flags & EEA_ENABLE_F_NAPI) { 693 rx->flags &= ~EEA_ENABLE_F_NAPI; 694 695 disable_irq(rx->enet->irq_blks[rx->index].irq); 696 napi_disable(rx->napi); 697 698 page_pool_disable_direct_recycling(rx->pp); 699 netif_napi_del(rx->napi); 700 } 701 } 702 703 void enet_rx_start(struct eea_net_rx *rx) 704 { 705 netif_napi_add(rx->enet->netdev, rx->napi, eea_poll); 706 707 page_pool_enable_direct_recycling(rx->pp, rx->napi); 708 709 napi_enable(rx->napi); 710 711 rx->flags |= EEA_ENABLE_F_NAPI; 712 713 local_bh_disable(); 714 napi_schedule(rx->napi); 715 local_bh_enable(); 716 717 enable_irq(rx->enet->irq_blks[rx->index].irq); 718 } 719 720 /* Maybe called before eea_bind_q_and_cfg. So the cfg must be passed. */ 721 void eea_free_rx(struct eea_net_rx *rx, struct eea_net_cfg *cfg) 722 { 723 if (!rx) 724 return; 725 726 if (rx->ering) { 727 eea_ering_free(rx->ering); 728 rx->ering = NULL; 729 } 730 731 if (rx->meta) { 732 eea_free_rx_buffers(rx, cfg); 733 eea_free_rx_hdr(rx, cfg); 734 kvfree(rx->meta); 735 rx->meta = NULL; 736 } 737 738 if (rx->pp) { 739 eea_destroy_page_pool(rx); 740 rx->pp = NULL; 741 } 742 743 kfree(rx); 744 } 745 746 static void eea_rx_meta_init(struct eea_net_rx *rx, u32 num) 747 { 748 struct eea_rx_meta *meta; 749 int i; 750 751 rx->free = NULL; 752 753 for (i = 0; i < num; ++i) { 754 meta = &rx->meta[i]; 755 meta->id = i; 756 meta->next = rx->free; 757 rx->free = meta; 758 } 759 } 760 761 struct eea_net_rx *eea_alloc_rx(struct eea_net_init_ctx *ctx, u32 idx) 762 { 763 struct eea_ring *ering; 764 struct eea_net_rx *rx; 765 int err; 766 767 rx = kzalloc(sizeof(*rx), GFP_KERNEL); 768 if (!rx) 769 return rx; 770 771 rx->index = idx; 772 snprintf(rx->name, sizeof(rx->name), "rx.%u", idx); 773 774 u64_stats_init(&rx->stats.syncp); 775 776 /* ering */ 777 ering = eea_ering_alloc(idx * 2, ctx->cfg.rx_ring_depth, ctx->edev, 778 ctx->cfg.rx_sq_desc_size, 779 ctx->cfg.rx_cq_desc_size, 780 rx->name); 781 if (!ering) 782 goto err_free_rx; 783 784 rx->ering = ering; 785 786 rx->dma_dev = ctx->edev->dma_dev; 787 788 /* meta */ 789 rx->meta = kvcalloc(ctx->cfg.rx_ring_depth, 790 sizeof(*rx->meta), GFP_KERNEL); 791 if (!rx->meta) 792 goto err_free_rx; 793 794 eea_rx_meta_init(rx, ctx->cfg.rx_ring_depth); 795 796 if (ctx->cfg.split_hdr) { 797 err = eea_alloc_rx_hdr(ctx, rx); 798 if (err) 799 goto err_free_rx; 800 } 801 802 rx->pp = eea_create_pp(ctx, idx); 803 if (IS_ERR(rx->pp)) { 804 err = PTR_ERR(rx->pp); 805 rx->pp = NULL; 806 goto err_free_rx; 807 } 808 809 return rx; 810 811 err_free_rx: 812 eea_free_rx(rx, &ctx->cfg); 813 return NULL; 814 } 815