1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. 3 4 #include <linux/etherdevice.h> 5 #include <linux/if_vlan.h> 6 #include <linux/netdevice.h> 7 #include <net/gro.h> 8 #include <net/page_pool/helpers.h> 9 10 #include "hinic3_hwdev.h" 11 #include "hinic3_nic_dev.h" 12 #include "hinic3_nic_io.h" 13 #include "hinic3_rx.h" 14 15 #define HINIC3_RX_HDR_SIZE 256 16 #define HINIC3_RX_BUFFER_WRITE 16 17 18 #define HINIC3_RX_TCP_PKT 0x3 19 #define HINIC3_RX_UDP_PKT 0x4 20 #define HINIC3_RX_SCTP_PKT 0x7 21 22 #define HINIC3_RX_IPV4_PKT 0 23 #define HINIC3_RX_IPV6_PKT 1 24 #define HINIC3_RX_INVALID_IP_TYPE 2 25 26 #define HINIC3_RX_PKT_FORMAT_NON_TUNNEL 0 27 #define HINIC3_RX_PKT_FORMAT_VXLAN 1 28 29 #define HINIC3_LRO_PKT_HDR_LEN_IPV4 66 30 #define HINIC3_LRO_PKT_HDR_LEN_IPV6 86 31 #define HINIC3_LRO_PKT_HDR_LEN(cqe) \ 32 (RQ_CQE_OFFOLAD_TYPE_GET((cqe)->offload_type, IP_TYPE) == \ 33 HINIC3_RX_IPV6_PKT ? HINIC3_LRO_PKT_HDR_LEN_IPV6 : \ 34 HINIC3_LRO_PKT_HDR_LEN_IPV4) 35 36 int hinic3_alloc_rxqs(struct net_device *netdev) 37 { 38 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 39 struct pci_dev *pdev = nic_dev->pdev; 40 u16 num_rxqs = nic_dev->max_qps; 41 struct hinic3_rxq *rxq; 42 u16 q_id; 43 44 nic_dev->rxqs = kcalloc(num_rxqs, sizeof(*nic_dev->rxqs), GFP_KERNEL); 45 if (!nic_dev->rxqs) 46 return -ENOMEM; 47 48 for (q_id = 0; q_id < num_rxqs; q_id++) { 49 rxq = &nic_dev->rxqs[q_id]; 50 rxq->netdev = netdev; 51 rxq->dev = &pdev->dev; 52 rxq->q_id = q_id; 53 rxq->buf_len = nic_dev->rx_buf_len; 54 rxq->buf_len_shift = ilog2(nic_dev->rx_buf_len); 55 rxq->q_depth = nic_dev->q_params.rq_depth; 56 rxq->q_mask = nic_dev->q_params.rq_depth - 1; 57 } 58 59 return 0; 60 } 61 62 void hinic3_free_rxqs(struct net_device *netdev) 63 { 64 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 65 66 kfree(nic_dev->rxqs); 67 } 68 69 static int rx_alloc_mapped_page(struct page_pool *page_pool, 70 struct hinic3_rx_info *rx_info, u16 buf_len) 71 { 72 struct page *page; 73 u32 page_offset; 74 75 if (likely(rx_info->page)) 76 return 0; 77 78 page = page_pool_dev_alloc_frag(page_pool, &page_offset, buf_len); 79 if (unlikely(!page)) 80 return -ENOMEM; 81 82 rx_info->page = page; 83 rx_info->page_offset = page_offset; 84 85 return 0; 86 } 87 88 /* Associate fixed completion element to every wqe in the rq. Every rq wqe will 89 * always post completion to the same place. 90 */ 91 static void rq_associate_cqes(struct hinic3_rxq *rxq) 92 { 93 struct hinic3_queue_pages *qpages; 94 struct hinic3_rq_wqe *rq_wqe; 95 dma_addr_t cqe_dma; 96 u32 i; 97 98 qpages = &rxq->rq->wq.qpages; 99 100 for (i = 0; i < rxq->q_depth; i++) { 101 rq_wqe = get_q_element(qpages, i, NULL); 102 cqe_dma = rxq->cqe_start_paddr + 103 i * sizeof(struct hinic3_rq_cqe); 104 rq_wqe->cqe_hi_addr = cpu_to_le32(upper_32_bits(cqe_dma)); 105 rq_wqe->cqe_lo_addr = cpu_to_le32(lower_32_bits(cqe_dma)); 106 } 107 } 108 109 static void rq_wqe_buf_set(struct hinic3_io_queue *rq, uint32_t wqe_idx, 110 dma_addr_t dma_addr, u16 len) 111 { 112 struct hinic3_rq_wqe *rq_wqe; 113 114 rq_wqe = get_q_element(&rq->wq.qpages, wqe_idx, NULL); 115 rq_wqe->buf_hi_addr = cpu_to_le32(upper_32_bits(dma_addr)); 116 rq_wqe->buf_lo_addr = cpu_to_le32(lower_32_bits(dma_addr)); 117 } 118 119 static u32 hinic3_rx_fill_buffers(struct hinic3_rxq *rxq) 120 { 121 u32 i, free_wqebbs = rxq->delta - 1; 122 struct hinic3_rx_info *rx_info; 123 dma_addr_t dma_addr; 124 int err; 125 126 for (i = 0; i < free_wqebbs; i++) { 127 rx_info = &rxq->rx_info[rxq->next_to_update]; 128 129 err = rx_alloc_mapped_page(rxq->page_pool, rx_info, 130 rxq->buf_len); 131 if (unlikely(err)) 132 break; 133 134 dma_addr = page_pool_get_dma_addr(rx_info->page) + 135 rx_info->page_offset; 136 rq_wqe_buf_set(rxq->rq, rxq->next_to_update, dma_addr, 137 rxq->buf_len); 138 rxq->next_to_update = (rxq->next_to_update + 1) & rxq->q_mask; 139 } 140 141 if (likely(i)) { 142 hinic3_write_db(rxq->rq, rxq->q_id & 3, DB_CFLAG_DP_RQ, 143 rxq->next_to_update << HINIC3_NORMAL_RQ_WQE); 144 rxq->delta -= i; 145 rxq->next_to_alloc = rxq->next_to_update; 146 } 147 148 return i; 149 } 150 151 static u32 hinic3_alloc_rx_buffers(struct hinic3_dyna_rxq_res *rqres, 152 u32 rq_depth, u16 buf_len) 153 { 154 u32 free_wqebbs = rq_depth - 1; 155 u32 idx; 156 int err; 157 158 for (idx = 0; idx < free_wqebbs; idx++) { 159 err = rx_alloc_mapped_page(rqres->page_pool, 160 &rqres->rx_info[idx], buf_len); 161 if (err) 162 break; 163 } 164 165 return idx; 166 } 167 168 static void hinic3_free_rx_buffers(struct hinic3_dyna_rxq_res *rqres, 169 u32 q_depth) 170 { 171 struct hinic3_rx_info *rx_info; 172 u32 i; 173 174 /* Free all the Rx ring sk_buffs */ 175 for (i = 0; i < q_depth; i++) { 176 rx_info = &rqres->rx_info[i]; 177 178 if (rx_info->page) { 179 page_pool_put_full_page(rqres->page_pool, 180 rx_info->page, false); 181 rx_info->page = NULL; 182 } 183 } 184 } 185 186 static void hinic3_add_rx_frag(struct hinic3_rxq *rxq, 187 struct hinic3_rx_info *rx_info, 188 struct sk_buff *skb, u32 size) 189 { 190 struct page *page; 191 u8 *va; 192 193 page = rx_info->page; 194 va = (u8 *)page_address(page) + rx_info->page_offset; 195 net_prefetch(va); 196 197 page_pool_dma_sync_for_cpu(rxq->page_pool, page, rx_info->page_offset, 198 rxq->buf_len); 199 200 if (size <= HINIC3_RX_HDR_SIZE && !skb_is_nonlinear(skb)) { 201 memcpy(__skb_put(skb, size), va, 202 ALIGN(size, sizeof(long))); 203 page_pool_put_full_page(rxq->page_pool, page, false); 204 205 return; 206 } 207 208 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 209 rx_info->page_offset, size, rxq->buf_len); 210 skb_mark_for_recycle(skb); 211 } 212 213 static void packaging_skb(struct hinic3_rxq *rxq, struct sk_buff *skb, 214 u32 sge_num, u32 pkt_len) 215 { 216 struct hinic3_rx_info *rx_info; 217 u32 temp_pkt_len = pkt_len; 218 u32 temp_sge_num = sge_num; 219 u32 sw_ci; 220 u32 size; 221 222 sw_ci = rxq->cons_idx & rxq->q_mask; 223 while (temp_sge_num) { 224 rx_info = &rxq->rx_info[sw_ci]; 225 sw_ci = (sw_ci + 1) & rxq->q_mask; 226 if (unlikely(temp_pkt_len > rxq->buf_len)) { 227 size = rxq->buf_len; 228 temp_pkt_len -= rxq->buf_len; 229 } else { 230 size = temp_pkt_len; 231 } 232 233 hinic3_add_rx_frag(rxq, rx_info, skb, size); 234 235 /* clear contents of buffer_info */ 236 rx_info->page = NULL; 237 temp_sge_num--; 238 } 239 } 240 241 static u32 hinic3_get_sge_num(struct hinic3_rxq *rxq, u32 pkt_len) 242 { 243 u32 sge_num; 244 245 sge_num = pkt_len >> rxq->buf_len_shift; 246 sge_num += (pkt_len & (rxq->buf_len - 1)) ? 1 : 0; 247 248 return sge_num; 249 } 250 251 static struct sk_buff *hinic3_fetch_rx_buffer(struct hinic3_rxq *rxq, 252 u32 pkt_len) 253 { 254 struct sk_buff *skb; 255 u32 sge_num; 256 257 skb = napi_alloc_skb(&rxq->irq_cfg->napi, HINIC3_RX_HDR_SIZE); 258 if (unlikely(!skb)) 259 return NULL; 260 261 sge_num = hinic3_get_sge_num(rxq, pkt_len); 262 263 net_prefetchw(skb->data); 264 packaging_skb(rxq, skb, sge_num, pkt_len); 265 266 rxq->cons_idx += sge_num; 267 rxq->delta += sge_num; 268 269 return skb; 270 } 271 272 static void hinic3_pull_tail(struct sk_buff *skb) 273 { 274 skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; 275 unsigned int pull_len; 276 unsigned char *va; 277 278 va = skb_frag_address(frag); 279 280 /* we need the header to contain the greater of either ETH_HLEN or 281 * 60 bytes if the skb->len is less than 60 for skb_pad. 282 */ 283 pull_len = eth_get_headlen(skb->dev, va, HINIC3_RX_HDR_SIZE); 284 285 /* align pull length to size of long to optimize memcpy performance */ 286 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 287 288 /* update all of the pointers */ 289 skb_frag_size_sub(frag, pull_len); 290 skb_frag_off_add(frag, pull_len); 291 292 skb->data_len -= pull_len; 293 skb->tail += pull_len; 294 } 295 296 static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type, 297 u32 status, struct sk_buff *skb) 298 { 299 u32 pkt_fmt = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, TUNNEL_PKT_FORMAT); 300 u32 pkt_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE); 301 u32 ip_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, IP_TYPE); 302 u32 csum_err = RQ_CQE_STATUS_GET(status, CSUM_ERR); 303 struct net_device *netdev = rxq->netdev; 304 305 if (!(netdev->features & NETIF_F_RXCSUM)) 306 return; 307 308 if (unlikely(csum_err)) { 309 /* pkt type is recognized by HW, and csum is wrong */ 310 skb->ip_summed = CHECKSUM_NONE; 311 return; 312 } 313 314 if (ip_type == HINIC3_RX_INVALID_IP_TYPE || 315 !(pkt_fmt == HINIC3_RX_PKT_FORMAT_NON_TUNNEL || 316 pkt_fmt == HINIC3_RX_PKT_FORMAT_VXLAN)) { 317 skb->ip_summed = CHECKSUM_NONE; 318 return; 319 } 320 321 switch (pkt_type) { 322 case HINIC3_RX_TCP_PKT: 323 case HINIC3_RX_UDP_PKT: 324 case HINIC3_RX_SCTP_PKT: 325 skb->ip_summed = CHECKSUM_UNNECESSARY; 326 break; 327 default: 328 skb->ip_summed = CHECKSUM_NONE; 329 break; 330 } 331 } 332 333 static void hinic3_lro_set_gso_params(struct sk_buff *skb, u16 num_lro) 334 { 335 struct ethhdr *eth = (struct ethhdr *)(skb->data); 336 __be16 proto; 337 338 proto = __vlan_get_protocol(skb, eth->h_proto, NULL); 339 340 skb_shinfo(skb)->gso_size = DIV_ROUND_UP(skb->len - skb_headlen(skb), 341 num_lro); 342 skb_shinfo(skb)->gso_type = proto == htons(ETH_P_IP) ? 343 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 344 skb_shinfo(skb)->gso_segs = num_lro; 345 } 346 347 static int recv_one_pkt(struct hinic3_rxq *rxq, struct hinic3_rq_cqe *rx_cqe, 348 u32 pkt_len, u32 vlan_len, u32 status) 349 { 350 struct net_device *netdev = rxq->netdev; 351 struct sk_buff *skb; 352 u32 offload_type; 353 u16 num_lro; 354 355 skb = hinic3_fetch_rx_buffer(rxq, pkt_len); 356 if (unlikely(!skb)) 357 return -ENOMEM; 358 359 /* place header in linear portion of buffer */ 360 if (skb_is_nonlinear(skb)) 361 hinic3_pull_tail(skb); 362 363 offload_type = le32_to_cpu(rx_cqe->offload_type); 364 hinic3_rx_csum(rxq, offload_type, status, skb); 365 366 num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO); 367 if (num_lro) 368 hinic3_lro_set_gso_params(skb, num_lro); 369 370 skb_record_rx_queue(skb, rxq->q_id); 371 skb->protocol = eth_type_trans(skb, netdev); 372 373 if (skb_has_frag_list(skb)) { 374 napi_gro_flush(&rxq->irq_cfg->napi, false); 375 netif_receive_skb(skb); 376 } else { 377 napi_gro_receive(&rxq->irq_cfg->napi, skb); 378 } 379 380 return 0; 381 } 382 383 int hinic3_alloc_rxqs_res(struct net_device *netdev, u16 num_rq, 384 u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res) 385 { 386 u64 cqe_mem_size = sizeof(struct hinic3_rq_cqe) * rq_depth; 387 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 388 struct page_pool_params pp_params = {}; 389 struct hinic3_dyna_rxq_res *rqres; 390 u32 pkt_idx; 391 int idx; 392 393 for (idx = 0; idx < num_rq; idx++) { 394 rqres = &rxqs_res[idx]; 395 rqres->rx_info = kcalloc(rq_depth, sizeof(*rqres->rx_info), 396 GFP_KERNEL); 397 if (!rqres->rx_info) 398 goto err_free_rqres; 399 400 rqres->cqe_start_vaddr = 401 dma_alloc_coherent(&nic_dev->pdev->dev, cqe_mem_size, 402 &rqres->cqe_start_paddr, GFP_KERNEL); 403 if (!rqres->cqe_start_vaddr) { 404 netdev_err(netdev, "Failed to alloc rxq%d rx cqe\n", 405 idx); 406 goto err_free_rx_info; 407 } 408 409 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 410 pp_params.pool_size = rq_depth * nic_dev->rx_buf_len / 411 PAGE_SIZE; 412 pp_params.nid = dev_to_node(&nic_dev->pdev->dev); 413 pp_params.dev = &nic_dev->pdev->dev; 414 pp_params.dma_dir = DMA_FROM_DEVICE; 415 pp_params.max_len = PAGE_SIZE; 416 rqres->page_pool = page_pool_create(&pp_params); 417 if (IS_ERR(rqres->page_pool)) { 418 netdev_err(netdev, "Failed to create rxq%d page pool\n", 419 idx); 420 goto err_free_cqe; 421 } 422 423 pkt_idx = hinic3_alloc_rx_buffers(rqres, rq_depth, 424 nic_dev->rx_buf_len); 425 if (!pkt_idx) { 426 netdev_err(netdev, "Failed to alloc rxq%d rx buffers\n", 427 idx); 428 goto err_destroy_page_pool; 429 } 430 rqres->next_to_alloc = pkt_idx; 431 } 432 433 return 0; 434 435 err_destroy_page_pool: 436 page_pool_destroy(rqres->page_pool); 437 err_free_cqe: 438 dma_free_coherent(&nic_dev->pdev->dev, cqe_mem_size, 439 rqres->cqe_start_vaddr, 440 rqres->cqe_start_paddr); 441 err_free_rx_info: 442 kfree(rqres->rx_info); 443 err_free_rqres: 444 hinic3_free_rxqs_res(netdev, idx, rq_depth, rxqs_res); 445 446 return -ENOMEM; 447 } 448 449 void hinic3_free_rxqs_res(struct net_device *netdev, u16 num_rq, 450 u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res) 451 { 452 u64 cqe_mem_size = sizeof(struct hinic3_rq_cqe) * rq_depth; 453 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 454 struct hinic3_dyna_rxq_res *rqres; 455 int idx; 456 457 for (idx = 0; idx < num_rq; idx++) { 458 rqres = &rxqs_res[idx]; 459 460 hinic3_free_rx_buffers(rqres, rq_depth); 461 page_pool_destroy(rqres->page_pool); 462 dma_free_coherent(&nic_dev->pdev->dev, cqe_mem_size, 463 rqres->cqe_start_vaddr, 464 rqres->cqe_start_paddr); 465 kfree(rqres->rx_info); 466 } 467 } 468 469 int hinic3_configure_rxqs(struct net_device *netdev, u16 num_rq, 470 u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res) 471 { 472 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 473 struct hinic3_dyna_rxq_res *rqres; 474 struct msix_entry *msix_entry; 475 struct hinic3_rxq *rxq; 476 u16 q_id; 477 u32 pkts; 478 479 for (q_id = 0; q_id < num_rq; q_id++) { 480 rxq = &nic_dev->rxqs[q_id]; 481 rqres = &rxqs_res[q_id]; 482 msix_entry = &nic_dev->qps_msix_entries[q_id]; 483 484 rxq->irq_id = msix_entry->vector; 485 rxq->msix_entry_idx = msix_entry->entry; 486 rxq->next_to_update = 0; 487 rxq->next_to_alloc = rqres->next_to_alloc; 488 rxq->q_depth = rq_depth; 489 rxq->delta = rxq->q_depth; 490 rxq->q_mask = rxq->q_depth - 1; 491 rxq->cons_idx = 0; 492 493 rxq->cqe_arr = rqres->cqe_start_vaddr; 494 rxq->cqe_start_paddr = rqres->cqe_start_paddr; 495 rxq->rx_info = rqres->rx_info; 496 rxq->page_pool = rqres->page_pool; 497 498 rxq->rq = &nic_dev->nic_io->rq[rxq->q_id]; 499 500 rq_associate_cqes(rxq); 501 502 pkts = hinic3_rx_fill_buffers(rxq); 503 if (!pkts) { 504 netdev_err(netdev, "Failed to fill Rx buffer\n"); 505 return -ENOMEM; 506 } 507 } 508 509 return 0; 510 } 511 512 int hinic3_rx_poll(struct hinic3_rxq *rxq, int budget) 513 { 514 struct hinic3_nic_dev *nic_dev = netdev_priv(rxq->netdev); 515 u32 sw_ci, status, pkt_len, vlan_len; 516 struct hinic3_rq_cqe *rx_cqe; 517 u32 num_wqe = 0; 518 int nr_pkts = 0; 519 u16 num_lro; 520 521 while (likely(nr_pkts < budget)) { 522 sw_ci = rxq->cons_idx & rxq->q_mask; 523 rx_cqe = rxq->cqe_arr + sw_ci; 524 status = le32_to_cpu(rx_cqe->status); 525 if (!RQ_CQE_STATUS_GET(status, RXDONE)) 526 break; 527 528 /* make sure we read rx_done before packet length */ 529 rmb(); 530 531 vlan_len = le32_to_cpu(rx_cqe->vlan_len); 532 pkt_len = RQ_CQE_SGE_GET(vlan_len, LEN); 533 if (recv_one_pkt(rxq, rx_cqe, pkt_len, vlan_len, status)) 534 break; 535 536 nr_pkts++; 537 num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO); 538 if (num_lro) 539 num_wqe += hinic3_get_sge_num(rxq, pkt_len); 540 541 rx_cqe->status = 0; 542 543 if (num_wqe >= nic_dev->lro_replenish_thld) 544 break; 545 } 546 547 if (rxq->delta >= HINIC3_RX_BUFFER_WRITE) 548 hinic3_rx_fill_buffers(rxq); 549 550 return nr_pkts; 551 } 552