1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. 3 4 #include <linux/if_vlan.h> 5 #include <linux/iopoll.h> 6 #include <net/ip6_checksum.h> 7 #include <net/ipv6.h> 8 #include <net/netdev_queues.h> 9 10 #include "hinic3_hwdev.h" 11 #include "hinic3_nic_cfg.h" 12 #include "hinic3_nic_dev.h" 13 #include "hinic3_nic_io.h" 14 #include "hinic3_tx.h" 15 #include "hinic3_wq.h" 16 17 #define MIN_SKB_LEN 32 18 19 int hinic3_alloc_txqs(struct net_device *netdev) 20 { 21 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 22 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 23 u16 q_id, num_txqs = nic_dev->max_qps; 24 struct pci_dev *pdev = nic_dev->pdev; 25 struct hinic3_txq *txq; 26 27 if (!num_txqs) { 28 dev_err(hwdev->dev, "Cannot allocate zero size txqs\n"); 29 return -EINVAL; 30 } 31 32 nic_dev->txqs = kcalloc(num_txqs, sizeof(*nic_dev->txqs), GFP_KERNEL); 33 if (!nic_dev->txqs) 34 return -ENOMEM; 35 36 for (q_id = 0; q_id < num_txqs; q_id++) { 37 txq = &nic_dev->txqs[q_id]; 38 txq->netdev = netdev; 39 txq->q_id = q_id; 40 txq->q_depth = nic_dev->q_params.sq_depth; 41 txq->q_mask = nic_dev->q_params.sq_depth - 1; 42 txq->dev = &pdev->dev; 43 } 44 45 return 0; 46 } 47 48 void hinic3_free_txqs(struct net_device *netdev) 49 { 50 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 51 52 kfree(nic_dev->txqs); 53 } 54 55 static void hinic3_set_buf_desc(struct hinic3_sq_bufdesc *buf_descs, 56 dma_addr_t addr, u32 len) 57 { 58 buf_descs->hi_addr = cpu_to_le32(upper_32_bits(addr)); 59 buf_descs->lo_addr = cpu_to_le32(lower_32_bits(addr)); 60 buf_descs->len = cpu_to_le32(len); 61 } 62 63 static int hinic3_tx_map_skb(struct net_device *netdev, struct sk_buff *skb, 64 struct hinic3_txq *txq, 65 struct hinic3_tx_info *tx_info, 66 struct hinic3_sq_wqe_combo *wqe_combo) 67 { 68 struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0; 69 struct hinic3_sq_bufdesc *buf_desc = wqe_combo->bds_head; 70 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 71 struct hinic3_dma_info *dma_info = tx_info->dma_info; 72 struct pci_dev *pdev = nic_dev->pdev; 73 skb_frag_t *frag; 74 u32 i, idx; 75 int err; 76 77 dma_info[0].dma = dma_map_single(&pdev->dev, skb->data, 78 skb_headlen(skb), DMA_TO_DEVICE); 79 if (dma_mapping_error(&pdev->dev, dma_info[0].dma)) 80 return -EFAULT; 81 82 dma_info[0].len = skb_headlen(skb); 83 84 wqe_desc->hi_addr = cpu_to_le32(upper_32_bits(dma_info[0].dma)); 85 wqe_desc->lo_addr = cpu_to_le32(lower_32_bits(dma_info[0].dma)); 86 87 wqe_desc->ctrl_len = cpu_to_le32(dma_info[0].len); 88 89 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 90 frag = &(skb_shinfo(skb)->frags[i]); 91 if (unlikely(i == wqe_combo->first_bds_num)) 92 buf_desc = wqe_combo->bds_sec2; 93 94 idx = i + 1; 95 dma_info[idx].dma = skb_frag_dma_map(&pdev->dev, frag, 0, 96 skb_frag_size(frag), 97 DMA_TO_DEVICE); 98 if (dma_mapping_error(&pdev->dev, dma_info[idx].dma)) { 99 err = -EFAULT; 100 goto err_unmap_page; 101 } 102 dma_info[idx].len = skb_frag_size(frag); 103 104 hinic3_set_buf_desc(buf_desc, dma_info[idx].dma, 105 dma_info[idx].len); 106 buf_desc++; 107 } 108 109 return 0; 110 111 err_unmap_page: 112 while (idx > 1) { 113 idx--; 114 dma_unmap_page(&pdev->dev, dma_info[idx].dma, 115 dma_info[idx].len, DMA_TO_DEVICE); 116 } 117 dma_unmap_single(&pdev->dev, dma_info[0].dma, dma_info[0].len, 118 DMA_TO_DEVICE); 119 120 return err; 121 } 122 123 static void hinic3_tx_unmap_skb(struct net_device *netdev, 124 struct sk_buff *skb, 125 struct hinic3_dma_info *dma_info) 126 { 127 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 128 struct pci_dev *pdev = nic_dev->pdev; 129 int i; 130 131 for (i = 0; i < skb_shinfo(skb)->nr_frags;) { 132 i++; 133 dma_unmap_page(&pdev->dev, 134 dma_info[i].dma, 135 dma_info[i].len, DMA_TO_DEVICE); 136 } 137 138 dma_unmap_single(&pdev->dev, dma_info[0].dma, 139 dma_info[0].len, DMA_TO_DEVICE); 140 } 141 142 static void free_all_tx_skbs(struct net_device *netdev, u32 sq_depth, 143 struct hinic3_tx_info *tx_info_arr) 144 { 145 struct hinic3_tx_info *tx_info; 146 u32 idx; 147 148 for (idx = 0; idx < sq_depth; idx++) { 149 tx_info = &tx_info_arr[idx]; 150 if (tx_info->skb) { 151 hinic3_tx_unmap_skb(netdev, tx_info->skb, 152 tx_info->dma_info); 153 dev_kfree_skb_any(tx_info->skb); 154 tx_info->skb = NULL; 155 } 156 } 157 } 158 159 union hinic3_ip { 160 struct iphdr *v4; 161 struct ipv6hdr *v6; 162 unsigned char *hdr; 163 }; 164 165 union hinic3_l4 { 166 struct tcphdr *tcp; 167 struct udphdr *udp; 168 unsigned char *hdr; 169 }; 170 171 enum hinic3_l3_type { 172 HINIC3_L3_UNKNOWN = 0, 173 HINIC3_L3_IP6_PKT = 1, 174 HINIC3_L3_IP4_PKT_NO_CSUM = 2, 175 HINIC3_L3_IP4_PKT_CSUM = 3, 176 }; 177 178 enum hinic3_l4_offload_type { 179 HINIC3_L4_OFFLOAD_DISABLE = 0, 180 HINIC3_L4_OFFLOAD_TCP = 1, 181 HINIC3_L4_OFFLOAD_STCP = 2, 182 HINIC3_L4_OFFLOAD_UDP = 3, 183 }; 184 185 /* initialize l4 offset and offload */ 186 static void get_inner_l4_info(struct sk_buff *skb, union hinic3_l4 *l4, 187 u8 l4_proto, u32 *offset, 188 enum hinic3_l4_offload_type *l4_offload) 189 { 190 switch (l4_proto) { 191 case IPPROTO_TCP: 192 *l4_offload = HINIC3_L4_OFFLOAD_TCP; 193 /* To be same with TSO, payload offset begins from payload */ 194 *offset = (l4->tcp->doff << TCP_HDR_DATA_OFF_UNIT_SHIFT) + 195 TRANSPORT_OFFSET(l4->hdr, skb); 196 break; 197 198 case IPPROTO_UDP: 199 *l4_offload = HINIC3_L4_OFFLOAD_UDP; 200 *offset = TRANSPORT_OFFSET(l4->hdr, skb); 201 break; 202 default: 203 *l4_offload = HINIC3_L4_OFFLOAD_DISABLE; 204 *offset = 0; 205 } 206 } 207 208 static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task, 209 struct sk_buff *skb) 210 { 211 if (skb->ip_summed != CHECKSUM_PARTIAL) 212 return 0; 213 214 if (skb->encapsulation) { 215 union hinic3_ip ip; 216 u8 l4_proto; 217 218 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, 219 TUNNEL_FLAG)); 220 221 ip.hdr = skb_network_header(skb); 222 if (ip.v4->version == 4) { 223 l4_proto = ip.v4->protocol; 224 } else if (ip.v4->version == 6) { 225 union hinic3_l4 l4; 226 unsigned char *exthdr; 227 __be16 frag_off; 228 229 exthdr = ip.hdr + sizeof(*ip.v6); 230 l4_proto = ip.v6->nexthdr; 231 l4.hdr = skb_transport_header(skb); 232 if (l4.hdr != exthdr) 233 ipv6_skip_exthdr(skb, exthdr - skb->data, 234 &l4_proto, &frag_off); 235 } else { 236 l4_proto = IPPROTO_RAW; 237 } 238 239 if (l4_proto != IPPROTO_UDP || 240 ((struct udphdr *)skb_transport_header(skb))->dest != 241 VXLAN_OFFLOAD_PORT_LE) { 242 /* Unsupported tunnel packet, disable csum offload */ 243 skb_checksum_help(skb); 244 return 0; 245 } 246 } 247 248 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L4_EN)); 249 250 return 1; 251 } 252 253 static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic3_ip *ip, 254 union hinic3_l4 *l4, 255 enum hinic3_l3_type *l3_type, u8 *l4_proto) 256 { 257 unsigned char *exthdr; 258 __be16 frag_off; 259 260 if (ip->v4->version == 4) { 261 *l3_type = HINIC3_L3_IP4_PKT_CSUM; 262 *l4_proto = ip->v4->protocol; 263 } else if (ip->v4->version == 6) { 264 *l3_type = HINIC3_L3_IP6_PKT; 265 exthdr = ip->hdr + sizeof(*ip->v6); 266 *l4_proto = ip->v6->nexthdr; 267 if (exthdr != l4->hdr) { 268 ipv6_skip_exthdr(skb, exthdr - skb->data, 269 l4_proto, &frag_off); 270 } 271 } else { 272 *l3_type = HINIC3_L3_UNKNOWN; 273 *l4_proto = 0; 274 } 275 } 276 277 static void hinic3_set_tso_info(struct hinic3_sq_task *task, __le32 *queue_info, 278 enum hinic3_l4_offload_type l4_offload, 279 u32 offset, u32 mss) 280 { 281 if (l4_offload == HINIC3_L4_OFFLOAD_TCP) { 282 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, TSO)); 283 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, 284 INNER_L4_EN)); 285 } else if (l4_offload == HINIC3_L4_OFFLOAD_UDP) { 286 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UFO)); 287 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, 288 INNER_L4_EN)); 289 } 290 291 /* enable L3 calculation */ 292 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L3_EN)); 293 294 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(offset >> 1, PLDOFF)); 295 296 /* set MSS value */ 297 *queue_info &= cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK); 298 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(mss, MSS)); 299 } 300 301 static __sum16 csum_magic(union hinic3_ip *ip, unsigned short proto) 302 { 303 return (ip->v4->version == 4) ? 304 csum_tcpudp_magic(ip->v4->saddr, ip->v4->daddr, 0, proto, 0) : 305 csum_ipv6_magic(&ip->v6->saddr, &ip->v6->daddr, 0, proto, 0); 306 } 307 308 static int hinic3_tso(struct hinic3_sq_task *task, __le32 *queue_info, 309 struct sk_buff *skb) 310 { 311 enum hinic3_l4_offload_type l4_offload; 312 enum hinic3_l3_type l3_type; 313 union hinic3_ip ip; 314 union hinic3_l4 l4; 315 u8 l4_proto; 316 u32 offset; 317 int err; 318 319 if (!skb_is_gso(skb)) 320 return 0; 321 322 err = skb_cow_head(skb, 0); 323 if (err < 0) 324 return err; 325 326 if (skb->encapsulation) { 327 u32 gso_type = skb_shinfo(skb)->gso_type; 328 /* L3 checksum is always enabled */ 329 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L3_EN)); 330 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, 331 TUNNEL_FLAG)); 332 333 l4.hdr = skb_transport_header(skb); 334 ip.hdr = skb_network_header(skb); 335 336 if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) { 337 l4.udp->check = ~csum_magic(&ip, IPPROTO_UDP); 338 task->pkt_info0 |= 339 cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L4_EN)); 340 } 341 342 ip.hdr = skb_inner_network_header(skb); 343 l4.hdr = skb_inner_transport_header(skb); 344 } else { 345 ip.hdr = skb_network_header(skb); 346 l4.hdr = skb_transport_header(skb); 347 } 348 349 get_inner_l3_l4_type(skb, &ip, &l4, &l3_type, &l4_proto); 350 351 if (l4_proto == IPPROTO_TCP) 352 l4.tcp->check = ~csum_magic(&ip, IPPROTO_TCP); 353 354 get_inner_l4_info(skb, &l4, l4_proto, &offset, &l4_offload); 355 356 hinic3_set_tso_info(task, queue_info, l4_offload, offset, 357 skb_shinfo(skb)->gso_size); 358 359 return 1; 360 } 361 362 static void hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task, 363 u16 vlan_tag, u8 vlan_tpid) 364 { 365 /* vlan_tpid: 0=select TPID0 in IPSU, 1=select TPID1 in IPSU 366 * 2=select TPID2 in IPSU, 3=select TPID3 in IPSU, 367 * 4=select TPID4 in IPSU 368 */ 369 task->vlan_offload = 370 cpu_to_le32(SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) | 371 SQ_TASK_INFO3_SET(vlan_tpid, VLAN_TPID) | 372 SQ_TASK_INFO3_SET(1, VLAN_TAG_VALID)); 373 } 374 375 static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task, 376 __le32 *queue_info, struct hinic3_txq *txq) 377 { 378 u32 offload = 0; 379 int tso_cs_en; 380 381 task->pkt_info0 = 0; 382 task->ip_identify = 0; 383 task->rsvd = 0; 384 task->vlan_offload = 0; 385 386 tso_cs_en = hinic3_tso(task, queue_info, skb); 387 if (tso_cs_en < 0) { 388 offload = HINIC3_TX_OFFLOAD_INVALID; 389 return offload; 390 } else if (tso_cs_en) { 391 offload |= HINIC3_TX_OFFLOAD_TSO; 392 } else { 393 tso_cs_en = hinic3_tx_csum(txq, task, skb); 394 if (tso_cs_en) 395 offload |= HINIC3_TX_OFFLOAD_CSUM; 396 } 397 398 #define VLAN_INSERT_MODE_MAX 5 399 if (unlikely(skb_vlan_tag_present(skb))) { 400 /* select vlan insert mode by qid, default 802.1Q Tag type */ 401 hinic3_set_vlan_tx_offload(task, skb_vlan_tag_get(skb), 402 txq->q_id % VLAN_INSERT_MODE_MAX); 403 offload |= HINIC3_TX_OFFLOAD_VLAN; 404 } 405 406 if (unlikely(SQ_CTRL_QUEUE_INFO_GET(*queue_info, PLDOFF) > 407 SQ_CTRL_MAX_PLDOFF)) { 408 offload = HINIC3_TX_OFFLOAD_INVALID; 409 return offload; 410 } 411 412 return offload; 413 } 414 415 static u16 hinic3_get_and_update_sq_owner(struct hinic3_io_queue *sq, 416 u16 curr_pi, u16 wqebb_cnt) 417 { 418 u16 owner = sq->owner; 419 420 if (unlikely(curr_pi + wqebb_cnt >= sq->wq.q_depth)) 421 sq->owner = !sq->owner; 422 423 return owner; 424 } 425 426 static u16 hinic3_set_wqe_combo(struct hinic3_txq *txq, 427 struct hinic3_sq_wqe_combo *wqe_combo, 428 u32 offload, u16 num_sge, u16 *curr_pi) 429 { 430 struct hinic3_sq_bufdesc *first_part_wqebbs, *second_part_wqebbs; 431 u16 first_part_wqebbs_num, tmp_pi; 432 433 wqe_combo->ctrl_bd0 = hinic3_wq_get_one_wqebb(&txq->sq->wq, curr_pi); 434 if (!offload && num_sge == 1) { 435 wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE; 436 return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi, 1); 437 } 438 439 wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; 440 441 if (offload) { 442 wqe_combo->task = hinic3_wq_get_one_wqebb(&txq->sq->wq, 443 &tmp_pi); 444 wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES; 445 } else { 446 wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS; 447 } 448 449 if (num_sge > 1) { 450 /* first wqebb contain bd0, and bd size is equal to sq wqebb 451 * size, so we use (num_sge - 1) as wanted weqbb_cnt 452 */ 453 hinic3_wq_get_multi_wqebbs(&txq->sq->wq, num_sge - 1, &tmp_pi, 454 &first_part_wqebbs, 455 &second_part_wqebbs, 456 &first_part_wqebbs_num); 457 wqe_combo->bds_head = first_part_wqebbs; 458 wqe_combo->bds_sec2 = second_part_wqebbs; 459 wqe_combo->first_bds_num = first_part_wqebbs_num; 460 } 461 462 return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi, 463 num_sge + !!offload); 464 } 465 466 static void hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, 467 __le32 queue_info, int nr_descs, u16 owner) 468 { 469 struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0; 470 471 if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE) { 472 wqe_desc->ctrl_len |= 473 cpu_to_le32(SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | 474 SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) | 475 SQ_CTRL_SET(owner, OWNER)); 476 477 /* compact wqe queue_info will transfer to chip */ 478 wqe_desc->queue_info = 0; 479 return; 480 } 481 482 wqe_desc->ctrl_len |= 483 cpu_to_le32(SQ_CTRL_SET(nr_descs, BUFDESC_NUM) | 484 SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) | 485 SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | 486 SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) | 487 SQ_CTRL_SET(owner, OWNER)); 488 489 wqe_desc->queue_info = queue_info; 490 wqe_desc->queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UC)); 491 492 if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) { 493 wqe_desc->queue_info |= 494 cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_DEFAULT, MSS)); 495 } else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) < 496 HINIC3_TX_MSS_MIN) { 497 /* mss should not be less than 80 */ 498 wqe_desc->queue_info &= 499 cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK); 500 wqe_desc->queue_info |= 501 cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_MIN, MSS)); 502 } 503 } 504 505 static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, 506 struct net_device *netdev, 507 struct hinic3_txq *txq) 508 { 509 struct hinic3_sq_wqe_combo wqe_combo = {}; 510 struct hinic3_tx_info *tx_info; 511 struct hinic3_sq_task task; 512 u16 wqebb_cnt, num_sge; 513 __le32 queue_info = 0; 514 u16 saved_wq_prod_idx; 515 u16 owner, pi = 0; 516 u8 saved_sq_owner; 517 u32 offload; 518 int err; 519 520 if (unlikely(skb->len < MIN_SKB_LEN)) { 521 if (skb_pad(skb, MIN_SKB_LEN - skb->len)) 522 goto err_out; 523 524 skb->len = MIN_SKB_LEN; 525 } 526 527 num_sge = skb_shinfo(skb)->nr_frags + 1; 528 /* assume normal wqe format + 1 wqebb for task info */ 529 wqebb_cnt = num_sge + 1; 530 531 if (unlikely(hinic3_wq_free_wqebbs(&txq->sq->wq) < wqebb_cnt)) { 532 if (likely(wqebb_cnt > txq->tx_stop_thrs)) 533 txq->tx_stop_thrs = min(wqebb_cnt, txq->tx_start_thrs); 534 535 netif_subqueue_try_stop(netdev, txq->sq->q_id, 536 hinic3_wq_free_wqebbs(&txq->sq->wq), 537 txq->tx_start_thrs); 538 539 return NETDEV_TX_BUSY; 540 } 541 542 offload = hinic3_tx_offload(skb, &task, &queue_info, txq); 543 if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) { 544 goto err_drop_pkt; 545 } else if (!offload) { 546 wqebb_cnt -= 1; 547 if (unlikely(num_sge == 1 && 548 skb->len > HINIC3_COMPACT_WQEE_SKB_MAX_LEN)) 549 goto err_drop_pkt; 550 } 551 552 saved_wq_prod_idx = txq->sq->wq.prod_idx; 553 saved_sq_owner = txq->sq->owner; 554 555 owner = hinic3_set_wqe_combo(txq, &wqe_combo, offload, num_sge, &pi); 556 if (offload) 557 *wqe_combo.task = task; 558 559 tx_info = &txq->tx_info[pi]; 560 tx_info->skb = skb; 561 tx_info->wqebb_cnt = wqebb_cnt; 562 563 err = hinic3_tx_map_skb(netdev, skb, txq, tx_info, &wqe_combo); 564 if (err) { 565 /* Rollback work queue to reclaim the wqebb we did not use */ 566 txq->sq->wq.prod_idx = saved_wq_prod_idx; 567 txq->sq->owner = saved_sq_owner; 568 goto err_drop_pkt; 569 } 570 571 netif_subqueue_sent(netdev, txq->sq->q_id, skb->len); 572 netif_subqueue_maybe_stop(netdev, txq->sq->q_id, 573 hinic3_wq_free_wqebbs(&txq->sq->wq), 574 txq->tx_stop_thrs, 575 txq->tx_start_thrs); 576 577 hinic3_prepare_sq_ctrl(&wqe_combo, queue_info, num_sge, owner); 578 hinic3_write_db(txq->sq, 0, DB_CFLAG_DP_SQ, 579 hinic3_get_sq_local_pi(txq->sq)); 580 581 return NETDEV_TX_OK; 582 583 err_drop_pkt: 584 dev_kfree_skb_any(skb); 585 586 err_out: 587 return NETDEV_TX_OK; 588 } 589 590 netdev_tx_t hinic3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 591 { 592 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 593 u16 q_id = skb_get_queue_mapping(skb); 594 595 if (unlikely(!netif_carrier_ok(netdev))) 596 goto err_drop_pkt; 597 598 if (unlikely(q_id >= nic_dev->q_params.num_qps)) 599 goto err_drop_pkt; 600 601 return hinic3_send_one_skb(skb, netdev, &nic_dev->txqs[q_id]); 602 603 err_drop_pkt: 604 dev_kfree_skb_any(skb); 605 606 return NETDEV_TX_OK; 607 } 608 609 static bool is_hw_complete_sq_process(struct hinic3_io_queue *sq) 610 { 611 u16 sw_pi, hw_ci; 612 613 sw_pi = hinic3_get_sq_local_pi(sq); 614 hw_ci = hinic3_get_sq_hw_ci(sq); 615 616 return sw_pi == hw_ci; 617 } 618 619 #define HINIC3_FLUSH_QUEUE_POLL_SLEEP_US 10000 620 #define HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US 10000000 621 static int hinic3_stop_sq(struct hinic3_txq *txq) 622 { 623 struct hinic3_nic_dev *nic_dev = netdev_priv(txq->netdev); 624 int err, rc; 625 626 err = read_poll_timeout(hinic3_force_drop_tx_pkt, rc, 627 is_hw_complete_sq_process(txq->sq) || rc, 628 HINIC3_FLUSH_QUEUE_POLL_SLEEP_US, 629 HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US, 630 true, nic_dev->hwdev); 631 if (rc) 632 return rc; 633 else 634 return err; 635 } 636 637 /* packet transmission should be stopped before calling this function */ 638 void hinic3_flush_txqs(struct net_device *netdev) 639 { 640 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 641 u16 qid; 642 int err; 643 644 for (qid = 0; qid < nic_dev->q_params.num_qps; qid++) { 645 err = hinic3_stop_sq(&nic_dev->txqs[qid]); 646 netdev_tx_reset_subqueue(netdev, qid); 647 if (err) 648 netdev_err(netdev, "Failed to stop sq%u\n", qid); 649 } 650 } 651 652 #define HINIC3_BDS_PER_SQ_WQEBB \ 653 (HINIC3_SQ_WQEBB_SIZE / sizeof(struct hinic3_sq_bufdesc)) 654 655 int hinic3_alloc_txqs_res(struct net_device *netdev, u16 num_sq, 656 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res) 657 { 658 struct hinic3_dyna_txq_res *tqres; 659 int idx; 660 661 for (idx = 0; idx < num_sq; idx++) { 662 tqres = &txqs_res[idx]; 663 664 tqres->tx_info = kcalloc(sq_depth, sizeof(*tqres->tx_info), 665 GFP_KERNEL); 666 if (!tqres->tx_info) 667 goto err_free_tqres; 668 669 tqres->bds = kcalloc(sq_depth * HINIC3_BDS_PER_SQ_WQEBB + 670 HINIC3_MAX_SQ_SGE, sizeof(*tqres->bds), 671 GFP_KERNEL); 672 if (!tqres->bds) { 673 kfree(tqres->tx_info); 674 goto err_free_tqres; 675 } 676 } 677 678 return 0; 679 680 err_free_tqres: 681 while (idx > 0) { 682 idx--; 683 tqres = &txqs_res[idx]; 684 685 kfree(tqres->bds); 686 kfree(tqres->tx_info); 687 } 688 689 return -ENOMEM; 690 } 691 692 void hinic3_free_txqs_res(struct net_device *netdev, u16 num_sq, 693 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res) 694 { 695 struct hinic3_dyna_txq_res *tqres; 696 int idx; 697 698 for (idx = 0; idx < num_sq; idx++) { 699 tqres = &txqs_res[idx]; 700 701 free_all_tx_skbs(netdev, sq_depth, tqres->tx_info); 702 kfree(tqres->bds); 703 kfree(tqres->tx_info); 704 } 705 } 706 707 int hinic3_configure_txqs(struct net_device *netdev, u16 num_sq, 708 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res) 709 { 710 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); 711 struct hinic3_dyna_txq_res *tqres; 712 struct hinic3_txq *txq; 713 u16 q_id; 714 u32 idx; 715 716 for (q_id = 0; q_id < num_sq; q_id++) { 717 txq = &nic_dev->txqs[q_id]; 718 tqres = &txqs_res[q_id]; 719 720 txq->q_depth = sq_depth; 721 txq->q_mask = sq_depth - 1; 722 723 txq->tx_stop_thrs = min(HINIC3_DEFAULT_STOP_THRS, 724 sq_depth / 20); 725 txq->tx_start_thrs = min(HINIC3_DEFAULT_START_THRS, 726 sq_depth / 10); 727 728 txq->tx_info = tqres->tx_info; 729 for (idx = 0; idx < sq_depth; idx++) 730 txq->tx_info[idx].dma_info = 731 &tqres->bds[idx * HINIC3_BDS_PER_SQ_WQEBB]; 732 733 txq->sq = &nic_dev->nic_io->sq[q_id]; 734 } 735 736 return 0; 737 } 738 739 bool hinic3_tx_poll(struct hinic3_txq *txq, int budget) 740 { 741 struct net_device *netdev = txq->netdev; 742 u16 hw_ci, sw_ci, q_id = txq->sq->q_id; 743 struct hinic3_tx_info *tx_info; 744 unsigned int bytes_compl = 0; 745 unsigned int pkts = 0; 746 u16 wqebb_cnt = 0; 747 748 hw_ci = hinic3_get_sq_hw_ci(txq->sq); 749 dma_rmb(); 750 sw_ci = hinic3_get_sq_local_ci(txq->sq); 751 752 do { 753 tx_info = &txq->tx_info[sw_ci]; 754 755 /* Did all wqebb of this wqe complete? */ 756 if (hw_ci == sw_ci || 757 ((hw_ci - sw_ci) & txq->q_mask) < tx_info->wqebb_cnt) 758 break; 759 760 sw_ci = (sw_ci + tx_info->wqebb_cnt) & txq->q_mask; 761 net_prefetch(&txq->tx_info[sw_ci]); 762 763 wqebb_cnt += tx_info->wqebb_cnt; 764 bytes_compl += tx_info->skb->len; 765 pkts++; 766 767 hinic3_tx_unmap_skb(netdev, tx_info->skb, tx_info->dma_info); 768 napi_consume_skb(tx_info->skb, budget); 769 tx_info->skb = NULL; 770 } while (likely(pkts < HINIC3_TX_POLL_WEIGHT)); 771 772 hinic3_wq_put_wqebbs(&txq->sq->wq, wqebb_cnt); 773 774 netif_subqueue_completed_wake(netdev, q_id, pkts, bytes_compl, 775 hinic3_wq_free_wqebbs(&txq->sq->wq), 776 txq->tx_start_thrs); 777 778 return pkts == HINIC3_TX_POLL_WEIGHT; 779 } 780