1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
3
4 #include <linux/if_vlan.h>
5 #include <linux/iopoll.h>
6 #include <net/ip6_checksum.h>
7 #include <net/ipv6.h>
8 #include <net/netdev_queues.h>
9
10 #include "hinic3_hwdev.h"
11 #include "hinic3_nic_cfg.h"
12 #include "hinic3_nic_dev.h"
13 #include "hinic3_nic_io.h"
14 #include "hinic3_tx.h"
15 #include "hinic3_wq.h"
16
17 #define MIN_SKB_LEN 32
18
hinic3_txq_clean_stats(struct hinic3_txq_stats * txq_stats)19 static void hinic3_txq_clean_stats(struct hinic3_txq_stats *txq_stats)
20 {
21 u64_stats_update_begin(&txq_stats->syncp);
22 txq_stats->bytes = 0;
23 txq_stats->packets = 0;
24 txq_stats->busy = 0;
25 txq_stats->dropped = 0;
26
27 txq_stats->skb_pad_err = 0;
28 txq_stats->frag_len_overflow = 0;
29 txq_stats->offload_cow_skb_err = 0;
30 txq_stats->map_frag_err = 0;
31 txq_stats->unknown_tunnel_pkt = 0;
32 txq_stats->frag_size_err = 0;
33 u64_stats_update_end(&txq_stats->syncp);
34 }
35
hinic3_txq_stats_init(struct hinic3_txq * txq)36 static void hinic3_txq_stats_init(struct hinic3_txq *txq)
37 {
38 struct hinic3_txq_stats *txq_stats = &txq->txq_stats;
39
40 u64_stats_init(&txq_stats->syncp);
41 hinic3_txq_clean_stats(txq_stats);
42 }
43
hinic3_alloc_txqs(struct net_device * netdev)44 int hinic3_alloc_txqs(struct net_device *netdev)
45 {
46 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
47 u16 q_id, num_txqs = nic_dev->max_qps;
48 struct pci_dev *pdev = nic_dev->pdev;
49 struct hinic3_txq *txq;
50
51 nic_dev->txqs = kzalloc_objs(*nic_dev->txqs, num_txqs);
52 if (!nic_dev->txqs)
53 return -ENOMEM;
54
55 for (q_id = 0; q_id < num_txqs; q_id++) {
56 txq = &nic_dev->txqs[q_id];
57 txq->netdev = netdev;
58 txq->q_id = q_id;
59 txq->q_depth = nic_dev->q_params.sq_depth;
60 txq->q_mask = nic_dev->q_params.sq_depth - 1;
61 txq->dev = &pdev->dev;
62
63 hinic3_txq_stats_init(txq);
64 }
65
66 return 0;
67 }
68
hinic3_free_txqs(struct net_device * netdev)69 void hinic3_free_txqs(struct net_device *netdev)
70 {
71 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
72
73 kfree(nic_dev->txqs);
74 }
75
hinic3_set_buf_desc(struct hinic3_sq_bufdesc * buf_descs,dma_addr_t addr,u32 len)76 static void hinic3_set_buf_desc(struct hinic3_sq_bufdesc *buf_descs,
77 dma_addr_t addr, u32 len)
78 {
79 buf_descs->hi_addr = cpu_to_le32(upper_32_bits(addr));
80 buf_descs->lo_addr = cpu_to_le32(lower_32_bits(addr));
81 buf_descs->len = cpu_to_le32(len);
82 }
83
hinic3_tx_map_skb(struct net_device * netdev,struct sk_buff * skb,struct hinic3_txq * txq,struct hinic3_tx_info * tx_info,struct hinic3_sq_wqe_combo * wqe_combo)84 static int hinic3_tx_map_skb(struct net_device *netdev, struct sk_buff *skb,
85 struct hinic3_txq *txq,
86 struct hinic3_tx_info *tx_info,
87 struct hinic3_sq_wqe_combo *wqe_combo)
88 {
89 struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0;
90 struct hinic3_sq_bufdesc *buf_desc = wqe_combo->bds_head;
91 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
92 struct hinic3_dma_info *dma_info = tx_info->dma_info;
93 struct pci_dev *pdev = nic_dev->pdev;
94 skb_frag_t *frag;
95 u32 i, idx;
96 int err;
97
98 dma_info[0].dma = dma_map_single(&pdev->dev, skb->data,
99 skb_headlen(skb), DMA_TO_DEVICE);
100 if (dma_mapping_error(&pdev->dev, dma_info[0].dma))
101 return -EFAULT;
102
103 dma_info[0].len = skb_headlen(skb);
104
105 wqe_desc->hi_addr = cpu_to_le32(upper_32_bits(dma_info[0].dma));
106 wqe_desc->lo_addr = cpu_to_le32(lower_32_bits(dma_info[0].dma));
107
108 wqe_desc->ctrl_len = cpu_to_le32(dma_info[0].len);
109
110 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
111 frag = &(skb_shinfo(skb)->frags[i]);
112 if (unlikely(i == wqe_combo->first_bds_num))
113 buf_desc = wqe_combo->bds_sec2;
114
115 idx = i + 1;
116 dma_info[idx].dma = skb_frag_dma_map(&pdev->dev, frag, 0,
117 skb_frag_size(frag),
118 DMA_TO_DEVICE);
119 if (dma_mapping_error(&pdev->dev, dma_info[idx].dma)) {
120 err = -EFAULT;
121 goto err_unmap_page;
122 }
123 dma_info[idx].len = skb_frag_size(frag);
124
125 hinic3_set_buf_desc(buf_desc, dma_info[idx].dma,
126 dma_info[idx].len);
127 buf_desc++;
128 }
129
130 return 0;
131
132 err_unmap_page:
133 while (idx > 1) {
134 idx--;
135 dma_unmap_page(&pdev->dev, dma_info[idx].dma,
136 dma_info[idx].len, DMA_TO_DEVICE);
137 }
138 dma_unmap_single(&pdev->dev, dma_info[0].dma, dma_info[0].len,
139 DMA_TO_DEVICE);
140
141 return err;
142 }
143
hinic3_tx_unmap_skb(struct net_device * netdev,struct sk_buff * skb,struct hinic3_dma_info * dma_info)144 static void hinic3_tx_unmap_skb(struct net_device *netdev,
145 struct sk_buff *skb,
146 struct hinic3_dma_info *dma_info)
147 {
148 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
149 struct pci_dev *pdev = nic_dev->pdev;
150 int i;
151
152 for (i = 0; i < skb_shinfo(skb)->nr_frags;) {
153 i++;
154 dma_unmap_page(&pdev->dev,
155 dma_info[i].dma,
156 dma_info[i].len, DMA_TO_DEVICE);
157 }
158
159 dma_unmap_single(&pdev->dev, dma_info[0].dma,
160 dma_info[0].len, DMA_TO_DEVICE);
161 }
162
free_all_tx_skbs(struct net_device * netdev,u32 sq_depth,struct hinic3_tx_info * tx_info_arr)163 static void free_all_tx_skbs(struct net_device *netdev, u32 sq_depth,
164 struct hinic3_tx_info *tx_info_arr)
165 {
166 struct hinic3_tx_info *tx_info;
167 u32 idx;
168
169 for (idx = 0; idx < sq_depth; idx++) {
170 tx_info = &tx_info_arr[idx];
171 if (tx_info->skb) {
172 hinic3_tx_unmap_skb(netdev, tx_info->skb,
173 tx_info->dma_info);
174 dev_kfree_skb_any(tx_info->skb);
175 tx_info->skb = NULL;
176 }
177 }
178 }
179
180 union hinic3_ip {
181 struct iphdr *v4;
182 struct ipv6hdr *v6;
183 unsigned char *hdr;
184 };
185
186 union hinic3_l4 {
187 struct tcphdr *tcp;
188 struct udphdr *udp;
189 unsigned char *hdr;
190 };
191
192 enum hinic3_l3_type {
193 HINIC3_L3_UNKNOWN = 0,
194 HINIC3_L3_IP6_PKT = 1,
195 HINIC3_L3_IP4_PKT_NO_CSUM = 2,
196 HINIC3_L3_IP4_PKT_CSUM = 3,
197 };
198
199 enum hinic3_l4_offload_type {
200 HINIC3_L4_OFFLOAD_DISABLE = 0,
201 HINIC3_L4_OFFLOAD_TCP = 1,
202 HINIC3_L4_OFFLOAD_STCP = 2,
203 HINIC3_L4_OFFLOAD_UDP = 3,
204 };
205
206 /* initialize l4 offset and offload */
get_inner_l4_info(struct sk_buff * skb,union hinic3_l4 * l4,u8 l4_proto,u32 * offset,enum hinic3_l4_offload_type * l4_offload)207 static void get_inner_l4_info(struct sk_buff *skb, union hinic3_l4 *l4,
208 u8 l4_proto, u32 *offset,
209 enum hinic3_l4_offload_type *l4_offload)
210 {
211 switch (l4_proto) {
212 case IPPROTO_TCP:
213 *l4_offload = HINIC3_L4_OFFLOAD_TCP;
214 /* To be same with TSO, payload offset begins from payload */
215 *offset = (l4->tcp->doff << TCP_HDR_DATA_OFF_UNIT_SHIFT) +
216 TRANSPORT_OFFSET(l4->hdr, skb);
217 break;
218
219 case IPPROTO_UDP:
220 *l4_offload = HINIC3_L4_OFFLOAD_UDP;
221 *offset = TRANSPORT_OFFSET(l4->hdr, skb);
222 break;
223 default:
224 *l4_offload = HINIC3_L4_OFFLOAD_DISABLE;
225 *offset = 0;
226 }
227 }
228
hinic3_tx_csum(struct hinic3_txq * txq,struct hinic3_sq_task * task,struct sk_buff * skb)229 static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task,
230 struct sk_buff *skb)
231 {
232 if (skb->ip_summed != CHECKSUM_PARTIAL)
233 return 0;
234
235 if (skb->encapsulation) {
236 union hinic3_ip ip;
237 u8 l4_proto;
238
239 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
240 TUNNEL_FLAG));
241
242 ip.hdr = skb_network_header(skb);
243 if (ip.v4->version == 4) {
244 l4_proto = ip.v4->protocol;
245 } else if (ip.v4->version == 6) {
246 union hinic3_l4 l4;
247 unsigned char *exthdr;
248 __be16 frag_off;
249
250 exthdr = ip.hdr + sizeof(*ip.v6);
251 l4_proto = ip.v6->nexthdr;
252 l4.hdr = skb_transport_header(skb);
253 if (l4.hdr != exthdr)
254 ipv6_skip_exthdr(skb, exthdr - skb->data,
255 &l4_proto, &frag_off);
256 } else {
257 l4_proto = IPPROTO_RAW;
258 }
259
260 if (l4_proto != IPPROTO_UDP ||
261 ((struct udphdr *)skb_transport_header(skb))->dest !=
262 VXLAN_OFFLOAD_PORT_LE) {
263 /* Unsupported tunnel packet, disable csum offload */
264 return skb_checksum_help(skb);
265 }
266 }
267
268 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L4_EN));
269
270 return 1;
271 }
272
get_inner_l3_l4_type(struct sk_buff * skb,union hinic3_ip * ip,union hinic3_l4 * l4,enum hinic3_l3_type * l3_type,u8 * l4_proto)273 static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic3_ip *ip,
274 union hinic3_l4 *l4,
275 enum hinic3_l3_type *l3_type, u8 *l4_proto)
276 {
277 unsigned char *exthdr;
278 __be16 frag_off;
279
280 if (ip->v4->version == 4) {
281 *l3_type = HINIC3_L3_IP4_PKT_CSUM;
282 *l4_proto = ip->v4->protocol;
283 } else if (ip->v4->version == 6) {
284 *l3_type = HINIC3_L3_IP6_PKT;
285 exthdr = ip->hdr + sizeof(*ip->v6);
286 *l4_proto = ip->v6->nexthdr;
287 if (exthdr != l4->hdr) {
288 ipv6_skip_exthdr(skb, exthdr - skb->data,
289 l4_proto, &frag_off);
290 }
291 } else {
292 *l3_type = HINIC3_L3_UNKNOWN;
293 *l4_proto = 0;
294 }
295 }
296
hinic3_set_tso_info(struct hinic3_sq_task * task,__le32 * queue_info,enum hinic3_l4_offload_type l4_offload,u32 offset,u32 mss)297 static void hinic3_set_tso_info(struct hinic3_sq_task *task, __le32 *queue_info,
298 enum hinic3_l4_offload_type l4_offload,
299 u32 offset, u32 mss)
300 {
301 if (l4_offload == HINIC3_L4_OFFLOAD_TCP) {
302 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, TSO));
303 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
304 INNER_L4_EN));
305 } else if (l4_offload == HINIC3_L4_OFFLOAD_UDP) {
306 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UFO));
307 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
308 INNER_L4_EN));
309 }
310
311 /* enable L3 calculation */
312 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L3_EN));
313
314 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(offset >> 1, PLDOFF));
315
316 /* set MSS value */
317 *queue_info &= cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK);
318 *queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(mss, MSS));
319 }
320
csum_magic(union hinic3_ip * ip,unsigned short proto)321 static __sum16 csum_magic(union hinic3_ip *ip, unsigned short proto)
322 {
323 return (ip->v4->version == 4) ?
324 csum_tcpudp_magic(ip->v4->saddr, ip->v4->daddr, 0, proto, 0) :
325 csum_ipv6_magic(&ip->v6->saddr, &ip->v6->daddr, 0, proto, 0);
326 }
327
hinic3_tso(struct hinic3_sq_task * task,__le32 * queue_info,struct sk_buff * skb)328 static int hinic3_tso(struct hinic3_sq_task *task, __le32 *queue_info,
329 struct sk_buff *skb)
330 {
331 enum hinic3_l4_offload_type l4_offload;
332 enum hinic3_l3_type l3_type;
333 union hinic3_ip ip;
334 union hinic3_l4 l4;
335 u8 l4_proto;
336 u32 offset;
337 int err;
338
339 if (!skb_is_gso(skb))
340 return 0;
341
342 err = skb_cow_head(skb, 0);
343 if (err < 0)
344 return err;
345
346 if (skb->encapsulation) {
347 u32 gso_type = skb_shinfo(skb)->gso_type;
348 /* L3 checksum is always enabled */
349 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L3_EN));
350 task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
351 TUNNEL_FLAG));
352
353 l4.hdr = skb_transport_header(skb);
354 ip.hdr = skb_network_header(skb);
355
356 if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) {
357 l4.udp->check = ~csum_magic(&ip, IPPROTO_UDP);
358 task->pkt_info0 |=
359 cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L4_EN));
360 }
361
362 ip.hdr = skb_inner_network_header(skb);
363 l4.hdr = skb_inner_transport_header(skb);
364 } else {
365 ip.hdr = skb_network_header(skb);
366 l4.hdr = skb_transport_header(skb);
367 }
368
369 get_inner_l3_l4_type(skb, &ip, &l4, &l3_type, &l4_proto);
370
371 if (l4_proto == IPPROTO_TCP)
372 l4.tcp->check = ~csum_magic(&ip, IPPROTO_TCP);
373
374 get_inner_l4_info(skb, &l4, l4_proto, &offset, &l4_offload);
375
376 hinic3_set_tso_info(task, queue_info, l4_offload, offset,
377 skb_shinfo(skb)->gso_size);
378
379 return 1;
380 }
381
hinic3_set_vlan_tx_offload(struct hinic3_sq_task * task,u16 vlan_tag,u8 vlan_tpid)382 static void hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task,
383 u16 vlan_tag, u8 vlan_tpid)
384 {
385 /* vlan_tpid: 0=select TPID0 in IPSU, 1=select TPID1 in IPSU
386 * 2=select TPID2 in IPSU, 3=select TPID3 in IPSU,
387 * 4=select TPID4 in IPSU
388 */
389 task->vlan_offload =
390 cpu_to_le32(SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) |
391 SQ_TASK_INFO3_SET(vlan_tpid, VLAN_TPID) |
392 SQ_TASK_INFO3_SET(1, VLAN_TAG_VALID));
393 }
394
hinic3_tx_offload(struct sk_buff * skb,struct hinic3_sq_task * task,__le32 * queue_info,struct hinic3_txq * txq)395 static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task,
396 __le32 *queue_info, struct hinic3_txq *txq)
397 {
398 u32 offload = 0;
399 int tso_cs_en;
400
401 task->pkt_info0 = 0;
402 task->ip_identify = 0;
403 task->rsvd = 0;
404 task->vlan_offload = 0;
405
406 tso_cs_en = hinic3_tso(task, queue_info, skb);
407 if (tso_cs_en < 0) {
408 offload = HINIC3_TX_OFFLOAD_INVALID;
409 return offload;
410 } else if (tso_cs_en) {
411 offload |= HINIC3_TX_OFFLOAD_TSO;
412 } else {
413 tso_cs_en = hinic3_tx_csum(txq, task, skb);
414 if (tso_cs_en < 0) {
415 offload = HINIC3_TX_OFFLOAD_INVALID;
416 return offload;
417 }
418 if (tso_cs_en)
419 offload |= HINIC3_TX_OFFLOAD_CSUM;
420 }
421
422 #define VLAN_INSERT_MODE_MAX 5
423 if (unlikely(skb_vlan_tag_present(skb))) {
424 /* select vlan insert mode by qid, default 802.1Q Tag type */
425 hinic3_set_vlan_tx_offload(task, skb_vlan_tag_get(skb),
426 txq->q_id % VLAN_INSERT_MODE_MAX);
427 offload |= HINIC3_TX_OFFLOAD_VLAN;
428 }
429
430 if (unlikely(SQ_CTRL_QUEUE_INFO_GET(*queue_info, PLDOFF) >
431 SQ_CTRL_MAX_PLDOFF)) {
432 offload = HINIC3_TX_OFFLOAD_INVALID;
433 return offload;
434 }
435
436 return offload;
437 }
438
hinic3_get_and_update_sq_owner(struct hinic3_io_queue * sq,u16 curr_pi,u16 wqebb_cnt)439 static u16 hinic3_get_and_update_sq_owner(struct hinic3_io_queue *sq,
440 u16 curr_pi, u16 wqebb_cnt)
441 {
442 u16 owner = sq->owner;
443
444 if (unlikely(curr_pi + wqebb_cnt >= sq->wq.q_depth))
445 sq->owner = !sq->owner;
446
447 return owner;
448 }
449
hinic3_set_wqe_combo(struct hinic3_txq * txq,struct hinic3_sq_wqe_combo * wqe_combo,u32 offload,u16 num_sge,u16 * curr_pi)450 static u16 hinic3_set_wqe_combo(struct hinic3_txq *txq,
451 struct hinic3_sq_wqe_combo *wqe_combo,
452 u32 offload, u16 num_sge, u16 *curr_pi)
453 {
454 struct hinic3_sq_bufdesc *first_part_wqebbs, *second_part_wqebbs;
455 u16 first_part_wqebbs_num, tmp_pi;
456
457 wqe_combo->ctrl_bd0 = hinic3_wq_get_one_wqebb(&txq->sq->wq, curr_pi);
458 if (!offload && num_sge == 1) {
459 wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE;
460 return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi, 1);
461 }
462
463 wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
464
465 if (offload) {
466 wqe_combo->task = hinic3_wq_get_one_wqebb(&txq->sq->wq,
467 &tmp_pi);
468 wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES;
469 } else {
470 wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS;
471 }
472
473 if (num_sge > 1) {
474 /* first wqebb contain bd0, and bd size is equal to sq wqebb
475 * size, so we use (num_sge - 1) as wanted weqbb_cnt
476 */
477 hinic3_wq_get_multi_wqebbs(&txq->sq->wq, num_sge - 1, &tmp_pi,
478 &first_part_wqebbs,
479 &second_part_wqebbs,
480 &first_part_wqebbs_num);
481 wqe_combo->bds_head = first_part_wqebbs;
482 wqe_combo->bds_sec2 = second_part_wqebbs;
483 wqe_combo->first_bds_num = first_part_wqebbs_num;
484 }
485
486 return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi,
487 num_sge + !!offload);
488 }
489
hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo * wqe_combo,__le32 queue_info,int nr_descs,u16 owner)490 static void hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo,
491 __le32 queue_info, int nr_descs, u16 owner)
492 {
493 struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0;
494
495 if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE) {
496 wqe_desc->ctrl_len |=
497 cpu_to_le32(SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
498 SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
499 SQ_CTRL_SET(owner, OWNER));
500
501 /* compact wqe queue_info will transfer to chip */
502 wqe_desc->queue_info = 0;
503 return;
504 }
505
506 wqe_desc->ctrl_len |=
507 cpu_to_le32(SQ_CTRL_SET(nr_descs, BUFDESC_NUM) |
508 SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) |
509 SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
510 SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
511 SQ_CTRL_SET(owner, OWNER));
512
513 wqe_desc->queue_info = queue_info;
514 wqe_desc->queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UC));
515
516 if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) {
517 wqe_desc->queue_info |=
518 cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_DEFAULT, MSS));
519 } else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) <
520 HINIC3_TX_MSS_MIN) {
521 /* mss should not be less than 80 */
522 wqe_desc->queue_info &=
523 cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK);
524 wqe_desc->queue_info |=
525 cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_MIN, MSS));
526 }
527 }
528
hinic3_send_one_skb(struct sk_buff * skb,struct net_device * netdev,struct hinic3_txq * txq)529 static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb,
530 struct net_device *netdev,
531 struct hinic3_txq *txq)
532 {
533 struct hinic3_sq_wqe_combo wqe_combo = {};
534 struct hinic3_tx_info *tx_info;
535 struct hinic3_sq_task task;
536 u16 wqebb_cnt, num_sge;
537 __le32 queue_info = 0;
538 u16 saved_wq_prod_idx;
539 u16 owner, pi = 0;
540 u8 saved_sq_owner;
541 u32 offload;
542 int err;
543
544 if (unlikely(skb->len < MIN_SKB_LEN)) {
545 if (skb_pad(skb, MIN_SKB_LEN - skb->len))
546 goto err_out;
547
548 skb->len = MIN_SKB_LEN;
549 }
550
551 offload = hinic3_tx_offload(skb, &task, &queue_info, txq);
552 num_sge = skb_shinfo(skb)->nr_frags + 1;
553 /* assume normal wqe format + 1 wqebb for task info */
554 wqebb_cnt = num_sge + 1;
555
556 if (unlikely(hinic3_wq_free_wqebbs(&txq->sq->wq) < wqebb_cnt)) {
557 if (likely(wqebb_cnt > txq->tx_stop_thrs))
558 txq->tx_stop_thrs = min(wqebb_cnt, txq->tx_start_thrs);
559
560 netif_subqueue_try_stop(netdev, txq->sq->q_id,
561 hinic3_wq_free_wqebbs(&txq->sq->wq),
562 txq->tx_start_thrs);
563
564 return NETDEV_TX_BUSY;
565 }
566
567 if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) {
568 goto err_drop_pkt;
569 } else if (!offload) {
570 wqebb_cnt -= 1;
571 if (unlikely(num_sge == 1 &&
572 skb->len > HINIC3_COMPACT_WQEE_SKB_MAX_LEN))
573 goto err_drop_pkt;
574 }
575
576 saved_wq_prod_idx = txq->sq->wq.prod_idx;
577 saved_sq_owner = txq->sq->owner;
578
579 owner = hinic3_set_wqe_combo(txq, &wqe_combo, offload, num_sge, &pi);
580 if (offload)
581 *wqe_combo.task = task;
582
583 tx_info = &txq->tx_info[pi];
584
585 err = hinic3_tx_map_skb(netdev, skb, txq, tx_info, &wqe_combo);
586 if (err) {
587 /* Rollback work queue to reclaim the wqebb we did not use */
588 txq->sq->wq.prod_idx = saved_wq_prod_idx;
589 txq->sq->owner = saved_sq_owner;
590 goto err_drop_pkt;
591 }
592
593 tx_info->skb = skb;
594 tx_info->wqebb_cnt = wqebb_cnt;
595
596 netif_subqueue_sent(netdev, txq->sq->q_id, skb->len);
597 netif_subqueue_maybe_stop(netdev, txq->sq->q_id,
598 hinic3_wq_free_wqebbs(&txq->sq->wq),
599 txq->tx_stop_thrs,
600 txq->tx_start_thrs);
601
602 hinic3_prepare_sq_ctrl(&wqe_combo, queue_info, num_sge, owner);
603 hinic3_write_db(txq->sq, 0, DB_CFLAG_DP_SQ,
604 hinic3_get_sq_local_pi(txq->sq));
605
606 return NETDEV_TX_OK;
607
608 err_drop_pkt:
609 dev_kfree_skb_any(skb);
610 err_out:
611 return NETDEV_TX_OK;
612 }
613
hinic3_xmit_frame(struct sk_buff * skb,struct net_device * netdev)614 netdev_tx_t hinic3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
615 {
616 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
617 u16 q_id = skb_get_queue_mapping(skb);
618
619 if (unlikely(!netif_carrier_ok(netdev)))
620 goto err_drop_pkt;
621
622 if (unlikely(q_id >= nic_dev->q_params.num_qps))
623 goto err_drop_pkt;
624
625 return hinic3_send_one_skb(skb, netdev, &nic_dev->txqs[q_id]);
626
627 err_drop_pkt:
628 dev_kfree_skb_any(skb);
629
630 return NETDEV_TX_OK;
631 }
632
is_hw_complete_sq_process(struct hinic3_io_queue * sq)633 static bool is_hw_complete_sq_process(struct hinic3_io_queue *sq)
634 {
635 u16 sw_pi, hw_ci;
636
637 sw_pi = hinic3_get_sq_local_pi(sq);
638 hw_ci = hinic3_get_sq_hw_ci(sq);
639
640 return sw_pi == hw_ci;
641 }
642
643 #define HINIC3_FLUSH_QUEUE_POLL_SLEEP_US 10000
644 #define HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US 10000000
hinic3_stop_sq(struct hinic3_txq * txq)645 static int hinic3_stop_sq(struct hinic3_txq *txq)
646 {
647 struct hinic3_nic_dev *nic_dev = netdev_priv(txq->netdev);
648 int err, rc;
649
650 err = read_poll_timeout(hinic3_force_drop_tx_pkt, rc,
651 is_hw_complete_sq_process(txq->sq) || rc,
652 HINIC3_FLUSH_QUEUE_POLL_SLEEP_US,
653 HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US,
654 true, nic_dev->hwdev);
655 if (rc)
656 return rc;
657 else
658 return err;
659 }
660
661 /* packet transmission should be stopped before calling this function */
hinic3_flush_txqs(struct net_device * netdev)662 void hinic3_flush_txqs(struct net_device *netdev)
663 {
664 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
665 u16 qid;
666 int err;
667
668 for (qid = 0; qid < nic_dev->q_params.num_qps; qid++) {
669 err = hinic3_stop_sq(&nic_dev->txqs[qid]);
670 netdev_tx_reset_subqueue(netdev, qid);
671 if (err)
672 netdev_err(netdev, "Failed to stop sq%u\n", qid);
673 }
674 }
675
676 #define HINIC3_BDS_PER_SQ_WQEBB \
677 (HINIC3_SQ_WQEBB_SIZE / sizeof(struct hinic3_sq_bufdesc))
678
hinic3_alloc_txqs_res(struct net_device * netdev,u16 num_sq,u32 sq_depth,struct hinic3_dyna_txq_res * txqs_res)679 int hinic3_alloc_txqs_res(struct net_device *netdev, u16 num_sq,
680 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
681 {
682 struct hinic3_dyna_txq_res *tqres;
683 int idx;
684
685 for (idx = 0; idx < num_sq; idx++) {
686 tqres = &txqs_res[idx];
687
688 tqres->tx_info = kzalloc_objs(*tqres->tx_info, sq_depth);
689 if (!tqres->tx_info)
690 goto err_free_tqres;
691
692 tqres->bds = kzalloc_objs(*tqres->bds,
693 sq_depth * HINIC3_BDS_PER_SQ_WQEBB + HINIC3_MAX_SQ_SGE);
694 if (!tqres->bds) {
695 kfree(tqres->tx_info);
696 goto err_free_tqres;
697 }
698 }
699
700 return 0;
701
702 err_free_tqres:
703 while (idx > 0) {
704 idx--;
705 tqres = &txqs_res[idx];
706
707 kfree(tqres->bds);
708 kfree(tqres->tx_info);
709 }
710
711 return -ENOMEM;
712 }
713
hinic3_free_txqs_res(struct net_device * netdev,u16 num_sq,u32 sq_depth,struct hinic3_dyna_txq_res * txqs_res)714 void hinic3_free_txqs_res(struct net_device *netdev, u16 num_sq,
715 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
716 {
717 struct hinic3_dyna_txq_res *tqres;
718 int idx;
719
720 for (idx = 0; idx < num_sq; idx++) {
721 tqres = &txqs_res[idx];
722
723 free_all_tx_skbs(netdev, sq_depth, tqres->tx_info);
724 kfree(tqres->bds);
725 kfree(tqres->tx_info);
726 }
727 }
728
hinic3_configure_txqs(struct net_device * netdev,u16 num_sq,u32 sq_depth,struct hinic3_dyna_txq_res * txqs_res)729 int hinic3_configure_txqs(struct net_device *netdev, u16 num_sq,
730 u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
731 {
732 struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
733 struct hinic3_dyna_txq_res *tqres;
734 struct hinic3_txq *txq;
735 u16 q_id;
736 u32 idx;
737
738 for (q_id = 0; q_id < num_sq; q_id++) {
739 txq = &nic_dev->txqs[q_id];
740 tqres = &txqs_res[q_id];
741
742 txq->q_depth = sq_depth;
743 txq->q_mask = sq_depth - 1;
744
745 txq->tx_stop_thrs = min(HINIC3_DEFAULT_STOP_THRS,
746 sq_depth / 20);
747 txq->tx_start_thrs = min(HINIC3_DEFAULT_START_THRS,
748 sq_depth / 10);
749
750 txq->tx_info = tqres->tx_info;
751 for (idx = 0; idx < sq_depth; idx++)
752 txq->tx_info[idx].dma_info =
753 &tqres->bds[idx * HINIC3_BDS_PER_SQ_WQEBB];
754
755 txq->sq = &nic_dev->nic_io->sq[q_id];
756 }
757
758 return 0;
759 }
760
hinic3_tx_poll(struct hinic3_txq * txq,int budget)761 bool hinic3_tx_poll(struct hinic3_txq *txq, int budget)
762 {
763 struct net_device *netdev = txq->netdev;
764 u16 hw_ci, sw_ci, q_id = txq->sq->q_id;
765 struct hinic3_tx_info *tx_info;
766 unsigned int bytes_compl = 0;
767 unsigned int pkts = 0;
768 u16 wqebb_cnt = 0;
769
770 hw_ci = hinic3_get_sq_hw_ci(txq->sq);
771 dma_rmb();
772 sw_ci = hinic3_get_sq_local_ci(txq->sq);
773
774 do {
775 tx_info = &txq->tx_info[sw_ci];
776
777 /* Did all wqebb of this wqe complete? */
778 if (hw_ci == sw_ci ||
779 ((hw_ci - sw_ci) & txq->q_mask) < tx_info->wqebb_cnt)
780 break;
781
782 sw_ci = (sw_ci + tx_info->wqebb_cnt) & txq->q_mask;
783 net_prefetch(&txq->tx_info[sw_ci]);
784
785 wqebb_cnt += tx_info->wqebb_cnt;
786 bytes_compl += tx_info->skb->len;
787 pkts++;
788
789 hinic3_tx_unmap_skb(netdev, tx_info->skb, tx_info->dma_info);
790 napi_consume_skb(tx_info->skb, budget);
791 tx_info->skb = NULL;
792 } while (likely(pkts < HINIC3_TX_POLL_WEIGHT));
793
794 hinic3_wq_put_wqebbs(&txq->sq->wq, wqebb_cnt);
795
796 netif_subqueue_completed_wake(netdev, q_id, pkts, bytes_compl,
797 hinic3_wq_free_wqebbs(&txq->sq->wq),
798 txq->tx_start_thrs);
799
800 return pkts == HINIC3_TX_POLL_WEIGHT;
801 }
802