1 // SPDX-License-Identifier: GPL-2.0-only
2 // Copyright 2024 Cisco Systems, Inc. All rights reserved.
3
4 #include <linux/skbuff.h>
5 #include <linux/if_vlan.h>
6 #include <net/busy_poll.h>
7 #include "enic.h"
8 #include "enic_res.h"
9 #include "enic_rq.h"
10 #include "vnic_rq.h"
11 #include "cq_enet_desc.h"
12
13 #define ENIC_LARGE_PKT_THRESHOLD 1000
14
enic_intr_update_pkt_size(struct vnic_rx_bytes_counter * pkt_size,u32 pkt_len)15 static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size,
16 u32 pkt_len)
17 {
18 if (pkt_len > ENIC_LARGE_PKT_THRESHOLD)
19 pkt_size->large_pkt_bytes_cnt += pkt_len;
20 else
21 pkt_size->small_pkt_bytes_cnt += pkt_len;
22 }
23
enic_rq_cq_desc_dec(void * cq_desc,u8 cq_desc_size,u8 * type,u8 * color,u16 * q_number,u16 * completed_index)24 static void enic_rq_cq_desc_dec(void *cq_desc, u8 cq_desc_size, u8 *type,
25 u8 *color, u16 *q_number, u16 *completed_index)
26 {
27 /* type_color is the last field for all cq structs */
28 u8 type_color;
29
30 switch (cq_desc_size) {
31 case VNIC_RQ_CQ_ENTRY_SIZE_16: {
32 struct cq_enet_rq_desc *desc =
33 (struct cq_enet_rq_desc *)cq_desc;
34 type_color = desc->type_color;
35
36 /* Make sure color bit is read from desc *before* other fields
37 * are read from desc. Hardware guarantees color bit is last
38 * bit (byte) written. Adding the rmb() prevents the compiler
39 * and/or CPU from reordering the reads which would potentially
40 * result in reading stale values.
41 */
42 rmb();
43
44 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
45 CQ_DESC_Q_NUM_MASK;
46 *completed_index = le16_to_cpu(desc->completed_index_flags) &
47 CQ_DESC_COMP_NDX_MASK;
48 break;
49 }
50 case VNIC_RQ_CQ_ENTRY_SIZE_32: {
51 struct cq_enet_rq_desc_32 *desc =
52 (struct cq_enet_rq_desc_32 *)cq_desc;
53 type_color = desc->type_color;
54
55 /* Make sure color bit is read from desc *before* other fields
56 * are read from desc. Hardware guarantees color bit is last
57 * bit (byte) written. Adding the rmb() prevents the compiler
58 * and/or CPU from reordering the reads which would potentially
59 * result in reading stale values.
60 */
61 rmb();
62
63 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
64 CQ_DESC_Q_NUM_MASK;
65 *completed_index = le16_to_cpu(desc->completed_index_flags) &
66 CQ_DESC_COMP_NDX_MASK;
67 *completed_index |= (desc->fetch_index_flags & CQ_DESC_32_FI_MASK) <<
68 CQ_DESC_COMP_NDX_BITS;
69 break;
70 }
71 case VNIC_RQ_CQ_ENTRY_SIZE_64: {
72 struct cq_enet_rq_desc_64 *desc =
73 (struct cq_enet_rq_desc_64 *)cq_desc;
74 type_color = desc->type_color;
75
76 /* Make sure color bit is read from desc *before* other fields
77 * are read from desc. Hardware guarantees color bit is last
78 * bit (byte) written. Adding the rmb() prevents the compiler
79 * and/or CPU from reordering the reads which would potentially
80 * result in reading stale values.
81 */
82 rmb();
83
84 *q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
85 CQ_DESC_Q_NUM_MASK;
86 *completed_index = le16_to_cpu(desc->completed_index_flags) &
87 CQ_DESC_COMP_NDX_MASK;
88 *completed_index |= (desc->fetch_index_flags & CQ_DESC_64_FI_MASK) <<
89 CQ_DESC_COMP_NDX_BITS;
90 break;
91 }
92 }
93
94 *color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
95 *type = type_color & CQ_DESC_TYPE_MASK;
96 }
97
enic_rq_set_skb_flags(struct vnic_rq * vrq,u8 type,u32 rss_hash,u8 rss_type,u8 fcoe,u8 fcoe_fc_crc_ok,u8 vlan_stripped,u8 csum_not_calc,u8 tcp_udp_csum_ok,u8 ipv6,u8 ipv4_csum_ok,u16 vlan_tci,struct sk_buff * skb)98 static void enic_rq_set_skb_flags(struct vnic_rq *vrq, u8 type, u32 rss_hash,
99 u8 rss_type, u8 fcoe, u8 fcoe_fc_crc_ok,
100 u8 vlan_stripped, u8 csum_not_calc,
101 u8 tcp_udp_csum_ok, u8 ipv6, u8 ipv4_csum_ok,
102 u16 vlan_tci, struct sk_buff *skb)
103 {
104 struct enic *enic = vnic_dev_priv(vrq->vdev);
105 struct net_device *netdev = enic->netdev;
106 struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats;
107 bool outer_csum_ok = true, encap = false;
108
109 if ((netdev->features & NETIF_F_RXHASH) && rss_hash && type == 3) {
110 switch (rss_type) {
111 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4:
112 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6:
113 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX:
114 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4);
115 rqstats->l4_rss_hash++;
116 break;
117 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4:
118 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6:
119 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX:
120 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3);
121 rqstats->l3_rss_hash++;
122 break;
123 }
124 }
125 if (enic->vxlan.vxlan_udp_port_number) {
126 switch (enic->vxlan.patch_level) {
127 case 0:
128 if (fcoe) {
129 encap = true;
130 outer_csum_ok = fcoe_fc_crc_ok;
131 }
132 break;
133 case 2:
134 if (type == 7 && (rss_hash & BIT(0))) {
135 encap = true;
136 outer_csum_ok = (rss_hash & BIT(1)) &&
137 (rss_hash & BIT(2));
138 }
139 break;
140 }
141 }
142
143 /* Hardware does not provide whole packet checksum. It only
144 * provides pseudo checksum. Since hw validates the packet
145 * checksum but not provide us the checksum value. use
146 * CHECSUM_UNNECESSARY.
147 *
148 * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is
149 * inner csum_ok. outer_csum_ok is set by hw when outer udp
150 * csum is correct or is zero.
151 */
152 if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc &&
153 tcp_udp_csum_ok && outer_csum_ok && (ipv4_csum_ok || ipv6)) {
154 skb->ip_summed = CHECKSUM_UNNECESSARY;
155 skb->csum_level = encap;
156 if (encap)
157 rqstats->csum_unnecessary_encap++;
158 else
159 rqstats->csum_unnecessary++;
160 }
161
162 if (vlan_stripped) {
163 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
164 rqstats->vlan_stripped++;
165 }
166 }
167
168 /*
169 * cq_enet_rq_desc accesses section uses only the 1st 15 bytes of the cq which
170 * is identical for all type (16,32 and 64 byte) of cqs.
171 */
cq_enet_rq_desc_dec(struct cq_enet_rq_desc * desc,u8 * ingress_port,u8 * fcoe,u8 * eop,u8 * sop,u8 * rss_type,u8 * csum_not_calc,u32 * rss_hash,u16 * bytes_written,u8 * packet_error,u8 * vlan_stripped,u16 * vlan_tci,u16 * checksum,u8 * fcoe_sof,u8 * fcoe_fc_crc_ok,u8 * fcoe_enc_error,u8 * fcoe_eof,u8 * tcp_udp_csum_ok,u8 * udp,u8 * tcp,u8 * ipv4_csum_ok,u8 * ipv6,u8 * ipv4,u8 * ipv4_fragment,u8 * fcs_ok)172 static void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, u8 *ingress_port,
173 u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type,
174 u8 *csum_not_calc, u32 *rss_hash,
175 u16 *bytes_written, u8 *packet_error,
176 u8 *vlan_stripped, u16 *vlan_tci,
177 u16 *checksum, u8 *fcoe_sof,
178 u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error,
179 u8 *fcoe_eof, u8 *tcp_udp_csum_ok, u8 *udp,
180 u8 *tcp, u8 *ipv4_csum_ok, u8 *ipv6, u8 *ipv4,
181 u8 *ipv4_fragment, u8 *fcs_ok)
182 {
183 u16 completed_index_flags;
184 u16 q_number_rss_type_flags;
185 u16 bytes_written_flags;
186
187 completed_index_flags = le16_to_cpu(desc->completed_index_flags);
188 q_number_rss_type_flags =
189 le16_to_cpu(desc->q_number_rss_type_flags);
190 bytes_written_flags = le16_to_cpu(desc->bytes_written_flags);
191
192 *ingress_port = (completed_index_flags &
193 CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0;
194 *fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ?
195 1 : 0;
196 *eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ?
197 1 : 0;
198 *sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ?
199 1 : 0;
200
201 *rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) &
202 CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
203 *csum_not_calc = (q_number_rss_type_flags &
204 CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0;
205
206 *rss_hash = le32_to_cpu(desc->rss_hash);
207
208 *bytes_written = bytes_written_flags &
209 CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
210 *packet_error = (bytes_written_flags &
211 CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0;
212 *vlan_stripped = (bytes_written_flags &
213 CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0;
214
215 /*
216 * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12)
217 */
218 *vlan_tci = le16_to_cpu(desc->vlan);
219
220 if (*fcoe) {
221 *fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) &
222 CQ_ENET_RQ_DESC_FCOE_SOF_MASK);
223 *fcoe_fc_crc_ok = (desc->flags &
224 CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0;
225 *fcoe_enc_error = (desc->flags &
226 CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0;
227 *fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >>
228 CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) &
229 CQ_ENET_RQ_DESC_FCOE_EOF_MASK);
230 *checksum = 0;
231 } else {
232 *fcoe_sof = 0;
233 *fcoe_fc_crc_ok = 0;
234 *fcoe_enc_error = 0;
235 *fcoe_eof = 0;
236 *checksum = le16_to_cpu(desc->checksum_fcoe);
237 }
238
239 *tcp_udp_csum_ok =
240 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0;
241 *udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0;
242 *tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0;
243 *ipv4_csum_ok =
244 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0;
245 *ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0;
246 *ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0;
247 *ipv4_fragment =
248 (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0;
249 *fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0;
250 }
251
enic_rq_pkt_error(struct vnic_rq * vrq,u8 packet_error,u8 fcs_ok,u16 bytes_written)252 static bool enic_rq_pkt_error(struct vnic_rq *vrq, u8 packet_error, u8 fcs_ok,
253 u16 bytes_written)
254 {
255 struct enic *enic = vnic_dev_priv(vrq->vdev);
256 struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats;
257
258 if (packet_error) {
259 if (!fcs_ok) {
260 if (bytes_written > 0)
261 rqstats->bad_fcs++;
262 else if (bytes_written == 0)
263 rqstats->pkt_truncated++;
264 }
265 return true;
266 }
267 return false;
268 }
269
enic_rq_alloc_buf(struct vnic_rq * rq)270 int enic_rq_alloc_buf(struct vnic_rq *rq)
271 {
272 struct enic *enic = vnic_dev_priv(rq->vdev);
273 struct net_device *netdev = enic->netdev;
274 struct enic_rq *erq = &enic->rq[rq->index];
275 struct enic_rq_stats *rqstats = &erq->stats;
276 unsigned int offset = 0;
277 unsigned int len = netdev->mtu + VLAN_ETH_HLEN;
278 unsigned int os_buf_index = 0;
279 dma_addr_t dma_addr;
280 struct vnic_rq_buf *buf = rq->to_use;
281 struct page *page;
282 unsigned int truesize = len;
283
284 if (buf->os_buf) {
285 enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr,
286 buf->len);
287
288 return 0;
289 }
290
291 page = page_pool_dev_alloc(erq->pool, &offset, &truesize);
292 if (unlikely(!page)) {
293 rqstats->pp_alloc_fail++;
294 return -ENOMEM;
295 }
296 buf->offset = offset;
297 buf->truesize = truesize;
298 dma_addr = page_pool_get_dma_addr(page) + offset;
299 enic_queue_rq_desc(rq, (void *)page, os_buf_index, dma_addr, len);
300
301 return 0;
302 }
303
enic_free_rq_buf(struct vnic_rq * rq,struct vnic_rq_buf * buf)304 void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf)
305 {
306 struct enic *enic = vnic_dev_priv(rq->vdev);
307 struct enic_rq *erq = &enic->rq[rq->index];
308
309 if (!buf->os_buf)
310 return;
311
312 page_pool_put_full_page(erq->pool, (struct page *)buf->os_buf, true);
313 buf->os_buf = NULL;
314 }
315
enic_rq_indicate_buf(struct enic * enic,struct vnic_rq * rq,struct vnic_rq_buf * buf,void * cq_desc,u8 type,u16 q_number,u16 completed_index)316 static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq,
317 struct vnic_rq_buf *buf, void *cq_desc,
318 u8 type, u16 q_number, u16 completed_index)
319 {
320 struct sk_buff *skb;
321 struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
322 struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats;
323 struct napi_struct *napi;
324
325 u8 eop, sop, ingress_port, vlan_stripped;
326 u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
327 u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
328 u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
329 u8 packet_error;
330 u16 bytes_written, vlan_tci, checksum;
331 u32 rss_hash;
332
333 rqstats->packets++;
334
335 cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, &ingress_port,
336 &fcoe, &eop, &sop, &rss_type, &csum_not_calc,
337 &rss_hash, &bytes_written, &packet_error,
338 &vlan_stripped, &vlan_tci, &checksum, &fcoe_sof,
339 &fcoe_fc_crc_ok, &fcoe_enc_error, &fcoe_eof,
340 &tcp_udp_csum_ok, &udp, &tcp, &ipv4_csum_ok, &ipv6,
341 &ipv4, &ipv4_fragment, &fcs_ok);
342
343 if (enic_rq_pkt_error(rq, packet_error, fcs_ok, bytes_written))
344 return;
345
346 if (eop && bytes_written > 0) {
347 /* Good receive
348 */
349 rqstats->bytes += bytes_written;
350 napi = &enic->napi[rq->index];
351 skb = napi_get_frags(napi);
352 if (unlikely(!skb)) {
353 net_warn_ratelimited("%s: skb alloc error rq[%d], desc[%d]\n",
354 enic->netdev->name, rq->index,
355 completed_index);
356 rqstats->no_skb++;
357 return;
358 }
359
360 prefetch(skb->data - NET_IP_ALIGN);
361
362 dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr,
363 bytes_written, DMA_FROM_DEVICE);
364 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
365 (struct page *)buf->os_buf, buf->offset,
366 bytes_written, buf->truesize);
367 skb_record_rx_queue(skb, q_number);
368 enic_rq_set_skb_flags(rq, type, rss_hash, rss_type, fcoe,
369 fcoe_fc_crc_ok, vlan_stripped,
370 csum_not_calc, tcp_udp_csum_ok, ipv6,
371 ipv4_csum_ok, vlan_tci, skb);
372 skb_mark_for_recycle(skb);
373 napi_gro_frags(napi);
374 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
375 enic_intr_update_pkt_size(&cq->pkt_size_counter,
376 bytes_written);
377 buf->os_buf = NULL;
378 buf->dma_addr = 0;
379 buf = buf->next;
380 } else {
381 /* Buffer overflow
382 */
383 rqstats->pkt_truncated++;
384 }
385 }
386
enic_rq_service(struct enic * enic,void * cq_desc,u8 type,u16 q_number,u16 completed_index)387 static void enic_rq_service(struct enic *enic, void *cq_desc, u8 type,
388 u16 q_number, u16 completed_index)
389 {
390 struct enic_rq_stats *rqstats = &enic->rq[q_number].stats;
391 struct vnic_rq *vrq = &enic->rq[q_number].vrq;
392 struct vnic_rq_buf *vrq_buf = vrq->to_clean;
393 int skipped;
394
395 while (1) {
396 skipped = (vrq_buf->index != completed_index);
397 if (!skipped)
398 enic_rq_indicate_buf(enic, vrq, vrq_buf, cq_desc, type,
399 q_number, completed_index);
400 else
401 rqstats->desc_skip++;
402
403 vrq->ring.desc_avail++;
404 vrq->to_clean = vrq_buf->next;
405 vrq_buf = vrq_buf->next;
406 if (!skipped)
407 break;
408 }
409 }
410
enic_rq_cq_service(struct enic * enic,unsigned int cq_index,unsigned int work_to_do)411 unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index,
412 unsigned int work_to_do)
413 {
414 struct vnic_cq *cq = &enic->cq[cq_index];
415 void *cq_desc = vnic_cq_to_clean(cq);
416 u16 q_number, completed_index;
417 unsigned int work_done = 0;
418 u8 type, color;
419
420 enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, &q_number,
421 &completed_index);
422
423 while (color != cq->last_color) {
424 enic_rq_service(enic, cq_desc, type, q_number, completed_index);
425 vnic_cq_inc_to_clean(cq);
426
427 if (++work_done >= work_to_do)
428 break;
429
430 cq_desc = vnic_cq_to_clean(cq);
431 enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color,
432 &q_number, &completed_index);
433 }
434
435 return work_done;
436 }
437