1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/workqueue.h>
9 #include <linux/iopoll.h>
10 #include <linux/sctp.h>
11 #include <linux/pci.h>
12 #include <net/tcp.h>
13 #include <net/ip.h>
14
15 #include "wx_type.h"
16 #include "wx_lib.h"
17 #include "wx_err.h"
18 #include "wx_ptp.h"
19 #include "wx_hw.h"
20 #include "wx_vf_lib.h"
21
22 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
23 static struct wx_dec_ptype wx_ptype_lookup[256] = {
24 /* L2: mac */
25 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
26 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
27 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
28 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
29 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
30 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
31 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
32
33 /* L2: ethertype filter */
34 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
35
36 /* L3: ip non-tunnel */
37 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
38 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
39 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
40 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
41 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
42 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
43 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
44 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
45 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
46 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
47
48 /* L2: fcoe */
49 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
50 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
51
52 /* IPv4 --> IPv4/IPv6 */
53 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
54 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
55 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
56 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
57 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
58 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
59 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
60 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
61 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
62 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
63
64 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
65 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
66 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
67 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
68 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
69 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
70 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
71 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
72 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
73 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
74 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
75 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
76
77 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
78 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
79 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
80 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
81 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
82 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
83 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
84 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
85 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
86 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
87 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
88 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
89
90 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
91 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
92 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
93 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
94 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
95 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
96 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
97 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
98 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
99 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
100 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
101 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
102
103 /* IPv6 --> IPv4/IPv6 */
104 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
105 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
106 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
107 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
108 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
109 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
110 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
111 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
112 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
113 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
114
115 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
116 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
117 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
118 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
119 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
120 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
121 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
122 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
123 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
124 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
125 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
126 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
127
128 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
129 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
130 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
131 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
132 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
133 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
134 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
135 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
136 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
137 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
138 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
139 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
140
141 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
142 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
143 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
144 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
145 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
146 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
147 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
148 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
149 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
150 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
151 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
152 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
153 };
154
wx_decode_ptype(const u8 ptype)155 struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
156 {
157 return wx_ptype_lookup[ptype];
158 }
159 EXPORT_SYMBOL(wx_decode_ptype);
160
161 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)162 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
163 const u32 stat_err_bits)
164 {
165 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
166 }
167
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)168 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
169 struct wx_rx_buffer *rx_buffer)
170 {
171 struct sk_buff *skb = rx_buffer->skb;
172 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
173
174 dma_sync_single_range_for_cpu(rx_ring->dev,
175 WX_CB(skb)->dma,
176 skb_frag_off(frag),
177 skb_frag_size(frag),
178 DMA_FROM_DEVICE);
179 }
180
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb)181 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
182 union wx_rx_desc *rx_desc,
183 struct sk_buff **skb)
184 {
185 struct wx_rx_buffer *rx_buffer;
186 unsigned int size;
187
188 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
189 size = le16_to_cpu(rx_desc->wb.upper.length);
190
191 prefetchw(rx_buffer->page);
192 *skb = rx_buffer->skb;
193
194 /* Delay unmapping of the first packet. It carries the header
195 * information, HW may still access the header after the writeback.
196 * Only unmap it when EOP is reached
197 */
198 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
199 if (!*skb)
200 goto skip_sync;
201 } else {
202 if (*skb)
203 wx_dma_sync_frag(rx_ring, rx_buffer);
204 }
205
206 /* we are reusing so sync this buffer for CPU use */
207 dma_sync_single_range_for_cpu(rx_ring->dev,
208 rx_buffer->dma,
209 rx_buffer->page_offset,
210 size,
211 DMA_FROM_DEVICE);
212 skip_sync:
213 return rx_buffer;
214 }
215
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb)216 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
217 struct wx_rx_buffer *rx_buffer,
218 struct sk_buff *skb)
219 {
220 /* clear contents of rx_buffer */
221 rx_buffer->page = NULL;
222 rx_buffer->skb = NULL;
223 }
224
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)225 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
226 struct wx_rx_buffer *rx_buffer,
227 union wx_rx_desc *rx_desc)
228 {
229 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
230 #if (PAGE_SIZE < 8192)
231 unsigned int truesize = wx_rx_pg_size(rx_ring) / 2;
232 #else
233 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
234 #endif
235 struct sk_buff *skb = rx_buffer->skb;
236
237 if (!skb) {
238 void *page_addr = page_address(rx_buffer->page) +
239 rx_buffer->page_offset;
240
241 /* prefetch first cache line of first page */
242 net_prefetch(page_addr);
243
244 /* allocate a skb to store the frags */
245 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
246 if (unlikely(!skb))
247 return NULL;
248
249 /* we will be copying header into skb->data in
250 * pskb_may_pull so it is in our interest to prefetch
251 * it now to avoid a possible cache miss
252 */
253 prefetchw(skb->data);
254
255 if (size <= WX_RXBUFFER_256) {
256 memcpy(__skb_put(skb, size), page_addr,
257 ALIGN(size, sizeof(long)));
258 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
259 return skb;
260 }
261
262 skb_mark_for_recycle(skb);
263
264 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
265 WX_CB(skb)->dma = rx_buffer->dma;
266
267 skb_add_rx_frag(skb, 0, rx_buffer->page,
268 rx_buffer->page_offset,
269 size, truesize);
270 goto out;
271
272 } else {
273 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
274 rx_buffer->page_offset, size, truesize);
275 }
276
277 out:
278 #if (PAGE_SIZE < 8192)
279 /* flip page offset to other buffer */
280 rx_buffer->page_offset ^= truesize;
281 #else
282 /* move offset up to the next cache line */
283 rx_buffer->page_offset += truesize;
284 #endif
285
286 return skb;
287 }
288
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)289 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
290 struct wx_rx_buffer *bi)
291 {
292 struct page *page = bi->page;
293 dma_addr_t dma;
294
295 /* since we are recycling buffers we should seldom need to alloc */
296 if (likely(page))
297 return true;
298
299 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
300 if (unlikely(!page))
301 return false;
302 dma = page_pool_get_dma_addr(page);
303
304 bi->dma = dma;
305 bi->page = page;
306 bi->page_offset = 0;
307
308 return true;
309 }
310
311 /**
312 * wx_alloc_rx_buffers - Replace used receive buffers
313 * @rx_ring: ring to place buffers on
314 * @cleaned_count: number of buffers to replace
315 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)316 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
317 {
318 u16 i = rx_ring->next_to_use;
319 union wx_rx_desc *rx_desc;
320 struct wx_rx_buffer *bi;
321
322 /* nothing to do */
323 if (!cleaned_count)
324 return;
325
326 rx_desc = WX_RX_DESC(rx_ring, i);
327 bi = &rx_ring->rx_buffer_info[i];
328 i -= rx_ring->count;
329
330 do {
331 if (!wx_alloc_mapped_page(rx_ring, bi))
332 break;
333
334 /* sync the buffer for use by the device */
335 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
336 bi->page_offset,
337 rx_ring->rx_buf_len,
338 DMA_FROM_DEVICE);
339
340 rx_desc->read.pkt_addr =
341 cpu_to_le64(bi->dma + bi->page_offset);
342
343 rx_desc++;
344 bi++;
345 i++;
346 if (unlikely(!i)) {
347 rx_desc = WX_RX_DESC(rx_ring, 0);
348 bi = rx_ring->rx_buffer_info;
349 i -= rx_ring->count;
350 }
351
352 /* clear the status bits for the next_to_use descriptor */
353 rx_desc->wb.upper.status_error = 0;
354 /* clear the length for the next_to_use descriptor */
355 rx_desc->wb.upper.length = 0;
356
357 cleaned_count--;
358 } while (cleaned_count);
359
360 i += rx_ring->count;
361
362 if (rx_ring->next_to_use != i) {
363 rx_ring->next_to_use = i;
364 /* update next to alloc since we have filled the ring */
365 rx_ring->next_to_alloc = i;
366
367 /* Force memory writes to complete before letting h/w
368 * know there are new descriptors to fetch. (Only
369 * applicable for weak-ordered memory model archs,
370 * such as IA-64).
371 */
372 wmb();
373 writel(i, rx_ring->tail);
374 }
375 }
376
wx_desc_unused(struct wx_ring * ring)377 u16 wx_desc_unused(struct wx_ring *ring)
378 {
379 u16 ntc = ring->next_to_clean;
380 u16 ntu = ring->next_to_use;
381
382 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
383 }
384
385 /**
386 * wx_is_non_eop - process handling of non-EOP buffers
387 * @rx_ring: Rx ring being processed
388 * @rx_desc: Rx descriptor for current buffer
389 * @skb: Current socket buffer containing buffer in progress
390 *
391 * This function updates next to clean. If the buffer is an EOP buffer
392 * this function exits returning false, otherwise it will place the
393 * sk_buff in the next buffer to be chained and return true indicating
394 * that this is in fact a non-EOP buffer.
395 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)396 static bool wx_is_non_eop(struct wx_ring *rx_ring,
397 union wx_rx_desc *rx_desc,
398 struct sk_buff *skb)
399 {
400 struct wx *wx = rx_ring->q_vector->wx;
401 u32 ntc = rx_ring->next_to_clean + 1;
402
403 /* fetch, update, and store next to clean */
404 ntc = (ntc < rx_ring->count) ? ntc : 0;
405 rx_ring->next_to_clean = ntc;
406
407 prefetch(WX_RX_DESC(rx_ring, ntc));
408
409 /* update RSC append count if present */
410 if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) {
411 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
412 cpu_to_le32(WX_RXD_RSCCNT_MASK);
413
414 if (unlikely(rsc_enabled)) {
415 u32 rsc_cnt = le32_to_cpu(rsc_enabled);
416
417 rsc_cnt >>= WX_RXD_RSCCNT_SHIFT;
418 WX_CB(skb)->append_cnt += rsc_cnt - 1;
419
420 /* update ntc based on RSC value */
421 ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
422 ntc &= WX_RXD_NEXTP_MASK;
423 ntc >>= WX_RXD_NEXTP_SHIFT;
424 }
425 }
426
427 /* if we are the last buffer then there is nothing else to do */
428 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
429 return false;
430
431 rx_ring->rx_buffer_info[ntc].skb = skb;
432 rx_ring->rx_stats.non_eop_descs++;
433
434 return true;
435 }
436
wx_pull_tail(struct sk_buff * skb)437 static void wx_pull_tail(struct sk_buff *skb)
438 {
439 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
440 unsigned int pull_len;
441 unsigned char *va;
442
443 /* it is valid to use page_address instead of kmap since we are
444 * working with pages allocated out of the lomem pool per
445 * alloc_page(GFP_ATOMIC)
446 */
447 va = skb_frag_address(frag);
448
449 /* we need the header to contain the greater of either ETH_HLEN or
450 * 60 bytes if the skb->len is less than 60 for skb_pad.
451 */
452 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
453
454 /* align pull length to size of long to optimize memcpy performance */
455 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
456
457 /* update all of the pointers */
458 skb_frag_size_sub(frag, pull_len);
459 skb_frag_off_add(frag, pull_len);
460 skb->data_len -= pull_len;
461 skb->tail += pull_len;
462 }
463
464 /**
465 * wx_cleanup_headers - Correct corrupted or empty headers
466 * @rx_ring: rx descriptor ring packet is being transacted on
467 * @rx_desc: pointer to the EOP Rx descriptor
468 * @skb: pointer to current skb being fixed
469 *
470 * Check for corrupted packet headers caused by senders on the local L2
471 * embedded NIC switch not setting up their Tx Descriptors right. These
472 * should be very rare.
473 *
474 * Also address the case where we are pulling data in on pages only
475 * and as such no data is present in the skb header.
476 *
477 * In addition if skb is not at least 60 bytes we need to pad it so that
478 * it is large enough to qualify as a valid Ethernet frame.
479 *
480 * Returns true if an error was encountered and skb was freed.
481 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)482 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
483 union wx_rx_desc *rx_desc,
484 struct sk_buff *skb)
485 {
486 struct net_device *netdev = rx_ring->netdev;
487
488 /* verify that the packet does not have any known errors */
489 if (!netdev ||
490 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
491 !(netdev->features & NETIF_F_RXALL))) {
492 dev_kfree_skb_any(skb);
493 return true;
494 }
495
496 /* place header in linear portion of buffer */
497 if (!skb_headlen(skb))
498 wx_pull_tail(skb);
499
500 /* if eth_skb_pad returns an error the skb was freed */
501 if (eth_skb_pad(skb))
502 return true;
503
504 return false;
505 }
506
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)507 static void wx_rx_hash(struct wx_ring *ring,
508 union wx_rx_desc *rx_desc,
509 struct sk_buff *skb)
510 {
511 u16 rss_type;
512
513 if (!(ring->netdev->features & NETIF_F_RXHASH))
514 return;
515
516 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
517 WX_RXD_RSSTYPE_MASK;
518
519 if (!rss_type)
520 return;
521
522 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
523 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
524 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
525 }
526
527 /**
528 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
529 * @ring: structure containing ring specific data
530 * @rx_desc: current Rx descriptor being processed
531 * @skb: skb currently being received and modified
532 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)533 static void wx_rx_checksum(struct wx_ring *ring,
534 union wx_rx_desc *rx_desc,
535 struct sk_buff *skb)
536 {
537 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
538
539 skb_checksum_none_assert(skb);
540 /* Rx csum disabled */
541 if (!(ring->netdev->features & NETIF_F_RXCSUM))
542 return;
543
544 /* if IPv4 header checksum error */
545 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
546 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
547 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
548 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
549 ring->rx_stats.csum_err++;
550 return;
551 }
552
553 /* L4 checksum offload flag must set for the below code to work */
554 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
555 return;
556
557 /* Hardware can't guarantee csum if IPv6 Dest Header found */
558 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP &&
559 wx_test_staterr(rx_desc, WX_RXD_STAT_IPV6EX))
560 return;
561
562 /* if L4 checksum error */
563 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
564 ring->rx_stats.csum_err++;
565 return;
566 }
567
568 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
569 skb->ip_summed = CHECKSUM_UNNECESSARY;
570
571 /* If there is an outer header present that might contain a checksum
572 * we need to bump the checksum level by 1 to reflect the fact that
573 * we are indicating we validated the inner checksum.
574 */
575 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
576 __skb_incr_checksum_unnecessary(skb);
577 ring->rx_stats.csum_good_cnt++;
578 }
579
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)580 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
581 struct sk_buff *skb)
582 {
583 u16 ethertype;
584 u8 idx = 0;
585
586 if ((ring->netdev->features &
587 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
588 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
589 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
590 0x1c0) >> 6;
591 ethertype = ring->q_vector->wx->tpid[idx];
592 __vlan_hwaccel_put_tag(skb, htons(ethertype),
593 le16_to_cpu(rx_desc->wb.upper.vlan));
594 }
595 }
596
wx_set_rsc_gso_size(struct wx_ring * ring,struct sk_buff * skb)597 static void wx_set_rsc_gso_size(struct wx_ring *ring,
598 struct sk_buff *skb)
599 {
600 u16 hdr_len = skb_headlen(skb);
601
602 /* set gso_size to avoid messing up TCP MSS */
603 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
604 WX_CB(skb)->append_cnt);
605 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
606 }
607
wx_update_rsc_stats(struct wx_ring * rx_ring,struct sk_buff * skb)608 static void wx_update_rsc_stats(struct wx_ring *rx_ring,
609 struct sk_buff *skb)
610 {
611 /* if append_cnt is 0 then frame is not RSC */
612 if (!WX_CB(skb)->append_cnt)
613 return;
614
615 rx_ring->rx_stats.rsc_count += WX_CB(skb)->append_cnt;
616 rx_ring->rx_stats.rsc_flush++;
617
618 wx_set_rsc_gso_size(rx_ring, skb);
619
620 /* gso_size is computed using append_cnt so always clear it last */
621 WX_CB(skb)->append_cnt = 0;
622 }
623
624 /**
625 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
626 * @rx_ring: rx descriptor ring packet is being transacted on
627 * @rx_desc: pointer to the EOP Rx descriptor
628 * @skb: pointer to current skb being populated
629 *
630 * This function checks the ring, descriptor, and packet information in
631 * order to populate the hash, checksum, protocol, and
632 * other fields within the skb.
633 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)634 static void wx_process_skb_fields(struct wx_ring *rx_ring,
635 union wx_rx_desc *rx_desc,
636 struct sk_buff *skb)
637 {
638 struct wx *wx = netdev_priv(rx_ring->netdev);
639
640 if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags))
641 wx_update_rsc_stats(rx_ring, skb);
642
643 wx_rx_hash(rx_ring, rx_desc, skb);
644 wx_rx_checksum(rx_ring, rx_desc, skb);
645
646 if (unlikely(test_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags)) &&
647 unlikely(wx_test_staterr(rx_desc, WX_RXD_STAT_TS))) {
648 wx_ptp_rx_hwtstamp(rx_ring->q_vector->wx, skb);
649 rx_ring->last_rx_timestamp = jiffies;
650 }
651
652 wx_rx_vlan(rx_ring, rx_desc, skb);
653 skb_record_rx_queue(skb, rx_ring->queue_index);
654 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
655 }
656
657 /**
658 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
659 * @q_vector: structure containing interrupt and ring information
660 * @rx_ring: rx descriptor ring to transact packets on
661 * @budget: Total limit on number of packets to process
662 *
663 * This function provides a "bounce buffer" approach to Rx interrupt
664 * processing. The advantage to this is that on systems that have
665 * expensive overhead for IOMMU access this provides a means of avoiding
666 * it by maintaining the mapping of the page to the system.
667 *
668 * Returns amount of work completed.
669 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)670 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
671 struct wx_ring *rx_ring,
672 int budget)
673 {
674 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
675 u16 cleaned_count = wx_desc_unused(rx_ring);
676
677 do {
678 struct wx_rx_buffer *rx_buffer;
679 union wx_rx_desc *rx_desc;
680 struct sk_buff *skb;
681
682 /* return some buffers to hardware, one at a time is too slow */
683 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
684 wx_alloc_rx_buffers(rx_ring, cleaned_count);
685 cleaned_count = 0;
686 }
687
688 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
689 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
690 break;
691
692 /* This memory barrier is needed to keep us from reading
693 * any other fields out of the rx_desc until we know the
694 * descriptor has been written back
695 */
696 dma_rmb();
697
698 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb);
699
700 /* retrieve a buffer from the ring */
701 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
702
703 /* exit if we failed to retrieve a buffer */
704 if (!skb) {
705 rx_ring->rx_stats.alloc_rx_buff_failed++;
706 break;
707 }
708
709 wx_put_rx_buffer(rx_ring, rx_buffer, skb);
710 cleaned_count++;
711
712 /* place incomplete frames back on ring for completion */
713 if (wx_is_non_eop(rx_ring, rx_desc, skb))
714 continue;
715
716 /* verify the packet layout is correct */
717 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
718 continue;
719
720 /* probably a little skewed due to removing CRC */
721 total_rx_bytes += skb->len;
722
723 /* populate checksum, timestamp, VLAN, and protocol */
724 wx_process_skb_fields(rx_ring, rx_desc, skb);
725 napi_gro_receive(&q_vector->napi, skb);
726
727 /* update budget accounting */
728 total_rx_packets++;
729 } while (likely(total_rx_packets < budget));
730
731 u64_stats_update_begin(&rx_ring->syncp);
732 rx_ring->stats.packets += total_rx_packets;
733 rx_ring->stats.bytes += total_rx_bytes;
734 u64_stats_update_end(&rx_ring->syncp);
735 q_vector->rx.total_packets += total_rx_packets;
736 q_vector->rx.total_bytes += total_rx_bytes;
737
738 return total_rx_packets;
739 }
740
wx_txring_txq(const struct wx_ring * ring)741 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
742 {
743 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
744 }
745
wx_get_tx_pending(struct wx_ring * ring)746 static u32 wx_get_tx_pending(struct wx_ring *ring)
747 {
748 unsigned int head, tail;
749
750 head = ring->next_to_clean;
751 tail = ring->next_to_use;
752
753 return ((head <= tail) ? tail : tail + ring->count) - head;
754 }
755
wx_check_tx_hang(struct wx_ring * ring)756 static bool wx_check_tx_hang(struct wx_ring *ring)
757 {
758 u32 tx_done_old = ring->tx_stats.tx_done_old;
759 u32 tx_pending = wx_get_tx_pending(ring);
760 u32 tx_done = ring->stats.packets;
761
762 if (!test_and_clear_bit(WX_TX_DETECT_HANG, ring->state))
763 return false;
764
765 if (tx_done_old == tx_done && tx_pending)
766 /* make sure it is true for two checks in a row */
767 return test_and_set_bit(WX_HANG_CHECK_ARMED, ring->state);
768
769 /* update completed stats and continue */
770 ring->tx_stats.tx_done_old = tx_done;
771 /* reset the countdown */
772 clear_bit(WX_HANG_CHECK_ARMED, ring->state);
773
774 return false;
775 }
776
777 /**
778 * wx_clean_tx_irq - Reclaim resources after transmit completes
779 * @q_vector: structure containing interrupt and ring information
780 * @tx_ring: tx ring to clean
781 * @napi_budget: Used to determine if we are in netpoll
782 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)783 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
784 struct wx_ring *tx_ring, int napi_budget)
785 {
786 unsigned int budget = q_vector->wx->tx_work_limit;
787 unsigned int total_bytes = 0, total_packets = 0;
788 struct wx *wx = netdev_priv(tx_ring->netdev);
789 unsigned int i = tx_ring->next_to_clean;
790 struct wx_tx_buffer *tx_buffer;
791 union wx_tx_desc *tx_desc;
792
793 if (!netif_carrier_ok(tx_ring->netdev))
794 return true;
795
796 tx_buffer = &tx_ring->tx_buffer_info[i];
797 tx_desc = WX_TX_DESC(tx_ring, i);
798 i -= tx_ring->count;
799
800 do {
801 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
802
803 /* if next_to_watch is not set then there is no work pending */
804 if (!eop_desc)
805 break;
806
807 /* prevent any other reads prior to eop_desc */
808 smp_rmb();
809
810 if (tx_ring->headwb_mem) {
811 u32 head = *tx_ring->headwb_mem;
812
813 if (head == tx_ring->next_to_clean)
814 break;
815 else if (head > tx_ring->next_to_clean &&
816 !(tx_buffer->next_eop >= tx_ring->next_to_clean &&
817 tx_buffer->next_eop < head))
818 break;
819 else if (!(tx_buffer->next_eop >= tx_ring->next_to_clean ||
820 tx_buffer->next_eop < head))
821 break;
822 } else if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD))) {
823 /* if DD is not set pending work has not been completed */
824 break;
825 }
826
827 /* clear next_to_watch to prevent false hangs */
828 tx_buffer->next_to_watch = NULL;
829
830 /* update the statistics for this packet */
831 total_bytes += tx_buffer->bytecount;
832 total_packets += tx_buffer->gso_segs;
833
834 /* schedule check for Tx timestamp */
835 if (unlikely(test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) &&
836 skb_shinfo(tx_buffer->skb)->tx_flags & SKBTX_IN_PROGRESS)
837 ptp_schedule_worker(wx->ptp_clock, 0);
838
839 /* free the skb */
840 napi_consume_skb(tx_buffer->skb, napi_budget);
841
842 /* unmap skb header data */
843 dma_unmap_single(tx_ring->dev,
844 dma_unmap_addr(tx_buffer, dma),
845 dma_unmap_len(tx_buffer, len),
846 DMA_TO_DEVICE);
847
848 /* clear tx_buffer data */
849 dma_unmap_len_set(tx_buffer, len, 0);
850
851 /* unmap remaining buffers */
852 while (tx_desc != eop_desc) {
853 tx_buffer++;
854 tx_desc++;
855 i++;
856 if (unlikely(!i)) {
857 i -= tx_ring->count;
858 tx_buffer = tx_ring->tx_buffer_info;
859 tx_desc = WX_TX_DESC(tx_ring, 0);
860 }
861
862 /* unmap any remaining paged data */
863 if (dma_unmap_len(tx_buffer, len)) {
864 dma_unmap_page(tx_ring->dev,
865 dma_unmap_addr(tx_buffer, dma),
866 dma_unmap_len(tx_buffer, len),
867 DMA_TO_DEVICE);
868 dma_unmap_len_set(tx_buffer, len, 0);
869 }
870 }
871
872 /* move us one more past the eop_desc for start of next pkt */
873 tx_buffer++;
874 tx_desc++;
875 i++;
876 if (unlikely(!i)) {
877 i -= tx_ring->count;
878 tx_buffer = tx_ring->tx_buffer_info;
879 tx_desc = WX_TX_DESC(tx_ring, 0);
880 }
881
882 /* issue prefetch for next Tx descriptor */
883 prefetch(tx_desc);
884
885 /* update budget accounting */
886 budget--;
887 } while (likely(budget));
888
889 i += tx_ring->count;
890 tx_ring->next_to_clean = i;
891 u64_stats_update_begin(&tx_ring->syncp);
892 tx_ring->stats.bytes += total_bytes;
893 tx_ring->stats.packets += total_packets;
894 u64_stats_update_end(&tx_ring->syncp);
895 q_vector->tx.total_bytes += total_bytes;
896 q_vector->tx.total_packets += total_packets;
897
898 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
899 total_packets, total_bytes);
900
901 if (wx_check_tx_hang(tx_ring)) {
902 wx_handle_tx_hang(tx_ring, i);
903 return true;
904 }
905
906 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
907 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
908 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
909 /* Make sure that anybody stopping the queue after this
910 * sees the new next_to_clean.
911 */
912 smp_mb();
913
914 if (__netif_subqueue_stopped(tx_ring->netdev,
915 tx_ring->queue_index) &&
916 !test_bit(WX_STATE_DOWN, wx->state)) {
917 netif_wake_subqueue(tx_ring->netdev,
918 tx_ring->queue_index);
919 ++tx_ring->tx_stats.restart_queue;
920 }
921 }
922
923 return !!budget;
924 }
925
wx_update_rx_dim_sample(struct wx_q_vector * q_vector)926 static void wx_update_rx_dim_sample(struct wx_q_vector *q_vector)
927 {
928 struct dim_sample sample = {};
929
930 dim_update_sample(q_vector->total_events,
931 q_vector->rx.total_packets,
932 q_vector->rx.total_bytes,
933 &sample);
934
935 net_dim(&q_vector->rx.dim, &sample);
936 }
937
wx_update_tx_dim_sample(struct wx_q_vector * q_vector)938 static void wx_update_tx_dim_sample(struct wx_q_vector *q_vector)
939 {
940 struct dim_sample sample = {};
941
942 dim_update_sample(q_vector->total_events,
943 q_vector->tx.total_packets,
944 q_vector->tx.total_bytes,
945 &sample);
946
947 net_dim(&q_vector->tx.dim, &sample);
948 }
949
wx_update_dim_sample(struct wx_q_vector * q_vector)950 static void wx_update_dim_sample(struct wx_q_vector *q_vector)
951 {
952 wx_update_rx_dim_sample(q_vector);
953 wx_update_tx_dim_sample(q_vector);
954 }
955
956 /**
957 * wx_poll - NAPI polling RX/TX cleanup routine
958 * @napi: napi struct with our devices info in it
959 * @budget: amount of work driver is allowed to do this pass, in packets
960 *
961 * This function will clean all queues associated with a q_vector.
962 **/
wx_poll(struct napi_struct * napi,int budget)963 static int wx_poll(struct napi_struct *napi, int budget)
964 {
965 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
966 int per_ring_budget, work_done = 0;
967 struct wx *wx = q_vector->wx;
968 bool clean_complete = true;
969 struct wx_ring *ring;
970
971 wx_for_each_ring(ring, q_vector->tx) {
972 if (!wx_clean_tx_irq(q_vector, ring, budget))
973 clean_complete = false;
974 }
975
976 /* Exit if we are called by netpoll */
977 if (budget <= 0)
978 return budget;
979
980 /* attempt to distribute budget to each queue fairly, but don't allow
981 * the budget to go below 1 because we'll exit polling
982 */
983 if (q_vector->rx.count > 1)
984 per_ring_budget = max(budget / q_vector->rx.count, 1);
985 else
986 per_ring_budget = budget;
987
988 wx_for_each_ring(ring, q_vector->rx) {
989 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
990
991 work_done += cleaned;
992 if (cleaned >= per_ring_budget)
993 clean_complete = false;
994 }
995
996 /* If all work not completed, return budget and keep polling */
997 if (!clean_complete)
998 return budget;
999
1000 /* all work done, exit the polling mode */
1001 if (likely(napi_complete_done(napi, work_done))) {
1002 if (wx->adaptive_itr)
1003 wx_update_dim_sample(q_vector);
1004 if (!test_bit(WX_STATE_DOWN, wx->state))
1005 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
1006 }
1007
1008 return min(work_done, budget - 1);
1009 }
1010
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)1011 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
1012 {
1013 if (likely(wx_desc_unused(tx_ring) >= size))
1014 return 0;
1015
1016 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
1017
1018 /* For the next check */
1019 smp_mb();
1020
1021 /* We need to check again in a case another CPU has just
1022 * made room available.
1023 */
1024 if (likely(wx_desc_unused(tx_ring) < size))
1025 return -EBUSY;
1026
1027 /* A reprieve! - use start_queue because it doesn't call schedule */
1028 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
1029 ++tx_ring->tx_stats.restart_queue;
1030
1031 return 0;
1032 }
1033
wx_tx_cmd_type(u32 tx_flags)1034 static u32 wx_tx_cmd_type(u32 tx_flags)
1035 {
1036 /* set type for advanced descriptor with frame checksum insertion */
1037 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
1038
1039 /* set HW vlan bit if vlan is present */
1040 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
1041 /* set segmentation enable bits for TSO/FSO */
1042 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
1043 /* set timestamp bit if present */
1044 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
1045 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
1046
1047 return cmd_type;
1048 }
1049
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)1050 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
1051 u32 tx_flags, unsigned int paylen)
1052 {
1053 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
1054
1055 /* enable L4 checksum for TSO and TX checksum offload */
1056 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
1057 /* enable IPv4 checksum for TSO */
1058 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
1059 /* enable outer IPv4 checksum for TSO */
1060 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
1061 WX_TXD_EIPCS);
1062 /* Check Context must be set if Tx switch is enabled, which it
1063 * always is for case where virtual functions are running
1064 */
1065 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
1066 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
1067 WX_TXD_IPSEC);
1068 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1069 }
1070
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)1071 static int wx_tx_map(struct wx_ring *tx_ring,
1072 struct wx_tx_buffer *first,
1073 const u8 hdr_len)
1074 {
1075 struct sk_buff *skb = first->skb;
1076 struct wx_tx_buffer *tx_buffer;
1077 u32 tx_flags = first->tx_flags;
1078 u16 i = tx_ring->next_to_use;
1079 unsigned int data_len, size;
1080 union wx_tx_desc *tx_desc;
1081 skb_frag_t *frag;
1082 dma_addr_t dma;
1083 u32 cmd_type;
1084
1085 cmd_type = wx_tx_cmd_type(tx_flags);
1086 tx_desc = WX_TX_DESC(tx_ring, i);
1087 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
1088
1089 size = skb_headlen(skb);
1090 data_len = skb->data_len;
1091 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1092
1093 tx_buffer = first;
1094
1095 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1096 if (dma_mapping_error(tx_ring->dev, dma))
1097 goto dma_error;
1098
1099 /* record length, and DMA address */
1100 dma_unmap_len_set(tx_buffer, len, size);
1101 dma_unmap_addr_set(tx_buffer, dma, dma);
1102
1103 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1104
1105 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
1106 tx_desc->read.cmd_type_len =
1107 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
1108
1109 i++;
1110 tx_desc++;
1111 if (i == tx_ring->count) {
1112 tx_desc = WX_TX_DESC(tx_ring, 0);
1113 i = 0;
1114 }
1115 tx_desc->read.olinfo_status = 0;
1116
1117 dma += WX_MAX_DATA_PER_TXD;
1118 size -= WX_MAX_DATA_PER_TXD;
1119
1120 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1121 }
1122
1123 if (likely(!data_len))
1124 break;
1125
1126 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1127
1128 i++;
1129 tx_desc++;
1130 if (i == tx_ring->count) {
1131 tx_desc = WX_TX_DESC(tx_ring, 0);
1132 i = 0;
1133 }
1134 tx_desc->read.olinfo_status = 0;
1135
1136 size = skb_frag_size(frag);
1137
1138 data_len -= size;
1139
1140 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1141 DMA_TO_DEVICE);
1142
1143 tx_buffer = &tx_ring->tx_buffer_info[i];
1144 }
1145
1146 /* write last descriptor with RS and EOP bits */
1147 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1148 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1149
1150 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1151
1152 /* set the timestamp */
1153 first->time_stamp = jiffies;
1154 skb_tx_timestamp(skb);
1155
1156 /* Force memory writes to complete before letting h/w know there
1157 * are new descriptors to fetch. (Only applicable for weak-ordered
1158 * memory model archs, such as IA-64).
1159 *
1160 * We also need this memory barrier to make certain all of the
1161 * status bits have been updated before next_to_watch is written.
1162 */
1163 wmb();
1164
1165 /* set next_to_watch value indicating a packet is present */
1166 first->next_to_watch = tx_desc;
1167
1168 /* set next_eop for amlite tx head wb */
1169 if (tx_ring->headwb_mem)
1170 first->next_eop = i;
1171
1172 i++;
1173 if (i == tx_ring->count)
1174 i = 0;
1175
1176 tx_ring->next_to_use = i;
1177
1178 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1179
1180 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1181 writel(i, tx_ring->tail);
1182
1183 return 0;
1184 dma_error:
1185 dev_err(tx_ring->dev, "TX DMA map failed\n");
1186
1187 /* clear dma mappings for failed tx_buffer_info map */
1188 for (;;) {
1189 tx_buffer = &tx_ring->tx_buffer_info[i];
1190 if (dma_unmap_len(tx_buffer, len))
1191 dma_unmap_page(tx_ring->dev,
1192 dma_unmap_addr(tx_buffer, dma),
1193 dma_unmap_len(tx_buffer, len),
1194 DMA_TO_DEVICE);
1195 dma_unmap_len_set(tx_buffer, len, 0);
1196 if (tx_buffer == first)
1197 break;
1198 if (i == 0)
1199 i += tx_ring->count;
1200 i--;
1201 }
1202
1203 /* first->skb is released by the caller, which keeps a reference on it
1204 * until the PTP cleanup has compared it against wx->ptp_tx_skb. That
1205 * prevents the address from being reused by a newer request while the
1206 * comparison is pending.
1207 */
1208 tx_ring->next_to_use = i;
1209
1210 return -ENOMEM;
1211 }
1212
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1213 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1214 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1215 {
1216 struct wx_tx_context_desc *context_desc;
1217 u16 i = tx_ring->next_to_use;
1218
1219 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1220 i++;
1221 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1222
1223 /* set bits to identify this as an advanced context descriptor */
1224 type_tucmd |= WX_TXD_DTYP_CTXT;
1225 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1226 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1227 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1228 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1229 }
1230
1231 union network_header {
1232 struct iphdr *ipv4;
1233 struct ipv6hdr *ipv6;
1234 void *raw;
1235 };
1236
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1237 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1238 {
1239 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1240 struct sk_buff *skb = first->skb;
1241 unsigned char *exthdr, *l4_hdr;
1242 __be16 frag_off;
1243
1244 if (skb->encapsulation) {
1245 union network_header hdr;
1246
1247 switch (first->protocol) {
1248 case htons(ETH_P_IP):
1249 tun_prot = ip_hdr(skb)->protocol;
1250 ptype = WX_PTYPE_TUN_IPV4;
1251 break;
1252 case htons(ETH_P_IPV6):
1253 l4_hdr = skb_transport_header(skb);
1254 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1255 tun_prot = ipv6_hdr(skb)->nexthdr;
1256 if (l4_hdr != exthdr)
1257 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1258 ptype = WX_PTYPE_TUN_IPV6;
1259 break;
1260 default:
1261 return ptype;
1262 }
1263
1264 if (tun_prot == IPPROTO_IPIP || tun_prot == IPPROTO_IPV6) {
1265 hdr.raw = (void *)inner_ip_hdr(skb);
1266 ptype |= WX_PTYPE_PKT_IPIP;
1267 } else if (tun_prot == IPPROTO_UDP) {
1268 hdr.raw = (void *)inner_ip_hdr(skb);
1269 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1270 skb->inner_protocol != htons(ETH_P_TEB)) {
1271 ptype |= WX_PTYPE_PKT_IG;
1272 } else {
1273 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1274 == htons(ETH_P_8021Q))
1275 ptype |= WX_PTYPE_PKT_IGMV;
1276 else
1277 ptype |= WX_PTYPE_PKT_IGM;
1278 }
1279
1280 } else if (tun_prot == IPPROTO_GRE) {
1281 hdr.raw = (void *)inner_ip_hdr(skb);
1282 if (skb->inner_protocol == htons(ETH_P_IP) ||
1283 skb->inner_protocol == htons(ETH_P_IPV6)) {
1284 ptype |= WX_PTYPE_PKT_IG;
1285 } else {
1286 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1287 == htons(ETH_P_8021Q))
1288 ptype |= WX_PTYPE_PKT_IGMV;
1289 else
1290 ptype |= WX_PTYPE_PKT_IGM;
1291 }
1292 } else {
1293 return ptype;
1294 }
1295
1296 switch (hdr.ipv4->version) {
1297 case IPVERSION:
1298 l4_prot = hdr.ipv4->protocol;
1299 break;
1300 case 6:
1301 l4_hdr = skb_inner_transport_header(skb);
1302 exthdr = skb_inner_network_header(skb) + sizeof(struct ipv6hdr);
1303 l4_prot = inner_ipv6_hdr(skb)->nexthdr;
1304 if (l4_hdr != exthdr)
1305 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1306 ptype |= WX_PTYPE_PKT_IPV6;
1307 break;
1308 default:
1309 return ptype;
1310 }
1311 } else {
1312 switch (first->protocol) {
1313 case htons(ETH_P_IP):
1314 l4_prot = ip_hdr(skb)->protocol;
1315 ptype = WX_PTYPE_PKT_IP;
1316 break;
1317 case htons(ETH_P_IPV6):
1318 l4_hdr = skb_transport_header(skb);
1319 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1320 l4_prot = ipv6_hdr(skb)->nexthdr;
1321 if (l4_hdr != exthdr)
1322 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1323 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1324 break;
1325 default:
1326 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1327 }
1328 }
1329 switch (l4_prot) {
1330 case IPPROTO_TCP:
1331 ptype |= WX_PTYPE_TYP_TCP;
1332 break;
1333 case IPPROTO_UDP:
1334 ptype |= WX_PTYPE_TYP_UDP;
1335 break;
1336 case IPPROTO_SCTP:
1337 ptype |= WX_PTYPE_TYP_SCTP;
1338 break;
1339 default:
1340 ptype |= WX_PTYPE_TYP_IP;
1341 break;
1342 }
1343
1344 return ptype;
1345 }
1346
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1347 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1348 u8 *hdr_len, u8 ptype)
1349 {
1350 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1351 struct net_device *netdev = tx_ring->netdev;
1352 u32 l4len, tunhdr_eiplen_tunlen = 0;
1353 struct sk_buff *skb = first->skb;
1354 bool enc = skb->encapsulation;
1355 struct ipv6hdr *ipv6h;
1356 struct tcphdr *tcph;
1357 struct iphdr *iph;
1358 u8 tun_prot = 0;
1359 int err;
1360
1361 if (skb->ip_summed != CHECKSUM_PARTIAL)
1362 return 0;
1363
1364 if (!skb_is_gso(skb))
1365 return 0;
1366
1367 err = skb_cow_head(skb, 0);
1368 if (err < 0)
1369 return err;
1370
1371 /* indicates the inner headers in the skbuff are valid. */
1372 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1373 if (iph->version == 4) {
1374 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1375 iph->tot_len = 0;
1376 iph->check = 0;
1377 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1378 iph->daddr, 0,
1379 IPPROTO_TCP, 0);
1380 first->tx_flags |= WX_TX_FLAGS_TSO |
1381 WX_TX_FLAGS_CSUM |
1382 WX_TX_FLAGS_IPV4 |
1383 WX_TX_FLAGS_CC;
1384 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1385 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1386 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1387 ipv6h->payload_len = 0;
1388 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1389 &ipv6h->daddr, 0,
1390 IPPROTO_TCP, 0);
1391 first->tx_flags |= WX_TX_FLAGS_TSO |
1392 WX_TX_FLAGS_CSUM |
1393 WX_TX_FLAGS_CC;
1394 }
1395
1396 /* compute header lengths */
1397 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1398 *hdr_len = enc ? skb_inner_transport_offset(skb) :
1399 skb_transport_offset(skb);
1400 *hdr_len += l4len;
1401
1402 /* update gso size and bytecount with header size */
1403 first->gso_segs = skb_shinfo(skb)->gso_segs;
1404 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1405
1406 /* mss_l4len_id: use 0 as index for TSO */
1407 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1408 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1409
1410 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1411 if (enc) {
1412 unsigned char *exthdr, *l4_hdr;
1413 __be16 frag_off;
1414
1415 switch (first->protocol) {
1416 case htons(ETH_P_IP):
1417 tun_prot = ip_hdr(skb)->protocol;
1418 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1419 break;
1420 case htons(ETH_P_IPV6):
1421 l4_hdr = skb_transport_header(skb);
1422 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1423 tun_prot = ipv6_hdr(skb)->nexthdr;
1424 if (l4_hdr != exthdr)
1425 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1426 break;
1427 default:
1428 break;
1429 }
1430 switch (tun_prot) {
1431 case IPPROTO_UDP:
1432 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1433 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1434 WX_TXD_OUTER_IPLEN_SHIFT) |
1435 (((skb_inner_mac_header(skb) -
1436 skb_transport_header(skb)) >> 1) <<
1437 WX_TXD_TUNNEL_LEN_SHIFT);
1438 break;
1439 case IPPROTO_GRE:
1440 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1441 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1442 WX_TXD_OUTER_IPLEN_SHIFT) |
1443 (((skb_inner_mac_header(skb) -
1444 skb_transport_header(skb)) >> 1) <<
1445 WX_TXD_TUNNEL_LEN_SHIFT);
1446 break;
1447 case IPPROTO_IPIP:
1448 case IPPROTO_IPV6:
1449 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1450 (char *)ip_hdr(skb)) >> 2) <<
1451 WX_TXD_OUTER_IPLEN_SHIFT;
1452 break;
1453 default:
1454 break;
1455 }
1456 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1457 } else {
1458 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1459 }
1460
1461 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1462 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1463
1464 type_tucmd = ptype << 24;
1465 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1466 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1467 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1468 WX_TX_FLAGS_HW_VLAN,
1469 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1470 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1471 type_tucmd, mss_l4len_idx);
1472
1473 return 1;
1474 }
1475
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1476 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1477 u8 ptype)
1478 {
1479 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1480 struct net_device *netdev = tx_ring->netdev;
1481 u32 mss_l4len_idx = 0, type_tucmd;
1482 struct sk_buff *skb = first->skb;
1483 u8 tun_prot = 0;
1484
1485 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1486 csum_failed:
1487 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1488 !(first->tx_flags & WX_TX_FLAGS_CC))
1489 return;
1490 vlan_macip_lens = skb_network_offset(skb) <<
1491 WX_TXD_MACLEN_SHIFT;
1492 } else {
1493 unsigned char *exthdr, *l4_hdr;
1494 __be16 frag_off;
1495 u8 l4_prot = 0;
1496 union {
1497 struct iphdr *ipv4;
1498 struct ipv6hdr *ipv6;
1499 u8 *raw;
1500 } network_hdr;
1501 union {
1502 struct tcphdr *tcphdr;
1503 u8 *raw;
1504 } transport_hdr;
1505
1506 if (skb->encapsulation) {
1507 network_hdr.raw = skb_inner_network_header(skb);
1508 transport_hdr.raw = skb_inner_transport_header(skb);
1509 vlan_macip_lens = skb_network_offset(skb) <<
1510 WX_TXD_MACLEN_SHIFT;
1511 switch (first->protocol) {
1512 case htons(ETH_P_IP):
1513 tun_prot = ip_hdr(skb)->protocol;
1514 break;
1515 case htons(ETH_P_IPV6):
1516 l4_hdr = skb_transport_header(skb);
1517 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1518 tun_prot = ipv6_hdr(skb)->nexthdr;
1519 if (l4_hdr != exthdr)
1520 ipv6_skip_exthdr(skb, exthdr - skb->data,
1521 &tun_prot, &frag_off);
1522 break;
1523 default:
1524 return;
1525 }
1526 switch (tun_prot) {
1527 case IPPROTO_UDP:
1528 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1529 tunhdr_eiplen_tunlen |=
1530 ((skb_network_header_len(skb) >> 2) <<
1531 WX_TXD_OUTER_IPLEN_SHIFT) |
1532 (((skb_inner_mac_header(skb) -
1533 skb_transport_header(skb)) >> 1) <<
1534 WX_TXD_TUNNEL_LEN_SHIFT);
1535 break;
1536 case IPPROTO_GRE:
1537 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1538 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1539 WX_TXD_OUTER_IPLEN_SHIFT) |
1540 (((skb_inner_mac_header(skb) -
1541 skb_transport_header(skb)) >> 1) <<
1542 WX_TXD_TUNNEL_LEN_SHIFT);
1543 break;
1544 case IPPROTO_IPIP:
1545 case IPPROTO_IPV6:
1546 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1547 (char *)ip_hdr(skb)) >> 2) <<
1548 WX_TXD_OUTER_IPLEN_SHIFT;
1549 break;
1550 default:
1551 break;
1552 }
1553
1554 } else {
1555 network_hdr.raw = skb_network_header(skb);
1556 transport_hdr.raw = skb_transport_header(skb);
1557 vlan_macip_lens = skb_network_offset(skb) <<
1558 WX_TXD_MACLEN_SHIFT;
1559 }
1560
1561 switch (network_hdr.ipv4->version) {
1562 case IPVERSION:
1563 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1564 l4_prot = network_hdr.ipv4->protocol;
1565 break;
1566 case 6:
1567 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1568 exthdr = network_hdr.raw + sizeof(struct ipv6hdr);
1569 l4_prot = network_hdr.ipv6->nexthdr;
1570 if (transport_hdr.raw != exthdr)
1571 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1572 break;
1573 default:
1574 break;
1575 }
1576
1577 switch (l4_prot) {
1578 case IPPROTO_TCP:
1579 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1580 WX_TXD_L4LEN_SHIFT;
1581 break;
1582 case IPPROTO_SCTP:
1583 mss_l4len_idx = sizeof(struct sctphdr) <<
1584 WX_TXD_L4LEN_SHIFT;
1585 break;
1586 case IPPROTO_UDP:
1587 mss_l4len_idx = sizeof(struct udphdr) <<
1588 WX_TXD_L4LEN_SHIFT;
1589 break;
1590 default:
1591 skb_checksum_help(skb);
1592 goto csum_failed;
1593 }
1594
1595 /* update TX checksum flag */
1596 first->tx_flags |= WX_TX_FLAGS_CSUM;
1597 }
1598 first->tx_flags |= WX_TX_FLAGS_CC;
1599 /* vlan_macip_lens: MACLEN, VLAN tag */
1600 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1601
1602 type_tucmd = ptype << 24;
1603 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1604 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1605 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1606 WX_TX_FLAGS_HW_VLAN,
1607 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1608 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1609 type_tucmd, mss_l4len_idx);
1610 }
1611
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1612 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1613 struct wx_ring *tx_ring)
1614 {
1615 struct wx *wx = netdev_priv(tx_ring->netdev);
1616 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1617 struct wx_tx_buffer *first;
1618 u8 hdr_len = 0, ptype;
1619 unsigned short f;
1620 u32 tx_flags = 0;
1621 int tso;
1622
1623 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1624 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1625 * + 2 desc gap to keep tail from touching head,
1626 * + 1 desc for context descriptor,
1627 * otherwise try next time
1628 */
1629 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1630 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1631 frags[f]));
1632
1633 if (wx_maybe_stop_tx(tx_ring, count + 3)) {
1634 tx_ring->tx_stats.tx_busy++;
1635 return NETDEV_TX_BUSY;
1636 }
1637
1638 /* record the location of the first descriptor for this packet */
1639 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1640 first->skb = skb;
1641 first->bytecount = skb->len;
1642 first->gso_segs = 1;
1643
1644 /* if we have a HW VLAN tag being added default to the HW one */
1645 if (skb_vlan_tag_present(skb)) {
1646 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1647 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1648 } else if (eth_type_vlan(skb->protocol)) {
1649 tx_flags |= WX_TX_FLAGS_SW_VLAN;
1650 }
1651
1652 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1653 wx->ptp_clock) {
1654 unsigned long flags;
1655
1656 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
1657 if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1658 !test_and_set_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) {
1659 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1660 tx_flags |= WX_TX_FLAGS_TSTAMP;
1661 wx->ptp_tx_skb = skb_get(skb);
1662 wx->ptp_tx_start = jiffies;
1663 } else {
1664 wx->tx_hwtstamp_skipped++;
1665 }
1666 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
1667 }
1668
1669 /* record initial flags and protocol */
1670 first->tx_flags = tx_flags;
1671 first->protocol = vlan_get_protocol(skb);
1672
1673 ptype = wx_encode_tx_desc_ptype(first);
1674
1675 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1676 if (tso < 0)
1677 goto out_drop;
1678 else if (!tso)
1679 wx_tx_csum(tx_ring, first, ptype);
1680
1681 if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)
1682 wx->atr(tx_ring, first, ptype);
1683
1684 if (wx_tx_map(tx_ring, first, hdr_len))
1685 goto out_drop;
1686
1687 return NETDEV_TX_OK;
1688 out_drop:
1689 /* The frame never reached the hardware, so no timestamp will ever be
1690 * reported for it and the request has to be cancelled. The slot is
1691 * shared, though: wx_ptp_clear_tx_timestamp() or wx_ptp_tx_hang() may
1692 * have dropped our request already, and a transmit on another queue
1693 * can have claimed the slot since. Only cancel it while it is still
1694 * ours, otherwise we would free somebody else's skb and release their
1695 * in-progress bit.
1696 */
1697 if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) {
1698 struct sk_buff *ptp_tx_skb = NULL;
1699 unsigned long flags;
1700
1701 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
1702 if (wx->ptp_tx_skb == skb) {
1703 ptp_tx_skb = wx->ptp_tx_skb;
1704 wx->ptp_tx_skb = NULL;
1705 clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
1706 wx->tx_hwtstamp_errors++;
1707 }
1708 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
1709
1710 dev_kfree_skb_any(ptp_tx_skb);
1711 }
1712 dev_kfree_skb_any(first->skb);
1713 first->skb = NULL;
1714
1715 return NETDEV_TX_OK;
1716 }
1717
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1718 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1719 struct net_device *netdev)
1720 {
1721 unsigned int r_idx = skb->queue_mapping;
1722 struct wx *wx = netdev_priv(netdev);
1723 struct wx_ring *tx_ring;
1724
1725 if (!netif_carrier_ok(netdev)) {
1726 dev_kfree_skb_any(skb);
1727 return NETDEV_TX_OK;
1728 }
1729
1730 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1731 * in order to meet this minimum size requirement.
1732 */
1733 if (skb_put_padto(skb, 17))
1734 return NETDEV_TX_OK;
1735
1736 if (r_idx >= wx->num_tx_queues)
1737 r_idx = r_idx % wx->num_tx_queues;
1738 tx_ring = wx->tx_ring[r_idx];
1739
1740 return wx_xmit_frame_ring(skb, tx_ring);
1741 }
1742 EXPORT_SYMBOL(wx_xmit_frame);
1743
wx_set_itr(struct wx_q_vector * q_vector)1744 static void wx_set_itr(struct wx_q_vector *q_vector)
1745 {
1746 struct wx *wx = q_vector->wx;
1747 u32 new_itr;
1748
1749 if (!wx->adaptive_itr)
1750 return;
1751
1752 /* use the smallest value of new ITR delay calculations */
1753 new_itr = min(q_vector->rx.itr, q_vector->tx.itr);
1754 new_itr <<= 2;
1755
1756 if (new_itr != q_vector->itr) {
1757 /* save the algorithm value here */
1758 q_vector->itr = new_itr;
1759
1760 if (wx->pdev->is_virtfn)
1761 wx_write_eitr_vf(q_vector);
1762 else
1763 wx_write_eitr(q_vector);
1764 }
1765 }
1766
wx_rx_dim_work(struct work_struct * work)1767 static void wx_rx_dim_work(struct work_struct *work)
1768 {
1769 struct dim *dim = container_of(work, struct dim, work);
1770 struct dim_cq_moder rx_moder;
1771 struct wx_ring_container *rx;
1772 struct wx_q_vector *q_vector;
1773
1774 rx = container_of(dim, struct wx_ring_container, dim);
1775
1776 rx_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1777 rx->itr = rx_moder.usec;
1778
1779 q_vector = container_of(rx, struct wx_q_vector, rx);
1780 wx_set_itr(q_vector);
1781
1782 dim->state = DIM_START_MEASURE;
1783 }
1784
wx_tx_dim_work(struct work_struct * work)1785 static void wx_tx_dim_work(struct work_struct *work)
1786 {
1787 struct dim *dim = container_of(work, struct dim, work);
1788 struct dim_cq_moder tx_moder;
1789 struct wx_ring_container *tx;
1790 struct wx_q_vector *q_vector;
1791
1792 tx = container_of(dim, struct wx_ring_container, dim);
1793
1794 tx_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
1795 tx->itr = tx_moder.usec;
1796
1797 q_vector = container_of(tx, struct wx_q_vector, tx);
1798 wx_set_itr(q_vector);
1799
1800 dim->state = DIM_START_MEASURE;
1801 }
1802
wx_napi_enable_all(struct wx * wx)1803 void wx_napi_enable_all(struct wx *wx)
1804 {
1805 struct wx_q_vector *q_vector;
1806 int q_idx;
1807
1808 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1809 q_vector = wx->q_vector[q_idx];
1810
1811 INIT_WORK(&q_vector->rx.dim.work, wx_rx_dim_work);
1812 INIT_WORK(&q_vector->tx.dim.work, wx_tx_dim_work);
1813 q_vector->rx.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1814 q_vector->tx.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1815 napi_enable(&q_vector->napi);
1816 }
1817 }
1818 EXPORT_SYMBOL(wx_napi_enable_all);
1819
wx_napi_disable_all(struct wx * wx)1820 void wx_napi_disable_all(struct wx *wx)
1821 {
1822 struct wx_q_vector *q_vector;
1823 int q_idx;
1824
1825 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1826 q_vector = wx->q_vector[q_idx];
1827 napi_disable(&q_vector->napi);
1828 disable_work_sync(&q_vector->rx.dim.work);
1829 disable_work_sync(&q_vector->tx.dim.work);
1830 }
1831 }
1832 EXPORT_SYMBOL(wx_napi_disable_all);
1833
wx_set_vmdq_queues(struct wx * wx)1834 static bool wx_set_vmdq_queues(struct wx *wx)
1835 {
1836 u16 vmdq_i = wx->ring_feature[RING_F_VMDQ].limit;
1837 u16 rss_i = wx->ring_feature[RING_F_RSS].limit;
1838 u16 rss_m = WX_RSS_DISABLED_MASK;
1839 u16 vmdq_m = 0;
1840
1841 /* only proceed if VMDq is enabled */
1842 if (!test_bit(WX_FLAG_VMDQ_ENABLED, wx->flags))
1843 return false;
1844 /* Add starting offset to total pool count */
1845 vmdq_i += wx->ring_feature[RING_F_VMDQ].offset;
1846
1847 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
1848 /* double check we are limited to maximum pools */
1849 vmdq_i = min_t(u16, 64, vmdq_i);
1850
1851 /* 64 pool mode with 2 queues per pool, or
1852 * 16/32/64 pool mode with 1 queue per pool
1853 */
1854 if (vmdq_i > 32 || rss_i < 4) {
1855 vmdq_m = WX_VMDQ_2Q_MASK;
1856 rss_m = WX_RSS_2Q_MASK;
1857 rss_i = min_t(u16, rss_i, 2);
1858 /* 32 pool mode with 4 queues per pool */
1859 } else {
1860 vmdq_m = WX_VMDQ_4Q_MASK;
1861 rss_m = WX_RSS_4Q_MASK;
1862 rss_i = 4;
1863 }
1864 } else {
1865 vmdq_m = WX_VMDQ_1Q_MASK;
1866 /* double check we are limited to maximum pools */
1867 vmdq_i = min_t(u16, 8, vmdq_i);
1868
1869 /* when VMDQ on, disable RSS */
1870 rss_i = 1;
1871 }
1872
1873 /* remove the starting offset from the pool count */
1874 vmdq_i -= wx->ring_feature[RING_F_VMDQ].offset;
1875
1876 /* save features for later use */
1877 wx->ring_feature[RING_F_VMDQ].indices = vmdq_i;
1878 wx->ring_feature[RING_F_VMDQ].mask = vmdq_m;
1879
1880 /* limit RSS based on user input and save for later use */
1881 wx->ring_feature[RING_F_RSS].indices = rss_i;
1882 wx->ring_feature[RING_F_RSS].mask = rss_m;
1883
1884 wx->queues_per_pool = rss_i;/*maybe same to num_rx_queues_per_pool*/
1885 wx->num_rx_pools = vmdq_i;
1886 wx->num_rx_queues_per_pool = rss_i;
1887
1888 wx->num_rx_queues = vmdq_i * rss_i;
1889 wx->num_tx_queues = vmdq_i * rss_i;
1890
1891 return true;
1892 }
1893
1894 /**
1895 * wx_set_rss_queues: Allocate queues for RSS
1896 * @wx: board private structure to initialize
1897 *
1898 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1899 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1900 *
1901 **/
wx_set_rss_queues(struct wx * wx)1902 static void wx_set_rss_queues(struct wx *wx)
1903 {
1904 struct wx_ring_feature *f;
1905
1906 /* set mask for 16 queue limit of RSS */
1907 f = &wx->ring_feature[RING_F_RSS];
1908 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags))
1909 f->mask = WX_RSS_64Q_MASK;
1910 else
1911 f->mask = WX_RSS_8Q_MASK;
1912 f->indices = f->limit;
1913
1914 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
1915 goto out;
1916
1917 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
1918
1919 wx->ring_feature[RING_F_FDIR].indices = 1;
1920 /* Use Flow Director in addition to RSS to ensure the best
1921 * distribution of flows across cores, even when an FDIR flow
1922 * isn't matched.
1923 */
1924 if (f->indices > 1) {
1925 f = &wx->ring_feature[RING_F_FDIR];
1926
1927 f->indices = f->limit;
1928
1929 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
1930 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
1931 }
1932
1933 out:
1934 wx->num_rx_queues = f->indices;
1935 wx->num_tx_queues = f->indices;
1936 }
1937
wx_set_num_queues(struct wx * wx)1938 static void wx_set_num_queues(struct wx *wx)
1939 {
1940 /* Start with base case */
1941 wx->num_rx_queues = 1;
1942 wx->num_tx_queues = 1;
1943 wx->queues_per_pool = 1;
1944
1945 if (wx_set_vmdq_queues(wx))
1946 return;
1947
1948 wx_set_rss_queues(wx);
1949 }
1950
1951 /**
1952 * wx_acquire_msix_vectors - acquire MSI-X vectors
1953 * @wx: board private structure
1954 *
1955 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1956 * return a negative error code if unable to acquire MSI-X vectors for any
1957 * reason.
1958 */
wx_acquire_msix_vectors(struct wx * wx)1959 static int wx_acquire_msix_vectors(struct wx *wx)
1960 {
1961 struct irq_affinity affd = { .post_vectors = 1 };
1962 int nvecs, i;
1963
1964 /* We start by asking for one vector per queue pair */
1965 nvecs = max(wx->num_rx_queues, wx->num_tx_queues);
1966 nvecs = min_t(int, nvecs, num_online_cpus());
1967 nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);
1968
1969 wx->msix_q_entries = kzalloc_objs(struct msix_entry, nvecs);
1970 if (!wx->msix_q_entries)
1971 return -ENOMEM;
1972
1973 /* One for non-queue interrupts */
1974 nvecs += 1;
1975
1976 wx->msix_entry = kzalloc_objs(struct msix_entry, 1);
1977 if (!wx->msix_entry) {
1978 kfree(wx->msix_q_entries);
1979 wx->msix_q_entries = NULL;
1980 return -ENOMEM;
1981 }
1982
1983 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1984 nvecs,
1985 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1986 &affd);
1987 if (nvecs < 0) {
1988 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1989 kfree(wx->msix_q_entries);
1990 wx->msix_q_entries = NULL;
1991 kfree(wx->msix_entry);
1992 wx->msix_entry = NULL;
1993 return nvecs;
1994 }
1995
1996 nvecs -= 1;
1997 for (i = 0; i < nvecs; i++) {
1998 wx->msix_q_entries[i].entry = i;
1999 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i);
2000 }
2001
2002 wx->num_q_vectors = nvecs;
2003
2004 wx->msix_entry->entry = nvecs;
2005 wx->msix_entry->vector = pci_irq_vector(wx->pdev, nvecs);
2006
2007 if (test_bit(WX_FLAG_IRQ_VECTOR_SHARED, wx->flags)) {
2008 wx->msix_entry->entry = 0;
2009 wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);
2010 wx->msix_q_entries[0].entry = 0;
2011 wx->msix_q_entries[0].vector = pci_irq_vector(wx->pdev, 1);
2012 }
2013
2014 return 0;
2015 }
2016
2017 /**
2018 * wx_set_interrupt_capability - set MSI-X or MSI if supported
2019 * @wx: board private structure to initialize
2020 *
2021 * Attempt to configure the interrupts using the best available
2022 * capabilities of the hardware and the kernel.
2023 **/
wx_set_interrupt_capability(struct wx * wx)2024 static int wx_set_interrupt_capability(struct wx *wx)
2025 {
2026 struct pci_dev *pdev = wx->pdev;
2027 int nvecs, ret;
2028
2029 /* We will try to get MSI-X interrupts first */
2030 ret = wx_acquire_msix_vectors(wx);
2031 if (ret == 0 || (ret == -ENOMEM) || pdev->is_virtfn)
2032 return ret;
2033
2034 /* Disable VMDq support */
2035 dev_warn(&wx->pdev->dev, "Disabling VMQQ support\n");
2036 clear_bit(WX_FLAG_VMDQ_ENABLED, wx->flags);
2037
2038 /* Disable RSS */
2039 dev_warn(&wx->pdev->dev, "Disabling RSS support\n");
2040 wx->ring_feature[RING_F_RSS].limit = 1;
2041
2042 wx_set_num_queues(wx);
2043
2044 /* minmum one for queue, one for misc*/
2045 nvecs = 1;
2046 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
2047 nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);
2048 if (nvecs == 1) {
2049 if (pdev->msi_enabled)
2050 wx_err(wx, "Fallback to MSI.\n");
2051 else
2052 wx_err(wx, "Fallback to INTx.\n");
2053 } else {
2054 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);
2055 return nvecs;
2056 }
2057
2058 pdev->irq = pci_irq_vector(pdev, 0);
2059 wx->num_q_vectors = 1;
2060
2061 return 0;
2062 }
2063
wx_cache_ring_vmdq(struct wx * wx)2064 static bool wx_cache_ring_vmdq(struct wx *wx)
2065 {
2066 struct wx_ring_feature *vmdq = &wx->ring_feature[RING_F_VMDQ];
2067 struct wx_ring_feature *rss = &wx->ring_feature[RING_F_RSS];
2068 u16 reg_idx;
2069 int i;
2070
2071 /* only proceed if VMDq is enabled */
2072 if (!test_bit(WX_FLAG_VMDQ_ENABLED, wx->flags))
2073 return false;
2074
2075 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
2076 /* start at VMDq register offset for SR-IOV enabled setups */
2077 reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
2078 for (i = 0; i < wx->num_rx_queues; i++, reg_idx++) {
2079 /* If we are greater than indices move to next pool */
2080 if ((reg_idx & ~vmdq->mask) >= rss->indices)
2081 reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
2082 wx->rx_ring[i]->reg_idx = reg_idx;
2083 }
2084 reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
2085 for (i = 0; i < wx->num_tx_queues; i++, reg_idx++) {
2086 /* If we are greater than indices move to next pool */
2087 if ((reg_idx & rss->mask) >= rss->indices)
2088 reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
2089 wx->tx_ring[i]->reg_idx = reg_idx;
2090 }
2091 } else {
2092 /* start at VMDq register offset for SR-IOV enabled setups */
2093 reg_idx = vmdq->offset;
2094 for (i = 0; i < wx->num_rx_queues; i++)
2095 /* If we are greater than indices move to next pool */
2096 wx->rx_ring[i]->reg_idx = reg_idx + i;
2097
2098 reg_idx = vmdq->offset;
2099 for (i = 0; i < wx->num_tx_queues; i++)
2100 /* If we are greater than indices move to next pool */
2101 wx->tx_ring[i]->reg_idx = reg_idx + i;
2102 }
2103
2104 return true;
2105 }
2106
2107 /**
2108 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
2109 * @wx: board private structure to initialize
2110 *
2111 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
2112 *
2113 **/
wx_cache_ring_rss(struct wx * wx)2114 static void wx_cache_ring_rss(struct wx *wx)
2115 {
2116 u16 i;
2117
2118 if (wx_cache_ring_vmdq(wx))
2119 return;
2120
2121 for (i = 0; i < wx->num_rx_queues; i++)
2122 wx->rx_ring[i]->reg_idx = i;
2123
2124 for (i = 0; i < wx->num_tx_queues; i++)
2125 wx->tx_ring[i]->reg_idx = i;
2126 }
2127
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)2128 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
2129 {
2130 ring->next = head->ring;
2131 head->ring = ring;
2132 head->count++;
2133 }
2134
2135 /**
2136 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
2137 * @wx: board private structure to initialize
2138 * @v_count: q_vectors allocated on wx, used for ring interleaving
2139 * @v_idx: index of vector in wx struct
2140 * @txr_count: total number of Tx rings to allocate
2141 * @txr_idx: index of first Tx ring to allocate
2142 * @rxr_count: total number of Rx rings to allocate
2143 * @rxr_idx: index of first Rx ring to allocate
2144 *
2145 * We allocate one q_vector. If allocation fails we return -ENOMEM.
2146 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)2147 static int wx_alloc_q_vector(struct wx *wx,
2148 unsigned int v_count, unsigned int v_idx,
2149 unsigned int txr_count, unsigned int txr_idx,
2150 unsigned int rxr_count, unsigned int rxr_idx)
2151 {
2152 struct wx_q_vector *q_vector;
2153 int ring_count, default_itr;
2154 struct wx_ring *ring;
2155
2156 /* note this will allocate space for the ring structure as well! */
2157 ring_count = txr_count + rxr_count;
2158
2159 q_vector = kzalloc_flex(*q_vector, ring, ring_count);
2160 if (!q_vector)
2161 return -ENOMEM;
2162
2163 /* initialize NAPI */
2164 netif_napi_add(wx->netdev, &q_vector->napi,
2165 wx_poll);
2166
2167 /* tie q_vector and wx together */
2168 wx->q_vector[v_idx] = q_vector;
2169 q_vector->wx = wx;
2170 q_vector->v_idx = v_idx;
2171 if (cpu_online(v_idx))
2172 q_vector->numa_node = cpu_to_node(v_idx);
2173
2174 /* initialize pointer to rings */
2175 ring = q_vector->ring;
2176
2177 switch (wx->mac.type) {
2178 case wx_mac_sp:
2179 case wx_mac_aml:
2180 case wx_mac_aml40:
2181 default_itr = WX_12K_ITR;
2182 break;
2183 default:
2184 default_itr = WX_7K_ITR;
2185 break;
2186 }
2187
2188 /* initialize ITR */
2189 if (txr_count && !rxr_count)
2190 /* tx only vector */
2191 q_vector->itr = wx->tx_itr_setting ?
2192 default_itr : wx->tx_itr_setting;
2193 else
2194 /* rx or rx/tx vector */
2195 q_vector->itr = wx->rx_itr_setting ?
2196 default_itr : wx->rx_itr_setting;
2197
2198 while (txr_count) {
2199 /* assign generic ring traits */
2200 ring->dev = &wx->pdev->dev;
2201 ring->netdev = wx->netdev;
2202
2203 /* configure backlink on ring */
2204 ring->q_vector = q_vector;
2205
2206 /* update q_vector Tx values */
2207 wx_add_ring(ring, &q_vector->tx);
2208
2209 /* apply Tx specific ring traits */
2210 ring->count = wx->tx_ring_count;
2211
2212 ring->queue_index = txr_idx;
2213
2214 /* assign ring to wx */
2215 wx->tx_ring[txr_idx] = ring;
2216
2217 /* update count and index */
2218 txr_count--;
2219 txr_idx += v_count;
2220
2221 /* push pointer to next ring */
2222 ring++;
2223 }
2224
2225 while (rxr_count) {
2226 /* assign generic ring traits */
2227 ring->dev = &wx->pdev->dev;
2228 ring->netdev = wx->netdev;
2229
2230 /* configure backlink on ring */
2231 ring->q_vector = q_vector;
2232
2233 /* update q_vector Rx values */
2234 wx_add_ring(ring, &q_vector->rx);
2235
2236 /* apply Rx specific ring traits */
2237 ring->count = wx->rx_ring_count;
2238 ring->queue_index = rxr_idx;
2239
2240 /* assign ring to wx */
2241 wx->rx_ring[rxr_idx] = ring;
2242
2243 /* update count and index */
2244 rxr_count--;
2245 rxr_idx += v_count;
2246
2247 /* push pointer to next ring */
2248 ring++;
2249 }
2250
2251 return 0;
2252 }
2253
2254 /**
2255 * wx_free_q_vector - Free memory allocated for specific interrupt vector
2256 * @wx: board private structure to initialize
2257 * @v_idx: Index of vector to be freed
2258 *
2259 * This function frees the memory allocated to the q_vector. In addition if
2260 * NAPI is enabled it will delete any references to the NAPI struct prior
2261 * to freeing the q_vector.
2262 **/
wx_free_q_vector(struct wx * wx,int v_idx)2263 static void wx_free_q_vector(struct wx *wx, int v_idx)
2264 {
2265 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2266 struct wx_ring *ring;
2267
2268 wx_for_each_ring(ring, q_vector->tx)
2269 wx->tx_ring[ring->queue_index] = NULL;
2270
2271 wx_for_each_ring(ring, q_vector->rx)
2272 wx->rx_ring[ring->queue_index] = NULL;
2273
2274 wx->q_vector[v_idx] = NULL;
2275 netif_napi_del(&q_vector->napi);
2276 kfree_rcu(q_vector, rcu);
2277 }
2278
2279 /**
2280 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
2281 * @wx: board private structure to initialize
2282 *
2283 * We allocate one q_vector per queue interrupt. If allocation fails we
2284 * return -ENOMEM.
2285 **/
wx_alloc_q_vectors(struct wx * wx)2286 static int wx_alloc_q_vectors(struct wx *wx)
2287 {
2288 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
2289 unsigned int rxr_remaining = wx->num_rx_queues;
2290 unsigned int txr_remaining = wx->num_tx_queues;
2291 unsigned int q_vectors = wx->num_q_vectors;
2292 int rqpv, tqpv;
2293 int err;
2294
2295 for (; v_idx < q_vectors; v_idx++) {
2296 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
2297 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
2298 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
2299 tqpv, txr_idx,
2300 rqpv, rxr_idx);
2301
2302 if (err)
2303 goto err_out;
2304
2305 /* update counts and index */
2306 rxr_remaining -= rqpv;
2307 txr_remaining -= tqpv;
2308 rxr_idx++;
2309 txr_idx++;
2310 }
2311
2312 return 0;
2313
2314 err_out:
2315 wx->num_tx_queues = 0;
2316 wx->num_rx_queues = 0;
2317 wx->num_q_vectors = 0;
2318
2319 while (v_idx--)
2320 wx_free_q_vector(wx, v_idx);
2321
2322 return -ENOMEM;
2323 }
2324
2325 /**
2326 * wx_free_q_vectors - Free memory allocated for interrupt vectors
2327 * @wx: board private structure to initialize
2328 *
2329 * This function frees the memory allocated to the q_vectors. In addition if
2330 * NAPI is enabled it will delete any references to the NAPI struct prior
2331 * to freeing the q_vector.
2332 **/
wx_free_q_vectors(struct wx * wx)2333 static void wx_free_q_vectors(struct wx *wx)
2334 {
2335 int v_idx = wx->num_q_vectors;
2336
2337 wx->num_tx_queues = 0;
2338 wx->num_rx_queues = 0;
2339 wx->num_q_vectors = 0;
2340
2341 while (v_idx--)
2342 wx_free_q_vector(wx, v_idx);
2343 }
2344
wx_reset_interrupt_capability(struct wx * wx)2345 void wx_reset_interrupt_capability(struct wx *wx)
2346 {
2347 struct pci_dev *pdev = wx->pdev;
2348
2349 if (!pdev->msi_enabled && !pdev->msix_enabled)
2350 return;
2351
2352 if (pdev->msix_enabled) {
2353 kfree(wx->msix_q_entries);
2354 wx->msix_q_entries = NULL;
2355 kfree(wx->msix_entry);
2356 wx->msix_entry = NULL;
2357 }
2358 pci_free_irq_vectors(wx->pdev);
2359 }
2360 EXPORT_SYMBOL(wx_reset_interrupt_capability);
2361
2362 /**
2363 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
2364 * @wx: board private structure to clear interrupt scheme on
2365 *
2366 * We go through and clear interrupt specific resources and reset the structure
2367 * to pre-load conditions
2368 **/
wx_clear_interrupt_scheme(struct wx * wx)2369 void wx_clear_interrupt_scheme(struct wx *wx)
2370 {
2371 wx_free_q_vectors(wx);
2372 wx_reset_interrupt_capability(wx);
2373 }
2374 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
2375
wx_init_interrupt_scheme(struct wx * wx)2376 int wx_init_interrupt_scheme(struct wx *wx)
2377 {
2378 int ret;
2379
2380 /* Number of supported queues */
2381 if (wx->pdev->is_virtfn) {
2382 if (wx->set_num_queues)
2383 wx->set_num_queues(wx);
2384 } else {
2385 wx_set_num_queues(wx);
2386 }
2387
2388 /* Set interrupt mode */
2389 ret = wx_set_interrupt_capability(wx);
2390 if (ret) {
2391 wx_err(wx, "Allocate irq vectors for failed.\n");
2392 return ret;
2393 }
2394
2395 /* Allocate memory for queues */
2396 ret = wx_alloc_q_vectors(wx);
2397 if (ret) {
2398 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
2399 wx_reset_interrupt_capability(wx);
2400 return ret;
2401 }
2402
2403 wx_cache_ring_rss(wx);
2404
2405 set_bit(WX_STATE_DOWN, wx->state);
2406
2407 return 0;
2408 }
2409 EXPORT_SYMBOL(wx_init_interrupt_scheme);
2410
wx_msix_clean_rings(int __always_unused irq,void * data)2411 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
2412 {
2413 struct wx_q_vector *q_vector = data;
2414
2415 /* EIAM disabled interrupts (on this vector) for us */
2416 if (q_vector->rx.ring || q_vector->tx.ring) {
2417 napi_schedule_irqoff(&q_vector->napi);
2418 q_vector->total_events++;
2419 }
2420
2421 return IRQ_HANDLED;
2422 }
2423 EXPORT_SYMBOL(wx_msix_clean_rings);
2424
wx_free_irq(struct wx * wx)2425 void wx_free_irq(struct wx *wx)
2426 {
2427 struct pci_dev *pdev = wx->pdev;
2428 int vector;
2429
2430 if (!(pdev->msix_enabled)) {
2431 if (!wx->misc_irq_domain)
2432 free_irq(pdev->irq, wx);
2433 return;
2434 }
2435
2436 for (vector = 0; vector < wx->num_q_vectors; vector++) {
2437 struct wx_q_vector *q_vector = wx->q_vector[vector];
2438 struct msix_entry *entry = &wx->msix_q_entries[vector];
2439
2440 /* free only the irqs that were actually requested */
2441 if (!q_vector->rx.ring && !q_vector->tx.ring)
2442 continue;
2443
2444 free_irq(entry->vector, q_vector);
2445 }
2446
2447 if (!wx->misc_irq_domain)
2448 free_irq(wx->msix_entry->vector, wx);
2449 }
2450 EXPORT_SYMBOL(wx_free_irq);
2451
2452 /**
2453 * wx_setup_isb_resources - allocate interrupt status resources
2454 * @wx: board private structure
2455 *
2456 * Return 0 on success, negative on failure
2457 **/
wx_setup_isb_resources(struct wx * wx)2458 int wx_setup_isb_resources(struct wx *wx)
2459 {
2460 struct pci_dev *pdev = wx->pdev;
2461
2462 if (wx->isb_mem)
2463 return 0;
2464
2465 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2466 sizeof(u32) * 4,
2467 &wx->isb_dma,
2468 GFP_KERNEL);
2469 if (!wx->isb_mem) {
2470 wx_err(wx, "Alloc isb_mem failed\n");
2471 return -ENOMEM;
2472 }
2473
2474 return 0;
2475 }
2476 EXPORT_SYMBOL(wx_setup_isb_resources);
2477
2478 /**
2479 * wx_free_isb_resources - allocate all queues Rx resources
2480 * @wx: board private structure
2481 *
2482 * Return 0 on success, negative on failure
2483 **/
wx_free_isb_resources(struct wx * wx)2484 void wx_free_isb_resources(struct wx *wx)
2485 {
2486 struct pci_dev *pdev = wx->pdev;
2487
2488 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2489 wx->isb_mem, wx->isb_dma);
2490 wx->isb_mem = NULL;
2491 }
2492 EXPORT_SYMBOL(wx_free_isb_resources);
2493
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2494 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2495 {
2496 u32 cur_tag = 0;
2497
2498 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2499 wx->isb_tag[idx] = cur_tag;
2500
2501 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2502 }
2503 EXPORT_SYMBOL(wx_misc_isb);
2504
2505 /**
2506 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2507 * @wx: pointer to wx struct
2508 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2509 * @queue: queue to map the corresponding interrupt to
2510 * @msix_vector: the vector to map to the corresponding queue
2511 *
2512 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2513 static void wx_set_ivar(struct wx *wx, s8 direction,
2514 u16 queue, u16 msix_vector)
2515 {
2516 u32 ivar, index;
2517
2518 if (direction == -1) {
2519 /* other causes */
2520 if (test_bit(WX_FLAG_IRQ_VECTOR_SHARED, wx->flags))
2521 msix_vector = 0;
2522 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2523 index = 0;
2524 ivar = rd32(wx, WX_PX_MISC_IVAR);
2525 ivar &= ~(0xFF << index);
2526 ivar |= (msix_vector << index);
2527 wr32(wx, WX_PX_MISC_IVAR, ivar);
2528 } else {
2529 /* tx or rx causes */
2530 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2531 index = ((16 * (queue & 1)) + (8 * direction));
2532 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2533 ivar &= ~(0xFF << index);
2534 ivar |= (msix_vector << index);
2535 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2536 }
2537 }
2538
2539 /**
2540 * wx_write_eitr - write EITR register in hardware specific way
2541 * @q_vector: structure containing interrupt and ring information
2542 *
2543 * This function is made to be called by ethtool and by the driver
2544 * when it needs to update EITR registers at runtime. Hardware
2545 * specific quirks/differences are taken care of here.
2546 */
wx_write_eitr(struct wx_q_vector * q_vector)2547 void wx_write_eitr(struct wx_q_vector *q_vector)
2548 {
2549 struct wx *wx = q_vector->wx;
2550 int v_idx = q_vector->v_idx;
2551 u32 itr_reg;
2552
2553 switch (wx->mac.type) {
2554 case wx_mac_sp:
2555 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2556 break;
2557 case wx_mac_aml:
2558 case wx_mac_aml40:
2559 itr_reg = (q_vector->itr >> 3) & WX_AML_MAX_EITR;
2560 break;
2561 default:
2562 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2563 break;
2564 }
2565
2566 itr_reg |= WX_PX_ITR_CNT_WDIS;
2567
2568 wr32(wx, WX_PX_ITR(v_idx), itr_reg);
2569 }
2570
2571 /**
2572 * wx_configure_vectors - Configure vectors for hardware
2573 * @wx: board private structure
2574 *
2575 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx
2576 * interrupts.
2577 **/
wx_configure_vectors(struct wx * wx)2578 void wx_configure_vectors(struct wx *wx)
2579 {
2580 struct pci_dev *pdev = wx->pdev;
2581 u32 eitrsel = 0;
2582 u16 v_idx, i;
2583
2584 if (pdev->msix_enabled) {
2585 /* Populate MSIX to EITR Select */
2586 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
2587 if (wx->num_vfs >= 32)
2588 eitrsel = BIT(wx->num_vfs % 32) - 1;
2589 } else {
2590 for (i = 0; i < wx->num_vfs; i++)
2591 eitrsel |= BIT(i);
2592 }
2593 wr32(wx, WX_PX_ITRSEL, eitrsel);
2594 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2595 * this saves a register write for every interrupt
2596 */
2597 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2598 } else {
2599 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2600 * specifically only auto mask tx and rx interrupts.
2601 */
2602 wr32(wx, WX_PX_GPIE, 0);
2603 }
2604
2605 /* Populate the IVAR table and set the ITR values to the
2606 * corresponding register.
2607 */
2608 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2609 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2610 struct wx_ring *ring;
2611
2612 wx_for_each_ring(ring, q_vector->rx)
2613 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2614
2615 wx_for_each_ring(ring, q_vector->tx)
2616 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2617
2618 wx_write_eitr(q_vector);
2619 }
2620
2621 wx_set_ivar(wx, -1, 0, v_idx);
2622 if (pdev->msix_enabled)
2623 wr32(wx, WX_PX_ITR(v_idx), 1950);
2624 }
2625 EXPORT_SYMBOL(wx_configure_vectors);
2626
2627 /**
2628 * wx_clean_rx_ring - Free Rx Buffers per Queue
2629 * @rx_ring: ring to free buffers from
2630 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2631 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2632 {
2633 struct wx_rx_buffer *rx_buffer;
2634 u16 i = rx_ring->next_to_clean;
2635
2636 rx_buffer = &rx_ring->rx_buffer_info[i];
2637
2638 /* Free all the Rx ring sk_buffs */
2639 while (i != rx_ring->next_to_alloc) {
2640 if (rx_buffer->skb) {
2641 struct sk_buff *skb = rx_buffer->skb;
2642
2643 dev_kfree_skb(skb);
2644 }
2645
2646 /* Invalidate cache lines that may have been written to by
2647 * device so that we avoid corrupting memory.
2648 */
2649 dma_sync_single_range_for_cpu(rx_ring->dev,
2650 rx_buffer->dma,
2651 rx_buffer->page_offset,
2652 rx_ring->rx_buf_len,
2653 DMA_FROM_DEVICE);
2654
2655 /* free resources associated with mapping */
2656 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2657
2658 i++;
2659 rx_buffer++;
2660 if (i == rx_ring->count) {
2661 i = 0;
2662 rx_buffer = rx_ring->rx_buffer_info;
2663 }
2664 }
2665
2666 /* Zero out the descriptor ring */
2667 memset(rx_ring->desc, 0, rx_ring->size);
2668
2669 rx_ring->next_to_alloc = 0;
2670 rx_ring->next_to_clean = 0;
2671 rx_ring->next_to_use = 0;
2672 }
2673
2674 /**
2675 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2676 * @wx: board private structure
2677 **/
wx_clean_all_rx_rings(struct wx * wx)2678 void wx_clean_all_rx_rings(struct wx *wx)
2679 {
2680 int i;
2681
2682 for (i = 0; i < wx->num_rx_queues; i++)
2683 wx_clean_rx_ring(wx->rx_ring[i]);
2684 }
2685 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2686
2687 /**
2688 * wx_free_rx_resources - Free Rx Resources
2689 * @rx_ring: ring to clean the resources from
2690 *
2691 * Free all receive software resources
2692 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2693 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2694 {
2695 wx_clean_rx_ring(rx_ring);
2696 kvfree(rx_ring->rx_buffer_info);
2697 rx_ring->rx_buffer_info = NULL;
2698
2699 /* if not set, then don't free */
2700 if (!rx_ring->desc)
2701 return;
2702
2703 dma_free_coherent(rx_ring->dev, rx_ring->size,
2704 rx_ring->desc, rx_ring->dma);
2705
2706 rx_ring->desc = NULL;
2707
2708 if (rx_ring->page_pool) {
2709 page_pool_destroy(rx_ring->page_pool);
2710 rx_ring->page_pool = NULL;
2711 }
2712 }
2713
2714 /**
2715 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2716 * @wx: pointer to hardware structure
2717 *
2718 * Free all receive software resources
2719 **/
wx_free_all_rx_resources(struct wx * wx)2720 static void wx_free_all_rx_resources(struct wx *wx)
2721 {
2722 int i;
2723
2724 for (i = 0; i < wx->num_rx_queues; i++)
2725 wx_free_rx_resources(wx->rx_ring[i]);
2726 }
2727
2728 /**
2729 * wx_clean_tx_ring - Free Tx Buffers
2730 * @tx_ring: ring to be cleaned
2731 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2732 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2733 {
2734 struct wx_tx_buffer *tx_buffer;
2735 u16 i = tx_ring->next_to_clean;
2736
2737 tx_buffer = &tx_ring->tx_buffer_info[i];
2738
2739 while (i != tx_ring->next_to_use) {
2740 union wx_tx_desc *eop_desc, *tx_desc;
2741
2742 /* Free all the Tx ring sk_buffs */
2743 dev_kfree_skb_any(tx_buffer->skb);
2744
2745 /* unmap skb header data */
2746 dma_unmap_single(tx_ring->dev,
2747 dma_unmap_addr(tx_buffer, dma),
2748 dma_unmap_len(tx_buffer, len),
2749 DMA_TO_DEVICE);
2750
2751 /* check for eop_desc to determine the end of the packet */
2752 eop_desc = tx_buffer->next_to_watch;
2753 tx_desc = WX_TX_DESC(tx_ring, i);
2754
2755 /* unmap remaining buffers */
2756 while (tx_desc != eop_desc) {
2757 tx_buffer++;
2758 tx_desc++;
2759 i++;
2760 if (unlikely(i == tx_ring->count)) {
2761 i = 0;
2762 tx_buffer = tx_ring->tx_buffer_info;
2763 tx_desc = WX_TX_DESC(tx_ring, 0);
2764 }
2765
2766 /* unmap any remaining paged data */
2767 if (dma_unmap_len(tx_buffer, len))
2768 dma_unmap_page(tx_ring->dev,
2769 dma_unmap_addr(tx_buffer, dma),
2770 dma_unmap_len(tx_buffer, len),
2771 DMA_TO_DEVICE);
2772 }
2773
2774 /* move us one more past the eop_desc for start of next pkt */
2775 tx_buffer++;
2776 i++;
2777 if (unlikely(i == tx_ring->count)) {
2778 i = 0;
2779 tx_buffer = tx_ring->tx_buffer_info;
2780 }
2781 }
2782
2783 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2784
2785 /* reset next_to_use and next_to_clean */
2786 tx_ring->next_to_use = 0;
2787 tx_ring->next_to_clean = 0;
2788 }
2789
2790 /**
2791 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2792 * @wx: board private structure
2793 **/
wx_clean_all_tx_rings(struct wx * wx)2794 void wx_clean_all_tx_rings(struct wx *wx)
2795 {
2796 int i;
2797
2798 for (i = 0; i < wx->num_tx_queues; i++)
2799 wx_clean_tx_ring(wx->tx_ring[i]);
2800 }
2801 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2802
wx_free_headwb_resources(struct wx_ring * tx_ring)2803 static void wx_free_headwb_resources(struct wx_ring *tx_ring)
2804 {
2805 if (!tx_ring->headwb_mem)
2806 return;
2807
2808 dma_free_coherent(tx_ring->dev, sizeof(u32),
2809 tx_ring->headwb_mem, tx_ring->headwb_dma);
2810 tx_ring->headwb_mem = NULL;
2811 }
2812
2813 /**
2814 * wx_free_tx_resources - Free Tx Resources per Queue
2815 * @tx_ring: Tx descriptor ring for a specific queue
2816 *
2817 * Free all transmit software resources
2818 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2819 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2820 {
2821 wx_clean_tx_ring(tx_ring);
2822 kvfree(tx_ring->tx_buffer_info);
2823 tx_ring->tx_buffer_info = NULL;
2824
2825 /* if not set, then don't free */
2826 if (!tx_ring->desc)
2827 return;
2828
2829 dma_free_coherent(tx_ring->dev, tx_ring->size,
2830 tx_ring->desc, tx_ring->dma);
2831 tx_ring->desc = NULL;
2832
2833 wx_free_headwb_resources(tx_ring);
2834 }
2835
2836 /**
2837 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2838 * @wx: pointer to hardware structure
2839 *
2840 * Free all transmit software resources
2841 **/
wx_free_all_tx_resources(struct wx * wx)2842 static void wx_free_all_tx_resources(struct wx *wx)
2843 {
2844 int i;
2845
2846 for (i = 0; i < wx->num_tx_queues; i++)
2847 wx_free_tx_resources(wx->tx_ring[i]);
2848 }
2849
wx_free_resources(struct wx * wx)2850 void wx_free_resources(struct wx *wx)
2851 {
2852 wx_free_all_rx_resources(wx);
2853 wx_free_all_tx_resources(wx);
2854 }
2855 EXPORT_SYMBOL(wx_free_resources);
2856
wx_alloc_page_pool(struct wx_ring * rx_ring)2857 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2858 {
2859 int ret = 0;
2860
2861 struct page_pool_params pp_params = {
2862 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2863 .order = wx_rx_pg_order(rx_ring),
2864 .pool_size = rx_ring->count * rx_ring->rx_buf_len /
2865 wx_rx_pg_size(rx_ring),
2866 .nid = dev_to_node(rx_ring->dev),
2867 .dev = rx_ring->dev,
2868 .dma_dir = DMA_FROM_DEVICE,
2869 .offset = 0,
2870 .max_len = wx_rx_pg_size(rx_ring),
2871 };
2872
2873 rx_ring->page_pool = page_pool_create(&pp_params);
2874 if (IS_ERR(rx_ring->page_pool)) {
2875 ret = PTR_ERR(rx_ring->page_pool);
2876 rx_ring->page_pool = NULL;
2877 }
2878
2879 return ret;
2880 }
2881
2882 /**
2883 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2884 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2885 *
2886 * Returns 0 on success, negative on failure
2887 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2888 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2889 {
2890 struct device *dev = rx_ring->dev;
2891 int orig_node = dev_to_node(dev);
2892 int numa_node = NUMA_NO_NODE;
2893 int size, ret;
2894
2895 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2896
2897 if (rx_ring->q_vector)
2898 numa_node = rx_ring->q_vector->numa_node;
2899
2900 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2901 if (!rx_ring->rx_buffer_info)
2902 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2903 if (!rx_ring->rx_buffer_info)
2904 goto err;
2905
2906 /* Round up to nearest 4K */
2907 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2908 rx_ring->size = ALIGN(rx_ring->size, 4096);
2909
2910 set_dev_node(dev, numa_node);
2911 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2912 &rx_ring->dma, GFP_KERNEL);
2913 if (!rx_ring->desc) {
2914 set_dev_node(dev, orig_node);
2915 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2916 &rx_ring->dma, GFP_KERNEL);
2917 }
2918
2919 if (!rx_ring->desc)
2920 goto err;
2921
2922 rx_ring->next_to_clean = 0;
2923 rx_ring->next_to_use = 0;
2924
2925 ret = wx_alloc_page_pool(rx_ring);
2926 if (ret < 0) {
2927 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2928 goto err_desc;
2929 }
2930
2931 return 0;
2932
2933 err_desc:
2934 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2935 err:
2936 kvfree(rx_ring->rx_buffer_info);
2937 rx_ring->rx_buffer_info = NULL;
2938 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2939 return -ENOMEM;
2940 }
2941
2942 /**
2943 * wx_setup_all_rx_resources - allocate all queues Rx resources
2944 * @wx: pointer to hardware structure
2945 *
2946 * If this function returns with an error, then it's possible one or
2947 * more of the rings is populated (while the rest are not). It is the
2948 * callers duty to clean those orphaned rings.
2949 *
2950 * Return 0 on success, negative on failure
2951 **/
wx_setup_all_rx_resources(struct wx * wx)2952 static int wx_setup_all_rx_resources(struct wx *wx)
2953 {
2954 int i, err = 0;
2955
2956 for (i = 0; i < wx->num_rx_queues; i++) {
2957 err = wx_setup_rx_resources(wx->rx_ring[i]);
2958 if (!err)
2959 continue;
2960
2961 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2962 goto err_setup_rx;
2963 }
2964
2965 return 0;
2966 err_setup_rx:
2967 /* rewind the index freeing the rings as we go */
2968 while (i--)
2969 wx_free_rx_resources(wx->rx_ring[i]);
2970 return err;
2971 }
2972
wx_setup_headwb_resources(struct wx_ring * tx_ring)2973 static void wx_setup_headwb_resources(struct wx_ring *tx_ring)
2974 {
2975 struct wx *wx = netdev_priv(tx_ring->netdev);
2976
2977 if (!test_bit(WX_FLAG_TXHEAD_WB_ENABLED, wx->flags))
2978 return;
2979
2980 if (!tx_ring->q_vector)
2981 return;
2982
2983 tx_ring->headwb_mem = dma_alloc_coherent(tx_ring->dev,
2984 sizeof(u32),
2985 &tx_ring->headwb_dma,
2986 GFP_KERNEL);
2987 if (!tx_ring->headwb_mem)
2988 dev_info(tx_ring->dev, "Allocate headwb memory failed, disable it\n");
2989 }
2990
2991 /**
2992 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2993 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2994 *
2995 * Return 0 on success, negative on failure
2996 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2997 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2998 {
2999 struct device *dev = tx_ring->dev;
3000 int orig_node = dev_to_node(dev);
3001 int numa_node = NUMA_NO_NODE;
3002 int size;
3003
3004 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
3005
3006 if (tx_ring->q_vector)
3007 numa_node = tx_ring->q_vector->numa_node;
3008
3009 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
3010 if (!tx_ring->tx_buffer_info)
3011 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
3012 if (!tx_ring->tx_buffer_info)
3013 goto err;
3014
3015 /* round up to nearest 4K */
3016 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
3017 tx_ring->size = ALIGN(tx_ring->size, 4096);
3018
3019 set_dev_node(dev, numa_node);
3020 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3021 &tx_ring->dma, GFP_KERNEL);
3022 if (!tx_ring->desc) {
3023 set_dev_node(dev, orig_node);
3024 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3025 &tx_ring->dma, GFP_KERNEL);
3026 }
3027
3028 if (!tx_ring->desc)
3029 goto err;
3030
3031 wx_setup_headwb_resources(tx_ring);
3032
3033 tx_ring->next_to_use = 0;
3034 tx_ring->next_to_clean = 0;
3035
3036 return 0;
3037
3038 err:
3039 kvfree(tx_ring->tx_buffer_info);
3040 tx_ring->tx_buffer_info = NULL;
3041 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
3042 return -ENOMEM;
3043 }
3044
3045 /**
3046 * wx_setup_all_tx_resources - allocate all queues Tx resources
3047 * @wx: pointer to private structure
3048 *
3049 * If this function returns with an error, then it's possible one or
3050 * more of the rings is populated (while the rest are not). It is the
3051 * callers duty to clean those orphaned rings.
3052 *
3053 * Return 0 on success, negative on failure
3054 **/
wx_setup_all_tx_resources(struct wx * wx)3055 static int wx_setup_all_tx_resources(struct wx *wx)
3056 {
3057 int i, err = 0;
3058
3059 for (i = 0; i < wx->num_tx_queues; i++) {
3060 err = wx_setup_tx_resources(wx->tx_ring[i]);
3061 if (!err)
3062 continue;
3063
3064 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
3065 goto err_setup_tx;
3066 }
3067
3068 return 0;
3069 err_setup_tx:
3070 /* rewind the index freeing the rings as we go */
3071 while (i--)
3072 wx_free_tx_resources(wx->tx_ring[i]);
3073 return err;
3074 }
3075
wx_setup_resources(struct wx * wx)3076 int wx_setup_resources(struct wx *wx)
3077 {
3078 int err;
3079
3080 /* allocate transmit descriptors */
3081 err = wx_setup_all_tx_resources(wx);
3082 if (err)
3083 return err;
3084
3085 /* allocate receive descriptors */
3086 err = wx_setup_all_rx_resources(wx);
3087 if (err)
3088 goto err_free_tx;
3089
3090 err = wx_setup_isb_resources(wx);
3091 if (err)
3092 goto err_free_rx;
3093
3094 return 0;
3095
3096 err_free_rx:
3097 wx_free_all_rx_resources(wx);
3098 err_free_tx:
3099 wx_free_all_tx_resources(wx);
3100
3101 return err;
3102 }
3103 EXPORT_SYMBOL(wx_setup_resources);
3104
3105 /**
3106 * wx_get_stats64 - Get System Network Statistics
3107 * @netdev: network interface device structure
3108 * @stats: storage space for 64bit statistics
3109 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)3110 void wx_get_stats64(struct net_device *netdev,
3111 struct rtnl_link_stats64 *stats)
3112 {
3113 struct wx *wx = netdev_priv(netdev);
3114 struct wx_hw_stats *hwstats;
3115 int i;
3116
3117 wx_update_stats(wx);
3118
3119 rcu_read_lock();
3120 for (i = 0; i < wx->num_rx_queues; i++) {
3121 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
3122 u64 bytes, packets;
3123 unsigned int start;
3124
3125 if (ring) {
3126 do {
3127 start = u64_stats_fetch_begin(&ring->syncp);
3128 packets = ring->stats.packets;
3129 bytes = ring->stats.bytes;
3130 } while (u64_stats_fetch_retry(&ring->syncp, start));
3131 stats->rx_packets += packets;
3132 stats->rx_bytes += bytes;
3133 }
3134 }
3135
3136 for (i = 0; i < wx->num_tx_queues; i++) {
3137 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
3138 u64 bytes, packets;
3139 unsigned int start;
3140
3141 if (ring) {
3142 do {
3143 start = u64_stats_fetch_begin(&ring->syncp);
3144 packets = ring->stats.packets;
3145 bytes = ring->stats.bytes;
3146 } while (u64_stats_fetch_retry(&ring->syncp,
3147 start));
3148 stats->tx_packets += packets;
3149 stats->tx_bytes += bytes;
3150 }
3151 }
3152
3153 rcu_read_unlock();
3154
3155 hwstats = &wx->stats;
3156 stats->rx_errors = hwstats->crcerrs + hwstats->rlec;
3157 stats->multicast = hwstats->qmprc;
3158 stats->rx_length_errors = hwstats->rlec;
3159 stats->rx_crc_errors = hwstats->crcerrs;
3160 }
3161 EXPORT_SYMBOL(wx_get_stats64);
3162
wx_set_features(struct net_device * netdev,netdev_features_t features)3163 int wx_set_features(struct net_device *netdev, netdev_features_t features)
3164 {
3165 netdev_features_t changed = netdev->features ^ features;
3166 struct wx *wx = netdev_priv(netdev);
3167 bool need_reset = false;
3168
3169 wx->rss_enabled = !!(features & NETIF_F_RXHASH);
3170 wx_enable_rss(wx, wx->rss_enabled);
3171
3172 netdev->features = features;
3173
3174 if (changed & NETIF_F_HW_VLAN_CTAG_RX && wx->do_reset)
3175 wx->do_reset(netdev, true);
3176 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))
3177 wx_set_rx_mode(netdev);
3178
3179 if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) {
3180 if (!(features & NETIF_F_LRO)) {
3181 if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags))
3182 need_reset = true;
3183 clear_bit(WX_FLAG_RSC_ENABLED, wx->flags);
3184 } else if (!(test_bit(WX_FLAG_RSC_ENABLED, wx->flags))) {
3185 if (wx->rx_itr_setting == 1 ||
3186 wx->rx_itr_setting > WX_MIN_RSC_ITR) {
3187 set_bit(WX_FLAG_RSC_ENABLED, wx->flags);
3188 need_reset = true;
3189 } else if (changed & NETIF_F_LRO) {
3190 dev_info(&wx->pdev->dev,
3191 "rx-usecs set too low, disable RSC\n");
3192 }
3193 }
3194 }
3195
3196 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
3197 goto out;
3198
3199 /* Check if Flow Director n-tuple support was enabled or disabled. If
3200 * the state changed, we need to reset.
3201 */
3202 switch (features & NETIF_F_NTUPLE) {
3203 case NETIF_F_NTUPLE:
3204 /* turn off ATR, enable perfect filters and reset */
3205 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
3206 need_reset = true;
3207
3208 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
3209 break;
3210 default:
3211 /* turn off perfect filters, enable ATR and reset */
3212 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))
3213 need_reset = true;
3214
3215 /* We cannot enable ATR if RSS is disabled */
3216 if (wx->ring_feature[RING_F_RSS].limit <= 1)
3217 break;
3218
3219 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
3220 break;
3221 }
3222
3223 out:
3224 if (need_reset && wx->do_reset)
3225 wx->do_reset(netdev, true);
3226
3227 return 0;
3228 }
3229 EXPORT_SYMBOL(wx_set_features);
3230
3231 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
3232 NETIF_F_HW_VLAN_STAG_RX)
3233
3234 #define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \
3235 NETIF_F_HW_VLAN_STAG_TX)
3236
3237 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \
3238 NETIF_F_HW_VLAN_STAG_FILTER)
3239
wx_fix_features(struct net_device * netdev,netdev_features_t features)3240 netdev_features_t wx_fix_features(struct net_device *netdev,
3241 netdev_features_t features)
3242 {
3243 netdev_features_t changed = netdev->features ^ features;
3244 struct wx *wx = netdev_priv(netdev);
3245
3246 if (changed & NETIF_VLAN_STRIPPING_FEATURES) {
3247 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&
3248 (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {
3249 features &= ~NETIF_VLAN_STRIPPING_FEATURES;
3250 features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;
3251 wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");
3252 }
3253 }
3254
3255 if (changed & NETIF_VLAN_INSERTION_FEATURES) {
3256 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&
3257 (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {
3258 features &= ~NETIF_VLAN_INSERTION_FEATURES;
3259 features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;
3260 wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");
3261 }
3262 }
3263
3264 if (changed & NETIF_VLAN_FILTERING_FEATURES) {
3265 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&
3266 (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {
3267 features &= ~NETIF_VLAN_FILTERING_FEATURES;
3268 features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;
3269 wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");
3270 }
3271 }
3272
3273 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
3274 if (!(features & NETIF_F_RXCSUM))
3275 features &= ~NETIF_F_LRO;
3276
3277 /* Turn off LRO if not RSC capable */
3278 if (!test_bit(WX_FLAG_RSC_CAPABLE, wx->flags))
3279 features &= ~NETIF_F_LRO;
3280
3281 return features;
3282 }
3283 EXPORT_SYMBOL(wx_fix_features);
3284
3285 #define WX_MAX_TUNNEL_HDR_LEN 80
wx_features_check(struct sk_buff * skb,struct net_device * netdev,netdev_features_t features)3286 netdev_features_t wx_features_check(struct sk_buff *skb,
3287 struct net_device *netdev,
3288 netdev_features_t features)
3289 {
3290 struct wx *wx = netdev_priv(netdev);
3291 __be16 type = skb->protocol;
3292 u16 vlan_depth = ETH_HLEN;
3293 u32 vlan_num = 0;
3294
3295 if (skb_vlan_tag_present(skb))
3296 vlan_num++;
3297
3298 while (eth_type_vlan(type)) {
3299 struct vlan_hdr vhdr, *vh;
3300
3301 vh = skb_header_pointer(skb, vlan_depth, sizeof(vhdr), &vhdr);
3302 if (unlikely(!vh))
3303 break;
3304
3305 type = vh->h_vlan_encapsulated_proto;
3306 vlan_depth += VLAN_HLEN;
3307 vlan_num++;
3308
3309 if (vlan_num > 2) {
3310 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3311 NETIF_F_HW_VLAN_STAG_TX);
3312 break;
3313 }
3314 }
3315
3316 if (!skb->encapsulation)
3317 return features;
3318
3319 if (wx->mac.type == wx_mac_em)
3320 return features & ~NETIF_F_CSUM_MASK;
3321
3322 if (unlikely(skb_inner_mac_header(skb) - skb_transport_header(skb) >
3323 WX_MAX_TUNNEL_HDR_LEN))
3324 return features & ~NETIF_F_CSUM_MASK;
3325
3326 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER &&
3327 skb->inner_protocol != htons(ETH_P_IP) &&
3328 skb->inner_protocol != htons(ETH_P_IPV6) &&
3329 skb->inner_protocol != htons(ETH_P_TEB))
3330 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3331
3332 return features;
3333 }
3334 EXPORT_SYMBOL(wx_features_check);
3335
wx_set_ring(struct wx * wx,u32 new_tx_count,u32 new_rx_count,struct wx_ring * temp_ring)3336 int wx_set_ring(struct wx *wx, u32 new_tx_count,
3337 u32 new_rx_count, struct wx_ring *temp_ring)
3338 {
3339 int i, err = 0;
3340
3341 /* Setup new Tx resources and free the old Tx resources in that order.
3342 * We can then assign the new resources to the rings via a memcpy.
3343 * The advantage to this approach is that we are guaranteed to still
3344 * have resources even in the case of an allocation failure.
3345 */
3346 if (new_tx_count != wx->tx_ring_count) {
3347 for (i = 0; i < wx->num_tx_queues; i++) {
3348 memcpy(&temp_ring[i], wx->tx_ring[i],
3349 sizeof(struct wx_ring));
3350
3351 temp_ring[i].count = new_tx_count;
3352 err = wx_setup_tx_resources(&temp_ring[i]);
3353 if (err) {
3354 wx_err(wx, "setup new tx resources failed, keep using the old config\n");
3355 while (i) {
3356 i--;
3357 wx_free_tx_resources(&temp_ring[i]);
3358 }
3359 return err;
3360 }
3361 }
3362
3363 for (i = 0; i < wx->num_tx_queues; i++) {
3364 wx_free_tx_resources(wx->tx_ring[i]);
3365
3366 memcpy(wx->tx_ring[i], &temp_ring[i],
3367 sizeof(struct wx_ring));
3368 }
3369
3370 wx->tx_ring_count = new_tx_count;
3371 }
3372
3373 /* Repeat the process for the Rx rings if needed */
3374 if (new_rx_count != wx->rx_ring_count) {
3375 for (i = 0; i < wx->num_rx_queues; i++) {
3376 memcpy(&temp_ring[i], wx->rx_ring[i],
3377 sizeof(struct wx_ring));
3378
3379 temp_ring[i].count = new_rx_count;
3380 err = wx_setup_rx_resources(&temp_ring[i]);
3381 if (err) {
3382 wx_err(wx, "setup new rx resources failed, keep using the old config\n");
3383 while (i) {
3384 i--;
3385 wx_free_rx_resources(&temp_ring[i]);
3386 }
3387 return err;
3388 }
3389 }
3390
3391 for (i = 0; i < wx->num_rx_queues; i++) {
3392 wx_free_rx_resources(wx->rx_ring[i]);
3393 memcpy(wx->rx_ring[i], &temp_ring[i],
3394 sizeof(struct wx_ring));
3395 }
3396
3397 wx->rx_ring_count = new_rx_count;
3398 }
3399 return 0;
3400 }
3401 EXPORT_SYMBOL(wx_set_ring);
3402
wx_service_event_schedule(struct wx * wx)3403 void wx_service_event_schedule(struct wx *wx)
3404 {
3405 if (!test_bit(WX_STATE_DOWN, wx->state) &&
3406 !test_and_set_bit(WX_STATE_SERVICE_SCHED, wx->state))
3407 queue_work(system_power_efficient_wq, &wx->service_task);
3408 }
3409 EXPORT_SYMBOL(wx_service_event_schedule);
3410
wx_service_event_complete(struct wx * wx)3411 void wx_service_event_complete(struct wx *wx)
3412 {
3413 if (WARN_ON(!test_bit(WX_STATE_SERVICE_SCHED, wx->state)))
3414 return;
3415
3416 /* flush memory to make sure state is correct before next watchdog */
3417 smp_mb__before_atomic();
3418 clear_bit(WX_STATE_SERVICE_SCHED, wx->state);
3419 }
3420 EXPORT_SYMBOL(wx_service_event_complete);
3421
wx_service_timer(struct timer_list * t)3422 void wx_service_timer(struct timer_list *t)
3423 {
3424 struct wx *wx = timer_container_of(wx, t, service_timer);
3425 unsigned long next_event_offset = HZ * 2;
3426
3427 /* Reset the timer */
3428 mod_timer(&wx->service_timer, next_event_offset + jiffies);
3429
3430 wx_service_event_schedule(wx);
3431 }
3432 EXPORT_SYMBOL(wx_service_timer);
3433
wx_soft_quiesce(struct wx * wx)3434 void wx_soft_quiesce(struct wx *wx)
3435 {
3436 if (!netif_running(wx->netdev) ||
3437 test_and_set_bit(WX_STATE_DOWN, wx->state))
3438 return;
3439
3440 pci_clear_master(wx->pdev);
3441 netif_tx_stop_all_queues(wx->netdev);
3442 netif_carrier_off(wx->netdev);
3443 netif_tx_disable(wx->netdev);
3444 wx_napi_disable_all(wx);
3445 wx_ptp_quiesce(wx);
3446
3447 clear_bit(WX_FLAG_NEED_DO_RESET, wx->flags);
3448 timer_delete_sync(&wx->service_timer);
3449 }
3450 EXPORT_SYMBOL(wx_soft_quiesce);
3451
3452 MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");
3453 MODULE_LICENSE("GPL");
3454