1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/workqueue.h>
9 #include <linux/iopoll.h>
10 #include <linux/sctp.h>
11 #include <linux/pci.h>
12 #include <net/tcp.h>
13 #include <net/ip.h>
14
15 #include "wx_type.h"
16 #include "wx_lib.h"
17 #include "wx_err.h"
18 #include "wx_ptp.h"
19 #include "wx_hw.h"
20 #include "wx_vf_lib.h"
21
22 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
23 static struct wx_dec_ptype wx_ptype_lookup[256] = {
24 /* L2: mac */
25 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
26 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
27 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
28 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
29 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
30 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
31 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
32
33 /* L2: ethertype filter */
34 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
35
36 /* L3: ip non-tunnel */
37 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
38 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
39 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
40 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
41 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
42 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
43 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
44 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
45 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
46 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
47
48 /* L2: fcoe */
49 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
50 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
51
52 /* IPv4 --> IPv4/IPv6 */
53 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
54 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
55 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
56 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
57 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
58 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
59 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
60 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
61 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
62 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
63
64 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
65 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
66 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
67 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
68 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
69 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
70 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
71 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
72 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
73 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
74 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
75 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
76
77 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
78 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
79 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
80 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
81 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
82 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
83 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
84 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
85 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
86 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
87 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
88 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
89
90 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
91 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
92 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
93 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
94 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
95 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
96 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
97 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
98 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
99 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
100 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
101 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
102
103 /* IPv6 --> IPv4/IPv6 */
104 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
105 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
106 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
107 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
108 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
109 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
110 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
111 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
112 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
113 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
114
115 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
116 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
117 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
118 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
119 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
120 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
121 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
122 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
123 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
124 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
125 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
126 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
127
128 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
129 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
130 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
131 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
132 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
133 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
134 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
135 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
136 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
137 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
138 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
139 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
140
141 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
142 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
143 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
144 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
145 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
146 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
147 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
148 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
149 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
150 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
151 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
152 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
153 };
154
wx_decode_ptype(const u8 ptype)155 struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
156 {
157 return wx_ptype_lookup[ptype];
158 }
159 EXPORT_SYMBOL(wx_decode_ptype);
160
161 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)162 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
163 const u32 stat_err_bits)
164 {
165 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
166 }
167
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)168 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
169 struct wx_rx_buffer *rx_buffer)
170 {
171 struct sk_buff *skb = rx_buffer->skb;
172 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
173
174 dma_sync_single_range_for_cpu(rx_ring->dev,
175 WX_CB(skb)->dma,
176 skb_frag_off(frag),
177 skb_frag_size(frag),
178 DMA_FROM_DEVICE);
179 }
180
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb)181 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
182 union wx_rx_desc *rx_desc,
183 struct sk_buff **skb)
184 {
185 struct wx_rx_buffer *rx_buffer;
186 unsigned int size;
187
188 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
189 size = le16_to_cpu(rx_desc->wb.upper.length);
190
191 prefetchw(rx_buffer->page);
192 *skb = rx_buffer->skb;
193
194 /* Delay unmapping of the first packet. It carries the header
195 * information, HW may still access the header after the writeback.
196 * Only unmap it when EOP is reached
197 */
198 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
199 if (!*skb)
200 goto skip_sync;
201 } else {
202 if (*skb)
203 wx_dma_sync_frag(rx_ring, rx_buffer);
204 }
205
206 /* we are reusing so sync this buffer for CPU use */
207 dma_sync_single_range_for_cpu(rx_ring->dev,
208 rx_buffer->dma,
209 rx_buffer->page_offset,
210 size,
211 DMA_FROM_DEVICE);
212 skip_sync:
213 return rx_buffer;
214 }
215
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb)216 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
217 struct wx_rx_buffer *rx_buffer,
218 struct sk_buff *skb)
219 {
220 /* clear contents of rx_buffer */
221 rx_buffer->page = NULL;
222 rx_buffer->skb = NULL;
223 }
224
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)225 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
226 struct wx_rx_buffer *rx_buffer,
227 union wx_rx_desc *rx_desc)
228 {
229 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
230 #if (PAGE_SIZE < 8192)
231 unsigned int truesize = wx_rx_pg_size(rx_ring) / 2;
232 #else
233 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
234 #endif
235 struct sk_buff *skb = rx_buffer->skb;
236
237 if (!skb) {
238 void *page_addr = page_address(rx_buffer->page) +
239 rx_buffer->page_offset;
240
241 /* prefetch first cache line of first page */
242 net_prefetch(page_addr);
243
244 /* allocate a skb to store the frags */
245 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
246 if (unlikely(!skb))
247 return NULL;
248
249 /* we will be copying header into skb->data in
250 * pskb_may_pull so it is in our interest to prefetch
251 * it now to avoid a possible cache miss
252 */
253 prefetchw(skb->data);
254
255 if (size <= WX_RXBUFFER_256) {
256 memcpy(__skb_put(skb, size), page_addr,
257 ALIGN(size, sizeof(long)));
258 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
259 return skb;
260 }
261
262 skb_mark_for_recycle(skb);
263
264 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
265 WX_CB(skb)->dma = rx_buffer->dma;
266
267 skb_add_rx_frag(skb, 0, rx_buffer->page,
268 rx_buffer->page_offset,
269 size, truesize);
270 goto out;
271
272 } else {
273 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
274 rx_buffer->page_offset, size, truesize);
275 }
276
277 out:
278 #if (PAGE_SIZE < 8192)
279 /* flip page offset to other buffer */
280 rx_buffer->page_offset ^= truesize;
281 #else
282 /* move offset up to the next cache line */
283 rx_buffer->page_offset += truesize;
284 #endif
285
286 return skb;
287 }
288
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)289 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
290 struct wx_rx_buffer *bi)
291 {
292 struct page *page = bi->page;
293 dma_addr_t dma;
294
295 /* since we are recycling buffers we should seldom need to alloc */
296 if (likely(page))
297 return true;
298
299 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
300 if (unlikely(!page))
301 return false;
302 dma = page_pool_get_dma_addr(page);
303
304 bi->dma = dma;
305 bi->page = page;
306 bi->page_offset = 0;
307
308 return true;
309 }
310
311 /**
312 * wx_alloc_rx_buffers - Replace used receive buffers
313 * @rx_ring: ring to place buffers on
314 * @cleaned_count: number of buffers to replace
315 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)316 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
317 {
318 u16 i = rx_ring->next_to_use;
319 union wx_rx_desc *rx_desc;
320 struct wx_rx_buffer *bi;
321
322 /* nothing to do */
323 if (!cleaned_count)
324 return;
325
326 rx_desc = WX_RX_DESC(rx_ring, i);
327 bi = &rx_ring->rx_buffer_info[i];
328 i -= rx_ring->count;
329
330 do {
331 if (!wx_alloc_mapped_page(rx_ring, bi))
332 break;
333
334 /* sync the buffer for use by the device */
335 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
336 bi->page_offset,
337 rx_ring->rx_buf_len,
338 DMA_FROM_DEVICE);
339
340 rx_desc->read.pkt_addr =
341 cpu_to_le64(bi->dma + bi->page_offset);
342
343 rx_desc++;
344 bi++;
345 i++;
346 if (unlikely(!i)) {
347 rx_desc = WX_RX_DESC(rx_ring, 0);
348 bi = rx_ring->rx_buffer_info;
349 i -= rx_ring->count;
350 }
351
352 /* clear the status bits for the next_to_use descriptor */
353 rx_desc->wb.upper.status_error = 0;
354 /* clear the length for the next_to_use descriptor */
355 rx_desc->wb.upper.length = 0;
356
357 cleaned_count--;
358 } while (cleaned_count);
359
360 i += rx_ring->count;
361
362 if (rx_ring->next_to_use != i) {
363 rx_ring->next_to_use = i;
364 /* update next to alloc since we have filled the ring */
365 rx_ring->next_to_alloc = i;
366
367 /* Force memory writes to complete before letting h/w
368 * know there are new descriptors to fetch. (Only
369 * applicable for weak-ordered memory model archs,
370 * such as IA-64).
371 */
372 wmb();
373 writel(i, rx_ring->tail);
374 }
375 }
376
wx_desc_unused(struct wx_ring * ring)377 u16 wx_desc_unused(struct wx_ring *ring)
378 {
379 u16 ntc = ring->next_to_clean;
380 u16 ntu = ring->next_to_use;
381
382 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
383 }
384
385 /**
386 * wx_is_non_eop - process handling of non-EOP buffers
387 * @rx_ring: Rx ring being processed
388 * @rx_desc: Rx descriptor for current buffer
389 * @skb: Current socket buffer containing buffer in progress
390 *
391 * This function updates next to clean. If the buffer is an EOP buffer
392 * this function exits returning false, otherwise it will place the
393 * sk_buff in the next buffer to be chained and return true indicating
394 * that this is in fact a non-EOP buffer.
395 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)396 static bool wx_is_non_eop(struct wx_ring *rx_ring,
397 union wx_rx_desc *rx_desc,
398 struct sk_buff *skb)
399 {
400 struct wx *wx = rx_ring->q_vector->wx;
401 u32 ntc = rx_ring->next_to_clean + 1;
402
403 /* fetch, update, and store next to clean */
404 ntc = (ntc < rx_ring->count) ? ntc : 0;
405 rx_ring->next_to_clean = ntc;
406
407 prefetch(WX_RX_DESC(rx_ring, ntc));
408
409 /* update RSC append count if present */
410 if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) {
411 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
412 cpu_to_le32(WX_RXD_RSCCNT_MASK);
413
414 if (unlikely(rsc_enabled)) {
415 u32 rsc_cnt = le32_to_cpu(rsc_enabled);
416
417 rsc_cnt >>= WX_RXD_RSCCNT_SHIFT;
418 WX_CB(skb)->append_cnt += rsc_cnt - 1;
419
420 /* update ntc based on RSC value */
421 ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
422 ntc &= WX_RXD_NEXTP_MASK;
423 ntc >>= WX_RXD_NEXTP_SHIFT;
424 }
425 }
426
427 /* if we are the last buffer then there is nothing else to do */
428 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
429 return false;
430
431 rx_ring->rx_buffer_info[ntc].skb = skb;
432 rx_ring->rx_stats.non_eop_descs++;
433
434 return true;
435 }
436
wx_pull_tail(struct sk_buff * skb)437 static void wx_pull_tail(struct sk_buff *skb)
438 {
439 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
440 unsigned int pull_len;
441 unsigned char *va;
442
443 /* it is valid to use page_address instead of kmap since we are
444 * working with pages allocated out of the lomem pool per
445 * alloc_page(GFP_ATOMIC)
446 */
447 va = skb_frag_address(frag);
448
449 /* we need the header to contain the greater of either ETH_HLEN or
450 * 60 bytes if the skb->len is less than 60 for skb_pad.
451 */
452 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
453
454 /* align pull length to size of long to optimize memcpy performance */
455 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
456
457 /* update all of the pointers */
458 skb_frag_size_sub(frag, pull_len);
459 skb_frag_off_add(frag, pull_len);
460 skb->data_len -= pull_len;
461 skb->tail += pull_len;
462 }
463
464 /**
465 * wx_cleanup_headers - Correct corrupted or empty headers
466 * @rx_ring: rx descriptor ring packet is being transacted on
467 * @rx_desc: pointer to the EOP Rx descriptor
468 * @skb: pointer to current skb being fixed
469 *
470 * Check for corrupted packet headers caused by senders on the local L2
471 * embedded NIC switch not setting up their Tx Descriptors right. These
472 * should be very rare.
473 *
474 * Also address the case where we are pulling data in on pages only
475 * and as such no data is present in the skb header.
476 *
477 * In addition if skb is not at least 60 bytes we need to pad it so that
478 * it is large enough to qualify as a valid Ethernet frame.
479 *
480 * Returns true if an error was encountered and skb was freed.
481 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)482 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
483 union wx_rx_desc *rx_desc,
484 struct sk_buff *skb)
485 {
486 struct net_device *netdev = rx_ring->netdev;
487
488 /* verify that the packet does not have any known errors */
489 if (!netdev ||
490 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
491 !(netdev->features & NETIF_F_RXALL))) {
492 dev_kfree_skb_any(skb);
493 return true;
494 }
495
496 /* place header in linear portion of buffer */
497 if (!skb_headlen(skb))
498 wx_pull_tail(skb);
499
500 /* if eth_skb_pad returns an error the skb was freed */
501 if (eth_skb_pad(skb))
502 return true;
503
504 return false;
505 }
506
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)507 static void wx_rx_hash(struct wx_ring *ring,
508 union wx_rx_desc *rx_desc,
509 struct sk_buff *skb)
510 {
511 u16 rss_type;
512
513 if (!(ring->netdev->features & NETIF_F_RXHASH))
514 return;
515
516 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
517 WX_RXD_RSSTYPE_MASK;
518
519 if (!rss_type)
520 return;
521
522 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
523 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
524 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
525 }
526
527 /**
528 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
529 * @ring: structure containing ring specific data
530 * @rx_desc: current Rx descriptor being processed
531 * @skb: skb currently being received and modified
532 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)533 static void wx_rx_checksum(struct wx_ring *ring,
534 union wx_rx_desc *rx_desc,
535 struct sk_buff *skb)
536 {
537 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
538
539 skb_checksum_none_assert(skb);
540 /* Rx csum disabled */
541 if (!(ring->netdev->features & NETIF_F_RXCSUM))
542 return;
543
544 /* if IPv4 header checksum error */
545 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
546 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
547 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
548 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
549 ring->rx_stats.csum_err++;
550 return;
551 }
552
553 /* L4 checksum offload flag must set for the below code to work */
554 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
555 return;
556
557 /* Hardware can't guarantee csum if IPv6 Dest Header found */
558 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP &&
559 wx_test_staterr(rx_desc, WX_RXD_STAT_IPV6EX))
560 return;
561
562 /* if L4 checksum error */
563 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
564 ring->rx_stats.csum_err++;
565 return;
566 }
567
568 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
569 skb->ip_summed = CHECKSUM_UNNECESSARY;
570
571 /* If there is an outer header present that might contain a checksum
572 * we need to bump the checksum level by 1 to reflect the fact that
573 * we are indicating we validated the inner checksum.
574 */
575 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
576 __skb_incr_checksum_unnecessary(skb);
577 ring->rx_stats.csum_good_cnt++;
578 }
579
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)580 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
581 struct sk_buff *skb)
582 {
583 u16 ethertype;
584 u8 idx = 0;
585
586 if ((ring->netdev->features &
587 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
588 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
589 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
590 0x1c0) >> 6;
591 ethertype = ring->q_vector->wx->tpid[idx];
592 __vlan_hwaccel_put_tag(skb, htons(ethertype),
593 le16_to_cpu(rx_desc->wb.upper.vlan));
594 }
595 }
596
wx_set_rsc_gso_size(struct wx_ring * ring,struct sk_buff * skb)597 static void wx_set_rsc_gso_size(struct wx_ring *ring,
598 struct sk_buff *skb)
599 {
600 u16 hdr_len = skb_headlen(skb);
601
602 /* set gso_size to avoid messing up TCP MSS */
603 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
604 WX_CB(skb)->append_cnt);
605 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
606 }
607
wx_update_rsc_stats(struct wx_ring * rx_ring,struct sk_buff * skb)608 static void wx_update_rsc_stats(struct wx_ring *rx_ring,
609 struct sk_buff *skb)
610 {
611 /* if append_cnt is 0 then frame is not RSC */
612 if (!WX_CB(skb)->append_cnt)
613 return;
614
615 rx_ring->rx_stats.rsc_count += WX_CB(skb)->append_cnt;
616 rx_ring->rx_stats.rsc_flush++;
617
618 wx_set_rsc_gso_size(rx_ring, skb);
619
620 /* gso_size is computed using append_cnt so always clear it last */
621 WX_CB(skb)->append_cnt = 0;
622 }
623
624 /**
625 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
626 * @rx_ring: rx descriptor ring packet is being transacted on
627 * @rx_desc: pointer to the EOP Rx descriptor
628 * @skb: pointer to current skb being populated
629 *
630 * This function checks the ring, descriptor, and packet information in
631 * order to populate the hash, checksum, protocol, and
632 * other fields within the skb.
633 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)634 static void wx_process_skb_fields(struct wx_ring *rx_ring,
635 union wx_rx_desc *rx_desc,
636 struct sk_buff *skb)
637 {
638 struct wx *wx = netdev_priv(rx_ring->netdev);
639
640 if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags))
641 wx_update_rsc_stats(rx_ring, skb);
642
643 wx_rx_hash(rx_ring, rx_desc, skb);
644 wx_rx_checksum(rx_ring, rx_desc, skb);
645
646 if (unlikely(test_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags)) &&
647 unlikely(wx_test_staterr(rx_desc, WX_RXD_STAT_TS))) {
648 wx_ptp_rx_hwtstamp(rx_ring->q_vector->wx, skb);
649 rx_ring->last_rx_timestamp = jiffies;
650 }
651
652 wx_rx_vlan(rx_ring, rx_desc, skb);
653 skb_record_rx_queue(skb, rx_ring->queue_index);
654 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
655 }
656
657 /**
658 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
659 * @q_vector: structure containing interrupt and ring information
660 * @rx_ring: rx descriptor ring to transact packets on
661 * @budget: Total limit on number of packets to process
662 *
663 * This function provides a "bounce buffer" approach to Rx interrupt
664 * processing. The advantage to this is that on systems that have
665 * expensive overhead for IOMMU access this provides a means of avoiding
666 * it by maintaining the mapping of the page to the system.
667 *
668 * Returns amount of work completed.
669 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)670 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
671 struct wx_ring *rx_ring,
672 int budget)
673 {
674 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
675 u16 cleaned_count = wx_desc_unused(rx_ring);
676
677 do {
678 struct wx_rx_buffer *rx_buffer;
679 union wx_rx_desc *rx_desc;
680 struct sk_buff *skb;
681
682 /* return some buffers to hardware, one at a time is too slow */
683 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
684 wx_alloc_rx_buffers(rx_ring, cleaned_count);
685 cleaned_count = 0;
686 }
687
688 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
689 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
690 break;
691
692 /* This memory barrier is needed to keep us from reading
693 * any other fields out of the rx_desc until we know the
694 * descriptor has been written back
695 */
696 dma_rmb();
697
698 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb);
699
700 /* retrieve a buffer from the ring */
701 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
702
703 /* exit if we failed to retrieve a buffer */
704 if (!skb) {
705 rx_ring->rx_stats.alloc_rx_buff_failed++;
706 break;
707 }
708
709 wx_put_rx_buffer(rx_ring, rx_buffer, skb);
710 cleaned_count++;
711
712 /* place incomplete frames back on ring for completion */
713 if (wx_is_non_eop(rx_ring, rx_desc, skb))
714 continue;
715
716 /* verify the packet layout is correct */
717 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
718 continue;
719
720 /* probably a little skewed due to removing CRC */
721 total_rx_bytes += skb->len;
722
723 /* populate checksum, timestamp, VLAN, and protocol */
724 wx_process_skb_fields(rx_ring, rx_desc, skb);
725 napi_gro_receive(&q_vector->napi, skb);
726
727 /* update budget accounting */
728 total_rx_packets++;
729 } while (likely(total_rx_packets < budget));
730
731 u64_stats_update_begin(&rx_ring->syncp);
732 rx_ring->stats.packets += total_rx_packets;
733 rx_ring->stats.bytes += total_rx_bytes;
734 u64_stats_update_end(&rx_ring->syncp);
735 q_vector->rx.total_packets += total_rx_packets;
736 q_vector->rx.total_bytes += total_rx_bytes;
737
738 return total_rx_packets;
739 }
740
wx_txring_txq(const struct wx_ring * ring)741 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
742 {
743 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
744 }
745
wx_get_tx_pending(struct wx_ring * ring)746 static u32 wx_get_tx_pending(struct wx_ring *ring)
747 {
748 unsigned int head, tail;
749
750 head = ring->next_to_clean;
751 tail = ring->next_to_use;
752
753 return ((head <= tail) ? tail : tail + ring->count) - head;
754 }
755
wx_check_tx_hang(struct wx_ring * ring)756 static bool wx_check_tx_hang(struct wx_ring *ring)
757 {
758 u32 tx_done_old = ring->tx_stats.tx_done_old;
759 u32 tx_pending = wx_get_tx_pending(ring);
760 u32 tx_done = ring->stats.packets;
761
762 if (!test_and_clear_bit(WX_TX_DETECT_HANG, ring->state))
763 return false;
764
765 if (tx_done_old == tx_done && tx_pending)
766 /* make sure it is true for two checks in a row */
767 return test_and_set_bit(WX_HANG_CHECK_ARMED, ring->state);
768
769 /* update completed stats and continue */
770 ring->tx_stats.tx_done_old = tx_done;
771 /* reset the countdown */
772 clear_bit(WX_HANG_CHECK_ARMED, ring->state);
773
774 return false;
775 }
776
777 /**
778 * wx_clean_tx_irq - Reclaim resources after transmit completes
779 * @q_vector: structure containing interrupt and ring information
780 * @tx_ring: tx ring to clean
781 * @napi_budget: Used to determine if we are in netpoll
782 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)783 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
784 struct wx_ring *tx_ring, int napi_budget)
785 {
786 unsigned int budget = q_vector->wx->tx_work_limit;
787 unsigned int total_bytes = 0, total_packets = 0;
788 struct wx *wx = netdev_priv(tx_ring->netdev);
789 unsigned int i = tx_ring->next_to_clean;
790 struct wx_tx_buffer *tx_buffer;
791 union wx_tx_desc *tx_desc;
792
793 if (!netif_carrier_ok(tx_ring->netdev))
794 return true;
795
796 tx_buffer = &tx_ring->tx_buffer_info[i];
797 tx_desc = WX_TX_DESC(tx_ring, i);
798 i -= tx_ring->count;
799
800 do {
801 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
802
803 /* if next_to_watch is not set then there is no work pending */
804 if (!eop_desc)
805 break;
806
807 /* prevent any other reads prior to eop_desc */
808 smp_rmb();
809
810 if (tx_ring->headwb_mem) {
811 u32 head = *tx_ring->headwb_mem;
812
813 if (head == tx_ring->next_to_clean)
814 break;
815 else if (head > tx_ring->next_to_clean &&
816 !(tx_buffer->next_eop >= tx_ring->next_to_clean &&
817 tx_buffer->next_eop < head))
818 break;
819 else if (!(tx_buffer->next_eop >= tx_ring->next_to_clean ||
820 tx_buffer->next_eop < head))
821 break;
822 } else if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD))) {
823 /* if DD is not set pending work has not been completed */
824 break;
825 }
826
827 /* clear next_to_watch to prevent false hangs */
828 tx_buffer->next_to_watch = NULL;
829
830 /* update the statistics for this packet */
831 total_bytes += tx_buffer->bytecount;
832 total_packets += tx_buffer->gso_segs;
833
834 /* schedule check for Tx timestamp */
835 if (unlikely(test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) &&
836 skb_shinfo(tx_buffer->skb)->tx_flags & SKBTX_IN_PROGRESS)
837 ptp_schedule_worker(wx->ptp_clock, 0);
838
839 /* free the skb */
840 napi_consume_skb(tx_buffer->skb, napi_budget);
841
842 /* unmap skb header data */
843 dma_unmap_single(tx_ring->dev,
844 dma_unmap_addr(tx_buffer, dma),
845 dma_unmap_len(tx_buffer, len),
846 DMA_TO_DEVICE);
847
848 /* clear tx_buffer data */
849 dma_unmap_len_set(tx_buffer, len, 0);
850
851 /* unmap remaining buffers */
852 while (tx_desc != eop_desc) {
853 tx_buffer++;
854 tx_desc++;
855 i++;
856 if (unlikely(!i)) {
857 i -= tx_ring->count;
858 tx_buffer = tx_ring->tx_buffer_info;
859 tx_desc = WX_TX_DESC(tx_ring, 0);
860 }
861
862 /* unmap any remaining paged data */
863 if (dma_unmap_len(tx_buffer, len)) {
864 dma_unmap_page(tx_ring->dev,
865 dma_unmap_addr(tx_buffer, dma),
866 dma_unmap_len(tx_buffer, len),
867 DMA_TO_DEVICE);
868 dma_unmap_len_set(tx_buffer, len, 0);
869 }
870 }
871
872 /* move us one more past the eop_desc for start of next pkt */
873 tx_buffer++;
874 tx_desc++;
875 i++;
876 if (unlikely(!i)) {
877 i -= tx_ring->count;
878 tx_buffer = tx_ring->tx_buffer_info;
879 tx_desc = WX_TX_DESC(tx_ring, 0);
880 }
881
882 /* issue prefetch for next Tx descriptor */
883 prefetch(tx_desc);
884
885 /* update budget accounting */
886 budget--;
887 } while (likely(budget));
888
889 i += tx_ring->count;
890 tx_ring->next_to_clean = i;
891 u64_stats_update_begin(&tx_ring->syncp);
892 tx_ring->stats.bytes += total_bytes;
893 tx_ring->stats.packets += total_packets;
894 u64_stats_update_end(&tx_ring->syncp);
895 q_vector->tx.total_bytes += total_bytes;
896 q_vector->tx.total_packets += total_packets;
897
898 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
899 total_packets, total_bytes);
900
901 if (wx_check_tx_hang(tx_ring)) {
902 wx_handle_tx_hang(tx_ring, i);
903 return true;
904 }
905
906 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
907 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
908 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
909 /* Make sure that anybody stopping the queue after this
910 * sees the new next_to_clean.
911 */
912 smp_mb();
913
914 if (__netif_subqueue_stopped(tx_ring->netdev,
915 tx_ring->queue_index) &&
916 !test_bit(WX_STATE_DOWN, wx->state)) {
917 netif_wake_subqueue(tx_ring->netdev,
918 tx_ring->queue_index);
919 ++tx_ring->tx_stats.restart_queue;
920 }
921 }
922
923 return !!budget;
924 }
925
wx_update_rx_dim_sample(struct wx_q_vector * q_vector)926 static void wx_update_rx_dim_sample(struct wx_q_vector *q_vector)
927 {
928 struct dim_sample sample = {};
929
930 dim_update_sample(q_vector->total_events,
931 q_vector->rx.total_packets,
932 q_vector->rx.total_bytes,
933 &sample);
934
935 net_dim(&q_vector->rx.dim, &sample);
936 }
937
wx_update_tx_dim_sample(struct wx_q_vector * q_vector)938 static void wx_update_tx_dim_sample(struct wx_q_vector *q_vector)
939 {
940 struct dim_sample sample = {};
941
942 dim_update_sample(q_vector->total_events,
943 q_vector->tx.total_packets,
944 q_vector->tx.total_bytes,
945 &sample);
946
947 net_dim(&q_vector->tx.dim, &sample);
948 }
949
wx_update_dim_sample(struct wx_q_vector * q_vector)950 static void wx_update_dim_sample(struct wx_q_vector *q_vector)
951 {
952 wx_update_rx_dim_sample(q_vector);
953 wx_update_tx_dim_sample(q_vector);
954 }
955
956 /**
957 * wx_poll - NAPI polling RX/TX cleanup routine
958 * @napi: napi struct with our devices info in it
959 * @budget: amount of work driver is allowed to do this pass, in packets
960 *
961 * This function will clean all queues associated with a q_vector.
962 **/
wx_poll(struct napi_struct * napi,int budget)963 static int wx_poll(struct napi_struct *napi, int budget)
964 {
965 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
966 int per_ring_budget, work_done = 0;
967 struct wx *wx = q_vector->wx;
968 bool clean_complete = true;
969 struct wx_ring *ring;
970
971 wx_for_each_ring(ring, q_vector->tx) {
972 if (!wx_clean_tx_irq(q_vector, ring, budget))
973 clean_complete = false;
974 }
975
976 /* Exit if we are called by netpoll */
977 if (budget <= 0)
978 return budget;
979
980 /* attempt to distribute budget to each queue fairly, but don't allow
981 * the budget to go below 1 because we'll exit polling
982 */
983 if (q_vector->rx.count > 1)
984 per_ring_budget = max(budget / q_vector->rx.count, 1);
985 else
986 per_ring_budget = budget;
987
988 wx_for_each_ring(ring, q_vector->rx) {
989 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
990
991 work_done += cleaned;
992 if (cleaned >= per_ring_budget)
993 clean_complete = false;
994 }
995
996 /* If all work not completed, return budget and keep polling */
997 if (!clean_complete)
998 return budget;
999
1000 /* all work done, exit the polling mode */
1001 if (likely(napi_complete_done(napi, work_done))) {
1002 if (wx->adaptive_itr)
1003 wx_update_dim_sample(q_vector);
1004 if (!test_bit(WX_STATE_DOWN, wx->state))
1005 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
1006 }
1007
1008 return min(work_done, budget - 1);
1009 }
1010
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)1011 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
1012 {
1013 if (likely(wx_desc_unused(tx_ring) >= size))
1014 return 0;
1015
1016 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
1017
1018 /* For the next check */
1019 smp_mb();
1020
1021 /* We need to check again in a case another CPU has just
1022 * made room available.
1023 */
1024 if (likely(wx_desc_unused(tx_ring) < size))
1025 return -EBUSY;
1026
1027 /* A reprieve! - use start_queue because it doesn't call schedule */
1028 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
1029 ++tx_ring->tx_stats.restart_queue;
1030
1031 return 0;
1032 }
1033
wx_tx_cmd_type(u32 tx_flags)1034 static u32 wx_tx_cmd_type(u32 tx_flags)
1035 {
1036 /* set type for advanced descriptor with frame checksum insertion */
1037 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
1038
1039 /* set HW vlan bit if vlan is present */
1040 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
1041 /* set segmentation enable bits for TSO/FSO */
1042 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
1043 /* set timestamp bit if present */
1044 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
1045 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
1046
1047 return cmd_type;
1048 }
1049
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)1050 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
1051 u32 tx_flags, unsigned int paylen)
1052 {
1053 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
1054
1055 /* enable L4 checksum for TSO and TX checksum offload */
1056 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
1057 /* enable IPv4 checksum for TSO */
1058 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
1059 /* enable outer IPv4 checksum for TSO */
1060 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
1061 WX_TXD_EIPCS);
1062 /* Check Context must be set if Tx switch is enabled, which it
1063 * always is for case where virtual functions are running
1064 */
1065 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
1066 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
1067 WX_TXD_IPSEC);
1068 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1069 }
1070
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)1071 static int wx_tx_map(struct wx_ring *tx_ring,
1072 struct wx_tx_buffer *first,
1073 const u8 hdr_len)
1074 {
1075 struct sk_buff *skb = first->skb;
1076 struct wx_tx_buffer *tx_buffer;
1077 u32 tx_flags = first->tx_flags;
1078 u16 i = tx_ring->next_to_use;
1079 unsigned int data_len, size;
1080 union wx_tx_desc *tx_desc;
1081 skb_frag_t *frag;
1082 dma_addr_t dma;
1083 u32 cmd_type;
1084
1085 cmd_type = wx_tx_cmd_type(tx_flags);
1086 tx_desc = WX_TX_DESC(tx_ring, i);
1087 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
1088
1089 size = skb_headlen(skb);
1090 data_len = skb->data_len;
1091 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1092
1093 tx_buffer = first;
1094
1095 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1096 if (dma_mapping_error(tx_ring->dev, dma))
1097 goto dma_error;
1098
1099 /* record length, and DMA address */
1100 dma_unmap_len_set(tx_buffer, len, size);
1101 dma_unmap_addr_set(tx_buffer, dma, dma);
1102
1103 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1104
1105 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
1106 tx_desc->read.cmd_type_len =
1107 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
1108
1109 i++;
1110 tx_desc++;
1111 if (i == tx_ring->count) {
1112 tx_desc = WX_TX_DESC(tx_ring, 0);
1113 i = 0;
1114 }
1115 tx_desc->read.olinfo_status = 0;
1116
1117 dma += WX_MAX_DATA_PER_TXD;
1118 size -= WX_MAX_DATA_PER_TXD;
1119
1120 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1121 }
1122
1123 if (likely(!data_len))
1124 break;
1125
1126 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1127
1128 i++;
1129 tx_desc++;
1130 if (i == tx_ring->count) {
1131 tx_desc = WX_TX_DESC(tx_ring, 0);
1132 i = 0;
1133 }
1134 tx_desc->read.olinfo_status = 0;
1135
1136 size = skb_frag_size(frag);
1137
1138 data_len -= size;
1139
1140 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1141 DMA_TO_DEVICE);
1142
1143 tx_buffer = &tx_ring->tx_buffer_info[i];
1144 }
1145
1146 /* write last descriptor with RS and EOP bits */
1147 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1148 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1149
1150 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1151
1152 /* set the timestamp */
1153 first->time_stamp = jiffies;
1154 skb_tx_timestamp(skb);
1155
1156 /* Force memory writes to complete before letting h/w know there
1157 * are new descriptors to fetch. (Only applicable for weak-ordered
1158 * memory model archs, such as IA-64).
1159 *
1160 * We also need this memory barrier to make certain all of the
1161 * status bits have been updated before next_to_watch is written.
1162 */
1163 wmb();
1164
1165 /* set next_to_watch value indicating a packet is present */
1166 first->next_to_watch = tx_desc;
1167
1168 /* set next_eop for amlite tx head wb */
1169 if (tx_ring->headwb_mem)
1170 first->next_eop = i;
1171
1172 i++;
1173 if (i == tx_ring->count)
1174 i = 0;
1175
1176 tx_ring->next_to_use = i;
1177
1178 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1179
1180 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1181 writel(i, tx_ring->tail);
1182
1183 return 0;
1184 dma_error:
1185 dev_err(tx_ring->dev, "TX DMA map failed\n");
1186
1187 /* clear dma mappings for failed tx_buffer_info map */
1188 for (;;) {
1189 tx_buffer = &tx_ring->tx_buffer_info[i];
1190 if (dma_unmap_len(tx_buffer, len))
1191 dma_unmap_page(tx_ring->dev,
1192 dma_unmap_addr(tx_buffer, dma),
1193 dma_unmap_len(tx_buffer, len),
1194 DMA_TO_DEVICE);
1195 dma_unmap_len_set(tx_buffer, len, 0);
1196 if (tx_buffer == first)
1197 break;
1198 if (i == 0)
1199 i += tx_ring->count;
1200 i--;
1201 }
1202
1203 dev_kfree_skb_any(first->skb);
1204 first->skb = NULL;
1205
1206 tx_ring->next_to_use = i;
1207
1208 return -ENOMEM;
1209 }
1210
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1211 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1212 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1213 {
1214 struct wx_tx_context_desc *context_desc;
1215 u16 i = tx_ring->next_to_use;
1216
1217 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1218 i++;
1219 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1220
1221 /* set bits to identify this as an advanced context descriptor */
1222 type_tucmd |= WX_TXD_DTYP_CTXT;
1223 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1224 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1225 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1226 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1227 }
1228
1229 union network_header {
1230 struct iphdr *ipv4;
1231 struct ipv6hdr *ipv6;
1232 void *raw;
1233 };
1234
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1235 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1236 {
1237 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1238 struct sk_buff *skb = first->skb;
1239 unsigned char *exthdr, *l4_hdr;
1240 __be16 frag_off;
1241
1242 if (skb->encapsulation) {
1243 union network_header hdr;
1244
1245 switch (first->protocol) {
1246 case htons(ETH_P_IP):
1247 tun_prot = ip_hdr(skb)->protocol;
1248 ptype = WX_PTYPE_TUN_IPV4;
1249 break;
1250 case htons(ETH_P_IPV6):
1251 l4_hdr = skb_transport_header(skb);
1252 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1253 tun_prot = ipv6_hdr(skb)->nexthdr;
1254 if (l4_hdr != exthdr)
1255 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1256 ptype = WX_PTYPE_TUN_IPV6;
1257 break;
1258 default:
1259 return ptype;
1260 }
1261
1262 if (tun_prot == IPPROTO_IPIP || tun_prot == IPPROTO_IPV6) {
1263 hdr.raw = (void *)inner_ip_hdr(skb);
1264 ptype |= WX_PTYPE_PKT_IPIP;
1265 } else if (tun_prot == IPPROTO_UDP) {
1266 hdr.raw = (void *)inner_ip_hdr(skb);
1267 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1268 skb->inner_protocol != htons(ETH_P_TEB)) {
1269 ptype |= WX_PTYPE_PKT_IG;
1270 } else {
1271 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1272 == htons(ETH_P_8021Q))
1273 ptype |= WX_PTYPE_PKT_IGMV;
1274 else
1275 ptype |= WX_PTYPE_PKT_IGM;
1276 }
1277
1278 } else if (tun_prot == IPPROTO_GRE) {
1279 hdr.raw = (void *)inner_ip_hdr(skb);
1280 if (skb->inner_protocol == htons(ETH_P_IP) ||
1281 skb->inner_protocol == htons(ETH_P_IPV6)) {
1282 ptype |= WX_PTYPE_PKT_IG;
1283 } else {
1284 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1285 == htons(ETH_P_8021Q))
1286 ptype |= WX_PTYPE_PKT_IGMV;
1287 else
1288 ptype |= WX_PTYPE_PKT_IGM;
1289 }
1290 } else {
1291 return ptype;
1292 }
1293
1294 switch (hdr.ipv4->version) {
1295 case IPVERSION:
1296 l4_prot = hdr.ipv4->protocol;
1297 break;
1298 case 6:
1299 l4_hdr = skb_inner_transport_header(skb);
1300 exthdr = skb_inner_network_header(skb) + sizeof(struct ipv6hdr);
1301 l4_prot = inner_ipv6_hdr(skb)->nexthdr;
1302 if (l4_hdr != exthdr)
1303 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1304 ptype |= WX_PTYPE_PKT_IPV6;
1305 break;
1306 default:
1307 return ptype;
1308 }
1309 } else {
1310 switch (first->protocol) {
1311 case htons(ETH_P_IP):
1312 l4_prot = ip_hdr(skb)->protocol;
1313 ptype = WX_PTYPE_PKT_IP;
1314 break;
1315 case htons(ETH_P_IPV6):
1316 l4_hdr = skb_transport_header(skb);
1317 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1318 l4_prot = ipv6_hdr(skb)->nexthdr;
1319 if (l4_hdr != exthdr)
1320 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1321 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1322 break;
1323 default:
1324 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1325 }
1326 }
1327 switch (l4_prot) {
1328 case IPPROTO_TCP:
1329 ptype |= WX_PTYPE_TYP_TCP;
1330 break;
1331 case IPPROTO_UDP:
1332 ptype |= WX_PTYPE_TYP_UDP;
1333 break;
1334 case IPPROTO_SCTP:
1335 ptype |= WX_PTYPE_TYP_SCTP;
1336 break;
1337 default:
1338 ptype |= WX_PTYPE_TYP_IP;
1339 break;
1340 }
1341
1342 return ptype;
1343 }
1344
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1345 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1346 u8 *hdr_len, u8 ptype)
1347 {
1348 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1349 struct net_device *netdev = tx_ring->netdev;
1350 u32 l4len, tunhdr_eiplen_tunlen = 0;
1351 struct sk_buff *skb = first->skb;
1352 bool enc = skb->encapsulation;
1353 struct ipv6hdr *ipv6h;
1354 struct tcphdr *tcph;
1355 struct iphdr *iph;
1356 u8 tun_prot = 0;
1357 int err;
1358
1359 if (skb->ip_summed != CHECKSUM_PARTIAL)
1360 return 0;
1361
1362 if (!skb_is_gso(skb))
1363 return 0;
1364
1365 err = skb_cow_head(skb, 0);
1366 if (err < 0)
1367 return err;
1368
1369 /* indicates the inner headers in the skbuff are valid. */
1370 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1371 if (iph->version == 4) {
1372 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1373 iph->tot_len = 0;
1374 iph->check = 0;
1375 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1376 iph->daddr, 0,
1377 IPPROTO_TCP, 0);
1378 first->tx_flags |= WX_TX_FLAGS_TSO |
1379 WX_TX_FLAGS_CSUM |
1380 WX_TX_FLAGS_IPV4 |
1381 WX_TX_FLAGS_CC;
1382 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1383 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1384 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1385 ipv6h->payload_len = 0;
1386 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1387 &ipv6h->daddr, 0,
1388 IPPROTO_TCP, 0);
1389 first->tx_flags |= WX_TX_FLAGS_TSO |
1390 WX_TX_FLAGS_CSUM |
1391 WX_TX_FLAGS_CC;
1392 }
1393
1394 /* compute header lengths */
1395 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1396 *hdr_len = enc ? skb_inner_transport_offset(skb) :
1397 skb_transport_offset(skb);
1398 *hdr_len += l4len;
1399
1400 /* update gso size and bytecount with header size */
1401 first->gso_segs = skb_shinfo(skb)->gso_segs;
1402 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1403
1404 /* mss_l4len_id: use 0 as index for TSO */
1405 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1406 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1407
1408 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1409 if (enc) {
1410 unsigned char *exthdr, *l4_hdr;
1411 __be16 frag_off;
1412
1413 switch (first->protocol) {
1414 case htons(ETH_P_IP):
1415 tun_prot = ip_hdr(skb)->protocol;
1416 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1417 break;
1418 case htons(ETH_P_IPV6):
1419 l4_hdr = skb_transport_header(skb);
1420 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1421 tun_prot = ipv6_hdr(skb)->nexthdr;
1422 if (l4_hdr != exthdr)
1423 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1424 break;
1425 default:
1426 break;
1427 }
1428 switch (tun_prot) {
1429 case IPPROTO_UDP:
1430 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1431 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1432 WX_TXD_OUTER_IPLEN_SHIFT) |
1433 (((skb_inner_mac_header(skb) -
1434 skb_transport_header(skb)) >> 1) <<
1435 WX_TXD_TUNNEL_LEN_SHIFT);
1436 break;
1437 case IPPROTO_GRE:
1438 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1439 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1440 WX_TXD_OUTER_IPLEN_SHIFT) |
1441 (((skb_inner_mac_header(skb) -
1442 skb_transport_header(skb)) >> 1) <<
1443 WX_TXD_TUNNEL_LEN_SHIFT);
1444 break;
1445 case IPPROTO_IPIP:
1446 case IPPROTO_IPV6:
1447 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1448 (char *)ip_hdr(skb)) >> 2) <<
1449 WX_TXD_OUTER_IPLEN_SHIFT;
1450 break;
1451 default:
1452 break;
1453 }
1454 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1455 } else {
1456 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1457 }
1458
1459 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1460 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1461
1462 type_tucmd = ptype << 24;
1463 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1464 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1465 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1466 WX_TX_FLAGS_HW_VLAN,
1467 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1468 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1469 type_tucmd, mss_l4len_idx);
1470
1471 return 1;
1472 }
1473
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1474 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1475 u8 ptype)
1476 {
1477 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1478 struct net_device *netdev = tx_ring->netdev;
1479 u32 mss_l4len_idx = 0, type_tucmd;
1480 struct sk_buff *skb = first->skb;
1481 u8 tun_prot = 0;
1482
1483 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1484 csum_failed:
1485 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1486 !(first->tx_flags & WX_TX_FLAGS_CC))
1487 return;
1488 vlan_macip_lens = skb_network_offset(skb) <<
1489 WX_TXD_MACLEN_SHIFT;
1490 } else {
1491 unsigned char *exthdr, *l4_hdr;
1492 __be16 frag_off;
1493 u8 l4_prot = 0;
1494 union {
1495 struct iphdr *ipv4;
1496 struct ipv6hdr *ipv6;
1497 u8 *raw;
1498 } network_hdr;
1499 union {
1500 struct tcphdr *tcphdr;
1501 u8 *raw;
1502 } transport_hdr;
1503
1504 if (skb->encapsulation) {
1505 network_hdr.raw = skb_inner_network_header(skb);
1506 transport_hdr.raw = skb_inner_transport_header(skb);
1507 vlan_macip_lens = skb_network_offset(skb) <<
1508 WX_TXD_MACLEN_SHIFT;
1509 switch (first->protocol) {
1510 case htons(ETH_P_IP):
1511 tun_prot = ip_hdr(skb)->protocol;
1512 break;
1513 case htons(ETH_P_IPV6):
1514 l4_hdr = skb_transport_header(skb);
1515 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1516 tun_prot = ipv6_hdr(skb)->nexthdr;
1517 if (l4_hdr != exthdr)
1518 ipv6_skip_exthdr(skb, exthdr - skb->data,
1519 &tun_prot, &frag_off);
1520 break;
1521 default:
1522 return;
1523 }
1524 switch (tun_prot) {
1525 case IPPROTO_UDP:
1526 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1527 tunhdr_eiplen_tunlen |=
1528 ((skb_network_header_len(skb) >> 2) <<
1529 WX_TXD_OUTER_IPLEN_SHIFT) |
1530 (((skb_inner_mac_header(skb) -
1531 skb_transport_header(skb)) >> 1) <<
1532 WX_TXD_TUNNEL_LEN_SHIFT);
1533 break;
1534 case IPPROTO_GRE:
1535 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1536 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1537 WX_TXD_OUTER_IPLEN_SHIFT) |
1538 (((skb_inner_mac_header(skb) -
1539 skb_transport_header(skb)) >> 1) <<
1540 WX_TXD_TUNNEL_LEN_SHIFT);
1541 break;
1542 case IPPROTO_IPIP:
1543 case IPPROTO_IPV6:
1544 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1545 (char *)ip_hdr(skb)) >> 2) <<
1546 WX_TXD_OUTER_IPLEN_SHIFT;
1547 break;
1548 default:
1549 break;
1550 }
1551
1552 } else {
1553 network_hdr.raw = skb_network_header(skb);
1554 transport_hdr.raw = skb_transport_header(skb);
1555 vlan_macip_lens = skb_network_offset(skb) <<
1556 WX_TXD_MACLEN_SHIFT;
1557 }
1558
1559 switch (network_hdr.ipv4->version) {
1560 case IPVERSION:
1561 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1562 l4_prot = network_hdr.ipv4->protocol;
1563 break;
1564 case 6:
1565 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1566 exthdr = network_hdr.raw + sizeof(struct ipv6hdr);
1567 l4_prot = network_hdr.ipv6->nexthdr;
1568 if (transport_hdr.raw != exthdr)
1569 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1570 break;
1571 default:
1572 break;
1573 }
1574
1575 switch (l4_prot) {
1576 case IPPROTO_TCP:
1577 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1578 WX_TXD_L4LEN_SHIFT;
1579 break;
1580 case IPPROTO_SCTP:
1581 mss_l4len_idx = sizeof(struct sctphdr) <<
1582 WX_TXD_L4LEN_SHIFT;
1583 break;
1584 case IPPROTO_UDP:
1585 mss_l4len_idx = sizeof(struct udphdr) <<
1586 WX_TXD_L4LEN_SHIFT;
1587 break;
1588 default:
1589 skb_checksum_help(skb);
1590 goto csum_failed;
1591 }
1592
1593 /* update TX checksum flag */
1594 first->tx_flags |= WX_TX_FLAGS_CSUM;
1595 }
1596 first->tx_flags |= WX_TX_FLAGS_CC;
1597 /* vlan_macip_lens: MACLEN, VLAN tag */
1598 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1599
1600 type_tucmd = ptype << 24;
1601 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1602 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1603 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1604 WX_TX_FLAGS_HW_VLAN,
1605 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1606 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1607 type_tucmd, mss_l4len_idx);
1608 }
1609
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1610 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1611 struct wx_ring *tx_ring)
1612 {
1613 struct wx *wx = netdev_priv(tx_ring->netdev);
1614 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1615 struct wx_tx_buffer *first;
1616 u8 hdr_len = 0, ptype;
1617 unsigned short f;
1618 u32 tx_flags = 0;
1619 int tso;
1620
1621 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1622 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1623 * + 2 desc gap to keep tail from touching head,
1624 * + 1 desc for context descriptor,
1625 * otherwise try next time
1626 */
1627 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1628 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1629 frags[f]));
1630
1631 if (wx_maybe_stop_tx(tx_ring, count + 3)) {
1632 tx_ring->tx_stats.tx_busy++;
1633 return NETDEV_TX_BUSY;
1634 }
1635
1636 /* record the location of the first descriptor for this packet */
1637 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1638 first->skb = skb;
1639 first->bytecount = skb->len;
1640 first->gso_segs = 1;
1641
1642 /* if we have a HW VLAN tag being added default to the HW one */
1643 if (skb_vlan_tag_present(skb)) {
1644 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1645 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1646 } else if (eth_type_vlan(skb->protocol)) {
1647 tx_flags |= WX_TX_FLAGS_SW_VLAN;
1648 }
1649
1650 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1651 wx->ptp_clock) {
1652 if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1653 !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS,
1654 wx->state)) {
1655 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1656 tx_flags |= WX_TX_FLAGS_TSTAMP;
1657 wx->ptp_tx_skb = skb_get(skb);
1658 wx->ptp_tx_start = jiffies;
1659 } else {
1660 wx->tx_hwtstamp_skipped++;
1661 }
1662 }
1663
1664 /* record initial flags and protocol */
1665 first->tx_flags = tx_flags;
1666 first->protocol = vlan_get_protocol(skb);
1667
1668 ptype = wx_encode_tx_desc_ptype(first);
1669
1670 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1671 if (tso < 0)
1672 goto out_drop;
1673 else if (!tso)
1674 wx_tx_csum(tx_ring, first, ptype);
1675
1676 if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)
1677 wx->atr(tx_ring, first, ptype);
1678
1679 if (wx_tx_map(tx_ring, first, hdr_len))
1680 goto cleanup_tx_tstamp;
1681
1682 return NETDEV_TX_OK;
1683 out_drop:
1684 dev_kfree_skb_any(first->skb);
1685 first->skb = NULL;
1686 cleanup_tx_tstamp:
1687 if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) {
1688 dev_kfree_skb_any(wx->ptp_tx_skb);
1689 wx->ptp_tx_skb = NULL;
1690 wx->tx_hwtstamp_errors++;
1691 clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
1692 }
1693
1694 return NETDEV_TX_OK;
1695 }
1696
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1697 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1698 struct net_device *netdev)
1699 {
1700 unsigned int r_idx = skb->queue_mapping;
1701 struct wx *wx = netdev_priv(netdev);
1702 struct wx_ring *tx_ring;
1703
1704 if (!netif_carrier_ok(netdev)) {
1705 dev_kfree_skb_any(skb);
1706 return NETDEV_TX_OK;
1707 }
1708
1709 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1710 * in order to meet this minimum size requirement.
1711 */
1712 if (skb_put_padto(skb, 17))
1713 return NETDEV_TX_OK;
1714
1715 if (r_idx >= wx->num_tx_queues)
1716 r_idx = r_idx % wx->num_tx_queues;
1717 tx_ring = wx->tx_ring[r_idx];
1718
1719 return wx_xmit_frame_ring(skb, tx_ring);
1720 }
1721 EXPORT_SYMBOL(wx_xmit_frame);
1722
wx_set_itr(struct wx_q_vector * q_vector)1723 static void wx_set_itr(struct wx_q_vector *q_vector)
1724 {
1725 struct wx *wx = q_vector->wx;
1726 u32 new_itr;
1727
1728 if (!wx->adaptive_itr)
1729 return;
1730
1731 /* use the smallest value of new ITR delay calculations */
1732 new_itr = min(q_vector->rx.itr, q_vector->tx.itr);
1733 new_itr <<= 2;
1734
1735 if (new_itr != q_vector->itr) {
1736 /* save the algorithm value here */
1737 q_vector->itr = new_itr;
1738
1739 if (wx->pdev->is_virtfn)
1740 wx_write_eitr_vf(q_vector);
1741 else
1742 wx_write_eitr(q_vector);
1743 }
1744 }
1745
wx_rx_dim_work(struct work_struct * work)1746 static void wx_rx_dim_work(struct work_struct *work)
1747 {
1748 struct dim *dim = container_of(work, struct dim, work);
1749 struct dim_cq_moder rx_moder;
1750 struct wx_ring_container *rx;
1751 struct wx_q_vector *q_vector;
1752
1753 rx = container_of(dim, struct wx_ring_container, dim);
1754
1755 rx_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1756 rx->itr = rx_moder.usec;
1757
1758 q_vector = container_of(rx, struct wx_q_vector, rx);
1759 wx_set_itr(q_vector);
1760
1761 dim->state = DIM_START_MEASURE;
1762 }
1763
wx_tx_dim_work(struct work_struct * work)1764 static void wx_tx_dim_work(struct work_struct *work)
1765 {
1766 struct dim *dim = container_of(work, struct dim, work);
1767 struct dim_cq_moder tx_moder;
1768 struct wx_ring_container *tx;
1769 struct wx_q_vector *q_vector;
1770
1771 tx = container_of(dim, struct wx_ring_container, dim);
1772
1773 tx_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
1774 tx->itr = tx_moder.usec;
1775
1776 q_vector = container_of(tx, struct wx_q_vector, tx);
1777 wx_set_itr(q_vector);
1778
1779 dim->state = DIM_START_MEASURE;
1780 }
1781
wx_napi_enable_all(struct wx * wx)1782 void wx_napi_enable_all(struct wx *wx)
1783 {
1784 struct wx_q_vector *q_vector;
1785 int q_idx;
1786
1787 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1788 q_vector = wx->q_vector[q_idx];
1789
1790 INIT_WORK(&q_vector->rx.dim.work, wx_rx_dim_work);
1791 INIT_WORK(&q_vector->tx.dim.work, wx_tx_dim_work);
1792 q_vector->rx.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1793 q_vector->tx.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1794 napi_enable(&q_vector->napi);
1795 }
1796 }
1797 EXPORT_SYMBOL(wx_napi_enable_all);
1798
wx_napi_disable_all(struct wx * wx)1799 void wx_napi_disable_all(struct wx *wx)
1800 {
1801 struct wx_q_vector *q_vector;
1802 int q_idx;
1803
1804 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1805 q_vector = wx->q_vector[q_idx];
1806 napi_disable(&q_vector->napi);
1807 disable_work_sync(&q_vector->rx.dim.work);
1808 disable_work_sync(&q_vector->tx.dim.work);
1809 }
1810 }
1811 EXPORT_SYMBOL(wx_napi_disable_all);
1812
wx_set_vmdq_queues(struct wx * wx)1813 static bool wx_set_vmdq_queues(struct wx *wx)
1814 {
1815 u16 vmdq_i = wx->ring_feature[RING_F_VMDQ].limit;
1816 u16 rss_i = wx->ring_feature[RING_F_RSS].limit;
1817 u16 rss_m = WX_RSS_DISABLED_MASK;
1818 u16 vmdq_m = 0;
1819
1820 /* only proceed if VMDq is enabled */
1821 if (!test_bit(WX_FLAG_VMDQ_ENABLED, wx->flags))
1822 return false;
1823 /* Add starting offset to total pool count */
1824 vmdq_i += wx->ring_feature[RING_F_VMDQ].offset;
1825
1826 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
1827 /* double check we are limited to maximum pools */
1828 vmdq_i = min_t(u16, 64, vmdq_i);
1829
1830 /* 64 pool mode with 2 queues per pool, or
1831 * 16/32/64 pool mode with 1 queue per pool
1832 */
1833 if (vmdq_i > 32 || rss_i < 4) {
1834 vmdq_m = WX_VMDQ_2Q_MASK;
1835 rss_m = WX_RSS_2Q_MASK;
1836 rss_i = min_t(u16, rss_i, 2);
1837 /* 32 pool mode with 4 queues per pool */
1838 } else {
1839 vmdq_m = WX_VMDQ_4Q_MASK;
1840 rss_m = WX_RSS_4Q_MASK;
1841 rss_i = 4;
1842 }
1843 } else {
1844 vmdq_m = WX_VMDQ_1Q_MASK;
1845 /* double check we are limited to maximum pools */
1846 vmdq_i = min_t(u16, 8, vmdq_i);
1847
1848 /* when VMDQ on, disable RSS */
1849 rss_i = 1;
1850 }
1851
1852 /* remove the starting offset from the pool count */
1853 vmdq_i -= wx->ring_feature[RING_F_VMDQ].offset;
1854
1855 /* save features for later use */
1856 wx->ring_feature[RING_F_VMDQ].indices = vmdq_i;
1857 wx->ring_feature[RING_F_VMDQ].mask = vmdq_m;
1858
1859 /* limit RSS based on user input and save for later use */
1860 wx->ring_feature[RING_F_RSS].indices = rss_i;
1861 wx->ring_feature[RING_F_RSS].mask = rss_m;
1862
1863 wx->queues_per_pool = rss_i;/*maybe same to num_rx_queues_per_pool*/
1864 wx->num_rx_pools = vmdq_i;
1865 wx->num_rx_queues_per_pool = rss_i;
1866
1867 wx->num_rx_queues = vmdq_i * rss_i;
1868 wx->num_tx_queues = vmdq_i * rss_i;
1869
1870 return true;
1871 }
1872
1873 /**
1874 * wx_set_rss_queues: Allocate queues for RSS
1875 * @wx: board private structure to initialize
1876 *
1877 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1878 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1879 *
1880 **/
wx_set_rss_queues(struct wx * wx)1881 static void wx_set_rss_queues(struct wx *wx)
1882 {
1883 struct wx_ring_feature *f;
1884
1885 /* set mask for 16 queue limit of RSS */
1886 f = &wx->ring_feature[RING_F_RSS];
1887 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags))
1888 f->mask = WX_RSS_64Q_MASK;
1889 else
1890 f->mask = WX_RSS_8Q_MASK;
1891 f->indices = f->limit;
1892
1893 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
1894 goto out;
1895
1896 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
1897
1898 wx->ring_feature[RING_F_FDIR].indices = 1;
1899 /* Use Flow Director in addition to RSS to ensure the best
1900 * distribution of flows across cores, even when an FDIR flow
1901 * isn't matched.
1902 */
1903 if (f->indices > 1) {
1904 f = &wx->ring_feature[RING_F_FDIR];
1905
1906 f->indices = f->limit;
1907
1908 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
1909 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
1910 }
1911
1912 out:
1913 wx->num_rx_queues = f->indices;
1914 wx->num_tx_queues = f->indices;
1915 }
1916
wx_set_num_queues(struct wx * wx)1917 static void wx_set_num_queues(struct wx *wx)
1918 {
1919 /* Start with base case */
1920 wx->num_rx_queues = 1;
1921 wx->num_tx_queues = 1;
1922 wx->queues_per_pool = 1;
1923
1924 if (wx_set_vmdq_queues(wx))
1925 return;
1926
1927 wx_set_rss_queues(wx);
1928 }
1929
1930 /**
1931 * wx_acquire_msix_vectors - acquire MSI-X vectors
1932 * @wx: board private structure
1933 *
1934 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1935 * return a negative error code if unable to acquire MSI-X vectors for any
1936 * reason.
1937 */
wx_acquire_msix_vectors(struct wx * wx)1938 static int wx_acquire_msix_vectors(struct wx *wx)
1939 {
1940 struct irq_affinity affd = { .post_vectors = 1 };
1941 int nvecs, i;
1942
1943 /* We start by asking for one vector per queue pair */
1944 nvecs = max(wx->num_rx_queues, wx->num_tx_queues);
1945 nvecs = min_t(int, nvecs, num_online_cpus());
1946 nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);
1947
1948 wx->msix_q_entries = kzalloc_objs(struct msix_entry, nvecs);
1949 if (!wx->msix_q_entries)
1950 return -ENOMEM;
1951
1952 /* One for non-queue interrupts */
1953 nvecs += 1;
1954
1955 wx->msix_entry = kzalloc_objs(struct msix_entry, 1);
1956 if (!wx->msix_entry) {
1957 kfree(wx->msix_q_entries);
1958 wx->msix_q_entries = NULL;
1959 return -ENOMEM;
1960 }
1961
1962 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1963 nvecs,
1964 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1965 &affd);
1966 if (nvecs < 0) {
1967 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1968 kfree(wx->msix_q_entries);
1969 wx->msix_q_entries = NULL;
1970 kfree(wx->msix_entry);
1971 wx->msix_entry = NULL;
1972 return nvecs;
1973 }
1974
1975 nvecs -= 1;
1976 for (i = 0; i < nvecs; i++) {
1977 wx->msix_q_entries[i].entry = i;
1978 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i);
1979 }
1980
1981 wx->num_q_vectors = nvecs;
1982
1983 wx->msix_entry->entry = nvecs;
1984 wx->msix_entry->vector = pci_irq_vector(wx->pdev, nvecs);
1985
1986 if (test_bit(WX_FLAG_IRQ_VECTOR_SHARED, wx->flags)) {
1987 wx->msix_entry->entry = 0;
1988 wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);
1989 wx->msix_q_entries[0].entry = 0;
1990 wx->msix_q_entries[0].vector = pci_irq_vector(wx->pdev, 1);
1991 }
1992
1993 return 0;
1994 }
1995
1996 /**
1997 * wx_set_interrupt_capability - set MSI-X or MSI if supported
1998 * @wx: board private structure to initialize
1999 *
2000 * Attempt to configure the interrupts using the best available
2001 * capabilities of the hardware and the kernel.
2002 **/
wx_set_interrupt_capability(struct wx * wx)2003 static int wx_set_interrupt_capability(struct wx *wx)
2004 {
2005 struct pci_dev *pdev = wx->pdev;
2006 int nvecs, ret;
2007
2008 /* We will try to get MSI-X interrupts first */
2009 ret = wx_acquire_msix_vectors(wx);
2010 if (ret == 0 || (ret == -ENOMEM) || pdev->is_virtfn)
2011 return ret;
2012
2013 /* Disable VMDq support */
2014 dev_warn(&wx->pdev->dev, "Disabling VMQQ support\n");
2015 clear_bit(WX_FLAG_VMDQ_ENABLED, wx->flags);
2016
2017 /* Disable RSS */
2018 dev_warn(&wx->pdev->dev, "Disabling RSS support\n");
2019 wx->ring_feature[RING_F_RSS].limit = 1;
2020
2021 wx_set_num_queues(wx);
2022
2023 /* minmum one for queue, one for misc*/
2024 nvecs = 1;
2025 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
2026 nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);
2027 if (nvecs == 1) {
2028 if (pdev->msi_enabled)
2029 wx_err(wx, "Fallback to MSI.\n");
2030 else
2031 wx_err(wx, "Fallback to INTx.\n");
2032 } else {
2033 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);
2034 return nvecs;
2035 }
2036
2037 pdev->irq = pci_irq_vector(pdev, 0);
2038 wx->num_q_vectors = 1;
2039
2040 return 0;
2041 }
2042
wx_cache_ring_vmdq(struct wx * wx)2043 static bool wx_cache_ring_vmdq(struct wx *wx)
2044 {
2045 struct wx_ring_feature *vmdq = &wx->ring_feature[RING_F_VMDQ];
2046 struct wx_ring_feature *rss = &wx->ring_feature[RING_F_RSS];
2047 u16 reg_idx;
2048 int i;
2049
2050 /* only proceed if VMDq is enabled */
2051 if (!test_bit(WX_FLAG_VMDQ_ENABLED, wx->flags))
2052 return false;
2053
2054 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
2055 /* start at VMDq register offset for SR-IOV enabled setups */
2056 reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
2057 for (i = 0; i < wx->num_rx_queues; i++, reg_idx++) {
2058 /* If we are greater than indices move to next pool */
2059 if ((reg_idx & ~vmdq->mask) >= rss->indices)
2060 reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
2061 wx->rx_ring[i]->reg_idx = reg_idx;
2062 }
2063 reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
2064 for (i = 0; i < wx->num_tx_queues; i++, reg_idx++) {
2065 /* If we are greater than indices move to next pool */
2066 if ((reg_idx & rss->mask) >= rss->indices)
2067 reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
2068 wx->tx_ring[i]->reg_idx = reg_idx;
2069 }
2070 } else {
2071 /* start at VMDq register offset for SR-IOV enabled setups */
2072 reg_idx = vmdq->offset;
2073 for (i = 0; i < wx->num_rx_queues; i++)
2074 /* If we are greater than indices move to next pool */
2075 wx->rx_ring[i]->reg_idx = reg_idx + i;
2076
2077 reg_idx = vmdq->offset;
2078 for (i = 0; i < wx->num_tx_queues; i++)
2079 /* If we are greater than indices move to next pool */
2080 wx->tx_ring[i]->reg_idx = reg_idx + i;
2081 }
2082
2083 return true;
2084 }
2085
2086 /**
2087 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
2088 * @wx: board private structure to initialize
2089 *
2090 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
2091 *
2092 **/
wx_cache_ring_rss(struct wx * wx)2093 static void wx_cache_ring_rss(struct wx *wx)
2094 {
2095 u16 i;
2096
2097 if (wx_cache_ring_vmdq(wx))
2098 return;
2099
2100 for (i = 0; i < wx->num_rx_queues; i++)
2101 wx->rx_ring[i]->reg_idx = i;
2102
2103 for (i = 0; i < wx->num_tx_queues; i++)
2104 wx->tx_ring[i]->reg_idx = i;
2105 }
2106
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)2107 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
2108 {
2109 ring->next = head->ring;
2110 head->ring = ring;
2111 head->count++;
2112 }
2113
2114 /**
2115 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
2116 * @wx: board private structure to initialize
2117 * @v_count: q_vectors allocated on wx, used for ring interleaving
2118 * @v_idx: index of vector in wx struct
2119 * @txr_count: total number of Tx rings to allocate
2120 * @txr_idx: index of first Tx ring to allocate
2121 * @rxr_count: total number of Rx rings to allocate
2122 * @rxr_idx: index of first Rx ring to allocate
2123 *
2124 * We allocate one q_vector. If allocation fails we return -ENOMEM.
2125 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)2126 static int wx_alloc_q_vector(struct wx *wx,
2127 unsigned int v_count, unsigned int v_idx,
2128 unsigned int txr_count, unsigned int txr_idx,
2129 unsigned int rxr_count, unsigned int rxr_idx)
2130 {
2131 struct wx_q_vector *q_vector;
2132 int ring_count, default_itr;
2133 struct wx_ring *ring;
2134
2135 /* note this will allocate space for the ring structure as well! */
2136 ring_count = txr_count + rxr_count;
2137
2138 q_vector = kzalloc_flex(*q_vector, ring, ring_count);
2139 if (!q_vector)
2140 return -ENOMEM;
2141
2142 /* initialize NAPI */
2143 netif_napi_add(wx->netdev, &q_vector->napi,
2144 wx_poll);
2145
2146 /* tie q_vector and wx together */
2147 wx->q_vector[v_idx] = q_vector;
2148 q_vector->wx = wx;
2149 q_vector->v_idx = v_idx;
2150 if (cpu_online(v_idx))
2151 q_vector->numa_node = cpu_to_node(v_idx);
2152
2153 /* initialize pointer to rings */
2154 ring = q_vector->ring;
2155
2156 switch (wx->mac.type) {
2157 case wx_mac_sp:
2158 case wx_mac_aml:
2159 case wx_mac_aml40:
2160 default_itr = WX_12K_ITR;
2161 break;
2162 default:
2163 default_itr = WX_7K_ITR;
2164 break;
2165 }
2166
2167 /* initialize ITR */
2168 if (txr_count && !rxr_count)
2169 /* tx only vector */
2170 q_vector->itr = wx->tx_itr_setting ?
2171 default_itr : wx->tx_itr_setting;
2172 else
2173 /* rx or rx/tx vector */
2174 q_vector->itr = wx->rx_itr_setting ?
2175 default_itr : wx->rx_itr_setting;
2176
2177 while (txr_count) {
2178 /* assign generic ring traits */
2179 ring->dev = &wx->pdev->dev;
2180 ring->netdev = wx->netdev;
2181
2182 /* configure backlink on ring */
2183 ring->q_vector = q_vector;
2184
2185 /* update q_vector Tx values */
2186 wx_add_ring(ring, &q_vector->tx);
2187
2188 /* apply Tx specific ring traits */
2189 ring->count = wx->tx_ring_count;
2190
2191 ring->queue_index = txr_idx;
2192
2193 /* assign ring to wx */
2194 wx->tx_ring[txr_idx] = ring;
2195
2196 /* update count and index */
2197 txr_count--;
2198 txr_idx += v_count;
2199
2200 /* push pointer to next ring */
2201 ring++;
2202 }
2203
2204 while (rxr_count) {
2205 /* assign generic ring traits */
2206 ring->dev = &wx->pdev->dev;
2207 ring->netdev = wx->netdev;
2208
2209 /* configure backlink on ring */
2210 ring->q_vector = q_vector;
2211
2212 /* update q_vector Rx values */
2213 wx_add_ring(ring, &q_vector->rx);
2214
2215 /* apply Rx specific ring traits */
2216 ring->count = wx->rx_ring_count;
2217 ring->queue_index = rxr_idx;
2218
2219 /* assign ring to wx */
2220 wx->rx_ring[rxr_idx] = ring;
2221
2222 /* update count and index */
2223 rxr_count--;
2224 rxr_idx += v_count;
2225
2226 /* push pointer to next ring */
2227 ring++;
2228 }
2229
2230 return 0;
2231 }
2232
2233 /**
2234 * wx_free_q_vector - Free memory allocated for specific interrupt vector
2235 * @wx: board private structure to initialize
2236 * @v_idx: Index of vector to be freed
2237 *
2238 * This function frees the memory allocated to the q_vector. In addition if
2239 * NAPI is enabled it will delete any references to the NAPI struct prior
2240 * to freeing the q_vector.
2241 **/
wx_free_q_vector(struct wx * wx,int v_idx)2242 static void wx_free_q_vector(struct wx *wx, int v_idx)
2243 {
2244 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2245 struct wx_ring *ring;
2246
2247 wx_for_each_ring(ring, q_vector->tx)
2248 wx->tx_ring[ring->queue_index] = NULL;
2249
2250 wx_for_each_ring(ring, q_vector->rx)
2251 wx->rx_ring[ring->queue_index] = NULL;
2252
2253 wx->q_vector[v_idx] = NULL;
2254 netif_napi_del(&q_vector->napi);
2255 kfree_rcu(q_vector, rcu);
2256 }
2257
2258 /**
2259 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
2260 * @wx: board private structure to initialize
2261 *
2262 * We allocate one q_vector per queue interrupt. If allocation fails we
2263 * return -ENOMEM.
2264 **/
wx_alloc_q_vectors(struct wx * wx)2265 static int wx_alloc_q_vectors(struct wx *wx)
2266 {
2267 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
2268 unsigned int rxr_remaining = wx->num_rx_queues;
2269 unsigned int txr_remaining = wx->num_tx_queues;
2270 unsigned int q_vectors = wx->num_q_vectors;
2271 int rqpv, tqpv;
2272 int err;
2273
2274 for (; v_idx < q_vectors; v_idx++) {
2275 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
2276 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
2277 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
2278 tqpv, txr_idx,
2279 rqpv, rxr_idx);
2280
2281 if (err)
2282 goto err_out;
2283
2284 /* update counts and index */
2285 rxr_remaining -= rqpv;
2286 txr_remaining -= tqpv;
2287 rxr_idx++;
2288 txr_idx++;
2289 }
2290
2291 return 0;
2292
2293 err_out:
2294 wx->num_tx_queues = 0;
2295 wx->num_rx_queues = 0;
2296 wx->num_q_vectors = 0;
2297
2298 while (v_idx--)
2299 wx_free_q_vector(wx, v_idx);
2300
2301 return -ENOMEM;
2302 }
2303
2304 /**
2305 * wx_free_q_vectors - Free memory allocated for interrupt vectors
2306 * @wx: board private structure to initialize
2307 *
2308 * This function frees the memory allocated to the q_vectors. In addition if
2309 * NAPI is enabled it will delete any references to the NAPI struct prior
2310 * to freeing the q_vector.
2311 **/
wx_free_q_vectors(struct wx * wx)2312 static void wx_free_q_vectors(struct wx *wx)
2313 {
2314 int v_idx = wx->num_q_vectors;
2315
2316 wx->num_tx_queues = 0;
2317 wx->num_rx_queues = 0;
2318 wx->num_q_vectors = 0;
2319
2320 while (v_idx--)
2321 wx_free_q_vector(wx, v_idx);
2322 }
2323
wx_reset_interrupt_capability(struct wx * wx)2324 void wx_reset_interrupt_capability(struct wx *wx)
2325 {
2326 struct pci_dev *pdev = wx->pdev;
2327
2328 if (!pdev->msi_enabled && !pdev->msix_enabled)
2329 return;
2330
2331 if (pdev->msix_enabled) {
2332 kfree(wx->msix_q_entries);
2333 wx->msix_q_entries = NULL;
2334 kfree(wx->msix_entry);
2335 wx->msix_entry = NULL;
2336 }
2337 pci_free_irq_vectors(wx->pdev);
2338 }
2339 EXPORT_SYMBOL(wx_reset_interrupt_capability);
2340
2341 /**
2342 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
2343 * @wx: board private structure to clear interrupt scheme on
2344 *
2345 * We go through and clear interrupt specific resources and reset the structure
2346 * to pre-load conditions
2347 **/
wx_clear_interrupt_scheme(struct wx * wx)2348 void wx_clear_interrupt_scheme(struct wx *wx)
2349 {
2350 wx_free_q_vectors(wx);
2351 wx_reset_interrupt_capability(wx);
2352 }
2353 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
2354
wx_init_interrupt_scheme(struct wx * wx)2355 int wx_init_interrupt_scheme(struct wx *wx)
2356 {
2357 int ret;
2358
2359 /* Number of supported queues */
2360 if (wx->pdev->is_virtfn) {
2361 if (wx->set_num_queues)
2362 wx->set_num_queues(wx);
2363 } else {
2364 wx_set_num_queues(wx);
2365 }
2366
2367 /* Set interrupt mode */
2368 ret = wx_set_interrupt_capability(wx);
2369 if (ret) {
2370 wx_err(wx, "Allocate irq vectors for failed.\n");
2371 return ret;
2372 }
2373
2374 /* Allocate memory for queues */
2375 ret = wx_alloc_q_vectors(wx);
2376 if (ret) {
2377 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
2378 wx_reset_interrupt_capability(wx);
2379 return ret;
2380 }
2381
2382 wx_cache_ring_rss(wx);
2383
2384 set_bit(WX_STATE_DOWN, wx->state);
2385
2386 return 0;
2387 }
2388 EXPORT_SYMBOL(wx_init_interrupt_scheme);
2389
wx_msix_clean_rings(int __always_unused irq,void * data)2390 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
2391 {
2392 struct wx_q_vector *q_vector = data;
2393
2394 /* EIAM disabled interrupts (on this vector) for us */
2395 if (q_vector->rx.ring || q_vector->tx.ring) {
2396 napi_schedule_irqoff(&q_vector->napi);
2397 q_vector->total_events++;
2398 }
2399
2400 return IRQ_HANDLED;
2401 }
2402 EXPORT_SYMBOL(wx_msix_clean_rings);
2403
wx_free_irq(struct wx * wx)2404 void wx_free_irq(struct wx *wx)
2405 {
2406 struct pci_dev *pdev = wx->pdev;
2407 int vector;
2408
2409 if (!(pdev->msix_enabled)) {
2410 if (!wx->misc_irq_domain)
2411 free_irq(pdev->irq, wx);
2412 return;
2413 }
2414
2415 for (vector = 0; vector < wx->num_q_vectors; vector++) {
2416 struct wx_q_vector *q_vector = wx->q_vector[vector];
2417 struct msix_entry *entry = &wx->msix_q_entries[vector];
2418
2419 /* free only the irqs that were actually requested */
2420 if (!q_vector->rx.ring && !q_vector->tx.ring)
2421 continue;
2422
2423 free_irq(entry->vector, q_vector);
2424 }
2425
2426 if (!wx->misc_irq_domain)
2427 free_irq(wx->msix_entry->vector, wx);
2428 }
2429 EXPORT_SYMBOL(wx_free_irq);
2430
2431 /**
2432 * wx_setup_isb_resources - allocate interrupt status resources
2433 * @wx: board private structure
2434 *
2435 * Return 0 on success, negative on failure
2436 **/
wx_setup_isb_resources(struct wx * wx)2437 int wx_setup_isb_resources(struct wx *wx)
2438 {
2439 struct pci_dev *pdev = wx->pdev;
2440
2441 if (wx->isb_mem)
2442 return 0;
2443
2444 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2445 sizeof(u32) * 4,
2446 &wx->isb_dma,
2447 GFP_KERNEL);
2448 if (!wx->isb_mem) {
2449 wx_err(wx, "Alloc isb_mem failed\n");
2450 return -ENOMEM;
2451 }
2452
2453 return 0;
2454 }
2455 EXPORT_SYMBOL(wx_setup_isb_resources);
2456
2457 /**
2458 * wx_free_isb_resources - allocate all queues Rx resources
2459 * @wx: board private structure
2460 *
2461 * Return 0 on success, negative on failure
2462 **/
wx_free_isb_resources(struct wx * wx)2463 void wx_free_isb_resources(struct wx *wx)
2464 {
2465 struct pci_dev *pdev = wx->pdev;
2466
2467 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2468 wx->isb_mem, wx->isb_dma);
2469 wx->isb_mem = NULL;
2470 }
2471 EXPORT_SYMBOL(wx_free_isb_resources);
2472
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2473 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2474 {
2475 u32 cur_tag = 0;
2476
2477 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2478 wx->isb_tag[idx] = cur_tag;
2479
2480 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2481 }
2482 EXPORT_SYMBOL(wx_misc_isb);
2483
2484 /**
2485 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2486 * @wx: pointer to wx struct
2487 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2488 * @queue: queue to map the corresponding interrupt to
2489 * @msix_vector: the vector to map to the corresponding queue
2490 *
2491 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2492 static void wx_set_ivar(struct wx *wx, s8 direction,
2493 u16 queue, u16 msix_vector)
2494 {
2495 u32 ivar, index;
2496
2497 if (direction == -1) {
2498 /* other causes */
2499 if (test_bit(WX_FLAG_IRQ_VECTOR_SHARED, wx->flags))
2500 msix_vector = 0;
2501 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2502 index = 0;
2503 ivar = rd32(wx, WX_PX_MISC_IVAR);
2504 ivar &= ~(0xFF << index);
2505 ivar |= (msix_vector << index);
2506 wr32(wx, WX_PX_MISC_IVAR, ivar);
2507 } else {
2508 /* tx or rx causes */
2509 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2510 index = ((16 * (queue & 1)) + (8 * direction));
2511 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2512 ivar &= ~(0xFF << index);
2513 ivar |= (msix_vector << index);
2514 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2515 }
2516 }
2517
2518 /**
2519 * wx_write_eitr - write EITR register in hardware specific way
2520 * @q_vector: structure containing interrupt and ring information
2521 *
2522 * This function is made to be called by ethtool and by the driver
2523 * when it needs to update EITR registers at runtime. Hardware
2524 * specific quirks/differences are taken care of here.
2525 */
wx_write_eitr(struct wx_q_vector * q_vector)2526 void wx_write_eitr(struct wx_q_vector *q_vector)
2527 {
2528 struct wx *wx = q_vector->wx;
2529 int v_idx = q_vector->v_idx;
2530 u32 itr_reg;
2531
2532 switch (wx->mac.type) {
2533 case wx_mac_sp:
2534 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2535 break;
2536 case wx_mac_aml:
2537 case wx_mac_aml40:
2538 itr_reg = (q_vector->itr >> 3) & WX_AML_MAX_EITR;
2539 break;
2540 default:
2541 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2542 break;
2543 }
2544
2545 itr_reg |= WX_PX_ITR_CNT_WDIS;
2546
2547 wr32(wx, WX_PX_ITR(v_idx), itr_reg);
2548 }
2549
2550 /**
2551 * wx_configure_vectors - Configure vectors for hardware
2552 * @wx: board private structure
2553 *
2554 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx
2555 * interrupts.
2556 **/
wx_configure_vectors(struct wx * wx)2557 void wx_configure_vectors(struct wx *wx)
2558 {
2559 struct pci_dev *pdev = wx->pdev;
2560 u32 eitrsel = 0;
2561 u16 v_idx, i;
2562
2563 if (pdev->msix_enabled) {
2564 /* Populate MSIX to EITR Select */
2565 if (test_bit(WX_FLAG_MULTI_64_FUNC, wx->flags)) {
2566 if (wx->num_vfs >= 32)
2567 eitrsel = BIT(wx->num_vfs % 32) - 1;
2568 } else {
2569 for (i = 0; i < wx->num_vfs; i++)
2570 eitrsel |= BIT(i);
2571 }
2572 wr32(wx, WX_PX_ITRSEL, eitrsel);
2573 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2574 * this saves a register write for every interrupt
2575 */
2576 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2577 } else {
2578 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2579 * specifically only auto mask tx and rx interrupts.
2580 */
2581 wr32(wx, WX_PX_GPIE, 0);
2582 }
2583
2584 /* Populate the IVAR table and set the ITR values to the
2585 * corresponding register.
2586 */
2587 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2588 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2589 struct wx_ring *ring;
2590
2591 wx_for_each_ring(ring, q_vector->rx)
2592 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2593
2594 wx_for_each_ring(ring, q_vector->tx)
2595 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2596
2597 wx_write_eitr(q_vector);
2598 }
2599
2600 wx_set_ivar(wx, -1, 0, v_idx);
2601 if (pdev->msix_enabled)
2602 wr32(wx, WX_PX_ITR(v_idx), 1950);
2603 }
2604 EXPORT_SYMBOL(wx_configure_vectors);
2605
2606 /**
2607 * wx_clean_rx_ring - Free Rx Buffers per Queue
2608 * @rx_ring: ring to free buffers from
2609 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2610 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2611 {
2612 struct wx_rx_buffer *rx_buffer;
2613 u16 i = rx_ring->next_to_clean;
2614
2615 rx_buffer = &rx_ring->rx_buffer_info[i];
2616
2617 /* Free all the Rx ring sk_buffs */
2618 while (i != rx_ring->next_to_alloc) {
2619 if (rx_buffer->skb) {
2620 struct sk_buff *skb = rx_buffer->skb;
2621
2622 dev_kfree_skb(skb);
2623 }
2624
2625 /* Invalidate cache lines that may have been written to by
2626 * device so that we avoid corrupting memory.
2627 */
2628 dma_sync_single_range_for_cpu(rx_ring->dev,
2629 rx_buffer->dma,
2630 rx_buffer->page_offset,
2631 rx_ring->rx_buf_len,
2632 DMA_FROM_DEVICE);
2633
2634 /* free resources associated with mapping */
2635 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2636
2637 i++;
2638 rx_buffer++;
2639 if (i == rx_ring->count) {
2640 i = 0;
2641 rx_buffer = rx_ring->rx_buffer_info;
2642 }
2643 }
2644
2645 /* Zero out the descriptor ring */
2646 memset(rx_ring->desc, 0, rx_ring->size);
2647
2648 rx_ring->next_to_alloc = 0;
2649 rx_ring->next_to_clean = 0;
2650 rx_ring->next_to_use = 0;
2651 }
2652
2653 /**
2654 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2655 * @wx: board private structure
2656 **/
wx_clean_all_rx_rings(struct wx * wx)2657 void wx_clean_all_rx_rings(struct wx *wx)
2658 {
2659 int i;
2660
2661 for (i = 0; i < wx->num_rx_queues; i++)
2662 wx_clean_rx_ring(wx->rx_ring[i]);
2663 }
2664 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2665
2666 /**
2667 * wx_free_rx_resources - Free Rx Resources
2668 * @rx_ring: ring to clean the resources from
2669 *
2670 * Free all receive software resources
2671 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2672 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2673 {
2674 wx_clean_rx_ring(rx_ring);
2675 kvfree(rx_ring->rx_buffer_info);
2676 rx_ring->rx_buffer_info = NULL;
2677
2678 /* if not set, then don't free */
2679 if (!rx_ring->desc)
2680 return;
2681
2682 dma_free_coherent(rx_ring->dev, rx_ring->size,
2683 rx_ring->desc, rx_ring->dma);
2684
2685 rx_ring->desc = NULL;
2686
2687 if (rx_ring->page_pool) {
2688 page_pool_destroy(rx_ring->page_pool);
2689 rx_ring->page_pool = NULL;
2690 }
2691 }
2692
2693 /**
2694 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2695 * @wx: pointer to hardware structure
2696 *
2697 * Free all receive software resources
2698 **/
wx_free_all_rx_resources(struct wx * wx)2699 static void wx_free_all_rx_resources(struct wx *wx)
2700 {
2701 int i;
2702
2703 for (i = 0; i < wx->num_rx_queues; i++)
2704 wx_free_rx_resources(wx->rx_ring[i]);
2705 }
2706
2707 /**
2708 * wx_clean_tx_ring - Free Tx Buffers
2709 * @tx_ring: ring to be cleaned
2710 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2711 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2712 {
2713 struct wx_tx_buffer *tx_buffer;
2714 u16 i = tx_ring->next_to_clean;
2715
2716 tx_buffer = &tx_ring->tx_buffer_info[i];
2717
2718 while (i != tx_ring->next_to_use) {
2719 union wx_tx_desc *eop_desc, *tx_desc;
2720
2721 /* Free all the Tx ring sk_buffs */
2722 dev_kfree_skb_any(tx_buffer->skb);
2723
2724 /* unmap skb header data */
2725 dma_unmap_single(tx_ring->dev,
2726 dma_unmap_addr(tx_buffer, dma),
2727 dma_unmap_len(tx_buffer, len),
2728 DMA_TO_DEVICE);
2729
2730 /* check for eop_desc to determine the end of the packet */
2731 eop_desc = tx_buffer->next_to_watch;
2732 tx_desc = WX_TX_DESC(tx_ring, i);
2733
2734 /* unmap remaining buffers */
2735 while (tx_desc != eop_desc) {
2736 tx_buffer++;
2737 tx_desc++;
2738 i++;
2739 if (unlikely(i == tx_ring->count)) {
2740 i = 0;
2741 tx_buffer = tx_ring->tx_buffer_info;
2742 tx_desc = WX_TX_DESC(tx_ring, 0);
2743 }
2744
2745 /* unmap any remaining paged data */
2746 if (dma_unmap_len(tx_buffer, len))
2747 dma_unmap_page(tx_ring->dev,
2748 dma_unmap_addr(tx_buffer, dma),
2749 dma_unmap_len(tx_buffer, len),
2750 DMA_TO_DEVICE);
2751 }
2752
2753 /* move us one more past the eop_desc for start of next pkt */
2754 tx_buffer++;
2755 i++;
2756 if (unlikely(i == tx_ring->count)) {
2757 i = 0;
2758 tx_buffer = tx_ring->tx_buffer_info;
2759 }
2760 }
2761
2762 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2763
2764 /* reset next_to_use and next_to_clean */
2765 tx_ring->next_to_use = 0;
2766 tx_ring->next_to_clean = 0;
2767 }
2768
2769 /**
2770 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2771 * @wx: board private structure
2772 **/
wx_clean_all_tx_rings(struct wx * wx)2773 void wx_clean_all_tx_rings(struct wx *wx)
2774 {
2775 int i;
2776
2777 for (i = 0; i < wx->num_tx_queues; i++)
2778 wx_clean_tx_ring(wx->tx_ring[i]);
2779 }
2780 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2781
wx_free_headwb_resources(struct wx_ring * tx_ring)2782 static void wx_free_headwb_resources(struct wx_ring *tx_ring)
2783 {
2784 if (!tx_ring->headwb_mem)
2785 return;
2786
2787 dma_free_coherent(tx_ring->dev, sizeof(u32),
2788 tx_ring->headwb_mem, tx_ring->headwb_dma);
2789 tx_ring->headwb_mem = NULL;
2790 }
2791
2792 /**
2793 * wx_free_tx_resources - Free Tx Resources per Queue
2794 * @tx_ring: Tx descriptor ring for a specific queue
2795 *
2796 * Free all transmit software resources
2797 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2798 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2799 {
2800 wx_clean_tx_ring(tx_ring);
2801 kvfree(tx_ring->tx_buffer_info);
2802 tx_ring->tx_buffer_info = NULL;
2803
2804 /* if not set, then don't free */
2805 if (!tx_ring->desc)
2806 return;
2807
2808 dma_free_coherent(tx_ring->dev, tx_ring->size,
2809 tx_ring->desc, tx_ring->dma);
2810 tx_ring->desc = NULL;
2811
2812 wx_free_headwb_resources(tx_ring);
2813 }
2814
2815 /**
2816 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2817 * @wx: pointer to hardware structure
2818 *
2819 * Free all transmit software resources
2820 **/
wx_free_all_tx_resources(struct wx * wx)2821 static void wx_free_all_tx_resources(struct wx *wx)
2822 {
2823 int i;
2824
2825 for (i = 0; i < wx->num_tx_queues; i++)
2826 wx_free_tx_resources(wx->tx_ring[i]);
2827 }
2828
wx_free_resources(struct wx * wx)2829 void wx_free_resources(struct wx *wx)
2830 {
2831 wx_free_all_rx_resources(wx);
2832 wx_free_all_tx_resources(wx);
2833 }
2834 EXPORT_SYMBOL(wx_free_resources);
2835
wx_alloc_page_pool(struct wx_ring * rx_ring)2836 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2837 {
2838 int ret = 0;
2839
2840 struct page_pool_params pp_params = {
2841 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2842 .order = wx_rx_pg_order(rx_ring),
2843 .pool_size = rx_ring->count * rx_ring->rx_buf_len /
2844 wx_rx_pg_size(rx_ring),
2845 .nid = dev_to_node(rx_ring->dev),
2846 .dev = rx_ring->dev,
2847 .dma_dir = DMA_FROM_DEVICE,
2848 .offset = 0,
2849 .max_len = wx_rx_pg_size(rx_ring),
2850 };
2851
2852 rx_ring->page_pool = page_pool_create(&pp_params);
2853 if (IS_ERR(rx_ring->page_pool)) {
2854 ret = PTR_ERR(rx_ring->page_pool);
2855 rx_ring->page_pool = NULL;
2856 }
2857
2858 return ret;
2859 }
2860
2861 /**
2862 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2863 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2864 *
2865 * Returns 0 on success, negative on failure
2866 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2867 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2868 {
2869 struct device *dev = rx_ring->dev;
2870 int orig_node = dev_to_node(dev);
2871 int numa_node = NUMA_NO_NODE;
2872 int size, ret;
2873
2874 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2875
2876 if (rx_ring->q_vector)
2877 numa_node = rx_ring->q_vector->numa_node;
2878
2879 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2880 if (!rx_ring->rx_buffer_info)
2881 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2882 if (!rx_ring->rx_buffer_info)
2883 goto err;
2884
2885 /* Round up to nearest 4K */
2886 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2887 rx_ring->size = ALIGN(rx_ring->size, 4096);
2888
2889 set_dev_node(dev, numa_node);
2890 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2891 &rx_ring->dma, GFP_KERNEL);
2892 if (!rx_ring->desc) {
2893 set_dev_node(dev, orig_node);
2894 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2895 &rx_ring->dma, GFP_KERNEL);
2896 }
2897
2898 if (!rx_ring->desc)
2899 goto err;
2900
2901 rx_ring->next_to_clean = 0;
2902 rx_ring->next_to_use = 0;
2903
2904 ret = wx_alloc_page_pool(rx_ring);
2905 if (ret < 0) {
2906 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2907 goto err_desc;
2908 }
2909
2910 return 0;
2911
2912 err_desc:
2913 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2914 err:
2915 kvfree(rx_ring->rx_buffer_info);
2916 rx_ring->rx_buffer_info = NULL;
2917 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2918 return -ENOMEM;
2919 }
2920
2921 /**
2922 * wx_setup_all_rx_resources - allocate all queues Rx resources
2923 * @wx: pointer to hardware structure
2924 *
2925 * If this function returns with an error, then it's possible one or
2926 * more of the rings is populated (while the rest are not). It is the
2927 * callers duty to clean those orphaned rings.
2928 *
2929 * Return 0 on success, negative on failure
2930 **/
wx_setup_all_rx_resources(struct wx * wx)2931 static int wx_setup_all_rx_resources(struct wx *wx)
2932 {
2933 int i, err = 0;
2934
2935 for (i = 0; i < wx->num_rx_queues; i++) {
2936 err = wx_setup_rx_resources(wx->rx_ring[i]);
2937 if (!err)
2938 continue;
2939
2940 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2941 goto err_setup_rx;
2942 }
2943
2944 return 0;
2945 err_setup_rx:
2946 /* rewind the index freeing the rings as we go */
2947 while (i--)
2948 wx_free_rx_resources(wx->rx_ring[i]);
2949 return err;
2950 }
2951
wx_setup_headwb_resources(struct wx_ring * tx_ring)2952 static void wx_setup_headwb_resources(struct wx_ring *tx_ring)
2953 {
2954 struct wx *wx = netdev_priv(tx_ring->netdev);
2955
2956 if (!test_bit(WX_FLAG_TXHEAD_WB_ENABLED, wx->flags))
2957 return;
2958
2959 if (!tx_ring->q_vector)
2960 return;
2961
2962 tx_ring->headwb_mem = dma_alloc_coherent(tx_ring->dev,
2963 sizeof(u32),
2964 &tx_ring->headwb_dma,
2965 GFP_KERNEL);
2966 if (!tx_ring->headwb_mem)
2967 dev_info(tx_ring->dev, "Allocate headwb memory failed, disable it\n");
2968 }
2969
2970 /**
2971 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2972 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2973 *
2974 * Return 0 on success, negative on failure
2975 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2976 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2977 {
2978 struct device *dev = tx_ring->dev;
2979 int orig_node = dev_to_node(dev);
2980 int numa_node = NUMA_NO_NODE;
2981 int size;
2982
2983 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2984
2985 if (tx_ring->q_vector)
2986 numa_node = tx_ring->q_vector->numa_node;
2987
2988 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2989 if (!tx_ring->tx_buffer_info)
2990 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2991 if (!tx_ring->tx_buffer_info)
2992 goto err;
2993
2994 /* round up to nearest 4K */
2995 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2996 tx_ring->size = ALIGN(tx_ring->size, 4096);
2997
2998 set_dev_node(dev, numa_node);
2999 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3000 &tx_ring->dma, GFP_KERNEL);
3001 if (!tx_ring->desc) {
3002 set_dev_node(dev, orig_node);
3003 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3004 &tx_ring->dma, GFP_KERNEL);
3005 }
3006
3007 if (!tx_ring->desc)
3008 goto err;
3009
3010 wx_setup_headwb_resources(tx_ring);
3011
3012 tx_ring->next_to_use = 0;
3013 tx_ring->next_to_clean = 0;
3014
3015 return 0;
3016
3017 err:
3018 kvfree(tx_ring->tx_buffer_info);
3019 tx_ring->tx_buffer_info = NULL;
3020 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
3021 return -ENOMEM;
3022 }
3023
3024 /**
3025 * wx_setup_all_tx_resources - allocate all queues Tx resources
3026 * @wx: pointer to private structure
3027 *
3028 * If this function returns with an error, then it's possible one or
3029 * more of the rings is populated (while the rest are not). It is the
3030 * callers duty to clean those orphaned rings.
3031 *
3032 * Return 0 on success, negative on failure
3033 **/
wx_setup_all_tx_resources(struct wx * wx)3034 static int wx_setup_all_tx_resources(struct wx *wx)
3035 {
3036 int i, err = 0;
3037
3038 for (i = 0; i < wx->num_tx_queues; i++) {
3039 err = wx_setup_tx_resources(wx->tx_ring[i]);
3040 if (!err)
3041 continue;
3042
3043 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
3044 goto err_setup_tx;
3045 }
3046
3047 return 0;
3048 err_setup_tx:
3049 /* rewind the index freeing the rings as we go */
3050 while (i--)
3051 wx_free_tx_resources(wx->tx_ring[i]);
3052 return err;
3053 }
3054
wx_setup_resources(struct wx * wx)3055 int wx_setup_resources(struct wx *wx)
3056 {
3057 int err;
3058
3059 /* allocate transmit descriptors */
3060 err = wx_setup_all_tx_resources(wx);
3061 if (err)
3062 return err;
3063
3064 /* allocate receive descriptors */
3065 err = wx_setup_all_rx_resources(wx);
3066 if (err)
3067 goto err_free_tx;
3068
3069 err = wx_setup_isb_resources(wx);
3070 if (err)
3071 goto err_free_rx;
3072
3073 return 0;
3074
3075 err_free_rx:
3076 wx_free_all_rx_resources(wx);
3077 err_free_tx:
3078 wx_free_all_tx_resources(wx);
3079
3080 return err;
3081 }
3082 EXPORT_SYMBOL(wx_setup_resources);
3083
3084 /**
3085 * wx_get_stats64 - Get System Network Statistics
3086 * @netdev: network interface device structure
3087 * @stats: storage space for 64bit statistics
3088 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)3089 void wx_get_stats64(struct net_device *netdev,
3090 struct rtnl_link_stats64 *stats)
3091 {
3092 struct wx *wx = netdev_priv(netdev);
3093 struct wx_hw_stats *hwstats;
3094 int i;
3095
3096 wx_update_stats(wx);
3097
3098 rcu_read_lock();
3099 for (i = 0; i < wx->num_rx_queues; i++) {
3100 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
3101 u64 bytes, packets;
3102 unsigned int start;
3103
3104 if (ring) {
3105 do {
3106 start = u64_stats_fetch_begin(&ring->syncp);
3107 packets = ring->stats.packets;
3108 bytes = ring->stats.bytes;
3109 } while (u64_stats_fetch_retry(&ring->syncp, start));
3110 stats->rx_packets += packets;
3111 stats->rx_bytes += bytes;
3112 }
3113 }
3114
3115 for (i = 0; i < wx->num_tx_queues; i++) {
3116 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
3117 u64 bytes, packets;
3118 unsigned int start;
3119
3120 if (ring) {
3121 do {
3122 start = u64_stats_fetch_begin(&ring->syncp);
3123 packets = ring->stats.packets;
3124 bytes = ring->stats.bytes;
3125 } while (u64_stats_fetch_retry(&ring->syncp,
3126 start));
3127 stats->tx_packets += packets;
3128 stats->tx_bytes += bytes;
3129 }
3130 }
3131
3132 rcu_read_unlock();
3133
3134 hwstats = &wx->stats;
3135 stats->rx_errors = hwstats->crcerrs + hwstats->rlec;
3136 stats->multicast = hwstats->qmprc;
3137 stats->rx_length_errors = hwstats->rlec;
3138 stats->rx_crc_errors = hwstats->crcerrs;
3139 }
3140 EXPORT_SYMBOL(wx_get_stats64);
3141
wx_set_features(struct net_device * netdev,netdev_features_t features)3142 int wx_set_features(struct net_device *netdev, netdev_features_t features)
3143 {
3144 netdev_features_t changed = netdev->features ^ features;
3145 struct wx *wx = netdev_priv(netdev);
3146 bool need_reset = false;
3147
3148 wx->rss_enabled = !!(features & NETIF_F_RXHASH);
3149 wx_enable_rss(wx, wx->rss_enabled);
3150
3151 netdev->features = features;
3152
3153 if (changed & NETIF_F_HW_VLAN_CTAG_RX && wx->do_reset)
3154 wx->do_reset(netdev, true);
3155 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))
3156 wx_set_rx_mode(netdev);
3157
3158 if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) {
3159 if (!(features & NETIF_F_LRO)) {
3160 if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags))
3161 need_reset = true;
3162 clear_bit(WX_FLAG_RSC_ENABLED, wx->flags);
3163 } else if (!(test_bit(WX_FLAG_RSC_ENABLED, wx->flags))) {
3164 if (wx->rx_itr_setting == 1 ||
3165 wx->rx_itr_setting > WX_MIN_RSC_ITR) {
3166 set_bit(WX_FLAG_RSC_ENABLED, wx->flags);
3167 need_reset = true;
3168 } else if (changed & NETIF_F_LRO) {
3169 dev_info(&wx->pdev->dev,
3170 "rx-usecs set too low, disable RSC\n");
3171 }
3172 }
3173 }
3174
3175 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
3176 goto out;
3177
3178 /* Check if Flow Director n-tuple support was enabled or disabled. If
3179 * the state changed, we need to reset.
3180 */
3181 switch (features & NETIF_F_NTUPLE) {
3182 case NETIF_F_NTUPLE:
3183 /* turn off ATR, enable perfect filters and reset */
3184 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
3185 need_reset = true;
3186
3187 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
3188 break;
3189 default:
3190 /* turn off perfect filters, enable ATR and reset */
3191 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))
3192 need_reset = true;
3193
3194 /* We cannot enable ATR if RSS is disabled */
3195 if (wx->ring_feature[RING_F_RSS].limit <= 1)
3196 break;
3197
3198 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
3199 break;
3200 }
3201
3202 out:
3203 if (need_reset && wx->do_reset)
3204 wx->do_reset(netdev, true);
3205
3206 return 0;
3207 }
3208 EXPORT_SYMBOL(wx_set_features);
3209
3210 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
3211 NETIF_F_HW_VLAN_STAG_RX)
3212
3213 #define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \
3214 NETIF_F_HW_VLAN_STAG_TX)
3215
3216 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \
3217 NETIF_F_HW_VLAN_STAG_FILTER)
3218
wx_fix_features(struct net_device * netdev,netdev_features_t features)3219 netdev_features_t wx_fix_features(struct net_device *netdev,
3220 netdev_features_t features)
3221 {
3222 netdev_features_t changed = netdev->features ^ features;
3223 struct wx *wx = netdev_priv(netdev);
3224
3225 if (changed & NETIF_VLAN_STRIPPING_FEATURES) {
3226 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&
3227 (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {
3228 features &= ~NETIF_VLAN_STRIPPING_FEATURES;
3229 features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;
3230 wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");
3231 }
3232 }
3233
3234 if (changed & NETIF_VLAN_INSERTION_FEATURES) {
3235 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&
3236 (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {
3237 features &= ~NETIF_VLAN_INSERTION_FEATURES;
3238 features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;
3239 wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");
3240 }
3241 }
3242
3243 if (changed & NETIF_VLAN_FILTERING_FEATURES) {
3244 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&
3245 (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {
3246 features &= ~NETIF_VLAN_FILTERING_FEATURES;
3247 features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;
3248 wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");
3249 }
3250 }
3251
3252 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
3253 if (!(features & NETIF_F_RXCSUM))
3254 features &= ~NETIF_F_LRO;
3255
3256 /* Turn off LRO if not RSC capable */
3257 if (!test_bit(WX_FLAG_RSC_CAPABLE, wx->flags))
3258 features &= ~NETIF_F_LRO;
3259
3260 return features;
3261 }
3262 EXPORT_SYMBOL(wx_fix_features);
3263
3264 #define WX_MAX_TUNNEL_HDR_LEN 80
wx_features_check(struct sk_buff * skb,struct net_device * netdev,netdev_features_t features)3265 netdev_features_t wx_features_check(struct sk_buff *skb,
3266 struct net_device *netdev,
3267 netdev_features_t features)
3268 {
3269 struct wx *wx = netdev_priv(netdev);
3270 __be16 type = skb->protocol;
3271 u16 vlan_depth = ETH_HLEN;
3272 u32 vlan_num = 0;
3273
3274 if (skb_vlan_tag_present(skb))
3275 vlan_num++;
3276
3277 while (eth_type_vlan(type)) {
3278 struct vlan_hdr vhdr, *vh;
3279
3280 vh = skb_header_pointer(skb, vlan_depth, sizeof(vhdr), &vhdr);
3281 if (unlikely(!vh))
3282 break;
3283
3284 type = vh->h_vlan_encapsulated_proto;
3285 vlan_depth += VLAN_HLEN;
3286 vlan_num++;
3287
3288 if (vlan_num > 2) {
3289 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3290 NETIF_F_HW_VLAN_STAG_TX);
3291 break;
3292 }
3293 }
3294
3295 if (!skb->encapsulation)
3296 return features;
3297
3298 if (wx->mac.type == wx_mac_em)
3299 return features & ~NETIF_F_CSUM_MASK;
3300
3301 if (unlikely(skb_inner_mac_header(skb) - skb_transport_header(skb) >
3302 WX_MAX_TUNNEL_HDR_LEN))
3303 return features & ~NETIF_F_CSUM_MASK;
3304
3305 if (skb->inner_protocol_type == ENCAP_TYPE_ETHER &&
3306 skb->inner_protocol != htons(ETH_P_IP) &&
3307 skb->inner_protocol != htons(ETH_P_IPV6) &&
3308 skb->inner_protocol != htons(ETH_P_TEB))
3309 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3310
3311 return features;
3312 }
3313 EXPORT_SYMBOL(wx_features_check);
3314
wx_set_ring(struct wx * wx,u32 new_tx_count,u32 new_rx_count,struct wx_ring * temp_ring)3315 int wx_set_ring(struct wx *wx, u32 new_tx_count,
3316 u32 new_rx_count, struct wx_ring *temp_ring)
3317 {
3318 int i, err = 0;
3319
3320 /* Setup new Tx resources and free the old Tx resources in that order.
3321 * We can then assign the new resources to the rings via a memcpy.
3322 * The advantage to this approach is that we are guaranteed to still
3323 * have resources even in the case of an allocation failure.
3324 */
3325 if (new_tx_count != wx->tx_ring_count) {
3326 for (i = 0; i < wx->num_tx_queues; i++) {
3327 memcpy(&temp_ring[i], wx->tx_ring[i],
3328 sizeof(struct wx_ring));
3329
3330 temp_ring[i].count = new_tx_count;
3331 err = wx_setup_tx_resources(&temp_ring[i]);
3332 if (err) {
3333 wx_err(wx, "setup new tx resources failed, keep using the old config\n");
3334 while (i) {
3335 i--;
3336 wx_free_tx_resources(&temp_ring[i]);
3337 }
3338 return err;
3339 }
3340 }
3341
3342 for (i = 0; i < wx->num_tx_queues; i++) {
3343 wx_free_tx_resources(wx->tx_ring[i]);
3344
3345 memcpy(wx->tx_ring[i], &temp_ring[i],
3346 sizeof(struct wx_ring));
3347 }
3348
3349 wx->tx_ring_count = new_tx_count;
3350 }
3351
3352 /* Repeat the process for the Rx rings if needed */
3353 if (new_rx_count != wx->rx_ring_count) {
3354 for (i = 0; i < wx->num_rx_queues; i++) {
3355 memcpy(&temp_ring[i], wx->rx_ring[i],
3356 sizeof(struct wx_ring));
3357
3358 temp_ring[i].count = new_rx_count;
3359 err = wx_setup_rx_resources(&temp_ring[i]);
3360 if (err) {
3361 wx_err(wx, "setup new rx resources failed, keep using the old config\n");
3362 while (i) {
3363 i--;
3364 wx_free_rx_resources(&temp_ring[i]);
3365 }
3366 return err;
3367 }
3368 }
3369
3370 for (i = 0; i < wx->num_rx_queues; i++) {
3371 wx_free_rx_resources(wx->rx_ring[i]);
3372 memcpy(wx->rx_ring[i], &temp_ring[i],
3373 sizeof(struct wx_ring));
3374 }
3375
3376 wx->rx_ring_count = new_rx_count;
3377 }
3378 return 0;
3379 }
3380 EXPORT_SYMBOL(wx_set_ring);
3381
wx_service_event_schedule(struct wx * wx)3382 void wx_service_event_schedule(struct wx *wx)
3383 {
3384 if (!test_bit(WX_STATE_DOWN, wx->state) &&
3385 !test_and_set_bit(WX_STATE_SERVICE_SCHED, wx->state))
3386 queue_work(system_power_efficient_wq, &wx->service_task);
3387 }
3388 EXPORT_SYMBOL(wx_service_event_schedule);
3389
wx_service_event_complete(struct wx * wx)3390 void wx_service_event_complete(struct wx *wx)
3391 {
3392 if (WARN_ON(!test_bit(WX_STATE_SERVICE_SCHED, wx->state)))
3393 return;
3394
3395 /* flush memory to make sure state is correct before next watchdog */
3396 smp_mb__before_atomic();
3397 clear_bit(WX_STATE_SERVICE_SCHED, wx->state);
3398 }
3399 EXPORT_SYMBOL(wx_service_event_complete);
3400
wx_service_timer(struct timer_list * t)3401 void wx_service_timer(struct timer_list *t)
3402 {
3403 struct wx *wx = timer_container_of(wx, t, service_timer);
3404 unsigned long next_event_offset = HZ * 2;
3405
3406 /* Reset the timer */
3407 mod_timer(&wx->service_timer, next_event_offset + jiffies);
3408
3409 wx_service_event_schedule(wx);
3410 }
3411 EXPORT_SYMBOL(wx_service_timer);
3412
wx_soft_quiesce(struct wx * wx)3413 void wx_soft_quiesce(struct wx *wx)
3414 {
3415 if (!netif_running(wx->netdev) ||
3416 test_and_set_bit(WX_STATE_DOWN, wx->state))
3417 return;
3418
3419 pci_clear_master(wx->pdev);
3420 netif_tx_stop_all_queues(wx->netdev);
3421 netif_carrier_off(wx->netdev);
3422 netif_tx_disable(wx->netdev);
3423 wx_napi_disable_all(wx);
3424 wx_ptp_quiesce(wx);
3425
3426 clear_bit(WX_FLAG_NEED_DO_RESET, wx->flags);
3427 timer_delete_sync(&wx->service_timer);
3428 }
3429 EXPORT_SYMBOL(wx_soft_quiesce);
3430
3431 MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");
3432 MODULE_LICENSE("GPL");
3433