1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
3
4 #include <linux/etherdevice.h>
5 #include <net/ip6_checksum.h>
6 #include <net/page_pool/helpers.h>
7 #include <net/inet_ecn.h>
8 #include <linux/iopoll.h>
9 #include <linux/sctp.h>
10 #include <linux/pci.h>
11 #include <net/tcp.h>
12 #include <net/ip.h>
13
14 #include "wx_type.h"
15 #include "wx_lib.h"
16 #include "wx_ptp.h"
17 #include "wx_hw.h"
18
19 /* Lookup table mapping the HW PTYPE to the bit field for decoding */
20 static struct wx_dec_ptype wx_ptype_lookup[256] = {
21 /* L2: mac */
22 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
23 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),
24 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
25 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
26 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
27 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),
28 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
29
30 /* L2: ethertype filter */
31 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),
32
33 /* L3: ip non-tunnel */
34 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),
35 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),
36 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),
37 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),
38 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),
39 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),
40 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),
41 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),
42 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),
43 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),
44
45 /* L2: fcoe */
46 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
47 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),
48
49 /* IPv4 --> IPv4/IPv6 */
50 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),
51 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),
52 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),
53 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),
54 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),
55 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),
56 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),
57 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),
58 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),
59 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),
60
61 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */
62 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),
63 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),
64 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),
65 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),
66 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),
67 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),
68 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),
69 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),
70 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),
71 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),
72 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),
73
74 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */
75 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),
76 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),
77 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),
78 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),
79 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),
80 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),
81 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),
82 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),
83 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),
84 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),
85 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),
86
87 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */
88 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),
89 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),
90 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),
91 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),
92 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),
93 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),
94 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),
95 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),
96 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),
97 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),
98 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),
99
100 /* IPv6 --> IPv4/IPv6 */
101 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),
102 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),
103 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),
104 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),
105 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),
106 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),
107 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),
108 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),
109 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),
110 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),
111
112 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */
113 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),
114 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),
115 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),
116 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),
117 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),
118 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),
119 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),
120 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),
121 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),
122 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),
123 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),
124
125 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */
126 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),
127 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),
128 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),
129 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),
130 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),
131 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),
132 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),
133 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),
134 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),
135 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),
136 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),
137
138 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */
139 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),
140 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),
141 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),
142 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),
143 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),
144 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),
145 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),
146 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),
147 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),
148 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),
149 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),
150 };
151
wx_decode_ptype(const u8 ptype)152 struct wx_dec_ptype wx_decode_ptype(const u8 ptype)
153 {
154 return wx_ptype_lookup[ptype];
155 }
156 EXPORT_SYMBOL(wx_decode_ptype);
157
158 /* wx_test_staterr - tests bits in Rx descriptor status and error fields */
wx_test_staterr(union wx_rx_desc * rx_desc,const u32 stat_err_bits)159 static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,
160 const u32 stat_err_bits)
161 {
162 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
163 }
164
wx_dma_sync_frag(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer)165 static void wx_dma_sync_frag(struct wx_ring *rx_ring,
166 struct wx_rx_buffer *rx_buffer)
167 {
168 struct sk_buff *skb = rx_buffer->skb;
169 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
170
171 dma_sync_single_range_for_cpu(rx_ring->dev,
172 WX_CB(skb)->dma,
173 skb_frag_off(frag),
174 skb_frag_size(frag),
175 DMA_FROM_DEVICE);
176
177 /* If the page was released, just unmap it. */
178 if (unlikely(WX_CB(skb)->page_released))
179 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
180 }
181
wx_get_rx_buffer(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff ** skb,int * rx_buffer_pgcnt)182 static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,
183 union wx_rx_desc *rx_desc,
184 struct sk_buff **skb,
185 int *rx_buffer_pgcnt)
186 {
187 struct wx_rx_buffer *rx_buffer;
188 unsigned int size;
189
190 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
191 size = le16_to_cpu(rx_desc->wb.upper.length);
192
193 #if (PAGE_SIZE < 8192)
194 *rx_buffer_pgcnt = page_count(rx_buffer->page);
195 #else
196 *rx_buffer_pgcnt = 0;
197 #endif
198
199 prefetchw(rx_buffer->page);
200 *skb = rx_buffer->skb;
201
202 /* Delay unmapping of the first packet. It carries the header
203 * information, HW may still access the header after the writeback.
204 * Only unmap it when EOP is reached
205 */
206 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {
207 if (!*skb)
208 goto skip_sync;
209 } else {
210 if (*skb)
211 wx_dma_sync_frag(rx_ring, rx_buffer);
212 }
213
214 /* we are reusing so sync this buffer for CPU use */
215 dma_sync_single_range_for_cpu(rx_ring->dev,
216 rx_buffer->dma,
217 rx_buffer->page_offset,
218 size,
219 DMA_FROM_DEVICE);
220 skip_sync:
221 return rx_buffer;
222 }
223
wx_put_rx_buffer(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,struct sk_buff * skb,int rx_buffer_pgcnt)224 static void wx_put_rx_buffer(struct wx_ring *rx_ring,
225 struct wx_rx_buffer *rx_buffer,
226 struct sk_buff *skb,
227 int rx_buffer_pgcnt)
228 {
229 if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma)
230 /* the page has been released from the ring */
231 WX_CB(skb)->page_released = true;
232
233 /* clear contents of rx_buffer */
234 rx_buffer->page = NULL;
235 rx_buffer->skb = NULL;
236 }
237
wx_build_skb(struct wx_ring * rx_ring,struct wx_rx_buffer * rx_buffer,union wx_rx_desc * rx_desc)238 static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,
239 struct wx_rx_buffer *rx_buffer,
240 union wx_rx_desc *rx_desc)
241 {
242 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
243 #if (PAGE_SIZE < 8192)
244 unsigned int truesize = WX_RX_BUFSZ;
245 #else
246 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
247 #endif
248 struct sk_buff *skb = rx_buffer->skb;
249
250 if (!skb) {
251 void *page_addr = page_address(rx_buffer->page) +
252 rx_buffer->page_offset;
253
254 /* prefetch first cache line of first page */
255 net_prefetch(page_addr);
256
257 /* allocate a skb to store the frags */
258 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);
259 if (unlikely(!skb))
260 return NULL;
261
262 /* we will be copying header into skb->data in
263 * pskb_may_pull so it is in our interest to prefetch
264 * it now to avoid a possible cache miss
265 */
266 prefetchw(skb->data);
267
268 if (size <= WX_RXBUFFER_256) {
269 memcpy(__skb_put(skb, size), page_addr,
270 ALIGN(size, sizeof(long)));
271 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);
272 return skb;
273 }
274
275 skb_mark_for_recycle(skb);
276
277 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))
278 WX_CB(skb)->dma = rx_buffer->dma;
279
280 skb_add_rx_frag(skb, 0, rx_buffer->page,
281 rx_buffer->page_offset,
282 size, truesize);
283 goto out;
284
285 } else {
286 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
287 rx_buffer->page_offset, size, truesize);
288 }
289
290 out:
291 #if (PAGE_SIZE < 8192)
292 /* flip page offset to other buffer */
293 rx_buffer->page_offset ^= truesize;
294 #else
295 /* move offset up to the next cache line */
296 rx_buffer->page_offset += truesize;
297 #endif
298
299 return skb;
300 }
301
wx_alloc_mapped_page(struct wx_ring * rx_ring,struct wx_rx_buffer * bi)302 static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,
303 struct wx_rx_buffer *bi)
304 {
305 struct page *page = bi->page;
306 dma_addr_t dma;
307
308 /* since we are recycling buffers we should seldom need to alloc */
309 if (likely(page))
310 return true;
311
312 page = page_pool_dev_alloc_pages(rx_ring->page_pool);
313 WARN_ON(!page);
314 dma = page_pool_get_dma_addr(page);
315
316 bi->page_dma = dma;
317 bi->page = page;
318 bi->page_offset = 0;
319
320 return true;
321 }
322
323 /**
324 * wx_alloc_rx_buffers - Replace used receive buffers
325 * @rx_ring: ring to place buffers on
326 * @cleaned_count: number of buffers to replace
327 **/
wx_alloc_rx_buffers(struct wx_ring * rx_ring,u16 cleaned_count)328 void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)
329 {
330 u16 i = rx_ring->next_to_use;
331 union wx_rx_desc *rx_desc;
332 struct wx_rx_buffer *bi;
333
334 /* nothing to do */
335 if (!cleaned_count)
336 return;
337
338 rx_desc = WX_RX_DESC(rx_ring, i);
339 bi = &rx_ring->rx_buffer_info[i];
340 i -= rx_ring->count;
341
342 do {
343 if (!wx_alloc_mapped_page(rx_ring, bi))
344 break;
345
346 /* sync the buffer for use by the device */
347 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
348 bi->page_offset,
349 WX_RX_BUFSZ,
350 DMA_FROM_DEVICE);
351
352 rx_desc->read.pkt_addr =
353 cpu_to_le64(bi->page_dma + bi->page_offset);
354
355 rx_desc++;
356 bi++;
357 i++;
358 if (unlikely(!i)) {
359 rx_desc = WX_RX_DESC(rx_ring, 0);
360 bi = rx_ring->rx_buffer_info;
361 i -= rx_ring->count;
362 }
363
364 /* clear the status bits for the next_to_use descriptor */
365 rx_desc->wb.upper.status_error = 0;
366
367 cleaned_count--;
368 } while (cleaned_count);
369
370 i += rx_ring->count;
371
372 if (rx_ring->next_to_use != i) {
373 rx_ring->next_to_use = i;
374 /* update next to alloc since we have filled the ring */
375 rx_ring->next_to_alloc = i;
376
377 /* Force memory writes to complete before letting h/w
378 * know there are new descriptors to fetch. (Only
379 * applicable for weak-ordered memory model archs,
380 * such as IA-64).
381 */
382 wmb();
383 writel(i, rx_ring->tail);
384 }
385 }
386
wx_desc_unused(struct wx_ring * ring)387 u16 wx_desc_unused(struct wx_ring *ring)
388 {
389 u16 ntc = ring->next_to_clean;
390 u16 ntu = ring->next_to_use;
391
392 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
393 }
394
395 /**
396 * wx_is_non_eop - process handling of non-EOP buffers
397 * @rx_ring: Rx ring being processed
398 * @rx_desc: Rx descriptor for current buffer
399 * @skb: Current socket buffer containing buffer in progress
400 *
401 * This function updates next to clean. If the buffer is an EOP buffer
402 * this function exits returning false, otherwise it will place the
403 * sk_buff in the next buffer to be chained and return true indicating
404 * that this is in fact a non-EOP buffer.
405 **/
wx_is_non_eop(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)406 static bool wx_is_non_eop(struct wx_ring *rx_ring,
407 union wx_rx_desc *rx_desc,
408 struct sk_buff *skb)
409 {
410 u32 ntc = rx_ring->next_to_clean + 1;
411
412 /* fetch, update, and store next to clean */
413 ntc = (ntc < rx_ring->count) ? ntc : 0;
414 rx_ring->next_to_clean = ntc;
415
416 prefetch(WX_RX_DESC(rx_ring, ntc));
417
418 /* if we are the last buffer then there is nothing else to do */
419 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))
420 return false;
421
422 rx_ring->rx_buffer_info[ntc].skb = skb;
423 rx_ring->rx_stats.non_eop_descs++;
424
425 return true;
426 }
427
wx_pull_tail(struct sk_buff * skb)428 static void wx_pull_tail(struct sk_buff *skb)
429 {
430 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
431 unsigned int pull_len;
432 unsigned char *va;
433
434 /* it is valid to use page_address instead of kmap since we are
435 * working with pages allocated out of the lomem pool per
436 * alloc_page(GFP_ATOMIC)
437 */
438 va = skb_frag_address(frag);
439
440 /* we need the header to contain the greater of either ETH_HLEN or
441 * 60 bytes if the skb->len is less than 60 for skb_pad.
442 */
443 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);
444
445 /* align pull length to size of long to optimize memcpy performance */
446 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
447
448 /* update all of the pointers */
449 skb_frag_size_sub(frag, pull_len);
450 skb_frag_off_add(frag, pull_len);
451 skb->data_len -= pull_len;
452 skb->tail += pull_len;
453 }
454
455 /**
456 * wx_cleanup_headers - Correct corrupted or empty headers
457 * @rx_ring: rx descriptor ring packet is being transacted on
458 * @rx_desc: pointer to the EOP Rx descriptor
459 * @skb: pointer to current skb being fixed
460 *
461 * Check for corrupted packet headers caused by senders on the local L2
462 * embedded NIC switch not setting up their Tx Descriptors right. These
463 * should be very rare.
464 *
465 * Also address the case where we are pulling data in on pages only
466 * and as such no data is present in the skb header.
467 *
468 * In addition if skb is not at least 60 bytes we need to pad it so that
469 * it is large enough to qualify as a valid Ethernet frame.
470 *
471 * Returns true if an error was encountered and skb was freed.
472 **/
wx_cleanup_headers(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)473 static bool wx_cleanup_headers(struct wx_ring *rx_ring,
474 union wx_rx_desc *rx_desc,
475 struct sk_buff *skb)
476 {
477 struct net_device *netdev = rx_ring->netdev;
478
479 /* verify that the packet does not have any known errors */
480 if (!netdev ||
481 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&
482 !(netdev->features & NETIF_F_RXALL))) {
483 dev_kfree_skb_any(skb);
484 return true;
485 }
486
487 /* place header in linear portion of buffer */
488 if (!skb_headlen(skb))
489 wx_pull_tail(skb);
490
491 /* if eth_skb_pad returns an error the skb was freed */
492 if (eth_skb_pad(skb))
493 return true;
494
495 return false;
496 }
497
wx_rx_hash(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)498 static void wx_rx_hash(struct wx_ring *ring,
499 union wx_rx_desc *rx_desc,
500 struct sk_buff *skb)
501 {
502 u16 rss_type;
503
504 if (!(ring->netdev->features & NETIF_F_RXHASH))
505 return;
506
507 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
508 WX_RXD_RSSTYPE_MASK;
509
510 if (!rss_type)
511 return;
512
513 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
514 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
515 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
516 }
517
518 /**
519 * wx_rx_checksum - indicate in skb if hw indicated a good cksum
520 * @ring: structure containing ring specific data
521 * @rx_desc: current Rx descriptor being processed
522 * @skb: skb currently being received and modified
523 **/
wx_rx_checksum(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)524 static void wx_rx_checksum(struct wx_ring *ring,
525 union wx_rx_desc *rx_desc,
526 struct sk_buff *skb)
527 {
528 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));
529
530 skb_checksum_none_assert(skb);
531 /* Rx csum disabled */
532 if (!(ring->netdev->features & NETIF_F_RXCSUM))
533 return;
534
535 /* if IPv4 header checksum error */
536 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&
537 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||
538 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&
539 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {
540 ring->rx_stats.csum_err++;
541 return;
542 }
543
544 /* L4 checksum offload flag must set for the below code to work */
545 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))
546 return;
547
548 /* Hardware can't guarantee csum if IPv6 Dest Header found */
549 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP && WX_RXD_IPV6EX(rx_desc))
550 return;
551
552 /* if L4 checksum error */
553 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {
554 ring->rx_stats.csum_err++;
555 return;
556 }
557
558 /* It must be a TCP or UDP or SCTP packet with a valid checksum */
559 skb->ip_summed = CHECKSUM_UNNECESSARY;
560
561 /* If there is an outer header present that might contain a checksum
562 * we need to bump the checksum level by 1 to reflect the fact that
563 * we are indicating we validated the inner checksum.
564 */
565 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)
566 __skb_incr_checksum_unnecessary(skb);
567 ring->rx_stats.csum_good_cnt++;
568 }
569
wx_rx_vlan(struct wx_ring * ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)570 static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,
571 struct sk_buff *skb)
572 {
573 u16 ethertype;
574 u8 idx = 0;
575
576 if ((ring->netdev->features &
577 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&
578 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {
579 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
580 0x1c0) >> 6;
581 ethertype = ring->q_vector->wx->tpid[idx];
582 __vlan_hwaccel_put_tag(skb, htons(ethertype),
583 le16_to_cpu(rx_desc->wb.upper.vlan));
584 }
585 }
586
587 /**
588 * wx_process_skb_fields - Populate skb header fields from Rx descriptor
589 * @rx_ring: rx descriptor ring packet is being transacted on
590 * @rx_desc: pointer to the EOP Rx descriptor
591 * @skb: pointer to current skb being populated
592 *
593 * This function checks the ring, descriptor, and packet information in
594 * order to populate the hash, checksum, protocol, and
595 * other fields within the skb.
596 **/
wx_process_skb_fields(struct wx_ring * rx_ring,union wx_rx_desc * rx_desc,struct sk_buff * skb)597 static void wx_process_skb_fields(struct wx_ring *rx_ring,
598 union wx_rx_desc *rx_desc,
599 struct sk_buff *skb)
600 {
601 struct wx *wx = netdev_priv(rx_ring->netdev);
602
603 wx_rx_hash(rx_ring, rx_desc, skb);
604 wx_rx_checksum(rx_ring, rx_desc, skb);
605
606 if (unlikely(test_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags)) &&
607 unlikely(wx_test_staterr(rx_desc, WX_RXD_STAT_TS))) {
608 wx_ptp_rx_hwtstamp(rx_ring->q_vector->wx, skb);
609 rx_ring->last_rx_timestamp = jiffies;
610 }
611
612 wx_rx_vlan(rx_ring, rx_desc, skb);
613 skb_record_rx_queue(skb, rx_ring->queue_index);
614 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
615 }
616
617 /**
618 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
619 * @q_vector: structure containing interrupt and ring information
620 * @rx_ring: rx descriptor ring to transact packets on
621 * @budget: Total limit on number of packets to process
622 *
623 * This function provides a "bounce buffer" approach to Rx interrupt
624 * processing. The advantage to this is that on systems that have
625 * expensive overhead for IOMMU access this provides a means of avoiding
626 * it by maintaining the mapping of the page to the system.
627 *
628 * Returns amount of work completed.
629 **/
wx_clean_rx_irq(struct wx_q_vector * q_vector,struct wx_ring * rx_ring,int budget)630 static int wx_clean_rx_irq(struct wx_q_vector *q_vector,
631 struct wx_ring *rx_ring,
632 int budget)
633 {
634 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
635 u16 cleaned_count = wx_desc_unused(rx_ring);
636
637 do {
638 struct wx_rx_buffer *rx_buffer;
639 union wx_rx_desc *rx_desc;
640 struct sk_buff *skb;
641 int rx_buffer_pgcnt;
642
643 /* return some buffers to hardware, one at a time is too slow */
644 if (cleaned_count >= WX_RX_BUFFER_WRITE) {
645 wx_alloc_rx_buffers(rx_ring, cleaned_count);
646 cleaned_count = 0;
647 }
648
649 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);
650 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))
651 break;
652
653 /* This memory barrier is needed to keep us from reading
654 * any other fields out of the rx_desc until we know the
655 * descriptor has been written back
656 */
657 dma_rmb();
658
659 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);
660
661 /* retrieve a buffer from the ring */
662 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);
663
664 /* exit if we failed to retrieve a buffer */
665 if (!skb) {
666 rx_ring->rx_stats.alloc_rx_buff_failed++;
667 break;
668 }
669
670 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
671 cleaned_count++;
672
673 /* place incomplete frames back on ring for completion */
674 if (wx_is_non_eop(rx_ring, rx_desc, skb))
675 continue;
676
677 /* verify the packet layout is correct */
678 if (wx_cleanup_headers(rx_ring, rx_desc, skb))
679 continue;
680
681 /* probably a little skewed due to removing CRC */
682 total_rx_bytes += skb->len;
683
684 /* populate checksum, timestamp, VLAN, and protocol */
685 wx_process_skb_fields(rx_ring, rx_desc, skb);
686 napi_gro_receive(&q_vector->napi, skb);
687
688 /* update budget accounting */
689 total_rx_packets++;
690 } while (likely(total_rx_packets < budget));
691
692 u64_stats_update_begin(&rx_ring->syncp);
693 rx_ring->stats.packets += total_rx_packets;
694 rx_ring->stats.bytes += total_rx_bytes;
695 u64_stats_update_end(&rx_ring->syncp);
696 q_vector->rx.total_packets += total_rx_packets;
697 q_vector->rx.total_bytes += total_rx_bytes;
698
699 return total_rx_packets;
700 }
701
wx_txring_txq(const struct wx_ring * ring)702 static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)
703 {
704 return netdev_get_tx_queue(ring->netdev, ring->queue_index);
705 }
706
707 /**
708 * wx_clean_tx_irq - Reclaim resources after transmit completes
709 * @q_vector: structure containing interrupt and ring information
710 * @tx_ring: tx ring to clean
711 * @napi_budget: Used to determine if we are in netpoll
712 **/
wx_clean_tx_irq(struct wx_q_vector * q_vector,struct wx_ring * tx_ring,int napi_budget)713 static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,
714 struct wx_ring *tx_ring, int napi_budget)
715 {
716 unsigned int budget = q_vector->wx->tx_work_limit;
717 unsigned int total_bytes = 0, total_packets = 0;
718 struct wx *wx = netdev_priv(tx_ring->netdev);
719 unsigned int i = tx_ring->next_to_clean;
720 struct wx_tx_buffer *tx_buffer;
721 union wx_tx_desc *tx_desc;
722
723 if (!netif_carrier_ok(tx_ring->netdev))
724 return true;
725
726 tx_buffer = &tx_ring->tx_buffer_info[i];
727 tx_desc = WX_TX_DESC(tx_ring, i);
728 i -= tx_ring->count;
729
730 do {
731 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;
732
733 /* if next_to_watch is not set then there is no work pending */
734 if (!eop_desc)
735 break;
736
737 /* prevent any other reads prior to eop_desc */
738 smp_rmb();
739
740 /* if DD is not set pending work has not been completed */
741 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))
742 break;
743
744 /* clear next_to_watch to prevent false hangs */
745 tx_buffer->next_to_watch = NULL;
746
747 /* update the statistics for this packet */
748 total_bytes += tx_buffer->bytecount;
749 total_packets += tx_buffer->gso_segs;
750
751 /* schedule check for Tx timestamp */
752 if (unlikely(test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) &&
753 skb_shinfo(tx_buffer->skb)->tx_flags & SKBTX_IN_PROGRESS)
754 ptp_schedule_worker(wx->ptp_clock, 0);
755
756 /* free the skb */
757 napi_consume_skb(tx_buffer->skb, napi_budget);
758
759 /* unmap skb header data */
760 dma_unmap_single(tx_ring->dev,
761 dma_unmap_addr(tx_buffer, dma),
762 dma_unmap_len(tx_buffer, len),
763 DMA_TO_DEVICE);
764
765 /* clear tx_buffer data */
766 dma_unmap_len_set(tx_buffer, len, 0);
767
768 /* unmap remaining buffers */
769 while (tx_desc != eop_desc) {
770 tx_buffer++;
771 tx_desc++;
772 i++;
773 if (unlikely(!i)) {
774 i -= tx_ring->count;
775 tx_buffer = tx_ring->tx_buffer_info;
776 tx_desc = WX_TX_DESC(tx_ring, 0);
777 }
778
779 /* unmap any remaining paged data */
780 if (dma_unmap_len(tx_buffer, len)) {
781 dma_unmap_page(tx_ring->dev,
782 dma_unmap_addr(tx_buffer, dma),
783 dma_unmap_len(tx_buffer, len),
784 DMA_TO_DEVICE);
785 dma_unmap_len_set(tx_buffer, len, 0);
786 }
787 }
788
789 /* move us one more past the eop_desc for start of next pkt */
790 tx_buffer++;
791 tx_desc++;
792 i++;
793 if (unlikely(!i)) {
794 i -= tx_ring->count;
795 tx_buffer = tx_ring->tx_buffer_info;
796 tx_desc = WX_TX_DESC(tx_ring, 0);
797 }
798
799 /* issue prefetch for next Tx descriptor */
800 prefetch(tx_desc);
801
802 /* update budget accounting */
803 budget--;
804 } while (likely(budget));
805
806 i += tx_ring->count;
807 tx_ring->next_to_clean = i;
808 u64_stats_update_begin(&tx_ring->syncp);
809 tx_ring->stats.bytes += total_bytes;
810 tx_ring->stats.packets += total_packets;
811 u64_stats_update_end(&tx_ring->syncp);
812 q_vector->tx.total_bytes += total_bytes;
813 q_vector->tx.total_packets += total_packets;
814
815 netdev_tx_completed_queue(wx_txring_txq(tx_ring),
816 total_packets, total_bytes);
817
818 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
819 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
820 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
821 /* Make sure that anybody stopping the queue after this
822 * sees the new next_to_clean.
823 */
824 smp_mb();
825
826 if (__netif_subqueue_stopped(tx_ring->netdev,
827 tx_ring->queue_index) &&
828 netif_running(tx_ring->netdev)) {
829 netif_wake_subqueue(tx_ring->netdev,
830 tx_ring->queue_index);
831 ++tx_ring->tx_stats.restart_queue;
832 }
833 }
834
835 return !!budget;
836 }
837
838 /**
839 * wx_poll - NAPI polling RX/TX cleanup routine
840 * @napi: napi struct with our devices info in it
841 * @budget: amount of work driver is allowed to do this pass, in packets
842 *
843 * This function will clean all queues associated with a q_vector.
844 **/
wx_poll(struct napi_struct * napi,int budget)845 static int wx_poll(struct napi_struct *napi, int budget)
846 {
847 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);
848 int per_ring_budget, work_done = 0;
849 struct wx *wx = q_vector->wx;
850 bool clean_complete = true;
851 struct wx_ring *ring;
852
853 wx_for_each_ring(ring, q_vector->tx) {
854 if (!wx_clean_tx_irq(q_vector, ring, budget))
855 clean_complete = false;
856 }
857
858 /* Exit if we are called by netpoll */
859 if (budget <= 0)
860 return budget;
861
862 /* attempt to distribute budget to each queue fairly, but don't allow
863 * the budget to go below 1 because we'll exit polling
864 */
865 if (q_vector->rx.count > 1)
866 per_ring_budget = max(budget / q_vector->rx.count, 1);
867 else
868 per_ring_budget = budget;
869
870 wx_for_each_ring(ring, q_vector->rx) {
871 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);
872
873 work_done += cleaned;
874 if (cleaned >= per_ring_budget)
875 clean_complete = false;
876 }
877
878 /* If all work not completed, return budget and keep polling */
879 if (!clean_complete)
880 return budget;
881
882 /* all work done, exit the polling mode */
883 if (likely(napi_complete_done(napi, work_done))) {
884 if (netif_running(wx->netdev))
885 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));
886 }
887
888 return min(work_done, budget - 1);
889 }
890
wx_maybe_stop_tx(struct wx_ring * tx_ring,u16 size)891 static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)
892 {
893 if (likely(wx_desc_unused(tx_ring) >= size))
894 return 0;
895
896 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
897
898 /* For the next check */
899 smp_mb();
900
901 /* We need to check again in a case another CPU has just
902 * made room available.
903 */
904 if (likely(wx_desc_unused(tx_ring) < size))
905 return -EBUSY;
906
907 /* A reprieve! - use start_queue because it doesn't call schedule */
908 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
909 ++tx_ring->tx_stats.restart_queue;
910
911 return 0;
912 }
913
wx_tx_cmd_type(u32 tx_flags)914 static u32 wx_tx_cmd_type(u32 tx_flags)
915 {
916 /* set type for advanced descriptor with frame checksum insertion */
917 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;
918
919 /* set HW vlan bit if vlan is present */
920 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);
921 /* set segmentation enable bits for TSO/FSO */
922 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);
923 /* set timestamp bit if present */
924 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);
925 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);
926
927 return cmd_type;
928 }
929
wx_tx_olinfo_status(union wx_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)930 static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,
931 u32 tx_flags, unsigned int paylen)
932 {
933 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;
934
935 /* enable L4 checksum for TSO and TX checksum offload */
936 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);
937 /* enable IPv4 checksum for TSO */
938 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);
939 /* enable outer IPv4 checksum for TSO */
940 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,
941 WX_TXD_EIPCS);
942 /* Check Context must be set if Tx switch is enabled, which it
943 * always is for case where virtual functions are running
944 */
945 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);
946 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,
947 WX_TXD_IPSEC);
948 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
949 }
950
wx_tx_map(struct wx_ring * tx_ring,struct wx_tx_buffer * first,const u8 hdr_len)951 static int wx_tx_map(struct wx_ring *tx_ring,
952 struct wx_tx_buffer *first,
953 const u8 hdr_len)
954 {
955 struct sk_buff *skb = first->skb;
956 struct wx_tx_buffer *tx_buffer;
957 u32 tx_flags = first->tx_flags;
958 u16 i = tx_ring->next_to_use;
959 unsigned int data_len, size;
960 union wx_tx_desc *tx_desc;
961 skb_frag_t *frag;
962 dma_addr_t dma;
963 u32 cmd_type;
964
965 cmd_type = wx_tx_cmd_type(tx_flags);
966 tx_desc = WX_TX_DESC(tx_ring, i);
967 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
968
969 size = skb_headlen(skb);
970 data_len = skb->data_len;
971 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
972
973 tx_buffer = first;
974
975 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
976 if (dma_mapping_error(tx_ring->dev, dma))
977 goto dma_error;
978
979 /* record length, and DMA address */
980 dma_unmap_len_set(tx_buffer, len, size);
981 dma_unmap_addr_set(tx_buffer, dma, dma);
982
983 tx_desc->read.buffer_addr = cpu_to_le64(dma);
984
985 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {
986 tx_desc->read.cmd_type_len =
987 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);
988
989 i++;
990 tx_desc++;
991 if (i == tx_ring->count) {
992 tx_desc = WX_TX_DESC(tx_ring, 0);
993 i = 0;
994 }
995 tx_desc->read.olinfo_status = 0;
996
997 dma += WX_MAX_DATA_PER_TXD;
998 size -= WX_MAX_DATA_PER_TXD;
999
1000 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1001 }
1002
1003 if (likely(!data_len))
1004 break;
1005
1006 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1007
1008 i++;
1009 tx_desc++;
1010 if (i == tx_ring->count) {
1011 tx_desc = WX_TX_DESC(tx_ring, 0);
1012 i = 0;
1013 }
1014 tx_desc->read.olinfo_status = 0;
1015
1016 size = skb_frag_size(frag);
1017
1018 data_len -= size;
1019
1020 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1021 DMA_TO_DEVICE);
1022
1023 tx_buffer = &tx_ring->tx_buffer_info[i];
1024 }
1025
1026 /* write last descriptor with RS and EOP bits */
1027 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;
1028 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1029
1030 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);
1031
1032 /* set the timestamp */
1033 first->time_stamp = jiffies;
1034 skb_tx_timestamp(skb);
1035
1036 /* Force memory writes to complete before letting h/w know there
1037 * are new descriptors to fetch. (Only applicable for weak-ordered
1038 * memory model archs, such as IA-64).
1039 *
1040 * We also need this memory barrier to make certain all of the
1041 * status bits have been updated before next_to_watch is written.
1042 */
1043 wmb();
1044
1045 /* set next_to_watch value indicating a packet is present */
1046 first->next_to_watch = tx_desc;
1047
1048 i++;
1049 if (i == tx_ring->count)
1050 i = 0;
1051
1052 tx_ring->next_to_use = i;
1053
1054 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);
1055
1056 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())
1057 writel(i, tx_ring->tail);
1058
1059 return 0;
1060 dma_error:
1061 dev_err(tx_ring->dev, "TX DMA map failed\n");
1062
1063 /* clear dma mappings for failed tx_buffer_info map */
1064 for (;;) {
1065 tx_buffer = &tx_ring->tx_buffer_info[i];
1066 if (dma_unmap_len(tx_buffer, len))
1067 dma_unmap_page(tx_ring->dev,
1068 dma_unmap_addr(tx_buffer, dma),
1069 dma_unmap_len(tx_buffer, len),
1070 DMA_TO_DEVICE);
1071 dma_unmap_len_set(tx_buffer, len, 0);
1072 if (tx_buffer == first)
1073 break;
1074 if (i == 0)
1075 i += tx_ring->count;
1076 i--;
1077 }
1078
1079 dev_kfree_skb_any(first->skb);
1080 first->skb = NULL;
1081
1082 tx_ring->next_to_use = i;
1083
1084 return -ENOMEM;
1085 }
1086
wx_tx_ctxtdesc(struct wx_ring * tx_ring,u32 vlan_macip_lens,u32 fcoe_sof_eof,u32 type_tucmd,u32 mss_l4len_idx)1087 static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,
1088 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
1089 {
1090 struct wx_tx_context_desc *context_desc;
1091 u16 i = tx_ring->next_to_use;
1092
1093 context_desc = WX_TX_CTXTDESC(tx_ring, i);
1094 i++;
1095 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1096
1097 /* set bits to identify this as an advanced context descriptor */
1098 type_tucmd |= WX_TXD_DTYP_CTXT;
1099 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1100 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
1101 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1102 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1103 }
1104
1105 union network_header {
1106 struct iphdr *ipv4;
1107 struct ipv6hdr *ipv6;
1108 void *raw;
1109 };
1110
wx_encode_tx_desc_ptype(const struct wx_tx_buffer * first)1111 static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)
1112 {
1113 u8 tun_prot = 0, l4_prot = 0, ptype = 0;
1114 struct sk_buff *skb = first->skb;
1115 unsigned char *exthdr, *l4_hdr;
1116 __be16 frag_off;
1117
1118 if (skb->encapsulation) {
1119 union network_header hdr;
1120
1121 switch (first->protocol) {
1122 case htons(ETH_P_IP):
1123 tun_prot = ip_hdr(skb)->protocol;
1124 ptype = WX_PTYPE_TUN_IPV4;
1125 break;
1126 case htons(ETH_P_IPV6):
1127 l4_hdr = skb_transport_header(skb);
1128 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1129 tun_prot = ipv6_hdr(skb)->nexthdr;
1130 if (l4_hdr != exthdr)
1131 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1132 ptype = WX_PTYPE_TUN_IPV6;
1133 break;
1134 default:
1135 return ptype;
1136 }
1137
1138 if (tun_prot == IPPROTO_IPIP || tun_prot == IPPROTO_IPV6) {
1139 hdr.raw = (void *)inner_ip_hdr(skb);
1140 ptype |= WX_PTYPE_PKT_IPIP;
1141 } else if (tun_prot == IPPROTO_UDP) {
1142 hdr.raw = (void *)inner_ip_hdr(skb);
1143 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1144 skb->inner_protocol != htons(ETH_P_TEB)) {
1145 ptype |= WX_PTYPE_PKT_IG;
1146 } else {
1147 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1148 == htons(ETH_P_8021Q))
1149 ptype |= WX_PTYPE_PKT_IGMV;
1150 else
1151 ptype |= WX_PTYPE_PKT_IGM;
1152 }
1153
1154 } else if (tun_prot == IPPROTO_GRE) {
1155 hdr.raw = (void *)inner_ip_hdr(skb);
1156 if (skb->inner_protocol == htons(ETH_P_IP) ||
1157 skb->inner_protocol == htons(ETH_P_IPV6)) {
1158 ptype |= WX_PTYPE_PKT_IG;
1159 } else {
1160 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto
1161 == htons(ETH_P_8021Q))
1162 ptype |= WX_PTYPE_PKT_IGMV;
1163 else
1164 ptype |= WX_PTYPE_PKT_IGM;
1165 }
1166 } else {
1167 return ptype;
1168 }
1169
1170 switch (hdr.ipv4->version) {
1171 case IPVERSION:
1172 l4_prot = hdr.ipv4->protocol;
1173 break;
1174 case 6:
1175 l4_hdr = skb_inner_transport_header(skb);
1176 exthdr = skb_inner_network_header(skb) + sizeof(struct ipv6hdr);
1177 l4_prot = inner_ipv6_hdr(skb)->nexthdr;
1178 if (l4_hdr != exthdr)
1179 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1180 ptype |= WX_PTYPE_PKT_IPV6;
1181 break;
1182 default:
1183 return ptype;
1184 }
1185 } else {
1186 switch (first->protocol) {
1187 case htons(ETH_P_IP):
1188 l4_prot = ip_hdr(skb)->protocol;
1189 ptype = WX_PTYPE_PKT_IP;
1190 break;
1191 case htons(ETH_P_IPV6):
1192 l4_hdr = skb_transport_header(skb);
1193 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1194 l4_prot = ipv6_hdr(skb)->nexthdr;
1195 if (l4_hdr != exthdr)
1196 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1197 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;
1198 break;
1199 default:
1200 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;
1201 }
1202 }
1203 switch (l4_prot) {
1204 case IPPROTO_TCP:
1205 ptype |= WX_PTYPE_TYP_TCP;
1206 break;
1207 case IPPROTO_UDP:
1208 ptype |= WX_PTYPE_TYP_UDP;
1209 break;
1210 case IPPROTO_SCTP:
1211 ptype |= WX_PTYPE_TYP_SCTP;
1212 break;
1213 default:
1214 ptype |= WX_PTYPE_TYP_IP;
1215 break;
1216 }
1217
1218 return ptype;
1219 }
1220
wx_tso(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 * hdr_len,u8 ptype)1221 static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1222 u8 *hdr_len, u8 ptype)
1223 {
1224 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1225 struct net_device *netdev = tx_ring->netdev;
1226 u32 l4len, tunhdr_eiplen_tunlen = 0;
1227 struct sk_buff *skb = first->skb;
1228 bool enc = skb->encapsulation;
1229 struct ipv6hdr *ipv6h;
1230 struct tcphdr *tcph;
1231 struct iphdr *iph;
1232 u8 tun_prot = 0;
1233 int err;
1234
1235 if (skb->ip_summed != CHECKSUM_PARTIAL)
1236 return 0;
1237
1238 if (!skb_is_gso(skb))
1239 return 0;
1240
1241 err = skb_cow_head(skb, 0);
1242 if (err < 0)
1243 return err;
1244
1245 /* indicates the inner headers in the skbuff are valid. */
1246 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);
1247 if (iph->version == 4) {
1248 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1249 iph->tot_len = 0;
1250 iph->check = 0;
1251 tcph->check = ~csum_tcpudp_magic(iph->saddr,
1252 iph->daddr, 0,
1253 IPPROTO_TCP, 0);
1254 first->tx_flags |= WX_TX_FLAGS_TSO |
1255 WX_TX_FLAGS_CSUM |
1256 WX_TX_FLAGS_IPV4 |
1257 WX_TX_FLAGS_CC;
1258 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {
1259 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);
1260 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1261 ipv6h->payload_len = 0;
1262 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1263 &ipv6h->daddr, 0,
1264 IPPROTO_TCP, 0);
1265 first->tx_flags |= WX_TX_FLAGS_TSO |
1266 WX_TX_FLAGS_CSUM |
1267 WX_TX_FLAGS_CC;
1268 }
1269
1270 /* compute header lengths */
1271 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1272 *hdr_len = enc ? skb_inner_transport_offset(skb) :
1273 skb_transport_offset(skb);
1274 *hdr_len += l4len;
1275
1276 /* update gso size and bytecount with header size */
1277 first->gso_segs = skb_shinfo(skb)->gso_segs;
1278 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1279
1280 /* mss_l4len_id: use 0 as index for TSO */
1281 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;
1282 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;
1283
1284 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
1285 if (enc) {
1286 unsigned char *exthdr, *l4_hdr;
1287 __be16 frag_off;
1288
1289 switch (first->protocol) {
1290 case htons(ETH_P_IP):
1291 tun_prot = ip_hdr(skb)->protocol;
1292 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;
1293 break;
1294 case htons(ETH_P_IPV6):
1295 l4_hdr = skb_transport_header(skb);
1296 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1297 tun_prot = ipv6_hdr(skb)->nexthdr;
1298 if (l4_hdr != exthdr)
1299 ipv6_skip_exthdr(skb, exthdr - skb->data, &tun_prot, &frag_off);
1300 break;
1301 default:
1302 break;
1303 }
1304 switch (tun_prot) {
1305 case IPPROTO_UDP:
1306 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1307 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1308 WX_TXD_OUTER_IPLEN_SHIFT) |
1309 (((skb_inner_mac_header(skb) -
1310 skb_transport_header(skb)) >> 1) <<
1311 WX_TXD_TUNNEL_LEN_SHIFT);
1312 break;
1313 case IPPROTO_GRE:
1314 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1315 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1316 WX_TXD_OUTER_IPLEN_SHIFT) |
1317 (((skb_inner_mac_header(skb) -
1318 skb_transport_header(skb)) >> 1) <<
1319 WX_TXD_TUNNEL_LEN_SHIFT);
1320 break;
1321 case IPPROTO_IPIP:
1322 case IPPROTO_IPV6:
1323 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1324 (char *)ip_hdr(skb)) >> 2) <<
1325 WX_TXD_OUTER_IPLEN_SHIFT;
1326 break;
1327 default:
1328 break;
1329 }
1330 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;
1331 } else {
1332 vlan_macip_lens = skb_network_header_len(skb) >> 1;
1333 }
1334
1335 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;
1336 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1337
1338 type_tucmd = ptype << 24;
1339 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1340 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1341 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1342 WX_TX_FLAGS_HW_VLAN,
1343 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1344 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1345 type_tucmd, mss_l4len_idx);
1346
1347 return 1;
1348 }
1349
wx_tx_csum(struct wx_ring * tx_ring,struct wx_tx_buffer * first,u8 ptype)1350 static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,
1351 u8 ptype)
1352 {
1353 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;
1354 struct net_device *netdev = tx_ring->netdev;
1355 u32 mss_l4len_idx = 0, type_tucmd;
1356 struct sk_buff *skb = first->skb;
1357 u8 tun_prot = 0;
1358
1359 if (skb->ip_summed != CHECKSUM_PARTIAL) {
1360 csum_failed:
1361 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&
1362 !(first->tx_flags & WX_TX_FLAGS_CC))
1363 return;
1364 vlan_macip_lens = skb_network_offset(skb) <<
1365 WX_TXD_MACLEN_SHIFT;
1366 } else {
1367 unsigned char *exthdr, *l4_hdr;
1368 __be16 frag_off;
1369 u8 l4_prot = 0;
1370 union {
1371 struct iphdr *ipv4;
1372 struct ipv6hdr *ipv6;
1373 u8 *raw;
1374 } network_hdr;
1375 union {
1376 struct tcphdr *tcphdr;
1377 u8 *raw;
1378 } transport_hdr;
1379
1380 if (skb->encapsulation) {
1381 network_hdr.raw = skb_inner_network_header(skb);
1382 transport_hdr.raw = skb_inner_transport_header(skb);
1383 vlan_macip_lens = skb_network_offset(skb) <<
1384 WX_TXD_MACLEN_SHIFT;
1385 switch (first->protocol) {
1386 case htons(ETH_P_IP):
1387 tun_prot = ip_hdr(skb)->protocol;
1388 break;
1389 case htons(ETH_P_IPV6):
1390 l4_hdr = skb_transport_header(skb);
1391 exthdr = skb_network_header(skb) + sizeof(struct ipv6hdr);
1392 tun_prot = ipv6_hdr(skb)->nexthdr;
1393 if (l4_hdr != exthdr)
1394 ipv6_skip_exthdr(skb, exthdr - skb->data,
1395 &tun_prot, &frag_off);
1396 break;
1397 default:
1398 return;
1399 }
1400 switch (tun_prot) {
1401 case IPPROTO_UDP:
1402 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;
1403 tunhdr_eiplen_tunlen |=
1404 ((skb_network_header_len(skb) >> 2) <<
1405 WX_TXD_OUTER_IPLEN_SHIFT) |
1406 (((skb_inner_mac_header(skb) -
1407 skb_transport_header(skb)) >> 1) <<
1408 WX_TXD_TUNNEL_LEN_SHIFT);
1409 break;
1410 case IPPROTO_GRE:
1411 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;
1412 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<
1413 WX_TXD_OUTER_IPLEN_SHIFT) |
1414 (((skb_inner_mac_header(skb) -
1415 skb_transport_header(skb)) >> 1) <<
1416 WX_TXD_TUNNEL_LEN_SHIFT);
1417 break;
1418 case IPPROTO_IPIP:
1419 case IPPROTO_IPV6:
1420 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -
1421 (char *)ip_hdr(skb)) >> 2) <<
1422 WX_TXD_OUTER_IPLEN_SHIFT;
1423 break;
1424 default:
1425 break;
1426 }
1427
1428 } else {
1429 network_hdr.raw = skb_network_header(skb);
1430 transport_hdr.raw = skb_transport_header(skb);
1431 vlan_macip_lens = skb_network_offset(skb) <<
1432 WX_TXD_MACLEN_SHIFT;
1433 }
1434
1435 switch (network_hdr.ipv4->version) {
1436 case IPVERSION:
1437 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1438 l4_prot = network_hdr.ipv4->protocol;
1439 break;
1440 case 6:
1441 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;
1442 exthdr = network_hdr.raw + sizeof(struct ipv6hdr);
1443 l4_prot = network_hdr.ipv6->nexthdr;
1444 if (transport_hdr.raw != exthdr)
1445 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_prot, &frag_off);
1446 break;
1447 default:
1448 break;
1449 }
1450
1451 switch (l4_prot) {
1452 case IPPROTO_TCP:
1453 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<
1454 WX_TXD_L4LEN_SHIFT;
1455 break;
1456 case IPPROTO_SCTP:
1457 mss_l4len_idx = sizeof(struct sctphdr) <<
1458 WX_TXD_L4LEN_SHIFT;
1459 break;
1460 case IPPROTO_UDP:
1461 mss_l4len_idx = sizeof(struct udphdr) <<
1462 WX_TXD_L4LEN_SHIFT;
1463 break;
1464 default:
1465 skb_checksum_help(skb);
1466 goto csum_failed;
1467 }
1468
1469 /* update TX checksum flag */
1470 first->tx_flags |= WX_TX_FLAGS_CSUM;
1471 }
1472 first->tx_flags |= WX_TX_FLAGS_CC;
1473 /* vlan_macip_lens: MACLEN, VLAN tag */
1474 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;
1475
1476 type_tucmd = ptype << 24;
1477 if (skb->vlan_proto == htons(ETH_P_8021AD) &&
1478 netdev->features & NETIF_F_HW_VLAN_STAG_TX)
1479 type_tucmd |= WX_SET_FLAG(first->tx_flags,
1480 WX_TX_FLAGS_HW_VLAN,
1481 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);
1482 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,
1483 type_tucmd, mss_l4len_idx);
1484 }
1485
wx_xmit_frame_ring(struct sk_buff * skb,struct wx_ring * tx_ring)1486 static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
1487 struct wx_ring *tx_ring)
1488 {
1489 struct wx *wx = netdev_priv(tx_ring->netdev);
1490 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1491 struct wx_tx_buffer *first;
1492 u8 hdr_len = 0, ptype;
1493 unsigned short f;
1494 u32 tx_flags = 0;
1495 int tso;
1496
1497 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,
1498 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,
1499 * + 2 desc gap to keep tail from touching head,
1500 * + 1 desc for context descriptor,
1501 * otherwise try next time
1502 */
1503 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1504 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->
1505 frags[f]));
1506
1507 if (wx_maybe_stop_tx(tx_ring, count + 3)) {
1508 tx_ring->tx_stats.tx_busy++;
1509 return NETDEV_TX_BUSY;
1510 }
1511
1512 /* record the location of the first descriptor for this packet */
1513 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1514 first->skb = skb;
1515 first->bytecount = skb->len;
1516 first->gso_segs = 1;
1517
1518 /* if we have a HW VLAN tag being added default to the HW one */
1519 if (skb_vlan_tag_present(skb)) {
1520 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;
1521 tx_flags |= WX_TX_FLAGS_HW_VLAN;
1522 }
1523
1524 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1525 wx->ptp_clock) {
1526 if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1527 !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS,
1528 wx->state)) {
1529 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1530 tx_flags |= WX_TX_FLAGS_TSTAMP;
1531 wx->ptp_tx_skb = skb_get(skb);
1532 wx->ptp_tx_start = jiffies;
1533 } else {
1534 wx->tx_hwtstamp_skipped++;
1535 }
1536 }
1537
1538 /* record initial flags and protocol */
1539 first->tx_flags = tx_flags;
1540 first->protocol = vlan_get_protocol(skb);
1541
1542 ptype = wx_encode_tx_desc_ptype(first);
1543
1544 tso = wx_tso(tx_ring, first, &hdr_len, ptype);
1545 if (tso < 0)
1546 goto out_drop;
1547 else if (!tso)
1548 wx_tx_csum(tx_ring, first, ptype);
1549
1550 if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)
1551 wx->atr(tx_ring, first, ptype);
1552
1553 if (wx_tx_map(tx_ring, first, hdr_len))
1554 goto cleanup_tx_tstamp;
1555
1556 return NETDEV_TX_OK;
1557 out_drop:
1558 dev_kfree_skb_any(first->skb);
1559 first->skb = NULL;
1560 cleanup_tx_tstamp:
1561 if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) {
1562 dev_kfree_skb_any(wx->ptp_tx_skb);
1563 wx->ptp_tx_skb = NULL;
1564 wx->tx_hwtstamp_errors++;
1565 clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
1566 }
1567
1568 return NETDEV_TX_OK;
1569 }
1570
wx_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1571 netdev_tx_t wx_xmit_frame(struct sk_buff *skb,
1572 struct net_device *netdev)
1573 {
1574 unsigned int r_idx = skb->queue_mapping;
1575 struct wx *wx = netdev_priv(netdev);
1576 struct wx_ring *tx_ring;
1577
1578 if (!netif_carrier_ok(netdev)) {
1579 dev_kfree_skb_any(skb);
1580 return NETDEV_TX_OK;
1581 }
1582
1583 /* The minimum packet size for olinfo paylen is 17 so pad the skb
1584 * in order to meet this minimum size requirement.
1585 */
1586 if (skb_put_padto(skb, 17))
1587 return NETDEV_TX_OK;
1588
1589 if (r_idx >= wx->num_tx_queues)
1590 r_idx = r_idx % wx->num_tx_queues;
1591 tx_ring = wx->tx_ring[r_idx];
1592
1593 return wx_xmit_frame_ring(skb, tx_ring);
1594 }
1595 EXPORT_SYMBOL(wx_xmit_frame);
1596
wx_napi_enable_all(struct wx * wx)1597 void wx_napi_enable_all(struct wx *wx)
1598 {
1599 struct wx_q_vector *q_vector;
1600 int q_idx;
1601
1602 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1603 q_vector = wx->q_vector[q_idx];
1604 napi_enable(&q_vector->napi);
1605 }
1606 }
1607 EXPORT_SYMBOL(wx_napi_enable_all);
1608
wx_napi_disable_all(struct wx * wx)1609 void wx_napi_disable_all(struct wx *wx)
1610 {
1611 struct wx_q_vector *q_vector;
1612 int q_idx;
1613
1614 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {
1615 q_vector = wx->q_vector[q_idx];
1616 napi_disable(&q_vector->napi);
1617 }
1618 }
1619 EXPORT_SYMBOL(wx_napi_disable_all);
1620
1621 /**
1622 * wx_set_rss_queues: Allocate queues for RSS
1623 * @wx: board private structure to initialize
1624 *
1625 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
1626 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1627 *
1628 **/
wx_set_rss_queues(struct wx * wx)1629 static void wx_set_rss_queues(struct wx *wx)
1630 {
1631 struct wx_ring_feature *f;
1632
1633 /* set mask for 16 queue limit of RSS */
1634 f = &wx->ring_feature[RING_F_RSS];
1635 f->indices = f->limit;
1636
1637 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
1638 goto out;
1639
1640 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
1641
1642 /* Use Flow Director in addition to RSS to ensure the best
1643 * distribution of flows across cores, even when an FDIR flow
1644 * isn't matched.
1645 */
1646 if (f->indices > 1) {
1647 f = &wx->ring_feature[RING_F_FDIR];
1648
1649 f->indices = f->limit;
1650
1651 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
1652 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
1653 }
1654
1655 out:
1656 wx->num_rx_queues = f->indices;
1657 wx->num_tx_queues = f->indices;
1658 }
1659
wx_set_num_queues(struct wx * wx)1660 static void wx_set_num_queues(struct wx *wx)
1661 {
1662 /* Start with base case */
1663 wx->num_rx_queues = 1;
1664 wx->num_tx_queues = 1;
1665 wx->queues_per_pool = 1;
1666
1667 wx_set_rss_queues(wx);
1668 }
1669
1670 /**
1671 * wx_acquire_msix_vectors - acquire MSI-X vectors
1672 * @wx: board private structure
1673 *
1674 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will
1675 * return a negative error code if unable to acquire MSI-X vectors for any
1676 * reason.
1677 */
wx_acquire_msix_vectors(struct wx * wx)1678 static int wx_acquire_msix_vectors(struct wx *wx)
1679 {
1680 struct irq_affinity affd = { .pre_vectors = 1 };
1681 int nvecs, i;
1682
1683 /* We start by asking for one vector per queue pair */
1684 nvecs = max(wx->num_rx_queues, wx->num_tx_queues);
1685 nvecs = min_t(int, nvecs, num_online_cpus());
1686 nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);
1687
1688 wx->msix_q_entries = kcalloc(nvecs, sizeof(struct msix_entry),
1689 GFP_KERNEL);
1690 if (!wx->msix_q_entries)
1691 return -ENOMEM;
1692
1693 /* One for non-queue interrupts */
1694 nvecs += 1;
1695
1696 wx->msix_entry = kcalloc(1, sizeof(struct msix_entry),
1697 GFP_KERNEL);
1698 if (!wx->msix_entry) {
1699 kfree(wx->msix_q_entries);
1700 wx->msix_q_entries = NULL;
1701 return -ENOMEM;
1702 }
1703
1704 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,
1705 nvecs,
1706 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
1707 &affd);
1708 if (nvecs < 0) {
1709 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);
1710 kfree(wx->msix_q_entries);
1711 wx->msix_q_entries = NULL;
1712 kfree(wx->msix_entry);
1713 wx->msix_entry = NULL;
1714 return nvecs;
1715 }
1716
1717 wx->msix_entry->entry = 0;
1718 wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);
1719 nvecs -= 1;
1720 for (i = 0; i < nvecs; i++) {
1721 wx->msix_q_entries[i].entry = i;
1722 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i + 1);
1723 }
1724
1725 wx->num_q_vectors = nvecs;
1726
1727 return 0;
1728 }
1729
1730 /**
1731 * wx_set_interrupt_capability - set MSI-X or MSI if supported
1732 * @wx: board private structure to initialize
1733 *
1734 * Attempt to configure the interrupts using the best available
1735 * capabilities of the hardware and the kernel.
1736 **/
wx_set_interrupt_capability(struct wx * wx)1737 static int wx_set_interrupt_capability(struct wx *wx)
1738 {
1739 struct pci_dev *pdev = wx->pdev;
1740 int nvecs, ret;
1741
1742 /* We will try to get MSI-X interrupts first */
1743 ret = wx_acquire_msix_vectors(wx);
1744 if (ret == 0 || (ret == -ENOMEM))
1745 return ret;
1746
1747 /* Disable RSS */
1748 dev_warn(&wx->pdev->dev, "Disabling RSS support\n");
1749 wx->ring_feature[RING_F_RSS].limit = 1;
1750
1751 wx_set_num_queues(wx);
1752
1753 /* minmum one for queue, one for misc*/
1754 nvecs = 1;
1755 nvecs = pci_alloc_irq_vectors(pdev, nvecs,
1756 nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);
1757 if (nvecs == 1) {
1758 if (pdev->msi_enabled)
1759 wx_err(wx, "Fallback to MSI.\n");
1760 else
1761 wx_err(wx, "Fallback to INTx.\n");
1762 } else {
1763 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);
1764 return nvecs;
1765 }
1766
1767 pdev->irq = pci_irq_vector(pdev, 0);
1768 wx->num_q_vectors = 1;
1769
1770 return 0;
1771 }
1772
1773 /**
1774 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS
1775 * @wx: board private structure to initialize
1776 *
1777 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.
1778 *
1779 **/
wx_cache_ring_rss(struct wx * wx)1780 static void wx_cache_ring_rss(struct wx *wx)
1781 {
1782 u16 i;
1783
1784 for (i = 0; i < wx->num_rx_queues; i++)
1785 wx->rx_ring[i]->reg_idx = i;
1786
1787 for (i = 0; i < wx->num_tx_queues; i++)
1788 wx->tx_ring[i]->reg_idx = i;
1789 }
1790
wx_add_ring(struct wx_ring * ring,struct wx_ring_container * head)1791 static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)
1792 {
1793 ring->next = head->ring;
1794 head->ring = ring;
1795 head->count++;
1796 }
1797
1798 /**
1799 * wx_alloc_q_vector - Allocate memory for a single interrupt vector
1800 * @wx: board private structure to initialize
1801 * @v_count: q_vectors allocated on wx, used for ring interleaving
1802 * @v_idx: index of vector in wx struct
1803 * @txr_count: total number of Tx rings to allocate
1804 * @txr_idx: index of first Tx ring to allocate
1805 * @rxr_count: total number of Rx rings to allocate
1806 * @rxr_idx: index of first Rx ring to allocate
1807 *
1808 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1809 **/
wx_alloc_q_vector(struct wx * wx,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)1810 static int wx_alloc_q_vector(struct wx *wx,
1811 unsigned int v_count, unsigned int v_idx,
1812 unsigned int txr_count, unsigned int txr_idx,
1813 unsigned int rxr_count, unsigned int rxr_idx)
1814 {
1815 struct wx_q_vector *q_vector;
1816 int ring_count, default_itr;
1817 struct wx_ring *ring;
1818
1819 /* note this will allocate space for the ring structure as well! */
1820 ring_count = txr_count + rxr_count;
1821
1822 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
1823 GFP_KERNEL);
1824 if (!q_vector)
1825 return -ENOMEM;
1826
1827 /* initialize NAPI */
1828 netif_napi_add(wx->netdev, &q_vector->napi,
1829 wx_poll);
1830
1831 /* tie q_vector and wx together */
1832 wx->q_vector[v_idx] = q_vector;
1833 q_vector->wx = wx;
1834 q_vector->v_idx = v_idx;
1835 if (cpu_online(v_idx))
1836 q_vector->numa_node = cpu_to_node(v_idx);
1837
1838 /* initialize pointer to rings */
1839 ring = q_vector->ring;
1840
1841 switch (wx->mac.type) {
1842 case wx_mac_sp:
1843 case wx_mac_aml:
1844 default_itr = WX_12K_ITR;
1845 break;
1846 default:
1847 default_itr = WX_7K_ITR;
1848 break;
1849 }
1850
1851 /* initialize ITR */
1852 if (txr_count && !rxr_count)
1853 /* tx only vector */
1854 q_vector->itr = wx->tx_itr_setting ?
1855 default_itr : wx->tx_itr_setting;
1856 else
1857 /* rx or rx/tx vector */
1858 q_vector->itr = wx->rx_itr_setting ?
1859 default_itr : wx->rx_itr_setting;
1860
1861 while (txr_count) {
1862 /* assign generic ring traits */
1863 ring->dev = &wx->pdev->dev;
1864 ring->netdev = wx->netdev;
1865
1866 /* configure backlink on ring */
1867 ring->q_vector = q_vector;
1868
1869 /* update q_vector Tx values */
1870 wx_add_ring(ring, &q_vector->tx);
1871
1872 /* apply Tx specific ring traits */
1873 ring->count = wx->tx_ring_count;
1874
1875 ring->queue_index = txr_idx;
1876
1877 /* assign ring to wx */
1878 wx->tx_ring[txr_idx] = ring;
1879
1880 /* update count and index */
1881 txr_count--;
1882 txr_idx += v_count;
1883
1884 /* push pointer to next ring */
1885 ring++;
1886 }
1887
1888 while (rxr_count) {
1889 /* assign generic ring traits */
1890 ring->dev = &wx->pdev->dev;
1891 ring->netdev = wx->netdev;
1892
1893 /* configure backlink on ring */
1894 ring->q_vector = q_vector;
1895
1896 /* update q_vector Rx values */
1897 wx_add_ring(ring, &q_vector->rx);
1898
1899 /* apply Rx specific ring traits */
1900 ring->count = wx->rx_ring_count;
1901 ring->queue_index = rxr_idx;
1902
1903 /* assign ring to wx */
1904 wx->rx_ring[rxr_idx] = ring;
1905
1906 /* update count and index */
1907 rxr_count--;
1908 rxr_idx += v_count;
1909
1910 /* push pointer to next ring */
1911 ring++;
1912 }
1913
1914 return 0;
1915 }
1916
1917 /**
1918 * wx_free_q_vector - Free memory allocated for specific interrupt vector
1919 * @wx: board private structure to initialize
1920 * @v_idx: Index of vector to be freed
1921 *
1922 * This function frees the memory allocated to the q_vector. In addition if
1923 * NAPI is enabled it will delete any references to the NAPI struct prior
1924 * to freeing the q_vector.
1925 **/
wx_free_q_vector(struct wx * wx,int v_idx)1926 static void wx_free_q_vector(struct wx *wx, int v_idx)
1927 {
1928 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
1929 struct wx_ring *ring;
1930
1931 wx_for_each_ring(ring, q_vector->tx)
1932 wx->tx_ring[ring->queue_index] = NULL;
1933
1934 wx_for_each_ring(ring, q_vector->rx)
1935 wx->rx_ring[ring->queue_index] = NULL;
1936
1937 wx->q_vector[v_idx] = NULL;
1938 netif_napi_del(&q_vector->napi);
1939 kfree_rcu(q_vector, rcu);
1940 }
1941
1942 /**
1943 * wx_alloc_q_vectors - Allocate memory for interrupt vectors
1944 * @wx: board private structure to initialize
1945 *
1946 * We allocate one q_vector per queue interrupt. If allocation fails we
1947 * return -ENOMEM.
1948 **/
wx_alloc_q_vectors(struct wx * wx)1949 static int wx_alloc_q_vectors(struct wx *wx)
1950 {
1951 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1952 unsigned int rxr_remaining = wx->num_rx_queues;
1953 unsigned int txr_remaining = wx->num_tx_queues;
1954 unsigned int q_vectors = wx->num_q_vectors;
1955 int rqpv, tqpv;
1956 int err;
1957
1958 for (; v_idx < q_vectors; v_idx++) {
1959 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1960 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1961 err = wx_alloc_q_vector(wx, q_vectors, v_idx,
1962 tqpv, txr_idx,
1963 rqpv, rxr_idx);
1964
1965 if (err)
1966 goto err_out;
1967
1968 /* update counts and index */
1969 rxr_remaining -= rqpv;
1970 txr_remaining -= tqpv;
1971 rxr_idx++;
1972 txr_idx++;
1973 }
1974
1975 return 0;
1976
1977 err_out:
1978 wx->num_tx_queues = 0;
1979 wx->num_rx_queues = 0;
1980 wx->num_q_vectors = 0;
1981
1982 while (v_idx--)
1983 wx_free_q_vector(wx, v_idx);
1984
1985 return -ENOMEM;
1986 }
1987
1988 /**
1989 * wx_free_q_vectors - Free memory allocated for interrupt vectors
1990 * @wx: board private structure to initialize
1991 *
1992 * This function frees the memory allocated to the q_vectors. In addition if
1993 * NAPI is enabled it will delete any references to the NAPI struct prior
1994 * to freeing the q_vector.
1995 **/
wx_free_q_vectors(struct wx * wx)1996 static void wx_free_q_vectors(struct wx *wx)
1997 {
1998 int v_idx = wx->num_q_vectors;
1999
2000 wx->num_tx_queues = 0;
2001 wx->num_rx_queues = 0;
2002 wx->num_q_vectors = 0;
2003
2004 while (v_idx--)
2005 wx_free_q_vector(wx, v_idx);
2006 }
2007
wx_reset_interrupt_capability(struct wx * wx)2008 void wx_reset_interrupt_capability(struct wx *wx)
2009 {
2010 struct pci_dev *pdev = wx->pdev;
2011
2012 if (!pdev->msi_enabled && !pdev->msix_enabled)
2013 return;
2014
2015 if (pdev->msix_enabled) {
2016 kfree(wx->msix_q_entries);
2017 wx->msix_q_entries = NULL;
2018 kfree(wx->msix_entry);
2019 wx->msix_entry = NULL;
2020 }
2021 pci_free_irq_vectors(wx->pdev);
2022 }
2023 EXPORT_SYMBOL(wx_reset_interrupt_capability);
2024
2025 /**
2026 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings
2027 * @wx: board private structure to clear interrupt scheme on
2028 *
2029 * We go through and clear interrupt specific resources and reset the structure
2030 * to pre-load conditions
2031 **/
wx_clear_interrupt_scheme(struct wx * wx)2032 void wx_clear_interrupt_scheme(struct wx *wx)
2033 {
2034 wx_free_q_vectors(wx);
2035 wx_reset_interrupt_capability(wx);
2036 }
2037 EXPORT_SYMBOL(wx_clear_interrupt_scheme);
2038
wx_init_interrupt_scheme(struct wx * wx)2039 int wx_init_interrupt_scheme(struct wx *wx)
2040 {
2041 int ret;
2042
2043 /* Number of supported queues */
2044 wx_set_num_queues(wx);
2045
2046 /* Set interrupt mode */
2047 ret = wx_set_interrupt_capability(wx);
2048 if (ret) {
2049 wx_err(wx, "Allocate irq vectors for failed.\n");
2050 return ret;
2051 }
2052
2053 /* Allocate memory for queues */
2054 ret = wx_alloc_q_vectors(wx);
2055 if (ret) {
2056 wx_err(wx, "Unable to allocate memory for queue vectors.\n");
2057 wx_reset_interrupt_capability(wx);
2058 return ret;
2059 }
2060
2061 wx_cache_ring_rss(wx);
2062
2063 return 0;
2064 }
2065 EXPORT_SYMBOL(wx_init_interrupt_scheme);
2066
wx_msix_clean_rings(int __always_unused irq,void * data)2067 irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)
2068 {
2069 struct wx_q_vector *q_vector = data;
2070
2071 /* EIAM disabled interrupts (on this vector) for us */
2072 if (q_vector->rx.ring || q_vector->tx.ring)
2073 napi_schedule_irqoff(&q_vector->napi);
2074
2075 return IRQ_HANDLED;
2076 }
2077 EXPORT_SYMBOL(wx_msix_clean_rings);
2078
wx_free_irq(struct wx * wx)2079 void wx_free_irq(struct wx *wx)
2080 {
2081 struct pci_dev *pdev = wx->pdev;
2082 int vector;
2083
2084 if (!(pdev->msix_enabled)) {
2085 if (!wx->misc_irq_domain)
2086 free_irq(pdev->irq, wx);
2087 return;
2088 }
2089
2090 for (vector = 0; vector < wx->num_q_vectors; vector++) {
2091 struct wx_q_vector *q_vector = wx->q_vector[vector];
2092 struct msix_entry *entry = &wx->msix_q_entries[vector];
2093
2094 /* free only the irqs that were actually requested */
2095 if (!q_vector->rx.ring && !q_vector->tx.ring)
2096 continue;
2097
2098 free_irq(entry->vector, q_vector);
2099 }
2100
2101 if (!wx->misc_irq_domain)
2102 free_irq(wx->msix_entry->vector, wx);
2103 }
2104 EXPORT_SYMBOL(wx_free_irq);
2105
2106 /**
2107 * wx_setup_isb_resources - allocate interrupt status resources
2108 * @wx: board private structure
2109 *
2110 * Return 0 on success, negative on failure
2111 **/
wx_setup_isb_resources(struct wx * wx)2112 int wx_setup_isb_resources(struct wx *wx)
2113 {
2114 struct pci_dev *pdev = wx->pdev;
2115
2116 if (wx->isb_mem)
2117 return 0;
2118
2119 wx->isb_mem = dma_alloc_coherent(&pdev->dev,
2120 sizeof(u32) * 4,
2121 &wx->isb_dma,
2122 GFP_KERNEL);
2123 if (!wx->isb_mem) {
2124 wx_err(wx, "Alloc isb_mem failed\n");
2125 return -ENOMEM;
2126 }
2127
2128 return 0;
2129 }
2130 EXPORT_SYMBOL(wx_setup_isb_resources);
2131
2132 /**
2133 * wx_free_isb_resources - allocate all queues Rx resources
2134 * @wx: board private structure
2135 *
2136 * Return 0 on success, negative on failure
2137 **/
wx_free_isb_resources(struct wx * wx)2138 void wx_free_isb_resources(struct wx *wx)
2139 {
2140 struct pci_dev *pdev = wx->pdev;
2141
2142 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,
2143 wx->isb_mem, wx->isb_dma);
2144 wx->isb_mem = NULL;
2145 }
2146 EXPORT_SYMBOL(wx_free_isb_resources);
2147
wx_misc_isb(struct wx * wx,enum wx_isb_idx idx)2148 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)
2149 {
2150 u32 cur_tag = 0;
2151
2152 cur_tag = wx->isb_mem[WX_ISB_HEADER];
2153 wx->isb_tag[idx] = cur_tag;
2154
2155 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);
2156 }
2157 EXPORT_SYMBOL(wx_misc_isb);
2158
2159 /**
2160 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
2161 * @wx: pointer to wx struct
2162 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
2163 * @queue: queue to map the corresponding interrupt to
2164 * @msix_vector: the vector to map to the corresponding queue
2165 *
2166 **/
wx_set_ivar(struct wx * wx,s8 direction,u16 queue,u16 msix_vector)2167 static void wx_set_ivar(struct wx *wx, s8 direction,
2168 u16 queue, u16 msix_vector)
2169 {
2170 u32 ivar, index;
2171
2172 if (direction == -1) {
2173 /* other causes */
2174 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2175 index = 0;
2176 ivar = rd32(wx, WX_PX_MISC_IVAR);
2177 ivar &= ~(0xFF << index);
2178 ivar |= (msix_vector << index);
2179 wr32(wx, WX_PX_MISC_IVAR, ivar);
2180 } else {
2181 /* tx or rx causes */
2182 msix_vector += 1; /* offset for queue vectors */
2183 msix_vector |= WX_PX_IVAR_ALLOC_VAL;
2184 index = ((16 * (queue & 1)) + (8 * direction));
2185 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));
2186 ivar &= ~(0xFF << index);
2187 ivar |= (msix_vector << index);
2188 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);
2189 }
2190 }
2191
2192 /**
2193 * wx_write_eitr - write EITR register in hardware specific way
2194 * @q_vector: structure containing interrupt and ring information
2195 *
2196 * This function is made to be called by ethtool and by the driver
2197 * when it needs to update EITR registers at runtime. Hardware
2198 * specific quirks/differences are taken care of here.
2199 */
wx_write_eitr(struct wx_q_vector * q_vector)2200 void wx_write_eitr(struct wx_q_vector *q_vector)
2201 {
2202 struct wx *wx = q_vector->wx;
2203 int v_idx = q_vector->v_idx;
2204 u32 itr_reg;
2205
2206 switch (wx->mac.type) {
2207 case wx_mac_sp:
2208 itr_reg = q_vector->itr & WX_SP_MAX_EITR;
2209 break;
2210 case wx_mac_aml:
2211 itr_reg = (q_vector->itr >> 3) & WX_AML_MAX_EITR;
2212 break;
2213 default:
2214 itr_reg = q_vector->itr & WX_EM_MAX_EITR;
2215 break;
2216 }
2217
2218 itr_reg |= WX_PX_ITR_CNT_WDIS;
2219
2220 wr32(wx, WX_PX_ITR(v_idx + 1), itr_reg);
2221 }
2222
2223 /**
2224 * wx_configure_vectors - Configure vectors for hardware
2225 * @wx: board private structure
2226 *
2227 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx
2228 * interrupts.
2229 **/
wx_configure_vectors(struct wx * wx)2230 void wx_configure_vectors(struct wx *wx)
2231 {
2232 struct pci_dev *pdev = wx->pdev;
2233 u32 eitrsel = 0;
2234 u16 v_idx;
2235
2236 if (pdev->msix_enabled) {
2237 /* Populate MSIX to EITR Select */
2238 wr32(wx, WX_PX_ITRSEL, eitrsel);
2239 /* use EIAM to auto-mask when MSI-X interrupt is asserted
2240 * this saves a register write for every interrupt
2241 */
2242 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);
2243 } else {
2244 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
2245 * specifically only auto mask tx and rx interrupts.
2246 */
2247 wr32(wx, WX_PX_GPIE, 0);
2248 }
2249
2250 /* Populate the IVAR table and set the ITR values to the
2251 * corresponding register.
2252 */
2253 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {
2254 struct wx_q_vector *q_vector = wx->q_vector[v_idx];
2255 struct wx_ring *ring;
2256
2257 wx_for_each_ring(ring, q_vector->rx)
2258 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);
2259
2260 wx_for_each_ring(ring, q_vector->tx)
2261 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);
2262
2263 wx_write_eitr(q_vector);
2264 }
2265
2266 wx_set_ivar(wx, -1, 0, 0);
2267 if (pdev->msix_enabled)
2268 wr32(wx, WX_PX_ITR(0), 1950);
2269 }
2270 EXPORT_SYMBOL(wx_configure_vectors);
2271
2272 /**
2273 * wx_clean_rx_ring - Free Rx Buffers per Queue
2274 * @rx_ring: ring to free buffers from
2275 **/
wx_clean_rx_ring(struct wx_ring * rx_ring)2276 static void wx_clean_rx_ring(struct wx_ring *rx_ring)
2277 {
2278 struct wx_rx_buffer *rx_buffer;
2279 u16 i = rx_ring->next_to_clean;
2280
2281 rx_buffer = &rx_ring->rx_buffer_info[i];
2282
2283 /* Free all the Rx ring sk_buffs */
2284 while (i != rx_ring->next_to_alloc) {
2285 if (rx_buffer->skb) {
2286 struct sk_buff *skb = rx_buffer->skb;
2287
2288 if (WX_CB(skb)->page_released)
2289 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2290
2291 dev_kfree_skb(skb);
2292 }
2293
2294 /* Invalidate cache lines that may have been written to by
2295 * device so that we avoid corrupting memory.
2296 */
2297 dma_sync_single_range_for_cpu(rx_ring->dev,
2298 rx_buffer->dma,
2299 rx_buffer->page_offset,
2300 WX_RX_BUFSZ,
2301 DMA_FROM_DEVICE);
2302
2303 /* free resources associated with mapping */
2304 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);
2305
2306 i++;
2307 rx_buffer++;
2308 if (i == rx_ring->count) {
2309 i = 0;
2310 rx_buffer = rx_ring->rx_buffer_info;
2311 }
2312 }
2313
2314 rx_ring->next_to_alloc = 0;
2315 rx_ring->next_to_clean = 0;
2316 rx_ring->next_to_use = 0;
2317 }
2318
2319 /**
2320 * wx_clean_all_rx_rings - Free Rx Buffers for all queues
2321 * @wx: board private structure
2322 **/
wx_clean_all_rx_rings(struct wx * wx)2323 void wx_clean_all_rx_rings(struct wx *wx)
2324 {
2325 int i;
2326
2327 for (i = 0; i < wx->num_rx_queues; i++)
2328 wx_clean_rx_ring(wx->rx_ring[i]);
2329 }
2330 EXPORT_SYMBOL(wx_clean_all_rx_rings);
2331
2332 /**
2333 * wx_free_rx_resources - Free Rx Resources
2334 * @rx_ring: ring to clean the resources from
2335 *
2336 * Free all receive software resources
2337 **/
wx_free_rx_resources(struct wx_ring * rx_ring)2338 static void wx_free_rx_resources(struct wx_ring *rx_ring)
2339 {
2340 wx_clean_rx_ring(rx_ring);
2341 kvfree(rx_ring->rx_buffer_info);
2342 rx_ring->rx_buffer_info = NULL;
2343
2344 /* if not set, then don't free */
2345 if (!rx_ring->desc)
2346 return;
2347
2348 dma_free_coherent(rx_ring->dev, rx_ring->size,
2349 rx_ring->desc, rx_ring->dma);
2350
2351 rx_ring->desc = NULL;
2352
2353 if (rx_ring->page_pool) {
2354 page_pool_destroy(rx_ring->page_pool);
2355 rx_ring->page_pool = NULL;
2356 }
2357 }
2358
2359 /**
2360 * wx_free_all_rx_resources - Free Rx Resources for All Queues
2361 * @wx: pointer to hardware structure
2362 *
2363 * Free all receive software resources
2364 **/
wx_free_all_rx_resources(struct wx * wx)2365 static void wx_free_all_rx_resources(struct wx *wx)
2366 {
2367 int i;
2368
2369 for (i = 0; i < wx->num_rx_queues; i++)
2370 wx_free_rx_resources(wx->rx_ring[i]);
2371 }
2372
2373 /**
2374 * wx_clean_tx_ring - Free Tx Buffers
2375 * @tx_ring: ring to be cleaned
2376 **/
wx_clean_tx_ring(struct wx_ring * tx_ring)2377 static void wx_clean_tx_ring(struct wx_ring *tx_ring)
2378 {
2379 struct wx_tx_buffer *tx_buffer;
2380 u16 i = tx_ring->next_to_clean;
2381
2382 tx_buffer = &tx_ring->tx_buffer_info[i];
2383
2384 while (i != tx_ring->next_to_use) {
2385 union wx_tx_desc *eop_desc, *tx_desc;
2386
2387 /* Free all the Tx ring sk_buffs */
2388 dev_kfree_skb_any(tx_buffer->skb);
2389
2390 /* unmap skb header data */
2391 dma_unmap_single(tx_ring->dev,
2392 dma_unmap_addr(tx_buffer, dma),
2393 dma_unmap_len(tx_buffer, len),
2394 DMA_TO_DEVICE);
2395
2396 /* check for eop_desc to determine the end of the packet */
2397 eop_desc = tx_buffer->next_to_watch;
2398 tx_desc = WX_TX_DESC(tx_ring, i);
2399
2400 /* unmap remaining buffers */
2401 while (tx_desc != eop_desc) {
2402 tx_buffer++;
2403 tx_desc++;
2404 i++;
2405 if (unlikely(i == tx_ring->count)) {
2406 i = 0;
2407 tx_buffer = tx_ring->tx_buffer_info;
2408 tx_desc = WX_TX_DESC(tx_ring, 0);
2409 }
2410
2411 /* unmap any remaining paged data */
2412 if (dma_unmap_len(tx_buffer, len))
2413 dma_unmap_page(tx_ring->dev,
2414 dma_unmap_addr(tx_buffer, dma),
2415 dma_unmap_len(tx_buffer, len),
2416 DMA_TO_DEVICE);
2417 }
2418
2419 /* move us one more past the eop_desc for start of next pkt */
2420 tx_buffer++;
2421 i++;
2422 if (unlikely(i == tx_ring->count)) {
2423 i = 0;
2424 tx_buffer = tx_ring->tx_buffer_info;
2425 }
2426 }
2427
2428 netdev_tx_reset_queue(wx_txring_txq(tx_ring));
2429
2430 /* reset next_to_use and next_to_clean */
2431 tx_ring->next_to_use = 0;
2432 tx_ring->next_to_clean = 0;
2433 }
2434
2435 /**
2436 * wx_clean_all_tx_rings - Free Tx Buffers for all queues
2437 * @wx: board private structure
2438 **/
wx_clean_all_tx_rings(struct wx * wx)2439 void wx_clean_all_tx_rings(struct wx *wx)
2440 {
2441 int i;
2442
2443 for (i = 0; i < wx->num_tx_queues; i++)
2444 wx_clean_tx_ring(wx->tx_ring[i]);
2445 }
2446 EXPORT_SYMBOL(wx_clean_all_tx_rings);
2447
2448 /**
2449 * wx_free_tx_resources - Free Tx Resources per Queue
2450 * @tx_ring: Tx descriptor ring for a specific queue
2451 *
2452 * Free all transmit software resources
2453 **/
wx_free_tx_resources(struct wx_ring * tx_ring)2454 static void wx_free_tx_resources(struct wx_ring *tx_ring)
2455 {
2456 wx_clean_tx_ring(tx_ring);
2457 kvfree(tx_ring->tx_buffer_info);
2458 tx_ring->tx_buffer_info = NULL;
2459
2460 /* if not set, then don't free */
2461 if (!tx_ring->desc)
2462 return;
2463
2464 dma_free_coherent(tx_ring->dev, tx_ring->size,
2465 tx_ring->desc, tx_ring->dma);
2466 tx_ring->desc = NULL;
2467 }
2468
2469 /**
2470 * wx_free_all_tx_resources - Free Tx Resources for All Queues
2471 * @wx: pointer to hardware structure
2472 *
2473 * Free all transmit software resources
2474 **/
wx_free_all_tx_resources(struct wx * wx)2475 static void wx_free_all_tx_resources(struct wx *wx)
2476 {
2477 int i;
2478
2479 for (i = 0; i < wx->num_tx_queues; i++)
2480 wx_free_tx_resources(wx->tx_ring[i]);
2481 }
2482
wx_free_resources(struct wx * wx)2483 void wx_free_resources(struct wx *wx)
2484 {
2485 wx_free_all_rx_resources(wx);
2486 wx_free_all_tx_resources(wx);
2487 }
2488 EXPORT_SYMBOL(wx_free_resources);
2489
wx_alloc_page_pool(struct wx_ring * rx_ring)2490 static int wx_alloc_page_pool(struct wx_ring *rx_ring)
2491 {
2492 int ret = 0;
2493
2494 struct page_pool_params pp_params = {
2495 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
2496 .order = 0,
2497 .pool_size = rx_ring->size,
2498 .nid = dev_to_node(rx_ring->dev),
2499 .dev = rx_ring->dev,
2500 .dma_dir = DMA_FROM_DEVICE,
2501 .offset = 0,
2502 .max_len = PAGE_SIZE,
2503 };
2504
2505 rx_ring->page_pool = page_pool_create(&pp_params);
2506 if (IS_ERR(rx_ring->page_pool)) {
2507 ret = PTR_ERR(rx_ring->page_pool);
2508 rx_ring->page_pool = NULL;
2509 }
2510
2511 return ret;
2512 }
2513
2514 /**
2515 * wx_setup_rx_resources - allocate Rx resources (Descriptors)
2516 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2517 *
2518 * Returns 0 on success, negative on failure
2519 **/
wx_setup_rx_resources(struct wx_ring * rx_ring)2520 static int wx_setup_rx_resources(struct wx_ring *rx_ring)
2521 {
2522 struct device *dev = rx_ring->dev;
2523 int orig_node = dev_to_node(dev);
2524 int numa_node = NUMA_NO_NODE;
2525 int size, ret;
2526
2527 size = sizeof(struct wx_rx_buffer) * rx_ring->count;
2528
2529 if (rx_ring->q_vector)
2530 numa_node = rx_ring->q_vector->numa_node;
2531
2532 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2533 if (!rx_ring->rx_buffer_info)
2534 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);
2535 if (!rx_ring->rx_buffer_info)
2536 goto err;
2537
2538 /* Round up to nearest 4K */
2539 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);
2540 rx_ring->size = ALIGN(rx_ring->size, 4096);
2541
2542 set_dev_node(dev, numa_node);
2543 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2544 &rx_ring->dma, GFP_KERNEL);
2545 if (!rx_ring->desc) {
2546 set_dev_node(dev, orig_node);
2547 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
2548 &rx_ring->dma, GFP_KERNEL);
2549 }
2550
2551 if (!rx_ring->desc)
2552 goto err;
2553
2554 rx_ring->next_to_clean = 0;
2555 rx_ring->next_to_use = 0;
2556
2557 ret = wx_alloc_page_pool(rx_ring);
2558 if (ret < 0) {
2559 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);
2560 goto err_desc;
2561 }
2562
2563 return 0;
2564
2565 err_desc:
2566 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
2567 err:
2568 kvfree(rx_ring->rx_buffer_info);
2569 rx_ring->rx_buffer_info = NULL;
2570 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2571 return -ENOMEM;
2572 }
2573
2574 /**
2575 * wx_setup_all_rx_resources - allocate all queues Rx resources
2576 * @wx: pointer to hardware structure
2577 *
2578 * If this function returns with an error, then it's possible one or
2579 * more of the rings is populated (while the rest are not). It is the
2580 * callers duty to clean those orphaned rings.
2581 *
2582 * Return 0 on success, negative on failure
2583 **/
wx_setup_all_rx_resources(struct wx * wx)2584 static int wx_setup_all_rx_resources(struct wx *wx)
2585 {
2586 int i, err = 0;
2587
2588 for (i = 0; i < wx->num_rx_queues; i++) {
2589 err = wx_setup_rx_resources(wx->rx_ring[i]);
2590 if (!err)
2591 continue;
2592
2593 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);
2594 goto err_setup_rx;
2595 }
2596
2597 return 0;
2598 err_setup_rx:
2599 /* rewind the index freeing the rings as we go */
2600 while (i--)
2601 wx_free_rx_resources(wx->rx_ring[i]);
2602 return err;
2603 }
2604
2605 /**
2606 * wx_setup_tx_resources - allocate Tx resources (Descriptors)
2607 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2608 *
2609 * Return 0 on success, negative on failure
2610 **/
wx_setup_tx_resources(struct wx_ring * tx_ring)2611 static int wx_setup_tx_resources(struct wx_ring *tx_ring)
2612 {
2613 struct device *dev = tx_ring->dev;
2614 int orig_node = dev_to_node(dev);
2615 int numa_node = NUMA_NO_NODE;
2616 int size;
2617
2618 size = sizeof(struct wx_tx_buffer) * tx_ring->count;
2619
2620 if (tx_ring->q_vector)
2621 numa_node = tx_ring->q_vector->numa_node;
2622
2623 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);
2624 if (!tx_ring->tx_buffer_info)
2625 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);
2626 if (!tx_ring->tx_buffer_info)
2627 goto err;
2628
2629 /* round up to nearest 4K */
2630 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);
2631 tx_ring->size = ALIGN(tx_ring->size, 4096);
2632
2633 set_dev_node(dev, numa_node);
2634 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2635 &tx_ring->dma, GFP_KERNEL);
2636 if (!tx_ring->desc) {
2637 set_dev_node(dev, orig_node);
2638 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
2639 &tx_ring->dma, GFP_KERNEL);
2640 }
2641
2642 if (!tx_ring->desc)
2643 goto err;
2644
2645 tx_ring->next_to_use = 0;
2646 tx_ring->next_to_clean = 0;
2647
2648 return 0;
2649
2650 err:
2651 kvfree(tx_ring->tx_buffer_info);
2652 tx_ring->tx_buffer_info = NULL;
2653 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2654 return -ENOMEM;
2655 }
2656
2657 /**
2658 * wx_setup_all_tx_resources - allocate all queues Tx resources
2659 * @wx: pointer to private structure
2660 *
2661 * If this function returns with an error, then it's possible one or
2662 * more of the rings is populated (while the rest are not). It is the
2663 * callers duty to clean those orphaned rings.
2664 *
2665 * Return 0 on success, negative on failure
2666 **/
wx_setup_all_tx_resources(struct wx * wx)2667 static int wx_setup_all_tx_resources(struct wx *wx)
2668 {
2669 int i, err = 0;
2670
2671 for (i = 0; i < wx->num_tx_queues; i++) {
2672 err = wx_setup_tx_resources(wx->tx_ring[i]);
2673 if (!err)
2674 continue;
2675
2676 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);
2677 goto err_setup_tx;
2678 }
2679
2680 return 0;
2681 err_setup_tx:
2682 /* rewind the index freeing the rings as we go */
2683 while (i--)
2684 wx_free_tx_resources(wx->tx_ring[i]);
2685 return err;
2686 }
2687
wx_setup_resources(struct wx * wx)2688 int wx_setup_resources(struct wx *wx)
2689 {
2690 int err;
2691
2692 /* allocate transmit descriptors */
2693 err = wx_setup_all_tx_resources(wx);
2694 if (err)
2695 return err;
2696
2697 /* allocate receive descriptors */
2698 err = wx_setup_all_rx_resources(wx);
2699 if (err)
2700 goto err_free_tx;
2701
2702 err = wx_setup_isb_resources(wx);
2703 if (err)
2704 goto err_free_rx;
2705
2706 return 0;
2707
2708 err_free_rx:
2709 wx_free_all_rx_resources(wx);
2710 err_free_tx:
2711 wx_free_all_tx_resources(wx);
2712
2713 return err;
2714 }
2715 EXPORT_SYMBOL(wx_setup_resources);
2716
2717 /**
2718 * wx_get_stats64 - Get System Network Statistics
2719 * @netdev: network interface device structure
2720 * @stats: storage space for 64bit statistics
2721 */
wx_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2722 void wx_get_stats64(struct net_device *netdev,
2723 struct rtnl_link_stats64 *stats)
2724 {
2725 struct wx *wx = netdev_priv(netdev);
2726 struct wx_hw_stats *hwstats;
2727 int i;
2728
2729 wx_update_stats(wx);
2730
2731 rcu_read_lock();
2732 for (i = 0; i < wx->num_rx_queues; i++) {
2733 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);
2734 u64 bytes, packets;
2735 unsigned int start;
2736
2737 if (ring) {
2738 do {
2739 start = u64_stats_fetch_begin(&ring->syncp);
2740 packets = ring->stats.packets;
2741 bytes = ring->stats.bytes;
2742 } while (u64_stats_fetch_retry(&ring->syncp, start));
2743 stats->rx_packets += packets;
2744 stats->rx_bytes += bytes;
2745 }
2746 }
2747
2748 for (i = 0; i < wx->num_tx_queues; i++) {
2749 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);
2750 u64 bytes, packets;
2751 unsigned int start;
2752
2753 if (ring) {
2754 do {
2755 start = u64_stats_fetch_begin(&ring->syncp);
2756 packets = ring->stats.packets;
2757 bytes = ring->stats.bytes;
2758 } while (u64_stats_fetch_retry(&ring->syncp,
2759 start));
2760 stats->tx_packets += packets;
2761 stats->tx_bytes += bytes;
2762 }
2763 }
2764
2765 rcu_read_unlock();
2766
2767 hwstats = &wx->stats;
2768 stats->rx_errors = hwstats->crcerrs + hwstats->rlec;
2769 stats->multicast = hwstats->qmprc;
2770 stats->rx_length_errors = hwstats->rlec;
2771 stats->rx_crc_errors = hwstats->crcerrs;
2772 }
2773 EXPORT_SYMBOL(wx_get_stats64);
2774
wx_set_features(struct net_device * netdev,netdev_features_t features)2775 int wx_set_features(struct net_device *netdev, netdev_features_t features)
2776 {
2777 netdev_features_t changed = netdev->features ^ features;
2778 struct wx *wx = netdev_priv(netdev);
2779 bool need_reset = false;
2780
2781 if (features & NETIF_F_RXHASH) {
2782 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,
2783 WX_RDB_RA_CTL_RSS_EN);
2784 wx->rss_enabled = true;
2785 } else {
2786 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);
2787 wx->rss_enabled = false;
2788 }
2789
2790 netdev->features = features;
2791
2792 if (changed & NETIF_F_HW_VLAN_CTAG_RX && wx->do_reset)
2793 wx->do_reset(netdev);
2794 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))
2795 wx_set_rx_mode(netdev);
2796
2797 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))
2798 return 0;
2799
2800 /* Check if Flow Director n-tuple support was enabled or disabled. If
2801 * the state changed, we need to reset.
2802 */
2803 switch (features & NETIF_F_NTUPLE) {
2804 case NETIF_F_NTUPLE:
2805 /* turn off ATR, enable perfect filters and reset */
2806 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))
2807 need_reset = true;
2808
2809 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);
2810 break;
2811 default:
2812 /* turn off perfect filters, enable ATR and reset */
2813 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))
2814 need_reset = true;
2815
2816 /* We cannot enable ATR if RSS is disabled */
2817 if (wx->ring_feature[RING_F_RSS].limit <= 1)
2818 break;
2819
2820 set_bit(WX_FLAG_FDIR_HASH, wx->flags);
2821 break;
2822 }
2823
2824 if (need_reset && wx->do_reset)
2825 wx->do_reset(netdev);
2826
2827 return 0;
2828 }
2829 EXPORT_SYMBOL(wx_set_features);
2830
2831 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
2832 NETIF_F_HW_VLAN_STAG_RX)
2833
2834 #define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \
2835 NETIF_F_HW_VLAN_STAG_TX)
2836
2837 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \
2838 NETIF_F_HW_VLAN_STAG_FILTER)
2839
wx_fix_features(struct net_device * netdev,netdev_features_t features)2840 netdev_features_t wx_fix_features(struct net_device *netdev,
2841 netdev_features_t features)
2842 {
2843 netdev_features_t changed = netdev->features ^ features;
2844 struct wx *wx = netdev_priv(netdev);
2845
2846 if (changed & NETIF_VLAN_STRIPPING_FEATURES) {
2847 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&
2848 (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {
2849 features &= ~NETIF_VLAN_STRIPPING_FEATURES;
2850 features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;
2851 wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");
2852 }
2853 }
2854
2855 if (changed & NETIF_VLAN_INSERTION_FEATURES) {
2856 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&
2857 (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {
2858 features &= ~NETIF_VLAN_INSERTION_FEATURES;
2859 features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;
2860 wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");
2861 }
2862 }
2863
2864 if (changed & NETIF_VLAN_FILTERING_FEATURES) {
2865 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&
2866 (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {
2867 features &= ~NETIF_VLAN_FILTERING_FEATURES;
2868 features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;
2869 wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");
2870 }
2871 }
2872
2873 return features;
2874 }
2875 EXPORT_SYMBOL(wx_fix_features);
2876
wx_set_ring(struct wx * wx,u32 new_tx_count,u32 new_rx_count,struct wx_ring * temp_ring)2877 void wx_set_ring(struct wx *wx, u32 new_tx_count,
2878 u32 new_rx_count, struct wx_ring *temp_ring)
2879 {
2880 int i, err = 0;
2881
2882 /* Setup new Tx resources and free the old Tx resources in that order.
2883 * We can then assign the new resources to the rings via a memcpy.
2884 * The advantage to this approach is that we are guaranteed to still
2885 * have resources even in the case of an allocation failure.
2886 */
2887 if (new_tx_count != wx->tx_ring_count) {
2888 for (i = 0; i < wx->num_tx_queues; i++) {
2889 memcpy(&temp_ring[i], wx->tx_ring[i],
2890 sizeof(struct wx_ring));
2891
2892 temp_ring[i].count = new_tx_count;
2893 err = wx_setup_tx_resources(&temp_ring[i]);
2894 if (err) {
2895 wx_err(wx, "setup new tx resources failed, keep using the old config\n");
2896 while (i) {
2897 i--;
2898 wx_free_tx_resources(&temp_ring[i]);
2899 }
2900 return;
2901 }
2902 }
2903
2904 for (i = 0; i < wx->num_tx_queues; i++) {
2905 wx_free_tx_resources(wx->tx_ring[i]);
2906
2907 memcpy(wx->tx_ring[i], &temp_ring[i],
2908 sizeof(struct wx_ring));
2909 }
2910
2911 wx->tx_ring_count = new_tx_count;
2912 }
2913
2914 /* Repeat the process for the Rx rings if needed */
2915 if (new_rx_count != wx->rx_ring_count) {
2916 for (i = 0; i < wx->num_rx_queues; i++) {
2917 memcpy(&temp_ring[i], wx->rx_ring[i],
2918 sizeof(struct wx_ring));
2919
2920 temp_ring[i].count = new_rx_count;
2921 err = wx_setup_rx_resources(&temp_ring[i]);
2922 if (err) {
2923 wx_err(wx, "setup new rx resources failed, keep using the old config\n");
2924 while (i) {
2925 i--;
2926 wx_free_rx_resources(&temp_ring[i]);
2927 }
2928 return;
2929 }
2930 }
2931
2932 for (i = 0; i < wx->num_rx_queues; i++) {
2933 wx_free_rx_resources(wx->rx_ring[i]);
2934 memcpy(wx->rx_ring[i], &temp_ring[i],
2935 sizeof(struct wx_ring));
2936 }
2937
2938 wx->rx_ring_count = new_rx_count;
2939 }
2940 }
2941 EXPORT_SYMBOL(wx_set_ring);
2942
2943 MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");
2944 MODULE_LICENSE("GPL");
2945