1 /******************************************************************************
2
3 Copyright (c) 2013-2018, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33
34 #ifndef _IXL_H_
35 #define _IXL_H_
36
37 #include "opt_inet.h"
38 #include "opt_inet6.h"
39 #include "opt_rss.h"
40 #include "opt_ixl.h"
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/buf_ring.h>
45 #include <sys/mbuf.h>
46 #include <sys/protosw.h>
47 #include <sys/socket.h>
48 #include <sys/malloc.h>
49 #include <sys/kernel.h>
50 #include <sys/module.h>
51 #include <sys/sockio.h>
52 #include <sys/eventhandler.h>
53 #include <sys/syslog.h>
54 #include <sys/priv.h>
55 #include <sys/bitstring.h>
56
57 #include <net/if.h>
58 #include <net/if_var.h>
59 #include <net/if_arp.h>
60 #include <net/bpf.h>
61 #include <net/ethernet.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
64 #include <net/iflib.h>
65
66 #include <net/bpf.h>
67 #include <net/if_types.h>
68 #include <net/if_vlan_var.h>
69
70 #include <netinet/in_systm.h>
71 #include <netinet/in.h>
72 #include <netinet/if_ether.h>
73 #include <netinet/ip.h>
74 #include <netinet/ip6.h>
75 #include <netinet/tcp.h>
76 #include <netinet/tcp_lro.h>
77 #include <netinet/udp.h>
78 #include <netinet/sctp.h>
79
80 #include <machine/in_cksum.h>
81
82 #include <sys/bus.h>
83 #include <machine/bus.h>
84 #include <sys/rman.h>
85 #include <machine/resource.h>
86 #include <vm/vm.h>
87 #include <vm/pmap.h>
88 #include <machine/clock.h>
89 #include <dev/pci/pcivar.h>
90 #include <dev/pci/pcireg.h>
91 #include <sys/proc.h>
92 #include <sys/stdarg.h>
93 #include <sys/sysctl.h>
94 #include <sys/endian.h>
95 #include <sys/taskqueue.h>
96 #include <sys/pcpu.h>
97 #include <sys/smp.h>
98 #include <sys/sbuf.h>
99 #include <machine/smp.h>
100
101 #include <net/rss_config.h>
102 #include <netinet/in_rss.h>
103
104 #include "ifdi_if.h"
105 #include "i40e_type.h"
106 #include "i40e_prototype.h"
107 #include "ixl_debug.h"
108
109 #define PVIDV(vendor, devid, name) \
110 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING)
111
112 /* Tunables */
113
114 /*
115 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the
116 * number of tx/rx descriptors allocated by the driver. Increasing this
117 * value allows the driver to queue more operations.
118 *
119 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes.
120 * The driver currently always uses 32 byte Rx descriptors.
121 */
122 #define IXL_DEFAULT_RING 1024
123 #define IXL_MAX_RING 4096
124 #define IXL_MIN_RING 64
125 #define IXL_RING_INCREMENT 32
126
127 #define IXL_AQ_LEN 256
128 #define IXL_AQ_LEN_MAX 1024
129
130 /* Alignment for rings */
131 #define DBA_ALIGN 128
132
133 #define MAX_MULTICAST_ADDR 128
134
135 #define IXL_MSIX_BAR 3
136 #define IXL_ADM_LIMIT 2
137 #define IXL_TSO_SIZE ((255*1024)-1)
138 #define IXL_TX_BUF_SZ ((u32) 1514)
139 #define IXL_AQ_BUF_SZ ((u32) 4096)
140 #define IXL_RX_ITR 0
141 #define IXL_TX_ITR 1
142 #define IXL_ITR_NONE 3
143 #define IXL_QUEUE_EOL 0x7FF
144 #define IXL_MIN_FRAME 17
145 #define IXL_MAX_FRAME 9728
146 #define IXL_MAX_TX_SEGS 8
147 #define IXL_MAX_RX_SEGS 5
148 #define IXL_MAX_TSO_SEGS 128
149 #define IXL_SPARSE_CHAIN 7
150 #define IXL_MIN_TSO_MSS 64
151 #define IXL_MAX_TSO_MSS 9668
152 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1)
153
154 #define IXL_RSS_KEY_SIZE_REG 13
155 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4)
156 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */
157 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F
158 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF
159
160 #define IXL_VF_MAX_BUFFER 0x3F80
161 #define IXL_VF_MAX_HDR_BUFFER 0x840
162 #define IXL_VF_MAX_FRAME 0x3FFF
163
164 #define IXL_NVM_VERSION_LO_SHIFT 0
165 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT)
166 #define IXL_NVM_VERSION_HI_SHIFT 12
167 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT)
168
169 /*
170 * Interrupt Moderation parameters
171 * Multiply ITR values by 2 for real ITR value
172 */
173 #define IXL_MAX_ITR 0x0FF0
174 #define IXL_ITR_100K 0x0005
175 #define IXL_ITR_20K 0x0019
176 #define IXL_ITR_8K 0x003E
177 #define IXL_ITR_4K 0x007A
178 #define IXL_ITR_1K 0x01F4
179 #define IXL_ITR_DYNAMIC 0x8000
180 #define IXL_LOW_LATENCY 0
181 #define IXL_AVE_LATENCY 1
182 #define IXL_BULK_LATENCY 2
183
184 /* MacVlan Flags */
185 #define IXL_FILTER_VLAN (u16)(1 << 0)
186 #define IXL_FILTER_MC (u16)(1 << 1)
187
188 /* used in the vlan field of the filter when not a vlan */
189 #define IXL_VLAN_ANY -1
190
191 /* Maximum number of MAC/VLAN filters supported by HW */
192 #define IXL_MAX_VLAN_FILTERS 255
193
194 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
195 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6)
196 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO)
197
198 /* Misc flags for ixl_vsi.flags */
199 #define IXL_FLAGS_KEEP_TSO4 (1 << 0)
200 #define IXL_FLAGS_KEEP_TSO6 (1 << 1)
201 #define IXL_FLAGS_USES_MSIX (1 << 2)
202 #define IXL_FLAGS_IS_VF (1 << 3)
203 #define IXL_FLAGS_MC_PROMISC (1 << 4)
204
205 #define IXL_VSI_IS_PF(v) ((v->flags & IXL_FLAGS_IS_VF) == 0)
206 #define IXL_VSI_IS_VF(v) ((v->flags & IXL_FLAGS_IS_VF) != 0)
207
208 #define IXL_VF_RESET_TIMEOUT 100
209
210 #define IXL_VSI_DATA_PORT 0x01
211
212 #define IAVF_MAX_QUEUES 16
213 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1))
214
215 #define IXL_RX_CTX_BASE_UNITS 128
216 #define IXL_TX_CTX_BASE_UNITS 128
217
218 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA
219
220 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20
221
222 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32)
223 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32))
224
225 #define IXL_MAX_ITR_IDX 3
226
227 #define IXL_END_OF_INTR_LNKLST 0x7FF
228
229 #define IXL_DEFAULT_RSS_HENA_BASE (\
230 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \
231 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \
232 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
233 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
234 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \
235 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \
236 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \
237 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
238 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
239 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \
240 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD))
241
242 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE
243
244 #define IXL_DEFAULT_RSS_HENA_X722 (\
245 IXL_DEFAULT_RSS_HENA_BASE | \
246 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \
247 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \
248 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
249 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \
250 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \
251 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
252
253 #define IXL_CAPS \
254 (IFCAP_TSO4 | IFCAP_TSO6 | \
255 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \
256 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \
257 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \
258 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \
259 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO)
260
261 #define IXL_CSUM_TCP \
262 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP)
263 #define IXL_CSUM_UDP \
264 (CSUM_IP_UDP|CSUM_IP6_UDP)
265 #define IXL_CSUM_SCTP \
266 (CSUM_IP_SCTP|CSUM_IP6_SCTP)
267 #define IXL_CSUM_IPV4 \
268 (CSUM_IP|CSUM_IP_TSO)
269
270 /* Pre-11 counter(9) compatibility */
271 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count)
272 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count)
273 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count)
274 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count)
275 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */
276 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count)
277 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count)
278 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count)
279 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count)
280 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count)
281 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count)
282 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
283
284 /* For stats sysctl naming */
285 #define IXL_QUEUE_NAME_LEN 32
286
287 #define IXL_PF_MAX_LINK_POLL SBT_1S * 5
288
289 MALLOC_DECLARE(M_IXL);
290
291 #define IXL_DEV_ERR(_dev, _format, ...) \
292 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__)
293
294 /*
295 *****************************************************************************
296 * vendor_info_array
297 *
298 * This array contains the list of Subvendor/Subdevice IDs on which the driver
299 * should load.
300 *
301 *****************************************************************************
302 */
303 typedef struct _ixl_vendor_info_t {
304 unsigned int vendor_id;
305 unsigned int device_id;
306 unsigned int subvendor_id;
307 unsigned int subdevice_id;
308 unsigned int index;
309 } ixl_vendor_info_t;
310
311 /*
312 ** This struct has multiple uses, multicast
313 ** addresses, vlans, and mac filters all use it.
314 */
315 struct ixl_mac_filter {
316 LIST_ENTRY(ixl_mac_filter) ftle;
317 u8 macaddr[ETHER_ADDR_LEN];
318 s16 vlan;
319 u16 flags;
320 };
321
322 /*
323 * The Transmit ring control struct
324 */
325 struct tx_ring {
326 struct ixl_tx_queue *que;
327 u32 tail;
328 struct i40e_tx_desc *tx_base;
329 u64 tx_paddr;
330 u32 latency;
331 u32 packets;
332 u32 me;
333 /*
334 * For reporting completed packet status
335 * in descriptor writeback mode
336 */
337 qidx_t *tx_rsq;
338 qidx_t tx_rs_cidx;
339 qidx_t tx_rs_pidx;
340 qidx_t tx_cidx_processed;
341
342 /* Used for Dynamic ITR calculation */
343 u32 itr;
344 u32 bytes;
345
346 /* Soft Stats */
347 u64 tx_bytes;
348 u64 tx_packets;
349 u64 mss_too_small;
350 };
351
352
353 /*
354 * The Receive ring control struct
355 */
356 struct rx_ring {
357 struct ixl_rx_queue *que;
358 union i40e_rx_desc *rx_base;
359 uint64_t rx_paddr;
360 bool discard;
361 u32 itr;
362 u32 latency;
363 u32 mbuf_sz;
364 u32 tail;
365 u32 me;
366
367 /* Used for Dynamic ITR calculation */
368 u32 packets;
369 u32 bytes;
370
371 /* Soft stats */
372 u64 rx_packets;
373 u64 rx_bytes;
374 u64 desc_errs;
375 u64 csum_errs;
376 };
377
378 /*
379 ** Driver queue structs
380 */
381 struct ixl_tx_queue {
382 struct ixl_vsi *vsi;
383 struct tx_ring txr;
384 struct if_irq que_irq;
385 u32 msix;
386 /* Stats */
387 u64 irqs;
388 u64 tso;
389 };
390
391 struct ixl_rx_queue {
392 struct ixl_vsi *vsi;
393 struct rx_ring rxr;
394 struct if_irq que_irq;
395 u32 msix; /* This queue's MSIX vector */
396 /* Stats */
397 u64 irqs;
398 };
399
400 /*
401 ** Virtual Station Interface
402 */
403 LIST_HEAD(ixl_ftl_head, ixl_mac_filter);
404 struct ixl_vsi {
405 if_ctx_t ctx;
406 if_softc_ctx_t shared;
407 if_t ifp;
408 device_t dev;
409 struct i40e_hw *hw;
410 struct ifmedia *media;
411
412 int num_rx_queues;
413 int num_tx_queues;
414
415 void *back;
416 enum i40e_vsi_type type;
417 int id;
418 u32 rx_itr_setting;
419 u32 tx_itr_setting;
420 bool enable_head_writeback;
421
422 u16 vsi_num;
423 bool link_active;
424 u16 seid;
425 u16 uplink_seid;
426 u16 downlink_seid;
427
428 struct ixl_tx_queue *tx_queues; /* TX queue array */
429 struct ixl_rx_queue *rx_queues; /* RX queue array */
430 struct if_irq irq;
431 u32 link_speed;
432
433 /* MAC/VLAN Filter list */
434 struct ixl_ftl_head ftl;
435 u16 num_macs;
436 u64 num_hw_filters;
437
438 /* Contains readylist & stat counter id */
439 struct i40e_aqc_vsi_properties_data info;
440
441 #define IXL_VLANS_MAP_LEN EVL_VLID_MASK + 1
442 bitstr_t bit_decl(vlans_map, IXL_VLANS_MAP_LEN);
443 u16 num_vlans;
444
445 /* Per-VSI stats from hardware */
446 struct i40e_eth_stats eth_stats;
447 struct i40e_eth_stats eth_stats_offsets;
448 bool stat_offsets_loaded;
449 /* VSI stat counters */
450 u64 ipackets;
451 u64 ierrors;
452 u64 opackets;
453 u64 oerrors;
454 u64 ibytes;
455 u64 obytes;
456 u64 imcasts;
457 u64 omcasts;
458 u64 iqdrops;
459 u64 oqdrops;
460 u64 noproto;
461
462 /* Misc. */
463 u64 flags;
464 /* Stats sysctls for this VSI */
465 struct sysctl_oid *vsi_node;
466 struct sysctl_ctx_list sysctl_ctx;
467 };
468
469 struct ixl_add_maddr_arg {
470 struct ixl_ftl_head to_add;
471 struct ixl_vsi *vsi;
472 };
473
474 /*
475 ** Compare two ethernet addresses
476 */
477 static inline bool
ixl_ether_is_equal(const u8 * ea1,const u8 * ea2)478 ixl_ether_is_equal(const u8 *ea1, const u8 *ea2)
479 {
480 return (bcmp(ea1, ea2, ETHER_ADDR_LEN) == 0);
481 }
482
483 /*
484 * Return next largest power of 2, unsigned
485 *
486 * Public domain, from Bit Twiddling Hacks
487 */
488 static inline u32
next_power_of_two(u32 n)489 next_power_of_two(u32 n)
490 {
491 n--;
492 n |= n >> 1;
493 n |= n >> 2;
494 n |= n >> 4;
495 n |= n >> 8;
496 n |= n >> 16;
497 n++;
498
499 /* Next power of two > 0 is 1 */
500 n += (n == 0);
501
502 return (n);
503 }
504
505 /*
506 * Info for stats sysctls
507 */
508 struct ixl_sysctl_info {
509 u64 *stat;
510 char *name;
511 char *description;
512 };
513
514 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN];
515
516 /* Common function prototypes between PF/VF driver */
517 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...);
518 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que);
519 void ixl_get_default_rss_key(u32 *);
520 const char * i40e_vc_stat_str(struct i40e_hw *hw,
521 enum virtchnl_status_code stat_err);
522 void ixl_init_tx_rsqs(struct ixl_vsi *vsi);
523 void ixl_init_tx_cidx(struct ixl_vsi *vsi);
524 u64 ixl_max_vc_speed_to_value(u8 link_speeds);
525 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi,
526 struct sysctl_ctx_list *ctx, const char *sysctl_name);
527 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
528 struct sysctl_oid_list *child,
529 struct i40e_eth_stats *eth_stats);
530 void ixl_vsi_add_queues_stats(struct ixl_vsi *vsi,
531 struct sysctl_ctx_list *ctx);
532 #endif /* _IXL_H_ */
533