1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
4 */
5
6 #ifndef _ENIC_H
7 #define _ENIC_H
8
9 #include <sys/param.h>
10 #include <sys/socket.h>
11 #include <sys/sysctl.h>
12 #include <sys/taskqueue.h>
13
14 #include <machine/bus.h>
15
16 #include <net/ethernet.h>
17 #include <net/if.h>
18 #include <net/if_var.h>
19 #include <net/iflib.h>
20
21 #define u8 uint8_t
22 #define u16 uint16_t
23 #define u32 uint32_t
24 #define u64 uint64_t
25
26 struct enic_bar_info {
27 struct resource *res;
28 bus_space_tag_t tag;
29 bus_space_handle_t handle;
30 bus_size_t size;
31 int rid;
32 int offset;
33 };
34
35 #define ENIC_BUS_WRITE_8(res, index, value) \
36 bus_space_write_8(res->bar.tag, res->bar.handle, \
37 res->bar.offset + (index), value)
38 #define ENIC_BUS_WRITE_4(res, index, value) \
39 bus_space_write_4(res->bar.tag, res->bar.handle, \
40 res->bar.offset + (index), value)
41 #define ENIC_BUS_WRITE_REGION_4(res, index, values, count) \
42 bus_space_write_region_4(res->bar.tag, res->bar.handle, \
43 res->bar.offset + (index), values, count);
44
45 #define ENIC_BUS_READ_8(res, index) \
46 bus_space_read_8(res->bar.tag, res->bar.handle, \
47 res->bar.offset + (index))
48 #define ENIC_BUS_READ_4(res, index) \
49 bus_space_read_4(res->bar.tag, res->bar.handle, \
50 res->bar.offset + (index))
51 #define ENIC_BUS_READ_REGION_4(res, type, index, values, count) \
52 bus_space_read_region_4(res->type.tag, res->type.handle, \
53 res->type.offset + (index), values, count);
54
55 struct vnic_res {
56 unsigned int count;
57 struct enic_bar_info bar;
58 };
59
60 #include "vnic_enet.h"
61 #include "vnic_dev.h"
62 #include "vnic_wq.h"
63 #include "vnic_rq.h"
64 #include "vnic_cq.h"
65 #include "vnic_intr.h"
66 #include "vnic_stats.h"
67 #include "vnic_nic.h"
68 #include "vnic_rss.h"
69 #include "enic_res.h"
70 #include "cq_enet_desc.h"
71
72 #define ENIC_LOCK(_softc) mtx_lock(&(_softc)->enic_lock)
73 #define ENIC_UNLOCK(_softc) mtx_unlock(&(_softc)->enic_lock)
74
75 #define DRV_NAME "enic"
76 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC"
77 #define DRV_COPYRIGHT "Copyright 2008-2015 Cisco Systems, Inc"
78
79 #define ENIC_MAX_MAC_ADDR 64
80
81 #define VLAN_ETH_HLEN 18
82
83 #define ENICPMD_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
84
85 #define ENICPMD_BDF_LENGTH 13 /* 0000:00:00.0'\0' */
86 #define ENIC_CALC_IP_CKSUM 1
87 #define ENIC_CALC_TCP_UDP_CKSUM 2
88 #define ENIC_MAX_MTU 9000
89 #define ENIC_PAGE_SIZE 4096
90 #define PAGE_ROUND_UP(x) \
91 ((((unsigned long)(x)) + ENIC_PAGE_SIZE-1) & (~(ENIC_PAGE_SIZE-1)))
92
93 /* must be >= VNIC_COUNTER_DMA_MIN_PERIOD */
94 #define VNIC_FLOW_COUNTER_UPDATE_MSECS 500
95
96 /* PCI IDs */
97 #define CISCO_VENDOR_ID 0x1137
98
99 #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */
100 #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */
101
102 /* Special Filter id for non-specific packet flagging. Don't change value */
103 #define ENIC_MAGIC_FILTER_ID 0xffff
104
105 #define ENICPMD_FDIR_MAX 64
106
107 /* HW default VXLAN port */
108 #define ENIC_DEFAULT_VXLAN_PORT 4789
109
110 /*
111 * Interrupt 1: rx queue 0
112 * Interrupt 2: rx queue 1
113 * ...
114 * Interrupt x: LSC and errors
115 */
116 #define ENICPMD_LSC_INTR_OFFSET 0
117 #define ENICPMD_RXQ_INTR_OFFSET 0
118
119 #include "vnic_devcmd.h"
120
121 enum vnic_proxy_type {
122 PROXY_NONE,
123 PROXY_BY_BDF,
124 PROXY_BY_INDEX,
125 };
126
127 struct vnic_intr_coal_timer_info {
128 u32 mul;
129 u32 div;
130 u32 max_usec;
131 };
132
133 struct enic_softc;
134 struct vnic_dev {
135 void *priv;
136 struct rte_pci_device *pdev;
137 struct vnic_res res[RES_TYPE_MAX];
138 enum vnic_dev_intr_mode intr_mode;
139 struct vnic_res __iomem *devcmd;
140 struct vnic_devcmd_notify *notify;
141 struct vnic_devcmd_notify notify_copy;
142 bus_addr_t notify_pa;
143 struct iflib_dma_info notify_res;
144 u32 notify_sz;
145 struct iflib_dma_info linkstatus_res;
146 struct vnic_stats *stats;
147 struct iflib_dma_info stats_res;
148 struct vnic_devcmd_fw_info *fw_info;
149 struct iflib_dma_info fw_info_res;
150 enum vnic_proxy_type proxy;
151 u32 proxy_index;
152 u64 args[VNIC_DEVCMD_NARGS];
153 int in_reset;
154 struct vnic_intr_coal_timer_info intr_coal_timer_info;
155 struct devcmd2_controller *devcmd2;
156 int (*devcmd_rtn)(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
157 int wait);
158 void *(*alloc_consistent)(void *priv, size_t size,
159 bus_addr_t *dma_handle, struct iflib_dma_info *res, u8 *name);
160 void (*free_consistent)(void *priv, size_t size, void *vaddr,
161 bus_addr_t dma_handle, struct iflib_dma_info *res);
162 struct vnic_counter_counts *flow_counters;
163 struct iflib_dma_info flow_counters_res;
164 u8 flow_counters_dma_active;
165 struct enic_softc *softc;
166 };
167
168 struct enic_soft_stats {
169 uint64_t rx_nombuf;
170 uint64_t rx_packet_errors;
171 uint64_t tx_oversized;
172 };
173
174 struct intr_queue {
175 struct if_irq intr_irq;
176 struct resource *res;
177 int rid;
178 struct enic_softc *softc;
179 };
180
181 #define ENIC_MAX_LINK_SPEEDS 3
182 #define ENIC_LINK_SPEED_10G 10000
183 #define ENIC_LINK_SPEED_4G 4000
184 #define ENIC_LINK_40G_INDEX 2
185 #define ENIC_LINK_10G_INDEX 1
186 #define ENIC_LINK_4G_INDEX 0
187 #define ENIC_RX_COALESCE_RANGE_END 125
188 #define ENIC_AIC_TS_BREAK 100
189
190 struct enic_rx_coal {
191 u32 small_pkt_range_start;
192 u32 large_pkt_range_start;
193 u32 range_end;
194 u32 use_adaptive_rx_coalesce;
195 };
196
197 /* Store only the lower range. Higher range is given by fw. */
198 struct enic_intr_mod_range {
199 u32 small_pkt_range_start;
200 u32 large_pkt_range_start;
201 };
202
203 struct enic {
204 struct enic *next;
205 struct rte_pci_device *pdev;
206 struct vnic_enet_config config;
207 struct vnic_dev_bar bar0;
208 struct vnic_dev *vdev;
209
210 /*
211 * mbuf_initializer contains 64 bits of mbuf rearm_data, used by
212 * the avx2 handler at this time.
213 */
214 uint64_t mbuf_initializer;
215 unsigned int port_id;
216 bool overlay_offload;
217 char bdf_name[ENICPMD_BDF_LENGTH];
218 int dev_fd;
219 int iommu_group_fd;
220 int iommu_groupid;
221 int eventfd;
222 uint8_t mac_addr[ETH_ALEN];
223 pthread_t err_intr_thread;
224 u8 ig_vlan_strip_en;
225 int link_status;
226 u8 hw_ip_checksum;
227 u16 max_mtu;
228 u8 adv_filters;
229 u32 flow_filter_mode;
230 u8 filter_actions; /* HW supported actions */
231 bool vxlan;
232 bool disable_overlay; /* devargs disable_overlay=1 */
233 uint8_t enable_avx2_rx; /* devargs enable-avx2-rx=1 */
234 bool nic_cfg_chk; /* NIC_CFG_CHK available */
235 bool udp_rss_weak; /* Bodega style UDP RSS */
236 uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */
237 uint16_t vxlan_port; /* current vxlan port pushed to NIC */
238
239 unsigned int flags;
240 unsigned int priv_flags;
241
242 /* work queue (len = conf_wq_count) */
243 struct vnic_wq *wq;
244 unsigned int wq_count; /* equals eth_dev nb_tx_queues */
245
246 /* receive queue (len = conf_rq_count) */
247 struct vnic_rq *rq;
248 unsigned int rq_count; /* equals eth_dev nb_rx_queues */
249
250 /* completion queue (len = conf_cq_count) */
251 struct vnic_cq *cq;
252 unsigned int cq_count; /* equals rq_count + wq_count */
253
254 /* interrupt vectors (len = conf_intr_count) */
255 struct vnic_intr *intr;
256 struct intr_queue *intr_queues;
257 unsigned int intr_count; /* equals enabled interrupts (lsc + rxqs) */
258
259
260 /* software counters */
261 struct enic_soft_stats soft_stats;
262
263 /* configured resources on vic */
264 unsigned int conf_rq_count;
265 unsigned int conf_wq_count;
266 unsigned int conf_cq_count;
267 unsigned int conf_intr_count;
268
269 /* linked list storing memory allocations */
270 LIST_HEAD(enic_memzone_list, enic_memzone_entry) memzone_list;
271
272 LIST_HEAD(enic_flows, rte_flow) flows;
273 int max_flow_counter;
274
275 /* RSS */
276 uint16_t reta_size;
277 uint8_t hash_key_size;
278 uint64_t flow_type_rss_offloads; /* 0 indicates RSS not supported */
279 /*
280 * Keep a copy of current RSS config for queries, as we cannot retrieve
281 * it from the NIC.
282 */
283 uint8_t rss_hash_type; /* NIC_CFG_RSS_HASH_TYPE flags */
284 uint8_t rss_enable;
285 uint64_t rss_hf; /* ETH_RSS flags */
286 union vnic_rss_key rss_key;
287 union vnic_rss_cpu rss_cpu;
288
289 uint64_t rx_offload_capa; /* DEV_RX_OFFLOAD flags */
290 uint64_t tx_offload_capa; /* DEV_TX_OFFLOAD flags */
291 uint64_t tx_queue_offload_capa; /* DEV_TX_OFFLOAD flags */
292 uint64_t tx_offload_mask; /* PKT_TX flags accepted */
293 struct enic_softc *softc;
294 int port_mtu;
295 struct enic_rx_coal rx_coalesce_setting;
296 u32 rx_coalesce_usecs;
297 u32 tx_coalesce_usecs;
298 };
299
300 struct enic_softc {
301 device_t dev;
302 if_ctx_t ctx;
303 if_softc_ctx_t scctx;
304 if_shared_ctx_t sctx;
305 struct ifmedia *media;
306 if_t ifp;
307
308 struct mtx enic_lock;
309
310 struct enic_bar_info mem;
311 struct enic_bar_info io;
312
313 struct vnic_dev vdev;
314 struct enic enic;
315
316 int ntxqsets;
317 int nrxqsets;
318
319 struct if_irq enic_event_intr_irq;
320 struct if_irq enic_err_intr_irq;
321 uint8_t lladdr[ETHER_ADDR_LEN];
322 int link_active;
323 int stopped;
324 uint8_t mac_addr[ETHER_ADDR_LEN];
325
326 int directed;
327 int multicast;
328 int broadcast;
329 int promisc;
330 int allmulti;
331
332 u_int mc_count;
333 uint8_t *mta;
334 };
335
336 /* Per-instance private data structure */
337
enic_cq_rq(struct enic * enic,unsigned int rq)338 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq)
339 {
340 return rq;
341 }
342
enic_cq_wq(struct enic * enic,unsigned int wq)343 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq)
344 {
345 return enic->rq_count + wq;
346 }
347
348 static inline uint32_t
enic_ring_incr(uint32_t n_descriptors,uint32_t idx)349 enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
350 {
351 idx++;
352 if (unlikely(idx == n_descriptors))
353 idx = 0;
354 return idx;
355 }
356
357 int enic_setup_finish(struct enic *enic);
358 void enic_start_wq(struct enic *enic, uint16_t queue_idx);
359 int enic_stop_wq(struct enic *enic, uint16_t queue_idx);
360 void enic_start_rq(struct enic *enic, uint16_t queue_idx);
361 int enic_stop_rq(struct enic *enic, uint16_t queue_idx);
362 void enic_dev_disable(struct enic *enic);
363 int enic_enable(struct enic *enic);
364 int enic_disable(struct enic *enic);
365 int enic_link_update(struct enic *enic);
366 bool enic_use_vector_rx_handler(struct enic *enic);
367 void enic_fdir_info(struct enic *enic);
368 void enic_prep_wq_for_simple_tx(struct enic *, uint16_t);
369
370 struct enic_ring {
371 uint64_t paddr;
372 caddr_t vaddr;
373 struct enic_softc *softc;
374 uint32_t ring_size; /* Must be a power of two */
375 uint16_t id; /* Logical ID */
376 uint16_t phys_id;
377 };
378
379 struct enic_cp_ring {
380 struct enic_ring ring;
381 struct if_irq irq;
382 uint32_t cons;
383 bool v_bit; /* Value of valid bit */
384 struct ctx_hw_stats *stats;
385 uint32_t stats_ctx_id;
386 uint32_t last_idx; /* Used by RX rings only
387 * set to the last read pidx
388 */
389 };
390
391 #endif /* _ENIC_H_ */
392