1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
4 */
5
6 #ifndef _ENIC_H
7 #define _ENIC_H
8
9 #include <sys/param.h>
10 #include <sys/socket.h>
11 #include <sys/sysctl.h>
12 #include <sys/taskqueue.h>
13
14 #include <machine/bus.h>
15
16 #include <net/ethernet.h>
17 #include <net/if.h>
18 #include <net/if_var.h>
19 #include <net/iflib.h>
20
21 #define u8 uint8_t
22 #define u16 uint16_t
23 #define u32 uint32_t
24 #define u64 uint64_t
25
26 struct enic_bar_info {
27 struct resource *res;
28 bus_space_tag_t tag;
29 bus_space_handle_t handle;
30 bus_size_t size;
31 int rid;
32 int offset;
33 };
34
35 #define ENIC_BUS_WRITE_8(res, index, value) \
36 bus_space_write_8(res->bar.tag, res->bar.handle, \
37 res->bar.offset + (index), value)
38 #define ENIC_BUS_WRITE_4(res, index, value) \
39 bus_space_write_4(res->bar.tag, res->bar.handle, \
40 res->bar.offset + (index), value)
41 #define ENIC_BUS_WRITE_REGION_4(res, index, values, count) \
42 bus_space_write_region_4(res->bar.tag, res->bar.handle, \
43 res->bar.offset + (index), values, count);
44
45 #define ENIC_BUS_READ_8(res, index) \
46 bus_space_read_8(res->bar.tag, res->bar.handle, \
47 res->bar.offset + (index))
48 #define ENIC_BUS_READ_4(res, index) \
49 bus_space_read_4(res->bar.tag, res->bar.handle, \
50 res->bar.offset + (index))
51 #define ENIC_BUS_READ_REGION_4(res, type, index, values, count) \
52 bus_space_read_region_4(res->type.tag, res->type.handle, \
53 res->type.offset + (index), values, count);
54
55 struct vnic_res {
56 unsigned int count;
57 struct enic_bar_info bar;
58 };
59
60 #include "vnic_enet.h"
61 #include "vnic_dev.h"
62 #include "vnic_wq.h"
63 #include "vnic_rq.h"
64 #include "vnic_cq.h"
65 #include "vnic_intr.h"
66 #include "vnic_stats.h"
67 #include "vnic_nic.h"
68 #include "vnic_rss.h"
69 #include "enic_res.h"
70 #include "cq_enet_desc.h"
71
72 #define ENIC_LOCK(_softc) mtx_lock(&(_softc)->enic_lock)
73 #define ENIC_UNLOCK(_softc) mtx_unlock(&(_softc)->enic_lock)
74
75 #define DRV_NAME "enic"
76 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC"
77 #define DRV_COPYRIGHT "Copyright 2008-2015 Cisco Systems, Inc"
78
79 #define ENIC_MAX_MAC_ADDR 64
80
81 #define VLAN_ETH_HLEN 18
82
83 #define ENICPMD_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
84
85 #define ENICPMD_BDF_LENGTH 13 /* 0000:00:00.0'\0' */
86 #define ENIC_CALC_IP_CKSUM 1
87 #define ENIC_CALC_TCP_UDP_CKSUM 2
88 #define ENIC_MAX_MTU 9000
89 #define ENIC_PAGE_SIZE 4096
90 #define PAGE_ROUND_UP(x) \
91 ((((unsigned long)(x)) + ENIC_PAGE_SIZE-1) & (~(ENIC_PAGE_SIZE-1)))
92
93 /* must be >= VNIC_COUNTER_DMA_MIN_PERIOD */
94 #define VNIC_FLOW_COUNTER_UPDATE_MSECS 500
95
96 /* PCI IDs */
97 #define CISCO_VENDOR_ID 0x1137
98
99 #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */
100 #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */
101
102 /* Special Filter id for non-specific packet flagging. Don't change value */
103 #define ENIC_MAGIC_FILTER_ID 0xffff
104
105 #define ENICPMD_FDIR_MAX 64
106
107 /* HW default VXLAN port */
108 #define ENIC_DEFAULT_VXLAN_PORT 4789
109
110 /*
111 * Interrupt 0: LSC and errors
112 * Interrupt 1: rx queue 0
113 * Interrupt 2: rx queue 1
114 * ...
115 */
116 #define ENICPMD_LSC_INTR_OFFSET 0
117 #define ENICPMD_RXQ_INTR_OFFSET 1
118
119 #include "vnic_devcmd.h"
120
121 enum vnic_proxy_type {
122 PROXY_NONE,
123 PROXY_BY_BDF,
124 PROXY_BY_INDEX,
125 };
126
127 struct vnic_intr_coal_timer_info {
128 u32 mul;
129 u32 div;
130 u32 max_usec;
131 };
132
133 struct enic_softc;
134 struct vnic_dev {
135 void *priv;
136 struct rte_pci_device *pdev;
137 struct vnic_res res[RES_TYPE_MAX];
138 enum vnic_dev_intr_mode intr_mode;
139 struct vnic_res __iomem *devcmd;
140 struct vnic_devcmd_notify *notify;
141 struct vnic_devcmd_notify notify_copy;
142 bus_addr_t notify_pa;
143 struct iflib_dma_info notify_res;
144 u32 notify_sz;
145 struct iflib_dma_info linkstatus_res;
146 struct vnic_stats *stats;
147 struct iflib_dma_info stats_res;
148 struct vnic_devcmd_fw_info *fw_info;
149 struct iflib_dma_info fw_info_res;
150 enum vnic_proxy_type proxy;
151 u32 proxy_index;
152 u64 args[VNIC_DEVCMD_NARGS];
153 int in_reset;
154 struct vnic_intr_coal_timer_info intr_coal_timer_info;
155 void *(*alloc_consistent)(void *priv, size_t size,
156 bus_addr_t *dma_handle, struct iflib_dma_info *res, u8 *name);
157 void (*free_consistent)(void *priv, size_t size, void *vaddr,
158 bus_addr_t dma_handle, struct iflib_dma_info *res);
159 struct vnic_counter_counts *flow_counters;
160 struct iflib_dma_info flow_counters_res;
161 u8 flow_counters_dma_active;
162 struct enic_softc *softc;
163 };
164
165 struct enic_soft_stats {
166 uint64_t rx_nombuf;
167 uint64_t rx_packet_errors;
168 uint64_t tx_oversized;
169 };
170
171 struct intr_queue {
172 struct if_irq intr_irq;
173 struct resource *res;
174 int rid;
175 struct enic_softc *softc;
176 };
177
178 struct enic {
179 struct enic *next;
180 struct rte_pci_device *pdev;
181 struct vnic_enet_config config;
182 struct vnic_dev_bar bar0;
183 struct vnic_dev *vdev;
184
185 /*
186 * mbuf_initializer contains 64 bits of mbuf rearm_data, used by
187 * the avx2 handler at this time.
188 */
189 uint64_t mbuf_initializer;
190 unsigned int port_id;
191 bool overlay_offload;
192 char bdf_name[ENICPMD_BDF_LENGTH];
193 int dev_fd;
194 int iommu_group_fd;
195 int iommu_groupid;
196 int eventfd;
197 uint8_t mac_addr[ETH_ALEN];
198 pthread_t err_intr_thread;
199 u8 ig_vlan_strip_en;
200 int link_status;
201 u8 hw_ip_checksum;
202 u16 max_mtu;
203 u8 adv_filters;
204 u32 flow_filter_mode;
205 u8 filter_actions; /* HW supported actions */
206 bool vxlan;
207 bool disable_overlay; /* devargs disable_overlay=1 */
208 uint8_t enable_avx2_rx; /* devargs enable-avx2-rx=1 */
209 bool nic_cfg_chk; /* NIC_CFG_CHK available */
210 bool udp_rss_weak; /* Bodega style UDP RSS */
211 uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */
212 uint16_t vxlan_port; /* current vxlan port pushed to NIC */
213
214 unsigned int flags;
215 unsigned int priv_flags;
216
217 /* work queue (len = conf_wq_count) */
218 struct vnic_wq *wq;
219 unsigned int wq_count; /* equals eth_dev nb_tx_queues */
220
221 /* receive queue (len = conf_rq_count) */
222 struct vnic_rq *rq;
223 unsigned int rq_count; /* equals eth_dev nb_rx_queues */
224
225 /* completion queue (len = conf_cq_count) */
226 struct vnic_cq *cq;
227 unsigned int cq_count; /* equals rq_count + wq_count */
228
229 /* interrupt vectors (len = conf_intr_count) */
230 struct vnic_intr *intr;
231 struct intr_queue *intr_queues;
232 unsigned int intr_count; /* equals enabled interrupts (lsc + rxqs) */
233
234
235 /* software counters */
236 struct enic_soft_stats soft_stats;
237
238 /* configured resources on vic */
239 unsigned int conf_rq_count;
240 unsigned int conf_wq_count;
241 unsigned int conf_cq_count;
242 unsigned int conf_intr_count;
243
244 /* linked list storing memory allocations */
245 LIST_HEAD(enic_memzone_list, enic_memzone_entry) memzone_list;
246
247 LIST_HEAD(enic_flows, rte_flow) flows;
248 int max_flow_counter;
249
250 /* RSS */
251 uint16_t reta_size;
252 uint8_t hash_key_size;
253 uint64_t flow_type_rss_offloads; /* 0 indicates RSS not supported */
254 /*
255 * Keep a copy of current RSS config for queries, as we cannot retrieve
256 * it from the NIC.
257 */
258 uint8_t rss_hash_type; /* NIC_CFG_RSS_HASH_TYPE flags */
259 uint8_t rss_enable;
260 uint64_t rss_hf; /* ETH_RSS flags */
261 union vnic_rss_key rss_key;
262 union vnic_rss_cpu rss_cpu;
263
264 uint64_t rx_offload_capa; /* DEV_RX_OFFLOAD flags */
265 uint64_t tx_offload_capa; /* DEV_TX_OFFLOAD flags */
266 uint64_t tx_queue_offload_capa; /* DEV_TX_OFFLOAD flags */
267 uint64_t tx_offload_mask; /* PKT_TX flags accepted */
268 struct enic_softc *softc;
269 int port_mtu;
270 };
271
272 struct enic_softc {
273 device_t dev;
274 if_ctx_t ctx;
275 if_softc_ctx_t scctx;
276 if_shared_ctx_t sctx;
277 struct ifmedia *media;
278 if_t ifp;
279
280 struct mtx enic_lock;
281
282 struct enic_bar_info mem;
283 struct enic_bar_info io;
284
285 struct vnic_dev vdev;
286 struct enic enic;
287
288 int ntxqsets;
289 int nrxqsets;
290
291 struct if_irq enic_event_intr_irq;
292 struct if_irq enic_err_intr_irq;
293 uint8_t lladdr[ETHER_ADDR_LEN];
294 int link_active;
295 int stopped;
296 uint8_t mac_addr[ETHER_ADDR_LEN];
297
298 int directed;
299 int multicast;
300 int broadcast;
301 int promisc;
302 int allmulti;
303
304 u_int mc_count;
305 uint8_t *mta;
306 };
307
308 /* Per-instance private data structure */
309
enic_vnic_rq_count(struct enic * enic)310 static inline unsigned int enic_vnic_rq_count(struct enic *enic)
311 {
312 return enic->rq_count;
313 }
314
enic_cq_rq(struct enic * enic,unsigned int rq)315 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq)
316 {
317 return rq;
318 }
319
enic_cq_wq(struct enic * enic,unsigned int wq)320 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq)
321 {
322 return enic->rq_count + wq;
323 }
324
325 static inline uint32_t
enic_ring_add(uint32_t n_descriptors,uint32_t i0,uint32_t i1)326 enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
327 {
328 uint32_t d = i0 + i1;
329 d -= (d >= n_descriptors) ? n_descriptors : 0;
330 return d;
331 }
332
333 static inline uint32_t
enic_ring_sub(uint32_t n_descriptors,uint32_t i0,uint32_t i1)334 enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
335 {
336 int32_t d = i1 - i0;
337 return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d);
338 }
339
340 static inline uint32_t
enic_ring_incr(uint32_t n_descriptors,uint32_t idx)341 enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
342 {
343 idx++;
344 if (unlikely(idx == n_descriptors))
345 idx = 0;
346 return idx;
347 }
348
349 void enic_free_wq(void *txq);
350 int enic_alloc_intr_resources(struct enic *enic);
351 int enic_setup_finish(struct enic *enic);
352 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
353 unsigned int socket_id, uint16_t nb_desc);
354 void enic_start_wq(struct enic *enic, uint16_t queue_idx);
355 int enic_stop_wq(struct enic *enic, uint16_t queue_idx);
356 void enic_start_rq(struct enic *enic, uint16_t queue_idx);
357 void enic_free_rq(void *rxq);
358 int enic_set_vnic_res(struct enic *enic);
359 int enic_init_rss_nic_cfg(struct enic *enic);
360 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu);
361 int enic_set_vlan_strip(struct enic *enic);
362 int enic_enable(struct enic *enic);
363 int enic_disable(struct enic *enic);
364 void enic_remove(struct enic *enic);
365 int enic_get_link_status(struct enic *enic);
366 void enic_dev_stats_clear(struct enic *enic);
367 void enic_add_packet_filter(struct enic *enic);
368 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr);
369 int enic_del_mac_address(struct enic *enic, int mac_index);
370 unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq);
371
372 void enic_post_wq_index(struct vnic_wq *wq);
373 int enic_probe(struct enic *enic);
374 int enic_clsf_init(struct enic *enic);
375 void enic_clsf_destroy(struct enic *enic);
376 int enic_set_mtu(struct enic *enic, uint16_t new_mtu);
377 int enic_link_update(struct enic *enic);
378 bool enic_use_vector_rx_handler(struct enic *enic);
379 void enic_fdir_info(struct enic *enic);
380 void enic_prep_wq_for_simple_tx(struct enic *, uint16_t);
381
382 struct enic_ring {
383 uint64_t paddr;
384 caddr_t vaddr;
385 struct enic_softc *softc;
386 uint32_t ring_size; /* Must be a power of two */
387 uint16_t id; /* Logical ID */
388 uint16_t phys_id;
389 };
390
391 struct enic_cp_ring {
392 struct enic_ring ring;
393 struct if_irq irq;
394 uint32_t cons;
395 bool v_bit; /* Value of valid bit */
396 struct ctx_hw_stats *stats;
397 uint32_t stats_ctx_id;
398 uint32_t last_idx; /* Used by RX rings only
399 * set to the last read pidx
400 */
401 };
402
403 #endif /* _ENIC_H_ */
404