1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved.
4 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 */
6
7 #ifndef _ENIC_H_
8 #define _ENIC_H_
9
10 #include "vnic_enet.h"
11 #include "vnic_dev.h"
12 #include "vnic_wq.h"
13 #include "vnic_rq.h"
14 #include "vnic_cq.h"
15 #include "vnic_intr.h"
16 #include "vnic_stats.h"
17 #include "vnic_nic.h"
18 #include "vnic_rss.h"
19 #include <linux/irq.h>
20 #include <net/page_pool/helpers.h>
21
22 #define DRV_NAME "enic"
23 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Driver"
24
25 #define PCI_SUBDEV_ID_CISCO_VIC_1225 0x085
26 #define PCI_SUBDEV_ID_CISCO_VIC_1225T 0x0CE
27 #define PCI_SUBDEV_ID_CISCO_VIC_1227 0x12E
28 #define PCI_SUBDEV_ID_CISCO_VIC_1227T 0x139
29 #define PCI_SUBDEV_ID_CISCO_VIC_1240 0x084
30 #define PCI_SUBDEV_ID_CISCO_VIC_1280 0x04F
31 #define PCI_SUBDEV_ID_CISCO_VIC_1285 0x0CD
32
33 #define PCI_SUBDEV_ID_CISCO_VIC_1340 0x12C
34 #define PCI_SUBDEV_ID_CISCO_VIC_1380 0x137
35 #define PCI_SUBDEV_ID_CISCO_VIC_1385 0x14D
36 #define PCI_SUBDEV_ID_CISCO_VIC_1387 0x15D
37
38 #define PCI_SUBDEV_ID_CISCO_VIC_1440 0x0215
39 #define PCI_SUBDEV_ID_CISCO_VIC_1455 0x0217
40 #define PCI_SUBDEV_ID_CISCO_VIC_1457 0x0218
41 #define PCI_SUBDEV_ID_CISCO_VIC_1467 0x02AF
42 #define PCI_SUBDEV_ID_CISCO_VIC_1477 0x2B0
43 #define PCI_SUBDEV_ID_CISCO_VIC_1480 0x0216
44 #define PCI_SUBDEV_ID_CISCO_VIC_1485 0x0219
45 #define PCI_SUBDEV_ID_CISCO_VIC_1487 0x021A
46 #define PCI_SUBDEV_ID_CISCO_VIC_1495 0x024A
47 #define PCI_SUBDEV_ID_CISCO_VIC_1497 0x024B
48 #define PCI_SUBDEV_ID_CISCO_VIC_14425 0x02CF
49 #define PCI_SUBDEV_ID_CISCO_VIC_14825 0x02D0
50
51 #define PCI_SUBDEV_ID_CISCO_VIC_15230 0x02DF
52 #define PCI_SUBDEV_ID_CISCO_VIC_15231 0x02DB
53 #define PCI_SUBDEV_ID_CISCO_VIC_15235 0x02E4
54 #define PCI_SUBDEV_ID_CISCO_VIC_15237 0x02F3
55 #define PCI_SUBDEV_ID_CISCO_VIC_15238 0x02E8
56 #define PCI_SUBDEV_ID_CISCO_VIC_15411 0x02DC
57 #define PCI_SUBDEV_ID_CISCO_VIC_15412 0x02E2
58 #define PCI_SUBDEV_ID_CISCO_VIC_15420 0x02DE
59 #define PCI_SUBDEV_ID_CISCO_VIC_15422 0x02E1
60 #define PCI_SUBDEV_ID_CISCO_VIC_15425 0x02F2
61 #define PCI_SUBDEV_ID_CISCO_VIC_15427 0x02E0
62 #define PCI_SUBDEV_ID_CISCO_VIC_15428 0x02DD
63
64 #define ENIC_BARS_MAX 6
65
66 #define ENIC_WQ_MAX 256
67 #define ENIC_RQ_MAX 256
68 #define ENIC_RQ_MIN_DEFAULT 8
69
70 #define ENIC_WQ_NAPI_BUDGET 256
71
72 #define ENIC_AIC_LARGE_PKT_DIFF 3
73
74 enum ext_cq {
75 ENIC_RQ_CQ_ENTRY_SIZE_16,
76 ENIC_RQ_CQ_ENTRY_SIZE_32,
77 ENIC_RQ_CQ_ENTRY_SIZE_64,
78 ENIC_RQ_CQ_ENTRY_SIZE_MAX,
79 };
80
81 struct enic_msix_entry {
82 int requested;
83 char devname[IFNAMSIZ + 8];
84 irqreturn_t (*isr)(int, void *);
85 void *devid;
86 cpumask_var_t affinity_mask;
87 };
88
89 /* Store only the lower range. Higher range is given by fw. */
90 struct enic_intr_mod_range {
91 u32 small_pkt_range_start;
92 u32 large_pkt_range_start;
93 };
94
95 struct enic_intr_mod_table {
96 u32 rx_rate;
97 u32 range_percent;
98 };
99
100 #define ENIC_MAX_LINK_SPEEDS 3
101 #define ENIC_LINK_SPEED_10G 10000
102 #define ENIC_LINK_SPEED_4G 4000
103 #define ENIC_LINK_40G_INDEX 2
104 #define ENIC_LINK_10G_INDEX 1
105 #define ENIC_LINK_4G_INDEX 0
106 #define ENIC_RX_COALESCE_RANGE_END 125
107 #define ENIC_AIC_TS_BREAK 100
108
109 struct enic_rx_coal {
110 u32 small_pkt_range_start;
111 u32 large_pkt_range_start;
112 u32 range_end;
113 u32 use_adaptive_rx_coalesce;
114 };
115
116 /* priv_flags */
117 #define ENIC_SRIOV_ENABLED (1 << 0)
118
119 /* enic port profile set flags */
120 #define ENIC_PORT_REQUEST_APPLIED (1 << 0)
121 #define ENIC_SET_REQUEST (1 << 1)
122 #define ENIC_SET_NAME (1 << 2)
123 #define ENIC_SET_INSTANCE (1 << 3)
124 #define ENIC_SET_HOST (1 << 4)
125
126 #define MAX_TSO BIT(16)
127 #define WQ_ENET_MAX_DESC_LEN BIT(WQ_ENET_LEN_BITS)
128 #define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1)
129
130 struct enic_port_profile {
131 u32 set;
132 u8 request;
133 char name[PORT_PROFILE_MAX];
134 u8 instance_uuid[PORT_UUID_MAX];
135 u8 host_uuid[PORT_UUID_MAX];
136 u8 vf_mac[ETH_ALEN];
137 u8 mac_addr[ETH_ALEN];
138 };
139
140 /* enic_rfs_fltr_node - rfs filter node in hash table
141 * @@keys: IPv4 5 tuple
142 * @flow_id: flow_id of clsf filter provided by kernel
143 * @fltr_id: filter id of clsf filter returned by adaptor
144 * @rq_id: desired rq index
145 * @node: hlist_node
146 */
147 struct enic_rfs_fltr_node {
148 struct flow_keys keys;
149 u32 flow_id;
150 u16 fltr_id;
151 u16 rq_id;
152 struct hlist_node node;
153 };
154
155 /* enic_rfs_flw_tbl - rfs flow table
156 * @max: Maximum number of filters vNIC supports
157 * @free: Number of free filters available
158 * @toclean: hash table index to clean next
159 * @ht_head: hash table list head
160 * @lock: spin lock
161 * @rfs_may_expire: timer function for enic_rps_may_expire_flow
162 */
163 struct enic_rfs_flw_tbl {
164 u16 max;
165 int free;
166
167 #define ENIC_RFS_FLW_BITSHIFT (10)
168 #define ENIC_RFS_FLW_MASK ((1 << ENIC_RFS_FLW_BITSHIFT) - 1)
169 u16 toclean:ENIC_RFS_FLW_BITSHIFT;
170 struct hlist_head ht_head[1 << ENIC_RFS_FLW_BITSHIFT];
171 spinlock_t lock;
172 struct timer_list rfs_may_expire;
173 };
174
175 struct vxlan_offload {
176 u16 vxlan_udp_port_number;
177 u8 patch_level;
178 u8 flags;
179 };
180
181 struct enic_wq_stats {
182 u64 packets; /* pkts queued for Tx */
183 u64 stopped; /* Tx ring almost full, queue stopped */
184 u64 wake; /* Tx ring no longer full, queue woken up*/
185 u64 tso; /* non-encap tso pkt */
186 u64 encap_tso; /* encap tso pkt */
187 u64 encap_csum; /* encap HW csum */
188 u64 csum_partial; /* skb->ip_summed = CHECKSUM_PARTIAL */
189 u64 csum_none; /* HW csum not required */
190 u64 bytes; /* bytes queued for Tx */
191 u64 add_vlan; /* HW adds vlan tag */
192 u64 cq_work; /* Tx completions processed */
193 u64 cq_bytes; /* Tx bytes processed */
194 u64 null_pkt; /* skb length <= 0 */
195 u64 skb_linear_fail; /* linearize failures */
196 u64 desc_full_awake; /* TX ring full while queue awake */
197 };
198
199 struct enic_rq_stats {
200 u64 packets; /* pkts received */
201 u64 bytes; /* bytes received */
202 u64 l4_rss_hash; /* hashed on l4 */
203 u64 l3_rss_hash; /* hashed on l3 */
204 u64 csum_unnecessary; /* HW verified csum */
205 u64 csum_unnecessary_encap; /* HW verified csum on encap packet */
206 u64 vlan_stripped; /* HW stripped vlan */
207 u64 napi_complete; /* napi complete intr reenabled */
208 u64 napi_repoll; /* napi poll again */
209 u64 bad_fcs; /* bad pkts */
210 u64 pkt_truncated; /* truncated pkts */
211 u64 no_skb; /* out of skbs */
212 u64 desc_skip; /* Rx pkt went into later buffer */
213 u64 pp_alloc_fail; /* page pool alloc failure */
214 };
215
216 struct enic_wq {
217 spinlock_t lock; /* spinlock for wq */
218 struct vnic_wq vwq;
219 struct enic_wq_stats stats;
220 } ____cacheline_aligned;
221
222 struct enic_rq {
223 struct vnic_rq vrq;
224 struct enic_rq_stats stats;
225 struct page_pool *pool;
226 } ____cacheline_aligned;
227
228 enum enic_vf_type {
229 ENIC_VF_TYPE_NONE,
230 ENIC_VF_TYPE_V1,
231 ENIC_VF_TYPE_USNIC,
232 ENIC_VF_TYPE_V2,
233 };
234
235 /* Per-instance private data structure */
236 struct enic {
237 struct net_device *netdev;
238 struct pci_dev *pdev;
239 struct vnic_enet_config config;
240 struct vnic_dev_bar bar[ENIC_BARS_MAX];
241 struct vnic_dev *vdev;
242 struct timer_list notify_timer;
243 struct work_struct reset;
244 struct work_struct tx_hang_reset;
245 struct work_struct change_mtu_work;
246 struct msix_entry *msix_entry;
247 struct enic_msix_entry *msix;
248 u32 msg_enable;
249 spinlock_t devcmd_lock;
250 u8 mac_addr[ETH_ALEN];
251 unsigned int flags;
252 unsigned int priv_flags;
253 unsigned int mc_count;
254 unsigned int uc_count;
255 u32 port_mtu;
256 struct enic_rx_coal rx_coalesce_setting;
257 u32 rx_coalesce_usecs;
258 u32 tx_coalesce_usecs;
259 u16 num_vfs;
260 enum enic_vf_type vf_type;
261 bool vf_registered;
262 u32 pf_cap_version;
263 unsigned int enable_count;
264 spinlock_t enic_api_lock;
265 bool enic_api_busy;
266 struct enic_port_profile *pp;
267
268 struct enic_wq *wq;
269 unsigned int wq_avail;
270 unsigned int wq_count;
271 u16 loop_enable;
272 u16 loop_tag;
273
274 struct enic_rq *rq;
275 unsigned int rq_avail;
276 unsigned int rq_count;
277 struct vxlan_offload vxlan;
278 struct napi_struct *napi;
279
280 struct vnic_intr *intr;
281 unsigned int intr_avail;
282 unsigned int intr_count;
283 u32 __iomem *legacy_pba; /* memory-mapped */
284
285 struct vnic_cq *cq;
286 unsigned int cq_avail;
287 unsigned int cq_count;
288 struct enic_rfs_flw_tbl rfs_h;
289 u8 rss_key[ENIC_RSS_LEN];
290 struct vnic_gen_stats gen_stats;
291 enum ext_cq ext_cq;
292
293 /* Admin channel resources for SR-IOV MBOX */
294 bool has_admin_channel;
295 /* true only while the admin WQ/RQ/CQ are allocated and enabled; gates
296 * enic_admin_channel_close() so it is a no-op after a failed (re)open
297 * left the resources freed.
298 */
299 bool admin_chan_up;
300 /* set on send timeout; cleared on channel re-open */
301 bool mbox_send_disabled;
302 struct vnic_wq admin_wq;
303 struct vnic_rq admin_rq;
304 struct vnic_cq admin_cq[2];
305 struct vnic_intr admin_intr;
306 struct delayed_work admin_poll_work;
307 unsigned int admin_intr_index;
308 struct work_struct link_notify_work;
309 struct work_struct admin_msg_work;
310 spinlock_t admin_msg_lock; /* protects admin_msg_list */
311 struct list_head admin_msg_list;
312 unsigned int admin_msg_count; /* current depth of admin_msg_list */
313 void (*admin_rq_handler)(struct enic *enic, void *buf,
314 unsigned int len);
315
316 /* MBOX protocol state — mbox_lock serializes admin WQ sends */
317 struct mutex mbox_lock;
318 u64 mbox_msg_num;
319 /* MBOX request-reply state. mbox_expected_reply is written and
320 * cleared by the process-context request helpers (capability/register/
321 * unregister) and only read by the admin_msg_work receive handlers, so
322 * it is annotated with READ_ONCE()/WRITE_ONCE() rather than locked:
323 * only one request is in flight at a time (requesters run under RTNL or
324 * single-threaded probe/remove), so each request is serialized and its
325 * reply completes mbox_comp before the next request is issued.
326 */
327 struct completion mbox_comp;
328 u8 mbox_expected_reply;
329 bool mbox_initialized;
330
331 /* PF: per-VF MBOX state, allocated when SRIOV V2 is enabled */
332 struct enic_vf_state {
333 bool registered;
334 } *vf_state;
335 };
336
vnic_get_netdev(struct vnic_dev * vdev)337 static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev)
338 {
339 struct enic *enic = vdev->priv;
340
341 return enic->netdev;
342 }
343
344 /* wrappers function for kernel log
345 */
346 #define vdev_err(vdev, fmt, ...) \
347 dev_err(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
348 #define vdev_warn(vdev, fmt, ...) \
349 dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
350 #define vdev_info(vdev, fmt, ...) \
351 dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
352 #define vdev_dbg(vdev, fmt, ...) \
353 dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
354
355 #define vdev_neterr(vdev, fmt, ...) \
356 netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
357 #define vdev_netwarn(vdev, fmt, ...) \
358 netdev_warn(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
359 #define vdev_netinfo(vdev, fmt, ...) \
360 netdev_info(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
361
enic_get_dev(struct enic * enic)362 static inline struct device *enic_get_dev(struct enic *enic)
363 {
364 return &(enic->pdev->dev);
365 }
366
enic_cq_rq(struct enic * enic,unsigned int rq)367 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq)
368 {
369 return rq;
370 }
371
enic_cq_wq(struct enic * enic,unsigned int wq)372 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq)
373 {
374 return enic->rq_count + wq;
375 }
376
enic_msix_rq_intr(struct enic * enic,unsigned int rq)377 static inline unsigned int enic_msix_rq_intr(struct enic *enic,
378 unsigned int rq)
379 {
380 return enic->cq[enic_cq_rq(enic, rq)].interrupt_offset;
381 }
382
enic_msix_wq_intr(struct enic * enic,unsigned int wq)383 static inline unsigned int enic_msix_wq_intr(struct enic *enic,
384 unsigned int wq)
385 {
386 return enic->cq[enic_cq_wq(enic, wq)].interrupt_offset;
387 }
388
389 /* MSIX interrupts are organized as the error interrupt, then the notify
390 * interrupt followed by all the I/O interrupts. The error interrupt needs
391 * to fit in 7 bits due to hardware constraints
392 */
393 #define ENIC_MSIX_RESERVED_INTR 2
394 #define ENIC_MSIX_ERR_INTR 0
395 #define ENIC_MSIX_NOTIFY_INTR 1
396 #define ENIC_MSIX_IO_INTR_BASE ENIC_MSIX_RESERVED_INTR
397 #define ENIC_MSIX_MIN_INTR (ENIC_MSIX_RESERVED_INTR + 2)
398
399 #define ENIC_LEGACY_IO_INTR 0
400 #define ENIC_LEGACY_ERR_INTR 1
401 #define ENIC_LEGACY_NOTIFY_INTR 2
402
enic_msix_err_intr(struct enic * enic)403 static inline unsigned int enic_msix_err_intr(struct enic *enic)
404 {
405 return ENIC_MSIX_ERR_INTR;
406 }
407
enic_msix_notify_intr(struct enic * enic)408 static inline unsigned int enic_msix_notify_intr(struct enic *enic)
409 {
410 return ENIC_MSIX_NOTIFY_INTR;
411 }
412
enic_is_err_intr(struct enic * enic,int intr)413 static inline bool enic_is_err_intr(struct enic *enic, int intr)
414 {
415 switch (vnic_dev_get_intr_mode(enic->vdev)) {
416 case VNIC_DEV_INTR_MODE_INTX:
417 return intr == ENIC_LEGACY_ERR_INTR;
418 case VNIC_DEV_INTR_MODE_MSIX:
419 return intr == enic_msix_err_intr(enic);
420 case VNIC_DEV_INTR_MODE_MSI:
421 default:
422 return false;
423 }
424 }
425
enic_is_notify_intr(struct enic * enic,int intr)426 static inline bool enic_is_notify_intr(struct enic *enic, int intr)
427 {
428 switch (vnic_dev_get_intr_mode(enic->vdev)) {
429 case VNIC_DEV_INTR_MODE_INTX:
430 return intr == ENIC_LEGACY_NOTIFY_INTR;
431 case VNIC_DEV_INTR_MODE_MSIX:
432 return intr == enic_msix_notify_intr(enic);
433 case VNIC_DEV_INTR_MODE_MSI:
434 default:
435 return false;
436 }
437 }
438
enic_dma_map_check(struct enic * enic,dma_addr_t dma_addr)439 static inline int enic_dma_map_check(struct enic *enic, dma_addr_t dma_addr)
440 {
441 if (unlikely(dma_mapping_error(&enic->pdev->dev, dma_addr))) {
442 net_warn_ratelimited("%s: PCI dma mapping failed!\n",
443 enic->netdev->name);
444 enic->gen_stats.dma_map_error++;
445
446 return -ENOMEM;
447 }
448
449 return 0;
450 }
451
452 void enic_reset_addr_lists(struct enic *enic);
453 int enic_sriov_enabled(struct enic *enic);
454 int enic_is_valid_vf(struct enic *enic, int vf);
455 int enic_is_dynamic(struct enic *enic);
456 int enic_is_sriov_vf_v2(struct enic *enic);
457 void enic_set_ethtool_ops(struct net_device *netdev);
458 int __enic_set_rsskey(struct enic *enic);
459 void enic_ext_cq(struct enic *enic);
460
461 #endif /* _ENIC_H_ */
462