1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved.
4 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 */
6
7 #ifndef _ENIC_H_
8 #define _ENIC_H_
9
10 #include "vnic_enet.h"
11 #include "vnic_dev.h"
12 #include "vnic_wq.h"
13 #include "vnic_rq.h"
14 #include "vnic_cq.h"
15 #include "vnic_intr.h"
16 #include "vnic_stats.h"
17 #include "vnic_nic.h"
18 #include "vnic_rss.h"
19 #include <linux/irq.h>
20 #include <net/page_pool/helpers.h>
21
22 #define DRV_NAME "enic"
23 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Driver"
24
25 #define PCI_SUBDEV_ID_CISCO_VIC_1225 0x085
26 #define PCI_SUBDEV_ID_CISCO_VIC_1225T 0x0CE
27 #define PCI_SUBDEV_ID_CISCO_VIC_1227 0x12E
28 #define PCI_SUBDEV_ID_CISCO_VIC_1227T 0x139
29 #define PCI_SUBDEV_ID_CISCO_VIC_1240 0x084
30 #define PCI_SUBDEV_ID_CISCO_VIC_1280 0x04F
31 #define PCI_SUBDEV_ID_CISCO_VIC_1285 0x0CD
32
33 #define PCI_SUBDEV_ID_CISCO_VIC_1340 0x12C
34 #define PCI_SUBDEV_ID_CISCO_VIC_1380 0x137
35 #define PCI_SUBDEV_ID_CISCO_VIC_1385 0x14D
36 #define PCI_SUBDEV_ID_CISCO_VIC_1387 0x15D
37
38 #define PCI_SUBDEV_ID_CISCO_VIC_1440 0x0215
39 #define PCI_SUBDEV_ID_CISCO_VIC_1455 0x0217
40 #define PCI_SUBDEV_ID_CISCO_VIC_1457 0x0218
41 #define PCI_SUBDEV_ID_CISCO_VIC_1467 0x02AF
42 #define PCI_SUBDEV_ID_CISCO_VIC_1477 0x2B0
43 #define PCI_SUBDEV_ID_CISCO_VIC_1480 0x0216
44 #define PCI_SUBDEV_ID_CISCO_VIC_1485 0x0219
45 #define PCI_SUBDEV_ID_CISCO_VIC_1487 0x021A
46 #define PCI_SUBDEV_ID_CISCO_VIC_1495 0x024A
47 #define PCI_SUBDEV_ID_CISCO_VIC_1497 0x024B
48 #define PCI_SUBDEV_ID_CISCO_VIC_14425 0x02CF
49 #define PCI_SUBDEV_ID_CISCO_VIC_14825 0x02D0
50
51 #define PCI_SUBDEV_ID_CISCO_VIC_15230 0x02DF
52 #define PCI_SUBDEV_ID_CISCO_VIC_15231 0x02DB
53 #define PCI_SUBDEV_ID_CISCO_VIC_15235 0x02E4
54 #define PCI_SUBDEV_ID_CISCO_VIC_15237 0x02F3
55 #define PCI_SUBDEV_ID_CISCO_VIC_15238 0x02E8
56 #define PCI_SUBDEV_ID_CISCO_VIC_15411 0x02DC
57 #define PCI_SUBDEV_ID_CISCO_VIC_15412 0x02E2
58 #define PCI_SUBDEV_ID_CISCO_VIC_15420 0x02DE
59 #define PCI_SUBDEV_ID_CISCO_VIC_15422 0x02E1
60 #define PCI_SUBDEV_ID_CISCO_VIC_15425 0x02F2
61 #define PCI_SUBDEV_ID_CISCO_VIC_15427 0x02E0
62 #define PCI_SUBDEV_ID_CISCO_VIC_15428 0x02DD
63
64 #define ENIC_BARS_MAX 6
65
66 #define ENIC_WQ_MAX 256
67 #define ENIC_RQ_MAX 256
68 #define ENIC_RQ_MIN_DEFAULT 8
69
70 #define ENIC_WQ_NAPI_BUDGET 256
71
72 #define ENIC_AIC_LARGE_PKT_DIFF 3
73
74 enum ext_cq {
75 ENIC_RQ_CQ_ENTRY_SIZE_16,
76 ENIC_RQ_CQ_ENTRY_SIZE_32,
77 ENIC_RQ_CQ_ENTRY_SIZE_64,
78 ENIC_RQ_CQ_ENTRY_SIZE_MAX,
79 };
80
81 struct enic_msix_entry {
82 int requested;
83 char devname[IFNAMSIZ + 8];
84 irqreturn_t (*isr)(int, void *);
85 void *devid;
86 cpumask_var_t affinity_mask;
87 };
88
89 /* Store only the lower range. Higher range is given by fw. */
90 struct enic_intr_mod_range {
91 u32 small_pkt_range_start;
92 u32 large_pkt_range_start;
93 };
94
95 struct enic_intr_mod_table {
96 u32 rx_rate;
97 u32 range_percent;
98 };
99
100 #define ENIC_MAX_LINK_SPEEDS 3
101 #define ENIC_LINK_SPEED_10G 10000
102 #define ENIC_LINK_SPEED_4G 4000
103 #define ENIC_LINK_40G_INDEX 2
104 #define ENIC_LINK_10G_INDEX 1
105 #define ENIC_LINK_4G_INDEX 0
106 #define ENIC_RX_COALESCE_RANGE_END 125
107 #define ENIC_AIC_TS_BREAK 100
108
109 struct enic_rx_coal {
110 u32 small_pkt_range_start;
111 u32 large_pkt_range_start;
112 u32 range_end;
113 u32 use_adaptive_rx_coalesce;
114 };
115
116 /* priv_flags */
117 #define ENIC_SRIOV_ENABLED (1 << 0)
118
119 /* enic port profile set flags */
120 #define ENIC_PORT_REQUEST_APPLIED (1 << 0)
121 #define ENIC_SET_REQUEST (1 << 1)
122 #define ENIC_SET_NAME (1 << 2)
123 #define ENIC_SET_INSTANCE (1 << 3)
124 #define ENIC_SET_HOST (1 << 4)
125
126 #define MAX_TSO BIT(16)
127 #define WQ_ENET_MAX_DESC_LEN BIT(WQ_ENET_LEN_BITS)
128 #define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1)
129
130 struct enic_port_profile {
131 u32 set;
132 u8 request;
133 char name[PORT_PROFILE_MAX];
134 u8 instance_uuid[PORT_UUID_MAX];
135 u8 host_uuid[PORT_UUID_MAX];
136 u8 vf_mac[ETH_ALEN];
137 u8 mac_addr[ETH_ALEN];
138 };
139
140 enum enic_vf_link_state {
141 ENIC_VF_LINK_STATE_UNKNOWN,
142 ENIC_VF_LINK_STATE_DOWN,
143 ENIC_VF_LINK_STATE_UP,
144 };
145
146 /* enic_rfs_fltr_node - rfs filter node in hash table
147 * @@keys: IPv4 5 tuple
148 * @flow_id: flow_id of clsf filter provided by kernel
149 * @fltr_id: filter id of clsf filter returned by adaptor
150 * @rq_id: desired rq index
151 * @node: hlist_node
152 */
153 struct enic_rfs_fltr_node {
154 struct flow_keys keys;
155 u32 flow_id;
156 u16 fltr_id;
157 u16 rq_id;
158 struct hlist_node node;
159 };
160
161 /* enic_rfs_flw_tbl - rfs flow table
162 * @max: Maximum number of filters vNIC supports
163 * @free: Number of free filters available
164 * @toclean: hash table index to clean next
165 * @ht_head: hash table list head
166 * @lock: spin lock
167 * @rfs_may_expire: timer function for enic_rps_may_expire_flow
168 */
169 struct enic_rfs_flw_tbl {
170 u16 max;
171 int free;
172
173 #define ENIC_RFS_FLW_BITSHIFT (10)
174 #define ENIC_RFS_FLW_MASK ((1 << ENIC_RFS_FLW_BITSHIFT) - 1)
175 u16 toclean:ENIC_RFS_FLW_BITSHIFT;
176 struct hlist_head ht_head[1 << ENIC_RFS_FLW_BITSHIFT];
177 spinlock_t lock;
178 struct timer_list rfs_may_expire;
179 };
180
181 struct vxlan_offload {
182 u16 vxlan_udp_port_number;
183 u8 patch_level;
184 u8 flags;
185 };
186
187 struct enic_wq_stats {
188 u64 packets; /* pkts queued for Tx */
189 u64 stopped; /* Tx ring almost full, queue stopped */
190 u64 wake; /* Tx ring no longer full, queue woken up*/
191 u64 tso; /* non-encap tso pkt */
192 u64 encap_tso; /* encap tso pkt */
193 u64 encap_csum; /* encap HW csum */
194 u64 csum_partial; /* skb->ip_summed = CHECKSUM_PARTIAL */
195 u64 csum_none; /* HW csum not required */
196 u64 bytes; /* bytes queued for Tx */
197 u64 add_vlan; /* HW adds vlan tag */
198 u64 cq_work; /* Tx completions processed */
199 u64 cq_bytes; /* Tx bytes processed */
200 u64 null_pkt; /* skb length <= 0 */
201 u64 skb_linear_fail; /* linearize failures */
202 u64 desc_full_awake; /* TX ring full while queue awake */
203 };
204
205 struct enic_rq_stats {
206 u64 packets; /* pkts received */
207 u64 bytes; /* bytes received */
208 u64 l4_rss_hash; /* hashed on l4 */
209 u64 l3_rss_hash; /* hashed on l3 */
210 u64 csum_unnecessary; /* HW verified csum */
211 u64 csum_unnecessary_encap; /* HW verified csum on encap packet */
212 u64 vlan_stripped; /* HW stripped vlan */
213 u64 napi_complete; /* napi complete intr reenabled */
214 u64 napi_repoll; /* napi poll again */
215 u64 bad_fcs; /* bad pkts */
216 u64 pkt_truncated; /* truncated pkts */
217 u64 no_skb; /* out of skbs */
218 u64 desc_skip; /* Rx pkt went into later buffer */
219 u64 pp_alloc_fail; /* page pool alloc failure */
220 };
221
222 struct enic_wq {
223 spinlock_t lock; /* spinlock for wq */
224 struct vnic_wq vwq;
225 struct enic_wq_stats stats;
226 } ____cacheline_aligned;
227
228 struct enic_rq {
229 struct vnic_rq vrq;
230 struct enic_rq_stats stats;
231 struct page_pool *pool;
232 } ____cacheline_aligned;
233
234 enum enic_vf_type {
235 ENIC_VF_TYPE_NONE,
236 ENIC_VF_TYPE_V1,
237 ENIC_VF_TYPE_USNIC,
238 ENIC_VF_TYPE_V2,
239 };
240
241 /* Per-instance private data structure */
242 struct enic {
243 struct net_device *netdev;
244 struct pci_dev *pdev;
245 struct vnic_enet_config config;
246 struct vnic_dev_bar bar[ENIC_BARS_MAX];
247 struct vnic_dev *vdev;
248 struct timer_list notify_timer;
249 struct work_struct reset;
250 struct work_struct tx_hang_reset;
251 struct work_struct change_mtu_work;
252 struct msix_entry *msix_entry;
253 struct enic_msix_entry *msix;
254 u32 msg_enable;
255 spinlock_t devcmd_lock;
256 u8 mac_addr[ETH_ALEN];
257 unsigned int flags;
258 unsigned int priv_flags;
259 unsigned int mc_count;
260 unsigned int uc_count;
261 u32 port_mtu;
262 struct enic_rx_coal rx_coalesce_setting;
263 u32 rx_coalesce_usecs;
264 u32 tx_coalesce_usecs;
265 u16 num_vfs;
266 enum enic_vf_type vf_type;
267 bool vf_registered;
268 u32 pf_cap_version;
269 unsigned int enable_count;
270 spinlock_t enic_api_lock;
271 bool enic_api_busy;
272 struct enic_port_profile *pp;
273
274 struct enic_wq *wq;
275 unsigned int wq_avail;
276 unsigned int wq_count;
277 u16 loop_enable;
278 u16 loop_tag;
279
280 struct enic_rq *rq;
281 unsigned int rq_avail;
282 unsigned int rq_count;
283 struct vxlan_offload vxlan;
284 struct napi_struct *napi;
285
286 struct vnic_intr *intr;
287 unsigned int intr_avail;
288 unsigned int intr_count;
289 u32 __iomem *legacy_pba; /* memory-mapped */
290
291 struct vnic_cq *cq;
292 unsigned int cq_avail;
293 unsigned int cq_count;
294 struct enic_rfs_flw_tbl rfs_h;
295 u8 rss_key[ENIC_RSS_LEN];
296 struct vnic_gen_stats gen_stats;
297 enum ext_cq ext_cq;
298
299 /* Admin channel resources for SR-IOV MBOX */
300 bool has_admin_channel;
301 /* true only while the admin WQ/RQ/CQ are allocated and enabled; gates
302 * enic_admin_channel_close() so it is a no-op after a failed (re)open
303 * left the resources freed.
304 */
305 bool admin_chan_up;
306 /* set on send timeout; cleared on channel re-open */
307 bool mbox_send_disabled;
308 struct vnic_wq admin_wq;
309 struct vnic_rq admin_rq;
310 struct vnic_cq admin_cq[2];
311 struct vnic_intr admin_intr;
312 struct delayed_work admin_poll_work;
313 unsigned int admin_intr_index;
314 struct work_struct link_notify_work;
315 struct work_struct admin_msg_work;
316 spinlock_t admin_msg_lock; /* protects admin_msg_list */
317 struct list_head admin_msg_list;
318 unsigned int admin_msg_count; /* current depth of admin_msg_list */
319 void (*admin_rq_handler)(struct enic *enic, void *buf,
320 unsigned int len);
321 /* The PF is authoritative for a V2 VF's carrier. Keep the last
322 * notification across an ordinary netdev close/open and serialize it
323 * against the open/stop carrier transition.
324 */
325 spinlock_t vf_link_state_lock;
326 enum enic_vf_link_state vf_link_state;
327 bool vf_link_running;
328
329 /* MBOX protocol state — mbox_lock serializes admin WQ sends */
330 struct mutex mbox_lock;
331 u64 mbox_msg_num;
332 /* MBOX request-reply state. Existing request callers allow only one
333 * request in flight. The state lock arbitrates reply acceptance against
334 * timeout invalidation, while mbox_comp publishes the accepted result to
335 * the requester.
336 */
337 struct completion mbox_comp;
338 spinlock_t mbox_state_lock; /* protects expected reply state */
339 u64 mbox_expected_msg_num;
340 u8 mbox_expected_reply;
341 bool mbox_initialized;
342
343 /* PF: per-VF MBOX state, allocated when SRIOV V2 is enabled */
344 struct enic_vf_state {
345 bool registered;
346 } *vf_state;
347 };
348
vnic_get_netdev(struct vnic_dev * vdev)349 static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev)
350 {
351 struct enic *enic = vdev->priv;
352
353 return enic->netdev;
354 }
355
356 /* wrappers function for kernel log
357 */
358 #define vdev_err(vdev, fmt, ...) \
359 dev_err(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
360 #define vdev_warn(vdev, fmt, ...) \
361 dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
362 #define vdev_info(vdev, fmt, ...) \
363 dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
364 #define vdev_dbg(vdev, fmt, ...) \
365 dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
366
367 #define vdev_neterr(vdev, fmt, ...) \
368 netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
369 #define vdev_netwarn(vdev, fmt, ...) \
370 netdev_warn(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
371 #define vdev_netinfo(vdev, fmt, ...) \
372 netdev_info(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
373
enic_get_dev(struct enic * enic)374 static inline struct device *enic_get_dev(struct enic *enic)
375 {
376 return &(enic->pdev->dev);
377 }
378
enic_cq_rq(struct enic * enic,unsigned int rq)379 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq)
380 {
381 return rq;
382 }
383
enic_cq_wq(struct enic * enic,unsigned int wq)384 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq)
385 {
386 return enic->rq_count + wq;
387 }
388
enic_msix_rq_intr(struct enic * enic,unsigned int rq)389 static inline unsigned int enic_msix_rq_intr(struct enic *enic,
390 unsigned int rq)
391 {
392 return enic->cq[enic_cq_rq(enic, rq)].interrupt_offset;
393 }
394
enic_msix_wq_intr(struct enic * enic,unsigned int wq)395 static inline unsigned int enic_msix_wq_intr(struct enic *enic,
396 unsigned int wq)
397 {
398 return enic->cq[enic_cq_wq(enic, wq)].interrupt_offset;
399 }
400
401 /* MSIX interrupts are organized as the error interrupt, then the notify
402 * interrupt followed by all the I/O interrupts. The error interrupt needs
403 * to fit in 7 bits due to hardware constraints
404 */
405 #define ENIC_MSIX_RESERVED_INTR 2
406 #define ENIC_MSIX_ERR_INTR 0
407 #define ENIC_MSIX_NOTIFY_INTR 1
408 #define ENIC_MSIX_IO_INTR_BASE ENIC_MSIX_RESERVED_INTR
409 #define ENIC_MSIX_MIN_INTR (ENIC_MSIX_RESERVED_INTR + 2)
410
411 #define ENIC_LEGACY_IO_INTR 0
412 #define ENIC_LEGACY_ERR_INTR 1
413 #define ENIC_LEGACY_NOTIFY_INTR 2
414
enic_msix_err_intr(struct enic * enic)415 static inline unsigned int enic_msix_err_intr(struct enic *enic)
416 {
417 return ENIC_MSIX_ERR_INTR;
418 }
419
enic_msix_notify_intr(struct enic * enic)420 static inline unsigned int enic_msix_notify_intr(struct enic *enic)
421 {
422 return ENIC_MSIX_NOTIFY_INTR;
423 }
424
enic_is_err_intr(struct enic * enic,int intr)425 static inline bool enic_is_err_intr(struct enic *enic, int intr)
426 {
427 switch (vnic_dev_get_intr_mode(enic->vdev)) {
428 case VNIC_DEV_INTR_MODE_INTX:
429 return intr == ENIC_LEGACY_ERR_INTR;
430 case VNIC_DEV_INTR_MODE_MSIX:
431 return intr == enic_msix_err_intr(enic);
432 case VNIC_DEV_INTR_MODE_MSI:
433 default:
434 return false;
435 }
436 }
437
enic_is_notify_intr(struct enic * enic,int intr)438 static inline bool enic_is_notify_intr(struct enic *enic, int intr)
439 {
440 switch (vnic_dev_get_intr_mode(enic->vdev)) {
441 case VNIC_DEV_INTR_MODE_INTX:
442 return intr == ENIC_LEGACY_NOTIFY_INTR;
443 case VNIC_DEV_INTR_MODE_MSIX:
444 return intr == enic_msix_notify_intr(enic);
445 case VNIC_DEV_INTR_MODE_MSI:
446 default:
447 return false;
448 }
449 }
450
enic_dma_map_check(struct enic * enic,dma_addr_t dma_addr)451 static inline int enic_dma_map_check(struct enic *enic, dma_addr_t dma_addr)
452 {
453 if (unlikely(dma_mapping_error(&enic->pdev->dev, dma_addr))) {
454 net_warn_ratelimited("%s: PCI dma mapping failed!\n",
455 enic->netdev->name);
456 enic->gen_stats.dma_map_error++;
457
458 return -ENOMEM;
459 }
460
461 return 0;
462 }
463
464 void enic_reset_addr_lists(struct enic *enic);
465 int enic_sriov_enabled(struct enic *enic);
466 int enic_is_valid_vf(struct enic *enic, int vf);
467 int enic_is_dynamic(struct enic *enic);
468 int enic_is_sriov_vf_v2(struct enic *enic);
469 void enic_set_ethtool_ops(struct net_device *netdev);
470 int __enic_set_rsskey(struct enic *enic);
471 void enic_ext_cq(struct enic *enic);
472
473 #endif /* _ENIC_H_ */
474