xref: /linux/drivers/net/ethernet/broadcom/bnxt/bnxt.c (revision ef9226cd56b718c79184a3466d32984a51cb449c)
1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2019 Broadcom Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  */
10 
11 #include <linux/module.h>
12 
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/gro.h>
41 #include <net/ip.h>
42 #include <net/tcp.h>
43 #include <net/udp.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 #include <net/udp_tunnel.h>
47 #include <linux/workqueue.h>
48 #include <linux/prefetch.h>
49 #include <linux/cache.h>
50 #include <linux/log2.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <net/page_pool/helpers.h>
56 #include <linux/align.h>
57 #include <net/netdev_queues.h>
58 
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_hwrm.h"
62 #include "bnxt_ulp.h"
63 #include "bnxt_sriov.h"
64 #include "bnxt_ethtool.h"
65 #include "bnxt_dcb.h"
66 #include "bnxt_xdp.h"
67 #include "bnxt_ptp.h"
68 #include "bnxt_vfr.h"
69 #include "bnxt_tc.h"
70 #include "bnxt_devlink.h"
71 #include "bnxt_debugfs.h"
72 #include "bnxt_hwmon.h"
73 
74 #define BNXT_TX_TIMEOUT		(5 * HZ)
75 #define BNXT_DEF_MSG_ENABLE	(NETIF_MSG_DRV | NETIF_MSG_HW | \
76 				 NETIF_MSG_TX_ERR)
77 
78 MODULE_LICENSE("GPL");
79 MODULE_DESCRIPTION("Broadcom NetXtreme network driver");
80 
81 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
82 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
83 #define BNXT_RX_COPY_THRESH 256
84 
85 #define BNXT_TX_PUSH_THRESH 164
86 
87 /* indexed by enum board_idx */
88 static const struct {
89 	char *name;
90 } board_info[] = {
91 	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
92 	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
93 	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
94 	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
95 	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
96 	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
97 	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
98 	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
99 	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
100 	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
101 	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
102 	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
103 	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
104 	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
105 	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
106 	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
107 	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
108 	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
109 	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
110 	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
111 	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
112 	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
113 	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
114 	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
115 	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
116 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
117 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
118 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
119 	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
120 	[BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
121 	[BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 	[BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
123 	[BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
124 	[BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
125 	[BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" },
126 	[BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
127 	[BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
128 	[BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
129 	[BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
130 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
131 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
132 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
133 	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
134 	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
135 	[NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
136 	[NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
137 	[NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
138 	[NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
139 	[NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
140 };
141 
142 static const struct pci_device_id bnxt_pci_tbl[] = {
143 	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
144 	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
145 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
146 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
147 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
148 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
149 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
150 	{ PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
151 	{ PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
152 	{ PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
153 	{ PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
154 	{ PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
155 	{ PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
156 	{ PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
157 	{ PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
158 	{ PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
159 	{ PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
160 	{ PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
161 	{ PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
162 	{ PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
163 	{ PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
164 	{ PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
165 	{ PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
166 	{ PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
167 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
168 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
169 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
170 	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
171 	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
172 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
173 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
174 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
175 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
176 	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
177 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
178 	{ PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
179 	{ PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
180 	{ PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
181 	{ PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 },
182 	{ PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 },
183 	{ PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 },
184 	{ PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 },
185 	{ PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
186 	{ PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
187 	{ PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
188 	{ PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
189 	{ PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
190 	{ PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
191 	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
192 	{ PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
193 #ifdef CONFIG_BNXT_SRIOV
194 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
195 	{ PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
196 	{ PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
197 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
198 	{ PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
199 	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
200 	{ PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
201 	{ PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
202 	{ PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
203 	{ PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
204 	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
205 	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
206 	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
207 	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
208 	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
209 	{ PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
210 	{ PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
211 	{ PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
212 	{ PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
213 	{ PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
214 	{ PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
215 #endif
216 	{ 0 }
217 };
218 
219 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
220 
221 static const u16 bnxt_vf_req_snif[] = {
222 	HWRM_FUNC_CFG,
223 	HWRM_FUNC_VF_CFG,
224 	HWRM_PORT_PHY_QCFG,
225 	HWRM_CFA_L2_FILTER_ALLOC,
226 };
227 
228 static const u16 bnxt_async_events_arr[] = {
229 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
230 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
231 	ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
232 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
233 	ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
234 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
235 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
236 	ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
237 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
238 	ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
239 	ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
240 	ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
241 	ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
242 	ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
243 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
244 	ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
245 };
246 
247 static struct workqueue_struct *bnxt_pf_wq;
248 
249 #define BNXT_IPV6_MASK_ALL {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, \
250 			       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }}}
251 #define BNXT_IPV6_MASK_NONE {{{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}}
252 
253 const struct bnxt_flow_masks BNXT_FLOW_MASK_NONE = {
254 	.ports = {
255 		.src = 0,
256 		.dst = 0,
257 	},
258 	.addrs = {
259 		.v6addrs = {
260 			.src = BNXT_IPV6_MASK_NONE,
261 			.dst = BNXT_IPV6_MASK_NONE,
262 		},
263 	},
264 };
265 
266 const struct bnxt_flow_masks BNXT_FLOW_IPV6_MASK_ALL = {
267 	.ports = {
268 		.src = cpu_to_be16(0xffff),
269 		.dst = cpu_to_be16(0xffff),
270 	},
271 	.addrs = {
272 		.v6addrs = {
273 			.src = BNXT_IPV6_MASK_ALL,
274 			.dst = BNXT_IPV6_MASK_ALL,
275 		},
276 	},
277 };
278 
279 const struct bnxt_flow_masks BNXT_FLOW_IPV4_MASK_ALL = {
280 	.ports = {
281 		.src = cpu_to_be16(0xffff),
282 		.dst = cpu_to_be16(0xffff),
283 	},
284 	.addrs = {
285 		.v4addrs = {
286 			.src = cpu_to_be32(0xffffffff),
287 			.dst = cpu_to_be32(0xffffffff),
288 		},
289 	},
290 };
291 
292 static bool bnxt_vf_pciid(enum board_idx idx)
293 {
294 	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
295 		idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
296 		idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
297 		idx == NETXTREME_E_P5_VF_HV);
298 }
299 
300 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
301 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
302 #define DB_CP_IRQ_DIS_FLAGS	(DB_KEY_CP | DB_IRQ_DIS)
303 
304 #define BNXT_CP_DB_IRQ_DIS(db)						\
305 		writel(DB_CP_IRQ_DIS_FLAGS, db)
306 
307 #define BNXT_DB_CQ(db, idx)						\
308 	writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
309 
310 #define BNXT_DB_NQ_P5(db, idx)						\
311 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
312 		    (db)->doorbell)
313 
314 #define BNXT_DB_NQ_P7(db, idx)						\
315 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK |		\
316 		    DB_RING_IDX(db, idx), (db)->doorbell)
317 
318 #define BNXT_DB_CQ_ARM(db, idx)						\
319 	writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
320 
321 #define BNXT_DB_NQ_ARM_P5(db, idx)					\
322 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM |		\
323 		    DB_RING_IDX(db, idx), (db)->doorbell)
324 
325 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
326 {
327 	if (bp->flags & BNXT_FLAG_CHIP_P7)
328 		BNXT_DB_NQ_P7(db, idx);
329 	else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
330 		BNXT_DB_NQ_P5(db, idx);
331 	else
332 		BNXT_DB_CQ(db, idx);
333 }
334 
335 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
336 {
337 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
338 		BNXT_DB_NQ_ARM_P5(db, idx);
339 	else
340 		BNXT_DB_CQ_ARM(db, idx);
341 }
342 
343 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
344 {
345 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
346 		bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
347 			    DB_RING_IDX(db, idx), db->doorbell);
348 	else
349 		BNXT_DB_CQ(db, idx);
350 }
351 
352 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
353 {
354 	if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
355 		return;
356 
357 	if (BNXT_PF(bp))
358 		queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
359 	else
360 		schedule_delayed_work(&bp->fw_reset_task, delay);
361 }
362 
363 static void __bnxt_queue_sp_work(struct bnxt *bp)
364 {
365 	if (BNXT_PF(bp))
366 		queue_work(bnxt_pf_wq, &bp->sp_task);
367 	else
368 		schedule_work(&bp->sp_task);
369 }
370 
371 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
372 {
373 	set_bit(event, &bp->sp_event);
374 	__bnxt_queue_sp_work(bp);
375 }
376 
377 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
378 {
379 	if (!rxr->bnapi->in_reset) {
380 		rxr->bnapi->in_reset = true;
381 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
382 			set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
383 		else
384 			set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
385 		__bnxt_queue_sp_work(bp);
386 	}
387 	rxr->rx_next_cons = 0xffff;
388 }
389 
390 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
391 			  u16 curr)
392 {
393 	struct bnxt_napi *bnapi = txr->bnapi;
394 
395 	if (bnapi->tx_fault)
396 		return;
397 
398 	netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
399 		   txr->txq_index, txr->tx_hw_cons,
400 		   txr->tx_cons, txr->tx_prod, curr);
401 	WARN_ON_ONCE(1);
402 	bnapi->tx_fault = 1;
403 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
404 }
405 
406 const u16 bnxt_lhint_arr[] = {
407 	TX_BD_FLAGS_LHINT_512_AND_SMALLER,
408 	TX_BD_FLAGS_LHINT_512_TO_1023,
409 	TX_BD_FLAGS_LHINT_1024_TO_2047,
410 	TX_BD_FLAGS_LHINT_1024_TO_2047,
411 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
412 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
413 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
414 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
415 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
416 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
417 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
418 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
419 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
420 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
421 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
422 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
423 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
424 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
425 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
426 };
427 
428 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
429 {
430 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
431 
432 	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
433 		return 0;
434 
435 	return md_dst->u.port_info.port_id;
436 }
437 
438 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
439 			     u16 prod)
440 {
441 	/* Sync BD data before updating doorbell */
442 	wmb();
443 	bnxt_db_write(bp, &txr->tx_db, prod);
444 	txr->kick_pending = 0;
445 }
446 
447 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
448 {
449 	struct bnxt *bp = netdev_priv(dev);
450 	struct tx_bd *txbd, *txbd0;
451 	struct tx_bd_ext *txbd1;
452 	struct netdev_queue *txq;
453 	int i;
454 	dma_addr_t mapping;
455 	unsigned int length, pad = 0;
456 	u32 len, free_size, vlan_tag_flags, cfa_action, flags;
457 	u16 prod, last_frag;
458 	struct pci_dev *pdev = bp->pdev;
459 	struct bnxt_tx_ring_info *txr;
460 	struct bnxt_sw_tx_bd *tx_buf;
461 	__le32 lflags = 0;
462 
463 	i = skb_get_queue_mapping(skb);
464 	if (unlikely(i >= bp->tx_nr_rings)) {
465 		dev_kfree_skb_any(skb);
466 		dev_core_stats_tx_dropped_inc(dev);
467 		return NETDEV_TX_OK;
468 	}
469 
470 	txq = netdev_get_tx_queue(dev, i);
471 	txr = &bp->tx_ring[bp->tx_ring_map[i]];
472 	prod = txr->tx_prod;
473 
474 	free_size = bnxt_tx_avail(bp, txr);
475 	if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
476 		/* We must have raced with NAPI cleanup */
477 		if (net_ratelimit() && txr->kick_pending)
478 			netif_warn(bp, tx_err, dev,
479 				   "bnxt: ring busy w/ flush pending!\n");
480 		if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
481 					bp->tx_wake_thresh))
482 			return NETDEV_TX_BUSY;
483 	}
484 
485 	if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
486 		goto tx_free;
487 
488 	length = skb->len;
489 	len = skb_headlen(skb);
490 	last_frag = skb_shinfo(skb)->nr_frags;
491 
492 	txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
493 
494 	tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
495 	tx_buf->skb = skb;
496 	tx_buf->nr_frags = last_frag;
497 
498 	vlan_tag_flags = 0;
499 	cfa_action = bnxt_xmit_get_cfa_action(skb);
500 	if (skb_vlan_tag_present(skb)) {
501 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
502 				 skb_vlan_tag_get(skb);
503 		/* Currently supports 8021Q, 8021AD vlan offloads
504 		 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
505 		 */
506 		if (skb->vlan_proto == htons(ETH_P_8021Q))
507 			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
508 	}
509 
510 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
511 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
512 
513 		if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
514 		    atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
515 			if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
516 					    &ptp->tx_hdr_off)) {
517 				if (vlan_tag_flags)
518 					ptp->tx_hdr_off += VLAN_HLEN;
519 				lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
520 				skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
521 			} else {
522 				atomic_inc(&bp->ptp_cfg->tx_avail);
523 			}
524 		}
525 	}
526 
527 	if (unlikely(skb->no_fcs))
528 		lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
529 
530 	if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
531 	    !lflags) {
532 		struct tx_push_buffer *tx_push_buf = txr->tx_push;
533 		struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
534 		struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
535 		void __iomem *db = txr->tx_db.doorbell;
536 		void *pdata = tx_push_buf->data;
537 		u64 *end;
538 		int j, push_len;
539 
540 		/* Set COAL_NOW to be ready quickly for the next push */
541 		tx_push->tx_bd_len_flags_type =
542 			cpu_to_le32((length << TX_BD_LEN_SHIFT) |
543 					TX_BD_TYPE_LONG_TX_BD |
544 					TX_BD_FLAGS_LHINT_512_AND_SMALLER |
545 					TX_BD_FLAGS_COAL_NOW |
546 					TX_BD_FLAGS_PACKET_END |
547 					(2 << TX_BD_FLAGS_BD_CNT_SHIFT));
548 
549 		if (skb->ip_summed == CHECKSUM_PARTIAL)
550 			tx_push1->tx_bd_hsize_lflags =
551 					cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
552 		else
553 			tx_push1->tx_bd_hsize_lflags = 0;
554 
555 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
556 		tx_push1->tx_bd_cfa_action =
557 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
558 
559 		end = pdata + length;
560 		end = PTR_ALIGN(end, 8) - 1;
561 		*end = 0;
562 
563 		skb_copy_from_linear_data(skb, pdata, len);
564 		pdata += len;
565 		for (j = 0; j < last_frag; j++) {
566 			skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
567 			void *fptr;
568 
569 			fptr = skb_frag_address_safe(frag);
570 			if (!fptr)
571 				goto normal_tx;
572 
573 			memcpy(pdata, fptr, skb_frag_size(frag));
574 			pdata += skb_frag_size(frag);
575 		}
576 
577 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
578 		txbd->tx_bd_haddr = txr->data_mapping;
579 		txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
580 		prod = NEXT_TX(prod);
581 		tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
582 		txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
583 		memcpy(txbd, tx_push1, sizeof(*txbd));
584 		prod = NEXT_TX(prod);
585 		tx_push->doorbell =
586 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH |
587 				    DB_RING_IDX(&txr->tx_db, prod));
588 		WRITE_ONCE(txr->tx_prod, prod);
589 
590 		tx_buf->is_push = 1;
591 		netdev_tx_sent_queue(txq, skb->len);
592 		wmb();	/* Sync is_push and byte queue before pushing data */
593 
594 		push_len = (length + sizeof(*tx_push) + 7) / 8;
595 		if (push_len > 16) {
596 			__iowrite64_copy(db, tx_push_buf, 16);
597 			__iowrite32_copy(db + 4, tx_push_buf + 1,
598 					 (push_len - 16) << 1);
599 		} else {
600 			__iowrite64_copy(db, tx_push_buf, push_len);
601 		}
602 
603 		goto tx_done;
604 	}
605 
606 normal_tx:
607 	if (length < BNXT_MIN_PKT_SIZE) {
608 		pad = BNXT_MIN_PKT_SIZE - length;
609 		if (skb_pad(skb, pad))
610 			/* SKB already freed. */
611 			goto tx_kick_pending;
612 		length = BNXT_MIN_PKT_SIZE;
613 	}
614 
615 	mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
616 
617 	if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
618 		goto tx_free;
619 
620 	dma_unmap_addr_set(tx_buf, mapping, mapping);
621 	flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
622 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
623 
624 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
625 	txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
626 
627 	prod = NEXT_TX(prod);
628 	txbd1 = (struct tx_bd_ext *)
629 		&txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
630 
631 	txbd1->tx_bd_hsize_lflags = lflags;
632 	if (skb_is_gso(skb)) {
633 		bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
634 		u32 hdr_len;
635 
636 		if (skb->encapsulation) {
637 			if (udp_gso)
638 				hdr_len = skb_inner_transport_offset(skb) +
639 					  sizeof(struct udphdr);
640 			else
641 				hdr_len = skb_inner_tcp_all_headers(skb);
642 		} else if (udp_gso) {
643 			hdr_len = skb_transport_offset(skb) +
644 				  sizeof(struct udphdr);
645 		} else {
646 			hdr_len = skb_tcp_all_headers(skb);
647 		}
648 
649 		txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
650 					TX_BD_FLAGS_T_IPID |
651 					(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
652 		length = skb_shinfo(skb)->gso_size;
653 		txbd1->tx_bd_mss = cpu_to_le32(length);
654 		length += hdr_len;
655 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
656 		txbd1->tx_bd_hsize_lflags |=
657 			cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
658 		txbd1->tx_bd_mss = 0;
659 	}
660 
661 	length >>= 9;
662 	if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
663 		dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
664 				     skb->len);
665 		i = 0;
666 		goto tx_dma_error;
667 	}
668 	flags |= bnxt_lhint_arr[length];
669 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
670 
671 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
672 	txbd1->tx_bd_cfa_action =
673 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
674 	txbd0 = txbd;
675 	for (i = 0; i < last_frag; i++) {
676 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
677 
678 		prod = NEXT_TX(prod);
679 		txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
680 
681 		len = skb_frag_size(frag);
682 		mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
683 					   DMA_TO_DEVICE);
684 
685 		if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
686 			goto tx_dma_error;
687 
688 		tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
689 		dma_unmap_addr_set(tx_buf, mapping, mapping);
690 
691 		txbd->tx_bd_haddr = cpu_to_le64(mapping);
692 
693 		flags = len << TX_BD_LEN_SHIFT;
694 		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
695 	}
696 
697 	flags &= ~TX_BD_LEN;
698 	txbd->tx_bd_len_flags_type =
699 		cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
700 			    TX_BD_FLAGS_PACKET_END);
701 
702 	netdev_tx_sent_queue(txq, skb->len);
703 
704 	skb_tx_timestamp(skb);
705 
706 	prod = NEXT_TX(prod);
707 	WRITE_ONCE(txr->tx_prod, prod);
708 
709 	if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
710 		bnxt_txr_db_kick(bp, txr, prod);
711 	} else {
712 		if (free_size >= bp->tx_wake_thresh)
713 			txbd0->tx_bd_len_flags_type |=
714 				cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
715 		txr->kick_pending = 1;
716 	}
717 
718 tx_done:
719 
720 	if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
721 		if (netdev_xmit_more() && !tx_buf->is_push) {
722 			txbd0->tx_bd_len_flags_type &=
723 				cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
724 			bnxt_txr_db_kick(bp, txr, prod);
725 		}
726 
727 		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
728 				   bp->tx_wake_thresh);
729 	}
730 	return NETDEV_TX_OK;
731 
732 tx_dma_error:
733 	if (BNXT_TX_PTP_IS_SET(lflags))
734 		atomic_inc(&bp->ptp_cfg->tx_avail);
735 
736 	last_frag = i;
737 
738 	/* start back at beginning and unmap skb */
739 	prod = txr->tx_prod;
740 	tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
741 	dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
742 			 skb_headlen(skb), DMA_TO_DEVICE);
743 	prod = NEXT_TX(prod);
744 
745 	/* unmap remaining mapped pages */
746 	for (i = 0; i < last_frag; i++) {
747 		prod = NEXT_TX(prod);
748 		tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
749 		dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
750 			       skb_frag_size(&skb_shinfo(skb)->frags[i]),
751 			       DMA_TO_DEVICE);
752 	}
753 
754 tx_free:
755 	dev_kfree_skb_any(skb);
756 tx_kick_pending:
757 	if (txr->kick_pending)
758 		bnxt_txr_db_kick(bp, txr, txr->tx_prod);
759 	txr->tx_buf_ring[txr->tx_prod].skb = NULL;
760 	dev_core_stats_tx_dropped_inc(dev);
761 	return NETDEV_TX_OK;
762 }
763 
764 static void __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
765 			  int budget)
766 {
767 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
768 	struct pci_dev *pdev = bp->pdev;
769 	u16 hw_cons = txr->tx_hw_cons;
770 	unsigned int tx_bytes = 0;
771 	u16 cons = txr->tx_cons;
772 	int tx_pkts = 0;
773 
774 	while (RING_TX(bp, cons) != hw_cons) {
775 		struct bnxt_sw_tx_bd *tx_buf;
776 		struct sk_buff *skb;
777 		int j, last;
778 
779 		tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
780 		cons = NEXT_TX(cons);
781 		skb = tx_buf->skb;
782 		tx_buf->skb = NULL;
783 
784 		if (unlikely(!skb)) {
785 			bnxt_sched_reset_txr(bp, txr, cons);
786 			return;
787 		}
788 
789 		tx_pkts++;
790 		tx_bytes += skb->len;
791 
792 		if (tx_buf->is_push) {
793 			tx_buf->is_push = 0;
794 			goto next_tx_int;
795 		}
796 
797 		dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
798 				 skb_headlen(skb), DMA_TO_DEVICE);
799 		last = tx_buf->nr_frags;
800 
801 		for (j = 0; j < last; j++) {
802 			cons = NEXT_TX(cons);
803 			tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
804 			dma_unmap_page(
805 				&pdev->dev,
806 				dma_unmap_addr(tx_buf, mapping),
807 				skb_frag_size(&skb_shinfo(skb)->frags[j]),
808 				DMA_TO_DEVICE);
809 		}
810 		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
811 			if (BNXT_CHIP_P5(bp)) {
812 				/* PTP worker takes ownership of the skb */
813 				if (!bnxt_get_tx_ts_p5(bp, skb))
814 					skb = NULL;
815 				else
816 					atomic_inc(&bp->ptp_cfg->tx_avail);
817 			}
818 		}
819 
820 next_tx_int:
821 		cons = NEXT_TX(cons);
822 
823 		dev_consume_skb_any(skb);
824 	}
825 
826 	WRITE_ONCE(txr->tx_cons, cons);
827 
828 	__netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
829 				   bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
830 				   READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
831 }
832 
833 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
834 {
835 	struct bnxt_tx_ring_info *txr;
836 	int i;
837 
838 	bnxt_for_each_napi_tx(i, bnapi, txr) {
839 		if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons))
840 			__bnxt_tx_int(bp, txr, budget);
841 	}
842 	bnapi->events &= ~BNXT_TX_CMP_EVENT;
843 }
844 
845 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
846 					 struct bnxt_rx_ring_info *rxr,
847 					 unsigned int *offset,
848 					 gfp_t gfp)
849 {
850 	struct page *page;
851 
852 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
853 		page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
854 						BNXT_RX_PAGE_SIZE);
855 	} else {
856 		page = page_pool_dev_alloc_pages(rxr->page_pool);
857 		*offset = 0;
858 	}
859 	if (!page)
860 		return NULL;
861 
862 	*mapping = page_pool_get_dma_addr(page) + *offset;
863 	return page;
864 }
865 
866 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
867 				       gfp_t gfp)
868 {
869 	u8 *data;
870 	struct pci_dev *pdev = bp->pdev;
871 
872 	if (gfp == GFP_ATOMIC)
873 		data = napi_alloc_frag(bp->rx_buf_size);
874 	else
875 		data = netdev_alloc_frag(bp->rx_buf_size);
876 	if (!data)
877 		return NULL;
878 
879 	*mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
880 					bp->rx_buf_use_size, bp->rx_dir,
881 					DMA_ATTR_WEAK_ORDERING);
882 
883 	if (dma_mapping_error(&pdev->dev, *mapping)) {
884 		skb_free_frag(data);
885 		data = NULL;
886 	}
887 	return data;
888 }
889 
890 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
891 		       u16 prod, gfp_t gfp)
892 {
893 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
894 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
895 	dma_addr_t mapping;
896 
897 	if (BNXT_RX_PAGE_MODE(bp)) {
898 		unsigned int offset;
899 		struct page *page =
900 			__bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
901 
902 		if (!page)
903 			return -ENOMEM;
904 
905 		mapping += bp->rx_dma_offset;
906 		rx_buf->data = page;
907 		rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
908 	} else {
909 		u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
910 
911 		if (!data)
912 			return -ENOMEM;
913 
914 		rx_buf->data = data;
915 		rx_buf->data_ptr = data + bp->rx_offset;
916 	}
917 	rx_buf->mapping = mapping;
918 
919 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
920 	return 0;
921 }
922 
923 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
924 {
925 	u16 prod = rxr->rx_prod;
926 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
927 	struct bnxt *bp = rxr->bnapi->bp;
928 	struct rx_bd *cons_bd, *prod_bd;
929 
930 	prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
931 	cons_rx_buf = &rxr->rx_buf_ring[cons];
932 
933 	prod_rx_buf->data = data;
934 	prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
935 
936 	prod_rx_buf->mapping = cons_rx_buf->mapping;
937 
938 	prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
939 	cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)];
940 
941 	prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
942 }
943 
944 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
945 {
946 	u16 next, max = rxr->rx_agg_bmap_size;
947 
948 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
949 	if (next >= max)
950 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
951 	return next;
952 }
953 
954 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
955 				     struct bnxt_rx_ring_info *rxr,
956 				     u16 prod, gfp_t gfp)
957 {
958 	struct rx_bd *rxbd =
959 		&rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
960 	struct bnxt_sw_rx_agg_bd *rx_agg_buf;
961 	struct page *page;
962 	dma_addr_t mapping;
963 	u16 sw_prod = rxr->rx_sw_agg_prod;
964 	unsigned int offset = 0;
965 
966 	page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
967 
968 	if (!page)
969 		return -ENOMEM;
970 
971 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
972 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
973 
974 	__set_bit(sw_prod, rxr->rx_agg_bmap);
975 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
976 	rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
977 
978 	rx_agg_buf->page = page;
979 	rx_agg_buf->offset = offset;
980 	rx_agg_buf->mapping = mapping;
981 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
982 	rxbd->rx_bd_opaque = sw_prod;
983 	return 0;
984 }
985 
986 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
987 				       struct bnxt_cp_ring_info *cpr,
988 				       u16 cp_cons, u16 curr)
989 {
990 	struct rx_agg_cmp *agg;
991 
992 	cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
993 	agg = (struct rx_agg_cmp *)
994 		&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
995 	return agg;
996 }
997 
998 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
999 					      struct bnxt_rx_ring_info *rxr,
1000 					      u16 agg_id, u16 curr)
1001 {
1002 	struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
1003 
1004 	return &tpa_info->agg_arr[curr];
1005 }
1006 
1007 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
1008 				   u16 start, u32 agg_bufs, bool tpa)
1009 {
1010 	struct bnxt_napi *bnapi = cpr->bnapi;
1011 	struct bnxt *bp = bnapi->bp;
1012 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1013 	u16 prod = rxr->rx_agg_prod;
1014 	u16 sw_prod = rxr->rx_sw_agg_prod;
1015 	bool p5_tpa = false;
1016 	u32 i;
1017 
1018 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1019 		p5_tpa = true;
1020 
1021 	for (i = 0; i < agg_bufs; i++) {
1022 		u16 cons;
1023 		struct rx_agg_cmp *agg;
1024 		struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
1025 		struct rx_bd *prod_bd;
1026 		struct page *page;
1027 
1028 		if (p5_tpa)
1029 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
1030 		else
1031 			agg = bnxt_get_agg(bp, cpr, idx, start + i);
1032 		cons = agg->rx_agg_cmp_opaque;
1033 		__clear_bit(cons, rxr->rx_agg_bmap);
1034 
1035 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1036 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
1037 
1038 		__set_bit(sw_prod, rxr->rx_agg_bmap);
1039 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
1040 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1041 
1042 		/* It is possible for sw_prod to be equal to cons, so
1043 		 * set cons_rx_buf->page to NULL first.
1044 		 */
1045 		page = cons_rx_buf->page;
1046 		cons_rx_buf->page = NULL;
1047 		prod_rx_buf->page = page;
1048 		prod_rx_buf->offset = cons_rx_buf->offset;
1049 
1050 		prod_rx_buf->mapping = cons_rx_buf->mapping;
1051 
1052 		prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
1053 
1054 		prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
1055 		prod_bd->rx_bd_opaque = sw_prod;
1056 
1057 		prod = NEXT_RX_AGG(prod);
1058 		sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1059 	}
1060 	rxr->rx_agg_prod = prod;
1061 	rxr->rx_sw_agg_prod = sw_prod;
1062 }
1063 
1064 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
1065 					      struct bnxt_rx_ring_info *rxr,
1066 					      u16 cons, void *data, u8 *data_ptr,
1067 					      dma_addr_t dma_addr,
1068 					      unsigned int offset_and_len)
1069 {
1070 	unsigned int len = offset_and_len & 0xffff;
1071 	struct page *page = data;
1072 	u16 prod = rxr->rx_prod;
1073 	struct sk_buff *skb;
1074 	int err;
1075 
1076 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1077 	if (unlikely(err)) {
1078 		bnxt_reuse_rx_data(rxr, cons, data);
1079 		return NULL;
1080 	}
1081 	dma_addr -= bp->rx_dma_offset;
1082 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1083 				bp->rx_dir);
1084 	skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
1085 	if (!skb) {
1086 		page_pool_recycle_direct(rxr->page_pool, page);
1087 		return NULL;
1088 	}
1089 	skb_mark_for_recycle(skb);
1090 	skb_reserve(skb, bp->rx_offset);
1091 	__skb_put(skb, len);
1092 
1093 	return skb;
1094 }
1095 
1096 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1097 					struct bnxt_rx_ring_info *rxr,
1098 					u16 cons, void *data, u8 *data_ptr,
1099 					dma_addr_t dma_addr,
1100 					unsigned int offset_and_len)
1101 {
1102 	unsigned int payload = offset_and_len >> 16;
1103 	unsigned int len = offset_and_len & 0xffff;
1104 	skb_frag_t *frag;
1105 	struct page *page = data;
1106 	u16 prod = rxr->rx_prod;
1107 	struct sk_buff *skb;
1108 	int off, err;
1109 
1110 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1111 	if (unlikely(err)) {
1112 		bnxt_reuse_rx_data(rxr, cons, data);
1113 		return NULL;
1114 	}
1115 	dma_addr -= bp->rx_dma_offset;
1116 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1117 				bp->rx_dir);
1118 
1119 	if (unlikely(!payload))
1120 		payload = eth_get_headlen(bp->dev, data_ptr, len);
1121 
1122 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1123 	if (!skb) {
1124 		page_pool_recycle_direct(rxr->page_pool, page);
1125 		return NULL;
1126 	}
1127 
1128 	skb_mark_for_recycle(skb);
1129 	off = (void *)data_ptr - page_address(page);
1130 	skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1131 	memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1132 	       payload + NET_IP_ALIGN);
1133 
1134 	frag = &skb_shinfo(skb)->frags[0];
1135 	skb_frag_size_sub(frag, payload);
1136 	skb_frag_off_add(frag, payload);
1137 	skb->data_len -= payload;
1138 	skb->tail += payload;
1139 
1140 	return skb;
1141 }
1142 
1143 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1144 				   struct bnxt_rx_ring_info *rxr, u16 cons,
1145 				   void *data, u8 *data_ptr,
1146 				   dma_addr_t dma_addr,
1147 				   unsigned int offset_and_len)
1148 {
1149 	u16 prod = rxr->rx_prod;
1150 	struct sk_buff *skb;
1151 	int err;
1152 
1153 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1154 	if (unlikely(err)) {
1155 		bnxt_reuse_rx_data(rxr, cons, data);
1156 		return NULL;
1157 	}
1158 
1159 	skb = napi_build_skb(data, bp->rx_buf_size);
1160 	dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1161 			       bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1162 	if (!skb) {
1163 		skb_free_frag(data);
1164 		return NULL;
1165 	}
1166 
1167 	skb_reserve(skb, bp->rx_offset);
1168 	skb_put(skb, offset_and_len & 0xffff);
1169 	return skb;
1170 }
1171 
1172 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1173 			       struct bnxt_cp_ring_info *cpr,
1174 			       struct skb_shared_info *shinfo,
1175 			       u16 idx, u32 agg_bufs, bool tpa,
1176 			       struct xdp_buff *xdp)
1177 {
1178 	struct bnxt_napi *bnapi = cpr->bnapi;
1179 	struct pci_dev *pdev = bp->pdev;
1180 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1181 	u16 prod = rxr->rx_agg_prod;
1182 	u32 i, total_frag_len = 0;
1183 	bool p5_tpa = false;
1184 
1185 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1186 		p5_tpa = true;
1187 
1188 	for (i = 0; i < agg_bufs; i++) {
1189 		skb_frag_t *frag = &shinfo->frags[i];
1190 		u16 cons, frag_len;
1191 		struct rx_agg_cmp *agg;
1192 		struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1193 		struct page *page;
1194 		dma_addr_t mapping;
1195 
1196 		if (p5_tpa)
1197 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1198 		else
1199 			agg = bnxt_get_agg(bp, cpr, idx, i);
1200 		cons = agg->rx_agg_cmp_opaque;
1201 		frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1202 			    RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1203 
1204 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1205 		skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1206 					cons_rx_buf->offset, frag_len);
1207 		shinfo->nr_frags = i + 1;
1208 		__clear_bit(cons, rxr->rx_agg_bmap);
1209 
1210 		/* It is possible for bnxt_alloc_rx_page() to allocate
1211 		 * a sw_prod index that equals the cons index, so we
1212 		 * need to clear the cons entry now.
1213 		 */
1214 		mapping = cons_rx_buf->mapping;
1215 		page = cons_rx_buf->page;
1216 		cons_rx_buf->page = NULL;
1217 
1218 		if (xdp && page_is_pfmemalloc(page))
1219 			xdp_buff_set_frag_pfmemalloc(xdp);
1220 
1221 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1222 			--shinfo->nr_frags;
1223 			cons_rx_buf->page = page;
1224 
1225 			/* Update prod since possibly some pages have been
1226 			 * allocated already.
1227 			 */
1228 			rxr->rx_agg_prod = prod;
1229 			bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1230 			return 0;
1231 		}
1232 
1233 		dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1234 					bp->rx_dir);
1235 
1236 		total_frag_len += frag_len;
1237 		prod = NEXT_RX_AGG(prod);
1238 	}
1239 	rxr->rx_agg_prod = prod;
1240 	return total_frag_len;
1241 }
1242 
1243 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1244 					     struct bnxt_cp_ring_info *cpr,
1245 					     struct sk_buff *skb, u16 idx,
1246 					     u32 agg_bufs, bool tpa)
1247 {
1248 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1249 	u32 total_frag_len = 0;
1250 
1251 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1252 					     agg_bufs, tpa, NULL);
1253 	if (!total_frag_len) {
1254 		skb_mark_for_recycle(skb);
1255 		dev_kfree_skb(skb);
1256 		return NULL;
1257 	}
1258 
1259 	skb->data_len += total_frag_len;
1260 	skb->len += total_frag_len;
1261 	skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1262 	return skb;
1263 }
1264 
1265 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1266 				 struct bnxt_cp_ring_info *cpr,
1267 				 struct xdp_buff *xdp, u16 idx,
1268 				 u32 agg_bufs, bool tpa)
1269 {
1270 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1271 	u32 total_frag_len = 0;
1272 
1273 	if (!xdp_buff_has_frags(xdp))
1274 		shinfo->nr_frags = 0;
1275 
1276 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1277 					     idx, agg_bufs, tpa, xdp);
1278 	if (total_frag_len) {
1279 		xdp_buff_set_frags_flag(xdp);
1280 		shinfo->nr_frags = agg_bufs;
1281 		shinfo->xdp_frags_size = total_frag_len;
1282 	}
1283 	return total_frag_len;
1284 }
1285 
1286 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1287 			       u8 agg_bufs, u32 *raw_cons)
1288 {
1289 	u16 last;
1290 	struct rx_agg_cmp *agg;
1291 
1292 	*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1293 	last = RING_CMP(*raw_cons);
1294 	agg = (struct rx_agg_cmp *)
1295 		&cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1296 	return RX_AGG_CMP_VALID(agg, *raw_cons);
1297 }
1298 
1299 static struct sk_buff *bnxt_copy_data(struct bnxt_napi *bnapi, u8 *data,
1300 				      unsigned int len,
1301 				      dma_addr_t mapping)
1302 {
1303 	struct bnxt *bp = bnapi->bp;
1304 	struct pci_dev *pdev = bp->pdev;
1305 	struct sk_buff *skb;
1306 
1307 	skb = napi_alloc_skb(&bnapi->napi, len);
1308 	if (!skb)
1309 		return NULL;
1310 
1311 	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1312 				bp->rx_dir);
1313 
1314 	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1315 	       len + NET_IP_ALIGN);
1316 
1317 	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1318 				   bp->rx_dir);
1319 
1320 	skb_put(skb, len);
1321 
1322 	return skb;
1323 }
1324 
1325 static struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1326 				     unsigned int len,
1327 				     dma_addr_t mapping)
1328 {
1329 	return bnxt_copy_data(bnapi, data, len, mapping);
1330 }
1331 
1332 static struct sk_buff *bnxt_copy_xdp(struct bnxt_napi *bnapi,
1333 				     struct xdp_buff *xdp,
1334 				     unsigned int len,
1335 				     dma_addr_t mapping)
1336 {
1337 	unsigned int metasize = 0;
1338 	u8 *data = xdp->data;
1339 	struct sk_buff *skb;
1340 
1341 	len = xdp->data_end - xdp->data_meta;
1342 	metasize = xdp->data - xdp->data_meta;
1343 	data = xdp->data_meta;
1344 
1345 	skb = bnxt_copy_data(bnapi, data, len, mapping);
1346 	if (!skb)
1347 		return skb;
1348 
1349 	if (metasize) {
1350 		skb_metadata_set(skb, metasize);
1351 		__skb_pull(skb, metasize);
1352 	}
1353 
1354 	return skb;
1355 }
1356 
1357 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1358 			   u32 *raw_cons, void *cmp)
1359 {
1360 	struct rx_cmp *rxcmp = cmp;
1361 	u32 tmp_raw_cons = *raw_cons;
1362 	u8 cmp_type, agg_bufs = 0;
1363 
1364 	cmp_type = RX_CMP_TYPE(rxcmp);
1365 
1366 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1367 		agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1368 			    RX_CMP_AGG_BUFS) >>
1369 			   RX_CMP_AGG_BUFS_SHIFT;
1370 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1371 		struct rx_tpa_end_cmp *tpa_end = cmp;
1372 
1373 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1374 			return 0;
1375 
1376 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1377 	}
1378 
1379 	if (agg_bufs) {
1380 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1381 			return -EBUSY;
1382 	}
1383 	*raw_cons = tmp_raw_cons;
1384 	return 0;
1385 }
1386 
1387 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1388 {
1389 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1390 	u16 idx = agg_id & MAX_TPA_P5_MASK;
1391 
1392 	if (test_bit(idx, map->agg_idx_bmap))
1393 		idx = find_first_zero_bit(map->agg_idx_bmap,
1394 					  BNXT_AGG_IDX_BMAP_SIZE);
1395 	__set_bit(idx, map->agg_idx_bmap);
1396 	map->agg_id_tbl[agg_id] = idx;
1397 	return idx;
1398 }
1399 
1400 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1401 {
1402 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1403 
1404 	__clear_bit(idx, map->agg_idx_bmap);
1405 }
1406 
1407 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1408 {
1409 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1410 
1411 	return map->agg_id_tbl[agg_id];
1412 }
1413 
1414 static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info,
1415 			      struct rx_tpa_start_cmp *tpa_start,
1416 			      struct rx_tpa_start_cmp_ext *tpa_start1)
1417 {
1418 	tpa_info->cfa_code_valid = 1;
1419 	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1420 	tpa_info->vlan_valid = 0;
1421 	if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1422 		tpa_info->vlan_valid = 1;
1423 		tpa_info->metadata =
1424 			le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1425 	}
1426 }
1427 
1428 static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info,
1429 				 struct rx_tpa_start_cmp *tpa_start,
1430 				 struct rx_tpa_start_cmp_ext *tpa_start1)
1431 {
1432 	tpa_info->vlan_valid = 0;
1433 	if (TPA_START_VLAN_VALID(tpa_start)) {
1434 		u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
1435 		u32 vlan_proto = ETH_P_8021Q;
1436 
1437 		tpa_info->vlan_valid = 1;
1438 		if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
1439 			vlan_proto = ETH_P_8021AD;
1440 		tpa_info->metadata = vlan_proto << 16 |
1441 				     TPA_START_METADATA0_TCI(tpa_start1);
1442 	}
1443 }
1444 
1445 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1446 			   u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
1447 			   struct rx_tpa_start_cmp_ext *tpa_start1)
1448 {
1449 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1450 	struct bnxt_tpa_info *tpa_info;
1451 	u16 cons, prod, agg_id;
1452 	struct rx_bd *prod_bd;
1453 	dma_addr_t mapping;
1454 
1455 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1456 		agg_id = TPA_START_AGG_ID_P5(tpa_start);
1457 		agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1458 	} else {
1459 		agg_id = TPA_START_AGG_ID(tpa_start);
1460 	}
1461 	cons = tpa_start->rx_tpa_start_cmp_opaque;
1462 	prod = rxr->rx_prod;
1463 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1464 	prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
1465 	tpa_info = &rxr->rx_tpa[agg_id];
1466 
1467 	if (unlikely(cons != rxr->rx_next_cons ||
1468 		     TPA_START_ERROR(tpa_start))) {
1469 		netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1470 			    cons, rxr->rx_next_cons,
1471 			    TPA_START_ERROR_CODE(tpa_start1));
1472 		bnxt_sched_reset_rxr(bp, rxr);
1473 		return;
1474 	}
1475 	prod_rx_buf->data = tpa_info->data;
1476 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
1477 
1478 	mapping = tpa_info->mapping;
1479 	prod_rx_buf->mapping = mapping;
1480 
1481 	prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
1482 
1483 	prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1484 
1485 	tpa_info->data = cons_rx_buf->data;
1486 	tpa_info->data_ptr = cons_rx_buf->data_ptr;
1487 	cons_rx_buf->data = NULL;
1488 	tpa_info->mapping = cons_rx_buf->mapping;
1489 
1490 	tpa_info->len =
1491 		le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1492 				RX_TPA_START_CMP_LEN_SHIFT;
1493 	if (likely(TPA_START_HASH_VALID(tpa_start))) {
1494 		tpa_info->hash_type = PKT_HASH_TYPE_L4;
1495 		tpa_info->gso_type = SKB_GSO_TCPV4;
1496 		if (TPA_START_IS_IPV6(tpa_start1))
1497 			tpa_info->gso_type = SKB_GSO_TCPV6;
1498 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1499 		else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP &&
1500 			 TPA_START_HASH_TYPE(tpa_start) == 3)
1501 			tpa_info->gso_type = SKB_GSO_TCPV6;
1502 		tpa_info->rss_hash =
1503 			le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1504 	} else {
1505 		tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1506 		tpa_info->gso_type = 0;
1507 		netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1508 	}
1509 	tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1510 	tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1511 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
1512 		bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1);
1513 	else
1514 		bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
1515 	tpa_info->agg_count = 0;
1516 
1517 	rxr->rx_prod = NEXT_RX(prod);
1518 	cons = RING_RX(bp, NEXT_RX(cons));
1519 	rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
1520 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1521 
1522 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1523 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1524 	cons_rx_buf->data = NULL;
1525 }
1526 
1527 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1528 {
1529 	if (agg_bufs)
1530 		bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1531 }
1532 
1533 #ifdef CONFIG_INET
1534 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1535 {
1536 	struct udphdr *uh = NULL;
1537 
1538 	if (ip_proto == htons(ETH_P_IP)) {
1539 		struct iphdr *iph = (struct iphdr *)skb->data;
1540 
1541 		if (iph->protocol == IPPROTO_UDP)
1542 			uh = (struct udphdr *)(iph + 1);
1543 	} else {
1544 		struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1545 
1546 		if (iph->nexthdr == IPPROTO_UDP)
1547 			uh = (struct udphdr *)(iph + 1);
1548 	}
1549 	if (uh) {
1550 		if (uh->check)
1551 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1552 		else
1553 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1554 	}
1555 }
1556 #endif
1557 
1558 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1559 					   int payload_off, int tcp_ts,
1560 					   struct sk_buff *skb)
1561 {
1562 #ifdef CONFIG_INET
1563 	struct tcphdr *th;
1564 	int len, nw_off;
1565 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1566 	u32 hdr_info = tpa_info->hdr_info;
1567 	bool loopback = false;
1568 
1569 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1570 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1571 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1572 
1573 	/* If the packet is an internal loopback packet, the offsets will
1574 	 * have an extra 4 bytes.
1575 	 */
1576 	if (inner_mac_off == 4) {
1577 		loopback = true;
1578 	} else if (inner_mac_off > 4) {
1579 		__be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1580 					    ETH_HLEN - 2));
1581 
1582 		/* We only support inner iPv4/ipv6.  If we don't see the
1583 		 * correct protocol ID, it must be a loopback packet where
1584 		 * the offsets are off by 4.
1585 		 */
1586 		if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1587 			loopback = true;
1588 	}
1589 	if (loopback) {
1590 		/* internal loopback packet, subtract all offsets by 4 */
1591 		inner_ip_off -= 4;
1592 		inner_mac_off -= 4;
1593 		outer_ip_off -= 4;
1594 	}
1595 
1596 	nw_off = inner_ip_off - ETH_HLEN;
1597 	skb_set_network_header(skb, nw_off);
1598 	if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1599 		struct ipv6hdr *iph = ipv6_hdr(skb);
1600 
1601 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1602 		len = skb->len - skb_transport_offset(skb);
1603 		th = tcp_hdr(skb);
1604 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1605 	} else {
1606 		struct iphdr *iph = ip_hdr(skb);
1607 
1608 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1609 		len = skb->len - skb_transport_offset(skb);
1610 		th = tcp_hdr(skb);
1611 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1612 	}
1613 
1614 	if (inner_mac_off) { /* tunnel */
1615 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1616 					    ETH_HLEN - 2));
1617 
1618 		bnxt_gro_tunnel(skb, proto);
1619 	}
1620 #endif
1621 	return skb;
1622 }
1623 
1624 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1625 					   int payload_off, int tcp_ts,
1626 					   struct sk_buff *skb)
1627 {
1628 #ifdef CONFIG_INET
1629 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1630 	u32 hdr_info = tpa_info->hdr_info;
1631 	int iphdr_len, nw_off;
1632 
1633 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1634 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1635 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1636 
1637 	nw_off = inner_ip_off - ETH_HLEN;
1638 	skb_set_network_header(skb, nw_off);
1639 	iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1640 		     sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1641 	skb_set_transport_header(skb, nw_off + iphdr_len);
1642 
1643 	if (inner_mac_off) { /* tunnel */
1644 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1645 					    ETH_HLEN - 2));
1646 
1647 		bnxt_gro_tunnel(skb, proto);
1648 	}
1649 #endif
1650 	return skb;
1651 }
1652 
1653 #define BNXT_IPV4_HDR_SIZE	(sizeof(struct iphdr) + sizeof(struct tcphdr))
1654 #define BNXT_IPV6_HDR_SIZE	(sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1655 
1656 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1657 					   int payload_off, int tcp_ts,
1658 					   struct sk_buff *skb)
1659 {
1660 #ifdef CONFIG_INET
1661 	struct tcphdr *th;
1662 	int len, nw_off, tcp_opt_len = 0;
1663 
1664 	if (tcp_ts)
1665 		tcp_opt_len = 12;
1666 
1667 	if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1668 		struct iphdr *iph;
1669 
1670 		nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1671 			 ETH_HLEN;
1672 		skb_set_network_header(skb, nw_off);
1673 		iph = ip_hdr(skb);
1674 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1675 		len = skb->len - skb_transport_offset(skb);
1676 		th = tcp_hdr(skb);
1677 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1678 	} else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1679 		struct ipv6hdr *iph;
1680 
1681 		nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1682 			 ETH_HLEN;
1683 		skb_set_network_header(skb, nw_off);
1684 		iph = ipv6_hdr(skb);
1685 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1686 		len = skb->len - skb_transport_offset(skb);
1687 		th = tcp_hdr(skb);
1688 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1689 	} else {
1690 		dev_kfree_skb_any(skb);
1691 		return NULL;
1692 	}
1693 
1694 	if (nw_off) /* tunnel */
1695 		bnxt_gro_tunnel(skb, skb->protocol);
1696 #endif
1697 	return skb;
1698 }
1699 
1700 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1701 					   struct bnxt_tpa_info *tpa_info,
1702 					   struct rx_tpa_end_cmp *tpa_end,
1703 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1704 					   struct sk_buff *skb)
1705 {
1706 #ifdef CONFIG_INET
1707 	int payload_off;
1708 	u16 segs;
1709 
1710 	segs = TPA_END_TPA_SEGS(tpa_end);
1711 	if (segs == 1)
1712 		return skb;
1713 
1714 	NAPI_GRO_CB(skb)->count = segs;
1715 	skb_shinfo(skb)->gso_size =
1716 		le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1717 	skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1718 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1719 		payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1720 	else
1721 		payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1722 	skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1723 	if (likely(skb))
1724 		tcp_gro_complete(skb);
1725 #endif
1726 	return skb;
1727 }
1728 
1729 /* Given the cfa_code of a received packet determine which
1730  * netdev (vf-rep or PF) the packet is destined to.
1731  */
1732 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1733 {
1734 	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1735 
1736 	/* if vf-rep dev is NULL, the must belongs to the PF */
1737 	return dev ? dev : bp->dev;
1738 }
1739 
1740 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1741 					   struct bnxt_cp_ring_info *cpr,
1742 					   u32 *raw_cons,
1743 					   struct rx_tpa_end_cmp *tpa_end,
1744 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1745 					   u8 *event)
1746 {
1747 	struct bnxt_napi *bnapi = cpr->bnapi;
1748 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1749 	struct net_device *dev = bp->dev;
1750 	u8 *data_ptr, agg_bufs;
1751 	unsigned int len;
1752 	struct bnxt_tpa_info *tpa_info;
1753 	dma_addr_t mapping;
1754 	struct sk_buff *skb;
1755 	u16 idx = 0, agg_id;
1756 	void *data;
1757 	bool gro;
1758 
1759 	if (unlikely(bnapi->in_reset)) {
1760 		int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1761 
1762 		if (rc < 0)
1763 			return ERR_PTR(-EBUSY);
1764 		return NULL;
1765 	}
1766 
1767 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1768 		agg_id = TPA_END_AGG_ID_P5(tpa_end);
1769 		agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1770 		agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1771 		tpa_info = &rxr->rx_tpa[agg_id];
1772 		if (unlikely(agg_bufs != tpa_info->agg_count)) {
1773 			netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1774 				    agg_bufs, tpa_info->agg_count);
1775 			agg_bufs = tpa_info->agg_count;
1776 		}
1777 		tpa_info->agg_count = 0;
1778 		*event |= BNXT_AGG_EVENT;
1779 		bnxt_free_agg_idx(rxr, agg_id);
1780 		idx = agg_id;
1781 		gro = !!(bp->flags & BNXT_FLAG_GRO);
1782 	} else {
1783 		agg_id = TPA_END_AGG_ID(tpa_end);
1784 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1785 		tpa_info = &rxr->rx_tpa[agg_id];
1786 		idx = RING_CMP(*raw_cons);
1787 		if (agg_bufs) {
1788 			if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1789 				return ERR_PTR(-EBUSY);
1790 
1791 			*event |= BNXT_AGG_EVENT;
1792 			idx = NEXT_CMP(idx);
1793 		}
1794 		gro = !!TPA_END_GRO(tpa_end);
1795 	}
1796 	data = tpa_info->data;
1797 	data_ptr = tpa_info->data_ptr;
1798 	prefetch(data_ptr);
1799 	len = tpa_info->len;
1800 	mapping = tpa_info->mapping;
1801 
1802 	if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1803 		bnxt_abort_tpa(cpr, idx, agg_bufs);
1804 		if (agg_bufs > MAX_SKB_FRAGS)
1805 			netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1806 				    agg_bufs, (int)MAX_SKB_FRAGS);
1807 		return NULL;
1808 	}
1809 
1810 	if (len <= bp->rx_copy_thresh) {
1811 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1812 		if (!skb) {
1813 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1814 			cpr->sw_stats.rx.rx_oom_discards += 1;
1815 			return NULL;
1816 		}
1817 	} else {
1818 		u8 *new_data;
1819 		dma_addr_t new_mapping;
1820 
1821 		new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1822 		if (!new_data) {
1823 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1824 			cpr->sw_stats.rx.rx_oom_discards += 1;
1825 			return NULL;
1826 		}
1827 
1828 		tpa_info->data = new_data;
1829 		tpa_info->data_ptr = new_data + bp->rx_offset;
1830 		tpa_info->mapping = new_mapping;
1831 
1832 		skb = napi_build_skb(data, bp->rx_buf_size);
1833 		dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1834 				       bp->rx_buf_use_size, bp->rx_dir,
1835 				       DMA_ATTR_WEAK_ORDERING);
1836 
1837 		if (!skb) {
1838 			skb_free_frag(data);
1839 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1840 			cpr->sw_stats.rx.rx_oom_discards += 1;
1841 			return NULL;
1842 		}
1843 		skb_reserve(skb, bp->rx_offset);
1844 		skb_put(skb, len);
1845 	}
1846 
1847 	if (agg_bufs) {
1848 		skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1849 		if (!skb) {
1850 			/* Page reuse already handled by bnxt_rx_pages(). */
1851 			cpr->sw_stats.rx.rx_oom_discards += 1;
1852 			return NULL;
1853 		}
1854 	}
1855 
1856 	if (tpa_info->cfa_code_valid)
1857 		dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code);
1858 	skb->protocol = eth_type_trans(skb, dev);
1859 
1860 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1861 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1862 
1863 	if (tpa_info->vlan_valid &&
1864 	    (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1865 		__be16 vlan_proto = htons(tpa_info->metadata >>
1866 					  RX_CMP_FLAGS2_METADATA_TPID_SFT);
1867 		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1868 
1869 		if (eth_type_vlan(vlan_proto)) {
1870 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1871 		} else {
1872 			dev_kfree_skb(skb);
1873 			return NULL;
1874 		}
1875 	}
1876 
1877 	skb_checksum_none_assert(skb);
1878 	if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1879 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1880 		skb->csum_level =
1881 			(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1882 	}
1883 
1884 	if (gro)
1885 		skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1886 
1887 	return skb;
1888 }
1889 
1890 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1891 			 struct rx_agg_cmp *rx_agg)
1892 {
1893 	u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1894 	struct bnxt_tpa_info *tpa_info;
1895 
1896 	agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1897 	tpa_info = &rxr->rx_tpa[agg_id];
1898 	BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1899 	tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1900 }
1901 
1902 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1903 			     struct sk_buff *skb)
1904 {
1905 	skb_mark_for_recycle(skb);
1906 
1907 	if (skb->dev != bp->dev) {
1908 		/* this packet belongs to a vf-rep */
1909 		bnxt_vf_rep_rx(bp, skb);
1910 		return;
1911 	}
1912 	skb_record_rx_queue(skb, bnapi->index);
1913 	napi_gro_receive(&bnapi->napi, skb);
1914 }
1915 
1916 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1917 			     struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1918 {
1919 	u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1920 
1921 	if (BNXT_PTP_RX_TS_VALID(flags))
1922 		goto ts_valid;
1923 	if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1924 		return false;
1925 
1926 ts_valid:
1927 	*cmpl_ts = ts;
1928 	return true;
1929 }
1930 
1931 static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type,
1932 				    struct rx_cmp *rxcmp,
1933 				    struct rx_cmp_ext *rxcmp1)
1934 {
1935 	__be16 vlan_proto;
1936 	u16 vtag;
1937 
1938 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1939 		__le32 flags2 = rxcmp1->rx_cmp_flags2;
1940 		u32 meta_data;
1941 
1942 		if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
1943 			return skb;
1944 
1945 		meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1946 		vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1947 		vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
1948 		if (eth_type_vlan(vlan_proto))
1949 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1950 		else
1951 			goto vlan_err;
1952 	} else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1953 		if (RX_CMP_VLAN_VALID(rxcmp)) {
1954 			u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
1955 
1956 			if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
1957 				vlan_proto = htons(ETH_P_8021Q);
1958 			else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
1959 				vlan_proto = htons(ETH_P_8021AD);
1960 			else
1961 				goto vlan_err;
1962 			vtag = RX_CMP_METADATA0_TCI(rxcmp1);
1963 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1964 		}
1965 	}
1966 	return skb;
1967 vlan_err:
1968 	dev_kfree_skb(skb);
1969 	return NULL;
1970 }
1971 
1972 static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp,
1973 					   struct rx_cmp *rxcmp)
1974 {
1975 	u8 ext_op;
1976 
1977 	ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp);
1978 	switch (ext_op) {
1979 	case EXT_OP_INNER_4:
1980 	case EXT_OP_OUTER_4:
1981 	case EXT_OP_INNFL_3:
1982 	case EXT_OP_OUTFL_3:
1983 		return PKT_HASH_TYPE_L4;
1984 	default:
1985 		return PKT_HASH_TYPE_L3;
1986 	}
1987 }
1988 
1989 /* returns the following:
1990  * 1       - 1 packet successfully received
1991  * 0       - successful TPA_START, packet not completed yet
1992  * -EBUSY  - completion ring does not have all the agg buffers yet
1993  * -ENOMEM - packet aborted due to out of memory
1994  * -EIO    - packet aborted due to hw error indicated in BD
1995  */
1996 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1997 		       u32 *raw_cons, u8 *event)
1998 {
1999 	struct bnxt_napi *bnapi = cpr->bnapi;
2000 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2001 	struct net_device *dev = bp->dev;
2002 	struct rx_cmp *rxcmp;
2003 	struct rx_cmp_ext *rxcmp1;
2004 	u32 tmp_raw_cons = *raw_cons;
2005 	u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
2006 	struct bnxt_sw_rx_bd *rx_buf;
2007 	unsigned int len;
2008 	u8 *data_ptr, agg_bufs, cmp_type;
2009 	bool xdp_active = false;
2010 	dma_addr_t dma_addr;
2011 	struct sk_buff *skb;
2012 	struct xdp_buff xdp;
2013 	u32 flags, misc;
2014 	u32 cmpl_ts;
2015 	void *data;
2016 	int rc = 0;
2017 
2018 	rxcmp = (struct rx_cmp *)
2019 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2020 
2021 	cmp_type = RX_CMP_TYPE(rxcmp);
2022 
2023 	if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
2024 		bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
2025 		goto next_rx_no_prod_no_len;
2026 	}
2027 
2028 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2029 	cp_cons = RING_CMP(tmp_raw_cons);
2030 	rxcmp1 = (struct rx_cmp_ext *)
2031 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2032 
2033 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2034 		return -EBUSY;
2035 
2036 	/* The valid test of the entry must be done first before
2037 	 * reading any further.
2038 	 */
2039 	dma_rmb();
2040 	prod = rxr->rx_prod;
2041 
2042 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
2043 	    cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2044 		bnxt_tpa_start(bp, rxr, cmp_type,
2045 			       (struct rx_tpa_start_cmp *)rxcmp,
2046 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
2047 
2048 		*event |= BNXT_RX_EVENT;
2049 		goto next_rx_no_prod_no_len;
2050 
2051 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2052 		skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
2053 				   (struct rx_tpa_end_cmp *)rxcmp,
2054 				   (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
2055 
2056 		if (IS_ERR(skb))
2057 			return -EBUSY;
2058 
2059 		rc = -ENOMEM;
2060 		if (likely(skb)) {
2061 			bnxt_deliver_skb(bp, bnapi, skb);
2062 			rc = 1;
2063 		}
2064 		*event |= BNXT_RX_EVENT;
2065 		goto next_rx_no_prod_no_len;
2066 	}
2067 
2068 	cons = rxcmp->rx_cmp_opaque;
2069 	if (unlikely(cons != rxr->rx_next_cons)) {
2070 		int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
2071 
2072 		/* 0xffff is forced error, don't print it */
2073 		if (rxr->rx_next_cons != 0xffff)
2074 			netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
2075 				    cons, rxr->rx_next_cons);
2076 		bnxt_sched_reset_rxr(bp, rxr);
2077 		if (rc1)
2078 			return rc1;
2079 		goto next_rx_no_prod_no_len;
2080 	}
2081 	rx_buf = &rxr->rx_buf_ring[cons];
2082 	data = rx_buf->data;
2083 	data_ptr = rx_buf->data_ptr;
2084 	prefetch(data_ptr);
2085 
2086 	misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
2087 	agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
2088 
2089 	if (agg_bufs) {
2090 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
2091 			return -EBUSY;
2092 
2093 		cp_cons = NEXT_CMP(cp_cons);
2094 		*event |= BNXT_AGG_EVENT;
2095 	}
2096 	*event |= BNXT_RX_EVENT;
2097 
2098 	rx_buf->data = NULL;
2099 	if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
2100 		u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
2101 
2102 		bnxt_reuse_rx_data(rxr, cons, data);
2103 		if (agg_bufs)
2104 			bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
2105 					       false);
2106 
2107 		rc = -EIO;
2108 		if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
2109 			bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
2110 			if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
2111 			    !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
2112 				netdev_warn_once(bp->dev, "RX buffer error %x\n",
2113 						 rx_err);
2114 				bnxt_sched_reset_rxr(bp, rxr);
2115 			}
2116 		}
2117 		goto next_rx_no_len;
2118 	}
2119 
2120 	flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
2121 	len = flags >> RX_CMP_LEN_SHIFT;
2122 	dma_addr = rx_buf->mapping;
2123 
2124 	if (bnxt_xdp_attached(bp, rxr)) {
2125 		bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
2126 		if (agg_bufs) {
2127 			u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
2128 							     cp_cons, agg_bufs,
2129 							     false);
2130 			if (!frag_len) {
2131 				cpr->sw_stats.rx.rx_oom_discards += 1;
2132 				rc = -ENOMEM;
2133 				goto next_rx;
2134 			}
2135 		}
2136 		xdp_active = true;
2137 	}
2138 
2139 	if (xdp_active) {
2140 		if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) {
2141 			rc = 1;
2142 			goto next_rx;
2143 		}
2144 	}
2145 
2146 	if (len <= bp->rx_copy_thresh) {
2147 		if (!xdp_active)
2148 			skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
2149 		else
2150 			skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr);
2151 		bnxt_reuse_rx_data(rxr, cons, data);
2152 		if (!skb) {
2153 			if (agg_bufs) {
2154 				if (!xdp_active)
2155 					bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
2156 							       agg_bufs, false);
2157 				else
2158 					bnxt_xdp_buff_frags_free(rxr, &xdp);
2159 			}
2160 			cpr->sw_stats.rx.rx_oom_discards += 1;
2161 			rc = -ENOMEM;
2162 			goto next_rx;
2163 		}
2164 	} else {
2165 		u32 payload;
2166 
2167 		if (rx_buf->data_ptr == data_ptr)
2168 			payload = misc & RX_CMP_PAYLOAD_OFFSET;
2169 		else
2170 			payload = 0;
2171 		skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
2172 				      payload | len);
2173 		if (!skb) {
2174 			cpr->sw_stats.rx.rx_oom_discards += 1;
2175 			rc = -ENOMEM;
2176 			goto next_rx;
2177 		}
2178 	}
2179 
2180 	if (agg_bufs) {
2181 		if (!xdp_active) {
2182 			skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
2183 			if (!skb) {
2184 				cpr->sw_stats.rx.rx_oom_discards += 1;
2185 				rc = -ENOMEM;
2186 				goto next_rx;
2187 			}
2188 		} else {
2189 			skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
2190 			if (!skb) {
2191 				/* we should be able to free the old skb here */
2192 				bnxt_xdp_buff_frags_free(rxr, &xdp);
2193 				cpr->sw_stats.rx.rx_oom_discards += 1;
2194 				rc = -ENOMEM;
2195 				goto next_rx;
2196 			}
2197 		}
2198 	}
2199 
2200 	if (RX_CMP_HASH_VALID(rxcmp)) {
2201 		enum pkt_hash_types type;
2202 
2203 		if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2204 			type = bnxt_rss_ext_op(bp, rxcmp);
2205 		} else {
2206 			u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
2207 
2208 			/* RSS profiles 1 and 3 with extract code 0 for inner
2209 			 * 4-tuple
2210 			 */
2211 			if (hash_type != 1 && hash_type != 3)
2212 				type = PKT_HASH_TYPE_L3;
2213 			else
2214 				type = PKT_HASH_TYPE_L4;
2215 		}
2216 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2217 	}
2218 
2219 	if (cmp_type == CMP_TYPE_RX_L2_CMP)
2220 		dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1));
2221 	skb->protocol = eth_type_trans(skb, dev);
2222 
2223 	if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) {
2224 		skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
2225 		if (!skb)
2226 			goto next_rx;
2227 	}
2228 
2229 	skb_checksum_none_assert(skb);
2230 	if (RX_CMP_L4_CS_OK(rxcmp1)) {
2231 		if (dev->features & NETIF_F_RXCSUM) {
2232 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2233 			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2234 		}
2235 	} else {
2236 		if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2237 			if (dev->features & NETIF_F_RXCSUM)
2238 				bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2239 		}
2240 	}
2241 
2242 	if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2243 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
2244 			u64 ns, ts;
2245 
2246 			if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2247 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2248 
2249 				spin_lock_bh(&ptp->ptp_lock);
2250 				ns = timecounter_cyc2time(&ptp->tc, ts);
2251 				spin_unlock_bh(&ptp->ptp_lock);
2252 				memset(skb_hwtstamps(skb), 0,
2253 				       sizeof(*skb_hwtstamps(skb)));
2254 				skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2255 			}
2256 		}
2257 	}
2258 	bnxt_deliver_skb(bp, bnapi, skb);
2259 	rc = 1;
2260 
2261 next_rx:
2262 	cpr->rx_packets += 1;
2263 	cpr->rx_bytes += len;
2264 
2265 next_rx_no_len:
2266 	rxr->rx_prod = NEXT_RX(prod);
2267 	rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
2268 
2269 next_rx_no_prod_no_len:
2270 	*raw_cons = tmp_raw_cons;
2271 
2272 	return rc;
2273 }
2274 
2275 /* In netpoll mode, if we are using a combined completion ring, we need to
2276  * discard the rx packets and recycle the buffers.
2277  */
2278 static int bnxt_force_rx_discard(struct bnxt *bp,
2279 				 struct bnxt_cp_ring_info *cpr,
2280 				 u32 *raw_cons, u8 *event)
2281 {
2282 	u32 tmp_raw_cons = *raw_cons;
2283 	struct rx_cmp_ext *rxcmp1;
2284 	struct rx_cmp *rxcmp;
2285 	u16 cp_cons;
2286 	u8 cmp_type;
2287 	int rc;
2288 
2289 	cp_cons = RING_CMP(tmp_raw_cons);
2290 	rxcmp = (struct rx_cmp *)
2291 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2292 
2293 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2294 	cp_cons = RING_CMP(tmp_raw_cons);
2295 	rxcmp1 = (struct rx_cmp_ext *)
2296 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2297 
2298 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2299 		return -EBUSY;
2300 
2301 	/* The valid test of the entry must be done first before
2302 	 * reading any further.
2303 	 */
2304 	dma_rmb();
2305 	cmp_type = RX_CMP_TYPE(rxcmp);
2306 	if (cmp_type == CMP_TYPE_RX_L2_CMP ||
2307 	    cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2308 		rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2309 			cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2310 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2311 		struct rx_tpa_end_cmp_ext *tpa_end1;
2312 
2313 		tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2314 		tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2315 			cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2316 	}
2317 	rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2318 	if (rc && rc != -EBUSY)
2319 		cpr->sw_stats.rx.rx_netpoll_discards += 1;
2320 	return rc;
2321 }
2322 
2323 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2324 {
2325 	struct bnxt_fw_health *fw_health = bp->fw_health;
2326 	u32 reg = fw_health->regs[reg_idx];
2327 	u32 reg_type, reg_off, val = 0;
2328 
2329 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2330 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2331 	switch (reg_type) {
2332 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
2333 		pci_read_config_dword(bp->pdev, reg_off, &val);
2334 		break;
2335 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
2336 		reg_off = fw_health->mapped_regs[reg_idx];
2337 		fallthrough;
2338 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2339 		val = readl(bp->bar0 + reg_off);
2340 		break;
2341 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2342 		val = readl(bp->bar1 + reg_off);
2343 		break;
2344 	}
2345 	if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2346 		val &= fw_health->fw_reset_inprog_reg_mask;
2347 	return val;
2348 }
2349 
2350 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2351 {
2352 	int i;
2353 
2354 	for (i = 0; i < bp->rx_nr_rings; i++) {
2355 		u16 grp_idx = bp->rx_ring[i].bnapi->index;
2356 		struct bnxt_ring_grp_info *grp_info;
2357 
2358 		grp_info = &bp->grp_info[grp_idx];
2359 		if (grp_info->agg_fw_ring_id == ring_id)
2360 			return grp_idx;
2361 	}
2362 	return INVALID_HW_RING_ID;
2363 }
2364 
2365 static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info)
2366 {
2367 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2368 
2369 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
2370 		return link_info->force_link_speed2;
2371 	if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4)
2372 		return link_info->force_pam4_link_speed;
2373 	return link_info->force_link_speed;
2374 }
2375 
2376 static void bnxt_set_force_speed(struct bnxt_link_info *link_info)
2377 {
2378 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2379 
2380 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2381 		link_info->req_link_speed = link_info->force_link_speed2;
2382 		link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2383 		switch (link_info->req_link_speed) {
2384 		case BNXT_LINK_SPEED_50GB_PAM4:
2385 		case BNXT_LINK_SPEED_100GB_PAM4:
2386 		case BNXT_LINK_SPEED_200GB_PAM4:
2387 		case BNXT_LINK_SPEED_400GB_PAM4:
2388 			link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2389 			break;
2390 		case BNXT_LINK_SPEED_100GB_PAM4_112:
2391 		case BNXT_LINK_SPEED_200GB_PAM4_112:
2392 		case BNXT_LINK_SPEED_400GB_PAM4_112:
2393 			link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112;
2394 			break;
2395 		default:
2396 			link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2397 		}
2398 		return;
2399 	}
2400 	link_info->req_link_speed = link_info->force_link_speed;
2401 	link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2402 	if (link_info->force_pam4_link_speed) {
2403 		link_info->req_link_speed = link_info->force_pam4_link_speed;
2404 		link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2405 	}
2406 }
2407 
2408 static void bnxt_set_auto_speed(struct bnxt_link_info *link_info)
2409 {
2410 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2411 
2412 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2413 		link_info->advertising = link_info->auto_link_speeds2;
2414 		return;
2415 	}
2416 	link_info->advertising = link_info->auto_link_speeds;
2417 	link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
2418 }
2419 
2420 static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info)
2421 {
2422 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2423 
2424 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2425 		if (link_info->req_link_speed != link_info->force_link_speed2)
2426 			return true;
2427 		return false;
2428 	}
2429 	if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
2430 	    link_info->req_link_speed != link_info->force_link_speed)
2431 		return true;
2432 	if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
2433 	    link_info->req_link_speed != link_info->force_pam4_link_speed)
2434 		return true;
2435 	return false;
2436 }
2437 
2438 static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info)
2439 {
2440 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2441 
2442 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2443 		if (link_info->advertising != link_info->auto_link_speeds2)
2444 			return true;
2445 		return false;
2446 	}
2447 	if (link_info->advertising != link_info->auto_link_speeds ||
2448 	    link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
2449 		return true;
2450 	return false;
2451 }
2452 
2453 #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2)				\
2454 	((data2) &							\
2455 	  ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK)
2456 
2457 #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)			\
2458 	(((data2) &							\
2459 	  ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\
2460 	 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT)
2461 
2462 #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1)			\
2463 	((data1) &							\
2464 	 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK)
2465 
2466 #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)		\
2467 	(((data1) &							\
2468 	  ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\
2469 	 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING)
2470 
2471 /* Return true if the workqueue has to be scheduled */
2472 static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2473 {
2474 	u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2475 
2476 	switch (err_type) {
2477 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2478 		netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2479 			   BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2480 		break;
2481 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2482 		netdev_warn(bp->dev, "Pause Storm detected!\n");
2483 		break;
2484 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2485 		netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2486 		break;
2487 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: {
2488 		u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1);
2489 		char *threshold_type;
2490 		bool notify = false;
2491 		char *dir_str;
2492 
2493 		switch (type) {
2494 		case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN:
2495 			threshold_type = "warning";
2496 			break;
2497 		case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL:
2498 			threshold_type = "critical";
2499 			break;
2500 		case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL:
2501 			threshold_type = "fatal";
2502 			break;
2503 		case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN:
2504 			threshold_type = "shutdown";
2505 			break;
2506 		default:
2507 			netdev_err(bp->dev, "Unknown Thermal threshold type event\n");
2508 			return false;
2509 		}
2510 		if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) {
2511 			dir_str = "above";
2512 			notify = true;
2513 		} else {
2514 			dir_str = "below";
2515 		}
2516 		netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n",
2517 			    dir_str, threshold_type);
2518 		netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n",
2519 			    BNXT_EVENT_THERMAL_CURRENT_TEMP(data2),
2520 			    BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2));
2521 		if (notify) {
2522 			bp->thermal_threshold_type = type;
2523 			set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event);
2524 			return true;
2525 		}
2526 		return false;
2527 	}
2528 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED:
2529 		netdev_warn(bp->dev, "Speed change not supported with dual rate transceivers on this board\n");
2530 		break;
2531 	default:
2532 		netdev_err(bp->dev, "FW reported unknown error type %u\n",
2533 			   err_type);
2534 		break;
2535 	}
2536 	return false;
2537 }
2538 
2539 #define BNXT_GET_EVENT_PORT(data)	\
2540 	((data) &			\
2541 	 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2542 
2543 #define BNXT_EVENT_RING_TYPE(data2)	\
2544 	((data2) &			\
2545 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2546 
2547 #define BNXT_EVENT_RING_TYPE_RX(data2)	\
2548 	(BNXT_EVENT_RING_TYPE(data2) ==	\
2549 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2550 
2551 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)	\
2552 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2553 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2554 
2555 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)	\
2556 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2557 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2558 
2559 #define BNXT_PHC_BITS	48
2560 
2561 static int bnxt_async_event_process(struct bnxt *bp,
2562 				    struct hwrm_async_event_cmpl *cmpl)
2563 {
2564 	u16 event_id = le16_to_cpu(cmpl->event_id);
2565 	u32 data1 = le32_to_cpu(cmpl->event_data1);
2566 	u32 data2 = le32_to_cpu(cmpl->event_data2);
2567 
2568 	netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2569 		   event_id, data1, data2);
2570 
2571 	/* TODO CHIMP_FW: Define event id's for link change, error etc */
2572 	switch (event_id) {
2573 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2574 		struct bnxt_link_info *link_info = &bp->link_info;
2575 
2576 		if (BNXT_VF(bp))
2577 			goto async_event_process_exit;
2578 
2579 		/* print unsupported speed warning in forced speed mode only */
2580 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2581 		    (data1 & 0x20000)) {
2582 			u16 fw_speed = bnxt_get_force_speed(link_info);
2583 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2584 
2585 			if (speed != SPEED_UNKNOWN)
2586 				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2587 					    speed);
2588 		}
2589 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2590 	}
2591 		fallthrough;
2592 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2593 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2594 		set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2595 		fallthrough;
2596 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2597 		set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2598 		break;
2599 	case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2600 		set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2601 		break;
2602 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2603 		u16 port_id = BNXT_GET_EVENT_PORT(data1);
2604 
2605 		if (BNXT_VF(bp))
2606 			break;
2607 
2608 		if (bp->pf.port_id != port_id)
2609 			break;
2610 
2611 		set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2612 		break;
2613 	}
2614 	case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2615 		if (BNXT_PF(bp))
2616 			goto async_event_process_exit;
2617 		set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2618 		break;
2619 	case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2620 		char *type_str = "Solicited";
2621 
2622 		if (!bp->fw_health)
2623 			goto async_event_process_exit;
2624 
2625 		bp->fw_reset_timestamp = jiffies;
2626 		bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2627 		if (!bp->fw_reset_min_dsecs)
2628 			bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2629 		bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2630 		if (!bp->fw_reset_max_dsecs)
2631 			bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2632 		if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2633 			set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2634 		} else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2635 			type_str = "Fatal";
2636 			bp->fw_health->fatalities++;
2637 			set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2638 		} else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2639 			   EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2640 			type_str = "Non-fatal";
2641 			bp->fw_health->survivals++;
2642 			set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2643 		}
2644 		netif_warn(bp, hw, bp->dev,
2645 			   "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2646 			   type_str, data1, data2,
2647 			   bp->fw_reset_min_dsecs * 100,
2648 			   bp->fw_reset_max_dsecs * 100);
2649 		set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2650 		break;
2651 	}
2652 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2653 		struct bnxt_fw_health *fw_health = bp->fw_health;
2654 		char *status_desc = "healthy";
2655 		u32 status;
2656 
2657 		if (!fw_health)
2658 			goto async_event_process_exit;
2659 
2660 		if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2661 			fw_health->enabled = false;
2662 			netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2663 			break;
2664 		}
2665 		fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2666 		fw_health->tmr_multiplier =
2667 			DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2668 				     bp->current_interval * 10);
2669 		fw_health->tmr_counter = fw_health->tmr_multiplier;
2670 		if (!fw_health->enabled)
2671 			fw_health->last_fw_heartbeat =
2672 				bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2673 		fw_health->last_fw_reset_cnt =
2674 			bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2675 		status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2676 		if (status != BNXT_FW_STATUS_HEALTHY)
2677 			status_desc = "unhealthy";
2678 		netif_info(bp, drv, bp->dev,
2679 			   "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2680 			   fw_health->primary ? "primary" : "backup", status,
2681 			   status_desc, fw_health->last_fw_reset_cnt);
2682 		if (!fw_health->enabled) {
2683 			/* Make sure tmr_counter is set and visible to
2684 			 * bnxt_health_check() before setting enabled to true.
2685 			 */
2686 			smp_wmb();
2687 			fw_health->enabled = true;
2688 		}
2689 		goto async_event_process_exit;
2690 	}
2691 	case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2692 		netif_notice(bp, hw, bp->dev,
2693 			     "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2694 			     data1, data2);
2695 		goto async_event_process_exit;
2696 	case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2697 		struct bnxt_rx_ring_info *rxr;
2698 		u16 grp_idx;
2699 
2700 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
2701 			goto async_event_process_exit;
2702 
2703 		netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2704 			    BNXT_EVENT_RING_TYPE(data2), data1);
2705 		if (!BNXT_EVENT_RING_TYPE_RX(data2))
2706 			goto async_event_process_exit;
2707 
2708 		grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2709 		if (grp_idx == INVALID_HW_RING_ID) {
2710 			netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2711 				    data1);
2712 			goto async_event_process_exit;
2713 		}
2714 		rxr = bp->bnapi[grp_idx]->rx_ring;
2715 		bnxt_sched_reset_rxr(bp, rxr);
2716 		goto async_event_process_exit;
2717 	}
2718 	case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2719 		struct bnxt_fw_health *fw_health = bp->fw_health;
2720 
2721 		netif_notice(bp, hw, bp->dev,
2722 			     "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2723 			     data1, data2);
2724 		if (fw_health) {
2725 			fw_health->echo_req_data1 = data1;
2726 			fw_health->echo_req_data2 = data2;
2727 			set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2728 			break;
2729 		}
2730 		goto async_event_process_exit;
2731 	}
2732 	case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2733 		bnxt_ptp_pps_event(bp, data1, data2);
2734 		goto async_event_process_exit;
2735 	}
2736 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2737 		if (bnxt_event_error_report(bp, data1, data2))
2738 			break;
2739 		goto async_event_process_exit;
2740 	}
2741 	case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2742 		switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2743 		case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2744 			if (BNXT_PTP_USE_RTC(bp)) {
2745 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2746 				u64 ns;
2747 
2748 				if (!ptp)
2749 					goto async_event_process_exit;
2750 
2751 				spin_lock_bh(&ptp->ptp_lock);
2752 				bnxt_ptp_update_current_time(bp);
2753 				ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2754 				       BNXT_PHC_BITS) | ptp->current_time);
2755 				bnxt_ptp_rtc_timecounter_init(ptp, ns);
2756 				spin_unlock_bh(&ptp->ptp_lock);
2757 			}
2758 			break;
2759 		}
2760 		goto async_event_process_exit;
2761 	}
2762 	case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2763 		u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2764 
2765 		hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2766 		goto async_event_process_exit;
2767 	}
2768 	default:
2769 		goto async_event_process_exit;
2770 	}
2771 	__bnxt_queue_sp_work(bp);
2772 async_event_process_exit:
2773 	return 0;
2774 }
2775 
2776 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2777 {
2778 	u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2779 	struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2780 	struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2781 				(struct hwrm_fwd_req_cmpl *)txcmp;
2782 
2783 	switch (cmpl_type) {
2784 	case CMPL_BASE_TYPE_HWRM_DONE:
2785 		seq_id = le16_to_cpu(h_cmpl->sequence_id);
2786 		hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2787 		break;
2788 
2789 	case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2790 		vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2791 
2792 		if ((vf_id < bp->pf.first_vf_id) ||
2793 		    (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2794 			netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2795 				   vf_id);
2796 			return -EINVAL;
2797 		}
2798 
2799 		set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2800 		bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2801 		break;
2802 
2803 	case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2804 		bnxt_async_event_process(bp,
2805 					 (struct hwrm_async_event_cmpl *)txcmp);
2806 		break;
2807 
2808 	default:
2809 		break;
2810 	}
2811 
2812 	return 0;
2813 }
2814 
2815 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2816 {
2817 	struct bnxt_napi *bnapi = dev_instance;
2818 	struct bnxt *bp = bnapi->bp;
2819 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2820 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2821 
2822 	cpr->event_ctr++;
2823 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2824 	napi_schedule(&bnapi->napi);
2825 	return IRQ_HANDLED;
2826 }
2827 
2828 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2829 {
2830 	u32 raw_cons = cpr->cp_raw_cons;
2831 	u16 cons = RING_CMP(raw_cons);
2832 	struct tx_cmp *txcmp;
2833 
2834 	txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2835 
2836 	return TX_CMP_VALID(txcmp, raw_cons);
2837 }
2838 
2839 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2840 {
2841 	struct bnxt_napi *bnapi = dev_instance;
2842 	struct bnxt *bp = bnapi->bp;
2843 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2844 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2845 	u32 int_status;
2846 
2847 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2848 
2849 	if (!bnxt_has_work(bp, cpr)) {
2850 		int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2851 		/* return if erroneous interrupt */
2852 		if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2853 			return IRQ_NONE;
2854 	}
2855 
2856 	/* disable ring IRQ */
2857 	BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2858 
2859 	/* Return here if interrupt is shared and is disabled. */
2860 	if (unlikely(atomic_read(&bp->intr_sem) != 0))
2861 		return IRQ_HANDLED;
2862 
2863 	napi_schedule(&bnapi->napi);
2864 	return IRQ_HANDLED;
2865 }
2866 
2867 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2868 			    int budget)
2869 {
2870 	struct bnxt_napi *bnapi = cpr->bnapi;
2871 	u32 raw_cons = cpr->cp_raw_cons;
2872 	u32 cons;
2873 	int rx_pkts = 0;
2874 	u8 event = 0;
2875 	struct tx_cmp *txcmp;
2876 
2877 	cpr->has_more_work = 0;
2878 	cpr->had_work_done = 1;
2879 	while (1) {
2880 		u8 cmp_type;
2881 		int rc;
2882 
2883 		cons = RING_CMP(raw_cons);
2884 		txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2885 
2886 		if (!TX_CMP_VALID(txcmp, raw_cons))
2887 			break;
2888 
2889 		/* The valid test of the entry must be done first before
2890 		 * reading any further.
2891 		 */
2892 		dma_rmb();
2893 		cmp_type = TX_CMP_TYPE(txcmp);
2894 		if (cmp_type == CMP_TYPE_TX_L2_CMP ||
2895 		    cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
2896 			u32 opaque = txcmp->tx_cmp_opaque;
2897 			struct bnxt_tx_ring_info *txr;
2898 			u16 tx_freed;
2899 
2900 			txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
2901 			event |= BNXT_TX_CMP_EVENT;
2902 			if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
2903 				txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
2904 			else
2905 				txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
2906 			tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
2907 				   bp->tx_ring_mask;
2908 			/* return full budget so NAPI will complete. */
2909 			if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
2910 				rx_pkts = budget;
2911 				raw_cons = NEXT_RAW_CMP(raw_cons);
2912 				if (budget)
2913 					cpr->has_more_work = 1;
2914 				break;
2915 			}
2916 		} else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
2917 			   cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2918 			if (likely(budget))
2919 				rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2920 			else
2921 				rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2922 							   &event);
2923 			if (likely(rc >= 0))
2924 				rx_pkts += rc;
2925 			/* Increment rx_pkts when rc is -ENOMEM to count towards
2926 			 * the NAPI budget.  Otherwise, we may potentially loop
2927 			 * here forever if we consistently cannot allocate
2928 			 * buffers.
2929 			 */
2930 			else if (rc == -ENOMEM && budget)
2931 				rx_pkts++;
2932 			else if (rc == -EBUSY)	/* partial completion */
2933 				break;
2934 		} else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
2935 				    cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
2936 				    cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) {
2937 			bnxt_hwrm_handler(bp, txcmp);
2938 		}
2939 		raw_cons = NEXT_RAW_CMP(raw_cons);
2940 
2941 		if (rx_pkts && rx_pkts == budget) {
2942 			cpr->has_more_work = 1;
2943 			break;
2944 		}
2945 	}
2946 
2947 	if (event & BNXT_REDIRECT_EVENT)
2948 		xdp_do_flush();
2949 
2950 	if (event & BNXT_TX_EVENT) {
2951 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
2952 		u16 prod = txr->tx_prod;
2953 
2954 		/* Sync BD data before updating doorbell */
2955 		wmb();
2956 
2957 		bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2958 	}
2959 
2960 	cpr->cp_raw_cons = raw_cons;
2961 	bnapi->events |= event;
2962 	return rx_pkts;
2963 }
2964 
2965 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2966 				  int budget)
2967 {
2968 	if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
2969 		bnapi->tx_int(bp, bnapi, budget);
2970 
2971 	if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2972 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2973 
2974 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2975 	}
2976 	if (bnapi->events & BNXT_AGG_EVENT) {
2977 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2978 
2979 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2980 	}
2981 	bnapi->events &= BNXT_TX_CMP_EVENT;
2982 }
2983 
2984 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2985 			  int budget)
2986 {
2987 	struct bnxt_napi *bnapi = cpr->bnapi;
2988 	int rx_pkts;
2989 
2990 	rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2991 
2992 	/* ACK completion ring before freeing tx ring and producing new
2993 	 * buffers in rx/agg rings to prevent overflowing the completion
2994 	 * ring.
2995 	 */
2996 	bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2997 
2998 	__bnxt_poll_work_done(bp, bnapi, budget);
2999 	return rx_pkts;
3000 }
3001 
3002 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
3003 {
3004 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3005 	struct bnxt *bp = bnapi->bp;
3006 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3007 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
3008 	struct tx_cmp *txcmp;
3009 	struct rx_cmp_ext *rxcmp1;
3010 	u32 cp_cons, tmp_raw_cons;
3011 	u32 raw_cons = cpr->cp_raw_cons;
3012 	bool flush_xdp = false;
3013 	u32 rx_pkts = 0;
3014 	u8 event = 0;
3015 
3016 	while (1) {
3017 		int rc;
3018 
3019 		cp_cons = RING_CMP(raw_cons);
3020 		txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3021 
3022 		if (!TX_CMP_VALID(txcmp, raw_cons))
3023 			break;
3024 
3025 		/* The valid test of the entry must be done first before
3026 		 * reading any further.
3027 		 */
3028 		dma_rmb();
3029 		if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
3030 			tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
3031 			cp_cons = RING_CMP(tmp_raw_cons);
3032 			rxcmp1 = (struct rx_cmp_ext *)
3033 			  &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3034 
3035 			if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
3036 				break;
3037 
3038 			/* force an error to recycle the buffer */
3039 			rxcmp1->rx_cmp_cfa_code_errors_v2 |=
3040 				cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
3041 
3042 			rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
3043 			if (likely(rc == -EIO) && budget)
3044 				rx_pkts++;
3045 			else if (rc == -EBUSY)	/* partial completion */
3046 				break;
3047 			if (event & BNXT_REDIRECT_EVENT)
3048 				flush_xdp = true;
3049 		} else if (unlikely(TX_CMP_TYPE(txcmp) ==
3050 				    CMPL_BASE_TYPE_HWRM_DONE)) {
3051 			bnxt_hwrm_handler(bp, txcmp);
3052 		} else {
3053 			netdev_err(bp->dev,
3054 				   "Invalid completion received on special ring\n");
3055 		}
3056 		raw_cons = NEXT_RAW_CMP(raw_cons);
3057 
3058 		if (rx_pkts == budget)
3059 			break;
3060 	}
3061 
3062 	cpr->cp_raw_cons = raw_cons;
3063 	BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
3064 	bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
3065 
3066 	if (event & BNXT_AGG_EVENT)
3067 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
3068 	if (flush_xdp)
3069 		xdp_do_flush();
3070 
3071 	if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
3072 		napi_complete_done(napi, rx_pkts);
3073 		BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3074 	}
3075 	return rx_pkts;
3076 }
3077 
3078 static int bnxt_poll(struct napi_struct *napi, int budget)
3079 {
3080 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3081 	struct bnxt *bp = bnapi->bp;
3082 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3083 	int work_done = 0;
3084 
3085 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3086 		napi_complete(napi);
3087 		return 0;
3088 	}
3089 	while (1) {
3090 		work_done += bnxt_poll_work(bp, cpr, budget - work_done);
3091 
3092 		if (work_done >= budget) {
3093 			if (!budget)
3094 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3095 			break;
3096 		}
3097 
3098 		if (!bnxt_has_work(bp, cpr)) {
3099 			if (napi_complete_done(napi, work_done))
3100 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3101 			break;
3102 		}
3103 	}
3104 	if (bp->flags & BNXT_FLAG_DIM) {
3105 		struct dim_sample dim_sample = {};
3106 
3107 		dim_update_sample(cpr->event_ctr,
3108 				  cpr->rx_packets,
3109 				  cpr->rx_bytes,
3110 				  &dim_sample);
3111 		net_dim(&cpr->dim, dim_sample);
3112 	}
3113 	return work_done;
3114 }
3115 
3116 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
3117 {
3118 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3119 	int i, work_done = 0;
3120 
3121 	for (i = 0; i < cpr->cp_ring_count; i++) {
3122 		struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3123 
3124 		if (cpr2->had_nqe_notify) {
3125 			work_done += __bnxt_poll_work(bp, cpr2,
3126 						      budget - work_done);
3127 			cpr->has_more_work |= cpr2->has_more_work;
3128 		}
3129 	}
3130 	return work_done;
3131 }
3132 
3133 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3134 				 u64 dbr_type, int budget)
3135 {
3136 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3137 	int i;
3138 
3139 	for (i = 0; i < cpr->cp_ring_count; i++) {
3140 		struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3141 		struct bnxt_db_info *db;
3142 
3143 		if (cpr2->had_work_done) {
3144 			u32 tgl = 0;
3145 
3146 			if (dbr_type == DBR_TYPE_CQ_ARMALL) {
3147 				cpr2->had_nqe_notify = 0;
3148 				tgl = cpr2->toggle;
3149 			}
3150 			db = &cpr2->cp_db;
3151 			bnxt_writeq(bp,
3152 				    db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
3153 				    DB_RING_IDX(db, cpr2->cp_raw_cons),
3154 				    db->doorbell);
3155 			cpr2->had_work_done = 0;
3156 		}
3157 	}
3158 	__bnxt_poll_work_done(bp, bnapi, budget);
3159 }
3160 
3161 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
3162 {
3163 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3164 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3165 	struct bnxt_cp_ring_info *cpr_rx;
3166 	u32 raw_cons = cpr->cp_raw_cons;
3167 	struct bnxt *bp = bnapi->bp;
3168 	struct nqe_cn *nqcmp;
3169 	int work_done = 0;
3170 	u32 cons;
3171 
3172 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3173 		napi_complete(napi);
3174 		return 0;
3175 	}
3176 	if (cpr->has_more_work) {
3177 		cpr->has_more_work = 0;
3178 		work_done = __bnxt_poll_cqs(bp, bnapi, budget);
3179 	}
3180 	while (1) {
3181 		u16 type;
3182 
3183 		cons = RING_CMP(raw_cons);
3184 		nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
3185 
3186 		if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
3187 			if (cpr->has_more_work)
3188 				break;
3189 
3190 			__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
3191 					     budget);
3192 			cpr->cp_raw_cons = raw_cons;
3193 			if (napi_complete_done(napi, work_done))
3194 				BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
3195 						  cpr->cp_raw_cons);
3196 			goto poll_done;
3197 		}
3198 
3199 		/* The valid test of the entry must be done first before
3200 		 * reading any further.
3201 		 */
3202 		dma_rmb();
3203 
3204 		type = le16_to_cpu(nqcmp->type);
3205 		if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
3206 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
3207 			u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
3208 			struct bnxt_cp_ring_info *cpr2;
3209 
3210 			/* No more budget for RX work */
3211 			if (budget && work_done >= budget &&
3212 			    cq_type == BNXT_NQ_HDL_TYPE_RX)
3213 				break;
3214 
3215 			idx = BNXT_NQ_HDL_IDX(idx);
3216 			cpr2 = &cpr->cp_ring_arr[idx];
3217 			cpr2->had_nqe_notify = 1;
3218 			cpr2->toggle = NQE_CN_TOGGLE(type);
3219 			work_done += __bnxt_poll_work(bp, cpr2,
3220 						      budget - work_done);
3221 			cpr->has_more_work |= cpr2->has_more_work;
3222 		} else {
3223 			bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
3224 		}
3225 		raw_cons = NEXT_RAW_CMP(raw_cons);
3226 	}
3227 	__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
3228 	if (raw_cons != cpr->cp_raw_cons) {
3229 		cpr->cp_raw_cons = raw_cons;
3230 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
3231 	}
3232 poll_done:
3233 	cpr_rx = &cpr->cp_ring_arr[0];
3234 	if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
3235 	    (bp->flags & BNXT_FLAG_DIM)) {
3236 		struct dim_sample dim_sample = {};
3237 
3238 		dim_update_sample(cpr->event_ctr,
3239 				  cpr_rx->rx_packets,
3240 				  cpr_rx->rx_bytes,
3241 				  &dim_sample);
3242 		net_dim(&cpr->dim, dim_sample);
3243 	}
3244 	return work_done;
3245 }
3246 
3247 static void bnxt_free_tx_skbs(struct bnxt *bp)
3248 {
3249 	int i, max_idx;
3250 	struct pci_dev *pdev = bp->pdev;
3251 
3252 	if (!bp->tx_ring)
3253 		return;
3254 
3255 	max_idx = bp->tx_nr_pages * TX_DESC_CNT;
3256 	for (i = 0; i < bp->tx_nr_rings; i++) {
3257 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3258 		int j;
3259 
3260 		if (!txr->tx_buf_ring)
3261 			continue;
3262 
3263 		for (j = 0; j < max_idx;) {
3264 			struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
3265 			struct sk_buff *skb;
3266 			int k, last;
3267 
3268 			if (i < bp->tx_nr_rings_xdp &&
3269 			    tx_buf->action == XDP_REDIRECT) {
3270 				dma_unmap_single(&pdev->dev,
3271 					dma_unmap_addr(tx_buf, mapping),
3272 					dma_unmap_len(tx_buf, len),
3273 					DMA_TO_DEVICE);
3274 				xdp_return_frame(tx_buf->xdpf);
3275 				tx_buf->action = 0;
3276 				tx_buf->xdpf = NULL;
3277 				j++;
3278 				continue;
3279 			}
3280 
3281 			skb = tx_buf->skb;
3282 			if (!skb) {
3283 				j++;
3284 				continue;
3285 			}
3286 
3287 			tx_buf->skb = NULL;
3288 
3289 			if (tx_buf->is_push) {
3290 				dev_kfree_skb(skb);
3291 				j += 2;
3292 				continue;
3293 			}
3294 
3295 			dma_unmap_single(&pdev->dev,
3296 					 dma_unmap_addr(tx_buf, mapping),
3297 					 skb_headlen(skb),
3298 					 DMA_TO_DEVICE);
3299 
3300 			last = tx_buf->nr_frags;
3301 			j += 2;
3302 			for (k = 0; k < last; k++, j++) {
3303 				int ring_idx = j & bp->tx_ring_mask;
3304 				skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3305 
3306 				tx_buf = &txr->tx_buf_ring[ring_idx];
3307 				dma_unmap_page(
3308 					&pdev->dev,
3309 					dma_unmap_addr(tx_buf, mapping),
3310 					skb_frag_size(frag), DMA_TO_DEVICE);
3311 			}
3312 			dev_kfree_skb(skb);
3313 		}
3314 		netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
3315 	}
3316 }
3317 
3318 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
3319 {
3320 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3321 	struct pci_dev *pdev = bp->pdev;
3322 	struct bnxt_tpa_idx_map *map;
3323 	int i, max_idx, max_agg_idx;
3324 
3325 	max_idx = bp->rx_nr_pages * RX_DESC_CNT;
3326 	max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
3327 	if (!rxr->rx_tpa)
3328 		goto skip_rx_tpa_free;
3329 
3330 	for (i = 0; i < bp->max_tpa; i++) {
3331 		struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
3332 		u8 *data = tpa_info->data;
3333 
3334 		if (!data)
3335 			continue;
3336 
3337 		dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
3338 				       bp->rx_buf_use_size, bp->rx_dir,
3339 				       DMA_ATTR_WEAK_ORDERING);
3340 
3341 		tpa_info->data = NULL;
3342 
3343 		skb_free_frag(data);
3344 	}
3345 
3346 skip_rx_tpa_free:
3347 	if (!rxr->rx_buf_ring)
3348 		goto skip_rx_buf_free;
3349 
3350 	for (i = 0; i < max_idx; i++) {
3351 		struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
3352 		dma_addr_t mapping = rx_buf->mapping;
3353 		void *data = rx_buf->data;
3354 
3355 		if (!data)
3356 			continue;
3357 
3358 		rx_buf->data = NULL;
3359 		if (BNXT_RX_PAGE_MODE(bp)) {
3360 			page_pool_recycle_direct(rxr->page_pool, data);
3361 		} else {
3362 			dma_unmap_single_attrs(&pdev->dev, mapping,
3363 					       bp->rx_buf_use_size, bp->rx_dir,
3364 					       DMA_ATTR_WEAK_ORDERING);
3365 			skb_free_frag(data);
3366 		}
3367 	}
3368 
3369 skip_rx_buf_free:
3370 	if (!rxr->rx_agg_ring)
3371 		goto skip_rx_agg_free;
3372 
3373 	for (i = 0; i < max_agg_idx; i++) {
3374 		struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
3375 		struct page *page = rx_agg_buf->page;
3376 
3377 		if (!page)
3378 			continue;
3379 
3380 		rx_agg_buf->page = NULL;
3381 		__clear_bit(i, rxr->rx_agg_bmap);
3382 
3383 		page_pool_recycle_direct(rxr->page_pool, page);
3384 	}
3385 
3386 skip_rx_agg_free:
3387 	map = rxr->rx_tpa_idx_map;
3388 	if (map)
3389 		memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3390 }
3391 
3392 static void bnxt_free_rx_skbs(struct bnxt *bp)
3393 {
3394 	int i;
3395 
3396 	if (!bp->rx_ring)
3397 		return;
3398 
3399 	for (i = 0; i < bp->rx_nr_rings; i++)
3400 		bnxt_free_one_rx_ring_skbs(bp, i);
3401 }
3402 
3403 static void bnxt_free_skbs(struct bnxt *bp)
3404 {
3405 	bnxt_free_tx_skbs(bp);
3406 	bnxt_free_rx_skbs(bp);
3407 }
3408 
3409 static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len)
3410 {
3411 	u8 init_val = ctxm->init_value;
3412 	u16 offset = ctxm->init_offset;
3413 	u8 *p2 = p;
3414 	int i;
3415 
3416 	if (!init_val)
3417 		return;
3418 	if (offset == BNXT_CTX_INIT_INVALID_OFFSET) {
3419 		memset(p, init_val, len);
3420 		return;
3421 	}
3422 	for (i = 0; i < len; i += ctxm->entry_size)
3423 		*(p2 + i + offset) = init_val;
3424 }
3425 
3426 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3427 {
3428 	struct pci_dev *pdev = bp->pdev;
3429 	int i;
3430 
3431 	if (!rmem->pg_arr)
3432 		goto skip_pages;
3433 
3434 	for (i = 0; i < rmem->nr_pages; i++) {
3435 		if (!rmem->pg_arr[i])
3436 			continue;
3437 
3438 		dma_free_coherent(&pdev->dev, rmem->page_size,
3439 				  rmem->pg_arr[i], rmem->dma_arr[i]);
3440 
3441 		rmem->pg_arr[i] = NULL;
3442 	}
3443 skip_pages:
3444 	if (rmem->pg_tbl) {
3445 		size_t pg_tbl_size = rmem->nr_pages * 8;
3446 
3447 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3448 			pg_tbl_size = rmem->page_size;
3449 		dma_free_coherent(&pdev->dev, pg_tbl_size,
3450 				  rmem->pg_tbl, rmem->pg_tbl_map);
3451 		rmem->pg_tbl = NULL;
3452 	}
3453 	if (rmem->vmem_size && *rmem->vmem) {
3454 		vfree(*rmem->vmem);
3455 		*rmem->vmem = NULL;
3456 	}
3457 }
3458 
3459 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3460 {
3461 	struct pci_dev *pdev = bp->pdev;
3462 	u64 valid_bit = 0;
3463 	int i;
3464 
3465 	if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3466 		valid_bit = PTU_PTE_VALID;
3467 	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3468 		size_t pg_tbl_size = rmem->nr_pages * 8;
3469 
3470 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3471 			pg_tbl_size = rmem->page_size;
3472 		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3473 						  &rmem->pg_tbl_map,
3474 						  GFP_KERNEL);
3475 		if (!rmem->pg_tbl)
3476 			return -ENOMEM;
3477 	}
3478 
3479 	for (i = 0; i < rmem->nr_pages; i++) {
3480 		u64 extra_bits = valid_bit;
3481 
3482 		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3483 						     rmem->page_size,
3484 						     &rmem->dma_arr[i],
3485 						     GFP_KERNEL);
3486 		if (!rmem->pg_arr[i])
3487 			return -ENOMEM;
3488 
3489 		if (rmem->ctx_mem)
3490 			bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
3491 					  rmem->page_size);
3492 		if (rmem->nr_pages > 1 || rmem->depth > 0) {
3493 			if (i == rmem->nr_pages - 2 &&
3494 			    (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3495 				extra_bits |= PTU_PTE_NEXT_TO_LAST;
3496 			else if (i == rmem->nr_pages - 1 &&
3497 				 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3498 				extra_bits |= PTU_PTE_LAST;
3499 			rmem->pg_tbl[i] =
3500 				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3501 		}
3502 	}
3503 
3504 	if (rmem->vmem_size) {
3505 		*rmem->vmem = vzalloc(rmem->vmem_size);
3506 		if (!(*rmem->vmem))
3507 			return -ENOMEM;
3508 	}
3509 	return 0;
3510 }
3511 
3512 static void bnxt_free_tpa_info(struct bnxt *bp)
3513 {
3514 	int i, j;
3515 
3516 	for (i = 0; i < bp->rx_nr_rings; i++) {
3517 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3518 
3519 		kfree(rxr->rx_tpa_idx_map);
3520 		rxr->rx_tpa_idx_map = NULL;
3521 		if (rxr->rx_tpa) {
3522 			for (j = 0; j < bp->max_tpa; j++) {
3523 				kfree(rxr->rx_tpa[j].agg_arr);
3524 				rxr->rx_tpa[j].agg_arr = NULL;
3525 			}
3526 		}
3527 		kfree(rxr->rx_tpa);
3528 		rxr->rx_tpa = NULL;
3529 	}
3530 }
3531 
3532 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3533 {
3534 	int i, j;
3535 
3536 	bp->max_tpa = MAX_TPA;
3537 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
3538 		if (!bp->max_tpa_v2)
3539 			return 0;
3540 		bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3541 	}
3542 
3543 	for (i = 0; i < bp->rx_nr_rings; i++) {
3544 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3545 		struct rx_agg_cmp *agg;
3546 
3547 		rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3548 				      GFP_KERNEL);
3549 		if (!rxr->rx_tpa)
3550 			return -ENOMEM;
3551 
3552 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3553 			continue;
3554 		for (j = 0; j < bp->max_tpa; j++) {
3555 			agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3556 			if (!agg)
3557 				return -ENOMEM;
3558 			rxr->rx_tpa[j].agg_arr = agg;
3559 		}
3560 		rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3561 					      GFP_KERNEL);
3562 		if (!rxr->rx_tpa_idx_map)
3563 			return -ENOMEM;
3564 	}
3565 	return 0;
3566 }
3567 
3568 static void bnxt_free_rx_rings(struct bnxt *bp)
3569 {
3570 	int i;
3571 
3572 	if (!bp->rx_ring)
3573 		return;
3574 
3575 	bnxt_free_tpa_info(bp);
3576 	for (i = 0; i < bp->rx_nr_rings; i++) {
3577 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3578 		struct bnxt_ring_struct *ring;
3579 
3580 		if (rxr->xdp_prog)
3581 			bpf_prog_put(rxr->xdp_prog);
3582 
3583 		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3584 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3585 
3586 		page_pool_destroy(rxr->page_pool);
3587 		rxr->page_pool = NULL;
3588 
3589 		kfree(rxr->rx_agg_bmap);
3590 		rxr->rx_agg_bmap = NULL;
3591 
3592 		ring = &rxr->rx_ring_struct;
3593 		bnxt_free_ring(bp, &ring->ring_mem);
3594 
3595 		ring = &rxr->rx_agg_ring_struct;
3596 		bnxt_free_ring(bp, &ring->ring_mem);
3597 	}
3598 }
3599 
3600 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3601 				   struct bnxt_rx_ring_info *rxr,
3602 				   int numa_node)
3603 {
3604 	struct page_pool_params pp = { 0 };
3605 
3606 	pp.pool_size = bp->rx_agg_ring_size;
3607 	if (BNXT_RX_PAGE_MODE(bp))
3608 		pp.pool_size += bp->rx_ring_size;
3609 	pp.nid = numa_node;
3610 	pp.napi = &rxr->bnapi->napi;
3611 	pp.netdev = bp->dev;
3612 	pp.dev = &bp->pdev->dev;
3613 	pp.dma_dir = bp->rx_dir;
3614 	pp.max_len = PAGE_SIZE;
3615 	pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3616 
3617 	rxr->page_pool = page_pool_create(&pp);
3618 	if (IS_ERR(rxr->page_pool)) {
3619 		int err = PTR_ERR(rxr->page_pool);
3620 
3621 		rxr->page_pool = NULL;
3622 		return err;
3623 	}
3624 	return 0;
3625 }
3626 
3627 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3628 {
3629 	int numa_node = dev_to_node(&bp->pdev->dev);
3630 	int i, rc = 0, agg_rings = 0, cpu;
3631 
3632 	if (!bp->rx_ring)
3633 		return -ENOMEM;
3634 
3635 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
3636 		agg_rings = 1;
3637 
3638 	for (i = 0; i < bp->rx_nr_rings; i++) {
3639 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3640 		struct bnxt_ring_struct *ring;
3641 		int cpu_node;
3642 
3643 		ring = &rxr->rx_ring_struct;
3644 
3645 		cpu = cpumask_local_spread(i, numa_node);
3646 		cpu_node = cpu_to_node(cpu);
3647 		netdev_dbg(bp->dev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n",
3648 			   i, cpu_node);
3649 		rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node);
3650 		if (rc)
3651 			return rc;
3652 
3653 		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3654 		if (rc < 0)
3655 			return rc;
3656 
3657 		rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3658 						MEM_TYPE_PAGE_POOL,
3659 						rxr->page_pool);
3660 		if (rc) {
3661 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3662 			return rc;
3663 		}
3664 
3665 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3666 		if (rc)
3667 			return rc;
3668 
3669 		ring->grp_idx = i;
3670 		if (agg_rings) {
3671 			u16 mem_size;
3672 
3673 			ring = &rxr->rx_agg_ring_struct;
3674 			rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3675 			if (rc)
3676 				return rc;
3677 
3678 			ring->grp_idx = i;
3679 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3680 			mem_size = rxr->rx_agg_bmap_size / 8;
3681 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3682 			if (!rxr->rx_agg_bmap)
3683 				return -ENOMEM;
3684 		}
3685 	}
3686 	if (bp->flags & BNXT_FLAG_TPA)
3687 		rc = bnxt_alloc_tpa_info(bp);
3688 	return rc;
3689 }
3690 
3691 static void bnxt_free_tx_rings(struct bnxt *bp)
3692 {
3693 	int i;
3694 	struct pci_dev *pdev = bp->pdev;
3695 
3696 	if (!bp->tx_ring)
3697 		return;
3698 
3699 	for (i = 0; i < bp->tx_nr_rings; i++) {
3700 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3701 		struct bnxt_ring_struct *ring;
3702 
3703 		if (txr->tx_push) {
3704 			dma_free_coherent(&pdev->dev, bp->tx_push_size,
3705 					  txr->tx_push, txr->tx_push_mapping);
3706 			txr->tx_push = NULL;
3707 		}
3708 
3709 		ring = &txr->tx_ring_struct;
3710 
3711 		bnxt_free_ring(bp, &ring->ring_mem);
3712 	}
3713 }
3714 
3715 #define BNXT_TC_TO_RING_BASE(bp, tc)	\
3716 	((tc) * (bp)->tx_nr_rings_per_tc)
3717 
3718 #define BNXT_RING_TO_TC_OFF(bp, tx)	\
3719 	((tx) % (bp)->tx_nr_rings_per_tc)
3720 
3721 #define BNXT_RING_TO_TC(bp, tx)		\
3722 	((tx) / (bp)->tx_nr_rings_per_tc)
3723 
3724 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3725 {
3726 	int i, j, rc;
3727 	struct pci_dev *pdev = bp->pdev;
3728 
3729 	bp->tx_push_size = 0;
3730 	if (bp->tx_push_thresh) {
3731 		int push_size;
3732 
3733 		push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3734 					bp->tx_push_thresh);
3735 
3736 		if (push_size > 256) {
3737 			push_size = 0;
3738 			bp->tx_push_thresh = 0;
3739 		}
3740 
3741 		bp->tx_push_size = push_size;
3742 	}
3743 
3744 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3745 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3746 		struct bnxt_ring_struct *ring;
3747 		u8 qidx;
3748 
3749 		ring = &txr->tx_ring_struct;
3750 
3751 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3752 		if (rc)
3753 			return rc;
3754 
3755 		ring->grp_idx = txr->bnapi->index;
3756 		if (bp->tx_push_size) {
3757 			dma_addr_t mapping;
3758 
3759 			/* One pre-allocated DMA buffer to backup
3760 			 * TX push operation
3761 			 */
3762 			txr->tx_push = dma_alloc_coherent(&pdev->dev,
3763 						bp->tx_push_size,
3764 						&txr->tx_push_mapping,
3765 						GFP_KERNEL);
3766 
3767 			if (!txr->tx_push)
3768 				return -ENOMEM;
3769 
3770 			mapping = txr->tx_push_mapping +
3771 				sizeof(struct tx_push_bd);
3772 			txr->data_mapping = cpu_to_le64(mapping);
3773 		}
3774 		qidx = bp->tc_to_qidx[j];
3775 		ring->queue_id = bp->q_info[qidx].queue_id;
3776 		spin_lock_init(&txr->xdp_tx_lock);
3777 		if (i < bp->tx_nr_rings_xdp)
3778 			continue;
3779 		if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
3780 			j++;
3781 	}
3782 	return 0;
3783 }
3784 
3785 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3786 {
3787 	struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3788 
3789 	kfree(cpr->cp_desc_ring);
3790 	cpr->cp_desc_ring = NULL;
3791 	ring->ring_mem.pg_arr = NULL;
3792 	kfree(cpr->cp_desc_mapping);
3793 	cpr->cp_desc_mapping = NULL;
3794 	ring->ring_mem.dma_arr = NULL;
3795 }
3796 
3797 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3798 {
3799 	cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3800 	if (!cpr->cp_desc_ring)
3801 		return -ENOMEM;
3802 	cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3803 				       GFP_KERNEL);
3804 	if (!cpr->cp_desc_mapping)
3805 		return -ENOMEM;
3806 	return 0;
3807 }
3808 
3809 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3810 {
3811 	int i;
3812 
3813 	if (!bp->bnapi)
3814 		return;
3815 	for (i = 0; i < bp->cp_nr_rings; i++) {
3816 		struct bnxt_napi *bnapi = bp->bnapi[i];
3817 
3818 		if (!bnapi)
3819 			continue;
3820 		bnxt_free_cp_arrays(&bnapi->cp_ring);
3821 	}
3822 }
3823 
3824 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3825 {
3826 	int i, n = bp->cp_nr_pages;
3827 
3828 	for (i = 0; i < bp->cp_nr_rings; i++) {
3829 		struct bnxt_napi *bnapi = bp->bnapi[i];
3830 		int rc;
3831 
3832 		if (!bnapi)
3833 			continue;
3834 		rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3835 		if (rc)
3836 			return rc;
3837 	}
3838 	return 0;
3839 }
3840 
3841 static void bnxt_free_cp_rings(struct bnxt *bp)
3842 {
3843 	int i;
3844 
3845 	if (!bp->bnapi)
3846 		return;
3847 
3848 	for (i = 0; i < bp->cp_nr_rings; i++) {
3849 		struct bnxt_napi *bnapi = bp->bnapi[i];
3850 		struct bnxt_cp_ring_info *cpr;
3851 		struct bnxt_ring_struct *ring;
3852 		int j;
3853 
3854 		if (!bnapi)
3855 			continue;
3856 
3857 		cpr = &bnapi->cp_ring;
3858 		ring = &cpr->cp_ring_struct;
3859 
3860 		bnxt_free_ring(bp, &ring->ring_mem);
3861 
3862 		if (!cpr->cp_ring_arr)
3863 			continue;
3864 
3865 		for (j = 0; j < cpr->cp_ring_count; j++) {
3866 			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
3867 
3868 			ring = &cpr2->cp_ring_struct;
3869 			bnxt_free_ring(bp, &ring->ring_mem);
3870 			bnxt_free_cp_arrays(cpr2);
3871 		}
3872 		kfree(cpr->cp_ring_arr);
3873 		cpr->cp_ring_arr = NULL;
3874 		cpr->cp_ring_count = 0;
3875 	}
3876 }
3877 
3878 static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
3879 				  struct bnxt_cp_ring_info *cpr)
3880 {
3881 	struct bnxt_ring_mem_info *rmem;
3882 	struct bnxt_ring_struct *ring;
3883 	int rc;
3884 
3885 	rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3886 	if (rc) {
3887 		bnxt_free_cp_arrays(cpr);
3888 		return -ENOMEM;
3889 	}
3890 	ring = &cpr->cp_ring_struct;
3891 	rmem = &ring->ring_mem;
3892 	rmem->nr_pages = bp->cp_nr_pages;
3893 	rmem->page_size = HW_CMPD_RING_SIZE;
3894 	rmem->pg_arr = (void **)cpr->cp_desc_ring;
3895 	rmem->dma_arr = cpr->cp_desc_mapping;
3896 	rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3897 	rc = bnxt_alloc_ring(bp, rmem);
3898 	if (rc) {
3899 		bnxt_free_ring(bp, rmem);
3900 		bnxt_free_cp_arrays(cpr);
3901 	}
3902 	return rc;
3903 }
3904 
3905 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3906 {
3907 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3908 	int i, j, rc, ulp_msix;
3909 	int tcs = bp->num_tc;
3910 
3911 	if (!tcs)
3912 		tcs = 1;
3913 	ulp_msix = bnxt_get_ulp_msix_num(bp);
3914 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
3915 		struct bnxt_napi *bnapi = bp->bnapi[i];
3916 		struct bnxt_cp_ring_info *cpr, *cpr2;
3917 		struct bnxt_ring_struct *ring;
3918 		int cp_count = 0, k;
3919 		int rx = 0, tx = 0;
3920 
3921 		if (!bnapi)
3922 			continue;
3923 
3924 		cpr = &bnapi->cp_ring;
3925 		cpr->bnapi = bnapi;
3926 		ring = &cpr->cp_ring_struct;
3927 
3928 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3929 		if (rc)
3930 			return rc;
3931 
3932 		ring->map_idx = ulp_msix + i;
3933 
3934 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3935 			continue;
3936 
3937 		if (i < bp->rx_nr_rings) {
3938 			cp_count++;
3939 			rx = 1;
3940 		}
3941 		if (i < bp->tx_nr_rings_xdp) {
3942 			cp_count++;
3943 			tx = 1;
3944 		} else if ((sh && i < bp->tx_nr_rings) ||
3945 			 (!sh && i >= bp->rx_nr_rings)) {
3946 			cp_count += tcs;
3947 			tx = 1;
3948 		}
3949 
3950 		cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
3951 					   GFP_KERNEL);
3952 		if (!cpr->cp_ring_arr)
3953 			return -ENOMEM;
3954 		cpr->cp_ring_count = cp_count;
3955 
3956 		for (k = 0; k < cp_count; k++) {
3957 			cpr2 = &cpr->cp_ring_arr[k];
3958 			rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
3959 			if (rc)
3960 				return rc;
3961 			cpr2->bnapi = bnapi;
3962 			cpr2->cp_idx = k;
3963 			if (!k && rx) {
3964 				bp->rx_ring[i].rx_cpr = cpr2;
3965 				cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
3966 			} else {
3967 				int n, tc = k - rx;
3968 
3969 				n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
3970 				bp->tx_ring[n].tx_cpr = cpr2;
3971 				cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
3972 			}
3973 		}
3974 		if (tx)
3975 			j++;
3976 	}
3977 	return 0;
3978 }
3979 
3980 static void bnxt_init_ring_struct(struct bnxt *bp)
3981 {
3982 	int i, j;
3983 
3984 	for (i = 0; i < bp->cp_nr_rings; i++) {
3985 		struct bnxt_napi *bnapi = bp->bnapi[i];
3986 		struct bnxt_ring_mem_info *rmem;
3987 		struct bnxt_cp_ring_info *cpr;
3988 		struct bnxt_rx_ring_info *rxr;
3989 		struct bnxt_tx_ring_info *txr;
3990 		struct bnxt_ring_struct *ring;
3991 
3992 		if (!bnapi)
3993 			continue;
3994 
3995 		cpr = &bnapi->cp_ring;
3996 		ring = &cpr->cp_ring_struct;
3997 		rmem = &ring->ring_mem;
3998 		rmem->nr_pages = bp->cp_nr_pages;
3999 		rmem->page_size = HW_CMPD_RING_SIZE;
4000 		rmem->pg_arr = (void **)cpr->cp_desc_ring;
4001 		rmem->dma_arr = cpr->cp_desc_mapping;
4002 		rmem->vmem_size = 0;
4003 
4004 		rxr = bnapi->rx_ring;
4005 		if (!rxr)
4006 			goto skip_rx;
4007 
4008 		ring = &rxr->rx_ring_struct;
4009 		rmem = &ring->ring_mem;
4010 		rmem->nr_pages = bp->rx_nr_pages;
4011 		rmem->page_size = HW_RXBD_RING_SIZE;
4012 		rmem->pg_arr = (void **)rxr->rx_desc_ring;
4013 		rmem->dma_arr = rxr->rx_desc_mapping;
4014 		rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
4015 		rmem->vmem = (void **)&rxr->rx_buf_ring;
4016 
4017 		ring = &rxr->rx_agg_ring_struct;
4018 		rmem = &ring->ring_mem;
4019 		rmem->nr_pages = bp->rx_agg_nr_pages;
4020 		rmem->page_size = HW_RXBD_RING_SIZE;
4021 		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
4022 		rmem->dma_arr = rxr->rx_agg_desc_mapping;
4023 		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
4024 		rmem->vmem = (void **)&rxr->rx_agg_ring;
4025 
4026 skip_rx:
4027 		bnxt_for_each_napi_tx(j, bnapi, txr) {
4028 			ring = &txr->tx_ring_struct;
4029 			rmem = &ring->ring_mem;
4030 			rmem->nr_pages = bp->tx_nr_pages;
4031 			rmem->page_size = HW_TXBD_RING_SIZE;
4032 			rmem->pg_arr = (void **)txr->tx_desc_ring;
4033 			rmem->dma_arr = txr->tx_desc_mapping;
4034 			rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
4035 			rmem->vmem = (void **)&txr->tx_buf_ring;
4036 		}
4037 	}
4038 }
4039 
4040 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
4041 {
4042 	int i;
4043 	u32 prod;
4044 	struct rx_bd **rx_buf_ring;
4045 
4046 	rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
4047 	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
4048 		int j;
4049 		struct rx_bd *rxbd;
4050 
4051 		rxbd = rx_buf_ring[i];
4052 		if (!rxbd)
4053 			continue;
4054 
4055 		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
4056 			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
4057 			rxbd->rx_bd_opaque = prod;
4058 		}
4059 	}
4060 }
4061 
4062 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
4063 {
4064 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
4065 	struct net_device *dev = bp->dev;
4066 	u32 prod;
4067 	int i;
4068 
4069 	prod = rxr->rx_prod;
4070 	for (i = 0; i < bp->rx_ring_size; i++) {
4071 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
4072 			netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
4073 				    ring_nr, i, bp->rx_ring_size);
4074 			break;
4075 		}
4076 		prod = NEXT_RX(prod);
4077 	}
4078 	rxr->rx_prod = prod;
4079 
4080 	if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
4081 		return 0;
4082 
4083 	prod = rxr->rx_agg_prod;
4084 	for (i = 0; i < bp->rx_agg_ring_size; i++) {
4085 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
4086 			netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
4087 				    ring_nr, i, bp->rx_ring_size);
4088 			break;
4089 		}
4090 		prod = NEXT_RX_AGG(prod);
4091 	}
4092 	rxr->rx_agg_prod = prod;
4093 
4094 	if (rxr->rx_tpa) {
4095 		dma_addr_t mapping;
4096 		u8 *data;
4097 
4098 		for (i = 0; i < bp->max_tpa; i++) {
4099 			data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
4100 			if (!data)
4101 				return -ENOMEM;
4102 
4103 			rxr->rx_tpa[i].data = data;
4104 			rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
4105 			rxr->rx_tpa[i].mapping = mapping;
4106 		}
4107 	}
4108 	return 0;
4109 }
4110 
4111 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
4112 {
4113 	struct bnxt_rx_ring_info *rxr;
4114 	struct bnxt_ring_struct *ring;
4115 	u32 type;
4116 
4117 	type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
4118 		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
4119 
4120 	if (NET_IP_ALIGN == 2)
4121 		type |= RX_BD_FLAGS_SOP;
4122 
4123 	rxr = &bp->rx_ring[ring_nr];
4124 	ring = &rxr->rx_ring_struct;
4125 	bnxt_init_rxbd_pages(ring, type);
4126 
4127 	netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX,
4128 			     &rxr->bnapi->napi);
4129 
4130 	if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
4131 		bpf_prog_add(bp->xdp_prog, 1);
4132 		rxr->xdp_prog = bp->xdp_prog;
4133 	}
4134 	ring->fw_ring_id = INVALID_HW_RING_ID;
4135 
4136 	ring = &rxr->rx_agg_ring_struct;
4137 	ring->fw_ring_id = INVALID_HW_RING_ID;
4138 
4139 	if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
4140 		type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
4141 			RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
4142 
4143 		bnxt_init_rxbd_pages(ring, type);
4144 	}
4145 
4146 	return bnxt_alloc_one_rx_ring(bp, ring_nr);
4147 }
4148 
4149 static void bnxt_init_cp_rings(struct bnxt *bp)
4150 {
4151 	int i, j;
4152 
4153 	for (i = 0; i < bp->cp_nr_rings; i++) {
4154 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
4155 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4156 
4157 		ring->fw_ring_id = INVALID_HW_RING_ID;
4158 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4159 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4160 		if (!cpr->cp_ring_arr)
4161 			continue;
4162 		for (j = 0; j < cpr->cp_ring_count; j++) {
4163 			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
4164 
4165 			ring = &cpr2->cp_ring_struct;
4166 			ring->fw_ring_id = INVALID_HW_RING_ID;
4167 			cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4168 			cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4169 		}
4170 	}
4171 }
4172 
4173 static int bnxt_init_rx_rings(struct bnxt *bp)
4174 {
4175 	int i, rc = 0;
4176 
4177 	if (BNXT_RX_PAGE_MODE(bp)) {
4178 		bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
4179 		bp->rx_dma_offset = XDP_PACKET_HEADROOM;
4180 	} else {
4181 		bp->rx_offset = BNXT_RX_OFFSET;
4182 		bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
4183 	}
4184 
4185 	for (i = 0; i < bp->rx_nr_rings; i++) {
4186 		rc = bnxt_init_one_rx_ring(bp, i);
4187 		if (rc)
4188 			break;
4189 	}
4190 
4191 	return rc;
4192 }
4193 
4194 static int bnxt_init_tx_rings(struct bnxt *bp)
4195 {
4196 	u16 i;
4197 
4198 	bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
4199 				   BNXT_MIN_TX_DESC_CNT);
4200 
4201 	for (i = 0; i < bp->tx_nr_rings; i++) {
4202 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4203 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4204 
4205 		ring->fw_ring_id = INVALID_HW_RING_ID;
4206 
4207 		if (i >= bp->tx_nr_rings_xdp)
4208 			netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp,
4209 					     NETDEV_QUEUE_TYPE_TX,
4210 					     &txr->bnapi->napi);
4211 	}
4212 
4213 	return 0;
4214 }
4215 
4216 static void bnxt_free_ring_grps(struct bnxt *bp)
4217 {
4218 	kfree(bp->grp_info);
4219 	bp->grp_info = NULL;
4220 }
4221 
4222 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
4223 {
4224 	int i;
4225 
4226 	if (irq_re_init) {
4227 		bp->grp_info = kcalloc(bp->cp_nr_rings,
4228 				       sizeof(struct bnxt_ring_grp_info),
4229 				       GFP_KERNEL);
4230 		if (!bp->grp_info)
4231 			return -ENOMEM;
4232 	}
4233 	for (i = 0; i < bp->cp_nr_rings; i++) {
4234 		if (irq_re_init)
4235 			bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
4236 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4237 		bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
4238 		bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
4239 		bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4240 	}
4241 	return 0;
4242 }
4243 
4244 static void bnxt_free_vnics(struct bnxt *bp)
4245 {
4246 	kfree(bp->vnic_info);
4247 	bp->vnic_info = NULL;
4248 	bp->nr_vnics = 0;
4249 }
4250 
4251 static int bnxt_alloc_vnics(struct bnxt *bp)
4252 {
4253 	int num_vnics = 1;
4254 
4255 #ifdef CONFIG_RFS_ACCEL
4256 	if (bp->flags & BNXT_FLAG_RFS) {
4257 		if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
4258 			num_vnics++;
4259 		else if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4260 			num_vnics += bp->rx_nr_rings;
4261 	}
4262 #endif
4263 
4264 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4265 		num_vnics++;
4266 
4267 	bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
4268 				GFP_KERNEL);
4269 	if (!bp->vnic_info)
4270 		return -ENOMEM;
4271 
4272 	bp->nr_vnics = num_vnics;
4273 	return 0;
4274 }
4275 
4276 static void bnxt_init_vnics(struct bnxt *bp)
4277 {
4278 	struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
4279 	int i;
4280 
4281 	for (i = 0; i < bp->nr_vnics; i++) {
4282 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4283 		int j;
4284 
4285 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
4286 		vnic->vnic_id = i;
4287 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
4288 			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
4289 
4290 		vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
4291 
4292 		if (bp->vnic_info[i].rss_hash_key) {
4293 			if (i == BNXT_VNIC_DEFAULT) {
4294 				u8 *key = (void *)vnic->rss_hash_key;
4295 				int k;
4296 
4297 				if (!bp->rss_hash_key_valid &&
4298 				    !bp->rss_hash_key_updated) {
4299 					get_random_bytes(bp->rss_hash_key,
4300 							 HW_HASH_KEY_SIZE);
4301 					bp->rss_hash_key_updated = true;
4302 				}
4303 
4304 				memcpy(vnic->rss_hash_key, bp->rss_hash_key,
4305 				       HW_HASH_KEY_SIZE);
4306 
4307 				if (!bp->rss_hash_key_updated)
4308 					continue;
4309 
4310 				bp->rss_hash_key_updated = false;
4311 				bp->rss_hash_key_valid = true;
4312 
4313 				bp->toeplitz_prefix = 0;
4314 				for (k = 0; k < 8; k++) {
4315 					bp->toeplitz_prefix <<= 8;
4316 					bp->toeplitz_prefix |= key[k];
4317 				}
4318 			} else {
4319 				memcpy(vnic->rss_hash_key, vnic0->rss_hash_key,
4320 				       HW_HASH_KEY_SIZE);
4321 			}
4322 		}
4323 	}
4324 }
4325 
4326 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
4327 {
4328 	int pages;
4329 
4330 	pages = ring_size / desc_per_pg;
4331 
4332 	if (!pages)
4333 		return 1;
4334 
4335 	pages++;
4336 
4337 	while (pages & (pages - 1))
4338 		pages++;
4339 
4340 	return pages;
4341 }
4342 
4343 void bnxt_set_tpa_flags(struct bnxt *bp)
4344 {
4345 	bp->flags &= ~BNXT_FLAG_TPA;
4346 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
4347 		return;
4348 	if (bp->dev->features & NETIF_F_LRO)
4349 		bp->flags |= BNXT_FLAG_LRO;
4350 	else if (bp->dev->features & NETIF_F_GRO_HW)
4351 		bp->flags |= BNXT_FLAG_GRO;
4352 }
4353 
4354 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
4355  * be set on entry.
4356  */
4357 void bnxt_set_ring_params(struct bnxt *bp)
4358 {
4359 	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
4360 	u32 agg_factor = 0, agg_ring_size = 0;
4361 
4362 	/* 8 for CRC and VLAN */
4363 	rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
4364 
4365 	rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
4366 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4367 
4368 	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
4369 	ring_size = bp->rx_ring_size;
4370 	bp->rx_agg_ring_size = 0;
4371 	bp->rx_agg_nr_pages = 0;
4372 
4373 	if (bp->flags & BNXT_FLAG_TPA)
4374 		agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
4375 
4376 	bp->flags &= ~BNXT_FLAG_JUMBO;
4377 	if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
4378 		u32 jumbo_factor;
4379 
4380 		bp->flags |= BNXT_FLAG_JUMBO;
4381 		jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
4382 		if (jumbo_factor > agg_factor)
4383 			agg_factor = jumbo_factor;
4384 	}
4385 	if (agg_factor) {
4386 		if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
4387 			ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
4388 			netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
4389 				    bp->rx_ring_size, ring_size);
4390 			bp->rx_ring_size = ring_size;
4391 		}
4392 		agg_ring_size = ring_size * agg_factor;
4393 
4394 		bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
4395 							RX_DESC_CNT);
4396 		if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
4397 			u32 tmp = agg_ring_size;
4398 
4399 			bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
4400 			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
4401 			netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
4402 				    tmp, agg_ring_size);
4403 		}
4404 		bp->rx_agg_ring_size = agg_ring_size;
4405 		bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
4406 
4407 		if (BNXT_RX_PAGE_MODE(bp)) {
4408 			rx_space = PAGE_SIZE;
4409 			rx_size = PAGE_SIZE -
4410 				  ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
4411 				  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4412 		} else {
4413 			rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
4414 			rx_space = rx_size + NET_SKB_PAD +
4415 				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4416 		}
4417 	}
4418 
4419 	bp->rx_buf_use_size = rx_size;
4420 	bp->rx_buf_size = rx_space;
4421 
4422 	bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
4423 	bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
4424 
4425 	ring_size = bp->tx_ring_size;
4426 	bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
4427 	bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
4428 
4429 	max_rx_cmpl = bp->rx_ring_size;
4430 	/* MAX TPA needs to be added because TPA_START completions are
4431 	 * immediately recycled, so the TPA completions are not bound by
4432 	 * the RX ring size.
4433 	 */
4434 	if (bp->flags & BNXT_FLAG_TPA)
4435 		max_rx_cmpl += bp->max_tpa;
4436 	/* RX and TPA completions are 32-byte, all others are 16-byte */
4437 	ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
4438 	bp->cp_ring_size = ring_size;
4439 
4440 	bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4441 	if (bp->cp_nr_pages > MAX_CP_PAGES) {
4442 		bp->cp_nr_pages = MAX_CP_PAGES;
4443 		bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4444 		netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4445 			    ring_size, bp->cp_ring_size);
4446 	}
4447 	bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4448 	bp->cp_ring_mask = bp->cp_bit - 1;
4449 }
4450 
4451 /* Changing allocation mode of RX rings.
4452  * TODO: Update when extending xdp_rxq_info to support allocation modes.
4453  */
4454 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4455 {
4456 	struct net_device *dev = bp->dev;
4457 
4458 	if (page_mode) {
4459 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4460 		bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4461 
4462 		if (bp->xdp_prog->aux->xdp_has_frags)
4463 			dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4464 		else
4465 			dev->max_mtu =
4466 				min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4467 		if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4468 			bp->flags |= BNXT_FLAG_JUMBO;
4469 			bp->rx_skb_func = bnxt_rx_multi_page_skb;
4470 		} else {
4471 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4472 			bp->rx_skb_func = bnxt_rx_page_skb;
4473 		}
4474 		bp->rx_dir = DMA_BIDIRECTIONAL;
4475 		/* Disable LRO or GRO_HW */
4476 		netdev_update_features(dev);
4477 	} else {
4478 		dev->max_mtu = bp->max_mtu;
4479 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4480 		bp->rx_dir = DMA_FROM_DEVICE;
4481 		bp->rx_skb_func = bnxt_rx_skb;
4482 	}
4483 	return 0;
4484 }
4485 
4486 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4487 {
4488 	int i;
4489 	struct bnxt_vnic_info *vnic;
4490 	struct pci_dev *pdev = bp->pdev;
4491 
4492 	if (!bp->vnic_info)
4493 		return;
4494 
4495 	for (i = 0; i < bp->nr_vnics; i++) {
4496 		vnic = &bp->vnic_info[i];
4497 
4498 		kfree(vnic->fw_grp_ids);
4499 		vnic->fw_grp_ids = NULL;
4500 
4501 		kfree(vnic->uc_list);
4502 		vnic->uc_list = NULL;
4503 
4504 		if (vnic->mc_list) {
4505 			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4506 					  vnic->mc_list, vnic->mc_list_mapping);
4507 			vnic->mc_list = NULL;
4508 		}
4509 
4510 		if (vnic->rss_table) {
4511 			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4512 					  vnic->rss_table,
4513 					  vnic->rss_table_dma_addr);
4514 			vnic->rss_table = NULL;
4515 		}
4516 
4517 		vnic->rss_hash_key = NULL;
4518 		vnic->flags = 0;
4519 	}
4520 }
4521 
4522 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4523 {
4524 	int i, rc = 0, size;
4525 	struct bnxt_vnic_info *vnic;
4526 	struct pci_dev *pdev = bp->pdev;
4527 	int max_rings;
4528 
4529 	for (i = 0; i < bp->nr_vnics; i++) {
4530 		vnic = &bp->vnic_info[i];
4531 
4532 		if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4533 			int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4534 
4535 			if (mem_size > 0) {
4536 				vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4537 				if (!vnic->uc_list) {
4538 					rc = -ENOMEM;
4539 					goto out;
4540 				}
4541 			}
4542 		}
4543 
4544 		if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4545 			vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4546 			vnic->mc_list =
4547 				dma_alloc_coherent(&pdev->dev,
4548 						   vnic->mc_list_size,
4549 						   &vnic->mc_list_mapping,
4550 						   GFP_KERNEL);
4551 			if (!vnic->mc_list) {
4552 				rc = -ENOMEM;
4553 				goto out;
4554 			}
4555 		}
4556 
4557 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4558 			goto vnic_skip_grps;
4559 
4560 		if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4561 			max_rings = bp->rx_nr_rings;
4562 		else
4563 			max_rings = 1;
4564 
4565 		vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4566 		if (!vnic->fw_grp_ids) {
4567 			rc = -ENOMEM;
4568 			goto out;
4569 		}
4570 vnic_skip_grps:
4571 		if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
4572 		    !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4573 			continue;
4574 
4575 		/* Allocate rss table and hash key */
4576 		size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4577 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4578 			size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4579 
4580 		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4581 		vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4582 						     vnic->rss_table_size,
4583 						     &vnic->rss_table_dma_addr,
4584 						     GFP_KERNEL);
4585 		if (!vnic->rss_table) {
4586 			rc = -ENOMEM;
4587 			goto out;
4588 		}
4589 
4590 		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4591 		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4592 	}
4593 	return 0;
4594 
4595 out:
4596 	return rc;
4597 }
4598 
4599 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4600 {
4601 	struct bnxt_hwrm_wait_token *token;
4602 
4603 	dma_pool_destroy(bp->hwrm_dma_pool);
4604 	bp->hwrm_dma_pool = NULL;
4605 
4606 	rcu_read_lock();
4607 	hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4608 		WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4609 	rcu_read_unlock();
4610 }
4611 
4612 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4613 {
4614 	bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4615 					    BNXT_HWRM_DMA_SIZE,
4616 					    BNXT_HWRM_DMA_ALIGN, 0);
4617 	if (!bp->hwrm_dma_pool)
4618 		return -ENOMEM;
4619 
4620 	INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4621 
4622 	return 0;
4623 }
4624 
4625 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4626 {
4627 	kfree(stats->hw_masks);
4628 	stats->hw_masks = NULL;
4629 	kfree(stats->sw_stats);
4630 	stats->sw_stats = NULL;
4631 	if (stats->hw_stats) {
4632 		dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4633 				  stats->hw_stats_map);
4634 		stats->hw_stats = NULL;
4635 	}
4636 }
4637 
4638 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4639 				bool alloc_masks)
4640 {
4641 	stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4642 					     &stats->hw_stats_map, GFP_KERNEL);
4643 	if (!stats->hw_stats)
4644 		return -ENOMEM;
4645 
4646 	stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4647 	if (!stats->sw_stats)
4648 		goto stats_mem_err;
4649 
4650 	if (alloc_masks) {
4651 		stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4652 		if (!stats->hw_masks)
4653 			goto stats_mem_err;
4654 	}
4655 	return 0;
4656 
4657 stats_mem_err:
4658 	bnxt_free_stats_mem(bp, stats);
4659 	return -ENOMEM;
4660 }
4661 
4662 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4663 {
4664 	int i;
4665 
4666 	for (i = 0; i < count; i++)
4667 		mask_arr[i] = mask;
4668 }
4669 
4670 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4671 {
4672 	int i;
4673 
4674 	for (i = 0; i < count; i++)
4675 		mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4676 }
4677 
4678 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4679 				    struct bnxt_stats_mem *stats)
4680 {
4681 	struct hwrm_func_qstats_ext_output *resp;
4682 	struct hwrm_func_qstats_ext_input *req;
4683 	__le64 *hw_masks;
4684 	int rc;
4685 
4686 	if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4687 	    !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4688 		return -EOPNOTSUPP;
4689 
4690 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4691 	if (rc)
4692 		return rc;
4693 
4694 	req->fid = cpu_to_le16(0xffff);
4695 	req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4696 
4697 	resp = hwrm_req_hold(bp, req);
4698 	rc = hwrm_req_send(bp, req);
4699 	if (!rc) {
4700 		hw_masks = &resp->rx_ucast_pkts;
4701 		bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4702 	}
4703 	hwrm_req_drop(bp, req);
4704 	return rc;
4705 }
4706 
4707 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4708 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4709 
4710 static void bnxt_init_stats(struct bnxt *bp)
4711 {
4712 	struct bnxt_napi *bnapi = bp->bnapi[0];
4713 	struct bnxt_cp_ring_info *cpr;
4714 	struct bnxt_stats_mem *stats;
4715 	__le64 *rx_stats, *tx_stats;
4716 	int rc, rx_count, tx_count;
4717 	u64 *rx_masks, *tx_masks;
4718 	u64 mask;
4719 	u8 flags;
4720 
4721 	cpr = &bnapi->cp_ring;
4722 	stats = &cpr->stats;
4723 	rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4724 	if (rc) {
4725 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4726 			mask = (1ULL << 48) - 1;
4727 		else
4728 			mask = -1ULL;
4729 		bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4730 	}
4731 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
4732 		stats = &bp->port_stats;
4733 		rx_stats = stats->hw_stats;
4734 		rx_masks = stats->hw_masks;
4735 		rx_count = sizeof(struct rx_port_stats) / 8;
4736 		tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4737 		tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4738 		tx_count = sizeof(struct tx_port_stats) / 8;
4739 
4740 		flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4741 		rc = bnxt_hwrm_port_qstats(bp, flags);
4742 		if (rc) {
4743 			mask = (1ULL << 40) - 1;
4744 
4745 			bnxt_fill_masks(rx_masks, mask, rx_count);
4746 			bnxt_fill_masks(tx_masks, mask, tx_count);
4747 		} else {
4748 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4749 			bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4750 			bnxt_hwrm_port_qstats(bp, 0);
4751 		}
4752 	}
4753 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4754 		stats = &bp->rx_port_stats_ext;
4755 		rx_stats = stats->hw_stats;
4756 		rx_masks = stats->hw_masks;
4757 		rx_count = sizeof(struct rx_port_stats_ext) / 8;
4758 		stats = &bp->tx_port_stats_ext;
4759 		tx_stats = stats->hw_stats;
4760 		tx_masks = stats->hw_masks;
4761 		tx_count = sizeof(struct tx_port_stats_ext) / 8;
4762 
4763 		flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4764 		rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4765 		if (rc) {
4766 			mask = (1ULL << 40) - 1;
4767 
4768 			bnxt_fill_masks(rx_masks, mask, rx_count);
4769 			if (tx_stats)
4770 				bnxt_fill_masks(tx_masks, mask, tx_count);
4771 		} else {
4772 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4773 			if (tx_stats)
4774 				bnxt_copy_hw_masks(tx_masks, tx_stats,
4775 						   tx_count);
4776 			bnxt_hwrm_port_qstats_ext(bp, 0);
4777 		}
4778 	}
4779 }
4780 
4781 static void bnxt_free_port_stats(struct bnxt *bp)
4782 {
4783 	bp->flags &= ~BNXT_FLAG_PORT_STATS;
4784 	bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4785 
4786 	bnxt_free_stats_mem(bp, &bp->port_stats);
4787 	bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4788 	bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4789 }
4790 
4791 static void bnxt_free_ring_stats(struct bnxt *bp)
4792 {
4793 	int i;
4794 
4795 	if (!bp->bnapi)
4796 		return;
4797 
4798 	for (i = 0; i < bp->cp_nr_rings; i++) {
4799 		struct bnxt_napi *bnapi = bp->bnapi[i];
4800 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4801 
4802 		bnxt_free_stats_mem(bp, &cpr->stats);
4803 	}
4804 }
4805 
4806 static int bnxt_alloc_stats(struct bnxt *bp)
4807 {
4808 	u32 size, i;
4809 	int rc;
4810 
4811 	size = bp->hw_ring_stats_size;
4812 
4813 	for (i = 0; i < bp->cp_nr_rings; i++) {
4814 		struct bnxt_napi *bnapi = bp->bnapi[i];
4815 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4816 
4817 		cpr->stats.len = size;
4818 		rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4819 		if (rc)
4820 			return rc;
4821 
4822 		cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4823 	}
4824 
4825 	if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4826 		return 0;
4827 
4828 	if (bp->port_stats.hw_stats)
4829 		goto alloc_ext_stats;
4830 
4831 	bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4832 	rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4833 	if (rc)
4834 		return rc;
4835 
4836 	bp->flags |= BNXT_FLAG_PORT_STATS;
4837 
4838 alloc_ext_stats:
4839 	/* Display extended statistics only if FW supports it */
4840 	if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4841 		if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4842 			return 0;
4843 
4844 	if (bp->rx_port_stats_ext.hw_stats)
4845 		goto alloc_tx_ext_stats;
4846 
4847 	bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4848 	rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4849 	/* Extended stats are optional */
4850 	if (rc)
4851 		return 0;
4852 
4853 alloc_tx_ext_stats:
4854 	if (bp->tx_port_stats_ext.hw_stats)
4855 		return 0;
4856 
4857 	if (bp->hwrm_spec_code >= 0x10902 ||
4858 	    (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4859 		bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4860 		rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4861 		/* Extended stats are optional */
4862 		if (rc)
4863 			return 0;
4864 	}
4865 	bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4866 	return 0;
4867 }
4868 
4869 static void bnxt_clear_ring_indices(struct bnxt *bp)
4870 {
4871 	int i, j;
4872 
4873 	if (!bp->bnapi)
4874 		return;
4875 
4876 	for (i = 0; i < bp->cp_nr_rings; i++) {
4877 		struct bnxt_napi *bnapi = bp->bnapi[i];
4878 		struct bnxt_cp_ring_info *cpr;
4879 		struct bnxt_rx_ring_info *rxr;
4880 		struct bnxt_tx_ring_info *txr;
4881 
4882 		if (!bnapi)
4883 			continue;
4884 
4885 		cpr = &bnapi->cp_ring;
4886 		cpr->cp_raw_cons = 0;
4887 
4888 		bnxt_for_each_napi_tx(j, bnapi, txr) {
4889 			txr->tx_prod = 0;
4890 			txr->tx_cons = 0;
4891 			txr->tx_hw_cons = 0;
4892 		}
4893 
4894 		rxr = bnapi->rx_ring;
4895 		if (rxr) {
4896 			rxr->rx_prod = 0;
4897 			rxr->rx_agg_prod = 0;
4898 			rxr->rx_sw_agg_prod = 0;
4899 			rxr->rx_next_cons = 0;
4900 		}
4901 		bnapi->events = 0;
4902 	}
4903 }
4904 
4905 void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
4906 {
4907 	u8 type = fltr->type, flags = fltr->flags;
4908 
4909 	INIT_LIST_HEAD(&fltr->list);
4910 	if ((type == BNXT_FLTR_TYPE_L2 && flags & BNXT_ACT_RING_DST) ||
4911 	    (type == BNXT_FLTR_TYPE_NTUPLE && flags & BNXT_ACT_NO_AGING))
4912 		list_add_tail(&fltr->list, &bp->usr_fltr_list);
4913 }
4914 
4915 void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
4916 {
4917 	if (!list_empty(&fltr->list))
4918 		list_del_init(&fltr->list);
4919 }
4920 
4921 void bnxt_clear_usr_fltrs(struct bnxt *bp, bool all)
4922 {
4923 	struct bnxt_filter_base *usr_fltr, *tmp;
4924 
4925 	list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
4926 		if (!all && usr_fltr->type == BNXT_FLTR_TYPE_L2)
4927 			continue;
4928 		bnxt_del_one_usr_fltr(bp, usr_fltr);
4929 	}
4930 }
4931 
4932 static void bnxt_del_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
4933 {
4934 	hlist_del(&fltr->hash);
4935 	bnxt_del_one_usr_fltr(bp, fltr);
4936 	if (fltr->flags) {
4937 		clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
4938 		bp->ntp_fltr_count--;
4939 	}
4940 	kfree(fltr);
4941 }
4942 
4943 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all)
4944 {
4945 	int i;
4946 
4947 	/* Under rtnl_lock and all our NAPIs have been disabled.  It's
4948 	 * safe to delete the hash table.
4949 	 */
4950 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4951 		struct hlist_head *head;
4952 		struct hlist_node *tmp;
4953 		struct bnxt_ntuple_filter *fltr;
4954 
4955 		head = &bp->ntp_fltr_hash_tbl[i];
4956 		hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
4957 			bnxt_del_l2_filter(bp, fltr->l2_fltr);
4958 			if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
4959 				     !list_empty(&fltr->base.list)))
4960 				continue;
4961 			bnxt_del_fltr(bp, &fltr->base);
4962 		}
4963 	}
4964 	if (!all)
4965 		return;
4966 
4967 	bitmap_free(bp->ntp_fltr_bmap);
4968 	bp->ntp_fltr_bmap = NULL;
4969 	bp->ntp_fltr_count = 0;
4970 }
4971 
4972 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4973 {
4974 	int i, rc = 0;
4975 
4976 	if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap)
4977 		return 0;
4978 
4979 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4980 		INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4981 
4982 	bp->ntp_fltr_count = 0;
4983 	bp->ntp_fltr_bmap = bitmap_zalloc(bp->max_fltr, GFP_KERNEL);
4984 
4985 	if (!bp->ntp_fltr_bmap)
4986 		rc = -ENOMEM;
4987 
4988 	return rc;
4989 }
4990 
4991 static void bnxt_free_l2_filters(struct bnxt *bp, bool all)
4992 {
4993 	int i;
4994 
4995 	for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) {
4996 		struct hlist_head *head;
4997 		struct hlist_node *tmp;
4998 		struct bnxt_l2_filter *fltr;
4999 
5000 		head = &bp->l2_fltr_hash_tbl[i];
5001 		hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
5002 			if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
5003 				     !list_empty(&fltr->base.list)))
5004 				continue;
5005 			bnxt_del_fltr(bp, &fltr->base);
5006 		}
5007 	}
5008 }
5009 
5010 static void bnxt_init_l2_fltr_tbl(struct bnxt *bp)
5011 {
5012 	int i;
5013 
5014 	for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++)
5015 		INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]);
5016 	get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed));
5017 }
5018 
5019 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
5020 {
5021 	bnxt_free_vnic_attributes(bp);
5022 	bnxt_free_tx_rings(bp);
5023 	bnxt_free_rx_rings(bp);
5024 	bnxt_free_cp_rings(bp);
5025 	bnxt_free_all_cp_arrays(bp);
5026 	bnxt_free_ntp_fltrs(bp, false);
5027 	bnxt_free_l2_filters(bp, false);
5028 	if (irq_re_init) {
5029 		bnxt_free_ring_stats(bp);
5030 		if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
5031 		    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
5032 			bnxt_free_port_stats(bp);
5033 		bnxt_free_ring_grps(bp);
5034 		bnxt_free_vnics(bp);
5035 		kfree(bp->tx_ring_map);
5036 		bp->tx_ring_map = NULL;
5037 		kfree(bp->tx_ring);
5038 		bp->tx_ring = NULL;
5039 		kfree(bp->rx_ring);
5040 		bp->rx_ring = NULL;
5041 		kfree(bp->bnapi);
5042 		bp->bnapi = NULL;
5043 	} else {
5044 		bnxt_clear_ring_indices(bp);
5045 	}
5046 }
5047 
5048 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
5049 {
5050 	int i, j, rc, size, arr_size;
5051 	void *bnapi;
5052 
5053 	if (irq_re_init) {
5054 		/* Allocate bnapi mem pointer array and mem block for
5055 		 * all queues
5056 		 */
5057 		arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
5058 				bp->cp_nr_rings);
5059 		size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
5060 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
5061 		if (!bnapi)
5062 			return -ENOMEM;
5063 
5064 		bp->bnapi = bnapi;
5065 		bnapi += arr_size;
5066 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
5067 			bp->bnapi[i] = bnapi;
5068 			bp->bnapi[i]->index = i;
5069 			bp->bnapi[i]->bp = bp;
5070 			if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5071 				struct bnxt_cp_ring_info *cpr =
5072 					&bp->bnapi[i]->cp_ring;
5073 
5074 				cpr->cp_ring_struct.ring_mem.flags =
5075 					BNXT_RMEM_RING_PTE_FLAG;
5076 			}
5077 		}
5078 
5079 		bp->rx_ring = kcalloc(bp->rx_nr_rings,
5080 				      sizeof(struct bnxt_rx_ring_info),
5081 				      GFP_KERNEL);
5082 		if (!bp->rx_ring)
5083 			return -ENOMEM;
5084 
5085 		for (i = 0; i < bp->rx_nr_rings; i++) {
5086 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5087 
5088 			if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5089 				rxr->rx_ring_struct.ring_mem.flags =
5090 					BNXT_RMEM_RING_PTE_FLAG;
5091 				rxr->rx_agg_ring_struct.ring_mem.flags =
5092 					BNXT_RMEM_RING_PTE_FLAG;
5093 			} else {
5094 				rxr->rx_cpr =  &bp->bnapi[i]->cp_ring;
5095 			}
5096 			rxr->bnapi = bp->bnapi[i];
5097 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
5098 		}
5099 
5100 		bp->tx_ring = kcalloc(bp->tx_nr_rings,
5101 				      sizeof(struct bnxt_tx_ring_info),
5102 				      GFP_KERNEL);
5103 		if (!bp->tx_ring)
5104 			return -ENOMEM;
5105 
5106 		bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
5107 					  GFP_KERNEL);
5108 
5109 		if (!bp->tx_ring_map)
5110 			return -ENOMEM;
5111 
5112 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5113 			j = 0;
5114 		else
5115 			j = bp->rx_nr_rings;
5116 
5117 		for (i = 0; i < bp->tx_nr_rings; i++) {
5118 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5119 			struct bnxt_napi *bnapi2;
5120 
5121 			if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5122 				txr->tx_ring_struct.ring_mem.flags =
5123 					BNXT_RMEM_RING_PTE_FLAG;
5124 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
5125 			if (i >= bp->tx_nr_rings_xdp) {
5126 				int k = j + BNXT_RING_TO_TC_OFF(bp, i);
5127 
5128 				bnapi2 = bp->bnapi[k];
5129 				txr->txq_index = i - bp->tx_nr_rings_xdp;
5130 				txr->tx_napi_idx =
5131 					BNXT_RING_TO_TC(bp, txr->txq_index);
5132 				bnapi2->tx_ring[txr->tx_napi_idx] = txr;
5133 				bnapi2->tx_int = bnxt_tx_int;
5134 			} else {
5135 				bnapi2 = bp->bnapi[j];
5136 				bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
5137 				bnapi2->tx_ring[0] = txr;
5138 				bnapi2->tx_int = bnxt_tx_int_xdp;
5139 				j++;
5140 			}
5141 			txr->bnapi = bnapi2;
5142 			if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
5143 				txr->tx_cpr = &bnapi2->cp_ring;
5144 		}
5145 
5146 		rc = bnxt_alloc_stats(bp);
5147 		if (rc)
5148 			goto alloc_mem_err;
5149 		bnxt_init_stats(bp);
5150 
5151 		rc = bnxt_alloc_ntp_fltrs(bp);
5152 		if (rc)
5153 			goto alloc_mem_err;
5154 
5155 		rc = bnxt_alloc_vnics(bp);
5156 		if (rc)
5157 			goto alloc_mem_err;
5158 	}
5159 
5160 	rc = bnxt_alloc_all_cp_arrays(bp);
5161 	if (rc)
5162 		goto alloc_mem_err;
5163 
5164 	bnxt_init_ring_struct(bp);
5165 
5166 	rc = bnxt_alloc_rx_rings(bp);
5167 	if (rc)
5168 		goto alloc_mem_err;
5169 
5170 	rc = bnxt_alloc_tx_rings(bp);
5171 	if (rc)
5172 		goto alloc_mem_err;
5173 
5174 	rc = bnxt_alloc_cp_rings(bp);
5175 	if (rc)
5176 		goto alloc_mem_err;
5177 
5178 	bp->vnic_info[BNXT_VNIC_DEFAULT].flags |= BNXT_VNIC_RSS_FLAG |
5179 						  BNXT_VNIC_MCAST_FLAG |
5180 						  BNXT_VNIC_UCAST_FLAG;
5181 	if (BNXT_SUPPORTS_NTUPLE_VNIC(bp) && (bp->flags & BNXT_FLAG_RFS))
5182 		bp->vnic_info[BNXT_VNIC_NTUPLE].flags |=
5183 			BNXT_VNIC_RSS_FLAG | BNXT_VNIC_NTUPLE_FLAG;
5184 
5185 	rc = bnxt_alloc_vnic_attributes(bp);
5186 	if (rc)
5187 		goto alloc_mem_err;
5188 	return 0;
5189 
5190 alloc_mem_err:
5191 	bnxt_free_mem(bp, true);
5192 	return rc;
5193 }
5194 
5195 static void bnxt_disable_int(struct bnxt *bp)
5196 {
5197 	int i;
5198 
5199 	if (!bp->bnapi)
5200 		return;
5201 
5202 	for (i = 0; i < bp->cp_nr_rings; i++) {
5203 		struct bnxt_napi *bnapi = bp->bnapi[i];
5204 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5205 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5206 
5207 		if (ring->fw_ring_id != INVALID_HW_RING_ID)
5208 			bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5209 	}
5210 }
5211 
5212 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
5213 {
5214 	struct bnxt_napi *bnapi = bp->bnapi[n];
5215 	struct bnxt_cp_ring_info *cpr;
5216 
5217 	cpr = &bnapi->cp_ring;
5218 	return cpr->cp_ring_struct.map_idx;
5219 }
5220 
5221 static void bnxt_disable_int_sync(struct bnxt *bp)
5222 {
5223 	int i;
5224 
5225 	if (!bp->irq_tbl)
5226 		return;
5227 
5228 	atomic_inc(&bp->intr_sem);
5229 
5230 	bnxt_disable_int(bp);
5231 	for (i = 0; i < bp->cp_nr_rings; i++) {
5232 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5233 
5234 		synchronize_irq(bp->irq_tbl[map_idx].vector);
5235 	}
5236 }
5237 
5238 static void bnxt_enable_int(struct bnxt *bp)
5239 {
5240 	int i;
5241 
5242 	atomic_set(&bp->intr_sem, 0);
5243 	for (i = 0; i < bp->cp_nr_rings; i++) {
5244 		struct bnxt_napi *bnapi = bp->bnapi[i];
5245 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5246 
5247 		bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
5248 	}
5249 }
5250 
5251 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
5252 			    bool async_only)
5253 {
5254 	DECLARE_BITMAP(async_events_bmap, 256);
5255 	u32 *events = (u32 *)async_events_bmap;
5256 	struct hwrm_func_drv_rgtr_output *resp;
5257 	struct hwrm_func_drv_rgtr_input *req;
5258 	u32 flags;
5259 	int rc, i;
5260 
5261 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
5262 	if (rc)
5263 		return rc;
5264 
5265 	req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
5266 				   FUNC_DRV_RGTR_REQ_ENABLES_VER |
5267 				   FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5268 
5269 	req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
5270 	flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
5271 	if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
5272 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
5273 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
5274 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
5275 			 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
5276 	req->flags = cpu_to_le32(flags);
5277 	req->ver_maj_8b = DRV_VER_MAJ;
5278 	req->ver_min_8b = DRV_VER_MIN;
5279 	req->ver_upd_8b = DRV_VER_UPD;
5280 	req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
5281 	req->ver_min = cpu_to_le16(DRV_VER_MIN);
5282 	req->ver_upd = cpu_to_le16(DRV_VER_UPD);
5283 
5284 	if (BNXT_PF(bp)) {
5285 		u32 data[8];
5286 		int i;
5287 
5288 		memset(data, 0, sizeof(data));
5289 		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
5290 			u16 cmd = bnxt_vf_req_snif[i];
5291 			unsigned int bit, idx;
5292 
5293 			idx = cmd / 32;
5294 			bit = cmd % 32;
5295 			data[idx] |= 1 << bit;
5296 		}
5297 
5298 		for (i = 0; i < 8; i++)
5299 			req->vf_req_fwd[i] = cpu_to_le32(data[i]);
5300 
5301 		req->enables |=
5302 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
5303 	}
5304 
5305 	if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
5306 		req->flags |= cpu_to_le32(
5307 			FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
5308 
5309 	memset(async_events_bmap, 0, sizeof(async_events_bmap));
5310 	for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
5311 		u16 event_id = bnxt_async_events_arr[i];
5312 
5313 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
5314 		    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5315 			continue;
5316 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
5317 		    !bp->ptp_cfg)
5318 			continue;
5319 		__set_bit(bnxt_async_events_arr[i], async_events_bmap);
5320 	}
5321 	if (bmap && bmap_size) {
5322 		for (i = 0; i < bmap_size; i++) {
5323 			if (test_bit(i, bmap))
5324 				__set_bit(i, async_events_bmap);
5325 		}
5326 	}
5327 	for (i = 0; i < 8; i++)
5328 		req->async_event_fwd[i] |= cpu_to_le32(events[i]);
5329 
5330 	if (async_only)
5331 		req->enables =
5332 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5333 
5334 	resp = hwrm_req_hold(bp, req);
5335 	rc = hwrm_req_send(bp, req);
5336 	if (!rc) {
5337 		set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
5338 		if (resp->flags &
5339 		    cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
5340 			bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
5341 	}
5342 	hwrm_req_drop(bp, req);
5343 	return rc;
5344 }
5345 
5346 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
5347 {
5348 	struct hwrm_func_drv_unrgtr_input *req;
5349 	int rc;
5350 
5351 	if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
5352 		return 0;
5353 
5354 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
5355 	if (rc)
5356 		return rc;
5357 	return hwrm_req_send(bp, req);
5358 }
5359 
5360 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa);
5361 
5362 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
5363 {
5364 	struct hwrm_tunnel_dst_port_free_input *req;
5365 	int rc;
5366 
5367 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
5368 	    bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
5369 		return 0;
5370 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
5371 	    bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
5372 		return 0;
5373 
5374 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
5375 	if (rc)
5376 		return rc;
5377 
5378 	req->tunnel_type = tunnel_type;
5379 
5380 	switch (tunnel_type) {
5381 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
5382 		req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
5383 		bp->vxlan_port = 0;
5384 		bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
5385 		break;
5386 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
5387 		req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
5388 		bp->nge_port = 0;
5389 		bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
5390 		break;
5391 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE:
5392 		req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id);
5393 		bp->vxlan_gpe_port = 0;
5394 		bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID;
5395 		break;
5396 	default:
5397 		break;
5398 	}
5399 
5400 	rc = hwrm_req_send(bp, req);
5401 	if (rc)
5402 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
5403 			   rc);
5404 	if (bp->flags & BNXT_FLAG_TPA)
5405 		bnxt_set_tpa(bp, true);
5406 	return rc;
5407 }
5408 
5409 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
5410 					   u8 tunnel_type)
5411 {
5412 	struct hwrm_tunnel_dst_port_alloc_output *resp;
5413 	struct hwrm_tunnel_dst_port_alloc_input *req;
5414 	int rc;
5415 
5416 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
5417 	if (rc)
5418 		return rc;
5419 
5420 	req->tunnel_type = tunnel_type;
5421 	req->tunnel_dst_port_val = port;
5422 
5423 	resp = hwrm_req_hold(bp, req);
5424 	rc = hwrm_req_send(bp, req);
5425 	if (rc) {
5426 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
5427 			   rc);
5428 		goto err_out;
5429 	}
5430 
5431 	switch (tunnel_type) {
5432 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
5433 		bp->vxlan_port = port;
5434 		bp->vxlan_fw_dst_port_id =
5435 			le16_to_cpu(resp->tunnel_dst_port_id);
5436 		break;
5437 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
5438 		bp->nge_port = port;
5439 		bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
5440 		break;
5441 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE:
5442 		bp->vxlan_gpe_port = port;
5443 		bp->vxlan_gpe_fw_dst_port_id =
5444 			le16_to_cpu(resp->tunnel_dst_port_id);
5445 		break;
5446 	default:
5447 		break;
5448 	}
5449 	if (bp->flags & BNXT_FLAG_TPA)
5450 		bnxt_set_tpa(bp, true);
5451 
5452 err_out:
5453 	hwrm_req_drop(bp, req);
5454 	return rc;
5455 }
5456 
5457 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
5458 {
5459 	struct hwrm_cfa_l2_set_rx_mask_input *req;
5460 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5461 	int rc;
5462 
5463 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
5464 	if (rc)
5465 		return rc;
5466 
5467 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5468 	if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
5469 		req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
5470 		req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
5471 	}
5472 	req->mask = cpu_to_le32(vnic->rx_mask);
5473 	return hwrm_req_send_silent(bp, req);
5474 }
5475 
5476 void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5477 {
5478 	if (!atomic_dec_and_test(&fltr->refcnt))
5479 		return;
5480 	spin_lock_bh(&bp->ntp_fltr_lock);
5481 	if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
5482 		spin_unlock_bh(&bp->ntp_fltr_lock);
5483 		return;
5484 	}
5485 	hlist_del_rcu(&fltr->base.hash);
5486 	bnxt_del_one_usr_fltr(bp, &fltr->base);
5487 	if (fltr->base.flags) {
5488 		clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
5489 		bp->ntp_fltr_count--;
5490 	}
5491 	spin_unlock_bh(&bp->ntp_fltr_lock);
5492 	kfree_rcu(fltr, base.rcu);
5493 }
5494 
5495 static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp,
5496 						      struct bnxt_l2_key *key,
5497 						      u32 idx)
5498 {
5499 	struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx];
5500 	struct bnxt_l2_filter *fltr;
5501 
5502 	hlist_for_each_entry_rcu(fltr, head, base.hash) {
5503 		struct bnxt_l2_key *l2_key = &fltr->l2_key;
5504 
5505 		if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
5506 		    l2_key->vlan == key->vlan)
5507 			return fltr;
5508 	}
5509 	return NULL;
5510 }
5511 
5512 static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp,
5513 						    struct bnxt_l2_key *key,
5514 						    u32 idx)
5515 {
5516 	struct bnxt_l2_filter *fltr = NULL;
5517 
5518 	rcu_read_lock();
5519 	fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5520 	if (fltr)
5521 		atomic_inc(&fltr->refcnt);
5522 	rcu_read_unlock();
5523 	return fltr;
5524 }
5525 
5526 #define BNXT_IPV4_4TUPLE(bp, fkeys)					\
5527 	(((fkeys)->basic.ip_proto == IPPROTO_TCP &&			\
5528 	  (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) ||	\
5529 	 ((fkeys)->basic.ip_proto == IPPROTO_UDP &&			\
5530 	  (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4))
5531 
5532 #define BNXT_IPV6_4TUPLE(bp, fkeys)					\
5533 	(((fkeys)->basic.ip_proto == IPPROTO_TCP &&			\
5534 	  (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) ||	\
5535 	 ((fkeys)->basic.ip_proto == IPPROTO_UDP &&			\
5536 	  (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6))
5537 
5538 static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys)
5539 {
5540 	if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5541 		if (BNXT_IPV4_4TUPLE(bp, fkeys))
5542 			return sizeof(fkeys->addrs.v4addrs) +
5543 			       sizeof(fkeys->ports);
5544 
5545 		if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4)
5546 			return sizeof(fkeys->addrs.v4addrs);
5547 	}
5548 
5549 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
5550 		if (BNXT_IPV6_4TUPLE(bp, fkeys))
5551 			return sizeof(fkeys->addrs.v6addrs) +
5552 			       sizeof(fkeys->ports);
5553 
5554 		if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6)
5555 			return sizeof(fkeys->addrs.v6addrs);
5556 	}
5557 
5558 	return 0;
5559 }
5560 
5561 static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys,
5562 			 const unsigned char *key)
5563 {
5564 	u64 prefix = bp->toeplitz_prefix, hash = 0;
5565 	struct bnxt_ipv4_tuple tuple4;
5566 	struct bnxt_ipv6_tuple tuple6;
5567 	int i, j, len = 0;
5568 	u8 *four_tuple;
5569 
5570 	len = bnxt_get_rss_flow_tuple_len(bp, fkeys);
5571 	if (!len)
5572 		return 0;
5573 
5574 	if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5575 		tuple4.v4addrs = fkeys->addrs.v4addrs;
5576 		tuple4.ports = fkeys->ports;
5577 		four_tuple = (unsigned char *)&tuple4;
5578 	} else {
5579 		tuple6.v6addrs = fkeys->addrs.v6addrs;
5580 		tuple6.ports = fkeys->ports;
5581 		four_tuple = (unsigned char *)&tuple6;
5582 	}
5583 
5584 	for (i = 0, j = 8; i < len; i++, j++) {
5585 		u8 byte = four_tuple[i];
5586 		int bit;
5587 
5588 		for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) {
5589 			if (byte & 0x80)
5590 				hash ^= prefix;
5591 		}
5592 		prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0;
5593 	}
5594 
5595 	/* The valid part of the hash is in the upper 32 bits. */
5596 	return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK;
5597 }
5598 
5599 #ifdef CONFIG_RFS_ACCEL
5600 static struct bnxt_l2_filter *
5601 bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key)
5602 {
5603 	struct bnxt_l2_filter *fltr;
5604 	u32 idx;
5605 
5606 	idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5607 	      BNXT_L2_FLTR_HASH_MASK;
5608 	fltr = bnxt_lookup_l2_filter(bp, key, idx);
5609 	return fltr;
5610 }
5611 #endif
5612 
5613 static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr,
5614 			       struct bnxt_l2_key *key, u32 idx)
5615 {
5616 	struct hlist_head *head;
5617 
5618 	ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
5619 	fltr->l2_key.vlan = key->vlan;
5620 	fltr->base.type = BNXT_FLTR_TYPE_L2;
5621 	if (fltr->base.flags) {
5622 		int bit_id;
5623 
5624 		bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5625 						 bp->max_fltr, 0);
5626 		if (bit_id < 0)
5627 			return -ENOMEM;
5628 		fltr->base.sw_id = (u16)bit_id;
5629 		bp->ntp_fltr_count++;
5630 	}
5631 	head = &bp->l2_fltr_hash_tbl[idx];
5632 	hlist_add_head_rcu(&fltr->base.hash, head);
5633 	bnxt_insert_usr_fltr(bp, &fltr->base);
5634 	set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
5635 	atomic_set(&fltr->refcnt, 1);
5636 	return 0;
5637 }
5638 
5639 static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp,
5640 						   struct bnxt_l2_key *key,
5641 						   gfp_t gfp)
5642 {
5643 	struct bnxt_l2_filter *fltr;
5644 	u32 idx;
5645 	int rc;
5646 
5647 	idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5648 	      BNXT_L2_FLTR_HASH_MASK;
5649 	fltr = bnxt_lookup_l2_filter(bp, key, idx);
5650 	if (fltr)
5651 		return fltr;
5652 
5653 	fltr = kzalloc(sizeof(*fltr), gfp);
5654 	if (!fltr)
5655 		return ERR_PTR(-ENOMEM);
5656 	spin_lock_bh(&bp->ntp_fltr_lock);
5657 	rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5658 	spin_unlock_bh(&bp->ntp_fltr_lock);
5659 	if (rc) {
5660 		bnxt_del_l2_filter(bp, fltr);
5661 		fltr = ERR_PTR(rc);
5662 	}
5663 	return fltr;
5664 }
5665 
5666 struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp,
5667 						struct bnxt_l2_key *key,
5668 						u16 flags)
5669 {
5670 	struct bnxt_l2_filter *fltr;
5671 	u32 idx;
5672 	int rc;
5673 
5674 	idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5675 	      BNXT_L2_FLTR_HASH_MASK;
5676 	spin_lock_bh(&bp->ntp_fltr_lock);
5677 	fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5678 	if (fltr) {
5679 		fltr = ERR_PTR(-EEXIST);
5680 		goto l2_filter_exit;
5681 	}
5682 	fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC);
5683 	if (!fltr) {
5684 		fltr = ERR_PTR(-ENOMEM);
5685 		goto l2_filter_exit;
5686 	}
5687 	fltr->base.flags = flags;
5688 	rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5689 	if (rc) {
5690 		spin_unlock_bh(&bp->ntp_fltr_lock);
5691 		bnxt_del_l2_filter(bp, fltr);
5692 		return ERR_PTR(rc);
5693 	}
5694 
5695 l2_filter_exit:
5696 	spin_unlock_bh(&bp->ntp_fltr_lock);
5697 	return fltr;
5698 }
5699 
5700 static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx)
5701 {
5702 #ifdef CONFIG_BNXT_SRIOV
5703 	struct bnxt_vf_info *vf = &pf->vf[vf_idx];
5704 
5705 	return vf->fw_fid;
5706 #else
5707 	return INVALID_HW_RING_ID;
5708 #endif
5709 }
5710 
5711 int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5712 {
5713 	struct hwrm_cfa_l2_filter_free_input *req;
5714 	u16 target_id = 0xffff;
5715 	int rc;
5716 
5717 	if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5718 		struct bnxt_pf_info *pf = &bp->pf;
5719 
5720 		if (fltr->base.vf_idx >= pf->active_vfs)
5721 			return -EINVAL;
5722 
5723 		target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5724 		if (target_id == INVALID_HW_RING_ID)
5725 			return -EINVAL;
5726 	}
5727 
5728 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5729 	if (rc)
5730 		return rc;
5731 
5732 	req->target_id = cpu_to_le16(target_id);
5733 	req->l2_filter_id = fltr->base.filter_id;
5734 	return hwrm_req_send(bp, req);
5735 }
5736 
5737 int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5738 {
5739 	struct hwrm_cfa_l2_filter_alloc_output *resp;
5740 	struct hwrm_cfa_l2_filter_alloc_input *req;
5741 	u16 target_id = 0xffff;
5742 	int rc;
5743 
5744 	if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5745 		struct bnxt_pf_info *pf = &bp->pf;
5746 
5747 		if (fltr->base.vf_idx >= pf->active_vfs)
5748 			return -EINVAL;
5749 
5750 		target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5751 	}
5752 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5753 	if (rc)
5754 		return rc;
5755 
5756 	req->target_id = cpu_to_le16(target_id);
5757 	req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5758 
5759 	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5760 		req->flags |=
5761 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5762 	req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
5763 	req->enables =
5764 		cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5765 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5766 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5767 	ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
5768 	eth_broadcast_addr(req->l2_addr_mask);
5769 
5770 	if (fltr->l2_key.vlan) {
5771 		req->enables |=
5772 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
5773 				CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
5774 				CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
5775 		req->num_vlans = 1;
5776 		req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
5777 		req->l2_ivlan_mask = cpu_to_le16(0xfff);
5778 	}
5779 
5780 	resp = hwrm_req_hold(bp, req);
5781 	rc = hwrm_req_send(bp, req);
5782 	if (!rc) {
5783 		fltr->base.filter_id = resp->l2_filter_id;
5784 		set_bit(BNXT_FLTR_VALID, &fltr->base.state);
5785 	}
5786 	hwrm_req_drop(bp, req);
5787 	return rc;
5788 }
5789 
5790 int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
5791 				     struct bnxt_ntuple_filter *fltr)
5792 {
5793 	struct hwrm_cfa_ntuple_filter_free_input *req;
5794 	int rc;
5795 
5796 	set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state);
5797 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
5798 	if (rc)
5799 		return rc;
5800 
5801 	req->ntuple_filter_id = fltr->base.filter_id;
5802 	return hwrm_req_send(bp, req);
5803 }
5804 
5805 #define BNXT_NTP_FLTR_FLAGS					\
5806 	(CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |	\
5807 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |	\
5808 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |	\
5809 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |	\
5810 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |	\
5811 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |	\
5812 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |	\
5813 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |	\
5814 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |		\
5815 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |	\
5816 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |		\
5817 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |	\
5818 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
5819 
5820 #define BNXT_NTP_TUNNEL_FLTR_FLAG				\
5821 		CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
5822 
5823 void bnxt_fill_ipv6_mask(__be32 mask[4])
5824 {
5825 	int i;
5826 
5827 	for (i = 0; i < 4; i++)
5828 		mask[i] = cpu_to_be32(~0);
5829 }
5830 
5831 static void
5832 bnxt_cfg_rfs_ring_tbl_idx(struct bnxt *bp,
5833 			  struct hwrm_cfa_ntuple_filter_alloc_input *req,
5834 			  struct bnxt_ntuple_filter *fltr)
5835 {
5836 	struct bnxt_rss_ctx *rss_ctx, *tmp;
5837 	u16 rxq = fltr->base.rxq;
5838 
5839 	if (fltr->base.flags & BNXT_ACT_RSS_CTX) {
5840 		list_for_each_entry_safe(rss_ctx, tmp, &bp->rss_ctx_list, list) {
5841 			if (rss_ctx->index == fltr->base.fw_vnic_id) {
5842 				struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
5843 
5844 				req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5845 				break;
5846 			}
5847 		}
5848 		return;
5849 	}
5850 	if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
5851 		struct bnxt_vnic_info *vnic;
5852 		u32 enables;
5853 
5854 		vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
5855 		req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5856 		enables = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
5857 		req->enables |= cpu_to_le32(enables);
5858 		req->rfs_ring_tbl_idx = cpu_to_le16(rxq);
5859 	} else {
5860 		u32 flags;
5861 
5862 		flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
5863 		req->flags |= cpu_to_le32(flags);
5864 		req->dst_id = cpu_to_le16(rxq);
5865 	}
5866 }
5867 
5868 int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
5869 				      struct bnxt_ntuple_filter *fltr)
5870 {
5871 	struct hwrm_cfa_ntuple_filter_alloc_output *resp;
5872 	struct hwrm_cfa_ntuple_filter_alloc_input *req;
5873 	struct bnxt_flow_masks *masks = &fltr->fmasks;
5874 	struct flow_keys *keys = &fltr->fkeys;
5875 	struct bnxt_l2_filter *l2_fltr;
5876 	struct bnxt_vnic_info *vnic;
5877 	int rc;
5878 
5879 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
5880 	if (rc)
5881 		return rc;
5882 
5883 	l2_fltr = fltr->l2_fltr;
5884 	req->l2_filter_id = l2_fltr->base.filter_id;
5885 
5886 	if (fltr->base.flags & BNXT_ACT_DROP) {
5887 		req->flags =
5888 			cpu_to_le32(CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP);
5889 	} else if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
5890 		bnxt_cfg_rfs_ring_tbl_idx(bp, req, fltr);
5891 	} else {
5892 		vnic = &bp->vnic_info[fltr->base.rxq + 1];
5893 		req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5894 	}
5895 	req->enables |= cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
5896 
5897 	req->ethertype = htons(ETH_P_IP);
5898 	req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
5899 	req->ip_protocol = keys->basic.ip_proto;
5900 
5901 	if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
5902 		req->ethertype = htons(ETH_P_IPV6);
5903 		req->ip_addr_type =
5904 			CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
5905 		*(struct in6_addr *)&req->src_ipaddr[0] = keys->addrs.v6addrs.src;
5906 		*(struct in6_addr *)&req->src_ipaddr_mask[0] = masks->addrs.v6addrs.src;
5907 		*(struct in6_addr *)&req->dst_ipaddr[0] = keys->addrs.v6addrs.dst;
5908 		*(struct in6_addr *)&req->dst_ipaddr_mask[0] = masks->addrs.v6addrs.dst;
5909 	} else {
5910 		req->src_ipaddr[0] = keys->addrs.v4addrs.src;
5911 		req->src_ipaddr_mask[0] = masks->addrs.v4addrs.src;
5912 		req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
5913 		req->dst_ipaddr_mask[0] = masks->addrs.v4addrs.dst;
5914 	}
5915 	if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
5916 		req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
5917 		req->tunnel_type =
5918 			CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
5919 	}
5920 
5921 	req->src_port = keys->ports.src;
5922 	req->src_port_mask = masks->ports.src;
5923 	req->dst_port = keys->ports.dst;
5924 	req->dst_port_mask = masks->ports.dst;
5925 
5926 	resp = hwrm_req_hold(bp, req);
5927 	rc = hwrm_req_send(bp, req);
5928 	if (!rc)
5929 		fltr->base.filter_id = resp->ntuple_filter_id;
5930 	hwrm_req_drop(bp, req);
5931 	return rc;
5932 }
5933 
5934 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5935 				     const u8 *mac_addr)
5936 {
5937 	struct bnxt_l2_filter *fltr;
5938 	struct bnxt_l2_key key;
5939 	int rc;
5940 
5941 	ether_addr_copy(key.dst_mac_addr, mac_addr);
5942 	key.vlan = 0;
5943 	fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL);
5944 	if (IS_ERR(fltr))
5945 		return PTR_ERR(fltr);
5946 
5947 	fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id;
5948 	rc = bnxt_hwrm_l2_filter_alloc(bp, fltr);
5949 	if (rc)
5950 		bnxt_del_l2_filter(bp, fltr);
5951 	else
5952 		bp->vnic_info[vnic_id].l2_filters[idx] = fltr;
5953 	return rc;
5954 }
5955 
5956 static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5957 {
5958 	u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5959 
5960 	/* Any associated ntuple filters will also be cleared by firmware. */
5961 	for (i = 0; i < num_of_vnics; i++) {
5962 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5963 
5964 		for (j = 0; j < vnic->uc_filter_count; j++) {
5965 			struct bnxt_l2_filter *fltr = vnic->l2_filters[j];
5966 
5967 			bnxt_hwrm_l2_filter_free(bp, fltr);
5968 			bnxt_del_l2_filter(bp, fltr);
5969 		}
5970 		vnic->uc_filter_count = 0;
5971 	}
5972 }
5973 
5974 #define BNXT_DFLT_TUNL_TPA_BMAP				\
5975 	(VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE |	\
5976 	 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 |	\
5977 	 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
5978 
5979 static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp,
5980 					   struct hwrm_vnic_tpa_cfg_input *req)
5981 {
5982 	u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP;
5983 
5984 	if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA))
5985 		return;
5986 
5987 	if (bp->vxlan_port)
5988 		tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
5989 	if (bp->vxlan_gpe_port)
5990 		tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
5991 	if (bp->nge_port)
5992 		tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
5993 
5994 	req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
5995 	req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
5996 }
5997 
5998 int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, struct bnxt_vnic_info *vnic,
5999 			   u32 tpa_flags)
6000 {
6001 	u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
6002 	struct hwrm_vnic_tpa_cfg_input *req;
6003 	int rc;
6004 
6005 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
6006 		return 0;
6007 
6008 	rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
6009 	if (rc)
6010 		return rc;
6011 
6012 	if (tpa_flags) {
6013 		u16 mss = bp->dev->mtu - 40;
6014 		u32 nsegs, n, segs = 0, flags;
6015 
6016 		flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
6017 			VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
6018 			VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
6019 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
6020 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
6021 		if (tpa_flags & BNXT_FLAG_GRO)
6022 			flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
6023 
6024 		req->flags = cpu_to_le32(flags);
6025 
6026 		req->enables =
6027 			cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
6028 				    VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
6029 				    VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
6030 
6031 		/* Number of segs are log2 units, and first packet is not
6032 		 * included as part of this units.
6033 		 */
6034 		if (mss <= BNXT_RX_PAGE_SIZE) {
6035 			n = BNXT_RX_PAGE_SIZE / mss;
6036 			nsegs = (MAX_SKB_FRAGS - 1) * n;
6037 		} else {
6038 			n = mss / BNXT_RX_PAGE_SIZE;
6039 			if (mss & (BNXT_RX_PAGE_SIZE - 1))
6040 				n++;
6041 			nsegs = (MAX_SKB_FRAGS - n) / n;
6042 		}
6043 
6044 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6045 			segs = MAX_TPA_SEGS_P5;
6046 			max_aggs = bp->max_tpa;
6047 		} else {
6048 			segs = ilog2(nsegs);
6049 		}
6050 		req->max_agg_segs = cpu_to_le16(segs);
6051 		req->max_aggs = cpu_to_le16(max_aggs);
6052 
6053 		req->min_agg_len = cpu_to_le32(512);
6054 		bnxt_hwrm_vnic_update_tunl_tpa(bp, req);
6055 	}
6056 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6057 
6058 	return hwrm_req_send(bp, req);
6059 }
6060 
6061 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
6062 {
6063 	struct bnxt_ring_grp_info *grp_info;
6064 
6065 	grp_info = &bp->grp_info[ring->grp_idx];
6066 	return grp_info->cp_fw_ring_id;
6067 }
6068 
6069 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
6070 {
6071 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6072 		return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
6073 	else
6074 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
6075 }
6076 
6077 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
6078 {
6079 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6080 		return txr->tx_cpr->cp_ring_struct.fw_ring_id;
6081 	else
6082 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
6083 }
6084 
6085 int bnxt_alloc_rss_indir_tbl(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx)
6086 {
6087 	int entries;
6088 	u16 *tbl;
6089 
6090 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6091 		entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
6092 	else
6093 		entries = HW_HASH_INDEX_SIZE;
6094 
6095 	bp->rss_indir_tbl_entries = entries;
6096 	tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), GFP_KERNEL);
6097 	if (!tbl)
6098 		return -ENOMEM;
6099 
6100 	if (rss_ctx)
6101 		rss_ctx->rss_indir_tbl = tbl;
6102 	else
6103 		bp->rss_indir_tbl = tbl;
6104 
6105 	return 0;
6106 }
6107 
6108 void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx)
6109 {
6110 	u16 max_rings, max_entries, pad, i;
6111 	u16 *rss_indir_tbl;
6112 
6113 	if (!bp->rx_nr_rings)
6114 		return;
6115 
6116 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6117 		max_rings = bp->rx_nr_rings - 1;
6118 	else
6119 		max_rings = bp->rx_nr_rings;
6120 
6121 	max_entries = bnxt_get_rxfh_indir_size(bp->dev);
6122 	if (rss_ctx)
6123 		rss_indir_tbl = &rss_ctx->rss_indir_tbl[0];
6124 	else
6125 		rss_indir_tbl = &bp->rss_indir_tbl[0];
6126 
6127 	for (i = 0; i < max_entries; i++)
6128 		rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
6129 
6130 	pad = bp->rss_indir_tbl_entries - max_entries;
6131 	if (pad)
6132 		memset(&rss_indir_tbl[i], 0, pad * sizeof(u16));
6133 }
6134 
6135 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
6136 {
6137 	u16 i, tbl_size, max_ring = 0;
6138 
6139 	if (!bp->rss_indir_tbl)
6140 		return 0;
6141 
6142 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6143 	for (i = 0; i < tbl_size; i++)
6144 		max_ring = max(max_ring, bp->rss_indir_tbl[i]);
6145 	return max_ring;
6146 }
6147 
6148 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
6149 {
6150 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6151 		if (!rx_rings)
6152 			return 0;
6153 		return bnxt_calc_nr_ring_pages(rx_rings - 1,
6154 					       BNXT_RSS_TABLE_ENTRIES_P5);
6155 	}
6156 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6157 		return 2;
6158 	return 1;
6159 }
6160 
6161 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6162 {
6163 	bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
6164 	u16 i, j;
6165 
6166 	/* Fill the RSS indirection table with ring group ids */
6167 	for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
6168 		if (!no_rss)
6169 			j = bp->rss_indir_tbl[i];
6170 		vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
6171 	}
6172 }
6173 
6174 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
6175 				    struct bnxt_vnic_info *vnic)
6176 {
6177 	__le16 *ring_tbl = vnic->rss_table;
6178 	struct bnxt_rx_ring_info *rxr;
6179 	u16 tbl_size, i;
6180 
6181 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6182 
6183 	for (i = 0; i < tbl_size; i++) {
6184 		u16 ring_id, j;
6185 
6186 		if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG)
6187 			j = ethtool_rxfh_indir_default(i, bp->rx_nr_rings);
6188 		else if (vnic->flags & BNXT_VNIC_RSSCTX_FLAG)
6189 			j = vnic->rss_ctx->rss_indir_tbl[i];
6190 		else
6191 			j = bp->rss_indir_tbl[i];
6192 		rxr = &bp->rx_ring[j];
6193 
6194 		ring_id = rxr->rx_ring_struct.fw_ring_id;
6195 		*ring_tbl++ = cpu_to_le16(ring_id);
6196 		ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6197 		*ring_tbl++ = cpu_to_le16(ring_id);
6198 	}
6199 }
6200 
6201 static void
6202 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
6203 			 struct bnxt_vnic_info *vnic)
6204 {
6205 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6206 		bnxt_fill_hw_rss_tbl_p5(bp, vnic);
6207 		if (bp->flags & BNXT_FLAG_CHIP_P7)
6208 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT;
6209 	} else {
6210 		bnxt_fill_hw_rss_tbl(bp, vnic);
6211 	}
6212 
6213 	if (bp->rss_hash_delta) {
6214 		req->hash_type = cpu_to_le32(bp->rss_hash_delta);
6215 		if (bp->rss_hash_cfg & bp->rss_hash_delta)
6216 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
6217 		else
6218 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
6219 	} else {
6220 		req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
6221 	}
6222 	req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
6223 	req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
6224 	req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
6225 }
6226 
6227 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6228 				  bool set_rss)
6229 {
6230 	struct hwrm_vnic_rss_cfg_input *req;
6231 	int rc;
6232 
6233 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ||
6234 	    vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
6235 		return 0;
6236 
6237 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6238 	if (rc)
6239 		return rc;
6240 
6241 	if (set_rss)
6242 		__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6243 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6244 	return hwrm_req_send(bp, req);
6245 }
6246 
6247 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp,
6248 				     struct bnxt_vnic_info *vnic, bool set_rss)
6249 {
6250 	struct hwrm_vnic_rss_cfg_input *req;
6251 	dma_addr_t ring_tbl_map;
6252 	u32 i, nr_ctxs;
6253 	int rc;
6254 
6255 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6256 	if (rc)
6257 		return rc;
6258 
6259 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6260 	if (!set_rss)
6261 		return hwrm_req_send(bp, req);
6262 
6263 	__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6264 	ring_tbl_map = vnic->rss_table_dma_addr;
6265 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
6266 
6267 	hwrm_req_hold(bp, req);
6268 	for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
6269 		req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
6270 		req->ring_table_pair_index = i;
6271 		req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
6272 		rc = hwrm_req_send(bp, req);
6273 		if (rc)
6274 			goto exit;
6275 	}
6276 
6277 exit:
6278 	hwrm_req_drop(bp, req);
6279 	return rc;
6280 }
6281 
6282 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
6283 {
6284 	struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6285 	struct hwrm_vnic_rss_qcfg_output *resp;
6286 	struct hwrm_vnic_rss_qcfg_input *req;
6287 
6288 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
6289 		return;
6290 
6291 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6292 	/* all contexts configured to same hash_type, zero always exists */
6293 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6294 	resp = hwrm_req_hold(bp, req);
6295 	if (!hwrm_req_send(bp, req)) {
6296 		bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
6297 		bp->rss_hash_delta = 0;
6298 	}
6299 	hwrm_req_drop(bp, req);
6300 }
6301 
6302 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6303 {
6304 	struct hwrm_vnic_plcmodes_cfg_input *req;
6305 	int rc;
6306 
6307 	rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
6308 	if (rc)
6309 		return rc;
6310 
6311 	req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
6312 	req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
6313 
6314 	if (BNXT_RX_PAGE_MODE(bp)) {
6315 		req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
6316 	} else {
6317 		req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
6318 					  VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
6319 		req->enables |=
6320 			cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
6321 		req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
6322 		req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
6323 	}
6324 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6325 	return hwrm_req_send(bp, req);
6326 }
6327 
6328 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp,
6329 					struct bnxt_vnic_info *vnic,
6330 					u16 ctx_idx)
6331 {
6332 	struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
6333 
6334 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
6335 		return;
6336 
6337 	req->rss_cos_lb_ctx_id =
6338 		cpu_to_le16(vnic->fw_rss_cos_lb_ctx[ctx_idx]);
6339 
6340 	hwrm_req_send(bp, req);
6341 	vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
6342 }
6343 
6344 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
6345 {
6346 	int i, j;
6347 
6348 	for (i = 0; i < bp->nr_vnics; i++) {
6349 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6350 
6351 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
6352 			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
6353 				bnxt_hwrm_vnic_ctx_free_one(bp, vnic, j);
6354 		}
6355 	}
6356 	bp->rsscos_nr_ctxs = 0;
6357 }
6358 
6359 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp,
6360 				    struct bnxt_vnic_info *vnic, u16 ctx_idx)
6361 {
6362 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
6363 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
6364 	int rc;
6365 
6366 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
6367 	if (rc)
6368 		return rc;
6369 
6370 	resp = hwrm_req_hold(bp, req);
6371 	rc = hwrm_req_send(bp, req);
6372 	if (!rc)
6373 		vnic->fw_rss_cos_lb_ctx[ctx_idx] =
6374 			le16_to_cpu(resp->rss_cos_lb_ctx_id);
6375 	hwrm_req_drop(bp, req);
6376 
6377 	return rc;
6378 }
6379 
6380 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
6381 {
6382 	if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
6383 		return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
6384 	return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
6385 }
6386 
6387 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6388 {
6389 	struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6390 	struct hwrm_vnic_cfg_input *req;
6391 	unsigned int ring = 0, grp_idx;
6392 	u16 def_vlan = 0;
6393 	int rc;
6394 
6395 	rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
6396 	if (rc)
6397 		return rc;
6398 
6399 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6400 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
6401 
6402 		req->default_rx_ring_id =
6403 			cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
6404 		req->default_cmpl_ring_id =
6405 			cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
6406 		req->enables =
6407 			cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
6408 				    VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
6409 		goto vnic_mru;
6410 	}
6411 	req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
6412 	/* Only RSS support for now TBD: COS & LB */
6413 	if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
6414 		req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6415 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6416 					   VNIC_CFG_REQ_ENABLES_MRU);
6417 	} else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
6418 		req->rss_rule = cpu_to_le16(vnic0->fw_rss_cos_lb_ctx[0]);
6419 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6420 					   VNIC_CFG_REQ_ENABLES_MRU);
6421 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
6422 	} else {
6423 		req->rss_rule = cpu_to_le16(0xffff);
6424 	}
6425 
6426 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
6427 	    (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
6428 		req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
6429 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
6430 	} else {
6431 		req->cos_rule = cpu_to_le16(0xffff);
6432 	}
6433 
6434 	if (vnic->flags & BNXT_VNIC_RSS_FLAG)
6435 		ring = 0;
6436 	else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
6437 		ring = vnic->vnic_id - 1;
6438 	else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
6439 		ring = bp->rx_nr_rings - 1;
6440 
6441 	grp_idx = bp->rx_ring[ring].bnapi->index;
6442 	req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
6443 	req->lb_rule = cpu_to_le16(0xffff);
6444 vnic_mru:
6445 	req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
6446 
6447 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6448 #ifdef CONFIG_BNXT_SRIOV
6449 	if (BNXT_VF(bp))
6450 		def_vlan = bp->vf.vlan;
6451 #endif
6452 	if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
6453 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
6454 	if (vnic->vnic_id == BNXT_VNIC_DEFAULT && bnxt_ulp_registered(bp->edev))
6455 		req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
6456 
6457 	return hwrm_req_send(bp, req);
6458 }
6459 
6460 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp,
6461 				    struct bnxt_vnic_info *vnic)
6462 {
6463 	if (vnic->fw_vnic_id != INVALID_HW_RING_ID) {
6464 		struct hwrm_vnic_free_input *req;
6465 
6466 		if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
6467 			return;
6468 
6469 		req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6470 
6471 		hwrm_req_send(bp, req);
6472 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
6473 	}
6474 }
6475 
6476 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
6477 {
6478 	u16 i;
6479 
6480 	for (i = 0; i < bp->nr_vnics; i++)
6481 		bnxt_hwrm_vnic_free_one(bp, &bp->vnic_info[i]);
6482 }
6483 
6484 int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6485 			 unsigned int start_rx_ring_idx,
6486 			 unsigned int nr_rings)
6487 {
6488 	unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
6489 	struct hwrm_vnic_alloc_output *resp;
6490 	struct hwrm_vnic_alloc_input *req;
6491 	int rc;
6492 
6493 	rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
6494 	if (rc)
6495 		return rc;
6496 
6497 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6498 		goto vnic_no_ring_grps;
6499 
6500 	/* map ring groups to this vnic */
6501 	for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
6502 		grp_idx = bp->rx_ring[i].bnapi->index;
6503 		if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
6504 			netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
6505 				   j, nr_rings);
6506 			break;
6507 		}
6508 		vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
6509 	}
6510 
6511 vnic_no_ring_grps:
6512 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
6513 		vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
6514 	if (vnic->vnic_id == BNXT_VNIC_DEFAULT)
6515 		req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
6516 
6517 	resp = hwrm_req_hold(bp, req);
6518 	rc = hwrm_req_send(bp, req);
6519 	if (!rc)
6520 		vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
6521 	hwrm_req_drop(bp, req);
6522 	return rc;
6523 }
6524 
6525 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
6526 {
6527 	struct hwrm_vnic_qcaps_output *resp;
6528 	struct hwrm_vnic_qcaps_input *req;
6529 	int rc;
6530 
6531 	bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
6532 	bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP;
6533 	bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP;
6534 	if (bp->hwrm_spec_code < 0x10600)
6535 		return 0;
6536 
6537 	rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
6538 	if (rc)
6539 		return rc;
6540 
6541 	resp = hwrm_req_hold(bp, req);
6542 	rc = hwrm_req_send(bp, req);
6543 	if (!rc) {
6544 		u32 flags = le32_to_cpu(resp->flags);
6545 
6546 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
6547 		    (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
6548 			bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP;
6549 		if (flags &
6550 		    VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
6551 			bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
6552 
6553 		/* Older P5 fw before EXT_HW_STATS support did not set
6554 		 * VLAN_STRIP_CAP properly.
6555 		 */
6556 		if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
6557 		    (BNXT_CHIP_P5(bp) &&
6558 		     !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
6559 			bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
6560 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
6561 			bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA;
6562 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED)
6563 			bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM;
6564 		bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
6565 		if (bp->max_tpa_v2) {
6566 			if (BNXT_CHIP_P5(bp))
6567 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
6568 			else
6569 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7;
6570 		}
6571 		if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP)
6572 			bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA;
6573 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP)
6574 			bp->rss_cap |= BNXT_RSS_CAP_AH_V4_RSS_CAP;
6575 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP)
6576 			bp->rss_cap |= BNXT_RSS_CAP_AH_V6_RSS_CAP;
6577 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP)
6578 			bp->rss_cap |= BNXT_RSS_CAP_ESP_V4_RSS_CAP;
6579 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP)
6580 			bp->rss_cap |= BNXT_RSS_CAP_ESP_V6_RSS_CAP;
6581 	}
6582 	hwrm_req_drop(bp, req);
6583 	return rc;
6584 }
6585 
6586 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
6587 {
6588 	struct hwrm_ring_grp_alloc_output *resp;
6589 	struct hwrm_ring_grp_alloc_input *req;
6590 	int rc;
6591 	u16 i;
6592 
6593 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6594 		return 0;
6595 
6596 	rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
6597 	if (rc)
6598 		return rc;
6599 
6600 	resp = hwrm_req_hold(bp, req);
6601 	for (i = 0; i < bp->rx_nr_rings; i++) {
6602 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
6603 
6604 		req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
6605 		req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
6606 		req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
6607 		req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
6608 
6609 		rc = hwrm_req_send(bp, req);
6610 
6611 		if (rc)
6612 			break;
6613 
6614 		bp->grp_info[grp_idx].fw_grp_id =
6615 			le32_to_cpu(resp->ring_group_id);
6616 	}
6617 	hwrm_req_drop(bp, req);
6618 	return rc;
6619 }
6620 
6621 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
6622 {
6623 	struct hwrm_ring_grp_free_input *req;
6624 	u16 i;
6625 
6626 	if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
6627 		return;
6628 
6629 	if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
6630 		return;
6631 
6632 	hwrm_req_hold(bp, req);
6633 	for (i = 0; i < bp->cp_nr_rings; i++) {
6634 		if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
6635 			continue;
6636 		req->ring_group_id =
6637 			cpu_to_le32(bp->grp_info[i].fw_grp_id);
6638 
6639 		hwrm_req_send(bp, req);
6640 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
6641 	}
6642 	hwrm_req_drop(bp, req);
6643 }
6644 
6645 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
6646 				    struct bnxt_ring_struct *ring,
6647 				    u32 ring_type, u32 map_index)
6648 {
6649 	struct hwrm_ring_alloc_output *resp;
6650 	struct hwrm_ring_alloc_input *req;
6651 	struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
6652 	struct bnxt_ring_grp_info *grp_info;
6653 	int rc, err = 0;
6654 	u16 ring_id;
6655 
6656 	rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
6657 	if (rc)
6658 		goto exit;
6659 
6660 	req->enables = 0;
6661 	if (rmem->nr_pages > 1) {
6662 		req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
6663 		/* Page size is in log2 units */
6664 		req->page_size = BNXT_PAGE_SHIFT;
6665 		req->page_tbl_depth = 1;
6666 	} else {
6667 		req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
6668 	}
6669 	req->fbo = 0;
6670 	/* Association of ring index with doorbell index and MSIX number */
6671 	req->logical_id = cpu_to_le16(map_index);
6672 
6673 	switch (ring_type) {
6674 	case HWRM_RING_ALLOC_TX: {
6675 		struct bnxt_tx_ring_info *txr;
6676 
6677 		txr = container_of(ring, struct bnxt_tx_ring_info,
6678 				   tx_ring_struct);
6679 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
6680 		/* Association of transmit ring with completion ring */
6681 		grp_info = &bp->grp_info[ring->grp_idx];
6682 		req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
6683 		req->length = cpu_to_le32(bp->tx_ring_mask + 1);
6684 		req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6685 		req->queue_id = cpu_to_le16(ring->queue_id);
6686 		if (bp->flags & BNXT_FLAG_TX_COAL_CMPL)
6687 			req->cmpl_coal_cnt =
6688 				RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64;
6689 		break;
6690 	}
6691 	case HWRM_RING_ALLOC_RX:
6692 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6693 		req->length = cpu_to_le32(bp->rx_ring_mask + 1);
6694 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6695 			u16 flags = 0;
6696 
6697 			/* Association of rx ring with stats context */
6698 			grp_info = &bp->grp_info[ring->grp_idx];
6699 			req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
6700 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6701 			req->enables |= cpu_to_le32(
6702 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6703 			if (NET_IP_ALIGN == 2)
6704 				flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
6705 			req->flags = cpu_to_le16(flags);
6706 		}
6707 		break;
6708 	case HWRM_RING_ALLOC_AGG:
6709 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6710 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
6711 			/* Association of agg ring with rx ring */
6712 			grp_info = &bp->grp_info[ring->grp_idx];
6713 			req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
6714 			req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
6715 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6716 			req->enables |= cpu_to_le32(
6717 				RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
6718 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6719 		} else {
6720 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6721 		}
6722 		req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
6723 		break;
6724 	case HWRM_RING_ALLOC_CMPL:
6725 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
6726 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6727 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6728 			/* Association of cp ring with nq */
6729 			grp_info = &bp->grp_info[map_index];
6730 			req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
6731 			req->cq_handle = cpu_to_le64(ring->handle);
6732 			req->enables |= cpu_to_le32(
6733 				RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
6734 		} else if (bp->flags & BNXT_FLAG_USING_MSIX) {
6735 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6736 		}
6737 		break;
6738 	case HWRM_RING_ALLOC_NQ:
6739 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
6740 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6741 		if (bp->flags & BNXT_FLAG_USING_MSIX)
6742 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6743 		break;
6744 	default:
6745 		netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
6746 			   ring_type);
6747 		return -1;
6748 	}
6749 
6750 	resp = hwrm_req_hold(bp, req);
6751 	rc = hwrm_req_send(bp, req);
6752 	err = le16_to_cpu(resp->error_code);
6753 	ring_id = le16_to_cpu(resp->ring_id);
6754 	hwrm_req_drop(bp, req);
6755 
6756 exit:
6757 	if (rc || err) {
6758 		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
6759 			   ring_type, rc, err);
6760 		return -EIO;
6761 	}
6762 	ring->fw_ring_id = ring_id;
6763 	return rc;
6764 }
6765 
6766 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
6767 {
6768 	int rc;
6769 
6770 	if (BNXT_PF(bp)) {
6771 		struct hwrm_func_cfg_input *req;
6772 
6773 		rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
6774 		if (rc)
6775 			return rc;
6776 
6777 		req->fid = cpu_to_le16(0xffff);
6778 		req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6779 		req->async_event_cr = cpu_to_le16(idx);
6780 		return hwrm_req_send(bp, req);
6781 	} else {
6782 		struct hwrm_func_vf_cfg_input *req;
6783 
6784 		rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
6785 		if (rc)
6786 			return rc;
6787 
6788 		req->enables =
6789 			cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6790 		req->async_event_cr = cpu_to_le16(idx);
6791 		return hwrm_req_send(bp, req);
6792 	}
6793 }
6794 
6795 static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db,
6796 			     u32 ring_type)
6797 {
6798 	switch (ring_type) {
6799 	case HWRM_RING_ALLOC_TX:
6800 		db->db_ring_mask = bp->tx_ring_mask;
6801 		break;
6802 	case HWRM_RING_ALLOC_RX:
6803 		db->db_ring_mask = bp->rx_ring_mask;
6804 		break;
6805 	case HWRM_RING_ALLOC_AGG:
6806 		db->db_ring_mask = bp->rx_agg_ring_mask;
6807 		break;
6808 	case HWRM_RING_ALLOC_CMPL:
6809 	case HWRM_RING_ALLOC_NQ:
6810 		db->db_ring_mask = bp->cp_ring_mask;
6811 		break;
6812 	}
6813 	if (bp->flags & BNXT_FLAG_CHIP_P7) {
6814 		db->db_epoch_mask = db->db_ring_mask + 1;
6815 		db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
6816 	}
6817 }
6818 
6819 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
6820 			u32 map_idx, u32 xid)
6821 {
6822 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6823 		switch (ring_type) {
6824 		case HWRM_RING_ALLOC_TX:
6825 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
6826 			break;
6827 		case HWRM_RING_ALLOC_RX:
6828 		case HWRM_RING_ALLOC_AGG:
6829 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
6830 			break;
6831 		case HWRM_RING_ALLOC_CMPL:
6832 			db->db_key64 = DBR_PATH_L2;
6833 			break;
6834 		case HWRM_RING_ALLOC_NQ:
6835 			db->db_key64 = DBR_PATH_L2;
6836 			break;
6837 		}
6838 		db->db_key64 |= (u64)xid << DBR_XID_SFT;
6839 
6840 		if (bp->flags & BNXT_FLAG_CHIP_P7)
6841 			db->db_key64 |= DBR_VALID;
6842 
6843 		db->doorbell = bp->bar1 + bp->db_offset;
6844 	} else {
6845 		db->doorbell = bp->bar1 + map_idx * 0x80;
6846 		switch (ring_type) {
6847 		case HWRM_RING_ALLOC_TX:
6848 			db->db_key32 = DB_KEY_TX;
6849 			break;
6850 		case HWRM_RING_ALLOC_RX:
6851 		case HWRM_RING_ALLOC_AGG:
6852 			db->db_key32 = DB_KEY_RX;
6853 			break;
6854 		case HWRM_RING_ALLOC_CMPL:
6855 			db->db_key32 = DB_KEY_CP;
6856 			break;
6857 		}
6858 	}
6859 	bnxt_set_db_mask(bp, db, ring_type);
6860 }
6861 
6862 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
6863 {
6864 	bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
6865 	int i, rc = 0;
6866 	u32 type;
6867 
6868 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6869 		type = HWRM_RING_ALLOC_NQ;
6870 	else
6871 		type = HWRM_RING_ALLOC_CMPL;
6872 	for (i = 0; i < bp->cp_nr_rings; i++) {
6873 		struct bnxt_napi *bnapi = bp->bnapi[i];
6874 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6875 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
6876 		u32 map_idx = ring->map_idx;
6877 		unsigned int vector;
6878 
6879 		vector = bp->irq_tbl[map_idx].vector;
6880 		disable_irq_nosync(vector);
6881 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6882 		if (rc) {
6883 			enable_irq(vector);
6884 			goto err_out;
6885 		}
6886 		bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
6887 		bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
6888 		enable_irq(vector);
6889 		bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
6890 
6891 		if (!i) {
6892 			rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
6893 			if (rc)
6894 				netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
6895 		}
6896 	}
6897 
6898 	type = HWRM_RING_ALLOC_TX;
6899 	for (i = 0; i < bp->tx_nr_rings; i++) {
6900 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6901 		struct bnxt_ring_struct *ring;
6902 		u32 map_idx;
6903 
6904 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6905 			struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
6906 			struct bnxt_napi *bnapi = txr->bnapi;
6907 			u32 type2 = HWRM_RING_ALLOC_CMPL;
6908 
6909 			ring = &cpr2->cp_ring_struct;
6910 			ring->handle = BNXT_SET_NQ_HDL(cpr2);
6911 			map_idx = bnapi->index;
6912 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6913 			if (rc)
6914 				goto err_out;
6915 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6916 				    ring->fw_ring_id);
6917 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6918 		}
6919 		ring = &txr->tx_ring_struct;
6920 		map_idx = i;
6921 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6922 		if (rc)
6923 			goto err_out;
6924 		bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
6925 	}
6926 
6927 	type = HWRM_RING_ALLOC_RX;
6928 	for (i = 0; i < bp->rx_nr_rings; i++) {
6929 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6930 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6931 		struct bnxt_napi *bnapi = rxr->bnapi;
6932 		u32 map_idx = bnapi->index;
6933 
6934 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6935 		if (rc)
6936 			goto err_out;
6937 		bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
6938 		/* If we have agg rings, post agg buffers first. */
6939 		if (!agg_rings)
6940 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6941 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
6942 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6943 			struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
6944 			u32 type2 = HWRM_RING_ALLOC_CMPL;
6945 
6946 			ring = &cpr2->cp_ring_struct;
6947 			ring->handle = BNXT_SET_NQ_HDL(cpr2);
6948 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6949 			if (rc)
6950 				goto err_out;
6951 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6952 				    ring->fw_ring_id);
6953 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6954 		}
6955 	}
6956 
6957 	if (agg_rings) {
6958 		type = HWRM_RING_ALLOC_AGG;
6959 		for (i = 0; i < bp->rx_nr_rings; i++) {
6960 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6961 			struct bnxt_ring_struct *ring =
6962 						&rxr->rx_agg_ring_struct;
6963 			u32 grp_idx = ring->grp_idx;
6964 			u32 map_idx = grp_idx + bp->rx_nr_rings;
6965 
6966 			rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6967 			if (rc)
6968 				goto err_out;
6969 
6970 			bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
6971 				    ring->fw_ring_id);
6972 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
6973 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6974 			bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
6975 		}
6976 	}
6977 err_out:
6978 	return rc;
6979 }
6980 
6981 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6982 				   struct bnxt_ring_struct *ring,
6983 				   u32 ring_type, int cmpl_ring_id)
6984 {
6985 	struct hwrm_ring_free_output *resp;
6986 	struct hwrm_ring_free_input *req;
6987 	u16 error_code = 0;
6988 	int rc;
6989 
6990 	if (BNXT_NO_FW_ACCESS(bp))
6991 		return 0;
6992 
6993 	rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6994 	if (rc)
6995 		goto exit;
6996 
6997 	req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6998 	req->ring_type = ring_type;
6999 	req->ring_id = cpu_to_le16(ring->fw_ring_id);
7000 
7001 	resp = hwrm_req_hold(bp, req);
7002 	rc = hwrm_req_send(bp, req);
7003 	error_code = le16_to_cpu(resp->error_code);
7004 	hwrm_req_drop(bp, req);
7005 exit:
7006 	if (rc || error_code) {
7007 		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
7008 			   ring_type, rc, error_code);
7009 		return -EIO;
7010 	}
7011 	return 0;
7012 }
7013 
7014 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
7015 {
7016 	u32 type;
7017 	int i;
7018 
7019 	if (!bp->bnapi)
7020 		return;
7021 
7022 	for (i = 0; i < bp->tx_nr_rings; i++) {
7023 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7024 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
7025 
7026 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7027 			u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
7028 
7029 			hwrm_ring_free_send_msg(bp, ring,
7030 						RING_FREE_REQ_RING_TYPE_TX,
7031 						close_path ? cmpl_ring_id :
7032 						INVALID_HW_RING_ID);
7033 			ring->fw_ring_id = INVALID_HW_RING_ID;
7034 		}
7035 	}
7036 
7037 	for (i = 0; i < bp->rx_nr_rings; i++) {
7038 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
7039 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
7040 		u32 grp_idx = rxr->bnapi->index;
7041 
7042 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7043 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7044 
7045 			hwrm_ring_free_send_msg(bp, ring,
7046 						RING_FREE_REQ_RING_TYPE_RX,
7047 						close_path ? cmpl_ring_id :
7048 						INVALID_HW_RING_ID);
7049 			ring->fw_ring_id = INVALID_HW_RING_ID;
7050 			bp->grp_info[grp_idx].rx_fw_ring_id =
7051 				INVALID_HW_RING_ID;
7052 		}
7053 	}
7054 
7055 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7056 		type = RING_FREE_REQ_RING_TYPE_RX_AGG;
7057 	else
7058 		type = RING_FREE_REQ_RING_TYPE_RX;
7059 	for (i = 0; i < bp->rx_nr_rings; i++) {
7060 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
7061 		struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
7062 		u32 grp_idx = rxr->bnapi->index;
7063 
7064 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7065 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7066 
7067 			hwrm_ring_free_send_msg(bp, ring, type,
7068 						close_path ? cmpl_ring_id :
7069 						INVALID_HW_RING_ID);
7070 			ring->fw_ring_id = INVALID_HW_RING_ID;
7071 			bp->grp_info[grp_idx].agg_fw_ring_id =
7072 				INVALID_HW_RING_ID;
7073 		}
7074 	}
7075 
7076 	/* The completion rings are about to be freed.  After that the
7077 	 * IRQ doorbell will not work anymore.  So we need to disable
7078 	 * IRQ here.
7079 	 */
7080 	bnxt_disable_int_sync(bp);
7081 
7082 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7083 		type = RING_FREE_REQ_RING_TYPE_NQ;
7084 	else
7085 		type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
7086 	for (i = 0; i < bp->cp_nr_rings; i++) {
7087 		struct bnxt_napi *bnapi = bp->bnapi[i];
7088 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7089 		struct bnxt_ring_struct *ring;
7090 		int j;
7091 
7092 		for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
7093 			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
7094 
7095 			ring = &cpr2->cp_ring_struct;
7096 			if (ring->fw_ring_id == INVALID_HW_RING_ID)
7097 				continue;
7098 			hwrm_ring_free_send_msg(bp, ring,
7099 						RING_FREE_REQ_RING_TYPE_L2_CMPL,
7100 						INVALID_HW_RING_ID);
7101 			ring->fw_ring_id = INVALID_HW_RING_ID;
7102 		}
7103 		ring = &cpr->cp_ring_struct;
7104 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7105 			hwrm_ring_free_send_msg(bp, ring, type,
7106 						INVALID_HW_RING_ID);
7107 			ring->fw_ring_id = INVALID_HW_RING_ID;
7108 			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
7109 		}
7110 	}
7111 }
7112 
7113 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7114 			     bool shared);
7115 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7116 			   bool shared);
7117 
7118 static int bnxt_hwrm_get_rings(struct bnxt *bp)
7119 {
7120 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7121 	struct hwrm_func_qcfg_output *resp;
7122 	struct hwrm_func_qcfg_input *req;
7123 	int rc;
7124 
7125 	if (bp->hwrm_spec_code < 0x10601)
7126 		return 0;
7127 
7128 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7129 	if (rc)
7130 		return rc;
7131 
7132 	req->fid = cpu_to_le16(0xffff);
7133 	resp = hwrm_req_hold(bp, req);
7134 	rc = hwrm_req_send(bp, req);
7135 	if (rc) {
7136 		hwrm_req_drop(bp, req);
7137 		return rc;
7138 	}
7139 
7140 	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7141 	if (BNXT_NEW_RM(bp)) {
7142 		u16 cp, stats;
7143 
7144 		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
7145 		hw_resc->resv_hw_ring_grps =
7146 			le32_to_cpu(resp->alloc_hw_ring_grps);
7147 		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
7148 		hw_resc->resv_rsscos_ctxs = le16_to_cpu(resp->alloc_rsscos_ctx);
7149 		cp = le16_to_cpu(resp->alloc_cmpl_rings);
7150 		stats = le16_to_cpu(resp->alloc_stat_ctx);
7151 		hw_resc->resv_irqs = cp;
7152 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7153 			int rx = hw_resc->resv_rx_rings;
7154 			int tx = hw_resc->resv_tx_rings;
7155 
7156 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
7157 				rx >>= 1;
7158 			if (cp < (rx + tx)) {
7159 				rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false);
7160 				if (rc)
7161 					goto get_rings_exit;
7162 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
7163 					rx <<= 1;
7164 				hw_resc->resv_rx_rings = rx;
7165 				hw_resc->resv_tx_rings = tx;
7166 			}
7167 			hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
7168 			hw_resc->resv_hw_ring_grps = rx;
7169 		}
7170 		hw_resc->resv_cp_rings = cp;
7171 		hw_resc->resv_stat_ctxs = stats;
7172 	}
7173 get_rings_exit:
7174 	hwrm_req_drop(bp, req);
7175 	return rc;
7176 }
7177 
7178 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
7179 {
7180 	struct hwrm_func_qcfg_output *resp;
7181 	struct hwrm_func_qcfg_input *req;
7182 	int rc;
7183 
7184 	if (bp->hwrm_spec_code < 0x10601)
7185 		return 0;
7186 
7187 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7188 	if (rc)
7189 		return rc;
7190 
7191 	req->fid = cpu_to_le16(fid);
7192 	resp = hwrm_req_hold(bp, req);
7193 	rc = hwrm_req_send(bp, req);
7194 	if (!rc)
7195 		*tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7196 
7197 	hwrm_req_drop(bp, req);
7198 	return rc;
7199 }
7200 
7201 static bool bnxt_rfs_supported(struct bnxt *bp);
7202 
7203 static struct hwrm_func_cfg_input *
7204 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7205 {
7206 	struct hwrm_func_cfg_input *req;
7207 	u32 enables = 0;
7208 
7209 	if (bnxt_hwrm_func_cfg_short_req_init(bp, &req))
7210 		return NULL;
7211 
7212 	req->fid = cpu_to_le16(0xffff);
7213 	enables |= hwr->tx ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7214 	req->num_tx_rings = cpu_to_le16(hwr->tx);
7215 	if (BNXT_NEW_RM(bp)) {
7216 		enables |= hwr->rx ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
7217 		enables |= hwr->stat ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7218 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7219 			enables |= hwr->cp ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
7220 			enables |= hwr->cp_p5 ?
7221 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7222 		} else {
7223 			enables |= hwr->cp ?
7224 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7225 			enables |= hwr->grp ?
7226 				   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7227 		}
7228 		enables |= hwr->vnic ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
7229 		enables |= hwr->rss_ctx ? FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS :
7230 					  0;
7231 		req->num_rx_rings = cpu_to_le16(hwr->rx);
7232 		req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7233 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7234 			req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7235 			req->num_msix = cpu_to_le16(hwr->cp);
7236 		} else {
7237 			req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7238 			req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7239 		}
7240 		req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7241 		req->num_vnics = cpu_to_le16(hwr->vnic);
7242 	}
7243 	req->enables = cpu_to_le32(enables);
7244 	return req;
7245 }
7246 
7247 static struct hwrm_func_vf_cfg_input *
7248 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7249 {
7250 	struct hwrm_func_vf_cfg_input *req;
7251 	u32 enables = 0;
7252 
7253 	if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
7254 		return NULL;
7255 
7256 	enables |= hwr->tx ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7257 	enables |= hwr->rx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
7258 			     FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7259 	enables |= hwr->stat ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7260 	enables |= hwr->rss_ctx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7261 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7262 		enables |= hwr->cp_p5 ?
7263 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7264 	} else {
7265 		enables |= hwr->cp ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7266 		enables |= hwr->grp ?
7267 			   FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7268 	}
7269 	enables |= hwr->vnic ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
7270 	enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
7271 
7272 	req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
7273 	req->num_tx_rings = cpu_to_le16(hwr->tx);
7274 	req->num_rx_rings = cpu_to_le16(hwr->rx);
7275 	req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7276 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7277 		req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7278 	} else {
7279 		req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7280 		req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7281 	}
7282 	req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7283 	req->num_vnics = cpu_to_le16(hwr->vnic);
7284 
7285 	req->enables = cpu_to_le32(enables);
7286 	return req;
7287 }
7288 
7289 static int
7290 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7291 {
7292 	struct hwrm_func_cfg_input *req;
7293 	int rc;
7294 
7295 	req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7296 	if (!req)
7297 		return -ENOMEM;
7298 
7299 	if (!req->enables) {
7300 		hwrm_req_drop(bp, req);
7301 		return 0;
7302 	}
7303 
7304 	rc = hwrm_req_send(bp, req);
7305 	if (rc)
7306 		return rc;
7307 
7308 	if (bp->hwrm_spec_code < 0x10601)
7309 		bp->hw_resc.resv_tx_rings = hwr->tx;
7310 
7311 	return bnxt_hwrm_get_rings(bp);
7312 }
7313 
7314 static int
7315 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7316 {
7317 	struct hwrm_func_vf_cfg_input *req;
7318 	int rc;
7319 
7320 	if (!BNXT_NEW_RM(bp)) {
7321 		bp->hw_resc.resv_tx_rings = hwr->tx;
7322 		return 0;
7323 	}
7324 
7325 	req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7326 	if (!req)
7327 		return -ENOMEM;
7328 
7329 	rc = hwrm_req_send(bp, req);
7330 	if (rc)
7331 		return rc;
7332 
7333 	return bnxt_hwrm_get_rings(bp);
7334 }
7335 
7336 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7337 {
7338 	if (BNXT_PF(bp))
7339 		return bnxt_hwrm_reserve_pf_rings(bp, hwr);
7340 	else
7341 		return bnxt_hwrm_reserve_vf_rings(bp, hwr);
7342 }
7343 
7344 int bnxt_nq_rings_in_use(struct bnxt *bp)
7345 {
7346 	return bp->cp_nr_rings + bnxt_get_ulp_msix_num(bp);
7347 }
7348 
7349 static int bnxt_cp_rings_in_use(struct bnxt *bp)
7350 {
7351 	int cp;
7352 
7353 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7354 		return bnxt_nq_rings_in_use(bp);
7355 
7356 	cp = bp->tx_nr_rings + bp->rx_nr_rings;
7357 	return cp;
7358 }
7359 
7360 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
7361 {
7362 	return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp);
7363 }
7364 
7365 static int bnxt_get_total_rss_ctxs(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7366 {
7367 	if (!hwr->grp)
7368 		return 0;
7369 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7370 		int rss_ctx = bnxt_get_nr_rss_ctxs(bp, hwr->grp);
7371 
7372 		if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7373 			rss_ctx *= hwr->vnic;
7374 		return rss_ctx;
7375 	}
7376 	if (BNXT_VF(bp))
7377 		return BNXT_VF_MAX_RSS_CTX;
7378 	if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && bnxt_rfs_supported(bp))
7379 		return hwr->grp + 1;
7380 	return 1;
7381 }
7382 
7383 /* Check if a default RSS map needs to be setup.  This function is only
7384  * used on older firmware that does not require reserving RX rings.
7385  */
7386 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
7387 {
7388 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7389 
7390 	/* The RSS map is valid for RX rings set to resv_rx_rings */
7391 	if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
7392 		hw_resc->resv_rx_rings = bp->rx_nr_rings;
7393 		if (!netif_is_rxfh_configured(bp->dev))
7394 			bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7395 	}
7396 }
7397 
7398 static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings)
7399 {
7400 	if (bp->flags & BNXT_FLAG_RFS) {
7401 		if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7402 			return 2 + bp->num_rss_ctx;
7403 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7404 			return rx_rings + 1;
7405 	}
7406 	return 1;
7407 }
7408 
7409 static bool bnxt_need_reserve_rings(struct bnxt *bp)
7410 {
7411 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7412 	int cp = bnxt_cp_rings_in_use(bp);
7413 	int nq = bnxt_nq_rings_in_use(bp);
7414 	int rx = bp->rx_nr_rings, stat;
7415 	int vnic, grp = rx;
7416 
7417 	if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
7418 	    bp->hwrm_spec_code >= 0x10601)
7419 		return true;
7420 
7421 	/* Old firmware does not need RX ring reservations but we still
7422 	 * need to setup a default RSS map when needed.  With new firmware
7423 	 * we go through RX ring reservations first and then set up the
7424 	 * RSS map for the successfully reserved RX rings when needed.
7425 	 */
7426 	if (!BNXT_NEW_RM(bp)) {
7427 		bnxt_check_rss_tbl_no_rmgr(bp);
7428 		return false;
7429 	}
7430 
7431 	vnic = bnxt_get_total_vnics(bp, rx);
7432 
7433 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
7434 		rx <<= 1;
7435 	stat = bnxt_get_func_stat_ctxs(bp);
7436 	if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
7437 	    hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
7438 	    (hw_resc->resv_hw_ring_grps != grp &&
7439 	     !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)))
7440 		return true;
7441 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) &&
7442 	    hw_resc->resv_irqs != nq)
7443 		return true;
7444 	return false;
7445 }
7446 
7447 static void bnxt_copy_reserved_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7448 {
7449 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7450 
7451 	hwr->tx = hw_resc->resv_tx_rings;
7452 	if (BNXT_NEW_RM(bp)) {
7453 		hwr->rx = hw_resc->resv_rx_rings;
7454 		hwr->cp = hw_resc->resv_irqs;
7455 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7456 			hwr->cp_p5 = hw_resc->resv_cp_rings;
7457 		hwr->grp = hw_resc->resv_hw_ring_grps;
7458 		hwr->vnic = hw_resc->resv_vnics;
7459 		hwr->stat = hw_resc->resv_stat_ctxs;
7460 		hwr->rss_ctx = hw_resc->resv_rsscos_ctxs;
7461 	}
7462 }
7463 
7464 static bool bnxt_rings_ok(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7465 {
7466 	return hwr->tx && hwr->rx && hwr->cp && hwr->grp && hwr->vnic &&
7467 	       hwr->stat && (hwr->cp_p5 || !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS));
7468 }
7469 
7470 static int __bnxt_reserve_rings(struct bnxt *bp)
7471 {
7472 	struct bnxt_hw_rings hwr = {0};
7473 	int cp = bp->cp_nr_rings;
7474 	int rx_rings, rc;
7475 	int ulp_msix = 0;
7476 	bool sh = false;
7477 	int tx_cp;
7478 
7479 	if (!bnxt_need_reserve_rings(bp))
7480 		return 0;
7481 
7482 	if (!bnxt_ulp_registered(bp->edev)) {
7483 		ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
7484 		if (!ulp_msix)
7485 			bnxt_set_ulp_stat_ctxs(bp, 0);
7486 
7487 		if (ulp_msix > bp->ulp_num_msix_want)
7488 			ulp_msix = bp->ulp_num_msix_want;
7489 		hwr.cp = cp + ulp_msix;
7490 	} else {
7491 		hwr.cp = bnxt_nq_rings_in_use(bp);
7492 	}
7493 
7494 	hwr.tx = bp->tx_nr_rings;
7495 	hwr.rx = bp->rx_nr_rings;
7496 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7497 		sh = true;
7498 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7499 		hwr.cp_p5 = hwr.rx + hwr.tx;
7500 
7501 	hwr.vnic = bnxt_get_total_vnics(bp, hwr.rx);
7502 
7503 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
7504 		hwr.rx <<= 1;
7505 	hwr.grp = bp->rx_nr_rings;
7506 	hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
7507 	hwr.stat = bnxt_get_func_stat_ctxs(bp);
7508 
7509 	rc = bnxt_hwrm_reserve_rings(bp, &hwr);
7510 	if (rc)
7511 		return rc;
7512 
7513 	bnxt_copy_reserved_rings(bp, &hwr);
7514 
7515 	rx_rings = hwr.rx;
7516 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7517 		if (hwr.rx >= 2) {
7518 			rx_rings = hwr.rx >> 1;
7519 		} else {
7520 			if (netif_running(bp->dev))
7521 				return -ENOMEM;
7522 
7523 			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7524 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7525 			bp->dev->hw_features &= ~NETIF_F_LRO;
7526 			bp->dev->features &= ~NETIF_F_LRO;
7527 			bnxt_set_ring_params(bp);
7528 		}
7529 	}
7530 	rx_rings = min_t(int, rx_rings, hwr.grp);
7531 	hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings);
7532 	if (hwr.stat > bnxt_get_ulp_stat_ctxs(bp))
7533 		hwr.stat -= bnxt_get_ulp_stat_ctxs(bp);
7534 	hwr.cp = min_t(int, hwr.cp, hwr.stat);
7535 	rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh);
7536 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
7537 		hwr.rx = rx_rings << 1;
7538 	tx_cp = bnxt_num_tx_to_cp(bp, hwr.tx);
7539 	hwr.cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
7540 	bp->tx_nr_rings = hwr.tx;
7541 
7542 	/* If we cannot reserve all the RX rings, reset the RSS map only
7543 	 * if absolutely necessary
7544 	 */
7545 	if (rx_rings != bp->rx_nr_rings) {
7546 		netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
7547 			    rx_rings, bp->rx_nr_rings);
7548 		if (netif_is_rxfh_configured(bp->dev) &&
7549 		    (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
7550 		     bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
7551 		     bnxt_get_max_rss_ring(bp) >= rx_rings)) {
7552 			netdev_warn(bp->dev, "RSS table entries reverting to default\n");
7553 			bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
7554 		}
7555 	}
7556 	bp->rx_nr_rings = rx_rings;
7557 	bp->cp_nr_rings = hwr.cp;
7558 
7559 	if (!bnxt_rings_ok(bp, &hwr))
7560 		return -ENOMEM;
7561 
7562 	if (!netif_is_rxfh_configured(bp->dev))
7563 		bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7564 
7565 	if (!bnxt_ulp_registered(bp->edev) && BNXT_NEW_RM(bp)) {
7566 		int resv_msix, resv_ctx, ulp_ctxs;
7567 		struct bnxt_hw_resc *hw_resc;
7568 
7569 		hw_resc = &bp->hw_resc;
7570 		resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
7571 		ulp_msix = min_t(int, resv_msix, ulp_msix);
7572 		bnxt_set_ulp_msix_num(bp, ulp_msix);
7573 		resv_ctx = hw_resc->resv_stat_ctxs  - bp->cp_nr_rings;
7574 		ulp_ctxs = min(resv_ctx, bnxt_get_ulp_stat_ctxs(bp));
7575 		bnxt_set_ulp_stat_ctxs(bp, ulp_ctxs);
7576 	}
7577 
7578 	return rc;
7579 }
7580 
7581 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7582 {
7583 	struct hwrm_func_vf_cfg_input *req;
7584 	u32 flags;
7585 
7586 	if (!BNXT_NEW_RM(bp))
7587 		return 0;
7588 
7589 	req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7590 	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
7591 		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7592 		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7593 		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7594 		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
7595 		FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
7596 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7597 		flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7598 
7599 	req->flags = cpu_to_le32(flags);
7600 	return hwrm_req_send_silent(bp, req);
7601 }
7602 
7603 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7604 {
7605 	struct hwrm_func_cfg_input *req;
7606 	u32 flags;
7607 
7608 	req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7609 	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
7610 	if (BNXT_NEW_RM(bp)) {
7611 		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7612 			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7613 			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7614 			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
7615 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7616 			flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
7617 				 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
7618 		else
7619 			flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7620 	}
7621 
7622 	req->flags = cpu_to_le32(flags);
7623 	return hwrm_req_send_silent(bp, req);
7624 }
7625 
7626 static int bnxt_hwrm_check_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7627 {
7628 	if (bp->hwrm_spec_code < 0x10801)
7629 		return 0;
7630 
7631 	if (BNXT_PF(bp))
7632 		return bnxt_hwrm_check_pf_rings(bp, hwr);
7633 
7634 	return bnxt_hwrm_check_vf_rings(bp, hwr);
7635 }
7636 
7637 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
7638 {
7639 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7640 	struct hwrm_ring_aggint_qcaps_output *resp;
7641 	struct hwrm_ring_aggint_qcaps_input *req;
7642 	int rc;
7643 
7644 	coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
7645 	coal_cap->num_cmpl_dma_aggr_max = 63;
7646 	coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
7647 	coal_cap->cmpl_aggr_dma_tmr_max = 65535;
7648 	coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
7649 	coal_cap->int_lat_tmr_min_max = 65535;
7650 	coal_cap->int_lat_tmr_max_max = 65535;
7651 	coal_cap->num_cmpl_aggr_int_max = 65535;
7652 	coal_cap->timer_units = 80;
7653 
7654 	if (bp->hwrm_spec_code < 0x10902)
7655 		return;
7656 
7657 	if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
7658 		return;
7659 
7660 	resp = hwrm_req_hold(bp, req);
7661 	rc = hwrm_req_send_silent(bp, req);
7662 	if (!rc) {
7663 		coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
7664 		coal_cap->nq_params = le32_to_cpu(resp->nq_params);
7665 		coal_cap->num_cmpl_dma_aggr_max =
7666 			le16_to_cpu(resp->num_cmpl_dma_aggr_max);
7667 		coal_cap->num_cmpl_dma_aggr_during_int_max =
7668 			le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
7669 		coal_cap->cmpl_aggr_dma_tmr_max =
7670 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
7671 		coal_cap->cmpl_aggr_dma_tmr_during_int_max =
7672 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
7673 		coal_cap->int_lat_tmr_min_max =
7674 			le16_to_cpu(resp->int_lat_tmr_min_max);
7675 		coal_cap->int_lat_tmr_max_max =
7676 			le16_to_cpu(resp->int_lat_tmr_max_max);
7677 		coal_cap->num_cmpl_aggr_int_max =
7678 			le16_to_cpu(resp->num_cmpl_aggr_int_max);
7679 		coal_cap->timer_units = le16_to_cpu(resp->timer_units);
7680 	}
7681 	hwrm_req_drop(bp, req);
7682 }
7683 
7684 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
7685 {
7686 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7687 
7688 	return usec * 1000 / coal_cap->timer_units;
7689 }
7690 
7691 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
7692 	struct bnxt_coal *hw_coal,
7693 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7694 {
7695 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7696 	u16 val, tmr, max, flags = hw_coal->flags;
7697 	u32 cmpl_params = coal_cap->cmpl_params;
7698 
7699 	max = hw_coal->bufs_per_record * 128;
7700 	if (hw_coal->budget)
7701 		max = hw_coal->bufs_per_record * hw_coal->budget;
7702 	max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
7703 
7704 	val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
7705 	req->num_cmpl_aggr_int = cpu_to_le16(val);
7706 
7707 	val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
7708 	req->num_cmpl_dma_aggr = cpu_to_le16(val);
7709 
7710 	val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
7711 		      coal_cap->num_cmpl_dma_aggr_during_int_max);
7712 	req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
7713 
7714 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
7715 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
7716 	req->int_lat_tmr_max = cpu_to_le16(tmr);
7717 
7718 	/* min timer set to 1/2 of interrupt timer */
7719 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
7720 		val = tmr / 2;
7721 		val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
7722 		req->int_lat_tmr_min = cpu_to_le16(val);
7723 		req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7724 	}
7725 
7726 	/* buf timer set to 1/4 of interrupt timer */
7727 	val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
7728 	req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
7729 
7730 	if (cmpl_params &
7731 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
7732 		tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
7733 		val = clamp_t(u16, tmr, 1,
7734 			      coal_cap->cmpl_aggr_dma_tmr_during_int_max);
7735 		req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
7736 		req->enables |=
7737 			cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
7738 	}
7739 
7740 	if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
7741 	    hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
7742 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
7743 	req->flags = cpu_to_le16(flags);
7744 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
7745 }
7746 
7747 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
7748 				   struct bnxt_coal *hw_coal)
7749 {
7750 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
7751 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7752 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7753 	u32 nq_params = coal_cap->nq_params;
7754 	u16 tmr;
7755 	int rc;
7756 
7757 	if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
7758 		return 0;
7759 
7760 	rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7761 	if (rc)
7762 		return rc;
7763 
7764 	req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
7765 	req->flags =
7766 		cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
7767 
7768 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
7769 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
7770 	req->int_lat_tmr_min = cpu_to_le16(tmr);
7771 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7772 	return hwrm_req_send(bp, req);
7773 }
7774 
7775 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
7776 {
7777 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
7778 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7779 	struct bnxt_coal coal;
7780 	int rc;
7781 
7782 	/* Tick values in micro seconds.
7783 	 * 1 coal_buf x bufs_per_record = 1 completion record.
7784 	 */
7785 	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
7786 
7787 	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
7788 	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
7789 
7790 	if (!bnapi->rx_ring)
7791 		return -ENODEV;
7792 
7793 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7794 	if (rc)
7795 		return rc;
7796 
7797 	bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
7798 
7799 	req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
7800 
7801 	return hwrm_req_send(bp, req_rx);
7802 }
7803 
7804 static int
7805 bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7806 		      struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7807 {
7808 	u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
7809 
7810 	req->ring_id = cpu_to_le16(ring_id);
7811 	return hwrm_req_send(bp, req);
7812 }
7813 
7814 static int
7815 bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7816 		      struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7817 {
7818 	struct bnxt_tx_ring_info *txr;
7819 	int i, rc;
7820 
7821 	bnxt_for_each_napi_tx(i, bnapi, txr) {
7822 		u16 ring_id;
7823 
7824 		ring_id = bnxt_cp_ring_for_tx(bp, txr);
7825 		req->ring_id = cpu_to_le16(ring_id);
7826 		rc = hwrm_req_send(bp, req);
7827 		if (rc)
7828 			return rc;
7829 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7830 			return 0;
7831 	}
7832 	return 0;
7833 }
7834 
7835 int bnxt_hwrm_set_coal(struct bnxt *bp)
7836 {
7837 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
7838 	int i, rc;
7839 
7840 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7841 	if (rc)
7842 		return rc;
7843 
7844 	rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7845 	if (rc) {
7846 		hwrm_req_drop(bp, req_rx);
7847 		return rc;
7848 	}
7849 
7850 	bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
7851 	bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
7852 
7853 	hwrm_req_hold(bp, req_rx);
7854 	hwrm_req_hold(bp, req_tx);
7855 	for (i = 0; i < bp->cp_nr_rings; i++) {
7856 		struct bnxt_napi *bnapi = bp->bnapi[i];
7857 		struct bnxt_coal *hw_coal;
7858 
7859 		if (!bnapi->rx_ring)
7860 			rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7861 		else
7862 			rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
7863 		if (rc)
7864 			break;
7865 
7866 		if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7867 			continue;
7868 
7869 		if (bnapi->rx_ring && bnapi->tx_ring[0]) {
7870 			rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7871 			if (rc)
7872 				break;
7873 		}
7874 		if (bnapi->rx_ring)
7875 			hw_coal = &bp->rx_coal;
7876 		else
7877 			hw_coal = &bp->tx_coal;
7878 		__bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
7879 	}
7880 	hwrm_req_drop(bp, req_rx);
7881 	hwrm_req_drop(bp, req_tx);
7882 	return rc;
7883 }
7884 
7885 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
7886 {
7887 	struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
7888 	struct hwrm_stat_ctx_free_input *req;
7889 	int i;
7890 
7891 	if (!bp->bnapi)
7892 		return;
7893 
7894 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7895 		return;
7896 
7897 	if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
7898 		return;
7899 	if (BNXT_FW_MAJ(bp) <= 20) {
7900 		if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
7901 			hwrm_req_drop(bp, req);
7902 			return;
7903 		}
7904 		hwrm_req_hold(bp, req0);
7905 	}
7906 	hwrm_req_hold(bp, req);
7907 	for (i = 0; i < bp->cp_nr_rings; i++) {
7908 		struct bnxt_napi *bnapi = bp->bnapi[i];
7909 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7910 
7911 		if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
7912 			req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
7913 			if (req0) {
7914 				req0->stat_ctx_id = req->stat_ctx_id;
7915 				hwrm_req_send(bp, req0);
7916 			}
7917 			hwrm_req_send(bp, req);
7918 
7919 			cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
7920 		}
7921 	}
7922 	hwrm_req_drop(bp, req);
7923 	if (req0)
7924 		hwrm_req_drop(bp, req0);
7925 }
7926 
7927 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
7928 {
7929 	struct hwrm_stat_ctx_alloc_output *resp;
7930 	struct hwrm_stat_ctx_alloc_input *req;
7931 	int rc, i;
7932 
7933 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7934 		return 0;
7935 
7936 	rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
7937 	if (rc)
7938 		return rc;
7939 
7940 	req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
7941 	req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
7942 
7943 	resp = hwrm_req_hold(bp, req);
7944 	for (i = 0; i < bp->cp_nr_rings; i++) {
7945 		struct bnxt_napi *bnapi = bp->bnapi[i];
7946 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7947 
7948 		req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
7949 
7950 		rc = hwrm_req_send(bp, req);
7951 		if (rc)
7952 			break;
7953 
7954 		cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
7955 
7956 		bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
7957 	}
7958 	hwrm_req_drop(bp, req);
7959 	return rc;
7960 }
7961 
7962 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
7963 {
7964 	struct hwrm_func_qcfg_output *resp;
7965 	struct hwrm_func_qcfg_input *req;
7966 	u16 flags;
7967 	int rc;
7968 
7969 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7970 	if (rc)
7971 		return rc;
7972 
7973 	req->fid = cpu_to_le16(0xffff);
7974 	resp = hwrm_req_hold(bp, req);
7975 	rc = hwrm_req_send(bp, req);
7976 	if (rc)
7977 		goto func_qcfg_exit;
7978 
7979 #ifdef CONFIG_BNXT_SRIOV
7980 	if (BNXT_VF(bp)) {
7981 		struct bnxt_vf_info *vf = &bp->vf;
7982 
7983 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
7984 	} else {
7985 		bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
7986 	}
7987 #endif
7988 	flags = le16_to_cpu(resp->flags);
7989 	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
7990 		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
7991 		bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
7992 		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
7993 			bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
7994 	}
7995 	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
7996 		bp->flags |= BNXT_FLAG_MULTI_HOST;
7997 
7998 	if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
7999 		bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
8000 
8001 	switch (resp->port_partition_type) {
8002 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
8003 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
8004 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
8005 		bp->port_partition_type = resp->port_partition_type;
8006 		break;
8007 	}
8008 	if (bp->hwrm_spec_code < 0x10707 ||
8009 	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
8010 		bp->br_mode = BRIDGE_MODE_VEB;
8011 	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
8012 		bp->br_mode = BRIDGE_MODE_VEPA;
8013 	else
8014 		bp->br_mode = BRIDGE_MODE_UNDEF;
8015 
8016 	bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
8017 	if (!bp->max_mtu)
8018 		bp->max_mtu = BNXT_MAX_MTU;
8019 
8020 	if (bp->db_size)
8021 		goto func_qcfg_exit;
8022 
8023 	bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024;
8024 	if (BNXT_CHIP_P5(bp)) {
8025 		if (BNXT_PF(bp))
8026 			bp->db_offset = DB_PF_OFFSET_P5;
8027 		else
8028 			bp->db_offset = DB_VF_OFFSET_P5;
8029 	}
8030 	bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
8031 				 1024);
8032 	if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
8033 	    bp->db_size <= bp->db_offset)
8034 		bp->db_size = pci_resource_len(bp->pdev, 2);
8035 
8036 func_qcfg_exit:
8037 	hwrm_req_drop(bp, req);
8038 	return rc;
8039 }
8040 
8041 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm,
8042 				      u8 init_val, u8 init_offset,
8043 				      bool init_mask_set)
8044 {
8045 	ctxm->init_value = init_val;
8046 	ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET;
8047 	if (init_mask_set)
8048 		ctxm->init_offset = init_offset * 4;
8049 	else
8050 		ctxm->init_value = 0;
8051 }
8052 
8053 static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max)
8054 {
8055 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
8056 	u16 type;
8057 
8058 	for (type = 0; type < ctx_max; type++) {
8059 		struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8060 		int n = 1;
8061 
8062 		if (!ctxm->max_entries)
8063 			continue;
8064 
8065 		if (ctxm->instance_bmap)
8066 			n = hweight32(ctxm->instance_bmap);
8067 		ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL);
8068 		if (!ctxm->pg_info)
8069 			return -ENOMEM;
8070 	}
8071 	return 0;
8072 }
8073 
8074 #define BNXT_CTX_INIT_VALID(flags)	\
8075 	(!!((flags) &			\
8076 	    FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT))
8077 
8078 static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp)
8079 {
8080 	struct hwrm_func_backing_store_qcaps_v2_output *resp;
8081 	struct hwrm_func_backing_store_qcaps_v2_input *req;
8082 	struct bnxt_ctx_mem_info *ctx;
8083 	u16 type;
8084 	int rc;
8085 
8086 	rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2);
8087 	if (rc)
8088 		return rc;
8089 
8090 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8091 	if (!ctx)
8092 		return -ENOMEM;
8093 	bp->ctx = ctx;
8094 
8095 	resp = hwrm_req_hold(bp, req);
8096 
8097 	for (type = 0; type < BNXT_CTX_V2_MAX; ) {
8098 		struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8099 		u8 init_val, init_off, i;
8100 		__le32 *p;
8101 		u32 flags;
8102 
8103 		req->type = cpu_to_le16(type);
8104 		rc = hwrm_req_send(bp, req);
8105 		if (rc)
8106 			goto ctx_done;
8107 		flags = le32_to_cpu(resp->flags);
8108 		type = le16_to_cpu(resp->next_valid_type);
8109 		if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID))
8110 			continue;
8111 
8112 		ctxm->type = le16_to_cpu(resp->type);
8113 		ctxm->entry_size = le16_to_cpu(resp->entry_size);
8114 		ctxm->flags = flags;
8115 		ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map);
8116 		ctxm->entry_multiple = resp->entry_multiple;
8117 		ctxm->max_entries = le32_to_cpu(resp->max_num_entries);
8118 		ctxm->min_entries = le32_to_cpu(resp->min_num_entries);
8119 		init_val = resp->ctx_init_value;
8120 		init_off = resp->ctx_init_offset;
8121 		bnxt_init_ctx_initializer(ctxm, init_val, init_off,
8122 					  BNXT_CTX_INIT_VALID(flags));
8123 		ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt,
8124 					      BNXT_MAX_SPLIT_ENTRY);
8125 		for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt;
8126 		     i++, p++)
8127 			ctxm->split[i] = le32_to_cpu(*p);
8128 	}
8129 	rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX);
8130 
8131 ctx_done:
8132 	hwrm_req_drop(bp, req);
8133 	return rc;
8134 }
8135 
8136 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
8137 {
8138 	struct hwrm_func_backing_store_qcaps_output *resp;
8139 	struct hwrm_func_backing_store_qcaps_input *req;
8140 	int rc;
8141 
8142 	if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
8143 		return 0;
8144 
8145 	if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8146 		return bnxt_hwrm_func_backing_store_qcaps_v2(bp);
8147 
8148 	rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
8149 	if (rc)
8150 		return rc;
8151 
8152 	resp = hwrm_req_hold(bp, req);
8153 	rc = hwrm_req_send_silent(bp, req);
8154 	if (!rc) {
8155 		struct bnxt_ctx_mem_type *ctxm;
8156 		struct bnxt_ctx_mem_info *ctx;
8157 		u8 init_val, init_idx = 0;
8158 		u16 init_mask;
8159 
8160 		ctx = bp->ctx;
8161 		if (!ctx) {
8162 			ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8163 			if (!ctx) {
8164 				rc = -ENOMEM;
8165 				goto ctx_err;
8166 			}
8167 			bp->ctx = ctx;
8168 		}
8169 		init_val = resp->ctx_kind_initializer;
8170 		init_mask = le16_to_cpu(resp->ctx_init_mask);
8171 
8172 		ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8173 		ctxm->max_entries = le32_to_cpu(resp->qp_max_entries);
8174 		ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
8175 		ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
8176 		ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries);
8177 		ctxm->entry_size = le16_to_cpu(resp->qp_entry_size);
8178 		bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset,
8179 					  (init_mask & (1 << init_idx++)) != 0);
8180 
8181 		ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8182 		ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
8183 		ctxm->max_entries = le32_to_cpu(resp->srq_max_entries);
8184 		ctxm->entry_size = le16_to_cpu(resp->srq_entry_size);
8185 		bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset,
8186 					  (init_mask & (1 << init_idx++)) != 0);
8187 
8188 		ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8189 		ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
8190 		ctxm->max_entries = le32_to_cpu(resp->cq_max_entries);
8191 		ctxm->entry_size = le16_to_cpu(resp->cq_entry_size);
8192 		bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset,
8193 					  (init_mask & (1 << init_idx++)) != 0);
8194 
8195 		ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8196 		ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries);
8197 		ctxm->max_entries = ctxm->vnic_entries +
8198 			le16_to_cpu(resp->vnic_max_ring_table_entries);
8199 		ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size);
8200 		bnxt_init_ctx_initializer(ctxm, init_val,
8201 					  resp->vnic_init_offset,
8202 					  (init_mask & (1 << init_idx++)) != 0);
8203 
8204 		ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8205 		ctxm->max_entries = le32_to_cpu(resp->stat_max_entries);
8206 		ctxm->entry_size = le16_to_cpu(resp->stat_entry_size);
8207 		bnxt_init_ctx_initializer(ctxm, init_val,
8208 					  resp->stat_init_offset,
8209 					  (init_mask & (1 << init_idx++)) != 0);
8210 
8211 		ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8212 		ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size);
8213 		ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring);
8214 		ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring);
8215 		ctxm->entry_multiple = resp->tqm_entries_multiple;
8216 		if (!ctxm->entry_multiple)
8217 			ctxm->entry_multiple = 1;
8218 
8219 		memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm));
8220 
8221 		ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8222 		ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries);
8223 		ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size);
8224 		ctxm->mrav_num_entries_units =
8225 			le16_to_cpu(resp->mrav_num_entries_units);
8226 		bnxt_init_ctx_initializer(ctxm, init_val,
8227 					  resp->mrav_init_offset,
8228 					  (init_mask & (1 << init_idx++)) != 0);
8229 
8230 		ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8231 		ctxm->entry_size = le16_to_cpu(resp->tim_entry_size);
8232 		ctxm->max_entries = le32_to_cpu(resp->tim_max_entries);
8233 
8234 		ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
8235 		if (!ctx->tqm_fp_rings_count)
8236 			ctx->tqm_fp_rings_count = bp->max_q;
8237 		else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
8238 			ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
8239 
8240 		ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8241 		memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm));
8242 		ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1;
8243 
8244 		rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX);
8245 	} else {
8246 		rc = 0;
8247 	}
8248 ctx_err:
8249 	hwrm_req_drop(bp, req);
8250 	return rc;
8251 }
8252 
8253 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
8254 				  __le64 *pg_dir)
8255 {
8256 	if (!rmem->nr_pages)
8257 		return;
8258 
8259 	BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
8260 	if (rmem->depth >= 1) {
8261 		if (rmem->depth == 2)
8262 			*pg_attr |= 2;
8263 		else
8264 			*pg_attr |= 1;
8265 		*pg_dir = cpu_to_le64(rmem->pg_tbl_map);
8266 	} else {
8267 		*pg_dir = cpu_to_le64(rmem->dma_arr[0]);
8268 	}
8269 }
8270 
8271 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
8272 	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
8273 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
8274 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
8275 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
8276 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
8277 
8278 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
8279 {
8280 	struct hwrm_func_backing_store_cfg_input *req;
8281 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
8282 	struct bnxt_ctx_pg_info *ctx_pg;
8283 	struct bnxt_ctx_mem_type *ctxm;
8284 	void **__req = (void **)&req;
8285 	u32 req_len = sizeof(*req);
8286 	__le32 *num_entries;
8287 	__le64 *pg_dir;
8288 	u32 flags = 0;
8289 	u8 *pg_attr;
8290 	u32 ena;
8291 	int rc;
8292 	int i;
8293 
8294 	if (!ctx)
8295 		return 0;
8296 
8297 	if (req_len > bp->hwrm_max_ext_req_len)
8298 		req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
8299 	rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
8300 	if (rc)
8301 		return rc;
8302 
8303 	req->enables = cpu_to_le32(enables);
8304 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
8305 		ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8306 		ctx_pg = ctxm->pg_info;
8307 		req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
8308 		req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries);
8309 		req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries);
8310 		req->qp_entry_size = cpu_to_le16(ctxm->entry_size);
8311 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8312 				      &req->qpc_pg_size_qpc_lvl,
8313 				      &req->qpc_page_dir);
8314 
8315 		if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD)
8316 			req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries);
8317 	}
8318 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
8319 		ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8320 		ctx_pg = ctxm->pg_info;
8321 		req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
8322 		req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries);
8323 		req->srq_entry_size = cpu_to_le16(ctxm->entry_size);
8324 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8325 				      &req->srq_pg_size_srq_lvl,
8326 				      &req->srq_page_dir);
8327 	}
8328 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
8329 		ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8330 		ctx_pg = ctxm->pg_info;
8331 		req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
8332 		req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries);
8333 		req->cq_entry_size = cpu_to_le16(ctxm->entry_size);
8334 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8335 				      &req->cq_pg_size_cq_lvl,
8336 				      &req->cq_page_dir);
8337 	}
8338 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
8339 		ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8340 		ctx_pg = ctxm->pg_info;
8341 		req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries);
8342 		req->vnic_num_ring_table_entries =
8343 			cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries);
8344 		req->vnic_entry_size = cpu_to_le16(ctxm->entry_size);
8345 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8346 				      &req->vnic_pg_size_vnic_lvl,
8347 				      &req->vnic_page_dir);
8348 	}
8349 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
8350 		ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8351 		ctx_pg = ctxm->pg_info;
8352 		req->stat_num_entries = cpu_to_le32(ctxm->max_entries);
8353 		req->stat_entry_size = cpu_to_le16(ctxm->entry_size);
8354 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8355 				      &req->stat_pg_size_stat_lvl,
8356 				      &req->stat_page_dir);
8357 	}
8358 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
8359 		u32 units;
8360 
8361 		ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8362 		ctx_pg = ctxm->pg_info;
8363 		req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
8364 		units = ctxm->mrav_num_entries_units;
8365 		if (units) {
8366 			u32 num_mr, num_ah = ctxm->mrav_av_entries;
8367 			u32 entries;
8368 
8369 			num_mr = ctx_pg->entries - num_ah;
8370 			entries = ((num_mr / units) << 16) | (num_ah / units);
8371 			req->mrav_num_entries = cpu_to_le32(entries);
8372 			flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
8373 		}
8374 		req->mrav_entry_size = cpu_to_le16(ctxm->entry_size);
8375 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8376 				      &req->mrav_pg_size_mrav_lvl,
8377 				      &req->mrav_page_dir);
8378 	}
8379 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
8380 		ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8381 		ctx_pg = ctxm->pg_info;
8382 		req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
8383 		req->tim_entry_size = cpu_to_le16(ctxm->entry_size);
8384 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8385 				      &req->tim_pg_size_tim_lvl,
8386 				      &req->tim_page_dir);
8387 	}
8388 	ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8389 	for (i = 0, num_entries = &req->tqm_sp_num_entries,
8390 	     pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
8391 	     pg_dir = &req->tqm_sp_page_dir,
8392 	     ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP,
8393 	     ctx_pg = ctxm->pg_info;
8394 	     i < BNXT_MAX_TQM_RINGS;
8395 	     ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i],
8396 	     i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
8397 		if (!(enables & ena))
8398 			continue;
8399 
8400 		req->tqm_entry_size = cpu_to_le16(ctxm->entry_size);
8401 		*num_entries = cpu_to_le32(ctx_pg->entries);
8402 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
8403 	}
8404 	req->flags = cpu_to_le32(flags);
8405 	return hwrm_req_send(bp, req);
8406 }
8407 
8408 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
8409 				  struct bnxt_ctx_pg_info *ctx_pg)
8410 {
8411 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8412 
8413 	rmem->page_size = BNXT_PAGE_SIZE;
8414 	rmem->pg_arr = ctx_pg->ctx_pg_arr;
8415 	rmem->dma_arr = ctx_pg->ctx_dma_arr;
8416 	rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
8417 	if (rmem->depth >= 1)
8418 		rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
8419 	return bnxt_alloc_ring(bp, rmem);
8420 }
8421 
8422 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
8423 				  struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
8424 				  u8 depth, struct bnxt_ctx_mem_type *ctxm)
8425 {
8426 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8427 	int rc;
8428 
8429 	if (!mem_size)
8430 		return -EINVAL;
8431 
8432 	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8433 	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
8434 		ctx_pg->nr_pages = 0;
8435 		return -EINVAL;
8436 	}
8437 	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
8438 		int nr_tbls, i;
8439 
8440 		rmem->depth = 2;
8441 		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
8442 					     GFP_KERNEL);
8443 		if (!ctx_pg->ctx_pg_tbl)
8444 			return -ENOMEM;
8445 		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
8446 		rmem->nr_pages = nr_tbls;
8447 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8448 		if (rc)
8449 			return rc;
8450 		for (i = 0; i < nr_tbls; i++) {
8451 			struct bnxt_ctx_pg_info *pg_tbl;
8452 
8453 			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
8454 			if (!pg_tbl)
8455 				return -ENOMEM;
8456 			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
8457 			rmem = &pg_tbl->ring_mem;
8458 			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
8459 			rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
8460 			rmem->depth = 1;
8461 			rmem->nr_pages = MAX_CTX_PAGES;
8462 			rmem->ctx_mem = ctxm;
8463 			if (i == (nr_tbls - 1)) {
8464 				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
8465 
8466 				if (rem)
8467 					rmem->nr_pages = rem;
8468 			}
8469 			rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
8470 			if (rc)
8471 				break;
8472 		}
8473 	} else {
8474 		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8475 		if (rmem->nr_pages > 1 || depth)
8476 			rmem->depth = 1;
8477 		rmem->ctx_mem = ctxm;
8478 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8479 	}
8480 	return rc;
8481 }
8482 
8483 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
8484 				  struct bnxt_ctx_pg_info *ctx_pg)
8485 {
8486 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8487 
8488 	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
8489 	    ctx_pg->ctx_pg_tbl) {
8490 		int i, nr_tbls = rmem->nr_pages;
8491 
8492 		for (i = 0; i < nr_tbls; i++) {
8493 			struct bnxt_ctx_pg_info *pg_tbl;
8494 			struct bnxt_ring_mem_info *rmem2;
8495 
8496 			pg_tbl = ctx_pg->ctx_pg_tbl[i];
8497 			if (!pg_tbl)
8498 				continue;
8499 			rmem2 = &pg_tbl->ring_mem;
8500 			bnxt_free_ring(bp, rmem2);
8501 			ctx_pg->ctx_pg_arr[i] = NULL;
8502 			kfree(pg_tbl);
8503 			ctx_pg->ctx_pg_tbl[i] = NULL;
8504 		}
8505 		kfree(ctx_pg->ctx_pg_tbl);
8506 		ctx_pg->ctx_pg_tbl = NULL;
8507 	}
8508 	bnxt_free_ring(bp, rmem);
8509 	ctx_pg->nr_pages = 0;
8510 }
8511 
8512 static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp,
8513 				   struct bnxt_ctx_mem_type *ctxm, u32 entries,
8514 				   u8 pg_lvl)
8515 {
8516 	struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8517 	int i, rc = 0, n = 1;
8518 	u32 mem_size;
8519 
8520 	if (!ctxm->entry_size || !ctx_pg)
8521 		return -EINVAL;
8522 	if (ctxm->instance_bmap)
8523 		n = hweight32(ctxm->instance_bmap);
8524 	if (ctxm->entry_multiple)
8525 		entries = roundup(entries, ctxm->entry_multiple);
8526 	entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
8527 	mem_size = entries * ctxm->entry_size;
8528 	for (i = 0; i < n && !rc; i++) {
8529 		ctx_pg[i].entries = entries;
8530 		rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl,
8531 					    ctxm->init_value ? ctxm : NULL);
8532 	}
8533 	return rc;
8534 }
8535 
8536 static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp,
8537 					       struct bnxt_ctx_mem_type *ctxm,
8538 					       bool last)
8539 {
8540 	struct hwrm_func_backing_store_cfg_v2_input *req;
8541 	u32 instance_bmap = ctxm->instance_bmap;
8542 	int i, j, rc = 0, n = 1;
8543 	__le32 *p;
8544 
8545 	if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info)
8546 		return 0;
8547 
8548 	if (instance_bmap)
8549 		n = hweight32(ctxm->instance_bmap);
8550 	else
8551 		instance_bmap = 1;
8552 
8553 	rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2);
8554 	if (rc)
8555 		return rc;
8556 	hwrm_req_hold(bp, req);
8557 	req->type = cpu_to_le16(ctxm->type);
8558 	req->entry_size = cpu_to_le16(ctxm->entry_size);
8559 	req->subtype_valid_cnt = ctxm->split_entry_cnt;
8560 	for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++)
8561 		p[i] = cpu_to_le32(ctxm->split[i]);
8562 	for (i = 0, j = 0; j < n && !rc; i++) {
8563 		struct bnxt_ctx_pg_info *ctx_pg;
8564 
8565 		if (!(instance_bmap & (1 << i)))
8566 			continue;
8567 		req->instance = cpu_to_le16(i);
8568 		ctx_pg = &ctxm->pg_info[j++];
8569 		if (!ctx_pg->entries)
8570 			continue;
8571 		req->num_entries = cpu_to_le32(ctx_pg->entries);
8572 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8573 				      &req->page_size_pbl_level,
8574 				      &req->page_dir);
8575 		if (last && j == n)
8576 			req->flags =
8577 				cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE);
8578 		rc = hwrm_req_send(bp, req);
8579 	}
8580 	hwrm_req_drop(bp, req);
8581 	return rc;
8582 }
8583 
8584 static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena)
8585 {
8586 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
8587 	struct bnxt_ctx_mem_type *ctxm;
8588 	u16 last_type;
8589 	int rc = 0;
8590 	u16 type;
8591 
8592 	if (!ena)
8593 		return 0;
8594 	else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
8595 		last_type = BNXT_CTX_MAX - 1;
8596 	else
8597 		last_type = BNXT_CTX_L2_MAX - 1;
8598 	ctx->ctx_arr[last_type].last = 1;
8599 
8600 	for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) {
8601 		ctxm = &ctx->ctx_arr[type];
8602 
8603 		rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last);
8604 		if (rc)
8605 			return rc;
8606 	}
8607 	return 0;
8608 }
8609 
8610 void bnxt_free_ctx_mem(struct bnxt *bp)
8611 {
8612 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
8613 	u16 type;
8614 
8615 	if (!ctx)
8616 		return;
8617 
8618 	for (type = 0; type < BNXT_CTX_V2_MAX; type++) {
8619 		struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8620 		struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8621 		int i, n = 1;
8622 
8623 		if (!ctx_pg)
8624 			continue;
8625 		if (ctxm->instance_bmap)
8626 			n = hweight32(ctxm->instance_bmap);
8627 		for (i = 0; i < n; i++)
8628 			bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]);
8629 
8630 		kfree(ctx_pg);
8631 		ctxm->pg_info = NULL;
8632 	}
8633 
8634 	ctx->flags &= ~BNXT_CTX_FLAG_INITED;
8635 	kfree(ctx);
8636 	bp->ctx = NULL;
8637 }
8638 
8639 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
8640 {
8641 	struct bnxt_ctx_mem_type *ctxm;
8642 	struct bnxt_ctx_mem_info *ctx;
8643 	u32 l2_qps, qp1_qps, max_qps;
8644 	u32 ena, entries_sp, entries;
8645 	u32 srqs, max_srqs, min;
8646 	u32 num_mr, num_ah;
8647 	u32 extra_srqs = 0;
8648 	u32 extra_qps = 0;
8649 	u32 fast_qpmd_qps;
8650 	u8 pg_lvl = 1;
8651 	int i, rc;
8652 
8653 	rc = bnxt_hwrm_func_backing_store_qcaps(bp);
8654 	if (rc) {
8655 		netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
8656 			   rc);
8657 		return rc;
8658 	}
8659 	ctx = bp->ctx;
8660 	if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
8661 		return 0;
8662 
8663 	ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8664 	l2_qps = ctxm->qp_l2_entries;
8665 	qp1_qps = ctxm->qp_qp1_entries;
8666 	fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
8667 	max_qps = ctxm->max_entries;
8668 	ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8669 	srqs = ctxm->srq_l2_entries;
8670 	max_srqs = ctxm->max_entries;
8671 	ena = 0;
8672 	if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
8673 		pg_lvl = 2;
8674 		extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
8675 		/* allocate extra qps if fw supports RoCE fast qp destroy feature */
8676 		extra_qps += fast_qpmd_qps;
8677 		extra_srqs = min_t(u32, 8192, max_srqs - srqs);
8678 		if (fast_qpmd_qps)
8679 			ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
8680 	}
8681 
8682 	ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8683 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps,
8684 				     pg_lvl);
8685 	if (rc)
8686 		return rc;
8687 
8688 	ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8689 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl);
8690 	if (rc)
8691 		return rc;
8692 
8693 	ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8694 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries +
8695 				     extra_qps * 2, pg_lvl);
8696 	if (rc)
8697 		return rc;
8698 
8699 	ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8700 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8701 	if (rc)
8702 		return rc;
8703 
8704 	ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8705 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8706 	if (rc)
8707 		return rc;
8708 
8709 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
8710 		goto skip_rdma;
8711 
8712 	ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8713 	/* 128K extra is needed to accommodate static AH context
8714 	 * allocation by f/w.
8715 	 */
8716 	num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
8717 	num_ah = min_t(u32, num_mr, 1024 * 128);
8718 	ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1;
8719 	if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
8720 		ctxm->mrav_av_entries = num_ah;
8721 
8722 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2);
8723 	if (rc)
8724 		return rc;
8725 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
8726 
8727 	ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8728 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1);
8729 	if (rc)
8730 		return rc;
8731 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
8732 
8733 skip_rdma:
8734 	ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8735 	min = ctxm->min_entries;
8736 	entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps +
8737 		     2 * (extra_qps + qp1_qps) + min;
8738 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2);
8739 	if (rc)
8740 		return rc;
8741 
8742 	ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8743 	entries = l2_qps + 2 * (extra_qps + qp1_qps);
8744 	rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2);
8745 	if (rc)
8746 		return rc;
8747 	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
8748 		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
8749 	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
8750 
8751 	if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8752 		rc = bnxt_backing_store_cfg_v2(bp, ena);
8753 	else
8754 		rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
8755 	if (rc) {
8756 		netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
8757 			   rc);
8758 		return rc;
8759 	}
8760 	ctx->flags |= BNXT_CTX_FLAG_INITED;
8761 	return 0;
8762 }
8763 
8764 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
8765 {
8766 	struct hwrm_func_resource_qcaps_output *resp;
8767 	struct hwrm_func_resource_qcaps_input *req;
8768 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8769 	int rc;
8770 
8771 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
8772 	if (rc)
8773 		return rc;
8774 
8775 	req->fid = cpu_to_le16(0xffff);
8776 	resp = hwrm_req_hold(bp, req);
8777 	rc = hwrm_req_send_silent(bp, req);
8778 	if (rc)
8779 		goto hwrm_func_resc_qcaps_exit;
8780 
8781 	hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
8782 	if (!all)
8783 		goto hwrm_func_resc_qcaps_exit;
8784 
8785 	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
8786 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8787 	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
8788 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8789 	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
8790 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8791 	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
8792 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8793 	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
8794 	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
8795 	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
8796 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8797 	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
8798 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8799 	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
8800 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8801 
8802 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8803 		u16 max_msix = le16_to_cpu(resp->max_msix);
8804 
8805 		hw_resc->max_nqs = max_msix;
8806 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
8807 	}
8808 
8809 	if (BNXT_PF(bp)) {
8810 		struct bnxt_pf_info *pf = &bp->pf;
8811 
8812 		pf->vf_resv_strategy =
8813 			le16_to_cpu(resp->vf_reservation_strategy);
8814 		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
8815 			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
8816 	}
8817 hwrm_func_resc_qcaps_exit:
8818 	hwrm_req_drop(bp, req);
8819 	return rc;
8820 }
8821 
8822 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
8823 {
8824 	struct hwrm_port_mac_ptp_qcfg_output *resp;
8825 	struct hwrm_port_mac_ptp_qcfg_input *req;
8826 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
8827 	bool phc_cfg;
8828 	u8 flags;
8829 	int rc;
8830 
8831 	if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5(bp)) {
8832 		rc = -ENODEV;
8833 		goto no_ptp;
8834 	}
8835 
8836 	rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
8837 	if (rc)
8838 		goto no_ptp;
8839 
8840 	req->port_id = cpu_to_le16(bp->pf.port_id);
8841 	resp = hwrm_req_hold(bp, req);
8842 	rc = hwrm_req_send(bp, req);
8843 	if (rc)
8844 		goto exit;
8845 
8846 	flags = resp->flags;
8847 	if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
8848 		rc = -ENODEV;
8849 		goto exit;
8850 	}
8851 	if (!ptp) {
8852 		ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
8853 		if (!ptp) {
8854 			rc = -ENOMEM;
8855 			goto exit;
8856 		}
8857 		ptp->bp = bp;
8858 		bp->ptp_cfg = ptp;
8859 	}
8860 	if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
8861 		ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
8862 		ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
8863 	} else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8864 		ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
8865 		ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
8866 	} else {
8867 		rc = -ENODEV;
8868 		goto exit;
8869 	}
8870 	phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
8871 	rc = bnxt_ptp_init(bp, phc_cfg);
8872 	if (rc)
8873 		netdev_warn(bp->dev, "PTP initialization failed.\n");
8874 exit:
8875 	hwrm_req_drop(bp, req);
8876 	if (!rc)
8877 		return 0;
8878 
8879 no_ptp:
8880 	bnxt_ptp_clear(bp);
8881 	kfree(ptp);
8882 	bp->ptp_cfg = NULL;
8883 	return rc;
8884 }
8885 
8886 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
8887 {
8888 	struct hwrm_func_qcaps_output *resp;
8889 	struct hwrm_func_qcaps_input *req;
8890 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8891 	u32 flags, flags_ext, flags_ext2;
8892 	int rc;
8893 
8894 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
8895 	if (rc)
8896 		return rc;
8897 
8898 	req->fid = cpu_to_le16(0xffff);
8899 	resp = hwrm_req_hold(bp, req);
8900 	rc = hwrm_req_send(bp, req);
8901 	if (rc)
8902 		goto hwrm_func_qcaps_exit;
8903 
8904 	flags = le32_to_cpu(resp->flags);
8905 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
8906 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
8907 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
8908 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
8909 	if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
8910 		bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
8911 	if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
8912 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
8913 	if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
8914 		bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
8915 	if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
8916 		bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
8917 	if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
8918 		bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
8919 	if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
8920 		bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
8921 	if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
8922 		bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
8923 
8924 	flags_ext = le32_to_cpu(resp->flags_ext);
8925 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
8926 		bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
8927 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
8928 		bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
8929 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
8930 		bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
8931 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
8932 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
8933 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
8934 		bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
8935 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED)
8936 		bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2;
8937 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP)
8938 		bp->flags |= BNXT_FLAG_TX_COAL_CMPL;
8939 
8940 	flags_ext2 = le32_to_cpu(resp->flags_ext2);
8941 	if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
8942 		bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
8943 	if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED)
8944 		bp->flags |= BNXT_FLAG_UDP_GSO_CAP;
8945 
8946 	bp->tx_push_thresh = 0;
8947 	if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
8948 	    BNXT_FW_MAJ(bp) > 217)
8949 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
8950 
8951 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8952 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8953 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8954 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8955 	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
8956 	if (!hw_resc->max_hw_ring_grps)
8957 		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
8958 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8959 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8960 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8961 
8962 	hw_resc->max_encap_records = le32_to_cpu(resp->max_encap_records);
8963 	hw_resc->max_decap_records = le32_to_cpu(resp->max_decap_records);
8964 	hw_resc->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
8965 	hw_resc->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
8966 	hw_resc->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
8967 	hw_resc->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
8968 
8969 	if (BNXT_PF(bp)) {
8970 		struct bnxt_pf_info *pf = &bp->pf;
8971 
8972 		pf->fw_fid = le16_to_cpu(resp->fid);
8973 		pf->port_id = le16_to_cpu(resp->port_id);
8974 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
8975 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
8976 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
8977 		bp->flags &= ~BNXT_FLAG_WOL_CAP;
8978 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
8979 			bp->flags |= BNXT_FLAG_WOL_CAP;
8980 		if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
8981 			bp->fw_cap |= BNXT_FW_CAP_PTP;
8982 		} else {
8983 			bnxt_ptp_clear(bp);
8984 			kfree(bp->ptp_cfg);
8985 			bp->ptp_cfg = NULL;
8986 		}
8987 	} else {
8988 #ifdef CONFIG_BNXT_SRIOV
8989 		struct bnxt_vf_info *vf = &bp->vf;
8990 
8991 		vf->fw_fid = le16_to_cpu(resp->fid);
8992 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
8993 #endif
8994 	}
8995 
8996 hwrm_func_qcaps_exit:
8997 	hwrm_req_drop(bp, req);
8998 	return rc;
8999 }
9000 
9001 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
9002 {
9003 	struct hwrm_dbg_qcaps_output *resp;
9004 	struct hwrm_dbg_qcaps_input *req;
9005 	int rc;
9006 
9007 	bp->fw_dbg_cap = 0;
9008 	if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
9009 		return;
9010 
9011 	rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
9012 	if (rc)
9013 		return;
9014 
9015 	req->fid = cpu_to_le16(0xffff);
9016 	resp = hwrm_req_hold(bp, req);
9017 	rc = hwrm_req_send(bp, req);
9018 	if (rc)
9019 		goto hwrm_dbg_qcaps_exit;
9020 
9021 	bp->fw_dbg_cap = le32_to_cpu(resp->flags);
9022 
9023 hwrm_dbg_qcaps_exit:
9024 	hwrm_req_drop(bp, req);
9025 }
9026 
9027 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
9028 
9029 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
9030 {
9031 	int rc;
9032 
9033 	rc = __bnxt_hwrm_func_qcaps(bp);
9034 	if (rc)
9035 		return rc;
9036 
9037 	bnxt_hwrm_dbg_qcaps(bp);
9038 
9039 	rc = bnxt_hwrm_queue_qportcfg(bp);
9040 	if (rc) {
9041 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
9042 		return rc;
9043 	}
9044 	if (bp->hwrm_spec_code >= 0x10803) {
9045 		rc = bnxt_alloc_ctx_mem(bp);
9046 		if (rc)
9047 			return rc;
9048 		rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9049 		if (!rc)
9050 			bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
9051 	}
9052 	return 0;
9053 }
9054 
9055 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
9056 {
9057 	struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
9058 	struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
9059 	u32 flags;
9060 	int rc;
9061 
9062 	if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
9063 		return 0;
9064 
9065 	rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
9066 	if (rc)
9067 		return rc;
9068 
9069 	resp = hwrm_req_hold(bp, req);
9070 	rc = hwrm_req_send(bp, req);
9071 	if (rc)
9072 		goto hwrm_cfa_adv_qcaps_exit;
9073 
9074 	flags = le32_to_cpu(resp->flags);
9075 	if (flags &
9076 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
9077 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
9078 
9079 	if (flags &
9080 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED)
9081 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3;
9082 
9083 	if (flags &
9084 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED)
9085 		bp->fw_cap |= BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO;
9086 
9087 hwrm_cfa_adv_qcaps_exit:
9088 	hwrm_req_drop(bp, req);
9089 	return rc;
9090 }
9091 
9092 static int __bnxt_alloc_fw_health(struct bnxt *bp)
9093 {
9094 	if (bp->fw_health)
9095 		return 0;
9096 
9097 	bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
9098 	if (!bp->fw_health)
9099 		return -ENOMEM;
9100 
9101 	mutex_init(&bp->fw_health->lock);
9102 	return 0;
9103 }
9104 
9105 static int bnxt_alloc_fw_health(struct bnxt *bp)
9106 {
9107 	int rc;
9108 
9109 	if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
9110 	    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9111 		return 0;
9112 
9113 	rc = __bnxt_alloc_fw_health(bp);
9114 	if (rc) {
9115 		bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
9116 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9117 		return rc;
9118 	}
9119 
9120 	return 0;
9121 }
9122 
9123 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
9124 {
9125 	writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
9126 					 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
9127 					 BNXT_FW_HEALTH_WIN_MAP_OFF);
9128 }
9129 
9130 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
9131 {
9132 	struct bnxt_fw_health *fw_health = bp->fw_health;
9133 	u32 reg_type;
9134 
9135 	if (!fw_health)
9136 		return;
9137 
9138 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
9139 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9140 		fw_health->status_reliable = false;
9141 
9142 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
9143 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9144 		fw_health->resets_reliable = false;
9145 }
9146 
9147 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
9148 {
9149 	void __iomem *hs;
9150 	u32 status_loc;
9151 	u32 reg_type;
9152 	u32 sig;
9153 
9154 	if (bp->fw_health)
9155 		bp->fw_health->status_reliable = false;
9156 
9157 	__bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
9158 	hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
9159 
9160 	sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
9161 	if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
9162 		if (!bp->chip_num) {
9163 			__bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
9164 			bp->chip_num = readl(bp->bar0 +
9165 					     BNXT_FW_HEALTH_WIN_BASE +
9166 					     BNXT_GRC_REG_CHIP_NUM);
9167 		}
9168 		if (!BNXT_CHIP_P5(bp))
9169 			return;
9170 
9171 		status_loc = BNXT_GRC_REG_STATUS_P5 |
9172 			     BNXT_FW_HEALTH_REG_TYPE_BAR0;
9173 	} else {
9174 		status_loc = readl(hs + offsetof(struct hcomm_status,
9175 						 fw_status_loc));
9176 	}
9177 
9178 	if (__bnxt_alloc_fw_health(bp)) {
9179 		netdev_warn(bp->dev, "no memory for firmware status checks\n");
9180 		return;
9181 	}
9182 
9183 	bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
9184 	reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
9185 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
9186 		__bnxt_map_fw_health_reg(bp, status_loc);
9187 		bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
9188 			BNXT_FW_HEALTH_WIN_OFF(status_loc);
9189 	}
9190 
9191 	bp->fw_health->status_reliable = true;
9192 }
9193 
9194 static int bnxt_map_fw_health_regs(struct bnxt *bp)
9195 {
9196 	struct bnxt_fw_health *fw_health = bp->fw_health;
9197 	u32 reg_base = 0xffffffff;
9198 	int i;
9199 
9200 	bp->fw_health->status_reliable = false;
9201 	bp->fw_health->resets_reliable = false;
9202 	/* Only pre-map the monitoring GRC registers using window 3 */
9203 	for (i = 0; i < 4; i++) {
9204 		u32 reg = fw_health->regs[i];
9205 
9206 		if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
9207 			continue;
9208 		if (reg_base == 0xffffffff)
9209 			reg_base = reg & BNXT_GRC_BASE_MASK;
9210 		if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
9211 			return -ERANGE;
9212 		fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
9213 	}
9214 	bp->fw_health->status_reliable = true;
9215 	bp->fw_health->resets_reliable = true;
9216 	if (reg_base == 0xffffffff)
9217 		return 0;
9218 
9219 	__bnxt_map_fw_health_reg(bp, reg_base);
9220 	return 0;
9221 }
9222 
9223 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
9224 {
9225 	if (!bp->fw_health)
9226 		return;
9227 
9228 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
9229 		bp->fw_health->status_reliable = true;
9230 		bp->fw_health->resets_reliable = true;
9231 	} else {
9232 		bnxt_try_map_fw_health_reg(bp);
9233 	}
9234 }
9235 
9236 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
9237 {
9238 	struct bnxt_fw_health *fw_health = bp->fw_health;
9239 	struct hwrm_error_recovery_qcfg_output *resp;
9240 	struct hwrm_error_recovery_qcfg_input *req;
9241 	int rc, i;
9242 
9243 	if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9244 		return 0;
9245 
9246 	rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
9247 	if (rc)
9248 		return rc;
9249 
9250 	resp = hwrm_req_hold(bp, req);
9251 	rc = hwrm_req_send(bp, req);
9252 	if (rc)
9253 		goto err_recovery_out;
9254 	fw_health->flags = le32_to_cpu(resp->flags);
9255 	if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
9256 	    !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
9257 		rc = -EINVAL;
9258 		goto err_recovery_out;
9259 	}
9260 	fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
9261 	fw_health->master_func_wait_dsecs =
9262 		le32_to_cpu(resp->master_func_wait_period);
9263 	fw_health->normal_func_wait_dsecs =
9264 		le32_to_cpu(resp->normal_func_wait_period);
9265 	fw_health->post_reset_wait_dsecs =
9266 		le32_to_cpu(resp->master_func_wait_period_after_reset);
9267 	fw_health->post_reset_max_wait_dsecs =
9268 		le32_to_cpu(resp->max_bailout_time_after_reset);
9269 	fw_health->regs[BNXT_FW_HEALTH_REG] =
9270 		le32_to_cpu(resp->fw_health_status_reg);
9271 	fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
9272 		le32_to_cpu(resp->fw_heartbeat_reg);
9273 	fw_health->regs[BNXT_FW_RESET_CNT_REG] =
9274 		le32_to_cpu(resp->fw_reset_cnt_reg);
9275 	fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
9276 		le32_to_cpu(resp->reset_inprogress_reg);
9277 	fw_health->fw_reset_inprog_reg_mask =
9278 		le32_to_cpu(resp->reset_inprogress_reg_mask);
9279 	fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
9280 	if (fw_health->fw_reset_seq_cnt >= 16) {
9281 		rc = -EINVAL;
9282 		goto err_recovery_out;
9283 	}
9284 	for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
9285 		fw_health->fw_reset_seq_regs[i] =
9286 			le32_to_cpu(resp->reset_reg[i]);
9287 		fw_health->fw_reset_seq_vals[i] =
9288 			le32_to_cpu(resp->reset_reg_val[i]);
9289 		fw_health->fw_reset_seq_delay_msec[i] =
9290 			resp->delay_after_reset[i];
9291 	}
9292 err_recovery_out:
9293 	hwrm_req_drop(bp, req);
9294 	if (!rc)
9295 		rc = bnxt_map_fw_health_regs(bp);
9296 	if (rc)
9297 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9298 	return rc;
9299 }
9300 
9301 static int bnxt_hwrm_func_reset(struct bnxt *bp)
9302 {
9303 	struct hwrm_func_reset_input *req;
9304 	int rc;
9305 
9306 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
9307 	if (rc)
9308 		return rc;
9309 
9310 	req->enables = 0;
9311 	hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
9312 	return hwrm_req_send(bp, req);
9313 }
9314 
9315 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
9316 {
9317 	struct hwrm_nvm_get_dev_info_output nvm_info;
9318 
9319 	if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
9320 		snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
9321 			 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
9322 			 nvm_info.nvm_cfg_ver_upd);
9323 }
9324 
9325 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
9326 {
9327 	struct hwrm_queue_qportcfg_output *resp;
9328 	struct hwrm_queue_qportcfg_input *req;
9329 	u8 i, j, *qptr;
9330 	bool no_rdma;
9331 	int rc = 0;
9332 
9333 	rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
9334 	if (rc)
9335 		return rc;
9336 
9337 	resp = hwrm_req_hold(bp, req);
9338 	rc = hwrm_req_send(bp, req);
9339 	if (rc)
9340 		goto qportcfg_exit;
9341 
9342 	if (!resp->max_configurable_queues) {
9343 		rc = -EINVAL;
9344 		goto qportcfg_exit;
9345 	}
9346 	bp->max_tc = resp->max_configurable_queues;
9347 	bp->max_lltc = resp->max_configurable_lossless_queues;
9348 	if (bp->max_tc > BNXT_MAX_QUEUE)
9349 		bp->max_tc = BNXT_MAX_QUEUE;
9350 
9351 	no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
9352 	qptr = &resp->queue_id0;
9353 	for (i = 0, j = 0; i < bp->max_tc; i++) {
9354 		bp->q_info[j].queue_id = *qptr;
9355 		bp->q_ids[i] = *qptr++;
9356 		bp->q_info[j].queue_profile = *qptr++;
9357 		bp->tc_to_qidx[j] = j;
9358 		if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
9359 		    (no_rdma && BNXT_PF(bp)))
9360 			j++;
9361 	}
9362 	bp->max_q = bp->max_tc;
9363 	bp->max_tc = max_t(u8, j, 1);
9364 
9365 	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
9366 		bp->max_tc = 1;
9367 
9368 	if (bp->max_lltc > bp->max_tc)
9369 		bp->max_lltc = bp->max_tc;
9370 
9371 qportcfg_exit:
9372 	hwrm_req_drop(bp, req);
9373 	return rc;
9374 }
9375 
9376 static int bnxt_hwrm_poll(struct bnxt *bp)
9377 {
9378 	struct hwrm_ver_get_input *req;
9379 	int rc;
9380 
9381 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9382 	if (rc)
9383 		return rc;
9384 
9385 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9386 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
9387 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9388 
9389 	hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
9390 	rc = hwrm_req_send(bp, req);
9391 	return rc;
9392 }
9393 
9394 static int bnxt_hwrm_ver_get(struct bnxt *bp)
9395 {
9396 	struct hwrm_ver_get_output *resp;
9397 	struct hwrm_ver_get_input *req;
9398 	u16 fw_maj, fw_min, fw_bld, fw_rsv;
9399 	u32 dev_caps_cfg, hwrm_ver;
9400 	int rc, len;
9401 
9402 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9403 	if (rc)
9404 		return rc;
9405 
9406 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
9407 	bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
9408 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9409 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
9410 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9411 
9412 	resp = hwrm_req_hold(bp, req);
9413 	rc = hwrm_req_send(bp, req);
9414 	if (rc)
9415 		goto hwrm_ver_get_exit;
9416 
9417 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
9418 
9419 	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
9420 			     resp->hwrm_intf_min_8b << 8 |
9421 			     resp->hwrm_intf_upd_8b;
9422 	if (resp->hwrm_intf_maj_8b < 1) {
9423 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
9424 			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9425 			    resp->hwrm_intf_upd_8b);
9426 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
9427 	}
9428 
9429 	hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
9430 			HWRM_VERSION_UPDATE;
9431 
9432 	if (bp->hwrm_spec_code > hwrm_ver)
9433 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9434 			 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
9435 			 HWRM_VERSION_UPDATE);
9436 	else
9437 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9438 			 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9439 			 resp->hwrm_intf_upd_8b);
9440 
9441 	fw_maj = le16_to_cpu(resp->hwrm_fw_major);
9442 	if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
9443 		fw_min = le16_to_cpu(resp->hwrm_fw_minor);
9444 		fw_bld = le16_to_cpu(resp->hwrm_fw_build);
9445 		fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
9446 		len = FW_VER_STR_LEN;
9447 	} else {
9448 		fw_maj = resp->hwrm_fw_maj_8b;
9449 		fw_min = resp->hwrm_fw_min_8b;
9450 		fw_bld = resp->hwrm_fw_bld_8b;
9451 		fw_rsv = resp->hwrm_fw_rsvd_8b;
9452 		len = BC_HWRM_STR_LEN;
9453 	}
9454 	bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
9455 	snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
9456 		 fw_rsv);
9457 
9458 	if (strlen(resp->active_pkg_name)) {
9459 		int fw_ver_len = strlen(bp->fw_ver_str);
9460 
9461 		snprintf(bp->fw_ver_str + fw_ver_len,
9462 			 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
9463 			 resp->active_pkg_name);
9464 		bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
9465 	}
9466 
9467 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
9468 	if (!bp->hwrm_cmd_timeout)
9469 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
9470 	bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
9471 	if (!bp->hwrm_cmd_max_timeout)
9472 		bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
9473 	else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
9474 		netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
9475 			    bp->hwrm_cmd_max_timeout / 1000);
9476 
9477 	if (resp->hwrm_intf_maj_8b >= 1) {
9478 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
9479 		bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
9480 	}
9481 	if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
9482 		bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
9483 
9484 	bp->chip_num = le16_to_cpu(resp->chip_num);
9485 	bp->chip_rev = resp->chip_rev;
9486 	if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
9487 	    !resp->chip_metal)
9488 		bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
9489 
9490 	dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
9491 	if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
9492 	    (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
9493 		bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
9494 
9495 	if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
9496 		bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
9497 
9498 	if (dev_caps_cfg &
9499 	    VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
9500 		bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
9501 
9502 	if (dev_caps_cfg &
9503 	    VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
9504 		bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
9505 
9506 	if (dev_caps_cfg &
9507 	    VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
9508 		bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
9509 
9510 hwrm_ver_get_exit:
9511 	hwrm_req_drop(bp, req);
9512 	return rc;
9513 }
9514 
9515 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
9516 {
9517 	struct hwrm_fw_set_time_input *req;
9518 	struct tm tm;
9519 	time64_t now = ktime_get_real_seconds();
9520 	int rc;
9521 
9522 	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
9523 	    bp->hwrm_spec_code < 0x10400)
9524 		return -EOPNOTSUPP;
9525 
9526 	time64_to_tm(now, 0, &tm);
9527 	rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
9528 	if (rc)
9529 		return rc;
9530 
9531 	req->year = cpu_to_le16(1900 + tm.tm_year);
9532 	req->month = 1 + tm.tm_mon;
9533 	req->day = tm.tm_mday;
9534 	req->hour = tm.tm_hour;
9535 	req->minute = tm.tm_min;
9536 	req->second = tm.tm_sec;
9537 	return hwrm_req_send(bp, req);
9538 }
9539 
9540 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
9541 {
9542 	u64 sw_tmp;
9543 
9544 	hw &= mask;
9545 	sw_tmp = (*sw & ~mask) | hw;
9546 	if (hw < (*sw & mask))
9547 		sw_tmp += mask + 1;
9548 	WRITE_ONCE(*sw, sw_tmp);
9549 }
9550 
9551 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
9552 				    int count, bool ignore_zero)
9553 {
9554 	int i;
9555 
9556 	for (i = 0; i < count; i++) {
9557 		u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
9558 
9559 		if (ignore_zero && !hw)
9560 			continue;
9561 
9562 		if (masks[i] == -1ULL)
9563 			sw_stats[i] = hw;
9564 		else
9565 			bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
9566 	}
9567 }
9568 
9569 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
9570 {
9571 	if (!stats->hw_stats)
9572 		return;
9573 
9574 	__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9575 				stats->hw_masks, stats->len / 8, false);
9576 }
9577 
9578 static void bnxt_accumulate_all_stats(struct bnxt *bp)
9579 {
9580 	struct bnxt_stats_mem *ring0_stats;
9581 	bool ignore_zero = false;
9582 	int i;
9583 
9584 	/* Chip bug.  Counter intermittently becomes 0. */
9585 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9586 		ignore_zero = true;
9587 
9588 	for (i = 0; i < bp->cp_nr_rings; i++) {
9589 		struct bnxt_napi *bnapi = bp->bnapi[i];
9590 		struct bnxt_cp_ring_info *cpr;
9591 		struct bnxt_stats_mem *stats;
9592 
9593 		cpr = &bnapi->cp_ring;
9594 		stats = &cpr->stats;
9595 		if (!i)
9596 			ring0_stats = stats;
9597 		__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9598 					ring0_stats->hw_masks,
9599 					ring0_stats->len / 8, ignore_zero);
9600 	}
9601 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
9602 		struct bnxt_stats_mem *stats = &bp->port_stats;
9603 		__le64 *hw_stats = stats->hw_stats;
9604 		u64 *sw_stats = stats->sw_stats;
9605 		u64 *masks = stats->hw_masks;
9606 		int cnt;
9607 
9608 		cnt = sizeof(struct rx_port_stats) / 8;
9609 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9610 
9611 		hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9612 		sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9613 		masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9614 		cnt = sizeof(struct tx_port_stats) / 8;
9615 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9616 	}
9617 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
9618 		bnxt_accumulate_stats(&bp->rx_port_stats_ext);
9619 		bnxt_accumulate_stats(&bp->tx_port_stats_ext);
9620 	}
9621 }
9622 
9623 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
9624 {
9625 	struct hwrm_port_qstats_input *req;
9626 	struct bnxt_pf_info *pf = &bp->pf;
9627 	int rc;
9628 
9629 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
9630 		return 0;
9631 
9632 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9633 		return -EOPNOTSUPP;
9634 
9635 	rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
9636 	if (rc)
9637 		return rc;
9638 
9639 	req->flags = flags;
9640 	req->port_id = cpu_to_le16(pf->port_id);
9641 	req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
9642 					    BNXT_TX_PORT_STATS_BYTE_OFFSET);
9643 	req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
9644 	return hwrm_req_send(bp, req);
9645 }
9646 
9647 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
9648 {
9649 	struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
9650 	struct hwrm_queue_pri2cos_qcfg_input *req_qc;
9651 	struct hwrm_port_qstats_ext_output *resp_qs;
9652 	struct hwrm_port_qstats_ext_input *req_qs;
9653 	struct bnxt_pf_info *pf = &bp->pf;
9654 	u32 tx_stat_size;
9655 	int rc;
9656 
9657 	if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
9658 		return 0;
9659 
9660 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9661 		return -EOPNOTSUPP;
9662 
9663 	rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
9664 	if (rc)
9665 		return rc;
9666 
9667 	req_qs->flags = flags;
9668 	req_qs->port_id = cpu_to_le16(pf->port_id);
9669 	req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
9670 	req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
9671 	tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
9672 		       sizeof(struct tx_port_stats_ext) : 0;
9673 	req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
9674 	req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
9675 	resp_qs = hwrm_req_hold(bp, req_qs);
9676 	rc = hwrm_req_send(bp, req_qs);
9677 	if (!rc) {
9678 		bp->fw_rx_stats_ext_size =
9679 			le16_to_cpu(resp_qs->rx_stat_size) / 8;
9680 		if (BNXT_FW_MAJ(bp) < 220 &&
9681 		    bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
9682 			bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
9683 
9684 		bp->fw_tx_stats_ext_size = tx_stat_size ?
9685 			le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
9686 	} else {
9687 		bp->fw_rx_stats_ext_size = 0;
9688 		bp->fw_tx_stats_ext_size = 0;
9689 	}
9690 	hwrm_req_drop(bp, req_qs);
9691 
9692 	if (flags)
9693 		return rc;
9694 
9695 	if (bp->fw_tx_stats_ext_size <=
9696 	    offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
9697 		bp->pri2cos_valid = 0;
9698 		return rc;
9699 	}
9700 
9701 	rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
9702 	if (rc)
9703 		return rc;
9704 
9705 	req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
9706 
9707 	resp_qc = hwrm_req_hold(bp, req_qc);
9708 	rc = hwrm_req_send(bp, req_qc);
9709 	if (!rc) {
9710 		u8 *pri2cos;
9711 		int i, j;
9712 
9713 		pri2cos = &resp_qc->pri0_cos_queue_id;
9714 		for (i = 0; i < 8; i++) {
9715 			u8 queue_id = pri2cos[i];
9716 			u8 queue_idx;
9717 
9718 			/* Per port queue IDs start from 0, 10, 20, etc */
9719 			queue_idx = queue_id % 10;
9720 			if (queue_idx > BNXT_MAX_QUEUE) {
9721 				bp->pri2cos_valid = false;
9722 				hwrm_req_drop(bp, req_qc);
9723 				return rc;
9724 			}
9725 			for (j = 0; j < bp->max_q; j++) {
9726 				if (bp->q_ids[j] == queue_id)
9727 					bp->pri2cos_idx[i] = queue_idx;
9728 			}
9729 		}
9730 		bp->pri2cos_valid = true;
9731 	}
9732 	hwrm_req_drop(bp, req_qc);
9733 
9734 	return rc;
9735 }
9736 
9737 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
9738 {
9739 	bnxt_hwrm_tunnel_dst_port_free(bp,
9740 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9741 	bnxt_hwrm_tunnel_dst_port_free(bp,
9742 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9743 }
9744 
9745 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
9746 {
9747 	int rc, i;
9748 	u32 tpa_flags = 0;
9749 
9750 	if (set_tpa)
9751 		tpa_flags = bp->flags & BNXT_FLAG_TPA;
9752 	else if (BNXT_NO_FW_ACCESS(bp))
9753 		return 0;
9754 	for (i = 0; i < bp->nr_vnics; i++) {
9755 		rc = bnxt_hwrm_vnic_set_tpa(bp, &bp->vnic_info[i], tpa_flags);
9756 		if (rc) {
9757 			netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
9758 				   i, rc);
9759 			return rc;
9760 		}
9761 	}
9762 	return 0;
9763 }
9764 
9765 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
9766 {
9767 	int i;
9768 
9769 	for (i = 0; i < bp->nr_vnics; i++)
9770 		bnxt_hwrm_vnic_set_rss(bp, &bp->vnic_info[i], false);
9771 }
9772 
9773 static void bnxt_clear_vnic(struct bnxt *bp)
9774 {
9775 	if (!bp->vnic_info)
9776 		return;
9777 
9778 	bnxt_hwrm_clear_vnic_filter(bp);
9779 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) {
9780 		/* clear all RSS setting before free vnic ctx */
9781 		bnxt_hwrm_clear_vnic_rss(bp);
9782 		bnxt_hwrm_vnic_ctx_free(bp);
9783 	}
9784 	/* before free the vnic, undo the vnic tpa settings */
9785 	if (bp->flags & BNXT_FLAG_TPA)
9786 		bnxt_set_tpa(bp, false);
9787 	bnxt_hwrm_vnic_free(bp);
9788 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9789 		bnxt_hwrm_vnic_ctx_free(bp);
9790 }
9791 
9792 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
9793 				    bool irq_re_init)
9794 {
9795 	bnxt_clear_vnic(bp);
9796 	bnxt_hwrm_ring_free(bp, close_path);
9797 	bnxt_hwrm_ring_grp_free(bp);
9798 	if (irq_re_init) {
9799 		bnxt_hwrm_stat_ctx_free(bp);
9800 		bnxt_hwrm_free_tunnel_ports(bp);
9801 	}
9802 }
9803 
9804 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
9805 {
9806 	struct hwrm_func_cfg_input *req;
9807 	u8 evb_mode;
9808 	int rc;
9809 
9810 	if (br_mode == BRIDGE_MODE_VEB)
9811 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
9812 	else if (br_mode == BRIDGE_MODE_VEPA)
9813 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
9814 	else
9815 		return -EINVAL;
9816 
9817 	rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9818 	if (rc)
9819 		return rc;
9820 
9821 	req->fid = cpu_to_le16(0xffff);
9822 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
9823 	req->evb_mode = evb_mode;
9824 	return hwrm_req_send(bp, req);
9825 }
9826 
9827 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
9828 {
9829 	struct hwrm_func_cfg_input *req;
9830 	int rc;
9831 
9832 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
9833 		return 0;
9834 
9835 	rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9836 	if (rc)
9837 		return rc;
9838 
9839 	req->fid = cpu_to_le16(0xffff);
9840 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
9841 	req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
9842 	if (size == 128)
9843 		req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
9844 
9845 	return hwrm_req_send(bp, req);
9846 }
9847 
9848 static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
9849 {
9850 	int rc;
9851 
9852 	if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
9853 		goto skip_rss_ctx;
9854 
9855 	/* allocate context for vnic */
9856 	rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 0);
9857 	if (rc) {
9858 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9859 			   vnic->vnic_id, rc);
9860 		goto vnic_setup_err;
9861 	}
9862 	bp->rsscos_nr_ctxs++;
9863 
9864 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9865 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 1);
9866 		if (rc) {
9867 			netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
9868 				   vnic->vnic_id, rc);
9869 			goto vnic_setup_err;
9870 		}
9871 		bp->rsscos_nr_ctxs++;
9872 	}
9873 
9874 skip_rss_ctx:
9875 	/* configure default vnic, ring grp */
9876 	rc = bnxt_hwrm_vnic_cfg(bp, vnic);
9877 	if (rc) {
9878 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9879 			   vnic->vnic_id, rc);
9880 		goto vnic_setup_err;
9881 	}
9882 
9883 	/* Enable RSS hashing on vnic */
9884 	rc = bnxt_hwrm_vnic_set_rss(bp, vnic, true);
9885 	if (rc) {
9886 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
9887 			   vnic->vnic_id, rc);
9888 		goto vnic_setup_err;
9889 	}
9890 
9891 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9892 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
9893 		if (rc) {
9894 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9895 				   vnic->vnic_id, rc);
9896 		}
9897 	}
9898 
9899 vnic_setup_err:
9900 	return rc;
9901 }
9902 
9903 int bnxt_hwrm_vnic_rss_cfg_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
9904 {
9905 	int rc;
9906 
9907 	rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true);
9908 	if (rc) {
9909 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
9910 			   vnic->vnic_id, rc);
9911 		return rc;
9912 	}
9913 	rc = bnxt_hwrm_vnic_cfg(bp, vnic);
9914 	if (rc)
9915 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9916 			   vnic->vnic_id, rc);
9917 	return rc;
9918 }
9919 
9920 int __bnxt_setup_vnic_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
9921 {
9922 	int rc, i, nr_ctxs;
9923 
9924 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
9925 	for (i = 0; i < nr_ctxs; i++) {
9926 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, i);
9927 		if (rc) {
9928 			netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
9929 				   vnic->vnic_id, i, rc);
9930 			break;
9931 		}
9932 		bp->rsscos_nr_ctxs++;
9933 	}
9934 	if (i < nr_ctxs)
9935 		return -ENOMEM;
9936 
9937 	rc = bnxt_hwrm_vnic_rss_cfg_p5(bp, vnic);
9938 	if (rc)
9939 		return rc;
9940 
9941 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9942 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
9943 		if (rc) {
9944 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9945 				   vnic->vnic_id, rc);
9946 		}
9947 	}
9948 	return rc;
9949 }
9950 
9951 static int bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
9952 {
9953 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9954 		return __bnxt_setup_vnic_p5(bp, vnic);
9955 	else
9956 		return __bnxt_setup_vnic(bp, vnic);
9957 }
9958 
9959 static int bnxt_alloc_and_setup_vnic(struct bnxt *bp,
9960 				     struct bnxt_vnic_info *vnic,
9961 				     u16 start_rx_ring_idx, int rx_rings)
9962 {
9963 	int rc;
9964 
9965 	rc = bnxt_hwrm_vnic_alloc(bp, vnic, start_rx_ring_idx, rx_rings);
9966 	if (rc) {
9967 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9968 			   vnic->vnic_id, rc);
9969 		return rc;
9970 	}
9971 	return bnxt_setup_vnic(bp, vnic);
9972 }
9973 
9974 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
9975 {
9976 	struct bnxt_vnic_info *vnic;
9977 	int i, rc = 0;
9978 
9979 	if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
9980 		vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
9981 		return bnxt_alloc_and_setup_vnic(bp, vnic, 0, bp->rx_nr_rings);
9982 	}
9983 
9984 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9985 		return 0;
9986 
9987 	for (i = 0; i < bp->rx_nr_rings; i++) {
9988 		u16 vnic_id = i + 1;
9989 		u16 ring_id = i;
9990 
9991 		if (vnic_id >= bp->nr_vnics)
9992 			break;
9993 
9994 		vnic = &bp->vnic_info[vnic_id];
9995 		vnic->flags |= BNXT_VNIC_RFS_FLAG;
9996 		if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
9997 			vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
9998 		if (bnxt_alloc_and_setup_vnic(bp, &bp->vnic_info[vnic_id], ring_id, 1))
9999 			break;
10000 	}
10001 	return rc;
10002 }
10003 
10004 void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx,
10005 			  bool all)
10006 {
10007 	struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10008 	struct bnxt_filter_base *usr_fltr, *tmp;
10009 	struct bnxt_ntuple_filter *ntp_fltr;
10010 	int i;
10011 
10012 	bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic);
10013 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) {
10014 		if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID)
10015 			bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i);
10016 	}
10017 	if (!all)
10018 		return;
10019 
10020 	list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
10021 		if ((usr_fltr->flags & BNXT_ACT_RSS_CTX) &&
10022 		    usr_fltr->fw_vnic_id == rss_ctx->index) {
10023 			ntp_fltr = container_of(usr_fltr,
10024 						struct bnxt_ntuple_filter,
10025 						base);
10026 			bnxt_hwrm_cfa_ntuple_filter_free(bp, ntp_fltr);
10027 			bnxt_del_ntp_filter(bp, ntp_fltr);
10028 			bnxt_del_one_usr_fltr(bp, usr_fltr);
10029 		}
10030 	}
10031 
10032 	if (vnic->rss_table)
10033 		dma_free_coherent(&bp->pdev->dev, vnic->rss_table_size,
10034 				  vnic->rss_table,
10035 				  vnic->rss_table_dma_addr);
10036 	kfree(rss_ctx->rss_indir_tbl);
10037 	list_del(&rss_ctx->list);
10038 	bp->num_rss_ctx--;
10039 	clear_bit(rss_ctx->index, bp->rss_ctx_bmap);
10040 	kfree(rss_ctx);
10041 }
10042 
10043 static void bnxt_hwrm_realloc_rss_ctx_vnic(struct bnxt *bp)
10044 {
10045 	bool set_tpa = !!(bp->flags & BNXT_FLAG_TPA);
10046 	struct bnxt_rss_ctx *rss_ctx, *tmp;
10047 
10048 	list_for_each_entry_safe(rss_ctx, tmp, &bp->rss_ctx_list, list) {
10049 		struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10050 
10051 		if (bnxt_hwrm_vnic_alloc(bp, vnic, 0, bp->rx_nr_rings) ||
10052 		    bnxt_hwrm_vnic_set_tpa(bp, vnic, set_tpa) ||
10053 		    __bnxt_setup_vnic_p5(bp, vnic)) {
10054 			netdev_err(bp->dev, "Failed to restore RSS ctx %d\n",
10055 				   rss_ctx->index);
10056 			bnxt_del_one_rss_ctx(bp, rss_ctx, true);
10057 		}
10058 	}
10059 }
10060 
10061 struct bnxt_rss_ctx *bnxt_alloc_rss_ctx(struct bnxt *bp)
10062 {
10063 	struct bnxt_rss_ctx *rss_ctx = NULL;
10064 
10065 	rss_ctx = kzalloc(sizeof(*rss_ctx), GFP_KERNEL);
10066 	if (rss_ctx) {
10067 		rss_ctx->vnic.rss_ctx = rss_ctx;
10068 		list_add_tail(&rss_ctx->list, &bp->rss_ctx_list);
10069 		bp->num_rss_ctx++;
10070 	}
10071 	return rss_ctx;
10072 }
10073 
10074 void bnxt_clear_rss_ctxs(struct bnxt *bp, bool all)
10075 {
10076 	struct bnxt_rss_ctx *rss_ctx, *tmp;
10077 
10078 	list_for_each_entry_safe(rss_ctx, tmp, &bp->rss_ctx_list, list)
10079 		bnxt_del_one_rss_ctx(bp, rss_ctx, all);
10080 
10081 	if (all)
10082 		bitmap_free(bp->rss_ctx_bmap);
10083 }
10084 
10085 static void bnxt_init_multi_rss_ctx(struct bnxt *bp)
10086 {
10087 	bp->rss_ctx_bmap = bitmap_zalloc(BNXT_RSS_CTX_BMAP_LEN, GFP_KERNEL);
10088 	if (bp->rss_ctx_bmap) {
10089 		/* burn index 0 since we cannot have context 0 */
10090 		__set_bit(0, bp->rss_ctx_bmap);
10091 		INIT_LIST_HEAD(&bp->rss_ctx_list);
10092 		bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX;
10093 	}
10094 }
10095 
10096 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
10097 static bool bnxt_promisc_ok(struct bnxt *bp)
10098 {
10099 #ifdef CONFIG_BNXT_SRIOV
10100 	if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
10101 		return false;
10102 #endif
10103 	return true;
10104 }
10105 
10106 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
10107 {
10108 	struct bnxt_vnic_info *vnic = &bp->vnic_info[1];
10109 	unsigned int rc = 0;
10110 
10111 	rc = bnxt_hwrm_vnic_alloc(bp, vnic, bp->rx_nr_rings - 1, 1);
10112 	if (rc) {
10113 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10114 			   rc);
10115 		return rc;
10116 	}
10117 
10118 	rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10119 	if (rc) {
10120 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10121 			   rc);
10122 		return rc;
10123 	}
10124 	return rc;
10125 }
10126 
10127 static int bnxt_cfg_rx_mode(struct bnxt *);
10128 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
10129 
10130 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
10131 {
10132 	struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
10133 	int rc = 0;
10134 	unsigned int rx_nr_rings = bp->rx_nr_rings;
10135 
10136 	if (irq_re_init) {
10137 		rc = bnxt_hwrm_stat_ctx_alloc(bp);
10138 		if (rc) {
10139 			netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
10140 				   rc);
10141 			goto err_out;
10142 		}
10143 	}
10144 
10145 	rc = bnxt_hwrm_ring_alloc(bp);
10146 	if (rc) {
10147 		netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
10148 		goto err_out;
10149 	}
10150 
10151 	rc = bnxt_hwrm_ring_grp_alloc(bp);
10152 	if (rc) {
10153 		netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
10154 		goto err_out;
10155 	}
10156 
10157 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10158 		rx_nr_rings--;
10159 
10160 	/* default vnic 0 */
10161 	rc = bnxt_hwrm_vnic_alloc(bp, vnic, 0, rx_nr_rings);
10162 	if (rc) {
10163 		netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
10164 		goto err_out;
10165 	}
10166 
10167 	if (BNXT_VF(bp))
10168 		bnxt_hwrm_func_qcfg(bp);
10169 
10170 	rc = bnxt_setup_vnic(bp, vnic);
10171 	if (rc)
10172 		goto err_out;
10173 	if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
10174 		bnxt_hwrm_update_rss_hash_cfg(bp);
10175 
10176 	if (bp->flags & BNXT_FLAG_RFS) {
10177 		rc = bnxt_alloc_rfs_vnics(bp);
10178 		if (rc)
10179 			goto err_out;
10180 	}
10181 
10182 	if (bp->flags & BNXT_FLAG_TPA) {
10183 		rc = bnxt_set_tpa(bp, true);
10184 		if (rc)
10185 			goto err_out;
10186 	}
10187 
10188 	if (BNXT_VF(bp))
10189 		bnxt_update_vf_mac(bp);
10190 
10191 	/* Filter for default vnic 0 */
10192 	rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
10193 	if (rc) {
10194 		if (BNXT_VF(bp) && rc == -ENODEV)
10195 			netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
10196 		else
10197 			netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
10198 		goto err_out;
10199 	}
10200 	vnic->uc_filter_count = 1;
10201 
10202 	vnic->rx_mask = 0;
10203 	if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
10204 		goto skip_rx_mask;
10205 
10206 	if (bp->dev->flags & IFF_BROADCAST)
10207 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10208 
10209 	if (bp->dev->flags & IFF_PROMISC)
10210 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10211 
10212 	if (bp->dev->flags & IFF_ALLMULTI) {
10213 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10214 		vnic->mc_list_count = 0;
10215 	} else if (bp->dev->flags & IFF_MULTICAST) {
10216 		u32 mask = 0;
10217 
10218 		bnxt_mc_list_updated(bp, &mask);
10219 		vnic->rx_mask |= mask;
10220 	}
10221 
10222 	rc = bnxt_cfg_rx_mode(bp);
10223 	if (rc)
10224 		goto err_out;
10225 
10226 skip_rx_mask:
10227 	rc = bnxt_hwrm_set_coal(bp);
10228 	if (rc)
10229 		netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
10230 				rc);
10231 
10232 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10233 		rc = bnxt_setup_nitroa0_vnic(bp);
10234 		if (rc)
10235 			netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
10236 				   rc);
10237 	}
10238 
10239 	if (BNXT_VF(bp)) {
10240 		bnxt_hwrm_func_qcfg(bp);
10241 		netdev_update_features(bp->dev);
10242 	}
10243 
10244 	return 0;
10245 
10246 err_out:
10247 	bnxt_hwrm_resource_free(bp, 0, true);
10248 
10249 	return rc;
10250 }
10251 
10252 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
10253 {
10254 	bnxt_hwrm_resource_free(bp, 1, irq_re_init);
10255 	return 0;
10256 }
10257 
10258 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
10259 {
10260 	bnxt_init_cp_rings(bp);
10261 	bnxt_init_rx_rings(bp);
10262 	bnxt_init_tx_rings(bp);
10263 	bnxt_init_ring_grps(bp, irq_re_init);
10264 	bnxt_init_vnics(bp);
10265 
10266 	return bnxt_init_chip(bp, irq_re_init);
10267 }
10268 
10269 static int bnxt_set_real_num_queues(struct bnxt *bp)
10270 {
10271 	int rc;
10272 	struct net_device *dev = bp->dev;
10273 
10274 	rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
10275 					  bp->tx_nr_rings_xdp);
10276 	if (rc)
10277 		return rc;
10278 
10279 	rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
10280 	if (rc)
10281 		return rc;
10282 
10283 #ifdef CONFIG_RFS_ACCEL
10284 	if (bp->flags & BNXT_FLAG_RFS)
10285 		dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
10286 #endif
10287 
10288 	return rc;
10289 }
10290 
10291 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10292 			     bool shared)
10293 {
10294 	int _rx = *rx, _tx = *tx;
10295 
10296 	if (shared) {
10297 		*rx = min_t(int, _rx, max);
10298 		*tx = min_t(int, _tx, max);
10299 	} else {
10300 		if (max < 2)
10301 			return -ENOMEM;
10302 
10303 		while (_rx + _tx > max) {
10304 			if (_rx > _tx && _rx > 1)
10305 				_rx--;
10306 			else if (_tx > 1)
10307 				_tx--;
10308 		}
10309 		*rx = _rx;
10310 		*tx = _tx;
10311 	}
10312 	return 0;
10313 }
10314 
10315 static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
10316 {
10317 	return (tx - tx_xdp) / tx_sets + tx_xdp;
10318 }
10319 
10320 int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
10321 {
10322 	int tcs = bp->num_tc;
10323 
10324 	if (!tcs)
10325 		tcs = 1;
10326 	return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
10327 }
10328 
10329 static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
10330 {
10331 	int tcs = bp->num_tc;
10332 
10333 	return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
10334 	       bp->tx_nr_rings_xdp;
10335 }
10336 
10337 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10338 			   bool sh)
10339 {
10340 	int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
10341 
10342 	if (tx_cp != *tx) {
10343 		int tx_saved = tx_cp, rc;
10344 
10345 		rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
10346 		if (rc)
10347 			return rc;
10348 		if (tx_cp != tx_saved)
10349 			*tx = bnxt_num_cp_to_tx(bp, tx_cp);
10350 		return 0;
10351 	}
10352 	return __bnxt_trim_rings(bp, rx, tx, max, sh);
10353 }
10354 
10355 static void bnxt_setup_msix(struct bnxt *bp)
10356 {
10357 	const int len = sizeof(bp->irq_tbl[0].name);
10358 	struct net_device *dev = bp->dev;
10359 	int tcs, i;
10360 
10361 	tcs = bp->num_tc;
10362 	if (tcs) {
10363 		int i, off, count;
10364 
10365 		for (i = 0; i < tcs; i++) {
10366 			count = bp->tx_nr_rings_per_tc;
10367 			off = BNXT_TC_TO_RING_BASE(bp, i);
10368 			netdev_set_tc_queue(dev, i, count, off);
10369 		}
10370 	}
10371 
10372 	for (i = 0; i < bp->cp_nr_rings; i++) {
10373 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10374 		char *attr;
10375 
10376 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10377 			attr = "TxRx";
10378 		else if (i < bp->rx_nr_rings)
10379 			attr = "rx";
10380 		else
10381 			attr = "tx";
10382 
10383 		snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
10384 			 attr, i);
10385 		bp->irq_tbl[map_idx].handler = bnxt_msix;
10386 	}
10387 }
10388 
10389 static void bnxt_setup_inta(struct bnxt *bp)
10390 {
10391 	const int len = sizeof(bp->irq_tbl[0].name);
10392 
10393 	if (bp->num_tc) {
10394 		netdev_reset_tc(bp->dev);
10395 		bp->num_tc = 0;
10396 	}
10397 
10398 	snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
10399 		 0);
10400 	bp->irq_tbl[0].handler = bnxt_inta;
10401 }
10402 
10403 static int bnxt_init_int_mode(struct bnxt *bp);
10404 
10405 static int bnxt_setup_int_mode(struct bnxt *bp)
10406 {
10407 	int rc;
10408 
10409 	if (!bp->irq_tbl) {
10410 		rc = bnxt_init_int_mode(bp);
10411 		if (rc || !bp->irq_tbl)
10412 			return rc ?: -ENODEV;
10413 	}
10414 
10415 	if (bp->flags & BNXT_FLAG_USING_MSIX)
10416 		bnxt_setup_msix(bp);
10417 	else
10418 		bnxt_setup_inta(bp);
10419 
10420 	rc = bnxt_set_real_num_queues(bp);
10421 	return rc;
10422 }
10423 
10424 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
10425 {
10426 	return bp->hw_resc.max_rsscos_ctxs;
10427 }
10428 
10429 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
10430 {
10431 	return bp->hw_resc.max_vnics;
10432 }
10433 
10434 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
10435 {
10436 	return bp->hw_resc.max_stat_ctxs;
10437 }
10438 
10439 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
10440 {
10441 	return bp->hw_resc.max_cp_rings;
10442 }
10443 
10444 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
10445 {
10446 	unsigned int cp = bp->hw_resc.max_cp_rings;
10447 
10448 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10449 		cp -= bnxt_get_ulp_msix_num(bp);
10450 
10451 	return cp;
10452 }
10453 
10454 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
10455 {
10456 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10457 
10458 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10459 		return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
10460 
10461 	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
10462 }
10463 
10464 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
10465 {
10466 	bp->hw_resc.max_irqs = max_irqs;
10467 }
10468 
10469 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
10470 {
10471 	unsigned int cp;
10472 
10473 	cp = bnxt_get_max_func_cp_rings_for_en(bp);
10474 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10475 		return cp - bp->rx_nr_rings - bp->tx_nr_rings;
10476 	else
10477 		return cp - bp->cp_nr_rings;
10478 }
10479 
10480 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
10481 {
10482 	return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
10483 }
10484 
10485 int bnxt_get_avail_msix(struct bnxt *bp, int num)
10486 {
10487 	int max_cp = bnxt_get_max_func_cp_rings(bp);
10488 	int max_irq = bnxt_get_max_func_irqs(bp);
10489 	int total_req = bp->cp_nr_rings + num;
10490 	int max_idx, avail_msix;
10491 
10492 	max_idx = bp->total_irqs;
10493 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10494 		max_idx = min_t(int, bp->total_irqs, max_cp);
10495 	avail_msix = max_idx - bp->cp_nr_rings;
10496 	if (!BNXT_NEW_RM(bp) || avail_msix >= num)
10497 		return avail_msix;
10498 
10499 	if (max_irq < total_req) {
10500 		num = max_irq - bp->cp_nr_rings;
10501 		if (num <= 0)
10502 			return 0;
10503 	}
10504 	return num;
10505 }
10506 
10507 static int bnxt_get_num_msix(struct bnxt *bp)
10508 {
10509 	if (!BNXT_NEW_RM(bp))
10510 		return bnxt_get_max_func_irqs(bp);
10511 
10512 	return bnxt_nq_rings_in_use(bp);
10513 }
10514 
10515 static int bnxt_init_msix(struct bnxt *bp)
10516 {
10517 	int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp;
10518 	struct msix_entry *msix_ent;
10519 
10520 	total_vecs = bnxt_get_num_msix(bp);
10521 	max = bnxt_get_max_func_irqs(bp);
10522 	if (total_vecs > max)
10523 		total_vecs = max;
10524 
10525 	if (!total_vecs)
10526 		return 0;
10527 
10528 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
10529 	if (!msix_ent)
10530 		return -ENOMEM;
10531 
10532 	for (i = 0; i < total_vecs; i++) {
10533 		msix_ent[i].entry = i;
10534 		msix_ent[i].vector = 0;
10535 	}
10536 
10537 	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
10538 		min = 2;
10539 
10540 	total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
10541 	ulp_msix = bnxt_get_ulp_msix_num(bp);
10542 	if (total_vecs < 0 || total_vecs < ulp_msix) {
10543 		rc = -ENODEV;
10544 		goto msix_setup_exit;
10545 	}
10546 
10547 	bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
10548 	if (bp->irq_tbl) {
10549 		for (i = 0; i < total_vecs; i++)
10550 			bp->irq_tbl[i].vector = msix_ent[i].vector;
10551 
10552 		bp->total_irqs = total_vecs;
10553 		/* Trim rings based upon num of vectors allocated */
10554 		rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
10555 				     total_vecs - ulp_msix, min == 1);
10556 		if (rc)
10557 			goto msix_setup_exit;
10558 
10559 		tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
10560 		bp->cp_nr_rings = (min == 1) ?
10561 				  max_t(int, tx_cp, bp->rx_nr_rings) :
10562 				  tx_cp + bp->rx_nr_rings;
10563 
10564 	} else {
10565 		rc = -ENOMEM;
10566 		goto msix_setup_exit;
10567 	}
10568 	bp->flags |= BNXT_FLAG_USING_MSIX;
10569 	kfree(msix_ent);
10570 	return 0;
10571 
10572 msix_setup_exit:
10573 	netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
10574 	kfree(bp->irq_tbl);
10575 	bp->irq_tbl = NULL;
10576 	pci_disable_msix(bp->pdev);
10577 	kfree(msix_ent);
10578 	return rc;
10579 }
10580 
10581 static int bnxt_init_inta(struct bnxt *bp)
10582 {
10583 	bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
10584 	if (!bp->irq_tbl)
10585 		return -ENOMEM;
10586 
10587 	bp->total_irqs = 1;
10588 	bp->rx_nr_rings = 1;
10589 	bp->tx_nr_rings = 1;
10590 	bp->cp_nr_rings = 1;
10591 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
10592 	bp->irq_tbl[0].vector = bp->pdev->irq;
10593 	return 0;
10594 }
10595 
10596 static int bnxt_init_int_mode(struct bnxt *bp)
10597 {
10598 	int rc = -ENODEV;
10599 
10600 	if (bp->flags & BNXT_FLAG_MSIX_CAP)
10601 		rc = bnxt_init_msix(bp);
10602 
10603 	if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
10604 		/* fallback to INTA */
10605 		rc = bnxt_init_inta(bp);
10606 	}
10607 	return rc;
10608 }
10609 
10610 static void bnxt_clear_int_mode(struct bnxt *bp)
10611 {
10612 	if (bp->flags & BNXT_FLAG_USING_MSIX)
10613 		pci_disable_msix(bp->pdev);
10614 
10615 	kfree(bp->irq_tbl);
10616 	bp->irq_tbl = NULL;
10617 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
10618 }
10619 
10620 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
10621 {
10622 	bool irq_cleared = false;
10623 	int tcs = bp->num_tc;
10624 	int irqs_required;
10625 	int rc;
10626 
10627 	if (!bnxt_need_reserve_rings(bp))
10628 		return 0;
10629 
10630 	if (!bnxt_ulp_registered(bp->edev)) {
10631 		int ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
10632 
10633 		if (ulp_msix > bp->ulp_num_msix_want)
10634 			ulp_msix = bp->ulp_num_msix_want;
10635 		irqs_required = ulp_msix + bp->cp_nr_rings;
10636 	} else {
10637 		irqs_required = bnxt_get_num_msix(bp);
10638 	}
10639 
10640 	if (irq_re_init && BNXT_NEW_RM(bp) && irqs_required != bp->total_irqs) {
10641 		bnxt_ulp_irq_stop(bp);
10642 		bnxt_clear_int_mode(bp);
10643 		irq_cleared = true;
10644 	}
10645 	rc = __bnxt_reserve_rings(bp);
10646 	if (irq_cleared) {
10647 		if (!rc)
10648 			rc = bnxt_init_int_mode(bp);
10649 		bnxt_ulp_irq_restart(bp, rc);
10650 	}
10651 	if (rc) {
10652 		netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
10653 		return rc;
10654 	}
10655 	if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
10656 		    bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
10657 		netdev_err(bp->dev, "tx ring reservation failure\n");
10658 		netdev_reset_tc(bp->dev);
10659 		bp->num_tc = 0;
10660 		if (bp->tx_nr_rings_xdp)
10661 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
10662 		else
10663 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10664 		return -ENOMEM;
10665 	}
10666 	return 0;
10667 }
10668 
10669 static void bnxt_free_irq(struct bnxt *bp)
10670 {
10671 	struct bnxt_irq *irq;
10672 	int i;
10673 
10674 #ifdef CONFIG_RFS_ACCEL
10675 	free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
10676 	bp->dev->rx_cpu_rmap = NULL;
10677 #endif
10678 	if (!bp->irq_tbl || !bp->bnapi)
10679 		return;
10680 
10681 	for (i = 0; i < bp->cp_nr_rings; i++) {
10682 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10683 
10684 		irq = &bp->irq_tbl[map_idx];
10685 		if (irq->requested) {
10686 			if (irq->have_cpumask) {
10687 				irq_set_affinity_hint(irq->vector, NULL);
10688 				free_cpumask_var(irq->cpu_mask);
10689 				irq->have_cpumask = 0;
10690 			}
10691 			free_irq(irq->vector, bp->bnapi[i]);
10692 		}
10693 
10694 		irq->requested = 0;
10695 	}
10696 }
10697 
10698 static int bnxt_request_irq(struct bnxt *bp)
10699 {
10700 	int i, j, rc = 0;
10701 	unsigned long flags = 0;
10702 #ifdef CONFIG_RFS_ACCEL
10703 	struct cpu_rmap *rmap;
10704 #endif
10705 
10706 	rc = bnxt_setup_int_mode(bp);
10707 	if (rc) {
10708 		netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
10709 			   rc);
10710 		return rc;
10711 	}
10712 #ifdef CONFIG_RFS_ACCEL
10713 	rmap = bp->dev->rx_cpu_rmap;
10714 #endif
10715 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
10716 		flags = IRQF_SHARED;
10717 
10718 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
10719 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10720 		struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
10721 
10722 #ifdef CONFIG_RFS_ACCEL
10723 		if (rmap && bp->bnapi[i]->rx_ring) {
10724 			rc = irq_cpu_rmap_add(rmap, irq->vector);
10725 			if (rc)
10726 				netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
10727 					    j);
10728 			j++;
10729 		}
10730 #endif
10731 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
10732 				 bp->bnapi[i]);
10733 		if (rc)
10734 			break;
10735 
10736 		netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector);
10737 		irq->requested = 1;
10738 
10739 		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
10740 			int numa_node = dev_to_node(&bp->pdev->dev);
10741 
10742 			irq->have_cpumask = 1;
10743 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
10744 					irq->cpu_mask);
10745 			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
10746 			if (rc) {
10747 				netdev_warn(bp->dev,
10748 					    "Set affinity failed, IRQ = %d\n",
10749 					    irq->vector);
10750 				break;
10751 			}
10752 		}
10753 	}
10754 	return rc;
10755 }
10756 
10757 static void bnxt_del_napi(struct bnxt *bp)
10758 {
10759 	int i;
10760 
10761 	if (!bp->bnapi)
10762 		return;
10763 
10764 	for (i = 0; i < bp->rx_nr_rings; i++)
10765 		netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
10766 	for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++)
10767 		netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL);
10768 
10769 	for (i = 0; i < bp->cp_nr_rings; i++) {
10770 		struct bnxt_napi *bnapi = bp->bnapi[i];
10771 
10772 		__netif_napi_del(&bnapi->napi);
10773 	}
10774 	/* We called __netif_napi_del(), we need
10775 	 * to respect an RCU grace period before freeing napi structures.
10776 	 */
10777 	synchronize_net();
10778 }
10779 
10780 static void bnxt_init_napi(struct bnxt *bp)
10781 {
10782 	int i;
10783 	unsigned int cp_nr_rings = bp->cp_nr_rings;
10784 	struct bnxt_napi *bnapi;
10785 
10786 	if (bp->flags & BNXT_FLAG_USING_MSIX) {
10787 		int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
10788 
10789 		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10790 			poll_fn = bnxt_poll_p5;
10791 		else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10792 			cp_nr_rings--;
10793 		for (i = 0; i < cp_nr_rings; i++) {
10794 			bnapi = bp->bnapi[i];
10795 			netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
10796 		}
10797 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10798 			bnapi = bp->bnapi[cp_nr_rings];
10799 			netif_napi_add(bp->dev, &bnapi->napi,
10800 				       bnxt_poll_nitroa0);
10801 		}
10802 	} else {
10803 		bnapi = bp->bnapi[0];
10804 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
10805 	}
10806 }
10807 
10808 static void bnxt_disable_napi(struct bnxt *bp)
10809 {
10810 	int i;
10811 
10812 	if (!bp->bnapi ||
10813 	    test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
10814 		return;
10815 
10816 	for (i = 0; i < bp->cp_nr_rings; i++) {
10817 		struct bnxt_napi *bnapi = bp->bnapi[i];
10818 		struct bnxt_cp_ring_info *cpr;
10819 
10820 		cpr = &bnapi->cp_ring;
10821 		if (bnapi->tx_fault)
10822 			cpr->sw_stats.tx.tx_resets++;
10823 		if (bnapi->in_reset)
10824 			cpr->sw_stats.rx.rx_resets++;
10825 		napi_disable(&bnapi->napi);
10826 		if (bnapi->rx_ring)
10827 			cancel_work_sync(&cpr->dim.work);
10828 	}
10829 }
10830 
10831 static void bnxt_enable_napi(struct bnxt *bp)
10832 {
10833 	int i;
10834 
10835 	clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
10836 	for (i = 0; i < bp->cp_nr_rings; i++) {
10837 		struct bnxt_napi *bnapi = bp->bnapi[i];
10838 		struct bnxt_cp_ring_info *cpr;
10839 
10840 		bnapi->tx_fault = 0;
10841 
10842 		cpr = &bnapi->cp_ring;
10843 		bnapi->in_reset = false;
10844 
10845 		if (bnapi->rx_ring) {
10846 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
10847 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
10848 		}
10849 		napi_enable(&bnapi->napi);
10850 	}
10851 }
10852 
10853 void bnxt_tx_disable(struct bnxt *bp)
10854 {
10855 	int i;
10856 	struct bnxt_tx_ring_info *txr;
10857 
10858 	if (bp->tx_ring) {
10859 		for (i = 0; i < bp->tx_nr_rings; i++) {
10860 			txr = &bp->tx_ring[i];
10861 			WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
10862 		}
10863 	}
10864 	/* Make sure napi polls see @dev_state change */
10865 	synchronize_net();
10866 	/* Drop carrier first to prevent TX timeout */
10867 	netif_carrier_off(bp->dev);
10868 	/* Stop all TX queues */
10869 	netif_tx_disable(bp->dev);
10870 }
10871 
10872 void bnxt_tx_enable(struct bnxt *bp)
10873 {
10874 	int i;
10875 	struct bnxt_tx_ring_info *txr;
10876 
10877 	for (i = 0; i < bp->tx_nr_rings; i++) {
10878 		txr = &bp->tx_ring[i];
10879 		WRITE_ONCE(txr->dev_state, 0);
10880 	}
10881 	/* Make sure napi polls see @dev_state change */
10882 	synchronize_net();
10883 	netif_tx_wake_all_queues(bp->dev);
10884 	if (BNXT_LINK_IS_UP(bp))
10885 		netif_carrier_on(bp->dev);
10886 }
10887 
10888 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
10889 {
10890 	u8 active_fec = link_info->active_fec_sig_mode &
10891 			PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
10892 
10893 	switch (active_fec) {
10894 	default:
10895 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
10896 		return "None";
10897 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
10898 		return "Clause 74 BaseR";
10899 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
10900 		return "Clause 91 RS(528,514)";
10901 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
10902 		return "Clause 91 RS544_1XN";
10903 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
10904 		return "Clause 91 RS(544,514)";
10905 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
10906 		return "Clause 91 RS272_1XN";
10907 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
10908 		return "Clause 91 RS(272,257)";
10909 	}
10910 }
10911 
10912 void bnxt_report_link(struct bnxt *bp)
10913 {
10914 	if (BNXT_LINK_IS_UP(bp)) {
10915 		const char *signal = "";
10916 		const char *flow_ctrl;
10917 		const char *duplex;
10918 		u32 speed;
10919 		u16 fec;
10920 
10921 		netif_carrier_on(bp->dev);
10922 		speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
10923 		if (speed == SPEED_UNKNOWN) {
10924 			netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
10925 			return;
10926 		}
10927 		if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
10928 			duplex = "full";
10929 		else
10930 			duplex = "half";
10931 		if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
10932 			flow_ctrl = "ON - receive & transmit";
10933 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
10934 			flow_ctrl = "ON - transmit";
10935 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
10936 			flow_ctrl = "ON - receive";
10937 		else
10938 			flow_ctrl = "none";
10939 		if (bp->link_info.phy_qcfg_resp.option_flags &
10940 		    PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
10941 			u8 sig_mode = bp->link_info.active_fec_sig_mode &
10942 				      PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
10943 			switch (sig_mode) {
10944 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
10945 				signal = "(NRZ) ";
10946 				break;
10947 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
10948 				signal = "(PAM4 56Gbps) ";
10949 				break;
10950 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112:
10951 				signal = "(PAM4 112Gbps) ";
10952 				break;
10953 			default:
10954 				break;
10955 			}
10956 		}
10957 		netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
10958 			    speed, signal, duplex, flow_ctrl);
10959 		if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
10960 			netdev_info(bp->dev, "EEE is %s\n",
10961 				    bp->eee.eee_active ? "active" :
10962 							 "not active");
10963 		fec = bp->link_info.fec_cfg;
10964 		if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
10965 			netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
10966 				    (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
10967 				    bnxt_report_fec(&bp->link_info));
10968 	} else {
10969 		netif_carrier_off(bp->dev);
10970 		netdev_err(bp->dev, "NIC Link is Down\n");
10971 	}
10972 }
10973 
10974 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
10975 {
10976 	if (!resp->supported_speeds_auto_mode &&
10977 	    !resp->supported_speeds_force_mode &&
10978 	    !resp->supported_pam4_speeds_auto_mode &&
10979 	    !resp->supported_pam4_speeds_force_mode &&
10980 	    !resp->supported_speeds2_auto_mode &&
10981 	    !resp->supported_speeds2_force_mode)
10982 		return true;
10983 	return false;
10984 }
10985 
10986 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
10987 {
10988 	struct bnxt_link_info *link_info = &bp->link_info;
10989 	struct hwrm_port_phy_qcaps_output *resp;
10990 	struct hwrm_port_phy_qcaps_input *req;
10991 	int rc = 0;
10992 
10993 	if (bp->hwrm_spec_code < 0x10201)
10994 		return 0;
10995 
10996 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
10997 	if (rc)
10998 		return rc;
10999 
11000 	resp = hwrm_req_hold(bp, req);
11001 	rc = hwrm_req_send(bp, req);
11002 	if (rc)
11003 		goto hwrm_phy_qcaps_exit;
11004 
11005 	bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
11006 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
11007 		struct ethtool_keee *eee = &bp->eee;
11008 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
11009 
11010 		_bnxt_fw_to_linkmode(eee->supported, fw_speeds);
11011 		bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
11012 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
11013 		bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
11014 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
11015 	}
11016 
11017 	if (bp->hwrm_spec_code >= 0x10a01) {
11018 		if (bnxt_phy_qcaps_no_speed(resp)) {
11019 			link_info->phy_state = BNXT_PHY_STATE_DISABLED;
11020 			netdev_warn(bp->dev, "Ethernet link disabled\n");
11021 		} else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
11022 			link_info->phy_state = BNXT_PHY_STATE_ENABLED;
11023 			netdev_info(bp->dev, "Ethernet link enabled\n");
11024 			/* Phy re-enabled, reprobe the speeds */
11025 			link_info->support_auto_speeds = 0;
11026 			link_info->support_pam4_auto_speeds = 0;
11027 			link_info->support_auto_speeds2 = 0;
11028 		}
11029 	}
11030 	if (resp->supported_speeds_auto_mode)
11031 		link_info->support_auto_speeds =
11032 			le16_to_cpu(resp->supported_speeds_auto_mode);
11033 	if (resp->supported_pam4_speeds_auto_mode)
11034 		link_info->support_pam4_auto_speeds =
11035 			le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
11036 	if (resp->supported_speeds2_auto_mode)
11037 		link_info->support_auto_speeds2 =
11038 			le16_to_cpu(resp->supported_speeds2_auto_mode);
11039 
11040 	bp->port_count = resp->port_cnt;
11041 
11042 hwrm_phy_qcaps_exit:
11043 	hwrm_req_drop(bp, req);
11044 	return rc;
11045 }
11046 
11047 static bool bnxt_support_dropped(u16 advertising, u16 supported)
11048 {
11049 	u16 diff = advertising ^ supported;
11050 
11051 	return ((supported | diff) != supported);
11052 }
11053 
11054 static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info)
11055 {
11056 	struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
11057 
11058 	/* Check if any advertised speeds are no longer supported. The caller
11059 	 * holds the link_lock mutex, so we can modify link_info settings.
11060 	 */
11061 	if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11062 		if (bnxt_support_dropped(link_info->advertising,
11063 					 link_info->support_auto_speeds2)) {
11064 			link_info->advertising = link_info->support_auto_speeds2;
11065 			return true;
11066 		}
11067 		return false;
11068 	}
11069 	if (bnxt_support_dropped(link_info->advertising,
11070 				 link_info->support_auto_speeds)) {
11071 		link_info->advertising = link_info->support_auto_speeds;
11072 		return true;
11073 	}
11074 	if (bnxt_support_dropped(link_info->advertising_pam4,
11075 				 link_info->support_pam4_auto_speeds)) {
11076 		link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
11077 		return true;
11078 	}
11079 	return false;
11080 }
11081 
11082 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
11083 {
11084 	struct bnxt_link_info *link_info = &bp->link_info;
11085 	struct hwrm_port_phy_qcfg_output *resp;
11086 	struct hwrm_port_phy_qcfg_input *req;
11087 	u8 link_state = link_info->link_state;
11088 	bool support_changed;
11089 	int rc;
11090 
11091 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
11092 	if (rc)
11093 		return rc;
11094 
11095 	resp = hwrm_req_hold(bp, req);
11096 	rc = hwrm_req_send(bp, req);
11097 	if (rc) {
11098 		hwrm_req_drop(bp, req);
11099 		if (BNXT_VF(bp) && rc == -ENODEV) {
11100 			netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
11101 			rc = 0;
11102 		}
11103 		return rc;
11104 	}
11105 
11106 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
11107 	link_info->phy_link_status = resp->link;
11108 	link_info->duplex = resp->duplex_cfg;
11109 	if (bp->hwrm_spec_code >= 0x10800)
11110 		link_info->duplex = resp->duplex_state;
11111 	link_info->pause = resp->pause;
11112 	link_info->auto_mode = resp->auto_mode;
11113 	link_info->auto_pause_setting = resp->auto_pause;
11114 	link_info->lp_pause = resp->link_partner_adv_pause;
11115 	link_info->force_pause_setting = resp->force_pause;
11116 	link_info->duplex_setting = resp->duplex_cfg;
11117 	if (link_info->phy_link_status == BNXT_LINK_LINK) {
11118 		link_info->link_speed = le16_to_cpu(resp->link_speed);
11119 		if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
11120 			link_info->active_lanes = resp->active_lanes;
11121 	} else {
11122 		link_info->link_speed = 0;
11123 		link_info->active_lanes = 0;
11124 	}
11125 	link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
11126 	link_info->force_pam4_link_speed =
11127 		le16_to_cpu(resp->force_pam4_link_speed);
11128 	link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2);
11129 	link_info->support_speeds = le16_to_cpu(resp->support_speeds);
11130 	link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
11131 	link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2);
11132 	link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
11133 	link_info->auto_pam4_link_speeds =
11134 		le16_to_cpu(resp->auto_pam4_link_speed_mask);
11135 	link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2);
11136 	link_info->lp_auto_link_speeds =
11137 		le16_to_cpu(resp->link_partner_adv_speeds);
11138 	link_info->lp_auto_pam4_link_speeds =
11139 		resp->link_partner_pam4_adv_speeds;
11140 	link_info->preemphasis = le32_to_cpu(resp->preemphasis);
11141 	link_info->phy_ver[0] = resp->phy_maj;
11142 	link_info->phy_ver[1] = resp->phy_min;
11143 	link_info->phy_ver[2] = resp->phy_bld;
11144 	link_info->media_type = resp->media_type;
11145 	link_info->phy_type = resp->phy_type;
11146 	link_info->transceiver = resp->xcvr_pkg_type;
11147 	link_info->phy_addr = resp->eee_config_phy_addr &
11148 			      PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
11149 	link_info->module_status = resp->module_status;
11150 
11151 	if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
11152 		struct ethtool_keee *eee = &bp->eee;
11153 		u16 fw_speeds;
11154 
11155 		eee->eee_active = 0;
11156 		if (resp->eee_config_phy_addr &
11157 		    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
11158 			eee->eee_active = 1;
11159 			fw_speeds = le16_to_cpu(
11160 				resp->link_partner_adv_eee_link_speed_mask);
11161 			_bnxt_fw_to_linkmode(eee->lp_advertised, fw_speeds);
11162 		}
11163 
11164 		/* Pull initial EEE config */
11165 		if (!chng_link_state) {
11166 			if (resp->eee_config_phy_addr &
11167 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
11168 				eee->eee_enabled = 1;
11169 
11170 			fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
11171 			_bnxt_fw_to_linkmode(eee->advertised, fw_speeds);
11172 
11173 			if (resp->eee_config_phy_addr &
11174 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
11175 				__le32 tmr;
11176 
11177 				eee->tx_lpi_enabled = 1;
11178 				tmr = resp->xcvr_identifier_type_tx_lpi_timer;
11179 				eee->tx_lpi_timer = le32_to_cpu(tmr) &
11180 					PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
11181 			}
11182 		}
11183 	}
11184 
11185 	link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
11186 	if (bp->hwrm_spec_code >= 0x10504) {
11187 		link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
11188 		link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
11189 	}
11190 	/* TODO: need to add more logic to report VF link */
11191 	if (chng_link_state) {
11192 		if (link_info->phy_link_status == BNXT_LINK_LINK)
11193 			link_info->link_state = BNXT_LINK_STATE_UP;
11194 		else
11195 			link_info->link_state = BNXT_LINK_STATE_DOWN;
11196 		if (link_state != link_info->link_state)
11197 			bnxt_report_link(bp);
11198 	} else {
11199 		/* always link down if not require to update link state */
11200 		link_info->link_state = BNXT_LINK_STATE_DOWN;
11201 	}
11202 	hwrm_req_drop(bp, req);
11203 
11204 	if (!BNXT_PHY_CFG_ABLE(bp))
11205 		return 0;
11206 
11207 	support_changed = bnxt_support_speed_dropped(link_info);
11208 	if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
11209 		bnxt_hwrm_set_link_setting(bp, true, false);
11210 	return 0;
11211 }
11212 
11213 static void bnxt_get_port_module_status(struct bnxt *bp)
11214 {
11215 	struct bnxt_link_info *link_info = &bp->link_info;
11216 	struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
11217 	u8 module_status;
11218 
11219 	if (bnxt_update_link(bp, true))
11220 		return;
11221 
11222 	module_status = link_info->module_status;
11223 	switch (module_status) {
11224 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
11225 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
11226 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
11227 		netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
11228 			    bp->pf.port_id);
11229 		if (bp->hwrm_spec_code >= 0x10201) {
11230 			netdev_warn(bp->dev, "Module part number %s\n",
11231 				    resp->phy_vendor_partnumber);
11232 		}
11233 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
11234 			netdev_warn(bp->dev, "TX is disabled\n");
11235 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
11236 			netdev_warn(bp->dev, "SFP+ module is shutdown\n");
11237 	}
11238 }
11239 
11240 static void
11241 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11242 {
11243 	if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
11244 		if (bp->hwrm_spec_code >= 0x10201)
11245 			req->auto_pause =
11246 				PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
11247 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11248 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
11249 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11250 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
11251 		req->enables |=
11252 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11253 	} else {
11254 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11255 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
11256 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11257 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
11258 		req->enables |=
11259 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
11260 		if (bp->hwrm_spec_code >= 0x10201) {
11261 			req->auto_pause = req->force_pause;
11262 			req->enables |= cpu_to_le32(
11263 				PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11264 		}
11265 	}
11266 }
11267 
11268 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11269 {
11270 	if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
11271 		req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
11272 		if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11273 			req->enables |=
11274 				cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK);
11275 			req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising);
11276 		} else if (bp->link_info.advertising) {
11277 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
11278 			req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
11279 		}
11280 		if (bp->link_info.advertising_pam4) {
11281 			req->enables |=
11282 				cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
11283 			req->auto_link_pam4_speed_mask =
11284 				cpu_to_le16(bp->link_info.advertising_pam4);
11285 		}
11286 		req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
11287 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
11288 	} else {
11289 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
11290 		if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11291 			req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed);
11292 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2);
11293 			netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n",
11294 				   (u32)bp->link_info.req_link_speed);
11295 		} else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
11296 			req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11297 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
11298 		} else {
11299 			req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11300 		}
11301 	}
11302 
11303 	/* tell chimp that the setting takes effect immediately */
11304 	req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
11305 }
11306 
11307 int bnxt_hwrm_set_pause(struct bnxt *bp)
11308 {
11309 	struct hwrm_port_phy_cfg_input *req;
11310 	int rc;
11311 
11312 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11313 	if (rc)
11314 		return rc;
11315 
11316 	bnxt_hwrm_set_pause_common(bp, req);
11317 
11318 	if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
11319 	    bp->link_info.force_link_chng)
11320 		bnxt_hwrm_set_link_common(bp, req);
11321 
11322 	rc = hwrm_req_send(bp, req);
11323 	if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
11324 		/* since changing of pause setting doesn't trigger any link
11325 		 * change event, the driver needs to update the current pause
11326 		 * result upon successfully return of the phy_cfg command
11327 		 */
11328 		bp->link_info.pause =
11329 		bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
11330 		bp->link_info.auto_pause_setting = 0;
11331 		if (!bp->link_info.force_link_chng)
11332 			bnxt_report_link(bp);
11333 	}
11334 	bp->link_info.force_link_chng = false;
11335 	return rc;
11336 }
11337 
11338 static void bnxt_hwrm_set_eee(struct bnxt *bp,
11339 			      struct hwrm_port_phy_cfg_input *req)
11340 {
11341 	struct ethtool_keee *eee = &bp->eee;
11342 
11343 	if (eee->eee_enabled) {
11344 		u16 eee_speeds;
11345 		u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
11346 
11347 		if (eee->tx_lpi_enabled)
11348 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
11349 		else
11350 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
11351 
11352 		req->flags |= cpu_to_le32(flags);
11353 		eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
11354 		req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
11355 		req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
11356 	} else {
11357 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
11358 	}
11359 }
11360 
11361 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
11362 {
11363 	struct hwrm_port_phy_cfg_input *req;
11364 	int rc;
11365 
11366 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11367 	if (rc)
11368 		return rc;
11369 
11370 	if (set_pause)
11371 		bnxt_hwrm_set_pause_common(bp, req);
11372 
11373 	bnxt_hwrm_set_link_common(bp, req);
11374 
11375 	if (set_eee)
11376 		bnxt_hwrm_set_eee(bp, req);
11377 	return hwrm_req_send(bp, req);
11378 }
11379 
11380 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
11381 {
11382 	struct hwrm_port_phy_cfg_input *req;
11383 	int rc;
11384 
11385 	if (!BNXT_SINGLE_PF(bp))
11386 		return 0;
11387 
11388 	if (pci_num_vf(bp->pdev) &&
11389 	    !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
11390 		return 0;
11391 
11392 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11393 	if (rc)
11394 		return rc;
11395 
11396 	req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
11397 	rc = hwrm_req_send(bp, req);
11398 	if (!rc) {
11399 		mutex_lock(&bp->link_lock);
11400 		/* Device is not obliged link down in certain scenarios, even
11401 		 * when forced. Setting the state unknown is consistent with
11402 		 * driver startup and will force link state to be reported
11403 		 * during subsequent open based on PORT_PHY_QCFG.
11404 		 */
11405 		bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
11406 		mutex_unlock(&bp->link_lock);
11407 	}
11408 	return rc;
11409 }
11410 
11411 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11412 {
11413 #ifdef CONFIG_TEE_BNXT_FW
11414 	int rc = tee_bnxt_fw_load();
11415 
11416 	if (rc)
11417 		netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11418 
11419 	return rc;
11420 #else
11421 	netdev_err(bp->dev, "OP-TEE not supported\n");
11422 	return -ENODEV;
11423 #endif
11424 }
11425 
11426 static int bnxt_try_recover_fw(struct bnxt *bp)
11427 {
11428 	if (bp->fw_health && bp->fw_health->status_reliable) {
11429 		int retry = 0, rc;
11430 		u32 sts;
11431 
11432 		do {
11433 			sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11434 			rc = bnxt_hwrm_poll(bp);
11435 			if (!BNXT_FW_IS_BOOTING(sts) &&
11436 			    !BNXT_FW_IS_RECOVERING(sts))
11437 				break;
11438 			retry++;
11439 		} while (rc == -EBUSY && retry < BNXT_FW_RETRY);
11440 
11441 		if (!BNXT_FW_IS_HEALTHY(sts)) {
11442 			netdev_err(bp->dev,
11443 				   "Firmware not responding, status: 0x%x\n",
11444 				   sts);
11445 			rc = -ENODEV;
11446 		}
11447 		if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11448 			netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11449 			return bnxt_fw_reset_via_optee(bp);
11450 		}
11451 		return rc;
11452 	}
11453 
11454 	return -ENODEV;
11455 }
11456 
11457 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
11458 {
11459 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11460 
11461 	if (!BNXT_NEW_RM(bp))
11462 		return; /* no resource reservations required */
11463 
11464 	hw_resc->resv_cp_rings = 0;
11465 	hw_resc->resv_stat_ctxs = 0;
11466 	hw_resc->resv_irqs = 0;
11467 	hw_resc->resv_tx_rings = 0;
11468 	hw_resc->resv_rx_rings = 0;
11469 	hw_resc->resv_hw_ring_grps = 0;
11470 	hw_resc->resv_vnics = 0;
11471 	hw_resc->resv_rsscos_ctxs = 0;
11472 	if (!fw_reset) {
11473 		bp->tx_nr_rings = 0;
11474 		bp->rx_nr_rings = 0;
11475 	}
11476 }
11477 
11478 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
11479 {
11480 	int rc;
11481 
11482 	if (!BNXT_NEW_RM(bp))
11483 		return 0; /* no resource reservations required */
11484 
11485 	rc = bnxt_hwrm_func_resc_qcaps(bp, true);
11486 	if (rc)
11487 		netdev_err(bp->dev, "resc_qcaps failed\n");
11488 
11489 	bnxt_clear_reservations(bp, fw_reset);
11490 
11491 	return rc;
11492 }
11493 
11494 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
11495 {
11496 	struct hwrm_func_drv_if_change_output *resp;
11497 	struct hwrm_func_drv_if_change_input *req;
11498 	bool fw_reset = !bp->irq_tbl;
11499 	bool resc_reinit = false;
11500 	int rc, retry = 0;
11501 	u32 flags = 0;
11502 
11503 	if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
11504 		return 0;
11505 
11506 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
11507 	if (rc)
11508 		return rc;
11509 
11510 	if (up)
11511 		req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
11512 	resp = hwrm_req_hold(bp, req);
11513 
11514 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
11515 	while (retry < BNXT_FW_IF_RETRY) {
11516 		rc = hwrm_req_send(bp, req);
11517 		if (rc != -EAGAIN)
11518 			break;
11519 
11520 		msleep(50);
11521 		retry++;
11522 	}
11523 
11524 	if (rc == -EAGAIN) {
11525 		hwrm_req_drop(bp, req);
11526 		return rc;
11527 	} else if (!rc) {
11528 		flags = le32_to_cpu(resp->flags);
11529 	} else if (up) {
11530 		rc = bnxt_try_recover_fw(bp);
11531 		fw_reset = true;
11532 	}
11533 	hwrm_req_drop(bp, req);
11534 	if (rc)
11535 		return rc;
11536 
11537 	if (!up) {
11538 		bnxt_inv_fw_health_reg(bp);
11539 		return 0;
11540 	}
11541 
11542 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
11543 		resc_reinit = true;
11544 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
11545 	    test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
11546 		fw_reset = true;
11547 	else
11548 		bnxt_remap_fw_health_regs(bp);
11549 
11550 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
11551 		netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
11552 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11553 		return -ENODEV;
11554 	}
11555 	if (resc_reinit || fw_reset) {
11556 		if (fw_reset) {
11557 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11558 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11559 				bnxt_ulp_stop(bp);
11560 			bnxt_free_ctx_mem(bp);
11561 			bnxt_dcb_free(bp);
11562 			rc = bnxt_fw_init_one(bp);
11563 			if (rc) {
11564 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11565 				set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11566 				return rc;
11567 			}
11568 			bnxt_clear_int_mode(bp);
11569 			rc = bnxt_init_int_mode(bp);
11570 			if (rc) {
11571 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11572 				netdev_err(bp->dev, "init int mode failed\n");
11573 				return rc;
11574 			}
11575 		}
11576 		rc = bnxt_cancel_reservations(bp, fw_reset);
11577 	}
11578 	return rc;
11579 }
11580 
11581 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
11582 {
11583 	struct hwrm_port_led_qcaps_output *resp;
11584 	struct hwrm_port_led_qcaps_input *req;
11585 	struct bnxt_pf_info *pf = &bp->pf;
11586 	int rc;
11587 
11588 	bp->num_leds = 0;
11589 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
11590 		return 0;
11591 
11592 	rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
11593 	if (rc)
11594 		return rc;
11595 
11596 	req->port_id = cpu_to_le16(pf->port_id);
11597 	resp = hwrm_req_hold(bp, req);
11598 	rc = hwrm_req_send(bp, req);
11599 	if (rc) {
11600 		hwrm_req_drop(bp, req);
11601 		return rc;
11602 	}
11603 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
11604 		int i;
11605 
11606 		bp->num_leds = resp->num_leds;
11607 		memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
11608 						 bp->num_leds);
11609 		for (i = 0; i < bp->num_leds; i++) {
11610 			struct bnxt_led_info *led = &bp->leds[i];
11611 			__le16 caps = led->led_state_caps;
11612 
11613 			if (!led->led_group_id ||
11614 			    !BNXT_LED_ALT_BLINK_CAP(caps)) {
11615 				bp->num_leds = 0;
11616 				break;
11617 			}
11618 		}
11619 	}
11620 	hwrm_req_drop(bp, req);
11621 	return 0;
11622 }
11623 
11624 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
11625 {
11626 	struct hwrm_wol_filter_alloc_output *resp;
11627 	struct hwrm_wol_filter_alloc_input *req;
11628 	int rc;
11629 
11630 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
11631 	if (rc)
11632 		return rc;
11633 
11634 	req->port_id = cpu_to_le16(bp->pf.port_id);
11635 	req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
11636 	req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
11637 	memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
11638 
11639 	resp = hwrm_req_hold(bp, req);
11640 	rc = hwrm_req_send(bp, req);
11641 	if (!rc)
11642 		bp->wol_filter_id = resp->wol_filter_id;
11643 	hwrm_req_drop(bp, req);
11644 	return rc;
11645 }
11646 
11647 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
11648 {
11649 	struct hwrm_wol_filter_free_input *req;
11650 	int rc;
11651 
11652 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
11653 	if (rc)
11654 		return rc;
11655 
11656 	req->port_id = cpu_to_le16(bp->pf.port_id);
11657 	req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
11658 	req->wol_filter_id = bp->wol_filter_id;
11659 
11660 	return hwrm_req_send(bp, req);
11661 }
11662 
11663 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
11664 {
11665 	struct hwrm_wol_filter_qcfg_output *resp;
11666 	struct hwrm_wol_filter_qcfg_input *req;
11667 	u16 next_handle = 0;
11668 	int rc;
11669 
11670 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
11671 	if (rc)
11672 		return rc;
11673 
11674 	req->port_id = cpu_to_le16(bp->pf.port_id);
11675 	req->handle = cpu_to_le16(handle);
11676 	resp = hwrm_req_hold(bp, req);
11677 	rc = hwrm_req_send(bp, req);
11678 	if (!rc) {
11679 		next_handle = le16_to_cpu(resp->next_handle);
11680 		if (next_handle != 0) {
11681 			if (resp->wol_type ==
11682 			    WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
11683 				bp->wol = 1;
11684 				bp->wol_filter_id = resp->wol_filter_id;
11685 			}
11686 		}
11687 	}
11688 	hwrm_req_drop(bp, req);
11689 	return next_handle;
11690 }
11691 
11692 static void bnxt_get_wol_settings(struct bnxt *bp)
11693 {
11694 	u16 handle = 0;
11695 
11696 	bp->wol = 0;
11697 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
11698 		return;
11699 
11700 	do {
11701 		handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
11702 	} while (handle && handle != 0xffff);
11703 }
11704 
11705 static bool bnxt_eee_config_ok(struct bnxt *bp)
11706 {
11707 	struct ethtool_keee *eee = &bp->eee;
11708 	struct bnxt_link_info *link_info = &bp->link_info;
11709 
11710 	if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
11711 		return true;
11712 
11713 	if (eee->eee_enabled) {
11714 		__ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
11715 		__ETHTOOL_DECLARE_LINK_MODE_MASK(tmp);
11716 
11717 		_bnxt_fw_to_linkmode(advertising, link_info->advertising);
11718 
11719 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11720 			eee->eee_enabled = 0;
11721 			return false;
11722 		}
11723 		if (linkmode_andnot(tmp, eee->advertised, advertising)) {
11724 			linkmode_and(eee->advertised, advertising,
11725 				     eee->supported);
11726 			return false;
11727 		}
11728 	}
11729 	return true;
11730 }
11731 
11732 static int bnxt_update_phy_setting(struct bnxt *bp)
11733 {
11734 	int rc;
11735 	bool update_link = false;
11736 	bool update_pause = false;
11737 	bool update_eee = false;
11738 	struct bnxt_link_info *link_info = &bp->link_info;
11739 
11740 	rc = bnxt_update_link(bp, true);
11741 	if (rc) {
11742 		netdev_err(bp->dev, "failed to update link (rc: %x)\n",
11743 			   rc);
11744 		return rc;
11745 	}
11746 	if (!BNXT_SINGLE_PF(bp))
11747 		return 0;
11748 
11749 	if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11750 	    (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
11751 	    link_info->req_flow_ctrl)
11752 		update_pause = true;
11753 	if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11754 	    link_info->force_pause_setting != link_info->req_flow_ctrl)
11755 		update_pause = true;
11756 	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11757 		if (BNXT_AUTO_MODE(link_info->auto_mode))
11758 			update_link = true;
11759 		if (bnxt_force_speed_updated(link_info))
11760 			update_link = true;
11761 		if (link_info->req_duplex != link_info->duplex_setting)
11762 			update_link = true;
11763 	} else {
11764 		if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
11765 			update_link = true;
11766 		if (bnxt_auto_speed_updated(link_info))
11767 			update_link = true;
11768 	}
11769 
11770 	/* The last close may have shutdown the link, so need to call
11771 	 * PHY_CFG to bring it back up.
11772 	 */
11773 	if (!BNXT_LINK_IS_UP(bp))
11774 		update_link = true;
11775 
11776 	if (!bnxt_eee_config_ok(bp))
11777 		update_eee = true;
11778 
11779 	if (update_link)
11780 		rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
11781 	else if (update_pause)
11782 		rc = bnxt_hwrm_set_pause(bp);
11783 	if (rc) {
11784 		netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
11785 			   rc);
11786 		return rc;
11787 	}
11788 
11789 	return rc;
11790 }
11791 
11792 /* Common routine to pre-map certain register block to different GRC window.
11793  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
11794  * in PF and 3 windows in VF that can be customized to map in different
11795  * register blocks.
11796  */
11797 static void bnxt_preset_reg_win(struct bnxt *bp)
11798 {
11799 	if (BNXT_PF(bp)) {
11800 		/* CAG registers map to GRC window #4 */
11801 		writel(BNXT_CAG_REG_BASE,
11802 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
11803 	}
11804 }
11805 
11806 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
11807 
11808 static int bnxt_reinit_after_abort(struct bnxt *bp)
11809 {
11810 	int rc;
11811 
11812 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11813 		return -EBUSY;
11814 
11815 	if (bp->dev->reg_state == NETREG_UNREGISTERED)
11816 		return -ENODEV;
11817 
11818 	rc = bnxt_fw_init_one(bp);
11819 	if (!rc) {
11820 		bnxt_clear_int_mode(bp);
11821 		rc = bnxt_init_int_mode(bp);
11822 		if (!rc) {
11823 			clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11824 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11825 		}
11826 	}
11827 	return rc;
11828 }
11829 
11830 static void bnxt_cfg_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
11831 {
11832 	struct bnxt_ntuple_filter *ntp_fltr;
11833 	struct bnxt_l2_filter *l2_fltr;
11834 
11835 	if (list_empty(&fltr->list))
11836 		return;
11837 
11838 	if (fltr->type == BNXT_FLTR_TYPE_NTUPLE) {
11839 		ntp_fltr = container_of(fltr, struct bnxt_ntuple_filter, base);
11840 		l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
11841 		atomic_inc(&l2_fltr->refcnt);
11842 		ntp_fltr->l2_fltr = l2_fltr;
11843 		if (bnxt_hwrm_cfa_ntuple_filter_alloc(bp, ntp_fltr)) {
11844 			bnxt_del_ntp_filter(bp, ntp_fltr);
11845 			netdev_err(bp->dev, "restoring previously configured ntuple filter id %d failed\n",
11846 				   fltr->sw_id);
11847 		}
11848 	} else if (fltr->type == BNXT_FLTR_TYPE_L2) {
11849 		l2_fltr = container_of(fltr, struct bnxt_l2_filter, base);
11850 		if (bnxt_hwrm_l2_filter_alloc(bp, l2_fltr)) {
11851 			bnxt_del_l2_filter(bp, l2_fltr);
11852 			netdev_err(bp->dev, "restoring previously configured l2 filter id %d failed\n",
11853 				   fltr->sw_id);
11854 		}
11855 	}
11856 }
11857 
11858 static void bnxt_cfg_usr_fltrs(struct bnxt *bp)
11859 {
11860 	struct bnxt_filter_base *usr_fltr, *tmp;
11861 
11862 	list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list)
11863 		bnxt_cfg_one_usr_fltr(bp, usr_fltr);
11864 }
11865 
11866 static int bnxt_set_xps_mapping(struct bnxt *bp)
11867 {
11868 	int numa_node = dev_to_node(&bp->pdev->dev);
11869 	unsigned int q_idx, map_idx, cpu, i;
11870 	const struct cpumask *cpu_mask_ptr;
11871 	int nr_cpus = num_online_cpus();
11872 	cpumask_t *q_map;
11873 	int rc = 0;
11874 
11875 	q_map = kcalloc(bp->tx_nr_rings_per_tc, sizeof(*q_map), GFP_KERNEL);
11876 	if (!q_map)
11877 		return -ENOMEM;
11878 
11879 	/* Create CPU mask for all TX queues across MQPRIO traffic classes.
11880 	 * Each TC has the same number of TX queues. The nth TX queue for each
11881 	 * TC will have the same CPU mask.
11882 	 */
11883 	for (i = 0; i < nr_cpus; i++) {
11884 		map_idx = i % bp->tx_nr_rings_per_tc;
11885 		cpu = cpumask_local_spread(i, numa_node);
11886 		cpu_mask_ptr = get_cpu_mask(cpu);
11887 		cpumask_or(&q_map[map_idx], &q_map[map_idx], cpu_mask_ptr);
11888 	}
11889 
11890 	/* Register CPU mask for each TX queue except the ones marked for XDP */
11891 	for (q_idx = 0; q_idx < bp->dev->real_num_tx_queues; q_idx++) {
11892 		map_idx = q_idx % bp->tx_nr_rings_per_tc;
11893 		rc = netif_set_xps_queue(bp->dev, &q_map[map_idx], q_idx);
11894 		if (rc) {
11895 			netdev_warn(bp->dev, "Error setting XPS for q:%d\n",
11896 				    q_idx);
11897 			break;
11898 		}
11899 	}
11900 
11901 	kfree(q_map);
11902 
11903 	return rc;
11904 }
11905 
11906 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11907 {
11908 	int rc = 0;
11909 
11910 	bnxt_preset_reg_win(bp);
11911 	netif_carrier_off(bp->dev);
11912 	if (irq_re_init) {
11913 		/* Reserve rings now if none were reserved at driver probe. */
11914 		rc = bnxt_init_dflt_ring_mode(bp);
11915 		if (rc) {
11916 			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
11917 			return rc;
11918 		}
11919 	}
11920 	rc = bnxt_reserve_rings(bp, irq_re_init);
11921 	if (rc)
11922 		return rc;
11923 	if ((bp->flags & BNXT_FLAG_RFS) &&
11924 	    !(bp->flags & BNXT_FLAG_USING_MSIX)) {
11925 		/* disable RFS if falling back to INTA */
11926 		bp->dev->hw_features &= ~NETIF_F_NTUPLE;
11927 		bp->flags &= ~BNXT_FLAG_RFS;
11928 	}
11929 
11930 	rc = bnxt_alloc_mem(bp, irq_re_init);
11931 	if (rc) {
11932 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
11933 		goto open_err_free_mem;
11934 	}
11935 
11936 	if (irq_re_init) {
11937 		bnxt_init_napi(bp);
11938 		rc = bnxt_request_irq(bp);
11939 		if (rc) {
11940 			netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
11941 			goto open_err_irq;
11942 		}
11943 	}
11944 
11945 	rc = bnxt_init_nic(bp, irq_re_init);
11946 	if (rc) {
11947 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
11948 		goto open_err_irq;
11949 	}
11950 
11951 	bnxt_enable_napi(bp);
11952 	bnxt_debug_dev_init(bp);
11953 
11954 	if (link_re_init) {
11955 		mutex_lock(&bp->link_lock);
11956 		rc = bnxt_update_phy_setting(bp);
11957 		mutex_unlock(&bp->link_lock);
11958 		if (rc) {
11959 			netdev_warn(bp->dev, "failed to update phy settings\n");
11960 			if (BNXT_SINGLE_PF(bp)) {
11961 				bp->link_info.phy_retry = true;
11962 				bp->link_info.phy_retry_expires =
11963 					jiffies + 5 * HZ;
11964 			}
11965 		}
11966 	}
11967 
11968 	if (irq_re_init) {
11969 		udp_tunnel_nic_reset_ntf(bp->dev);
11970 		rc = bnxt_set_xps_mapping(bp);
11971 		if (rc)
11972 			netdev_warn(bp->dev, "failed to set xps mapping\n");
11973 	}
11974 
11975 	if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
11976 		if (!static_key_enabled(&bnxt_xdp_locking_key))
11977 			static_branch_enable(&bnxt_xdp_locking_key);
11978 	} else if (static_key_enabled(&bnxt_xdp_locking_key)) {
11979 		static_branch_disable(&bnxt_xdp_locking_key);
11980 	}
11981 	set_bit(BNXT_STATE_OPEN, &bp->state);
11982 	bnxt_enable_int(bp);
11983 	/* Enable TX queues */
11984 	bnxt_tx_enable(bp);
11985 	mod_timer(&bp->timer, jiffies + bp->current_interval);
11986 	/* Poll link status and check for SFP+ module status */
11987 	mutex_lock(&bp->link_lock);
11988 	bnxt_get_port_module_status(bp);
11989 	mutex_unlock(&bp->link_lock);
11990 
11991 	/* VF-reps may need to be re-opened after the PF is re-opened */
11992 	if (BNXT_PF(bp))
11993 		bnxt_vf_reps_open(bp);
11994 	if (bp->ptp_cfg)
11995 		atomic_set(&bp->ptp_cfg->tx_avail, BNXT_MAX_TX_TS);
11996 	bnxt_ptp_init_rtc(bp, true);
11997 	bnxt_ptp_cfg_tstamp_filters(bp);
11998 	if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
11999 		bnxt_hwrm_realloc_rss_ctx_vnic(bp);
12000 	bnxt_cfg_usr_fltrs(bp);
12001 	return 0;
12002 
12003 open_err_irq:
12004 	bnxt_del_napi(bp);
12005 
12006 open_err_free_mem:
12007 	bnxt_free_skbs(bp);
12008 	bnxt_free_irq(bp);
12009 	bnxt_free_mem(bp, true);
12010 	return rc;
12011 }
12012 
12013 /* rtnl_lock held */
12014 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12015 {
12016 	int rc = 0;
12017 
12018 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
12019 		rc = -EIO;
12020 	if (!rc)
12021 		rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
12022 	if (rc) {
12023 		netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
12024 		dev_close(bp->dev);
12025 	}
12026 	return rc;
12027 }
12028 
12029 /* rtnl_lock held, open the NIC half way by allocating all resources, but
12030  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
12031  * self tests.
12032  */
12033 int bnxt_half_open_nic(struct bnxt *bp)
12034 {
12035 	int rc = 0;
12036 
12037 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12038 		netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
12039 		rc = -ENODEV;
12040 		goto half_open_err;
12041 	}
12042 
12043 	rc = bnxt_alloc_mem(bp, true);
12044 	if (rc) {
12045 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
12046 		goto half_open_err;
12047 	}
12048 	bnxt_init_napi(bp);
12049 	set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12050 	rc = bnxt_init_nic(bp, true);
12051 	if (rc) {
12052 		clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12053 		bnxt_del_napi(bp);
12054 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
12055 		goto half_open_err;
12056 	}
12057 	return 0;
12058 
12059 half_open_err:
12060 	bnxt_free_skbs(bp);
12061 	bnxt_free_mem(bp, true);
12062 	dev_close(bp->dev);
12063 	return rc;
12064 }
12065 
12066 /* rtnl_lock held, this call can only be made after a previous successful
12067  * call to bnxt_half_open_nic().
12068  */
12069 void bnxt_half_close_nic(struct bnxt *bp)
12070 {
12071 	bnxt_hwrm_resource_free(bp, false, true);
12072 	bnxt_del_napi(bp);
12073 	bnxt_free_skbs(bp);
12074 	bnxt_free_mem(bp, true);
12075 	clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12076 }
12077 
12078 void bnxt_reenable_sriov(struct bnxt *bp)
12079 {
12080 	if (BNXT_PF(bp)) {
12081 		struct bnxt_pf_info *pf = &bp->pf;
12082 		int n = pf->active_vfs;
12083 
12084 		if (n)
12085 			bnxt_cfg_hw_sriov(bp, &n, true);
12086 	}
12087 }
12088 
12089 static int bnxt_open(struct net_device *dev)
12090 {
12091 	struct bnxt *bp = netdev_priv(dev);
12092 	int rc;
12093 
12094 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12095 		rc = bnxt_reinit_after_abort(bp);
12096 		if (rc) {
12097 			if (rc == -EBUSY)
12098 				netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
12099 			else
12100 				netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
12101 			return -ENODEV;
12102 		}
12103 	}
12104 
12105 	rc = bnxt_hwrm_if_change(bp, true);
12106 	if (rc)
12107 		return rc;
12108 
12109 	rc = __bnxt_open_nic(bp, true, true);
12110 	if (rc) {
12111 		bnxt_hwrm_if_change(bp, false);
12112 	} else {
12113 		if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
12114 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12115 				bnxt_ulp_start(bp, 0);
12116 				bnxt_reenable_sriov(bp);
12117 			}
12118 		}
12119 	}
12120 
12121 	return rc;
12122 }
12123 
12124 static bool bnxt_drv_busy(struct bnxt *bp)
12125 {
12126 	return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
12127 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
12128 }
12129 
12130 static void bnxt_get_ring_stats(struct bnxt *bp,
12131 				struct rtnl_link_stats64 *stats);
12132 
12133 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
12134 			     bool link_re_init)
12135 {
12136 	/* Close the VF-reps before closing PF */
12137 	if (BNXT_PF(bp))
12138 		bnxt_vf_reps_close(bp);
12139 
12140 	/* Change device state to avoid TX queue wake up's */
12141 	bnxt_tx_disable(bp);
12142 
12143 	clear_bit(BNXT_STATE_OPEN, &bp->state);
12144 	smp_mb__after_atomic();
12145 	while (bnxt_drv_busy(bp))
12146 		msleep(20);
12147 
12148 	if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
12149 		bnxt_clear_rss_ctxs(bp, false);
12150 	/* Flush rings and disable interrupts */
12151 	bnxt_shutdown_nic(bp, irq_re_init);
12152 
12153 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
12154 
12155 	bnxt_debug_dev_exit(bp);
12156 	bnxt_disable_napi(bp);
12157 	del_timer_sync(&bp->timer);
12158 	bnxt_free_skbs(bp);
12159 
12160 	/* Save ring stats before shutdown */
12161 	if (bp->bnapi && irq_re_init) {
12162 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
12163 		bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
12164 	}
12165 	if (irq_re_init) {
12166 		bnxt_free_irq(bp);
12167 		bnxt_del_napi(bp);
12168 	}
12169 	bnxt_free_mem(bp, irq_re_init);
12170 }
12171 
12172 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12173 {
12174 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12175 		/* If we get here, it means firmware reset is in progress
12176 		 * while we are trying to close.  We can safely proceed with
12177 		 * the close because we are holding rtnl_lock().  Some firmware
12178 		 * messages may fail as we proceed to close.  We set the
12179 		 * ABORT_ERR flag here so that the FW reset thread will later
12180 		 * abort when it gets the rtnl_lock() and sees the flag.
12181 		 */
12182 		netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
12183 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12184 	}
12185 
12186 #ifdef CONFIG_BNXT_SRIOV
12187 	if (bp->sriov_cfg) {
12188 		int rc;
12189 
12190 		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
12191 						      !bp->sriov_cfg,
12192 						      BNXT_SRIOV_CFG_WAIT_TMO);
12193 		if (!rc)
12194 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
12195 		else if (rc < 0)
12196 			netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
12197 	}
12198 #endif
12199 	__bnxt_close_nic(bp, irq_re_init, link_re_init);
12200 }
12201 
12202 static int bnxt_close(struct net_device *dev)
12203 {
12204 	struct bnxt *bp = netdev_priv(dev);
12205 
12206 	bnxt_close_nic(bp, true, true);
12207 	bnxt_hwrm_shutdown_link(bp);
12208 	bnxt_hwrm_if_change(bp, false);
12209 	return 0;
12210 }
12211 
12212 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
12213 				   u16 *val)
12214 {
12215 	struct hwrm_port_phy_mdio_read_output *resp;
12216 	struct hwrm_port_phy_mdio_read_input *req;
12217 	int rc;
12218 
12219 	if (bp->hwrm_spec_code < 0x10a00)
12220 		return -EOPNOTSUPP;
12221 
12222 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
12223 	if (rc)
12224 		return rc;
12225 
12226 	req->port_id = cpu_to_le16(bp->pf.port_id);
12227 	req->phy_addr = phy_addr;
12228 	req->reg_addr = cpu_to_le16(reg & 0x1f);
12229 	if (mdio_phy_id_is_c45(phy_addr)) {
12230 		req->cl45_mdio = 1;
12231 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
12232 		req->dev_addr = mdio_phy_id_devad(phy_addr);
12233 		req->reg_addr = cpu_to_le16(reg);
12234 	}
12235 
12236 	resp = hwrm_req_hold(bp, req);
12237 	rc = hwrm_req_send(bp, req);
12238 	if (!rc)
12239 		*val = le16_to_cpu(resp->reg_data);
12240 	hwrm_req_drop(bp, req);
12241 	return rc;
12242 }
12243 
12244 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
12245 				    u16 val)
12246 {
12247 	struct hwrm_port_phy_mdio_write_input *req;
12248 	int rc;
12249 
12250 	if (bp->hwrm_spec_code < 0x10a00)
12251 		return -EOPNOTSUPP;
12252 
12253 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
12254 	if (rc)
12255 		return rc;
12256 
12257 	req->port_id = cpu_to_le16(bp->pf.port_id);
12258 	req->phy_addr = phy_addr;
12259 	req->reg_addr = cpu_to_le16(reg & 0x1f);
12260 	if (mdio_phy_id_is_c45(phy_addr)) {
12261 		req->cl45_mdio = 1;
12262 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
12263 		req->dev_addr = mdio_phy_id_devad(phy_addr);
12264 		req->reg_addr = cpu_to_le16(reg);
12265 	}
12266 	req->reg_data = cpu_to_le16(val);
12267 
12268 	return hwrm_req_send(bp, req);
12269 }
12270 
12271 /* rtnl_lock held */
12272 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
12273 {
12274 	struct mii_ioctl_data *mdio = if_mii(ifr);
12275 	struct bnxt *bp = netdev_priv(dev);
12276 	int rc;
12277 
12278 	switch (cmd) {
12279 	case SIOCGMIIPHY:
12280 		mdio->phy_id = bp->link_info.phy_addr;
12281 
12282 		fallthrough;
12283 	case SIOCGMIIREG: {
12284 		u16 mii_regval = 0;
12285 
12286 		if (!netif_running(dev))
12287 			return -EAGAIN;
12288 
12289 		rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
12290 					     &mii_regval);
12291 		mdio->val_out = mii_regval;
12292 		return rc;
12293 	}
12294 
12295 	case SIOCSMIIREG:
12296 		if (!netif_running(dev))
12297 			return -EAGAIN;
12298 
12299 		return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
12300 						mdio->val_in);
12301 
12302 	case SIOCSHWTSTAMP:
12303 		return bnxt_hwtstamp_set(dev, ifr);
12304 
12305 	case SIOCGHWTSTAMP:
12306 		return bnxt_hwtstamp_get(dev, ifr);
12307 
12308 	default:
12309 		/* do nothing */
12310 		break;
12311 	}
12312 	return -EOPNOTSUPP;
12313 }
12314 
12315 static void bnxt_get_ring_stats(struct bnxt *bp,
12316 				struct rtnl_link_stats64 *stats)
12317 {
12318 	int i;
12319 
12320 	for (i = 0; i < bp->cp_nr_rings; i++) {
12321 		struct bnxt_napi *bnapi = bp->bnapi[i];
12322 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
12323 		u64 *sw = cpr->stats.sw_stats;
12324 
12325 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
12326 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12327 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
12328 
12329 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
12330 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
12331 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
12332 
12333 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
12334 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
12335 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
12336 
12337 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
12338 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
12339 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
12340 
12341 		stats->rx_missed_errors +=
12342 			BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
12343 
12344 		stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12345 
12346 		stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
12347 
12348 		stats->rx_dropped +=
12349 			cpr->sw_stats.rx.rx_netpoll_discards +
12350 			cpr->sw_stats.rx.rx_oom_discards;
12351 	}
12352 }
12353 
12354 static void bnxt_add_prev_stats(struct bnxt *bp,
12355 				struct rtnl_link_stats64 *stats)
12356 {
12357 	struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
12358 
12359 	stats->rx_packets += prev_stats->rx_packets;
12360 	stats->tx_packets += prev_stats->tx_packets;
12361 	stats->rx_bytes += prev_stats->rx_bytes;
12362 	stats->tx_bytes += prev_stats->tx_bytes;
12363 	stats->rx_missed_errors += prev_stats->rx_missed_errors;
12364 	stats->multicast += prev_stats->multicast;
12365 	stats->rx_dropped += prev_stats->rx_dropped;
12366 	stats->tx_dropped += prev_stats->tx_dropped;
12367 }
12368 
12369 static void
12370 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
12371 {
12372 	struct bnxt *bp = netdev_priv(dev);
12373 
12374 	set_bit(BNXT_STATE_READ_STATS, &bp->state);
12375 	/* Make sure bnxt_close_nic() sees that we are reading stats before
12376 	 * we check the BNXT_STATE_OPEN flag.
12377 	 */
12378 	smp_mb__after_atomic();
12379 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12380 		clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12381 		*stats = bp->net_stats_prev;
12382 		return;
12383 	}
12384 
12385 	bnxt_get_ring_stats(bp, stats);
12386 	bnxt_add_prev_stats(bp, stats);
12387 
12388 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
12389 		u64 *rx = bp->port_stats.sw_stats;
12390 		u64 *tx = bp->port_stats.sw_stats +
12391 			  BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
12392 
12393 		stats->rx_crc_errors =
12394 			BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
12395 		stats->rx_frame_errors =
12396 			BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
12397 		stats->rx_length_errors =
12398 			BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
12399 			BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
12400 			BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
12401 		stats->rx_errors =
12402 			BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
12403 			BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
12404 		stats->collisions =
12405 			BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
12406 		stats->tx_fifo_errors =
12407 			BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
12408 		stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
12409 	}
12410 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12411 }
12412 
12413 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
12414 					struct bnxt_total_ring_err_stats *stats,
12415 					struct bnxt_cp_ring_info *cpr)
12416 {
12417 	struct bnxt_sw_stats *sw_stats = &cpr->sw_stats;
12418 	u64 *hw_stats = cpr->stats.sw_stats;
12419 
12420 	stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
12421 	stats->rx_total_resets += sw_stats->rx.rx_resets;
12422 	stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
12423 	stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
12424 	stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
12425 	stats->rx_total_ring_discards +=
12426 		BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
12427 	stats->tx_total_resets += sw_stats->tx.tx_resets;
12428 	stats->tx_total_ring_discards +=
12429 		BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
12430 	stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
12431 }
12432 
12433 void bnxt_get_ring_err_stats(struct bnxt *bp,
12434 			     struct bnxt_total_ring_err_stats *stats)
12435 {
12436 	int i;
12437 
12438 	for (i = 0; i < bp->cp_nr_rings; i++)
12439 		bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
12440 }
12441 
12442 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
12443 {
12444 	struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12445 	struct net_device *dev = bp->dev;
12446 	struct netdev_hw_addr *ha;
12447 	u8 *haddr;
12448 	int mc_count = 0;
12449 	bool update = false;
12450 	int off = 0;
12451 
12452 	netdev_for_each_mc_addr(ha, dev) {
12453 		if (mc_count >= BNXT_MAX_MC_ADDRS) {
12454 			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12455 			vnic->mc_list_count = 0;
12456 			return false;
12457 		}
12458 		haddr = ha->addr;
12459 		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
12460 			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
12461 			update = true;
12462 		}
12463 		off += ETH_ALEN;
12464 		mc_count++;
12465 	}
12466 	if (mc_count)
12467 		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12468 
12469 	if (mc_count != vnic->mc_list_count) {
12470 		vnic->mc_list_count = mc_count;
12471 		update = true;
12472 	}
12473 	return update;
12474 }
12475 
12476 static bool bnxt_uc_list_updated(struct bnxt *bp)
12477 {
12478 	struct net_device *dev = bp->dev;
12479 	struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12480 	struct netdev_hw_addr *ha;
12481 	int off = 0;
12482 
12483 	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
12484 		return true;
12485 
12486 	netdev_for_each_uc_addr(ha, dev) {
12487 		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
12488 			return true;
12489 
12490 		off += ETH_ALEN;
12491 	}
12492 	return false;
12493 }
12494 
12495 static void bnxt_set_rx_mode(struct net_device *dev)
12496 {
12497 	struct bnxt *bp = netdev_priv(dev);
12498 	struct bnxt_vnic_info *vnic;
12499 	bool mc_update = false;
12500 	bool uc_update;
12501 	u32 mask;
12502 
12503 	if (!test_bit(BNXT_STATE_OPEN, &bp->state))
12504 		return;
12505 
12506 	vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12507 	mask = vnic->rx_mask;
12508 	mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
12509 		  CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
12510 		  CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
12511 		  CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
12512 
12513 	if (dev->flags & IFF_PROMISC)
12514 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12515 
12516 	uc_update = bnxt_uc_list_updated(bp);
12517 
12518 	if (dev->flags & IFF_BROADCAST)
12519 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
12520 	if (dev->flags & IFF_ALLMULTI) {
12521 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12522 		vnic->mc_list_count = 0;
12523 	} else if (dev->flags & IFF_MULTICAST) {
12524 		mc_update = bnxt_mc_list_updated(bp, &mask);
12525 	}
12526 
12527 	if (mask != vnic->rx_mask || uc_update || mc_update) {
12528 		vnic->rx_mask = mask;
12529 
12530 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12531 	}
12532 }
12533 
12534 static int bnxt_cfg_rx_mode(struct bnxt *bp)
12535 {
12536 	struct net_device *dev = bp->dev;
12537 	struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12538 	struct netdev_hw_addr *ha;
12539 	int i, off = 0, rc;
12540 	bool uc_update;
12541 
12542 	netif_addr_lock_bh(dev);
12543 	uc_update = bnxt_uc_list_updated(bp);
12544 	netif_addr_unlock_bh(dev);
12545 
12546 	if (!uc_update)
12547 		goto skip_uc;
12548 
12549 	for (i = 1; i < vnic->uc_filter_count; i++) {
12550 		struct bnxt_l2_filter *fltr = vnic->l2_filters[i];
12551 
12552 		bnxt_hwrm_l2_filter_free(bp, fltr);
12553 		bnxt_del_l2_filter(bp, fltr);
12554 	}
12555 
12556 	vnic->uc_filter_count = 1;
12557 
12558 	netif_addr_lock_bh(dev);
12559 	if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
12560 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12561 	} else {
12562 		netdev_for_each_uc_addr(ha, dev) {
12563 			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
12564 			off += ETH_ALEN;
12565 			vnic->uc_filter_count++;
12566 		}
12567 	}
12568 	netif_addr_unlock_bh(dev);
12569 
12570 	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
12571 		rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
12572 		if (rc) {
12573 			if (BNXT_VF(bp) && rc == -ENODEV) {
12574 				if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12575 					netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
12576 				else
12577 					netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
12578 				rc = 0;
12579 			} else {
12580 				netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
12581 			}
12582 			vnic->uc_filter_count = i;
12583 			return rc;
12584 		}
12585 	}
12586 	if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12587 		netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
12588 
12589 skip_uc:
12590 	if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
12591 	    !bnxt_promisc_ok(bp))
12592 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12593 	rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12594 	if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
12595 		netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
12596 			    rc);
12597 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12598 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12599 		vnic->mc_list_count = 0;
12600 		rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12601 	}
12602 	if (rc)
12603 		netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
12604 			   rc);
12605 
12606 	return rc;
12607 }
12608 
12609 static bool bnxt_can_reserve_rings(struct bnxt *bp)
12610 {
12611 #ifdef CONFIG_BNXT_SRIOV
12612 	if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
12613 		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12614 
12615 		/* No minimum rings were provisioned by the PF.  Don't
12616 		 * reserve rings by default when device is down.
12617 		 */
12618 		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
12619 			return true;
12620 
12621 		if (!netif_running(bp->dev))
12622 			return false;
12623 	}
12624 #endif
12625 	return true;
12626 }
12627 
12628 /* If the chip and firmware supports RFS */
12629 static bool bnxt_rfs_supported(struct bnxt *bp)
12630 {
12631 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
12632 		if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
12633 			return true;
12634 		return false;
12635 	}
12636 	/* 212 firmware is broken for aRFS */
12637 	if (BNXT_FW_MAJ(bp) == 212)
12638 		return false;
12639 	if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
12640 		return true;
12641 	if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12642 		return true;
12643 	return false;
12644 }
12645 
12646 /* If runtime conditions support RFS */
12647 bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx)
12648 {
12649 	struct bnxt_hw_rings hwr = {0};
12650 	int max_vnics, max_rss_ctxs;
12651 
12652 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
12653 	    !BNXT_SUPPORTS_NTUPLE_VNIC(bp))
12654 		return bnxt_rfs_supported(bp);
12655 
12656 	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
12657 		return false;
12658 
12659 	hwr.grp = bp->rx_nr_rings;
12660 	hwr.vnic = bnxt_get_total_vnics(bp, bp->rx_nr_rings);
12661 	if (new_rss_ctx)
12662 		hwr.vnic++;
12663 	hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
12664 	max_vnics = bnxt_get_max_func_vnics(bp);
12665 	max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
12666 
12667 	if (hwr.vnic > max_vnics || hwr.rss_ctx > max_rss_ctxs) {
12668 		if (bp->rx_nr_rings > 1)
12669 			netdev_warn(bp->dev,
12670 				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
12671 				    min(max_rss_ctxs - 1, max_vnics - 1));
12672 		return false;
12673 	}
12674 
12675 	if (!BNXT_NEW_RM(bp))
12676 		return true;
12677 
12678 	if (hwr.vnic == bp->hw_resc.resv_vnics &&
12679 	    hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
12680 		return true;
12681 
12682 	bnxt_hwrm_reserve_rings(bp, &hwr);
12683 	if (hwr.vnic <= bp->hw_resc.resv_vnics &&
12684 	    hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
12685 		return true;
12686 
12687 	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
12688 	hwr.vnic = 1;
12689 	hwr.rss_ctx = 0;
12690 	bnxt_hwrm_reserve_rings(bp, &hwr);
12691 	return false;
12692 }
12693 
12694 static netdev_features_t bnxt_fix_features(struct net_device *dev,
12695 					   netdev_features_t features)
12696 {
12697 	struct bnxt *bp = netdev_priv(dev);
12698 	netdev_features_t vlan_features;
12699 
12700 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false))
12701 		features &= ~NETIF_F_NTUPLE;
12702 
12703 	if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
12704 		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12705 
12706 	if (!(features & NETIF_F_GRO))
12707 		features &= ~NETIF_F_GRO_HW;
12708 
12709 	if (features & NETIF_F_GRO_HW)
12710 		features &= ~NETIF_F_LRO;
12711 
12712 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
12713 	 * turned on or off together.
12714 	 */
12715 	vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
12716 	if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
12717 		if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12718 			features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12719 		else if (vlan_features)
12720 			features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12721 	}
12722 #ifdef CONFIG_BNXT_SRIOV
12723 	if (BNXT_VF(bp) && bp->vf.vlan)
12724 		features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12725 #endif
12726 	return features;
12727 }
12728 
12729 static int bnxt_reinit_features(struct bnxt *bp, bool irq_re_init,
12730 				bool link_re_init, u32 flags, bool update_tpa)
12731 {
12732 	bnxt_close_nic(bp, irq_re_init, link_re_init);
12733 	bp->flags = flags;
12734 	if (update_tpa)
12735 		bnxt_set_ring_params(bp);
12736 	return bnxt_open_nic(bp, irq_re_init, link_re_init);
12737 }
12738 
12739 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
12740 {
12741 	bool update_tpa = false, update_ntuple = false;
12742 	struct bnxt *bp = netdev_priv(dev);
12743 	u32 flags = bp->flags;
12744 	u32 changes;
12745 	int rc = 0;
12746 	bool re_init = false;
12747 
12748 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
12749 	if (features & NETIF_F_GRO_HW)
12750 		flags |= BNXT_FLAG_GRO;
12751 	else if (features & NETIF_F_LRO)
12752 		flags |= BNXT_FLAG_LRO;
12753 
12754 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
12755 		flags &= ~BNXT_FLAG_TPA;
12756 
12757 	if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12758 		flags |= BNXT_FLAG_STRIP_VLAN;
12759 
12760 	if (features & NETIF_F_NTUPLE)
12761 		flags |= BNXT_FLAG_RFS;
12762 	else
12763 		bnxt_clear_usr_fltrs(bp, true);
12764 
12765 	changes = flags ^ bp->flags;
12766 	if (changes & BNXT_FLAG_TPA) {
12767 		update_tpa = true;
12768 		if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
12769 		    (flags & BNXT_FLAG_TPA) == 0 ||
12770 		    (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12771 			re_init = true;
12772 	}
12773 
12774 	if (changes & ~BNXT_FLAG_TPA)
12775 		re_init = true;
12776 
12777 	if (changes & BNXT_FLAG_RFS)
12778 		update_ntuple = true;
12779 
12780 	if (flags != bp->flags) {
12781 		u32 old_flags = bp->flags;
12782 
12783 		if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12784 			bp->flags = flags;
12785 			if (update_tpa)
12786 				bnxt_set_ring_params(bp);
12787 			return rc;
12788 		}
12789 
12790 		if (update_ntuple)
12791 			return bnxt_reinit_features(bp, true, false, flags, update_tpa);
12792 
12793 		if (re_init)
12794 			return bnxt_reinit_features(bp, false, false, flags, update_tpa);
12795 
12796 		if (update_tpa) {
12797 			bp->flags = flags;
12798 			rc = bnxt_set_tpa(bp,
12799 					  (flags & BNXT_FLAG_TPA) ?
12800 					  true : false);
12801 			if (rc)
12802 				bp->flags = old_flags;
12803 		}
12804 	}
12805 	return rc;
12806 }
12807 
12808 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
12809 			      u8 **nextp)
12810 {
12811 	struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
12812 	struct hop_jumbo_hdr *jhdr;
12813 	int hdr_count = 0;
12814 	u8 *nexthdr;
12815 	int start;
12816 
12817 	/* Check that there are at most 2 IPv6 extension headers, no
12818 	 * fragment header, and each is <= 64 bytes.
12819 	 */
12820 	start = nw_off + sizeof(*ip6h);
12821 	nexthdr = &ip6h->nexthdr;
12822 	while (ipv6_ext_hdr(*nexthdr)) {
12823 		struct ipv6_opt_hdr *hp;
12824 		int hdrlen;
12825 
12826 		if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
12827 		    *nexthdr == NEXTHDR_FRAGMENT)
12828 			return false;
12829 		hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
12830 					  skb_headlen(skb), NULL);
12831 		if (!hp)
12832 			return false;
12833 		if (*nexthdr == NEXTHDR_AUTH)
12834 			hdrlen = ipv6_authlen(hp);
12835 		else
12836 			hdrlen = ipv6_optlen(hp);
12837 
12838 		if (hdrlen > 64)
12839 			return false;
12840 
12841 		/* The ext header may be a hop-by-hop header inserted for
12842 		 * big TCP purposes. This will be removed before sending
12843 		 * from NIC, so do not count it.
12844 		 */
12845 		if (*nexthdr == NEXTHDR_HOP) {
12846 			if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
12847 				goto increment_hdr;
12848 
12849 			jhdr = (struct hop_jumbo_hdr *)hp;
12850 			if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
12851 			    jhdr->nexthdr != IPPROTO_TCP)
12852 				goto increment_hdr;
12853 
12854 			goto next_hdr;
12855 		}
12856 increment_hdr:
12857 		hdr_count++;
12858 next_hdr:
12859 		nexthdr = &hp->nexthdr;
12860 		start += hdrlen;
12861 	}
12862 	if (nextp) {
12863 		/* Caller will check inner protocol */
12864 		if (skb->encapsulation) {
12865 			*nextp = nexthdr;
12866 			return true;
12867 		}
12868 		*nextp = NULL;
12869 	}
12870 	/* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
12871 	return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
12872 }
12873 
12874 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
12875 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
12876 {
12877 	struct udphdr *uh = udp_hdr(skb);
12878 	__be16 udp_port = uh->dest;
12879 
12880 	if (udp_port != bp->vxlan_port && udp_port != bp->nge_port &&
12881 	    udp_port != bp->vxlan_gpe_port)
12882 		return false;
12883 	if (skb->inner_protocol == htons(ETH_P_TEB)) {
12884 		struct ethhdr *eh = inner_eth_hdr(skb);
12885 
12886 		switch (eh->h_proto) {
12887 		case htons(ETH_P_IP):
12888 			return true;
12889 		case htons(ETH_P_IPV6):
12890 			return bnxt_exthdr_check(bp, skb,
12891 						 skb_inner_network_offset(skb),
12892 						 NULL);
12893 		}
12894 	} else if (skb->inner_protocol == htons(ETH_P_IP)) {
12895 		return true;
12896 	} else if (skb->inner_protocol == htons(ETH_P_IPV6)) {
12897 		return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12898 					 NULL);
12899 	}
12900 	return false;
12901 }
12902 
12903 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
12904 {
12905 	switch (l4_proto) {
12906 	case IPPROTO_UDP:
12907 		return bnxt_udp_tunl_check(bp, skb);
12908 	case IPPROTO_IPIP:
12909 		return true;
12910 	case IPPROTO_GRE: {
12911 		switch (skb->inner_protocol) {
12912 		default:
12913 			return false;
12914 		case htons(ETH_P_IP):
12915 			return true;
12916 		case htons(ETH_P_IPV6):
12917 			fallthrough;
12918 		}
12919 	}
12920 	case IPPROTO_IPV6:
12921 		/* Check ext headers of inner ipv6 */
12922 		return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12923 					 NULL);
12924 	}
12925 	return false;
12926 }
12927 
12928 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
12929 					     struct net_device *dev,
12930 					     netdev_features_t features)
12931 {
12932 	struct bnxt *bp = netdev_priv(dev);
12933 	u8 *l4_proto;
12934 
12935 	features = vlan_features_check(skb, features);
12936 	switch (vlan_get_protocol(skb)) {
12937 	case htons(ETH_P_IP):
12938 		if (!skb->encapsulation)
12939 			return features;
12940 		l4_proto = &ip_hdr(skb)->protocol;
12941 		if (bnxt_tunl_check(bp, skb, *l4_proto))
12942 			return features;
12943 		break;
12944 	case htons(ETH_P_IPV6):
12945 		if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
12946 				       &l4_proto))
12947 			break;
12948 		if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
12949 			return features;
12950 		break;
12951 	}
12952 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12953 }
12954 
12955 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
12956 			 u32 *reg_buf)
12957 {
12958 	struct hwrm_dbg_read_direct_output *resp;
12959 	struct hwrm_dbg_read_direct_input *req;
12960 	__le32 *dbg_reg_buf;
12961 	dma_addr_t mapping;
12962 	int rc, i;
12963 
12964 	rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
12965 	if (rc)
12966 		return rc;
12967 
12968 	dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
12969 					 &mapping);
12970 	if (!dbg_reg_buf) {
12971 		rc = -ENOMEM;
12972 		goto dbg_rd_reg_exit;
12973 	}
12974 
12975 	req->host_dest_addr = cpu_to_le64(mapping);
12976 
12977 	resp = hwrm_req_hold(bp, req);
12978 	req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
12979 	req->read_len32 = cpu_to_le32(num_words);
12980 
12981 	rc = hwrm_req_send(bp, req);
12982 	if (rc || resp->error_code) {
12983 		rc = -EIO;
12984 		goto dbg_rd_reg_exit;
12985 	}
12986 	for (i = 0; i < num_words; i++)
12987 		reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
12988 
12989 dbg_rd_reg_exit:
12990 	hwrm_req_drop(bp, req);
12991 	return rc;
12992 }
12993 
12994 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
12995 				       u32 ring_id, u32 *prod, u32 *cons)
12996 {
12997 	struct hwrm_dbg_ring_info_get_output *resp;
12998 	struct hwrm_dbg_ring_info_get_input *req;
12999 	int rc;
13000 
13001 	rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
13002 	if (rc)
13003 		return rc;
13004 
13005 	req->ring_type = ring_type;
13006 	req->fw_ring_id = cpu_to_le32(ring_id);
13007 	resp = hwrm_req_hold(bp, req);
13008 	rc = hwrm_req_send(bp, req);
13009 	if (!rc) {
13010 		*prod = le32_to_cpu(resp->producer_index);
13011 		*cons = le32_to_cpu(resp->consumer_index);
13012 	}
13013 	hwrm_req_drop(bp, req);
13014 	return rc;
13015 }
13016 
13017 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
13018 {
13019 	struct bnxt_tx_ring_info *txr;
13020 	int i = bnapi->index, j;
13021 
13022 	bnxt_for_each_napi_tx(j, bnapi, txr)
13023 		netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
13024 			    i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
13025 			    txr->tx_cons);
13026 }
13027 
13028 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
13029 {
13030 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
13031 	int i = bnapi->index;
13032 
13033 	if (!rxr)
13034 		return;
13035 
13036 	netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
13037 		    i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
13038 		    rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
13039 		    rxr->rx_sw_agg_prod);
13040 }
13041 
13042 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
13043 {
13044 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
13045 	int i = bnapi->index;
13046 
13047 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
13048 		    i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
13049 }
13050 
13051 static void bnxt_dbg_dump_states(struct bnxt *bp)
13052 {
13053 	int i;
13054 	struct bnxt_napi *bnapi;
13055 
13056 	for (i = 0; i < bp->cp_nr_rings; i++) {
13057 		bnapi = bp->bnapi[i];
13058 		if (netif_msg_drv(bp)) {
13059 			bnxt_dump_tx_sw_state(bnapi);
13060 			bnxt_dump_rx_sw_state(bnapi);
13061 			bnxt_dump_cp_sw_state(bnapi);
13062 		}
13063 	}
13064 }
13065 
13066 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
13067 {
13068 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
13069 	struct hwrm_ring_reset_input *req;
13070 	struct bnxt_napi *bnapi = rxr->bnapi;
13071 	struct bnxt_cp_ring_info *cpr;
13072 	u16 cp_ring_id;
13073 	int rc;
13074 
13075 	rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
13076 	if (rc)
13077 		return rc;
13078 
13079 	cpr = &bnapi->cp_ring;
13080 	cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
13081 	req->cmpl_ring = cpu_to_le16(cp_ring_id);
13082 	req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
13083 	req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
13084 	return hwrm_req_send_silent(bp, req);
13085 }
13086 
13087 static void bnxt_reset_task(struct bnxt *bp, bool silent)
13088 {
13089 	if (!silent)
13090 		bnxt_dbg_dump_states(bp);
13091 	if (netif_running(bp->dev)) {
13092 		int rc;
13093 
13094 		if (silent) {
13095 			bnxt_close_nic(bp, false, false);
13096 			bnxt_open_nic(bp, false, false);
13097 		} else {
13098 			bnxt_ulp_stop(bp);
13099 			bnxt_close_nic(bp, true, false);
13100 			rc = bnxt_open_nic(bp, true, false);
13101 			bnxt_ulp_start(bp, rc);
13102 		}
13103 	}
13104 }
13105 
13106 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
13107 {
13108 	struct bnxt *bp = netdev_priv(dev);
13109 
13110 	netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
13111 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
13112 }
13113 
13114 static void bnxt_fw_health_check(struct bnxt *bp)
13115 {
13116 	struct bnxt_fw_health *fw_health = bp->fw_health;
13117 	struct pci_dev *pdev = bp->pdev;
13118 	u32 val;
13119 
13120 	if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13121 		return;
13122 
13123 	/* Make sure it is enabled before checking the tmr_counter. */
13124 	smp_rmb();
13125 	if (fw_health->tmr_counter) {
13126 		fw_health->tmr_counter--;
13127 		return;
13128 	}
13129 
13130 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13131 	if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
13132 		fw_health->arrests++;
13133 		goto fw_reset;
13134 	}
13135 
13136 	fw_health->last_fw_heartbeat = val;
13137 
13138 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13139 	if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
13140 		fw_health->discoveries++;
13141 		goto fw_reset;
13142 	}
13143 
13144 	fw_health->tmr_counter = fw_health->tmr_multiplier;
13145 	return;
13146 
13147 fw_reset:
13148 	bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
13149 }
13150 
13151 static void bnxt_timer(struct timer_list *t)
13152 {
13153 	struct bnxt *bp = from_timer(bp, t, timer);
13154 	struct net_device *dev = bp->dev;
13155 
13156 	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
13157 		return;
13158 
13159 	if (atomic_read(&bp->intr_sem) != 0)
13160 		goto bnxt_restart_timer;
13161 
13162 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
13163 		bnxt_fw_health_check(bp);
13164 
13165 	if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
13166 		bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
13167 
13168 	if (bnxt_tc_flower_enabled(bp))
13169 		bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
13170 
13171 #ifdef CONFIG_RFS_ACCEL
13172 	if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
13173 		bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
13174 #endif /*CONFIG_RFS_ACCEL*/
13175 
13176 	if (bp->link_info.phy_retry) {
13177 		if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
13178 			bp->link_info.phy_retry = false;
13179 			netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
13180 		} else {
13181 			bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
13182 		}
13183 	}
13184 
13185 	if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
13186 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
13187 
13188 	if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev))
13189 		bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
13190 
13191 bnxt_restart_timer:
13192 	mod_timer(&bp->timer, jiffies + bp->current_interval);
13193 }
13194 
13195 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
13196 {
13197 	/* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
13198 	 * set.  If the device is being closed, bnxt_close() may be holding
13199 	 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
13200 	 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
13201 	 */
13202 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13203 	rtnl_lock();
13204 }
13205 
13206 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
13207 {
13208 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13209 	rtnl_unlock();
13210 }
13211 
13212 /* Only called from bnxt_sp_task() */
13213 static void bnxt_reset(struct bnxt *bp, bool silent)
13214 {
13215 	bnxt_rtnl_lock_sp(bp);
13216 	if (test_bit(BNXT_STATE_OPEN, &bp->state))
13217 		bnxt_reset_task(bp, silent);
13218 	bnxt_rtnl_unlock_sp(bp);
13219 }
13220 
13221 /* Only called from bnxt_sp_task() */
13222 static void bnxt_rx_ring_reset(struct bnxt *bp)
13223 {
13224 	int i;
13225 
13226 	bnxt_rtnl_lock_sp(bp);
13227 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13228 		bnxt_rtnl_unlock_sp(bp);
13229 		return;
13230 	}
13231 	/* Disable and flush TPA before resetting the RX ring */
13232 	if (bp->flags & BNXT_FLAG_TPA)
13233 		bnxt_set_tpa(bp, false);
13234 	for (i = 0; i < bp->rx_nr_rings; i++) {
13235 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
13236 		struct bnxt_cp_ring_info *cpr;
13237 		int rc;
13238 
13239 		if (!rxr->bnapi->in_reset)
13240 			continue;
13241 
13242 		rc = bnxt_hwrm_rx_ring_reset(bp, i);
13243 		if (rc) {
13244 			if (rc == -EINVAL || rc == -EOPNOTSUPP)
13245 				netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
13246 			else
13247 				netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
13248 					    rc);
13249 			bnxt_reset_task(bp, true);
13250 			break;
13251 		}
13252 		bnxt_free_one_rx_ring_skbs(bp, i);
13253 		rxr->rx_prod = 0;
13254 		rxr->rx_agg_prod = 0;
13255 		rxr->rx_sw_agg_prod = 0;
13256 		rxr->rx_next_cons = 0;
13257 		rxr->bnapi->in_reset = false;
13258 		bnxt_alloc_one_rx_ring(bp, i);
13259 		cpr = &rxr->bnapi->cp_ring;
13260 		cpr->sw_stats.rx.rx_resets++;
13261 		if (bp->flags & BNXT_FLAG_AGG_RINGS)
13262 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
13263 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
13264 	}
13265 	if (bp->flags & BNXT_FLAG_TPA)
13266 		bnxt_set_tpa(bp, true);
13267 	bnxt_rtnl_unlock_sp(bp);
13268 }
13269 
13270 static void bnxt_fw_reset_close(struct bnxt *bp)
13271 {
13272 	bnxt_ulp_stop(bp);
13273 	/* When firmware is in fatal state, quiesce device and disable
13274 	 * bus master to prevent any potential bad DMAs before freeing
13275 	 * kernel memory.
13276 	 */
13277 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
13278 		u16 val = 0;
13279 
13280 		pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
13281 		if (val == 0xffff)
13282 			bp->fw_reset_min_dsecs = 0;
13283 		bnxt_tx_disable(bp);
13284 		bnxt_disable_napi(bp);
13285 		bnxt_disable_int_sync(bp);
13286 		bnxt_free_irq(bp);
13287 		bnxt_clear_int_mode(bp);
13288 		pci_disable_device(bp->pdev);
13289 	}
13290 	__bnxt_close_nic(bp, true, false);
13291 	bnxt_vf_reps_free(bp);
13292 	bnxt_clear_int_mode(bp);
13293 	bnxt_hwrm_func_drv_unrgtr(bp);
13294 	if (pci_is_enabled(bp->pdev))
13295 		pci_disable_device(bp->pdev);
13296 	bnxt_free_ctx_mem(bp);
13297 }
13298 
13299 static bool is_bnxt_fw_ok(struct bnxt *bp)
13300 {
13301 	struct bnxt_fw_health *fw_health = bp->fw_health;
13302 	bool no_heartbeat = false, has_reset = false;
13303 	u32 val;
13304 
13305 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13306 	if (val == fw_health->last_fw_heartbeat)
13307 		no_heartbeat = true;
13308 
13309 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13310 	if (val != fw_health->last_fw_reset_cnt)
13311 		has_reset = true;
13312 
13313 	if (!no_heartbeat && has_reset)
13314 		return true;
13315 
13316 	return false;
13317 }
13318 
13319 /* rtnl_lock is acquired before calling this function */
13320 static void bnxt_force_fw_reset(struct bnxt *bp)
13321 {
13322 	struct bnxt_fw_health *fw_health = bp->fw_health;
13323 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13324 	u32 wait_dsecs;
13325 
13326 	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
13327 	    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13328 		return;
13329 
13330 	if (ptp) {
13331 		spin_lock_bh(&ptp->ptp_lock);
13332 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13333 		spin_unlock_bh(&ptp->ptp_lock);
13334 	} else {
13335 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13336 	}
13337 	bnxt_fw_reset_close(bp);
13338 	wait_dsecs = fw_health->master_func_wait_dsecs;
13339 	if (fw_health->primary) {
13340 		if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
13341 			wait_dsecs = 0;
13342 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
13343 	} else {
13344 		bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
13345 		wait_dsecs = fw_health->normal_func_wait_dsecs;
13346 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13347 	}
13348 
13349 	bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
13350 	bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
13351 	bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
13352 }
13353 
13354 void bnxt_fw_exception(struct bnxt *bp)
13355 {
13356 	netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
13357 	set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
13358 	bnxt_rtnl_lock_sp(bp);
13359 	bnxt_force_fw_reset(bp);
13360 	bnxt_rtnl_unlock_sp(bp);
13361 }
13362 
13363 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
13364  * < 0 on error.
13365  */
13366 static int bnxt_get_registered_vfs(struct bnxt *bp)
13367 {
13368 #ifdef CONFIG_BNXT_SRIOV
13369 	int rc;
13370 
13371 	if (!BNXT_PF(bp))
13372 		return 0;
13373 
13374 	rc = bnxt_hwrm_func_qcfg(bp);
13375 	if (rc) {
13376 		netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
13377 		return rc;
13378 	}
13379 	if (bp->pf.registered_vfs)
13380 		return bp->pf.registered_vfs;
13381 	if (bp->sriov_cfg)
13382 		return 1;
13383 #endif
13384 	return 0;
13385 }
13386 
13387 void bnxt_fw_reset(struct bnxt *bp)
13388 {
13389 	bnxt_rtnl_lock_sp(bp);
13390 	if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
13391 	    !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13392 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13393 		int n = 0, tmo;
13394 
13395 		if (ptp) {
13396 			spin_lock_bh(&ptp->ptp_lock);
13397 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13398 			spin_unlock_bh(&ptp->ptp_lock);
13399 		} else {
13400 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13401 		}
13402 		if (bp->pf.active_vfs &&
13403 		    !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
13404 			n = bnxt_get_registered_vfs(bp);
13405 		if (n < 0) {
13406 			netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
13407 				   n);
13408 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13409 			dev_close(bp->dev);
13410 			goto fw_reset_exit;
13411 		} else if (n > 0) {
13412 			u16 vf_tmo_dsecs = n * 10;
13413 
13414 			if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
13415 				bp->fw_reset_max_dsecs = vf_tmo_dsecs;
13416 			bp->fw_reset_state =
13417 				BNXT_FW_RESET_STATE_POLL_VF;
13418 			bnxt_queue_fw_reset_work(bp, HZ / 10);
13419 			goto fw_reset_exit;
13420 		}
13421 		bnxt_fw_reset_close(bp);
13422 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13423 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
13424 			tmo = HZ / 10;
13425 		} else {
13426 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13427 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
13428 		}
13429 		bnxt_queue_fw_reset_work(bp, tmo);
13430 	}
13431 fw_reset_exit:
13432 	bnxt_rtnl_unlock_sp(bp);
13433 }
13434 
13435 static void bnxt_chk_missed_irq(struct bnxt *bp)
13436 {
13437 	int i;
13438 
13439 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
13440 		return;
13441 
13442 	for (i = 0; i < bp->cp_nr_rings; i++) {
13443 		struct bnxt_napi *bnapi = bp->bnapi[i];
13444 		struct bnxt_cp_ring_info *cpr;
13445 		u32 fw_ring_id;
13446 		int j;
13447 
13448 		if (!bnapi)
13449 			continue;
13450 
13451 		cpr = &bnapi->cp_ring;
13452 		for (j = 0; j < cpr->cp_ring_count; j++) {
13453 			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
13454 			u32 val[2];
13455 
13456 			if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
13457 				continue;
13458 
13459 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
13460 				cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
13461 				continue;
13462 			}
13463 			fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
13464 			bnxt_dbg_hwrm_ring_info_get(bp,
13465 				DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
13466 				fw_ring_id, &val[0], &val[1]);
13467 			cpr->sw_stats.cmn.missed_irqs++;
13468 		}
13469 	}
13470 }
13471 
13472 static void bnxt_cfg_ntp_filters(struct bnxt *);
13473 
13474 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
13475 {
13476 	struct bnxt_link_info *link_info = &bp->link_info;
13477 
13478 	if (BNXT_AUTO_MODE(link_info->auto_mode)) {
13479 		link_info->autoneg = BNXT_AUTONEG_SPEED;
13480 		if (bp->hwrm_spec_code >= 0x10201) {
13481 			if (link_info->auto_pause_setting &
13482 			    PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
13483 				link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13484 		} else {
13485 			link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13486 		}
13487 		bnxt_set_auto_speed(link_info);
13488 	} else {
13489 		bnxt_set_force_speed(link_info);
13490 		link_info->req_duplex = link_info->duplex_setting;
13491 	}
13492 	if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
13493 		link_info->req_flow_ctrl =
13494 			link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
13495 	else
13496 		link_info->req_flow_ctrl = link_info->force_pause_setting;
13497 }
13498 
13499 static void bnxt_fw_echo_reply(struct bnxt *bp)
13500 {
13501 	struct bnxt_fw_health *fw_health = bp->fw_health;
13502 	struct hwrm_func_echo_response_input *req;
13503 	int rc;
13504 
13505 	rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
13506 	if (rc)
13507 		return;
13508 	req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
13509 	req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
13510 	hwrm_req_send(bp, req);
13511 }
13512 
13513 static void bnxt_sp_task(struct work_struct *work)
13514 {
13515 	struct bnxt *bp = container_of(work, struct bnxt, sp_task);
13516 
13517 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13518 	smp_mb__after_atomic();
13519 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13520 		clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13521 		return;
13522 	}
13523 
13524 	if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
13525 		bnxt_cfg_rx_mode(bp);
13526 
13527 	if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
13528 		bnxt_cfg_ntp_filters(bp);
13529 	if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
13530 		bnxt_hwrm_exec_fwd_req(bp);
13531 	if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13532 		netdev_info(bp->dev, "Receive PF driver unload event!\n");
13533 	if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
13534 		bnxt_hwrm_port_qstats(bp, 0);
13535 		bnxt_hwrm_port_qstats_ext(bp, 0);
13536 		bnxt_accumulate_all_stats(bp);
13537 	}
13538 
13539 	if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
13540 		int rc;
13541 
13542 		mutex_lock(&bp->link_lock);
13543 		if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
13544 				       &bp->sp_event))
13545 			bnxt_hwrm_phy_qcaps(bp);
13546 
13547 		rc = bnxt_update_link(bp, true);
13548 		if (rc)
13549 			netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
13550 				   rc);
13551 
13552 		if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
13553 				       &bp->sp_event))
13554 			bnxt_init_ethtool_link_settings(bp);
13555 		mutex_unlock(&bp->link_lock);
13556 	}
13557 	if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
13558 		int rc;
13559 
13560 		mutex_lock(&bp->link_lock);
13561 		rc = bnxt_update_phy_setting(bp);
13562 		mutex_unlock(&bp->link_lock);
13563 		if (rc) {
13564 			netdev_warn(bp->dev, "update phy settings retry failed\n");
13565 		} else {
13566 			bp->link_info.phy_retry = false;
13567 			netdev_info(bp->dev, "update phy settings retry succeeded\n");
13568 		}
13569 	}
13570 	if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
13571 		mutex_lock(&bp->link_lock);
13572 		bnxt_get_port_module_status(bp);
13573 		mutex_unlock(&bp->link_lock);
13574 	}
13575 
13576 	if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
13577 		bnxt_tc_flow_stats_work(bp);
13578 
13579 	if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
13580 		bnxt_chk_missed_irq(bp);
13581 
13582 	if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
13583 		bnxt_fw_echo_reply(bp);
13584 
13585 	if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event))
13586 		bnxt_hwmon_notify_event(bp);
13587 
13588 	/* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
13589 	 * must be the last functions to be called before exiting.
13590 	 */
13591 	if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
13592 		bnxt_reset(bp, false);
13593 
13594 	if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
13595 		bnxt_reset(bp, true);
13596 
13597 	if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
13598 		bnxt_rx_ring_reset(bp);
13599 
13600 	if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
13601 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
13602 		    test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
13603 			bnxt_devlink_health_fw_report(bp);
13604 		else
13605 			bnxt_fw_reset(bp);
13606 	}
13607 
13608 	if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
13609 		if (!is_bnxt_fw_ok(bp))
13610 			bnxt_devlink_health_fw_report(bp);
13611 	}
13612 
13613 	smp_mb__before_atomic();
13614 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13615 }
13616 
13617 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13618 				int *max_cp);
13619 
13620 /* Under rtnl_lock */
13621 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
13622 		     int tx_xdp)
13623 {
13624 	int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
13625 	struct bnxt_hw_rings hwr = {0};
13626 	int rx_rings = rx;
13627 
13628 	if (tcs)
13629 		tx_sets = tcs;
13630 
13631 	_bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
13632 
13633 	if (max_rx < rx_rings)
13634 		return -ENOMEM;
13635 
13636 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
13637 		rx_rings <<= 1;
13638 
13639 	hwr.rx = rx_rings;
13640 	hwr.tx = tx * tx_sets + tx_xdp;
13641 	if (max_tx < hwr.tx)
13642 		return -ENOMEM;
13643 
13644 	hwr.vnic = bnxt_get_total_vnics(bp, rx);
13645 
13646 	tx_cp = __bnxt_num_tx_to_cp(bp, hwr.tx, tx_sets, tx_xdp);
13647 	hwr.cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
13648 	if (max_cp < hwr.cp)
13649 		return -ENOMEM;
13650 	hwr.stat = hwr.cp;
13651 	if (BNXT_NEW_RM(bp)) {
13652 		hwr.cp += bnxt_get_ulp_msix_num_in_use(bp);
13653 		hwr.stat += bnxt_get_ulp_stat_ctxs_in_use(bp);
13654 		hwr.grp = rx;
13655 		hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
13656 	}
13657 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
13658 		hwr.cp_p5 = hwr.tx + rx;
13659 	return bnxt_hwrm_check_rings(bp, &hwr);
13660 }
13661 
13662 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
13663 {
13664 	if (bp->bar2) {
13665 		pci_iounmap(pdev, bp->bar2);
13666 		bp->bar2 = NULL;
13667 	}
13668 
13669 	if (bp->bar1) {
13670 		pci_iounmap(pdev, bp->bar1);
13671 		bp->bar1 = NULL;
13672 	}
13673 
13674 	if (bp->bar0) {
13675 		pci_iounmap(pdev, bp->bar0);
13676 		bp->bar0 = NULL;
13677 	}
13678 }
13679 
13680 static void bnxt_cleanup_pci(struct bnxt *bp)
13681 {
13682 	bnxt_unmap_bars(bp, bp->pdev);
13683 	pci_release_regions(bp->pdev);
13684 	if (pci_is_enabled(bp->pdev))
13685 		pci_disable_device(bp->pdev);
13686 }
13687 
13688 static void bnxt_init_dflt_coal(struct bnxt *bp)
13689 {
13690 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
13691 	struct bnxt_coal *coal;
13692 	u16 flags = 0;
13693 
13694 	if (coal_cap->cmpl_params &
13695 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
13696 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
13697 
13698 	/* Tick values in micro seconds.
13699 	 * 1 coal_buf x bufs_per_record = 1 completion record.
13700 	 */
13701 	coal = &bp->rx_coal;
13702 	coal->coal_ticks = 10;
13703 	coal->coal_bufs = 30;
13704 	coal->coal_ticks_irq = 1;
13705 	coal->coal_bufs_irq = 2;
13706 	coal->idle_thresh = 50;
13707 	coal->bufs_per_record = 2;
13708 	coal->budget = 64;		/* NAPI budget */
13709 	coal->flags = flags;
13710 
13711 	coal = &bp->tx_coal;
13712 	coal->coal_ticks = 28;
13713 	coal->coal_bufs = 30;
13714 	coal->coal_ticks_irq = 2;
13715 	coal->coal_bufs_irq = 2;
13716 	coal->bufs_per_record = 1;
13717 	coal->flags = flags;
13718 
13719 	bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
13720 }
13721 
13722 /* FW that pre-reserves 1 VNIC per function */
13723 static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp)
13724 {
13725 	u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp);
13726 
13727 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13728 	    (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18)))
13729 		return true;
13730 	if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13731 	    (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172)))
13732 		return true;
13733 	return false;
13734 }
13735 
13736 static int bnxt_fw_init_one_p1(struct bnxt *bp)
13737 {
13738 	int rc;
13739 
13740 	bp->fw_cap = 0;
13741 	rc = bnxt_hwrm_ver_get(bp);
13742 	/* FW may be unresponsive after FLR. FLR must complete within 100 msec
13743 	 * so wait before continuing with recovery.
13744 	 */
13745 	if (rc)
13746 		msleep(100);
13747 	bnxt_try_map_fw_health_reg(bp);
13748 	if (rc) {
13749 		rc = bnxt_try_recover_fw(bp);
13750 		if (rc)
13751 			return rc;
13752 		rc = bnxt_hwrm_ver_get(bp);
13753 		if (rc)
13754 			return rc;
13755 	}
13756 
13757 	bnxt_nvm_cfg_ver_get(bp);
13758 
13759 	rc = bnxt_hwrm_func_reset(bp);
13760 	if (rc)
13761 		return -ENODEV;
13762 
13763 	bnxt_hwrm_fw_set_time(bp);
13764 	return 0;
13765 }
13766 
13767 static int bnxt_fw_init_one_p2(struct bnxt *bp)
13768 {
13769 	int rc;
13770 
13771 	/* Get the MAX capabilities for this function */
13772 	rc = bnxt_hwrm_func_qcaps(bp);
13773 	if (rc) {
13774 		netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
13775 			   rc);
13776 		return -ENODEV;
13777 	}
13778 
13779 	rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
13780 	if (rc)
13781 		netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
13782 			    rc);
13783 
13784 	if (bnxt_alloc_fw_health(bp)) {
13785 		netdev_warn(bp->dev, "no memory for firmware error recovery\n");
13786 	} else {
13787 		rc = bnxt_hwrm_error_recovery_qcfg(bp);
13788 		if (rc)
13789 			netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
13790 				    rc);
13791 	}
13792 
13793 	rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
13794 	if (rc)
13795 		return -ENODEV;
13796 
13797 	if (bnxt_fw_pre_resv_vnics(bp))
13798 		bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS;
13799 
13800 	bnxt_hwrm_func_qcfg(bp);
13801 	bnxt_hwrm_vnic_qcaps(bp);
13802 	bnxt_hwrm_port_led_qcaps(bp);
13803 	bnxt_ethtool_init(bp);
13804 	if (bp->fw_cap & BNXT_FW_CAP_PTP)
13805 		__bnxt_hwrm_ptp_qcfg(bp);
13806 	bnxt_dcb_init(bp);
13807 	bnxt_hwmon_init(bp);
13808 	return 0;
13809 }
13810 
13811 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
13812 {
13813 	bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP;
13814 	bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
13815 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
13816 			   VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
13817 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
13818 	if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
13819 		bp->rss_hash_delta = bp->rss_hash_cfg;
13820 	if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
13821 		bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP;
13822 		bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
13823 				    VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
13824 	}
13825 }
13826 
13827 static void bnxt_set_dflt_rfs(struct bnxt *bp)
13828 {
13829 	struct net_device *dev = bp->dev;
13830 
13831 	dev->hw_features &= ~NETIF_F_NTUPLE;
13832 	dev->features &= ~NETIF_F_NTUPLE;
13833 	bp->flags &= ~BNXT_FLAG_RFS;
13834 	if (bnxt_rfs_supported(bp)) {
13835 		dev->hw_features |= NETIF_F_NTUPLE;
13836 		if (bnxt_rfs_capable(bp, false)) {
13837 			bp->flags |= BNXT_FLAG_RFS;
13838 			dev->features |= NETIF_F_NTUPLE;
13839 		}
13840 	}
13841 }
13842 
13843 static void bnxt_fw_init_one_p3(struct bnxt *bp)
13844 {
13845 	struct pci_dev *pdev = bp->pdev;
13846 
13847 	bnxt_set_dflt_rss_hash_type(bp);
13848 	bnxt_set_dflt_rfs(bp);
13849 
13850 	bnxt_get_wol_settings(bp);
13851 	if (bp->flags & BNXT_FLAG_WOL_CAP)
13852 		device_set_wakeup_enable(&pdev->dev, bp->wol);
13853 	else
13854 		device_set_wakeup_capable(&pdev->dev, false);
13855 
13856 	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
13857 	bnxt_hwrm_coal_params_qcaps(bp);
13858 }
13859 
13860 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
13861 
13862 int bnxt_fw_init_one(struct bnxt *bp)
13863 {
13864 	int rc;
13865 
13866 	rc = bnxt_fw_init_one_p1(bp);
13867 	if (rc) {
13868 		netdev_err(bp->dev, "Firmware init phase 1 failed\n");
13869 		return rc;
13870 	}
13871 	rc = bnxt_fw_init_one_p2(bp);
13872 	if (rc) {
13873 		netdev_err(bp->dev, "Firmware init phase 2 failed\n");
13874 		return rc;
13875 	}
13876 	rc = bnxt_probe_phy(bp, false);
13877 	if (rc)
13878 		return rc;
13879 	rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
13880 	if (rc)
13881 		return rc;
13882 
13883 	bnxt_fw_init_one_p3(bp);
13884 	return 0;
13885 }
13886 
13887 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
13888 {
13889 	struct bnxt_fw_health *fw_health = bp->fw_health;
13890 	u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
13891 	u32 val = fw_health->fw_reset_seq_vals[reg_idx];
13892 	u32 reg_type, reg_off, delay_msecs;
13893 
13894 	delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
13895 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
13896 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
13897 	switch (reg_type) {
13898 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
13899 		pci_write_config_dword(bp->pdev, reg_off, val);
13900 		break;
13901 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
13902 		writel(reg_off & BNXT_GRC_BASE_MASK,
13903 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
13904 		reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
13905 		fallthrough;
13906 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
13907 		writel(val, bp->bar0 + reg_off);
13908 		break;
13909 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
13910 		writel(val, bp->bar1 + reg_off);
13911 		break;
13912 	}
13913 	if (delay_msecs) {
13914 		pci_read_config_dword(bp->pdev, 0, &val);
13915 		msleep(delay_msecs);
13916 	}
13917 }
13918 
13919 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
13920 {
13921 	struct hwrm_func_qcfg_output *resp;
13922 	struct hwrm_func_qcfg_input *req;
13923 	bool result = true; /* firmware will enforce if unknown */
13924 
13925 	if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
13926 		return result;
13927 
13928 	if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
13929 		return result;
13930 
13931 	req->fid = cpu_to_le16(0xffff);
13932 	resp = hwrm_req_hold(bp, req);
13933 	if (!hwrm_req_send(bp, req))
13934 		result = !!(le16_to_cpu(resp->flags) &
13935 			    FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
13936 	hwrm_req_drop(bp, req);
13937 	return result;
13938 }
13939 
13940 static void bnxt_reset_all(struct bnxt *bp)
13941 {
13942 	struct bnxt_fw_health *fw_health = bp->fw_health;
13943 	int i, rc;
13944 
13945 	if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13946 		bnxt_fw_reset_via_optee(bp);
13947 		bp->fw_reset_timestamp = jiffies;
13948 		return;
13949 	}
13950 
13951 	if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
13952 		for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
13953 			bnxt_fw_reset_writel(bp, i);
13954 	} else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
13955 		struct hwrm_fw_reset_input *req;
13956 
13957 		rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
13958 		if (!rc) {
13959 			req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
13960 			req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
13961 			req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
13962 			req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
13963 			rc = hwrm_req_send(bp, req);
13964 		}
13965 		if (rc != -ENODEV)
13966 			netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
13967 	}
13968 	bp->fw_reset_timestamp = jiffies;
13969 }
13970 
13971 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
13972 {
13973 	return time_after(jiffies, bp->fw_reset_timestamp +
13974 			  (bp->fw_reset_max_dsecs * HZ / 10));
13975 }
13976 
13977 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
13978 {
13979 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13980 	if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
13981 		bnxt_ulp_start(bp, rc);
13982 		bnxt_dl_health_fw_status_update(bp, false);
13983 	}
13984 	bp->fw_reset_state = 0;
13985 	dev_close(bp->dev);
13986 }
13987 
13988 static void bnxt_fw_reset_task(struct work_struct *work)
13989 {
13990 	struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
13991 	int rc = 0;
13992 
13993 	if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13994 		netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
13995 		return;
13996 	}
13997 
13998 	switch (bp->fw_reset_state) {
13999 	case BNXT_FW_RESET_STATE_POLL_VF: {
14000 		int n = bnxt_get_registered_vfs(bp);
14001 		int tmo;
14002 
14003 		if (n < 0) {
14004 			netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
14005 				   n, jiffies_to_msecs(jiffies -
14006 				   bp->fw_reset_timestamp));
14007 			goto fw_reset_abort;
14008 		} else if (n > 0) {
14009 			if (bnxt_fw_reset_timeout(bp)) {
14010 				clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14011 				bp->fw_reset_state = 0;
14012 				netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
14013 					   n);
14014 				return;
14015 			}
14016 			bnxt_queue_fw_reset_work(bp, HZ / 10);
14017 			return;
14018 		}
14019 		bp->fw_reset_timestamp = jiffies;
14020 		rtnl_lock();
14021 		if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
14022 			bnxt_fw_reset_abort(bp, rc);
14023 			rtnl_unlock();
14024 			return;
14025 		}
14026 		bnxt_fw_reset_close(bp);
14027 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
14028 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
14029 			tmo = HZ / 10;
14030 		} else {
14031 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14032 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
14033 		}
14034 		rtnl_unlock();
14035 		bnxt_queue_fw_reset_work(bp, tmo);
14036 		return;
14037 	}
14038 	case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
14039 		u32 val;
14040 
14041 		val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14042 		if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
14043 		    !bnxt_fw_reset_timeout(bp)) {
14044 			bnxt_queue_fw_reset_work(bp, HZ / 5);
14045 			return;
14046 		}
14047 
14048 		if (!bp->fw_health->primary) {
14049 			u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
14050 
14051 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14052 			bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
14053 			return;
14054 		}
14055 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
14056 	}
14057 		fallthrough;
14058 	case BNXT_FW_RESET_STATE_RESET_FW:
14059 		bnxt_reset_all(bp);
14060 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14061 		bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
14062 		return;
14063 	case BNXT_FW_RESET_STATE_ENABLE_DEV:
14064 		bnxt_inv_fw_health_reg(bp);
14065 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
14066 		    !bp->fw_reset_min_dsecs) {
14067 			u16 val;
14068 
14069 			pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
14070 			if (val == 0xffff) {
14071 				if (bnxt_fw_reset_timeout(bp)) {
14072 					netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
14073 					rc = -ETIMEDOUT;
14074 					goto fw_reset_abort;
14075 				}
14076 				bnxt_queue_fw_reset_work(bp, HZ / 1000);
14077 				return;
14078 			}
14079 		}
14080 		clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
14081 		clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
14082 		if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
14083 		    !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
14084 			bnxt_dl_remote_reload(bp);
14085 		if (pci_enable_device(bp->pdev)) {
14086 			netdev_err(bp->dev, "Cannot re-enable PCI device\n");
14087 			rc = -ENODEV;
14088 			goto fw_reset_abort;
14089 		}
14090 		pci_set_master(bp->pdev);
14091 		bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
14092 		fallthrough;
14093 	case BNXT_FW_RESET_STATE_POLL_FW:
14094 		bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
14095 		rc = bnxt_hwrm_poll(bp);
14096 		if (rc) {
14097 			if (bnxt_fw_reset_timeout(bp)) {
14098 				netdev_err(bp->dev, "Firmware reset aborted\n");
14099 				goto fw_reset_abort_status;
14100 			}
14101 			bnxt_queue_fw_reset_work(bp, HZ / 5);
14102 			return;
14103 		}
14104 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
14105 		bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
14106 		fallthrough;
14107 	case BNXT_FW_RESET_STATE_OPENING:
14108 		while (!rtnl_trylock()) {
14109 			bnxt_queue_fw_reset_work(bp, HZ / 10);
14110 			return;
14111 		}
14112 		rc = bnxt_open(bp->dev);
14113 		if (rc) {
14114 			netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
14115 			bnxt_fw_reset_abort(bp, rc);
14116 			rtnl_unlock();
14117 			return;
14118 		}
14119 
14120 		if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
14121 		    bp->fw_health->enabled) {
14122 			bp->fw_health->last_fw_reset_cnt =
14123 				bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
14124 		}
14125 		bp->fw_reset_state = 0;
14126 		/* Make sure fw_reset_state is 0 before clearing the flag */
14127 		smp_mb__before_atomic();
14128 		clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14129 		bnxt_ulp_start(bp, 0);
14130 		bnxt_reenable_sriov(bp);
14131 		bnxt_vf_reps_alloc(bp);
14132 		bnxt_vf_reps_open(bp);
14133 		bnxt_ptp_reapply_pps(bp);
14134 		clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
14135 		if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
14136 			bnxt_dl_health_fw_recovery_done(bp);
14137 			bnxt_dl_health_fw_status_update(bp, true);
14138 		}
14139 		rtnl_unlock();
14140 		break;
14141 	}
14142 	return;
14143 
14144 fw_reset_abort_status:
14145 	if (bp->fw_health->status_reliable ||
14146 	    (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
14147 		u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14148 
14149 		netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
14150 	}
14151 fw_reset_abort:
14152 	rtnl_lock();
14153 	bnxt_fw_reset_abort(bp, rc);
14154 	rtnl_unlock();
14155 }
14156 
14157 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
14158 {
14159 	int rc;
14160 	struct bnxt *bp = netdev_priv(dev);
14161 
14162 	SET_NETDEV_DEV(dev, &pdev->dev);
14163 
14164 	/* enable device (incl. PCI PM wakeup), and bus-mastering */
14165 	rc = pci_enable_device(pdev);
14166 	if (rc) {
14167 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
14168 		goto init_err;
14169 	}
14170 
14171 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
14172 		dev_err(&pdev->dev,
14173 			"Cannot find PCI device base address, aborting\n");
14174 		rc = -ENODEV;
14175 		goto init_err_disable;
14176 	}
14177 
14178 	rc = pci_request_regions(pdev, DRV_MODULE_NAME);
14179 	if (rc) {
14180 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
14181 		goto init_err_disable;
14182 	}
14183 
14184 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
14185 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
14186 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
14187 		rc = -EIO;
14188 		goto init_err_release;
14189 	}
14190 
14191 	pci_set_master(pdev);
14192 
14193 	bp->dev = dev;
14194 	bp->pdev = pdev;
14195 
14196 	/* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
14197 	 * determines the BAR size.
14198 	 */
14199 	bp->bar0 = pci_ioremap_bar(pdev, 0);
14200 	if (!bp->bar0) {
14201 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
14202 		rc = -ENOMEM;
14203 		goto init_err_release;
14204 	}
14205 
14206 	bp->bar2 = pci_ioremap_bar(pdev, 4);
14207 	if (!bp->bar2) {
14208 		dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
14209 		rc = -ENOMEM;
14210 		goto init_err_release;
14211 	}
14212 
14213 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
14214 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
14215 
14216 	spin_lock_init(&bp->ntp_fltr_lock);
14217 #if BITS_PER_LONG == 32
14218 	spin_lock_init(&bp->db_lock);
14219 #endif
14220 
14221 	bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
14222 	bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
14223 
14224 	timer_setup(&bp->timer, bnxt_timer, 0);
14225 	bp->current_interval = BNXT_TIMER_INTERVAL;
14226 
14227 	bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
14228 	bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
14229 
14230 	clear_bit(BNXT_STATE_OPEN, &bp->state);
14231 	return 0;
14232 
14233 init_err_release:
14234 	bnxt_unmap_bars(bp, pdev);
14235 	pci_release_regions(pdev);
14236 
14237 init_err_disable:
14238 	pci_disable_device(pdev);
14239 
14240 init_err:
14241 	return rc;
14242 }
14243 
14244 /* rtnl_lock held */
14245 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
14246 {
14247 	struct sockaddr *addr = p;
14248 	struct bnxt *bp = netdev_priv(dev);
14249 	int rc = 0;
14250 
14251 	if (!is_valid_ether_addr(addr->sa_data))
14252 		return -EADDRNOTAVAIL;
14253 
14254 	if (ether_addr_equal(addr->sa_data, dev->dev_addr))
14255 		return 0;
14256 
14257 	rc = bnxt_approve_mac(bp, addr->sa_data, true);
14258 	if (rc)
14259 		return rc;
14260 
14261 	eth_hw_addr_set(dev, addr->sa_data);
14262 	bnxt_clear_usr_fltrs(bp, true);
14263 	if (netif_running(dev)) {
14264 		bnxt_close_nic(bp, false, false);
14265 		rc = bnxt_open_nic(bp, false, false);
14266 	}
14267 
14268 	return rc;
14269 }
14270 
14271 /* rtnl_lock held */
14272 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
14273 {
14274 	struct bnxt *bp = netdev_priv(dev);
14275 
14276 	if (netif_running(dev))
14277 		bnxt_close_nic(bp, true, false);
14278 
14279 	dev->mtu = new_mtu;
14280 	bnxt_set_ring_params(bp);
14281 
14282 	if (netif_running(dev))
14283 		return bnxt_open_nic(bp, true, false);
14284 
14285 	return 0;
14286 }
14287 
14288 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
14289 {
14290 	struct bnxt *bp = netdev_priv(dev);
14291 	bool sh = false;
14292 	int rc, tx_cp;
14293 
14294 	if (tc > bp->max_tc) {
14295 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
14296 			   tc, bp->max_tc);
14297 		return -EINVAL;
14298 	}
14299 
14300 	if (bp->num_tc == tc)
14301 		return 0;
14302 
14303 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
14304 		sh = true;
14305 
14306 	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
14307 			      sh, tc, bp->tx_nr_rings_xdp);
14308 	if (rc)
14309 		return rc;
14310 
14311 	/* Needs to close the device and do hw resource re-allocations */
14312 	if (netif_running(bp->dev))
14313 		bnxt_close_nic(bp, true, false);
14314 
14315 	if (tc) {
14316 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
14317 		netdev_set_num_tc(dev, tc);
14318 		bp->num_tc = tc;
14319 	} else {
14320 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14321 		netdev_reset_tc(dev);
14322 		bp->num_tc = 0;
14323 	}
14324 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
14325 	tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
14326 	bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
14327 			       tx_cp + bp->rx_nr_rings;
14328 
14329 	if (netif_running(bp->dev))
14330 		return bnxt_open_nic(bp, true, false);
14331 
14332 	return 0;
14333 }
14334 
14335 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
14336 				  void *cb_priv)
14337 {
14338 	struct bnxt *bp = cb_priv;
14339 
14340 	if (!bnxt_tc_flower_enabled(bp) ||
14341 	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
14342 		return -EOPNOTSUPP;
14343 
14344 	switch (type) {
14345 	case TC_SETUP_CLSFLOWER:
14346 		return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
14347 	default:
14348 		return -EOPNOTSUPP;
14349 	}
14350 }
14351 
14352 LIST_HEAD(bnxt_block_cb_list);
14353 
14354 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
14355 			 void *type_data)
14356 {
14357 	struct bnxt *bp = netdev_priv(dev);
14358 
14359 	switch (type) {
14360 	case TC_SETUP_BLOCK:
14361 		return flow_block_cb_setup_simple(type_data,
14362 						  &bnxt_block_cb_list,
14363 						  bnxt_setup_tc_block_cb,
14364 						  bp, bp, true);
14365 	case TC_SETUP_QDISC_MQPRIO: {
14366 		struct tc_mqprio_qopt *mqprio = type_data;
14367 
14368 		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
14369 
14370 		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
14371 	}
14372 	default:
14373 		return -EOPNOTSUPP;
14374 	}
14375 }
14376 
14377 u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys,
14378 			    const struct sk_buff *skb)
14379 {
14380 	struct bnxt_vnic_info *vnic;
14381 
14382 	if (skb)
14383 		return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
14384 
14385 	vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
14386 	return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key);
14387 }
14388 
14389 int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr,
14390 			   u32 idx)
14391 {
14392 	struct hlist_head *head;
14393 	int bit_id;
14394 
14395 	spin_lock_bh(&bp->ntp_fltr_lock);
14396 	bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, bp->max_fltr, 0);
14397 	if (bit_id < 0) {
14398 		spin_unlock_bh(&bp->ntp_fltr_lock);
14399 		return -ENOMEM;
14400 	}
14401 
14402 	fltr->base.sw_id = (u16)bit_id;
14403 	fltr->base.type = BNXT_FLTR_TYPE_NTUPLE;
14404 	fltr->base.flags |= BNXT_ACT_RING_DST;
14405 	head = &bp->ntp_fltr_hash_tbl[idx];
14406 	hlist_add_head_rcu(&fltr->base.hash, head);
14407 	set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
14408 	bnxt_insert_usr_fltr(bp, &fltr->base);
14409 	bp->ntp_fltr_count++;
14410 	spin_unlock_bh(&bp->ntp_fltr_lock);
14411 	return 0;
14412 }
14413 
14414 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
14415 			    struct bnxt_ntuple_filter *f2)
14416 {
14417 	struct bnxt_flow_masks *masks1 = &f1->fmasks;
14418 	struct bnxt_flow_masks *masks2 = &f2->fmasks;
14419 	struct flow_keys *keys1 = &f1->fkeys;
14420 	struct flow_keys *keys2 = &f2->fkeys;
14421 
14422 	if (keys1->basic.n_proto != keys2->basic.n_proto ||
14423 	    keys1->basic.ip_proto != keys2->basic.ip_proto)
14424 		return false;
14425 
14426 	if (keys1->basic.n_proto == htons(ETH_P_IP)) {
14427 		if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
14428 		    masks1->addrs.v4addrs.src != masks2->addrs.v4addrs.src ||
14429 		    keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst ||
14430 		    masks1->addrs.v4addrs.dst != masks2->addrs.v4addrs.dst)
14431 			return false;
14432 	} else {
14433 		if (!ipv6_addr_equal(&keys1->addrs.v6addrs.src,
14434 				     &keys2->addrs.v6addrs.src) ||
14435 		    !ipv6_addr_equal(&masks1->addrs.v6addrs.src,
14436 				     &masks2->addrs.v6addrs.src) ||
14437 		    !ipv6_addr_equal(&keys1->addrs.v6addrs.dst,
14438 				     &keys2->addrs.v6addrs.dst) ||
14439 		    !ipv6_addr_equal(&masks1->addrs.v6addrs.dst,
14440 				     &masks2->addrs.v6addrs.dst))
14441 			return false;
14442 	}
14443 
14444 	return keys1->ports.src == keys2->ports.src &&
14445 	       masks1->ports.src == masks2->ports.src &&
14446 	       keys1->ports.dst == keys2->ports.dst &&
14447 	       masks1->ports.dst == masks2->ports.dst &&
14448 	       keys1->control.flags == keys2->control.flags &&
14449 	       f1->l2_fltr == f2->l2_fltr;
14450 }
14451 
14452 struct bnxt_ntuple_filter *
14453 bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp,
14454 				struct bnxt_ntuple_filter *fltr, u32 idx)
14455 {
14456 	struct bnxt_ntuple_filter *f;
14457 	struct hlist_head *head;
14458 
14459 	head = &bp->ntp_fltr_hash_tbl[idx];
14460 	hlist_for_each_entry_rcu(f, head, base.hash) {
14461 		if (bnxt_fltr_match(f, fltr))
14462 			return f;
14463 	}
14464 	return NULL;
14465 }
14466 
14467 #ifdef CONFIG_RFS_ACCEL
14468 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
14469 			      u16 rxq_index, u32 flow_id)
14470 {
14471 	struct bnxt *bp = netdev_priv(dev);
14472 	struct bnxt_ntuple_filter *fltr, *new_fltr;
14473 	struct flow_keys *fkeys;
14474 	struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
14475 	struct bnxt_l2_filter *l2_fltr;
14476 	int rc = 0, idx;
14477 	u32 flags;
14478 
14479 	if (ether_addr_equal(dev->dev_addr, eth->h_dest)) {
14480 		l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
14481 		atomic_inc(&l2_fltr->refcnt);
14482 	} else {
14483 		struct bnxt_l2_key key;
14484 
14485 		ether_addr_copy(key.dst_mac_addr, eth->h_dest);
14486 		key.vlan = 0;
14487 		l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key);
14488 		if (!l2_fltr)
14489 			return -EINVAL;
14490 		if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) {
14491 			bnxt_del_l2_filter(bp, l2_fltr);
14492 			return -EINVAL;
14493 		}
14494 	}
14495 	new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
14496 	if (!new_fltr) {
14497 		bnxt_del_l2_filter(bp, l2_fltr);
14498 		return -ENOMEM;
14499 	}
14500 
14501 	fkeys = &new_fltr->fkeys;
14502 	if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
14503 		rc = -EPROTONOSUPPORT;
14504 		goto err_free;
14505 	}
14506 
14507 	if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
14508 	     fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
14509 	    ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
14510 	     (fkeys->basic.ip_proto != IPPROTO_UDP))) {
14511 		rc = -EPROTONOSUPPORT;
14512 		goto err_free;
14513 	}
14514 	new_fltr->fmasks = BNXT_FLOW_IPV4_MASK_ALL;
14515 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
14516 		if (bp->hwrm_spec_code < 0x10601) {
14517 			rc = -EPROTONOSUPPORT;
14518 			goto err_free;
14519 		}
14520 		new_fltr->fmasks = BNXT_FLOW_IPV6_MASK_ALL;
14521 	}
14522 	flags = fkeys->control.flags;
14523 	if (((flags & FLOW_DIS_ENCAPSULATION) &&
14524 	     bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
14525 		rc = -EPROTONOSUPPORT;
14526 		goto err_free;
14527 	}
14528 	new_fltr->l2_fltr = l2_fltr;
14529 
14530 	idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb);
14531 	rcu_read_lock();
14532 	fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx);
14533 	if (fltr) {
14534 		rc = fltr->base.sw_id;
14535 		rcu_read_unlock();
14536 		goto err_free;
14537 	}
14538 	rcu_read_unlock();
14539 
14540 	new_fltr->flow_id = flow_id;
14541 	new_fltr->base.rxq = rxq_index;
14542 	rc = bnxt_insert_ntp_filter(bp, new_fltr, idx);
14543 	if (!rc) {
14544 		bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
14545 		return new_fltr->base.sw_id;
14546 	}
14547 
14548 err_free:
14549 	bnxt_del_l2_filter(bp, l2_fltr);
14550 	kfree(new_fltr);
14551 	return rc;
14552 }
14553 #endif
14554 
14555 void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr)
14556 {
14557 	spin_lock_bh(&bp->ntp_fltr_lock);
14558 	if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
14559 		spin_unlock_bh(&bp->ntp_fltr_lock);
14560 		return;
14561 	}
14562 	hlist_del_rcu(&fltr->base.hash);
14563 	bnxt_del_one_usr_fltr(bp, &fltr->base);
14564 	bp->ntp_fltr_count--;
14565 	spin_unlock_bh(&bp->ntp_fltr_lock);
14566 	bnxt_del_l2_filter(bp, fltr->l2_fltr);
14567 	clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
14568 	kfree_rcu(fltr, base.rcu);
14569 }
14570 
14571 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
14572 {
14573 #ifdef CONFIG_RFS_ACCEL
14574 	int i;
14575 
14576 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
14577 		struct hlist_head *head;
14578 		struct hlist_node *tmp;
14579 		struct bnxt_ntuple_filter *fltr;
14580 		int rc;
14581 
14582 		head = &bp->ntp_fltr_hash_tbl[i];
14583 		hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
14584 			bool del = false;
14585 
14586 			if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) {
14587 				if (fltr->base.flags & BNXT_ACT_NO_AGING)
14588 					continue;
14589 				if (rps_may_expire_flow(bp->dev, fltr->base.rxq,
14590 							fltr->flow_id,
14591 							fltr->base.sw_id)) {
14592 					bnxt_hwrm_cfa_ntuple_filter_free(bp,
14593 									 fltr);
14594 					del = true;
14595 				}
14596 			} else {
14597 				rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
14598 								       fltr);
14599 				if (rc)
14600 					del = true;
14601 				else
14602 					set_bit(BNXT_FLTR_VALID, &fltr->base.state);
14603 			}
14604 
14605 			if (del)
14606 				bnxt_del_ntp_filter(bp, fltr);
14607 		}
14608 	}
14609 #endif
14610 }
14611 
14612 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
14613 				    unsigned int entry, struct udp_tunnel_info *ti)
14614 {
14615 	struct bnxt *bp = netdev_priv(netdev);
14616 	unsigned int cmd;
14617 
14618 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14619 		cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
14620 	else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14621 		cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE;
14622 	else
14623 		cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE;
14624 
14625 	return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
14626 }
14627 
14628 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
14629 				      unsigned int entry, struct udp_tunnel_info *ti)
14630 {
14631 	struct bnxt *bp = netdev_priv(netdev);
14632 	unsigned int cmd;
14633 
14634 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14635 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
14636 	else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14637 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
14638 	else
14639 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE;
14640 
14641 	return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
14642 }
14643 
14644 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
14645 	.set_port	= bnxt_udp_tunnel_set_port,
14646 	.unset_port	= bnxt_udp_tunnel_unset_port,
14647 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14648 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14649 	.tables		= {
14650 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
14651 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14652 	},
14653 }, bnxt_udp_tunnels_p7 = {
14654 	.set_port	= bnxt_udp_tunnel_set_port,
14655 	.unset_port	= bnxt_udp_tunnel_unset_port,
14656 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14657 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14658 	.tables		= {
14659 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
14660 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14661 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, },
14662 	},
14663 };
14664 
14665 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
14666 			       struct net_device *dev, u32 filter_mask,
14667 			       int nlflags)
14668 {
14669 	struct bnxt *bp = netdev_priv(dev);
14670 
14671 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
14672 				       nlflags, filter_mask, NULL);
14673 }
14674 
14675 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
14676 			       u16 flags, struct netlink_ext_ack *extack)
14677 {
14678 	struct bnxt *bp = netdev_priv(dev);
14679 	struct nlattr *attr, *br_spec;
14680 	int rem, rc = 0;
14681 
14682 	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
14683 		return -EOPNOTSUPP;
14684 
14685 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
14686 	if (!br_spec)
14687 		return -EINVAL;
14688 
14689 	nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
14690 		u16 mode;
14691 
14692 		mode = nla_get_u16(attr);
14693 		if (mode == bp->br_mode)
14694 			break;
14695 
14696 		rc = bnxt_hwrm_set_br_mode(bp, mode);
14697 		if (!rc)
14698 			bp->br_mode = mode;
14699 		break;
14700 	}
14701 	return rc;
14702 }
14703 
14704 int bnxt_get_port_parent_id(struct net_device *dev,
14705 			    struct netdev_phys_item_id *ppid)
14706 {
14707 	struct bnxt *bp = netdev_priv(dev);
14708 
14709 	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
14710 		return -EOPNOTSUPP;
14711 
14712 	/* The PF and it's VF-reps only support the switchdev framework */
14713 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
14714 		return -EOPNOTSUPP;
14715 
14716 	ppid->id_len = sizeof(bp->dsn);
14717 	memcpy(ppid->id, bp->dsn, ppid->id_len);
14718 
14719 	return 0;
14720 }
14721 
14722 static const struct net_device_ops bnxt_netdev_ops = {
14723 	.ndo_open		= bnxt_open,
14724 	.ndo_start_xmit		= bnxt_start_xmit,
14725 	.ndo_stop		= bnxt_close,
14726 	.ndo_get_stats64	= bnxt_get_stats64,
14727 	.ndo_set_rx_mode	= bnxt_set_rx_mode,
14728 	.ndo_eth_ioctl		= bnxt_ioctl,
14729 	.ndo_validate_addr	= eth_validate_addr,
14730 	.ndo_set_mac_address	= bnxt_change_mac_addr,
14731 	.ndo_change_mtu		= bnxt_change_mtu,
14732 	.ndo_fix_features	= bnxt_fix_features,
14733 	.ndo_set_features	= bnxt_set_features,
14734 	.ndo_features_check	= bnxt_features_check,
14735 	.ndo_tx_timeout		= bnxt_tx_timeout,
14736 #ifdef CONFIG_BNXT_SRIOV
14737 	.ndo_get_vf_config	= bnxt_get_vf_config,
14738 	.ndo_set_vf_mac		= bnxt_set_vf_mac,
14739 	.ndo_set_vf_vlan	= bnxt_set_vf_vlan,
14740 	.ndo_set_vf_rate	= bnxt_set_vf_bw,
14741 	.ndo_set_vf_link_state	= bnxt_set_vf_link_state,
14742 	.ndo_set_vf_spoofchk	= bnxt_set_vf_spoofchk,
14743 	.ndo_set_vf_trust	= bnxt_set_vf_trust,
14744 #endif
14745 	.ndo_setup_tc           = bnxt_setup_tc,
14746 #ifdef CONFIG_RFS_ACCEL
14747 	.ndo_rx_flow_steer	= bnxt_rx_flow_steer,
14748 #endif
14749 	.ndo_bpf		= bnxt_xdp,
14750 	.ndo_xdp_xmit		= bnxt_xdp_xmit,
14751 	.ndo_bridge_getlink	= bnxt_bridge_getlink,
14752 	.ndo_bridge_setlink	= bnxt_bridge_setlink,
14753 };
14754 
14755 static void bnxt_get_queue_stats_rx(struct net_device *dev, int i,
14756 				    struct netdev_queue_stats_rx *stats)
14757 {
14758 	struct bnxt *bp = netdev_priv(dev);
14759 	struct bnxt_cp_ring_info *cpr;
14760 	u64 *sw;
14761 
14762 	cpr = &bp->bnapi[i]->cp_ring;
14763 	sw = cpr->stats.sw_stats;
14764 
14765 	stats->packets = 0;
14766 	stats->packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
14767 	stats->packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
14768 	stats->packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
14769 
14770 	stats->bytes = 0;
14771 	stats->bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
14772 	stats->bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
14773 	stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
14774 
14775 	stats->alloc_fail = cpr->sw_stats.rx.rx_oom_discards;
14776 }
14777 
14778 static void bnxt_get_queue_stats_tx(struct net_device *dev, int i,
14779 				    struct netdev_queue_stats_tx *stats)
14780 {
14781 	struct bnxt *bp = netdev_priv(dev);
14782 	struct bnxt_napi *bnapi;
14783 	u64 *sw;
14784 
14785 	bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi;
14786 	sw = bnapi->cp_ring.stats.sw_stats;
14787 
14788 	stats->packets = 0;
14789 	stats->packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
14790 	stats->packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
14791 	stats->packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
14792 
14793 	stats->bytes = 0;
14794 	stats->bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
14795 	stats->bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
14796 	stats->bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
14797 }
14798 
14799 static void bnxt_get_base_stats(struct net_device *dev,
14800 				struct netdev_queue_stats_rx *rx,
14801 				struct netdev_queue_stats_tx *tx)
14802 {
14803 	struct bnxt *bp = netdev_priv(dev);
14804 
14805 	rx->packets = bp->net_stats_prev.rx_packets;
14806 	rx->bytes = bp->net_stats_prev.rx_bytes;
14807 	rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards;
14808 
14809 	tx->packets = bp->net_stats_prev.tx_packets;
14810 	tx->bytes = bp->net_stats_prev.tx_bytes;
14811 }
14812 
14813 static const struct netdev_stat_ops bnxt_stat_ops = {
14814 	.get_queue_stats_rx	= bnxt_get_queue_stats_rx,
14815 	.get_queue_stats_tx	= bnxt_get_queue_stats_tx,
14816 	.get_base_stats		= bnxt_get_base_stats,
14817 };
14818 
14819 static void bnxt_remove_one(struct pci_dev *pdev)
14820 {
14821 	struct net_device *dev = pci_get_drvdata(pdev);
14822 	struct bnxt *bp = netdev_priv(dev);
14823 
14824 	if (BNXT_PF(bp))
14825 		bnxt_sriov_disable(bp);
14826 
14827 	bnxt_rdma_aux_device_del(bp);
14828 
14829 	bnxt_ptp_clear(bp);
14830 	unregister_netdev(dev);
14831 
14832 	bnxt_rdma_aux_device_uninit(bp);
14833 
14834 	bnxt_free_l2_filters(bp, true);
14835 	bnxt_free_ntp_fltrs(bp, true);
14836 	if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
14837 		bnxt_clear_rss_ctxs(bp, true);
14838 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14839 	/* Flush any pending tasks */
14840 	cancel_work_sync(&bp->sp_task);
14841 	cancel_delayed_work_sync(&bp->fw_reset_task);
14842 	bp->sp_event = 0;
14843 
14844 	bnxt_dl_fw_reporters_destroy(bp);
14845 	bnxt_dl_unregister(bp);
14846 	bnxt_shutdown_tc(bp);
14847 
14848 	bnxt_clear_int_mode(bp);
14849 	bnxt_hwrm_func_drv_unrgtr(bp);
14850 	bnxt_free_hwrm_resources(bp);
14851 	bnxt_hwmon_uninit(bp);
14852 	bnxt_ethtool_free(bp);
14853 	bnxt_dcb_free(bp);
14854 	kfree(bp->ptp_cfg);
14855 	bp->ptp_cfg = NULL;
14856 	kfree(bp->fw_health);
14857 	bp->fw_health = NULL;
14858 	bnxt_cleanup_pci(bp);
14859 	bnxt_free_ctx_mem(bp);
14860 	kfree(bp->rss_indir_tbl);
14861 	bp->rss_indir_tbl = NULL;
14862 	bnxt_free_port_stats(bp);
14863 	free_netdev(dev);
14864 }
14865 
14866 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
14867 {
14868 	int rc = 0;
14869 	struct bnxt_link_info *link_info = &bp->link_info;
14870 
14871 	bp->phy_flags = 0;
14872 	rc = bnxt_hwrm_phy_qcaps(bp);
14873 	if (rc) {
14874 		netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
14875 			   rc);
14876 		return rc;
14877 	}
14878 	if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
14879 		bp->dev->priv_flags |= IFF_SUPP_NOFCS;
14880 	else
14881 		bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
14882 	if (!fw_dflt)
14883 		return 0;
14884 
14885 	mutex_lock(&bp->link_lock);
14886 	rc = bnxt_update_link(bp, false);
14887 	if (rc) {
14888 		mutex_unlock(&bp->link_lock);
14889 		netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
14890 			   rc);
14891 		return rc;
14892 	}
14893 
14894 	/* Older firmware does not have supported_auto_speeds, so assume
14895 	 * that all supported speeds can be autonegotiated.
14896 	 */
14897 	if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
14898 		link_info->support_auto_speeds = link_info->support_speeds;
14899 
14900 	bnxt_init_ethtool_link_settings(bp);
14901 	mutex_unlock(&bp->link_lock);
14902 	return 0;
14903 }
14904 
14905 static int bnxt_get_max_irq(struct pci_dev *pdev)
14906 {
14907 	u16 ctrl;
14908 
14909 	if (!pdev->msix_cap)
14910 		return 1;
14911 
14912 	pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
14913 	return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
14914 }
14915 
14916 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14917 				int *max_cp)
14918 {
14919 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
14920 	int max_ring_grps = 0, max_irq;
14921 
14922 	*max_tx = hw_resc->max_tx_rings;
14923 	*max_rx = hw_resc->max_rx_rings;
14924 	*max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
14925 	max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
14926 			bnxt_get_ulp_msix_num_in_use(bp),
14927 			hw_resc->max_stat_ctxs -
14928 			bnxt_get_ulp_stat_ctxs_in_use(bp));
14929 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
14930 		*max_cp = min_t(int, *max_cp, max_irq);
14931 	max_ring_grps = hw_resc->max_hw_ring_grps;
14932 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
14933 		*max_cp -= 1;
14934 		*max_rx -= 2;
14935 	}
14936 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
14937 		*max_rx >>= 1;
14938 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
14939 		int rc;
14940 
14941 		rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
14942 		if (rc) {
14943 			*max_rx = 0;
14944 			*max_tx = 0;
14945 		}
14946 		/* On P5 chips, max_cp output param should be available NQs */
14947 		*max_cp = max_irq;
14948 	}
14949 	*max_rx = min_t(int, *max_rx, max_ring_grps);
14950 }
14951 
14952 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
14953 {
14954 	int rx, tx, cp;
14955 
14956 	_bnxt_get_max_rings(bp, &rx, &tx, &cp);
14957 	*max_rx = rx;
14958 	*max_tx = tx;
14959 	if (!rx || !tx || !cp)
14960 		return -ENOMEM;
14961 
14962 	return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
14963 }
14964 
14965 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14966 			       bool shared)
14967 {
14968 	int rc;
14969 
14970 	rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14971 	if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
14972 		/* Not enough rings, try disabling agg rings. */
14973 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
14974 		rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14975 		if (rc) {
14976 			/* set BNXT_FLAG_AGG_RINGS back for consistency */
14977 			bp->flags |= BNXT_FLAG_AGG_RINGS;
14978 			return rc;
14979 		}
14980 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
14981 		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14982 		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14983 		bnxt_set_ring_params(bp);
14984 	}
14985 
14986 	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
14987 		int max_cp, max_stat, max_irq;
14988 
14989 		/* Reserve minimum resources for RoCE */
14990 		max_cp = bnxt_get_max_func_cp_rings(bp);
14991 		max_stat = bnxt_get_max_func_stat_ctxs(bp);
14992 		max_irq = bnxt_get_max_func_irqs(bp);
14993 		if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
14994 		    max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
14995 		    max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
14996 			return 0;
14997 
14998 		max_cp -= BNXT_MIN_ROCE_CP_RINGS;
14999 		max_irq -= BNXT_MIN_ROCE_CP_RINGS;
15000 		max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
15001 		max_cp = min_t(int, max_cp, max_irq);
15002 		max_cp = min_t(int, max_cp, max_stat);
15003 		rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
15004 		if (rc)
15005 			rc = 0;
15006 	}
15007 	return rc;
15008 }
15009 
15010 /* In initial default shared ring setting, each shared ring must have a
15011  * RX/TX ring pair.
15012  */
15013 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
15014 {
15015 	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
15016 	bp->rx_nr_rings = bp->cp_nr_rings;
15017 	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
15018 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15019 }
15020 
15021 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
15022 {
15023 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
15024 	int avail_msix;
15025 
15026 	if (!bnxt_can_reserve_rings(bp))
15027 		return 0;
15028 
15029 	if (sh)
15030 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
15031 	dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
15032 	/* Reduce default rings on multi-port cards so that total default
15033 	 * rings do not exceed CPU count.
15034 	 */
15035 	if (bp->port_count > 1) {
15036 		int max_rings =
15037 			max_t(int, num_online_cpus() / bp->port_count, 1);
15038 
15039 		dflt_rings = min_t(int, dflt_rings, max_rings);
15040 	}
15041 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
15042 	if (rc)
15043 		return rc;
15044 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
15045 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
15046 	if (sh)
15047 		bnxt_trim_dflt_sh_rings(bp);
15048 	else
15049 		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
15050 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15051 
15052 	avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings;
15053 	if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) {
15054 		int ulp_num_msix = min(avail_msix, bp->ulp_num_msix_want);
15055 
15056 		bnxt_set_ulp_msix_num(bp, ulp_num_msix);
15057 		bnxt_set_dflt_ulp_stat_ctxs(bp);
15058 	}
15059 
15060 	rc = __bnxt_reserve_rings(bp);
15061 	if (rc && rc != -ENODEV)
15062 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
15063 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15064 	if (sh)
15065 		bnxt_trim_dflt_sh_rings(bp);
15066 
15067 	/* Rings may have been trimmed, re-reserve the trimmed rings. */
15068 	if (bnxt_need_reserve_rings(bp)) {
15069 		rc = __bnxt_reserve_rings(bp);
15070 		if (rc && rc != -ENODEV)
15071 			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
15072 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15073 	}
15074 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
15075 		bp->rx_nr_rings++;
15076 		bp->cp_nr_rings++;
15077 	}
15078 	if (rc) {
15079 		bp->tx_nr_rings = 0;
15080 		bp->rx_nr_rings = 0;
15081 	}
15082 	return rc;
15083 }
15084 
15085 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
15086 {
15087 	int rc;
15088 
15089 	if (bp->tx_nr_rings)
15090 		return 0;
15091 
15092 	bnxt_ulp_irq_stop(bp);
15093 	bnxt_clear_int_mode(bp);
15094 	rc = bnxt_set_dflt_rings(bp, true);
15095 	if (rc) {
15096 		if (BNXT_VF(bp) && rc == -ENODEV)
15097 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
15098 		else
15099 			netdev_err(bp->dev, "Not enough rings available.\n");
15100 		goto init_dflt_ring_err;
15101 	}
15102 	rc = bnxt_init_int_mode(bp);
15103 	if (rc)
15104 		goto init_dflt_ring_err;
15105 
15106 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15107 
15108 	bnxt_set_dflt_rfs(bp);
15109 
15110 init_dflt_ring_err:
15111 	bnxt_ulp_irq_restart(bp, rc);
15112 	return rc;
15113 }
15114 
15115 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
15116 {
15117 	int rc;
15118 
15119 	ASSERT_RTNL();
15120 	bnxt_hwrm_func_qcaps(bp);
15121 
15122 	if (netif_running(bp->dev))
15123 		__bnxt_close_nic(bp, true, false);
15124 
15125 	bnxt_ulp_irq_stop(bp);
15126 	bnxt_clear_int_mode(bp);
15127 	rc = bnxt_init_int_mode(bp);
15128 	bnxt_ulp_irq_restart(bp, rc);
15129 
15130 	if (netif_running(bp->dev)) {
15131 		if (rc)
15132 			dev_close(bp->dev);
15133 		else
15134 			rc = bnxt_open_nic(bp, true, false);
15135 	}
15136 
15137 	return rc;
15138 }
15139 
15140 static int bnxt_init_mac_addr(struct bnxt *bp)
15141 {
15142 	int rc = 0;
15143 
15144 	if (BNXT_PF(bp)) {
15145 		eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
15146 	} else {
15147 #ifdef CONFIG_BNXT_SRIOV
15148 		struct bnxt_vf_info *vf = &bp->vf;
15149 		bool strict_approval = true;
15150 
15151 		if (is_valid_ether_addr(vf->mac_addr)) {
15152 			/* overwrite netdev dev_addr with admin VF MAC */
15153 			eth_hw_addr_set(bp->dev, vf->mac_addr);
15154 			/* Older PF driver or firmware may not approve this
15155 			 * correctly.
15156 			 */
15157 			strict_approval = false;
15158 		} else {
15159 			eth_hw_addr_random(bp->dev);
15160 		}
15161 		rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
15162 #endif
15163 	}
15164 	return rc;
15165 }
15166 
15167 static void bnxt_vpd_read_info(struct bnxt *bp)
15168 {
15169 	struct pci_dev *pdev = bp->pdev;
15170 	unsigned int vpd_size, kw_len;
15171 	int pos, size;
15172 	u8 *vpd_data;
15173 
15174 	vpd_data = pci_vpd_alloc(pdev, &vpd_size);
15175 	if (IS_ERR(vpd_data)) {
15176 		pci_warn(pdev, "Unable to read VPD\n");
15177 		return;
15178 	}
15179 
15180 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15181 					   PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
15182 	if (pos < 0)
15183 		goto read_sn;
15184 
15185 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15186 	memcpy(bp->board_partno, &vpd_data[pos], size);
15187 
15188 read_sn:
15189 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15190 					   PCI_VPD_RO_KEYWORD_SERIALNO,
15191 					   &kw_len);
15192 	if (pos < 0)
15193 		goto exit;
15194 
15195 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15196 	memcpy(bp->board_serialno, &vpd_data[pos], size);
15197 exit:
15198 	kfree(vpd_data);
15199 }
15200 
15201 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
15202 {
15203 	struct pci_dev *pdev = bp->pdev;
15204 	u64 qword;
15205 
15206 	qword = pci_get_dsn(pdev);
15207 	if (!qword) {
15208 		netdev_info(bp->dev, "Unable to read adapter's DSN\n");
15209 		return -EOPNOTSUPP;
15210 	}
15211 
15212 	put_unaligned_le64(qword, dsn);
15213 
15214 	bp->flags |= BNXT_FLAG_DSN_VALID;
15215 	return 0;
15216 }
15217 
15218 static int bnxt_map_db_bar(struct bnxt *bp)
15219 {
15220 	if (!bp->db_size)
15221 		return -ENODEV;
15222 	bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
15223 	if (!bp->bar1)
15224 		return -ENOMEM;
15225 	return 0;
15226 }
15227 
15228 void bnxt_print_device_info(struct bnxt *bp)
15229 {
15230 	netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
15231 		    board_info[bp->board_idx].name,
15232 		    (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
15233 
15234 	pcie_print_link_status(bp->pdev);
15235 }
15236 
15237 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
15238 {
15239 	struct bnxt_hw_resc *hw_resc;
15240 	struct net_device *dev;
15241 	struct bnxt *bp;
15242 	int rc, max_irqs;
15243 
15244 	if (pci_is_bridge(pdev))
15245 		return -ENODEV;
15246 
15247 	/* Clear any pending DMA transactions from crash kernel
15248 	 * while loading driver in capture kernel.
15249 	 */
15250 	if (is_kdump_kernel()) {
15251 		pci_clear_master(pdev);
15252 		pcie_flr(pdev);
15253 	}
15254 
15255 	max_irqs = bnxt_get_max_irq(pdev);
15256 	dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
15257 				 max_irqs);
15258 	if (!dev)
15259 		return -ENOMEM;
15260 
15261 	bp = netdev_priv(dev);
15262 	bp->board_idx = ent->driver_data;
15263 	bp->msg_enable = BNXT_DEF_MSG_ENABLE;
15264 	bnxt_set_max_func_irqs(bp, max_irqs);
15265 
15266 	if (bnxt_vf_pciid(bp->board_idx))
15267 		bp->flags |= BNXT_FLAG_VF;
15268 
15269 	/* No devlink port registration in case of a VF */
15270 	if (BNXT_PF(bp))
15271 		SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
15272 
15273 	if (pdev->msix_cap)
15274 		bp->flags |= BNXT_FLAG_MSIX_CAP;
15275 
15276 	rc = bnxt_init_board(pdev, dev);
15277 	if (rc < 0)
15278 		goto init_err_free;
15279 
15280 	dev->netdev_ops = &bnxt_netdev_ops;
15281 	dev->stat_ops = &bnxt_stat_ops;
15282 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
15283 	dev->ethtool_ops = &bnxt_ethtool_ops;
15284 	pci_set_drvdata(pdev, dev);
15285 
15286 	rc = bnxt_alloc_hwrm_resources(bp);
15287 	if (rc)
15288 		goto init_err_pci_clean;
15289 
15290 	mutex_init(&bp->hwrm_cmd_lock);
15291 	mutex_init(&bp->link_lock);
15292 
15293 	rc = bnxt_fw_init_one_p1(bp);
15294 	if (rc)
15295 		goto init_err_pci_clean;
15296 
15297 	if (BNXT_PF(bp))
15298 		bnxt_vpd_read_info(bp);
15299 
15300 	if (BNXT_CHIP_P5_PLUS(bp)) {
15301 		bp->flags |= BNXT_FLAG_CHIP_P5_PLUS;
15302 		if (BNXT_CHIP_P7(bp))
15303 			bp->flags |= BNXT_FLAG_CHIP_P7;
15304 	}
15305 
15306 	rc = bnxt_alloc_rss_indir_tbl(bp, NULL);
15307 	if (rc)
15308 		goto init_err_pci_clean;
15309 
15310 	rc = bnxt_fw_init_one_p2(bp);
15311 	if (rc)
15312 		goto init_err_pci_clean;
15313 
15314 	rc = bnxt_map_db_bar(bp);
15315 	if (rc) {
15316 		dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
15317 			rc);
15318 		goto init_err_pci_clean;
15319 	}
15320 
15321 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
15322 			   NETIF_F_TSO | NETIF_F_TSO6 |
15323 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
15324 			   NETIF_F_GSO_IPXIP4 |
15325 			   NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
15326 			   NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
15327 			   NETIF_F_RXCSUM | NETIF_F_GRO;
15328 	if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
15329 		dev->hw_features |= NETIF_F_GSO_UDP_L4;
15330 
15331 	if (BNXT_SUPPORTS_TPA(bp))
15332 		dev->hw_features |= NETIF_F_LRO;
15333 
15334 	dev->hw_enc_features =
15335 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
15336 			NETIF_F_TSO | NETIF_F_TSO6 |
15337 			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
15338 			NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
15339 			NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
15340 	if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
15341 		dev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
15342 	if (bp->flags & BNXT_FLAG_CHIP_P7)
15343 		dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7;
15344 	else
15345 		dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
15346 
15347 	dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
15348 				    NETIF_F_GSO_GRE_CSUM;
15349 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
15350 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
15351 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
15352 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
15353 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
15354 	if (BNXT_SUPPORTS_TPA(bp))
15355 		dev->hw_features |= NETIF_F_GRO_HW;
15356 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
15357 	if (dev->features & NETIF_F_GRO_HW)
15358 		dev->features &= ~NETIF_F_LRO;
15359 	dev->priv_flags |= IFF_UNICAST_FLT;
15360 
15361 	netif_set_tso_max_size(dev, GSO_MAX_SIZE);
15362 
15363 	dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
15364 			    NETDEV_XDP_ACT_RX_SG;
15365 
15366 #ifdef CONFIG_BNXT_SRIOV
15367 	init_waitqueue_head(&bp->sriov_cfg_wait);
15368 #endif
15369 	if (BNXT_SUPPORTS_TPA(bp)) {
15370 		bp->gro_func = bnxt_gro_func_5730x;
15371 		if (BNXT_CHIP_P4(bp))
15372 			bp->gro_func = bnxt_gro_func_5731x;
15373 		else if (BNXT_CHIP_P5_PLUS(bp))
15374 			bp->gro_func = bnxt_gro_func_5750x;
15375 	}
15376 	if (!BNXT_CHIP_P4_PLUS(bp))
15377 		bp->flags |= BNXT_FLAG_DOUBLE_DB;
15378 
15379 	rc = bnxt_init_mac_addr(bp);
15380 	if (rc) {
15381 		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
15382 		rc = -EADDRNOTAVAIL;
15383 		goto init_err_pci_clean;
15384 	}
15385 
15386 	if (BNXT_PF(bp)) {
15387 		/* Read the adapter's DSN to use as the eswitch switch_id */
15388 		rc = bnxt_pcie_dsn_get(bp, bp->dsn);
15389 	}
15390 
15391 	/* MTU range: 60 - FW defined max */
15392 	dev->min_mtu = ETH_ZLEN;
15393 	dev->max_mtu = bp->max_mtu;
15394 
15395 	rc = bnxt_probe_phy(bp, true);
15396 	if (rc)
15397 		goto init_err_pci_clean;
15398 
15399 	hw_resc = &bp->hw_resc;
15400 	bp->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows +
15401 		       BNXT_L2_FLTR_MAX_FLTR;
15402 	/* Older firmware may not report these filters properly */
15403 	if (bp->max_fltr < BNXT_MAX_FLTR)
15404 		bp->max_fltr = BNXT_MAX_FLTR;
15405 	bnxt_init_l2_fltr_tbl(bp);
15406 	bnxt_set_rx_skb_mode(bp, false);
15407 	bnxt_set_tpa_flags(bp);
15408 	bnxt_set_ring_params(bp);
15409 	bnxt_rdma_aux_device_init(bp);
15410 	rc = bnxt_set_dflt_rings(bp, true);
15411 	if (rc) {
15412 		if (BNXT_VF(bp) && rc == -ENODEV) {
15413 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
15414 		} else {
15415 			netdev_err(bp->dev, "Not enough rings available.\n");
15416 			rc = -ENOMEM;
15417 		}
15418 		goto init_err_pci_clean;
15419 	}
15420 
15421 	bnxt_fw_init_one_p3(bp);
15422 
15423 	bnxt_init_dflt_coal(bp);
15424 
15425 	if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
15426 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
15427 
15428 	rc = bnxt_init_int_mode(bp);
15429 	if (rc)
15430 		goto init_err_pci_clean;
15431 
15432 	/* No TC has been set yet and rings may have been trimmed due to
15433 	 * limited MSIX, so we re-initialize the TX rings per TC.
15434 	 */
15435 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15436 
15437 	if (BNXT_PF(bp)) {
15438 		if (!bnxt_pf_wq) {
15439 			bnxt_pf_wq =
15440 				create_singlethread_workqueue("bnxt_pf_wq");
15441 			if (!bnxt_pf_wq) {
15442 				dev_err(&pdev->dev, "Unable to create workqueue.\n");
15443 				rc = -ENOMEM;
15444 				goto init_err_pci_clean;
15445 			}
15446 		}
15447 		rc = bnxt_init_tc(bp);
15448 		if (rc)
15449 			netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
15450 				   rc);
15451 	}
15452 
15453 	bnxt_inv_fw_health_reg(bp);
15454 	rc = bnxt_dl_register(bp);
15455 	if (rc)
15456 		goto init_err_dl;
15457 
15458 	INIT_LIST_HEAD(&bp->usr_fltr_list);
15459 
15460 	if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
15461 		bnxt_init_multi_rss_ctx(bp);
15462 
15463 
15464 	rc = register_netdev(dev);
15465 	if (rc)
15466 		goto init_err_cleanup;
15467 
15468 	bnxt_dl_fw_reporters_create(bp);
15469 
15470 	bnxt_rdma_aux_device_add(bp);
15471 
15472 	bnxt_print_device_info(bp);
15473 
15474 	pci_save_state(pdev);
15475 
15476 	return 0;
15477 init_err_cleanup:
15478 	bnxt_rdma_aux_device_uninit(bp);
15479 	bnxt_dl_unregister(bp);
15480 init_err_dl:
15481 	bnxt_shutdown_tc(bp);
15482 	bnxt_clear_int_mode(bp);
15483 
15484 init_err_pci_clean:
15485 	if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
15486 		bnxt_clear_rss_ctxs(bp, true);
15487 	bnxt_hwrm_func_drv_unrgtr(bp);
15488 	bnxt_free_hwrm_resources(bp);
15489 	bnxt_hwmon_uninit(bp);
15490 	bnxt_ethtool_free(bp);
15491 	bnxt_ptp_clear(bp);
15492 	kfree(bp->ptp_cfg);
15493 	bp->ptp_cfg = NULL;
15494 	kfree(bp->fw_health);
15495 	bp->fw_health = NULL;
15496 	bnxt_cleanup_pci(bp);
15497 	bnxt_free_ctx_mem(bp);
15498 	kfree(bp->rss_indir_tbl);
15499 	bp->rss_indir_tbl = NULL;
15500 
15501 init_err_free:
15502 	free_netdev(dev);
15503 	return rc;
15504 }
15505 
15506 static void bnxt_shutdown(struct pci_dev *pdev)
15507 {
15508 	struct net_device *dev = pci_get_drvdata(pdev);
15509 	struct bnxt *bp;
15510 
15511 	if (!dev)
15512 		return;
15513 
15514 	rtnl_lock();
15515 	bp = netdev_priv(dev);
15516 	if (!bp)
15517 		goto shutdown_exit;
15518 
15519 	if (netif_running(dev))
15520 		dev_close(dev);
15521 
15522 	bnxt_clear_int_mode(bp);
15523 	pci_disable_device(pdev);
15524 
15525 	if (system_state == SYSTEM_POWER_OFF) {
15526 		pci_wake_from_d3(pdev, bp->wol);
15527 		pci_set_power_state(pdev, PCI_D3hot);
15528 	}
15529 
15530 shutdown_exit:
15531 	rtnl_unlock();
15532 }
15533 
15534 #ifdef CONFIG_PM_SLEEP
15535 static int bnxt_suspend(struct device *device)
15536 {
15537 	struct net_device *dev = dev_get_drvdata(device);
15538 	struct bnxt *bp = netdev_priv(dev);
15539 	int rc = 0;
15540 
15541 	rtnl_lock();
15542 	bnxt_ulp_stop(bp);
15543 	if (netif_running(dev)) {
15544 		netif_device_detach(dev);
15545 		rc = bnxt_close(dev);
15546 	}
15547 	bnxt_hwrm_func_drv_unrgtr(bp);
15548 	pci_disable_device(bp->pdev);
15549 	bnxt_free_ctx_mem(bp);
15550 	rtnl_unlock();
15551 	return rc;
15552 }
15553 
15554 static int bnxt_resume(struct device *device)
15555 {
15556 	struct net_device *dev = dev_get_drvdata(device);
15557 	struct bnxt *bp = netdev_priv(dev);
15558 	int rc = 0;
15559 
15560 	rtnl_lock();
15561 	rc = pci_enable_device(bp->pdev);
15562 	if (rc) {
15563 		netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
15564 			   rc);
15565 		goto resume_exit;
15566 	}
15567 	pci_set_master(bp->pdev);
15568 	if (bnxt_hwrm_ver_get(bp)) {
15569 		rc = -ENODEV;
15570 		goto resume_exit;
15571 	}
15572 	rc = bnxt_hwrm_func_reset(bp);
15573 	if (rc) {
15574 		rc = -EBUSY;
15575 		goto resume_exit;
15576 	}
15577 
15578 	rc = bnxt_hwrm_func_qcaps(bp);
15579 	if (rc)
15580 		goto resume_exit;
15581 
15582 	bnxt_clear_reservations(bp, true);
15583 
15584 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
15585 		rc = -ENODEV;
15586 		goto resume_exit;
15587 	}
15588 
15589 	bnxt_get_wol_settings(bp);
15590 	if (netif_running(dev)) {
15591 		rc = bnxt_open(dev);
15592 		if (!rc)
15593 			netif_device_attach(dev);
15594 	}
15595 
15596 resume_exit:
15597 	bnxt_ulp_start(bp, rc);
15598 	if (!rc)
15599 		bnxt_reenable_sriov(bp);
15600 	rtnl_unlock();
15601 	return rc;
15602 }
15603 
15604 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
15605 #define BNXT_PM_OPS (&bnxt_pm_ops)
15606 
15607 #else
15608 
15609 #define BNXT_PM_OPS NULL
15610 
15611 #endif /* CONFIG_PM_SLEEP */
15612 
15613 /**
15614  * bnxt_io_error_detected - called when PCI error is detected
15615  * @pdev: Pointer to PCI device
15616  * @state: The current pci connection state
15617  *
15618  * This function is called after a PCI bus error affecting
15619  * this device has been detected.
15620  */
15621 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
15622 					       pci_channel_state_t state)
15623 {
15624 	struct net_device *netdev = pci_get_drvdata(pdev);
15625 	struct bnxt *bp = netdev_priv(netdev);
15626 
15627 	netdev_info(netdev, "PCI I/O error detected\n");
15628 
15629 	rtnl_lock();
15630 	netif_device_detach(netdev);
15631 
15632 	bnxt_ulp_stop(bp);
15633 
15634 	if (state == pci_channel_io_perm_failure) {
15635 		rtnl_unlock();
15636 		return PCI_ERS_RESULT_DISCONNECT;
15637 	}
15638 
15639 	if (state == pci_channel_io_frozen)
15640 		set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
15641 
15642 	if (netif_running(netdev))
15643 		bnxt_close(netdev);
15644 
15645 	if (pci_is_enabled(pdev))
15646 		pci_disable_device(pdev);
15647 	bnxt_free_ctx_mem(bp);
15648 	rtnl_unlock();
15649 
15650 	/* Request a slot slot reset. */
15651 	return PCI_ERS_RESULT_NEED_RESET;
15652 }
15653 
15654 /**
15655  * bnxt_io_slot_reset - called after the pci bus has been reset.
15656  * @pdev: Pointer to PCI device
15657  *
15658  * Restart the card from scratch, as if from a cold-boot.
15659  * At this point, the card has exprienced a hard reset,
15660  * followed by fixups by BIOS, and has its config space
15661  * set up identically to what it was at cold boot.
15662  */
15663 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
15664 {
15665 	pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
15666 	struct net_device *netdev = pci_get_drvdata(pdev);
15667 	struct bnxt *bp = netdev_priv(netdev);
15668 	int retry = 0;
15669 	int err = 0;
15670 	int off;
15671 
15672 	netdev_info(bp->dev, "PCI Slot Reset\n");
15673 
15674 	if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
15675 	    test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state))
15676 		msleep(900);
15677 
15678 	rtnl_lock();
15679 
15680 	if (pci_enable_device(pdev)) {
15681 		dev_err(&pdev->dev,
15682 			"Cannot re-enable PCI device after reset.\n");
15683 	} else {
15684 		pci_set_master(pdev);
15685 		/* Upon fatal error, our device internal logic that latches to
15686 		 * BAR value is getting reset and will restore only upon
15687 		 * rewritting the BARs.
15688 		 *
15689 		 * As pci_restore_state() does not re-write the BARs if the
15690 		 * value is same as saved value earlier, driver needs to
15691 		 * write the BARs to 0 to force restore, in case of fatal error.
15692 		 */
15693 		if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
15694 				       &bp->state)) {
15695 			for (off = PCI_BASE_ADDRESS_0;
15696 			     off <= PCI_BASE_ADDRESS_5; off += 4)
15697 				pci_write_config_dword(bp->pdev, off, 0);
15698 		}
15699 		pci_restore_state(pdev);
15700 		pci_save_state(pdev);
15701 
15702 		bnxt_inv_fw_health_reg(bp);
15703 		bnxt_try_map_fw_health_reg(bp);
15704 
15705 		/* In some PCIe AER scenarios, firmware may take up to
15706 		 * 10 seconds to become ready in the worst case.
15707 		 */
15708 		do {
15709 			err = bnxt_try_recover_fw(bp);
15710 			if (!err)
15711 				break;
15712 			retry++;
15713 		} while (retry < BNXT_FW_SLOT_RESET_RETRY);
15714 
15715 		if (err) {
15716 			dev_err(&pdev->dev, "Firmware not ready\n");
15717 			goto reset_exit;
15718 		}
15719 
15720 		err = bnxt_hwrm_func_reset(bp);
15721 		if (!err)
15722 			result = PCI_ERS_RESULT_RECOVERED;
15723 
15724 		bnxt_ulp_irq_stop(bp);
15725 		bnxt_clear_int_mode(bp);
15726 		err = bnxt_init_int_mode(bp);
15727 		bnxt_ulp_irq_restart(bp, err);
15728 	}
15729 
15730 reset_exit:
15731 	bnxt_clear_reservations(bp, true);
15732 	rtnl_unlock();
15733 
15734 	return result;
15735 }
15736 
15737 /**
15738  * bnxt_io_resume - called when traffic can start flowing again.
15739  * @pdev: Pointer to PCI device
15740  *
15741  * This callback is called when the error recovery driver tells
15742  * us that its OK to resume normal operation.
15743  */
15744 static void bnxt_io_resume(struct pci_dev *pdev)
15745 {
15746 	struct net_device *netdev = pci_get_drvdata(pdev);
15747 	struct bnxt *bp = netdev_priv(netdev);
15748 	int err;
15749 
15750 	netdev_info(bp->dev, "PCI Slot Resume\n");
15751 	rtnl_lock();
15752 
15753 	err = bnxt_hwrm_func_qcaps(bp);
15754 	if (!err && netif_running(netdev))
15755 		err = bnxt_open(netdev);
15756 
15757 	bnxt_ulp_start(bp, err);
15758 	if (!err) {
15759 		bnxt_reenable_sriov(bp);
15760 		netif_device_attach(netdev);
15761 	}
15762 
15763 	rtnl_unlock();
15764 }
15765 
15766 static const struct pci_error_handlers bnxt_err_handler = {
15767 	.error_detected	= bnxt_io_error_detected,
15768 	.slot_reset	= bnxt_io_slot_reset,
15769 	.resume		= bnxt_io_resume
15770 };
15771 
15772 static struct pci_driver bnxt_pci_driver = {
15773 	.name		= DRV_MODULE_NAME,
15774 	.id_table	= bnxt_pci_tbl,
15775 	.probe		= bnxt_init_one,
15776 	.remove		= bnxt_remove_one,
15777 	.shutdown	= bnxt_shutdown,
15778 	.driver.pm	= BNXT_PM_OPS,
15779 	.err_handler	= &bnxt_err_handler,
15780 #if defined(CONFIG_BNXT_SRIOV)
15781 	.sriov_configure = bnxt_sriov_configure,
15782 #endif
15783 };
15784 
15785 static int __init bnxt_init(void)
15786 {
15787 	int err;
15788 
15789 	bnxt_debug_init();
15790 	err = pci_register_driver(&bnxt_pci_driver);
15791 	if (err) {
15792 		bnxt_debug_exit();
15793 		return err;
15794 	}
15795 
15796 	return 0;
15797 }
15798 
15799 static void __exit bnxt_exit(void)
15800 {
15801 	pci_unregister_driver(&bnxt_pci_driver);
15802 	if (bnxt_pf_wq)
15803 		destroy_workqueue(bnxt_pf_wq);
15804 	bnxt_debug_exit();
15805 }
15806 
15807 module_init(bnxt_init);
15808 module_exit(bnxt_exit);
15809