1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 * Copyright (c) 2016-2019 Broadcom Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/gro.h>
41 #include <net/ip.h>
42 #include <net/tcp.h>
43 #include <net/udp.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 #include <net/udp_tunnel.h>
47 #include <linux/workqueue.h>
48 #include <linux/prefetch.h>
49 #include <linux/cache.h>
50 #include <linux/log2.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <net/page_pool/helpers.h>
56 #include <linux/align.h>
57 #include <net/netdev_queues.h>
58
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_hwrm.h"
62 #include "bnxt_ulp.h"
63 #include "bnxt_sriov.h"
64 #include "bnxt_ethtool.h"
65 #include "bnxt_dcb.h"
66 #include "bnxt_xdp.h"
67 #include "bnxt_ptp.h"
68 #include "bnxt_vfr.h"
69 #include "bnxt_tc.h"
70 #include "bnxt_devlink.h"
71 #include "bnxt_debugfs.h"
72 #include "bnxt_coredump.h"
73 #include "bnxt_hwmon.h"
74
75 #define BNXT_TX_TIMEOUT (5 * HZ)
76 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \
77 NETIF_MSG_TX_ERR)
78
79 MODULE_LICENSE("GPL");
80 MODULE_DESCRIPTION("Broadcom NetXtreme network driver");
81
82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
84 #define BNXT_RX_COPY_THRESH 256
85
86 #define BNXT_TX_PUSH_THRESH 164
87
88 /* indexed by enum board_idx */
89 static const struct {
90 char *name;
91 } board_info[] = {
92 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
93 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
94 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
95 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
96 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
97 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
98 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
99 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
100 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
101 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
102 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
103 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
104 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
105 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
106 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
107 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
108 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
109 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
110 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
111 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
112 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
113 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
114 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
115 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
116 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
117 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
118 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
119 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
120 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
121 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
123 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
124 [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
125 [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
126 [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" },
127 [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
128 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
129 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
130 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
131 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
132 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
133 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
134 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
135 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
136 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
137 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
138 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
139 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
140 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
141 [NETXTREME_E_P7_VF] = { "Broadcom BCM5760X Virtual Function" },
142 };
143
144 static const struct pci_device_id bnxt_pci_tbl[] = {
145 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
146 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
147 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
148 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
149 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
150 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
151 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
152 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
153 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
154 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
155 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
156 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
157 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
158 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
159 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
160 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
161 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
162 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
163 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
164 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
165 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
166 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
167 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
168 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
169 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
170 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
171 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
172 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
173 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
174 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
175 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
176 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
177 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
178 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
179 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
180 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
181 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
182 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
183 { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 },
184 { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 },
185 { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 },
186 { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 },
187 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
188 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
189 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
190 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
191 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
192 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
193 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
194 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
195 #ifdef CONFIG_BNXT_SRIOV
196 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
197 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
198 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
199 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
200 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
201 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
202 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
203 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
204 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
205 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
206 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
207 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
208 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
209 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
210 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
211 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
212 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
213 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
214 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
215 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
216 { PCI_VDEVICE(BROADCOM, 0x1819), .driver_data = NETXTREME_E_P7_VF },
217 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
218 #endif
219 { 0 }
220 };
221
222 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
223
224 static const u16 bnxt_vf_req_snif[] = {
225 HWRM_FUNC_CFG,
226 HWRM_FUNC_VF_CFG,
227 HWRM_PORT_PHY_QCFG,
228 HWRM_CFA_L2_FILTER_ALLOC,
229 };
230
231 static const u16 bnxt_async_events_arr[] = {
232 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
233 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
234 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
235 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
236 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
237 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
238 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
239 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
240 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
241 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
242 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
243 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
244 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
245 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
246 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
247 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
248 };
249
250 static struct workqueue_struct *bnxt_pf_wq;
251
252 #define BNXT_IPV6_MASK_ALL {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, \
253 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }}}
254 #define BNXT_IPV6_MASK_NONE {{{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}}
255
256 const struct bnxt_flow_masks BNXT_FLOW_MASK_NONE = {
257 .ports = {
258 .src = 0,
259 .dst = 0,
260 },
261 .addrs = {
262 .v6addrs = {
263 .src = BNXT_IPV6_MASK_NONE,
264 .dst = BNXT_IPV6_MASK_NONE,
265 },
266 },
267 };
268
269 const struct bnxt_flow_masks BNXT_FLOW_IPV6_MASK_ALL = {
270 .ports = {
271 .src = cpu_to_be16(0xffff),
272 .dst = cpu_to_be16(0xffff),
273 },
274 .addrs = {
275 .v6addrs = {
276 .src = BNXT_IPV6_MASK_ALL,
277 .dst = BNXT_IPV6_MASK_ALL,
278 },
279 },
280 };
281
282 const struct bnxt_flow_masks BNXT_FLOW_IPV4_MASK_ALL = {
283 .ports = {
284 .src = cpu_to_be16(0xffff),
285 .dst = cpu_to_be16(0xffff),
286 },
287 .addrs = {
288 .v4addrs = {
289 .src = cpu_to_be32(0xffffffff),
290 .dst = cpu_to_be32(0xffffffff),
291 },
292 },
293 };
294
bnxt_vf_pciid(enum board_idx idx)295 static bool bnxt_vf_pciid(enum board_idx idx)
296 {
297 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
298 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
299 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
300 idx == NETXTREME_E_P5_VF_HV || idx == NETXTREME_E_P7_VF);
301 }
302
303 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
304 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
305
306 #define BNXT_DB_CQ(db, idx) \
307 writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
308
309 #define BNXT_DB_NQ_P5(db, idx) \
310 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
311 (db)->doorbell)
312
313 #define BNXT_DB_NQ_P7(db, idx) \
314 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK | \
315 DB_RING_IDX(db, idx), (db)->doorbell)
316
317 #define BNXT_DB_CQ_ARM(db, idx) \
318 writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
319
320 #define BNXT_DB_NQ_ARM_P5(db, idx) \
321 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | \
322 DB_RING_IDX(db, idx), (db)->doorbell)
323
bnxt_db_nq(struct bnxt * bp,struct bnxt_db_info * db,u32 idx)324 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
325 {
326 if (bp->flags & BNXT_FLAG_CHIP_P7)
327 BNXT_DB_NQ_P7(db, idx);
328 else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
329 BNXT_DB_NQ_P5(db, idx);
330 else
331 BNXT_DB_CQ(db, idx);
332 }
333
bnxt_db_nq_arm(struct bnxt * bp,struct bnxt_db_info * db,u32 idx)334 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
335 {
336 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
337 BNXT_DB_NQ_ARM_P5(db, idx);
338 else
339 BNXT_DB_CQ_ARM(db, idx);
340 }
341
bnxt_db_cq(struct bnxt * bp,struct bnxt_db_info * db,u32 idx)342 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
343 {
344 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
345 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
346 DB_RING_IDX(db, idx), db->doorbell);
347 else
348 BNXT_DB_CQ(db, idx);
349 }
350
bnxt_queue_fw_reset_work(struct bnxt * bp,unsigned long delay)351 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
352 {
353 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
354 return;
355
356 if (BNXT_PF(bp))
357 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
358 else
359 schedule_delayed_work(&bp->fw_reset_task, delay);
360 }
361
__bnxt_queue_sp_work(struct bnxt * bp)362 static void __bnxt_queue_sp_work(struct bnxt *bp)
363 {
364 if (BNXT_PF(bp))
365 queue_work(bnxt_pf_wq, &bp->sp_task);
366 else
367 schedule_work(&bp->sp_task);
368 }
369
bnxt_queue_sp_work(struct bnxt * bp,unsigned int event)370 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
371 {
372 set_bit(event, &bp->sp_event);
373 __bnxt_queue_sp_work(bp);
374 }
375
bnxt_sched_reset_rxr(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)376 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
377 {
378 if (!rxr->bnapi->in_reset) {
379 rxr->bnapi->in_reset = true;
380 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
381 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
382 else
383 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
384 __bnxt_queue_sp_work(bp);
385 }
386 rxr->rx_next_cons = 0xffff;
387 }
388
bnxt_sched_reset_txr(struct bnxt * bp,struct bnxt_tx_ring_info * txr,u16 curr)389 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
390 u16 curr)
391 {
392 struct bnxt_napi *bnapi = txr->bnapi;
393
394 if (bnapi->tx_fault)
395 return;
396
397 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
398 txr->txq_index, txr->tx_hw_cons,
399 txr->tx_cons, txr->tx_prod, curr);
400 WARN_ON_ONCE(1);
401 bnapi->tx_fault = 1;
402 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
403 }
404
405 const u16 bnxt_lhint_arr[] = {
406 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
407 TX_BD_FLAGS_LHINT_512_TO_1023,
408 TX_BD_FLAGS_LHINT_1024_TO_2047,
409 TX_BD_FLAGS_LHINT_1024_TO_2047,
410 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
411 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
412 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
413 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
414 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
415 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
416 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
417 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
418 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
419 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
420 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
421 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
422 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
423 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
424 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
425 };
426
bnxt_xmit_get_cfa_action(struct sk_buff * skb)427 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
428 {
429 struct metadata_dst *md_dst = skb_metadata_dst(skb);
430
431 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
432 return 0;
433
434 return md_dst->u.port_info.port_id;
435 }
436
bnxt_txr_db_kick(struct bnxt * bp,struct bnxt_tx_ring_info * txr,u16 prod)437 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
438 u16 prod)
439 {
440 /* Sync BD data before updating doorbell */
441 wmb();
442 bnxt_db_write(bp, &txr->tx_db, prod);
443 txr->kick_pending = 0;
444 }
445
bnxt_start_xmit(struct sk_buff * skb,struct net_device * dev)446 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
447 {
448 struct bnxt *bp = netdev_priv(dev);
449 struct tx_bd *txbd, *txbd0;
450 struct tx_bd_ext *txbd1;
451 struct netdev_queue *txq;
452 int i;
453 dma_addr_t mapping;
454 unsigned int length, pad = 0;
455 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
456 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
457 struct pci_dev *pdev = bp->pdev;
458 u16 prod, last_frag, txts_prod;
459 struct bnxt_tx_ring_info *txr;
460 struct bnxt_sw_tx_bd *tx_buf;
461 __le32 lflags = 0;
462
463 i = skb_get_queue_mapping(skb);
464 if (unlikely(i >= bp->tx_nr_rings)) {
465 dev_kfree_skb_any(skb);
466 dev_core_stats_tx_dropped_inc(dev);
467 return NETDEV_TX_OK;
468 }
469
470 txq = netdev_get_tx_queue(dev, i);
471 txr = &bp->tx_ring[bp->tx_ring_map[i]];
472 prod = txr->tx_prod;
473
474 free_size = bnxt_tx_avail(bp, txr);
475 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
476 /* We must have raced with NAPI cleanup */
477 if (net_ratelimit() && txr->kick_pending)
478 netif_warn(bp, tx_err, dev,
479 "bnxt: ring busy w/ flush pending!\n");
480 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
481 bp->tx_wake_thresh))
482 return NETDEV_TX_BUSY;
483 }
484
485 if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
486 goto tx_free;
487
488 length = skb->len;
489 len = skb_headlen(skb);
490 last_frag = skb_shinfo(skb)->nr_frags;
491
492 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
493
494 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
495 tx_buf->skb = skb;
496 tx_buf->nr_frags = last_frag;
497
498 vlan_tag_flags = 0;
499 cfa_action = bnxt_xmit_get_cfa_action(skb);
500 if (skb_vlan_tag_present(skb)) {
501 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
502 skb_vlan_tag_get(skb);
503 /* Currently supports 8021Q, 8021AD vlan offloads
504 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
505 */
506 if (skb->vlan_proto == htons(ETH_P_8021Q))
507 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
508 }
509
510 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && ptp &&
511 ptp->tx_tstamp_en) {
512 if (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) {
513 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
514 tx_buf->is_ts_pkt = 1;
515 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
516 } else if (!skb_is_gso(skb)) {
517 u16 seq_id, hdr_off;
518
519 if (!bnxt_ptp_parse(skb, &seq_id, &hdr_off) &&
520 !bnxt_ptp_get_txts_prod(ptp, &txts_prod)) {
521 if (vlan_tag_flags)
522 hdr_off += VLAN_HLEN;
523 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
524 tx_buf->is_ts_pkt = 1;
525 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
526
527 ptp->txts_req[txts_prod].tx_seqid = seq_id;
528 ptp->txts_req[txts_prod].tx_hdr_off = hdr_off;
529 tx_buf->txts_prod = txts_prod;
530 }
531 }
532 }
533 if (unlikely(skb->no_fcs))
534 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
535
536 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
537 !lflags) {
538 struct tx_push_buffer *tx_push_buf = txr->tx_push;
539 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
540 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
541 void __iomem *db = txr->tx_db.doorbell;
542 void *pdata = tx_push_buf->data;
543 u64 *end;
544 int j, push_len;
545
546 /* Set COAL_NOW to be ready quickly for the next push */
547 tx_push->tx_bd_len_flags_type =
548 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
549 TX_BD_TYPE_LONG_TX_BD |
550 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
551 TX_BD_FLAGS_COAL_NOW |
552 TX_BD_FLAGS_PACKET_END |
553 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
554
555 if (skb->ip_summed == CHECKSUM_PARTIAL)
556 tx_push1->tx_bd_hsize_lflags =
557 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
558 else
559 tx_push1->tx_bd_hsize_lflags = 0;
560
561 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
562 tx_push1->tx_bd_cfa_action =
563 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
564
565 end = pdata + length;
566 end = PTR_ALIGN(end, 8) - 1;
567 *end = 0;
568
569 skb_copy_from_linear_data(skb, pdata, len);
570 pdata += len;
571 for (j = 0; j < last_frag; j++) {
572 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
573 void *fptr;
574
575 fptr = skb_frag_address_safe(frag);
576 if (!fptr)
577 goto normal_tx;
578
579 memcpy(pdata, fptr, skb_frag_size(frag));
580 pdata += skb_frag_size(frag);
581 }
582
583 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
584 txbd->tx_bd_haddr = txr->data_mapping;
585 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
586 prod = NEXT_TX(prod);
587 tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
588 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
589 memcpy(txbd, tx_push1, sizeof(*txbd));
590 prod = NEXT_TX(prod);
591 tx_push->doorbell =
592 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH |
593 DB_RING_IDX(&txr->tx_db, prod));
594 WRITE_ONCE(txr->tx_prod, prod);
595
596 tx_buf->is_push = 1;
597 netdev_tx_sent_queue(txq, skb->len);
598 wmb(); /* Sync is_push and byte queue before pushing data */
599
600 push_len = (length + sizeof(*tx_push) + 7) / 8;
601 if (push_len > 16) {
602 __iowrite64_copy(db, tx_push_buf, 16);
603 __iowrite32_copy(db + 4, tx_push_buf + 1,
604 (push_len - 16) << 1);
605 } else {
606 __iowrite64_copy(db, tx_push_buf, push_len);
607 }
608
609 goto tx_done;
610 }
611
612 normal_tx:
613 if (length < BNXT_MIN_PKT_SIZE) {
614 pad = BNXT_MIN_PKT_SIZE - length;
615 if (skb_pad(skb, pad))
616 /* SKB already freed. */
617 goto tx_kick_pending;
618 length = BNXT_MIN_PKT_SIZE;
619 }
620
621 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
622
623 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
624 goto tx_free;
625
626 dma_unmap_addr_set(tx_buf, mapping, mapping);
627 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
628 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
629
630 txbd->tx_bd_haddr = cpu_to_le64(mapping);
631 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
632
633 prod = NEXT_TX(prod);
634 txbd1 = (struct tx_bd_ext *)
635 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
636
637 txbd1->tx_bd_hsize_lflags = lflags;
638 if (skb_is_gso(skb)) {
639 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
640 u32 hdr_len;
641
642 if (skb->encapsulation) {
643 if (udp_gso)
644 hdr_len = skb_inner_transport_offset(skb) +
645 sizeof(struct udphdr);
646 else
647 hdr_len = skb_inner_tcp_all_headers(skb);
648 } else if (udp_gso) {
649 hdr_len = skb_transport_offset(skb) +
650 sizeof(struct udphdr);
651 } else {
652 hdr_len = skb_tcp_all_headers(skb);
653 }
654
655 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
656 TX_BD_FLAGS_T_IPID |
657 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
658 length = skb_shinfo(skb)->gso_size;
659 txbd1->tx_bd_mss = cpu_to_le32(length);
660 length += hdr_len;
661 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
662 txbd1->tx_bd_hsize_lflags |=
663 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
664 txbd1->tx_bd_mss = 0;
665 }
666
667 length >>= 9;
668 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
669 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
670 skb->len);
671 i = 0;
672 goto tx_dma_error;
673 }
674 flags |= bnxt_lhint_arr[length];
675 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
676
677 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
678 txbd1->tx_bd_cfa_action =
679 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
680 txbd0 = txbd;
681 for (i = 0; i < last_frag; i++) {
682 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
683
684 prod = NEXT_TX(prod);
685 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
686
687 len = skb_frag_size(frag);
688 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
689 DMA_TO_DEVICE);
690
691 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
692 goto tx_dma_error;
693
694 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
695 dma_unmap_addr_set(tx_buf, mapping, mapping);
696
697 txbd->tx_bd_haddr = cpu_to_le64(mapping);
698
699 flags = len << TX_BD_LEN_SHIFT;
700 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
701 }
702
703 flags &= ~TX_BD_LEN;
704 txbd->tx_bd_len_flags_type =
705 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
706 TX_BD_FLAGS_PACKET_END);
707
708 netdev_tx_sent_queue(txq, skb->len);
709
710 skb_tx_timestamp(skb);
711
712 prod = NEXT_TX(prod);
713 WRITE_ONCE(txr->tx_prod, prod);
714
715 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
716 bnxt_txr_db_kick(bp, txr, prod);
717 } else {
718 if (free_size >= bp->tx_wake_thresh)
719 txbd0->tx_bd_len_flags_type |=
720 cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
721 txr->kick_pending = 1;
722 }
723
724 tx_done:
725
726 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
727 if (netdev_xmit_more() && !tx_buf->is_push) {
728 txbd0->tx_bd_len_flags_type &=
729 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
730 bnxt_txr_db_kick(bp, txr, prod);
731 }
732
733 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
734 bp->tx_wake_thresh);
735 }
736 return NETDEV_TX_OK;
737
738 tx_dma_error:
739 last_frag = i;
740
741 /* start back at beginning and unmap skb */
742 prod = txr->tx_prod;
743 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
744 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
745 skb_headlen(skb), DMA_TO_DEVICE);
746 prod = NEXT_TX(prod);
747
748 /* unmap remaining mapped pages */
749 for (i = 0; i < last_frag; i++) {
750 prod = NEXT_TX(prod);
751 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
752 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
753 skb_frag_size(&skb_shinfo(skb)->frags[i]),
754 DMA_TO_DEVICE);
755 }
756
757 tx_free:
758 dev_kfree_skb_any(skb);
759 tx_kick_pending:
760 if (BNXT_TX_PTP_IS_SET(lflags)) {
761 txr->tx_buf_ring[txr->tx_prod].is_ts_pkt = 0;
762 atomic64_inc(&bp->ptp_cfg->stats.ts_err);
763 if (!(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP))
764 /* set SKB to err so PTP worker will clean up */
765 ptp->txts_req[txts_prod].tx_skb = ERR_PTR(-EIO);
766 }
767 if (txr->kick_pending)
768 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
769 txr->tx_buf_ring[txr->tx_prod].skb = NULL;
770 dev_core_stats_tx_dropped_inc(dev);
771 return NETDEV_TX_OK;
772 }
773
774 /* Returns true if some remaining TX packets not processed. */
__bnxt_tx_int(struct bnxt * bp,struct bnxt_tx_ring_info * txr,int budget)775 static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
776 int budget)
777 {
778 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
779 struct pci_dev *pdev = bp->pdev;
780 u16 hw_cons = txr->tx_hw_cons;
781 unsigned int tx_bytes = 0;
782 u16 cons = txr->tx_cons;
783 int tx_pkts = 0;
784 bool rc = false;
785
786 while (RING_TX(bp, cons) != hw_cons) {
787 struct bnxt_sw_tx_bd *tx_buf;
788 struct sk_buff *skb;
789 bool is_ts_pkt;
790 int j, last;
791
792 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
793 skb = tx_buf->skb;
794
795 if (unlikely(!skb)) {
796 bnxt_sched_reset_txr(bp, txr, cons);
797 return rc;
798 }
799
800 is_ts_pkt = tx_buf->is_ts_pkt;
801 if (is_ts_pkt && (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP)) {
802 rc = true;
803 break;
804 }
805
806 cons = NEXT_TX(cons);
807 tx_pkts++;
808 tx_bytes += skb->len;
809 tx_buf->skb = NULL;
810 tx_buf->is_ts_pkt = 0;
811
812 if (tx_buf->is_push) {
813 tx_buf->is_push = 0;
814 goto next_tx_int;
815 }
816
817 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
818 skb_headlen(skb), DMA_TO_DEVICE);
819 last = tx_buf->nr_frags;
820
821 for (j = 0; j < last; j++) {
822 cons = NEXT_TX(cons);
823 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
824 dma_unmap_page(
825 &pdev->dev,
826 dma_unmap_addr(tx_buf, mapping),
827 skb_frag_size(&skb_shinfo(skb)->frags[j]),
828 DMA_TO_DEVICE);
829 }
830 if (unlikely(is_ts_pkt)) {
831 if (BNXT_CHIP_P5(bp)) {
832 /* PTP worker takes ownership of the skb */
833 bnxt_get_tx_ts_p5(bp, skb, tx_buf->txts_prod);
834 skb = NULL;
835 }
836 }
837
838 next_tx_int:
839 cons = NEXT_TX(cons);
840
841 dev_consume_skb_any(skb);
842 }
843
844 WRITE_ONCE(txr->tx_cons, cons);
845
846 __netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
847 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
848 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
849
850 return rc;
851 }
852
bnxt_tx_int(struct bnxt * bp,struct bnxt_napi * bnapi,int budget)853 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
854 {
855 struct bnxt_tx_ring_info *txr;
856 bool more = false;
857 int i;
858
859 bnxt_for_each_napi_tx(i, bnapi, txr) {
860 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons))
861 more |= __bnxt_tx_int(bp, txr, budget);
862 }
863 if (!more)
864 bnapi->events &= ~BNXT_TX_CMP_EVENT;
865 }
866
__bnxt_alloc_rx_page(struct bnxt * bp,dma_addr_t * mapping,struct bnxt_rx_ring_info * rxr,unsigned int * offset,gfp_t gfp)867 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
868 struct bnxt_rx_ring_info *rxr,
869 unsigned int *offset,
870 gfp_t gfp)
871 {
872 struct page *page;
873
874 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
875 page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
876 BNXT_RX_PAGE_SIZE);
877 } else {
878 page = page_pool_dev_alloc_pages(rxr->page_pool);
879 *offset = 0;
880 }
881 if (!page)
882 return NULL;
883
884 *mapping = page_pool_get_dma_addr(page) + *offset;
885 return page;
886 }
887
__bnxt_alloc_rx_frag(struct bnxt * bp,dma_addr_t * mapping,gfp_t gfp)888 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
889 gfp_t gfp)
890 {
891 u8 *data;
892 struct pci_dev *pdev = bp->pdev;
893
894 if (gfp == GFP_ATOMIC)
895 data = napi_alloc_frag(bp->rx_buf_size);
896 else
897 data = netdev_alloc_frag(bp->rx_buf_size);
898 if (!data)
899 return NULL;
900
901 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
902 bp->rx_buf_use_size, bp->rx_dir,
903 DMA_ATTR_WEAK_ORDERING);
904
905 if (dma_mapping_error(&pdev->dev, *mapping)) {
906 skb_free_frag(data);
907 data = NULL;
908 }
909 return data;
910 }
911
bnxt_alloc_rx_data(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 prod,gfp_t gfp)912 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
913 u16 prod, gfp_t gfp)
914 {
915 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
916 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
917 dma_addr_t mapping;
918
919 if (BNXT_RX_PAGE_MODE(bp)) {
920 unsigned int offset;
921 struct page *page =
922 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
923
924 if (!page)
925 return -ENOMEM;
926
927 mapping += bp->rx_dma_offset;
928 rx_buf->data = page;
929 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
930 } else {
931 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
932
933 if (!data)
934 return -ENOMEM;
935
936 rx_buf->data = data;
937 rx_buf->data_ptr = data + bp->rx_offset;
938 }
939 rx_buf->mapping = mapping;
940
941 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
942 return 0;
943 }
944
bnxt_reuse_rx_data(struct bnxt_rx_ring_info * rxr,u16 cons,void * data)945 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
946 {
947 u16 prod = rxr->rx_prod;
948 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
949 struct bnxt *bp = rxr->bnapi->bp;
950 struct rx_bd *cons_bd, *prod_bd;
951
952 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
953 cons_rx_buf = &rxr->rx_buf_ring[cons];
954
955 prod_rx_buf->data = data;
956 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
957
958 prod_rx_buf->mapping = cons_rx_buf->mapping;
959
960 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
961 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)];
962
963 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
964 }
965
bnxt_find_next_agg_idx(struct bnxt_rx_ring_info * rxr,u16 idx)966 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
967 {
968 u16 next, max = rxr->rx_agg_bmap_size;
969
970 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
971 if (next >= max)
972 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
973 return next;
974 }
975
bnxt_alloc_rx_page(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 prod,gfp_t gfp)976 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
977 struct bnxt_rx_ring_info *rxr,
978 u16 prod, gfp_t gfp)
979 {
980 struct rx_bd *rxbd =
981 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
982 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
983 struct page *page;
984 dma_addr_t mapping;
985 u16 sw_prod = rxr->rx_sw_agg_prod;
986 unsigned int offset = 0;
987
988 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
989
990 if (!page)
991 return -ENOMEM;
992
993 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
994 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
995
996 __set_bit(sw_prod, rxr->rx_agg_bmap);
997 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
998 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
999
1000 rx_agg_buf->page = page;
1001 rx_agg_buf->offset = offset;
1002 rx_agg_buf->mapping = mapping;
1003 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
1004 rxbd->rx_bd_opaque = sw_prod;
1005 return 0;
1006 }
1007
bnxt_get_agg(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u16 cp_cons,u16 curr)1008 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
1009 struct bnxt_cp_ring_info *cpr,
1010 u16 cp_cons, u16 curr)
1011 {
1012 struct rx_agg_cmp *agg;
1013
1014 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
1015 agg = (struct rx_agg_cmp *)
1016 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1017 return agg;
1018 }
1019
bnxt_get_tpa_agg_p5(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 agg_id,u16 curr)1020 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
1021 struct bnxt_rx_ring_info *rxr,
1022 u16 agg_id, u16 curr)
1023 {
1024 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
1025
1026 return &tpa_info->agg_arr[curr];
1027 }
1028
bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info * cpr,u16 idx,u16 start,u32 agg_bufs,bool tpa)1029 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
1030 u16 start, u32 agg_bufs, bool tpa)
1031 {
1032 struct bnxt_napi *bnapi = cpr->bnapi;
1033 struct bnxt *bp = bnapi->bp;
1034 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1035 u16 prod = rxr->rx_agg_prod;
1036 u16 sw_prod = rxr->rx_sw_agg_prod;
1037 bool p5_tpa = false;
1038 u32 i;
1039
1040 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1041 p5_tpa = true;
1042
1043 for (i = 0; i < agg_bufs; i++) {
1044 u16 cons;
1045 struct rx_agg_cmp *agg;
1046 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
1047 struct rx_bd *prod_bd;
1048 struct page *page;
1049
1050 if (p5_tpa)
1051 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
1052 else
1053 agg = bnxt_get_agg(bp, cpr, idx, start + i);
1054 cons = agg->rx_agg_cmp_opaque;
1055 __clear_bit(cons, rxr->rx_agg_bmap);
1056
1057 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1058 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
1059
1060 __set_bit(sw_prod, rxr->rx_agg_bmap);
1061 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
1062 cons_rx_buf = &rxr->rx_agg_ring[cons];
1063
1064 /* It is possible for sw_prod to be equal to cons, so
1065 * set cons_rx_buf->page to NULL first.
1066 */
1067 page = cons_rx_buf->page;
1068 cons_rx_buf->page = NULL;
1069 prod_rx_buf->page = page;
1070 prod_rx_buf->offset = cons_rx_buf->offset;
1071
1072 prod_rx_buf->mapping = cons_rx_buf->mapping;
1073
1074 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
1075
1076 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
1077 prod_bd->rx_bd_opaque = sw_prod;
1078
1079 prod = NEXT_RX_AGG(prod);
1080 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1081 }
1082 rxr->rx_agg_prod = prod;
1083 rxr->rx_sw_agg_prod = sw_prod;
1084 }
1085
bnxt_rx_multi_page_skb(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 cons,void * data,u8 * data_ptr,dma_addr_t dma_addr,unsigned int offset_and_len)1086 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
1087 struct bnxt_rx_ring_info *rxr,
1088 u16 cons, void *data, u8 *data_ptr,
1089 dma_addr_t dma_addr,
1090 unsigned int offset_and_len)
1091 {
1092 unsigned int len = offset_and_len & 0xffff;
1093 struct page *page = data;
1094 u16 prod = rxr->rx_prod;
1095 struct sk_buff *skb;
1096 int err;
1097
1098 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1099 if (unlikely(err)) {
1100 bnxt_reuse_rx_data(rxr, cons, data);
1101 return NULL;
1102 }
1103 dma_addr -= bp->rx_dma_offset;
1104 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1105 bp->rx_dir);
1106 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
1107 if (!skb) {
1108 page_pool_recycle_direct(rxr->page_pool, page);
1109 return NULL;
1110 }
1111 skb_mark_for_recycle(skb);
1112 skb_reserve(skb, bp->rx_offset);
1113 __skb_put(skb, len);
1114
1115 return skb;
1116 }
1117
bnxt_rx_page_skb(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 cons,void * data,u8 * data_ptr,dma_addr_t dma_addr,unsigned int offset_and_len)1118 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1119 struct bnxt_rx_ring_info *rxr,
1120 u16 cons, void *data, u8 *data_ptr,
1121 dma_addr_t dma_addr,
1122 unsigned int offset_and_len)
1123 {
1124 unsigned int payload = offset_and_len >> 16;
1125 unsigned int len = offset_and_len & 0xffff;
1126 skb_frag_t *frag;
1127 struct page *page = data;
1128 u16 prod = rxr->rx_prod;
1129 struct sk_buff *skb;
1130 int off, err;
1131
1132 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1133 if (unlikely(err)) {
1134 bnxt_reuse_rx_data(rxr, cons, data);
1135 return NULL;
1136 }
1137 dma_addr -= bp->rx_dma_offset;
1138 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1139 bp->rx_dir);
1140
1141 if (unlikely(!payload))
1142 payload = eth_get_headlen(bp->dev, data_ptr, len);
1143
1144 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1145 if (!skb) {
1146 page_pool_recycle_direct(rxr->page_pool, page);
1147 return NULL;
1148 }
1149
1150 skb_mark_for_recycle(skb);
1151 off = (void *)data_ptr - page_address(page);
1152 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1153 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1154 payload + NET_IP_ALIGN);
1155
1156 frag = &skb_shinfo(skb)->frags[0];
1157 skb_frag_size_sub(frag, payload);
1158 skb_frag_off_add(frag, payload);
1159 skb->data_len -= payload;
1160 skb->tail += payload;
1161
1162 return skb;
1163 }
1164
bnxt_rx_skb(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u16 cons,void * data,u8 * data_ptr,dma_addr_t dma_addr,unsigned int offset_and_len)1165 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1166 struct bnxt_rx_ring_info *rxr, u16 cons,
1167 void *data, u8 *data_ptr,
1168 dma_addr_t dma_addr,
1169 unsigned int offset_and_len)
1170 {
1171 u16 prod = rxr->rx_prod;
1172 struct sk_buff *skb;
1173 int err;
1174
1175 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1176 if (unlikely(err)) {
1177 bnxt_reuse_rx_data(rxr, cons, data);
1178 return NULL;
1179 }
1180
1181 skb = napi_build_skb(data, bp->rx_buf_size);
1182 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1183 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1184 if (!skb) {
1185 skb_free_frag(data);
1186 return NULL;
1187 }
1188
1189 skb_reserve(skb, bp->rx_offset);
1190 skb_put(skb, offset_and_len & 0xffff);
1191 return skb;
1192 }
1193
__bnxt_rx_agg_pages(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,struct skb_shared_info * shinfo,u16 idx,u32 agg_bufs,bool tpa,struct xdp_buff * xdp)1194 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1195 struct bnxt_cp_ring_info *cpr,
1196 struct skb_shared_info *shinfo,
1197 u16 idx, u32 agg_bufs, bool tpa,
1198 struct xdp_buff *xdp)
1199 {
1200 struct bnxt_napi *bnapi = cpr->bnapi;
1201 struct pci_dev *pdev = bp->pdev;
1202 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1203 u16 prod = rxr->rx_agg_prod;
1204 u32 i, total_frag_len = 0;
1205 bool p5_tpa = false;
1206
1207 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1208 p5_tpa = true;
1209
1210 for (i = 0; i < agg_bufs; i++) {
1211 skb_frag_t *frag = &shinfo->frags[i];
1212 u16 cons, frag_len;
1213 struct rx_agg_cmp *agg;
1214 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1215 struct page *page;
1216 dma_addr_t mapping;
1217
1218 if (p5_tpa)
1219 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1220 else
1221 agg = bnxt_get_agg(bp, cpr, idx, i);
1222 cons = agg->rx_agg_cmp_opaque;
1223 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1224 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1225
1226 cons_rx_buf = &rxr->rx_agg_ring[cons];
1227 skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1228 cons_rx_buf->offset, frag_len);
1229 shinfo->nr_frags = i + 1;
1230 __clear_bit(cons, rxr->rx_agg_bmap);
1231
1232 /* It is possible for bnxt_alloc_rx_page() to allocate
1233 * a sw_prod index that equals the cons index, so we
1234 * need to clear the cons entry now.
1235 */
1236 mapping = cons_rx_buf->mapping;
1237 page = cons_rx_buf->page;
1238 cons_rx_buf->page = NULL;
1239
1240 if (xdp && page_is_pfmemalloc(page))
1241 xdp_buff_set_frag_pfmemalloc(xdp);
1242
1243 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1244 --shinfo->nr_frags;
1245 cons_rx_buf->page = page;
1246
1247 /* Update prod since possibly some pages have been
1248 * allocated already.
1249 */
1250 rxr->rx_agg_prod = prod;
1251 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1252 return 0;
1253 }
1254
1255 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1256 bp->rx_dir);
1257
1258 total_frag_len += frag_len;
1259 prod = NEXT_RX_AGG(prod);
1260 }
1261 rxr->rx_agg_prod = prod;
1262 return total_frag_len;
1263 }
1264
bnxt_rx_agg_pages_skb(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,struct sk_buff * skb,u16 idx,u32 agg_bufs,bool tpa)1265 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1266 struct bnxt_cp_ring_info *cpr,
1267 struct sk_buff *skb, u16 idx,
1268 u32 agg_bufs, bool tpa)
1269 {
1270 struct skb_shared_info *shinfo = skb_shinfo(skb);
1271 u32 total_frag_len = 0;
1272
1273 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1274 agg_bufs, tpa, NULL);
1275 if (!total_frag_len) {
1276 skb_mark_for_recycle(skb);
1277 dev_kfree_skb(skb);
1278 return NULL;
1279 }
1280
1281 skb->data_len += total_frag_len;
1282 skb->len += total_frag_len;
1283 skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1284 return skb;
1285 }
1286
bnxt_rx_agg_pages_xdp(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,struct xdp_buff * xdp,u16 idx,u32 agg_bufs,bool tpa)1287 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1288 struct bnxt_cp_ring_info *cpr,
1289 struct xdp_buff *xdp, u16 idx,
1290 u32 agg_bufs, bool tpa)
1291 {
1292 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1293 u32 total_frag_len = 0;
1294
1295 if (!xdp_buff_has_frags(xdp))
1296 shinfo->nr_frags = 0;
1297
1298 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1299 idx, agg_bufs, tpa, xdp);
1300 if (total_frag_len) {
1301 xdp_buff_set_frags_flag(xdp);
1302 shinfo->nr_frags = agg_bufs;
1303 shinfo->xdp_frags_size = total_frag_len;
1304 }
1305 return total_frag_len;
1306 }
1307
bnxt_agg_bufs_valid(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u8 agg_bufs,u32 * raw_cons)1308 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1309 u8 agg_bufs, u32 *raw_cons)
1310 {
1311 u16 last;
1312 struct rx_agg_cmp *agg;
1313
1314 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1315 last = RING_CMP(*raw_cons);
1316 agg = (struct rx_agg_cmp *)
1317 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1318 return RX_AGG_CMP_VALID(agg, *raw_cons);
1319 }
1320
bnxt_copy_data(struct bnxt_napi * bnapi,u8 * data,unsigned int len,dma_addr_t mapping)1321 static struct sk_buff *bnxt_copy_data(struct bnxt_napi *bnapi, u8 *data,
1322 unsigned int len,
1323 dma_addr_t mapping)
1324 {
1325 struct bnxt *bp = bnapi->bp;
1326 struct pci_dev *pdev = bp->pdev;
1327 struct sk_buff *skb;
1328
1329 skb = napi_alloc_skb(&bnapi->napi, len);
1330 if (!skb)
1331 return NULL;
1332
1333 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1334 bp->rx_dir);
1335
1336 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1337 len + NET_IP_ALIGN);
1338
1339 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1340 bp->rx_dir);
1341
1342 skb_put(skb, len);
1343
1344 return skb;
1345 }
1346
bnxt_copy_skb(struct bnxt_napi * bnapi,u8 * data,unsigned int len,dma_addr_t mapping)1347 static struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1348 unsigned int len,
1349 dma_addr_t mapping)
1350 {
1351 return bnxt_copy_data(bnapi, data, len, mapping);
1352 }
1353
bnxt_copy_xdp(struct bnxt_napi * bnapi,struct xdp_buff * xdp,unsigned int len,dma_addr_t mapping)1354 static struct sk_buff *bnxt_copy_xdp(struct bnxt_napi *bnapi,
1355 struct xdp_buff *xdp,
1356 unsigned int len,
1357 dma_addr_t mapping)
1358 {
1359 unsigned int metasize = 0;
1360 u8 *data = xdp->data;
1361 struct sk_buff *skb;
1362
1363 len = xdp->data_end - xdp->data_meta;
1364 metasize = xdp->data - xdp->data_meta;
1365 data = xdp->data_meta;
1366
1367 skb = bnxt_copy_data(bnapi, data, len, mapping);
1368 if (!skb)
1369 return skb;
1370
1371 if (metasize) {
1372 skb_metadata_set(skb, metasize);
1373 __skb_pull(skb, metasize);
1374 }
1375
1376 return skb;
1377 }
1378
bnxt_discard_rx(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u32 * raw_cons,void * cmp)1379 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1380 u32 *raw_cons, void *cmp)
1381 {
1382 struct rx_cmp *rxcmp = cmp;
1383 u32 tmp_raw_cons = *raw_cons;
1384 u8 cmp_type, agg_bufs = 0;
1385
1386 cmp_type = RX_CMP_TYPE(rxcmp);
1387
1388 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1389 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1390 RX_CMP_AGG_BUFS) >>
1391 RX_CMP_AGG_BUFS_SHIFT;
1392 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1393 struct rx_tpa_end_cmp *tpa_end = cmp;
1394
1395 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1396 return 0;
1397
1398 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1399 }
1400
1401 if (agg_bufs) {
1402 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1403 return -EBUSY;
1404 }
1405 *raw_cons = tmp_raw_cons;
1406 return 0;
1407 }
1408
bnxt_alloc_agg_idx(struct bnxt_rx_ring_info * rxr,u16 agg_id)1409 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1410 {
1411 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1412 u16 idx = agg_id & MAX_TPA_P5_MASK;
1413
1414 if (test_bit(idx, map->agg_idx_bmap))
1415 idx = find_first_zero_bit(map->agg_idx_bmap,
1416 BNXT_AGG_IDX_BMAP_SIZE);
1417 __set_bit(idx, map->agg_idx_bmap);
1418 map->agg_id_tbl[agg_id] = idx;
1419 return idx;
1420 }
1421
bnxt_free_agg_idx(struct bnxt_rx_ring_info * rxr,u16 idx)1422 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1423 {
1424 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1425
1426 __clear_bit(idx, map->agg_idx_bmap);
1427 }
1428
bnxt_lookup_agg_idx(struct bnxt_rx_ring_info * rxr,u16 agg_id)1429 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1430 {
1431 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1432
1433 return map->agg_id_tbl[agg_id];
1434 }
1435
bnxt_tpa_metadata(struct bnxt_tpa_info * tpa_info,struct rx_tpa_start_cmp * tpa_start,struct rx_tpa_start_cmp_ext * tpa_start1)1436 static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info,
1437 struct rx_tpa_start_cmp *tpa_start,
1438 struct rx_tpa_start_cmp_ext *tpa_start1)
1439 {
1440 tpa_info->cfa_code_valid = 1;
1441 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1442 tpa_info->vlan_valid = 0;
1443 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1444 tpa_info->vlan_valid = 1;
1445 tpa_info->metadata =
1446 le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1447 }
1448 }
1449
bnxt_tpa_metadata_v2(struct bnxt_tpa_info * tpa_info,struct rx_tpa_start_cmp * tpa_start,struct rx_tpa_start_cmp_ext * tpa_start1)1450 static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info,
1451 struct rx_tpa_start_cmp *tpa_start,
1452 struct rx_tpa_start_cmp_ext *tpa_start1)
1453 {
1454 tpa_info->vlan_valid = 0;
1455 if (TPA_START_VLAN_VALID(tpa_start)) {
1456 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
1457 u32 vlan_proto = ETH_P_8021Q;
1458
1459 tpa_info->vlan_valid = 1;
1460 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
1461 vlan_proto = ETH_P_8021AD;
1462 tpa_info->metadata = vlan_proto << 16 |
1463 TPA_START_METADATA0_TCI(tpa_start1);
1464 }
1465 }
1466
bnxt_tpa_start(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,u8 cmp_type,struct rx_tpa_start_cmp * tpa_start,struct rx_tpa_start_cmp_ext * tpa_start1)1467 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1468 u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
1469 struct rx_tpa_start_cmp_ext *tpa_start1)
1470 {
1471 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1472 struct bnxt_tpa_info *tpa_info;
1473 u16 cons, prod, agg_id;
1474 struct rx_bd *prod_bd;
1475 dma_addr_t mapping;
1476
1477 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1478 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1479 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1480 } else {
1481 agg_id = TPA_START_AGG_ID(tpa_start);
1482 }
1483 cons = tpa_start->rx_tpa_start_cmp_opaque;
1484 prod = rxr->rx_prod;
1485 cons_rx_buf = &rxr->rx_buf_ring[cons];
1486 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
1487 tpa_info = &rxr->rx_tpa[agg_id];
1488
1489 if (unlikely(cons != rxr->rx_next_cons ||
1490 TPA_START_ERROR(tpa_start))) {
1491 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1492 cons, rxr->rx_next_cons,
1493 TPA_START_ERROR_CODE(tpa_start1));
1494 bnxt_sched_reset_rxr(bp, rxr);
1495 return;
1496 }
1497 prod_rx_buf->data = tpa_info->data;
1498 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1499
1500 mapping = tpa_info->mapping;
1501 prod_rx_buf->mapping = mapping;
1502
1503 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
1504
1505 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1506
1507 tpa_info->data = cons_rx_buf->data;
1508 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1509 cons_rx_buf->data = NULL;
1510 tpa_info->mapping = cons_rx_buf->mapping;
1511
1512 tpa_info->len =
1513 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1514 RX_TPA_START_CMP_LEN_SHIFT;
1515 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1516 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1517 tpa_info->gso_type = SKB_GSO_TCPV4;
1518 if (TPA_START_IS_IPV6(tpa_start1))
1519 tpa_info->gso_type = SKB_GSO_TCPV6;
1520 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1521 else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP &&
1522 TPA_START_HASH_TYPE(tpa_start) == 3)
1523 tpa_info->gso_type = SKB_GSO_TCPV6;
1524 tpa_info->rss_hash =
1525 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1526 } else {
1527 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1528 tpa_info->gso_type = 0;
1529 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1530 }
1531 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1532 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1533 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
1534 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1);
1535 else
1536 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
1537 tpa_info->agg_count = 0;
1538
1539 rxr->rx_prod = NEXT_RX(prod);
1540 cons = RING_RX(bp, NEXT_RX(cons));
1541 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
1542 cons_rx_buf = &rxr->rx_buf_ring[cons];
1543
1544 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1545 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1546 cons_rx_buf->data = NULL;
1547 }
1548
bnxt_abort_tpa(struct bnxt_cp_ring_info * cpr,u16 idx,u32 agg_bufs)1549 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1550 {
1551 if (agg_bufs)
1552 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1553 }
1554
1555 #ifdef CONFIG_INET
bnxt_gro_tunnel(struct sk_buff * skb,__be16 ip_proto)1556 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1557 {
1558 struct udphdr *uh = NULL;
1559
1560 if (ip_proto == htons(ETH_P_IP)) {
1561 struct iphdr *iph = (struct iphdr *)skb->data;
1562
1563 if (iph->protocol == IPPROTO_UDP)
1564 uh = (struct udphdr *)(iph + 1);
1565 } else {
1566 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1567
1568 if (iph->nexthdr == IPPROTO_UDP)
1569 uh = (struct udphdr *)(iph + 1);
1570 }
1571 if (uh) {
1572 if (uh->check)
1573 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1574 else
1575 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1576 }
1577 }
1578 #endif
1579
bnxt_gro_func_5731x(struct bnxt_tpa_info * tpa_info,int payload_off,int tcp_ts,struct sk_buff * skb)1580 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1581 int payload_off, int tcp_ts,
1582 struct sk_buff *skb)
1583 {
1584 #ifdef CONFIG_INET
1585 struct tcphdr *th;
1586 int len, nw_off;
1587 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1588 u32 hdr_info = tpa_info->hdr_info;
1589 bool loopback = false;
1590
1591 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1592 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1593 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1594
1595 /* If the packet is an internal loopback packet, the offsets will
1596 * have an extra 4 bytes.
1597 */
1598 if (inner_mac_off == 4) {
1599 loopback = true;
1600 } else if (inner_mac_off > 4) {
1601 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1602 ETH_HLEN - 2));
1603
1604 /* We only support inner iPv4/ipv6. If we don't see the
1605 * correct protocol ID, it must be a loopback packet where
1606 * the offsets are off by 4.
1607 */
1608 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1609 loopback = true;
1610 }
1611 if (loopback) {
1612 /* internal loopback packet, subtract all offsets by 4 */
1613 inner_ip_off -= 4;
1614 inner_mac_off -= 4;
1615 outer_ip_off -= 4;
1616 }
1617
1618 nw_off = inner_ip_off - ETH_HLEN;
1619 skb_set_network_header(skb, nw_off);
1620 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1621 struct ipv6hdr *iph = ipv6_hdr(skb);
1622
1623 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1624 len = skb->len - skb_transport_offset(skb);
1625 th = tcp_hdr(skb);
1626 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1627 } else {
1628 struct iphdr *iph = ip_hdr(skb);
1629
1630 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1631 len = skb->len - skb_transport_offset(skb);
1632 th = tcp_hdr(skb);
1633 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1634 }
1635
1636 if (inner_mac_off) { /* tunnel */
1637 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1638 ETH_HLEN - 2));
1639
1640 bnxt_gro_tunnel(skb, proto);
1641 }
1642 #endif
1643 return skb;
1644 }
1645
bnxt_gro_func_5750x(struct bnxt_tpa_info * tpa_info,int payload_off,int tcp_ts,struct sk_buff * skb)1646 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1647 int payload_off, int tcp_ts,
1648 struct sk_buff *skb)
1649 {
1650 #ifdef CONFIG_INET
1651 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1652 u32 hdr_info = tpa_info->hdr_info;
1653 int iphdr_len, nw_off;
1654
1655 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1656 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1657 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1658
1659 nw_off = inner_ip_off - ETH_HLEN;
1660 skb_set_network_header(skb, nw_off);
1661 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1662 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1663 skb_set_transport_header(skb, nw_off + iphdr_len);
1664
1665 if (inner_mac_off) { /* tunnel */
1666 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1667 ETH_HLEN - 2));
1668
1669 bnxt_gro_tunnel(skb, proto);
1670 }
1671 #endif
1672 return skb;
1673 }
1674
1675 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1676 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1677
bnxt_gro_func_5730x(struct bnxt_tpa_info * tpa_info,int payload_off,int tcp_ts,struct sk_buff * skb)1678 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1679 int payload_off, int tcp_ts,
1680 struct sk_buff *skb)
1681 {
1682 #ifdef CONFIG_INET
1683 struct tcphdr *th;
1684 int len, nw_off, tcp_opt_len = 0;
1685
1686 if (tcp_ts)
1687 tcp_opt_len = 12;
1688
1689 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1690 struct iphdr *iph;
1691
1692 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1693 ETH_HLEN;
1694 skb_set_network_header(skb, nw_off);
1695 iph = ip_hdr(skb);
1696 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1697 len = skb->len - skb_transport_offset(skb);
1698 th = tcp_hdr(skb);
1699 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1700 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1701 struct ipv6hdr *iph;
1702
1703 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1704 ETH_HLEN;
1705 skb_set_network_header(skb, nw_off);
1706 iph = ipv6_hdr(skb);
1707 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1708 len = skb->len - skb_transport_offset(skb);
1709 th = tcp_hdr(skb);
1710 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1711 } else {
1712 dev_kfree_skb_any(skb);
1713 return NULL;
1714 }
1715
1716 if (nw_off) /* tunnel */
1717 bnxt_gro_tunnel(skb, skb->protocol);
1718 #endif
1719 return skb;
1720 }
1721
bnxt_gro_skb(struct bnxt * bp,struct bnxt_tpa_info * tpa_info,struct rx_tpa_end_cmp * tpa_end,struct rx_tpa_end_cmp_ext * tpa_end1,struct sk_buff * skb)1722 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1723 struct bnxt_tpa_info *tpa_info,
1724 struct rx_tpa_end_cmp *tpa_end,
1725 struct rx_tpa_end_cmp_ext *tpa_end1,
1726 struct sk_buff *skb)
1727 {
1728 #ifdef CONFIG_INET
1729 int payload_off;
1730 u16 segs;
1731
1732 segs = TPA_END_TPA_SEGS(tpa_end);
1733 if (segs == 1)
1734 return skb;
1735
1736 NAPI_GRO_CB(skb)->count = segs;
1737 skb_shinfo(skb)->gso_size =
1738 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1739 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1740 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1741 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1742 else
1743 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1744 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1745 if (likely(skb))
1746 tcp_gro_complete(skb);
1747 #endif
1748 return skb;
1749 }
1750
1751 /* Given the cfa_code of a received packet determine which
1752 * netdev (vf-rep or PF) the packet is destined to.
1753 */
bnxt_get_pkt_dev(struct bnxt * bp,u16 cfa_code)1754 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1755 {
1756 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1757
1758 /* if vf-rep dev is NULL, the must belongs to the PF */
1759 return dev ? dev : bp->dev;
1760 }
1761
bnxt_tpa_end(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u32 * raw_cons,struct rx_tpa_end_cmp * tpa_end,struct rx_tpa_end_cmp_ext * tpa_end1,u8 * event)1762 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1763 struct bnxt_cp_ring_info *cpr,
1764 u32 *raw_cons,
1765 struct rx_tpa_end_cmp *tpa_end,
1766 struct rx_tpa_end_cmp_ext *tpa_end1,
1767 u8 *event)
1768 {
1769 struct bnxt_napi *bnapi = cpr->bnapi;
1770 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1771 struct net_device *dev = bp->dev;
1772 u8 *data_ptr, agg_bufs;
1773 unsigned int len;
1774 struct bnxt_tpa_info *tpa_info;
1775 dma_addr_t mapping;
1776 struct sk_buff *skb;
1777 u16 idx = 0, agg_id;
1778 void *data;
1779 bool gro;
1780
1781 if (unlikely(bnapi->in_reset)) {
1782 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1783
1784 if (rc < 0)
1785 return ERR_PTR(-EBUSY);
1786 return NULL;
1787 }
1788
1789 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1790 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1791 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1792 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1793 tpa_info = &rxr->rx_tpa[agg_id];
1794 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1795 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1796 agg_bufs, tpa_info->agg_count);
1797 agg_bufs = tpa_info->agg_count;
1798 }
1799 tpa_info->agg_count = 0;
1800 *event |= BNXT_AGG_EVENT;
1801 bnxt_free_agg_idx(rxr, agg_id);
1802 idx = agg_id;
1803 gro = !!(bp->flags & BNXT_FLAG_GRO);
1804 } else {
1805 agg_id = TPA_END_AGG_ID(tpa_end);
1806 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1807 tpa_info = &rxr->rx_tpa[agg_id];
1808 idx = RING_CMP(*raw_cons);
1809 if (agg_bufs) {
1810 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1811 return ERR_PTR(-EBUSY);
1812
1813 *event |= BNXT_AGG_EVENT;
1814 idx = NEXT_CMP(idx);
1815 }
1816 gro = !!TPA_END_GRO(tpa_end);
1817 }
1818 data = tpa_info->data;
1819 data_ptr = tpa_info->data_ptr;
1820 prefetch(data_ptr);
1821 len = tpa_info->len;
1822 mapping = tpa_info->mapping;
1823
1824 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1825 bnxt_abort_tpa(cpr, idx, agg_bufs);
1826 if (agg_bufs > MAX_SKB_FRAGS)
1827 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1828 agg_bufs, (int)MAX_SKB_FRAGS);
1829 return NULL;
1830 }
1831
1832 if (len <= bp->rx_copy_thresh) {
1833 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1834 if (!skb) {
1835 bnxt_abort_tpa(cpr, idx, agg_bufs);
1836 cpr->sw_stats->rx.rx_oom_discards += 1;
1837 return NULL;
1838 }
1839 } else {
1840 u8 *new_data;
1841 dma_addr_t new_mapping;
1842
1843 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1844 if (!new_data) {
1845 bnxt_abort_tpa(cpr, idx, agg_bufs);
1846 cpr->sw_stats->rx.rx_oom_discards += 1;
1847 return NULL;
1848 }
1849
1850 tpa_info->data = new_data;
1851 tpa_info->data_ptr = new_data + bp->rx_offset;
1852 tpa_info->mapping = new_mapping;
1853
1854 skb = napi_build_skb(data, bp->rx_buf_size);
1855 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1856 bp->rx_buf_use_size, bp->rx_dir,
1857 DMA_ATTR_WEAK_ORDERING);
1858
1859 if (!skb) {
1860 skb_free_frag(data);
1861 bnxt_abort_tpa(cpr, idx, agg_bufs);
1862 cpr->sw_stats->rx.rx_oom_discards += 1;
1863 return NULL;
1864 }
1865 skb_reserve(skb, bp->rx_offset);
1866 skb_put(skb, len);
1867 }
1868
1869 if (agg_bufs) {
1870 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1871 if (!skb) {
1872 /* Page reuse already handled by bnxt_rx_pages(). */
1873 cpr->sw_stats->rx.rx_oom_discards += 1;
1874 return NULL;
1875 }
1876 }
1877
1878 if (tpa_info->cfa_code_valid)
1879 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code);
1880 skb->protocol = eth_type_trans(skb, dev);
1881
1882 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1883 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1884
1885 if (tpa_info->vlan_valid &&
1886 (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1887 __be16 vlan_proto = htons(tpa_info->metadata >>
1888 RX_CMP_FLAGS2_METADATA_TPID_SFT);
1889 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1890
1891 if (eth_type_vlan(vlan_proto)) {
1892 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1893 } else {
1894 dev_kfree_skb(skb);
1895 return NULL;
1896 }
1897 }
1898
1899 skb_checksum_none_assert(skb);
1900 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1901 skb->ip_summed = CHECKSUM_UNNECESSARY;
1902 skb->csum_level =
1903 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1904 }
1905
1906 if (gro)
1907 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1908
1909 return skb;
1910 }
1911
bnxt_tpa_agg(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,struct rx_agg_cmp * rx_agg)1912 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1913 struct rx_agg_cmp *rx_agg)
1914 {
1915 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1916 struct bnxt_tpa_info *tpa_info;
1917
1918 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1919 tpa_info = &rxr->rx_tpa[agg_id];
1920 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1921 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1922 }
1923
bnxt_deliver_skb(struct bnxt * bp,struct bnxt_napi * bnapi,struct sk_buff * skb)1924 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1925 struct sk_buff *skb)
1926 {
1927 skb_mark_for_recycle(skb);
1928
1929 if (skb->dev != bp->dev) {
1930 /* this packet belongs to a vf-rep */
1931 bnxt_vf_rep_rx(bp, skb);
1932 return;
1933 }
1934 skb_record_rx_queue(skb, bnapi->index);
1935 napi_gro_receive(&bnapi->napi, skb);
1936 }
1937
bnxt_rx_ts_valid(struct bnxt * bp,u32 flags,struct rx_cmp_ext * rxcmp1,u32 * cmpl_ts)1938 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1939 struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1940 {
1941 u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1942
1943 if (BNXT_PTP_RX_TS_VALID(flags))
1944 goto ts_valid;
1945 if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1946 return false;
1947
1948 ts_valid:
1949 *cmpl_ts = ts;
1950 return true;
1951 }
1952
bnxt_rx_vlan(struct sk_buff * skb,u8 cmp_type,struct rx_cmp * rxcmp,struct rx_cmp_ext * rxcmp1)1953 static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type,
1954 struct rx_cmp *rxcmp,
1955 struct rx_cmp_ext *rxcmp1)
1956 {
1957 __be16 vlan_proto;
1958 u16 vtag;
1959
1960 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1961 __le32 flags2 = rxcmp1->rx_cmp_flags2;
1962 u32 meta_data;
1963
1964 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
1965 return skb;
1966
1967 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1968 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1969 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
1970 if (eth_type_vlan(vlan_proto))
1971 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1972 else
1973 goto vlan_err;
1974 } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1975 if (RX_CMP_VLAN_VALID(rxcmp)) {
1976 u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
1977
1978 if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
1979 vlan_proto = htons(ETH_P_8021Q);
1980 else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
1981 vlan_proto = htons(ETH_P_8021AD);
1982 else
1983 goto vlan_err;
1984 vtag = RX_CMP_METADATA0_TCI(rxcmp1);
1985 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1986 }
1987 }
1988 return skb;
1989 vlan_err:
1990 dev_kfree_skb(skb);
1991 return NULL;
1992 }
1993
bnxt_rss_ext_op(struct bnxt * bp,struct rx_cmp * rxcmp)1994 static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp,
1995 struct rx_cmp *rxcmp)
1996 {
1997 u8 ext_op;
1998
1999 ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp);
2000 switch (ext_op) {
2001 case EXT_OP_INNER_4:
2002 case EXT_OP_OUTER_4:
2003 case EXT_OP_INNFL_3:
2004 case EXT_OP_OUTFL_3:
2005 return PKT_HASH_TYPE_L4;
2006 default:
2007 return PKT_HASH_TYPE_L3;
2008 }
2009 }
2010
2011 /* returns the following:
2012 * 1 - 1 packet successfully received
2013 * 0 - successful TPA_START, packet not completed yet
2014 * -EBUSY - completion ring does not have all the agg buffers yet
2015 * -ENOMEM - packet aborted due to out of memory
2016 * -EIO - packet aborted due to hw error indicated in BD
2017 */
bnxt_rx_pkt(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u32 * raw_cons,u8 * event)2018 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2019 u32 *raw_cons, u8 *event)
2020 {
2021 struct bnxt_napi *bnapi = cpr->bnapi;
2022 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2023 struct net_device *dev = bp->dev;
2024 struct rx_cmp *rxcmp;
2025 struct rx_cmp_ext *rxcmp1;
2026 u32 tmp_raw_cons = *raw_cons;
2027 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
2028 struct bnxt_sw_rx_bd *rx_buf;
2029 unsigned int len;
2030 u8 *data_ptr, agg_bufs, cmp_type;
2031 bool xdp_active = false;
2032 dma_addr_t dma_addr;
2033 struct sk_buff *skb;
2034 struct xdp_buff xdp;
2035 u32 flags, misc;
2036 u32 cmpl_ts;
2037 void *data;
2038 int rc = 0;
2039
2040 rxcmp = (struct rx_cmp *)
2041 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2042
2043 cmp_type = RX_CMP_TYPE(rxcmp);
2044
2045 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
2046 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
2047 goto next_rx_no_prod_no_len;
2048 }
2049
2050 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2051 cp_cons = RING_CMP(tmp_raw_cons);
2052 rxcmp1 = (struct rx_cmp_ext *)
2053 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2054
2055 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2056 return -EBUSY;
2057
2058 /* The valid test of the entry must be done first before
2059 * reading any further.
2060 */
2061 dma_rmb();
2062 prod = rxr->rx_prod;
2063
2064 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
2065 cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2066 bnxt_tpa_start(bp, rxr, cmp_type,
2067 (struct rx_tpa_start_cmp *)rxcmp,
2068 (struct rx_tpa_start_cmp_ext *)rxcmp1);
2069
2070 *event |= BNXT_RX_EVENT;
2071 goto next_rx_no_prod_no_len;
2072
2073 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2074 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
2075 (struct rx_tpa_end_cmp *)rxcmp,
2076 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
2077
2078 if (IS_ERR(skb))
2079 return -EBUSY;
2080
2081 rc = -ENOMEM;
2082 if (likely(skb)) {
2083 bnxt_deliver_skb(bp, bnapi, skb);
2084 rc = 1;
2085 }
2086 *event |= BNXT_RX_EVENT;
2087 goto next_rx_no_prod_no_len;
2088 }
2089
2090 cons = rxcmp->rx_cmp_opaque;
2091 if (unlikely(cons != rxr->rx_next_cons)) {
2092 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
2093
2094 /* 0xffff is forced error, don't print it */
2095 if (rxr->rx_next_cons != 0xffff)
2096 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
2097 cons, rxr->rx_next_cons);
2098 bnxt_sched_reset_rxr(bp, rxr);
2099 if (rc1)
2100 return rc1;
2101 goto next_rx_no_prod_no_len;
2102 }
2103 rx_buf = &rxr->rx_buf_ring[cons];
2104 data = rx_buf->data;
2105 data_ptr = rx_buf->data_ptr;
2106 prefetch(data_ptr);
2107
2108 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
2109 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
2110
2111 if (agg_bufs) {
2112 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
2113 return -EBUSY;
2114
2115 cp_cons = NEXT_CMP(cp_cons);
2116 *event |= BNXT_AGG_EVENT;
2117 }
2118 *event |= BNXT_RX_EVENT;
2119
2120 rx_buf->data = NULL;
2121 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
2122 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
2123
2124 bnxt_reuse_rx_data(rxr, cons, data);
2125 if (agg_bufs)
2126 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
2127 false);
2128
2129 rc = -EIO;
2130 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
2131 bnapi->cp_ring.sw_stats->rx.rx_buf_errors++;
2132 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
2133 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
2134 netdev_warn_once(bp->dev, "RX buffer error %x\n",
2135 rx_err);
2136 bnxt_sched_reset_rxr(bp, rxr);
2137 }
2138 }
2139 goto next_rx_no_len;
2140 }
2141
2142 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
2143 len = flags >> RX_CMP_LEN_SHIFT;
2144 dma_addr = rx_buf->mapping;
2145
2146 if (bnxt_xdp_attached(bp, rxr)) {
2147 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
2148 if (agg_bufs) {
2149 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
2150 cp_cons, agg_bufs,
2151 false);
2152 if (!frag_len)
2153 goto oom_next_rx;
2154 }
2155 xdp_active = true;
2156 }
2157
2158 if (xdp_active) {
2159 if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) {
2160 rc = 1;
2161 goto next_rx;
2162 }
2163 }
2164
2165 if (len <= bp->rx_copy_thresh) {
2166 if (!xdp_active)
2167 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
2168 else
2169 skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr);
2170 bnxt_reuse_rx_data(rxr, cons, data);
2171 if (!skb) {
2172 if (agg_bufs) {
2173 if (!xdp_active)
2174 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
2175 agg_bufs, false);
2176 else
2177 bnxt_xdp_buff_frags_free(rxr, &xdp);
2178 }
2179 goto oom_next_rx;
2180 }
2181 } else {
2182 u32 payload;
2183
2184 if (rx_buf->data_ptr == data_ptr)
2185 payload = misc & RX_CMP_PAYLOAD_OFFSET;
2186 else
2187 payload = 0;
2188 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
2189 payload | len);
2190 if (!skb)
2191 goto oom_next_rx;
2192 }
2193
2194 if (agg_bufs) {
2195 if (!xdp_active) {
2196 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
2197 if (!skb)
2198 goto oom_next_rx;
2199 } else {
2200 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
2201 if (!skb) {
2202 /* we should be able to free the old skb here */
2203 bnxt_xdp_buff_frags_free(rxr, &xdp);
2204 goto oom_next_rx;
2205 }
2206 }
2207 }
2208
2209 if (RX_CMP_HASH_VALID(rxcmp)) {
2210 enum pkt_hash_types type;
2211
2212 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2213 type = bnxt_rss_ext_op(bp, rxcmp);
2214 } else {
2215 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
2216
2217 /* RSS profiles 1 and 3 with extract code 0 for inner
2218 * 4-tuple
2219 */
2220 if (hash_type != 1 && hash_type != 3)
2221 type = PKT_HASH_TYPE_L3;
2222 else
2223 type = PKT_HASH_TYPE_L4;
2224 }
2225 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2226 }
2227
2228 if (cmp_type == CMP_TYPE_RX_L2_CMP)
2229 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1));
2230 skb->protocol = eth_type_trans(skb, dev);
2231
2232 if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) {
2233 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
2234 if (!skb)
2235 goto next_rx;
2236 }
2237
2238 skb_checksum_none_assert(skb);
2239 if (RX_CMP_L4_CS_OK(rxcmp1)) {
2240 if (dev->features & NETIF_F_RXCSUM) {
2241 skb->ip_summed = CHECKSUM_UNNECESSARY;
2242 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2243 }
2244 } else {
2245 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2246 if (dev->features & NETIF_F_RXCSUM)
2247 bnapi->cp_ring.sw_stats->rx.rx_l4_csum_errors++;
2248 }
2249 }
2250
2251 if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2252 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
2253 u64 ns, ts;
2254
2255 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2256 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2257 unsigned long flags;
2258
2259 spin_lock_irqsave(&ptp->ptp_lock, flags);
2260 ns = timecounter_cyc2time(&ptp->tc, ts);
2261 spin_unlock_irqrestore(&ptp->ptp_lock, flags);
2262 memset(skb_hwtstamps(skb), 0,
2263 sizeof(*skb_hwtstamps(skb)));
2264 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2265 }
2266 }
2267 }
2268 bnxt_deliver_skb(bp, bnapi, skb);
2269 rc = 1;
2270
2271 next_rx:
2272 cpr->rx_packets += 1;
2273 cpr->rx_bytes += len;
2274
2275 next_rx_no_len:
2276 rxr->rx_prod = NEXT_RX(prod);
2277 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
2278
2279 next_rx_no_prod_no_len:
2280 *raw_cons = tmp_raw_cons;
2281
2282 return rc;
2283
2284 oom_next_rx:
2285 cpr->sw_stats->rx.rx_oom_discards += 1;
2286 rc = -ENOMEM;
2287 goto next_rx;
2288 }
2289
2290 /* In netpoll mode, if we are using a combined completion ring, we need to
2291 * discard the rx packets and recycle the buffers.
2292 */
bnxt_force_rx_discard(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,u32 * raw_cons,u8 * event)2293 static int bnxt_force_rx_discard(struct bnxt *bp,
2294 struct bnxt_cp_ring_info *cpr,
2295 u32 *raw_cons, u8 *event)
2296 {
2297 u32 tmp_raw_cons = *raw_cons;
2298 struct rx_cmp_ext *rxcmp1;
2299 struct rx_cmp *rxcmp;
2300 u16 cp_cons;
2301 u8 cmp_type;
2302 int rc;
2303
2304 cp_cons = RING_CMP(tmp_raw_cons);
2305 rxcmp = (struct rx_cmp *)
2306 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2307
2308 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2309 cp_cons = RING_CMP(tmp_raw_cons);
2310 rxcmp1 = (struct rx_cmp_ext *)
2311 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2312
2313 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2314 return -EBUSY;
2315
2316 /* The valid test of the entry must be done first before
2317 * reading any further.
2318 */
2319 dma_rmb();
2320 cmp_type = RX_CMP_TYPE(rxcmp);
2321 if (cmp_type == CMP_TYPE_RX_L2_CMP ||
2322 cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2323 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2324 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2325 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2326 struct rx_tpa_end_cmp_ext *tpa_end1;
2327
2328 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2329 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2330 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2331 }
2332 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2333 if (rc && rc != -EBUSY)
2334 cpr->sw_stats->rx.rx_netpoll_discards += 1;
2335 return rc;
2336 }
2337
bnxt_fw_health_readl(struct bnxt * bp,int reg_idx)2338 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2339 {
2340 struct bnxt_fw_health *fw_health = bp->fw_health;
2341 u32 reg = fw_health->regs[reg_idx];
2342 u32 reg_type, reg_off, val = 0;
2343
2344 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2345 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2346 switch (reg_type) {
2347 case BNXT_FW_HEALTH_REG_TYPE_CFG:
2348 pci_read_config_dword(bp->pdev, reg_off, &val);
2349 break;
2350 case BNXT_FW_HEALTH_REG_TYPE_GRC:
2351 reg_off = fw_health->mapped_regs[reg_idx];
2352 fallthrough;
2353 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2354 val = readl(bp->bar0 + reg_off);
2355 break;
2356 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2357 val = readl(bp->bar1 + reg_off);
2358 break;
2359 }
2360 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2361 val &= fw_health->fw_reset_inprog_reg_mask;
2362 return val;
2363 }
2364
bnxt_agg_ring_id_to_grp_idx(struct bnxt * bp,u16 ring_id)2365 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2366 {
2367 int i;
2368
2369 for (i = 0; i < bp->rx_nr_rings; i++) {
2370 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2371 struct bnxt_ring_grp_info *grp_info;
2372
2373 grp_info = &bp->grp_info[grp_idx];
2374 if (grp_info->agg_fw_ring_id == ring_id)
2375 return grp_idx;
2376 }
2377 return INVALID_HW_RING_ID;
2378 }
2379
bnxt_get_force_speed(struct bnxt_link_info * link_info)2380 static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info)
2381 {
2382 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2383
2384 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
2385 return link_info->force_link_speed2;
2386 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4)
2387 return link_info->force_pam4_link_speed;
2388 return link_info->force_link_speed;
2389 }
2390
bnxt_set_force_speed(struct bnxt_link_info * link_info)2391 static void bnxt_set_force_speed(struct bnxt_link_info *link_info)
2392 {
2393 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2394
2395 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2396 link_info->req_link_speed = link_info->force_link_speed2;
2397 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2398 switch (link_info->req_link_speed) {
2399 case BNXT_LINK_SPEED_50GB_PAM4:
2400 case BNXT_LINK_SPEED_100GB_PAM4:
2401 case BNXT_LINK_SPEED_200GB_PAM4:
2402 case BNXT_LINK_SPEED_400GB_PAM4:
2403 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2404 break;
2405 case BNXT_LINK_SPEED_100GB_PAM4_112:
2406 case BNXT_LINK_SPEED_200GB_PAM4_112:
2407 case BNXT_LINK_SPEED_400GB_PAM4_112:
2408 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112;
2409 break;
2410 default:
2411 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2412 }
2413 return;
2414 }
2415 link_info->req_link_speed = link_info->force_link_speed;
2416 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2417 if (link_info->force_pam4_link_speed) {
2418 link_info->req_link_speed = link_info->force_pam4_link_speed;
2419 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2420 }
2421 }
2422
bnxt_set_auto_speed(struct bnxt_link_info * link_info)2423 static void bnxt_set_auto_speed(struct bnxt_link_info *link_info)
2424 {
2425 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2426
2427 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2428 link_info->advertising = link_info->auto_link_speeds2;
2429 return;
2430 }
2431 link_info->advertising = link_info->auto_link_speeds;
2432 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
2433 }
2434
bnxt_force_speed_updated(struct bnxt_link_info * link_info)2435 static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info)
2436 {
2437 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2438
2439 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2440 if (link_info->req_link_speed != link_info->force_link_speed2)
2441 return true;
2442 return false;
2443 }
2444 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
2445 link_info->req_link_speed != link_info->force_link_speed)
2446 return true;
2447 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
2448 link_info->req_link_speed != link_info->force_pam4_link_speed)
2449 return true;
2450 return false;
2451 }
2452
bnxt_auto_speed_updated(struct bnxt_link_info * link_info)2453 static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info)
2454 {
2455 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2456
2457 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2458 if (link_info->advertising != link_info->auto_link_speeds2)
2459 return true;
2460 return false;
2461 }
2462 if (link_info->advertising != link_info->auto_link_speeds ||
2463 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
2464 return true;
2465 return false;
2466 }
2467
2468 #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \
2469 ((data2) & \
2470 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK)
2471
2472 #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \
2473 (((data2) & \
2474 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\
2475 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT)
2476
2477 #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \
2478 ((data1) & \
2479 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK)
2480
2481 #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \
2482 (((data1) & \
2483 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\
2484 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING)
2485
2486 /* Return true if the workqueue has to be scheduled */
bnxt_event_error_report(struct bnxt * bp,u32 data1,u32 data2)2487 static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2488 {
2489 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2490
2491 switch (err_type) {
2492 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2493 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2494 BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2495 break;
2496 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2497 netdev_warn(bp->dev, "Pause Storm detected!\n");
2498 break;
2499 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2500 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2501 break;
2502 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: {
2503 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1);
2504 char *threshold_type;
2505 bool notify = false;
2506 char *dir_str;
2507
2508 switch (type) {
2509 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN:
2510 threshold_type = "warning";
2511 break;
2512 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL:
2513 threshold_type = "critical";
2514 break;
2515 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL:
2516 threshold_type = "fatal";
2517 break;
2518 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN:
2519 threshold_type = "shutdown";
2520 break;
2521 default:
2522 netdev_err(bp->dev, "Unknown Thermal threshold type event\n");
2523 return false;
2524 }
2525 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) {
2526 dir_str = "above";
2527 notify = true;
2528 } else {
2529 dir_str = "below";
2530 }
2531 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n",
2532 dir_str, threshold_type);
2533 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n",
2534 BNXT_EVENT_THERMAL_CURRENT_TEMP(data2),
2535 BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2));
2536 if (notify) {
2537 bp->thermal_threshold_type = type;
2538 set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event);
2539 return true;
2540 }
2541 return false;
2542 }
2543 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED:
2544 netdev_warn(bp->dev, "Speed change not supported with dual rate transceivers on this board\n");
2545 break;
2546 default:
2547 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2548 err_type);
2549 break;
2550 }
2551 return false;
2552 }
2553
2554 #define BNXT_GET_EVENT_PORT(data) \
2555 ((data) & \
2556 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2557
2558 #define BNXT_EVENT_RING_TYPE(data2) \
2559 ((data2) & \
2560 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2561
2562 #define BNXT_EVENT_RING_TYPE_RX(data2) \
2563 (BNXT_EVENT_RING_TYPE(data2) == \
2564 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2565
2566 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \
2567 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2568 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2569
2570 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \
2571 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2572 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2573
2574 #define BNXT_PHC_BITS 48
2575
bnxt_async_event_process(struct bnxt * bp,struct hwrm_async_event_cmpl * cmpl)2576 static int bnxt_async_event_process(struct bnxt *bp,
2577 struct hwrm_async_event_cmpl *cmpl)
2578 {
2579 u16 event_id = le16_to_cpu(cmpl->event_id);
2580 u32 data1 = le32_to_cpu(cmpl->event_data1);
2581 u32 data2 = le32_to_cpu(cmpl->event_data2);
2582
2583 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2584 event_id, data1, data2);
2585
2586 /* TODO CHIMP_FW: Define event id's for link change, error etc */
2587 switch (event_id) {
2588 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2589 struct bnxt_link_info *link_info = &bp->link_info;
2590
2591 if (BNXT_VF(bp))
2592 goto async_event_process_exit;
2593
2594 /* print unsupported speed warning in forced speed mode only */
2595 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2596 (data1 & 0x20000)) {
2597 u16 fw_speed = bnxt_get_force_speed(link_info);
2598 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2599
2600 if (speed != SPEED_UNKNOWN)
2601 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2602 speed);
2603 }
2604 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2605 }
2606 fallthrough;
2607 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2608 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2609 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2610 fallthrough;
2611 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2612 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2613 break;
2614 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2615 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2616 break;
2617 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2618 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2619
2620 if (BNXT_VF(bp))
2621 break;
2622
2623 if (bp->pf.port_id != port_id)
2624 break;
2625
2626 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2627 break;
2628 }
2629 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2630 if (BNXT_PF(bp))
2631 goto async_event_process_exit;
2632 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2633 break;
2634 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2635 char *type_str = "Solicited";
2636
2637 if (!bp->fw_health)
2638 goto async_event_process_exit;
2639
2640 bp->fw_reset_timestamp = jiffies;
2641 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2642 if (!bp->fw_reset_min_dsecs)
2643 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2644 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2645 if (!bp->fw_reset_max_dsecs)
2646 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2647 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2648 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2649 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2650 type_str = "Fatal";
2651 bp->fw_health->fatalities++;
2652 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2653 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2654 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2655 type_str = "Non-fatal";
2656 bp->fw_health->survivals++;
2657 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2658 }
2659 netif_warn(bp, hw, bp->dev,
2660 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2661 type_str, data1, data2,
2662 bp->fw_reset_min_dsecs * 100,
2663 bp->fw_reset_max_dsecs * 100);
2664 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2665 break;
2666 }
2667 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2668 struct bnxt_fw_health *fw_health = bp->fw_health;
2669 char *status_desc = "healthy";
2670 u32 status;
2671
2672 if (!fw_health)
2673 goto async_event_process_exit;
2674
2675 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2676 fw_health->enabled = false;
2677 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2678 break;
2679 }
2680 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2681 fw_health->tmr_multiplier =
2682 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2683 bp->current_interval * 10);
2684 fw_health->tmr_counter = fw_health->tmr_multiplier;
2685 if (!fw_health->enabled)
2686 fw_health->last_fw_heartbeat =
2687 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2688 fw_health->last_fw_reset_cnt =
2689 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2690 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2691 if (status != BNXT_FW_STATUS_HEALTHY)
2692 status_desc = "unhealthy";
2693 netif_info(bp, drv, bp->dev,
2694 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2695 fw_health->primary ? "primary" : "backup", status,
2696 status_desc, fw_health->last_fw_reset_cnt);
2697 if (!fw_health->enabled) {
2698 /* Make sure tmr_counter is set and visible to
2699 * bnxt_health_check() before setting enabled to true.
2700 */
2701 smp_wmb();
2702 fw_health->enabled = true;
2703 }
2704 goto async_event_process_exit;
2705 }
2706 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2707 netif_notice(bp, hw, bp->dev,
2708 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2709 data1, data2);
2710 goto async_event_process_exit;
2711 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2712 struct bnxt_rx_ring_info *rxr;
2713 u16 grp_idx;
2714
2715 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
2716 goto async_event_process_exit;
2717
2718 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2719 BNXT_EVENT_RING_TYPE(data2), data1);
2720 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2721 goto async_event_process_exit;
2722
2723 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2724 if (grp_idx == INVALID_HW_RING_ID) {
2725 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2726 data1);
2727 goto async_event_process_exit;
2728 }
2729 rxr = bp->bnapi[grp_idx]->rx_ring;
2730 bnxt_sched_reset_rxr(bp, rxr);
2731 goto async_event_process_exit;
2732 }
2733 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2734 struct bnxt_fw_health *fw_health = bp->fw_health;
2735
2736 netif_notice(bp, hw, bp->dev,
2737 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2738 data1, data2);
2739 if (fw_health) {
2740 fw_health->echo_req_data1 = data1;
2741 fw_health->echo_req_data2 = data2;
2742 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2743 break;
2744 }
2745 goto async_event_process_exit;
2746 }
2747 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2748 bnxt_ptp_pps_event(bp, data1, data2);
2749 goto async_event_process_exit;
2750 }
2751 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2752 if (bnxt_event_error_report(bp, data1, data2))
2753 break;
2754 goto async_event_process_exit;
2755 }
2756 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2757 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2758 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2759 if (BNXT_PTP_USE_RTC(bp)) {
2760 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2761 unsigned long flags;
2762 u64 ns;
2763
2764 if (!ptp)
2765 goto async_event_process_exit;
2766
2767 spin_lock_irqsave(&ptp->ptp_lock, flags);
2768 bnxt_ptp_update_current_time(bp);
2769 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2770 BNXT_PHC_BITS) | ptp->current_time);
2771 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2772 spin_unlock_irqrestore(&ptp->ptp_lock, flags);
2773 }
2774 break;
2775 }
2776 goto async_event_process_exit;
2777 }
2778 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2779 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2780
2781 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2782 goto async_event_process_exit;
2783 }
2784 default:
2785 goto async_event_process_exit;
2786 }
2787 __bnxt_queue_sp_work(bp);
2788 async_event_process_exit:
2789 return 0;
2790 }
2791
bnxt_hwrm_handler(struct bnxt * bp,struct tx_cmp * txcmp)2792 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2793 {
2794 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2795 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2796 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2797 (struct hwrm_fwd_req_cmpl *)txcmp;
2798
2799 switch (cmpl_type) {
2800 case CMPL_BASE_TYPE_HWRM_DONE:
2801 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2802 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2803 break;
2804
2805 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2806 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2807
2808 if ((vf_id < bp->pf.first_vf_id) ||
2809 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2810 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2811 vf_id);
2812 return -EINVAL;
2813 }
2814
2815 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2816 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2817 break;
2818
2819 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2820 bnxt_async_event_process(bp,
2821 (struct hwrm_async_event_cmpl *)txcmp);
2822 break;
2823
2824 default:
2825 break;
2826 }
2827
2828 return 0;
2829 }
2830
bnxt_msix(int irq,void * dev_instance)2831 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2832 {
2833 struct bnxt_napi *bnapi = dev_instance;
2834 struct bnxt *bp = bnapi->bp;
2835 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2836 u32 cons = RING_CMP(cpr->cp_raw_cons);
2837
2838 cpr->event_ctr++;
2839 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2840 napi_schedule(&bnapi->napi);
2841 return IRQ_HANDLED;
2842 }
2843
bnxt_has_work(struct bnxt * bp,struct bnxt_cp_ring_info * cpr)2844 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2845 {
2846 u32 raw_cons = cpr->cp_raw_cons;
2847 u16 cons = RING_CMP(raw_cons);
2848 struct tx_cmp *txcmp;
2849
2850 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2851
2852 return TX_CMP_VALID(txcmp, raw_cons);
2853 }
2854
__bnxt_poll_work(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,int budget)2855 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2856 int budget)
2857 {
2858 struct bnxt_napi *bnapi = cpr->bnapi;
2859 u32 raw_cons = cpr->cp_raw_cons;
2860 u32 cons;
2861 int rx_pkts = 0;
2862 u8 event = 0;
2863 struct tx_cmp *txcmp;
2864
2865 cpr->has_more_work = 0;
2866 cpr->had_work_done = 1;
2867 while (1) {
2868 u8 cmp_type;
2869 int rc;
2870
2871 cons = RING_CMP(raw_cons);
2872 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2873
2874 if (!TX_CMP_VALID(txcmp, raw_cons))
2875 break;
2876
2877 /* The valid test of the entry must be done first before
2878 * reading any further.
2879 */
2880 dma_rmb();
2881 cmp_type = TX_CMP_TYPE(txcmp);
2882 if (cmp_type == CMP_TYPE_TX_L2_CMP ||
2883 cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
2884 u32 opaque = txcmp->tx_cmp_opaque;
2885 struct bnxt_tx_ring_info *txr;
2886 u16 tx_freed;
2887
2888 txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
2889 event |= BNXT_TX_CMP_EVENT;
2890 if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
2891 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
2892 else
2893 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
2894 tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
2895 bp->tx_ring_mask;
2896 /* return full budget so NAPI will complete. */
2897 if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
2898 rx_pkts = budget;
2899 raw_cons = NEXT_RAW_CMP(raw_cons);
2900 if (budget)
2901 cpr->has_more_work = 1;
2902 break;
2903 }
2904 } else if (cmp_type == CMP_TYPE_TX_L2_PKT_TS_CMP) {
2905 bnxt_tx_ts_cmp(bp, bnapi, (struct tx_ts_cmp *)txcmp);
2906 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
2907 cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2908 if (likely(budget))
2909 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2910 else
2911 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2912 &event);
2913 if (likely(rc >= 0))
2914 rx_pkts += rc;
2915 /* Increment rx_pkts when rc is -ENOMEM to count towards
2916 * the NAPI budget. Otherwise, we may potentially loop
2917 * here forever if we consistently cannot allocate
2918 * buffers.
2919 */
2920 else if (rc == -ENOMEM && budget)
2921 rx_pkts++;
2922 else if (rc == -EBUSY) /* partial completion */
2923 break;
2924 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
2925 cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
2926 cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) {
2927 bnxt_hwrm_handler(bp, txcmp);
2928 }
2929 raw_cons = NEXT_RAW_CMP(raw_cons);
2930
2931 if (rx_pkts && rx_pkts == budget) {
2932 cpr->has_more_work = 1;
2933 break;
2934 }
2935 }
2936
2937 if (event & BNXT_REDIRECT_EVENT) {
2938 xdp_do_flush();
2939 event &= ~BNXT_REDIRECT_EVENT;
2940 }
2941
2942 if (event & BNXT_TX_EVENT) {
2943 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
2944 u16 prod = txr->tx_prod;
2945
2946 /* Sync BD data before updating doorbell */
2947 wmb();
2948
2949 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2950 event &= ~BNXT_TX_EVENT;
2951 }
2952
2953 cpr->cp_raw_cons = raw_cons;
2954 bnapi->events |= event;
2955 return rx_pkts;
2956 }
2957
__bnxt_poll_work_done(struct bnxt * bp,struct bnxt_napi * bnapi,int budget)2958 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2959 int budget)
2960 {
2961 if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
2962 bnapi->tx_int(bp, bnapi, budget);
2963
2964 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2965 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2966
2967 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2968 bnapi->events &= ~BNXT_RX_EVENT;
2969 }
2970 if (bnapi->events & BNXT_AGG_EVENT) {
2971 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2972
2973 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2974 bnapi->events &= ~BNXT_AGG_EVENT;
2975 }
2976 }
2977
bnxt_poll_work(struct bnxt * bp,struct bnxt_cp_ring_info * cpr,int budget)2978 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2979 int budget)
2980 {
2981 struct bnxt_napi *bnapi = cpr->bnapi;
2982 int rx_pkts;
2983
2984 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2985
2986 /* ACK completion ring before freeing tx ring and producing new
2987 * buffers in rx/agg rings to prevent overflowing the completion
2988 * ring.
2989 */
2990 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2991
2992 __bnxt_poll_work_done(bp, bnapi, budget);
2993 return rx_pkts;
2994 }
2995
bnxt_poll_nitroa0(struct napi_struct * napi,int budget)2996 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2997 {
2998 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2999 struct bnxt *bp = bnapi->bp;
3000 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3001 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
3002 struct tx_cmp *txcmp;
3003 struct rx_cmp_ext *rxcmp1;
3004 u32 cp_cons, tmp_raw_cons;
3005 u32 raw_cons = cpr->cp_raw_cons;
3006 bool flush_xdp = false;
3007 u32 rx_pkts = 0;
3008 u8 event = 0;
3009
3010 while (1) {
3011 int rc;
3012
3013 cp_cons = RING_CMP(raw_cons);
3014 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3015
3016 if (!TX_CMP_VALID(txcmp, raw_cons))
3017 break;
3018
3019 /* The valid test of the entry must be done first before
3020 * reading any further.
3021 */
3022 dma_rmb();
3023 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
3024 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
3025 cp_cons = RING_CMP(tmp_raw_cons);
3026 rxcmp1 = (struct rx_cmp_ext *)
3027 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3028
3029 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
3030 break;
3031
3032 /* force an error to recycle the buffer */
3033 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
3034 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
3035
3036 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
3037 if (likely(rc == -EIO) && budget)
3038 rx_pkts++;
3039 else if (rc == -EBUSY) /* partial completion */
3040 break;
3041 if (event & BNXT_REDIRECT_EVENT)
3042 flush_xdp = true;
3043 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
3044 CMPL_BASE_TYPE_HWRM_DONE)) {
3045 bnxt_hwrm_handler(bp, txcmp);
3046 } else {
3047 netdev_err(bp->dev,
3048 "Invalid completion received on special ring\n");
3049 }
3050 raw_cons = NEXT_RAW_CMP(raw_cons);
3051
3052 if (rx_pkts == budget)
3053 break;
3054 }
3055
3056 cpr->cp_raw_cons = raw_cons;
3057 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
3058 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
3059
3060 if (event & BNXT_AGG_EVENT)
3061 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
3062 if (flush_xdp)
3063 xdp_do_flush();
3064
3065 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
3066 napi_complete_done(napi, rx_pkts);
3067 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3068 }
3069 return rx_pkts;
3070 }
3071
bnxt_poll(struct napi_struct * napi,int budget)3072 static int bnxt_poll(struct napi_struct *napi, int budget)
3073 {
3074 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3075 struct bnxt *bp = bnapi->bp;
3076 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3077 int work_done = 0;
3078
3079 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3080 napi_complete(napi);
3081 return 0;
3082 }
3083 while (1) {
3084 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
3085
3086 if (work_done >= budget) {
3087 if (!budget)
3088 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3089 break;
3090 }
3091
3092 if (!bnxt_has_work(bp, cpr)) {
3093 if (napi_complete_done(napi, work_done))
3094 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3095 break;
3096 }
3097 }
3098 if (bp->flags & BNXT_FLAG_DIM) {
3099 struct dim_sample dim_sample = {};
3100
3101 dim_update_sample(cpr->event_ctr,
3102 cpr->rx_packets,
3103 cpr->rx_bytes,
3104 &dim_sample);
3105 net_dim(&cpr->dim, dim_sample);
3106 }
3107 return work_done;
3108 }
3109
__bnxt_poll_cqs(struct bnxt * bp,struct bnxt_napi * bnapi,int budget)3110 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
3111 {
3112 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3113 int i, work_done = 0;
3114
3115 for (i = 0; i < cpr->cp_ring_count; i++) {
3116 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3117
3118 if (cpr2->had_nqe_notify) {
3119 work_done += __bnxt_poll_work(bp, cpr2,
3120 budget - work_done);
3121 cpr->has_more_work |= cpr2->has_more_work;
3122 }
3123 }
3124 return work_done;
3125 }
3126
__bnxt_poll_cqs_done(struct bnxt * bp,struct bnxt_napi * bnapi,u64 dbr_type,int budget)3127 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3128 u64 dbr_type, int budget)
3129 {
3130 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3131 int i;
3132
3133 for (i = 0; i < cpr->cp_ring_count; i++) {
3134 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3135 struct bnxt_db_info *db;
3136
3137 if (cpr2->had_work_done) {
3138 u32 tgl = 0;
3139
3140 if (dbr_type == DBR_TYPE_CQ_ARMALL) {
3141 cpr2->had_nqe_notify = 0;
3142 tgl = cpr2->toggle;
3143 }
3144 db = &cpr2->cp_db;
3145 bnxt_writeq(bp,
3146 db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
3147 DB_RING_IDX(db, cpr2->cp_raw_cons),
3148 db->doorbell);
3149 cpr2->had_work_done = 0;
3150 }
3151 }
3152 __bnxt_poll_work_done(bp, bnapi, budget);
3153 }
3154
bnxt_poll_p5(struct napi_struct * napi,int budget)3155 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
3156 {
3157 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3158 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3159 struct bnxt_cp_ring_info *cpr_rx;
3160 u32 raw_cons = cpr->cp_raw_cons;
3161 struct bnxt *bp = bnapi->bp;
3162 struct nqe_cn *nqcmp;
3163 int work_done = 0;
3164 u32 cons;
3165
3166 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3167 napi_complete(napi);
3168 return 0;
3169 }
3170 if (cpr->has_more_work) {
3171 cpr->has_more_work = 0;
3172 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
3173 }
3174 while (1) {
3175 u16 type;
3176
3177 cons = RING_CMP(raw_cons);
3178 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
3179
3180 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
3181 if (cpr->has_more_work)
3182 break;
3183
3184 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
3185 budget);
3186 cpr->cp_raw_cons = raw_cons;
3187 if (napi_complete_done(napi, work_done))
3188 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
3189 cpr->cp_raw_cons);
3190 goto poll_done;
3191 }
3192
3193 /* The valid test of the entry must be done first before
3194 * reading any further.
3195 */
3196 dma_rmb();
3197
3198 type = le16_to_cpu(nqcmp->type);
3199 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
3200 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
3201 u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
3202 struct bnxt_cp_ring_info *cpr2;
3203
3204 /* No more budget for RX work */
3205 if (budget && work_done >= budget &&
3206 cq_type == BNXT_NQ_HDL_TYPE_RX)
3207 break;
3208
3209 idx = BNXT_NQ_HDL_IDX(idx);
3210 cpr2 = &cpr->cp_ring_arr[idx];
3211 cpr2->had_nqe_notify = 1;
3212 cpr2->toggle = NQE_CN_TOGGLE(type);
3213 work_done += __bnxt_poll_work(bp, cpr2,
3214 budget - work_done);
3215 cpr->has_more_work |= cpr2->has_more_work;
3216 } else {
3217 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
3218 }
3219 raw_cons = NEXT_RAW_CMP(raw_cons);
3220 }
3221 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
3222 if (raw_cons != cpr->cp_raw_cons) {
3223 cpr->cp_raw_cons = raw_cons;
3224 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
3225 }
3226 poll_done:
3227 cpr_rx = &cpr->cp_ring_arr[0];
3228 if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
3229 (bp->flags & BNXT_FLAG_DIM)) {
3230 struct dim_sample dim_sample = {};
3231
3232 dim_update_sample(cpr->event_ctr,
3233 cpr_rx->rx_packets,
3234 cpr_rx->rx_bytes,
3235 &dim_sample);
3236 net_dim(&cpr->dim, dim_sample);
3237 }
3238 return work_done;
3239 }
3240
bnxt_free_tx_skbs(struct bnxt * bp)3241 static void bnxt_free_tx_skbs(struct bnxt *bp)
3242 {
3243 int i, max_idx;
3244 struct pci_dev *pdev = bp->pdev;
3245
3246 if (!bp->tx_ring)
3247 return;
3248
3249 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
3250 for (i = 0; i < bp->tx_nr_rings; i++) {
3251 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3252 int j;
3253
3254 if (!txr->tx_buf_ring)
3255 continue;
3256
3257 for (j = 0; j < max_idx;) {
3258 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
3259 struct sk_buff *skb;
3260 int k, last;
3261
3262 if (i < bp->tx_nr_rings_xdp &&
3263 tx_buf->action == XDP_REDIRECT) {
3264 dma_unmap_single(&pdev->dev,
3265 dma_unmap_addr(tx_buf, mapping),
3266 dma_unmap_len(tx_buf, len),
3267 DMA_TO_DEVICE);
3268 xdp_return_frame(tx_buf->xdpf);
3269 tx_buf->action = 0;
3270 tx_buf->xdpf = NULL;
3271 j++;
3272 continue;
3273 }
3274
3275 skb = tx_buf->skb;
3276 if (!skb) {
3277 j++;
3278 continue;
3279 }
3280
3281 tx_buf->skb = NULL;
3282
3283 if (tx_buf->is_push) {
3284 dev_kfree_skb(skb);
3285 j += 2;
3286 continue;
3287 }
3288
3289 dma_unmap_single(&pdev->dev,
3290 dma_unmap_addr(tx_buf, mapping),
3291 skb_headlen(skb),
3292 DMA_TO_DEVICE);
3293
3294 last = tx_buf->nr_frags;
3295 j += 2;
3296 for (k = 0; k < last; k++, j++) {
3297 int ring_idx = j & bp->tx_ring_mask;
3298 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3299
3300 tx_buf = &txr->tx_buf_ring[ring_idx];
3301 dma_unmap_page(
3302 &pdev->dev,
3303 dma_unmap_addr(tx_buf, mapping),
3304 skb_frag_size(frag), DMA_TO_DEVICE);
3305 }
3306 dev_kfree_skb(skb);
3307 }
3308 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
3309 }
3310 }
3311
bnxt_free_one_rx_ring(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)3312 static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
3313 {
3314 struct pci_dev *pdev = bp->pdev;
3315 int i, max_idx;
3316
3317 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
3318
3319 for (i = 0; i < max_idx; i++) {
3320 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
3321 dma_addr_t mapping = rx_buf->mapping;
3322 void *data = rx_buf->data;
3323
3324 if (!data)
3325 continue;
3326
3327 rx_buf->data = NULL;
3328 if (BNXT_RX_PAGE_MODE(bp)) {
3329 page_pool_recycle_direct(rxr->page_pool, data);
3330 } else {
3331 dma_unmap_single_attrs(&pdev->dev, mapping,
3332 bp->rx_buf_use_size, bp->rx_dir,
3333 DMA_ATTR_WEAK_ORDERING);
3334 skb_free_frag(data);
3335 }
3336 }
3337 }
3338
bnxt_free_one_rx_agg_ring(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)3339 static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
3340 {
3341 int i, max_idx;
3342
3343 max_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
3344
3345 for (i = 0; i < max_idx; i++) {
3346 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
3347 struct page *page = rx_agg_buf->page;
3348
3349 if (!page)
3350 continue;
3351
3352 rx_agg_buf->page = NULL;
3353 __clear_bit(i, rxr->rx_agg_bmap);
3354
3355 page_pool_recycle_direct(rxr->page_pool, page);
3356 }
3357 }
3358
bnxt_free_one_rx_ring_skbs(struct bnxt * bp,int ring_nr)3359 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
3360 {
3361 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3362 struct pci_dev *pdev = bp->pdev;
3363 struct bnxt_tpa_idx_map *map;
3364 int i;
3365
3366 if (!rxr->rx_tpa)
3367 goto skip_rx_tpa_free;
3368
3369 for (i = 0; i < bp->max_tpa; i++) {
3370 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
3371 u8 *data = tpa_info->data;
3372
3373 if (!data)
3374 continue;
3375
3376 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
3377 bp->rx_buf_use_size, bp->rx_dir,
3378 DMA_ATTR_WEAK_ORDERING);
3379
3380 tpa_info->data = NULL;
3381
3382 skb_free_frag(data);
3383 }
3384
3385 skip_rx_tpa_free:
3386 if (!rxr->rx_buf_ring)
3387 goto skip_rx_buf_free;
3388
3389 bnxt_free_one_rx_ring(bp, rxr);
3390
3391 skip_rx_buf_free:
3392 if (!rxr->rx_agg_ring)
3393 goto skip_rx_agg_free;
3394
3395 bnxt_free_one_rx_agg_ring(bp, rxr);
3396
3397 skip_rx_agg_free:
3398 map = rxr->rx_tpa_idx_map;
3399 if (map)
3400 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3401 }
3402
bnxt_free_rx_skbs(struct bnxt * bp)3403 static void bnxt_free_rx_skbs(struct bnxt *bp)
3404 {
3405 int i;
3406
3407 if (!bp->rx_ring)
3408 return;
3409
3410 for (i = 0; i < bp->rx_nr_rings; i++)
3411 bnxt_free_one_rx_ring_skbs(bp, i);
3412 }
3413
bnxt_free_skbs(struct bnxt * bp)3414 static void bnxt_free_skbs(struct bnxt *bp)
3415 {
3416 bnxt_free_tx_skbs(bp);
3417 bnxt_free_rx_skbs(bp);
3418 }
3419
bnxt_init_ctx_mem(struct bnxt_ctx_mem_type * ctxm,void * p,int len)3420 static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len)
3421 {
3422 u8 init_val = ctxm->init_value;
3423 u16 offset = ctxm->init_offset;
3424 u8 *p2 = p;
3425 int i;
3426
3427 if (!init_val)
3428 return;
3429 if (offset == BNXT_CTX_INIT_INVALID_OFFSET) {
3430 memset(p, init_val, len);
3431 return;
3432 }
3433 for (i = 0; i < len; i += ctxm->entry_size)
3434 *(p2 + i + offset) = init_val;
3435 }
3436
bnxt_free_ring(struct bnxt * bp,struct bnxt_ring_mem_info * rmem)3437 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3438 {
3439 struct pci_dev *pdev = bp->pdev;
3440 int i;
3441
3442 if (!rmem->pg_arr)
3443 goto skip_pages;
3444
3445 for (i = 0; i < rmem->nr_pages; i++) {
3446 if (!rmem->pg_arr[i])
3447 continue;
3448
3449 dma_free_coherent(&pdev->dev, rmem->page_size,
3450 rmem->pg_arr[i], rmem->dma_arr[i]);
3451
3452 rmem->pg_arr[i] = NULL;
3453 }
3454 skip_pages:
3455 if (rmem->pg_tbl) {
3456 size_t pg_tbl_size = rmem->nr_pages * 8;
3457
3458 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3459 pg_tbl_size = rmem->page_size;
3460 dma_free_coherent(&pdev->dev, pg_tbl_size,
3461 rmem->pg_tbl, rmem->pg_tbl_map);
3462 rmem->pg_tbl = NULL;
3463 }
3464 if (rmem->vmem_size && *rmem->vmem) {
3465 vfree(*rmem->vmem);
3466 *rmem->vmem = NULL;
3467 }
3468 }
3469
bnxt_alloc_ring(struct bnxt * bp,struct bnxt_ring_mem_info * rmem)3470 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3471 {
3472 struct pci_dev *pdev = bp->pdev;
3473 u64 valid_bit = 0;
3474 int i;
3475
3476 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3477 valid_bit = PTU_PTE_VALID;
3478 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3479 size_t pg_tbl_size = rmem->nr_pages * 8;
3480
3481 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3482 pg_tbl_size = rmem->page_size;
3483 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3484 &rmem->pg_tbl_map,
3485 GFP_KERNEL);
3486 if (!rmem->pg_tbl)
3487 return -ENOMEM;
3488 }
3489
3490 for (i = 0; i < rmem->nr_pages; i++) {
3491 u64 extra_bits = valid_bit;
3492
3493 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3494 rmem->page_size,
3495 &rmem->dma_arr[i],
3496 GFP_KERNEL);
3497 if (!rmem->pg_arr[i])
3498 return -ENOMEM;
3499
3500 if (rmem->ctx_mem)
3501 bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
3502 rmem->page_size);
3503 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3504 if (i == rmem->nr_pages - 2 &&
3505 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3506 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3507 else if (i == rmem->nr_pages - 1 &&
3508 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3509 extra_bits |= PTU_PTE_LAST;
3510 rmem->pg_tbl[i] =
3511 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3512 }
3513 }
3514
3515 if (rmem->vmem_size) {
3516 *rmem->vmem = vzalloc(rmem->vmem_size);
3517 if (!(*rmem->vmem))
3518 return -ENOMEM;
3519 }
3520 return 0;
3521 }
3522
bnxt_free_tpa_info(struct bnxt * bp)3523 static void bnxt_free_tpa_info(struct bnxt *bp)
3524 {
3525 int i, j;
3526
3527 for (i = 0; i < bp->rx_nr_rings; i++) {
3528 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3529
3530 kfree(rxr->rx_tpa_idx_map);
3531 rxr->rx_tpa_idx_map = NULL;
3532 if (rxr->rx_tpa) {
3533 for (j = 0; j < bp->max_tpa; j++) {
3534 kfree(rxr->rx_tpa[j].agg_arr);
3535 rxr->rx_tpa[j].agg_arr = NULL;
3536 }
3537 }
3538 kfree(rxr->rx_tpa);
3539 rxr->rx_tpa = NULL;
3540 }
3541 }
3542
bnxt_alloc_tpa_info(struct bnxt * bp)3543 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3544 {
3545 int i, j;
3546
3547 bp->max_tpa = MAX_TPA;
3548 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
3549 if (!bp->max_tpa_v2)
3550 return 0;
3551 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3552 }
3553
3554 for (i = 0; i < bp->rx_nr_rings; i++) {
3555 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3556 struct rx_agg_cmp *agg;
3557
3558 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3559 GFP_KERNEL);
3560 if (!rxr->rx_tpa)
3561 return -ENOMEM;
3562
3563 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3564 continue;
3565 for (j = 0; j < bp->max_tpa; j++) {
3566 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3567 if (!agg)
3568 return -ENOMEM;
3569 rxr->rx_tpa[j].agg_arr = agg;
3570 }
3571 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3572 GFP_KERNEL);
3573 if (!rxr->rx_tpa_idx_map)
3574 return -ENOMEM;
3575 }
3576 return 0;
3577 }
3578
bnxt_free_rx_rings(struct bnxt * bp)3579 static void bnxt_free_rx_rings(struct bnxt *bp)
3580 {
3581 int i;
3582
3583 if (!bp->rx_ring)
3584 return;
3585
3586 bnxt_free_tpa_info(bp);
3587 for (i = 0; i < bp->rx_nr_rings; i++) {
3588 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3589 struct bnxt_ring_struct *ring;
3590
3591 if (rxr->xdp_prog)
3592 bpf_prog_put(rxr->xdp_prog);
3593
3594 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3595 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3596
3597 page_pool_destroy(rxr->page_pool);
3598 rxr->page_pool = NULL;
3599
3600 kfree(rxr->rx_agg_bmap);
3601 rxr->rx_agg_bmap = NULL;
3602
3603 ring = &rxr->rx_ring_struct;
3604 bnxt_free_ring(bp, &ring->ring_mem);
3605
3606 ring = &rxr->rx_agg_ring_struct;
3607 bnxt_free_ring(bp, &ring->ring_mem);
3608 }
3609 }
3610
bnxt_alloc_rx_page_pool(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,int numa_node)3611 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3612 struct bnxt_rx_ring_info *rxr,
3613 int numa_node)
3614 {
3615 struct page_pool_params pp = { 0 };
3616
3617 pp.pool_size = bp->rx_agg_ring_size;
3618 if (BNXT_RX_PAGE_MODE(bp))
3619 pp.pool_size += bp->rx_ring_size;
3620 pp.nid = numa_node;
3621 pp.napi = &rxr->bnapi->napi;
3622 pp.netdev = bp->dev;
3623 pp.dev = &bp->pdev->dev;
3624 pp.dma_dir = bp->rx_dir;
3625 pp.max_len = PAGE_SIZE;
3626 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3627
3628 rxr->page_pool = page_pool_create(&pp);
3629 if (IS_ERR(rxr->page_pool)) {
3630 int err = PTR_ERR(rxr->page_pool);
3631
3632 rxr->page_pool = NULL;
3633 return err;
3634 }
3635 return 0;
3636 }
3637
bnxt_alloc_rx_rings(struct bnxt * bp)3638 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3639 {
3640 int numa_node = dev_to_node(&bp->pdev->dev);
3641 int i, rc = 0, agg_rings = 0, cpu;
3642
3643 if (!bp->rx_ring)
3644 return -ENOMEM;
3645
3646 if (bp->flags & BNXT_FLAG_AGG_RINGS)
3647 agg_rings = 1;
3648
3649 for (i = 0; i < bp->rx_nr_rings; i++) {
3650 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3651 struct bnxt_ring_struct *ring;
3652 int cpu_node;
3653
3654 ring = &rxr->rx_ring_struct;
3655
3656 cpu = cpumask_local_spread(i, numa_node);
3657 cpu_node = cpu_to_node(cpu);
3658 netdev_dbg(bp->dev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n",
3659 i, cpu_node);
3660 rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node);
3661 if (rc)
3662 return rc;
3663
3664 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3665 if (rc < 0)
3666 return rc;
3667
3668 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3669 MEM_TYPE_PAGE_POOL,
3670 rxr->page_pool);
3671 if (rc) {
3672 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3673 return rc;
3674 }
3675
3676 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3677 if (rc)
3678 return rc;
3679
3680 ring->grp_idx = i;
3681 if (agg_rings) {
3682 u16 mem_size;
3683
3684 ring = &rxr->rx_agg_ring_struct;
3685 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3686 if (rc)
3687 return rc;
3688
3689 ring->grp_idx = i;
3690 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3691 mem_size = rxr->rx_agg_bmap_size / 8;
3692 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3693 if (!rxr->rx_agg_bmap)
3694 return -ENOMEM;
3695 }
3696 }
3697 if (bp->flags & BNXT_FLAG_TPA)
3698 rc = bnxt_alloc_tpa_info(bp);
3699 return rc;
3700 }
3701
bnxt_free_tx_rings(struct bnxt * bp)3702 static void bnxt_free_tx_rings(struct bnxt *bp)
3703 {
3704 int i;
3705 struct pci_dev *pdev = bp->pdev;
3706
3707 if (!bp->tx_ring)
3708 return;
3709
3710 for (i = 0; i < bp->tx_nr_rings; i++) {
3711 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3712 struct bnxt_ring_struct *ring;
3713
3714 if (txr->tx_push) {
3715 dma_free_coherent(&pdev->dev, bp->tx_push_size,
3716 txr->tx_push, txr->tx_push_mapping);
3717 txr->tx_push = NULL;
3718 }
3719
3720 ring = &txr->tx_ring_struct;
3721
3722 bnxt_free_ring(bp, &ring->ring_mem);
3723 }
3724 }
3725
3726 #define BNXT_TC_TO_RING_BASE(bp, tc) \
3727 ((tc) * (bp)->tx_nr_rings_per_tc)
3728
3729 #define BNXT_RING_TO_TC_OFF(bp, tx) \
3730 ((tx) % (bp)->tx_nr_rings_per_tc)
3731
3732 #define BNXT_RING_TO_TC(bp, tx) \
3733 ((tx) / (bp)->tx_nr_rings_per_tc)
3734
bnxt_alloc_tx_rings(struct bnxt * bp)3735 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3736 {
3737 int i, j, rc;
3738 struct pci_dev *pdev = bp->pdev;
3739
3740 bp->tx_push_size = 0;
3741 if (bp->tx_push_thresh) {
3742 int push_size;
3743
3744 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3745 bp->tx_push_thresh);
3746
3747 if (push_size > 256) {
3748 push_size = 0;
3749 bp->tx_push_thresh = 0;
3750 }
3751
3752 bp->tx_push_size = push_size;
3753 }
3754
3755 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3756 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3757 struct bnxt_ring_struct *ring;
3758 u8 qidx;
3759
3760 ring = &txr->tx_ring_struct;
3761
3762 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3763 if (rc)
3764 return rc;
3765
3766 ring->grp_idx = txr->bnapi->index;
3767 if (bp->tx_push_size) {
3768 dma_addr_t mapping;
3769
3770 /* One pre-allocated DMA buffer to backup
3771 * TX push operation
3772 */
3773 txr->tx_push = dma_alloc_coherent(&pdev->dev,
3774 bp->tx_push_size,
3775 &txr->tx_push_mapping,
3776 GFP_KERNEL);
3777
3778 if (!txr->tx_push)
3779 return -ENOMEM;
3780
3781 mapping = txr->tx_push_mapping +
3782 sizeof(struct tx_push_bd);
3783 txr->data_mapping = cpu_to_le64(mapping);
3784 }
3785 qidx = bp->tc_to_qidx[j];
3786 ring->queue_id = bp->q_info[qidx].queue_id;
3787 spin_lock_init(&txr->xdp_tx_lock);
3788 if (i < bp->tx_nr_rings_xdp)
3789 continue;
3790 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
3791 j++;
3792 }
3793 return 0;
3794 }
3795
bnxt_free_cp_arrays(struct bnxt_cp_ring_info * cpr)3796 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3797 {
3798 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3799
3800 kfree(cpr->cp_desc_ring);
3801 cpr->cp_desc_ring = NULL;
3802 ring->ring_mem.pg_arr = NULL;
3803 kfree(cpr->cp_desc_mapping);
3804 cpr->cp_desc_mapping = NULL;
3805 ring->ring_mem.dma_arr = NULL;
3806 }
3807
bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info * cpr,int n)3808 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3809 {
3810 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3811 if (!cpr->cp_desc_ring)
3812 return -ENOMEM;
3813 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3814 GFP_KERNEL);
3815 if (!cpr->cp_desc_mapping)
3816 return -ENOMEM;
3817 return 0;
3818 }
3819
bnxt_free_all_cp_arrays(struct bnxt * bp)3820 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3821 {
3822 int i;
3823
3824 if (!bp->bnapi)
3825 return;
3826 for (i = 0; i < bp->cp_nr_rings; i++) {
3827 struct bnxt_napi *bnapi = bp->bnapi[i];
3828
3829 if (!bnapi)
3830 continue;
3831 bnxt_free_cp_arrays(&bnapi->cp_ring);
3832 }
3833 }
3834
bnxt_alloc_all_cp_arrays(struct bnxt * bp)3835 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3836 {
3837 int i, n = bp->cp_nr_pages;
3838
3839 for (i = 0; i < bp->cp_nr_rings; i++) {
3840 struct bnxt_napi *bnapi = bp->bnapi[i];
3841 int rc;
3842
3843 if (!bnapi)
3844 continue;
3845 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3846 if (rc)
3847 return rc;
3848 }
3849 return 0;
3850 }
3851
bnxt_free_cp_rings(struct bnxt * bp)3852 static void bnxt_free_cp_rings(struct bnxt *bp)
3853 {
3854 int i;
3855
3856 if (!bp->bnapi)
3857 return;
3858
3859 for (i = 0; i < bp->cp_nr_rings; i++) {
3860 struct bnxt_napi *bnapi = bp->bnapi[i];
3861 struct bnxt_cp_ring_info *cpr;
3862 struct bnxt_ring_struct *ring;
3863 int j;
3864
3865 if (!bnapi)
3866 continue;
3867
3868 cpr = &bnapi->cp_ring;
3869 ring = &cpr->cp_ring_struct;
3870
3871 bnxt_free_ring(bp, &ring->ring_mem);
3872
3873 if (!cpr->cp_ring_arr)
3874 continue;
3875
3876 for (j = 0; j < cpr->cp_ring_count; j++) {
3877 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
3878
3879 ring = &cpr2->cp_ring_struct;
3880 bnxt_free_ring(bp, &ring->ring_mem);
3881 bnxt_free_cp_arrays(cpr2);
3882 }
3883 kfree(cpr->cp_ring_arr);
3884 cpr->cp_ring_arr = NULL;
3885 cpr->cp_ring_count = 0;
3886 }
3887 }
3888
bnxt_alloc_cp_sub_ring(struct bnxt * bp,struct bnxt_cp_ring_info * cpr)3889 static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
3890 struct bnxt_cp_ring_info *cpr)
3891 {
3892 struct bnxt_ring_mem_info *rmem;
3893 struct bnxt_ring_struct *ring;
3894 int rc;
3895
3896 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3897 if (rc) {
3898 bnxt_free_cp_arrays(cpr);
3899 return -ENOMEM;
3900 }
3901 ring = &cpr->cp_ring_struct;
3902 rmem = &ring->ring_mem;
3903 rmem->nr_pages = bp->cp_nr_pages;
3904 rmem->page_size = HW_CMPD_RING_SIZE;
3905 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3906 rmem->dma_arr = cpr->cp_desc_mapping;
3907 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3908 rc = bnxt_alloc_ring(bp, rmem);
3909 if (rc) {
3910 bnxt_free_ring(bp, rmem);
3911 bnxt_free_cp_arrays(cpr);
3912 }
3913 return rc;
3914 }
3915
bnxt_alloc_cp_rings(struct bnxt * bp)3916 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3917 {
3918 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3919 int i, j, rc, ulp_msix;
3920 int tcs = bp->num_tc;
3921
3922 if (!tcs)
3923 tcs = 1;
3924 ulp_msix = bnxt_get_ulp_msix_num(bp);
3925 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
3926 struct bnxt_napi *bnapi = bp->bnapi[i];
3927 struct bnxt_cp_ring_info *cpr, *cpr2;
3928 struct bnxt_ring_struct *ring;
3929 int cp_count = 0, k;
3930 int rx = 0, tx = 0;
3931
3932 if (!bnapi)
3933 continue;
3934
3935 cpr = &bnapi->cp_ring;
3936 cpr->bnapi = bnapi;
3937 ring = &cpr->cp_ring_struct;
3938
3939 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3940 if (rc)
3941 return rc;
3942
3943 ring->map_idx = ulp_msix + i;
3944
3945 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3946 continue;
3947
3948 if (i < bp->rx_nr_rings) {
3949 cp_count++;
3950 rx = 1;
3951 }
3952 if (i < bp->tx_nr_rings_xdp) {
3953 cp_count++;
3954 tx = 1;
3955 } else if ((sh && i < bp->tx_nr_rings) ||
3956 (!sh && i >= bp->rx_nr_rings)) {
3957 cp_count += tcs;
3958 tx = 1;
3959 }
3960
3961 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
3962 GFP_KERNEL);
3963 if (!cpr->cp_ring_arr)
3964 return -ENOMEM;
3965 cpr->cp_ring_count = cp_count;
3966
3967 for (k = 0; k < cp_count; k++) {
3968 cpr2 = &cpr->cp_ring_arr[k];
3969 rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
3970 if (rc)
3971 return rc;
3972 cpr2->bnapi = bnapi;
3973 cpr2->sw_stats = cpr->sw_stats;
3974 cpr2->cp_idx = k;
3975 if (!k && rx) {
3976 bp->rx_ring[i].rx_cpr = cpr2;
3977 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
3978 } else {
3979 int n, tc = k - rx;
3980
3981 n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
3982 bp->tx_ring[n].tx_cpr = cpr2;
3983 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
3984 }
3985 }
3986 if (tx)
3987 j++;
3988 }
3989 return 0;
3990 }
3991
bnxt_init_rx_ring_struct(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)3992 static void bnxt_init_rx_ring_struct(struct bnxt *bp,
3993 struct bnxt_rx_ring_info *rxr)
3994 {
3995 struct bnxt_ring_mem_info *rmem;
3996 struct bnxt_ring_struct *ring;
3997
3998 ring = &rxr->rx_ring_struct;
3999 rmem = &ring->ring_mem;
4000 rmem->nr_pages = bp->rx_nr_pages;
4001 rmem->page_size = HW_RXBD_RING_SIZE;
4002 rmem->pg_arr = (void **)rxr->rx_desc_ring;
4003 rmem->dma_arr = rxr->rx_desc_mapping;
4004 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
4005 rmem->vmem = (void **)&rxr->rx_buf_ring;
4006
4007 ring = &rxr->rx_agg_ring_struct;
4008 rmem = &ring->ring_mem;
4009 rmem->nr_pages = bp->rx_agg_nr_pages;
4010 rmem->page_size = HW_RXBD_RING_SIZE;
4011 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
4012 rmem->dma_arr = rxr->rx_agg_desc_mapping;
4013 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
4014 rmem->vmem = (void **)&rxr->rx_agg_ring;
4015 }
4016
bnxt_reset_rx_ring_struct(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)4017 static void bnxt_reset_rx_ring_struct(struct bnxt *bp,
4018 struct bnxt_rx_ring_info *rxr)
4019 {
4020 struct bnxt_ring_mem_info *rmem;
4021 struct bnxt_ring_struct *ring;
4022 int i;
4023
4024 rxr->page_pool->p.napi = NULL;
4025 rxr->page_pool = NULL;
4026 memset(&rxr->xdp_rxq, 0, sizeof(struct xdp_rxq_info));
4027
4028 ring = &rxr->rx_ring_struct;
4029 rmem = &ring->ring_mem;
4030 rmem->pg_tbl = NULL;
4031 rmem->pg_tbl_map = 0;
4032 for (i = 0; i < rmem->nr_pages; i++) {
4033 rmem->pg_arr[i] = NULL;
4034 rmem->dma_arr[i] = 0;
4035 }
4036 *rmem->vmem = NULL;
4037
4038 ring = &rxr->rx_agg_ring_struct;
4039 rmem = &ring->ring_mem;
4040 rmem->pg_tbl = NULL;
4041 rmem->pg_tbl_map = 0;
4042 for (i = 0; i < rmem->nr_pages; i++) {
4043 rmem->pg_arr[i] = NULL;
4044 rmem->dma_arr[i] = 0;
4045 }
4046 *rmem->vmem = NULL;
4047 }
4048
bnxt_init_ring_struct(struct bnxt * bp)4049 static void bnxt_init_ring_struct(struct bnxt *bp)
4050 {
4051 int i, j;
4052
4053 for (i = 0; i < bp->cp_nr_rings; i++) {
4054 struct bnxt_napi *bnapi = bp->bnapi[i];
4055 struct bnxt_ring_mem_info *rmem;
4056 struct bnxt_cp_ring_info *cpr;
4057 struct bnxt_rx_ring_info *rxr;
4058 struct bnxt_tx_ring_info *txr;
4059 struct bnxt_ring_struct *ring;
4060
4061 if (!bnapi)
4062 continue;
4063
4064 cpr = &bnapi->cp_ring;
4065 ring = &cpr->cp_ring_struct;
4066 rmem = &ring->ring_mem;
4067 rmem->nr_pages = bp->cp_nr_pages;
4068 rmem->page_size = HW_CMPD_RING_SIZE;
4069 rmem->pg_arr = (void **)cpr->cp_desc_ring;
4070 rmem->dma_arr = cpr->cp_desc_mapping;
4071 rmem->vmem_size = 0;
4072
4073 rxr = bnapi->rx_ring;
4074 if (!rxr)
4075 goto skip_rx;
4076
4077 ring = &rxr->rx_ring_struct;
4078 rmem = &ring->ring_mem;
4079 rmem->nr_pages = bp->rx_nr_pages;
4080 rmem->page_size = HW_RXBD_RING_SIZE;
4081 rmem->pg_arr = (void **)rxr->rx_desc_ring;
4082 rmem->dma_arr = rxr->rx_desc_mapping;
4083 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
4084 rmem->vmem = (void **)&rxr->rx_buf_ring;
4085
4086 ring = &rxr->rx_agg_ring_struct;
4087 rmem = &ring->ring_mem;
4088 rmem->nr_pages = bp->rx_agg_nr_pages;
4089 rmem->page_size = HW_RXBD_RING_SIZE;
4090 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
4091 rmem->dma_arr = rxr->rx_agg_desc_mapping;
4092 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
4093 rmem->vmem = (void **)&rxr->rx_agg_ring;
4094
4095 skip_rx:
4096 bnxt_for_each_napi_tx(j, bnapi, txr) {
4097 ring = &txr->tx_ring_struct;
4098 rmem = &ring->ring_mem;
4099 rmem->nr_pages = bp->tx_nr_pages;
4100 rmem->page_size = HW_TXBD_RING_SIZE;
4101 rmem->pg_arr = (void **)txr->tx_desc_ring;
4102 rmem->dma_arr = txr->tx_desc_mapping;
4103 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
4104 rmem->vmem = (void **)&txr->tx_buf_ring;
4105 }
4106 }
4107 }
4108
bnxt_init_rxbd_pages(struct bnxt_ring_struct * ring,u32 type)4109 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
4110 {
4111 int i;
4112 u32 prod;
4113 struct rx_bd **rx_buf_ring;
4114
4115 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
4116 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
4117 int j;
4118 struct rx_bd *rxbd;
4119
4120 rxbd = rx_buf_ring[i];
4121 if (!rxbd)
4122 continue;
4123
4124 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
4125 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
4126 rxbd->rx_bd_opaque = prod;
4127 }
4128 }
4129 }
4130
bnxt_alloc_one_rx_ring_skb(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,int ring_nr)4131 static void bnxt_alloc_one_rx_ring_skb(struct bnxt *bp,
4132 struct bnxt_rx_ring_info *rxr,
4133 int ring_nr)
4134 {
4135 u32 prod;
4136 int i;
4137
4138 prod = rxr->rx_prod;
4139 for (i = 0; i < bp->rx_ring_size; i++) {
4140 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
4141 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d skbs only\n",
4142 ring_nr, i, bp->rx_ring_size);
4143 break;
4144 }
4145 prod = NEXT_RX(prod);
4146 }
4147 rxr->rx_prod = prod;
4148 }
4149
bnxt_alloc_one_rx_ring_page(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,int ring_nr)4150 static void bnxt_alloc_one_rx_ring_page(struct bnxt *bp,
4151 struct bnxt_rx_ring_info *rxr,
4152 int ring_nr)
4153 {
4154 u32 prod;
4155 int i;
4156
4157 prod = rxr->rx_agg_prod;
4158 for (i = 0; i < bp->rx_agg_ring_size; i++) {
4159 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
4160 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d pages only\n",
4161 ring_nr, i, bp->rx_ring_size);
4162 break;
4163 }
4164 prod = NEXT_RX_AGG(prod);
4165 }
4166 rxr->rx_agg_prod = prod;
4167 }
4168
bnxt_alloc_one_rx_ring(struct bnxt * bp,int ring_nr)4169 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
4170 {
4171 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
4172 int i;
4173
4174 bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr);
4175
4176 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
4177 return 0;
4178
4179 bnxt_alloc_one_rx_ring_page(bp, rxr, ring_nr);
4180
4181 if (rxr->rx_tpa) {
4182 dma_addr_t mapping;
4183 u8 *data;
4184
4185 for (i = 0; i < bp->max_tpa; i++) {
4186 data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
4187 if (!data)
4188 return -ENOMEM;
4189
4190 rxr->rx_tpa[i].data = data;
4191 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
4192 rxr->rx_tpa[i].mapping = mapping;
4193 }
4194 }
4195 return 0;
4196 }
4197
bnxt_init_one_rx_ring_rxbd(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)4198 static void bnxt_init_one_rx_ring_rxbd(struct bnxt *bp,
4199 struct bnxt_rx_ring_info *rxr)
4200 {
4201 struct bnxt_ring_struct *ring;
4202 u32 type;
4203
4204 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
4205 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
4206
4207 if (NET_IP_ALIGN == 2)
4208 type |= RX_BD_FLAGS_SOP;
4209
4210 ring = &rxr->rx_ring_struct;
4211 bnxt_init_rxbd_pages(ring, type);
4212 ring->fw_ring_id = INVALID_HW_RING_ID;
4213 }
4214
bnxt_init_one_rx_agg_ring_rxbd(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)4215 static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp,
4216 struct bnxt_rx_ring_info *rxr)
4217 {
4218 struct bnxt_ring_struct *ring;
4219 u32 type;
4220
4221 ring = &rxr->rx_agg_ring_struct;
4222 ring->fw_ring_id = INVALID_HW_RING_ID;
4223 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
4224 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
4225 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
4226
4227 bnxt_init_rxbd_pages(ring, type);
4228 }
4229 }
4230
bnxt_init_one_rx_ring(struct bnxt * bp,int ring_nr)4231 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
4232 {
4233 struct bnxt_rx_ring_info *rxr;
4234
4235 rxr = &bp->rx_ring[ring_nr];
4236 bnxt_init_one_rx_ring_rxbd(bp, rxr);
4237
4238 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX,
4239 &rxr->bnapi->napi);
4240
4241 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
4242 bpf_prog_add(bp->xdp_prog, 1);
4243 rxr->xdp_prog = bp->xdp_prog;
4244 }
4245
4246 bnxt_init_one_rx_agg_ring_rxbd(bp, rxr);
4247
4248 return bnxt_alloc_one_rx_ring(bp, ring_nr);
4249 }
4250
bnxt_init_cp_rings(struct bnxt * bp)4251 static void bnxt_init_cp_rings(struct bnxt *bp)
4252 {
4253 int i, j;
4254
4255 for (i = 0; i < bp->cp_nr_rings; i++) {
4256 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
4257 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4258
4259 ring->fw_ring_id = INVALID_HW_RING_ID;
4260 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4261 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4262 if (!cpr->cp_ring_arr)
4263 continue;
4264 for (j = 0; j < cpr->cp_ring_count; j++) {
4265 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
4266
4267 ring = &cpr2->cp_ring_struct;
4268 ring->fw_ring_id = INVALID_HW_RING_ID;
4269 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4270 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4271 }
4272 }
4273 }
4274
bnxt_init_rx_rings(struct bnxt * bp)4275 static int bnxt_init_rx_rings(struct bnxt *bp)
4276 {
4277 int i, rc = 0;
4278
4279 if (BNXT_RX_PAGE_MODE(bp)) {
4280 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
4281 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
4282 } else {
4283 bp->rx_offset = BNXT_RX_OFFSET;
4284 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
4285 }
4286
4287 for (i = 0; i < bp->rx_nr_rings; i++) {
4288 rc = bnxt_init_one_rx_ring(bp, i);
4289 if (rc)
4290 break;
4291 }
4292
4293 return rc;
4294 }
4295
bnxt_init_tx_rings(struct bnxt * bp)4296 static int bnxt_init_tx_rings(struct bnxt *bp)
4297 {
4298 u16 i;
4299
4300 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
4301 BNXT_MIN_TX_DESC_CNT);
4302
4303 for (i = 0; i < bp->tx_nr_rings; i++) {
4304 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4305 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4306
4307 ring->fw_ring_id = INVALID_HW_RING_ID;
4308
4309 if (i >= bp->tx_nr_rings_xdp)
4310 netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp,
4311 NETDEV_QUEUE_TYPE_TX,
4312 &txr->bnapi->napi);
4313 }
4314
4315 return 0;
4316 }
4317
bnxt_free_ring_grps(struct bnxt * bp)4318 static void bnxt_free_ring_grps(struct bnxt *bp)
4319 {
4320 kfree(bp->grp_info);
4321 bp->grp_info = NULL;
4322 }
4323
bnxt_init_ring_grps(struct bnxt * bp,bool irq_re_init)4324 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
4325 {
4326 int i;
4327
4328 if (irq_re_init) {
4329 bp->grp_info = kcalloc(bp->cp_nr_rings,
4330 sizeof(struct bnxt_ring_grp_info),
4331 GFP_KERNEL);
4332 if (!bp->grp_info)
4333 return -ENOMEM;
4334 }
4335 for (i = 0; i < bp->cp_nr_rings; i++) {
4336 if (irq_re_init)
4337 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
4338 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4339 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
4340 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
4341 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4342 }
4343 return 0;
4344 }
4345
bnxt_free_vnics(struct bnxt * bp)4346 static void bnxt_free_vnics(struct bnxt *bp)
4347 {
4348 kfree(bp->vnic_info);
4349 bp->vnic_info = NULL;
4350 bp->nr_vnics = 0;
4351 }
4352
bnxt_alloc_vnics(struct bnxt * bp)4353 static int bnxt_alloc_vnics(struct bnxt *bp)
4354 {
4355 int num_vnics = 1;
4356
4357 #ifdef CONFIG_RFS_ACCEL
4358 if (bp->flags & BNXT_FLAG_RFS) {
4359 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
4360 num_vnics++;
4361 else if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4362 num_vnics += bp->rx_nr_rings;
4363 }
4364 #endif
4365
4366 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4367 num_vnics++;
4368
4369 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
4370 GFP_KERNEL);
4371 if (!bp->vnic_info)
4372 return -ENOMEM;
4373
4374 bp->nr_vnics = num_vnics;
4375 return 0;
4376 }
4377
bnxt_init_vnics(struct bnxt * bp)4378 static void bnxt_init_vnics(struct bnxt *bp)
4379 {
4380 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
4381 int i;
4382
4383 for (i = 0; i < bp->nr_vnics; i++) {
4384 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4385 int j;
4386
4387 vnic->fw_vnic_id = INVALID_HW_RING_ID;
4388 vnic->vnic_id = i;
4389 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
4390 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
4391
4392 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
4393
4394 if (bp->vnic_info[i].rss_hash_key) {
4395 if (i == BNXT_VNIC_DEFAULT) {
4396 u8 *key = (void *)vnic->rss_hash_key;
4397 int k;
4398
4399 if (!bp->rss_hash_key_valid &&
4400 !bp->rss_hash_key_updated) {
4401 get_random_bytes(bp->rss_hash_key,
4402 HW_HASH_KEY_SIZE);
4403 bp->rss_hash_key_updated = true;
4404 }
4405
4406 memcpy(vnic->rss_hash_key, bp->rss_hash_key,
4407 HW_HASH_KEY_SIZE);
4408
4409 if (!bp->rss_hash_key_updated)
4410 continue;
4411
4412 bp->rss_hash_key_updated = false;
4413 bp->rss_hash_key_valid = true;
4414
4415 bp->toeplitz_prefix = 0;
4416 for (k = 0; k < 8; k++) {
4417 bp->toeplitz_prefix <<= 8;
4418 bp->toeplitz_prefix |= key[k];
4419 }
4420 } else {
4421 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key,
4422 HW_HASH_KEY_SIZE);
4423 }
4424 }
4425 }
4426 }
4427
bnxt_calc_nr_ring_pages(u32 ring_size,int desc_per_pg)4428 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
4429 {
4430 int pages;
4431
4432 pages = ring_size / desc_per_pg;
4433
4434 if (!pages)
4435 return 1;
4436
4437 pages++;
4438
4439 while (pages & (pages - 1))
4440 pages++;
4441
4442 return pages;
4443 }
4444
bnxt_set_tpa_flags(struct bnxt * bp)4445 void bnxt_set_tpa_flags(struct bnxt *bp)
4446 {
4447 bp->flags &= ~BNXT_FLAG_TPA;
4448 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
4449 return;
4450 if (bp->dev->features & NETIF_F_LRO)
4451 bp->flags |= BNXT_FLAG_LRO;
4452 else if (bp->dev->features & NETIF_F_GRO_HW)
4453 bp->flags |= BNXT_FLAG_GRO;
4454 }
4455
4456 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
4457 * be set on entry.
4458 */
bnxt_set_ring_params(struct bnxt * bp)4459 void bnxt_set_ring_params(struct bnxt *bp)
4460 {
4461 u32 ring_size, rx_size, rx_space, max_rx_cmpl;
4462 u32 agg_factor = 0, agg_ring_size = 0;
4463
4464 /* 8 for CRC and VLAN */
4465 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
4466
4467 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
4468 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4469
4470 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
4471 ring_size = bp->rx_ring_size;
4472 bp->rx_agg_ring_size = 0;
4473 bp->rx_agg_nr_pages = 0;
4474
4475 if (bp->flags & BNXT_FLAG_TPA)
4476 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
4477
4478 bp->flags &= ~BNXT_FLAG_JUMBO;
4479 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
4480 u32 jumbo_factor;
4481
4482 bp->flags |= BNXT_FLAG_JUMBO;
4483 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
4484 if (jumbo_factor > agg_factor)
4485 agg_factor = jumbo_factor;
4486 }
4487 if (agg_factor) {
4488 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
4489 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
4490 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
4491 bp->rx_ring_size, ring_size);
4492 bp->rx_ring_size = ring_size;
4493 }
4494 agg_ring_size = ring_size * agg_factor;
4495
4496 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
4497 RX_DESC_CNT);
4498 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
4499 u32 tmp = agg_ring_size;
4500
4501 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
4502 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
4503 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
4504 tmp, agg_ring_size);
4505 }
4506 bp->rx_agg_ring_size = agg_ring_size;
4507 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
4508
4509 if (BNXT_RX_PAGE_MODE(bp)) {
4510 rx_space = PAGE_SIZE;
4511 rx_size = PAGE_SIZE -
4512 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
4513 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4514 } else {
4515 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
4516 rx_space = rx_size + NET_SKB_PAD +
4517 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4518 }
4519 }
4520
4521 bp->rx_buf_use_size = rx_size;
4522 bp->rx_buf_size = rx_space;
4523
4524 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
4525 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
4526
4527 ring_size = bp->tx_ring_size;
4528 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
4529 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
4530
4531 max_rx_cmpl = bp->rx_ring_size;
4532 /* MAX TPA needs to be added because TPA_START completions are
4533 * immediately recycled, so the TPA completions are not bound by
4534 * the RX ring size.
4535 */
4536 if (bp->flags & BNXT_FLAG_TPA)
4537 max_rx_cmpl += bp->max_tpa;
4538 /* RX and TPA completions are 32-byte, all others are 16-byte */
4539 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
4540 bp->cp_ring_size = ring_size;
4541
4542 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4543 if (bp->cp_nr_pages > MAX_CP_PAGES) {
4544 bp->cp_nr_pages = MAX_CP_PAGES;
4545 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4546 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4547 ring_size, bp->cp_ring_size);
4548 }
4549 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4550 bp->cp_ring_mask = bp->cp_bit - 1;
4551 }
4552
4553 /* Changing allocation mode of RX rings.
4554 * TODO: Update when extending xdp_rxq_info to support allocation modes.
4555 */
bnxt_set_rx_skb_mode(struct bnxt * bp,bool page_mode)4556 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4557 {
4558 struct net_device *dev = bp->dev;
4559
4560 if (page_mode) {
4561 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4562 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4563
4564 if (bp->xdp_prog->aux->xdp_has_frags)
4565 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4566 else
4567 dev->max_mtu =
4568 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4569 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4570 bp->flags |= BNXT_FLAG_JUMBO;
4571 bp->rx_skb_func = bnxt_rx_multi_page_skb;
4572 } else {
4573 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4574 bp->rx_skb_func = bnxt_rx_page_skb;
4575 }
4576 bp->rx_dir = DMA_BIDIRECTIONAL;
4577 /* Disable LRO or GRO_HW */
4578 netdev_update_features(dev);
4579 } else {
4580 dev->max_mtu = bp->max_mtu;
4581 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4582 bp->rx_dir = DMA_FROM_DEVICE;
4583 bp->rx_skb_func = bnxt_rx_skb;
4584 }
4585 return 0;
4586 }
4587
bnxt_free_vnic_attributes(struct bnxt * bp)4588 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4589 {
4590 int i;
4591 struct bnxt_vnic_info *vnic;
4592 struct pci_dev *pdev = bp->pdev;
4593
4594 if (!bp->vnic_info)
4595 return;
4596
4597 for (i = 0; i < bp->nr_vnics; i++) {
4598 vnic = &bp->vnic_info[i];
4599
4600 kfree(vnic->fw_grp_ids);
4601 vnic->fw_grp_ids = NULL;
4602
4603 kfree(vnic->uc_list);
4604 vnic->uc_list = NULL;
4605
4606 if (vnic->mc_list) {
4607 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4608 vnic->mc_list, vnic->mc_list_mapping);
4609 vnic->mc_list = NULL;
4610 }
4611
4612 if (vnic->rss_table) {
4613 dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4614 vnic->rss_table,
4615 vnic->rss_table_dma_addr);
4616 vnic->rss_table = NULL;
4617 }
4618
4619 vnic->rss_hash_key = NULL;
4620 vnic->flags = 0;
4621 }
4622 }
4623
bnxt_alloc_vnic_attributes(struct bnxt * bp)4624 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4625 {
4626 int i, rc = 0, size;
4627 struct bnxt_vnic_info *vnic;
4628 struct pci_dev *pdev = bp->pdev;
4629 int max_rings;
4630
4631 for (i = 0; i < bp->nr_vnics; i++) {
4632 vnic = &bp->vnic_info[i];
4633
4634 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4635 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4636
4637 if (mem_size > 0) {
4638 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4639 if (!vnic->uc_list) {
4640 rc = -ENOMEM;
4641 goto out;
4642 }
4643 }
4644 }
4645
4646 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4647 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4648 vnic->mc_list =
4649 dma_alloc_coherent(&pdev->dev,
4650 vnic->mc_list_size,
4651 &vnic->mc_list_mapping,
4652 GFP_KERNEL);
4653 if (!vnic->mc_list) {
4654 rc = -ENOMEM;
4655 goto out;
4656 }
4657 }
4658
4659 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4660 goto vnic_skip_grps;
4661
4662 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4663 max_rings = bp->rx_nr_rings;
4664 else
4665 max_rings = 1;
4666
4667 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4668 if (!vnic->fw_grp_ids) {
4669 rc = -ENOMEM;
4670 goto out;
4671 }
4672 vnic_skip_grps:
4673 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
4674 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4675 continue;
4676
4677 /* Allocate rss table and hash key */
4678 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4679 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4680 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4681
4682 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4683 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4684 vnic->rss_table_size,
4685 &vnic->rss_table_dma_addr,
4686 GFP_KERNEL);
4687 if (!vnic->rss_table) {
4688 rc = -ENOMEM;
4689 goto out;
4690 }
4691
4692 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4693 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4694 }
4695 return 0;
4696
4697 out:
4698 return rc;
4699 }
4700
bnxt_free_hwrm_resources(struct bnxt * bp)4701 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4702 {
4703 struct bnxt_hwrm_wait_token *token;
4704
4705 dma_pool_destroy(bp->hwrm_dma_pool);
4706 bp->hwrm_dma_pool = NULL;
4707
4708 rcu_read_lock();
4709 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4710 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4711 rcu_read_unlock();
4712 }
4713
bnxt_alloc_hwrm_resources(struct bnxt * bp)4714 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4715 {
4716 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4717 BNXT_HWRM_DMA_SIZE,
4718 BNXT_HWRM_DMA_ALIGN, 0);
4719 if (!bp->hwrm_dma_pool)
4720 return -ENOMEM;
4721
4722 INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4723
4724 return 0;
4725 }
4726
bnxt_free_stats_mem(struct bnxt * bp,struct bnxt_stats_mem * stats)4727 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4728 {
4729 kfree(stats->hw_masks);
4730 stats->hw_masks = NULL;
4731 kfree(stats->sw_stats);
4732 stats->sw_stats = NULL;
4733 if (stats->hw_stats) {
4734 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4735 stats->hw_stats_map);
4736 stats->hw_stats = NULL;
4737 }
4738 }
4739
bnxt_alloc_stats_mem(struct bnxt * bp,struct bnxt_stats_mem * stats,bool alloc_masks)4740 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4741 bool alloc_masks)
4742 {
4743 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4744 &stats->hw_stats_map, GFP_KERNEL);
4745 if (!stats->hw_stats)
4746 return -ENOMEM;
4747
4748 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4749 if (!stats->sw_stats)
4750 goto stats_mem_err;
4751
4752 if (alloc_masks) {
4753 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4754 if (!stats->hw_masks)
4755 goto stats_mem_err;
4756 }
4757 return 0;
4758
4759 stats_mem_err:
4760 bnxt_free_stats_mem(bp, stats);
4761 return -ENOMEM;
4762 }
4763
bnxt_fill_masks(u64 * mask_arr,u64 mask,int count)4764 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4765 {
4766 int i;
4767
4768 for (i = 0; i < count; i++)
4769 mask_arr[i] = mask;
4770 }
4771
bnxt_copy_hw_masks(u64 * mask_arr,__le64 * hw_mask_arr,int count)4772 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4773 {
4774 int i;
4775
4776 for (i = 0; i < count; i++)
4777 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4778 }
4779
bnxt_hwrm_func_qstat_ext(struct bnxt * bp,struct bnxt_stats_mem * stats)4780 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4781 struct bnxt_stats_mem *stats)
4782 {
4783 struct hwrm_func_qstats_ext_output *resp;
4784 struct hwrm_func_qstats_ext_input *req;
4785 __le64 *hw_masks;
4786 int rc;
4787
4788 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4789 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4790 return -EOPNOTSUPP;
4791
4792 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4793 if (rc)
4794 return rc;
4795
4796 req->fid = cpu_to_le16(0xffff);
4797 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4798
4799 resp = hwrm_req_hold(bp, req);
4800 rc = hwrm_req_send(bp, req);
4801 if (!rc) {
4802 hw_masks = &resp->rx_ucast_pkts;
4803 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4804 }
4805 hwrm_req_drop(bp, req);
4806 return rc;
4807 }
4808
4809 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4810 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4811
bnxt_init_stats(struct bnxt * bp)4812 static void bnxt_init_stats(struct bnxt *bp)
4813 {
4814 struct bnxt_napi *bnapi = bp->bnapi[0];
4815 struct bnxt_cp_ring_info *cpr;
4816 struct bnxt_stats_mem *stats;
4817 __le64 *rx_stats, *tx_stats;
4818 int rc, rx_count, tx_count;
4819 u64 *rx_masks, *tx_masks;
4820 u64 mask;
4821 u8 flags;
4822
4823 cpr = &bnapi->cp_ring;
4824 stats = &cpr->stats;
4825 rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4826 if (rc) {
4827 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4828 mask = (1ULL << 48) - 1;
4829 else
4830 mask = -1ULL;
4831 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4832 }
4833 if (bp->flags & BNXT_FLAG_PORT_STATS) {
4834 stats = &bp->port_stats;
4835 rx_stats = stats->hw_stats;
4836 rx_masks = stats->hw_masks;
4837 rx_count = sizeof(struct rx_port_stats) / 8;
4838 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4839 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4840 tx_count = sizeof(struct tx_port_stats) / 8;
4841
4842 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4843 rc = bnxt_hwrm_port_qstats(bp, flags);
4844 if (rc) {
4845 mask = (1ULL << 40) - 1;
4846
4847 bnxt_fill_masks(rx_masks, mask, rx_count);
4848 bnxt_fill_masks(tx_masks, mask, tx_count);
4849 } else {
4850 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4851 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4852 bnxt_hwrm_port_qstats(bp, 0);
4853 }
4854 }
4855 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4856 stats = &bp->rx_port_stats_ext;
4857 rx_stats = stats->hw_stats;
4858 rx_masks = stats->hw_masks;
4859 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4860 stats = &bp->tx_port_stats_ext;
4861 tx_stats = stats->hw_stats;
4862 tx_masks = stats->hw_masks;
4863 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4864
4865 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4866 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4867 if (rc) {
4868 mask = (1ULL << 40) - 1;
4869
4870 bnxt_fill_masks(rx_masks, mask, rx_count);
4871 if (tx_stats)
4872 bnxt_fill_masks(tx_masks, mask, tx_count);
4873 } else {
4874 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4875 if (tx_stats)
4876 bnxt_copy_hw_masks(tx_masks, tx_stats,
4877 tx_count);
4878 bnxt_hwrm_port_qstats_ext(bp, 0);
4879 }
4880 }
4881 }
4882
bnxt_free_port_stats(struct bnxt * bp)4883 static void bnxt_free_port_stats(struct bnxt *bp)
4884 {
4885 bp->flags &= ~BNXT_FLAG_PORT_STATS;
4886 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4887
4888 bnxt_free_stats_mem(bp, &bp->port_stats);
4889 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4890 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4891 }
4892
bnxt_free_ring_stats(struct bnxt * bp)4893 static void bnxt_free_ring_stats(struct bnxt *bp)
4894 {
4895 int i;
4896
4897 if (!bp->bnapi)
4898 return;
4899
4900 for (i = 0; i < bp->cp_nr_rings; i++) {
4901 struct bnxt_napi *bnapi = bp->bnapi[i];
4902 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4903
4904 bnxt_free_stats_mem(bp, &cpr->stats);
4905
4906 kfree(cpr->sw_stats);
4907 cpr->sw_stats = NULL;
4908 }
4909 }
4910
bnxt_alloc_stats(struct bnxt * bp)4911 static int bnxt_alloc_stats(struct bnxt *bp)
4912 {
4913 u32 size, i;
4914 int rc;
4915
4916 size = bp->hw_ring_stats_size;
4917
4918 for (i = 0; i < bp->cp_nr_rings; i++) {
4919 struct bnxt_napi *bnapi = bp->bnapi[i];
4920 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4921
4922 cpr->sw_stats = kzalloc(sizeof(*cpr->sw_stats), GFP_KERNEL);
4923 if (!cpr->sw_stats)
4924 return -ENOMEM;
4925
4926 cpr->stats.len = size;
4927 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4928 if (rc)
4929 return rc;
4930
4931 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4932 }
4933
4934 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4935 return 0;
4936
4937 if (bp->port_stats.hw_stats)
4938 goto alloc_ext_stats;
4939
4940 bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4941 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4942 if (rc)
4943 return rc;
4944
4945 bp->flags |= BNXT_FLAG_PORT_STATS;
4946
4947 alloc_ext_stats:
4948 /* Display extended statistics only if FW supports it */
4949 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4950 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4951 return 0;
4952
4953 if (bp->rx_port_stats_ext.hw_stats)
4954 goto alloc_tx_ext_stats;
4955
4956 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4957 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4958 /* Extended stats are optional */
4959 if (rc)
4960 return 0;
4961
4962 alloc_tx_ext_stats:
4963 if (bp->tx_port_stats_ext.hw_stats)
4964 return 0;
4965
4966 if (bp->hwrm_spec_code >= 0x10902 ||
4967 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4968 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4969 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4970 /* Extended stats are optional */
4971 if (rc)
4972 return 0;
4973 }
4974 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4975 return 0;
4976 }
4977
bnxt_clear_ring_indices(struct bnxt * bp)4978 static void bnxt_clear_ring_indices(struct bnxt *bp)
4979 {
4980 int i, j;
4981
4982 if (!bp->bnapi)
4983 return;
4984
4985 for (i = 0; i < bp->cp_nr_rings; i++) {
4986 struct bnxt_napi *bnapi = bp->bnapi[i];
4987 struct bnxt_cp_ring_info *cpr;
4988 struct bnxt_rx_ring_info *rxr;
4989 struct bnxt_tx_ring_info *txr;
4990
4991 if (!bnapi)
4992 continue;
4993
4994 cpr = &bnapi->cp_ring;
4995 cpr->cp_raw_cons = 0;
4996
4997 bnxt_for_each_napi_tx(j, bnapi, txr) {
4998 txr->tx_prod = 0;
4999 txr->tx_cons = 0;
5000 txr->tx_hw_cons = 0;
5001 }
5002
5003 rxr = bnapi->rx_ring;
5004 if (rxr) {
5005 rxr->rx_prod = 0;
5006 rxr->rx_agg_prod = 0;
5007 rxr->rx_sw_agg_prod = 0;
5008 rxr->rx_next_cons = 0;
5009 }
5010 bnapi->events = 0;
5011 }
5012 }
5013
bnxt_insert_usr_fltr(struct bnxt * bp,struct bnxt_filter_base * fltr)5014 void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5015 {
5016 u8 type = fltr->type, flags = fltr->flags;
5017
5018 INIT_LIST_HEAD(&fltr->list);
5019 if ((type == BNXT_FLTR_TYPE_L2 && flags & BNXT_ACT_RING_DST) ||
5020 (type == BNXT_FLTR_TYPE_NTUPLE && flags & BNXT_ACT_NO_AGING))
5021 list_add_tail(&fltr->list, &bp->usr_fltr_list);
5022 }
5023
bnxt_del_one_usr_fltr(struct bnxt * bp,struct bnxt_filter_base * fltr)5024 void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5025 {
5026 if (!list_empty(&fltr->list))
5027 list_del_init(&fltr->list);
5028 }
5029
bnxt_clear_usr_fltrs(struct bnxt * bp,bool all)5030 static void bnxt_clear_usr_fltrs(struct bnxt *bp, bool all)
5031 {
5032 struct bnxt_filter_base *usr_fltr, *tmp;
5033
5034 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
5035 if (!all && usr_fltr->type == BNXT_FLTR_TYPE_L2)
5036 continue;
5037 bnxt_del_one_usr_fltr(bp, usr_fltr);
5038 }
5039 }
5040
bnxt_del_fltr(struct bnxt * bp,struct bnxt_filter_base * fltr)5041 static void bnxt_del_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5042 {
5043 hlist_del(&fltr->hash);
5044 bnxt_del_one_usr_fltr(bp, fltr);
5045 if (fltr->flags) {
5046 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
5047 bp->ntp_fltr_count--;
5048 }
5049 kfree(fltr);
5050 }
5051
bnxt_free_ntp_fltrs(struct bnxt * bp,bool all)5052 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all)
5053 {
5054 int i;
5055
5056 /* Under rtnl_lock and all our NAPIs have been disabled. It's
5057 * safe to delete the hash table.
5058 */
5059 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
5060 struct hlist_head *head;
5061 struct hlist_node *tmp;
5062 struct bnxt_ntuple_filter *fltr;
5063
5064 head = &bp->ntp_fltr_hash_tbl[i];
5065 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
5066 bnxt_del_l2_filter(bp, fltr->l2_fltr);
5067 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
5068 !list_empty(&fltr->base.list)))
5069 continue;
5070 bnxt_del_fltr(bp, &fltr->base);
5071 }
5072 }
5073 if (!all)
5074 return;
5075
5076 bitmap_free(bp->ntp_fltr_bmap);
5077 bp->ntp_fltr_bmap = NULL;
5078 bp->ntp_fltr_count = 0;
5079 }
5080
bnxt_alloc_ntp_fltrs(struct bnxt * bp)5081 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
5082 {
5083 int i, rc = 0;
5084
5085 if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap)
5086 return 0;
5087
5088 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
5089 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
5090
5091 bp->ntp_fltr_count = 0;
5092 bp->ntp_fltr_bmap = bitmap_zalloc(bp->max_fltr, GFP_KERNEL);
5093
5094 if (!bp->ntp_fltr_bmap)
5095 rc = -ENOMEM;
5096
5097 return rc;
5098 }
5099
bnxt_free_l2_filters(struct bnxt * bp,bool all)5100 static void bnxt_free_l2_filters(struct bnxt *bp, bool all)
5101 {
5102 int i;
5103
5104 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) {
5105 struct hlist_head *head;
5106 struct hlist_node *tmp;
5107 struct bnxt_l2_filter *fltr;
5108
5109 head = &bp->l2_fltr_hash_tbl[i];
5110 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
5111 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
5112 !list_empty(&fltr->base.list)))
5113 continue;
5114 bnxt_del_fltr(bp, &fltr->base);
5115 }
5116 }
5117 }
5118
bnxt_init_l2_fltr_tbl(struct bnxt * bp)5119 static void bnxt_init_l2_fltr_tbl(struct bnxt *bp)
5120 {
5121 int i;
5122
5123 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++)
5124 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]);
5125 get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed));
5126 }
5127
bnxt_free_mem(struct bnxt * bp,bool irq_re_init)5128 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
5129 {
5130 bnxt_free_vnic_attributes(bp);
5131 bnxt_free_tx_rings(bp);
5132 bnxt_free_rx_rings(bp);
5133 bnxt_free_cp_rings(bp);
5134 bnxt_free_all_cp_arrays(bp);
5135 bnxt_free_ntp_fltrs(bp, false);
5136 bnxt_free_l2_filters(bp, false);
5137 if (irq_re_init) {
5138 bnxt_free_ring_stats(bp);
5139 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
5140 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
5141 bnxt_free_port_stats(bp);
5142 bnxt_free_ring_grps(bp);
5143 bnxt_free_vnics(bp);
5144 kfree(bp->tx_ring_map);
5145 bp->tx_ring_map = NULL;
5146 kfree(bp->tx_ring);
5147 bp->tx_ring = NULL;
5148 kfree(bp->rx_ring);
5149 bp->rx_ring = NULL;
5150 kfree(bp->bnapi);
5151 bp->bnapi = NULL;
5152 } else {
5153 bnxt_clear_ring_indices(bp);
5154 }
5155 }
5156
bnxt_alloc_mem(struct bnxt * bp,bool irq_re_init)5157 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
5158 {
5159 int i, j, rc, size, arr_size;
5160 void *bnapi;
5161
5162 if (irq_re_init) {
5163 /* Allocate bnapi mem pointer array and mem block for
5164 * all queues
5165 */
5166 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
5167 bp->cp_nr_rings);
5168 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
5169 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
5170 if (!bnapi)
5171 return -ENOMEM;
5172
5173 bp->bnapi = bnapi;
5174 bnapi += arr_size;
5175 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
5176 bp->bnapi[i] = bnapi;
5177 bp->bnapi[i]->index = i;
5178 bp->bnapi[i]->bp = bp;
5179 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5180 struct bnxt_cp_ring_info *cpr =
5181 &bp->bnapi[i]->cp_ring;
5182
5183 cpr->cp_ring_struct.ring_mem.flags =
5184 BNXT_RMEM_RING_PTE_FLAG;
5185 }
5186 }
5187
5188 bp->rx_ring = kcalloc(bp->rx_nr_rings,
5189 sizeof(struct bnxt_rx_ring_info),
5190 GFP_KERNEL);
5191 if (!bp->rx_ring)
5192 return -ENOMEM;
5193
5194 for (i = 0; i < bp->rx_nr_rings; i++) {
5195 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5196
5197 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5198 rxr->rx_ring_struct.ring_mem.flags =
5199 BNXT_RMEM_RING_PTE_FLAG;
5200 rxr->rx_agg_ring_struct.ring_mem.flags =
5201 BNXT_RMEM_RING_PTE_FLAG;
5202 } else {
5203 rxr->rx_cpr = &bp->bnapi[i]->cp_ring;
5204 }
5205 rxr->bnapi = bp->bnapi[i];
5206 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
5207 }
5208
5209 bp->tx_ring = kcalloc(bp->tx_nr_rings,
5210 sizeof(struct bnxt_tx_ring_info),
5211 GFP_KERNEL);
5212 if (!bp->tx_ring)
5213 return -ENOMEM;
5214
5215 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
5216 GFP_KERNEL);
5217
5218 if (!bp->tx_ring_map)
5219 return -ENOMEM;
5220
5221 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5222 j = 0;
5223 else
5224 j = bp->rx_nr_rings;
5225
5226 for (i = 0; i < bp->tx_nr_rings; i++) {
5227 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5228 struct bnxt_napi *bnapi2;
5229
5230 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5231 txr->tx_ring_struct.ring_mem.flags =
5232 BNXT_RMEM_RING_PTE_FLAG;
5233 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
5234 if (i >= bp->tx_nr_rings_xdp) {
5235 int k = j + BNXT_RING_TO_TC_OFF(bp, i);
5236
5237 bnapi2 = bp->bnapi[k];
5238 txr->txq_index = i - bp->tx_nr_rings_xdp;
5239 txr->tx_napi_idx =
5240 BNXT_RING_TO_TC(bp, txr->txq_index);
5241 bnapi2->tx_ring[txr->tx_napi_idx] = txr;
5242 bnapi2->tx_int = bnxt_tx_int;
5243 } else {
5244 bnapi2 = bp->bnapi[j];
5245 bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
5246 bnapi2->tx_ring[0] = txr;
5247 bnapi2->tx_int = bnxt_tx_int_xdp;
5248 j++;
5249 }
5250 txr->bnapi = bnapi2;
5251 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
5252 txr->tx_cpr = &bnapi2->cp_ring;
5253 }
5254
5255 rc = bnxt_alloc_stats(bp);
5256 if (rc)
5257 goto alloc_mem_err;
5258 bnxt_init_stats(bp);
5259
5260 rc = bnxt_alloc_ntp_fltrs(bp);
5261 if (rc)
5262 goto alloc_mem_err;
5263
5264 rc = bnxt_alloc_vnics(bp);
5265 if (rc)
5266 goto alloc_mem_err;
5267 }
5268
5269 rc = bnxt_alloc_all_cp_arrays(bp);
5270 if (rc)
5271 goto alloc_mem_err;
5272
5273 bnxt_init_ring_struct(bp);
5274
5275 rc = bnxt_alloc_rx_rings(bp);
5276 if (rc)
5277 goto alloc_mem_err;
5278
5279 rc = bnxt_alloc_tx_rings(bp);
5280 if (rc)
5281 goto alloc_mem_err;
5282
5283 rc = bnxt_alloc_cp_rings(bp);
5284 if (rc)
5285 goto alloc_mem_err;
5286
5287 bp->vnic_info[BNXT_VNIC_DEFAULT].flags |= BNXT_VNIC_RSS_FLAG |
5288 BNXT_VNIC_MCAST_FLAG |
5289 BNXT_VNIC_UCAST_FLAG;
5290 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp) && (bp->flags & BNXT_FLAG_RFS))
5291 bp->vnic_info[BNXT_VNIC_NTUPLE].flags |=
5292 BNXT_VNIC_RSS_FLAG | BNXT_VNIC_NTUPLE_FLAG;
5293
5294 rc = bnxt_alloc_vnic_attributes(bp);
5295 if (rc)
5296 goto alloc_mem_err;
5297 return 0;
5298
5299 alloc_mem_err:
5300 bnxt_free_mem(bp, true);
5301 return rc;
5302 }
5303
bnxt_disable_int(struct bnxt * bp)5304 static void bnxt_disable_int(struct bnxt *bp)
5305 {
5306 int i;
5307
5308 if (!bp->bnapi)
5309 return;
5310
5311 for (i = 0; i < bp->cp_nr_rings; i++) {
5312 struct bnxt_napi *bnapi = bp->bnapi[i];
5313 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5314 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5315
5316 if (ring->fw_ring_id != INVALID_HW_RING_ID)
5317 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5318 }
5319 }
5320
bnxt_cp_num_to_irq_num(struct bnxt * bp,int n)5321 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
5322 {
5323 struct bnxt_napi *bnapi = bp->bnapi[n];
5324 struct bnxt_cp_ring_info *cpr;
5325
5326 cpr = &bnapi->cp_ring;
5327 return cpr->cp_ring_struct.map_idx;
5328 }
5329
bnxt_disable_int_sync(struct bnxt * bp)5330 static void bnxt_disable_int_sync(struct bnxt *bp)
5331 {
5332 int i;
5333
5334 if (!bp->irq_tbl)
5335 return;
5336
5337 atomic_inc(&bp->intr_sem);
5338
5339 bnxt_disable_int(bp);
5340 for (i = 0; i < bp->cp_nr_rings; i++) {
5341 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5342
5343 synchronize_irq(bp->irq_tbl[map_idx].vector);
5344 }
5345 }
5346
bnxt_enable_int(struct bnxt * bp)5347 static void bnxt_enable_int(struct bnxt *bp)
5348 {
5349 int i;
5350
5351 atomic_set(&bp->intr_sem, 0);
5352 for (i = 0; i < bp->cp_nr_rings; i++) {
5353 struct bnxt_napi *bnapi = bp->bnapi[i];
5354 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5355
5356 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
5357 }
5358 }
5359
bnxt_hwrm_func_drv_rgtr(struct bnxt * bp,unsigned long * bmap,int bmap_size,bool async_only)5360 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
5361 bool async_only)
5362 {
5363 DECLARE_BITMAP(async_events_bmap, 256);
5364 u32 *events = (u32 *)async_events_bmap;
5365 struct hwrm_func_drv_rgtr_output *resp;
5366 struct hwrm_func_drv_rgtr_input *req;
5367 u32 flags;
5368 int rc, i;
5369
5370 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
5371 if (rc)
5372 return rc;
5373
5374 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
5375 FUNC_DRV_RGTR_REQ_ENABLES_VER |
5376 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5377
5378 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
5379 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
5380 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
5381 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
5382 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
5383 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
5384 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
5385 req->flags = cpu_to_le32(flags);
5386 req->ver_maj_8b = DRV_VER_MAJ;
5387 req->ver_min_8b = DRV_VER_MIN;
5388 req->ver_upd_8b = DRV_VER_UPD;
5389 req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
5390 req->ver_min = cpu_to_le16(DRV_VER_MIN);
5391 req->ver_upd = cpu_to_le16(DRV_VER_UPD);
5392
5393 if (BNXT_PF(bp)) {
5394 u32 data[8];
5395 int i;
5396
5397 memset(data, 0, sizeof(data));
5398 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
5399 u16 cmd = bnxt_vf_req_snif[i];
5400 unsigned int bit, idx;
5401
5402 idx = cmd / 32;
5403 bit = cmd % 32;
5404 data[idx] |= 1 << bit;
5405 }
5406
5407 for (i = 0; i < 8; i++)
5408 req->vf_req_fwd[i] = cpu_to_le32(data[i]);
5409
5410 req->enables |=
5411 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
5412 }
5413
5414 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
5415 req->flags |= cpu_to_le32(
5416 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
5417
5418 memset(async_events_bmap, 0, sizeof(async_events_bmap));
5419 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
5420 u16 event_id = bnxt_async_events_arr[i];
5421
5422 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
5423 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5424 continue;
5425 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
5426 !bp->ptp_cfg)
5427 continue;
5428 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
5429 }
5430 if (bmap && bmap_size) {
5431 for (i = 0; i < bmap_size; i++) {
5432 if (test_bit(i, bmap))
5433 __set_bit(i, async_events_bmap);
5434 }
5435 }
5436 for (i = 0; i < 8; i++)
5437 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
5438
5439 if (async_only)
5440 req->enables =
5441 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5442
5443 resp = hwrm_req_hold(bp, req);
5444 rc = hwrm_req_send(bp, req);
5445 if (!rc) {
5446 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
5447 if (resp->flags &
5448 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
5449 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
5450 }
5451 hwrm_req_drop(bp, req);
5452 return rc;
5453 }
5454
bnxt_hwrm_func_drv_unrgtr(struct bnxt * bp)5455 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
5456 {
5457 struct hwrm_func_drv_unrgtr_input *req;
5458 int rc;
5459
5460 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
5461 return 0;
5462
5463 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
5464 if (rc)
5465 return rc;
5466 return hwrm_req_send(bp, req);
5467 }
5468
5469 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa);
5470
bnxt_hwrm_tunnel_dst_port_free(struct bnxt * bp,u8 tunnel_type)5471 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
5472 {
5473 struct hwrm_tunnel_dst_port_free_input *req;
5474 int rc;
5475
5476 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
5477 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
5478 return 0;
5479 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
5480 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
5481 return 0;
5482
5483 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
5484 if (rc)
5485 return rc;
5486
5487 req->tunnel_type = tunnel_type;
5488
5489 switch (tunnel_type) {
5490 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
5491 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
5492 bp->vxlan_port = 0;
5493 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
5494 break;
5495 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
5496 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
5497 bp->nge_port = 0;
5498 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
5499 break;
5500 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE:
5501 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id);
5502 bp->vxlan_gpe_port = 0;
5503 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID;
5504 break;
5505 default:
5506 break;
5507 }
5508
5509 rc = hwrm_req_send(bp, req);
5510 if (rc)
5511 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
5512 rc);
5513 if (bp->flags & BNXT_FLAG_TPA)
5514 bnxt_set_tpa(bp, true);
5515 return rc;
5516 }
5517
bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt * bp,__be16 port,u8 tunnel_type)5518 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
5519 u8 tunnel_type)
5520 {
5521 struct hwrm_tunnel_dst_port_alloc_output *resp;
5522 struct hwrm_tunnel_dst_port_alloc_input *req;
5523 int rc;
5524
5525 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
5526 if (rc)
5527 return rc;
5528
5529 req->tunnel_type = tunnel_type;
5530 req->tunnel_dst_port_val = port;
5531
5532 resp = hwrm_req_hold(bp, req);
5533 rc = hwrm_req_send(bp, req);
5534 if (rc) {
5535 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
5536 rc);
5537 goto err_out;
5538 }
5539
5540 switch (tunnel_type) {
5541 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
5542 bp->vxlan_port = port;
5543 bp->vxlan_fw_dst_port_id =
5544 le16_to_cpu(resp->tunnel_dst_port_id);
5545 break;
5546 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
5547 bp->nge_port = port;
5548 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
5549 break;
5550 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE:
5551 bp->vxlan_gpe_port = port;
5552 bp->vxlan_gpe_fw_dst_port_id =
5553 le16_to_cpu(resp->tunnel_dst_port_id);
5554 break;
5555 default:
5556 break;
5557 }
5558 if (bp->flags & BNXT_FLAG_TPA)
5559 bnxt_set_tpa(bp, true);
5560
5561 err_out:
5562 hwrm_req_drop(bp, req);
5563 return rc;
5564 }
5565
bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt * bp,u16 vnic_id)5566 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
5567 {
5568 struct hwrm_cfa_l2_set_rx_mask_input *req;
5569 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5570 int rc;
5571
5572 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
5573 if (rc)
5574 return rc;
5575
5576 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5577 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
5578 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
5579 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
5580 }
5581 req->mask = cpu_to_le32(vnic->rx_mask);
5582 return hwrm_req_send_silent(bp, req);
5583 }
5584
bnxt_del_l2_filter(struct bnxt * bp,struct bnxt_l2_filter * fltr)5585 void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5586 {
5587 if (!atomic_dec_and_test(&fltr->refcnt))
5588 return;
5589 spin_lock_bh(&bp->ntp_fltr_lock);
5590 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
5591 spin_unlock_bh(&bp->ntp_fltr_lock);
5592 return;
5593 }
5594 hlist_del_rcu(&fltr->base.hash);
5595 bnxt_del_one_usr_fltr(bp, &fltr->base);
5596 if (fltr->base.flags) {
5597 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
5598 bp->ntp_fltr_count--;
5599 }
5600 spin_unlock_bh(&bp->ntp_fltr_lock);
5601 kfree_rcu(fltr, base.rcu);
5602 }
5603
__bnxt_lookup_l2_filter(struct bnxt * bp,struct bnxt_l2_key * key,u32 idx)5604 static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp,
5605 struct bnxt_l2_key *key,
5606 u32 idx)
5607 {
5608 struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx];
5609 struct bnxt_l2_filter *fltr;
5610
5611 hlist_for_each_entry_rcu(fltr, head, base.hash) {
5612 struct bnxt_l2_key *l2_key = &fltr->l2_key;
5613
5614 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
5615 l2_key->vlan == key->vlan)
5616 return fltr;
5617 }
5618 return NULL;
5619 }
5620
bnxt_lookup_l2_filter(struct bnxt * bp,struct bnxt_l2_key * key,u32 idx)5621 static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp,
5622 struct bnxt_l2_key *key,
5623 u32 idx)
5624 {
5625 struct bnxt_l2_filter *fltr = NULL;
5626
5627 rcu_read_lock();
5628 fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5629 if (fltr)
5630 atomic_inc(&fltr->refcnt);
5631 rcu_read_unlock();
5632 return fltr;
5633 }
5634
5635 #define BNXT_IPV4_4TUPLE(bp, fkeys) \
5636 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5637 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) || \
5638 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5639 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4))
5640
5641 #define BNXT_IPV6_4TUPLE(bp, fkeys) \
5642 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5643 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) || \
5644 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5645 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6))
5646
bnxt_get_rss_flow_tuple_len(struct bnxt * bp,struct flow_keys * fkeys)5647 static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys)
5648 {
5649 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5650 if (BNXT_IPV4_4TUPLE(bp, fkeys))
5651 return sizeof(fkeys->addrs.v4addrs) +
5652 sizeof(fkeys->ports);
5653
5654 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4)
5655 return sizeof(fkeys->addrs.v4addrs);
5656 }
5657
5658 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
5659 if (BNXT_IPV6_4TUPLE(bp, fkeys))
5660 return sizeof(fkeys->addrs.v6addrs) +
5661 sizeof(fkeys->ports);
5662
5663 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6)
5664 return sizeof(fkeys->addrs.v6addrs);
5665 }
5666
5667 return 0;
5668 }
5669
bnxt_toeplitz(struct bnxt * bp,struct flow_keys * fkeys,const unsigned char * key)5670 static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys,
5671 const unsigned char *key)
5672 {
5673 u64 prefix = bp->toeplitz_prefix, hash = 0;
5674 struct bnxt_ipv4_tuple tuple4;
5675 struct bnxt_ipv6_tuple tuple6;
5676 int i, j, len = 0;
5677 u8 *four_tuple;
5678
5679 len = bnxt_get_rss_flow_tuple_len(bp, fkeys);
5680 if (!len)
5681 return 0;
5682
5683 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5684 tuple4.v4addrs = fkeys->addrs.v4addrs;
5685 tuple4.ports = fkeys->ports;
5686 four_tuple = (unsigned char *)&tuple4;
5687 } else {
5688 tuple6.v6addrs = fkeys->addrs.v6addrs;
5689 tuple6.ports = fkeys->ports;
5690 four_tuple = (unsigned char *)&tuple6;
5691 }
5692
5693 for (i = 0, j = 8; i < len; i++, j++) {
5694 u8 byte = four_tuple[i];
5695 int bit;
5696
5697 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) {
5698 if (byte & 0x80)
5699 hash ^= prefix;
5700 }
5701 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0;
5702 }
5703
5704 /* The valid part of the hash is in the upper 32 bits. */
5705 return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK;
5706 }
5707
5708 #ifdef CONFIG_RFS_ACCEL
5709 static struct bnxt_l2_filter *
bnxt_lookup_l2_filter_from_key(struct bnxt * bp,struct bnxt_l2_key * key)5710 bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key)
5711 {
5712 struct bnxt_l2_filter *fltr;
5713 u32 idx;
5714
5715 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5716 BNXT_L2_FLTR_HASH_MASK;
5717 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5718 return fltr;
5719 }
5720 #endif
5721
bnxt_init_l2_filter(struct bnxt * bp,struct bnxt_l2_filter * fltr,struct bnxt_l2_key * key,u32 idx)5722 static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr,
5723 struct bnxt_l2_key *key, u32 idx)
5724 {
5725 struct hlist_head *head;
5726
5727 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
5728 fltr->l2_key.vlan = key->vlan;
5729 fltr->base.type = BNXT_FLTR_TYPE_L2;
5730 if (fltr->base.flags) {
5731 int bit_id;
5732
5733 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5734 bp->max_fltr, 0);
5735 if (bit_id < 0)
5736 return -ENOMEM;
5737 fltr->base.sw_id = (u16)bit_id;
5738 bp->ntp_fltr_count++;
5739 }
5740 head = &bp->l2_fltr_hash_tbl[idx];
5741 hlist_add_head_rcu(&fltr->base.hash, head);
5742 bnxt_insert_usr_fltr(bp, &fltr->base);
5743 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
5744 atomic_set(&fltr->refcnt, 1);
5745 return 0;
5746 }
5747
bnxt_alloc_l2_filter(struct bnxt * bp,struct bnxt_l2_key * key,gfp_t gfp)5748 static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp,
5749 struct bnxt_l2_key *key,
5750 gfp_t gfp)
5751 {
5752 struct bnxt_l2_filter *fltr;
5753 u32 idx;
5754 int rc;
5755
5756 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5757 BNXT_L2_FLTR_HASH_MASK;
5758 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5759 if (fltr)
5760 return fltr;
5761
5762 fltr = kzalloc(sizeof(*fltr), gfp);
5763 if (!fltr)
5764 return ERR_PTR(-ENOMEM);
5765 spin_lock_bh(&bp->ntp_fltr_lock);
5766 rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5767 spin_unlock_bh(&bp->ntp_fltr_lock);
5768 if (rc) {
5769 bnxt_del_l2_filter(bp, fltr);
5770 fltr = ERR_PTR(rc);
5771 }
5772 return fltr;
5773 }
5774
bnxt_alloc_new_l2_filter(struct bnxt * bp,struct bnxt_l2_key * key,u16 flags)5775 struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp,
5776 struct bnxt_l2_key *key,
5777 u16 flags)
5778 {
5779 struct bnxt_l2_filter *fltr;
5780 u32 idx;
5781 int rc;
5782
5783 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5784 BNXT_L2_FLTR_HASH_MASK;
5785 spin_lock_bh(&bp->ntp_fltr_lock);
5786 fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5787 if (fltr) {
5788 fltr = ERR_PTR(-EEXIST);
5789 goto l2_filter_exit;
5790 }
5791 fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC);
5792 if (!fltr) {
5793 fltr = ERR_PTR(-ENOMEM);
5794 goto l2_filter_exit;
5795 }
5796 fltr->base.flags = flags;
5797 rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5798 if (rc) {
5799 spin_unlock_bh(&bp->ntp_fltr_lock);
5800 bnxt_del_l2_filter(bp, fltr);
5801 return ERR_PTR(rc);
5802 }
5803
5804 l2_filter_exit:
5805 spin_unlock_bh(&bp->ntp_fltr_lock);
5806 return fltr;
5807 }
5808
bnxt_vf_target_id(struct bnxt_pf_info * pf,u16 vf_idx)5809 static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx)
5810 {
5811 #ifdef CONFIG_BNXT_SRIOV
5812 struct bnxt_vf_info *vf = &pf->vf[vf_idx];
5813
5814 return vf->fw_fid;
5815 #else
5816 return INVALID_HW_RING_ID;
5817 #endif
5818 }
5819
bnxt_hwrm_l2_filter_free(struct bnxt * bp,struct bnxt_l2_filter * fltr)5820 int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5821 {
5822 struct hwrm_cfa_l2_filter_free_input *req;
5823 u16 target_id = 0xffff;
5824 int rc;
5825
5826 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5827 struct bnxt_pf_info *pf = &bp->pf;
5828
5829 if (fltr->base.vf_idx >= pf->active_vfs)
5830 return -EINVAL;
5831
5832 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5833 if (target_id == INVALID_HW_RING_ID)
5834 return -EINVAL;
5835 }
5836
5837 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5838 if (rc)
5839 return rc;
5840
5841 req->target_id = cpu_to_le16(target_id);
5842 req->l2_filter_id = fltr->base.filter_id;
5843 return hwrm_req_send(bp, req);
5844 }
5845
bnxt_hwrm_l2_filter_alloc(struct bnxt * bp,struct bnxt_l2_filter * fltr)5846 int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5847 {
5848 struct hwrm_cfa_l2_filter_alloc_output *resp;
5849 struct hwrm_cfa_l2_filter_alloc_input *req;
5850 u16 target_id = 0xffff;
5851 int rc;
5852
5853 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5854 struct bnxt_pf_info *pf = &bp->pf;
5855
5856 if (fltr->base.vf_idx >= pf->active_vfs)
5857 return -EINVAL;
5858
5859 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5860 }
5861 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5862 if (rc)
5863 return rc;
5864
5865 req->target_id = cpu_to_le16(target_id);
5866 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5867
5868 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5869 req->flags |=
5870 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5871 req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
5872 req->enables =
5873 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5874 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5875 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5876 ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
5877 eth_broadcast_addr(req->l2_addr_mask);
5878
5879 if (fltr->l2_key.vlan) {
5880 req->enables |=
5881 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
5882 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
5883 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
5884 req->num_vlans = 1;
5885 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
5886 req->l2_ivlan_mask = cpu_to_le16(0xfff);
5887 }
5888
5889 resp = hwrm_req_hold(bp, req);
5890 rc = hwrm_req_send(bp, req);
5891 if (!rc) {
5892 fltr->base.filter_id = resp->l2_filter_id;
5893 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
5894 }
5895 hwrm_req_drop(bp, req);
5896 return rc;
5897 }
5898
bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt * bp,struct bnxt_ntuple_filter * fltr)5899 int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
5900 struct bnxt_ntuple_filter *fltr)
5901 {
5902 struct hwrm_cfa_ntuple_filter_free_input *req;
5903 int rc;
5904
5905 set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state);
5906 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
5907 if (rc)
5908 return rc;
5909
5910 req->ntuple_filter_id = fltr->base.filter_id;
5911 return hwrm_req_send(bp, req);
5912 }
5913
5914 #define BNXT_NTP_FLTR_FLAGS \
5915 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
5916 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
5917 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
5918 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
5919 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
5920 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
5921 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
5922 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
5923 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
5924 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
5925 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
5926 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
5927 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
5928
5929 #define BNXT_NTP_TUNNEL_FLTR_FLAG \
5930 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
5931
bnxt_fill_ipv6_mask(__be32 mask[4])5932 void bnxt_fill_ipv6_mask(__be32 mask[4])
5933 {
5934 int i;
5935
5936 for (i = 0; i < 4; i++)
5937 mask[i] = cpu_to_be32(~0);
5938 }
5939
5940 static void
bnxt_cfg_rfs_ring_tbl_idx(struct bnxt * bp,struct hwrm_cfa_ntuple_filter_alloc_input * req,struct bnxt_ntuple_filter * fltr)5941 bnxt_cfg_rfs_ring_tbl_idx(struct bnxt *bp,
5942 struct hwrm_cfa_ntuple_filter_alloc_input *req,
5943 struct bnxt_ntuple_filter *fltr)
5944 {
5945 u16 rxq = fltr->base.rxq;
5946
5947 if (fltr->base.flags & BNXT_ACT_RSS_CTX) {
5948 struct ethtool_rxfh_context *ctx;
5949 struct bnxt_rss_ctx *rss_ctx;
5950 struct bnxt_vnic_info *vnic;
5951
5952 ctx = xa_load(&bp->dev->ethtool->rss_ctx,
5953 fltr->base.fw_vnic_id);
5954 if (ctx) {
5955 rss_ctx = ethtool_rxfh_context_priv(ctx);
5956 vnic = &rss_ctx->vnic;
5957
5958 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5959 }
5960 return;
5961 }
5962 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
5963 struct bnxt_vnic_info *vnic;
5964 u32 enables;
5965
5966 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
5967 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5968 enables = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
5969 req->enables |= cpu_to_le32(enables);
5970 req->rfs_ring_tbl_idx = cpu_to_le16(rxq);
5971 } else {
5972 u32 flags;
5973
5974 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
5975 req->flags |= cpu_to_le32(flags);
5976 req->dst_id = cpu_to_le16(rxq);
5977 }
5978 }
5979
bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt * bp,struct bnxt_ntuple_filter * fltr)5980 int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
5981 struct bnxt_ntuple_filter *fltr)
5982 {
5983 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
5984 struct hwrm_cfa_ntuple_filter_alloc_input *req;
5985 struct bnxt_flow_masks *masks = &fltr->fmasks;
5986 struct flow_keys *keys = &fltr->fkeys;
5987 struct bnxt_l2_filter *l2_fltr;
5988 struct bnxt_vnic_info *vnic;
5989 int rc;
5990
5991 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
5992 if (rc)
5993 return rc;
5994
5995 l2_fltr = fltr->l2_fltr;
5996 req->l2_filter_id = l2_fltr->base.filter_id;
5997
5998 if (fltr->base.flags & BNXT_ACT_DROP) {
5999 req->flags =
6000 cpu_to_le32(CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP);
6001 } else if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
6002 bnxt_cfg_rfs_ring_tbl_idx(bp, req, fltr);
6003 } else {
6004 vnic = &bp->vnic_info[fltr->base.rxq + 1];
6005 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
6006 }
6007 req->enables |= cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
6008
6009 req->ethertype = htons(ETH_P_IP);
6010 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
6011 req->ip_protocol = keys->basic.ip_proto;
6012
6013 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
6014 req->ethertype = htons(ETH_P_IPV6);
6015 req->ip_addr_type =
6016 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
6017 *(struct in6_addr *)&req->src_ipaddr[0] = keys->addrs.v6addrs.src;
6018 *(struct in6_addr *)&req->src_ipaddr_mask[0] = masks->addrs.v6addrs.src;
6019 *(struct in6_addr *)&req->dst_ipaddr[0] = keys->addrs.v6addrs.dst;
6020 *(struct in6_addr *)&req->dst_ipaddr_mask[0] = masks->addrs.v6addrs.dst;
6021 } else {
6022 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
6023 req->src_ipaddr_mask[0] = masks->addrs.v4addrs.src;
6024 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
6025 req->dst_ipaddr_mask[0] = masks->addrs.v4addrs.dst;
6026 }
6027 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
6028 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
6029 req->tunnel_type =
6030 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
6031 }
6032
6033 req->src_port = keys->ports.src;
6034 req->src_port_mask = masks->ports.src;
6035 req->dst_port = keys->ports.dst;
6036 req->dst_port_mask = masks->ports.dst;
6037
6038 resp = hwrm_req_hold(bp, req);
6039 rc = hwrm_req_send(bp, req);
6040 if (!rc)
6041 fltr->base.filter_id = resp->ntuple_filter_id;
6042 hwrm_req_drop(bp, req);
6043 return rc;
6044 }
6045
bnxt_hwrm_set_vnic_filter(struct bnxt * bp,u16 vnic_id,u16 idx,const u8 * mac_addr)6046 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
6047 const u8 *mac_addr)
6048 {
6049 struct bnxt_l2_filter *fltr;
6050 struct bnxt_l2_key key;
6051 int rc;
6052
6053 ether_addr_copy(key.dst_mac_addr, mac_addr);
6054 key.vlan = 0;
6055 fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL);
6056 if (IS_ERR(fltr))
6057 return PTR_ERR(fltr);
6058
6059 fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id;
6060 rc = bnxt_hwrm_l2_filter_alloc(bp, fltr);
6061 if (rc)
6062 bnxt_del_l2_filter(bp, fltr);
6063 else
6064 bp->vnic_info[vnic_id].l2_filters[idx] = fltr;
6065 return rc;
6066 }
6067
bnxt_hwrm_clear_vnic_filter(struct bnxt * bp)6068 static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
6069 {
6070 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
6071
6072 /* Any associated ntuple filters will also be cleared by firmware. */
6073 for (i = 0; i < num_of_vnics; i++) {
6074 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6075
6076 for (j = 0; j < vnic->uc_filter_count; j++) {
6077 struct bnxt_l2_filter *fltr = vnic->l2_filters[j];
6078
6079 bnxt_hwrm_l2_filter_free(bp, fltr);
6080 bnxt_del_l2_filter(bp, fltr);
6081 }
6082 vnic->uc_filter_count = 0;
6083 }
6084 }
6085
6086 #define BNXT_DFLT_TUNL_TPA_BMAP \
6087 (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \
6088 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \
6089 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
6090
bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt * bp,struct hwrm_vnic_tpa_cfg_input * req)6091 static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp,
6092 struct hwrm_vnic_tpa_cfg_input *req)
6093 {
6094 u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP;
6095
6096 if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA))
6097 return;
6098
6099 if (bp->vxlan_port)
6100 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
6101 if (bp->vxlan_gpe_port)
6102 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
6103 if (bp->nge_port)
6104 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
6105
6106 req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
6107 req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
6108 }
6109
bnxt_hwrm_vnic_set_tpa(struct bnxt * bp,struct bnxt_vnic_info * vnic,u32 tpa_flags)6110 int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6111 u32 tpa_flags)
6112 {
6113 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
6114 struct hwrm_vnic_tpa_cfg_input *req;
6115 int rc;
6116
6117 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
6118 return 0;
6119
6120 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
6121 if (rc)
6122 return rc;
6123
6124 if (tpa_flags) {
6125 u16 mss = bp->dev->mtu - 40;
6126 u32 nsegs, n, segs = 0, flags;
6127
6128 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
6129 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
6130 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
6131 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
6132 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
6133 if (tpa_flags & BNXT_FLAG_GRO)
6134 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
6135
6136 req->flags = cpu_to_le32(flags);
6137
6138 req->enables =
6139 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
6140 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
6141 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
6142
6143 /* Number of segs are log2 units, and first packet is not
6144 * included as part of this units.
6145 */
6146 if (mss <= BNXT_RX_PAGE_SIZE) {
6147 n = BNXT_RX_PAGE_SIZE / mss;
6148 nsegs = (MAX_SKB_FRAGS - 1) * n;
6149 } else {
6150 n = mss / BNXT_RX_PAGE_SIZE;
6151 if (mss & (BNXT_RX_PAGE_SIZE - 1))
6152 n++;
6153 nsegs = (MAX_SKB_FRAGS - n) / n;
6154 }
6155
6156 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6157 segs = MAX_TPA_SEGS_P5;
6158 max_aggs = bp->max_tpa;
6159 } else {
6160 segs = ilog2(nsegs);
6161 }
6162 req->max_agg_segs = cpu_to_le16(segs);
6163 req->max_aggs = cpu_to_le16(max_aggs);
6164
6165 req->min_agg_len = cpu_to_le32(512);
6166 bnxt_hwrm_vnic_update_tunl_tpa(bp, req);
6167 }
6168 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6169
6170 return hwrm_req_send(bp, req);
6171 }
6172
bnxt_cp_ring_from_grp(struct bnxt * bp,struct bnxt_ring_struct * ring)6173 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
6174 {
6175 struct bnxt_ring_grp_info *grp_info;
6176
6177 grp_info = &bp->grp_info[ring->grp_idx];
6178 return grp_info->cp_fw_ring_id;
6179 }
6180
bnxt_cp_ring_for_rx(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)6181 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
6182 {
6183 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6184 return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
6185 else
6186 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
6187 }
6188
bnxt_cp_ring_for_tx(struct bnxt * bp,struct bnxt_tx_ring_info * txr)6189 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
6190 {
6191 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6192 return txr->tx_cpr->cp_ring_struct.fw_ring_id;
6193 else
6194 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
6195 }
6196
bnxt_alloc_rss_indir_tbl(struct bnxt * bp)6197 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
6198 {
6199 int entries;
6200
6201 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6202 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
6203 else
6204 entries = HW_HASH_INDEX_SIZE;
6205
6206 bp->rss_indir_tbl_entries = entries;
6207 bp->rss_indir_tbl =
6208 kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), GFP_KERNEL);
6209 if (!bp->rss_indir_tbl)
6210 return -ENOMEM;
6211
6212 return 0;
6213 }
6214
bnxt_set_dflt_rss_indir_tbl(struct bnxt * bp,struct ethtool_rxfh_context * rss_ctx)6215 void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp,
6216 struct ethtool_rxfh_context *rss_ctx)
6217 {
6218 u16 max_rings, max_entries, pad, i;
6219 u32 *rss_indir_tbl;
6220
6221 if (!bp->rx_nr_rings)
6222 return;
6223
6224 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6225 max_rings = bp->rx_nr_rings - 1;
6226 else
6227 max_rings = bp->rx_nr_rings;
6228
6229 max_entries = bnxt_get_rxfh_indir_size(bp->dev);
6230 if (rss_ctx)
6231 rss_indir_tbl = ethtool_rxfh_context_indir(rss_ctx);
6232 else
6233 rss_indir_tbl = &bp->rss_indir_tbl[0];
6234
6235 for (i = 0; i < max_entries; i++)
6236 rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
6237
6238 pad = bp->rss_indir_tbl_entries - max_entries;
6239 if (pad)
6240 memset(&rss_indir_tbl[i], 0, pad * sizeof(*rss_indir_tbl));
6241 }
6242
bnxt_get_max_rss_ring(struct bnxt * bp)6243 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
6244 {
6245 u32 i, tbl_size, max_ring = 0;
6246
6247 if (!bp->rss_indir_tbl)
6248 return 0;
6249
6250 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6251 for (i = 0; i < tbl_size; i++)
6252 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
6253 return max_ring;
6254 }
6255
bnxt_get_nr_rss_ctxs(struct bnxt * bp,int rx_rings)6256 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
6257 {
6258 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6259 if (!rx_rings)
6260 return 0;
6261 return bnxt_calc_nr_ring_pages(rx_rings - 1,
6262 BNXT_RSS_TABLE_ENTRIES_P5);
6263 }
6264 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6265 return 2;
6266 return 1;
6267 }
6268
bnxt_fill_hw_rss_tbl(struct bnxt * bp,struct bnxt_vnic_info * vnic)6269 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6270 {
6271 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
6272 u16 i, j;
6273
6274 /* Fill the RSS indirection table with ring group ids */
6275 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
6276 if (!no_rss)
6277 j = bp->rss_indir_tbl[i];
6278 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
6279 }
6280 }
6281
bnxt_fill_hw_rss_tbl_p5(struct bnxt * bp,struct bnxt_vnic_info * vnic)6282 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
6283 struct bnxt_vnic_info *vnic)
6284 {
6285 __le16 *ring_tbl = vnic->rss_table;
6286 struct bnxt_rx_ring_info *rxr;
6287 u16 tbl_size, i;
6288
6289 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6290
6291 for (i = 0; i < tbl_size; i++) {
6292 u16 ring_id, j;
6293
6294 if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG)
6295 j = ethtool_rxfh_indir_default(i, bp->rx_nr_rings);
6296 else if (vnic->flags & BNXT_VNIC_RSSCTX_FLAG)
6297 j = ethtool_rxfh_context_indir(vnic->rss_ctx)[i];
6298 else
6299 j = bp->rss_indir_tbl[i];
6300 rxr = &bp->rx_ring[j];
6301
6302 ring_id = rxr->rx_ring_struct.fw_ring_id;
6303 *ring_tbl++ = cpu_to_le16(ring_id);
6304 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6305 *ring_tbl++ = cpu_to_le16(ring_id);
6306 }
6307 }
6308
6309 static void
__bnxt_hwrm_vnic_set_rss(struct bnxt * bp,struct hwrm_vnic_rss_cfg_input * req,struct bnxt_vnic_info * vnic)6310 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
6311 struct bnxt_vnic_info *vnic)
6312 {
6313 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6314 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
6315 if (bp->flags & BNXT_FLAG_CHIP_P7)
6316 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT;
6317 } else {
6318 bnxt_fill_hw_rss_tbl(bp, vnic);
6319 }
6320
6321 if (bp->rss_hash_delta) {
6322 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
6323 if (bp->rss_hash_cfg & bp->rss_hash_delta)
6324 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
6325 else
6326 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
6327 } else {
6328 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
6329 }
6330 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
6331 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
6332 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
6333 }
6334
bnxt_hwrm_vnic_set_rss(struct bnxt * bp,struct bnxt_vnic_info * vnic,bool set_rss)6335 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6336 bool set_rss)
6337 {
6338 struct hwrm_vnic_rss_cfg_input *req;
6339 int rc;
6340
6341 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ||
6342 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
6343 return 0;
6344
6345 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6346 if (rc)
6347 return rc;
6348
6349 if (set_rss)
6350 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6351 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6352 return hwrm_req_send(bp, req);
6353 }
6354
bnxt_hwrm_vnic_set_rss_p5(struct bnxt * bp,struct bnxt_vnic_info * vnic,bool set_rss)6355 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp,
6356 struct bnxt_vnic_info *vnic, bool set_rss)
6357 {
6358 struct hwrm_vnic_rss_cfg_input *req;
6359 dma_addr_t ring_tbl_map;
6360 u32 i, nr_ctxs;
6361 int rc;
6362
6363 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6364 if (rc)
6365 return rc;
6366
6367 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6368 if (!set_rss)
6369 return hwrm_req_send(bp, req);
6370
6371 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6372 ring_tbl_map = vnic->rss_table_dma_addr;
6373 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
6374
6375 hwrm_req_hold(bp, req);
6376 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
6377 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
6378 req->ring_table_pair_index = i;
6379 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
6380 rc = hwrm_req_send(bp, req);
6381 if (rc)
6382 goto exit;
6383 }
6384
6385 exit:
6386 hwrm_req_drop(bp, req);
6387 return rc;
6388 }
6389
bnxt_hwrm_update_rss_hash_cfg(struct bnxt * bp)6390 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
6391 {
6392 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6393 struct hwrm_vnic_rss_qcfg_output *resp;
6394 struct hwrm_vnic_rss_qcfg_input *req;
6395
6396 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
6397 return;
6398
6399 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6400 /* all contexts configured to same hash_type, zero always exists */
6401 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6402 resp = hwrm_req_hold(bp, req);
6403 if (!hwrm_req_send(bp, req)) {
6404 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
6405 bp->rss_hash_delta = 0;
6406 }
6407 hwrm_req_drop(bp, req);
6408 }
6409
bnxt_hwrm_vnic_set_hds(struct bnxt * bp,struct bnxt_vnic_info * vnic)6410 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6411 {
6412 struct hwrm_vnic_plcmodes_cfg_input *req;
6413 int rc;
6414
6415 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
6416 if (rc)
6417 return rc;
6418
6419 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
6420 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
6421
6422 if (BNXT_RX_PAGE_MODE(bp)) {
6423 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
6424 } else {
6425 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
6426 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
6427 req->enables |=
6428 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
6429 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
6430 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
6431 }
6432 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6433 return hwrm_req_send(bp, req);
6434 }
6435
bnxt_hwrm_vnic_ctx_free_one(struct bnxt * bp,struct bnxt_vnic_info * vnic,u16 ctx_idx)6436 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp,
6437 struct bnxt_vnic_info *vnic,
6438 u16 ctx_idx)
6439 {
6440 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
6441
6442 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
6443 return;
6444
6445 req->rss_cos_lb_ctx_id =
6446 cpu_to_le16(vnic->fw_rss_cos_lb_ctx[ctx_idx]);
6447
6448 hwrm_req_send(bp, req);
6449 vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
6450 }
6451
bnxt_hwrm_vnic_ctx_free(struct bnxt * bp)6452 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
6453 {
6454 int i, j;
6455
6456 for (i = 0; i < bp->nr_vnics; i++) {
6457 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6458
6459 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
6460 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
6461 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, j);
6462 }
6463 }
6464 bp->rsscos_nr_ctxs = 0;
6465 }
6466
bnxt_hwrm_vnic_ctx_alloc(struct bnxt * bp,struct bnxt_vnic_info * vnic,u16 ctx_idx)6467 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp,
6468 struct bnxt_vnic_info *vnic, u16 ctx_idx)
6469 {
6470 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
6471 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
6472 int rc;
6473
6474 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
6475 if (rc)
6476 return rc;
6477
6478 resp = hwrm_req_hold(bp, req);
6479 rc = hwrm_req_send(bp, req);
6480 if (!rc)
6481 vnic->fw_rss_cos_lb_ctx[ctx_idx] =
6482 le16_to_cpu(resp->rss_cos_lb_ctx_id);
6483 hwrm_req_drop(bp, req);
6484
6485 return rc;
6486 }
6487
bnxt_get_roce_vnic_mode(struct bnxt * bp)6488 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
6489 {
6490 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
6491 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
6492 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
6493 }
6494
bnxt_hwrm_vnic_cfg(struct bnxt * bp,struct bnxt_vnic_info * vnic)6495 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6496 {
6497 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6498 struct hwrm_vnic_cfg_input *req;
6499 unsigned int ring = 0, grp_idx;
6500 u16 def_vlan = 0;
6501 int rc;
6502
6503 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
6504 if (rc)
6505 return rc;
6506
6507 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6508 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
6509
6510 req->default_rx_ring_id =
6511 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
6512 req->default_cmpl_ring_id =
6513 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
6514 req->enables =
6515 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
6516 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
6517 goto vnic_mru;
6518 }
6519 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
6520 /* Only RSS support for now TBD: COS & LB */
6521 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
6522 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6523 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6524 VNIC_CFG_REQ_ENABLES_MRU);
6525 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
6526 req->rss_rule = cpu_to_le16(vnic0->fw_rss_cos_lb_ctx[0]);
6527 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6528 VNIC_CFG_REQ_ENABLES_MRU);
6529 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
6530 } else {
6531 req->rss_rule = cpu_to_le16(0xffff);
6532 }
6533
6534 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
6535 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
6536 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
6537 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
6538 } else {
6539 req->cos_rule = cpu_to_le16(0xffff);
6540 }
6541
6542 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
6543 ring = 0;
6544 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
6545 ring = vnic->vnic_id - 1;
6546 else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
6547 ring = bp->rx_nr_rings - 1;
6548
6549 grp_idx = bp->rx_ring[ring].bnapi->index;
6550 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
6551 req->lb_rule = cpu_to_le16(0xffff);
6552 vnic_mru:
6553 vnic->mru = bp->dev->mtu + ETH_HLEN + VLAN_HLEN;
6554 req->mru = cpu_to_le16(vnic->mru);
6555
6556 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6557 #ifdef CONFIG_BNXT_SRIOV
6558 if (BNXT_VF(bp))
6559 def_vlan = bp->vf.vlan;
6560 #endif
6561 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
6562 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
6563 if (vnic->vnic_id == BNXT_VNIC_DEFAULT && bnxt_ulp_registered(bp->edev))
6564 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
6565
6566 return hwrm_req_send(bp, req);
6567 }
6568
bnxt_hwrm_vnic_free_one(struct bnxt * bp,struct bnxt_vnic_info * vnic)6569 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp,
6570 struct bnxt_vnic_info *vnic)
6571 {
6572 if (vnic->fw_vnic_id != INVALID_HW_RING_ID) {
6573 struct hwrm_vnic_free_input *req;
6574
6575 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
6576 return;
6577
6578 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6579
6580 hwrm_req_send(bp, req);
6581 vnic->fw_vnic_id = INVALID_HW_RING_ID;
6582 }
6583 }
6584
bnxt_hwrm_vnic_free(struct bnxt * bp)6585 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
6586 {
6587 u16 i;
6588
6589 for (i = 0; i < bp->nr_vnics; i++)
6590 bnxt_hwrm_vnic_free_one(bp, &bp->vnic_info[i]);
6591 }
6592
bnxt_hwrm_vnic_alloc(struct bnxt * bp,struct bnxt_vnic_info * vnic,unsigned int start_rx_ring_idx,unsigned int nr_rings)6593 int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6594 unsigned int start_rx_ring_idx,
6595 unsigned int nr_rings)
6596 {
6597 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
6598 struct hwrm_vnic_alloc_output *resp;
6599 struct hwrm_vnic_alloc_input *req;
6600 int rc;
6601
6602 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
6603 if (rc)
6604 return rc;
6605
6606 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6607 goto vnic_no_ring_grps;
6608
6609 /* map ring groups to this vnic */
6610 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
6611 grp_idx = bp->rx_ring[i].bnapi->index;
6612 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
6613 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
6614 j, nr_rings);
6615 break;
6616 }
6617 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
6618 }
6619
6620 vnic_no_ring_grps:
6621 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
6622 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
6623 if (vnic->vnic_id == BNXT_VNIC_DEFAULT)
6624 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
6625
6626 resp = hwrm_req_hold(bp, req);
6627 rc = hwrm_req_send(bp, req);
6628 if (!rc)
6629 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
6630 hwrm_req_drop(bp, req);
6631 return rc;
6632 }
6633
bnxt_hwrm_vnic_qcaps(struct bnxt * bp)6634 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
6635 {
6636 struct hwrm_vnic_qcaps_output *resp;
6637 struct hwrm_vnic_qcaps_input *req;
6638 int rc;
6639
6640 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
6641 bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP;
6642 bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP;
6643 if (bp->hwrm_spec_code < 0x10600)
6644 return 0;
6645
6646 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
6647 if (rc)
6648 return rc;
6649
6650 resp = hwrm_req_hold(bp, req);
6651 rc = hwrm_req_send(bp, req);
6652 if (!rc) {
6653 u32 flags = le32_to_cpu(resp->flags);
6654
6655 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
6656 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
6657 bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP;
6658 if (flags &
6659 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
6660 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
6661
6662 /* Older P5 fw before EXT_HW_STATS support did not set
6663 * VLAN_STRIP_CAP properly.
6664 */
6665 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
6666 (BNXT_CHIP_P5(bp) &&
6667 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
6668 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
6669 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
6670 bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA;
6671 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED)
6672 bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM;
6673 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
6674 if (bp->max_tpa_v2) {
6675 if (BNXT_CHIP_P5(bp))
6676 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
6677 else
6678 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7;
6679 }
6680 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP)
6681 bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA;
6682 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP)
6683 bp->rss_cap |= BNXT_RSS_CAP_AH_V4_RSS_CAP;
6684 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP)
6685 bp->rss_cap |= BNXT_RSS_CAP_AH_V6_RSS_CAP;
6686 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP)
6687 bp->rss_cap |= BNXT_RSS_CAP_ESP_V4_RSS_CAP;
6688 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP)
6689 bp->rss_cap |= BNXT_RSS_CAP_ESP_V6_RSS_CAP;
6690 if (flags & VNIC_QCAPS_RESP_FLAGS_RE_FLUSH_CAP)
6691 bp->fw_cap |= BNXT_FW_CAP_VNIC_RE_FLUSH;
6692 }
6693 hwrm_req_drop(bp, req);
6694 return rc;
6695 }
6696
bnxt_hwrm_ring_grp_alloc(struct bnxt * bp)6697 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
6698 {
6699 struct hwrm_ring_grp_alloc_output *resp;
6700 struct hwrm_ring_grp_alloc_input *req;
6701 int rc;
6702 u16 i;
6703
6704 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6705 return 0;
6706
6707 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
6708 if (rc)
6709 return rc;
6710
6711 resp = hwrm_req_hold(bp, req);
6712 for (i = 0; i < bp->rx_nr_rings; i++) {
6713 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
6714
6715 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
6716 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
6717 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
6718 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
6719
6720 rc = hwrm_req_send(bp, req);
6721
6722 if (rc)
6723 break;
6724
6725 bp->grp_info[grp_idx].fw_grp_id =
6726 le32_to_cpu(resp->ring_group_id);
6727 }
6728 hwrm_req_drop(bp, req);
6729 return rc;
6730 }
6731
bnxt_hwrm_ring_grp_free(struct bnxt * bp)6732 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
6733 {
6734 struct hwrm_ring_grp_free_input *req;
6735 u16 i;
6736
6737 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
6738 return;
6739
6740 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
6741 return;
6742
6743 hwrm_req_hold(bp, req);
6744 for (i = 0; i < bp->cp_nr_rings; i++) {
6745 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
6746 continue;
6747 req->ring_group_id =
6748 cpu_to_le32(bp->grp_info[i].fw_grp_id);
6749
6750 hwrm_req_send(bp, req);
6751 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
6752 }
6753 hwrm_req_drop(bp, req);
6754 }
6755
hwrm_ring_alloc_send_msg(struct bnxt * bp,struct bnxt_ring_struct * ring,u32 ring_type,u32 map_index)6756 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
6757 struct bnxt_ring_struct *ring,
6758 u32 ring_type, u32 map_index)
6759 {
6760 struct hwrm_ring_alloc_output *resp;
6761 struct hwrm_ring_alloc_input *req;
6762 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
6763 struct bnxt_ring_grp_info *grp_info;
6764 int rc, err = 0;
6765 u16 ring_id;
6766
6767 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
6768 if (rc)
6769 goto exit;
6770
6771 req->enables = 0;
6772 if (rmem->nr_pages > 1) {
6773 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
6774 /* Page size is in log2 units */
6775 req->page_size = BNXT_PAGE_SHIFT;
6776 req->page_tbl_depth = 1;
6777 } else {
6778 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
6779 }
6780 req->fbo = 0;
6781 /* Association of ring index with doorbell index and MSIX number */
6782 req->logical_id = cpu_to_le16(map_index);
6783
6784 switch (ring_type) {
6785 case HWRM_RING_ALLOC_TX: {
6786 struct bnxt_tx_ring_info *txr;
6787 u16 flags = 0;
6788
6789 txr = container_of(ring, struct bnxt_tx_ring_info,
6790 tx_ring_struct);
6791 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
6792 /* Association of transmit ring with completion ring */
6793 grp_info = &bp->grp_info[ring->grp_idx];
6794 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
6795 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
6796 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6797 req->queue_id = cpu_to_le16(ring->queue_id);
6798 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL)
6799 req->cmpl_coal_cnt =
6800 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64;
6801 if ((bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) && bp->ptp_cfg)
6802 flags |= RING_ALLOC_REQ_FLAGS_TX_PKT_TS_CMPL_ENABLE;
6803 req->flags = cpu_to_le16(flags);
6804 break;
6805 }
6806 case HWRM_RING_ALLOC_RX:
6807 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6808 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
6809 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6810 u16 flags = 0;
6811
6812 /* Association of rx ring with stats context */
6813 grp_info = &bp->grp_info[ring->grp_idx];
6814 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
6815 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6816 req->enables |= cpu_to_le32(
6817 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6818 if (NET_IP_ALIGN == 2)
6819 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
6820 req->flags = cpu_to_le16(flags);
6821 }
6822 break;
6823 case HWRM_RING_ALLOC_AGG:
6824 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6825 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
6826 /* Association of agg ring with rx ring */
6827 grp_info = &bp->grp_info[ring->grp_idx];
6828 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
6829 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
6830 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6831 req->enables |= cpu_to_le32(
6832 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
6833 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6834 } else {
6835 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6836 }
6837 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
6838 break;
6839 case HWRM_RING_ALLOC_CMPL:
6840 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
6841 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6842 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6843 /* Association of cp ring with nq */
6844 grp_info = &bp->grp_info[map_index];
6845 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
6846 req->cq_handle = cpu_to_le64(ring->handle);
6847 req->enables |= cpu_to_le32(
6848 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
6849 } else {
6850 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6851 }
6852 break;
6853 case HWRM_RING_ALLOC_NQ:
6854 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
6855 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6856 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6857 break;
6858 default:
6859 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
6860 ring_type);
6861 return -1;
6862 }
6863
6864 resp = hwrm_req_hold(bp, req);
6865 rc = hwrm_req_send(bp, req);
6866 err = le16_to_cpu(resp->error_code);
6867 ring_id = le16_to_cpu(resp->ring_id);
6868 hwrm_req_drop(bp, req);
6869
6870 exit:
6871 if (rc || err) {
6872 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
6873 ring_type, rc, err);
6874 return -EIO;
6875 }
6876 ring->fw_ring_id = ring_id;
6877 return rc;
6878 }
6879
bnxt_hwrm_set_async_event_cr(struct bnxt * bp,int idx)6880 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
6881 {
6882 int rc;
6883
6884 if (BNXT_PF(bp)) {
6885 struct hwrm_func_cfg_input *req;
6886
6887 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
6888 if (rc)
6889 return rc;
6890
6891 req->fid = cpu_to_le16(0xffff);
6892 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6893 req->async_event_cr = cpu_to_le16(idx);
6894 return hwrm_req_send(bp, req);
6895 } else {
6896 struct hwrm_func_vf_cfg_input *req;
6897
6898 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
6899 if (rc)
6900 return rc;
6901
6902 req->enables =
6903 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6904 req->async_event_cr = cpu_to_le16(idx);
6905 return hwrm_req_send(bp, req);
6906 }
6907 }
6908
bnxt_set_db_mask(struct bnxt * bp,struct bnxt_db_info * db,u32 ring_type)6909 static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db,
6910 u32 ring_type)
6911 {
6912 switch (ring_type) {
6913 case HWRM_RING_ALLOC_TX:
6914 db->db_ring_mask = bp->tx_ring_mask;
6915 break;
6916 case HWRM_RING_ALLOC_RX:
6917 db->db_ring_mask = bp->rx_ring_mask;
6918 break;
6919 case HWRM_RING_ALLOC_AGG:
6920 db->db_ring_mask = bp->rx_agg_ring_mask;
6921 break;
6922 case HWRM_RING_ALLOC_CMPL:
6923 case HWRM_RING_ALLOC_NQ:
6924 db->db_ring_mask = bp->cp_ring_mask;
6925 break;
6926 }
6927 if (bp->flags & BNXT_FLAG_CHIP_P7) {
6928 db->db_epoch_mask = db->db_ring_mask + 1;
6929 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
6930 }
6931 }
6932
bnxt_set_db(struct bnxt * bp,struct bnxt_db_info * db,u32 ring_type,u32 map_idx,u32 xid)6933 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
6934 u32 map_idx, u32 xid)
6935 {
6936 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6937 switch (ring_type) {
6938 case HWRM_RING_ALLOC_TX:
6939 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
6940 break;
6941 case HWRM_RING_ALLOC_RX:
6942 case HWRM_RING_ALLOC_AGG:
6943 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
6944 break;
6945 case HWRM_RING_ALLOC_CMPL:
6946 db->db_key64 = DBR_PATH_L2;
6947 break;
6948 case HWRM_RING_ALLOC_NQ:
6949 db->db_key64 = DBR_PATH_L2;
6950 break;
6951 }
6952 db->db_key64 |= (u64)xid << DBR_XID_SFT;
6953
6954 if (bp->flags & BNXT_FLAG_CHIP_P7)
6955 db->db_key64 |= DBR_VALID;
6956
6957 db->doorbell = bp->bar1 + bp->db_offset;
6958 } else {
6959 db->doorbell = bp->bar1 + map_idx * 0x80;
6960 switch (ring_type) {
6961 case HWRM_RING_ALLOC_TX:
6962 db->db_key32 = DB_KEY_TX;
6963 break;
6964 case HWRM_RING_ALLOC_RX:
6965 case HWRM_RING_ALLOC_AGG:
6966 db->db_key32 = DB_KEY_RX;
6967 break;
6968 case HWRM_RING_ALLOC_CMPL:
6969 db->db_key32 = DB_KEY_CP;
6970 break;
6971 }
6972 }
6973 bnxt_set_db_mask(bp, db, ring_type);
6974 }
6975
bnxt_hwrm_rx_ring_alloc(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)6976 static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp,
6977 struct bnxt_rx_ring_info *rxr)
6978 {
6979 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6980 struct bnxt_napi *bnapi = rxr->bnapi;
6981 u32 type = HWRM_RING_ALLOC_RX;
6982 u32 map_idx = bnapi->index;
6983 int rc;
6984
6985 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6986 if (rc)
6987 return rc;
6988
6989 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
6990 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
6991
6992 return 0;
6993 }
6994
bnxt_hwrm_rx_agg_ring_alloc(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)6995 static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp,
6996 struct bnxt_rx_ring_info *rxr)
6997 {
6998 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6999 u32 type = HWRM_RING_ALLOC_AGG;
7000 u32 grp_idx = ring->grp_idx;
7001 u32 map_idx;
7002 int rc;
7003
7004 map_idx = grp_idx + bp->rx_nr_rings;
7005 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7006 if (rc)
7007 return rc;
7008
7009 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
7010 ring->fw_ring_id);
7011 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
7012 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
7013 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
7014
7015 return 0;
7016 }
7017
bnxt_hwrm_ring_alloc(struct bnxt * bp)7018 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
7019 {
7020 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
7021 int i, rc = 0;
7022 u32 type;
7023
7024 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7025 type = HWRM_RING_ALLOC_NQ;
7026 else
7027 type = HWRM_RING_ALLOC_CMPL;
7028 for (i = 0; i < bp->cp_nr_rings; i++) {
7029 struct bnxt_napi *bnapi = bp->bnapi[i];
7030 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7031 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
7032 u32 map_idx = ring->map_idx;
7033 unsigned int vector;
7034
7035 vector = bp->irq_tbl[map_idx].vector;
7036 disable_irq_nosync(vector);
7037 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7038 if (rc) {
7039 enable_irq(vector);
7040 goto err_out;
7041 }
7042 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
7043 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
7044 enable_irq(vector);
7045 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
7046
7047 if (!i) {
7048 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
7049 if (rc)
7050 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
7051 }
7052 }
7053
7054 type = HWRM_RING_ALLOC_TX;
7055 for (i = 0; i < bp->tx_nr_rings; i++) {
7056 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7057 struct bnxt_ring_struct *ring;
7058 u32 map_idx;
7059
7060 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7061 struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
7062 struct bnxt_napi *bnapi = txr->bnapi;
7063 u32 type2 = HWRM_RING_ALLOC_CMPL;
7064
7065 ring = &cpr2->cp_ring_struct;
7066 ring->handle = BNXT_SET_NQ_HDL(cpr2);
7067 map_idx = bnapi->index;
7068 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
7069 if (rc)
7070 goto err_out;
7071 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
7072 ring->fw_ring_id);
7073 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
7074 }
7075 ring = &txr->tx_ring_struct;
7076 map_idx = i;
7077 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7078 if (rc)
7079 goto err_out;
7080 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
7081 }
7082
7083 for (i = 0; i < bp->rx_nr_rings; i++) {
7084 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
7085
7086 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr);
7087 if (rc)
7088 goto err_out;
7089 /* If we have agg rings, post agg buffers first. */
7090 if (!agg_rings)
7091 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
7092 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7093 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
7094 struct bnxt_napi *bnapi = rxr->bnapi;
7095 u32 type2 = HWRM_RING_ALLOC_CMPL;
7096 struct bnxt_ring_struct *ring;
7097 u32 map_idx = bnapi->index;
7098
7099 ring = &cpr2->cp_ring_struct;
7100 ring->handle = BNXT_SET_NQ_HDL(cpr2);
7101 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
7102 if (rc)
7103 goto err_out;
7104 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
7105 ring->fw_ring_id);
7106 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
7107 }
7108 }
7109
7110 if (agg_rings) {
7111 for (i = 0; i < bp->rx_nr_rings; i++) {
7112 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, &bp->rx_ring[i]);
7113 if (rc)
7114 goto err_out;
7115 }
7116 }
7117 err_out:
7118 return rc;
7119 }
7120
hwrm_ring_free_send_msg(struct bnxt * bp,struct bnxt_ring_struct * ring,u32 ring_type,int cmpl_ring_id)7121 static int hwrm_ring_free_send_msg(struct bnxt *bp,
7122 struct bnxt_ring_struct *ring,
7123 u32 ring_type, int cmpl_ring_id)
7124 {
7125 struct hwrm_ring_free_output *resp;
7126 struct hwrm_ring_free_input *req;
7127 u16 error_code = 0;
7128 int rc;
7129
7130 if (BNXT_NO_FW_ACCESS(bp))
7131 return 0;
7132
7133 rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
7134 if (rc)
7135 goto exit;
7136
7137 req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
7138 req->ring_type = ring_type;
7139 req->ring_id = cpu_to_le16(ring->fw_ring_id);
7140
7141 resp = hwrm_req_hold(bp, req);
7142 rc = hwrm_req_send(bp, req);
7143 error_code = le16_to_cpu(resp->error_code);
7144 hwrm_req_drop(bp, req);
7145 exit:
7146 if (rc || error_code) {
7147 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
7148 ring_type, rc, error_code);
7149 return -EIO;
7150 }
7151 return 0;
7152 }
7153
bnxt_hwrm_rx_ring_free(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,bool close_path)7154 static void bnxt_hwrm_rx_ring_free(struct bnxt *bp,
7155 struct bnxt_rx_ring_info *rxr,
7156 bool close_path)
7157 {
7158 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
7159 u32 grp_idx = rxr->bnapi->index;
7160 u32 cmpl_ring_id;
7161
7162 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7163 return;
7164
7165 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7166 hwrm_ring_free_send_msg(bp, ring,
7167 RING_FREE_REQ_RING_TYPE_RX,
7168 close_path ? cmpl_ring_id :
7169 INVALID_HW_RING_ID);
7170 ring->fw_ring_id = INVALID_HW_RING_ID;
7171 bp->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID;
7172 }
7173
bnxt_hwrm_rx_agg_ring_free(struct bnxt * bp,struct bnxt_rx_ring_info * rxr,bool close_path)7174 static void bnxt_hwrm_rx_agg_ring_free(struct bnxt *bp,
7175 struct bnxt_rx_ring_info *rxr,
7176 bool close_path)
7177 {
7178 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
7179 u32 grp_idx = rxr->bnapi->index;
7180 u32 type, cmpl_ring_id;
7181
7182 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7183 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
7184 else
7185 type = RING_FREE_REQ_RING_TYPE_RX;
7186
7187 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7188 return;
7189
7190 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7191 hwrm_ring_free_send_msg(bp, ring, type,
7192 close_path ? cmpl_ring_id :
7193 INVALID_HW_RING_ID);
7194 ring->fw_ring_id = INVALID_HW_RING_ID;
7195 bp->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID;
7196 }
7197
bnxt_hwrm_ring_free(struct bnxt * bp,bool close_path)7198 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
7199 {
7200 u32 type;
7201 int i;
7202
7203 if (!bp->bnapi)
7204 return;
7205
7206 for (i = 0; i < bp->tx_nr_rings; i++) {
7207 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7208 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
7209
7210 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7211 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
7212
7213 hwrm_ring_free_send_msg(bp, ring,
7214 RING_FREE_REQ_RING_TYPE_TX,
7215 close_path ? cmpl_ring_id :
7216 INVALID_HW_RING_ID);
7217 ring->fw_ring_id = INVALID_HW_RING_ID;
7218 }
7219 }
7220
7221 for (i = 0; i < bp->rx_nr_rings; i++) {
7222 bnxt_hwrm_rx_ring_free(bp, &bp->rx_ring[i], close_path);
7223 bnxt_hwrm_rx_agg_ring_free(bp, &bp->rx_ring[i], close_path);
7224 }
7225
7226 /* The completion rings are about to be freed. After that the
7227 * IRQ doorbell will not work anymore. So we need to disable
7228 * IRQ here.
7229 */
7230 bnxt_disable_int_sync(bp);
7231
7232 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7233 type = RING_FREE_REQ_RING_TYPE_NQ;
7234 else
7235 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
7236 for (i = 0; i < bp->cp_nr_rings; i++) {
7237 struct bnxt_napi *bnapi = bp->bnapi[i];
7238 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7239 struct bnxt_ring_struct *ring;
7240 int j;
7241
7242 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
7243 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
7244
7245 ring = &cpr2->cp_ring_struct;
7246 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7247 continue;
7248 hwrm_ring_free_send_msg(bp, ring,
7249 RING_FREE_REQ_RING_TYPE_L2_CMPL,
7250 INVALID_HW_RING_ID);
7251 ring->fw_ring_id = INVALID_HW_RING_ID;
7252 }
7253 ring = &cpr->cp_ring_struct;
7254 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7255 hwrm_ring_free_send_msg(bp, ring, type,
7256 INVALID_HW_RING_ID);
7257 ring->fw_ring_id = INVALID_HW_RING_ID;
7258 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
7259 }
7260 }
7261 }
7262
7263 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7264 bool shared);
7265 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7266 bool shared);
7267
bnxt_hwrm_get_rings(struct bnxt * bp)7268 static int bnxt_hwrm_get_rings(struct bnxt *bp)
7269 {
7270 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7271 struct hwrm_func_qcfg_output *resp;
7272 struct hwrm_func_qcfg_input *req;
7273 int rc;
7274
7275 if (bp->hwrm_spec_code < 0x10601)
7276 return 0;
7277
7278 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7279 if (rc)
7280 return rc;
7281
7282 req->fid = cpu_to_le16(0xffff);
7283 resp = hwrm_req_hold(bp, req);
7284 rc = hwrm_req_send(bp, req);
7285 if (rc) {
7286 hwrm_req_drop(bp, req);
7287 return rc;
7288 }
7289
7290 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7291 if (BNXT_NEW_RM(bp)) {
7292 u16 cp, stats;
7293
7294 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
7295 hw_resc->resv_hw_ring_grps =
7296 le32_to_cpu(resp->alloc_hw_ring_grps);
7297 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
7298 hw_resc->resv_rsscos_ctxs = le16_to_cpu(resp->alloc_rsscos_ctx);
7299 cp = le16_to_cpu(resp->alloc_cmpl_rings);
7300 stats = le16_to_cpu(resp->alloc_stat_ctx);
7301 hw_resc->resv_irqs = cp;
7302 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7303 int rx = hw_resc->resv_rx_rings;
7304 int tx = hw_resc->resv_tx_rings;
7305
7306 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7307 rx >>= 1;
7308 if (cp < (rx + tx)) {
7309 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false);
7310 if (rc)
7311 goto get_rings_exit;
7312 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7313 rx <<= 1;
7314 hw_resc->resv_rx_rings = rx;
7315 hw_resc->resv_tx_rings = tx;
7316 }
7317 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
7318 hw_resc->resv_hw_ring_grps = rx;
7319 }
7320 hw_resc->resv_cp_rings = cp;
7321 hw_resc->resv_stat_ctxs = stats;
7322 }
7323 get_rings_exit:
7324 hwrm_req_drop(bp, req);
7325 return rc;
7326 }
7327
__bnxt_hwrm_get_tx_rings(struct bnxt * bp,u16 fid,int * tx_rings)7328 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
7329 {
7330 struct hwrm_func_qcfg_output *resp;
7331 struct hwrm_func_qcfg_input *req;
7332 int rc;
7333
7334 if (bp->hwrm_spec_code < 0x10601)
7335 return 0;
7336
7337 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7338 if (rc)
7339 return rc;
7340
7341 req->fid = cpu_to_le16(fid);
7342 resp = hwrm_req_hold(bp, req);
7343 rc = hwrm_req_send(bp, req);
7344 if (!rc)
7345 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7346
7347 hwrm_req_drop(bp, req);
7348 return rc;
7349 }
7350
7351 static bool bnxt_rfs_supported(struct bnxt *bp);
7352
7353 static struct hwrm_func_cfg_input *
__bnxt_hwrm_reserve_pf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7354 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7355 {
7356 struct hwrm_func_cfg_input *req;
7357 u32 enables = 0;
7358
7359 if (bnxt_hwrm_func_cfg_short_req_init(bp, &req))
7360 return NULL;
7361
7362 req->fid = cpu_to_le16(0xffff);
7363 enables |= hwr->tx ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7364 req->num_tx_rings = cpu_to_le16(hwr->tx);
7365 if (BNXT_NEW_RM(bp)) {
7366 enables |= hwr->rx ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
7367 enables |= hwr->stat ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7368 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7369 enables |= hwr->cp ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
7370 enables |= hwr->cp_p5 ?
7371 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7372 } else {
7373 enables |= hwr->cp ?
7374 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7375 enables |= hwr->grp ?
7376 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7377 }
7378 enables |= hwr->vnic ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
7379 enables |= hwr->rss_ctx ? FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS :
7380 0;
7381 req->num_rx_rings = cpu_to_le16(hwr->rx);
7382 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7383 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7384 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7385 req->num_msix = cpu_to_le16(hwr->cp);
7386 } else {
7387 req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7388 req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7389 }
7390 req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7391 req->num_vnics = cpu_to_le16(hwr->vnic);
7392 }
7393 req->enables = cpu_to_le32(enables);
7394 return req;
7395 }
7396
7397 static struct hwrm_func_vf_cfg_input *
__bnxt_hwrm_reserve_vf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7398 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7399 {
7400 struct hwrm_func_vf_cfg_input *req;
7401 u32 enables = 0;
7402
7403 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
7404 return NULL;
7405
7406 enables |= hwr->tx ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7407 enables |= hwr->rx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
7408 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7409 enables |= hwr->stat ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7410 enables |= hwr->rss_ctx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7411 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7412 enables |= hwr->cp_p5 ?
7413 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7414 } else {
7415 enables |= hwr->cp ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7416 enables |= hwr->grp ?
7417 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7418 }
7419 enables |= hwr->vnic ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
7420 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
7421
7422 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
7423 req->num_tx_rings = cpu_to_le16(hwr->tx);
7424 req->num_rx_rings = cpu_to_le16(hwr->rx);
7425 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7426 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7427 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7428 } else {
7429 req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7430 req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7431 }
7432 req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7433 req->num_vnics = cpu_to_le16(hwr->vnic);
7434
7435 req->enables = cpu_to_le32(enables);
7436 return req;
7437 }
7438
7439 static int
bnxt_hwrm_reserve_pf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7440 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7441 {
7442 struct hwrm_func_cfg_input *req;
7443 int rc;
7444
7445 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7446 if (!req)
7447 return -ENOMEM;
7448
7449 if (!req->enables) {
7450 hwrm_req_drop(bp, req);
7451 return 0;
7452 }
7453
7454 rc = hwrm_req_send(bp, req);
7455 if (rc)
7456 return rc;
7457
7458 if (bp->hwrm_spec_code < 0x10601)
7459 bp->hw_resc.resv_tx_rings = hwr->tx;
7460
7461 return bnxt_hwrm_get_rings(bp);
7462 }
7463
7464 static int
bnxt_hwrm_reserve_vf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7465 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7466 {
7467 struct hwrm_func_vf_cfg_input *req;
7468 int rc;
7469
7470 if (!BNXT_NEW_RM(bp)) {
7471 bp->hw_resc.resv_tx_rings = hwr->tx;
7472 return 0;
7473 }
7474
7475 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7476 if (!req)
7477 return -ENOMEM;
7478
7479 rc = hwrm_req_send(bp, req);
7480 if (rc)
7481 return rc;
7482
7483 return bnxt_hwrm_get_rings(bp);
7484 }
7485
bnxt_hwrm_reserve_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7486 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7487 {
7488 if (BNXT_PF(bp))
7489 return bnxt_hwrm_reserve_pf_rings(bp, hwr);
7490 else
7491 return bnxt_hwrm_reserve_vf_rings(bp, hwr);
7492 }
7493
bnxt_nq_rings_in_use(struct bnxt * bp)7494 int bnxt_nq_rings_in_use(struct bnxt *bp)
7495 {
7496 return bp->cp_nr_rings + bnxt_get_ulp_msix_num(bp);
7497 }
7498
bnxt_cp_rings_in_use(struct bnxt * bp)7499 static int bnxt_cp_rings_in_use(struct bnxt *bp)
7500 {
7501 int cp;
7502
7503 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7504 return bnxt_nq_rings_in_use(bp);
7505
7506 cp = bp->tx_nr_rings + bp->rx_nr_rings;
7507 return cp;
7508 }
7509
bnxt_get_func_stat_ctxs(struct bnxt * bp)7510 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
7511 {
7512 return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp);
7513 }
7514
bnxt_get_total_rss_ctxs(struct bnxt * bp,struct bnxt_hw_rings * hwr)7515 static int bnxt_get_total_rss_ctxs(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7516 {
7517 if (!hwr->grp)
7518 return 0;
7519 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7520 int rss_ctx = bnxt_get_nr_rss_ctxs(bp, hwr->grp);
7521
7522 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7523 rss_ctx *= hwr->vnic;
7524 return rss_ctx;
7525 }
7526 if (BNXT_VF(bp))
7527 return BNXT_VF_MAX_RSS_CTX;
7528 if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && bnxt_rfs_supported(bp))
7529 return hwr->grp + 1;
7530 return 1;
7531 }
7532
7533 /* Check if a default RSS map needs to be setup. This function is only
7534 * used on older firmware that does not require reserving RX rings.
7535 */
bnxt_check_rss_tbl_no_rmgr(struct bnxt * bp)7536 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
7537 {
7538 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7539
7540 /* The RSS map is valid for RX rings set to resv_rx_rings */
7541 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
7542 hw_resc->resv_rx_rings = bp->rx_nr_rings;
7543 if (!netif_is_rxfh_configured(bp->dev))
7544 bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7545 }
7546 }
7547
bnxt_get_total_vnics(struct bnxt * bp,int rx_rings)7548 static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings)
7549 {
7550 if (bp->flags & BNXT_FLAG_RFS) {
7551 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7552 return 2 + bp->num_rss_ctx;
7553 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7554 return rx_rings + 1;
7555 }
7556 return 1;
7557 }
7558
bnxt_need_reserve_rings(struct bnxt * bp)7559 static bool bnxt_need_reserve_rings(struct bnxt *bp)
7560 {
7561 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7562 int cp = bnxt_cp_rings_in_use(bp);
7563 int nq = bnxt_nq_rings_in_use(bp);
7564 int rx = bp->rx_nr_rings, stat;
7565 int vnic, grp = rx;
7566
7567 /* Old firmware does not need RX ring reservations but we still
7568 * need to setup a default RSS map when needed. With new firmware
7569 * we go through RX ring reservations first and then set up the
7570 * RSS map for the successfully reserved RX rings when needed.
7571 */
7572 if (!BNXT_NEW_RM(bp))
7573 bnxt_check_rss_tbl_no_rmgr(bp);
7574
7575 if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
7576 bp->hwrm_spec_code >= 0x10601)
7577 return true;
7578
7579 if (!BNXT_NEW_RM(bp))
7580 return false;
7581
7582 vnic = bnxt_get_total_vnics(bp, rx);
7583
7584 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7585 rx <<= 1;
7586 stat = bnxt_get_func_stat_ctxs(bp);
7587 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
7588 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
7589 (hw_resc->resv_hw_ring_grps != grp &&
7590 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)))
7591 return true;
7592 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) &&
7593 hw_resc->resv_irqs != nq)
7594 return true;
7595 return false;
7596 }
7597
bnxt_copy_reserved_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7598 static void bnxt_copy_reserved_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7599 {
7600 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7601
7602 hwr->tx = hw_resc->resv_tx_rings;
7603 if (BNXT_NEW_RM(bp)) {
7604 hwr->rx = hw_resc->resv_rx_rings;
7605 hwr->cp = hw_resc->resv_irqs;
7606 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7607 hwr->cp_p5 = hw_resc->resv_cp_rings;
7608 hwr->grp = hw_resc->resv_hw_ring_grps;
7609 hwr->vnic = hw_resc->resv_vnics;
7610 hwr->stat = hw_resc->resv_stat_ctxs;
7611 hwr->rss_ctx = hw_resc->resv_rsscos_ctxs;
7612 }
7613 }
7614
bnxt_rings_ok(struct bnxt * bp,struct bnxt_hw_rings * hwr)7615 static bool bnxt_rings_ok(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7616 {
7617 return hwr->tx && hwr->rx && hwr->cp && hwr->grp && hwr->vnic &&
7618 hwr->stat && (hwr->cp_p5 || !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS));
7619 }
7620
7621 static int bnxt_get_avail_msix(struct bnxt *bp, int num);
7622
__bnxt_reserve_rings(struct bnxt * bp)7623 static int __bnxt_reserve_rings(struct bnxt *bp)
7624 {
7625 struct bnxt_hw_rings hwr = {0};
7626 int rx_rings, old_rx_rings, rc;
7627 int cp = bp->cp_nr_rings;
7628 int ulp_msix = 0;
7629 bool sh = false;
7630 int tx_cp;
7631
7632 if (!bnxt_need_reserve_rings(bp))
7633 return 0;
7634
7635 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) {
7636 ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
7637 if (!ulp_msix)
7638 bnxt_set_ulp_stat_ctxs(bp, 0);
7639
7640 if (ulp_msix > bp->ulp_num_msix_want)
7641 ulp_msix = bp->ulp_num_msix_want;
7642 hwr.cp = cp + ulp_msix;
7643 } else {
7644 hwr.cp = bnxt_nq_rings_in_use(bp);
7645 }
7646
7647 hwr.tx = bp->tx_nr_rings;
7648 hwr.rx = bp->rx_nr_rings;
7649 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7650 sh = true;
7651 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7652 hwr.cp_p5 = hwr.rx + hwr.tx;
7653
7654 hwr.vnic = bnxt_get_total_vnics(bp, hwr.rx);
7655
7656 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7657 hwr.rx <<= 1;
7658 hwr.grp = bp->rx_nr_rings;
7659 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
7660 hwr.stat = bnxt_get_func_stat_ctxs(bp);
7661 old_rx_rings = bp->hw_resc.resv_rx_rings;
7662
7663 rc = bnxt_hwrm_reserve_rings(bp, &hwr);
7664 if (rc)
7665 return rc;
7666
7667 bnxt_copy_reserved_rings(bp, &hwr);
7668
7669 rx_rings = hwr.rx;
7670 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7671 if (hwr.rx >= 2) {
7672 rx_rings = hwr.rx >> 1;
7673 } else {
7674 if (netif_running(bp->dev))
7675 return -ENOMEM;
7676
7677 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7678 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7679 bp->dev->hw_features &= ~NETIF_F_LRO;
7680 bp->dev->features &= ~NETIF_F_LRO;
7681 bnxt_set_ring_params(bp);
7682 }
7683 }
7684 rx_rings = min_t(int, rx_rings, hwr.grp);
7685 hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings);
7686 if (hwr.stat > bnxt_get_ulp_stat_ctxs(bp))
7687 hwr.stat -= bnxt_get_ulp_stat_ctxs(bp);
7688 hwr.cp = min_t(int, hwr.cp, hwr.stat);
7689 rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh);
7690 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7691 hwr.rx = rx_rings << 1;
7692 tx_cp = bnxt_num_tx_to_cp(bp, hwr.tx);
7693 hwr.cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
7694 bp->tx_nr_rings = hwr.tx;
7695
7696 /* If we cannot reserve all the RX rings, reset the RSS map only
7697 * if absolutely necessary
7698 */
7699 if (rx_rings != bp->rx_nr_rings) {
7700 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
7701 rx_rings, bp->rx_nr_rings);
7702 if (netif_is_rxfh_configured(bp->dev) &&
7703 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
7704 bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
7705 bnxt_get_max_rss_ring(bp) >= rx_rings)) {
7706 netdev_warn(bp->dev, "RSS table entries reverting to default\n");
7707 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
7708 }
7709 }
7710 bp->rx_nr_rings = rx_rings;
7711 bp->cp_nr_rings = hwr.cp;
7712
7713 if (!bnxt_rings_ok(bp, &hwr))
7714 return -ENOMEM;
7715
7716 if (old_rx_rings != bp->hw_resc.resv_rx_rings &&
7717 !netif_is_rxfh_configured(bp->dev))
7718 bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7719
7720 if (!bnxt_ulp_registered(bp->edev) && BNXT_NEW_RM(bp)) {
7721 int resv_msix, resv_ctx, ulp_ctxs;
7722 struct bnxt_hw_resc *hw_resc;
7723
7724 hw_resc = &bp->hw_resc;
7725 resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
7726 ulp_msix = min_t(int, resv_msix, ulp_msix);
7727 bnxt_set_ulp_msix_num(bp, ulp_msix);
7728 resv_ctx = hw_resc->resv_stat_ctxs - bp->cp_nr_rings;
7729 ulp_ctxs = min(resv_ctx, bnxt_get_ulp_stat_ctxs(bp));
7730 bnxt_set_ulp_stat_ctxs(bp, ulp_ctxs);
7731 }
7732
7733 return rc;
7734 }
7735
bnxt_hwrm_check_vf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7736 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7737 {
7738 struct hwrm_func_vf_cfg_input *req;
7739 u32 flags;
7740
7741 if (!BNXT_NEW_RM(bp))
7742 return 0;
7743
7744 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7745 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
7746 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7747 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7748 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7749 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
7750 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
7751 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7752 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7753
7754 req->flags = cpu_to_le32(flags);
7755 return hwrm_req_send_silent(bp, req);
7756 }
7757
bnxt_hwrm_check_pf_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7758 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7759 {
7760 struct hwrm_func_cfg_input *req;
7761 u32 flags;
7762
7763 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7764 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
7765 if (BNXT_NEW_RM(bp)) {
7766 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7767 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7768 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7769 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
7770 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7771 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
7772 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
7773 else
7774 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7775 }
7776
7777 req->flags = cpu_to_le32(flags);
7778 return hwrm_req_send_silent(bp, req);
7779 }
7780
bnxt_hwrm_check_rings(struct bnxt * bp,struct bnxt_hw_rings * hwr)7781 static int bnxt_hwrm_check_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7782 {
7783 if (bp->hwrm_spec_code < 0x10801)
7784 return 0;
7785
7786 if (BNXT_PF(bp))
7787 return bnxt_hwrm_check_pf_rings(bp, hwr);
7788
7789 return bnxt_hwrm_check_vf_rings(bp, hwr);
7790 }
7791
bnxt_hwrm_coal_params_qcaps(struct bnxt * bp)7792 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
7793 {
7794 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7795 struct hwrm_ring_aggint_qcaps_output *resp;
7796 struct hwrm_ring_aggint_qcaps_input *req;
7797 int rc;
7798
7799 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
7800 coal_cap->num_cmpl_dma_aggr_max = 63;
7801 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
7802 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
7803 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
7804 coal_cap->int_lat_tmr_min_max = 65535;
7805 coal_cap->int_lat_tmr_max_max = 65535;
7806 coal_cap->num_cmpl_aggr_int_max = 65535;
7807 coal_cap->timer_units = 80;
7808
7809 if (bp->hwrm_spec_code < 0x10902)
7810 return;
7811
7812 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
7813 return;
7814
7815 resp = hwrm_req_hold(bp, req);
7816 rc = hwrm_req_send_silent(bp, req);
7817 if (!rc) {
7818 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
7819 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
7820 coal_cap->num_cmpl_dma_aggr_max =
7821 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
7822 coal_cap->num_cmpl_dma_aggr_during_int_max =
7823 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
7824 coal_cap->cmpl_aggr_dma_tmr_max =
7825 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
7826 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
7827 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
7828 coal_cap->int_lat_tmr_min_max =
7829 le16_to_cpu(resp->int_lat_tmr_min_max);
7830 coal_cap->int_lat_tmr_max_max =
7831 le16_to_cpu(resp->int_lat_tmr_max_max);
7832 coal_cap->num_cmpl_aggr_int_max =
7833 le16_to_cpu(resp->num_cmpl_aggr_int_max);
7834 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
7835 }
7836 hwrm_req_drop(bp, req);
7837 }
7838
bnxt_usec_to_coal_tmr(struct bnxt * bp,u16 usec)7839 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
7840 {
7841 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7842
7843 return usec * 1000 / coal_cap->timer_units;
7844 }
7845
bnxt_hwrm_set_coal_params(struct bnxt * bp,struct bnxt_coal * hw_coal,struct hwrm_ring_cmpl_ring_cfg_aggint_params_input * req)7846 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
7847 struct bnxt_coal *hw_coal,
7848 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7849 {
7850 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7851 u16 val, tmr, max, flags = hw_coal->flags;
7852 u32 cmpl_params = coal_cap->cmpl_params;
7853
7854 max = hw_coal->bufs_per_record * 128;
7855 if (hw_coal->budget)
7856 max = hw_coal->bufs_per_record * hw_coal->budget;
7857 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
7858
7859 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
7860 req->num_cmpl_aggr_int = cpu_to_le16(val);
7861
7862 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
7863 req->num_cmpl_dma_aggr = cpu_to_le16(val);
7864
7865 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
7866 coal_cap->num_cmpl_dma_aggr_during_int_max);
7867 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
7868
7869 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
7870 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
7871 req->int_lat_tmr_max = cpu_to_le16(tmr);
7872
7873 /* min timer set to 1/2 of interrupt timer */
7874 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
7875 val = tmr / 2;
7876 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
7877 req->int_lat_tmr_min = cpu_to_le16(val);
7878 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7879 }
7880
7881 /* buf timer set to 1/4 of interrupt timer */
7882 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
7883 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
7884
7885 if (cmpl_params &
7886 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
7887 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
7888 val = clamp_t(u16, tmr, 1,
7889 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
7890 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
7891 req->enables |=
7892 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
7893 }
7894
7895 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
7896 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
7897 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
7898 req->flags = cpu_to_le16(flags);
7899 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
7900 }
7901
__bnxt_hwrm_set_coal_nq(struct bnxt * bp,struct bnxt_napi * bnapi,struct bnxt_coal * hw_coal)7902 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
7903 struct bnxt_coal *hw_coal)
7904 {
7905 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
7906 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7907 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7908 u32 nq_params = coal_cap->nq_params;
7909 u16 tmr;
7910 int rc;
7911
7912 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
7913 return 0;
7914
7915 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7916 if (rc)
7917 return rc;
7918
7919 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
7920 req->flags =
7921 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
7922
7923 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
7924 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
7925 req->int_lat_tmr_min = cpu_to_le16(tmr);
7926 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7927 return hwrm_req_send(bp, req);
7928 }
7929
bnxt_hwrm_set_ring_coal(struct bnxt * bp,struct bnxt_napi * bnapi)7930 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
7931 {
7932 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
7933 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7934 struct bnxt_coal coal;
7935 int rc;
7936
7937 /* Tick values in micro seconds.
7938 * 1 coal_buf x bufs_per_record = 1 completion record.
7939 */
7940 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
7941
7942 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
7943 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
7944
7945 if (!bnapi->rx_ring)
7946 return -ENODEV;
7947
7948 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7949 if (rc)
7950 return rc;
7951
7952 bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
7953
7954 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
7955
7956 return hwrm_req_send(bp, req_rx);
7957 }
7958
7959 static int
bnxt_hwrm_set_rx_coal(struct bnxt * bp,struct bnxt_napi * bnapi,struct hwrm_ring_cmpl_ring_cfg_aggint_params_input * req)7960 bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7961 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7962 {
7963 u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
7964
7965 req->ring_id = cpu_to_le16(ring_id);
7966 return hwrm_req_send(bp, req);
7967 }
7968
7969 static int
bnxt_hwrm_set_tx_coal(struct bnxt * bp,struct bnxt_napi * bnapi,struct hwrm_ring_cmpl_ring_cfg_aggint_params_input * req)7970 bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7971 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7972 {
7973 struct bnxt_tx_ring_info *txr;
7974 int i, rc;
7975
7976 bnxt_for_each_napi_tx(i, bnapi, txr) {
7977 u16 ring_id;
7978
7979 ring_id = bnxt_cp_ring_for_tx(bp, txr);
7980 req->ring_id = cpu_to_le16(ring_id);
7981 rc = hwrm_req_send(bp, req);
7982 if (rc)
7983 return rc;
7984 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7985 return 0;
7986 }
7987 return 0;
7988 }
7989
bnxt_hwrm_set_coal(struct bnxt * bp)7990 int bnxt_hwrm_set_coal(struct bnxt *bp)
7991 {
7992 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
7993 int i, rc;
7994
7995 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7996 if (rc)
7997 return rc;
7998
7999 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
8000 if (rc) {
8001 hwrm_req_drop(bp, req_rx);
8002 return rc;
8003 }
8004
8005 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
8006 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
8007
8008 hwrm_req_hold(bp, req_rx);
8009 hwrm_req_hold(bp, req_tx);
8010 for (i = 0; i < bp->cp_nr_rings; i++) {
8011 struct bnxt_napi *bnapi = bp->bnapi[i];
8012 struct bnxt_coal *hw_coal;
8013
8014 if (!bnapi->rx_ring)
8015 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
8016 else
8017 rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
8018 if (rc)
8019 break;
8020
8021 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
8022 continue;
8023
8024 if (bnapi->rx_ring && bnapi->tx_ring[0]) {
8025 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
8026 if (rc)
8027 break;
8028 }
8029 if (bnapi->rx_ring)
8030 hw_coal = &bp->rx_coal;
8031 else
8032 hw_coal = &bp->tx_coal;
8033 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
8034 }
8035 hwrm_req_drop(bp, req_rx);
8036 hwrm_req_drop(bp, req_tx);
8037 return rc;
8038 }
8039
bnxt_hwrm_stat_ctx_free(struct bnxt * bp)8040 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
8041 {
8042 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
8043 struct hwrm_stat_ctx_free_input *req;
8044 int i;
8045
8046 if (!bp->bnapi)
8047 return;
8048
8049 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8050 return;
8051
8052 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
8053 return;
8054 if (BNXT_FW_MAJ(bp) <= 20) {
8055 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
8056 hwrm_req_drop(bp, req);
8057 return;
8058 }
8059 hwrm_req_hold(bp, req0);
8060 }
8061 hwrm_req_hold(bp, req);
8062 for (i = 0; i < bp->cp_nr_rings; i++) {
8063 struct bnxt_napi *bnapi = bp->bnapi[i];
8064 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8065
8066 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
8067 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
8068 if (req0) {
8069 req0->stat_ctx_id = req->stat_ctx_id;
8070 hwrm_req_send(bp, req0);
8071 }
8072 hwrm_req_send(bp, req);
8073
8074 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
8075 }
8076 }
8077 hwrm_req_drop(bp, req);
8078 if (req0)
8079 hwrm_req_drop(bp, req0);
8080 }
8081
bnxt_hwrm_stat_ctx_alloc(struct bnxt * bp)8082 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
8083 {
8084 struct hwrm_stat_ctx_alloc_output *resp;
8085 struct hwrm_stat_ctx_alloc_input *req;
8086 int rc, i;
8087
8088 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8089 return 0;
8090
8091 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
8092 if (rc)
8093 return rc;
8094
8095 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
8096 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
8097
8098 resp = hwrm_req_hold(bp, req);
8099 for (i = 0; i < bp->cp_nr_rings; i++) {
8100 struct bnxt_napi *bnapi = bp->bnapi[i];
8101 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8102
8103 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
8104
8105 rc = hwrm_req_send(bp, req);
8106 if (rc)
8107 break;
8108
8109 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
8110
8111 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
8112 }
8113 hwrm_req_drop(bp, req);
8114 return rc;
8115 }
8116
bnxt_hwrm_func_qcfg(struct bnxt * bp)8117 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
8118 {
8119 struct hwrm_func_qcfg_output *resp;
8120 struct hwrm_func_qcfg_input *req;
8121 u16 flags;
8122 int rc;
8123
8124 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
8125 if (rc)
8126 return rc;
8127
8128 req->fid = cpu_to_le16(0xffff);
8129 resp = hwrm_req_hold(bp, req);
8130 rc = hwrm_req_send(bp, req);
8131 if (rc)
8132 goto func_qcfg_exit;
8133
8134 #ifdef CONFIG_BNXT_SRIOV
8135 if (BNXT_VF(bp)) {
8136 struct bnxt_vf_info *vf = &bp->vf;
8137
8138 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
8139 } else {
8140 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
8141 }
8142 #endif
8143 flags = le16_to_cpu(resp->flags);
8144 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
8145 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
8146 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
8147 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
8148 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
8149 }
8150 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
8151 bp->flags |= BNXT_FLAG_MULTI_HOST;
8152
8153 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
8154 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
8155
8156 switch (resp->port_partition_type) {
8157 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
8158 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
8159 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
8160 bp->port_partition_type = resp->port_partition_type;
8161 break;
8162 }
8163 if (bp->hwrm_spec_code < 0x10707 ||
8164 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
8165 bp->br_mode = BRIDGE_MODE_VEB;
8166 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
8167 bp->br_mode = BRIDGE_MODE_VEPA;
8168 else
8169 bp->br_mode = BRIDGE_MODE_UNDEF;
8170
8171 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
8172 if (!bp->max_mtu)
8173 bp->max_mtu = BNXT_MAX_MTU;
8174
8175 if (bp->db_size)
8176 goto func_qcfg_exit;
8177
8178 bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024;
8179 if (BNXT_CHIP_P5(bp)) {
8180 if (BNXT_PF(bp))
8181 bp->db_offset = DB_PF_OFFSET_P5;
8182 else
8183 bp->db_offset = DB_VF_OFFSET_P5;
8184 }
8185 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
8186 1024);
8187 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
8188 bp->db_size <= bp->db_offset)
8189 bp->db_size = pci_resource_len(bp->pdev, 2);
8190
8191 func_qcfg_exit:
8192 hwrm_req_drop(bp, req);
8193 return rc;
8194 }
8195
bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type * ctxm,u8 init_val,u8 init_offset,bool init_mask_set)8196 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm,
8197 u8 init_val, u8 init_offset,
8198 bool init_mask_set)
8199 {
8200 ctxm->init_value = init_val;
8201 ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET;
8202 if (init_mask_set)
8203 ctxm->init_offset = init_offset * 4;
8204 else
8205 ctxm->init_value = 0;
8206 }
8207
bnxt_alloc_all_ctx_pg_info(struct bnxt * bp,int ctx_max)8208 static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max)
8209 {
8210 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8211 u16 type;
8212
8213 for (type = 0; type < ctx_max; type++) {
8214 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8215 int n = 1;
8216
8217 if (!ctxm->max_entries)
8218 continue;
8219
8220 if (ctxm->instance_bmap)
8221 n = hweight32(ctxm->instance_bmap);
8222 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL);
8223 if (!ctxm->pg_info)
8224 return -ENOMEM;
8225 }
8226 return 0;
8227 }
8228
8229 #define BNXT_CTX_INIT_VALID(flags) \
8230 (!!((flags) & \
8231 FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT))
8232
bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt * bp)8233 static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp)
8234 {
8235 struct hwrm_func_backing_store_qcaps_v2_output *resp;
8236 struct hwrm_func_backing_store_qcaps_v2_input *req;
8237 struct bnxt_ctx_mem_info *ctx;
8238 u16 type;
8239 int rc;
8240
8241 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2);
8242 if (rc)
8243 return rc;
8244
8245 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8246 if (!ctx)
8247 return -ENOMEM;
8248 bp->ctx = ctx;
8249
8250 resp = hwrm_req_hold(bp, req);
8251
8252 for (type = 0; type < BNXT_CTX_V2_MAX; ) {
8253 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8254 u8 init_val, init_off, i;
8255 __le32 *p;
8256 u32 flags;
8257
8258 req->type = cpu_to_le16(type);
8259 rc = hwrm_req_send(bp, req);
8260 if (rc)
8261 goto ctx_done;
8262 flags = le32_to_cpu(resp->flags);
8263 type = le16_to_cpu(resp->next_valid_type);
8264 if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID))
8265 continue;
8266
8267 ctxm->type = le16_to_cpu(resp->type);
8268 ctxm->entry_size = le16_to_cpu(resp->entry_size);
8269 ctxm->flags = flags;
8270 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map);
8271 ctxm->entry_multiple = resp->entry_multiple;
8272 ctxm->max_entries = le32_to_cpu(resp->max_num_entries);
8273 ctxm->min_entries = le32_to_cpu(resp->min_num_entries);
8274 init_val = resp->ctx_init_value;
8275 init_off = resp->ctx_init_offset;
8276 bnxt_init_ctx_initializer(ctxm, init_val, init_off,
8277 BNXT_CTX_INIT_VALID(flags));
8278 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt,
8279 BNXT_MAX_SPLIT_ENTRY);
8280 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt;
8281 i++, p++)
8282 ctxm->split[i] = le32_to_cpu(*p);
8283 }
8284 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX);
8285
8286 ctx_done:
8287 hwrm_req_drop(bp, req);
8288 return rc;
8289 }
8290
bnxt_hwrm_func_backing_store_qcaps(struct bnxt * bp)8291 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
8292 {
8293 struct hwrm_func_backing_store_qcaps_output *resp;
8294 struct hwrm_func_backing_store_qcaps_input *req;
8295 int rc;
8296
8297 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
8298 return 0;
8299
8300 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8301 return bnxt_hwrm_func_backing_store_qcaps_v2(bp);
8302
8303 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
8304 if (rc)
8305 return rc;
8306
8307 resp = hwrm_req_hold(bp, req);
8308 rc = hwrm_req_send_silent(bp, req);
8309 if (!rc) {
8310 struct bnxt_ctx_mem_type *ctxm;
8311 struct bnxt_ctx_mem_info *ctx;
8312 u8 init_val, init_idx = 0;
8313 u16 init_mask;
8314
8315 ctx = bp->ctx;
8316 if (!ctx) {
8317 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8318 if (!ctx) {
8319 rc = -ENOMEM;
8320 goto ctx_err;
8321 }
8322 bp->ctx = ctx;
8323 }
8324 init_val = resp->ctx_kind_initializer;
8325 init_mask = le16_to_cpu(resp->ctx_init_mask);
8326
8327 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8328 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries);
8329 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
8330 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
8331 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries);
8332 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size);
8333 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset,
8334 (init_mask & (1 << init_idx++)) != 0);
8335
8336 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8337 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
8338 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries);
8339 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size);
8340 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset,
8341 (init_mask & (1 << init_idx++)) != 0);
8342
8343 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8344 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
8345 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries);
8346 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size);
8347 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset,
8348 (init_mask & (1 << init_idx++)) != 0);
8349
8350 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8351 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries);
8352 ctxm->max_entries = ctxm->vnic_entries +
8353 le16_to_cpu(resp->vnic_max_ring_table_entries);
8354 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size);
8355 bnxt_init_ctx_initializer(ctxm, init_val,
8356 resp->vnic_init_offset,
8357 (init_mask & (1 << init_idx++)) != 0);
8358
8359 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8360 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries);
8361 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size);
8362 bnxt_init_ctx_initializer(ctxm, init_val,
8363 resp->stat_init_offset,
8364 (init_mask & (1 << init_idx++)) != 0);
8365
8366 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8367 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size);
8368 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring);
8369 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring);
8370 ctxm->entry_multiple = resp->tqm_entries_multiple;
8371 if (!ctxm->entry_multiple)
8372 ctxm->entry_multiple = 1;
8373
8374 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm));
8375
8376 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8377 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries);
8378 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size);
8379 ctxm->mrav_num_entries_units =
8380 le16_to_cpu(resp->mrav_num_entries_units);
8381 bnxt_init_ctx_initializer(ctxm, init_val,
8382 resp->mrav_init_offset,
8383 (init_mask & (1 << init_idx++)) != 0);
8384
8385 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8386 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size);
8387 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries);
8388
8389 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
8390 if (!ctx->tqm_fp_rings_count)
8391 ctx->tqm_fp_rings_count = bp->max_q;
8392 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
8393 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
8394
8395 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8396 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm));
8397 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1;
8398
8399 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX);
8400 } else {
8401 rc = 0;
8402 }
8403 ctx_err:
8404 hwrm_req_drop(bp, req);
8405 return rc;
8406 }
8407
bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info * rmem,u8 * pg_attr,__le64 * pg_dir)8408 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
8409 __le64 *pg_dir)
8410 {
8411 if (!rmem->nr_pages)
8412 return;
8413
8414 BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
8415 if (rmem->depth >= 1) {
8416 if (rmem->depth == 2)
8417 *pg_attr |= 2;
8418 else
8419 *pg_attr |= 1;
8420 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
8421 } else {
8422 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
8423 }
8424 }
8425
8426 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
8427 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
8428 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
8429 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
8430 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
8431 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
8432
bnxt_hwrm_func_backing_store_cfg(struct bnxt * bp,u32 enables)8433 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
8434 {
8435 struct hwrm_func_backing_store_cfg_input *req;
8436 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8437 struct bnxt_ctx_pg_info *ctx_pg;
8438 struct bnxt_ctx_mem_type *ctxm;
8439 void **__req = (void **)&req;
8440 u32 req_len = sizeof(*req);
8441 __le32 *num_entries;
8442 __le64 *pg_dir;
8443 u32 flags = 0;
8444 u8 *pg_attr;
8445 u32 ena;
8446 int rc;
8447 int i;
8448
8449 if (!ctx)
8450 return 0;
8451
8452 if (req_len > bp->hwrm_max_ext_req_len)
8453 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
8454 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
8455 if (rc)
8456 return rc;
8457
8458 req->enables = cpu_to_le32(enables);
8459 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
8460 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8461 ctx_pg = ctxm->pg_info;
8462 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
8463 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries);
8464 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries);
8465 req->qp_entry_size = cpu_to_le16(ctxm->entry_size);
8466 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8467 &req->qpc_pg_size_qpc_lvl,
8468 &req->qpc_page_dir);
8469
8470 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD)
8471 req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries);
8472 }
8473 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
8474 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8475 ctx_pg = ctxm->pg_info;
8476 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
8477 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries);
8478 req->srq_entry_size = cpu_to_le16(ctxm->entry_size);
8479 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8480 &req->srq_pg_size_srq_lvl,
8481 &req->srq_page_dir);
8482 }
8483 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
8484 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8485 ctx_pg = ctxm->pg_info;
8486 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
8487 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries);
8488 req->cq_entry_size = cpu_to_le16(ctxm->entry_size);
8489 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8490 &req->cq_pg_size_cq_lvl,
8491 &req->cq_page_dir);
8492 }
8493 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
8494 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8495 ctx_pg = ctxm->pg_info;
8496 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries);
8497 req->vnic_num_ring_table_entries =
8498 cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries);
8499 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size);
8500 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8501 &req->vnic_pg_size_vnic_lvl,
8502 &req->vnic_page_dir);
8503 }
8504 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
8505 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8506 ctx_pg = ctxm->pg_info;
8507 req->stat_num_entries = cpu_to_le32(ctxm->max_entries);
8508 req->stat_entry_size = cpu_to_le16(ctxm->entry_size);
8509 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8510 &req->stat_pg_size_stat_lvl,
8511 &req->stat_page_dir);
8512 }
8513 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
8514 u32 units;
8515
8516 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8517 ctx_pg = ctxm->pg_info;
8518 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
8519 units = ctxm->mrav_num_entries_units;
8520 if (units) {
8521 u32 num_mr, num_ah = ctxm->mrav_av_entries;
8522 u32 entries;
8523
8524 num_mr = ctx_pg->entries - num_ah;
8525 entries = ((num_mr / units) << 16) | (num_ah / units);
8526 req->mrav_num_entries = cpu_to_le32(entries);
8527 flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
8528 }
8529 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size);
8530 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8531 &req->mrav_pg_size_mrav_lvl,
8532 &req->mrav_page_dir);
8533 }
8534 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
8535 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8536 ctx_pg = ctxm->pg_info;
8537 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
8538 req->tim_entry_size = cpu_to_le16(ctxm->entry_size);
8539 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8540 &req->tim_pg_size_tim_lvl,
8541 &req->tim_page_dir);
8542 }
8543 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8544 for (i = 0, num_entries = &req->tqm_sp_num_entries,
8545 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
8546 pg_dir = &req->tqm_sp_page_dir,
8547 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP,
8548 ctx_pg = ctxm->pg_info;
8549 i < BNXT_MAX_TQM_RINGS;
8550 ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i],
8551 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
8552 if (!(enables & ena))
8553 continue;
8554
8555 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size);
8556 *num_entries = cpu_to_le32(ctx_pg->entries);
8557 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
8558 }
8559 req->flags = cpu_to_le32(flags);
8560 return hwrm_req_send(bp, req);
8561 }
8562
bnxt_alloc_ctx_mem_blk(struct bnxt * bp,struct bnxt_ctx_pg_info * ctx_pg)8563 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
8564 struct bnxt_ctx_pg_info *ctx_pg)
8565 {
8566 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8567
8568 rmem->page_size = BNXT_PAGE_SIZE;
8569 rmem->pg_arr = ctx_pg->ctx_pg_arr;
8570 rmem->dma_arr = ctx_pg->ctx_dma_arr;
8571 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
8572 if (rmem->depth >= 1)
8573 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
8574 return bnxt_alloc_ring(bp, rmem);
8575 }
8576
bnxt_alloc_ctx_pg_tbls(struct bnxt * bp,struct bnxt_ctx_pg_info * ctx_pg,u32 mem_size,u8 depth,struct bnxt_ctx_mem_type * ctxm)8577 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
8578 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
8579 u8 depth, struct bnxt_ctx_mem_type *ctxm)
8580 {
8581 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8582 int rc;
8583
8584 if (!mem_size)
8585 return -EINVAL;
8586
8587 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8588 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
8589 ctx_pg->nr_pages = 0;
8590 return -EINVAL;
8591 }
8592 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
8593 int nr_tbls, i;
8594
8595 rmem->depth = 2;
8596 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
8597 GFP_KERNEL);
8598 if (!ctx_pg->ctx_pg_tbl)
8599 return -ENOMEM;
8600 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
8601 rmem->nr_pages = nr_tbls;
8602 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8603 if (rc)
8604 return rc;
8605 for (i = 0; i < nr_tbls; i++) {
8606 struct bnxt_ctx_pg_info *pg_tbl;
8607
8608 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
8609 if (!pg_tbl)
8610 return -ENOMEM;
8611 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
8612 rmem = &pg_tbl->ring_mem;
8613 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
8614 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
8615 rmem->depth = 1;
8616 rmem->nr_pages = MAX_CTX_PAGES;
8617 rmem->ctx_mem = ctxm;
8618 if (i == (nr_tbls - 1)) {
8619 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
8620
8621 if (rem)
8622 rmem->nr_pages = rem;
8623 }
8624 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
8625 if (rc)
8626 break;
8627 }
8628 } else {
8629 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8630 if (rmem->nr_pages > 1 || depth)
8631 rmem->depth = 1;
8632 rmem->ctx_mem = ctxm;
8633 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8634 }
8635 return rc;
8636 }
8637
bnxt_free_ctx_pg_tbls(struct bnxt * bp,struct bnxt_ctx_pg_info * ctx_pg)8638 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
8639 struct bnxt_ctx_pg_info *ctx_pg)
8640 {
8641 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8642
8643 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
8644 ctx_pg->ctx_pg_tbl) {
8645 int i, nr_tbls = rmem->nr_pages;
8646
8647 for (i = 0; i < nr_tbls; i++) {
8648 struct bnxt_ctx_pg_info *pg_tbl;
8649 struct bnxt_ring_mem_info *rmem2;
8650
8651 pg_tbl = ctx_pg->ctx_pg_tbl[i];
8652 if (!pg_tbl)
8653 continue;
8654 rmem2 = &pg_tbl->ring_mem;
8655 bnxt_free_ring(bp, rmem2);
8656 ctx_pg->ctx_pg_arr[i] = NULL;
8657 kfree(pg_tbl);
8658 ctx_pg->ctx_pg_tbl[i] = NULL;
8659 }
8660 kfree(ctx_pg->ctx_pg_tbl);
8661 ctx_pg->ctx_pg_tbl = NULL;
8662 }
8663 bnxt_free_ring(bp, rmem);
8664 ctx_pg->nr_pages = 0;
8665 }
8666
bnxt_setup_ctxm_pg_tbls(struct bnxt * bp,struct bnxt_ctx_mem_type * ctxm,u32 entries,u8 pg_lvl)8667 static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp,
8668 struct bnxt_ctx_mem_type *ctxm, u32 entries,
8669 u8 pg_lvl)
8670 {
8671 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8672 int i, rc = 0, n = 1;
8673 u32 mem_size;
8674
8675 if (!ctxm->entry_size || !ctx_pg)
8676 return -EINVAL;
8677 if (ctxm->instance_bmap)
8678 n = hweight32(ctxm->instance_bmap);
8679 if (ctxm->entry_multiple)
8680 entries = roundup(entries, ctxm->entry_multiple);
8681 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
8682 mem_size = entries * ctxm->entry_size;
8683 for (i = 0; i < n && !rc; i++) {
8684 ctx_pg[i].entries = entries;
8685 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl,
8686 ctxm->init_value ? ctxm : NULL);
8687 }
8688 return rc;
8689 }
8690
bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt * bp,struct bnxt_ctx_mem_type * ctxm,bool last)8691 static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp,
8692 struct bnxt_ctx_mem_type *ctxm,
8693 bool last)
8694 {
8695 struct hwrm_func_backing_store_cfg_v2_input *req;
8696 u32 instance_bmap = ctxm->instance_bmap;
8697 int i, j, rc = 0, n = 1;
8698 __le32 *p;
8699
8700 if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info)
8701 return 0;
8702
8703 if (instance_bmap)
8704 n = hweight32(ctxm->instance_bmap);
8705 else
8706 instance_bmap = 1;
8707
8708 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2);
8709 if (rc)
8710 return rc;
8711 hwrm_req_hold(bp, req);
8712 req->type = cpu_to_le16(ctxm->type);
8713 req->entry_size = cpu_to_le16(ctxm->entry_size);
8714 req->subtype_valid_cnt = ctxm->split_entry_cnt;
8715 for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++)
8716 p[i] = cpu_to_le32(ctxm->split[i]);
8717 for (i = 0, j = 0; j < n && !rc; i++) {
8718 struct bnxt_ctx_pg_info *ctx_pg;
8719
8720 if (!(instance_bmap & (1 << i)))
8721 continue;
8722 req->instance = cpu_to_le16(i);
8723 ctx_pg = &ctxm->pg_info[j++];
8724 if (!ctx_pg->entries)
8725 continue;
8726 req->num_entries = cpu_to_le32(ctx_pg->entries);
8727 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8728 &req->page_size_pbl_level,
8729 &req->page_dir);
8730 if (last && j == n)
8731 req->flags =
8732 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE);
8733 rc = hwrm_req_send(bp, req);
8734 }
8735 hwrm_req_drop(bp, req);
8736 return rc;
8737 }
8738
bnxt_backing_store_cfg_v2(struct bnxt * bp,u32 ena)8739 static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena)
8740 {
8741 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8742 struct bnxt_ctx_mem_type *ctxm;
8743 u16 last_type;
8744 int rc = 0;
8745 u16 type;
8746
8747 if (!ena)
8748 return 0;
8749 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
8750 last_type = BNXT_CTX_MAX - 1;
8751 else
8752 last_type = BNXT_CTX_L2_MAX - 1;
8753 ctx->ctx_arr[last_type].last = 1;
8754
8755 for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) {
8756 ctxm = &ctx->ctx_arr[type];
8757
8758 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last);
8759 if (rc)
8760 return rc;
8761 }
8762 return 0;
8763 }
8764
bnxt_free_ctx_mem(struct bnxt * bp)8765 void bnxt_free_ctx_mem(struct bnxt *bp)
8766 {
8767 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8768 u16 type;
8769
8770 if (!ctx)
8771 return;
8772
8773 for (type = 0; type < BNXT_CTX_V2_MAX; type++) {
8774 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8775 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8776 int i, n = 1;
8777
8778 if (!ctx_pg)
8779 continue;
8780 if (ctxm->instance_bmap)
8781 n = hweight32(ctxm->instance_bmap);
8782 for (i = 0; i < n; i++)
8783 bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]);
8784
8785 kfree(ctx_pg);
8786 ctxm->pg_info = NULL;
8787 }
8788
8789 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
8790 kfree(ctx);
8791 bp->ctx = NULL;
8792 }
8793
bnxt_alloc_ctx_mem(struct bnxt * bp)8794 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
8795 {
8796 struct bnxt_ctx_mem_type *ctxm;
8797 struct bnxt_ctx_mem_info *ctx;
8798 u32 l2_qps, qp1_qps, max_qps;
8799 u32 ena, entries_sp, entries;
8800 u32 srqs, max_srqs, min;
8801 u32 num_mr, num_ah;
8802 u32 extra_srqs = 0;
8803 u32 extra_qps = 0;
8804 u32 fast_qpmd_qps;
8805 u8 pg_lvl = 1;
8806 int i, rc;
8807
8808 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
8809 if (rc) {
8810 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
8811 rc);
8812 return rc;
8813 }
8814 ctx = bp->ctx;
8815 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
8816 return 0;
8817
8818 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8819 l2_qps = ctxm->qp_l2_entries;
8820 qp1_qps = ctxm->qp_qp1_entries;
8821 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
8822 max_qps = ctxm->max_entries;
8823 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8824 srqs = ctxm->srq_l2_entries;
8825 max_srqs = ctxm->max_entries;
8826 ena = 0;
8827 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
8828 pg_lvl = 2;
8829 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
8830 /* allocate extra qps if fw supports RoCE fast qp destroy feature */
8831 extra_qps += fast_qpmd_qps;
8832 extra_srqs = min_t(u32, 8192, max_srqs - srqs);
8833 if (fast_qpmd_qps)
8834 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
8835 }
8836
8837 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8838 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps,
8839 pg_lvl);
8840 if (rc)
8841 return rc;
8842
8843 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8844 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl);
8845 if (rc)
8846 return rc;
8847
8848 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8849 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries +
8850 extra_qps * 2, pg_lvl);
8851 if (rc)
8852 return rc;
8853
8854 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8855 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8856 if (rc)
8857 return rc;
8858
8859 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8860 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8861 if (rc)
8862 return rc;
8863
8864 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
8865 goto skip_rdma;
8866
8867 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8868 /* 128K extra is needed to accommodate static AH context
8869 * allocation by f/w.
8870 */
8871 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
8872 num_ah = min_t(u32, num_mr, 1024 * 128);
8873 ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1;
8874 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
8875 ctxm->mrav_av_entries = num_ah;
8876
8877 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2);
8878 if (rc)
8879 return rc;
8880 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
8881
8882 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8883 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1);
8884 if (rc)
8885 return rc;
8886 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
8887
8888 skip_rdma:
8889 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8890 min = ctxm->min_entries;
8891 entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps +
8892 2 * (extra_qps + qp1_qps) + min;
8893 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2);
8894 if (rc)
8895 return rc;
8896
8897 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8898 entries = l2_qps + 2 * (extra_qps + qp1_qps);
8899 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2);
8900 if (rc)
8901 return rc;
8902 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
8903 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
8904 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
8905
8906 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8907 rc = bnxt_backing_store_cfg_v2(bp, ena);
8908 else
8909 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
8910 if (rc) {
8911 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
8912 rc);
8913 return rc;
8914 }
8915 ctx->flags |= BNXT_CTX_FLAG_INITED;
8916 return 0;
8917 }
8918
bnxt_hwrm_crash_dump_mem_cfg(struct bnxt * bp)8919 static int bnxt_hwrm_crash_dump_mem_cfg(struct bnxt *bp)
8920 {
8921 struct hwrm_dbg_crashdump_medium_cfg_input *req;
8922 u16 page_attr;
8923 int rc;
8924
8925 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR))
8926 return 0;
8927
8928 rc = hwrm_req_init(bp, req, HWRM_DBG_CRASHDUMP_MEDIUM_CFG);
8929 if (rc)
8930 return rc;
8931
8932 if (BNXT_PAGE_SIZE == 0x2000)
8933 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_8K;
8934 else if (BNXT_PAGE_SIZE == 0x10000)
8935 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_64K;
8936 else
8937 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_4K;
8938 req->pg_size_lvl = cpu_to_le16(page_attr |
8939 bp->fw_crash_mem->ring_mem.depth);
8940 req->pbl = cpu_to_le64(bp->fw_crash_mem->ring_mem.pg_tbl_map);
8941 req->size = cpu_to_le32(bp->fw_crash_len);
8942 req->output_dest_flags = cpu_to_le16(BNXT_DBG_CR_DUMP_MDM_CFG_DDR);
8943 return hwrm_req_send(bp, req);
8944 }
8945
bnxt_free_crash_dump_mem(struct bnxt * bp)8946 static void bnxt_free_crash_dump_mem(struct bnxt *bp)
8947 {
8948 if (bp->fw_crash_mem) {
8949 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem);
8950 kfree(bp->fw_crash_mem);
8951 bp->fw_crash_mem = NULL;
8952 }
8953 }
8954
bnxt_alloc_crash_dump_mem(struct bnxt * bp)8955 static int bnxt_alloc_crash_dump_mem(struct bnxt *bp)
8956 {
8957 u32 mem_size = 0;
8958 int rc;
8959
8960 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR))
8961 return 0;
8962
8963 rc = bnxt_hwrm_get_dump_len(bp, BNXT_DUMP_CRASH, &mem_size);
8964 if (rc)
8965 return rc;
8966
8967 mem_size = round_up(mem_size, 4);
8968
8969 /* keep and use the existing pages */
8970 if (bp->fw_crash_mem &&
8971 mem_size <= bp->fw_crash_mem->nr_pages * BNXT_PAGE_SIZE)
8972 goto alloc_done;
8973
8974 if (bp->fw_crash_mem)
8975 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem);
8976 else
8977 bp->fw_crash_mem = kzalloc(sizeof(*bp->fw_crash_mem),
8978 GFP_KERNEL);
8979 if (!bp->fw_crash_mem)
8980 return -ENOMEM;
8981
8982 rc = bnxt_alloc_ctx_pg_tbls(bp, bp->fw_crash_mem, mem_size, 1, NULL);
8983 if (rc) {
8984 bnxt_free_crash_dump_mem(bp);
8985 return rc;
8986 }
8987
8988 alloc_done:
8989 bp->fw_crash_len = mem_size;
8990 return 0;
8991 }
8992
bnxt_hwrm_func_resc_qcaps(struct bnxt * bp,bool all)8993 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
8994 {
8995 struct hwrm_func_resource_qcaps_output *resp;
8996 struct hwrm_func_resource_qcaps_input *req;
8997 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8998 int rc;
8999
9000 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
9001 if (rc)
9002 return rc;
9003
9004 req->fid = cpu_to_le16(0xffff);
9005 resp = hwrm_req_hold(bp, req);
9006 rc = hwrm_req_send_silent(bp, req);
9007 if (rc)
9008 goto hwrm_func_resc_qcaps_exit;
9009
9010 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
9011 if (!all)
9012 goto hwrm_func_resc_qcaps_exit;
9013
9014 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
9015 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
9016 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
9017 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
9018 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
9019 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
9020 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
9021 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
9022 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
9023 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
9024 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
9025 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
9026 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
9027 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
9028 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
9029 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
9030
9031 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
9032 u16 max_msix = le16_to_cpu(resp->max_msix);
9033
9034 hw_resc->max_nqs = max_msix;
9035 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
9036 }
9037
9038 if (BNXT_PF(bp)) {
9039 struct bnxt_pf_info *pf = &bp->pf;
9040
9041 pf->vf_resv_strategy =
9042 le16_to_cpu(resp->vf_reservation_strategy);
9043 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
9044 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
9045 }
9046 hwrm_func_resc_qcaps_exit:
9047 hwrm_req_drop(bp, req);
9048 return rc;
9049 }
9050
__bnxt_hwrm_ptp_qcfg(struct bnxt * bp)9051 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
9052 {
9053 struct hwrm_port_mac_ptp_qcfg_output *resp;
9054 struct hwrm_port_mac_ptp_qcfg_input *req;
9055 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
9056 bool phc_cfg;
9057 u8 flags;
9058 int rc;
9059
9060 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_PLUS(bp)) {
9061 rc = -ENODEV;
9062 goto no_ptp;
9063 }
9064
9065 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
9066 if (rc)
9067 goto no_ptp;
9068
9069 req->port_id = cpu_to_le16(bp->pf.port_id);
9070 resp = hwrm_req_hold(bp, req);
9071 rc = hwrm_req_send(bp, req);
9072 if (rc)
9073 goto exit;
9074
9075 flags = resp->flags;
9076 if (BNXT_CHIP_P5_AND_MINUS(bp) &&
9077 !(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
9078 rc = -ENODEV;
9079 goto exit;
9080 }
9081 if (!ptp) {
9082 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
9083 if (!ptp) {
9084 rc = -ENOMEM;
9085 goto exit;
9086 }
9087 ptp->bp = bp;
9088 bp->ptp_cfg = ptp;
9089 }
9090
9091 if (flags &
9092 (PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK |
9093 PORT_MAC_PTP_QCFG_RESP_FLAGS_64B_PHC_TIME)) {
9094 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
9095 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
9096 } else if (BNXT_CHIP_P5(bp)) {
9097 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
9098 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
9099 } else {
9100 rc = -ENODEV;
9101 goto exit;
9102 }
9103 phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
9104 rc = bnxt_ptp_init(bp, phc_cfg);
9105 if (rc)
9106 netdev_warn(bp->dev, "PTP initialization failed.\n");
9107 exit:
9108 hwrm_req_drop(bp, req);
9109 if (!rc)
9110 return 0;
9111
9112 no_ptp:
9113 bnxt_ptp_clear(bp);
9114 kfree(ptp);
9115 bp->ptp_cfg = NULL;
9116 return rc;
9117 }
9118
__bnxt_hwrm_func_qcaps(struct bnxt * bp)9119 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
9120 {
9121 struct hwrm_func_qcaps_output *resp;
9122 struct hwrm_func_qcaps_input *req;
9123 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9124 u32 flags, flags_ext, flags_ext2;
9125 int rc;
9126
9127 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
9128 if (rc)
9129 return rc;
9130
9131 req->fid = cpu_to_le16(0xffff);
9132 resp = hwrm_req_hold(bp, req);
9133 rc = hwrm_req_send(bp, req);
9134 if (rc)
9135 goto hwrm_func_qcaps_exit;
9136
9137 flags = le32_to_cpu(resp->flags);
9138 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
9139 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
9140 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
9141 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
9142 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
9143 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
9144 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
9145 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
9146 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
9147 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
9148 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
9149 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
9150 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
9151 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
9152 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
9153 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
9154 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
9155 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
9156
9157 flags_ext = le32_to_cpu(resp->flags_ext);
9158 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
9159 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
9160 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
9161 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
9162 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
9163 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
9164 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
9165 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
9166 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
9167 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
9168 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED))
9169 bp->fw_cap |= BNXT_FW_CAP_DFLT_VLAN_TPID_PCP;
9170 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED)
9171 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2;
9172 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP)
9173 bp->flags |= BNXT_FLAG_TX_COAL_CMPL;
9174
9175 flags_ext2 = le32_to_cpu(resp->flags_ext2);
9176 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
9177 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
9178 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED)
9179 bp->flags |= BNXT_FLAG_UDP_GSO_CAP;
9180 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED)
9181 bp->fw_cap |= BNXT_FW_CAP_TX_TS_CMP;
9182
9183 bp->tx_push_thresh = 0;
9184 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
9185 BNXT_FW_MAJ(bp) > 217)
9186 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
9187
9188 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
9189 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
9190 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
9191 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
9192 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
9193 if (!hw_resc->max_hw_ring_grps)
9194 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
9195 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
9196 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
9197 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
9198
9199 hw_resc->max_encap_records = le32_to_cpu(resp->max_encap_records);
9200 hw_resc->max_decap_records = le32_to_cpu(resp->max_decap_records);
9201 hw_resc->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
9202 hw_resc->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
9203 hw_resc->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
9204 hw_resc->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
9205
9206 if (BNXT_PF(bp)) {
9207 struct bnxt_pf_info *pf = &bp->pf;
9208
9209 pf->fw_fid = le16_to_cpu(resp->fid);
9210 pf->port_id = le16_to_cpu(resp->port_id);
9211 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
9212 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
9213 pf->max_vfs = le16_to_cpu(resp->max_vfs);
9214 bp->flags &= ~BNXT_FLAG_WOL_CAP;
9215 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
9216 bp->flags |= BNXT_FLAG_WOL_CAP;
9217 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
9218 bp->fw_cap |= BNXT_FW_CAP_PTP;
9219 } else {
9220 bnxt_ptp_clear(bp);
9221 kfree(bp->ptp_cfg);
9222 bp->ptp_cfg = NULL;
9223 }
9224 } else {
9225 #ifdef CONFIG_BNXT_SRIOV
9226 struct bnxt_vf_info *vf = &bp->vf;
9227
9228 vf->fw_fid = le16_to_cpu(resp->fid);
9229 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
9230 #endif
9231 }
9232 bp->tso_max_segs = le16_to_cpu(resp->max_tso_segs);
9233
9234 hwrm_func_qcaps_exit:
9235 hwrm_req_drop(bp, req);
9236 return rc;
9237 }
9238
bnxt_hwrm_dbg_qcaps(struct bnxt * bp)9239 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
9240 {
9241 struct hwrm_dbg_qcaps_output *resp;
9242 struct hwrm_dbg_qcaps_input *req;
9243 int rc;
9244
9245 bp->fw_dbg_cap = 0;
9246 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
9247 return;
9248
9249 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
9250 if (rc)
9251 return;
9252
9253 req->fid = cpu_to_le16(0xffff);
9254 resp = hwrm_req_hold(bp, req);
9255 rc = hwrm_req_send(bp, req);
9256 if (rc)
9257 goto hwrm_dbg_qcaps_exit;
9258
9259 bp->fw_dbg_cap = le32_to_cpu(resp->flags);
9260
9261 hwrm_dbg_qcaps_exit:
9262 hwrm_req_drop(bp, req);
9263 }
9264
9265 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
9266
bnxt_hwrm_func_qcaps(struct bnxt * bp)9267 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
9268 {
9269 int rc;
9270
9271 rc = __bnxt_hwrm_func_qcaps(bp);
9272 if (rc)
9273 return rc;
9274
9275 bnxt_hwrm_dbg_qcaps(bp);
9276
9277 rc = bnxt_hwrm_queue_qportcfg(bp);
9278 if (rc) {
9279 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
9280 return rc;
9281 }
9282 if (bp->hwrm_spec_code >= 0x10803) {
9283 rc = bnxt_alloc_ctx_mem(bp);
9284 if (rc)
9285 return rc;
9286 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9287 if (!rc)
9288 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
9289 }
9290 return 0;
9291 }
9292
bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt * bp)9293 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
9294 {
9295 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
9296 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
9297 u32 flags;
9298 int rc;
9299
9300 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
9301 return 0;
9302
9303 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
9304 if (rc)
9305 return rc;
9306
9307 resp = hwrm_req_hold(bp, req);
9308 rc = hwrm_req_send(bp, req);
9309 if (rc)
9310 goto hwrm_cfa_adv_qcaps_exit;
9311
9312 flags = le32_to_cpu(resp->flags);
9313 if (flags &
9314 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
9315 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
9316
9317 if (flags &
9318 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED)
9319 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3;
9320
9321 if (flags &
9322 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED)
9323 bp->fw_cap |= BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO;
9324
9325 hwrm_cfa_adv_qcaps_exit:
9326 hwrm_req_drop(bp, req);
9327 return rc;
9328 }
9329
__bnxt_alloc_fw_health(struct bnxt * bp)9330 static int __bnxt_alloc_fw_health(struct bnxt *bp)
9331 {
9332 if (bp->fw_health)
9333 return 0;
9334
9335 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
9336 if (!bp->fw_health)
9337 return -ENOMEM;
9338
9339 mutex_init(&bp->fw_health->lock);
9340 return 0;
9341 }
9342
bnxt_alloc_fw_health(struct bnxt * bp)9343 static int bnxt_alloc_fw_health(struct bnxt *bp)
9344 {
9345 int rc;
9346
9347 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
9348 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9349 return 0;
9350
9351 rc = __bnxt_alloc_fw_health(bp);
9352 if (rc) {
9353 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
9354 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9355 return rc;
9356 }
9357
9358 return 0;
9359 }
9360
__bnxt_map_fw_health_reg(struct bnxt * bp,u32 reg)9361 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
9362 {
9363 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
9364 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
9365 BNXT_FW_HEALTH_WIN_MAP_OFF);
9366 }
9367
bnxt_inv_fw_health_reg(struct bnxt * bp)9368 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
9369 {
9370 struct bnxt_fw_health *fw_health = bp->fw_health;
9371 u32 reg_type;
9372
9373 if (!fw_health)
9374 return;
9375
9376 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
9377 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9378 fw_health->status_reliable = false;
9379
9380 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
9381 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9382 fw_health->resets_reliable = false;
9383 }
9384
bnxt_try_map_fw_health_reg(struct bnxt * bp)9385 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
9386 {
9387 void __iomem *hs;
9388 u32 status_loc;
9389 u32 reg_type;
9390 u32 sig;
9391
9392 if (bp->fw_health)
9393 bp->fw_health->status_reliable = false;
9394
9395 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
9396 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
9397
9398 sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
9399 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
9400 if (!bp->chip_num) {
9401 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
9402 bp->chip_num = readl(bp->bar0 +
9403 BNXT_FW_HEALTH_WIN_BASE +
9404 BNXT_GRC_REG_CHIP_NUM);
9405 }
9406 if (!BNXT_CHIP_P5_PLUS(bp))
9407 return;
9408
9409 status_loc = BNXT_GRC_REG_STATUS_P5 |
9410 BNXT_FW_HEALTH_REG_TYPE_BAR0;
9411 } else {
9412 status_loc = readl(hs + offsetof(struct hcomm_status,
9413 fw_status_loc));
9414 }
9415
9416 if (__bnxt_alloc_fw_health(bp)) {
9417 netdev_warn(bp->dev, "no memory for firmware status checks\n");
9418 return;
9419 }
9420
9421 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
9422 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
9423 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
9424 __bnxt_map_fw_health_reg(bp, status_loc);
9425 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
9426 BNXT_FW_HEALTH_WIN_OFF(status_loc);
9427 }
9428
9429 bp->fw_health->status_reliable = true;
9430 }
9431
bnxt_map_fw_health_regs(struct bnxt * bp)9432 static int bnxt_map_fw_health_regs(struct bnxt *bp)
9433 {
9434 struct bnxt_fw_health *fw_health = bp->fw_health;
9435 u32 reg_base = 0xffffffff;
9436 int i;
9437
9438 bp->fw_health->status_reliable = false;
9439 bp->fw_health->resets_reliable = false;
9440 /* Only pre-map the monitoring GRC registers using window 3 */
9441 for (i = 0; i < 4; i++) {
9442 u32 reg = fw_health->regs[i];
9443
9444 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
9445 continue;
9446 if (reg_base == 0xffffffff)
9447 reg_base = reg & BNXT_GRC_BASE_MASK;
9448 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
9449 return -ERANGE;
9450 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
9451 }
9452 bp->fw_health->status_reliable = true;
9453 bp->fw_health->resets_reliable = true;
9454 if (reg_base == 0xffffffff)
9455 return 0;
9456
9457 __bnxt_map_fw_health_reg(bp, reg_base);
9458 return 0;
9459 }
9460
bnxt_remap_fw_health_regs(struct bnxt * bp)9461 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
9462 {
9463 if (!bp->fw_health)
9464 return;
9465
9466 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
9467 bp->fw_health->status_reliable = true;
9468 bp->fw_health->resets_reliable = true;
9469 } else {
9470 bnxt_try_map_fw_health_reg(bp);
9471 }
9472 }
9473
bnxt_hwrm_error_recovery_qcfg(struct bnxt * bp)9474 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
9475 {
9476 struct bnxt_fw_health *fw_health = bp->fw_health;
9477 struct hwrm_error_recovery_qcfg_output *resp;
9478 struct hwrm_error_recovery_qcfg_input *req;
9479 int rc, i;
9480
9481 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9482 return 0;
9483
9484 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
9485 if (rc)
9486 return rc;
9487
9488 resp = hwrm_req_hold(bp, req);
9489 rc = hwrm_req_send(bp, req);
9490 if (rc)
9491 goto err_recovery_out;
9492 fw_health->flags = le32_to_cpu(resp->flags);
9493 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
9494 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
9495 rc = -EINVAL;
9496 goto err_recovery_out;
9497 }
9498 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
9499 fw_health->master_func_wait_dsecs =
9500 le32_to_cpu(resp->master_func_wait_period);
9501 fw_health->normal_func_wait_dsecs =
9502 le32_to_cpu(resp->normal_func_wait_period);
9503 fw_health->post_reset_wait_dsecs =
9504 le32_to_cpu(resp->master_func_wait_period_after_reset);
9505 fw_health->post_reset_max_wait_dsecs =
9506 le32_to_cpu(resp->max_bailout_time_after_reset);
9507 fw_health->regs[BNXT_FW_HEALTH_REG] =
9508 le32_to_cpu(resp->fw_health_status_reg);
9509 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
9510 le32_to_cpu(resp->fw_heartbeat_reg);
9511 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
9512 le32_to_cpu(resp->fw_reset_cnt_reg);
9513 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
9514 le32_to_cpu(resp->reset_inprogress_reg);
9515 fw_health->fw_reset_inprog_reg_mask =
9516 le32_to_cpu(resp->reset_inprogress_reg_mask);
9517 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
9518 if (fw_health->fw_reset_seq_cnt >= 16) {
9519 rc = -EINVAL;
9520 goto err_recovery_out;
9521 }
9522 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
9523 fw_health->fw_reset_seq_regs[i] =
9524 le32_to_cpu(resp->reset_reg[i]);
9525 fw_health->fw_reset_seq_vals[i] =
9526 le32_to_cpu(resp->reset_reg_val[i]);
9527 fw_health->fw_reset_seq_delay_msec[i] =
9528 resp->delay_after_reset[i];
9529 }
9530 err_recovery_out:
9531 hwrm_req_drop(bp, req);
9532 if (!rc)
9533 rc = bnxt_map_fw_health_regs(bp);
9534 if (rc)
9535 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9536 return rc;
9537 }
9538
bnxt_hwrm_func_reset(struct bnxt * bp)9539 static int bnxt_hwrm_func_reset(struct bnxt *bp)
9540 {
9541 struct hwrm_func_reset_input *req;
9542 int rc;
9543
9544 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
9545 if (rc)
9546 return rc;
9547
9548 req->enables = 0;
9549 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
9550 return hwrm_req_send(bp, req);
9551 }
9552
bnxt_nvm_cfg_ver_get(struct bnxt * bp)9553 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
9554 {
9555 struct hwrm_nvm_get_dev_info_output nvm_info;
9556
9557 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
9558 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
9559 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
9560 nvm_info.nvm_cfg_ver_upd);
9561 }
9562
bnxt_hwrm_queue_qportcfg(struct bnxt * bp)9563 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
9564 {
9565 struct hwrm_queue_qportcfg_output *resp;
9566 struct hwrm_queue_qportcfg_input *req;
9567 u8 i, j, *qptr;
9568 bool no_rdma;
9569 int rc = 0;
9570
9571 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
9572 if (rc)
9573 return rc;
9574
9575 resp = hwrm_req_hold(bp, req);
9576 rc = hwrm_req_send(bp, req);
9577 if (rc)
9578 goto qportcfg_exit;
9579
9580 if (!resp->max_configurable_queues) {
9581 rc = -EINVAL;
9582 goto qportcfg_exit;
9583 }
9584 bp->max_tc = resp->max_configurable_queues;
9585 bp->max_lltc = resp->max_configurable_lossless_queues;
9586 if (bp->max_tc > BNXT_MAX_QUEUE)
9587 bp->max_tc = BNXT_MAX_QUEUE;
9588
9589 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
9590 qptr = &resp->queue_id0;
9591 for (i = 0, j = 0; i < bp->max_tc; i++) {
9592 bp->q_info[j].queue_id = *qptr;
9593 bp->q_ids[i] = *qptr++;
9594 bp->q_info[j].queue_profile = *qptr++;
9595 bp->tc_to_qidx[j] = j;
9596 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
9597 (no_rdma && BNXT_PF(bp)))
9598 j++;
9599 }
9600 bp->max_q = bp->max_tc;
9601 bp->max_tc = max_t(u8, j, 1);
9602
9603 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
9604 bp->max_tc = 1;
9605
9606 if (bp->max_lltc > bp->max_tc)
9607 bp->max_lltc = bp->max_tc;
9608
9609 qportcfg_exit:
9610 hwrm_req_drop(bp, req);
9611 return rc;
9612 }
9613
bnxt_hwrm_poll(struct bnxt * bp)9614 static int bnxt_hwrm_poll(struct bnxt *bp)
9615 {
9616 struct hwrm_ver_get_input *req;
9617 int rc;
9618
9619 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9620 if (rc)
9621 return rc;
9622
9623 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9624 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9625 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9626
9627 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
9628 rc = hwrm_req_send(bp, req);
9629 return rc;
9630 }
9631
bnxt_hwrm_ver_get(struct bnxt * bp)9632 static int bnxt_hwrm_ver_get(struct bnxt *bp)
9633 {
9634 struct hwrm_ver_get_output *resp;
9635 struct hwrm_ver_get_input *req;
9636 u16 fw_maj, fw_min, fw_bld, fw_rsv;
9637 u32 dev_caps_cfg, hwrm_ver;
9638 int rc, len;
9639
9640 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9641 if (rc)
9642 return rc;
9643
9644 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
9645 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
9646 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9647 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9648 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9649
9650 resp = hwrm_req_hold(bp, req);
9651 rc = hwrm_req_send(bp, req);
9652 if (rc)
9653 goto hwrm_ver_get_exit;
9654
9655 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
9656
9657 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
9658 resp->hwrm_intf_min_8b << 8 |
9659 resp->hwrm_intf_upd_8b;
9660 if (resp->hwrm_intf_maj_8b < 1) {
9661 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
9662 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9663 resp->hwrm_intf_upd_8b);
9664 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
9665 }
9666
9667 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
9668 HWRM_VERSION_UPDATE;
9669
9670 if (bp->hwrm_spec_code > hwrm_ver)
9671 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9672 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
9673 HWRM_VERSION_UPDATE);
9674 else
9675 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9676 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9677 resp->hwrm_intf_upd_8b);
9678
9679 fw_maj = le16_to_cpu(resp->hwrm_fw_major);
9680 if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
9681 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
9682 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
9683 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
9684 len = FW_VER_STR_LEN;
9685 } else {
9686 fw_maj = resp->hwrm_fw_maj_8b;
9687 fw_min = resp->hwrm_fw_min_8b;
9688 fw_bld = resp->hwrm_fw_bld_8b;
9689 fw_rsv = resp->hwrm_fw_rsvd_8b;
9690 len = BC_HWRM_STR_LEN;
9691 }
9692 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
9693 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
9694 fw_rsv);
9695
9696 if (strlen(resp->active_pkg_name)) {
9697 int fw_ver_len = strlen(bp->fw_ver_str);
9698
9699 snprintf(bp->fw_ver_str + fw_ver_len,
9700 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
9701 resp->active_pkg_name);
9702 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
9703 }
9704
9705 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
9706 if (!bp->hwrm_cmd_timeout)
9707 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
9708 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
9709 if (!bp->hwrm_cmd_max_timeout)
9710 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
9711 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
9712 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
9713 bp->hwrm_cmd_max_timeout / 1000);
9714
9715 if (resp->hwrm_intf_maj_8b >= 1) {
9716 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
9717 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
9718 }
9719 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
9720 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
9721
9722 bp->chip_num = le16_to_cpu(resp->chip_num);
9723 bp->chip_rev = resp->chip_rev;
9724 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
9725 !resp->chip_metal)
9726 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
9727
9728 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
9729 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
9730 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
9731 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
9732
9733 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
9734 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
9735
9736 if (dev_caps_cfg &
9737 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
9738 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
9739
9740 if (dev_caps_cfg &
9741 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
9742 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
9743
9744 if (dev_caps_cfg &
9745 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
9746 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
9747
9748 hwrm_ver_get_exit:
9749 hwrm_req_drop(bp, req);
9750 return rc;
9751 }
9752
bnxt_hwrm_fw_set_time(struct bnxt * bp)9753 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
9754 {
9755 struct hwrm_fw_set_time_input *req;
9756 struct tm tm;
9757 time64_t now = ktime_get_real_seconds();
9758 int rc;
9759
9760 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
9761 bp->hwrm_spec_code < 0x10400)
9762 return -EOPNOTSUPP;
9763
9764 time64_to_tm(now, 0, &tm);
9765 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
9766 if (rc)
9767 return rc;
9768
9769 req->year = cpu_to_le16(1900 + tm.tm_year);
9770 req->month = 1 + tm.tm_mon;
9771 req->day = tm.tm_mday;
9772 req->hour = tm.tm_hour;
9773 req->minute = tm.tm_min;
9774 req->second = tm.tm_sec;
9775 return hwrm_req_send(bp, req);
9776 }
9777
bnxt_add_one_ctr(u64 hw,u64 * sw,u64 mask)9778 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
9779 {
9780 u64 sw_tmp;
9781
9782 hw &= mask;
9783 sw_tmp = (*sw & ~mask) | hw;
9784 if (hw < (*sw & mask))
9785 sw_tmp += mask + 1;
9786 WRITE_ONCE(*sw, sw_tmp);
9787 }
9788
__bnxt_accumulate_stats(__le64 * hw_stats,u64 * sw_stats,u64 * masks,int count,bool ignore_zero)9789 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
9790 int count, bool ignore_zero)
9791 {
9792 int i;
9793
9794 for (i = 0; i < count; i++) {
9795 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
9796
9797 if (ignore_zero && !hw)
9798 continue;
9799
9800 if (masks[i] == -1ULL)
9801 sw_stats[i] = hw;
9802 else
9803 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
9804 }
9805 }
9806
bnxt_accumulate_stats(struct bnxt_stats_mem * stats)9807 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
9808 {
9809 if (!stats->hw_stats)
9810 return;
9811
9812 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9813 stats->hw_masks, stats->len / 8, false);
9814 }
9815
bnxt_accumulate_all_stats(struct bnxt * bp)9816 static void bnxt_accumulate_all_stats(struct bnxt *bp)
9817 {
9818 struct bnxt_stats_mem *ring0_stats;
9819 bool ignore_zero = false;
9820 int i;
9821
9822 /* Chip bug. Counter intermittently becomes 0. */
9823 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9824 ignore_zero = true;
9825
9826 for (i = 0; i < bp->cp_nr_rings; i++) {
9827 struct bnxt_napi *bnapi = bp->bnapi[i];
9828 struct bnxt_cp_ring_info *cpr;
9829 struct bnxt_stats_mem *stats;
9830
9831 cpr = &bnapi->cp_ring;
9832 stats = &cpr->stats;
9833 if (!i)
9834 ring0_stats = stats;
9835 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9836 ring0_stats->hw_masks,
9837 ring0_stats->len / 8, ignore_zero);
9838 }
9839 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9840 struct bnxt_stats_mem *stats = &bp->port_stats;
9841 __le64 *hw_stats = stats->hw_stats;
9842 u64 *sw_stats = stats->sw_stats;
9843 u64 *masks = stats->hw_masks;
9844 int cnt;
9845
9846 cnt = sizeof(struct rx_port_stats) / 8;
9847 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9848
9849 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9850 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9851 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9852 cnt = sizeof(struct tx_port_stats) / 8;
9853 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9854 }
9855 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
9856 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
9857 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
9858 }
9859 }
9860
bnxt_hwrm_port_qstats(struct bnxt * bp,u8 flags)9861 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
9862 {
9863 struct hwrm_port_qstats_input *req;
9864 struct bnxt_pf_info *pf = &bp->pf;
9865 int rc;
9866
9867 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
9868 return 0;
9869
9870 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9871 return -EOPNOTSUPP;
9872
9873 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
9874 if (rc)
9875 return rc;
9876
9877 req->flags = flags;
9878 req->port_id = cpu_to_le16(pf->port_id);
9879 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
9880 BNXT_TX_PORT_STATS_BYTE_OFFSET);
9881 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
9882 return hwrm_req_send(bp, req);
9883 }
9884
bnxt_hwrm_port_qstats_ext(struct bnxt * bp,u8 flags)9885 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
9886 {
9887 struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
9888 struct hwrm_queue_pri2cos_qcfg_input *req_qc;
9889 struct hwrm_port_qstats_ext_output *resp_qs;
9890 struct hwrm_port_qstats_ext_input *req_qs;
9891 struct bnxt_pf_info *pf = &bp->pf;
9892 u32 tx_stat_size;
9893 int rc;
9894
9895 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
9896 return 0;
9897
9898 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9899 return -EOPNOTSUPP;
9900
9901 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
9902 if (rc)
9903 return rc;
9904
9905 req_qs->flags = flags;
9906 req_qs->port_id = cpu_to_le16(pf->port_id);
9907 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
9908 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
9909 tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
9910 sizeof(struct tx_port_stats_ext) : 0;
9911 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
9912 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
9913 resp_qs = hwrm_req_hold(bp, req_qs);
9914 rc = hwrm_req_send(bp, req_qs);
9915 if (!rc) {
9916 bp->fw_rx_stats_ext_size =
9917 le16_to_cpu(resp_qs->rx_stat_size) / 8;
9918 if (BNXT_FW_MAJ(bp) < 220 &&
9919 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
9920 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
9921
9922 bp->fw_tx_stats_ext_size = tx_stat_size ?
9923 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
9924 } else {
9925 bp->fw_rx_stats_ext_size = 0;
9926 bp->fw_tx_stats_ext_size = 0;
9927 }
9928 hwrm_req_drop(bp, req_qs);
9929
9930 if (flags)
9931 return rc;
9932
9933 if (bp->fw_tx_stats_ext_size <=
9934 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
9935 bp->pri2cos_valid = 0;
9936 return rc;
9937 }
9938
9939 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
9940 if (rc)
9941 return rc;
9942
9943 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
9944
9945 resp_qc = hwrm_req_hold(bp, req_qc);
9946 rc = hwrm_req_send(bp, req_qc);
9947 if (!rc) {
9948 u8 *pri2cos;
9949 int i, j;
9950
9951 pri2cos = &resp_qc->pri0_cos_queue_id;
9952 for (i = 0; i < 8; i++) {
9953 u8 queue_id = pri2cos[i];
9954 u8 queue_idx;
9955
9956 /* Per port queue IDs start from 0, 10, 20, etc */
9957 queue_idx = queue_id % 10;
9958 if (queue_idx > BNXT_MAX_QUEUE) {
9959 bp->pri2cos_valid = false;
9960 hwrm_req_drop(bp, req_qc);
9961 return rc;
9962 }
9963 for (j = 0; j < bp->max_q; j++) {
9964 if (bp->q_ids[j] == queue_id)
9965 bp->pri2cos_idx[i] = queue_idx;
9966 }
9967 }
9968 bp->pri2cos_valid = true;
9969 }
9970 hwrm_req_drop(bp, req_qc);
9971
9972 return rc;
9973 }
9974
bnxt_hwrm_free_tunnel_ports(struct bnxt * bp)9975 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
9976 {
9977 bnxt_hwrm_tunnel_dst_port_free(bp,
9978 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9979 bnxt_hwrm_tunnel_dst_port_free(bp,
9980 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9981 }
9982
bnxt_set_tpa(struct bnxt * bp,bool set_tpa)9983 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
9984 {
9985 int rc, i;
9986 u32 tpa_flags = 0;
9987
9988 if (set_tpa)
9989 tpa_flags = bp->flags & BNXT_FLAG_TPA;
9990 else if (BNXT_NO_FW_ACCESS(bp))
9991 return 0;
9992 for (i = 0; i < bp->nr_vnics; i++) {
9993 rc = bnxt_hwrm_vnic_set_tpa(bp, &bp->vnic_info[i], tpa_flags);
9994 if (rc) {
9995 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
9996 i, rc);
9997 return rc;
9998 }
9999 }
10000 return 0;
10001 }
10002
bnxt_hwrm_clear_vnic_rss(struct bnxt * bp)10003 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
10004 {
10005 int i;
10006
10007 for (i = 0; i < bp->nr_vnics; i++)
10008 bnxt_hwrm_vnic_set_rss(bp, &bp->vnic_info[i], false);
10009 }
10010
bnxt_clear_vnic(struct bnxt * bp)10011 static void bnxt_clear_vnic(struct bnxt *bp)
10012 {
10013 if (!bp->vnic_info)
10014 return;
10015
10016 bnxt_hwrm_clear_vnic_filter(bp);
10017 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) {
10018 /* clear all RSS setting before free vnic ctx */
10019 bnxt_hwrm_clear_vnic_rss(bp);
10020 bnxt_hwrm_vnic_ctx_free(bp);
10021 }
10022 /* before free the vnic, undo the vnic tpa settings */
10023 if (bp->flags & BNXT_FLAG_TPA)
10024 bnxt_set_tpa(bp, false);
10025 bnxt_hwrm_vnic_free(bp);
10026 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10027 bnxt_hwrm_vnic_ctx_free(bp);
10028 }
10029
bnxt_hwrm_resource_free(struct bnxt * bp,bool close_path,bool irq_re_init)10030 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
10031 bool irq_re_init)
10032 {
10033 bnxt_clear_vnic(bp);
10034 bnxt_hwrm_ring_free(bp, close_path);
10035 bnxt_hwrm_ring_grp_free(bp);
10036 if (irq_re_init) {
10037 bnxt_hwrm_stat_ctx_free(bp);
10038 bnxt_hwrm_free_tunnel_ports(bp);
10039 }
10040 }
10041
bnxt_hwrm_set_br_mode(struct bnxt * bp,u16 br_mode)10042 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
10043 {
10044 struct hwrm_func_cfg_input *req;
10045 u8 evb_mode;
10046 int rc;
10047
10048 if (br_mode == BRIDGE_MODE_VEB)
10049 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
10050 else if (br_mode == BRIDGE_MODE_VEPA)
10051 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
10052 else
10053 return -EINVAL;
10054
10055 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
10056 if (rc)
10057 return rc;
10058
10059 req->fid = cpu_to_le16(0xffff);
10060 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
10061 req->evb_mode = evb_mode;
10062 return hwrm_req_send(bp, req);
10063 }
10064
bnxt_hwrm_set_cache_line_size(struct bnxt * bp,int size)10065 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
10066 {
10067 struct hwrm_func_cfg_input *req;
10068 int rc;
10069
10070 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
10071 return 0;
10072
10073 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
10074 if (rc)
10075 return rc;
10076
10077 req->fid = cpu_to_le16(0xffff);
10078 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
10079 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
10080 if (size == 128)
10081 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
10082
10083 return hwrm_req_send(bp, req);
10084 }
10085
__bnxt_setup_vnic(struct bnxt * bp,struct bnxt_vnic_info * vnic)10086 static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10087 {
10088 int rc;
10089
10090 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
10091 goto skip_rss_ctx;
10092
10093 /* allocate context for vnic */
10094 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 0);
10095 if (rc) {
10096 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
10097 vnic->vnic_id, rc);
10098 goto vnic_setup_err;
10099 }
10100 bp->rsscos_nr_ctxs++;
10101
10102 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10103 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 1);
10104 if (rc) {
10105 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
10106 vnic->vnic_id, rc);
10107 goto vnic_setup_err;
10108 }
10109 bp->rsscos_nr_ctxs++;
10110 }
10111
10112 skip_rss_ctx:
10113 /* configure default vnic, ring grp */
10114 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10115 if (rc) {
10116 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
10117 vnic->vnic_id, rc);
10118 goto vnic_setup_err;
10119 }
10120
10121 /* Enable RSS hashing on vnic */
10122 rc = bnxt_hwrm_vnic_set_rss(bp, vnic, true);
10123 if (rc) {
10124 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
10125 vnic->vnic_id, rc);
10126 goto vnic_setup_err;
10127 }
10128
10129 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
10130 rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
10131 if (rc) {
10132 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
10133 vnic->vnic_id, rc);
10134 }
10135 }
10136
10137 vnic_setup_err:
10138 return rc;
10139 }
10140
bnxt_hwrm_vnic_update(struct bnxt * bp,struct bnxt_vnic_info * vnic,u8 valid)10141 int bnxt_hwrm_vnic_update(struct bnxt *bp, struct bnxt_vnic_info *vnic,
10142 u8 valid)
10143 {
10144 struct hwrm_vnic_update_input *req;
10145 int rc;
10146
10147 rc = hwrm_req_init(bp, req, HWRM_VNIC_UPDATE);
10148 if (rc)
10149 return rc;
10150
10151 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
10152
10153 if (valid & VNIC_UPDATE_REQ_ENABLES_MRU_VALID)
10154 req->mru = cpu_to_le16(vnic->mru);
10155
10156 req->enables = cpu_to_le32(valid);
10157
10158 return hwrm_req_send(bp, req);
10159 }
10160
bnxt_hwrm_vnic_rss_cfg_p5(struct bnxt * bp,struct bnxt_vnic_info * vnic)10161 int bnxt_hwrm_vnic_rss_cfg_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10162 {
10163 int rc;
10164
10165 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true);
10166 if (rc) {
10167 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
10168 vnic->vnic_id, rc);
10169 return rc;
10170 }
10171 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10172 if (rc)
10173 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
10174 vnic->vnic_id, rc);
10175 return rc;
10176 }
10177
__bnxt_setup_vnic_p5(struct bnxt * bp,struct bnxt_vnic_info * vnic)10178 int __bnxt_setup_vnic_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10179 {
10180 int rc, i, nr_ctxs;
10181
10182 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
10183 for (i = 0; i < nr_ctxs; i++) {
10184 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, i);
10185 if (rc) {
10186 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
10187 vnic->vnic_id, i, rc);
10188 break;
10189 }
10190 bp->rsscos_nr_ctxs++;
10191 }
10192 if (i < nr_ctxs)
10193 return -ENOMEM;
10194
10195 rc = bnxt_hwrm_vnic_rss_cfg_p5(bp, vnic);
10196 if (rc)
10197 return rc;
10198
10199 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
10200 rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
10201 if (rc) {
10202 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
10203 vnic->vnic_id, rc);
10204 }
10205 }
10206 return rc;
10207 }
10208
bnxt_setup_vnic(struct bnxt * bp,struct bnxt_vnic_info * vnic)10209 static int bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10210 {
10211 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10212 return __bnxt_setup_vnic_p5(bp, vnic);
10213 else
10214 return __bnxt_setup_vnic(bp, vnic);
10215 }
10216
bnxt_alloc_and_setup_vnic(struct bnxt * bp,struct bnxt_vnic_info * vnic,u16 start_rx_ring_idx,int rx_rings)10217 static int bnxt_alloc_and_setup_vnic(struct bnxt *bp,
10218 struct bnxt_vnic_info *vnic,
10219 u16 start_rx_ring_idx, int rx_rings)
10220 {
10221 int rc;
10222
10223 rc = bnxt_hwrm_vnic_alloc(bp, vnic, start_rx_ring_idx, rx_rings);
10224 if (rc) {
10225 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
10226 vnic->vnic_id, rc);
10227 return rc;
10228 }
10229 return bnxt_setup_vnic(bp, vnic);
10230 }
10231
bnxt_alloc_rfs_vnics(struct bnxt * bp)10232 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
10233 {
10234 struct bnxt_vnic_info *vnic;
10235 int i, rc = 0;
10236
10237 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
10238 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
10239 return bnxt_alloc_and_setup_vnic(bp, vnic, 0, bp->rx_nr_rings);
10240 }
10241
10242 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10243 return 0;
10244
10245 for (i = 0; i < bp->rx_nr_rings; i++) {
10246 u16 vnic_id = i + 1;
10247 u16 ring_id = i;
10248
10249 if (vnic_id >= bp->nr_vnics)
10250 break;
10251
10252 vnic = &bp->vnic_info[vnic_id];
10253 vnic->flags |= BNXT_VNIC_RFS_FLAG;
10254 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
10255 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
10256 if (bnxt_alloc_and_setup_vnic(bp, &bp->vnic_info[vnic_id], ring_id, 1))
10257 break;
10258 }
10259 return rc;
10260 }
10261
bnxt_del_one_rss_ctx(struct bnxt * bp,struct bnxt_rss_ctx * rss_ctx,bool all)10262 void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx,
10263 bool all)
10264 {
10265 struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10266 struct bnxt_filter_base *usr_fltr, *tmp;
10267 struct bnxt_ntuple_filter *ntp_fltr;
10268 int i;
10269
10270 if (netif_running(bp->dev)) {
10271 bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic);
10272 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) {
10273 if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID)
10274 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i);
10275 }
10276 }
10277 if (!all)
10278 return;
10279
10280 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
10281 if ((usr_fltr->flags & BNXT_ACT_RSS_CTX) &&
10282 usr_fltr->fw_vnic_id == rss_ctx->index) {
10283 ntp_fltr = container_of(usr_fltr,
10284 struct bnxt_ntuple_filter,
10285 base);
10286 bnxt_hwrm_cfa_ntuple_filter_free(bp, ntp_fltr);
10287 bnxt_del_ntp_filter(bp, ntp_fltr);
10288 bnxt_del_one_usr_fltr(bp, usr_fltr);
10289 }
10290 }
10291
10292 if (vnic->rss_table)
10293 dma_free_coherent(&bp->pdev->dev, vnic->rss_table_size,
10294 vnic->rss_table,
10295 vnic->rss_table_dma_addr);
10296 bp->num_rss_ctx--;
10297 }
10298
bnxt_hwrm_realloc_rss_ctx_vnic(struct bnxt * bp)10299 static void bnxt_hwrm_realloc_rss_ctx_vnic(struct bnxt *bp)
10300 {
10301 bool set_tpa = !!(bp->flags & BNXT_FLAG_TPA);
10302 struct ethtool_rxfh_context *ctx;
10303 unsigned long context;
10304
10305 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) {
10306 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx);
10307 struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10308
10309 if (bnxt_hwrm_vnic_alloc(bp, vnic, 0, bp->rx_nr_rings) ||
10310 bnxt_hwrm_vnic_set_tpa(bp, vnic, set_tpa) ||
10311 __bnxt_setup_vnic_p5(bp, vnic)) {
10312 netdev_err(bp->dev, "Failed to restore RSS ctx %d\n",
10313 rss_ctx->index);
10314 bnxt_del_one_rss_ctx(bp, rss_ctx, true);
10315 ethtool_rxfh_context_lost(bp->dev, rss_ctx->index);
10316 }
10317 }
10318 }
10319
bnxt_clear_rss_ctxs(struct bnxt * bp)10320 static void bnxt_clear_rss_ctxs(struct bnxt *bp)
10321 {
10322 struct ethtool_rxfh_context *ctx;
10323 unsigned long context;
10324
10325 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) {
10326 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx);
10327
10328 bnxt_del_one_rss_ctx(bp, rss_ctx, false);
10329 }
10330 }
10331
10332 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
bnxt_promisc_ok(struct bnxt * bp)10333 static bool bnxt_promisc_ok(struct bnxt *bp)
10334 {
10335 #ifdef CONFIG_BNXT_SRIOV
10336 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
10337 return false;
10338 #endif
10339 return true;
10340 }
10341
bnxt_setup_nitroa0_vnic(struct bnxt * bp)10342 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
10343 {
10344 struct bnxt_vnic_info *vnic = &bp->vnic_info[1];
10345 unsigned int rc = 0;
10346
10347 rc = bnxt_hwrm_vnic_alloc(bp, vnic, bp->rx_nr_rings - 1, 1);
10348 if (rc) {
10349 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10350 rc);
10351 return rc;
10352 }
10353
10354 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10355 if (rc) {
10356 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10357 rc);
10358 return rc;
10359 }
10360 return rc;
10361 }
10362
10363 static int bnxt_cfg_rx_mode(struct bnxt *);
10364 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
10365
bnxt_init_chip(struct bnxt * bp,bool irq_re_init)10366 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
10367 {
10368 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
10369 int rc = 0;
10370 unsigned int rx_nr_rings = bp->rx_nr_rings;
10371
10372 if (irq_re_init) {
10373 rc = bnxt_hwrm_stat_ctx_alloc(bp);
10374 if (rc) {
10375 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
10376 rc);
10377 goto err_out;
10378 }
10379 }
10380
10381 rc = bnxt_hwrm_ring_alloc(bp);
10382 if (rc) {
10383 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
10384 goto err_out;
10385 }
10386
10387 rc = bnxt_hwrm_ring_grp_alloc(bp);
10388 if (rc) {
10389 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
10390 goto err_out;
10391 }
10392
10393 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10394 rx_nr_rings--;
10395
10396 /* default vnic 0 */
10397 rc = bnxt_hwrm_vnic_alloc(bp, vnic, 0, rx_nr_rings);
10398 if (rc) {
10399 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
10400 goto err_out;
10401 }
10402
10403 if (BNXT_VF(bp))
10404 bnxt_hwrm_func_qcfg(bp);
10405
10406 rc = bnxt_setup_vnic(bp, vnic);
10407 if (rc)
10408 goto err_out;
10409 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
10410 bnxt_hwrm_update_rss_hash_cfg(bp);
10411
10412 if (bp->flags & BNXT_FLAG_RFS) {
10413 rc = bnxt_alloc_rfs_vnics(bp);
10414 if (rc)
10415 goto err_out;
10416 }
10417
10418 if (bp->flags & BNXT_FLAG_TPA) {
10419 rc = bnxt_set_tpa(bp, true);
10420 if (rc)
10421 goto err_out;
10422 }
10423
10424 if (BNXT_VF(bp))
10425 bnxt_update_vf_mac(bp);
10426
10427 /* Filter for default vnic 0 */
10428 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
10429 if (rc) {
10430 if (BNXT_VF(bp) && rc == -ENODEV)
10431 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
10432 else
10433 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
10434 goto err_out;
10435 }
10436 vnic->uc_filter_count = 1;
10437
10438 vnic->rx_mask = 0;
10439 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
10440 goto skip_rx_mask;
10441
10442 if (bp->dev->flags & IFF_BROADCAST)
10443 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10444
10445 if (bp->dev->flags & IFF_PROMISC)
10446 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10447
10448 if (bp->dev->flags & IFF_ALLMULTI) {
10449 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10450 vnic->mc_list_count = 0;
10451 } else if (bp->dev->flags & IFF_MULTICAST) {
10452 u32 mask = 0;
10453
10454 bnxt_mc_list_updated(bp, &mask);
10455 vnic->rx_mask |= mask;
10456 }
10457
10458 rc = bnxt_cfg_rx_mode(bp);
10459 if (rc)
10460 goto err_out;
10461
10462 skip_rx_mask:
10463 rc = bnxt_hwrm_set_coal(bp);
10464 if (rc)
10465 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
10466 rc);
10467
10468 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10469 rc = bnxt_setup_nitroa0_vnic(bp);
10470 if (rc)
10471 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
10472 rc);
10473 }
10474
10475 if (BNXT_VF(bp)) {
10476 bnxt_hwrm_func_qcfg(bp);
10477 netdev_update_features(bp->dev);
10478 }
10479
10480 return 0;
10481
10482 err_out:
10483 bnxt_hwrm_resource_free(bp, 0, true);
10484
10485 return rc;
10486 }
10487
bnxt_shutdown_nic(struct bnxt * bp,bool irq_re_init)10488 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
10489 {
10490 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
10491 return 0;
10492 }
10493
bnxt_init_nic(struct bnxt * bp,bool irq_re_init)10494 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
10495 {
10496 bnxt_init_cp_rings(bp);
10497 bnxt_init_rx_rings(bp);
10498 bnxt_init_tx_rings(bp);
10499 bnxt_init_ring_grps(bp, irq_re_init);
10500 bnxt_init_vnics(bp);
10501
10502 return bnxt_init_chip(bp, irq_re_init);
10503 }
10504
bnxt_set_real_num_queues(struct bnxt * bp)10505 static int bnxt_set_real_num_queues(struct bnxt *bp)
10506 {
10507 int rc;
10508 struct net_device *dev = bp->dev;
10509
10510 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
10511 bp->tx_nr_rings_xdp);
10512 if (rc)
10513 return rc;
10514
10515 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
10516 if (rc)
10517 return rc;
10518
10519 #ifdef CONFIG_RFS_ACCEL
10520 if (bp->flags & BNXT_FLAG_RFS)
10521 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
10522 #endif
10523
10524 return rc;
10525 }
10526
__bnxt_trim_rings(struct bnxt * bp,int * rx,int * tx,int max,bool shared)10527 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10528 bool shared)
10529 {
10530 int _rx = *rx, _tx = *tx;
10531
10532 if (shared) {
10533 *rx = min_t(int, _rx, max);
10534 *tx = min_t(int, _tx, max);
10535 } else {
10536 if (max < 2)
10537 return -ENOMEM;
10538
10539 while (_rx + _tx > max) {
10540 if (_rx > _tx && _rx > 1)
10541 _rx--;
10542 else if (_tx > 1)
10543 _tx--;
10544 }
10545 *rx = _rx;
10546 *tx = _tx;
10547 }
10548 return 0;
10549 }
10550
__bnxt_num_tx_to_cp(struct bnxt * bp,int tx,int tx_sets,int tx_xdp)10551 static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
10552 {
10553 return (tx - tx_xdp) / tx_sets + tx_xdp;
10554 }
10555
bnxt_num_tx_to_cp(struct bnxt * bp,int tx)10556 int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
10557 {
10558 int tcs = bp->num_tc;
10559
10560 if (!tcs)
10561 tcs = 1;
10562 return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
10563 }
10564
bnxt_num_cp_to_tx(struct bnxt * bp,int tx_cp)10565 static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
10566 {
10567 int tcs = bp->num_tc;
10568
10569 return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
10570 bp->tx_nr_rings_xdp;
10571 }
10572
bnxt_trim_rings(struct bnxt * bp,int * rx,int * tx,int max,bool sh)10573 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10574 bool sh)
10575 {
10576 int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
10577
10578 if (tx_cp != *tx) {
10579 int tx_saved = tx_cp, rc;
10580
10581 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
10582 if (rc)
10583 return rc;
10584 if (tx_cp != tx_saved)
10585 *tx = bnxt_num_cp_to_tx(bp, tx_cp);
10586 return 0;
10587 }
10588 return __bnxt_trim_rings(bp, rx, tx, max, sh);
10589 }
10590
bnxt_setup_msix(struct bnxt * bp)10591 static void bnxt_setup_msix(struct bnxt *bp)
10592 {
10593 const int len = sizeof(bp->irq_tbl[0].name);
10594 struct net_device *dev = bp->dev;
10595 int tcs, i;
10596
10597 tcs = bp->num_tc;
10598 if (tcs) {
10599 int i, off, count;
10600
10601 for (i = 0; i < tcs; i++) {
10602 count = bp->tx_nr_rings_per_tc;
10603 off = BNXT_TC_TO_RING_BASE(bp, i);
10604 netdev_set_tc_queue(dev, i, count, off);
10605 }
10606 }
10607
10608 for (i = 0; i < bp->cp_nr_rings; i++) {
10609 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10610 char *attr;
10611
10612 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10613 attr = "TxRx";
10614 else if (i < bp->rx_nr_rings)
10615 attr = "rx";
10616 else
10617 attr = "tx";
10618
10619 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
10620 attr, i);
10621 bp->irq_tbl[map_idx].handler = bnxt_msix;
10622 }
10623 }
10624
10625 static int bnxt_init_int_mode(struct bnxt *bp);
10626
bnxt_change_msix(struct bnxt * bp,int total)10627 static int bnxt_change_msix(struct bnxt *bp, int total)
10628 {
10629 struct msi_map map;
10630 int i;
10631
10632 /* add MSIX to the end if needed */
10633 for (i = bp->total_irqs; i < total; i++) {
10634 map = pci_msix_alloc_irq_at(bp->pdev, i, NULL);
10635 if (map.index < 0)
10636 return bp->total_irqs;
10637 bp->irq_tbl[i].vector = map.virq;
10638 bp->total_irqs++;
10639 }
10640
10641 /* trim MSIX from the end if needed */
10642 for (i = bp->total_irqs; i > total; i--) {
10643 map.index = i - 1;
10644 map.virq = bp->irq_tbl[i - 1].vector;
10645 pci_msix_free_irq(bp->pdev, map);
10646 bp->total_irqs--;
10647 }
10648 return bp->total_irqs;
10649 }
10650
bnxt_setup_int_mode(struct bnxt * bp)10651 static int bnxt_setup_int_mode(struct bnxt *bp)
10652 {
10653 int rc;
10654
10655 if (!bp->irq_tbl) {
10656 rc = bnxt_init_int_mode(bp);
10657 if (rc || !bp->irq_tbl)
10658 return rc ?: -ENODEV;
10659 }
10660
10661 bnxt_setup_msix(bp);
10662
10663 rc = bnxt_set_real_num_queues(bp);
10664 return rc;
10665 }
10666
bnxt_get_max_func_rss_ctxs(struct bnxt * bp)10667 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
10668 {
10669 return bp->hw_resc.max_rsscos_ctxs;
10670 }
10671
bnxt_get_max_func_vnics(struct bnxt * bp)10672 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
10673 {
10674 return bp->hw_resc.max_vnics;
10675 }
10676
bnxt_get_max_func_stat_ctxs(struct bnxt * bp)10677 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
10678 {
10679 return bp->hw_resc.max_stat_ctxs;
10680 }
10681
bnxt_get_max_func_cp_rings(struct bnxt * bp)10682 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
10683 {
10684 return bp->hw_resc.max_cp_rings;
10685 }
10686
bnxt_get_max_func_cp_rings_for_en(struct bnxt * bp)10687 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
10688 {
10689 unsigned int cp = bp->hw_resc.max_cp_rings;
10690
10691 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10692 cp -= bnxt_get_ulp_msix_num(bp);
10693
10694 return cp;
10695 }
10696
bnxt_get_max_func_irqs(struct bnxt * bp)10697 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
10698 {
10699 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10700
10701 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10702 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
10703
10704 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
10705 }
10706
bnxt_set_max_func_irqs(struct bnxt * bp,unsigned int max_irqs)10707 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
10708 {
10709 bp->hw_resc.max_irqs = max_irqs;
10710 }
10711
bnxt_get_avail_cp_rings_for_en(struct bnxt * bp)10712 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
10713 {
10714 unsigned int cp;
10715
10716 cp = bnxt_get_max_func_cp_rings_for_en(bp);
10717 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10718 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
10719 else
10720 return cp - bp->cp_nr_rings;
10721 }
10722
bnxt_get_avail_stat_ctxs_for_en(struct bnxt * bp)10723 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
10724 {
10725 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
10726 }
10727
bnxt_get_avail_msix(struct bnxt * bp,int num)10728 static int bnxt_get_avail_msix(struct bnxt *bp, int num)
10729 {
10730 int max_irq = bnxt_get_max_func_irqs(bp);
10731 int total_req = bp->cp_nr_rings + num;
10732
10733 if (max_irq < total_req) {
10734 num = max_irq - bp->cp_nr_rings;
10735 if (num <= 0)
10736 return 0;
10737 }
10738 return num;
10739 }
10740
bnxt_get_num_msix(struct bnxt * bp)10741 static int bnxt_get_num_msix(struct bnxt *bp)
10742 {
10743 if (!BNXT_NEW_RM(bp))
10744 return bnxt_get_max_func_irqs(bp);
10745
10746 return bnxt_nq_rings_in_use(bp);
10747 }
10748
bnxt_init_int_mode(struct bnxt * bp)10749 static int bnxt_init_int_mode(struct bnxt *bp)
10750 {
10751 int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp, tbl_size;
10752
10753 total_vecs = bnxt_get_num_msix(bp);
10754 max = bnxt_get_max_func_irqs(bp);
10755 if (total_vecs > max)
10756 total_vecs = max;
10757
10758 if (!total_vecs)
10759 return 0;
10760
10761 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
10762 min = 2;
10763
10764 total_vecs = pci_alloc_irq_vectors(bp->pdev, min, total_vecs,
10765 PCI_IRQ_MSIX);
10766 ulp_msix = bnxt_get_ulp_msix_num(bp);
10767 if (total_vecs < 0 || total_vecs < ulp_msix) {
10768 rc = -ENODEV;
10769 goto msix_setup_exit;
10770 }
10771
10772 tbl_size = total_vecs;
10773 if (pci_msix_can_alloc_dyn(bp->pdev))
10774 tbl_size = max;
10775 bp->irq_tbl = kcalloc(tbl_size, sizeof(*bp->irq_tbl), GFP_KERNEL);
10776 if (bp->irq_tbl) {
10777 for (i = 0; i < total_vecs; i++)
10778 bp->irq_tbl[i].vector = pci_irq_vector(bp->pdev, i);
10779
10780 bp->total_irqs = total_vecs;
10781 /* Trim rings based upon num of vectors allocated */
10782 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
10783 total_vecs - ulp_msix, min == 1);
10784 if (rc)
10785 goto msix_setup_exit;
10786
10787 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
10788 bp->cp_nr_rings = (min == 1) ?
10789 max_t(int, tx_cp, bp->rx_nr_rings) :
10790 tx_cp + bp->rx_nr_rings;
10791
10792 } else {
10793 rc = -ENOMEM;
10794 goto msix_setup_exit;
10795 }
10796 return 0;
10797
10798 msix_setup_exit:
10799 netdev_err(bp->dev, "bnxt_init_int_mode err: %x\n", rc);
10800 kfree(bp->irq_tbl);
10801 bp->irq_tbl = NULL;
10802 pci_free_irq_vectors(bp->pdev);
10803 return rc;
10804 }
10805
bnxt_clear_int_mode(struct bnxt * bp)10806 static void bnxt_clear_int_mode(struct bnxt *bp)
10807 {
10808 pci_free_irq_vectors(bp->pdev);
10809
10810 kfree(bp->irq_tbl);
10811 bp->irq_tbl = NULL;
10812 }
10813
bnxt_reserve_rings(struct bnxt * bp,bool irq_re_init)10814 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
10815 {
10816 bool irq_cleared = false;
10817 bool irq_change = false;
10818 int tcs = bp->num_tc;
10819 int irqs_required;
10820 int rc;
10821
10822 if (!bnxt_need_reserve_rings(bp))
10823 return 0;
10824
10825 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) {
10826 int ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
10827
10828 if (ulp_msix > bp->ulp_num_msix_want)
10829 ulp_msix = bp->ulp_num_msix_want;
10830 irqs_required = ulp_msix + bp->cp_nr_rings;
10831 } else {
10832 irqs_required = bnxt_get_num_msix(bp);
10833 }
10834
10835 if (irq_re_init && BNXT_NEW_RM(bp) && irqs_required != bp->total_irqs) {
10836 irq_change = true;
10837 if (!pci_msix_can_alloc_dyn(bp->pdev)) {
10838 bnxt_ulp_irq_stop(bp);
10839 bnxt_clear_int_mode(bp);
10840 irq_cleared = true;
10841 }
10842 }
10843 rc = __bnxt_reserve_rings(bp);
10844 if (irq_cleared) {
10845 if (!rc)
10846 rc = bnxt_init_int_mode(bp);
10847 bnxt_ulp_irq_restart(bp, rc);
10848 } else if (irq_change && !rc) {
10849 if (bnxt_change_msix(bp, irqs_required) != irqs_required)
10850 rc = -ENOSPC;
10851 }
10852 if (rc) {
10853 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
10854 return rc;
10855 }
10856 if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
10857 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
10858 netdev_err(bp->dev, "tx ring reservation failure\n");
10859 netdev_reset_tc(bp->dev);
10860 bp->num_tc = 0;
10861 if (bp->tx_nr_rings_xdp)
10862 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
10863 else
10864 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10865 return -ENOMEM;
10866 }
10867 return 0;
10868 }
10869
bnxt_free_irq(struct bnxt * bp)10870 static void bnxt_free_irq(struct bnxt *bp)
10871 {
10872 struct bnxt_irq *irq;
10873 int i;
10874
10875 #ifdef CONFIG_RFS_ACCEL
10876 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
10877 bp->dev->rx_cpu_rmap = NULL;
10878 #endif
10879 if (!bp->irq_tbl || !bp->bnapi)
10880 return;
10881
10882 for (i = 0; i < bp->cp_nr_rings; i++) {
10883 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10884
10885 irq = &bp->irq_tbl[map_idx];
10886 if (irq->requested) {
10887 if (irq->have_cpumask) {
10888 irq_set_affinity_hint(irq->vector, NULL);
10889 free_cpumask_var(irq->cpu_mask);
10890 irq->have_cpumask = 0;
10891 }
10892 free_irq(irq->vector, bp->bnapi[i]);
10893 }
10894
10895 irq->requested = 0;
10896 }
10897 }
10898
bnxt_request_irq(struct bnxt * bp)10899 static int bnxt_request_irq(struct bnxt *bp)
10900 {
10901 int i, j, rc = 0;
10902 unsigned long flags = 0;
10903 #ifdef CONFIG_RFS_ACCEL
10904 struct cpu_rmap *rmap;
10905 #endif
10906
10907 rc = bnxt_setup_int_mode(bp);
10908 if (rc) {
10909 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
10910 rc);
10911 return rc;
10912 }
10913 #ifdef CONFIG_RFS_ACCEL
10914 rmap = bp->dev->rx_cpu_rmap;
10915 #endif
10916 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
10917 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10918 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
10919
10920 #ifdef CONFIG_RFS_ACCEL
10921 if (rmap && bp->bnapi[i]->rx_ring) {
10922 rc = irq_cpu_rmap_add(rmap, irq->vector);
10923 if (rc)
10924 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
10925 j);
10926 j++;
10927 }
10928 #endif
10929 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
10930 bp->bnapi[i]);
10931 if (rc)
10932 break;
10933
10934 netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector);
10935 irq->requested = 1;
10936
10937 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
10938 int numa_node = dev_to_node(&bp->pdev->dev);
10939
10940 irq->have_cpumask = 1;
10941 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
10942 irq->cpu_mask);
10943 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
10944 if (rc) {
10945 netdev_warn(bp->dev,
10946 "Set affinity failed, IRQ = %d\n",
10947 irq->vector);
10948 break;
10949 }
10950 }
10951 }
10952 return rc;
10953 }
10954
bnxt_del_napi(struct bnxt * bp)10955 static void bnxt_del_napi(struct bnxt *bp)
10956 {
10957 int i;
10958
10959 if (!bp->bnapi)
10960 return;
10961
10962 for (i = 0; i < bp->rx_nr_rings; i++)
10963 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
10964 for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++)
10965 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL);
10966
10967 for (i = 0; i < bp->cp_nr_rings; i++) {
10968 struct bnxt_napi *bnapi = bp->bnapi[i];
10969
10970 __netif_napi_del(&bnapi->napi);
10971 }
10972 /* We called __netif_napi_del(), we need
10973 * to respect an RCU grace period before freeing napi structures.
10974 */
10975 synchronize_net();
10976 }
10977
bnxt_init_napi(struct bnxt * bp)10978 static void bnxt_init_napi(struct bnxt *bp)
10979 {
10980 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
10981 unsigned int cp_nr_rings = bp->cp_nr_rings;
10982 struct bnxt_napi *bnapi;
10983 int i;
10984
10985 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10986 poll_fn = bnxt_poll_p5;
10987 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10988 cp_nr_rings--;
10989 for (i = 0; i < cp_nr_rings; i++) {
10990 bnapi = bp->bnapi[i];
10991 netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
10992 }
10993 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10994 bnapi = bp->bnapi[cp_nr_rings];
10995 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll_nitroa0);
10996 }
10997 }
10998
bnxt_disable_napi(struct bnxt * bp)10999 static void bnxt_disable_napi(struct bnxt *bp)
11000 {
11001 int i;
11002
11003 if (!bp->bnapi ||
11004 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
11005 return;
11006
11007 for (i = 0; i < bp->cp_nr_rings; i++) {
11008 struct bnxt_napi *bnapi = bp->bnapi[i];
11009 struct bnxt_cp_ring_info *cpr;
11010
11011 cpr = &bnapi->cp_ring;
11012 if (bnapi->tx_fault)
11013 cpr->sw_stats->tx.tx_resets++;
11014 if (bnapi->in_reset)
11015 cpr->sw_stats->rx.rx_resets++;
11016 napi_disable(&bnapi->napi);
11017 if (bnapi->rx_ring)
11018 cancel_work_sync(&cpr->dim.work);
11019 }
11020 }
11021
bnxt_enable_napi(struct bnxt * bp)11022 static void bnxt_enable_napi(struct bnxt *bp)
11023 {
11024 int i;
11025
11026 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
11027 for (i = 0; i < bp->cp_nr_rings; i++) {
11028 struct bnxt_napi *bnapi = bp->bnapi[i];
11029 struct bnxt_cp_ring_info *cpr;
11030
11031 bnapi->tx_fault = 0;
11032
11033 cpr = &bnapi->cp_ring;
11034 bnapi->in_reset = false;
11035
11036 if (bnapi->rx_ring) {
11037 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
11038 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
11039 }
11040 napi_enable(&bnapi->napi);
11041 }
11042 }
11043
bnxt_tx_disable(struct bnxt * bp)11044 void bnxt_tx_disable(struct bnxt *bp)
11045 {
11046 int i;
11047 struct bnxt_tx_ring_info *txr;
11048
11049 if (bp->tx_ring) {
11050 for (i = 0; i < bp->tx_nr_rings; i++) {
11051 txr = &bp->tx_ring[i];
11052 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
11053 }
11054 }
11055 /* Make sure napi polls see @dev_state change */
11056 synchronize_net();
11057 /* Drop carrier first to prevent TX timeout */
11058 netif_carrier_off(bp->dev);
11059 /* Stop all TX queues */
11060 netif_tx_disable(bp->dev);
11061 }
11062
bnxt_tx_enable(struct bnxt * bp)11063 void bnxt_tx_enable(struct bnxt *bp)
11064 {
11065 int i;
11066 struct bnxt_tx_ring_info *txr;
11067
11068 for (i = 0; i < bp->tx_nr_rings; i++) {
11069 txr = &bp->tx_ring[i];
11070 WRITE_ONCE(txr->dev_state, 0);
11071 }
11072 /* Make sure napi polls see @dev_state change */
11073 synchronize_net();
11074 netif_tx_wake_all_queues(bp->dev);
11075 if (BNXT_LINK_IS_UP(bp))
11076 netif_carrier_on(bp->dev);
11077 }
11078
bnxt_report_fec(struct bnxt_link_info * link_info)11079 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
11080 {
11081 u8 active_fec = link_info->active_fec_sig_mode &
11082 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
11083
11084 switch (active_fec) {
11085 default:
11086 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
11087 return "None";
11088 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
11089 return "Clause 74 BaseR";
11090 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
11091 return "Clause 91 RS(528,514)";
11092 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
11093 return "Clause 91 RS544_1XN";
11094 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
11095 return "Clause 91 RS(544,514)";
11096 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
11097 return "Clause 91 RS272_1XN";
11098 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
11099 return "Clause 91 RS(272,257)";
11100 }
11101 }
11102
bnxt_report_link(struct bnxt * bp)11103 void bnxt_report_link(struct bnxt *bp)
11104 {
11105 if (BNXT_LINK_IS_UP(bp)) {
11106 const char *signal = "";
11107 const char *flow_ctrl;
11108 const char *duplex;
11109 u32 speed;
11110 u16 fec;
11111
11112 netif_carrier_on(bp->dev);
11113 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
11114 if (speed == SPEED_UNKNOWN) {
11115 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
11116 return;
11117 }
11118 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
11119 duplex = "full";
11120 else
11121 duplex = "half";
11122 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
11123 flow_ctrl = "ON - receive & transmit";
11124 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
11125 flow_ctrl = "ON - transmit";
11126 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
11127 flow_ctrl = "ON - receive";
11128 else
11129 flow_ctrl = "none";
11130 if (bp->link_info.phy_qcfg_resp.option_flags &
11131 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
11132 u8 sig_mode = bp->link_info.active_fec_sig_mode &
11133 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
11134 switch (sig_mode) {
11135 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
11136 signal = "(NRZ) ";
11137 break;
11138 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
11139 signal = "(PAM4 56Gbps) ";
11140 break;
11141 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112:
11142 signal = "(PAM4 112Gbps) ";
11143 break;
11144 default:
11145 break;
11146 }
11147 }
11148 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
11149 speed, signal, duplex, flow_ctrl);
11150 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
11151 netdev_info(bp->dev, "EEE is %s\n",
11152 bp->eee.eee_active ? "active" :
11153 "not active");
11154 fec = bp->link_info.fec_cfg;
11155 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
11156 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
11157 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
11158 bnxt_report_fec(&bp->link_info));
11159 } else {
11160 netif_carrier_off(bp->dev);
11161 netdev_err(bp->dev, "NIC Link is Down\n");
11162 }
11163 }
11164
bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output * resp)11165 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
11166 {
11167 if (!resp->supported_speeds_auto_mode &&
11168 !resp->supported_speeds_force_mode &&
11169 !resp->supported_pam4_speeds_auto_mode &&
11170 !resp->supported_pam4_speeds_force_mode &&
11171 !resp->supported_speeds2_auto_mode &&
11172 !resp->supported_speeds2_force_mode)
11173 return true;
11174 return false;
11175 }
11176
bnxt_hwrm_phy_qcaps(struct bnxt * bp)11177 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
11178 {
11179 struct bnxt_link_info *link_info = &bp->link_info;
11180 struct hwrm_port_phy_qcaps_output *resp;
11181 struct hwrm_port_phy_qcaps_input *req;
11182 int rc = 0;
11183
11184 if (bp->hwrm_spec_code < 0x10201)
11185 return 0;
11186
11187 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
11188 if (rc)
11189 return rc;
11190
11191 resp = hwrm_req_hold(bp, req);
11192 rc = hwrm_req_send(bp, req);
11193 if (rc)
11194 goto hwrm_phy_qcaps_exit;
11195
11196 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
11197 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
11198 struct ethtool_keee *eee = &bp->eee;
11199 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
11200
11201 _bnxt_fw_to_linkmode(eee->supported, fw_speeds);
11202 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
11203 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
11204 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
11205 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
11206 }
11207
11208 if (bp->hwrm_spec_code >= 0x10a01) {
11209 if (bnxt_phy_qcaps_no_speed(resp)) {
11210 link_info->phy_state = BNXT_PHY_STATE_DISABLED;
11211 netdev_warn(bp->dev, "Ethernet link disabled\n");
11212 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
11213 link_info->phy_state = BNXT_PHY_STATE_ENABLED;
11214 netdev_info(bp->dev, "Ethernet link enabled\n");
11215 /* Phy re-enabled, reprobe the speeds */
11216 link_info->support_auto_speeds = 0;
11217 link_info->support_pam4_auto_speeds = 0;
11218 link_info->support_auto_speeds2 = 0;
11219 }
11220 }
11221 if (resp->supported_speeds_auto_mode)
11222 link_info->support_auto_speeds =
11223 le16_to_cpu(resp->supported_speeds_auto_mode);
11224 if (resp->supported_pam4_speeds_auto_mode)
11225 link_info->support_pam4_auto_speeds =
11226 le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
11227 if (resp->supported_speeds2_auto_mode)
11228 link_info->support_auto_speeds2 =
11229 le16_to_cpu(resp->supported_speeds2_auto_mode);
11230
11231 bp->port_count = resp->port_cnt;
11232
11233 hwrm_phy_qcaps_exit:
11234 hwrm_req_drop(bp, req);
11235 return rc;
11236 }
11237
bnxt_support_dropped(u16 advertising,u16 supported)11238 static bool bnxt_support_dropped(u16 advertising, u16 supported)
11239 {
11240 u16 diff = advertising ^ supported;
11241
11242 return ((supported | diff) != supported);
11243 }
11244
bnxt_support_speed_dropped(struct bnxt_link_info * link_info)11245 static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info)
11246 {
11247 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
11248
11249 /* Check if any advertised speeds are no longer supported. The caller
11250 * holds the link_lock mutex, so we can modify link_info settings.
11251 */
11252 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11253 if (bnxt_support_dropped(link_info->advertising,
11254 link_info->support_auto_speeds2)) {
11255 link_info->advertising = link_info->support_auto_speeds2;
11256 return true;
11257 }
11258 return false;
11259 }
11260 if (bnxt_support_dropped(link_info->advertising,
11261 link_info->support_auto_speeds)) {
11262 link_info->advertising = link_info->support_auto_speeds;
11263 return true;
11264 }
11265 if (bnxt_support_dropped(link_info->advertising_pam4,
11266 link_info->support_pam4_auto_speeds)) {
11267 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
11268 return true;
11269 }
11270 return false;
11271 }
11272
bnxt_update_link(struct bnxt * bp,bool chng_link_state)11273 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
11274 {
11275 struct bnxt_link_info *link_info = &bp->link_info;
11276 struct hwrm_port_phy_qcfg_output *resp;
11277 struct hwrm_port_phy_qcfg_input *req;
11278 u8 link_state = link_info->link_state;
11279 bool support_changed;
11280 int rc;
11281
11282 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
11283 if (rc)
11284 return rc;
11285
11286 resp = hwrm_req_hold(bp, req);
11287 rc = hwrm_req_send(bp, req);
11288 if (rc) {
11289 hwrm_req_drop(bp, req);
11290 if (BNXT_VF(bp) && rc == -ENODEV) {
11291 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
11292 rc = 0;
11293 }
11294 return rc;
11295 }
11296
11297 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
11298 link_info->phy_link_status = resp->link;
11299 link_info->duplex = resp->duplex_cfg;
11300 if (bp->hwrm_spec_code >= 0x10800)
11301 link_info->duplex = resp->duplex_state;
11302 link_info->pause = resp->pause;
11303 link_info->auto_mode = resp->auto_mode;
11304 link_info->auto_pause_setting = resp->auto_pause;
11305 link_info->lp_pause = resp->link_partner_adv_pause;
11306 link_info->force_pause_setting = resp->force_pause;
11307 link_info->duplex_setting = resp->duplex_cfg;
11308 if (link_info->phy_link_status == BNXT_LINK_LINK) {
11309 link_info->link_speed = le16_to_cpu(resp->link_speed);
11310 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
11311 link_info->active_lanes = resp->active_lanes;
11312 } else {
11313 link_info->link_speed = 0;
11314 link_info->active_lanes = 0;
11315 }
11316 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
11317 link_info->force_pam4_link_speed =
11318 le16_to_cpu(resp->force_pam4_link_speed);
11319 link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2);
11320 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
11321 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
11322 link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2);
11323 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
11324 link_info->auto_pam4_link_speeds =
11325 le16_to_cpu(resp->auto_pam4_link_speed_mask);
11326 link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2);
11327 link_info->lp_auto_link_speeds =
11328 le16_to_cpu(resp->link_partner_adv_speeds);
11329 link_info->lp_auto_pam4_link_speeds =
11330 resp->link_partner_pam4_adv_speeds;
11331 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
11332 link_info->phy_ver[0] = resp->phy_maj;
11333 link_info->phy_ver[1] = resp->phy_min;
11334 link_info->phy_ver[2] = resp->phy_bld;
11335 link_info->media_type = resp->media_type;
11336 link_info->phy_type = resp->phy_type;
11337 link_info->transceiver = resp->xcvr_pkg_type;
11338 link_info->phy_addr = resp->eee_config_phy_addr &
11339 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
11340 link_info->module_status = resp->module_status;
11341
11342 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
11343 struct ethtool_keee *eee = &bp->eee;
11344 u16 fw_speeds;
11345
11346 eee->eee_active = 0;
11347 if (resp->eee_config_phy_addr &
11348 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
11349 eee->eee_active = 1;
11350 fw_speeds = le16_to_cpu(
11351 resp->link_partner_adv_eee_link_speed_mask);
11352 _bnxt_fw_to_linkmode(eee->lp_advertised, fw_speeds);
11353 }
11354
11355 /* Pull initial EEE config */
11356 if (!chng_link_state) {
11357 if (resp->eee_config_phy_addr &
11358 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
11359 eee->eee_enabled = 1;
11360
11361 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
11362 _bnxt_fw_to_linkmode(eee->advertised, fw_speeds);
11363
11364 if (resp->eee_config_phy_addr &
11365 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
11366 __le32 tmr;
11367
11368 eee->tx_lpi_enabled = 1;
11369 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
11370 eee->tx_lpi_timer = le32_to_cpu(tmr) &
11371 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
11372 }
11373 }
11374 }
11375
11376 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
11377 if (bp->hwrm_spec_code >= 0x10504) {
11378 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
11379 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
11380 }
11381 /* TODO: need to add more logic to report VF link */
11382 if (chng_link_state) {
11383 if (link_info->phy_link_status == BNXT_LINK_LINK)
11384 link_info->link_state = BNXT_LINK_STATE_UP;
11385 else
11386 link_info->link_state = BNXT_LINK_STATE_DOWN;
11387 if (link_state != link_info->link_state)
11388 bnxt_report_link(bp);
11389 } else {
11390 /* always link down if not require to update link state */
11391 link_info->link_state = BNXT_LINK_STATE_DOWN;
11392 }
11393 hwrm_req_drop(bp, req);
11394
11395 if (!BNXT_PHY_CFG_ABLE(bp))
11396 return 0;
11397
11398 support_changed = bnxt_support_speed_dropped(link_info);
11399 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
11400 bnxt_hwrm_set_link_setting(bp, true, false);
11401 return 0;
11402 }
11403
bnxt_get_port_module_status(struct bnxt * bp)11404 static void bnxt_get_port_module_status(struct bnxt *bp)
11405 {
11406 struct bnxt_link_info *link_info = &bp->link_info;
11407 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
11408 u8 module_status;
11409
11410 if (bnxt_update_link(bp, true))
11411 return;
11412
11413 module_status = link_info->module_status;
11414 switch (module_status) {
11415 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
11416 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
11417 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
11418 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
11419 bp->pf.port_id);
11420 if (bp->hwrm_spec_code >= 0x10201) {
11421 netdev_warn(bp->dev, "Module part number %s\n",
11422 resp->phy_vendor_partnumber);
11423 }
11424 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
11425 netdev_warn(bp->dev, "TX is disabled\n");
11426 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
11427 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
11428 }
11429 }
11430
11431 static void
bnxt_hwrm_set_pause_common(struct bnxt * bp,struct hwrm_port_phy_cfg_input * req)11432 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11433 {
11434 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
11435 if (bp->hwrm_spec_code >= 0x10201)
11436 req->auto_pause =
11437 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
11438 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11439 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
11440 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11441 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
11442 req->enables |=
11443 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11444 } else {
11445 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11446 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
11447 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11448 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
11449 req->enables |=
11450 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
11451 if (bp->hwrm_spec_code >= 0x10201) {
11452 req->auto_pause = req->force_pause;
11453 req->enables |= cpu_to_le32(
11454 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11455 }
11456 }
11457 }
11458
bnxt_hwrm_set_link_common(struct bnxt * bp,struct hwrm_port_phy_cfg_input * req)11459 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11460 {
11461 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
11462 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
11463 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11464 req->enables |=
11465 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK);
11466 req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising);
11467 } else if (bp->link_info.advertising) {
11468 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
11469 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
11470 }
11471 if (bp->link_info.advertising_pam4) {
11472 req->enables |=
11473 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
11474 req->auto_link_pam4_speed_mask =
11475 cpu_to_le16(bp->link_info.advertising_pam4);
11476 }
11477 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
11478 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
11479 } else {
11480 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
11481 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11482 req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed);
11483 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2);
11484 netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n",
11485 (u32)bp->link_info.req_link_speed);
11486 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
11487 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11488 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
11489 } else {
11490 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11491 }
11492 }
11493
11494 /* tell chimp that the setting takes effect immediately */
11495 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
11496 }
11497
bnxt_hwrm_set_pause(struct bnxt * bp)11498 int bnxt_hwrm_set_pause(struct bnxt *bp)
11499 {
11500 struct hwrm_port_phy_cfg_input *req;
11501 int rc;
11502
11503 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11504 if (rc)
11505 return rc;
11506
11507 bnxt_hwrm_set_pause_common(bp, req);
11508
11509 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
11510 bp->link_info.force_link_chng)
11511 bnxt_hwrm_set_link_common(bp, req);
11512
11513 rc = hwrm_req_send(bp, req);
11514 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
11515 /* since changing of pause setting doesn't trigger any link
11516 * change event, the driver needs to update the current pause
11517 * result upon successfully return of the phy_cfg command
11518 */
11519 bp->link_info.pause =
11520 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
11521 bp->link_info.auto_pause_setting = 0;
11522 if (!bp->link_info.force_link_chng)
11523 bnxt_report_link(bp);
11524 }
11525 bp->link_info.force_link_chng = false;
11526 return rc;
11527 }
11528
bnxt_hwrm_set_eee(struct bnxt * bp,struct hwrm_port_phy_cfg_input * req)11529 static void bnxt_hwrm_set_eee(struct bnxt *bp,
11530 struct hwrm_port_phy_cfg_input *req)
11531 {
11532 struct ethtool_keee *eee = &bp->eee;
11533
11534 if (eee->eee_enabled) {
11535 u16 eee_speeds;
11536 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
11537
11538 if (eee->tx_lpi_enabled)
11539 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
11540 else
11541 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
11542
11543 req->flags |= cpu_to_le32(flags);
11544 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
11545 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
11546 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
11547 } else {
11548 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
11549 }
11550 }
11551
bnxt_hwrm_set_link_setting(struct bnxt * bp,bool set_pause,bool set_eee)11552 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
11553 {
11554 struct hwrm_port_phy_cfg_input *req;
11555 int rc;
11556
11557 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11558 if (rc)
11559 return rc;
11560
11561 if (set_pause)
11562 bnxt_hwrm_set_pause_common(bp, req);
11563
11564 bnxt_hwrm_set_link_common(bp, req);
11565
11566 if (set_eee)
11567 bnxt_hwrm_set_eee(bp, req);
11568 return hwrm_req_send(bp, req);
11569 }
11570
bnxt_hwrm_shutdown_link(struct bnxt * bp)11571 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
11572 {
11573 struct hwrm_port_phy_cfg_input *req;
11574 int rc;
11575
11576 if (!BNXT_SINGLE_PF(bp))
11577 return 0;
11578
11579 if (pci_num_vf(bp->pdev) &&
11580 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
11581 return 0;
11582
11583 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11584 if (rc)
11585 return rc;
11586
11587 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
11588 rc = hwrm_req_send(bp, req);
11589 if (!rc) {
11590 mutex_lock(&bp->link_lock);
11591 /* Device is not obliged link down in certain scenarios, even
11592 * when forced. Setting the state unknown is consistent with
11593 * driver startup and will force link state to be reported
11594 * during subsequent open based on PORT_PHY_QCFG.
11595 */
11596 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
11597 mutex_unlock(&bp->link_lock);
11598 }
11599 return rc;
11600 }
11601
bnxt_fw_reset_via_optee(struct bnxt * bp)11602 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11603 {
11604 #ifdef CONFIG_TEE_BNXT_FW
11605 int rc = tee_bnxt_fw_load();
11606
11607 if (rc)
11608 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11609
11610 return rc;
11611 #else
11612 netdev_err(bp->dev, "OP-TEE not supported\n");
11613 return -ENODEV;
11614 #endif
11615 }
11616
bnxt_try_recover_fw(struct bnxt * bp)11617 static int bnxt_try_recover_fw(struct bnxt *bp)
11618 {
11619 if (bp->fw_health && bp->fw_health->status_reliable) {
11620 int retry = 0, rc;
11621 u32 sts;
11622
11623 do {
11624 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11625 rc = bnxt_hwrm_poll(bp);
11626 if (!BNXT_FW_IS_BOOTING(sts) &&
11627 !BNXT_FW_IS_RECOVERING(sts))
11628 break;
11629 retry++;
11630 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
11631
11632 if (!BNXT_FW_IS_HEALTHY(sts)) {
11633 netdev_err(bp->dev,
11634 "Firmware not responding, status: 0x%x\n",
11635 sts);
11636 rc = -ENODEV;
11637 }
11638 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11639 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11640 return bnxt_fw_reset_via_optee(bp);
11641 }
11642 return rc;
11643 }
11644
11645 return -ENODEV;
11646 }
11647
bnxt_clear_reservations(struct bnxt * bp,bool fw_reset)11648 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
11649 {
11650 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11651
11652 if (!BNXT_NEW_RM(bp))
11653 return; /* no resource reservations required */
11654
11655 hw_resc->resv_cp_rings = 0;
11656 hw_resc->resv_stat_ctxs = 0;
11657 hw_resc->resv_irqs = 0;
11658 hw_resc->resv_tx_rings = 0;
11659 hw_resc->resv_rx_rings = 0;
11660 hw_resc->resv_hw_ring_grps = 0;
11661 hw_resc->resv_vnics = 0;
11662 hw_resc->resv_rsscos_ctxs = 0;
11663 if (!fw_reset) {
11664 bp->tx_nr_rings = 0;
11665 bp->rx_nr_rings = 0;
11666 }
11667 }
11668
bnxt_cancel_reservations(struct bnxt * bp,bool fw_reset)11669 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
11670 {
11671 int rc;
11672
11673 if (!BNXT_NEW_RM(bp))
11674 return 0; /* no resource reservations required */
11675
11676 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
11677 if (rc)
11678 netdev_err(bp->dev, "resc_qcaps failed\n");
11679
11680 bnxt_clear_reservations(bp, fw_reset);
11681
11682 return rc;
11683 }
11684
bnxt_hwrm_if_change(struct bnxt * bp,bool up)11685 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
11686 {
11687 struct hwrm_func_drv_if_change_output *resp;
11688 struct hwrm_func_drv_if_change_input *req;
11689 bool fw_reset = !bp->irq_tbl;
11690 bool resc_reinit = false;
11691 int rc, retry = 0;
11692 u32 flags = 0;
11693
11694 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
11695 return 0;
11696
11697 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
11698 if (rc)
11699 return rc;
11700
11701 if (up)
11702 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
11703 resp = hwrm_req_hold(bp, req);
11704
11705 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
11706 while (retry < BNXT_FW_IF_RETRY) {
11707 rc = hwrm_req_send(bp, req);
11708 if (rc != -EAGAIN)
11709 break;
11710
11711 msleep(50);
11712 retry++;
11713 }
11714
11715 if (rc == -EAGAIN) {
11716 hwrm_req_drop(bp, req);
11717 return rc;
11718 } else if (!rc) {
11719 flags = le32_to_cpu(resp->flags);
11720 } else if (up) {
11721 rc = bnxt_try_recover_fw(bp);
11722 fw_reset = true;
11723 }
11724 hwrm_req_drop(bp, req);
11725 if (rc)
11726 return rc;
11727
11728 if (!up) {
11729 bnxt_inv_fw_health_reg(bp);
11730 return 0;
11731 }
11732
11733 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
11734 resc_reinit = true;
11735 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
11736 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
11737 fw_reset = true;
11738 else
11739 bnxt_remap_fw_health_regs(bp);
11740
11741 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
11742 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
11743 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11744 return -ENODEV;
11745 }
11746 if (resc_reinit || fw_reset) {
11747 if (fw_reset) {
11748 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11749 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11750 bnxt_ulp_irq_stop(bp);
11751 bnxt_free_ctx_mem(bp);
11752 bnxt_dcb_free(bp);
11753 rc = bnxt_fw_init_one(bp);
11754 if (rc) {
11755 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11756 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11757 return rc;
11758 }
11759 bnxt_clear_int_mode(bp);
11760 rc = bnxt_init_int_mode(bp);
11761 if (rc) {
11762 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11763 netdev_err(bp->dev, "init int mode failed\n");
11764 return rc;
11765 }
11766 }
11767 rc = bnxt_cancel_reservations(bp, fw_reset);
11768 }
11769 return rc;
11770 }
11771
bnxt_hwrm_port_led_qcaps(struct bnxt * bp)11772 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
11773 {
11774 struct hwrm_port_led_qcaps_output *resp;
11775 struct hwrm_port_led_qcaps_input *req;
11776 struct bnxt_pf_info *pf = &bp->pf;
11777 int rc;
11778
11779 bp->num_leds = 0;
11780 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
11781 return 0;
11782
11783 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
11784 if (rc)
11785 return rc;
11786
11787 req->port_id = cpu_to_le16(pf->port_id);
11788 resp = hwrm_req_hold(bp, req);
11789 rc = hwrm_req_send(bp, req);
11790 if (rc) {
11791 hwrm_req_drop(bp, req);
11792 return rc;
11793 }
11794 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
11795 int i;
11796
11797 bp->num_leds = resp->num_leds;
11798 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
11799 bp->num_leds);
11800 for (i = 0; i < bp->num_leds; i++) {
11801 struct bnxt_led_info *led = &bp->leds[i];
11802 __le16 caps = led->led_state_caps;
11803
11804 if (!led->led_group_id ||
11805 !BNXT_LED_ALT_BLINK_CAP(caps)) {
11806 bp->num_leds = 0;
11807 break;
11808 }
11809 }
11810 }
11811 hwrm_req_drop(bp, req);
11812 return 0;
11813 }
11814
bnxt_hwrm_alloc_wol_fltr(struct bnxt * bp)11815 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
11816 {
11817 struct hwrm_wol_filter_alloc_output *resp;
11818 struct hwrm_wol_filter_alloc_input *req;
11819 int rc;
11820
11821 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
11822 if (rc)
11823 return rc;
11824
11825 req->port_id = cpu_to_le16(bp->pf.port_id);
11826 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
11827 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
11828 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
11829
11830 resp = hwrm_req_hold(bp, req);
11831 rc = hwrm_req_send(bp, req);
11832 if (!rc)
11833 bp->wol_filter_id = resp->wol_filter_id;
11834 hwrm_req_drop(bp, req);
11835 return rc;
11836 }
11837
bnxt_hwrm_free_wol_fltr(struct bnxt * bp)11838 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
11839 {
11840 struct hwrm_wol_filter_free_input *req;
11841 int rc;
11842
11843 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
11844 if (rc)
11845 return rc;
11846
11847 req->port_id = cpu_to_le16(bp->pf.port_id);
11848 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
11849 req->wol_filter_id = bp->wol_filter_id;
11850
11851 return hwrm_req_send(bp, req);
11852 }
11853
bnxt_hwrm_get_wol_fltrs(struct bnxt * bp,u16 handle)11854 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
11855 {
11856 struct hwrm_wol_filter_qcfg_output *resp;
11857 struct hwrm_wol_filter_qcfg_input *req;
11858 u16 next_handle = 0;
11859 int rc;
11860
11861 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
11862 if (rc)
11863 return rc;
11864
11865 req->port_id = cpu_to_le16(bp->pf.port_id);
11866 req->handle = cpu_to_le16(handle);
11867 resp = hwrm_req_hold(bp, req);
11868 rc = hwrm_req_send(bp, req);
11869 if (!rc) {
11870 next_handle = le16_to_cpu(resp->next_handle);
11871 if (next_handle != 0) {
11872 if (resp->wol_type ==
11873 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
11874 bp->wol = 1;
11875 bp->wol_filter_id = resp->wol_filter_id;
11876 }
11877 }
11878 }
11879 hwrm_req_drop(bp, req);
11880 return next_handle;
11881 }
11882
bnxt_get_wol_settings(struct bnxt * bp)11883 static void bnxt_get_wol_settings(struct bnxt *bp)
11884 {
11885 u16 handle = 0;
11886
11887 bp->wol = 0;
11888 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
11889 return;
11890
11891 do {
11892 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
11893 } while (handle && handle != 0xffff);
11894 }
11895
bnxt_eee_config_ok(struct bnxt * bp)11896 static bool bnxt_eee_config_ok(struct bnxt *bp)
11897 {
11898 struct ethtool_keee *eee = &bp->eee;
11899 struct bnxt_link_info *link_info = &bp->link_info;
11900
11901 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
11902 return true;
11903
11904 if (eee->eee_enabled) {
11905 __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
11906 __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp);
11907
11908 _bnxt_fw_to_linkmode(advertising, link_info->advertising);
11909
11910 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11911 eee->eee_enabled = 0;
11912 return false;
11913 }
11914 if (linkmode_andnot(tmp, eee->advertised, advertising)) {
11915 linkmode_and(eee->advertised, advertising,
11916 eee->supported);
11917 return false;
11918 }
11919 }
11920 return true;
11921 }
11922
bnxt_update_phy_setting(struct bnxt * bp)11923 static int bnxt_update_phy_setting(struct bnxt *bp)
11924 {
11925 int rc;
11926 bool update_link = false;
11927 bool update_pause = false;
11928 bool update_eee = false;
11929 struct bnxt_link_info *link_info = &bp->link_info;
11930
11931 rc = bnxt_update_link(bp, true);
11932 if (rc) {
11933 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
11934 rc);
11935 return rc;
11936 }
11937 if (!BNXT_SINGLE_PF(bp))
11938 return 0;
11939
11940 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11941 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
11942 link_info->req_flow_ctrl)
11943 update_pause = true;
11944 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11945 link_info->force_pause_setting != link_info->req_flow_ctrl)
11946 update_pause = true;
11947 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11948 if (BNXT_AUTO_MODE(link_info->auto_mode))
11949 update_link = true;
11950 if (bnxt_force_speed_updated(link_info))
11951 update_link = true;
11952 if (link_info->req_duplex != link_info->duplex_setting)
11953 update_link = true;
11954 } else {
11955 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
11956 update_link = true;
11957 if (bnxt_auto_speed_updated(link_info))
11958 update_link = true;
11959 }
11960
11961 /* The last close may have shutdown the link, so need to call
11962 * PHY_CFG to bring it back up.
11963 */
11964 if (!BNXT_LINK_IS_UP(bp))
11965 update_link = true;
11966
11967 if (!bnxt_eee_config_ok(bp))
11968 update_eee = true;
11969
11970 if (update_link)
11971 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
11972 else if (update_pause)
11973 rc = bnxt_hwrm_set_pause(bp);
11974 if (rc) {
11975 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
11976 rc);
11977 return rc;
11978 }
11979
11980 return rc;
11981 }
11982
11983 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
11984
bnxt_reinit_after_abort(struct bnxt * bp)11985 static int bnxt_reinit_after_abort(struct bnxt *bp)
11986 {
11987 int rc;
11988
11989 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11990 return -EBUSY;
11991
11992 if (bp->dev->reg_state == NETREG_UNREGISTERED)
11993 return -ENODEV;
11994
11995 rc = bnxt_fw_init_one(bp);
11996 if (!rc) {
11997 bnxt_clear_int_mode(bp);
11998 rc = bnxt_init_int_mode(bp);
11999 if (!rc) {
12000 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12001 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
12002 }
12003 }
12004 return rc;
12005 }
12006
bnxt_cfg_one_usr_fltr(struct bnxt * bp,struct bnxt_filter_base * fltr)12007 static void bnxt_cfg_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
12008 {
12009 struct bnxt_ntuple_filter *ntp_fltr;
12010 struct bnxt_l2_filter *l2_fltr;
12011
12012 if (list_empty(&fltr->list))
12013 return;
12014
12015 if (fltr->type == BNXT_FLTR_TYPE_NTUPLE) {
12016 ntp_fltr = container_of(fltr, struct bnxt_ntuple_filter, base);
12017 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
12018 atomic_inc(&l2_fltr->refcnt);
12019 ntp_fltr->l2_fltr = l2_fltr;
12020 if (bnxt_hwrm_cfa_ntuple_filter_alloc(bp, ntp_fltr)) {
12021 bnxt_del_ntp_filter(bp, ntp_fltr);
12022 netdev_err(bp->dev, "restoring previously configured ntuple filter id %d failed\n",
12023 fltr->sw_id);
12024 }
12025 } else if (fltr->type == BNXT_FLTR_TYPE_L2) {
12026 l2_fltr = container_of(fltr, struct bnxt_l2_filter, base);
12027 if (bnxt_hwrm_l2_filter_alloc(bp, l2_fltr)) {
12028 bnxt_del_l2_filter(bp, l2_fltr);
12029 netdev_err(bp->dev, "restoring previously configured l2 filter id %d failed\n",
12030 fltr->sw_id);
12031 }
12032 }
12033 }
12034
bnxt_cfg_usr_fltrs(struct bnxt * bp)12035 static void bnxt_cfg_usr_fltrs(struct bnxt *bp)
12036 {
12037 struct bnxt_filter_base *usr_fltr, *tmp;
12038
12039 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list)
12040 bnxt_cfg_one_usr_fltr(bp, usr_fltr);
12041 }
12042
bnxt_set_xps_mapping(struct bnxt * bp)12043 static int bnxt_set_xps_mapping(struct bnxt *bp)
12044 {
12045 int numa_node = dev_to_node(&bp->pdev->dev);
12046 unsigned int q_idx, map_idx, cpu, i;
12047 const struct cpumask *cpu_mask_ptr;
12048 int nr_cpus = num_online_cpus();
12049 cpumask_t *q_map;
12050 int rc = 0;
12051
12052 q_map = kcalloc(bp->tx_nr_rings_per_tc, sizeof(*q_map), GFP_KERNEL);
12053 if (!q_map)
12054 return -ENOMEM;
12055
12056 /* Create CPU mask for all TX queues across MQPRIO traffic classes.
12057 * Each TC has the same number of TX queues. The nth TX queue for each
12058 * TC will have the same CPU mask.
12059 */
12060 for (i = 0; i < nr_cpus; i++) {
12061 map_idx = i % bp->tx_nr_rings_per_tc;
12062 cpu = cpumask_local_spread(i, numa_node);
12063 cpu_mask_ptr = get_cpu_mask(cpu);
12064 cpumask_or(&q_map[map_idx], &q_map[map_idx], cpu_mask_ptr);
12065 }
12066
12067 /* Register CPU mask for each TX queue except the ones marked for XDP */
12068 for (q_idx = 0; q_idx < bp->dev->real_num_tx_queues; q_idx++) {
12069 map_idx = q_idx % bp->tx_nr_rings_per_tc;
12070 rc = netif_set_xps_queue(bp->dev, &q_map[map_idx], q_idx);
12071 if (rc) {
12072 netdev_warn(bp->dev, "Error setting XPS for q:%d\n",
12073 q_idx);
12074 break;
12075 }
12076 }
12077
12078 kfree(q_map);
12079
12080 return rc;
12081 }
12082
__bnxt_open_nic(struct bnxt * bp,bool irq_re_init,bool link_re_init)12083 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12084 {
12085 int rc = 0;
12086
12087 netif_carrier_off(bp->dev);
12088 if (irq_re_init) {
12089 /* Reserve rings now if none were reserved at driver probe. */
12090 rc = bnxt_init_dflt_ring_mode(bp);
12091 if (rc) {
12092 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
12093 return rc;
12094 }
12095 }
12096 rc = bnxt_reserve_rings(bp, irq_re_init);
12097 if (rc)
12098 return rc;
12099
12100 rc = bnxt_alloc_mem(bp, irq_re_init);
12101 if (rc) {
12102 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
12103 goto open_err_free_mem;
12104 }
12105
12106 if (irq_re_init) {
12107 bnxt_init_napi(bp);
12108 rc = bnxt_request_irq(bp);
12109 if (rc) {
12110 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
12111 goto open_err_irq;
12112 }
12113 }
12114
12115 rc = bnxt_init_nic(bp, irq_re_init);
12116 if (rc) {
12117 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
12118 goto open_err_irq;
12119 }
12120
12121 bnxt_enable_napi(bp);
12122 bnxt_debug_dev_init(bp);
12123
12124 if (link_re_init) {
12125 mutex_lock(&bp->link_lock);
12126 rc = bnxt_update_phy_setting(bp);
12127 mutex_unlock(&bp->link_lock);
12128 if (rc) {
12129 netdev_warn(bp->dev, "failed to update phy settings\n");
12130 if (BNXT_SINGLE_PF(bp)) {
12131 bp->link_info.phy_retry = true;
12132 bp->link_info.phy_retry_expires =
12133 jiffies + 5 * HZ;
12134 }
12135 }
12136 }
12137
12138 if (irq_re_init) {
12139 udp_tunnel_nic_reset_ntf(bp->dev);
12140 rc = bnxt_set_xps_mapping(bp);
12141 if (rc)
12142 netdev_warn(bp->dev, "failed to set xps mapping\n");
12143 }
12144
12145 if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
12146 if (!static_key_enabled(&bnxt_xdp_locking_key))
12147 static_branch_enable(&bnxt_xdp_locking_key);
12148 } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
12149 static_branch_disable(&bnxt_xdp_locking_key);
12150 }
12151 set_bit(BNXT_STATE_OPEN, &bp->state);
12152 bnxt_enable_int(bp);
12153 /* Enable TX queues */
12154 bnxt_tx_enable(bp);
12155 mod_timer(&bp->timer, jiffies + bp->current_interval);
12156 /* Poll link status and check for SFP+ module status */
12157 mutex_lock(&bp->link_lock);
12158 bnxt_get_port_module_status(bp);
12159 mutex_unlock(&bp->link_lock);
12160
12161 /* VF-reps may need to be re-opened after the PF is re-opened */
12162 if (BNXT_PF(bp))
12163 bnxt_vf_reps_open(bp);
12164 if (bp->ptp_cfg && !(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP))
12165 WRITE_ONCE(bp->ptp_cfg->tx_avail, BNXT_MAX_TX_TS);
12166 bnxt_ptp_init_rtc(bp, true);
12167 bnxt_ptp_cfg_tstamp_filters(bp);
12168 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
12169 bnxt_hwrm_realloc_rss_ctx_vnic(bp);
12170 bnxt_cfg_usr_fltrs(bp);
12171 return 0;
12172
12173 open_err_irq:
12174 bnxt_del_napi(bp);
12175
12176 open_err_free_mem:
12177 bnxt_free_skbs(bp);
12178 bnxt_free_irq(bp);
12179 bnxt_free_mem(bp, true);
12180 return rc;
12181 }
12182
12183 /* rtnl_lock held */
bnxt_open_nic(struct bnxt * bp,bool irq_re_init,bool link_re_init)12184 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12185 {
12186 int rc = 0;
12187
12188 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
12189 rc = -EIO;
12190 if (!rc)
12191 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
12192 if (rc) {
12193 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
12194 dev_close(bp->dev);
12195 }
12196 return rc;
12197 }
12198
12199 /* rtnl_lock held, open the NIC half way by allocating all resources, but
12200 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
12201 * self tests.
12202 */
bnxt_half_open_nic(struct bnxt * bp)12203 int bnxt_half_open_nic(struct bnxt *bp)
12204 {
12205 int rc = 0;
12206
12207 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12208 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
12209 rc = -ENODEV;
12210 goto half_open_err;
12211 }
12212
12213 rc = bnxt_alloc_mem(bp, true);
12214 if (rc) {
12215 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
12216 goto half_open_err;
12217 }
12218 bnxt_init_napi(bp);
12219 set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12220 rc = bnxt_init_nic(bp, true);
12221 if (rc) {
12222 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12223 bnxt_del_napi(bp);
12224 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
12225 goto half_open_err;
12226 }
12227 return 0;
12228
12229 half_open_err:
12230 bnxt_free_skbs(bp);
12231 bnxt_free_mem(bp, true);
12232 dev_close(bp->dev);
12233 return rc;
12234 }
12235
12236 /* rtnl_lock held, this call can only be made after a previous successful
12237 * call to bnxt_half_open_nic().
12238 */
bnxt_half_close_nic(struct bnxt * bp)12239 void bnxt_half_close_nic(struct bnxt *bp)
12240 {
12241 bnxt_hwrm_resource_free(bp, false, true);
12242 bnxt_del_napi(bp);
12243 bnxt_free_skbs(bp);
12244 bnxt_free_mem(bp, true);
12245 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12246 }
12247
bnxt_reenable_sriov(struct bnxt * bp)12248 void bnxt_reenable_sriov(struct bnxt *bp)
12249 {
12250 if (BNXT_PF(bp)) {
12251 struct bnxt_pf_info *pf = &bp->pf;
12252 int n = pf->active_vfs;
12253
12254 if (n)
12255 bnxt_cfg_hw_sriov(bp, &n, true);
12256 }
12257 }
12258
bnxt_open(struct net_device * dev)12259 static int bnxt_open(struct net_device *dev)
12260 {
12261 struct bnxt *bp = netdev_priv(dev);
12262 int rc;
12263
12264 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12265 rc = bnxt_reinit_after_abort(bp);
12266 if (rc) {
12267 if (rc == -EBUSY)
12268 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
12269 else
12270 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
12271 return -ENODEV;
12272 }
12273 }
12274
12275 rc = bnxt_hwrm_if_change(bp, true);
12276 if (rc)
12277 return rc;
12278
12279 rc = __bnxt_open_nic(bp, true, true);
12280 if (rc) {
12281 bnxt_hwrm_if_change(bp, false);
12282 } else {
12283 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
12284 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12285 bnxt_queue_sp_work(bp,
12286 BNXT_RESTART_ULP_SP_EVENT);
12287 }
12288 }
12289
12290 return rc;
12291 }
12292
bnxt_drv_busy(struct bnxt * bp)12293 static bool bnxt_drv_busy(struct bnxt *bp)
12294 {
12295 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
12296 test_bit(BNXT_STATE_READ_STATS, &bp->state));
12297 }
12298
12299 static void bnxt_get_ring_stats(struct bnxt *bp,
12300 struct rtnl_link_stats64 *stats);
12301
__bnxt_close_nic(struct bnxt * bp,bool irq_re_init,bool link_re_init)12302 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
12303 bool link_re_init)
12304 {
12305 /* Close the VF-reps before closing PF */
12306 if (BNXT_PF(bp))
12307 bnxt_vf_reps_close(bp);
12308
12309 /* Change device state to avoid TX queue wake up's */
12310 bnxt_tx_disable(bp);
12311
12312 clear_bit(BNXT_STATE_OPEN, &bp->state);
12313 smp_mb__after_atomic();
12314 while (bnxt_drv_busy(bp))
12315 msleep(20);
12316
12317 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
12318 bnxt_clear_rss_ctxs(bp);
12319 /* Flush rings and disable interrupts */
12320 bnxt_shutdown_nic(bp, irq_re_init);
12321
12322 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
12323
12324 bnxt_debug_dev_exit(bp);
12325 bnxt_disable_napi(bp);
12326 del_timer_sync(&bp->timer);
12327 bnxt_free_skbs(bp);
12328
12329 /* Save ring stats before shutdown */
12330 if (bp->bnapi && irq_re_init) {
12331 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
12332 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
12333 }
12334 if (irq_re_init) {
12335 bnxt_free_irq(bp);
12336 bnxt_del_napi(bp);
12337 }
12338 bnxt_free_mem(bp, irq_re_init);
12339 }
12340
bnxt_close_nic(struct bnxt * bp,bool irq_re_init,bool link_re_init)12341 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12342 {
12343 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12344 /* If we get here, it means firmware reset is in progress
12345 * while we are trying to close. We can safely proceed with
12346 * the close because we are holding rtnl_lock(). Some firmware
12347 * messages may fail as we proceed to close. We set the
12348 * ABORT_ERR flag here so that the FW reset thread will later
12349 * abort when it gets the rtnl_lock() and sees the flag.
12350 */
12351 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
12352 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12353 }
12354
12355 #ifdef CONFIG_BNXT_SRIOV
12356 if (bp->sriov_cfg) {
12357 int rc;
12358
12359 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
12360 !bp->sriov_cfg,
12361 BNXT_SRIOV_CFG_WAIT_TMO);
12362 if (!rc)
12363 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
12364 else if (rc < 0)
12365 netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
12366 }
12367 #endif
12368 __bnxt_close_nic(bp, irq_re_init, link_re_init);
12369 }
12370
bnxt_close(struct net_device * dev)12371 static int bnxt_close(struct net_device *dev)
12372 {
12373 struct bnxt *bp = netdev_priv(dev);
12374
12375 bnxt_close_nic(bp, true, true);
12376 bnxt_hwrm_shutdown_link(bp);
12377 bnxt_hwrm_if_change(bp, false);
12378 return 0;
12379 }
12380
bnxt_hwrm_port_phy_read(struct bnxt * bp,u16 phy_addr,u16 reg,u16 * val)12381 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
12382 u16 *val)
12383 {
12384 struct hwrm_port_phy_mdio_read_output *resp;
12385 struct hwrm_port_phy_mdio_read_input *req;
12386 int rc;
12387
12388 if (bp->hwrm_spec_code < 0x10a00)
12389 return -EOPNOTSUPP;
12390
12391 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
12392 if (rc)
12393 return rc;
12394
12395 req->port_id = cpu_to_le16(bp->pf.port_id);
12396 req->phy_addr = phy_addr;
12397 req->reg_addr = cpu_to_le16(reg & 0x1f);
12398 if (mdio_phy_id_is_c45(phy_addr)) {
12399 req->cl45_mdio = 1;
12400 req->phy_addr = mdio_phy_id_prtad(phy_addr);
12401 req->dev_addr = mdio_phy_id_devad(phy_addr);
12402 req->reg_addr = cpu_to_le16(reg);
12403 }
12404
12405 resp = hwrm_req_hold(bp, req);
12406 rc = hwrm_req_send(bp, req);
12407 if (!rc)
12408 *val = le16_to_cpu(resp->reg_data);
12409 hwrm_req_drop(bp, req);
12410 return rc;
12411 }
12412
bnxt_hwrm_port_phy_write(struct bnxt * bp,u16 phy_addr,u16 reg,u16 val)12413 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
12414 u16 val)
12415 {
12416 struct hwrm_port_phy_mdio_write_input *req;
12417 int rc;
12418
12419 if (bp->hwrm_spec_code < 0x10a00)
12420 return -EOPNOTSUPP;
12421
12422 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
12423 if (rc)
12424 return rc;
12425
12426 req->port_id = cpu_to_le16(bp->pf.port_id);
12427 req->phy_addr = phy_addr;
12428 req->reg_addr = cpu_to_le16(reg & 0x1f);
12429 if (mdio_phy_id_is_c45(phy_addr)) {
12430 req->cl45_mdio = 1;
12431 req->phy_addr = mdio_phy_id_prtad(phy_addr);
12432 req->dev_addr = mdio_phy_id_devad(phy_addr);
12433 req->reg_addr = cpu_to_le16(reg);
12434 }
12435 req->reg_data = cpu_to_le16(val);
12436
12437 return hwrm_req_send(bp, req);
12438 }
12439
12440 /* rtnl_lock held */
bnxt_ioctl(struct net_device * dev,struct ifreq * ifr,int cmd)12441 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
12442 {
12443 struct mii_ioctl_data *mdio = if_mii(ifr);
12444 struct bnxt *bp = netdev_priv(dev);
12445 int rc;
12446
12447 switch (cmd) {
12448 case SIOCGMIIPHY:
12449 mdio->phy_id = bp->link_info.phy_addr;
12450
12451 fallthrough;
12452 case SIOCGMIIREG: {
12453 u16 mii_regval = 0;
12454
12455 if (!netif_running(dev))
12456 return -EAGAIN;
12457
12458 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
12459 &mii_regval);
12460 mdio->val_out = mii_regval;
12461 return rc;
12462 }
12463
12464 case SIOCSMIIREG:
12465 if (!netif_running(dev))
12466 return -EAGAIN;
12467
12468 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
12469 mdio->val_in);
12470
12471 case SIOCSHWTSTAMP:
12472 return bnxt_hwtstamp_set(dev, ifr);
12473
12474 case SIOCGHWTSTAMP:
12475 return bnxt_hwtstamp_get(dev, ifr);
12476
12477 default:
12478 /* do nothing */
12479 break;
12480 }
12481 return -EOPNOTSUPP;
12482 }
12483
bnxt_get_ring_stats(struct bnxt * bp,struct rtnl_link_stats64 * stats)12484 static void bnxt_get_ring_stats(struct bnxt *bp,
12485 struct rtnl_link_stats64 *stats)
12486 {
12487 int i;
12488
12489 for (i = 0; i < bp->cp_nr_rings; i++) {
12490 struct bnxt_napi *bnapi = bp->bnapi[i];
12491 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
12492 u64 *sw = cpr->stats.sw_stats;
12493
12494 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
12495 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12496 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
12497
12498 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
12499 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
12500 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
12501
12502 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
12503 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
12504 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
12505
12506 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
12507 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
12508 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
12509
12510 stats->rx_missed_errors +=
12511 BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
12512
12513 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12514
12515 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
12516
12517 stats->rx_dropped +=
12518 cpr->sw_stats->rx.rx_netpoll_discards +
12519 cpr->sw_stats->rx.rx_oom_discards;
12520 }
12521 }
12522
bnxt_add_prev_stats(struct bnxt * bp,struct rtnl_link_stats64 * stats)12523 static void bnxt_add_prev_stats(struct bnxt *bp,
12524 struct rtnl_link_stats64 *stats)
12525 {
12526 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
12527
12528 stats->rx_packets += prev_stats->rx_packets;
12529 stats->tx_packets += prev_stats->tx_packets;
12530 stats->rx_bytes += prev_stats->rx_bytes;
12531 stats->tx_bytes += prev_stats->tx_bytes;
12532 stats->rx_missed_errors += prev_stats->rx_missed_errors;
12533 stats->multicast += prev_stats->multicast;
12534 stats->rx_dropped += prev_stats->rx_dropped;
12535 stats->tx_dropped += prev_stats->tx_dropped;
12536 }
12537
12538 static void
bnxt_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats)12539 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
12540 {
12541 struct bnxt *bp = netdev_priv(dev);
12542
12543 set_bit(BNXT_STATE_READ_STATS, &bp->state);
12544 /* Make sure bnxt_close_nic() sees that we are reading stats before
12545 * we check the BNXT_STATE_OPEN flag.
12546 */
12547 smp_mb__after_atomic();
12548 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12549 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12550 *stats = bp->net_stats_prev;
12551 return;
12552 }
12553
12554 bnxt_get_ring_stats(bp, stats);
12555 bnxt_add_prev_stats(bp, stats);
12556
12557 if (bp->flags & BNXT_FLAG_PORT_STATS) {
12558 u64 *rx = bp->port_stats.sw_stats;
12559 u64 *tx = bp->port_stats.sw_stats +
12560 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
12561
12562 stats->rx_crc_errors =
12563 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
12564 stats->rx_frame_errors =
12565 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
12566 stats->rx_length_errors =
12567 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
12568 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
12569 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
12570 stats->rx_errors =
12571 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
12572 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
12573 stats->collisions =
12574 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
12575 stats->tx_fifo_errors =
12576 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
12577 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
12578 }
12579 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12580 }
12581
bnxt_get_one_ring_err_stats(struct bnxt * bp,struct bnxt_total_ring_err_stats * stats,struct bnxt_cp_ring_info * cpr)12582 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
12583 struct bnxt_total_ring_err_stats *stats,
12584 struct bnxt_cp_ring_info *cpr)
12585 {
12586 struct bnxt_sw_stats *sw_stats = cpr->sw_stats;
12587 u64 *hw_stats = cpr->stats.sw_stats;
12588
12589 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
12590 stats->rx_total_resets += sw_stats->rx.rx_resets;
12591 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
12592 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
12593 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
12594 stats->rx_total_ring_discards +=
12595 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
12596 stats->tx_total_resets += sw_stats->tx.tx_resets;
12597 stats->tx_total_ring_discards +=
12598 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
12599 stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
12600 }
12601
bnxt_get_ring_err_stats(struct bnxt * bp,struct bnxt_total_ring_err_stats * stats)12602 void bnxt_get_ring_err_stats(struct bnxt *bp,
12603 struct bnxt_total_ring_err_stats *stats)
12604 {
12605 int i;
12606
12607 for (i = 0; i < bp->cp_nr_rings; i++)
12608 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
12609 }
12610
bnxt_mc_list_updated(struct bnxt * bp,u32 * rx_mask)12611 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
12612 {
12613 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12614 struct net_device *dev = bp->dev;
12615 struct netdev_hw_addr *ha;
12616 u8 *haddr;
12617 int mc_count = 0;
12618 bool update = false;
12619 int off = 0;
12620
12621 netdev_for_each_mc_addr(ha, dev) {
12622 if (mc_count >= BNXT_MAX_MC_ADDRS) {
12623 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12624 vnic->mc_list_count = 0;
12625 return false;
12626 }
12627 haddr = ha->addr;
12628 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
12629 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
12630 update = true;
12631 }
12632 off += ETH_ALEN;
12633 mc_count++;
12634 }
12635 if (mc_count)
12636 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12637
12638 if (mc_count != vnic->mc_list_count) {
12639 vnic->mc_list_count = mc_count;
12640 update = true;
12641 }
12642 return update;
12643 }
12644
bnxt_uc_list_updated(struct bnxt * bp)12645 static bool bnxt_uc_list_updated(struct bnxt *bp)
12646 {
12647 struct net_device *dev = bp->dev;
12648 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12649 struct netdev_hw_addr *ha;
12650 int off = 0;
12651
12652 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
12653 return true;
12654
12655 netdev_for_each_uc_addr(ha, dev) {
12656 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
12657 return true;
12658
12659 off += ETH_ALEN;
12660 }
12661 return false;
12662 }
12663
bnxt_set_rx_mode(struct net_device * dev)12664 static void bnxt_set_rx_mode(struct net_device *dev)
12665 {
12666 struct bnxt *bp = netdev_priv(dev);
12667 struct bnxt_vnic_info *vnic;
12668 bool mc_update = false;
12669 bool uc_update;
12670 u32 mask;
12671
12672 if (!test_bit(BNXT_STATE_OPEN, &bp->state))
12673 return;
12674
12675 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12676 mask = vnic->rx_mask;
12677 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
12678 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
12679 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
12680 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
12681
12682 if (dev->flags & IFF_PROMISC)
12683 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12684
12685 uc_update = bnxt_uc_list_updated(bp);
12686
12687 if (dev->flags & IFF_BROADCAST)
12688 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
12689 if (dev->flags & IFF_ALLMULTI) {
12690 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12691 vnic->mc_list_count = 0;
12692 } else if (dev->flags & IFF_MULTICAST) {
12693 mc_update = bnxt_mc_list_updated(bp, &mask);
12694 }
12695
12696 if (mask != vnic->rx_mask || uc_update || mc_update) {
12697 vnic->rx_mask = mask;
12698
12699 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12700 }
12701 }
12702
bnxt_cfg_rx_mode(struct bnxt * bp)12703 static int bnxt_cfg_rx_mode(struct bnxt *bp)
12704 {
12705 struct net_device *dev = bp->dev;
12706 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12707 struct netdev_hw_addr *ha;
12708 int i, off = 0, rc;
12709 bool uc_update;
12710
12711 netif_addr_lock_bh(dev);
12712 uc_update = bnxt_uc_list_updated(bp);
12713 netif_addr_unlock_bh(dev);
12714
12715 if (!uc_update)
12716 goto skip_uc;
12717
12718 for (i = 1; i < vnic->uc_filter_count; i++) {
12719 struct bnxt_l2_filter *fltr = vnic->l2_filters[i];
12720
12721 bnxt_hwrm_l2_filter_free(bp, fltr);
12722 bnxt_del_l2_filter(bp, fltr);
12723 }
12724
12725 vnic->uc_filter_count = 1;
12726
12727 netif_addr_lock_bh(dev);
12728 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
12729 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12730 } else {
12731 netdev_for_each_uc_addr(ha, dev) {
12732 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
12733 off += ETH_ALEN;
12734 vnic->uc_filter_count++;
12735 }
12736 }
12737 netif_addr_unlock_bh(dev);
12738
12739 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
12740 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
12741 if (rc) {
12742 if (BNXT_VF(bp) && rc == -ENODEV) {
12743 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12744 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
12745 else
12746 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
12747 rc = 0;
12748 } else {
12749 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
12750 }
12751 vnic->uc_filter_count = i;
12752 return rc;
12753 }
12754 }
12755 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12756 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
12757
12758 skip_uc:
12759 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
12760 !bnxt_promisc_ok(bp))
12761 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12762 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12763 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
12764 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
12765 rc);
12766 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12767 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12768 vnic->mc_list_count = 0;
12769 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12770 }
12771 if (rc)
12772 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
12773 rc);
12774
12775 return rc;
12776 }
12777
bnxt_can_reserve_rings(struct bnxt * bp)12778 static bool bnxt_can_reserve_rings(struct bnxt *bp)
12779 {
12780 #ifdef CONFIG_BNXT_SRIOV
12781 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
12782 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12783
12784 /* No minimum rings were provisioned by the PF. Don't
12785 * reserve rings by default when device is down.
12786 */
12787 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
12788 return true;
12789
12790 if (!netif_running(bp->dev))
12791 return false;
12792 }
12793 #endif
12794 return true;
12795 }
12796
12797 /* If the chip and firmware supports RFS */
bnxt_rfs_supported(struct bnxt * bp)12798 static bool bnxt_rfs_supported(struct bnxt *bp)
12799 {
12800 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
12801 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
12802 return true;
12803 return false;
12804 }
12805 /* 212 firmware is broken for aRFS */
12806 if (BNXT_FW_MAJ(bp) == 212)
12807 return false;
12808 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
12809 return true;
12810 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12811 return true;
12812 return false;
12813 }
12814
12815 /* If runtime conditions support RFS */
bnxt_rfs_capable(struct bnxt * bp,bool new_rss_ctx)12816 bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx)
12817 {
12818 struct bnxt_hw_rings hwr = {0};
12819 int max_vnics, max_rss_ctxs;
12820
12821 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
12822 !BNXT_SUPPORTS_NTUPLE_VNIC(bp))
12823 return bnxt_rfs_supported(bp);
12824
12825 if (!bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
12826 return false;
12827
12828 hwr.grp = bp->rx_nr_rings;
12829 hwr.vnic = bnxt_get_total_vnics(bp, bp->rx_nr_rings);
12830 if (new_rss_ctx)
12831 hwr.vnic++;
12832 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
12833 max_vnics = bnxt_get_max_func_vnics(bp);
12834 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
12835
12836 if (hwr.vnic > max_vnics || hwr.rss_ctx > max_rss_ctxs) {
12837 if (bp->rx_nr_rings > 1)
12838 netdev_warn(bp->dev,
12839 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
12840 min(max_rss_ctxs - 1, max_vnics - 1));
12841 return false;
12842 }
12843
12844 if (!BNXT_NEW_RM(bp))
12845 return true;
12846
12847 /* Do not reduce VNIC and RSS ctx reservations. There is a FW
12848 * issue that will mess up the default VNIC if we reduce the
12849 * reservations.
12850 */
12851 if (hwr.vnic <= bp->hw_resc.resv_vnics &&
12852 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
12853 return true;
12854
12855 bnxt_hwrm_reserve_rings(bp, &hwr);
12856 if (hwr.vnic <= bp->hw_resc.resv_vnics &&
12857 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
12858 return true;
12859
12860 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
12861 hwr.vnic = 1;
12862 hwr.rss_ctx = 0;
12863 bnxt_hwrm_reserve_rings(bp, &hwr);
12864 return false;
12865 }
12866
bnxt_fix_features(struct net_device * dev,netdev_features_t features)12867 static netdev_features_t bnxt_fix_features(struct net_device *dev,
12868 netdev_features_t features)
12869 {
12870 struct bnxt *bp = netdev_priv(dev);
12871 netdev_features_t vlan_features;
12872
12873 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false))
12874 features &= ~NETIF_F_NTUPLE;
12875
12876 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
12877 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12878
12879 if (!(features & NETIF_F_GRO))
12880 features &= ~NETIF_F_GRO_HW;
12881
12882 if (features & NETIF_F_GRO_HW)
12883 features &= ~NETIF_F_LRO;
12884
12885 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
12886 * turned on or off together.
12887 */
12888 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
12889 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
12890 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12891 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12892 else if (vlan_features)
12893 features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12894 }
12895 #ifdef CONFIG_BNXT_SRIOV
12896 if (BNXT_VF(bp) && bp->vf.vlan)
12897 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12898 #endif
12899 return features;
12900 }
12901
bnxt_reinit_features(struct bnxt * bp,bool irq_re_init,bool link_re_init,u32 flags,bool update_tpa)12902 static int bnxt_reinit_features(struct bnxt *bp, bool irq_re_init,
12903 bool link_re_init, u32 flags, bool update_tpa)
12904 {
12905 bnxt_close_nic(bp, irq_re_init, link_re_init);
12906 bp->flags = flags;
12907 if (update_tpa)
12908 bnxt_set_ring_params(bp);
12909 return bnxt_open_nic(bp, irq_re_init, link_re_init);
12910 }
12911
bnxt_set_features(struct net_device * dev,netdev_features_t features)12912 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
12913 {
12914 bool update_tpa = false, update_ntuple = false;
12915 struct bnxt *bp = netdev_priv(dev);
12916 u32 flags = bp->flags;
12917 u32 changes;
12918 int rc = 0;
12919 bool re_init = false;
12920
12921 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
12922 if (features & NETIF_F_GRO_HW)
12923 flags |= BNXT_FLAG_GRO;
12924 else if (features & NETIF_F_LRO)
12925 flags |= BNXT_FLAG_LRO;
12926
12927 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
12928 flags &= ~BNXT_FLAG_TPA;
12929
12930 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12931 flags |= BNXT_FLAG_STRIP_VLAN;
12932
12933 if (features & NETIF_F_NTUPLE)
12934 flags |= BNXT_FLAG_RFS;
12935 else
12936 bnxt_clear_usr_fltrs(bp, true);
12937
12938 changes = flags ^ bp->flags;
12939 if (changes & BNXT_FLAG_TPA) {
12940 update_tpa = true;
12941 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
12942 (flags & BNXT_FLAG_TPA) == 0 ||
12943 (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12944 re_init = true;
12945 }
12946
12947 if (changes & ~BNXT_FLAG_TPA)
12948 re_init = true;
12949
12950 if (changes & BNXT_FLAG_RFS)
12951 update_ntuple = true;
12952
12953 if (flags != bp->flags) {
12954 u32 old_flags = bp->flags;
12955
12956 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12957 bp->flags = flags;
12958 if (update_tpa)
12959 bnxt_set_ring_params(bp);
12960 return rc;
12961 }
12962
12963 if (update_ntuple)
12964 return bnxt_reinit_features(bp, true, false, flags, update_tpa);
12965
12966 if (re_init)
12967 return bnxt_reinit_features(bp, false, false, flags, update_tpa);
12968
12969 if (update_tpa) {
12970 bp->flags = flags;
12971 rc = bnxt_set_tpa(bp,
12972 (flags & BNXT_FLAG_TPA) ?
12973 true : false);
12974 if (rc)
12975 bp->flags = old_flags;
12976 }
12977 }
12978 return rc;
12979 }
12980
bnxt_exthdr_check(struct bnxt * bp,struct sk_buff * skb,int nw_off,u8 ** nextp)12981 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
12982 u8 **nextp)
12983 {
12984 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
12985 struct hop_jumbo_hdr *jhdr;
12986 int hdr_count = 0;
12987 u8 *nexthdr;
12988 int start;
12989
12990 /* Check that there are at most 2 IPv6 extension headers, no
12991 * fragment header, and each is <= 64 bytes.
12992 */
12993 start = nw_off + sizeof(*ip6h);
12994 nexthdr = &ip6h->nexthdr;
12995 while (ipv6_ext_hdr(*nexthdr)) {
12996 struct ipv6_opt_hdr *hp;
12997 int hdrlen;
12998
12999 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
13000 *nexthdr == NEXTHDR_FRAGMENT)
13001 return false;
13002 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
13003 skb_headlen(skb), NULL);
13004 if (!hp)
13005 return false;
13006 if (*nexthdr == NEXTHDR_AUTH)
13007 hdrlen = ipv6_authlen(hp);
13008 else
13009 hdrlen = ipv6_optlen(hp);
13010
13011 if (hdrlen > 64)
13012 return false;
13013
13014 /* The ext header may be a hop-by-hop header inserted for
13015 * big TCP purposes. This will be removed before sending
13016 * from NIC, so do not count it.
13017 */
13018 if (*nexthdr == NEXTHDR_HOP) {
13019 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
13020 goto increment_hdr;
13021
13022 jhdr = (struct hop_jumbo_hdr *)hp;
13023 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
13024 jhdr->nexthdr != IPPROTO_TCP)
13025 goto increment_hdr;
13026
13027 goto next_hdr;
13028 }
13029 increment_hdr:
13030 hdr_count++;
13031 next_hdr:
13032 nexthdr = &hp->nexthdr;
13033 start += hdrlen;
13034 }
13035 if (nextp) {
13036 /* Caller will check inner protocol */
13037 if (skb->encapsulation) {
13038 *nextp = nexthdr;
13039 return true;
13040 }
13041 *nextp = NULL;
13042 }
13043 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
13044 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
13045 }
13046
13047 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
bnxt_udp_tunl_check(struct bnxt * bp,struct sk_buff * skb)13048 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
13049 {
13050 struct udphdr *uh = udp_hdr(skb);
13051 __be16 udp_port = uh->dest;
13052
13053 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port &&
13054 udp_port != bp->vxlan_gpe_port)
13055 return false;
13056 if (skb->inner_protocol == htons(ETH_P_TEB)) {
13057 struct ethhdr *eh = inner_eth_hdr(skb);
13058
13059 switch (eh->h_proto) {
13060 case htons(ETH_P_IP):
13061 return true;
13062 case htons(ETH_P_IPV6):
13063 return bnxt_exthdr_check(bp, skb,
13064 skb_inner_network_offset(skb),
13065 NULL);
13066 }
13067 } else if (skb->inner_protocol == htons(ETH_P_IP)) {
13068 return true;
13069 } else if (skb->inner_protocol == htons(ETH_P_IPV6)) {
13070 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
13071 NULL);
13072 }
13073 return false;
13074 }
13075
bnxt_tunl_check(struct bnxt * bp,struct sk_buff * skb,u8 l4_proto)13076 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
13077 {
13078 switch (l4_proto) {
13079 case IPPROTO_UDP:
13080 return bnxt_udp_tunl_check(bp, skb);
13081 case IPPROTO_IPIP:
13082 return true;
13083 case IPPROTO_GRE: {
13084 switch (skb->inner_protocol) {
13085 default:
13086 return false;
13087 case htons(ETH_P_IP):
13088 return true;
13089 case htons(ETH_P_IPV6):
13090 fallthrough;
13091 }
13092 }
13093 case IPPROTO_IPV6:
13094 /* Check ext headers of inner ipv6 */
13095 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
13096 NULL);
13097 }
13098 return false;
13099 }
13100
bnxt_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)13101 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
13102 struct net_device *dev,
13103 netdev_features_t features)
13104 {
13105 struct bnxt *bp = netdev_priv(dev);
13106 u8 *l4_proto;
13107
13108 features = vlan_features_check(skb, features);
13109 switch (vlan_get_protocol(skb)) {
13110 case htons(ETH_P_IP):
13111 if (!skb->encapsulation)
13112 return features;
13113 l4_proto = &ip_hdr(skb)->protocol;
13114 if (bnxt_tunl_check(bp, skb, *l4_proto))
13115 return features;
13116 break;
13117 case htons(ETH_P_IPV6):
13118 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
13119 &l4_proto))
13120 break;
13121 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
13122 return features;
13123 break;
13124 }
13125 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13126 }
13127
bnxt_dbg_hwrm_rd_reg(struct bnxt * bp,u32 reg_off,u16 num_words,u32 * reg_buf)13128 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
13129 u32 *reg_buf)
13130 {
13131 struct hwrm_dbg_read_direct_output *resp;
13132 struct hwrm_dbg_read_direct_input *req;
13133 __le32 *dbg_reg_buf;
13134 dma_addr_t mapping;
13135 int rc, i;
13136
13137 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
13138 if (rc)
13139 return rc;
13140
13141 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
13142 &mapping);
13143 if (!dbg_reg_buf) {
13144 rc = -ENOMEM;
13145 goto dbg_rd_reg_exit;
13146 }
13147
13148 req->host_dest_addr = cpu_to_le64(mapping);
13149
13150 resp = hwrm_req_hold(bp, req);
13151 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
13152 req->read_len32 = cpu_to_le32(num_words);
13153
13154 rc = hwrm_req_send(bp, req);
13155 if (rc || resp->error_code) {
13156 rc = -EIO;
13157 goto dbg_rd_reg_exit;
13158 }
13159 for (i = 0; i < num_words; i++)
13160 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
13161
13162 dbg_rd_reg_exit:
13163 hwrm_req_drop(bp, req);
13164 return rc;
13165 }
13166
bnxt_dbg_hwrm_ring_info_get(struct bnxt * bp,u8 ring_type,u32 ring_id,u32 * prod,u32 * cons)13167 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
13168 u32 ring_id, u32 *prod, u32 *cons)
13169 {
13170 struct hwrm_dbg_ring_info_get_output *resp;
13171 struct hwrm_dbg_ring_info_get_input *req;
13172 int rc;
13173
13174 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
13175 if (rc)
13176 return rc;
13177
13178 req->ring_type = ring_type;
13179 req->fw_ring_id = cpu_to_le32(ring_id);
13180 resp = hwrm_req_hold(bp, req);
13181 rc = hwrm_req_send(bp, req);
13182 if (!rc) {
13183 *prod = le32_to_cpu(resp->producer_index);
13184 *cons = le32_to_cpu(resp->consumer_index);
13185 }
13186 hwrm_req_drop(bp, req);
13187 return rc;
13188 }
13189
bnxt_dump_tx_sw_state(struct bnxt_napi * bnapi)13190 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
13191 {
13192 struct bnxt_tx_ring_info *txr;
13193 int i = bnapi->index, j;
13194
13195 bnxt_for_each_napi_tx(j, bnapi, txr)
13196 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
13197 i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
13198 txr->tx_cons);
13199 }
13200
bnxt_dump_rx_sw_state(struct bnxt_napi * bnapi)13201 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
13202 {
13203 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
13204 int i = bnapi->index;
13205
13206 if (!rxr)
13207 return;
13208
13209 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
13210 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
13211 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
13212 rxr->rx_sw_agg_prod);
13213 }
13214
bnxt_dump_cp_sw_state(struct bnxt_napi * bnapi)13215 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
13216 {
13217 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
13218 int i = bnapi->index;
13219
13220 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
13221 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
13222 }
13223
bnxt_dbg_dump_states(struct bnxt * bp)13224 static void bnxt_dbg_dump_states(struct bnxt *bp)
13225 {
13226 int i;
13227 struct bnxt_napi *bnapi;
13228
13229 for (i = 0; i < bp->cp_nr_rings; i++) {
13230 bnapi = bp->bnapi[i];
13231 if (netif_msg_drv(bp)) {
13232 bnxt_dump_tx_sw_state(bnapi);
13233 bnxt_dump_rx_sw_state(bnapi);
13234 bnxt_dump_cp_sw_state(bnapi);
13235 }
13236 }
13237 }
13238
bnxt_hwrm_rx_ring_reset(struct bnxt * bp,int ring_nr)13239 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
13240 {
13241 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
13242 struct hwrm_ring_reset_input *req;
13243 struct bnxt_napi *bnapi = rxr->bnapi;
13244 struct bnxt_cp_ring_info *cpr;
13245 u16 cp_ring_id;
13246 int rc;
13247
13248 rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
13249 if (rc)
13250 return rc;
13251
13252 cpr = &bnapi->cp_ring;
13253 cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
13254 req->cmpl_ring = cpu_to_le16(cp_ring_id);
13255 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
13256 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
13257 return hwrm_req_send_silent(bp, req);
13258 }
13259
bnxt_reset_task(struct bnxt * bp,bool silent)13260 static void bnxt_reset_task(struct bnxt *bp, bool silent)
13261 {
13262 if (!silent)
13263 bnxt_dbg_dump_states(bp);
13264 if (netif_running(bp->dev)) {
13265 bnxt_close_nic(bp, !silent, false);
13266 bnxt_open_nic(bp, !silent, false);
13267 }
13268 }
13269
bnxt_tx_timeout(struct net_device * dev,unsigned int txqueue)13270 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
13271 {
13272 struct bnxt *bp = netdev_priv(dev);
13273
13274 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
13275 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
13276 }
13277
bnxt_fw_health_check(struct bnxt * bp)13278 static void bnxt_fw_health_check(struct bnxt *bp)
13279 {
13280 struct bnxt_fw_health *fw_health = bp->fw_health;
13281 struct pci_dev *pdev = bp->pdev;
13282 u32 val;
13283
13284 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13285 return;
13286
13287 /* Make sure it is enabled before checking the tmr_counter. */
13288 smp_rmb();
13289 if (fw_health->tmr_counter) {
13290 fw_health->tmr_counter--;
13291 return;
13292 }
13293
13294 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13295 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
13296 fw_health->arrests++;
13297 goto fw_reset;
13298 }
13299
13300 fw_health->last_fw_heartbeat = val;
13301
13302 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13303 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
13304 fw_health->discoveries++;
13305 goto fw_reset;
13306 }
13307
13308 fw_health->tmr_counter = fw_health->tmr_multiplier;
13309 return;
13310
13311 fw_reset:
13312 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
13313 }
13314
bnxt_timer(struct timer_list * t)13315 static void bnxt_timer(struct timer_list *t)
13316 {
13317 struct bnxt *bp = from_timer(bp, t, timer);
13318 struct net_device *dev = bp->dev;
13319
13320 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
13321 return;
13322
13323 if (atomic_read(&bp->intr_sem) != 0)
13324 goto bnxt_restart_timer;
13325
13326 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
13327 bnxt_fw_health_check(bp);
13328
13329 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
13330 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
13331
13332 if (bnxt_tc_flower_enabled(bp))
13333 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
13334
13335 #ifdef CONFIG_RFS_ACCEL
13336 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
13337 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
13338 #endif /*CONFIG_RFS_ACCEL*/
13339
13340 if (bp->link_info.phy_retry) {
13341 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
13342 bp->link_info.phy_retry = false;
13343 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
13344 } else {
13345 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
13346 }
13347 }
13348
13349 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
13350 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
13351
13352 if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev))
13353 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
13354
13355 bnxt_restart_timer:
13356 mod_timer(&bp->timer, jiffies + bp->current_interval);
13357 }
13358
bnxt_rtnl_lock_sp(struct bnxt * bp)13359 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
13360 {
13361 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
13362 * set. If the device is being closed, bnxt_close() may be holding
13363 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
13364 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
13365 */
13366 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13367 rtnl_lock();
13368 }
13369
bnxt_rtnl_unlock_sp(struct bnxt * bp)13370 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
13371 {
13372 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13373 rtnl_unlock();
13374 }
13375
13376 /* Only called from bnxt_sp_task() */
bnxt_reset(struct bnxt * bp,bool silent)13377 static void bnxt_reset(struct bnxt *bp, bool silent)
13378 {
13379 bnxt_rtnl_lock_sp(bp);
13380 if (test_bit(BNXT_STATE_OPEN, &bp->state))
13381 bnxt_reset_task(bp, silent);
13382 bnxt_rtnl_unlock_sp(bp);
13383 }
13384
13385 /* Only called from bnxt_sp_task() */
bnxt_rx_ring_reset(struct bnxt * bp)13386 static void bnxt_rx_ring_reset(struct bnxt *bp)
13387 {
13388 int i;
13389
13390 bnxt_rtnl_lock_sp(bp);
13391 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13392 bnxt_rtnl_unlock_sp(bp);
13393 return;
13394 }
13395 /* Disable and flush TPA before resetting the RX ring */
13396 if (bp->flags & BNXT_FLAG_TPA)
13397 bnxt_set_tpa(bp, false);
13398 for (i = 0; i < bp->rx_nr_rings; i++) {
13399 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
13400 struct bnxt_cp_ring_info *cpr;
13401 int rc;
13402
13403 if (!rxr->bnapi->in_reset)
13404 continue;
13405
13406 rc = bnxt_hwrm_rx_ring_reset(bp, i);
13407 if (rc) {
13408 if (rc == -EINVAL || rc == -EOPNOTSUPP)
13409 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
13410 else
13411 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
13412 rc);
13413 bnxt_reset_task(bp, true);
13414 break;
13415 }
13416 bnxt_free_one_rx_ring_skbs(bp, i);
13417 rxr->rx_prod = 0;
13418 rxr->rx_agg_prod = 0;
13419 rxr->rx_sw_agg_prod = 0;
13420 rxr->rx_next_cons = 0;
13421 rxr->bnapi->in_reset = false;
13422 bnxt_alloc_one_rx_ring(bp, i);
13423 cpr = &rxr->bnapi->cp_ring;
13424 cpr->sw_stats->rx.rx_resets++;
13425 if (bp->flags & BNXT_FLAG_AGG_RINGS)
13426 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
13427 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
13428 }
13429 if (bp->flags & BNXT_FLAG_TPA)
13430 bnxt_set_tpa(bp, true);
13431 bnxt_rtnl_unlock_sp(bp);
13432 }
13433
bnxt_fw_fatal_close(struct bnxt * bp)13434 static void bnxt_fw_fatal_close(struct bnxt *bp)
13435 {
13436 bnxt_tx_disable(bp);
13437 bnxt_disable_napi(bp);
13438 bnxt_disable_int_sync(bp);
13439 bnxt_free_irq(bp);
13440 bnxt_clear_int_mode(bp);
13441 pci_disable_device(bp->pdev);
13442 }
13443
bnxt_fw_reset_close(struct bnxt * bp)13444 static void bnxt_fw_reset_close(struct bnxt *bp)
13445 {
13446 /* When firmware is in fatal state, quiesce device and disable
13447 * bus master to prevent any potential bad DMAs before freeing
13448 * kernel memory.
13449 */
13450 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
13451 u16 val = 0;
13452
13453 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
13454 if (val == 0xffff)
13455 bp->fw_reset_min_dsecs = 0;
13456 bnxt_fw_fatal_close(bp);
13457 }
13458 __bnxt_close_nic(bp, true, false);
13459 bnxt_vf_reps_free(bp);
13460 bnxt_clear_int_mode(bp);
13461 bnxt_hwrm_func_drv_unrgtr(bp);
13462 if (pci_is_enabled(bp->pdev))
13463 pci_disable_device(bp->pdev);
13464 bnxt_free_ctx_mem(bp);
13465 }
13466
is_bnxt_fw_ok(struct bnxt * bp)13467 static bool is_bnxt_fw_ok(struct bnxt *bp)
13468 {
13469 struct bnxt_fw_health *fw_health = bp->fw_health;
13470 bool no_heartbeat = false, has_reset = false;
13471 u32 val;
13472
13473 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13474 if (val == fw_health->last_fw_heartbeat)
13475 no_heartbeat = true;
13476
13477 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13478 if (val != fw_health->last_fw_reset_cnt)
13479 has_reset = true;
13480
13481 if (!no_heartbeat && has_reset)
13482 return true;
13483
13484 return false;
13485 }
13486
13487 /* rtnl_lock is acquired before calling this function */
bnxt_force_fw_reset(struct bnxt * bp)13488 static void bnxt_force_fw_reset(struct bnxt *bp)
13489 {
13490 struct bnxt_fw_health *fw_health = bp->fw_health;
13491 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13492 u32 wait_dsecs;
13493
13494 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
13495 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13496 return;
13497
13498 if (ptp) {
13499 unsigned long flags;
13500
13501 spin_lock_irqsave(&ptp->ptp_lock, flags);
13502 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13503 spin_unlock_irqrestore(&ptp->ptp_lock, flags);
13504 } else {
13505 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13506 }
13507 bnxt_fw_reset_close(bp);
13508 wait_dsecs = fw_health->master_func_wait_dsecs;
13509 if (fw_health->primary) {
13510 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
13511 wait_dsecs = 0;
13512 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
13513 } else {
13514 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
13515 wait_dsecs = fw_health->normal_func_wait_dsecs;
13516 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13517 }
13518
13519 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
13520 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
13521 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
13522 }
13523
bnxt_fw_exception(struct bnxt * bp)13524 void bnxt_fw_exception(struct bnxt *bp)
13525 {
13526 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
13527 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
13528 bnxt_ulp_stop(bp);
13529 bnxt_rtnl_lock_sp(bp);
13530 bnxt_force_fw_reset(bp);
13531 bnxt_rtnl_unlock_sp(bp);
13532 }
13533
13534 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
13535 * < 0 on error.
13536 */
bnxt_get_registered_vfs(struct bnxt * bp)13537 static int bnxt_get_registered_vfs(struct bnxt *bp)
13538 {
13539 #ifdef CONFIG_BNXT_SRIOV
13540 int rc;
13541
13542 if (!BNXT_PF(bp))
13543 return 0;
13544
13545 rc = bnxt_hwrm_func_qcfg(bp);
13546 if (rc) {
13547 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
13548 return rc;
13549 }
13550 if (bp->pf.registered_vfs)
13551 return bp->pf.registered_vfs;
13552 if (bp->sriov_cfg)
13553 return 1;
13554 #endif
13555 return 0;
13556 }
13557
bnxt_fw_reset(struct bnxt * bp)13558 void bnxt_fw_reset(struct bnxt *bp)
13559 {
13560 bnxt_ulp_stop(bp);
13561 bnxt_rtnl_lock_sp(bp);
13562 if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
13563 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13564 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13565 int n = 0, tmo;
13566
13567 if (ptp) {
13568 unsigned long flags;
13569
13570 spin_lock_irqsave(&ptp->ptp_lock, flags);
13571 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13572 spin_unlock_irqrestore(&ptp->ptp_lock, flags);
13573 } else {
13574 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13575 }
13576 if (bp->pf.active_vfs &&
13577 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
13578 n = bnxt_get_registered_vfs(bp);
13579 if (n < 0) {
13580 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
13581 n);
13582 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13583 dev_close(bp->dev);
13584 goto fw_reset_exit;
13585 } else if (n > 0) {
13586 u16 vf_tmo_dsecs = n * 10;
13587
13588 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
13589 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
13590 bp->fw_reset_state =
13591 BNXT_FW_RESET_STATE_POLL_VF;
13592 bnxt_queue_fw_reset_work(bp, HZ / 10);
13593 goto fw_reset_exit;
13594 }
13595 bnxt_fw_reset_close(bp);
13596 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13597 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
13598 tmo = HZ / 10;
13599 } else {
13600 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13601 tmo = bp->fw_reset_min_dsecs * HZ / 10;
13602 }
13603 bnxt_queue_fw_reset_work(bp, tmo);
13604 }
13605 fw_reset_exit:
13606 bnxt_rtnl_unlock_sp(bp);
13607 }
13608
bnxt_chk_missed_irq(struct bnxt * bp)13609 static void bnxt_chk_missed_irq(struct bnxt *bp)
13610 {
13611 int i;
13612
13613 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
13614 return;
13615
13616 for (i = 0; i < bp->cp_nr_rings; i++) {
13617 struct bnxt_napi *bnapi = bp->bnapi[i];
13618 struct bnxt_cp_ring_info *cpr;
13619 u32 fw_ring_id;
13620 int j;
13621
13622 if (!bnapi)
13623 continue;
13624
13625 cpr = &bnapi->cp_ring;
13626 for (j = 0; j < cpr->cp_ring_count; j++) {
13627 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
13628 u32 val[2];
13629
13630 if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
13631 continue;
13632
13633 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
13634 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
13635 continue;
13636 }
13637 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
13638 bnxt_dbg_hwrm_ring_info_get(bp,
13639 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
13640 fw_ring_id, &val[0], &val[1]);
13641 cpr->sw_stats->cmn.missed_irqs++;
13642 }
13643 }
13644 }
13645
13646 static void bnxt_cfg_ntp_filters(struct bnxt *);
13647
bnxt_init_ethtool_link_settings(struct bnxt * bp)13648 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
13649 {
13650 struct bnxt_link_info *link_info = &bp->link_info;
13651
13652 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
13653 link_info->autoneg = BNXT_AUTONEG_SPEED;
13654 if (bp->hwrm_spec_code >= 0x10201) {
13655 if (link_info->auto_pause_setting &
13656 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
13657 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13658 } else {
13659 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13660 }
13661 bnxt_set_auto_speed(link_info);
13662 } else {
13663 bnxt_set_force_speed(link_info);
13664 link_info->req_duplex = link_info->duplex_setting;
13665 }
13666 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
13667 link_info->req_flow_ctrl =
13668 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
13669 else
13670 link_info->req_flow_ctrl = link_info->force_pause_setting;
13671 }
13672
bnxt_fw_echo_reply(struct bnxt * bp)13673 static void bnxt_fw_echo_reply(struct bnxt *bp)
13674 {
13675 struct bnxt_fw_health *fw_health = bp->fw_health;
13676 struct hwrm_func_echo_response_input *req;
13677 int rc;
13678
13679 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
13680 if (rc)
13681 return;
13682 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
13683 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
13684 hwrm_req_send(bp, req);
13685 }
13686
bnxt_ulp_restart(struct bnxt * bp)13687 static void bnxt_ulp_restart(struct bnxt *bp)
13688 {
13689 bnxt_ulp_stop(bp);
13690 bnxt_ulp_start(bp, 0);
13691 }
13692
bnxt_sp_task(struct work_struct * work)13693 static void bnxt_sp_task(struct work_struct *work)
13694 {
13695 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
13696
13697 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13698 smp_mb__after_atomic();
13699 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13700 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13701 return;
13702 }
13703
13704 if (test_and_clear_bit(BNXT_RESTART_ULP_SP_EVENT, &bp->sp_event)) {
13705 bnxt_ulp_restart(bp);
13706 bnxt_reenable_sriov(bp);
13707 }
13708
13709 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
13710 bnxt_cfg_rx_mode(bp);
13711
13712 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
13713 bnxt_cfg_ntp_filters(bp);
13714 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
13715 bnxt_hwrm_exec_fwd_req(bp);
13716 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13717 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13718 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
13719 bnxt_hwrm_port_qstats(bp, 0);
13720 bnxt_hwrm_port_qstats_ext(bp, 0);
13721 bnxt_accumulate_all_stats(bp);
13722 }
13723
13724 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
13725 int rc;
13726
13727 mutex_lock(&bp->link_lock);
13728 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
13729 &bp->sp_event))
13730 bnxt_hwrm_phy_qcaps(bp);
13731
13732 rc = bnxt_update_link(bp, true);
13733 if (rc)
13734 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
13735 rc);
13736
13737 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
13738 &bp->sp_event))
13739 bnxt_init_ethtool_link_settings(bp);
13740 mutex_unlock(&bp->link_lock);
13741 }
13742 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
13743 int rc;
13744
13745 mutex_lock(&bp->link_lock);
13746 rc = bnxt_update_phy_setting(bp);
13747 mutex_unlock(&bp->link_lock);
13748 if (rc) {
13749 netdev_warn(bp->dev, "update phy settings retry failed\n");
13750 } else {
13751 bp->link_info.phy_retry = false;
13752 netdev_info(bp->dev, "update phy settings retry succeeded\n");
13753 }
13754 }
13755 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
13756 mutex_lock(&bp->link_lock);
13757 bnxt_get_port_module_status(bp);
13758 mutex_unlock(&bp->link_lock);
13759 }
13760
13761 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
13762 bnxt_tc_flow_stats_work(bp);
13763
13764 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
13765 bnxt_chk_missed_irq(bp);
13766
13767 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
13768 bnxt_fw_echo_reply(bp);
13769
13770 if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event))
13771 bnxt_hwmon_notify_event(bp);
13772
13773 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
13774 * must be the last functions to be called before exiting.
13775 */
13776 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
13777 bnxt_reset(bp, false);
13778
13779 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
13780 bnxt_reset(bp, true);
13781
13782 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
13783 bnxt_rx_ring_reset(bp);
13784
13785 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
13786 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
13787 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
13788 bnxt_devlink_health_fw_report(bp);
13789 else
13790 bnxt_fw_reset(bp);
13791 }
13792
13793 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
13794 if (!is_bnxt_fw_ok(bp))
13795 bnxt_devlink_health_fw_report(bp);
13796 }
13797
13798 smp_mb__before_atomic();
13799 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13800 }
13801
13802 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13803 int *max_cp);
13804
13805 /* Under rtnl_lock */
bnxt_check_rings(struct bnxt * bp,int tx,int rx,bool sh,int tcs,int tx_xdp)13806 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
13807 int tx_xdp)
13808 {
13809 int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
13810 struct bnxt_hw_rings hwr = {0};
13811 int rx_rings = rx;
13812 int rc;
13813
13814 if (tcs)
13815 tx_sets = tcs;
13816
13817 _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
13818
13819 if (max_rx < rx_rings)
13820 return -ENOMEM;
13821
13822 if (bp->flags & BNXT_FLAG_AGG_RINGS)
13823 rx_rings <<= 1;
13824
13825 hwr.rx = rx_rings;
13826 hwr.tx = tx * tx_sets + tx_xdp;
13827 if (max_tx < hwr.tx)
13828 return -ENOMEM;
13829
13830 hwr.vnic = bnxt_get_total_vnics(bp, rx);
13831
13832 tx_cp = __bnxt_num_tx_to_cp(bp, hwr.tx, tx_sets, tx_xdp);
13833 hwr.cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
13834 if (max_cp < hwr.cp)
13835 return -ENOMEM;
13836 hwr.stat = hwr.cp;
13837 if (BNXT_NEW_RM(bp)) {
13838 hwr.cp += bnxt_get_ulp_msix_num_in_use(bp);
13839 hwr.stat += bnxt_get_ulp_stat_ctxs_in_use(bp);
13840 hwr.grp = rx;
13841 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
13842 }
13843 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
13844 hwr.cp_p5 = hwr.tx + rx;
13845 rc = bnxt_hwrm_check_rings(bp, &hwr);
13846 if (!rc && pci_msix_can_alloc_dyn(bp->pdev)) {
13847 if (!bnxt_ulp_registered(bp->edev)) {
13848 hwr.cp += bnxt_get_ulp_msix_num(bp);
13849 hwr.cp = min_t(int, hwr.cp, bnxt_get_max_func_irqs(bp));
13850 }
13851 if (hwr.cp > bp->total_irqs) {
13852 int total_msix = bnxt_change_msix(bp, hwr.cp);
13853
13854 if (total_msix < hwr.cp) {
13855 netdev_warn(bp->dev, "Unable to allocate %d MSIX vectors, maximum available %d\n",
13856 hwr.cp, total_msix);
13857 rc = -ENOSPC;
13858 }
13859 }
13860 }
13861 return rc;
13862 }
13863
bnxt_unmap_bars(struct bnxt * bp,struct pci_dev * pdev)13864 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
13865 {
13866 if (bp->bar2) {
13867 pci_iounmap(pdev, bp->bar2);
13868 bp->bar2 = NULL;
13869 }
13870
13871 if (bp->bar1) {
13872 pci_iounmap(pdev, bp->bar1);
13873 bp->bar1 = NULL;
13874 }
13875
13876 if (bp->bar0) {
13877 pci_iounmap(pdev, bp->bar0);
13878 bp->bar0 = NULL;
13879 }
13880 }
13881
bnxt_cleanup_pci(struct bnxt * bp)13882 static void bnxt_cleanup_pci(struct bnxt *bp)
13883 {
13884 bnxt_unmap_bars(bp, bp->pdev);
13885 pci_release_regions(bp->pdev);
13886 if (pci_is_enabled(bp->pdev))
13887 pci_disable_device(bp->pdev);
13888 }
13889
bnxt_init_dflt_coal(struct bnxt * bp)13890 static void bnxt_init_dflt_coal(struct bnxt *bp)
13891 {
13892 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
13893 struct bnxt_coal *coal;
13894 u16 flags = 0;
13895
13896 if (coal_cap->cmpl_params &
13897 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
13898 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
13899
13900 /* Tick values in micro seconds.
13901 * 1 coal_buf x bufs_per_record = 1 completion record.
13902 */
13903 coal = &bp->rx_coal;
13904 coal->coal_ticks = 10;
13905 coal->coal_bufs = 30;
13906 coal->coal_ticks_irq = 1;
13907 coal->coal_bufs_irq = 2;
13908 coal->idle_thresh = 50;
13909 coal->bufs_per_record = 2;
13910 coal->budget = 64; /* NAPI budget */
13911 coal->flags = flags;
13912
13913 coal = &bp->tx_coal;
13914 coal->coal_ticks = 28;
13915 coal->coal_bufs = 30;
13916 coal->coal_ticks_irq = 2;
13917 coal->coal_bufs_irq = 2;
13918 coal->bufs_per_record = 1;
13919 coal->flags = flags;
13920
13921 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
13922 }
13923
13924 /* FW that pre-reserves 1 VNIC per function */
bnxt_fw_pre_resv_vnics(struct bnxt * bp)13925 static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp)
13926 {
13927 u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp);
13928
13929 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13930 (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18)))
13931 return true;
13932 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13933 (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172)))
13934 return true;
13935 return false;
13936 }
13937
bnxt_fw_init_one_p1(struct bnxt * bp)13938 static int bnxt_fw_init_one_p1(struct bnxt *bp)
13939 {
13940 int rc;
13941
13942 bp->fw_cap = 0;
13943 rc = bnxt_hwrm_ver_get(bp);
13944 /* FW may be unresponsive after FLR. FLR must complete within 100 msec
13945 * so wait before continuing with recovery.
13946 */
13947 if (rc)
13948 msleep(100);
13949 bnxt_try_map_fw_health_reg(bp);
13950 if (rc) {
13951 rc = bnxt_try_recover_fw(bp);
13952 if (rc)
13953 return rc;
13954 rc = bnxt_hwrm_ver_get(bp);
13955 if (rc)
13956 return rc;
13957 }
13958
13959 bnxt_nvm_cfg_ver_get(bp);
13960
13961 rc = bnxt_hwrm_func_reset(bp);
13962 if (rc)
13963 return -ENODEV;
13964
13965 bnxt_hwrm_fw_set_time(bp);
13966 return 0;
13967 }
13968
bnxt_fw_init_one_p2(struct bnxt * bp)13969 static int bnxt_fw_init_one_p2(struct bnxt *bp)
13970 {
13971 int rc;
13972
13973 /* Get the MAX capabilities for this function */
13974 rc = bnxt_hwrm_func_qcaps(bp);
13975 if (rc) {
13976 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
13977 rc);
13978 return -ENODEV;
13979 }
13980
13981 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
13982 if (rc)
13983 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
13984 rc);
13985
13986 if (bnxt_alloc_fw_health(bp)) {
13987 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
13988 } else {
13989 rc = bnxt_hwrm_error_recovery_qcfg(bp);
13990 if (rc)
13991 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
13992 rc);
13993 }
13994
13995 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
13996 if (rc)
13997 return -ENODEV;
13998
13999 rc = bnxt_alloc_crash_dump_mem(bp);
14000 if (rc)
14001 netdev_warn(bp->dev, "crash dump mem alloc failure rc: %d\n",
14002 rc);
14003 if (!rc) {
14004 rc = bnxt_hwrm_crash_dump_mem_cfg(bp);
14005 if (rc) {
14006 bnxt_free_crash_dump_mem(bp);
14007 netdev_warn(bp->dev,
14008 "hwrm crash dump mem failure rc: %d\n", rc);
14009 }
14010 }
14011
14012 if (bnxt_fw_pre_resv_vnics(bp))
14013 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS;
14014
14015 bnxt_hwrm_func_qcfg(bp);
14016 bnxt_hwrm_vnic_qcaps(bp);
14017 bnxt_hwrm_port_led_qcaps(bp);
14018 bnxt_ethtool_init(bp);
14019 if (bp->fw_cap & BNXT_FW_CAP_PTP)
14020 __bnxt_hwrm_ptp_qcfg(bp);
14021 bnxt_dcb_init(bp);
14022 bnxt_hwmon_init(bp);
14023 return 0;
14024 }
14025
bnxt_set_dflt_rss_hash_type(struct bnxt * bp)14026 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
14027 {
14028 bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP;
14029 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
14030 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
14031 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
14032 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
14033 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
14034 bp->rss_hash_delta = bp->rss_hash_cfg;
14035 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
14036 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP;
14037 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
14038 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
14039 }
14040 }
14041
bnxt_set_dflt_rfs(struct bnxt * bp)14042 static void bnxt_set_dflt_rfs(struct bnxt *bp)
14043 {
14044 struct net_device *dev = bp->dev;
14045
14046 dev->hw_features &= ~NETIF_F_NTUPLE;
14047 dev->features &= ~NETIF_F_NTUPLE;
14048 bp->flags &= ~BNXT_FLAG_RFS;
14049 if (bnxt_rfs_supported(bp)) {
14050 dev->hw_features |= NETIF_F_NTUPLE;
14051 if (bnxt_rfs_capable(bp, false)) {
14052 bp->flags |= BNXT_FLAG_RFS;
14053 dev->features |= NETIF_F_NTUPLE;
14054 }
14055 }
14056 }
14057
bnxt_fw_init_one_p3(struct bnxt * bp)14058 static void bnxt_fw_init_one_p3(struct bnxt *bp)
14059 {
14060 struct pci_dev *pdev = bp->pdev;
14061
14062 bnxt_set_dflt_rss_hash_type(bp);
14063 bnxt_set_dflt_rfs(bp);
14064
14065 bnxt_get_wol_settings(bp);
14066 if (bp->flags & BNXT_FLAG_WOL_CAP)
14067 device_set_wakeup_enable(&pdev->dev, bp->wol);
14068 else
14069 device_set_wakeup_capable(&pdev->dev, false);
14070
14071 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
14072 bnxt_hwrm_coal_params_qcaps(bp);
14073 }
14074
14075 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
14076
bnxt_fw_init_one(struct bnxt * bp)14077 int bnxt_fw_init_one(struct bnxt *bp)
14078 {
14079 int rc;
14080
14081 rc = bnxt_fw_init_one_p1(bp);
14082 if (rc) {
14083 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
14084 return rc;
14085 }
14086 rc = bnxt_fw_init_one_p2(bp);
14087 if (rc) {
14088 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
14089 return rc;
14090 }
14091 rc = bnxt_probe_phy(bp, false);
14092 if (rc)
14093 return rc;
14094 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
14095 if (rc)
14096 return rc;
14097
14098 bnxt_fw_init_one_p3(bp);
14099 return 0;
14100 }
14101
bnxt_fw_reset_writel(struct bnxt * bp,int reg_idx)14102 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
14103 {
14104 struct bnxt_fw_health *fw_health = bp->fw_health;
14105 u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
14106 u32 val = fw_health->fw_reset_seq_vals[reg_idx];
14107 u32 reg_type, reg_off, delay_msecs;
14108
14109 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
14110 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
14111 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
14112 switch (reg_type) {
14113 case BNXT_FW_HEALTH_REG_TYPE_CFG:
14114 pci_write_config_dword(bp->pdev, reg_off, val);
14115 break;
14116 case BNXT_FW_HEALTH_REG_TYPE_GRC:
14117 writel(reg_off & BNXT_GRC_BASE_MASK,
14118 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
14119 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
14120 fallthrough;
14121 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
14122 writel(val, bp->bar0 + reg_off);
14123 break;
14124 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
14125 writel(val, bp->bar1 + reg_off);
14126 break;
14127 }
14128 if (delay_msecs) {
14129 pci_read_config_dword(bp->pdev, 0, &val);
14130 msleep(delay_msecs);
14131 }
14132 }
14133
bnxt_hwrm_reset_permitted(struct bnxt * bp)14134 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
14135 {
14136 struct hwrm_func_qcfg_output *resp;
14137 struct hwrm_func_qcfg_input *req;
14138 bool result = true; /* firmware will enforce if unknown */
14139
14140 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
14141 return result;
14142
14143 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
14144 return result;
14145
14146 req->fid = cpu_to_le16(0xffff);
14147 resp = hwrm_req_hold(bp, req);
14148 if (!hwrm_req_send(bp, req))
14149 result = !!(le16_to_cpu(resp->flags) &
14150 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
14151 hwrm_req_drop(bp, req);
14152 return result;
14153 }
14154
bnxt_reset_all(struct bnxt * bp)14155 static void bnxt_reset_all(struct bnxt *bp)
14156 {
14157 struct bnxt_fw_health *fw_health = bp->fw_health;
14158 int i, rc;
14159
14160 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
14161 bnxt_fw_reset_via_optee(bp);
14162 bp->fw_reset_timestamp = jiffies;
14163 return;
14164 }
14165
14166 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
14167 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
14168 bnxt_fw_reset_writel(bp, i);
14169 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
14170 struct hwrm_fw_reset_input *req;
14171
14172 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
14173 if (!rc) {
14174 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
14175 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
14176 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
14177 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
14178 rc = hwrm_req_send(bp, req);
14179 }
14180 if (rc != -ENODEV)
14181 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
14182 }
14183 bp->fw_reset_timestamp = jiffies;
14184 }
14185
bnxt_fw_reset_timeout(struct bnxt * bp)14186 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
14187 {
14188 return time_after(jiffies, bp->fw_reset_timestamp +
14189 (bp->fw_reset_max_dsecs * HZ / 10));
14190 }
14191
bnxt_fw_reset_abort(struct bnxt * bp,int rc)14192 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
14193 {
14194 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14195 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
14196 bnxt_dl_health_fw_status_update(bp, false);
14197 bp->fw_reset_state = 0;
14198 dev_close(bp->dev);
14199 }
14200
bnxt_fw_reset_task(struct work_struct * work)14201 static void bnxt_fw_reset_task(struct work_struct *work)
14202 {
14203 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
14204 int rc = 0;
14205
14206 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
14207 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
14208 return;
14209 }
14210
14211 switch (bp->fw_reset_state) {
14212 case BNXT_FW_RESET_STATE_POLL_VF: {
14213 int n = bnxt_get_registered_vfs(bp);
14214 int tmo;
14215
14216 if (n < 0) {
14217 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
14218 n, jiffies_to_msecs(jiffies -
14219 bp->fw_reset_timestamp));
14220 goto fw_reset_abort;
14221 } else if (n > 0) {
14222 if (bnxt_fw_reset_timeout(bp)) {
14223 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14224 bp->fw_reset_state = 0;
14225 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
14226 n);
14227 goto ulp_start;
14228 }
14229 bnxt_queue_fw_reset_work(bp, HZ / 10);
14230 return;
14231 }
14232 bp->fw_reset_timestamp = jiffies;
14233 rtnl_lock();
14234 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
14235 bnxt_fw_reset_abort(bp, rc);
14236 rtnl_unlock();
14237 goto ulp_start;
14238 }
14239 bnxt_fw_reset_close(bp);
14240 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
14241 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
14242 tmo = HZ / 10;
14243 } else {
14244 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14245 tmo = bp->fw_reset_min_dsecs * HZ / 10;
14246 }
14247 rtnl_unlock();
14248 bnxt_queue_fw_reset_work(bp, tmo);
14249 return;
14250 }
14251 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
14252 u32 val;
14253
14254 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14255 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
14256 !bnxt_fw_reset_timeout(bp)) {
14257 bnxt_queue_fw_reset_work(bp, HZ / 5);
14258 return;
14259 }
14260
14261 if (!bp->fw_health->primary) {
14262 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
14263
14264 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14265 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
14266 return;
14267 }
14268 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
14269 }
14270 fallthrough;
14271 case BNXT_FW_RESET_STATE_RESET_FW:
14272 bnxt_reset_all(bp);
14273 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14274 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
14275 return;
14276 case BNXT_FW_RESET_STATE_ENABLE_DEV:
14277 bnxt_inv_fw_health_reg(bp);
14278 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
14279 !bp->fw_reset_min_dsecs) {
14280 u16 val;
14281
14282 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
14283 if (val == 0xffff) {
14284 if (bnxt_fw_reset_timeout(bp)) {
14285 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
14286 rc = -ETIMEDOUT;
14287 goto fw_reset_abort;
14288 }
14289 bnxt_queue_fw_reset_work(bp, HZ / 1000);
14290 return;
14291 }
14292 }
14293 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
14294 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
14295 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
14296 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
14297 bnxt_dl_remote_reload(bp);
14298 if (pci_enable_device(bp->pdev)) {
14299 netdev_err(bp->dev, "Cannot re-enable PCI device\n");
14300 rc = -ENODEV;
14301 goto fw_reset_abort;
14302 }
14303 pci_set_master(bp->pdev);
14304 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
14305 fallthrough;
14306 case BNXT_FW_RESET_STATE_POLL_FW:
14307 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
14308 rc = bnxt_hwrm_poll(bp);
14309 if (rc) {
14310 if (bnxt_fw_reset_timeout(bp)) {
14311 netdev_err(bp->dev, "Firmware reset aborted\n");
14312 goto fw_reset_abort_status;
14313 }
14314 bnxt_queue_fw_reset_work(bp, HZ / 5);
14315 return;
14316 }
14317 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
14318 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
14319 fallthrough;
14320 case BNXT_FW_RESET_STATE_OPENING:
14321 while (!rtnl_trylock()) {
14322 bnxt_queue_fw_reset_work(bp, HZ / 10);
14323 return;
14324 }
14325 rc = bnxt_open(bp->dev);
14326 if (rc) {
14327 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
14328 bnxt_fw_reset_abort(bp, rc);
14329 rtnl_unlock();
14330 goto ulp_start;
14331 }
14332
14333 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
14334 bp->fw_health->enabled) {
14335 bp->fw_health->last_fw_reset_cnt =
14336 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
14337 }
14338 bp->fw_reset_state = 0;
14339 /* Make sure fw_reset_state is 0 before clearing the flag */
14340 smp_mb__before_atomic();
14341 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14342 bnxt_ptp_reapply_pps(bp);
14343 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
14344 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
14345 bnxt_dl_health_fw_recovery_done(bp);
14346 bnxt_dl_health_fw_status_update(bp, true);
14347 }
14348 rtnl_unlock();
14349 bnxt_ulp_start(bp, 0);
14350 bnxt_reenable_sriov(bp);
14351 rtnl_lock();
14352 bnxt_vf_reps_alloc(bp);
14353 bnxt_vf_reps_open(bp);
14354 rtnl_unlock();
14355 break;
14356 }
14357 return;
14358
14359 fw_reset_abort_status:
14360 if (bp->fw_health->status_reliable ||
14361 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
14362 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14363
14364 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
14365 }
14366 fw_reset_abort:
14367 rtnl_lock();
14368 bnxt_fw_reset_abort(bp, rc);
14369 rtnl_unlock();
14370 ulp_start:
14371 bnxt_ulp_start(bp, rc);
14372 }
14373
bnxt_init_board(struct pci_dev * pdev,struct net_device * dev)14374 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
14375 {
14376 int rc;
14377 struct bnxt *bp = netdev_priv(dev);
14378
14379 SET_NETDEV_DEV(dev, &pdev->dev);
14380
14381 /* enable device (incl. PCI PM wakeup), and bus-mastering */
14382 rc = pci_enable_device(pdev);
14383 if (rc) {
14384 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
14385 goto init_err;
14386 }
14387
14388 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
14389 dev_err(&pdev->dev,
14390 "Cannot find PCI device base address, aborting\n");
14391 rc = -ENODEV;
14392 goto init_err_disable;
14393 }
14394
14395 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
14396 if (rc) {
14397 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
14398 goto init_err_disable;
14399 }
14400
14401 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
14402 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
14403 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
14404 rc = -EIO;
14405 goto init_err_release;
14406 }
14407
14408 pci_set_master(pdev);
14409
14410 bp->dev = dev;
14411 bp->pdev = pdev;
14412
14413 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
14414 * determines the BAR size.
14415 */
14416 bp->bar0 = pci_ioremap_bar(pdev, 0);
14417 if (!bp->bar0) {
14418 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
14419 rc = -ENOMEM;
14420 goto init_err_release;
14421 }
14422
14423 bp->bar2 = pci_ioremap_bar(pdev, 4);
14424 if (!bp->bar2) {
14425 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
14426 rc = -ENOMEM;
14427 goto init_err_release;
14428 }
14429
14430 INIT_WORK(&bp->sp_task, bnxt_sp_task);
14431 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
14432
14433 spin_lock_init(&bp->ntp_fltr_lock);
14434 #if BITS_PER_LONG == 32
14435 spin_lock_init(&bp->db_lock);
14436 #endif
14437
14438 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
14439 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
14440
14441 timer_setup(&bp->timer, bnxt_timer, 0);
14442 bp->current_interval = BNXT_TIMER_INTERVAL;
14443
14444 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
14445 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
14446
14447 clear_bit(BNXT_STATE_OPEN, &bp->state);
14448 return 0;
14449
14450 init_err_release:
14451 bnxt_unmap_bars(bp, pdev);
14452 pci_release_regions(pdev);
14453
14454 init_err_disable:
14455 pci_disable_device(pdev);
14456
14457 init_err:
14458 return rc;
14459 }
14460
14461 /* rtnl_lock held */
bnxt_change_mac_addr(struct net_device * dev,void * p)14462 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
14463 {
14464 struct sockaddr *addr = p;
14465 struct bnxt *bp = netdev_priv(dev);
14466 int rc = 0;
14467
14468 if (!is_valid_ether_addr(addr->sa_data))
14469 return -EADDRNOTAVAIL;
14470
14471 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
14472 return 0;
14473
14474 rc = bnxt_approve_mac(bp, addr->sa_data, true);
14475 if (rc)
14476 return rc;
14477
14478 eth_hw_addr_set(dev, addr->sa_data);
14479 bnxt_clear_usr_fltrs(bp, true);
14480 if (netif_running(dev)) {
14481 bnxt_close_nic(bp, false, false);
14482 rc = bnxt_open_nic(bp, false, false);
14483 }
14484
14485 return rc;
14486 }
14487
14488 /* rtnl_lock held */
bnxt_change_mtu(struct net_device * dev,int new_mtu)14489 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
14490 {
14491 struct bnxt *bp = netdev_priv(dev);
14492
14493 if (netif_running(dev))
14494 bnxt_close_nic(bp, true, false);
14495
14496 WRITE_ONCE(dev->mtu, new_mtu);
14497 bnxt_set_ring_params(bp);
14498
14499 if (netif_running(dev))
14500 return bnxt_open_nic(bp, true, false);
14501
14502 return 0;
14503 }
14504
bnxt_setup_mq_tc(struct net_device * dev,u8 tc)14505 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
14506 {
14507 struct bnxt *bp = netdev_priv(dev);
14508 bool sh = false;
14509 int rc, tx_cp;
14510
14511 if (tc > bp->max_tc) {
14512 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
14513 tc, bp->max_tc);
14514 return -EINVAL;
14515 }
14516
14517 if (bp->num_tc == tc)
14518 return 0;
14519
14520 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
14521 sh = true;
14522
14523 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
14524 sh, tc, bp->tx_nr_rings_xdp);
14525 if (rc)
14526 return rc;
14527
14528 /* Needs to close the device and do hw resource re-allocations */
14529 if (netif_running(bp->dev))
14530 bnxt_close_nic(bp, true, false);
14531
14532 if (tc) {
14533 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
14534 netdev_set_num_tc(dev, tc);
14535 bp->num_tc = tc;
14536 } else {
14537 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14538 netdev_reset_tc(dev);
14539 bp->num_tc = 0;
14540 }
14541 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
14542 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
14543 bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
14544 tx_cp + bp->rx_nr_rings;
14545
14546 if (netif_running(bp->dev))
14547 return bnxt_open_nic(bp, true, false);
14548
14549 return 0;
14550 }
14551
bnxt_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)14552 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
14553 void *cb_priv)
14554 {
14555 struct bnxt *bp = cb_priv;
14556
14557 if (!bnxt_tc_flower_enabled(bp) ||
14558 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
14559 return -EOPNOTSUPP;
14560
14561 switch (type) {
14562 case TC_SETUP_CLSFLOWER:
14563 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
14564 default:
14565 return -EOPNOTSUPP;
14566 }
14567 }
14568
14569 LIST_HEAD(bnxt_block_cb_list);
14570
bnxt_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)14571 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
14572 void *type_data)
14573 {
14574 struct bnxt *bp = netdev_priv(dev);
14575
14576 switch (type) {
14577 case TC_SETUP_BLOCK:
14578 return flow_block_cb_setup_simple(type_data,
14579 &bnxt_block_cb_list,
14580 bnxt_setup_tc_block_cb,
14581 bp, bp, true);
14582 case TC_SETUP_QDISC_MQPRIO: {
14583 struct tc_mqprio_qopt *mqprio = type_data;
14584
14585 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
14586
14587 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
14588 }
14589 default:
14590 return -EOPNOTSUPP;
14591 }
14592 }
14593
bnxt_get_ntp_filter_idx(struct bnxt * bp,struct flow_keys * fkeys,const struct sk_buff * skb)14594 u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys,
14595 const struct sk_buff *skb)
14596 {
14597 struct bnxt_vnic_info *vnic;
14598
14599 if (skb)
14600 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
14601
14602 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
14603 return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key);
14604 }
14605
bnxt_insert_ntp_filter(struct bnxt * bp,struct bnxt_ntuple_filter * fltr,u32 idx)14606 int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr,
14607 u32 idx)
14608 {
14609 struct hlist_head *head;
14610 int bit_id;
14611
14612 spin_lock_bh(&bp->ntp_fltr_lock);
14613 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, bp->max_fltr, 0);
14614 if (bit_id < 0) {
14615 spin_unlock_bh(&bp->ntp_fltr_lock);
14616 return -ENOMEM;
14617 }
14618
14619 fltr->base.sw_id = (u16)bit_id;
14620 fltr->base.type = BNXT_FLTR_TYPE_NTUPLE;
14621 fltr->base.flags |= BNXT_ACT_RING_DST;
14622 head = &bp->ntp_fltr_hash_tbl[idx];
14623 hlist_add_head_rcu(&fltr->base.hash, head);
14624 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
14625 bnxt_insert_usr_fltr(bp, &fltr->base);
14626 bp->ntp_fltr_count++;
14627 spin_unlock_bh(&bp->ntp_fltr_lock);
14628 return 0;
14629 }
14630
bnxt_fltr_match(struct bnxt_ntuple_filter * f1,struct bnxt_ntuple_filter * f2)14631 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
14632 struct bnxt_ntuple_filter *f2)
14633 {
14634 struct bnxt_flow_masks *masks1 = &f1->fmasks;
14635 struct bnxt_flow_masks *masks2 = &f2->fmasks;
14636 struct flow_keys *keys1 = &f1->fkeys;
14637 struct flow_keys *keys2 = &f2->fkeys;
14638
14639 if (keys1->basic.n_proto != keys2->basic.n_proto ||
14640 keys1->basic.ip_proto != keys2->basic.ip_proto)
14641 return false;
14642
14643 if (keys1->basic.n_proto == htons(ETH_P_IP)) {
14644 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
14645 masks1->addrs.v4addrs.src != masks2->addrs.v4addrs.src ||
14646 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst ||
14647 masks1->addrs.v4addrs.dst != masks2->addrs.v4addrs.dst)
14648 return false;
14649 } else {
14650 if (!ipv6_addr_equal(&keys1->addrs.v6addrs.src,
14651 &keys2->addrs.v6addrs.src) ||
14652 !ipv6_addr_equal(&masks1->addrs.v6addrs.src,
14653 &masks2->addrs.v6addrs.src) ||
14654 !ipv6_addr_equal(&keys1->addrs.v6addrs.dst,
14655 &keys2->addrs.v6addrs.dst) ||
14656 !ipv6_addr_equal(&masks1->addrs.v6addrs.dst,
14657 &masks2->addrs.v6addrs.dst))
14658 return false;
14659 }
14660
14661 return keys1->ports.src == keys2->ports.src &&
14662 masks1->ports.src == masks2->ports.src &&
14663 keys1->ports.dst == keys2->ports.dst &&
14664 masks1->ports.dst == masks2->ports.dst &&
14665 keys1->control.flags == keys2->control.flags &&
14666 f1->l2_fltr == f2->l2_fltr;
14667 }
14668
14669 struct bnxt_ntuple_filter *
bnxt_lookup_ntp_filter_from_idx(struct bnxt * bp,struct bnxt_ntuple_filter * fltr,u32 idx)14670 bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp,
14671 struct bnxt_ntuple_filter *fltr, u32 idx)
14672 {
14673 struct bnxt_ntuple_filter *f;
14674 struct hlist_head *head;
14675
14676 head = &bp->ntp_fltr_hash_tbl[idx];
14677 hlist_for_each_entry_rcu(f, head, base.hash) {
14678 if (bnxt_fltr_match(f, fltr))
14679 return f;
14680 }
14681 return NULL;
14682 }
14683
14684 #ifdef CONFIG_RFS_ACCEL
bnxt_rx_flow_steer(struct net_device * dev,const struct sk_buff * skb,u16 rxq_index,u32 flow_id)14685 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
14686 u16 rxq_index, u32 flow_id)
14687 {
14688 struct bnxt *bp = netdev_priv(dev);
14689 struct bnxt_ntuple_filter *fltr, *new_fltr;
14690 struct flow_keys *fkeys;
14691 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
14692 struct bnxt_l2_filter *l2_fltr;
14693 int rc = 0, idx;
14694 u32 flags;
14695
14696 if (ether_addr_equal(dev->dev_addr, eth->h_dest)) {
14697 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
14698 atomic_inc(&l2_fltr->refcnt);
14699 } else {
14700 struct bnxt_l2_key key;
14701
14702 ether_addr_copy(key.dst_mac_addr, eth->h_dest);
14703 key.vlan = 0;
14704 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key);
14705 if (!l2_fltr)
14706 return -EINVAL;
14707 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) {
14708 bnxt_del_l2_filter(bp, l2_fltr);
14709 return -EINVAL;
14710 }
14711 }
14712 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
14713 if (!new_fltr) {
14714 bnxt_del_l2_filter(bp, l2_fltr);
14715 return -ENOMEM;
14716 }
14717
14718 fkeys = &new_fltr->fkeys;
14719 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
14720 rc = -EPROTONOSUPPORT;
14721 goto err_free;
14722 }
14723
14724 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
14725 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
14726 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
14727 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
14728 rc = -EPROTONOSUPPORT;
14729 goto err_free;
14730 }
14731 new_fltr->fmasks = BNXT_FLOW_IPV4_MASK_ALL;
14732 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
14733 if (bp->hwrm_spec_code < 0x10601) {
14734 rc = -EPROTONOSUPPORT;
14735 goto err_free;
14736 }
14737 new_fltr->fmasks = BNXT_FLOW_IPV6_MASK_ALL;
14738 }
14739 flags = fkeys->control.flags;
14740 if (((flags & FLOW_DIS_ENCAPSULATION) &&
14741 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
14742 rc = -EPROTONOSUPPORT;
14743 goto err_free;
14744 }
14745 new_fltr->l2_fltr = l2_fltr;
14746
14747 idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb);
14748 rcu_read_lock();
14749 fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx);
14750 if (fltr) {
14751 rc = fltr->base.sw_id;
14752 rcu_read_unlock();
14753 goto err_free;
14754 }
14755 rcu_read_unlock();
14756
14757 new_fltr->flow_id = flow_id;
14758 new_fltr->base.rxq = rxq_index;
14759 rc = bnxt_insert_ntp_filter(bp, new_fltr, idx);
14760 if (!rc) {
14761 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
14762 return new_fltr->base.sw_id;
14763 }
14764
14765 err_free:
14766 bnxt_del_l2_filter(bp, l2_fltr);
14767 kfree(new_fltr);
14768 return rc;
14769 }
14770 #endif
14771
bnxt_del_ntp_filter(struct bnxt * bp,struct bnxt_ntuple_filter * fltr)14772 void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr)
14773 {
14774 spin_lock_bh(&bp->ntp_fltr_lock);
14775 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
14776 spin_unlock_bh(&bp->ntp_fltr_lock);
14777 return;
14778 }
14779 hlist_del_rcu(&fltr->base.hash);
14780 bnxt_del_one_usr_fltr(bp, &fltr->base);
14781 bp->ntp_fltr_count--;
14782 spin_unlock_bh(&bp->ntp_fltr_lock);
14783 bnxt_del_l2_filter(bp, fltr->l2_fltr);
14784 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
14785 kfree_rcu(fltr, base.rcu);
14786 }
14787
bnxt_cfg_ntp_filters(struct bnxt * bp)14788 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
14789 {
14790 #ifdef CONFIG_RFS_ACCEL
14791 int i;
14792
14793 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
14794 struct hlist_head *head;
14795 struct hlist_node *tmp;
14796 struct bnxt_ntuple_filter *fltr;
14797 int rc;
14798
14799 head = &bp->ntp_fltr_hash_tbl[i];
14800 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
14801 bool del = false;
14802
14803 if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) {
14804 if (fltr->base.flags & BNXT_ACT_NO_AGING)
14805 continue;
14806 if (rps_may_expire_flow(bp->dev, fltr->base.rxq,
14807 fltr->flow_id,
14808 fltr->base.sw_id)) {
14809 bnxt_hwrm_cfa_ntuple_filter_free(bp,
14810 fltr);
14811 del = true;
14812 }
14813 } else {
14814 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
14815 fltr);
14816 if (rc)
14817 del = true;
14818 else
14819 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
14820 }
14821
14822 if (del)
14823 bnxt_del_ntp_filter(bp, fltr);
14824 }
14825 }
14826 #endif
14827 }
14828
bnxt_udp_tunnel_set_port(struct net_device * netdev,unsigned int table,unsigned int entry,struct udp_tunnel_info * ti)14829 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
14830 unsigned int entry, struct udp_tunnel_info *ti)
14831 {
14832 struct bnxt *bp = netdev_priv(netdev);
14833 unsigned int cmd;
14834
14835 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14836 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
14837 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14838 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE;
14839 else
14840 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE;
14841
14842 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
14843 }
14844
bnxt_udp_tunnel_unset_port(struct net_device * netdev,unsigned int table,unsigned int entry,struct udp_tunnel_info * ti)14845 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
14846 unsigned int entry, struct udp_tunnel_info *ti)
14847 {
14848 struct bnxt *bp = netdev_priv(netdev);
14849 unsigned int cmd;
14850
14851 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14852 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
14853 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14854 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
14855 else
14856 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE;
14857
14858 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
14859 }
14860
14861 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
14862 .set_port = bnxt_udp_tunnel_set_port,
14863 .unset_port = bnxt_udp_tunnel_unset_port,
14864 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14865 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14866 .tables = {
14867 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
14868 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14869 },
14870 }, bnxt_udp_tunnels_p7 = {
14871 .set_port = bnxt_udp_tunnel_set_port,
14872 .unset_port = bnxt_udp_tunnel_unset_port,
14873 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14874 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14875 .tables = {
14876 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
14877 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14878 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, },
14879 },
14880 };
14881
bnxt_bridge_getlink(struct sk_buff * skb,u32 pid,u32 seq,struct net_device * dev,u32 filter_mask,int nlflags)14882 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
14883 struct net_device *dev, u32 filter_mask,
14884 int nlflags)
14885 {
14886 struct bnxt *bp = netdev_priv(dev);
14887
14888 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
14889 nlflags, filter_mask, NULL);
14890 }
14891
bnxt_bridge_setlink(struct net_device * dev,struct nlmsghdr * nlh,u16 flags,struct netlink_ext_ack * extack)14892 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
14893 u16 flags, struct netlink_ext_ack *extack)
14894 {
14895 struct bnxt *bp = netdev_priv(dev);
14896 struct nlattr *attr, *br_spec;
14897 int rem, rc = 0;
14898
14899 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
14900 return -EOPNOTSUPP;
14901
14902 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
14903 if (!br_spec)
14904 return -EINVAL;
14905
14906 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
14907 u16 mode;
14908
14909 mode = nla_get_u16(attr);
14910 if (mode == bp->br_mode)
14911 break;
14912
14913 rc = bnxt_hwrm_set_br_mode(bp, mode);
14914 if (!rc)
14915 bp->br_mode = mode;
14916 break;
14917 }
14918 return rc;
14919 }
14920
bnxt_get_port_parent_id(struct net_device * dev,struct netdev_phys_item_id * ppid)14921 int bnxt_get_port_parent_id(struct net_device *dev,
14922 struct netdev_phys_item_id *ppid)
14923 {
14924 struct bnxt *bp = netdev_priv(dev);
14925
14926 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
14927 return -EOPNOTSUPP;
14928
14929 /* The PF and it's VF-reps only support the switchdev framework */
14930 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
14931 return -EOPNOTSUPP;
14932
14933 ppid->id_len = sizeof(bp->dsn);
14934 memcpy(ppid->id, bp->dsn, ppid->id_len);
14935
14936 return 0;
14937 }
14938
14939 static const struct net_device_ops bnxt_netdev_ops = {
14940 .ndo_open = bnxt_open,
14941 .ndo_start_xmit = bnxt_start_xmit,
14942 .ndo_stop = bnxt_close,
14943 .ndo_get_stats64 = bnxt_get_stats64,
14944 .ndo_set_rx_mode = bnxt_set_rx_mode,
14945 .ndo_eth_ioctl = bnxt_ioctl,
14946 .ndo_validate_addr = eth_validate_addr,
14947 .ndo_set_mac_address = bnxt_change_mac_addr,
14948 .ndo_change_mtu = bnxt_change_mtu,
14949 .ndo_fix_features = bnxt_fix_features,
14950 .ndo_set_features = bnxt_set_features,
14951 .ndo_features_check = bnxt_features_check,
14952 .ndo_tx_timeout = bnxt_tx_timeout,
14953 #ifdef CONFIG_BNXT_SRIOV
14954 .ndo_get_vf_config = bnxt_get_vf_config,
14955 .ndo_set_vf_mac = bnxt_set_vf_mac,
14956 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
14957 .ndo_set_vf_rate = bnxt_set_vf_bw,
14958 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
14959 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
14960 .ndo_set_vf_trust = bnxt_set_vf_trust,
14961 #endif
14962 .ndo_setup_tc = bnxt_setup_tc,
14963 #ifdef CONFIG_RFS_ACCEL
14964 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
14965 #endif
14966 .ndo_bpf = bnxt_xdp,
14967 .ndo_xdp_xmit = bnxt_xdp_xmit,
14968 .ndo_bridge_getlink = bnxt_bridge_getlink,
14969 .ndo_bridge_setlink = bnxt_bridge_setlink,
14970 };
14971
bnxt_get_queue_stats_rx(struct net_device * dev,int i,struct netdev_queue_stats_rx * stats)14972 static void bnxt_get_queue_stats_rx(struct net_device *dev, int i,
14973 struct netdev_queue_stats_rx *stats)
14974 {
14975 struct bnxt *bp = netdev_priv(dev);
14976 struct bnxt_cp_ring_info *cpr;
14977 u64 *sw;
14978
14979 cpr = &bp->bnapi[i]->cp_ring;
14980 sw = cpr->stats.sw_stats;
14981
14982 stats->packets = 0;
14983 stats->packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
14984 stats->packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
14985 stats->packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
14986
14987 stats->bytes = 0;
14988 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
14989 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
14990 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
14991
14992 stats->alloc_fail = cpr->sw_stats->rx.rx_oom_discards;
14993 }
14994
bnxt_get_queue_stats_tx(struct net_device * dev,int i,struct netdev_queue_stats_tx * stats)14995 static void bnxt_get_queue_stats_tx(struct net_device *dev, int i,
14996 struct netdev_queue_stats_tx *stats)
14997 {
14998 struct bnxt *bp = netdev_priv(dev);
14999 struct bnxt_napi *bnapi;
15000 u64 *sw;
15001
15002 bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi;
15003 sw = bnapi->cp_ring.stats.sw_stats;
15004
15005 stats->packets = 0;
15006 stats->packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
15007 stats->packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
15008 stats->packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
15009
15010 stats->bytes = 0;
15011 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
15012 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
15013 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
15014 }
15015
bnxt_get_base_stats(struct net_device * dev,struct netdev_queue_stats_rx * rx,struct netdev_queue_stats_tx * tx)15016 static void bnxt_get_base_stats(struct net_device *dev,
15017 struct netdev_queue_stats_rx *rx,
15018 struct netdev_queue_stats_tx *tx)
15019 {
15020 struct bnxt *bp = netdev_priv(dev);
15021
15022 rx->packets = bp->net_stats_prev.rx_packets;
15023 rx->bytes = bp->net_stats_prev.rx_bytes;
15024 rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards;
15025
15026 tx->packets = bp->net_stats_prev.tx_packets;
15027 tx->bytes = bp->net_stats_prev.tx_bytes;
15028 }
15029
15030 static const struct netdev_stat_ops bnxt_stat_ops = {
15031 .get_queue_stats_rx = bnxt_get_queue_stats_rx,
15032 .get_queue_stats_tx = bnxt_get_queue_stats_tx,
15033 .get_base_stats = bnxt_get_base_stats,
15034 };
15035
bnxt_alloc_rx_agg_bmap(struct bnxt * bp,struct bnxt_rx_ring_info * rxr)15036 static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
15037 {
15038 u16 mem_size;
15039
15040 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
15041 mem_size = rxr->rx_agg_bmap_size / 8;
15042 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
15043 if (!rxr->rx_agg_bmap)
15044 return -ENOMEM;
15045
15046 return 0;
15047 }
15048
bnxt_queue_mem_alloc(struct net_device * dev,void * qmem,int idx)15049 static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx)
15050 {
15051 struct bnxt_rx_ring_info *rxr, *clone;
15052 struct bnxt *bp = netdev_priv(dev);
15053 struct bnxt_ring_struct *ring;
15054 int rc;
15055
15056 rxr = &bp->rx_ring[idx];
15057 clone = qmem;
15058 memcpy(clone, rxr, sizeof(*rxr));
15059 bnxt_init_rx_ring_struct(bp, clone);
15060 bnxt_reset_rx_ring_struct(bp, clone);
15061
15062 clone->rx_prod = 0;
15063 clone->rx_agg_prod = 0;
15064 clone->rx_sw_agg_prod = 0;
15065 clone->rx_next_cons = 0;
15066
15067 rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid);
15068 if (rc)
15069 return rc;
15070
15071 rc = xdp_rxq_info_reg(&clone->xdp_rxq, bp->dev, idx, 0);
15072 if (rc < 0)
15073 goto err_page_pool_destroy;
15074
15075 rc = xdp_rxq_info_reg_mem_model(&clone->xdp_rxq,
15076 MEM_TYPE_PAGE_POOL,
15077 clone->page_pool);
15078 if (rc)
15079 goto err_rxq_info_unreg;
15080
15081 ring = &clone->rx_ring_struct;
15082 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
15083 if (rc)
15084 goto err_free_rx_ring;
15085
15086 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
15087 ring = &clone->rx_agg_ring_struct;
15088 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
15089 if (rc)
15090 goto err_free_rx_agg_ring;
15091
15092 rc = bnxt_alloc_rx_agg_bmap(bp, clone);
15093 if (rc)
15094 goto err_free_rx_agg_ring;
15095 }
15096
15097 bnxt_init_one_rx_ring_rxbd(bp, clone);
15098 bnxt_init_one_rx_agg_ring_rxbd(bp, clone);
15099
15100 bnxt_alloc_one_rx_ring_skb(bp, clone, idx);
15101 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15102 bnxt_alloc_one_rx_ring_page(bp, clone, idx);
15103
15104 return 0;
15105
15106 err_free_rx_agg_ring:
15107 bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem);
15108 err_free_rx_ring:
15109 bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem);
15110 err_rxq_info_unreg:
15111 xdp_rxq_info_unreg(&clone->xdp_rxq);
15112 err_page_pool_destroy:
15113 clone->page_pool->p.napi = NULL;
15114 page_pool_destroy(clone->page_pool);
15115 clone->page_pool = NULL;
15116 return rc;
15117 }
15118
bnxt_queue_mem_free(struct net_device * dev,void * qmem)15119 static void bnxt_queue_mem_free(struct net_device *dev, void *qmem)
15120 {
15121 struct bnxt_rx_ring_info *rxr = qmem;
15122 struct bnxt *bp = netdev_priv(dev);
15123 struct bnxt_ring_struct *ring;
15124
15125 bnxt_free_one_rx_ring(bp, rxr);
15126 bnxt_free_one_rx_agg_ring(bp, rxr);
15127
15128 xdp_rxq_info_unreg(&rxr->xdp_rxq);
15129
15130 page_pool_destroy(rxr->page_pool);
15131 rxr->page_pool = NULL;
15132
15133 ring = &rxr->rx_ring_struct;
15134 bnxt_free_ring(bp, &ring->ring_mem);
15135
15136 ring = &rxr->rx_agg_ring_struct;
15137 bnxt_free_ring(bp, &ring->ring_mem);
15138
15139 kfree(rxr->rx_agg_bmap);
15140 rxr->rx_agg_bmap = NULL;
15141 }
15142
bnxt_copy_rx_ring(struct bnxt * bp,struct bnxt_rx_ring_info * dst,struct bnxt_rx_ring_info * src)15143 static void bnxt_copy_rx_ring(struct bnxt *bp,
15144 struct bnxt_rx_ring_info *dst,
15145 struct bnxt_rx_ring_info *src)
15146 {
15147 struct bnxt_ring_mem_info *dst_rmem, *src_rmem;
15148 struct bnxt_ring_struct *dst_ring, *src_ring;
15149 int i;
15150
15151 dst_ring = &dst->rx_ring_struct;
15152 dst_rmem = &dst_ring->ring_mem;
15153 src_ring = &src->rx_ring_struct;
15154 src_rmem = &src_ring->ring_mem;
15155
15156 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages);
15157 WARN_ON(dst_rmem->page_size != src_rmem->page_size);
15158 WARN_ON(dst_rmem->flags != src_rmem->flags);
15159 WARN_ON(dst_rmem->depth != src_rmem->depth);
15160 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size);
15161 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem);
15162
15163 dst_rmem->pg_tbl = src_rmem->pg_tbl;
15164 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map;
15165 *dst_rmem->vmem = *src_rmem->vmem;
15166 for (i = 0; i < dst_rmem->nr_pages; i++) {
15167 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i];
15168 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i];
15169 }
15170
15171 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
15172 return;
15173
15174 dst_ring = &dst->rx_agg_ring_struct;
15175 dst_rmem = &dst_ring->ring_mem;
15176 src_ring = &src->rx_agg_ring_struct;
15177 src_rmem = &src_ring->ring_mem;
15178
15179 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages);
15180 WARN_ON(dst_rmem->page_size != src_rmem->page_size);
15181 WARN_ON(dst_rmem->flags != src_rmem->flags);
15182 WARN_ON(dst_rmem->depth != src_rmem->depth);
15183 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size);
15184 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem);
15185 WARN_ON(dst->rx_agg_bmap_size != src->rx_agg_bmap_size);
15186
15187 dst_rmem->pg_tbl = src_rmem->pg_tbl;
15188 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map;
15189 *dst_rmem->vmem = *src_rmem->vmem;
15190 for (i = 0; i < dst_rmem->nr_pages; i++) {
15191 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i];
15192 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i];
15193 }
15194
15195 dst->rx_agg_bmap = src->rx_agg_bmap;
15196 }
15197
bnxt_queue_start(struct net_device * dev,void * qmem,int idx)15198 static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx)
15199 {
15200 struct bnxt *bp = netdev_priv(dev);
15201 struct bnxt_rx_ring_info *rxr, *clone;
15202 struct bnxt_cp_ring_info *cpr;
15203 struct bnxt_vnic_info *vnic;
15204 int i, rc;
15205
15206 rxr = &bp->rx_ring[idx];
15207 clone = qmem;
15208
15209 rxr->rx_prod = clone->rx_prod;
15210 rxr->rx_agg_prod = clone->rx_agg_prod;
15211 rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod;
15212 rxr->rx_next_cons = clone->rx_next_cons;
15213 rxr->page_pool = clone->page_pool;
15214 rxr->xdp_rxq = clone->xdp_rxq;
15215
15216 bnxt_copy_rx_ring(bp, rxr, clone);
15217
15218 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr);
15219 if (rc)
15220 return rc;
15221 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr);
15222 if (rc)
15223 goto err_free_hwrm_rx_ring;
15224
15225 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
15226 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15227 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
15228
15229 cpr = &rxr->bnapi->cp_ring;
15230 cpr->sw_stats->rx.rx_resets++;
15231
15232 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) {
15233 vnic = &bp->vnic_info[i];
15234 vnic->mru = bp->dev->mtu + ETH_HLEN + VLAN_HLEN;
15235 bnxt_hwrm_vnic_update(bp, vnic,
15236 VNIC_UPDATE_REQ_ENABLES_MRU_VALID);
15237 }
15238
15239 return 0;
15240
15241 err_free_hwrm_rx_ring:
15242 bnxt_hwrm_rx_ring_free(bp, rxr, false);
15243 return rc;
15244 }
15245
bnxt_queue_stop(struct net_device * dev,void * qmem,int idx)15246 static int bnxt_queue_stop(struct net_device *dev, void *qmem, int idx)
15247 {
15248 struct bnxt *bp = netdev_priv(dev);
15249 struct bnxt_rx_ring_info *rxr;
15250 struct bnxt_vnic_info *vnic;
15251 int i;
15252
15253 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) {
15254 vnic = &bp->vnic_info[i];
15255 vnic->mru = 0;
15256 bnxt_hwrm_vnic_update(bp, vnic,
15257 VNIC_UPDATE_REQ_ENABLES_MRU_VALID);
15258 }
15259
15260 rxr = &bp->rx_ring[idx];
15261 bnxt_hwrm_rx_ring_free(bp, rxr, false);
15262 bnxt_hwrm_rx_agg_ring_free(bp, rxr, false);
15263 rxr->rx_next_cons = 0;
15264 page_pool_disable_direct_recycling(rxr->page_pool);
15265
15266 memcpy(qmem, rxr, sizeof(*rxr));
15267 bnxt_init_rx_ring_struct(bp, qmem);
15268
15269 return 0;
15270 }
15271
15272 static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = {
15273 .ndo_queue_mem_size = sizeof(struct bnxt_rx_ring_info),
15274 .ndo_queue_mem_alloc = bnxt_queue_mem_alloc,
15275 .ndo_queue_mem_free = bnxt_queue_mem_free,
15276 .ndo_queue_start = bnxt_queue_start,
15277 .ndo_queue_stop = bnxt_queue_stop,
15278 };
15279
bnxt_remove_one(struct pci_dev * pdev)15280 static void bnxt_remove_one(struct pci_dev *pdev)
15281 {
15282 struct net_device *dev = pci_get_drvdata(pdev);
15283 struct bnxt *bp = netdev_priv(dev);
15284
15285 if (BNXT_PF(bp))
15286 bnxt_sriov_disable(bp);
15287
15288 bnxt_rdma_aux_device_del(bp);
15289
15290 bnxt_ptp_clear(bp);
15291 unregister_netdev(dev);
15292
15293 bnxt_rdma_aux_device_uninit(bp);
15294
15295 bnxt_free_l2_filters(bp, true);
15296 bnxt_free_ntp_fltrs(bp, true);
15297 WARN_ON(bp->num_rss_ctx);
15298 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
15299 /* Flush any pending tasks */
15300 cancel_work_sync(&bp->sp_task);
15301 cancel_delayed_work_sync(&bp->fw_reset_task);
15302 bp->sp_event = 0;
15303
15304 bnxt_dl_fw_reporters_destroy(bp);
15305 bnxt_dl_unregister(bp);
15306 bnxt_shutdown_tc(bp);
15307
15308 bnxt_clear_int_mode(bp);
15309 bnxt_hwrm_func_drv_unrgtr(bp);
15310 bnxt_free_hwrm_resources(bp);
15311 bnxt_hwmon_uninit(bp);
15312 bnxt_ethtool_free(bp);
15313 bnxt_dcb_free(bp);
15314 kfree(bp->ptp_cfg);
15315 bp->ptp_cfg = NULL;
15316 kfree(bp->fw_health);
15317 bp->fw_health = NULL;
15318 bnxt_cleanup_pci(bp);
15319 bnxt_free_ctx_mem(bp);
15320 bnxt_free_crash_dump_mem(bp);
15321 kfree(bp->rss_indir_tbl);
15322 bp->rss_indir_tbl = NULL;
15323 bnxt_free_port_stats(bp);
15324 free_netdev(dev);
15325 }
15326
bnxt_probe_phy(struct bnxt * bp,bool fw_dflt)15327 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
15328 {
15329 int rc = 0;
15330 struct bnxt_link_info *link_info = &bp->link_info;
15331
15332 bp->phy_flags = 0;
15333 rc = bnxt_hwrm_phy_qcaps(bp);
15334 if (rc) {
15335 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
15336 rc);
15337 return rc;
15338 }
15339 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
15340 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
15341 else
15342 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
15343 if (!fw_dflt)
15344 return 0;
15345
15346 mutex_lock(&bp->link_lock);
15347 rc = bnxt_update_link(bp, false);
15348 if (rc) {
15349 mutex_unlock(&bp->link_lock);
15350 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
15351 rc);
15352 return rc;
15353 }
15354
15355 /* Older firmware does not have supported_auto_speeds, so assume
15356 * that all supported speeds can be autonegotiated.
15357 */
15358 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
15359 link_info->support_auto_speeds = link_info->support_speeds;
15360
15361 bnxt_init_ethtool_link_settings(bp);
15362 mutex_unlock(&bp->link_lock);
15363 return 0;
15364 }
15365
bnxt_get_max_irq(struct pci_dev * pdev)15366 static int bnxt_get_max_irq(struct pci_dev *pdev)
15367 {
15368 u16 ctrl;
15369
15370 if (!pdev->msix_cap)
15371 return 1;
15372
15373 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
15374 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
15375 }
15376
_bnxt_get_max_rings(struct bnxt * bp,int * max_rx,int * max_tx,int * max_cp)15377 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
15378 int *max_cp)
15379 {
15380 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
15381 int max_ring_grps = 0, max_irq;
15382
15383 *max_tx = hw_resc->max_tx_rings;
15384 *max_rx = hw_resc->max_rx_rings;
15385 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
15386 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
15387 bnxt_get_ulp_msix_num_in_use(bp),
15388 hw_resc->max_stat_ctxs -
15389 bnxt_get_ulp_stat_ctxs_in_use(bp));
15390 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
15391 *max_cp = min_t(int, *max_cp, max_irq);
15392 max_ring_grps = hw_resc->max_hw_ring_grps;
15393 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
15394 *max_cp -= 1;
15395 *max_rx -= 2;
15396 }
15397 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15398 *max_rx >>= 1;
15399 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
15400 int rc;
15401
15402 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
15403 if (rc) {
15404 *max_rx = 0;
15405 *max_tx = 0;
15406 }
15407 /* On P5 chips, max_cp output param should be available NQs */
15408 *max_cp = max_irq;
15409 }
15410 *max_rx = min_t(int, *max_rx, max_ring_grps);
15411 }
15412
bnxt_get_max_rings(struct bnxt * bp,int * max_rx,int * max_tx,bool shared)15413 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
15414 {
15415 int rx, tx, cp;
15416
15417 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
15418 *max_rx = rx;
15419 *max_tx = tx;
15420 if (!rx || !tx || !cp)
15421 return -ENOMEM;
15422
15423 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
15424 }
15425
bnxt_get_dflt_rings(struct bnxt * bp,int * max_rx,int * max_tx,bool shared)15426 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
15427 bool shared)
15428 {
15429 int rc;
15430
15431 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
15432 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
15433 /* Not enough rings, try disabling agg rings. */
15434 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
15435 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
15436 if (rc) {
15437 /* set BNXT_FLAG_AGG_RINGS back for consistency */
15438 bp->flags |= BNXT_FLAG_AGG_RINGS;
15439 return rc;
15440 }
15441 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
15442 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
15443 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
15444 bnxt_set_ring_params(bp);
15445 }
15446
15447 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
15448 int max_cp, max_stat, max_irq;
15449
15450 /* Reserve minimum resources for RoCE */
15451 max_cp = bnxt_get_max_func_cp_rings(bp);
15452 max_stat = bnxt_get_max_func_stat_ctxs(bp);
15453 max_irq = bnxt_get_max_func_irqs(bp);
15454 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
15455 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
15456 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
15457 return 0;
15458
15459 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
15460 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
15461 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
15462 max_cp = min_t(int, max_cp, max_irq);
15463 max_cp = min_t(int, max_cp, max_stat);
15464 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
15465 if (rc)
15466 rc = 0;
15467 }
15468 return rc;
15469 }
15470
15471 /* In initial default shared ring setting, each shared ring must have a
15472 * RX/TX ring pair.
15473 */
bnxt_trim_dflt_sh_rings(struct bnxt * bp)15474 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
15475 {
15476 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
15477 bp->rx_nr_rings = bp->cp_nr_rings;
15478 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
15479 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15480 }
15481
bnxt_set_dflt_rings(struct bnxt * bp,bool sh)15482 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
15483 {
15484 int dflt_rings, max_rx_rings, max_tx_rings, rc;
15485 int avail_msix;
15486
15487 if (!bnxt_can_reserve_rings(bp))
15488 return 0;
15489
15490 if (sh)
15491 bp->flags |= BNXT_FLAG_SHARED_RINGS;
15492 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
15493 /* Reduce default rings on multi-port cards so that total default
15494 * rings do not exceed CPU count.
15495 */
15496 if (bp->port_count > 1) {
15497 int max_rings =
15498 max_t(int, num_online_cpus() / bp->port_count, 1);
15499
15500 dflt_rings = min_t(int, dflt_rings, max_rings);
15501 }
15502 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
15503 if (rc)
15504 return rc;
15505 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
15506 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
15507 if (sh)
15508 bnxt_trim_dflt_sh_rings(bp);
15509 else
15510 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
15511 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15512
15513 avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings;
15514 if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) {
15515 int ulp_num_msix = min(avail_msix, bp->ulp_num_msix_want);
15516
15517 bnxt_set_ulp_msix_num(bp, ulp_num_msix);
15518 bnxt_set_dflt_ulp_stat_ctxs(bp);
15519 }
15520
15521 rc = __bnxt_reserve_rings(bp);
15522 if (rc && rc != -ENODEV)
15523 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
15524 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15525 if (sh)
15526 bnxt_trim_dflt_sh_rings(bp);
15527
15528 /* Rings may have been trimmed, re-reserve the trimmed rings. */
15529 if (bnxt_need_reserve_rings(bp)) {
15530 rc = __bnxt_reserve_rings(bp);
15531 if (rc && rc != -ENODEV)
15532 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
15533 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15534 }
15535 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
15536 bp->rx_nr_rings++;
15537 bp->cp_nr_rings++;
15538 }
15539 if (rc) {
15540 bp->tx_nr_rings = 0;
15541 bp->rx_nr_rings = 0;
15542 }
15543 return rc;
15544 }
15545
bnxt_init_dflt_ring_mode(struct bnxt * bp)15546 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
15547 {
15548 int rc;
15549
15550 if (bp->tx_nr_rings)
15551 return 0;
15552
15553 bnxt_ulp_irq_stop(bp);
15554 bnxt_clear_int_mode(bp);
15555 rc = bnxt_set_dflt_rings(bp, true);
15556 if (rc) {
15557 if (BNXT_VF(bp) && rc == -ENODEV)
15558 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
15559 else
15560 netdev_err(bp->dev, "Not enough rings available.\n");
15561 goto init_dflt_ring_err;
15562 }
15563 rc = bnxt_init_int_mode(bp);
15564 if (rc)
15565 goto init_dflt_ring_err;
15566
15567 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15568
15569 bnxt_set_dflt_rfs(bp);
15570
15571 init_dflt_ring_err:
15572 bnxt_ulp_irq_restart(bp, rc);
15573 return rc;
15574 }
15575
bnxt_restore_pf_fw_resources(struct bnxt * bp)15576 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
15577 {
15578 int rc;
15579
15580 ASSERT_RTNL();
15581 bnxt_hwrm_func_qcaps(bp);
15582
15583 if (netif_running(bp->dev))
15584 __bnxt_close_nic(bp, true, false);
15585
15586 bnxt_ulp_irq_stop(bp);
15587 bnxt_clear_int_mode(bp);
15588 rc = bnxt_init_int_mode(bp);
15589 bnxt_ulp_irq_restart(bp, rc);
15590
15591 if (netif_running(bp->dev)) {
15592 if (rc)
15593 dev_close(bp->dev);
15594 else
15595 rc = bnxt_open_nic(bp, true, false);
15596 }
15597
15598 return rc;
15599 }
15600
bnxt_init_mac_addr(struct bnxt * bp)15601 static int bnxt_init_mac_addr(struct bnxt *bp)
15602 {
15603 int rc = 0;
15604
15605 if (BNXT_PF(bp)) {
15606 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
15607 } else {
15608 #ifdef CONFIG_BNXT_SRIOV
15609 struct bnxt_vf_info *vf = &bp->vf;
15610 bool strict_approval = true;
15611
15612 if (is_valid_ether_addr(vf->mac_addr)) {
15613 /* overwrite netdev dev_addr with admin VF MAC */
15614 eth_hw_addr_set(bp->dev, vf->mac_addr);
15615 /* Older PF driver or firmware may not approve this
15616 * correctly.
15617 */
15618 strict_approval = false;
15619 } else {
15620 eth_hw_addr_random(bp->dev);
15621 }
15622 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
15623 #endif
15624 }
15625 return rc;
15626 }
15627
bnxt_vpd_read_info(struct bnxt * bp)15628 static void bnxt_vpd_read_info(struct bnxt *bp)
15629 {
15630 struct pci_dev *pdev = bp->pdev;
15631 unsigned int vpd_size, kw_len;
15632 int pos, size;
15633 u8 *vpd_data;
15634
15635 vpd_data = pci_vpd_alloc(pdev, &vpd_size);
15636 if (IS_ERR(vpd_data)) {
15637 pci_warn(pdev, "Unable to read VPD\n");
15638 return;
15639 }
15640
15641 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15642 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
15643 if (pos < 0)
15644 goto read_sn;
15645
15646 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15647 memcpy(bp->board_partno, &vpd_data[pos], size);
15648
15649 read_sn:
15650 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15651 PCI_VPD_RO_KEYWORD_SERIALNO,
15652 &kw_len);
15653 if (pos < 0)
15654 goto exit;
15655
15656 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15657 memcpy(bp->board_serialno, &vpd_data[pos], size);
15658 exit:
15659 kfree(vpd_data);
15660 }
15661
bnxt_pcie_dsn_get(struct bnxt * bp,u8 dsn[])15662 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
15663 {
15664 struct pci_dev *pdev = bp->pdev;
15665 u64 qword;
15666
15667 qword = pci_get_dsn(pdev);
15668 if (!qword) {
15669 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
15670 return -EOPNOTSUPP;
15671 }
15672
15673 put_unaligned_le64(qword, dsn);
15674
15675 bp->flags |= BNXT_FLAG_DSN_VALID;
15676 return 0;
15677 }
15678
bnxt_map_db_bar(struct bnxt * bp)15679 static int bnxt_map_db_bar(struct bnxt *bp)
15680 {
15681 if (!bp->db_size)
15682 return -ENODEV;
15683 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
15684 if (!bp->bar1)
15685 return -ENOMEM;
15686 return 0;
15687 }
15688
bnxt_print_device_info(struct bnxt * bp)15689 void bnxt_print_device_info(struct bnxt *bp)
15690 {
15691 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
15692 board_info[bp->board_idx].name,
15693 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
15694
15695 pcie_print_link_status(bp->pdev);
15696 }
15697
bnxt_init_one(struct pci_dev * pdev,const struct pci_device_id * ent)15698 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
15699 {
15700 struct bnxt_hw_resc *hw_resc;
15701 struct net_device *dev;
15702 struct bnxt *bp;
15703 int rc, max_irqs;
15704
15705 if (pci_is_bridge(pdev))
15706 return -ENODEV;
15707
15708 if (!pdev->msix_cap) {
15709 dev_err(&pdev->dev, "MSIX capability not found, aborting\n");
15710 return -ENODEV;
15711 }
15712
15713 /* Clear any pending DMA transactions from crash kernel
15714 * while loading driver in capture kernel.
15715 */
15716 if (is_kdump_kernel()) {
15717 pci_clear_master(pdev);
15718 pcie_flr(pdev);
15719 }
15720
15721 max_irqs = bnxt_get_max_irq(pdev);
15722 dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
15723 max_irqs);
15724 if (!dev)
15725 return -ENOMEM;
15726
15727 bp = netdev_priv(dev);
15728 bp->board_idx = ent->driver_data;
15729 bp->msg_enable = BNXT_DEF_MSG_ENABLE;
15730 bnxt_set_max_func_irqs(bp, max_irqs);
15731
15732 if (bnxt_vf_pciid(bp->board_idx))
15733 bp->flags |= BNXT_FLAG_VF;
15734
15735 /* No devlink port registration in case of a VF */
15736 if (BNXT_PF(bp))
15737 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
15738
15739 rc = bnxt_init_board(pdev, dev);
15740 if (rc < 0)
15741 goto init_err_free;
15742
15743 dev->netdev_ops = &bnxt_netdev_ops;
15744 dev->stat_ops = &bnxt_stat_ops;
15745 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
15746 dev->ethtool_ops = &bnxt_ethtool_ops;
15747 pci_set_drvdata(pdev, dev);
15748
15749 rc = bnxt_alloc_hwrm_resources(bp);
15750 if (rc)
15751 goto init_err_pci_clean;
15752
15753 mutex_init(&bp->hwrm_cmd_lock);
15754 mutex_init(&bp->link_lock);
15755
15756 rc = bnxt_fw_init_one_p1(bp);
15757 if (rc)
15758 goto init_err_pci_clean;
15759
15760 if (BNXT_PF(bp))
15761 bnxt_vpd_read_info(bp);
15762
15763 if (BNXT_CHIP_P5_PLUS(bp)) {
15764 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS;
15765 if (BNXT_CHIP_P7(bp))
15766 bp->flags |= BNXT_FLAG_CHIP_P7;
15767 }
15768
15769 rc = bnxt_alloc_rss_indir_tbl(bp);
15770 if (rc)
15771 goto init_err_pci_clean;
15772
15773 rc = bnxt_fw_init_one_p2(bp);
15774 if (rc)
15775 goto init_err_pci_clean;
15776
15777 rc = bnxt_map_db_bar(bp);
15778 if (rc) {
15779 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
15780 rc);
15781 goto init_err_pci_clean;
15782 }
15783
15784 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
15785 NETIF_F_TSO | NETIF_F_TSO6 |
15786 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
15787 NETIF_F_GSO_IPXIP4 |
15788 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
15789 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
15790 NETIF_F_RXCSUM | NETIF_F_GRO;
15791 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
15792 dev->hw_features |= NETIF_F_GSO_UDP_L4;
15793
15794 if (BNXT_SUPPORTS_TPA(bp))
15795 dev->hw_features |= NETIF_F_LRO;
15796
15797 dev->hw_enc_features =
15798 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
15799 NETIF_F_TSO | NETIF_F_TSO6 |
15800 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
15801 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
15802 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
15803 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
15804 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
15805 if (bp->flags & BNXT_FLAG_CHIP_P7)
15806 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7;
15807 else
15808 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
15809
15810 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
15811 NETIF_F_GSO_GRE_CSUM;
15812 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
15813 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
15814 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
15815 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
15816 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
15817 if (BNXT_SUPPORTS_TPA(bp))
15818 dev->hw_features |= NETIF_F_GRO_HW;
15819 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
15820 if (dev->features & NETIF_F_GRO_HW)
15821 dev->features &= ~NETIF_F_LRO;
15822 dev->priv_flags |= IFF_UNICAST_FLT;
15823
15824 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
15825 if (bp->tso_max_segs)
15826 netif_set_tso_max_segs(dev, bp->tso_max_segs);
15827
15828 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
15829 NETDEV_XDP_ACT_RX_SG;
15830
15831 #ifdef CONFIG_BNXT_SRIOV
15832 init_waitqueue_head(&bp->sriov_cfg_wait);
15833 #endif
15834 if (BNXT_SUPPORTS_TPA(bp)) {
15835 bp->gro_func = bnxt_gro_func_5730x;
15836 if (BNXT_CHIP_P4(bp))
15837 bp->gro_func = bnxt_gro_func_5731x;
15838 else if (BNXT_CHIP_P5_PLUS(bp))
15839 bp->gro_func = bnxt_gro_func_5750x;
15840 }
15841 if (!BNXT_CHIP_P4_PLUS(bp))
15842 bp->flags |= BNXT_FLAG_DOUBLE_DB;
15843
15844 rc = bnxt_init_mac_addr(bp);
15845 if (rc) {
15846 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
15847 rc = -EADDRNOTAVAIL;
15848 goto init_err_pci_clean;
15849 }
15850
15851 if (BNXT_PF(bp)) {
15852 /* Read the adapter's DSN to use as the eswitch switch_id */
15853 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
15854 }
15855
15856 /* MTU range: 60 - FW defined max */
15857 dev->min_mtu = ETH_ZLEN;
15858 dev->max_mtu = bp->max_mtu;
15859
15860 rc = bnxt_probe_phy(bp, true);
15861 if (rc)
15862 goto init_err_pci_clean;
15863
15864 hw_resc = &bp->hw_resc;
15865 bp->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows +
15866 BNXT_L2_FLTR_MAX_FLTR;
15867 /* Older firmware may not report these filters properly */
15868 if (bp->max_fltr < BNXT_MAX_FLTR)
15869 bp->max_fltr = BNXT_MAX_FLTR;
15870 bnxt_init_l2_fltr_tbl(bp);
15871 bnxt_set_rx_skb_mode(bp, false);
15872 bnxt_set_tpa_flags(bp);
15873 bnxt_set_ring_params(bp);
15874 bnxt_rdma_aux_device_init(bp);
15875 rc = bnxt_set_dflt_rings(bp, true);
15876 if (rc) {
15877 if (BNXT_VF(bp) && rc == -ENODEV) {
15878 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
15879 } else {
15880 netdev_err(bp->dev, "Not enough rings available.\n");
15881 rc = -ENOMEM;
15882 }
15883 goto init_err_pci_clean;
15884 }
15885
15886 bnxt_fw_init_one_p3(bp);
15887
15888 bnxt_init_dflt_coal(bp);
15889
15890 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
15891 bp->flags |= BNXT_FLAG_STRIP_VLAN;
15892
15893 rc = bnxt_init_int_mode(bp);
15894 if (rc)
15895 goto init_err_pci_clean;
15896
15897 /* No TC has been set yet and rings may have been trimmed due to
15898 * limited MSIX, so we re-initialize the TX rings per TC.
15899 */
15900 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15901
15902 if (BNXT_PF(bp)) {
15903 if (!bnxt_pf_wq) {
15904 bnxt_pf_wq =
15905 create_singlethread_workqueue("bnxt_pf_wq");
15906 if (!bnxt_pf_wq) {
15907 dev_err(&pdev->dev, "Unable to create workqueue.\n");
15908 rc = -ENOMEM;
15909 goto init_err_pci_clean;
15910 }
15911 }
15912 rc = bnxt_init_tc(bp);
15913 if (rc)
15914 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
15915 rc);
15916 }
15917
15918 bnxt_inv_fw_health_reg(bp);
15919 rc = bnxt_dl_register(bp);
15920 if (rc)
15921 goto init_err_dl;
15922
15923 INIT_LIST_HEAD(&bp->usr_fltr_list);
15924
15925 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
15926 bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX;
15927 if (BNXT_SUPPORTS_QUEUE_API(bp))
15928 dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops;
15929
15930 rc = register_netdev(dev);
15931 if (rc)
15932 goto init_err_cleanup;
15933
15934 bnxt_dl_fw_reporters_create(bp);
15935
15936 bnxt_rdma_aux_device_add(bp);
15937
15938 bnxt_print_device_info(bp);
15939
15940 pci_save_state(pdev);
15941
15942 return 0;
15943 init_err_cleanup:
15944 bnxt_rdma_aux_device_uninit(bp);
15945 bnxt_dl_unregister(bp);
15946 init_err_dl:
15947 bnxt_shutdown_tc(bp);
15948 bnxt_clear_int_mode(bp);
15949
15950 init_err_pci_clean:
15951 bnxt_hwrm_func_drv_unrgtr(bp);
15952 bnxt_free_hwrm_resources(bp);
15953 bnxt_hwmon_uninit(bp);
15954 bnxt_ethtool_free(bp);
15955 bnxt_ptp_clear(bp);
15956 kfree(bp->ptp_cfg);
15957 bp->ptp_cfg = NULL;
15958 kfree(bp->fw_health);
15959 bp->fw_health = NULL;
15960 bnxt_cleanup_pci(bp);
15961 bnxt_free_ctx_mem(bp);
15962 bnxt_free_crash_dump_mem(bp);
15963 kfree(bp->rss_indir_tbl);
15964 bp->rss_indir_tbl = NULL;
15965
15966 init_err_free:
15967 free_netdev(dev);
15968 return rc;
15969 }
15970
bnxt_shutdown(struct pci_dev * pdev)15971 static void bnxt_shutdown(struct pci_dev *pdev)
15972 {
15973 struct net_device *dev = pci_get_drvdata(pdev);
15974 struct bnxt *bp;
15975
15976 if (!dev)
15977 return;
15978
15979 rtnl_lock();
15980 bp = netdev_priv(dev);
15981 if (!bp)
15982 goto shutdown_exit;
15983
15984 if (netif_running(dev))
15985 dev_close(dev);
15986
15987 bnxt_clear_int_mode(bp);
15988 pci_disable_device(pdev);
15989
15990 if (system_state == SYSTEM_POWER_OFF) {
15991 pci_wake_from_d3(pdev, bp->wol);
15992 pci_set_power_state(pdev, PCI_D3hot);
15993 }
15994
15995 shutdown_exit:
15996 rtnl_unlock();
15997 }
15998
15999 #ifdef CONFIG_PM_SLEEP
bnxt_suspend(struct device * device)16000 static int bnxt_suspend(struct device *device)
16001 {
16002 struct net_device *dev = dev_get_drvdata(device);
16003 struct bnxt *bp = netdev_priv(dev);
16004 int rc = 0;
16005
16006 bnxt_ulp_stop(bp);
16007
16008 rtnl_lock();
16009 if (netif_running(dev)) {
16010 netif_device_detach(dev);
16011 rc = bnxt_close(dev);
16012 }
16013 bnxt_hwrm_func_drv_unrgtr(bp);
16014 pci_disable_device(bp->pdev);
16015 bnxt_free_ctx_mem(bp);
16016 rtnl_unlock();
16017 return rc;
16018 }
16019
bnxt_resume(struct device * device)16020 static int bnxt_resume(struct device *device)
16021 {
16022 struct net_device *dev = dev_get_drvdata(device);
16023 struct bnxt *bp = netdev_priv(dev);
16024 int rc = 0;
16025
16026 rtnl_lock();
16027 rc = pci_enable_device(bp->pdev);
16028 if (rc) {
16029 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
16030 rc);
16031 goto resume_exit;
16032 }
16033 pci_set_master(bp->pdev);
16034 if (bnxt_hwrm_ver_get(bp)) {
16035 rc = -ENODEV;
16036 goto resume_exit;
16037 }
16038 rc = bnxt_hwrm_func_reset(bp);
16039 if (rc) {
16040 rc = -EBUSY;
16041 goto resume_exit;
16042 }
16043
16044 rc = bnxt_hwrm_func_qcaps(bp);
16045 if (rc)
16046 goto resume_exit;
16047
16048 bnxt_clear_reservations(bp, true);
16049
16050 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
16051 rc = -ENODEV;
16052 goto resume_exit;
16053 }
16054 if (bp->fw_crash_mem)
16055 bnxt_hwrm_crash_dump_mem_cfg(bp);
16056
16057 bnxt_get_wol_settings(bp);
16058 if (netif_running(dev)) {
16059 rc = bnxt_open(dev);
16060 if (!rc)
16061 netif_device_attach(dev);
16062 }
16063
16064 resume_exit:
16065 rtnl_unlock();
16066 bnxt_ulp_start(bp, rc);
16067 if (!rc)
16068 bnxt_reenable_sriov(bp);
16069 return rc;
16070 }
16071
16072 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
16073 #define BNXT_PM_OPS (&bnxt_pm_ops)
16074
16075 #else
16076
16077 #define BNXT_PM_OPS NULL
16078
16079 #endif /* CONFIG_PM_SLEEP */
16080
16081 /**
16082 * bnxt_io_error_detected - called when PCI error is detected
16083 * @pdev: Pointer to PCI device
16084 * @state: The current pci connection state
16085 *
16086 * This function is called after a PCI bus error affecting
16087 * this device has been detected.
16088 */
bnxt_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)16089 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
16090 pci_channel_state_t state)
16091 {
16092 struct net_device *netdev = pci_get_drvdata(pdev);
16093 struct bnxt *bp = netdev_priv(netdev);
16094 bool abort = false;
16095
16096 netdev_info(netdev, "PCI I/O error detected\n");
16097
16098 bnxt_ulp_stop(bp);
16099
16100 rtnl_lock();
16101 netif_device_detach(netdev);
16102
16103 if (test_and_set_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
16104 netdev_err(bp->dev, "Firmware reset already in progress\n");
16105 abort = true;
16106 }
16107
16108 if (abort || state == pci_channel_io_perm_failure) {
16109 rtnl_unlock();
16110 return PCI_ERS_RESULT_DISCONNECT;
16111 }
16112
16113 /* Link is not reliable anymore if state is pci_channel_io_frozen
16114 * so we disable bus master to prevent any potential bad DMAs before
16115 * freeing kernel memory.
16116 */
16117 if (state == pci_channel_io_frozen) {
16118 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
16119 bnxt_fw_fatal_close(bp);
16120 }
16121
16122 if (netif_running(netdev))
16123 __bnxt_close_nic(bp, true, true);
16124
16125 if (pci_is_enabled(pdev))
16126 pci_disable_device(pdev);
16127 bnxt_free_ctx_mem(bp);
16128 rtnl_unlock();
16129
16130 /* Request a slot slot reset. */
16131 return PCI_ERS_RESULT_NEED_RESET;
16132 }
16133
16134 /**
16135 * bnxt_io_slot_reset - called after the pci bus has been reset.
16136 * @pdev: Pointer to PCI device
16137 *
16138 * Restart the card from scratch, as if from a cold-boot.
16139 * At this point, the card has exprienced a hard reset,
16140 * followed by fixups by BIOS, and has its config space
16141 * set up identically to what it was at cold boot.
16142 */
bnxt_io_slot_reset(struct pci_dev * pdev)16143 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
16144 {
16145 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
16146 struct net_device *netdev = pci_get_drvdata(pdev);
16147 struct bnxt *bp = netdev_priv(netdev);
16148 int retry = 0;
16149 int err = 0;
16150 int off;
16151
16152 netdev_info(bp->dev, "PCI Slot Reset\n");
16153
16154 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
16155 test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state))
16156 msleep(900);
16157
16158 rtnl_lock();
16159
16160 if (pci_enable_device(pdev)) {
16161 dev_err(&pdev->dev,
16162 "Cannot re-enable PCI device after reset.\n");
16163 } else {
16164 pci_set_master(pdev);
16165 /* Upon fatal error, our device internal logic that latches to
16166 * BAR value is getting reset and will restore only upon
16167 * rewritting the BARs.
16168 *
16169 * As pci_restore_state() does not re-write the BARs if the
16170 * value is same as saved value earlier, driver needs to
16171 * write the BARs to 0 to force restore, in case of fatal error.
16172 */
16173 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
16174 &bp->state)) {
16175 for (off = PCI_BASE_ADDRESS_0;
16176 off <= PCI_BASE_ADDRESS_5; off += 4)
16177 pci_write_config_dword(bp->pdev, off, 0);
16178 }
16179 pci_restore_state(pdev);
16180 pci_save_state(pdev);
16181
16182 bnxt_inv_fw_health_reg(bp);
16183 bnxt_try_map_fw_health_reg(bp);
16184
16185 /* In some PCIe AER scenarios, firmware may take up to
16186 * 10 seconds to become ready in the worst case.
16187 */
16188 do {
16189 err = bnxt_try_recover_fw(bp);
16190 if (!err)
16191 break;
16192 retry++;
16193 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
16194
16195 if (err) {
16196 dev_err(&pdev->dev, "Firmware not ready\n");
16197 goto reset_exit;
16198 }
16199
16200 err = bnxt_hwrm_func_reset(bp);
16201 if (!err)
16202 result = PCI_ERS_RESULT_RECOVERED;
16203
16204 bnxt_ulp_irq_stop(bp);
16205 bnxt_clear_int_mode(bp);
16206 err = bnxt_init_int_mode(bp);
16207 bnxt_ulp_irq_restart(bp, err);
16208 }
16209
16210 reset_exit:
16211 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
16212 bnxt_clear_reservations(bp, true);
16213 rtnl_unlock();
16214
16215 return result;
16216 }
16217
16218 /**
16219 * bnxt_io_resume - called when traffic can start flowing again.
16220 * @pdev: Pointer to PCI device
16221 *
16222 * This callback is called when the error recovery driver tells
16223 * us that its OK to resume normal operation.
16224 */
bnxt_io_resume(struct pci_dev * pdev)16225 static void bnxt_io_resume(struct pci_dev *pdev)
16226 {
16227 struct net_device *netdev = pci_get_drvdata(pdev);
16228 struct bnxt *bp = netdev_priv(netdev);
16229 int err;
16230
16231 netdev_info(bp->dev, "PCI Slot Resume\n");
16232 rtnl_lock();
16233
16234 err = bnxt_hwrm_func_qcaps(bp);
16235 if (!err && netif_running(netdev))
16236 err = bnxt_open(netdev);
16237
16238 if (!err)
16239 netif_device_attach(netdev);
16240
16241 rtnl_unlock();
16242 bnxt_ulp_start(bp, err);
16243 if (!err)
16244 bnxt_reenable_sriov(bp);
16245 }
16246
16247 static const struct pci_error_handlers bnxt_err_handler = {
16248 .error_detected = bnxt_io_error_detected,
16249 .slot_reset = bnxt_io_slot_reset,
16250 .resume = bnxt_io_resume
16251 };
16252
16253 static struct pci_driver bnxt_pci_driver = {
16254 .name = DRV_MODULE_NAME,
16255 .id_table = bnxt_pci_tbl,
16256 .probe = bnxt_init_one,
16257 .remove = bnxt_remove_one,
16258 .shutdown = bnxt_shutdown,
16259 .driver.pm = BNXT_PM_OPS,
16260 .err_handler = &bnxt_err_handler,
16261 #if defined(CONFIG_BNXT_SRIOV)
16262 .sriov_configure = bnxt_sriov_configure,
16263 #endif
16264 };
16265
bnxt_init(void)16266 static int __init bnxt_init(void)
16267 {
16268 int err;
16269
16270 bnxt_debug_init();
16271 err = pci_register_driver(&bnxt_pci_driver);
16272 if (err) {
16273 bnxt_debug_exit();
16274 return err;
16275 }
16276
16277 return 0;
16278 }
16279
bnxt_exit(void)16280 static void __exit bnxt_exit(void)
16281 {
16282 pci_unregister_driver(&bnxt_pci_driver);
16283 if (bnxt_pf_wq)
16284 destroy_workqueue(bnxt_pf_wq);
16285 bnxt_debug_exit();
16286 }
16287
16288 module_init(bnxt_init);
16289 module_exit(bnxt_exit);
16290