1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2025 Broadcom.
3
4 #include <asm/byteorder.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/dmapool.h>
7 #include <linux/delay.h>
8 #include <linux/errno.h>
9 #include <linux/kernel.h>
10 #include <linux/list.h>
11 #include <linux/pci.h>
12
13 #include "bnge.h"
14 #include "bnge_hwrm.h"
15
bnge_cal_sentinel(struct bnge_hwrm_ctx * ctx,u16 req_type)16 static u64 bnge_cal_sentinel(struct bnge_hwrm_ctx *ctx, u16 req_type)
17 {
18 return (((uintptr_t)ctx) + req_type) ^ BNGE_HWRM_SENTINEL;
19 }
20
bnge_hwrm_req_create(struct bnge_dev * bd,void ** req,u16 req_type,u32 req_len)21 int bnge_hwrm_req_create(struct bnge_dev *bd, void **req, u16 req_type,
22 u32 req_len)
23 {
24 struct bnge_hwrm_ctx *ctx;
25 dma_addr_t dma_handle;
26 u8 *req_addr;
27
28 if (req_len > BNGE_HWRM_CTX_OFFSET)
29 return -E2BIG;
30
31 req_addr = dma_pool_alloc(bd->hwrm_dma_pool, GFP_KERNEL | __GFP_ZERO,
32 &dma_handle);
33 if (!req_addr)
34 return -ENOMEM;
35
36 ctx = (struct bnge_hwrm_ctx *)(req_addr + BNGE_HWRM_CTX_OFFSET);
37 /* safety first, sentinel used to check for invalid requests */
38 ctx->sentinel = bnge_cal_sentinel(ctx, req_type);
39 ctx->req_len = req_len;
40 ctx->req = (struct input *)req_addr;
41 ctx->resp = (struct output *)(req_addr + BNGE_HWRM_RESP_OFFSET);
42 ctx->dma_handle = dma_handle;
43 ctx->flags = 0; /* __GFP_ZERO, but be explicit regarding ownership */
44 ctx->timeout = bd->hwrm_cmd_timeout ?: BNGE_DFLT_HWRM_CMD_TIMEOUT;
45 ctx->allocated = BNGE_HWRM_DMA_SIZE - BNGE_HWRM_CTX_OFFSET;
46 ctx->gfp = GFP_KERNEL;
47 ctx->slice_addr = NULL;
48
49 /* initialize common request fields */
50 ctx->req->req_type = cpu_to_le16(req_type);
51 ctx->req->resp_addr = cpu_to_le64(dma_handle + BNGE_HWRM_RESP_OFFSET);
52 ctx->req->cmpl_ring = cpu_to_le16(BNGE_HWRM_NO_CMPL_RING);
53 ctx->req->target_id = cpu_to_le16(BNGE_HWRM_TARGET);
54 *req = ctx->req;
55
56 return 0;
57 }
58
__hwrm_ctx_get(struct bnge_dev * bd,u8 * req_addr)59 static struct bnge_hwrm_ctx *__hwrm_ctx_get(struct bnge_dev *bd, u8 *req_addr)
60 {
61 void *ctx_addr = req_addr + BNGE_HWRM_CTX_OFFSET;
62 struct input *req = (struct input *)req_addr;
63 struct bnge_hwrm_ctx *ctx = ctx_addr;
64 u64 sentinel;
65
66 if (!req) {
67 dev_err(bd->dev, "null HWRM request");
68 dump_stack();
69 return NULL;
70 }
71
72 /* HWRM API has no type safety, verify sentinel to validate address */
73 sentinel = bnge_cal_sentinel(ctx, le16_to_cpu(req->req_type));
74 if (ctx->sentinel != sentinel) {
75 dev_err(bd->dev, "HWRM sentinel mismatch, req_type = %u\n",
76 (u32)le16_to_cpu(req->req_type));
77 dump_stack();
78 return NULL;
79 }
80
81 return ctx;
82 }
83
bnge_hwrm_req_timeout(struct bnge_dev * bd,void * req,unsigned int timeout)84 void bnge_hwrm_req_timeout(struct bnge_dev *bd,
85 void *req, unsigned int timeout)
86 {
87 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
88
89 if (ctx)
90 ctx->timeout = timeout;
91 }
92
bnge_hwrm_req_alloc_flags(struct bnge_dev * bd,void * req,gfp_t gfp)93 void bnge_hwrm_req_alloc_flags(struct bnge_dev *bd, void *req, gfp_t gfp)
94 {
95 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
96
97 if (ctx)
98 ctx->gfp = gfp;
99 }
100
bnge_hwrm_req_flags(struct bnge_dev * bd,void * req,enum bnge_hwrm_ctx_flags flags)101 void bnge_hwrm_req_flags(struct bnge_dev *bd, void *req,
102 enum bnge_hwrm_ctx_flags flags)
103 {
104 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
105
106 if (ctx)
107 ctx->flags |= (flags & BNGE_HWRM_API_FLAGS);
108 }
109
bnge_hwrm_req_hold(struct bnge_dev * bd,void * req)110 void *bnge_hwrm_req_hold(struct bnge_dev *bd, void *req)
111 {
112 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
113 struct input *input = (struct input *)req;
114
115 if (!ctx)
116 return NULL;
117
118 if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED) {
119 dev_err(bd->dev, "HWRM context already owned, req_type = %u\n",
120 (u32)le16_to_cpu(input->req_type));
121 dump_stack();
122 return NULL;
123 }
124
125 ctx->flags |= BNGE_HWRM_INTERNAL_CTX_OWNED;
126 return ((u8 *)req) + BNGE_HWRM_RESP_OFFSET;
127 }
128
__hwrm_ctx_invalidate(struct bnge_dev * bd,struct bnge_hwrm_ctx * ctx)129 static void __hwrm_ctx_invalidate(struct bnge_dev *bd,
130 struct bnge_hwrm_ctx *ctx)
131 {
132 void *addr = ((u8 *)ctx) - BNGE_HWRM_CTX_OFFSET;
133 dma_addr_t dma_handle = ctx->dma_handle; /* save before invalidate */
134
135 /* unmap any auxiliary DMA slice */
136 if (ctx->slice_addr)
137 dma_free_coherent(bd->dev, ctx->slice_size,
138 ctx->slice_addr, ctx->slice_handle);
139
140 /* invalidate, ensure ownership, sentinel and dma_handle are cleared */
141 memset(ctx, 0, sizeof(struct bnge_hwrm_ctx));
142
143 /* return the buffer to the DMA pool */
144 if (dma_handle)
145 dma_pool_free(bd->hwrm_dma_pool, addr, dma_handle);
146 }
147
bnge_hwrm_req_drop(struct bnge_dev * bd,void * req)148 void bnge_hwrm_req_drop(struct bnge_dev *bd, void *req)
149 {
150 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
151
152 if (ctx)
153 __hwrm_ctx_invalidate(bd, ctx);
154 }
155
bnge_map_hwrm_error(u32 hwrm_err)156 static int bnge_map_hwrm_error(u32 hwrm_err)
157 {
158 switch (hwrm_err) {
159 case HWRM_ERR_CODE_SUCCESS:
160 return 0;
161 case HWRM_ERR_CODE_RESOURCE_LOCKED:
162 return -EROFS;
163 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
164 return -EACCES;
165 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
166 return -ENOSPC;
167 case HWRM_ERR_CODE_INVALID_PARAMS:
168 case HWRM_ERR_CODE_INVALID_FLAGS:
169 case HWRM_ERR_CODE_INVALID_ENABLES:
170 case HWRM_ERR_CODE_UNSUPPORTED_TLV:
171 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
172 return -EINVAL;
173 case HWRM_ERR_CODE_NO_BUFFER:
174 return -ENOMEM;
175 case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
176 case HWRM_ERR_CODE_BUSY:
177 return -EAGAIN;
178 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
179 return -EOPNOTSUPP;
180 case HWRM_ERR_CODE_PF_UNAVAILABLE:
181 return -ENODEV;
182 default:
183 return -EIO;
184 }
185 }
186
187 static struct bnge_hwrm_wait_token *
bnge_hwrm_create_token(struct bnge_dev * bd,enum bnge_hwrm_chnl dst)188 bnge_hwrm_create_token(struct bnge_dev *bd, enum bnge_hwrm_chnl dst)
189 {
190 struct bnge_hwrm_wait_token *token;
191
192 token = kzalloc(sizeof(*token), GFP_KERNEL);
193 if (!token)
194 return NULL;
195
196 mutex_lock(&bd->hwrm_cmd_lock);
197
198 token->dst = dst;
199 token->state = BNGE_HWRM_PENDING;
200 if (dst == BNGE_HWRM_CHNL_CHIMP) {
201 token->seq_id = bd->hwrm_cmd_seq++;
202 hlist_add_head_rcu(&token->node, &bd->hwrm_pending_list);
203 } else {
204 token->seq_id = bd->hwrm_cmd_kong_seq++;
205 }
206
207 return token;
208 }
209
210 static void
bnge_hwrm_destroy_token(struct bnge_dev * bd,struct bnge_hwrm_wait_token * token)211 bnge_hwrm_destroy_token(struct bnge_dev *bd, struct bnge_hwrm_wait_token *token)
212 {
213 if (token->dst == BNGE_HWRM_CHNL_CHIMP) {
214 hlist_del_rcu(&token->node);
215 kfree_rcu(token, rcu);
216 } else {
217 kfree(token);
218 }
219 mutex_unlock(&bd->hwrm_cmd_lock);
220 }
221
bnge_hwrm_req_dbg(struct bnge_dev * bd,struct input * req)222 static void bnge_hwrm_req_dbg(struct bnge_dev *bd, struct input *req)
223 {
224 u32 ring = le16_to_cpu(req->cmpl_ring);
225 u32 type = le16_to_cpu(req->req_type);
226 u32 tgt = le16_to_cpu(req->target_id);
227 u32 seq = le16_to_cpu(req->seq_id);
228 char opt[32] = "\n";
229
230 if (unlikely(ring != (u16)BNGE_HWRM_NO_CMPL_RING))
231 snprintf(opt, 16, " ring %d\n", ring);
232
233 if (unlikely(tgt != BNGE_HWRM_TARGET))
234 snprintf(opt + strlen(opt) - 1, 16, " tgt 0x%x\n", tgt);
235
236 dev_dbg(bd->dev, "sent hwrm req_type 0x%x seq id 0x%x%s",
237 type, seq, opt);
238 }
239
240 #define bnge_hwrm_err(bd, ctx, fmt, ...) \
241 do { \
242 if ((ctx)->flags & BNGE_HWRM_CTX_SILENT) \
243 dev_dbg((bd)->dev, fmt, __VA_ARGS__); \
244 else \
245 dev_err((bd)->dev, fmt, __VA_ARGS__); \
246 } while (0)
247
__hwrm_send_ctx(struct bnge_dev * bd,struct bnge_hwrm_ctx * ctx)248 static int __hwrm_send_ctx(struct bnge_dev *bd, struct bnge_hwrm_ctx *ctx)
249 {
250 u32 doorbell_offset = BNGE_GRCPF_REG_CHIMP_COMM_TRIGGER;
251 enum bnge_hwrm_chnl dst = BNGE_HWRM_CHNL_CHIMP;
252 u32 bar_offset = BNGE_GRCPF_REG_CHIMP_COMM;
253 struct bnge_hwrm_wait_token *token = NULL;
254 u16 max_req_len = BNGE_HWRM_MAX_REQ_LEN;
255 unsigned int i, timeout, tmo_count;
256 u32 *data = (u32 *)ctx->req;
257 u32 msg_len = ctx->req_len;
258 int rc = -EBUSY;
259 u32 req_type;
260 u16 len = 0;
261 u8 *valid;
262
263 if (ctx->flags & BNGE_HWRM_INTERNAL_RESP_DIRTY)
264 memset(ctx->resp, 0, PAGE_SIZE);
265
266 req_type = le16_to_cpu(ctx->req->req_type);
267
268 if (msg_len > BNGE_HWRM_MAX_REQ_LEN &&
269 msg_len > bd->hwrm_max_ext_req_len) {
270 dev_warn(bd->dev, "oversized hwrm request, req_type 0x%x",
271 req_type);
272 rc = -E2BIG;
273 goto exit;
274 }
275
276 token = bnge_hwrm_create_token(bd, dst);
277 if (!token) {
278 rc = -ENOMEM;
279 goto exit;
280 }
281 ctx->req->seq_id = cpu_to_le16(token->seq_id);
282
283 /* Ensure any associated DMA buffers are written before doorbell */
284 wmb();
285
286 /* Write request msg to hwrm channel */
287 __iowrite32_copy(bd->bar0 + bar_offset, data, msg_len / 4);
288
289 for (i = msg_len; i < max_req_len; i += 4)
290 writel(0, bd->bar0 + bar_offset + i);
291
292 /* Ring channel doorbell */
293 writel(1, bd->bar0 + doorbell_offset);
294
295 bnge_hwrm_req_dbg(bd, ctx->req);
296
297 /* Limit timeout to an upper limit */
298 timeout = min(ctx->timeout,
299 bd->hwrm_cmd_max_timeout ?: BNGE_HWRM_CMD_MAX_TIMEOUT);
300 /* convert timeout to usec */
301 timeout *= 1000;
302
303 i = 0;
304 /* Short timeout for the first few iterations:
305 * number of loops = number of loops for short timeout +
306 * number of loops for standard timeout.
307 */
308 tmo_count = BNGE_HWRM_SHORT_TIMEOUT_COUNTER;
309 timeout = timeout - BNGE_HWRM_SHORT_MIN_TIMEOUT *
310 BNGE_HWRM_SHORT_TIMEOUT_COUNTER;
311 tmo_count += DIV_ROUND_UP(timeout, BNGE_HWRM_MIN_TIMEOUT);
312
313 if (le16_to_cpu(ctx->req->cmpl_ring) != INVALID_HW_RING_ID) {
314 /* Wait until hwrm response cmpl interrupt is processed */
315 while (READ_ONCE(token->state) < BNGE_HWRM_COMPLETE &&
316 i++ < tmo_count) {
317 /* on first few passes, just barely sleep */
318 if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) {
319 usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT,
320 BNGE_HWRM_SHORT_MAX_TIMEOUT);
321 } else {
322 usleep_range(BNGE_HWRM_MIN_TIMEOUT,
323 BNGE_HWRM_MAX_TIMEOUT);
324 }
325 }
326
327 if (READ_ONCE(token->state) != BNGE_HWRM_COMPLETE) {
328 bnge_hwrm_err(bd, ctx, "No hwrm cmpl received: 0x%x\n",
329 req_type);
330 goto exit;
331 }
332 len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len));
333 valid = ((u8 *)ctx->resp) + len - 1;
334 } else {
335 __le16 seen_out_of_seq = ctx->req->seq_id; /* will never see */
336 int j;
337
338 /* Check if response len is updated */
339 for (i = 0; i < tmo_count; i++) {
340 if (token &&
341 READ_ONCE(token->state) == BNGE_HWRM_DEFERRED) {
342 bnge_hwrm_destroy_token(bd, token);
343 token = NULL;
344 }
345
346 len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len));
347 if (len) {
348 __le16 resp_seq = READ_ONCE(ctx->resp->seq_id);
349
350 if (resp_seq == ctx->req->seq_id)
351 break;
352 if (resp_seq != seen_out_of_seq) {
353 dev_warn(bd->dev, "Discarding out of seq response: 0x%x for msg {0x%x 0x%x}\n",
354 le16_to_cpu(resp_seq), req_type, le16_to_cpu(ctx->req->seq_id));
355 seen_out_of_seq = resp_seq;
356 }
357 }
358
359 /* on first few passes, just barely sleep */
360 if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) {
361 usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT,
362 BNGE_HWRM_SHORT_MAX_TIMEOUT);
363 } else {
364 usleep_range(BNGE_HWRM_MIN_TIMEOUT,
365 BNGE_HWRM_MAX_TIMEOUT);
366 }
367 }
368
369 if (i >= tmo_count) {
370 bnge_hwrm_err(bd, ctx,
371 "Error (timeout: %u) msg {0x%x 0x%x} len:%d\n",
372 bnge_hwrm_timeout(i), req_type,
373 le16_to_cpu(ctx->req->seq_id), len);
374 goto exit;
375 }
376
377 /* Last byte of resp contains valid bit */
378 valid = ((u8 *)ctx->resp) + len - 1;
379 for (j = 0; j < BNGE_HWRM_FIN_WAIT_USEC; ) {
380 /* make sure we read from updated DMA memory */
381 dma_rmb();
382 if (*valid)
383 break;
384 if (j < 10) {
385 udelay(1);
386 j++;
387 } else {
388 usleep_range(20, 30);
389 j += 20;
390 }
391 }
392
393 if (j >= BNGE_HWRM_FIN_WAIT_USEC) {
394 bnge_hwrm_err(bd, ctx, "Error (timeout: %u) msg {0x%x 0x%x} len:%d v:%d\n",
395 bnge_hwrm_timeout(i) + j, req_type,
396 le16_to_cpu(ctx->req->seq_id), len, *valid);
397 goto exit;
398 }
399 }
400
401 /* Zero valid bit for compatibility. Valid bit in an older spec
402 * may become a new field in a newer spec. We must make sure that
403 * a new field not implemented by old spec will read zero.
404 */
405 *valid = 0;
406 rc = le16_to_cpu(ctx->resp->error_code);
407 if (rc == HWRM_ERR_CODE_BUSY && !(ctx->flags & BNGE_HWRM_CTX_SILENT))
408 dev_warn(bd->dev, "FW returned busy, hwrm req_type 0x%x\n",
409 req_type);
410 else if (rc && rc != HWRM_ERR_CODE_PF_UNAVAILABLE)
411 bnge_hwrm_err(bd, ctx, "hwrm req_type 0x%x seq id 0x%x error %d\n",
412 req_type, le16_to_cpu(ctx->req->seq_id), rc);
413 rc = bnge_map_hwrm_error(rc);
414
415 exit:
416 if (token)
417 bnge_hwrm_destroy_token(bd, token);
418 if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED)
419 ctx->flags |= BNGE_HWRM_INTERNAL_RESP_DIRTY;
420 else
421 __hwrm_ctx_invalidate(bd, ctx);
422 return rc;
423 }
424
bnge_hwrm_req_send(struct bnge_dev * bd,void * req)425 int bnge_hwrm_req_send(struct bnge_dev *bd, void *req)
426 {
427 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
428
429 if (!ctx)
430 return -EINVAL;
431
432 return __hwrm_send_ctx(bd, ctx);
433 }
434
bnge_hwrm_req_send_silent(struct bnge_dev * bd,void * req)435 int bnge_hwrm_req_send_silent(struct bnge_dev *bd, void *req)
436 {
437 bnge_hwrm_req_flags(bd, req, BNGE_HWRM_CTX_SILENT);
438 return bnge_hwrm_req_send(bd, req);
439 }
440
441 void *
bnge_hwrm_req_dma_slice(struct bnge_dev * bd,void * req,u32 size,dma_addr_t * dma_handle)442 bnge_hwrm_req_dma_slice(struct bnge_dev *bd, void *req, u32 size,
443 dma_addr_t *dma_handle)
444 {
445 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
446 u8 *end = ((u8 *)req) + BNGE_HWRM_DMA_SIZE;
447 struct input *input = req;
448 u8 *addr, *req_addr = req;
449 u32 max_offset, offset;
450
451 if (!ctx)
452 return NULL;
453
454 max_offset = BNGE_HWRM_DMA_SIZE - ctx->allocated;
455 offset = max_offset - size;
456 offset = ALIGN_DOWN(offset, BNGE_HWRM_DMA_ALIGN);
457 addr = req_addr + offset;
458
459 if (addr < req_addr + max_offset && req_addr + ctx->req_len <= addr) {
460 ctx->allocated = end - addr;
461 *dma_handle = ctx->dma_handle + offset;
462 return addr;
463 }
464
465 if (ctx->slice_addr) {
466 dev_err(bd->dev, "HWRM refusing to reallocate DMA slice, req_type = %u\n",
467 (u32)le16_to_cpu(input->req_type));
468 dump_stack();
469 return NULL;
470 }
471
472 addr = dma_alloc_coherent(bd->dev, size, dma_handle, ctx->gfp);
473 if (!addr)
474 return NULL;
475
476 ctx->slice_addr = addr;
477 ctx->slice_size = size;
478 ctx->slice_handle = *dma_handle;
479
480 return addr;
481 }
482
bnge_cleanup_hwrm_resources(struct bnge_dev * bd)483 void bnge_cleanup_hwrm_resources(struct bnge_dev *bd)
484 {
485 struct bnge_hwrm_wait_token *token;
486
487 dma_pool_destroy(bd->hwrm_dma_pool);
488 bd->hwrm_dma_pool = NULL;
489
490 rcu_read_lock();
491 hlist_for_each_entry_rcu(token, &bd->hwrm_pending_list, node)
492 WRITE_ONCE(token->state, BNGE_HWRM_CANCELLED);
493 rcu_read_unlock();
494 }
495
bnge_init_hwrm_resources(struct bnge_dev * bd)496 int bnge_init_hwrm_resources(struct bnge_dev *bd)
497 {
498 bd->hwrm_dma_pool = dma_pool_create("bnge_hwrm", bd->dev,
499 BNGE_HWRM_DMA_SIZE,
500 BNGE_HWRM_DMA_ALIGN, 0);
501 if (!bd->hwrm_dma_pool)
502 return -ENOMEM;
503
504 INIT_HLIST_HEAD(&bd->hwrm_pending_list);
505 mutex_init(&bd->hwrm_cmd_lock);
506
507 return 0;
508 }
509