1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <asm/byteorder.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/dmapool.h> 7 #include <linux/delay.h> 8 #include <linux/errno.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 13 #include "bnge.h" 14 #include "bnge_hwrm.h" 15 16 static u64 bnge_cal_sentinel(struct bnge_hwrm_ctx *ctx, u16 req_type) 17 { 18 return (((uintptr_t)ctx) + req_type) ^ BNGE_HWRM_SENTINEL; 19 } 20 21 int bnge_hwrm_req_create(struct bnge_dev *bd, void **req, u16 req_type, 22 u32 req_len) 23 { 24 struct bnge_hwrm_ctx *ctx; 25 dma_addr_t dma_handle; 26 u8 *req_addr; 27 28 if (req_len > BNGE_HWRM_CTX_OFFSET) 29 return -E2BIG; 30 31 req_addr = dma_pool_alloc(bd->hwrm_dma_pool, GFP_KERNEL | __GFP_ZERO, 32 &dma_handle); 33 if (!req_addr) 34 return -ENOMEM; 35 36 ctx = (struct bnge_hwrm_ctx *)(req_addr + BNGE_HWRM_CTX_OFFSET); 37 /* safety first, sentinel used to check for invalid requests */ 38 ctx->sentinel = bnge_cal_sentinel(ctx, req_type); 39 ctx->req_len = req_len; 40 ctx->req = (struct input *)req_addr; 41 ctx->resp = (struct output *)(req_addr + BNGE_HWRM_RESP_OFFSET); 42 ctx->dma_handle = dma_handle; 43 ctx->flags = 0; /* __GFP_ZERO, but be explicit regarding ownership */ 44 ctx->timeout = bd->hwrm_cmd_timeout ?: BNGE_DFLT_HWRM_CMD_TIMEOUT; 45 ctx->allocated = BNGE_HWRM_DMA_SIZE - BNGE_HWRM_CTX_OFFSET; 46 ctx->gfp = GFP_KERNEL; 47 ctx->slice_addr = NULL; 48 49 /* initialize common request fields */ 50 ctx->req->req_type = cpu_to_le16(req_type); 51 ctx->req->resp_addr = cpu_to_le64(dma_handle + BNGE_HWRM_RESP_OFFSET); 52 ctx->req->cmpl_ring = cpu_to_le16(BNGE_HWRM_NO_CMPL_RING); 53 ctx->req->target_id = cpu_to_le16(BNGE_HWRM_TARGET); 54 *req = ctx->req; 55 56 return 0; 57 } 58 59 static struct bnge_hwrm_ctx *__hwrm_ctx_get(struct bnge_dev *bd, u8 *req_addr) 60 { 61 void *ctx_addr = req_addr + BNGE_HWRM_CTX_OFFSET; 62 struct input *req = (struct input *)req_addr; 63 struct bnge_hwrm_ctx *ctx = ctx_addr; 64 u64 sentinel; 65 66 if (!req) { 67 dev_err(bd->dev, "null HWRM request"); 68 dump_stack(); 69 return NULL; 70 } 71 72 /* HWRM API has no type safety, verify sentinel to validate address */ 73 sentinel = bnge_cal_sentinel(ctx, le16_to_cpu(req->req_type)); 74 if (ctx->sentinel != sentinel) { 75 dev_err(bd->dev, "HWRM sentinel mismatch, req_type = %u\n", 76 (u32)le16_to_cpu(req->req_type)); 77 dump_stack(); 78 return NULL; 79 } 80 81 return ctx; 82 } 83 84 void bnge_hwrm_req_timeout(struct bnge_dev *bd, 85 void *req, unsigned int timeout) 86 { 87 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 88 89 if (ctx) 90 ctx->timeout = timeout; 91 } 92 93 void bnge_hwrm_req_alloc_flags(struct bnge_dev *bd, void *req, gfp_t gfp) 94 { 95 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 96 97 if (ctx) 98 ctx->gfp = gfp; 99 } 100 101 void bnge_hwrm_req_flags(struct bnge_dev *bd, void *req, 102 enum bnge_hwrm_ctx_flags flags) 103 { 104 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 105 106 if (ctx) 107 ctx->flags |= (flags & BNGE_HWRM_API_FLAGS); 108 } 109 110 void *bnge_hwrm_req_hold(struct bnge_dev *bd, void *req) 111 { 112 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 113 struct input *input = (struct input *)req; 114 115 if (!ctx) 116 return NULL; 117 118 if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED) { 119 dev_err(bd->dev, "HWRM context already owned, req_type = %u\n", 120 (u32)le16_to_cpu(input->req_type)); 121 dump_stack(); 122 return NULL; 123 } 124 125 ctx->flags |= BNGE_HWRM_INTERNAL_CTX_OWNED; 126 return ((u8 *)req) + BNGE_HWRM_RESP_OFFSET; 127 } 128 129 static void __hwrm_ctx_invalidate(struct bnge_dev *bd, 130 struct bnge_hwrm_ctx *ctx) 131 { 132 void *addr = ((u8 *)ctx) - BNGE_HWRM_CTX_OFFSET; 133 dma_addr_t dma_handle = ctx->dma_handle; /* save before invalidate */ 134 135 /* unmap any auxiliary DMA slice */ 136 if (ctx->slice_addr) 137 dma_free_coherent(bd->dev, ctx->slice_size, 138 ctx->slice_addr, ctx->slice_handle); 139 140 /* invalidate, ensure ownership, sentinel and dma_handle are cleared */ 141 memset(ctx, 0, sizeof(struct bnge_hwrm_ctx)); 142 143 /* return the buffer to the DMA pool */ 144 if (dma_handle) 145 dma_pool_free(bd->hwrm_dma_pool, addr, dma_handle); 146 } 147 148 void bnge_hwrm_req_drop(struct bnge_dev *bd, void *req) 149 { 150 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 151 152 if (ctx) 153 __hwrm_ctx_invalidate(bd, ctx); 154 } 155 156 static int bnge_map_hwrm_error(u32 hwrm_err) 157 { 158 switch (hwrm_err) { 159 case HWRM_ERR_CODE_SUCCESS: 160 return 0; 161 case HWRM_ERR_CODE_RESOURCE_LOCKED: 162 return -EROFS; 163 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: 164 return -EACCES; 165 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: 166 return -ENOSPC; 167 case HWRM_ERR_CODE_INVALID_PARAMS: 168 case HWRM_ERR_CODE_INVALID_FLAGS: 169 case HWRM_ERR_CODE_INVALID_ENABLES: 170 case HWRM_ERR_CODE_UNSUPPORTED_TLV: 171 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR: 172 return -EINVAL; 173 case HWRM_ERR_CODE_NO_BUFFER: 174 return -ENOMEM; 175 case HWRM_ERR_CODE_HOT_RESET_PROGRESS: 176 case HWRM_ERR_CODE_BUSY: 177 return -EAGAIN; 178 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: 179 return -EOPNOTSUPP; 180 case HWRM_ERR_CODE_PF_UNAVAILABLE: 181 return -ENODEV; 182 default: 183 return -EIO; 184 } 185 } 186 187 static struct bnge_hwrm_wait_token * 188 bnge_hwrm_create_token(struct bnge_dev *bd, enum bnge_hwrm_chnl dst) 189 { 190 struct bnge_hwrm_wait_token *token; 191 192 token = kzalloc(sizeof(*token), GFP_KERNEL); 193 if (!token) 194 return NULL; 195 196 mutex_lock(&bd->hwrm_cmd_lock); 197 198 token->dst = dst; 199 token->state = BNGE_HWRM_PENDING; 200 if (dst == BNGE_HWRM_CHNL_CHIMP) { 201 token->seq_id = bd->hwrm_cmd_seq++; 202 hlist_add_head_rcu(&token->node, &bd->hwrm_pending_list); 203 } else { 204 token->seq_id = bd->hwrm_cmd_kong_seq++; 205 } 206 207 return token; 208 } 209 210 static void 211 bnge_hwrm_destroy_token(struct bnge_dev *bd, struct bnge_hwrm_wait_token *token) 212 { 213 if (token->dst == BNGE_HWRM_CHNL_CHIMP) { 214 hlist_del_rcu(&token->node); 215 kfree_rcu(token, rcu); 216 } else { 217 kfree(token); 218 } 219 mutex_unlock(&bd->hwrm_cmd_lock); 220 } 221 222 static void bnge_hwrm_req_dbg(struct bnge_dev *bd, struct input *req) 223 { 224 u32 ring = le16_to_cpu(req->cmpl_ring); 225 u32 type = le16_to_cpu(req->req_type); 226 u32 tgt = le16_to_cpu(req->target_id); 227 u32 seq = le16_to_cpu(req->seq_id); 228 char opt[32] = "\n"; 229 230 if (unlikely(ring != (u16)BNGE_HWRM_NO_CMPL_RING)) 231 snprintf(opt, 16, " ring %d\n", ring); 232 233 if (unlikely(tgt != BNGE_HWRM_TARGET)) 234 snprintf(opt + strlen(opt) - 1, 16, " tgt 0x%x\n", tgt); 235 236 dev_dbg(bd->dev, "sent hwrm req_type 0x%x seq id 0x%x%s", 237 type, seq, opt); 238 } 239 240 #define bnge_hwrm_err(bd, ctx, fmt, ...) \ 241 do { \ 242 if ((ctx)->flags & BNGE_HWRM_CTX_SILENT) \ 243 dev_dbg((bd)->dev, fmt, __VA_ARGS__); \ 244 else \ 245 dev_err((bd)->dev, fmt, __VA_ARGS__); \ 246 } while (0) 247 248 static int __hwrm_send_ctx(struct bnge_dev *bd, struct bnge_hwrm_ctx *ctx) 249 { 250 u32 doorbell_offset = BNGE_GRCPF_REG_CHIMP_COMM_TRIGGER; 251 enum bnge_hwrm_chnl dst = BNGE_HWRM_CHNL_CHIMP; 252 u32 bar_offset = BNGE_GRCPF_REG_CHIMP_COMM; 253 struct bnge_hwrm_wait_token *token = NULL; 254 u16 max_req_len = BNGE_HWRM_MAX_REQ_LEN; 255 unsigned int i, timeout, tmo_count; 256 u32 *data = (u32 *)ctx->req; 257 u32 msg_len = ctx->req_len; 258 int rc = -EBUSY; 259 u32 req_type; 260 u16 len = 0; 261 u8 *valid; 262 263 if (ctx->flags & BNGE_HWRM_INTERNAL_RESP_DIRTY) 264 memset(ctx->resp, 0, PAGE_SIZE); 265 266 req_type = le16_to_cpu(ctx->req->req_type); 267 268 if (msg_len > BNGE_HWRM_MAX_REQ_LEN && 269 msg_len > bd->hwrm_max_ext_req_len) { 270 dev_warn(bd->dev, "oversized hwrm request, req_type 0x%x", 271 req_type); 272 rc = -E2BIG; 273 goto exit; 274 } 275 276 token = bnge_hwrm_create_token(bd, dst); 277 if (!token) { 278 rc = -ENOMEM; 279 goto exit; 280 } 281 ctx->req->seq_id = cpu_to_le16(token->seq_id); 282 283 /* Ensure any associated DMA buffers are written before doorbell */ 284 wmb(); 285 286 /* Write request msg to hwrm channel */ 287 __iowrite32_copy(bd->bar0 + bar_offset, data, msg_len / 4); 288 289 for (i = msg_len; i < max_req_len; i += 4) 290 writel(0, bd->bar0 + bar_offset + i); 291 292 /* Ring channel doorbell */ 293 writel(1, bd->bar0 + doorbell_offset); 294 295 bnge_hwrm_req_dbg(bd, ctx->req); 296 297 /* Limit timeout to an upper limit */ 298 timeout = min(ctx->timeout, 299 bd->hwrm_cmd_max_timeout ?: BNGE_HWRM_CMD_MAX_TIMEOUT); 300 /* convert timeout to usec */ 301 timeout *= 1000; 302 303 i = 0; 304 /* Short timeout for the first few iterations: 305 * number of loops = number of loops for short timeout + 306 * number of loops for standard timeout. 307 */ 308 tmo_count = BNGE_HWRM_SHORT_TIMEOUT_COUNTER; 309 timeout = timeout - BNGE_HWRM_SHORT_MIN_TIMEOUT * 310 BNGE_HWRM_SHORT_TIMEOUT_COUNTER; 311 tmo_count += DIV_ROUND_UP(timeout, BNGE_HWRM_MIN_TIMEOUT); 312 313 if (le16_to_cpu(ctx->req->cmpl_ring) != INVALID_HW_RING_ID) { 314 /* Wait until hwrm response cmpl interrupt is processed */ 315 while (READ_ONCE(token->state) < BNGE_HWRM_COMPLETE && 316 i++ < tmo_count) { 317 /* on first few passes, just barely sleep */ 318 if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) { 319 usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT, 320 BNGE_HWRM_SHORT_MAX_TIMEOUT); 321 } else { 322 usleep_range(BNGE_HWRM_MIN_TIMEOUT, 323 BNGE_HWRM_MAX_TIMEOUT); 324 } 325 } 326 327 if (READ_ONCE(token->state) != BNGE_HWRM_COMPLETE) { 328 bnge_hwrm_err(bd, ctx, "No hwrm cmpl received: 0x%x\n", 329 req_type); 330 goto exit; 331 } 332 len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len)); 333 valid = ((u8 *)ctx->resp) + len - 1; 334 } else { 335 __le16 seen_out_of_seq = ctx->req->seq_id; /* will never see */ 336 int j; 337 338 /* Check if response len is updated */ 339 for (i = 0; i < tmo_count; i++) { 340 if (token && 341 READ_ONCE(token->state) == BNGE_HWRM_DEFERRED) { 342 bnge_hwrm_destroy_token(bd, token); 343 token = NULL; 344 } 345 346 len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len)); 347 if (len) { 348 __le16 resp_seq = READ_ONCE(ctx->resp->seq_id); 349 350 if (resp_seq == ctx->req->seq_id) 351 break; 352 if (resp_seq != seen_out_of_seq) { 353 dev_warn(bd->dev, "Discarding out of seq response: 0x%x for msg {0x%x 0x%x}\n", 354 le16_to_cpu(resp_seq), req_type, le16_to_cpu(ctx->req->seq_id)); 355 seen_out_of_seq = resp_seq; 356 } 357 } 358 359 /* on first few passes, just barely sleep */ 360 if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) { 361 usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT, 362 BNGE_HWRM_SHORT_MAX_TIMEOUT); 363 } else { 364 usleep_range(BNGE_HWRM_MIN_TIMEOUT, 365 BNGE_HWRM_MAX_TIMEOUT); 366 } 367 } 368 369 if (i >= tmo_count) { 370 bnge_hwrm_err(bd, ctx, 371 "Error (timeout: %u) msg {0x%x 0x%x} len:%d\n", 372 bnge_hwrm_timeout(i), req_type, 373 le16_to_cpu(ctx->req->seq_id), len); 374 goto exit; 375 } 376 377 /* Last byte of resp contains valid bit */ 378 valid = ((u8 *)ctx->resp) + len - 1; 379 for (j = 0; j < BNGE_HWRM_FIN_WAIT_USEC; ) { 380 /* make sure we read from updated DMA memory */ 381 dma_rmb(); 382 if (*valid) 383 break; 384 if (j < 10) { 385 udelay(1); 386 j++; 387 } else { 388 usleep_range(20, 30); 389 j += 20; 390 } 391 } 392 393 if (j >= BNGE_HWRM_FIN_WAIT_USEC) { 394 bnge_hwrm_err(bd, ctx, "Error (timeout: %u) msg {0x%x 0x%x} len:%d v:%d\n", 395 bnge_hwrm_timeout(i) + j, req_type, 396 le16_to_cpu(ctx->req->seq_id), len, *valid); 397 goto exit; 398 } 399 } 400 401 /* Zero valid bit for compatibility. Valid bit in an older spec 402 * may become a new field in a newer spec. We must make sure that 403 * a new field not implemented by old spec will read zero. 404 */ 405 *valid = 0; 406 rc = le16_to_cpu(ctx->resp->error_code); 407 if (rc == HWRM_ERR_CODE_BUSY && !(ctx->flags & BNGE_HWRM_CTX_SILENT)) 408 dev_warn(bd->dev, "FW returned busy, hwrm req_type 0x%x\n", 409 req_type); 410 else if (rc && rc != HWRM_ERR_CODE_PF_UNAVAILABLE) 411 bnge_hwrm_err(bd, ctx, "hwrm req_type 0x%x seq id 0x%x error %d\n", 412 req_type, le16_to_cpu(ctx->req->seq_id), rc); 413 rc = bnge_map_hwrm_error(rc); 414 415 exit: 416 if (token) 417 bnge_hwrm_destroy_token(bd, token); 418 if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED) 419 ctx->flags |= BNGE_HWRM_INTERNAL_RESP_DIRTY; 420 else 421 __hwrm_ctx_invalidate(bd, ctx); 422 return rc; 423 } 424 425 int bnge_hwrm_req_send(struct bnge_dev *bd, void *req) 426 { 427 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 428 429 if (!ctx) 430 return -EINVAL; 431 432 return __hwrm_send_ctx(bd, ctx); 433 } 434 435 int bnge_hwrm_req_send_silent(struct bnge_dev *bd, void *req) 436 { 437 bnge_hwrm_req_flags(bd, req, BNGE_HWRM_CTX_SILENT); 438 return bnge_hwrm_req_send(bd, req); 439 } 440 441 void * 442 bnge_hwrm_req_dma_slice(struct bnge_dev *bd, void *req, u32 size, 443 dma_addr_t *dma_handle) 444 { 445 struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req); 446 u8 *end = ((u8 *)req) + BNGE_HWRM_DMA_SIZE; 447 struct input *input = req; 448 u8 *addr, *req_addr = req; 449 u32 max_offset, offset; 450 451 if (!ctx) 452 return NULL; 453 454 max_offset = BNGE_HWRM_DMA_SIZE - ctx->allocated; 455 offset = max_offset - size; 456 offset = ALIGN_DOWN(offset, BNGE_HWRM_DMA_ALIGN); 457 addr = req_addr + offset; 458 459 if (addr < req_addr + max_offset && req_addr + ctx->req_len <= addr) { 460 ctx->allocated = end - addr; 461 *dma_handle = ctx->dma_handle + offset; 462 return addr; 463 } 464 465 if (ctx->slice_addr) { 466 dev_err(bd->dev, "HWRM refusing to reallocate DMA slice, req_type = %u\n", 467 (u32)le16_to_cpu(input->req_type)); 468 dump_stack(); 469 return NULL; 470 } 471 472 addr = dma_alloc_coherent(bd->dev, size, dma_handle, ctx->gfp); 473 if (!addr) 474 return NULL; 475 476 ctx->slice_addr = addr; 477 ctx->slice_size = size; 478 ctx->slice_handle = *dma_handle; 479 480 return addr; 481 } 482 483 void bnge_cleanup_hwrm_resources(struct bnge_dev *bd) 484 { 485 struct bnge_hwrm_wait_token *token; 486 487 dma_pool_destroy(bd->hwrm_dma_pool); 488 bd->hwrm_dma_pool = NULL; 489 490 rcu_read_lock(); 491 hlist_for_each_entry_rcu(token, &bd->hwrm_pending_list, node) 492 WRITE_ONCE(token->state, BNGE_HWRM_CANCELLED); 493 rcu_read_unlock(); 494 } 495 496 int bnge_init_hwrm_resources(struct bnge_dev *bd) 497 { 498 bd->hwrm_dma_pool = dma_pool_create("bnge_hwrm", bd->dev, 499 BNGE_HWRM_DMA_SIZE, 500 BNGE_HWRM_DMA_ALIGN, 0); 501 if (!bd->hwrm_dma_pool) 502 return -ENOMEM; 503 504 INIT_HLIST_HEAD(&bd->hwrm_pending_list); 505 mutex_init(&bd->hwrm_cmd_lock); 506 507 return 0; 508 } 509