1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause 2 /* 3 * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/crc16.h> 7 #include <linux/log2.h> 8 9 #include "efa_com.h" 10 #include "efa_regs_defs.h" 11 12 #define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */ 13 14 #define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */ 15 #define EFA_MMIO_READ_INVALID 0xffffffff 16 17 #define EFA_POLL_INTERVAL_MS 100 /* msecs */ 18 19 #define EFA_ASYNC_QUEUE_DEPTH 16 20 #define EFA_ADMIN_QUEUE_DEPTH 32 21 22 #define EFA_CTRL_MAJOR 0 23 #define EFA_CTRL_MINOR 0 24 #define EFA_CTRL_SUB_MINOR 1 25 26 #define EFA_CRC16_INIT_VAL 0xffff 27 28 #define EFA_ADMIN_SQ_MAX_ENT_SIZE sizeof(struct efa_admin_aq_entry_v2) 29 30 #define EFA_CRC_MIN_API_VERSION_MAJOR 0 31 #define EFA_CRC_MIN_API_VERSION_MINOR 2 32 33 #define EFA_ADMIN_V2_MIN_API_VERSION_MAJOR 0 34 #define EFA_ADMIN_V2_MIN_API_VERSION_MINOR 3 35 36 #define EFA_MIN_API_VERSION_MAJOR 0 37 #define EFA_MIN_API_VERSION_MINOR 1 38 39 enum efa_cmd_status { 40 EFA_CMD_UNUSED, 41 EFA_CMD_ALLOCATED, 42 EFA_CMD_SUBMITTED, 43 EFA_CMD_COMPLETED, 44 }; 45 46 struct efa_comp_ctx { 47 struct completion wait_event; 48 struct efa_admin_acq_entry *user_cqe; 49 u32 comp_size; 50 enum efa_cmd_status status; 51 u16 cmd_id; 52 u8 cmd_opcode; 53 }; 54 55 static const char *efa_com_cmd_str(u8 cmd) 56 { 57 #define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd 58 59 switch (cmd) { 60 EFA_CMD_STR_CASE(CREATE_QP); 61 EFA_CMD_STR_CASE(MODIFY_QP); 62 EFA_CMD_STR_CASE(QUERY_QP); 63 EFA_CMD_STR_CASE(DESTROY_QP); 64 EFA_CMD_STR_CASE(CREATE_AH); 65 EFA_CMD_STR_CASE(DESTROY_AH); 66 EFA_CMD_STR_CASE(REG_MR); 67 EFA_CMD_STR_CASE(DEREG_MR); 68 EFA_CMD_STR_CASE(CREATE_CQ); 69 EFA_CMD_STR_CASE(DESTROY_CQ); 70 EFA_CMD_STR_CASE(GET_FEATURE); 71 EFA_CMD_STR_CASE(SET_FEATURE); 72 EFA_CMD_STR_CASE(GET_STATS); 73 EFA_CMD_STR_CASE(ALLOC_PD); 74 EFA_CMD_STR_CASE(DEALLOC_PD); 75 EFA_CMD_STR_CASE(ALLOC_UAR); 76 EFA_CMD_STR_CASE(DEALLOC_UAR); 77 EFA_CMD_STR_CASE(CREATE_EQ); 78 EFA_CMD_STR_CASE(DESTROY_EQ); 79 default: return "unknown command opcode"; 80 } 81 #undef EFA_CMD_STR_CASE 82 } 83 84 void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low) 85 { 86 *addr_low = lower_32_bits(addr); 87 *addr_high = upper_32_bits(addr); 88 } 89 90 static u32 efa_com_construct_ver(u32 major, u32 minor) 91 { 92 u32 ver = 0; 93 94 EFA_SET(&ver, EFA_REGS_VERSION_MAJOR_VERSION, major); 95 EFA_SET(&ver, EFA_REGS_VERSION_MINOR_VERSION, minor); 96 97 return ver; 98 } 99 100 static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset) 101 { 102 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 103 struct efa_admin_mmio_req_read_less_resp *read_resp; 104 unsigned long exp_time; 105 u32 mmio_read_reg = 0; 106 u32 err; 107 108 read_resp = mmio_read->read_resp; 109 110 spin_lock(&mmio_read->lock); 111 mmio_read->seq_num++; 112 113 /* trash DMA req_id to identify when hardware is done */ 114 read_resp->req_id = mmio_read->seq_num + 0x9aL; 115 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset); 116 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID, 117 mmio_read->seq_num); 118 119 writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF); 120 121 exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout); 122 do { 123 if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num) 124 break; 125 udelay(1); 126 } while (time_is_after_jiffies(exp_time)); 127 128 if (read_resp->req_id != mmio_read->seq_num) { 129 ibdev_err_ratelimited( 130 edev->efa_dev, 131 "Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n", 132 mmio_read->seq_num, offset, read_resp->req_id, 133 read_resp->reg_off); 134 err = EFA_MMIO_READ_INVALID; 135 goto out; 136 } 137 138 if (read_resp->reg_off != offset) { 139 ibdev_err_ratelimited( 140 edev->efa_dev, 141 "Reading register failed: wrong offset provided\n"); 142 err = EFA_MMIO_READ_INVALID; 143 goto out; 144 } 145 146 err = read_resp->reg_val; 147 out: 148 spin_unlock(&mmio_read->lock); 149 return err; 150 } 151 152 static int efa_com_admin_init_sq(struct efa_com_dev *edev) 153 { 154 struct efa_com_admin_queue *aq = &edev->aq; 155 struct efa_com_admin_sq *sq = &aq->sq; 156 u32 aq_caps = 0, admin_v2_min_ver = 0; 157 u32 addr_high, addr_low; 158 159 admin_v2_min_ver = efa_com_construct_ver(EFA_ADMIN_V2_MIN_API_VERSION_MAJOR, 160 EFA_ADMIN_V2_MIN_API_VERSION_MINOR); 161 if (edev->dev_api_ver >= admin_v2_min_ver) { 162 sq->entry_size = sizeof(struct efa_admin_aq_entry_v2); 163 sq->payload_offset = offsetof(struct efa_admin_aq_entry_v2, request_payload); 164 sq->proto_ver = EFA_ADMIN_V2_PROTO_VER; 165 } else { 166 sq->entry_size = sizeof(struct efa_admin_aq_entry); 167 sq->payload_offset = offsetof(struct efa_admin_aq_entry, request_payload); 168 sq->proto_ver = EFA_ADMIN_V1_PROTO_VER; 169 } 170 171 sq->max_payload_size = sq->entry_size - sq->payload_offset; 172 173 sq->buffer = dma_alloc_coherent(aq->dmadev, aq->depth * sq->entry_size, 174 &sq->dma_addr, GFP_KERNEL); 175 if (!sq->buffer) 176 return -ENOMEM; 177 178 spin_lock_init(&sq->lock); 179 180 sq->cc = 0; 181 sq->pc = 0; 182 sq->phase = 1; 183 184 sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF); 185 186 addr_high = upper_32_bits(sq->dma_addr); 187 addr_low = lower_32_bits(sq->dma_addr); 188 189 writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF); 190 writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF); 191 192 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth); 193 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE, sq->entry_size); 194 195 writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF); 196 197 return 0; 198 } 199 200 static int efa_com_admin_init_cq(struct efa_com_dev *edev) 201 { 202 struct efa_com_admin_queue *aq = &edev->aq; 203 struct efa_com_admin_cq *cq = &aq->cq; 204 u16 size = aq->depth * sizeof(*cq->entries); 205 u32 acq_caps = 0, crc_min_ver = 0; 206 u32 addr_high, addr_low; 207 208 cq->entries = 209 dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); 210 if (!cq->entries) 211 return -ENOMEM; 212 213 spin_lock_init(&cq->lock); 214 215 crc_min_ver = efa_com_construct_ver(EFA_CRC_MIN_API_VERSION_MAJOR, 216 EFA_CRC_MIN_API_VERSION_MINOR); 217 if (edev->dev_api_ver >= crc_min_ver) 218 cq->validate_checksum = true; 219 220 cq->cc = 0; 221 cq->phase = 1; 222 223 addr_high = upper_32_bits(cq->dma_addr); 224 addr_low = lower_32_bits(cq->dma_addr); 225 226 writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF); 227 writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF); 228 229 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth); 230 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE, 231 sizeof(struct efa_admin_acq_entry)); 232 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR, 233 aq->msix_vector_idx); 234 235 writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF); 236 237 return 0; 238 } 239 240 static int efa_com_admin_init_aenq(struct efa_com_dev *edev, 241 struct efa_aenq_handlers *aenq_handlers) 242 { 243 struct efa_com_aenq *aenq = &edev->aenq; 244 u32 addr_low, addr_high; 245 u32 aenq_caps = 0; 246 u16 size; 247 248 if (!aenq_handlers) { 249 ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n"); 250 return -EINVAL; 251 } 252 253 size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries); 254 aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr, 255 GFP_KERNEL); 256 if (!aenq->entries) 257 return -ENOMEM; 258 259 aenq->aenq_handlers = aenq_handlers; 260 aenq->depth = EFA_ASYNC_QUEUE_DEPTH; 261 aenq->cc = 0; 262 aenq->phase = 1; 263 264 addr_low = lower_32_bits(aenq->dma_addr); 265 addr_high = upper_32_bits(aenq->dma_addr); 266 267 writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF); 268 writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF); 269 270 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth); 271 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE, 272 sizeof(struct efa_admin_aenq_entry)); 273 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR, 274 aenq->msix_vector_idx); 275 writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF); 276 277 /* 278 * Init cons_db to mark that all entries in the queue 279 * are initially available 280 */ 281 writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 282 283 return 0; 284 } 285 286 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) 287 { 288 u16 ctx_id; 289 290 spin_lock(&aq->comp_ctx_lock); 291 ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next]; 292 aq->comp_ctx_pool_next++; 293 spin_unlock(&aq->comp_ctx_lock); 294 295 return ctx_id; 296 } 297 298 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, 299 u16 ctx_id) 300 { 301 spin_lock(&aq->comp_ctx_lock); 302 aq->comp_ctx_pool_next--; 303 aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id; 304 spin_unlock(&aq->comp_ctx_lock); 305 } 306 307 static struct efa_comp_ctx *efa_com_alloc_comp_ctx(struct efa_com_admin_queue *aq) 308 { 309 struct efa_comp_ctx *comp_ctx; 310 u16 ctx_id; 311 312 ctx_id = efa_com_alloc_ctx_id(aq); 313 314 comp_ctx = &aq->comp_ctx[ctx_id]; 315 if (comp_ctx->status != EFA_CMD_UNUSED) { 316 efa_com_dealloc_ctx_id(aq, ctx_id); 317 ibdev_err_ratelimited(aq->efa_dev, 318 "Completion context[%u] is used[%u]\n", 319 ctx_id, comp_ctx->status); 320 return NULL; 321 } 322 323 comp_ctx->status = EFA_CMD_ALLOCATED; 324 ibdev_dbg(aq->efa_dev, "Take completion context[%u]\n", ctx_id); 325 return comp_ctx; 326 } 327 328 static inline u16 efa_com_get_comp_ctx_id(struct efa_com_admin_queue *aq, 329 struct efa_comp_ctx *comp_ctx) 330 { 331 return comp_ctx - aq->comp_ctx; 332 } 333 334 static inline void efa_com_dealloc_comp_ctx(struct efa_com_admin_queue *aq, 335 struct efa_comp_ctx *comp_ctx) 336 { 337 u16 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 338 339 ibdev_dbg(aq->efa_dev, "Put completion context[%u]\n", ctx_id); 340 comp_ctx->status = EFA_CMD_UNUSED; 341 efa_com_dealloc_ctx_id(aq, ctx_id); 342 } 343 344 static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com_admin_queue *aq, 345 u16 cmd_id) 346 { 347 u16 ctx_id = cmd_id & (aq->depth - 1); 348 349 return &aq->comp_ctx[ctx_id]; 350 } 351 352 static u16 efa_com_calc_crc16_checksum(u8 *buff, u32 buff_size) 353 { 354 return crc16(EFA_CRC16_INIT_VAL, buff, buff_size) ^ EFA_CRC16_INIT_VAL; 355 } 356 357 static void efa_com_construct_aq_entry(struct efa_com_admin_queue *aq, u8 *aq_entry, u16 cmd_id, 358 u8 opcode, u8 flags, void *payload, size_t payload_size) 359 { 360 struct efa_admin_aq_common_desc_v2 *common_v2 = NULL; 361 struct efa_admin_aq_common_desc *common; 362 struct efa_com_admin_sq *sq = &aq->sq; 363 364 if (sq->proto_ver == EFA_ADMIN_V1_PROTO_VER) { 365 common = (struct efa_admin_aq_common_desc *)aq_entry; 366 } else { 367 common_v2 = (struct efa_admin_aq_common_desc_v2 *)aq_entry; 368 common = &common_v2->common; 369 } 370 371 common->command_id = cmd_id; 372 common->opcode = opcode; 373 common->flags = flags; 374 EFA_SET(&common->flags, EFA_ADMIN_AQ_COMMON_DESC_PHASE, sq->phase); 375 376 if (payload) 377 memcpy(aq_entry + sq->payload_offset, payload, payload_size); 378 379 if (common_v2) 380 common_v2->checksum = efa_com_calc_crc16_checksum(aq_entry, sq->entry_size); 381 } 382 383 static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 384 struct efa_comp_ctx *comp_ctx, 385 u8 opcode, u8 flags, 386 void *payload, size_t payload_size, 387 struct efa_admin_acq_entry *comp, 388 size_t comp_size_in_bytes) 389 { 390 u8 aq_entry[EFA_ADMIN_SQ_MAX_ENT_SIZE] __aligned(sizeof(u64)) = {}; 391 u16 queue_size_mask, cmd_id, ctx_id, pi; 392 struct efa_com_admin_sq *sq = &aq->sq; 393 u8 *aqe; 394 395 queue_size_mask = aq->depth - 1; 396 pi = sq->pc & queue_size_mask; 397 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 398 399 /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ 400 cmd_id = ctx_id & queue_size_mask; 401 cmd_id |= sq->pc << ilog2(aq->depth); 402 cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; 403 404 efa_com_construct_aq_entry(aq, aq_entry, cmd_id, opcode, flags, payload, payload_size); 405 406 comp_ctx->status = EFA_CMD_SUBMITTED; 407 comp_ctx->comp_size = comp_size_in_bytes; 408 comp_ctx->user_cqe = comp; 409 comp_ctx->cmd_opcode = opcode; 410 comp_ctx->cmd_id = cmd_id; 411 412 reinit_completion(&comp_ctx->wait_event); 413 414 aqe = sq->buffer + sq->entry_size * pi; 415 memset(aqe, 0, sq->entry_size); 416 memcpy(aqe, aq_entry, sq->entry_size); 417 418 sq->pc++; 419 atomic64_inc(&aq->stats.submitted_cmd); 420 421 if ((sq->pc & queue_size_mask) == 0) 422 sq->phase = !sq->phase; 423 424 /* barrier not needed in case of writel */ 425 writel(sq->pc, sq->db_addr); 426 } 427 428 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) 429 { 430 size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool); 431 size_t size = aq->depth * sizeof(struct efa_comp_ctx); 432 struct efa_comp_ctx *comp_ctx; 433 u16 i; 434 435 aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL); 436 aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL); 437 if (!aq->comp_ctx || !aq->comp_ctx_pool) { 438 devm_kfree(aq->dmadev, aq->comp_ctx_pool); 439 devm_kfree(aq->dmadev, aq->comp_ctx); 440 return -ENOMEM; 441 } 442 443 for (i = 0; i < aq->depth; i++) { 444 comp_ctx = &aq->comp_ctx[i]; 445 comp_ctx->status = EFA_CMD_UNUSED; 446 init_completion(&comp_ctx->wait_event); 447 448 aq->comp_ctx_pool[i] = i; 449 } 450 451 spin_lock_init(&aq->comp_ctx_lock); 452 453 aq->comp_ctx_pool_next = 0; 454 455 return 0; 456 } 457 458 static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 459 struct efa_comp_ctx *comp_ctx, 460 u8 opcode, u8 flags, 461 void *payload, size_t payload_size, 462 struct efa_admin_acq_entry *comp, 463 size_t comp_size_in_bytes) 464 { 465 spin_lock(&aq->sq.lock); 466 if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { 467 ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); 468 spin_unlock(&aq->sq.lock); 469 return -ENODEV; 470 } 471 472 __efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, 473 payload_size, comp, comp_size_in_bytes); 474 spin_unlock(&aq->sq.lock); 475 476 return 0; 477 } 478 479 static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq, 480 struct efa_admin_acq_entry *cqe) 481 { 482 u16 cqe_checksum = cqe->acq_common_descriptor.checksum; 483 u16 calc_checksum; 484 485 cqe->acq_common_descriptor.checksum = 0; 486 487 calc_checksum = efa_com_calc_crc16_checksum((u8 *)cqe, sizeof(*cqe)); 488 if (calc_checksum != cqe_checksum) { 489 ibdev_err(aq->efa_dev, 490 "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 491 cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc, 492 aq->cq.cc); 493 return false; 494 } 495 496 return true; 497 } 498 499 static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, 500 struct efa_admin_acq_entry *cqe) 501 { 502 struct efa_comp_ctx *comp_ctx; 503 u16 cmd_id; 504 505 if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe)) 506 return -EINVAL; 507 508 cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, 509 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); 510 511 comp_ctx = efa_com_get_comp_ctx_by_cmd_id(aq, cmd_id); 512 if (comp_ctx->status != EFA_CMD_SUBMITTED || comp_ctx->cmd_id != cmd_id) { 513 ibdev_err(aq->efa_dev, 514 "Received completion with unexpected command id[%x], status[%d] sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 515 cmd_id, comp_ctx->status, aq->sq.pc, aq->sq.cc, 516 aq->cq.cc); 517 return -EINVAL; 518 } 519 520 comp_ctx->status = EFA_CMD_COMPLETED; 521 memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size); 522 523 if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 524 complete(&comp_ctx->wait_event); 525 526 return 0; 527 } 528 529 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq) 530 { 531 struct efa_admin_acq_entry *cqe; 532 u16 queue_size_mask; 533 u16 comp_cmds = 0; 534 u8 phase; 535 int err; 536 u16 ci; 537 538 queue_size_mask = aq->depth - 1; 539 540 ci = aq->cq.cc & queue_size_mask; 541 phase = aq->cq.phase; 542 543 cqe = &aq->cq.entries[ci]; 544 545 /* Go over all the completions */ 546 while ((READ_ONCE(cqe->acq_common_descriptor.flags) & 547 EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { 548 /* 549 * Do not read the rest of the completion entry before the 550 * phase bit was validated 551 */ 552 dma_rmb(); 553 err = efa_com_handle_single_admin_completion(aq, cqe); 554 if (!err) 555 comp_cmds++; 556 557 aq->cq.cc++; 558 ci++; 559 if (ci == aq->depth) { 560 ci = 0; 561 phase = !phase; 562 } 563 564 cqe = &aq->cq.entries[ci]; 565 } 566 567 aq->cq.phase = phase; 568 aq->sq.cc += comp_cmds; 569 atomic64_add(comp_cmds, &aq->stats.completed_cmd); 570 } 571 572 static int efa_com_comp_status_to_errno(u8 comp_status) 573 { 574 switch (comp_status) { 575 case EFA_ADMIN_SUCCESS: 576 return 0; 577 case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE: 578 return -ENOMEM; 579 case EFA_ADMIN_UNSUPPORTED_OPCODE: 580 return -EOPNOTSUPP; 581 case EFA_ADMIN_BAD_OPCODE: 582 case EFA_ADMIN_MALFORMED_REQUEST: 583 case EFA_ADMIN_ILLEGAL_PARAMETER: 584 case EFA_ADMIN_UNKNOWN_ERROR: 585 return -EINVAL; 586 default: 587 return -EINVAL; 588 } 589 } 590 591 static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx, 592 struct efa_com_admin_queue *aq) 593 { 594 unsigned long timeout; 595 unsigned long flags; 596 597 timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); 598 599 while (1) { 600 spin_lock_irqsave(&aq->cq.lock, flags); 601 efa_com_handle_admin_completion(aq); 602 spin_unlock_irqrestore(&aq->cq.lock, flags); 603 604 if (comp_ctx->status != EFA_CMD_SUBMITTED) 605 break; 606 607 if (time_is_before_jiffies(timeout)) { 608 ibdev_err_ratelimited( 609 aq->efa_dev, 610 "Wait for completion (polling) timeout\n"); 611 /* EFA didn't have any completion */ 612 atomic64_inc(&aq->stats.no_completion); 613 614 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 615 return -ETIME; 616 } 617 618 msleep(aq->poll_interval); 619 } 620 621 return efa_com_comp_status_to_errno( 622 comp_ctx->user_cqe->acq_common_descriptor.status); 623 } 624 625 static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, 626 struct efa_com_admin_queue *aq) 627 { 628 unsigned long flags; 629 630 wait_for_completion_timeout(&comp_ctx->wait_event, 631 usecs_to_jiffies(aq->completion_timeout)); 632 633 /* 634 * In case the command wasn't completed find out the root cause. 635 * There might be 2 kinds of errors 636 * 1) No completion (timeout reached) 637 * 2) There is completion but the device didn't get any msi-x interrupt. 638 */ 639 if (comp_ctx->status == EFA_CMD_SUBMITTED) { 640 spin_lock_irqsave(&aq->cq.lock, flags); 641 efa_com_handle_admin_completion(aq); 642 spin_unlock_irqrestore(&aq->cq.lock, flags); 643 644 atomic64_inc(&aq->stats.no_completion); 645 646 if (comp_ctx->status == EFA_CMD_COMPLETED) 647 ibdev_err_ratelimited( 648 aq->efa_dev, 649 "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 650 efa_com_cmd_str(comp_ctx->cmd_opcode), 651 comp_ctx->cmd_opcode, comp_ctx->status, 652 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 653 aq->cq.cc); 654 else 655 ibdev_err_ratelimited( 656 aq->efa_dev, 657 "The device didn't send any completion for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 658 efa_com_cmd_str(comp_ctx->cmd_opcode), 659 comp_ctx->cmd_opcode, comp_ctx->status, 660 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 661 aq->cq.cc); 662 663 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 664 return -ETIME; 665 } 666 667 return efa_com_comp_status_to_errno( 668 comp_ctx->user_cqe->acq_common_descriptor.status); 669 } 670 671 /* 672 * There are two types to wait for completion. 673 * Polling mode - wait until the completion is available. 674 * Async mode - wait on wait queue until the completion is ready 675 * (or the timeout expired). 676 * It is expected that the IRQ called efa_com_handle_admin_completion 677 * to mark the completions. 678 */ 679 static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx, 680 struct efa_com_admin_queue *aq) 681 { 682 if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 683 return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq); 684 685 return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq); 686 } 687 688 /** 689 * efa_com_cmd_exec - Execute admin command 690 * @aq: admin queue. 691 * @opcode: the admin command opcode. 692 * @flags: the admin command header flags. 693 * @payload: the admin command payload. 694 * @payload_size: the payload size. 695 * @comp: command completion return entry. 696 * @comp_size: command completion size. 697 * Submit an admin command and then wait until the device will return a 698 * completion. 699 * The completion will be copied into comp. 700 * 701 * @return - 0 on success, negative value on failure. 702 */ 703 int efa_com_cmd_exec(struct efa_com_admin_queue *aq, 704 u8 opcode, u8 flags, 705 void *payload, size_t payload_size, 706 struct efa_admin_acq_entry *comp, size_t comp_size) 707 { 708 struct efa_comp_ctx *comp_ctx; 709 int err; 710 711 if (payload_size > aq->sq.max_payload_size) 712 return -EINVAL; 713 714 might_sleep(); 715 716 /* In case of queue FULL */ 717 down(&aq->avail_cmds); 718 719 ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", efa_com_cmd_str(opcode), 720 opcode); 721 722 comp_ctx = efa_com_alloc_comp_ctx(aq); 723 if (!comp_ctx) { 724 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 725 up(&aq->avail_cmds); 726 return -EINVAL; 727 } 728 729 err = efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, payload_size, comp, 730 comp_size); 731 if (err) { 732 ibdev_err_ratelimited( 733 aq->efa_dev, 734 "Failed to submit command %s (opcode %u) err %d\n", 735 efa_com_cmd_str(opcode), opcode, err); 736 737 efa_com_dealloc_comp_ctx(aq, comp_ctx); 738 up(&aq->avail_cmds); 739 atomic64_inc(&aq->stats.cmd_err); 740 return err; 741 } 742 743 err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); 744 if (err) { 745 ibdev_err_ratelimited( 746 aq->efa_dev, 747 "Failed to process command %s (opcode %u) err %d\n", 748 efa_com_cmd_str(opcode), opcode, err); 749 atomic64_inc(&aq->stats.cmd_err); 750 } 751 752 efa_com_dealloc_comp_ctx(aq, comp_ctx); 753 up(&aq->avail_cmds); 754 755 return err; 756 } 757 758 /** 759 * efa_com_admin_destroy - Destroy the admin and the async events queues. 760 * @edev: EFA communication layer struct 761 */ 762 void efa_com_admin_destroy(struct efa_com_dev *edev) 763 { 764 struct efa_com_admin_queue *aq = &edev->aq; 765 struct efa_com_aenq *aenq = &edev->aenq; 766 struct efa_com_admin_cq *cq = &aq->cq; 767 struct efa_com_admin_sq *sq = &aq->sq; 768 u16 size; 769 770 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 771 772 devm_kfree(edev->dmadev, aq->comp_ctx_pool); 773 devm_kfree(edev->dmadev, aq->comp_ctx); 774 775 size = aq->depth * sq->entry_size; 776 dma_free_coherent(edev->dmadev, size, sq->buffer, sq->dma_addr); 777 778 size = aq->depth * sizeof(*cq->entries); 779 dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr); 780 781 size = aenq->depth * sizeof(*aenq->entries); 782 dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr); 783 784 efa_ah_cache_destroy(&edev->ah_cache); 785 } 786 787 /** 788 * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode 789 * @edev: EFA communication layer struct 790 * @polling: Enable/Disable polling mode 791 * 792 * Set the admin completion mode. 793 */ 794 void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling) 795 { 796 u32 mask_value = 0; 797 798 if (polling) 799 EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1); 800 801 writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF); 802 if (polling) 803 set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 804 else 805 clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 806 } 807 808 static void efa_com_stats_init(struct efa_com_dev *edev) 809 { 810 atomic64_t *s = (atomic64_t *)&edev->aq.stats; 811 int i; 812 813 for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++) 814 atomic64_set(s, 0); 815 } 816 817 /** 818 * efa_com_admin_init - Init the admin and the async queues 819 * @edev: EFA communication layer struct 820 * @aenq_handlers: Those handlers to be called upon event. 821 * 822 * Initialize the admin submission and completion queues. 823 * Initialize the asynchronous events notification queues. 824 * 825 * @return - 0 on success, negative value on failure. 826 */ 827 int efa_com_admin_init(struct efa_com_dev *edev, 828 struct efa_aenq_handlers *aenq_handlers) 829 { 830 struct efa_com_admin_queue *aq = &edev->aq; 831 u32 timeout; 832 u32 dev_sts; 833 u32 cap; 834 int err; 835 836 dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 837 if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) { 838 ibdev_err(edev->efa_dev, 839 "Device isn't ready, abort com init %#x\n", dev_sts); 840 return -ENODEV; 841 } 842 843 err = efa_ah_cache_init(&edev->ah_cache); 844 if (err) { 845 ibdev_err(edev->efa_dev, "Failed to init AH cache\n"); 846 return err; 847 } 848 849 aq->depth = EFA_ADMIN_QUEUE_DEPTH; 850 851 aq->dmadev = edev->dmadev; 852 aq->efa_dev = edev->efa_dev; 853 set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state); 854 855 sema_init(&aq->avail_cmds, aq->depth); 856 857 efa_com_stats_init(edev); 858 859 err = efa_com_init_comp_ctxt(aq); 860 if (err) 861 goto err_destroy_ah_cache; 862 863 err = efa_com_admin_init_sq(edev); 864 if (err) 865 goto err_destroy_comp_ctxt; 866 867 err = efa_com_admin_init_cq(edev); 868 if (err) 869 goto err_destroy_sq; 870 871 efa_com_set_admin_polling_mode(edev, false); 872 873 err = efa_com_admin_init_aenq(edev, aenq_handlers); 874 if (err) 875 goto err_destroy_cq; 876 877 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 878 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 879 if (timeout) 880 /* the resolution of timeout reg is 100ms */ 881 aq->completion_timeout = timeout * 100000; 882 else 883 aq->completion_timeout = ADMIN_CMD_TIMEOUT_US; 884 885 aq->poll_interval = EFA_POLL_INTERVAL_MS; 886 887 set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 888 889 return 0; 890 891 err_destroy_cq: 892 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries), 893 aq->cq.entries, aq->cq.dma_addr); 894 err_destroy_sq: 895 dma_free_coherent(edev->dmadev, aq->depth * aq->sq.entry_size, 896 aq->sq.buffer, aq->sq.dma_addr); 897 err_destroy_comp_ctxt: 898 devm_kfree(edev->dmadev, aq->comp_ctx); 899 err_destroy_ah_cache: 900 efa_ah_cache_destroy(&edev->ah_cache); 901 902 return err; 903 } 904 905 /** 906 * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler 907 * @edev: EFA communication layer struct 908 * 909 * This method goes over the admin completion queue and wakes up 910 * all the pending threads that wait on the commands wait event. 911 * 912 * Note: Should be called after MSI-X interrupt. 913 */ 914 void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev) 915 { 916 unsigned long flags; 917 918 spin_lock_irqsave(&edev->aq.cq.lock, flags); 919 efa_com_handle_admin_completion(&edev->aq); 920 spin_unlock_irqrestore(&edev->aq.cq.lock, flags); 921 } 922 923 /* 924 * efa_handle_specific_aenq_event: 925 * return the handler that is relevant to the specific event group 926 */ 927 static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev, 928 u16 group) 929 { 930 struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers; 931 932 if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group]) 933 return aenq_handlers->handlers[group]; 934 935 return aenq_handlers->unimplemented_handler; 936 } 937 938 /** 939 * efa_com_aenq_intr_handler - AENQ interrupt handler 940 * @edev: EFA communication layer struct 941 * @data: Data of interrupt handler. 942 * 943 * Go over the async event notification queue and call the proper aenq handler. 944 */ 945 void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data) 946 { 947 struct efa_admin_aenq_common_desc *aenq_common; 948 struct efa_com_aenq *aenq = &edev->aenq; 949 struct efa_admin_aenq_entry *aenq_e; 950 efa_aenq_handler handler_cb; 951 u32 processed = 0; 952 u8 phase; 953 u32 ci; 954 955 ci = aenq->cc & (aenq->depth - 1); 956 phase = aenq->phase; 957 aenq_e = &aenq->entries[ci]; /* Get first entry */ 958 aenq_common = &aenq_e->aenq_common_desc; 959 960 /* Go over all the events */ 961 while ((READ_ONCE(aenq_common->flags) & 962 EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { 963 /* 964 * Do not read the rest of the completion entry before the 965 * phase bit was validated 966 */ 967 dma_rmb(); 968 969 /* Handle specific event*/ 970 handler_cb = efa_com_get_specific_aenq_cb(edev, 971 aenq_common->group); 972 handler_cb(data, aenq_e); /* call the actual event handler*/ 973 974 /* Get next event entry */ 975 ci++; 976 processed++; 977 978 if (ci == aenq->depth) { 979 ci = 0; 980 phase = !phase; 981 } 982 aenq_e = &aenq->entries[ci]; 983 aenq_common = &aenq_e->aenq_common_desc; 984 } 985 986 aenq->cc += processed; 987 aenq->phase = phase; 988 989 /* Don't update aenq doorbell if there weren't any processed events */ 990 if (!processed) 991 return; 992 993 /* barrier not needed in case of writel */ 994 writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 995 } 996 997 static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev) 998 { 999 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1000 u32 addr_high; 1001 u32 addr_low; 1002 1003 /* dma_addr_bits is unknown at this point */ 1004 addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0); 1005 addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0); 1006 1007 writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF); 1008 writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF); 1009 } 1010 1011 int efa_com_mmio_reg_read_init(struct efa_com_dev *edev) 1012 { 1013 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1014 1015 spin_lock_init(&mmio_read->lock); 1016 mmio_read->read_resp = 1017 dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1018 &mmio_read->read_resp_dma_addr, GFP_KERNEL); 1019 if (!mmio_read->read_resp) 1020 return -ENOMEM; 1021 1022 efa_com_mmio_reg_read_resp_addr_init(edev); 1023 1024 mmio_read->read_resp->req_id = 0; 1025 mmio_read->seq_num = 0; 1026 mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US; 1027 1028 return 0; 1029 } 1030 1031 void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev) 1032 { 1033 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1034 1035 dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1036 mmio_read->read_resp, mmio_read->read_resp_dma_addr); 1037 } 1038 1039 int efa_com_validate_version(struct efa_com_dev *edev) 1040 { 1041 u32 min_ctrl_ver = 0; 1042 u32 ctrl_ver_masked; 1043 u32 min_ver = 0; 1044 u32 ctrl_ver; 1045 u32 ver; 1046 1047 /* 1048 * Make sure the EFA version and the controller version are at least 1049 * as the driver expects 1050 */ 1051 ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF); 1052 ctrl_ver = efa_com_reg_read32(edev, 1053 EFA_REGS_CONTROLLER_VERSION_OFF); 1054 1055 ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n", 1056 EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION), 1057 EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION)); 1058 1059 min_ver = efa_com_construct_ver(EFA_MIN_API_VERSION_MAJOR, 1060 EFA_MIN_API_VERSION_MINOR); 1061 if (ver < min_ver) { 1062 ibdev_err(edev->efa_dev, 1063 "EFA version is lower than the minimal version the driver supports\n"); 1064 return -EOPNOTSUPP; 1065 } 1066 1067 edev->dev_api_ver = ver; 1068 1069 ibdev_dbg( 1070 edev->efa_dev, 1071 "efa controller version: %d.%d.%d implementation version %d\n", 1072 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION), 1073 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION), 1074 EFA_GET(&ctrl_ver, 1075 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION), 1076 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID)); 1077 1078 ctrl_ver_masked = 1079 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) | 1080 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) | 1081 EFA_GET(&ctrl_ver, 1082 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION); 1083 1084 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION, 1085 EFA_CTRL_MAJOR); 1086 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION, 1087 EFA_CTRL_MINOR); 1088 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION, 1089 EFA_CTRL_SUB_MINOR); 1090 /* Validate the ctrl version without the implementation ID */ 1091 if (ctrl_ver_masked < min_ctrl_ver) { 1092 ibdev_err(edev->efa_dev, 1093 "EFA ctrl version is lower than the minimal ctrl version the driver supports\n"); 1094 return -EOPNOTSUPP; 1095 } 1096 1097 return 0; 1098 } 1099 1100 /** 1101 * efa_com_get_dma_width - Retrieve physical dma address width the device 1102 * supports. 1103 * @edev: EFA communication layer struct 1104 * 1105 * Retrieve the maximum physical address bits the device can handle. 1106 * 1107 * @return: > 0 on Success and negative value otherwise. 1108 */ 1109 int efa_com_get_dma_width(struct efa_com_dev *edev) 1110 { 1111 u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1112 int width; 1113 1114 width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH); 1115 1116 ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width); 1117 1118 if (width < 32 || width > 64) { 1119 ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width); 1120 return -EINVAL; 1121 } 1122 1123 edev->dma_addr_bits = width; 1124 1125 return width; 1126 } 1127 1128 static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on) 1129 { 1130 u32 val, i; 1131 1132 for (i = 0; i < timeout; i++) { 1133 val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1134 1135 if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on) 1136 return 0; 1137 1138 ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val); 1139 msleep(EFA_POLL_INTERVAL_MS); 1140 } 1141 1142 return -ETIME; 1143 } 1144 1145 /** 1146 * efa_com_dev_reset - Perform device FLR to the device. 1147 * @edev: EFA communication layer struct 1148 * @reset_reason: Specify what is the trigger for the reset in case of an error. 1149 * 1150 * @return - 0 on success, negative value on failure. 1151 */ 1152 int efa_com_dev_reset(struct efa_com_dev *edev, 1153 enum efa_regs_reset_reason_types reset_reason) 1154 { 1155 u32 stat, timeout, cap; 1156 u32 reset_val = 0; 1157 int err; 1158 1159 stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1160 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1161 1162 if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) { 1163 ibdev_err(edev->efa_dev, 1164 "Device isn't ready, can't reset device\n"); 1165 return -EINVAL; 1166 } 1167 1168 timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT); 1169 if (!timeout) { 1170 ibdev_err(edev->efa_dev, "Invalid timeout value\n"); 1171 return -EINVAL; 1172 } 1173 1174 /* start reset */ 1175 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1); 1176 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason); 1177 writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1178 1179 /* reset clears the mmio readless address, restore it */ 1180 efa_com_mmio_reg_read_resp_addr_init(edev); 1181 1182 err = wait_for_reset_state(edev, timeout, 1); 1183 if (err) { 1184 ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n"); 1185 return err; 1186 } 1187 1188 /* reset done */ 1189 writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1190 err = wait_for_reset_state(edev, timeout, 0); 1191 if (err) { 1192 ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n"); 1193 return err; 1194 } 1195 1196 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 1197 if (timeout) 1198 /* the resolution of timeout reg is 100ms */ 1199 edev->aq.completion_timeout = timeout * 100000; 1200 else 1201 edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US; 1202 1203 return 0; 1204 } 1205 1206 static int efa_com_create_eq(struct efa_com_dev *edev, 1207 struct efa_com_create_eq_params *params, 1208 struct efa_com_create_eq_result *result) 1209 { 1210 struct efa_com_admin_queue *aq = &edev->aq; 1211 struct efa_admin_create_eq_resp resp = {}; 1212 struct efa_admin_create_eq_cmd cmd = {}; 1213 int err; 1214 1215 EFA_SET(&cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS, 1216 params->entry_size_in_bytes / 4); 1217 cmd.depth = params->depth; 1218 cmd.event_bitmask = params->event_bitmask; 1219 cmd.msix_vec = params->msix_vec; 1220 1221 efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high, 1222 &cmd.ba.mem_addr_low); 1223 1224 err = efa_com_cmd_exec(aq, EFA_ADMIN_CREATE_EQ, 0, 1225 &cmd, sizeof(cmd), 1226 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1227 if (err) { 1228 ibdev_err_ratelimited(edev->efa_dev, 1229 "Failed to create eq[%d]\n", err); 1230 return err; 1231 } 1232 1233 result->eqn = resp.eqn; 1234 1235 return 0; 1236 } 1237 1238 static void efa_com_destroy_eq(struct efa_com_dev *edev, 1239 struct efa_com_destroy_eq_params *params) 1240 { 1241 struct efa_com_admin_queue *aq = &edev->aq; 1242 struct efa_admin_destroy_eq_resp resp = {}; 1243 struct efa_admin_destroy_eq_cmd cmd = {}; 1244 int err; 1245 1246 cmd.eqn = params->eqn; 1247 1248 err = efa_com_cmd_exec(aq, EFA_ADMIN_DESTROY_EQ, 0, 1249 &cmd, sizeof(cmd), 1250 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1251 if (err) 1252 ibdev_err_ratelimited(edev->efa_dev, 1253 "Failed to destroy EQ-%u [%d]\n", cmd.eqn, 1254 err); 1255 } 1256 1257 static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1258 { 1259 u32 val = 0; 1260 1261 EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn); 1262 EFA_SET(&val, EFA_REGS_EQ_DB_ARM, 1); 1263 1264 writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF); 1265 } 1266 1267 void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev, 1268 struct efa_com_eq *eeq) 1269 { 1270 struct efa_admin_eqe *eqe; 1271 u32 processed = 0; 1272 u8 phase; 1273 u32 ci; 1274 1275 ci = eeq->cc & (eeq->depth - 1); 1276 phase = eeq->phase; 1277 eqe = &eeq->eqes[ci]; 1278 1279 /* Go over all the events */ 1280 while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) { 1281 /* 1282 * Do not read the rest of the completion entry before the 1283 * phase bit was validated 1284 */ 1285 dma_rmb(); 1286 1287 eeq->cb(eeq, eqe); 1288 1289 /* Get next event entry */ 1290 ci++; 1291 processed++; 1292 1293 if (ci == eeq->depth) { 1294 ci = 0; 1295 phase = !phase; 1296 } 1297 1298 eqe = &eeq->eqes[ci]; 1299 } 1300 1301 eeq->cc += processed; 1302 eeq->phase = phase; 1303 efa_com_arm_eq(eeq->edev, eeq); 1304 } 1305 1306 void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1307 { 1308 struct efa_com_destroy_eq_params params = { 1309 .eqn = eeq->eqn, 1310 }; 1311 1312 efa_com_destroy_eq(edev, ¶ms); 1313 dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes), 1314 eeq->eqes, eeq->dma_addr); 1315 } 1316 1317 int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq, 1318 efa_eqe_handler cb, u16 depth, u8 msix_vec) 1319 { 1320 struct efa_com_create_eq_params params = {}; 1321 struct efa_com_create_eq_result result = {}; 1322 int err; 1323 1324 params.depth = depth; 1325 params.entry_size_in_bytes = sizeof(*eeq->eqes); 1326 EFA_SET(¶ms.event_bitmask, 1327 EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1); 1328 params.msix_vec = msix_vec; 1329 1330 eeq->eqes = dma_alloc_coherent(edev->dmadev, 1331 params.depth * sizeof(*eeq->eqes), 1332 ¶ms.dma_addr, GFP_KERNEL); 1333 if (!eeq->eqes) 1334 return -ENOMEM; 1335 1336 err = efa_com_create_eq(edev, ¶ms, &result); 1337 if (err) 1338 goto err_free_coherent; 1339 1340 eeq->eqn = result.eqn; 1341 eeq->edev = edev; 1342 eeq->dma_addr = params.dma_addr; 1343 eeq->phase = 1; 1344 eeq->depth = params.depth; 1345 eeq->cb = cb; 1346 efa_com_arm_eq(edev, eeq); 1347 1348 return 0; 1349 1350 err_free_coherent: 1351 dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes), 1352 eeq->eqes, params.dma_addr); 1353 return err; 1354 } 1355