1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause 2 /* 3 * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/crc16.h> 7 #include <linux/log2.h> 8 9 #include "efa_com.h" 10 #include "efa_regs_defs.h" 11 12 #define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */ 13 14 #define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */ 15 #define EFA_MMIO_READ_INVALID 0xffffffff 16 17 #define EFA_POLL_INTERVAL_MS 100 /* msecs */ 18 19 #define EFA_ASYNC_QUEUE_DEPTH 16 20 #define EFA_ADMIN_QUEUE_DEPTH 32 21 22 #define EFA_CTRL_MAJOR 0 23 #define EFA_CTRL_MINOR 0 24 #define EFA_CTRL_SUB_MINOR 1 25 26 #define EFA_CRC16_INIT_VAL 0xffff 27 28 #define EFA_ADMIN_SQ_MAX_ENT_SIZE sizeof(struct efa_admin_aq_entry_v2) 29 30 #define EFA_CRC_MIN_API_VERSION_MAJOR 0 31 #define EFA_CRC_MIN_API_VERSION_MINOR 2 32 33 #define EFA_ADMIN_V2_MIN_API_VERSION_MAJOR 0 34 #define EFA_ADMIN_V2_MIN_API_VERSION_MINOR 3 35 36 #define EFA_MIN_API_VERSION_MAJOR 0 37 #define EFA_MIN_API_VERSION_MINOR 1 38 39 enum efa_cmd_status { 40 EFA_CMD_UNUSED, 41 EFA_CMD_ALLOCATED, 42 EFA_CMD_SUBMITTED, 43 EFA_CMD_COMPLETED, 44 }; 45 46 struct efa_comp_ctx { 47 struct completion wait_event; 48 struct efa_admin_acq_entry *user_cqe; 49 u32 comp_size; 50 enum efa_cmd_status status; 51 u16 cmd_id; 52 u8 cmd_opcode; 53 }; 54 55 static const char *efa_com_cmd_str(u8 cmd) 56 { 57 #define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd 58 59 switch (cmd) { 60 EFA_CMD_STR_CASE(CREATE_QP); 61 EFA_CMD_STR_CASE(MODIFY_QP); 62 EFA_CMD_STR_CASE(QUERY_QP); 63 EFA_CMD_STR_CASE(DESTROY_QP); 64 EFA_CMD_STR_CASE(CREATE_AH); 65 EFA_CMD_STR_CASE(DESTROY_AH); 66 EFA_CMD_STR_CASE(REG_MR); 67 EFA_CMD_STR_CASE(DEREG_MR); 68 EFA_CMD_STR_CASE(CREATE_CQ); 69 EFA_CMD_STR_CASE(DESTROY_CQ); 70 EFA_CMD_STR_CASE(GET_FEATURE); 71 EFA_CMD_STR_CASE(SET_FEATURE); 72 EFA_CMD_STR_CASE(GET_STATS); 73 EFA_CMD_STR_CASE(ALLOC_PD); 74 EFA_CMD_STR_CASE(DEALLOC_PD); 75 EFA_CMD_STR_CASE(ALLOC_UAR); 76 EFA_CMD_STR_CASE(DEALLOC_UAR); 77 EFA_CMD_STR_CASE(CREATE_EQ); 78 EFA_CMD_STR_CASE(DESTROY_EQ); 79 default: return "unknown command opcode"; 80 } 81 #undef EFA_CMD_STR_CASE 82 } 83 84 void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low) 85 { 86 *addr_low = lower_32_bits(addr); 87 *addr_high = upper_32_bits(addr); 88 } 89 90 static u32 efa_com_construct_ver(u32 major, u32 minor) 91 { 92 u32 ver = 0; 93 94 EFA_SET(&ver, EFA_REGS_VERSION_MAJOR_VERSION, major); 95 EFA_SET(&ver, EFA_REGS_VERSION_MINOR_VERSION, minor); 96 97 return ver; 98 } 99 100 static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset) 101 { 102 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 103 struct efa_admin_mmio_req_read_less_resp *read_resp; 104 unsigned long exp_time; 105 u32 mmio_read_reg = 0; 106 u32 err; 107 108 read_resp = mmio_read->read_resp; 109 110 spin_lock(&mmio_read->lock); 111 mmio_read->seq_num++; 112 113 /* trash DMA req_id to identify when hardware is done */ 114 read_resp->req_id = mmio_read->seq_num + 0x9aL; 115 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset); 116 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID, 117 mmio_read->seq_num); 118 119 writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF); 120 121 exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout); 122 do { 123 if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num) 124 break; 125 udelay(1); 126 } while (time_is_after_jiffies(exp_time)); 127 128 if (read_resp->req_id != mmio_read->seq_num) { 129 ibdev_err_ratelimited( 130 edev->efa_dev, 131 "Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n", 132 mmio_read->seq_num, offset, read_resp->req_id, 133 read_resp->reg_off); 134 err = EFA_MMIO_READ_INVALID; 135 goto out; 136 } 137 138 if (read_resp->reg_off != offset) { 139 ibdev_err_ratelimited( 140 edev->efa_dev, 141 "Reading register failed: wrong offset provided\n"); 142 err = EFA_MMIO_READ_INVALID; 143 goto out; 144 } 145 146 err = read_resp->reg_val; 147 out: 148 spin_unlock(&mmio_read->lock); 149 return err; 150 } 151 152 static int efa_com_admin_init_sq(struct efa_com_dev *edev) 153 { 154 struct efa_com_admin_queue *aq = &edev->aq; 155 struct efa_com_admin_sq *sq = &aq->sq; 156 u32 aq_caps = 0, admin_v2_min_ver = 0; 157 u32 addr_high, addr_low; 158 159 admin_v2_min_ver = efa_com_construct_ver(EFA_ADMIN_V2_MIN_API_VERSION_MAJOR, 160 EFA_ADMIN_V2_MIN_API_VERSION_MINOR); 161 if (edev->dev_api_ver >= admin_v2_min_ver) { 162 sq->entry_size = sizeof(struct efa_admin_aq_entry_v2); 163 sq->payload_offset = offsetof(struct efa_admin_aq_entry_v2, request_payload); 164 sq->proto_ver = EFA_ADMIN_V2_PROTO_VER; 165 } else { 166 sq->entry_size = sizeof(struct efa_admin_aq_entry); 167 sq->payload_offset = offsetof(struct efa_admin_aq_entry, request_payload); 168 sq->proto_ver = EFA_ADMIN_V1_PROTO_VER; 169 } 170 171 sq->max_payload_size = sq->entry_size - sq->payload_offset; 172 173 sq->buffer = dma_alloc_coherent(aq->dmadev, aq->depth * sq->entry_size, 174 &sq->dma_addr, GFP_KERNEL); 175 if (!sq->buffer) 176 return -ENOMEM; 177 178 spin_lock_init(&sq->lock); 179 180 sq->cc = 0; 181 sq->pc = 0; 182 sq->phase = 1; 183 184 sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF); 185 186 addr_high = upper_32_bits(sq->dma_addr); 187 addr_low = lower_32_bits(sq->dma_addr); 188 189 writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF); 190 writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF); 191 192 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth); 193 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE, sq->entry_size); 194 195 writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF); 196 197 return 0; 198 } 199 200 static int efa_com_admin_init_cq(struct efa_com_dev *edev) 201 { 202 struct efa_com_admin_queue *aq = &edev->aq; 203 struct efa_com_admin_cq *cq = &aq->cq; 204 u16 size = aq->depth * sizeof(*cq->entries); 205 u32 acq_caps = 0, crc_min_ver = 0; 206 u32 addr_high, addr_low; 207 208 cq->entries = 209 dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); 210 if (!cq->entries) 211 return -ENOMEM; 212 213 spin_lock_init(&cq->lock); 214 215 crc_min_ver = efa_com_construct_ver(EFA_CRC_MIN_API_VERSION_MAJOR, 216 EFA_CRC_MIN_API_VERSION_MINOR); 217 if (edev->dev_api_ver >= crc_min_ver) 218 cq->validate_checksum = true; 219 220 cq->cc = 0; 221 cq->phase = 1; 222 223 addr_high = upper_32_bits(cq->dma_addr); 224 addr_low = lower_32_bits(cq->dma_addr); 225 226 writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF); 227 writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF); 228 229 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth); 230 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE, 231 sizeof(struct efa_admin_acq_entry)); 232 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR, 233 aq->msix_vector_idx); 234 235 writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF); 236 237 return 0; 238 } 239 240 static int efa_com_admin_init_aenq(struct efa_com_dev *edev, 241 struct efa_aenq_handlers *aenq_handlers) 242 { 243 struct efa_com_aenq *aenq = &edev->aenq; 244 u32 addr_low, addr_high; 245 u32 aenq_caps = 0; 246 u16 size; 247 248 if (!aenq_handlers) { 249 ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n"); 250 return -EINVAL; 251 } 252 253 size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries); 254 aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr, 255 GFP_KERNEL); 256 if (!aenq->entries) 257 return -ENOMEM; 258 259 aenq->aenq_handlers = aenq_handlers; 260 aenq->depth = EFA_ASYNC_QUEUE_DEPTH; 261 aenq->cc = 0; 262 aenq->phase = 1; 263 264 addr_low = lower_32_bits(aenq->dma_addr); 265 addr_high = upper_32_bits(aenq->dma_addr); 266 267 writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF); 268 writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF); 269 270 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth); 271 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE, 272 sizeof(struct efa_admin_aenq_entry)); 273 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR, 274 aenq->msix_vector_idx); 275 writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF); 276 277 /* 278 * Init cons_db to mark that all entries in the queue 279 * are initially available 280 */ 281 writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 282 283 return 0; 284 } 285 286 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) 287 { 288 u16 ctx_id; 289 290 spin_lock(&aq->comp_ctx_lock); 291 ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next]; 292 aq->comp_ctx_pool_next++; 293 spin_unlock(&aq->comp_ctx_lock); 294 295 return ctx_id; 296 } 297 298 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, 299 u16 ctx_id) 300 { 301 spin_lock(&aq->comp_ctx_lock); 302 aq->comp_ctx_pool_next--; 303 aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id; 304 spin_unlock(&aq->comp_ctx_lock); 305 } 306 307 static struct efa_comp_ctx *efa_com_alloc_comp_ctx(struct efa_com_admin_queue *aq) 308 { 309 struct efa_comp_ctx *comp_ctx; 310 u16 ctx_id; 311 312 ctx_id = efa_com_alloc_ctx_id(aq); 313 314 comp_ctx = &aq->comp_ctx[ctx_id]; 315 if (comp_ctx->status != EFA_CMD_UNUSED) { 316 efa_com_dealloc_ctx_id(aq, ctx_id); 317 ibdev_err_ratelimited(aq->efa_dev, 318 "Completion context[%u] is used[%u]\n", 319 ctx_id, comp_ctx->status); 320 return NULL; 321 } 322 323 comp_ctx->status = EFA_CMD_ALLOCATED; 324 ibdev_dbg(aq->efa_dev, "Take completion context[%u]\n", ctx_id); 325 return comp_ctx; 326 } 327 328 static inline u16 efa_com_get_comp_ctx_id(struct efa_com_admin_queue *aq, 329 struct efa_comp_ctx *comp_ctx) 330 { 331 return comp_ctx - aq->comp_ctx; 332 } 333 334 static inline void efa_com_dealloc_comp_ctx(struct efa_com_admin_queue *aq, 335 struct efa_comp_ctx *comp_ctx) 336 { 337 u16 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 338 339 ibdev_dbg(aq->efa_dev, "Put completion context[%u]\n", ctx_id); 340 comp_ctx->status = EFA_CMD_UNUSED; 341 efa_com_dealloc_ctx_id(aq, ctx_id); 342 } 343 344 static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com_admin_queue *aq, 345 u16 cmd_id) 346 { 347 u16 ctx_id = cmd_id & (aq->depth - 1); 348 349 return &aq->comp_ctx[ctx_id]; 350 } 351 352 static u16 efa_com_calc_crc16_checksum(u8 *buff, u32 buff_size) 353 { 354 return crc16(EFA_CRC16_INIT_VAL, buff, buff_size) ^ EFA_CRC16_INIT_VAL; 355 } 356 357 static void efa_com_construct_aq_entry(struct efa_com_admin_queue *aq, u8 *aq_entry, u16 cmd_id, 358 u8 opcode, u8 flags, void *payload, size_t payload_size) 359 { 360 struct efa_admin_aq_common_desc_v2 *common_v2 = NULL; 361 struct efa_admin_aq_common_desc *common; 362 struct efa_com_admin_sq *sq = &aq->sq; 363 364 if (sq->proto_ver == EFA_ADMIN_V1_PROTO_VER) { 365 common = (struct efa_admin_aq_common_desc *)aq_entry; 366 } else { 367 common_v2 = (struct efa_admin_aq_common_desc_v2 *)aq_entry; 368 common = &common_v2->common; 369 } 370 371 common->command_id = cmd_id; 372 common->opcode = opcode; 373 common->flags = flags; 374 EFA_SET(&common->flags, EFA_ADMIN_AQ_COMMON_DESC_PHASE, sq->phase); 375 376 if (payload) 377 memcpy(aq_entry + sq->payload_offset, payload, payload_size); 378 379 if (common_v2) 380 common_v2->checksum = efa_com_calc_crc16_checksum(aq_entry, sq->entry_size); 381 } 382 383 static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 384 struct efa_comp_ctx *comp_ctx, 385 u8 opcode, u8 flags, 386 void *payload, size_t payload_size, 387 struct efa_admin_acq_entry *comp, 388 size_t comp_size_in_bytes) 389 { 390 u8 aq_entry[EFA_ADMIN_SQ_MAX_ENT_SIZE] __aligned(sizeof(u64)) = {}; 391 u16 queue_size_mask, cmd_id, ctx_id, pi; 392 struct efa_com_admin_sq *sq = &aq->sq; 393 u8 *aqe; 394 395 queue_size_mask = aq->depth - 1; 396 pi = sq->pc & queue_size_mask; 397 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 398 399 /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ 400 cmd_id = ctx_id & queue_size_mask; 401 cmd_id |= sq->pc << ilog2(aq->depth); 402 cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; 403 404 efa_com_construct_aq_entry(aq, aq_entry, cmd_id, opcode, flags, payload, payload_size); 405 406 comp_ctx->status = EFA_CMD_SUBMITTED; 407 comp_ctx->comp_size = comp_size_in_bytes; 408 comp_ctx->user_cqe = comp; 409 comp_ctx->cmd_opcode = opcode; 410 comp_ctx->cmd_id = cmd_id; 411 412 reinit_completion(&comp_ctx->wait_event); 413 414 aqe = sq->buffer + sq->entry_size * pi; 415 memset(aqe, 0, sq->entry_size); 416 memcpy(aqe, aq_entry, sq->entry_size); 417 418 sq->pc++; 419 atomic64_inc(&aq->stats.submitted_cmd); 420 421 if ((sq->pc & queue_size_mask) == 0) 422 sq->phase = !sq->phase; 423 424 /* barrier not needed in case of writel */ 425 writel(sq->pc, sq->db_addr); 426 } 427 428 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) 429 { 430 size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool); 431 size_t size = aq->depth * sizeof(struct efa_comp_ctx); 432 struct efa_comp_ctx *comp_ctx; 433 u16 i; 434 435 aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL); 436 aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL); 437 if (!aq->comp_ctx || !aq->comp_ctx_pool) { 438 devm_kfree(aq->dmadev, aq->comp_ctx_pool); 439 devm_kfree(aq->dmadev, aq->comp_ctx); 440 return -ENOMEM; 441 } 442 443 for (i = 0; i < aq->depth; i++) { 444 comp_ctx = &aq->comp_ctx[i]; 445 comp_ctx->status = EFA_CMD_UNUSED; 446 init_completion(&comp_ctx->wait_event); 447 448 aq->comp_ctx_pool[i] = i; 449 } 450 451 spin_lock_init(&aq->comp_ctx_lock); 452 453 aq->comp_ctx_pool_next = 0; 454 455 return 0; 456 } 457 458 static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 459 struct efa_comp_ctx *comp_ctx, 460 u8 opcode, u8 flags, 461 void *payload, size_t payload_size, 462 struct efa_admin_acq_entry *comp, 463 size_t comp_size_in_bytes) 464 { 465 spin_lock(&aq->sq.lock); 466 if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { 467 ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); 468 spin_unlock(&aq->sq.lock); 469 return -ENODEV; 470 } 471 472 __efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, 473 payload_size, comp, comp_size_in_bytes); 474 spin_unlock(&aq->sq.lock); 475 476 return 0; 477 } 478 479 static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq, 480 struct efa_admin_acq_entry *cqe) 481 { 482 u16 cqe_checksum = cqe->acq_common_descriptor.checksum; 483 u16 calc_checksum; 484 485 cqe->acq_common_descriptor.checksum = 0; 486 487 calc_checksum = efa_com_calc_crc16_checksum((u8 *)cqe, sizeof(*cqe)); 488 if (calc_checksum != cqe_checksum) { 489 ibdev_err(aq->efa_dev, 490 "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 491 cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc, 492 aq->cq.cc); 493 return false; 494 } 495 496 return true; 497 } 498 499 static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, 500 struct efa_admin_acq_entry *cqe) 501 { 502 struct efa_comp_ctx *comp_ctx; 503 u16 cmd_id; 504 505 if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe)) 506 return -EINVAL; 507 508 cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, 509 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); 510 511 comp_ctx = efa_com_get_comp_ctx_by_cmd_id(aq, cmd_id); 512 if (comp_ctx->status != EFA_CMD_SUBMITTED || comp_ctx->cmd_id != cmd_id) { 513 ibdev_err(aq->efa_dev, 514 "Received completion with unexpected command id[%x], status[%d] sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 515 cmd_id, comp_ctx->status, aq->sq.pc, aq->sq.cc, 516 aq->cq.cc); 517 return -EINVAL; 518 } 519 520 comp_ctx->status = EFA_CMD_COMPLETED; 521 memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size); 522 523 if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 524 complete(&comp_ctx->wait_event); 525 526 return 0; 527 } 528 529 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq) 530 { 531 struct efa_admin_acq_entry *cqe; 532 u16 queue_size_mask; 533 u16 comp_cmds = 0; 534 u8 phase; 535 int err; 536 u16 ci; 537 538 queue_size_mask = aq->depth - 1; 539 540 ci = aq->cq.cc & queue_size_mask; 541 phase = aq->cq.phase; 542 543 cqe = &aq->cq.entries[ci]; 544 545 /* Go over all the completions */ 546 while ((READ_ONCE(cqe->acq_common_descriptor.flags) & 547 EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { 548 /* 549 * Do not read the rest of the completion entry before the 550 * phase bit was validated 551 */ 552 dma_rmb(); 553 err = efa_com_handle_single_admin_completion(aq, cqe); 554 if (!err) 555 comp_cmds++; 556 557 aq->cq.cc++; 558 ci++; 559 if (ci == aq->depth) { 560 ci = 0; 561 phase = !phase; 562 } 563 564 cqe = &aq->cq.entries[ci]; 565 } 566 567 aq->cq.phase = phase; 568 aq->sq.cc += comp_cmds; 569 atomic64_add(comp_cmds, &aq->stats.completed_cmd); 570 } 571 572 static int efa_com_comp_status_to_errno(u8 comp_status) 573 { 574 switch (comp_status) { 575 case EFA_ADMIN_SUCCESS: 576 return 0; 577 case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE: 578 return -ENOMEM; 579 case EFA_ADMIN_UNSUPPORTED_OPCODE: 580 return -EOPNOTSUPP; 581 case EFA_ADMIN_BAD_OPCODE: 582 case EFA_ADMIN_MALFORMED_REQUEST: 583 case EFA_ADMIN_ILLEGAL_PARAMETER: 584 case EFA_ADMIN_UNKNOWN_ERROR: 585 return -EINVAL; 586 default: 587 return -EINVAL; 588 } 589 } 590 591 static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx, 592 struct efa_com_admin_queue *aq) 593 { 594 unsigned long timeout; 595 unsigned long flags; 596 597 timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); 598 599 while (1) { 600 spin_lock_irqsave(&aq->cq.lock, flags); 601 efa_com_handle_admin_completion(aq); 602 spin_unlock_irqrestore(&aq->cq.lock, flags); 603 604 if (comp_ctx->status != EFA_CMD_SUBMITTED) 605 break; 606 607 if (time_is_before_jiffies(timeout)) { 608 ibdev_err_ratelimited( 609 aq->efa_dev, 610 "Wait for completion (polling) timeout\n"); 611 /* EFA didn't have any completion */ 612 atomic64_inc(&aq->stats.no_completion); 613 614 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 615 return -ETIME; 616 } 617 618 msleep(aq->poll_interval); 619 } 620 621 return efa_com_comp_status_to_errno( 622 comp_ctx->user_cqe->acq_common_descriptor.status); 623 } 624 625 static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, 626 struct efa_com_admin_queue *aq) 627 { 628 unsigned long flags; 629 630 wait_for_completion_timeout(&comp_ctx->wait_event, 631 usecs_to_jiffies(aq->completion_timeout)); 632 633 /* 634 * In case the command wasn't completed find out the root cause. 635 * There might be 2 kinds of errors 636 * 1) No completion (timeout reached) 637 * 2) There is completion but the device didn't get any msi-x interrupt. 638 */ 639 if (comp_ctx->status == EFA_CMD_SUBMITTED) { 640 spin_lock_irqsave(&aq->cq.lock, flags); 641 efa_com_handle_admin_completion(aq); 642 spin_unlock_irqrestore(&aq->cq.lock, flags); 643 644 atomic64_inc(&aq->stats.no_completion); 645 646 if (comp_ctx->status == EFA_CMD_COMPLETED) 647 ibdev_err_ratelimited( 648 aq->efa_dev, 649 "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 650 efa_com_cmd_str(comp_ctx->cmd_opcode), 651 comp_ctx->cmd_opcode, comp_ctx->status, 652 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 653 aq->cq.cc); 654 else 655 ibdev_err_ratelimited( 656 aq->efa_dev, 657 "The device didn't send any completion for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 658 efa_com_cmd_str(comp_ctx->cmd_opcode), 659 comp_ctx->cmd_opcode, comp_ctx->status, 660 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 661 aq->cq.cc); 662 663 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 664 return -ETIME; 665 } 666 667 return efa_com_comp_status_to_errno( 668 comp_ctx->user_cqe->acq_common_descriptor.status); 669 } 670 671 /* 672 * There are two types to wait for completion. 673 * Polling mode - wait until the completion is available. 674 * Async mode - wait on wait queue until the completion is ready 675 * (or the timeout expired). 676 * It is expected that the IRQ called efa_com_handle_admin_completion 677 * to mark the completions. 678 */ 679 static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx, 680 struct efa_com_admin_queue *aq) 681 { 682 if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 683 return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq); 684 685 return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq); 686 } 687 688 /** 689 * efa_com_cmd_exec - Execute admin command 690 * @aq: admin queue. 691 * @opcode: the admin command opcode. 692 * @flags: the admin command header flags. 693 * @payload: the admin command payload. 694 * @payload_size: the payload size. 695 * @comp: command completion return entry. 696 * @comp_size: command completion size. 697 * Submit an admin command and then wait until the device will return a 698 * completion. 699 * The completion will be copied into comp. 700 * 701 * @return - 0 on success, negative value on failure. 702 */ 703 int efa_com_cmd_exec(struct efa_com_admin_queue *aq, 704 u8 opcode, u8 flags, 705 void *payload, size_t payload_size, 706 struct efa_admin_acq_entry *comp, size_t comp_size) 707 { 708 struct efa_comp_ctx *comp_ctx; 709 int err; 710 711 if (payload_size > aq->sq.max_payload_size) 712 return -EINVAL; 713 714 might_sleep(); 715 716 /* In case of queue FULL */ 717 down(&aq->avail_cmds); 718 719 ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", efa_com_cmd_str(opcode), 720 opcode); 721 722 comp_ctx = efa_com_alloc_comp_ctx(aq); 723 if (!comp_ctx) { 724 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 725 up(&aq->avail_cmds); 726 return -EINVAL; 727 } 728 729 err = efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, payload_size, comp, 730 comp_size); 731 if (err) { 732 ibdev_err_ratelimited( 733 aq->efa_dev, 734 "Failed to submit command %s (opcode %u) err %d\n", 735 efa_com_cmd_str(opcode), opcode, err); 736 737 efa_com_dealloc_comp_ctx(aq, comp_ctx); 738 up(&aq->avail_cmds); 739 atomic64_inc(&aq->stats.cmd_err); 740 return err; 741 } 742 743 err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); 744 if (err) { 745 ibdev_err_ratelimited( 746 aq->efa_dev, 747 "Failed to process command %s (opcode %u) err %d\n", 748 efa_com_cmd_str(opcode), opcode, err); 749 atomic64_inc(&aq->stats.cmd_err); 750 } 751 752 efa_com_dealloc_comp_ctx(aq, comp_ctx); 753 up(&aq->avail_cmds); 754 755 return err; 756 } 757 758 /** 759 * efa_com_admin_destroy - Destroy the admin and the async events queues. 760 * @edev: EFA communication layer struct 761 */ 762 void efa_com_admin_destroy(struct efa_com_dev *edev) 763 { 764 struct efa_com_admin_queue *aq = &edev->aq; 765 struct efa_com_aenq *aenq = &edev->aenq; 766 struct efa_com_admin_cq *cq = &aq->cq; 767 struct efa_com_admin_sq *sq = &aq->sq; 768 u16 size; 769 770 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 771 772 devm_kfree(edev->dmadev, aq->comp_ctx_pool); 773 devm_kfree(edev->dmadev, aq->comp_ctx); 774 775 size = aq->depth * sq->entry_size; 776 dma_free_coherent(edev->dmadev, size, sq->buffer, sq->dma_addr); 777 778 size = aq->depth * sizeof(*cq->entries); 779 dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr); 780 781 size = aenq->depth * sizeof(*aenq->entries); 782 dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr); 783 784 efa_ah_cache_destroy(&edev->ah_cache); 785 } 786 787 /** 788 * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode 789 * @edev: EFA communication layer struct 790 * @polling: Enable/Disable polling mode 791 * 792 * Set the admin completion mode. 793 */ 794 void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling) 795 { 796 u32 mask_value = 0; 797 798 if (polling) 799 EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1); 800 801 writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF); 802 if (polling) 803 set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 804 else 805 clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 806 } 807 808 static void efa_com_stats_init(struct efa_com_dev *edev) 809 { 810 atomic64_t *s = (atomic64_t *)&edev->aq.stats; 811 int i; 812 813 for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++) 814 atomic64_set(s, 0); 815 } 816 817 /** 818 * efa_com_admin_init - Init the admin and the async queues 819 * @edev: EFA communication layer struct 820 * @aenq_handlers: Those handlers to be called upon event. 821 * 822 * Initialize the admin submission and completion queues. 823 * Initialize the asynchronous events notification queues. 824 * 825 * @return - 0 on success, negative value on failure. 826 */ 827 int efa_com_admin_init(struct efa_com_dev *edev, 828 struct efa_aenq_handlers *aenq_handlers) 829 { 830 struct efa_com_admin_queue *aq = &edev->aq; 831 u32 timeout; 832 u32 dev_sts; 833 u32 cap; 834 int err; 835 836 dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 837 if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) { 838 ibdev_err(edev->efa_dev, 839 "Device isn't ready, abort com init %#x\n", dev_sts); 840 return -ENODEV; 841 } 842 843 err = efa_ah_cache_init(&edev->ah_cache); 844 if (err) { 845 ibdev_err(edev->efa_dev, "Failed to init AH cache\n"); 846 return err; 847 } 848 849 aq->depth = EFA_ADMIN_QUEUE_DEPTH; 850 851 aq->dmadev = edev->dmadev; 852 aq->efa_dev = edev->efa_dev; 853 efa_com_set_admin_polling_mode(edev, true); 854 855 sema_init(&aq->avail_cmds, aq->depth); 856 857 efa_com_stats_init(edev); 858 859 err = efa_com_init_comp_ctxt(aq); 860 if (err) 861 goto err_destroy_ah_cache; 862 863 err = efa_com_admin_init_sq(edev); 864 if (err) 865 goto err_destroy_comp_ctxt; 866 867 err = efa_com_admin_init_cq(edev); 868 if (err) 869 goto err_destroy_sq; 870 871 err = efa_com_admin_init_aenq(edev, aenq_handlers); 872 if (err) 873 goto err_destroy_cq; 874 875 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 876 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 877 if (timeout) 878 /* the resolution of timeout reg is 100ms */ 879 aq->completion_timeout = timeout * 100000; 880 else 881 aq->completion_timeout = ADMIN_CMD_TIMEOUT_US; 882 883 aq->poll_interval = EFA_POLL_INTERVAL_MS; 884 885 set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 886 887 return 0; 888 889 err_destroy_cq: 890 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries), 891 aq->cq.entries, aq->cq.dma_addr); 892 err_destroy_sq: 893 dma_free_coherent(edev->dmadev, aq->depth * aq->sq.entry_size, 894 aq->sq.buffer, aq->sq.dma_addr); 895 err_destroy_comp_ctxt: 896 devm_kfree(edev->dmadev, aq->comp_ctx); 897 err_destroy_ah_cache: 898 efa_ah_cache_destroy(&edev->ah_cache); 899 900 return err; 901 } 902 903 /** 904 * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler 905 * @edev: EFA communication layer struct 906 * 907 * This method goes over the admin completion queue and wakes up 908 * all the pending threads that wait on the commands wait event. 909 * 910 * Note: Should be called after MSI-X interrupt. 911 */ 912 void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev) 913 { 914 unsigned long flags; 915 916 spin_lock_irqsave(&edev->aq.cq.lock, flags); 917 efa_com_handle_admin_completion(&edev->aq); 918 spin_unlock_irqrestore(&edev->aq.cq.lock, flags); 919 } 920 921 /* 922 * efa_handle_specific_aenq_event: 923 * return the handler that is relevant to the specific event group 924 */ 925 static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev, 926 u16 group) 927 { 928 struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers; 929 930 if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group]) 931 return aenq_handlers->handlers[group]; 932 933 return aenq_handlers->unimplemented_handler; 934 } 935 936 /** 937 * efa_com_aenq_intr_handler - AENQ interrupt handler 938 * @edev: EFA communication layer struct 939 * @data: Data of interrupt handler. 940 * 941 * Go over the async event notification queue and call the proper aenq handler. 942 */ 943 void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data) 944 { 945 struct efa_admin_aenq_common_desc *aenq_common; 946 struct efa_com_aenq *aenq = &edev->aenq; 947 struct efa_admin_aenq_entry *aenq_e; 948 efa_aenq_handler handler_cb; 949 u32 processed = 0; 950 u8 phase; 951 u32 ci; 952 953 ci = aenq->cc & (aenq->depth - 1); 954 phase = aenq->phase; 955 aenq_e = &aenq->entries[ci]; /* Get first entry */ 956 aenq_common = &aenq_e->aenq_common_desc; 957 958 /* Go over all the events */ 959 while ((READ_ONCE(aenq_common->flags) & 960 EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { 961 /* 962 * Do not read the rest of the completion entry before the 963 * phase bit was validated 964 */ 965 dma_rmb(); 966 967 /* Handle specific event*/ 968 handler_cb = efa_com_get_specific_aenq_cb(edev, 969 aenq_common->group); 970 handler_cb(data, aenq_e); /* call the actual event handler*/ 971 972 /* Get next event entry */ 973 ci++; 974 processed++; 975 976 if (ci == aenq->depth) { 977 ci = 0; 978 phase = !phase; 979 } 980 aenq_e = &aenq->entries[ci]; 981 aenq_common = &aenq_e->aenq_common_desc; 982 } 983 984 aenq->cc += processed; 985 aenq->phase = phase; 986 987 /* Don't update aenq doorbell if there weren't any processed events */ 988 if (!processed) 989 return; 990 991 /* barrier not needed in case of writel */ 992 writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 993 } 994 995 static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev) 996 { 997 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 998 u32 addr_high; 999 u32 addr_low; 1000 1001 /* dma_addr_bits is unknown at this point */ 1002 addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0); 1003 addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0); 1004 1005 writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF); 1006 writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF); 1007 } 1008 1009 int efa_com_mmio_reg_read_init(struct efa_com_dev *edev) 1010 { 1011 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1012 1013 spin_lock_init(&mmio_read->lock); 1014 mmio_read->read_resp = 1015 dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1016 &mmio_read->read_resp_dma_addr, GFP_KERNEL); 1017 if (!mmio_read->read_resp) 1018 return -ENOMEM; 1019 1020 efa_com_mmio_reg_read_resp_addr_init(edev); 1021 1022 mmio_read->read_resp->req_id = 0; 1023 mmio_read->seq_num = 0; 1024 mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US; 1025 1026 return 0; 1027 } 1028 1029 void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev) 1030 { 1031 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1032 1033 dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1034 mmio_read->read_resp, mmio_read->read_resp_dma_addr); 1035 } 1036 1037 int efa_com_validate_version(struct efa_com_dev *edev) 1038 { 1039 u32 min_ctrl_ver = 0; 1040 u32 ctrl_ver_masked; 1041 u32 min_ver = 0; 1042 u32 ctrl_ver; 1043 u32 ver; 1044 1045 /* 1046 * Make sure the EFA version and the controller version are at least 1047 * as the driver expects 1048 */ 1049 ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF); 1050 ctrl_ver = efa_com_reg_read32(edev, 1051 EFA_REGS_CONTROLLER_VERSION_OFF); 1052 1053 ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n", 1054 EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION), 1055 EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION)); 1056 1057 min_ver = efa_com_construct_ver(EFA_MIN_API_VERSION_MAJOR, 1058 EFA_MIN_API_VERSION_MINOR); 1059 if (ver < min_ver) { 1060 ibdev_err(edev->efa_dev, 1061 "EFA version is lower than the minimal version the driver supports\n"); 1062 return -EOPNOTSUPP; 1063 } 1064 1065 edev->dev_api_ver = ver; 1066 1067 ibdev_dbg( 1068 edev->efa_dev, 1069 "efa controller version: %d.%d.%d implementation version %d\n", 1070 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION), 1071 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION), 1072 EFA_GET(&ctrl_ver, 1073 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION), 1074 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID)); 1075 1076 ctrl_ver_masked = 1077 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) | 1078 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) | 1079 EFA_GET(&ctrl_ver, 1080 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION); 1081 1082 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION, 1083 EFA_CTRL_MAJOR); 1084 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION, 1085 EFA_CTRL_MINOR); 1086 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION, 1087 EFA_CTRL_SUB_MINOR); 1088 /* Validate the ctrl version without the implementation ID */ 1089 if (ctrl_ver_masked < min_ctrl_ver) { 1090 ibdev_err(edev->efa_dev, 1091 "EFA ctrl version is lower than the minimal ctrl version the driver supports\n"); 1092 return -EOPNOTSUPP; 1093 } 1094 1095 return 0; 1096 } 1097 1098 /** 1099 * efa_com_get_dma_width - Retrieve physical dma address width the device 1100 * supports. 1101 * @edev: EFA communication layer struct 1102 * 1103 * Retrieve the maximum physical address bits the device can handle. 1104 * 1105 * @return: > 0 on Success and negative value otherwise. 1106 */ 1107 int efa_com_get_dma_width(struct efa_com_dev *edev) 1108 { 1109 u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1110 int width; 1111 1112 width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH); 1113 1114 ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width); 1115 1116 if (width < 32 || width > 64) { 1117 ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width); 1118 return -EINVAL; 1119 } 1120 1121 edev->dma_addr_bits = width; 1122 1123 return width; 1124 } 1125 1126 static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on) 1127 { 1128 u32 val, i; 1129 1130 for (i = 0; i < timeout; i++) { 1131 val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1132 1133 if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on) 1134 return 0; 1135 1136 ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val); 1137 msleep(EFA_POLL_INTERVAL_MS); 1138 } 1139 1140 return -ETIME; 1141 } 1142 1143 /** 1144 * efa_com_dev_reset - Perform device FLR to the device. 1145 * @edev: EFA communication layer struct 1146 * @reset_reason: Specify what is the trigger for the reset in case of an error. 1147 * 1148 * @return - 0 on success, negative value on failure. 1149 */ 1150 int efa_com_dev_reset(struct efa_com_dev *edev, 1151 enum efa_regs_reset_reason_types reset_reason) 1152 { 1153 u32 stat, timeout, cap; 1154 u32 reset_val = 0; 1155 int err; 1156 1157 stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1158 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1159 1160 if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) { 1161 ibdev_err(edev->efa_dev, 1162 "Device isn't ready, can't reset device\n"); 1163 return -EINVAL; 1164 } 1165 1166 timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT); 1167 if (!timeout) { 1168 ibdev_err(edev->efa_dev, "Invalid timeout value\n"); 1169 return -EINVAL; 1170 } 1171 1172 /* start reset */ 1173 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1); 1174 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason); 1175 writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1176 1177 /* reset clears the mmio readless address, restore it */ 1178 efa_com_mmio_reg_read_resp_addr_init(edev); 1179 1180 err = wait_for_reset_state(edev, timeout, 1); 1181 if (err) { 1182 ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n"); 1183 return err; 1184 } 1185 1186 /* reset done */ 1187 writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1188 err = wait_for_reset_state(edev, timeout, 0); 1189 if (err) { 1190 ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n"); 1191 return err; 1192 } 1193 1194 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 1195 if (timeout) 1196 /* the resolution of timeout reg is 100ms */ 1197 edev->aq.completion_timeout = timeout * 100000; 1198 else 1199 edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US; 1200 1201 return 0; 1202 } 1203 1204 static int efa_com_create_eq(struct efa_com_dev *edev, 1205 struct efa_com_create_eq_params *params, 1206 struct efa_com_create_eq_result *result) 1207 { 1208 struct efa_com_admin_queue *aq = &edev->aq; 1209 struct efa_admin_create_eq_resp resp = {}; 1210 struct efa_admin_create_eq_cmd cmd = {}; 1211 int err; 1212 1213 EFA_SET(&cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS, 1214 params->entry_size_in_bytes / 4); 1215 cmd.depth = params->depth; 1216 cmd.event_bitmask = params->event_bitmask; 1217 cmd.msix_vec = params->msix_vec; 1218 1219 efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high, 1220 &cmd.ba.mem_addr_low); 1221 1222 err = efa_com_cmd_exec(aq, EFA_ADMIN_CREATE_EQ, 0, 1223 &cmd, sizeof(cmd), 1224 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1225 if (err) { 1226 ibdev_err_ratelimited(edev->efa_dev, 1227 "Failed to create eq[%d]\n", err); 1228 return err; 1229 } 1230 1231 result->eqn = resp.eqn; 1232 1233 return 0; 1234 } 1235 1236 static void efa_com_destroy_eq(struct efa_com_dev *edev, 1237 struct efa_com_destroy_eq_params *params) 1238 { 1239 struct efa_com_admin_queue *aq = &edev->aq; 1240 struct efa_admin_destroy_eq_resp resp = {}; 1241 struct efa_admin_destroy_eq_cmd cmd = {}; 1242 int err; 1243 1244 cmd.eqn = params->eqn; 1245 1246 err = efa_com_cmd_exec(aq, EFA_ADMIN_DESTROY_EQ, 0, 1247 &cmd, sizeof(cmd), 1248 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1249 if (err) 1250 ibdev_err_ratelimited(edev->efa_dev, 1251 "Failed to destroy EQ-%u [%d]\n", cmd.eqn, 1252 err); 1253 } 1254 1255 void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1256 { 1257 u32 val = 0; 1258 1259 EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn); 1260 EFA_SET(&val, EFA_REGS_EQ_DB_ARM, 1); 1261 1262 writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF); 1263 } 1264 1265 void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev, 1266 struct efa_com_eq *eeq) 1267 { 1268 struct efa_admin_eqe *eqe; 1269 u32 processed = 0; 1270 u8 phase; 1271 u32 ci; 1272 1273 ci = eeq->cc & (eeq->depth - 1); 1274 phase = eeq->phase; 1275 eqe = &eeq->eqes[ci]; 1276 1277 /* Go over all the events */ 1278 while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) { 1279 /* 1280 * Do not read the rest of the completion entry before the 1281 * phase bit was validated 1282 */ 1283 dma_rmb(); 1284 1285 eeq->cb(eeq, eqe); 1286 1287 /* Get next event entry */ 1288 ci++; 1289 processed++; 1290 1291 if (ci == eeq->depth) { 1292 ci = 0; 1293 phase = !phase; 1294 } 1295 1296 eqe = &eeq->eqes[ci]; 1297 } 1298 1299 eeq->cc += processed; 1300 eeq->phase = phase; 1301 efa_com_arm_eq(eeq->edev, eeq); 1302 } 1303 1304 void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1305 { 1306 struct efa_com_destroy_eq_params params = { 1307 .eqn = eeq->eqn, 1308 }; 1309 1310 efa_com_destroy_eq(edev, ¶ms); 1311 dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes), 1312 eeq->eqes, eeq->dma_addr); 1313 } 1314 1315 int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq, 1316 efa_eqe_handler cb, u16 depth, u8 msix_vec) 1317 { 1318 struct efa_com_create_eq_params params = {}; 1319 struct efa_com_create_eq_result result = {}; 1320 int err; 1321 1322 params.depth = depth; 1323 params.entry_size_in_bytes = sizeof(*eeq->eqes); 1324 EFA_SET(¶ms.event_bitmask, 1325 EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1); 1326 params.msix_vec = msix_vec; 1327 1328 eeq->eqes = dma_alloc_coherent(edev->dmadev, 1329 params.depth * sizeof(*eeq->eqes), 1330 ¶ms.dma_addr, GFP_KERNEL); 1331 if (!eeq->eqes) 1332 return -ENOMEM; 1333 1334 err = efa_com_create_eq(edev, ¶ms, &result); 1335 if (err) 1336 goto err_free_coherent; 1337 1338 eeq->eqn = result.eqn; 1339 eeq->edev = edev; 1340 eeq->dma_addr = params.dma_addr; 1341 eeq->phase = 1; 1342 eeq->depth = params.depth; 1343 eeq->cb = cb; 1344 1345 return 0; 1346 1347 err_free_coherent: 1348 dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes), 1349 eeq->eqes, params.dma_addr); 1350 return err; 1351 } 1352