1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause 2 /* 3 * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/crc16.h> 7 #include <linux/log2.h> 8 9 #include "efa_com.h" 10 #include "efa_regs_defs.h" 11 12 #define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */ 13 14 #define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */ 15 #define EFA_MMIO_READ_INVALID 0xffffffff 16 17 #define EFA_POLL_INTERVAL_MS 100 /* msecs */ 18 19 #define EFA_ASYNC_QUEUE_DEPTH 16 20 #define EFA_ADMIN_QUEUE_DEPTH 32 21 22 #define EFA_CTRL_MAJOR 0 23 #define EFA_CTRL_MINOR 0 24 #define EFA_CTRL_SUB_MINOR 1 25 26 #define EFA_CRC16_INIT_VAL 0xffff 27 28 #define EFA_ADMIN_SQ_MAX_ENT_SIZE sizeof(struct efa_admin_aq_entry_v2) 29 30 #define EFA_CRC_MIN_API_VERSION_MAJOR 0 31 #define EFA_CRC_MIN_API_VERSION_MINOR 2 32 33 #define EFA_ADMIN_V2_MIN_API_VERSION_MAJOR 0 34 #define EFA_ADMIN_V2_MIN_API_VERSION_MINOR 3 35 36 #define EFA_MIN_API_VERSION_MAJOR 0 37 #define EFA_MIN_API_VERSION_MINOR 1 38 39 enum efa_cmd_status { 40 EFA_CMD_UNUSED, 41 EFA_CMD_ALLOCATED, 42 EFA_CMD_SUBMITTED, 43 EFA_CMD_COMPLETED, 44 EFA_CMD_TIMED_OUT, 45 }; 46 47 struct efa_comp_ctx { 48 struct completion wait_event; 49 struct efa_admin_acq_entry *user_cqe; 50 u32 comp_size; 51 enum efa_cmd_status status; 52 u16 cmd_id; 53 u8 cmd_opcode; 54 }; 55 56 static const char *efa_com_cmd_str(u8 cmd) 57 { 58 #define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd 59 60 switch (cmd) { 61 EFA_CMD_STR_CASE(CREATE_QP); 62 EFA_CMD_STR_CASE(MODIFY_QP); 63 EFA_CMD_STR_CASE(QUERY_QP); 64 EFA_CMD_STR_CASE(DESTROY_QP); 65 EFA_CMD_STR_CASE(CREATE_AH); 66 EFA_CMD_STR_CASE(DESTROY_AH); 67 EFA_CMD_STR_CASE(REG_MR); 68 EFA_CMD_STR_CASE(DEREG_MR); 69 EFA_CMD_STR_CASE(CREATE_CQ); 70 EFA_CMD_STR_CASE(DESTROY_CQ); 71 EFA_CMD_STR_CASE(GET_FEATURE); 72 EFA_CMD_STR_CASE(SET_FEATURE); 73 EFA_CMD_STR_CASE(GET_STATS); 74 EFA_CMD_STR_CASE(ALLOC_PD); 75 EFA_CMD_STR_CASE(DEALLOC_PD); 76 EFA_CMD_STR_CASE(ALLOC_UAR); 77 EFA_CMD_STR_CASE(DEALLOC_UAR); 78 EFA_CMD_STR_CASE(CREATE_EQ); 79 EFA_CMD_STR_CASE(DESTROY_EQ); 80 default: return "unknown command opcode"; 81 } 82 #undef EFA_CMD_STR_CASE 83 } 84 85 void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low) 86 { 87 *addr_low = lower_32_bits(addr); 88 *addr_high = upper_32_bits(addr); 89 } 90 91 static u32 efa_com_construct_ver(u32 major, u32 minor) 92 { 93 u32 ver = 0; 94 95 EFA_SET(&ver, EFA_REGS_VERSION_MAJOR_VERSION, major); 96 EFA_SET(&ver, EFA_REGS_VERSION_MINOR_VERSION, minor); 97 98 return ver; 99 } 100 101 static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset) 102 { 103 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 104 struct efa_admin_mmio_req_read_less_resp *read_resp; 105 unsigned long exp_time; 106 u32 mmio_read_reg = 0; 107 u32 err; 108 109 read_resp = mmio_read->read_resp; 110 111 spin_lock(&mmio_read->lock); 112 mmio_read->seq_num++; 113 114 /* trash DMA req_id to identify when hardware is done */ 115 read_resp->req_id = mmio_read->seq_num + 0x9aL; 116 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset); 117 EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID, 118 mmio_read->seq_num); 119 120 writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF); 121 122 exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout); 123 do { 124 if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num) 125 break; 126 udelay(1); 127 } while (time_is_after_jiffies(exp_time)); 128 129 if (read_resp->req_id != mmio_read->seq_num) { 130 ibdev_err_ratelimited( 131 edev->efa_dev, 132 "Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n", 133 mmio_read->seq_num, offset, read_resp->req_id, 134 read_resp->reg_off); 135 err = EFA_MMIO_READ_INVALID; 136 goto out; 137 } 138 139 if (read_resp->reg_off != offset) { 140 ibdev_err_ratelimited( 141 edev->efa_dev, 142 "Reading register failed: wrong offset provided\n"); 143 err = EFA_MMIO_READ_INVALID; 144 goto out; 145 } 146 147 err = read_resp->reg_val; 148 out: 149 spin_unlock(&mmio_read->lock); 150 return err; 151 } 152 153 static int efa_com_admin_init_sq(struct efa_com_dev *edev) 154 { 155 struct efa_com_admin_queue *aq = &edev->aq; 156 struct efa_com_admin_sq *sq = &aq->sq; 157 u32 aq_caps = 0, admin_v2_min_ver = 0; 158 u32 addr_high, addr_low; 159 160 admin_v2_min_ver = efa_com_construct_ver(EFA_ADMIN_V2_MIN_API_VERSION_MAJOR, 161 EFA_ADMIN_V2_MIN_API_VERSION_MINOR); 162 if (edev->dev_api_ver >= admin_v2_min_ver) { 163 sq->entry_size = sizeof(struct efa_admin_aq_entry_v2); 164 sq->payload_offset = offsetof(struct efa_admin_aq_entry_v2, request_payload); 165 sq->proto_ver = EFA_ADMIN_V2_PROTO_VER; 166 } else { 167 sq->entry_size = sizeof(struct efa_admin_aq_entry); 168 sq->payload_offset = offsetof(struct efa_admin_aq_entry, request_payload); 169 sq->proto_ver = EFA_ADMIN_V1_PROTO_VER; 170 } 171 172 sq->max_payload_size = sq->entry_size - sq->payload_offset; 173 174 sq->buffer = dma_alloc_coherent(aq->dmadev, aq->depth * sq->entry_size, 175 &sq->dma_addr, GFP_KERNEL); 176 if (!sq->buffer) 177 return -ENOMEM; 178 179 spin_lock_init(&sq->lock); 180 181 sq->cc = 0; 182 sq->pc = 0; 183 sq->phase = 1; 184 185 sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF); 186 187 addr_high = upper_32_bits(sq->dma_addr); 188 addr_low = lower_32_bits(sq->dma_addr); 189 190 writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF); 191 writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF); 192 193 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth); 194 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE, sq->entry_size); 195 196 writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF); 197 198 return 0; 199 } 200 201 static int efa_com_admin_init_cq(struct efa_com_dev *edev) 202 { 203 struct efa_com_admin_queue *aq = &edev->aq; 204 struct efa_com_admin_cq *cq = &aq->cq; 205 u16 size = aq->depth * sizeof(*cq->entries); 206 u32 acq_caps = 0, crc_min_ver = 0; 207 u32 addr_high, addr_low; 208 209 cq->entries = 210 dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); 211 if (!cq->entries) 212 return -ENOMEM; 213 214 spin_lock_init(&cq->lock); 215 216 crc_min_ver = efa_com_construct_ver(EFA_CRC_MIN_API_VERSION_MAJOR, 217 EFA_CRC_MIN_API_VERSION_MINOR); 218 if (edev->dev_api_ver >= crc_min_ver) 219 cq->validate_checksum = true; 220 221 cq->cc = 0; 222 cq->phase = 1; 223 224 addr_high = upper_32_bits(cq->dma_addr); 225 addr_low = lower_32_bits(cq->dma_addr); 226 227 writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF); 228 writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF); 229 230 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth); 231 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE, 232 sizeof(struct efa_admin_acq_entry)); 233 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR, 234 aq->msix_vector_idx); 235 236 writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF); 237 238 return 0; 239 } 240 241 static int efa_com_admin_init_aenq(struct efa_com_dev *edev, 242 struct efa_aenq_handlers *aenq_handlers) 243 { 244 struct efa_com_aenq *aenq = &edev->aenq; 245 u32 addr_low, addr_high; 246 u32 aenq_caps = 0; 247 u16 size; 248 249 if (!aenq_handlers) { 250 ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n"); 251 return -EINVAL; 252 } 253 254 size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries); 255 aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr, 256 GFP_KERNEL); 257 if (!aenq->entries) 258 return -ENOMEM; 259 260 aenq->aenq_handlers = aenq_handlers; 261 aenq->depth = EFA_ASYNC_QUEUE_DEPTH; 262 aenq->cc = 0; 263 aenq->phase = 1; 264 265 addr_low = lower_32_bits(aenq->dma_addr); 266 addr_high = upper_32_bits(aenq->dma_addr); 267 268 writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF); 269 writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF); 270 271 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth); 272 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE, 273 sizeof(struct efa_admin_aenq_entry)); 274 EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR, 275 aenq->msix_vector_idx); 276 writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF); 277 278 /* 279 * Init cons_db to mark that all entries in the queue 280 * are initially available 281 */ 282 writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 283 284 return 0; 285 } 286 287 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) 288 { 289 u16 ctx_id; 290 291 spin_lock(&aq->comp_ctx_lock); 292 ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next]; 293 aq->comp_ctx_pool_next++; 294 spin_unlock(&aq->comp_ctx_lock); 295 296 return ctx_id; 297 } 298 299 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, 300 u16 ctx_id) 301 { 302 spin_lock(&aq->comp_ctx_lock); 303 aq->comp_ctx_pool_next--; 304 aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id; 305 spin_unlock(&aq->comp_ctx_lock); 306 } 307 308 static struct efa_comp_ctx *efa_com_alloc_comp_ctx(struct efa_com_admin_queue *aq) 309 { 310 struct efa_comp_ctx *comp_ctx; 311 u16 ctx_id; 312 313 ctx_id = efa_com_alloc_ctx_id(aq); 314 315 comp_ctx = &aq->comp_ctx[ctx_id]; 316 if (comp_ctx->status != EFA_CMD_UNUSED) { 317 efa_com_dealloc_ctx_id(aq, ctx_id); 318 ibdev_err_ratelimited(aq->efa_dev, 319 "Completion context[%u] is used[%u]\n", 320 ctx_id, comp_ctx->status); 321 return NULL; 322 } 323 324 comp_ctx->status = EFA_CMD_ALLOCATED; 325 ibdev_dbg(aq->efa_dev, "Take completion context[%u]\n", ctx_id); 326 return comp_ctx; 327 } 328 329 static inline u16 efa_com_get_comp_ctx_id(struct efa_com_admin_queue *aq, 330 struct efa_comp_ctx *comp_ctx) 331 { 332 return comp_ctx - aq->comp_ctx; 333 } 334 335 static inline void efa_com_dealloc_comp_ctx(struct efa_com_admin_queue *aq, 336 struct efa_comp_ctx *comp_ctx) 337 { 338 u16 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 339 340 ibdev_dbg(aq->efa_dev, "Put completion context[%u]\n", ctx_id); 341 comp_ctx->status = EFA_CMD_UNUSED; 342 efa_com_dealloc_ctx_id(aq, ctx_id); 343 } 344 345 static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com_admin_queue *aq, 346 u16 cmd_id) 347 { 348 u16 ctx_id = cmd_id & (aq->depth - 1); 349 350 return &aq->comp_ctx[ctx_id]; 351 } 352 353 static u16 efa_com_calc_crc16_checksum(u8 *buff, u32 buff_size) 354 { 355 return crc16(EFA_CRC16_INIT_VAL, buff, buff_size) ^ EFA_CRC16_INIT_VAL; 356 } 357 358 static void efa_com_construct_aq_entry(struct efa_com_admin_queue *aq, u8 *aq_entry, u16 cmd_id, 359 u8 opcode, u8 flags, void *payload, size_t payload_size) 360 { 361 struct efa_admin_aq_common_desc_v2 *common_v2 = NULL; 362 struct efa_admin_aq_common_desc *common; 363 struct efa_com_admin_sq *sq = &aq->sq; 364 365 if (sq->proto_ver == EFA_ADMIN_V1_PROTO_VER) { 366 common = (struct efa_admin_aq_common_desc *)aq_entry; 367 } else { 368 common_v2 = (struct efa_admin_aq_common_desc_v2 *)aq_entry; 369 common = &common_v2->common; 370 } 371 372 common->command_id = cmd_id; 373 common->opcode = opcode; 374 common->flags = flags; 375 EFA_SET(&common->flags, EFA_ADMIN_AQ_COMMON_DESC_PHASE, sq->phase); 376 377 if (payload) 378 memcpy(aq_entry + sq->payload_offset, payload, payload_size); 379 380 if (common_v2) 381 common_v2->checksum = efa_com_calc_crc16_checksum(aq_entry, sq->entry_size); 382 } 383 384 static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 385 struct efa_comp_ctx *comp_ctx, 386 u8 opcode, u8 flags, 387 void *payload, size_t payload_size, 388 struct efa_admin_acq_entry *comp, 389 size_t comp_size_in_bytes) 390 { 391 u8 aq_entry[EFA_ADMIN_SQ_MAX_ENT_SIZE] __aligned(sizeof(u64)) = {}; 392 u16 queue_size_mask, cmd_id, ctx_id, pi; 393 struct efa_com_admin_sq *sq = &aq->sq; 394 u8 *aqe; 395 396 queue_size_mask = aq->depth - 1; 397 pi = sq->pc & queue_size_mask; 398 ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); 399 400 /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ 401 cmd_id = ctx_id & queue_size_mask; 402 cmd_id |= sq->pc << ilog2(aq->depth); 403 cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; 404 405 efa_com_construct_aq_entry(aq, aq_entry, cmd_id, opcode, flags, payload, payload_size); 406 407 comp_ctx->status = EFA_CMD_SUBMITTED; 408 comp_ctx->comp_size = comp_size_in_bytes; 409 comp_ctx->user_cqe = comp; 410 comp_ctx->cmd_opcode = opcode; 411 comp_ctx->cmd_id = cmd_id; 412 413 reinit_completion(&comp_ctx->wait_event); 414 415 aqe = sq->buffer + sq->entry_size * pi; 416 memset(aqe, 0, sq->entry_size); 417 memcpy(aqe, aq_entry, sq->entry_size); 418 419 sq->pc++; 420 atomic64_inc(&aq->stats.submitted_cmd); 421 422 if ((sq->pc & queue_size_mask) == 0) 423 sq->phase = !sq->phase; 424 425 /* barrier not needed in case of writel */ 426 writel(sq->pc, sq->db_addr); 427 } 428 429 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) 430 { 431 size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool); 432 size_t size = aq->depth * sizeof(struct efa_comp_ctx); 433 struct efa_comp_ctx *comp_ctx; 434 u16 i; 435 436 aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL); 437 aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL); 438 if (!aq->comp_ctx || !aq->comp_ctx_pool) { 439 devm_kfree(aq->dmadev, aq->comp_ctx_pool); 440 devm_kfree(aq->dmadev, aq->comp_ctx); 441 return -ENOMEM; 442 } 443 444 for (i = 0; i < aq->depth; i++) { 445 comp_ctx = &aq->comp_ctx[i]; 446 comp_ctx->status = EFA_CMD_UNUSED; 447 init_completion(&comp_ctx->wait_event); 448 449 aq->comp_ctx_pool[i] = i; 450 } 451 452 spin_lock_init(&aq->comp_ctx_lock); 453 454 aq->comp_ctx_pool_next = 0; 455 456 return 0; 457 } 458 459 static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, 460 struct efa_comp_ctx *comp_ctx, 461 u8 opcode, u8 flags, 462 void *payload, size_t payload_size, 463 struct efa_admin_acq_entry *comp, 464 size_t comp_size_in_bytes) 465 { 466 spin_lock(&aq->sq.lock); 467 if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { 468 ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); 469 spin_unlock(&aq->sq.lock); 470 return -ENODEV; 471 } 472 473 __efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, 474 payload_size, comp, comp_size_in_bytes); 475 spin_unlock(&aq->sq.lock); 476 477 return 0; 478 } 479 480 static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq, 481 struct efa_admin_acq_entry *cqe) 482 { 483 u16 cqe_checksum = cqe->acq_common_descriptor.checksum; 484 u16 calc_checksum; 485 486 cqe->acq_common_descriptor.checksum = 0; 487 488 calc_checksum = efa_com_calc_crc16_checksum((u8 *)cqe, sizeof(*cqe)); 489 if (calc_checksum != cqe_checksum) { 490 ibdev_err(aq->efa_dev, 491 "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 492 cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc, 493 aq->cq.cc); 494 return false; 495 } 496 497 return true; 498 } 499 500 static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, 501 struct efa_admin_acq_entry *cqe) 502 { 503 struct efa_comp_ctx *comp_ctx; 504 u16 cmd_id; 505 506 if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe)) 507 return -EINVAL; 508 509 cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, 510 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); 511 512 comp_ctx = efa_com_get_comp_ctx_by_cmd_id(aq, cmd_id); 513 if (comp_ctx->status != EFA_CMD_SUBMITTED || comp_ctx->cmd_id != cmd_id) { 514 ibdev_err(aq->efa_dev, 515 "Received completion with unexpected command id[%x], status[%d] sq producer[%d], sq consumer[%d], cq consumer[%d]\n", 516 cmd_id, comp_ctx->status, aq->sq.pc, aq->sq.cc, 517 aq->cq.cc); 518 return -EINVAL; 519 } 520 521 comp_ctx->status = EFA_CMD_COMPLETED; 522 memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size); 523 524 if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 525 complete(&comp_ctx->wait_event); 526 527 return 0; 528 } 529 530 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq) 531 { 532 struct efa_admin_acq_entry *cqe; 533 u16 queue_size_mask; 534 u16 comp_cmds = 0; 535 u8 phase; 536 int err; 537 u16 ci; 538 539 queue_size_mask = aq->depth - 1; 540 541 ci = aq->cq.cc & queue_size_mask; 542 phase = aq->cq.phase; 543 544 cqe = &aq->cq.entries[ci]; 545 546 /* Go over all the completions */ 547 while ((READ_ONCE(cqe->acq_common_descriptor.flags) & 548 EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { 549 /* 550 * Do not read the rest of the completion entry before the 551 * phase bit was validated 552 */ 553 dma_rmb(); 554 err = efa_com_handle_single_admin_completion(aq, cqe); 555 if (!err) 556 comp_cmds++; 557 558 aq->cq.cc++; 559 ci++; 560 if (ci == aq->depth) { 561 ci = 0; 562 phase = !phase; 563 } 564 565 cqe = &aq->cq.entries[ci]; 566 } 567 568 aq->cq.phase = phase; 569 aq->sq.cc += comp_cmds; 570 atomic64_add(comp_cmds, &aq->stats.completed_cmd); 571 } 572 573 static int efa_com_comp_status_to_errno(u8 comp_status) 574 { 575 switch (comp_status) { 576 case EFA_ADMIN_SUCCESS: 577 return 0; 578 case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE: 579 return -ENOMEM; 580 case EFA_ADMIN_UNSUPPORTED_OPCODE: 581 return -EOPNOTSUPP; 582 case EFA_ADMIN_BAD_OPCODE: 583 case EFA_ADMIN_MALFORMED_REQUEST: 584 case EFA_ADMIN_ILLEGAL_PARAMETER: 585 case EFA_ADMIN_UNKNOWN_ERROR: 586 return -EINVAL; 587 default: 588 return -EINVAL; 589 } 590 } 591 592 static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx, 593 struct efa_com_admin_queue *aq) 594 { 595 bool comp_timeout = false; 596 unsigned long timeout; 597 unsigned long flags; 598 599 timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); 600 601 while (1) { 602 spin_lock_irqsave(&aq->cq.lock, flags); 603 efa_com_handle_admin_completion(aq); 604 if (comp_ctx->status != EFA_CMD_SUBMITTED) { 605 spin_unlock_irqrestore(&aq->cq.lock, flags); 606 break; 607 } 608 609 if (time_is_before_jiffies(timeout)) { 610 comp_ctx->status = EFA_CMD_TIMED_OUT; 611 comp_timeout = true; 612 } 613 614 spin_unlock_irqrestore(&aq->cq.lock, flags); 615 616 if (comp_timeout) { 617 ibdev_err_ratelimited( 618 aq->efa_dev, 619 "Wait for completion (polling) timeout\n"); 620 /* EFA didn't have any completion */ 621 atomic64_inc(&aq->stats.no_completion); 622 623 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 624 return -ETIME; 625 } 626 627 msleep(aq->poll_interval); 628 } 629 630 return efa_com_comp_status_to_errno( 631 comp_ctx->user_cqe->acq_common_descriptor.status); 632 } 633 634 static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, 635 struct efa_com_admin_queue *aq) 636 { 637 unsigned long flags, remaining_jiffies; 638 bool completed; 639 640 remaining_jiffies = wait_for_completion_timeout( 641 &comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout)); 642 643 /* 644 * In case the command wasn't completed find out the root cause. 645 * There might be 2 kinds of errors 646 * 1) No completion (timeout reached) 647 * 2) There is completion but the device didn't get any msi-x interrupt. 648 */ 649 if (remaining_jiffies == 0) { 650 spin_lock_irqsave(&aq->cq.lock, flags); 651 efa_com_handle_admin_completion(aq); 652 completed = (comp_ctx->status == EFA_CMD_COMPLETED); 653 comp_ctx->status = EFA_CMD_TIMED_OUT; 654 spin_unlock_irqrestore(&aq->cq.lock, flags); 655 656 atomic64_inc(&aq->stats.no_completion); 657 658 if (completed) 659 ibdev_err_ratelimited( 660 aq->efa_dev, 661 "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 662 efa_com_cmd_str(comp_ctx->cmd_opcode), 663 comp_ctx->cmd_opcode, comp_ctx->status, 664 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 665 aq->cq.cc); 666 else 667 ibdev_err_ratelimited( 668 aq->efa_dev, 669 "The device didn't send any completion for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", 670 efa_com_cmd_str(comp_ctx->cmd_opcode), 671 comp_ctx->cmd_opcode, comp_ctx->status, 672 comp_ctx->cmd_id, aq->sq.pc, aq->sq.cc, 673 aq->cq.cc); 674 675 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 676 return -ETIME; 677 } 678 679 return efa_com_comp_status_to_errno( 680 comp_ctx->user_cqe->acq_common_descriptor.status); 681 } 682 683 /* 684 * There are two types to wait for completion. 685 * Polling mode - wait until the completion is available. 686 * Async mode - wait on wait queue until the completion is ready 687 * (or the timeout expired). 688 * It is expected that the IRQ called efa_com_handle_admin_completion 689 * to mark the completions. 690 */ 691 static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx, 692 struct efa_com_admin_queue *aq) 693 { 694 if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) 695 return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq); 696 697 return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq); 698 } 699 700 /** 701 * efa_com_cmd_exec - Execute admin command 702 * @aq: admin queue. 703 * @opcode: the admin command opcode. 704 * @flags: the admin command header flags. 705 * @payload: the admin command payload. 706 * @payload_size: the payload size. 707 * @comp: command completion return entry. 708 * @comp_size: command completion size. 709 * Submit an admin command and then wait until the device will return a 710 * completion. 711 * The completion will be copied into comp. 712 * 713 * @return - 0 on success, negative value on failure. 714 */ 715 int efa_com_cmd_exec(struct efa_com_admin_queue *aq, 716 u8 opcode, u8 flags, 717 void *payload, size_t payload_size, 718 struct efa_admin_acq_entry *comp, size_t comp_size) 719 { 720 struct efa_comp_ctx *comp_ctx; 721 int err; 722 723 if (payload_size > aq->sq.max_payload_size) 724 return -EINVAL; 725 726 might_sleep(); 727 728 /* In case of queue FULL */ 729 down(&aq->avail_cmds); 730 731 ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", efa_com_cmd_str(opcode), 732 opcode); 733 734 comp_ctx = efa_com_alloc_comp_ctx(aq); 735 if (!comp_ctx) { 736 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 737 up(&aq->avail_cmds); 738 return -EINVAL; 739 } 740 741 err = efa_com_submit_admin_cmd(aq, comp_ctx, opcode, flags, payload, payload_size, comp, 742 comp_size); 743 if (err) { 744 ibdev_err_ratelimited( 745 aq->efa_dev, 746 "Failed to submit command %s (opcode %u) err %d\n", 747 efa_com_cmd_str(opcode), opcode, err); 748 749 efa_com_dealloc_comp_ctx(aq, comp_ctx); 750 up(&aq->avail_cmds); 751 atomic64_inc(&aq->stats.cmd_err); 752 return err; 753 } 754 755 err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); 756 if (err) { 757 ibdev_err_ratelimited( 758 aq->efa_dev, 759 "Failed to process command %s (opcode %u) err %d\n", 760 efa_com_cmd_str(opcode), opcode, err); 761 atomic64_inc(&aq->stats.cmd_err); 762 } 763 764 efa_com_dealloc_comp_ctx(aq, comp_ctx); 765 up(&aq->avail_cmds); 766 767 return err; 768 } 769 770 /** 771 * efa_com_admin_destroy - Destroy the admin and the async events queues. 772 * @edev: EFA communication layer struct 773 */ 774 void efa_com_admin_destroy(struct efa_com_dev *edev) 775 { 776 struct efa_com_admin_queue *aq = &edev->aq; 777 struct efa_com_aenq *aenq = &edev->aenq; 778 struct efa_com_admin_cq *cq = &aq->cq; 779 struct efa_com_admin_sq *sq = &aq->sq; 780 u16 size; 781 782 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 783 784 devm_kfree(edev->dmadev, aq->comp_ctx_pool); 785 devm_kfree(edev->dmadev, aq->comp_ctx); 786 787 size = aq->depth * sq->entry_size; 788 dma_free_coherent(edev->dmadev, size, sq->buffer, sq->dma_addr); 789 790 size = aq->depth * sizeof(*cq->entries); 791 dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr); 792 793 size = aenq->depth * sizeof(*aenq->entries); 794 dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr); 795 796 efa_ah_cache_destroy(&edev->ah_cache); 797 } 798 799 /** 800 * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode 801 * @edev: EFA communication layer struct 802 * @polling: Enable/Disable polling mode 803 * 804 * Set the admin completion mode. 805 */ 806 void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling) 807 { 808 u32 mask_value = 0; 809 810 if (polling) 811 EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1); 812 813 writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF); 814 if (polling) 815 set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 816 else 817 clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); 818 } 819 820 static void efa_com_stats_init(struct efa_com_dev *edev) 821 { 822 atomic64_t *s = (atomic64_t *)&edev->aq.stats; 823 int i; 824 825 for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++) 826 atomic64_set(s, 0); 827 } 828 829 /** 830 * efa_com_admin_init - Init the admin and the async queues 831 * @edev: EFA communication layer struct 832 * @aenq_handlers: Those handlers to be called upon event. 833 * 834 * Initialize the admin submission and completion queues. 835 * Initialize the asynchronous events notification queues. 836 * 837 * @return - 0 on success, negative value on failure. 838 */ 839 int efa_com_admin_init(struct efa_com_dev *edev, 840 struct efa_aenq_handlers *aenq_handlers) 841 { 842 struct efa_com_admin_queue *aq = &edev->aq; 843 u32 timeout; 844 u32 dev_sts; 845 u32 cap; 846 int err; 847 848 dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 849 if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) { 850 ibdev_err(edev->efa_dev, 851 "Device isn't ready, abort com init %#x\n", dev_sts); 852 return -ENODEV; 853 } 854 855 err = efa_ah_cache_init(&edev->ah_cache); 856 if (err) { 857 ibdev_err(edev->efa_dev, "Failed to init AH cache\n"); 858 return err; 859 } 860 861 aq->depth = EFA_ADMIN_QUEUE_DEPTH; 862 863 aq->dmadev = edev->dmadev; 864 aq->efa_dev = edev->efa_dev; 865 efa_com_set_admin_polling_mode(edev, true); 866 867 sema_init(&aq->avail_cmds, aq->depth); 868 869 efa_com_stats_init(edev); 870 871 err = efa_com_init_comp_ctxt(aq); 872 if (err) 873 goto err_destroy_ah_cache; 874 875 err = efa_com_admin_init_sq(edev); 876 if (err) 877 goto err_destroy_comp_ctxt; 878 879 err = efa_com_admin_init_cq(edev); 880 if (err) 881 goto err_destroy_sq; 882 883 err = efa_com_admin_init_aenq(edev, aenq_handlers); 884 if (err) 885 goto err_destroy_cq; 886 887 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 888 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 889 if (timeout) 890 /* the resolution of timeout reg is 100ms */ 891 aq->completion_timeout = timeout * 100000; 892 else 893 aq->completion_timeout = ADMIN_CMD_TIMEOUT_US; 894 895 aq->poll_interval = EFA_POLL_INTERVAL_MS; 896 897 set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); 898 899 return 0; 900 901 err_destroy_cq: 902 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries), 903 aq->cq.entries, aq->cq.dma_addr); 904 err_destroy_sq: 905 dma_free_coherent(edev->dmadev, aq->depth * aq->sq.entry_size, 906 aq->sq.buffer, aq->sq.dma_addr); 907 err_destroy_comp_ctxt: 908 devm_kfree(edev->dmadev, aq->comp_ctx); 909 err_destroy_ah_cache: 910 efa_ah_cache_destroy(&edev->ah_cache); 911 912 return err; 913 } 914 915 /** 916 * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler 917 * @edev: EFA communication layer struct 918 * 919 * This method goes over the admin completion queue and wakes up 920 * all the pending threads that wait on the commands wait event. 921 * 922 * Note: Should be called after MSI-X interrupt. 923 */ 924 void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev) 925 { 926 unsigned long flags; 927 928 spin_lock_irqsave(&edev->aq.cq.lock, flags); 929 efa_com_handle_admin_completion(&edev->aq); 930 spin_unlock_irqrestore(&edev->aq.cq.lock, flags); 931 } 932 933 /* 934 * efa_handle_specific_aenq_event: 935 * return the handler that is relevant to the specific event group 936 */ 937 static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev, 938 u16 group) 939 { 940 struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers; 941 942 if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group]) 943 return aenq_handlers->handlers[group]; 944 945 return aenq_handlers->unimplemented_handler; 946 } 947 948 /** 949 * efa_com_aenq_intr_handler - AENQ interrupt handler 950 * @edev: EFA communication layer struct 951 * @data: Data of interrupt handler. 952 * 953 * Go over the async event notification queue and call the proper aenq handler. 954 */ 955 void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data) 956 { 957 struct efa_admin_aenq_common_desc *aenq_common; 958 struct efa_com_aenq *aenq = &edev->aenq; 959 struct efa_admin_aenq_entry *aenq_e; 960 efa_aenq_handler handler_cb; 961 u32 processed = 0; 962 u8 phase; 963 u32 ci; 964 965 ci = aenq->cc & (aenq->depth - 1); 966 phase = aenq->phase; 967 aenq_e = &aenq->entries[ci]; /* Get first entry */ 968 aenq_common = &aenq_e->aenq_common_desc; 969 970 /* Go over all the events */ 971 while ((READ_ONCE(aenq_common->flags) & 972 EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { 973 /* 974 * Do not read the rest of the completion entry before the 975 * phase bit was validated 976 */ 977 dma_rmb(); 978 979 /* Handle specific event*/ 980 handler_cb = efa_com_get_specific_aenq_cb(edev, 981 aenq_common->group); 982 handler_cb(data, aenq_e); /* call the actual event handler*/ 983 984 /* Get next event entry */ 985 ci++; 986 processed++; 987 988 if (ci == aenq->depth) { 989 ci = 0; 990 phase = !phase; 991 } 992 aenq_e = &aenq->entries[ci]; 993 aenq_common = &aenq_e->aenq_common_desc; 994 } 995 996 aenq->cc += processed; 997 aenq->phase = phase; 998 999 /* Don't update aenq doorbell if there weren't any processed events */ 1000 if (!processed) 1001 return; 1002 1003 /* barrier not needed in case of writel */ 1004 writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF); 1005 } 1006 1007 static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev) 1008 { 1009 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1010 u32 addr_high; 1011 u32 addr_low; 1012 1013 /* dma_addr_bits is unknown at this point */ 1014 addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0); 1015 addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0); 1016 1017 writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF); 1018 writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF); 1019 } 1020 1021 int efa_com_mmio_reg_read_init(struct efa_com_dev *edev) 1022 { 1023 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1024 1025 spin_lock_init(&mmio_read->lock); 1026 mmio_read->read_resp = 1027 dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1028 &mmio_read->read_resp_dma_addr, GFP_KERNEL); 1029 if (!mmio_read->read_resp) 1030 return -ENOMEM; 1031 1032 efa_com_mmio_reg_read_resp_addr_init(edev); 1033 1034 mmio_read->read_resp->req_id = 0; 1035 mmio_read->seq_num = 0; 1036 mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US; 1037 1038 return 0; 1039 } 1040 1041 void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev) 1042 { 1043 struct efa_com_mmio_read *mmio_read = &edev->mmio_read; 1044 1045 dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp), 1046 mmio_read->read_resp, mmio_read->read_resp_dma_addr); 1047 } 1048 1049 int efa_com_validate_version(struct efa_com_dev *edev) 1050 { 1051 u32 min_ctrl_ver = 0; 1052 u32 ctrl_ver_masked; 1053 u32 min_ver = 0; 1054 u32 ctrl_ver; 1055 u32 ver; 1056 1057 /* 1058 * Make sure the EFA version and the controller version are at least 1059 * as the driver expects 1060 */ 1061 ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF); 1062 ctrl_ver = efa_com_reg_read32(edev, 1063 EFA_REGS_CONTROLLER_VERSION_OFF); 1064 1065 ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n", 1066 EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION), 1067 EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION)); 1068 1069 min_ver = efa_com_construct_ver(EFA_MIN_API_VERSION_MAJOR, 1070 EFA_MIN_API_VERSION_MINOR); 1071 if (ver < min_ver) { 1072 ibdev_err(edev->efa_dev, 1073 "EFA version is lower than the minimal version the driver supports\n"); 1074 return -EOPNOTSUPP; 1075 } 1076 1077 edev->dev_api_ver = ver; 1078 1079 ibdev_dbg( 1080 edev->efa_dev, 1081 "efa controller version: %d.%d.%d implementation version %d\n", 1082 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION), 1083 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION), 1084 EFA_GET(&ctrl_ver, 1085 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION), 1086 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID)); 1087 1088 ctrl_ver_masked = 1089 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) | 1090 EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) | 1091 EFA_GET(&ctrl_ver, 1092 EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION); 1093 1094 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION, 1095 EFA_CTRL_MAJOR); 1096 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION, 1097 EFA_CTRL_MINOR); 1098 EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION, 1099 EFA_CTRL_SUB_MINOR); 1100 /* Validate the ctrl version without the implementation ID */ 1101 if (ctrl_ver_masked < min_ctrl_ver) { 1102 ibdev_err(edev->efa_dev, 1103 "EFA ctrl version is lower than the minimal ctrl version the driver supports\n"); 1104 return -EOPNOTSUPP; 1105 } 1106 1107 return 0; 1108 } 1109 1110 /** 1111 * efa_com_get_dma_width - Retrieve physical dma address width the device 1112 * supports. 1113 * @edev: EFA communication layer struct 1114 * 1115 * Retrieve the maximum physical address bits the device can handle. 1116 * 1117 * @return: > 0 on Success and negative value otherwise. 1118 */ 1119 int efa_com_get_dma_width(struct efa_com_dev *edev) 1120 { 1121 u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1122 int width; 1123 1124 width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH); 1125 1126 ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width); 1127 1128 if (width < 32 || width > 64) { 1129 ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width); 1130 return -EINVAL; 1131 } 1132 1133 edev->dma_addr_bits = width; 1134 1135 return width; 1136 } 1137 1138 static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on) 1139 { 1140 u32 val, i; 1141 1142 for (i = 0; i < timeout; i++) { 1143 val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1144 1145 if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on) 1146 return 0; 1147 1148 ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val); 1149 msleep(EFA_POLL_INTERVAL_MS); 1150 } 1151 1152 return -ETIME; 1153 } 1154 1155 /** 1156 * efa_com_dev_reset - Perform device FLR to the device. 1157 * @edev: EFA communication layer struct 1158 * @reset_reason: Specify what is the trigger for the reset in case of an error. 1159 * 1160 * @return - 0 on success, negative value on failure. 1161 */ 1162 int efa_com_dev_reset(struct efa_com_dev *edev, 1163 enum efa_regs_reset_reason_types reset_reason) 1164 { 1165 u32 stat, timeout, cap; 1166 u32 reset_val = 0; 1167 int err; 1168 1169 stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF); 1170 cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF); 1171 1172 if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) { 1173 ibdev_err(edev->efa_dev, 1174 "Device isn't ready, can't reset device\n"); 1175 return -EINVAL; 1176 } 1177 1178 timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT); 1179 if (!timeout) { 1180 ibdev_err(edev->efa_dev, "Invalid timeout value\n"); 1181 return -EINVAL; 1182 } 1183 1184 /* start reset */ 1185 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1); 1186 EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason); 1187 writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1188 1189 /* reset clears the mmio readless address, restore it */ 1190 efa_com_mmio_reg_read_resp_addr_init(edev); 1191 1192 err = wait_for_reset_state(edev, timeout, 1); 1193 if (err) { 1194 ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n"); 1195 return err; 1196 } 1197 1198 /* reset done */ 1199 writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF); 1200 err = wait_for_reset_state(edev, timeout, 0); 1201 if (err) { 1202 ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n"); 1203 return err; 1204 } 1205 1206 timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO); 1207 if (timeout) 1208 /* the resolution of timeout reg is 100ms */ 1209 edev->aq.completion_timeout = timeout * 100000; 1210 else 1211 edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US; 1212 1213 return 0; 1214 } 1215 1216 static int efa_com_create_eq(struct efa_com_dev *edev, 1217 struct efa_com_create_eq_params *params, 1218 struct efa_com_create_eq_result *result) 1219 { 1220 struct efa_com_admin_queue *aq = &edev->aq; 1221 struct efa_admin_create_eq_resp resp = {}; 1222 struct efa_admin_create_eq_cmd cmd = {}; 1223 int err; 1224 1225 EFA_SET(&cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS, 1226 params->entry_size_in_bytes / 4); 1227 cmd.depth = params->depth; 1228 cmd.event_bitmask = params->event_bitmask; 1229 cmd.msix_vec = params->msix_vec; 1230 1231 efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high, 1232 &cmd.ba.mem_addr_low); 1233 1234 err = efa_com_cmd_exec(aq, EFA_ADMIN_CREATE_EQ, 0, 1235 &cmd, sizeof(cmd), 1236 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1237 if (err) { 1238 ibdev_err_ratelimited(edev->efa_dev, 1239 "Failed to create eq[%d]\n", err); 1240 return err; 1241 } 1242 1243 result->eqn = resp.eqn; 1244 1245 return 0; 1246 } 1247 1248 static void efa_com_destroy_eq(struct efa_com_dev *edev, 1249 struct efa_com_destroy_eq_params *params) 1250 { 1251 struct efa_com_admin_queue *aq = &edev->aq; 1252 struct efa_admin_destroy_eq_resp resp = {}; 1253 struct efa_admin_destroy_eq_cmd cmd = {}; 1254 int err; 1255 1256 cmd.eqn = params->eqn; 1257 1258 err = efa_com_cmd_exec(aq, EFA_ADMIN_DESTROY_EQ, 0, 1259 &cmd, sizeof(cmd), 1260 (struct efa_admin_acq_entry *)&resp, sizeof(resp)); 1261 if (err) 1262 ibdev_err_ratelimited(edev->efa_dev, 1263 "Failed to destroy EQ-%u [%d]\n", cmd.eqn, 1264 err); 1265 } 1266 1267 void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1268 { 1269 u32 val = 0; 1270 1271 EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn); 1272 EFA_SET(&val, EFA_REGS_EQ_DB_ARM, 1); 1273 1274 writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF); 1275 } 1276 1277 void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev, 1278 struct efa_com_eq *eeq) 1279 { 1280 struct efa_admin_eqe *eqe; 1281 u32 processed = 0; 1282 u8 phase; 1283 u32 ci; 1284 1285 ci = eeq->cc & (eeq->depth - 1); 1286 phase = eeq->phase; 1287 eqe = &eeq->eqes[ci]; 1288 1289 /* Go over all the events */ 1290 while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) { 1291 /* 1292 * Do not read the rest of the completion entry before the 1293 * phase bit was validated 1294 */ 1295 dma_rmb(); 1296 1297 eeq->cb(eeq, eqe); 1298 1299 /* Get next event entry */ 1300 ci++; 1301 processed++; 1302 1303 if (ci == eeq->depth) { 1304 ci = 0; 1305 phase = !phase; 1306 } 1307 1308 eqe = &eeq->eqes[ci]; 1309 } 1310 1311 eeq->cc += processed; 1312 eeq->phase = phase; 1313 efa_com_arm_eq(eeq->edev, eeq); 1314 } 1315 1316 void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq) 1317 { 1318 struct efa_com_destroy_eq_params params = { 1319 .eqn = eeq->eqn, 1320 }; 1321 1322 efa_com_destroy_eq(edev, ¶ms); 1323 dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes), 1324 eeq->eqes, eeq->dma_addr); 1325 } 1326 1327 int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq, 1328 efa_eqe_handler cb, u16 depth, u8 msix_vec) 1329 { 1330 struct efa_com_create_eq_params params = {}; 1331 struct efa_com_create_eq_result result = {}; 1332 int err; 1333 1334 params.depth = depth; 1335 params.entry_size_in_bytes = sizeof(*eeq->eqes); 1336 EFA_SET(¶ms.event_bitmask, 1337 EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1); 1338 params.msix_vec = msix_vec; 1339 1340 eeq->eqes = dma_alloc_coherent(edev->dmadev, 1341 params.depth * sizeof(*eeq->eqes), 1342 ¶ms.dma_addr, GFP_KERNEL); 1343 if (!eeq->eqes) 1344 return -ENOMEM; 1345 1346 err = efa_com_create_eq(edev, ¶ms, &result); 1347 if (err) 1348 goto err_free_coherent; 1349 1350 eeq->eqn = result.eqn; 1351 eeq->edev = edev; 1352 eeq->dma_addr = params.dma_addr; 1353 eeq->phase = 1; 1354 eeq->depth = params.depth; 1355 eeq->cb = cb; 1356 1357 return 0; 1358 1359 err_free_coherent: 1360 dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes), 1361 eeq->eqes, params.dma_addr); 1362 return err; 1363 } 1364