1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/debugfs.h> 6 #include <linux/module.h> 7 #include <linux/pci.h> 8 #include <linux/sizes.h> 9 #include <linux/utsname.h> 10 #include <linux/version.h> 11 #include <linux/msi.h> 12 #include <linux/irqdomain.h> 13 #include <linux/export.h> 14 #include <linux/uaccess.h> 15 16 #include <net/mana/mana.h> 17 #include <net/mana/hw_channel.h> 18 19 struct dentry *mana_debugfs_root; 20 21 struct mana_dev_recovery { 22 struct list_head list; 23 struct pci_dev *pdev; 24 enum gdma_eqe_type type; 25 }; 26 27 static struct mana_dev_recovery_work { 28 struct list_head dev_list; 29 struct delayed_work work; 30 31 /* Lock for dev_list above */ 32 spinlock_t lock; 33 } mana_dev_recovery_work; 34 35 static u32 mana_gd_r32(struct gdma_context *g, u64 offset) 36 { 37 return readl(g->bar0_va + offset); 38 } 39 40 static u64 mana_gd_r64(struct gdma_context *g, u64 offset) 41 { 42 return readq(g->bar0_va + offset); 43 } 44 45 static int mana_gd_init_pf_regs(struct pci_dev *pdev) 46 { 47 struct gdma_context *gc = pci_get_drvdata(pdev); 48 u64 remaining_barsize; 49 u64 sriov_base_off; 50 u64 sriov_shm_off; 51 52 gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; 53 54 /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ 55 * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page 56 * size must be at least SZ_4K. 57 */ 58 if (gc->db_page_size < SZ_4K) { 59 dev_err(gc->dev, 60 "Doorbell page size %llu too small (min %u)\n", 61 gc->db_page_size, SZ_4K); 62 return -EPROTO; 63 } 64 65 gc->db_page_off = mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); 66 67 /* Validate doorbell offset is within BAR0 */ 68 if (gc->db_page_off >= gc->bar0_size) { 69 dev_err(gc->dev, 70 "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", 71 gc->db_page_off, (u64)gc->bar0_size); 72 return -EPROTO; 73 } 74 75 gc->db_page_base = gc->bar0_va + gc->db_page_off; 76 gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; 77 78 sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); 79 if (sriov_base_off >= gc->bar0_size || 80 gc->bar0_size - sriov_base_off < 81 GDMA_PF_REG_SHM_OFF + sizeof(u64) || 82 !IS_ALIGNED(sriov_base_off, sizeof(u64))) { 83 dev_err(gc->dev, 84 "SRIOV base offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", 85 sriov_base_off, (u64)gc->bar0_size); 86 return -EPROTO; 87 } 88 89 remaining_barsize = gc->bar0_size - sriov_base_off; 90 sriov_shm_off = mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF); 91 if (sriov_shm_off >= remaining_barsize || 92 remaining_barsize - sriov_shm_off < SMC_APERTURE_SIZE || 93 !IS_ALIGNED(sriov_shm_off, sizeof(u32))) { 94 dev_err(gc->dev, 95 "SRIOV SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", 96 sriov_shm_off, (u64)gc->bar0_size); 97 return -EPROTO; 98 } 99 100 gc->shm_base = gc->bar0_va + sriov_base_off + sriov_shm_off; 101 102 return 0; 103 } 104 105 static int mana_gd_init_vf_regs(struct pci_dev *pdev) 106 { 107 struct gdma_context *gc = pci_get_drvdata(pdev); 108 u64 shm_off; 109 110 gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; 111 112 /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ 113 * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page 114 * size must be at least SZ_4K. 115 */ 116 if (gc->db_page_size < SZ_4K) { 117 dev_err(gc->dev, 118 "Doorbell page size %llu too small (min %u)\n", 119 gc->db_page_size, SZ_4K); 120 return -EPROTO; 121 } 122 123 gc->db_page_off = mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); 124 125 /* Validate doorbell offset is within BAR0 */ 126 if (gc->db_page_off >= gc->bar0_size) { 127 dev_err(gc->dev, 128 "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", 129 gc->db_page_off, (u64)gc->bar0_size); 130 return -EPROTO; 131 } 132 133 gc->db_page_base = gc->bar0_va + gc->db_page_off; 134 gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; 135 136 shm_off = mana_gd_r64(gc, GDMA_REG_SHM_OFFSET); 137 if (shm_off >= gc->bar0_size || 138 gc->bar0_size - shm_off < SMC_APERTURE_SIZE || 139 !IS_ALIGNED(shm_off, sizeof(u32))) { 140 dev_err(gc->dev, 141 "SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", 142 shm_off, (u64)gc->bar0_size); 143 return -EPROTO; 144 } 145 146 gc->shm_base = gc->bar0_va + shm_off; 147 148 return 0; 149 } 150 151 static int mana_gd_init_registers(struct pci_dev *pdev) 152 { 153 struct gdma_context *gc = pci_get_drvdata(pdev); 154 155 if (gc->is_pf && !gc->is_pf2) 156 return mana_gd_init_pf_regs(pdev); 157 else 158 return mana_gd_init_vf_regs(pdev); 159 } 160 161 /* Suppress logging when we set timeout to zero */ 162 bool mana_need_log(struct gdma_context *gc, int err) 163 { 164 struct hw_channel_context *hwc; 165 166 if (err != -ETIMEDOUT) 167 return true; 168 169 if (!gc) 170 return true; 171 172 hwc = gc->hwc.driver_data; 173 if (hwc && hwc->hwc_timeout == 0) 174 return false; 175 176 return true; 177 } 178 179 static int mana_gd_query_max_resources(struct pci_dev *pdev) 180 { 181 struct gdma_context *gc = pci_get_drvdata(pdev); 182 struct gdma_query_max_resources_resp resp = {}; 183 struct gdma_general_req req = {}; 184 unsigned int max_num_queues; 185 u8 bm_hostmode; 186 u16 num_ports; 187 int err; 188 189 /* Reset msi_sharing so it is recomputed from current hardware 190 * state. On resume, num_online_cpus() or num_msix_usable may 191 * have changed, making dedicated MSI-X feasible where it was 192 * not before. Only reset on platforms that support dynamic 193 * MSI-X allocation; on non-dyn platforms msi_sharing is 194 * unconditionally true (set in mana_gd_setup_hwc_irqs). 195 */ 196 if (pci_msix_can_alloc_dyn(to_pci_dev(gc->dev))) 197 gc->msi_sharing = false; 198 199 mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, 200 sizeof(req), sizeof(resp)); 201 202 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 203 if (err || resp.hdr.status) { 204 dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n", 205 err, resp.hdr.status); 206 return err ? err : -EPROTO; 207 } 208 209 if (!pci_msix_can_alloc_dyn(pdev)) { 210 if (gc->num_msix_usable > resp.max_msix) 211 gc->num_msix_usable = resp.max_msix; 212 } else { 213 /* If dynamic allocation is enabled we have already allocated 214 * hwc msi 215 * Also, we make sure in this case the following is always true 216 * (num_msix_usable - 1 HWC) <= num_online_cpus() 217 */ 218 gc->num_msix_usable = min(resp.max_msix, num_online_cpus() + 1); 219 } 220 221 if (gc->num_msix_usable <= 1) 222 return -ENOSPC; 223 224 gc->max_num_queues = num_online_cpus(); 225 if (gc->max_num_queues > MANA_MAX_NUM_QUEUES) 226 gc->max_num_queues = MANA_MAX_NUM_QUEUES; 227 228 if (gc->max_num_queues > resp.max_eq) 229 gc->max_num_queues = resp.max_eq; 230 231 if (gc->max_num_queues > resp.max_cq) 232 gc->max_num_queues = resp.max_cq; 233 234 if (gc->max_num_queues > resp.max_sq) 235 gc->max_num_queues = resp.max_sq; 236 237 if (gc->max_num_queues > resp.max_rq) 238 gc->max_num_queues = resp.max_rq; 239 240 /* The Hardware Channel (HWC) used 1 MSI-X */ 241 if (gc->max_num_queues > gc->num_msix_usable - 1) 242 gc->max_num_queues = gc->num_msix_usable - 1; 243 244 if (gc->max_num_queues == 0) 245 return -ENOSPC; 246 247 debugfs_create_u32("num_msix_usable", 0400, gc->mana_pci_debugfs, 248 &gc->num_msix_usable); 249 debugfs_create_u32("max_num_queues", 0400, gc->mana_pci_debugfs, 250 &gc->max_num_queues); 251 252 err = mana_gd_query_device_cfg(gc, MANA_MAJOR_VERSION, 253 MANA_MINOR_VERSION, 254 MANA_MICRO_VERSION, 255 &num_ports, &bm_hostmode); 256 if (err) 257 return err; 258 259 if (!num_ports) { 260 dev_err(gc->dev, "Failed to detect any vPort\n"); 261 return -EINVAL; 262 } 263 264 /* Cap to the same limit used by mana_probe() for port instantiation, 265 * so MSI-X and queue budgeting matches the actual port count. 266 */ 267 if (num_ports > MAX_PORTS_IN_MANA_DEV) 268 num_ports = MAX_PORTS_IN_MANA_DEV; 269 270 /* 271 * Adjust the per-vPort max queue count to allow dedicated 272 * MSIx for each vPort. Prefer at least MANA_DEF_NUM_QUEUES, 273 * but the hardware max (gc->max_num_queues) takes precedence. 274 */ 275 max_num_queues = (gc->num_msix_usable - 1) / num_ports; 276 max_num_queues = rounddown_pow_of_two(max(max_num_queues, 1U)); 277 if (max_num_queues < MANA_DEF_NUM_QUEUES) 278 max_num_queues = MANA_DEF_NUM_QUEUES; 279 280 /* 281 * Use dedicated MSIx for EQs whenever possible, use MSIx sharing for 282 * Ethernet EQs when (max_num_queues * num_ports > num_msix_usable - 1). 283 */ 284 max_num_queues = min(gc->max_num_queues, max_num_queues); 285 if (max_num_queues * num_ports > gc->num_msix_usable - 1) 286 gc->msi_sharing = true; 287 288 /* If MSI is shared, use max allowed value */ 289 if (gc->msi_sharing) 290 gc->max_num_queues_vport = min(gc->num_msix_usable - 1, 291 gc->max_num_queues); 292 else 293 gc->max_num_queues_vport = max_num_queues; 294 295 dev_info(gc->dev, "MSI sharing mode %u max queues %u\n", 296 gc->msi_sharing, gc->max_num_queues_vport); 297 298 return 0; 299 } 300 301 static int mana_gd_query_hwc_timeout(struct pci_dev *pdev, u32 *timeout_val) 302 { 303 struct gdma_context *gc = pci_get_drvdata(pdev); 304 struct gdma_query_hwc_timeout_resp resp = {}; 305 struct gdma_query_hwc_timeout_req req = {}; 306 int err; 307 308 mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_HWC_TIMEOUT, 309 sizeof(req), sizeof(resp)); 310 req.timeout_ms = *timeout_val; 311 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 312 if (err || resp.hdr.status) 313 return err ? err : -EPROTO; 314 315 *timeout_val = resp.timeout_ms; 316 317 return 0; 318 } 319 320 static int mana_gd_detect_devices(struct pci_dev *pdev) 321 { 322 struct gdma_context *gc = pci_get_drvdata(pdev); 323 struct gdma_list_devices_resp resp = {}; 324 struct gdma_general_req req = {}; 325 struct gdma_dev_id dev; 326 int found_dev = 0; 327 u16 dev_type; 328 int err; 329 u32 i; 330 331 mana_gd_init_req_hdr(&req.hdr, GDMA_LIST_DEVICES, sizeof(req), 332 sizeof(resp)); 333 334 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 335 if (err || resp.hdr.status) { 336 dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err, 337 resp.hdr.status); 338 return err ? err : -EPROTO; 339 } 340 341 for (i = 0; i < GDMA_DEV_LIST_SIZE && 342 found_dev < resp.num_of_devs; i++) { 343 dev = resp.devs[i]; 344 dev_type = dev.type; 345 346 /* Skip empty devices */ 347 if (dev.as_uint32 == 0) 348 continue; 349 350 found_dev++; 351 352 /* HWC is already detected in mana_hwc_create_channel(). */ 353 if (dev_type == GDMA_DEVICE_HWC) 354 continue; 355 356 if (dev_type == GDMA_DEVICE_MANA) { 357 gc->mana.gdma_context = gc; 358 gc->mana.dev_id = dev; 359 } else if (dev_type == GDMA_DEVICE_MANA_IB) { 360 gc->mana_ib.dev_id = dev; 361 gc->mana_ib.gdma_context = gc; 362 } 363 } 364 365 return gc->mana.dev_id.type == 0 ? -ENODEV : 0; 366 } 367 368 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, 369 u32 resp_len, void *resp) 370 { 371 struct hw_channel_context *hwc = gc->hwc.driver_data; 372 373 return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); 374 } 375 EXPORT_SYMBOL_NS(mana_gd_send_request, "NET_MANA"); 376 377 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, 378 struct gdma_mem_info *gmi, bool allow_scatter) 379 { 380 unsigned int npages, i; 381 dma_addr_t dma_handle; 382 bool can_fallback; 383 void *buf; 384 385 if (length < MANA_PAGE_SIZE || !is_power_of_2(length)) 386 return -EINVAL; 387 388 gmi->dev = gc->dev; 389 390 /* An allocation that fits in one page does not benefit from 391 * fallback. 392 */ 393 can_fallback = allow_scatter && length > PAGE_SIZE; 394 395 /* Warn only when there is no fallback to rescue the failure. */ 396 buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, 397 GFP_KERNEL | 398 (can_fallback ? __GFP_NOWARN : 0)); 399 if (buf) { 400 gmi->dma_handle = dma_handle; 401 gmi->virt_addr = buf; 402 gmi->length = length; 403 gmi->nr_pages = 0; 404 return 0; 405 } 406 407 if (!can_fallback) 408 return -ENOMEM; 409 410 /* length is a power of 2 above PAGE_SIZE, so this divides exactly. */ 411 npages = length / PAGE_SIZE; 412 413 gmi->pages_va = kvcalloc(npages, sizeof(*gmi->pages_va), GFP_KERNEL); 414 if (!gmi->pages_va) 415 return -ENOMEM; 416 417 gmi->pages_dma = kvcalloc(npages, sizeof(*gmi->pages_dma), GFP_KERNEL); 418 if (!gmi->pages_dma) 419 goto free_va; 420 421 for (i = 0; i < npages; i++) { 422 gmi->pages_va[i] = dma_alloc_coherent(gmi->dev, PAGE_SIZE, 423 &gmi->pages_dma[i], 424 GFP_KERNEL); 425 if (!gmi->pages_va[i]) 426 goto free_pages; 427 } 428 429 dev_info_ratelimited(gmi->dev, 430 "contiguous %u-byte DMA alloc failed; using %u scattered pages\n", 431 length, npages); 432 433 gmi->virt_addr = NULL; 434 gmi->dma_handle = 0; 435 gmi->length = length; 436 gmi->nr_pages = npages; 437 438 return 0; 439 440 free_pages: 441 while (i--) 442 dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i], 443 gmi->pages_dma[i]); 444 kvfree(gmi->pages_dma); 445 gmi->pages_dma = NULL; 446 free_va: 447 kvfree(gmi->pages_va); 448 gmi->pages_va = NULL; 449 return -ENOMEM; 450 } 451 452 void mana_gd_free_memory(struct gdma_mem_info *gmi) 453 { 454 unsigned int i; 455 456 if (gmi->nr_pages > 0) { 457 for (i = 0; i < gmi->nr_pages; i++) 458 dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i], 459 gmi->pages_dma[i]); 460 kvfree(gmi->pages_va); 461 kvfree(gmi->pages_dma); 462 gmi->pages_va = NULL; 463 gmi->pages_dma = NULL; 464 gmi->nr_pages = 0; 465 return; 466 } 467 468 dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr, 469 gmi->dma_handle); 470 } 471 472 static int mana_gd_create_hw_eq(struct gdma_context *gc, 473 struct gdma_queue *queue) 474 { 475 struct gdma_create_queue_resp resp = {}; 476 struct gdma_create_queue_req req = {}; 477 int err; 478 479 if (queue->type != GDMA_EQ) 480 return -EINVAL; 481 482 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_QUEUE, 483 sizeof(req), sizeof(resp)); 484 485 req.hdr.dev_id = queue->gdma_dev->dev_id; 486 req.type = queue->type; 487 req.pdid = queue->gdma_dev->pdid; 488 req.doolbell_id = queue->gdma_dev->doorbell; 489 req.gdma_region = queue->mem_info.dma_region_handle; 490 req.queue_size = queue->queue_size; 491 req.log2_throttle_limit = queue->eq.log2_throttle_limit; 492 req.eq_pci_msix_index = queue->eq.msix_index; 493 494 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 495 if (err || resp.hdr.status) { 496 dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err, 497 resp.hdr.status); 498 return err ? err : -EPROTO; 499 } 500 501 queue->id = resp.queue_index; 502 queue->eq.disable_needed = true; 503 queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 504 return 0; 505 } 506 507 static int mana_gd_disable_queue(struct gdma_queue *queue) 508 { 509 struct gdma_context *gc = queue->gdma_dev->gdma_context; 510 struct gdma_disable_queue_req req = {}; 511 struct gdma_general_resp resp = {}; 512 int err; 513 514 WARN_ON(queue->type != GDMA_EQ); 515 516 mana_gd_init_req_hdr(&req.hdr, GDMA_DISABLE_QUEUE, 517 sizeof(req), sizeof(resp)); 518 519 req.hdr.dev_id = queue->gdma_dev->dev_id; 520 req.type = queue->type; 521 req.queue_index = queue->id; 522 req.alloc_res_id_on_creation = 1; 523 524 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 525 if (err || resp.hdr.status) { 526 if (mana_need_log(gc, err)) 527 dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err, 528 resp.hdr.status); 529 return err ? err : -EPROTO; 530 } 531 532 return 0; 533 } 534 535 #define DOORBELL_OFFSET_SQ 0x0 536 #define DOORBELL_OFFSET_RQ 0x400 537 #define DOORBELL_OFFSET_CQ 0x800 538 #define DOORBELL_OFFSET_EQ 0xFF8 539 #define DOORBELL_OFFSET_DIM 0x820 540 541 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, 542 enum gdma_queue_type q_type, u32 qid, 543 u32 tail_ptr, u8 num_req) 544 { 545 void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; 546 union gdma_doorbell_entry e = {}; 547 548 switch (q_type) { 549 case GDMA_EQ: 550 e.eq.id = qid; 551 e.eq.tail_ptr = tail_ptr; 552 e.eq.arm = num_req; 553 554 addr += DOORBELL_OFFSET_EQ; 555 break; 556 557 case GDMA_CQ: 558 e.cq.id = qid; 559 e.cq.tail_ptr = tail_ptr; 560 e.cq.arm = num_req; 561 562 addr += DOORBELL_OFFSET_CQ; 563 break; 564 565 case GDMA_RQ: 566 e.rq.id = qid; 567 e.rq.tail_ptr = tail_ptr; 568 e.rq.wqe_cnt = num_req; 569 570 addr += DOORBELL_OFFSET_RQ; 571 break; 572 573 case GDMA_SQ: 574 e.sq.id = qid; 575 e.sq.tail_ptr = tail_ptr; 576 577 addr += DOORBELL_OFFSET_SQ; 578 break; 579 580 case GDMA_DIM: 581 e.dim.id = qid; 582 e.dim.mod_usec = FIELD_GET(MANA_INTR_MODR_USEC_MAX, tail_ptr); 583 e.dim.mod_usec_vld = !!(tail_ptr & MANA_INTR_MODR_USEC_VLD); 584 e.dim.mod_comps = FIELD_GET(MANA_INTR_MODR_COMP_MASK, tail_ptr); 585 e.dim.mod_comps_vld = num_req; 586 587 addr += DOORBELL_OFFSET_DIM; 588 break; 589 590 default: 591 WARN_ON(1); 592 return; 593 } 594 595 /* Ensure all writes are done before ring doorbell */ 596 wmb(); 597 598 writeq(e.as_uint64, addr); 599 } 600 601 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) 602 { 603 /* Hardware Spec specifies that software client should set 0 for 604 * wqe_cnt for Receive Queues. This value is not used in Send Queues. 605 */ 606 mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, 607 queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); 608 } 609 EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell, "NET_MANA"); 610 611 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) 612 { 613 struct gdma_context *gc = cq->gdma_dev->gdma_context; 614 615 u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE; 616 617 u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); 618 619 mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, 620 head, arm_bit); 621 } 622 EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA"); 623 624 void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, 625 u32 mod_comps, bool mod_comps_vld) 626 { 627 struct gdma_context *gc = cq->gdma_dev->gdma_context; 628 u32 dim_val; 629 630 /* Convert the DIM values to doorbell parameters */ 631 dim_val = FIELD_PREP(MANA_INTR_MODR_USEC_MAX, mod_usec) | 632 FIELD_PREP(MANA_INTR_MODR_COMP_MASK, mod_comps); 633 if (mod_usec_vld) 634 dim_val |= MANA_INTR_MODR_USEC_VLD; 635 636 mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id, 637 dim_val, mod_comps_vld); 638 } 639 EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA"); 640 641 #define MANA_SERVICE_PERIOD 10 642 643 static void mana_serv_rescan(struct pci_dev *pdev) 644 { 645 struct pci_bus *parent; 646 647 pci_lock_rescan_remove(); 648 649 parent = pdev->bus; 650 if (!parent) { 651 dev_err(&pdev->dev, "MANA service: no parent bus\n"); 652 goto out; 653 } 654 655 pci_stop_and_remove_bus_device(pdev); 656 pci_rescan_bus(parent); 657 658 out: 659 pci_unlock_rescan_remove(); 660 } 661 662 static void mana_serv_fpga(struct pci_dev *pdev) 663 { 664 struct pci_bus *bus, *parent; 665 666 pci_lock_rescan_remove(); 667 668 bus = pdev->bus; 669 if (!bus) { 670 dev_err(&pdev->dev, "MANA service: no bus\n"); 671 goto out; 672 } 673 674 parent = bus->parent; 675 if (!parent) { 676 dev_err(&pdev->dev, "MANA service: no parent bus\n"); 677 goto out; 678 } 679 680 pci_stop_and_remove_bus_device(bus->self); 681 682 msleep(MANA_SERVICE_PERIOD * 1000); 683 684 pci_rescan_bus(parent); 685 686 out: 687 pci_unlock_rescan_remove(); 688 } 689 690 static void mana_serv_reset(struct pci_dev *pdev) 691 { 692 struct gdma_context *gc = pci_get_drvdata(pdev); 693 struct hw_channel_context *hwc; 694 int ret; 695 696 if (!gc) { 697 /* Perform PCI rescan on device if GC is not set up */ 698 dev_err(&pdev->dev, "MANA service: GC not setup, rescanning\n"); 699 mana_serv_rescan(pdev); 700 return; 701 } 702 703 hwc = gc->hwc.driver_data; 704 if (!hwc) { 705 dev_err(&pdev->dev, "MANA service: no HWC\n"); 706 goto out; 707 } 708 709 /* HWC is not responding in this case, so don't wait */ 710 hwc->hwc_timeout = 0; 711 712 dev_info(&pdev->dev, "MANA reset cycle start\n"); 713 714 mana_gd_suspend(pdev, PMSG_SUSPEND); 715 716 msleep(MANA_SERVICE_PERIOD * 1000); 717 718 ret = mana_gd_resume(pdev); 719 if (ret == -ETIMEDOUT || ret == -EPROTO) { 720 /* Perform PCI rescan on device if we failed on HWC */ 721 dev_err(&pdev->dev, "MANA service: resume failed, rescanning\n"); 722 mana_serv_rescan(pdev); 723 return; 724 } 725 726 if (ret) 727 dev_info(&pdev->dev, "MANA reset cycle failed err %d\n", ret); 728 else 729 dev_info(&pdev->dev, "MANA reset cycle completed\n"); 730 731 out: 732 clear_bit(GC_IN_SERVICE, &gc->flags); 733 } 734 735 static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) 736 { 737 switch (type) { 738 case GDMA_EQE_HWC_FPGA_RECONFIG: 739 mana_serv_fpga(pdev); 740 break; 741 742 case GDMA_EQE_HWC_RESET_REQUEST: 743 mana_serv_reset(pdev); 744 break; 745 746 default: 747 dev_err(&pdev->dev, "MANA service: unknown type %d\n", type); 748 break; 749 } 750 } 751 752 static void mana_recovery_delayed_func(struct work_struct *w) 753 { 754 struct mana_dev_recovery_work *work; 755 struct mana_dev_recovery *dev; 756 unsigned long flags; 757 758 work = container_of(w, struct mana_dev_recovery_work, work.work); 759 760 spin_lock_irqsave(&work->lock, flags); 761 762 while (!list_empty(&work->dev_list)) { 763 dev = list_first_entry(&work->dev_list, 764 struct mana_dev_recovery, list); 765 list_del(&dev->list); 766 spin_unlock_irqrestore(&work->lock, flags); 767 768 mana_do_service(dev->type, dev->pdev); 769 pci_dev_put(dev->pdev); 770 kfree(dev); 771 772 spin_lock_irqsave(&work->lock, flags); 773 } 774 775 spin_unlock_irqrestore(&work->lock, flags); 776 } 777 778 static void mana_serv_func(struct work_struct *w) 779 { 780 struct mana_serv_work *mns_wk; 781 struct pci_dev *pdev; 782 783 mns_wk = container_of(w, struct mana_serv_work, serv_work); 784 pdev = mns_wk->pdev; 785 786 if (pdev) 787 mana_do_service(mns_wk->type, pdev); 788 789 pci_dev_put(pdev); 790 kfree(mns_wk); 791 module_put(THIS_MODULE); 792 } 793 794 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) 795 { 796 struct mana_serv_work *mns_wk; 797 798 if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { 799 dev_info(gc->dev, "Already in service\n"); 800 return -EBUSY; 801 } 802 803 if (!try_module_get(THIS_MODULE)) { 804 dev_info(gc->dev, "Module is unloading\n"); 805 clear_bit(GC_IN_SERVICE, &gc->flags); 806 return -ENODEV; 807 } 808 809 mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC); 810 if (!mns_wk) { 811 module_put(THIS_MODULE); 812 clear_bit(GC_IN_SERVICE, &gc->flags); 813 return -ENOMEM; 814 } 815 816 dev_info(gc->dev, "Start MANA service type:%d\n", type); 817 mns_wk->pdev = to_pci_dev(gc->dev); 818 mns_wk->type = type; 819 pci_dev_get(mns_wk->pdev); 820 INIT_WORK(&mns_wk->serv_work, mana_serv_func); 821 schedule_work(&mns_wk->serv_work); 822 return 0; 823 } 824 825 /* Return the CPU address of byte @offset within a queue's ring buffer. */ 826 static void *mana_gd_ring_ptr(const struct gdma_queue *q, u32 offset) 827 { 828 const struct gdma_mem_info *gmi = &q->mem_info; 829 830 if (gmi->nr_pages > 0) 831 return (u8 *)gmi->pages_va[offset / PAGE_SIZE] + 832 (offset & (PAGE_SIZE - 1)); 833 834 return q->queue_mem_ptr + offset; 835 } 836 837 /* Number of bytes from @offset to the end of the CPU-contiguous region: the 838 * rest of the ring, or the rest of the current page when scattered. 839 */ 840 static u32 mana_gd_ring_contig_avail(const struct gdma_queue *q, u32 offset) 841 { 842 if (q->mem_info.nr_pages > 0) 843 return PAGE_SIZE - (offset & (PAGE_SIZE - 1)); 844 845 return q->queue_size - offset; 846 } 847 848 /* Copy up to @count bytes from ring offset *@pos of @q into user buffer @buf, 849 * so a scattered ring reads back as if it were contiguous. Returns bytes 850 * copied, 0 at end of ring, or a negative errno. 851 */ 852 ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf, 853 size_t count, loff_t *pos) 854 { 855 u32 size = q->queue_size; 856 loff_t off = *pos; 857 size_t copied = 0; 858 859 if (off < 0) 860 return -EINVAL; 861 if (off >= size || !count) 862 return 0; 863 count = min_t(size_t, count, size - off); 864 865 while (count) { 866 u32 offset = off; 867 u32 avail = mana_gd_ring_contig_avail(q, offset); 868 size_t chunk = min_t(size_t, count, avail); 869 size_t left = copy_to_user(buf, mana_gd_ring_ptr(q, offset), 870 chunk); 871 872 chunk -= left; 873 buf += chunk; 874 off += chunk; 875 copied += chunk; 876 count -= chunk; 877 if (left) 878 break; 879 } 880 881 if (!copied) 882 return -EFAULT; 883 884 *pos = off; 885 return copied; 886 } 887 888 static void mana_gd_process_eqe(struct gdma_queue *eq) 889 { 890 u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); 891 struct gdma_context *gc = eq->gdma_dev->gdma_context; 892 union gdma_eqe_info eqe_info; 893 enum gdma_eqe_type type; 894 struct gdma_event event; 895 struct gdma_queue *cq; 896 struct gdma_eqe *eqe; 897 u32 cq_id; 898 899 eqe = mana_gd_ring_ptr(eq, head * sizeof(*eqe)); 900 eqe_info.as_uint32 = eqe->eqe_info; 901 type = eqe_info.type; 902 903 switch (type) { 904 case GDMA_EQE_COMPLETION: 905 cq_id = eqe->details[0] & 0xFFFFFF; 906 if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) 907 break; 908 909 cq = gc->cq_table[cq_id]; 910 if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id)) 911 break; 912 913 if (cq->cq.callback) 914 cq->cq.callback(cq->cq.context, cq); 915 916 break; 917 918 case GDMA_EQE_TEST_EVENT: 919 gc->test_event_eq_id = eq->id; 920 complete(&gc->eq_test_event); 921 break; 922 923 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 924 case GDMA_EQE_HWC_INIT_DATA: 925 case GDMA_EQE_HWC_INIT_DONE: 926 case GDMA_EQE_HWC_SOC_SERVICE: 927 case GDMA_EQE_RNIC_QP_FATAL: 928 case GDMA_EQE_HWC_SOC_RECONFIG_DATA: 929 if (!eq->eq.callback) 930 break; 931 932 event.type = type; 933 memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE); 934 eq->eq.callback(eq->eq.context, eq, &event); 935 break; 936 937 case GDMA_EQE_HWC_FPGA_RECONFIG: 938 case GDMA_EQE_HWC_RESET_REQUEST: 939 dev_info(gc->dev, "Recv MANA service type:%d\n", type); 940 941 if (!test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { 942 /* 943 * Device is in probe and we received a hardware reset 944 * event, the probe function will detect that the flag 945 * has changed and perform service procedure. 946 */ 947 dev_info(gc->dev, 948 "Service is to be processed in probe\n"); 949 break; 950 } 951 mana_schedule_serv_work(gc, type); 952 break; 953 954 default: 955 break; 956 } 957 } 958 959 static void mana_gd_process_eq_events(void *arg) 960 { 961 u32 owner_bits, new_bits, old_bits; 962 union gdma_eqe_info eqe_info; 963 struct gdma_queue *eq = arg; 964 struct gdma_context *gc; 965 struct gdma_eqe *eqe; 966 u32 head, num_eqe; 967 int i; 968 969 gc = eq->gdma_dev->gdma_context; 970 971 num_eqe = eq->queue_size / GDMA_EQE_SIZE; 972 973 /* Process up to 5 EQEs at a time, and update the HW head. */ 974 for (i = 0; i < 5; i++) { 975 eqe = mana_gd_ring_ptr(eq, (eq->head % num_eqe) * sizeof(*eqe)); 976 eqe_info.as_uint32 = eqe->eqe_info; 977 owner_bits = eqe_info.owner_bits; 978 979 old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; 980 /* No more entries */ 981 if (owner_bits == old_bits) { 982 /* return here without ringing the doorbell */ 983 if (i == 0) 984 return; 985 break; 986 } 987 988 new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; 989 if (owner_bits != new_bits) { 990 dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id); 991 break; 992 } 993 994 /* Per GDMA spec, rmb is necessary after checking owner_bits, before 995 * reading eqe. 996 */ 997 rmb(); 998 999 mana_gd_process_eqe(eq); 1000 1001 eq->head++; 1002 } 1003 1004 head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); 1005 1006 mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, 1007 head, SET_ARM_BIT); 1008 } 1009 1010 static int mana_gd_register_irq(struct gdma_queue *queue, 1011 const struct gdma_queue_spec *spec) 1012 { 1013 struct gdma_dev *gd = queue->gdma_dev; 1014 struct gdma_irq_context *gic; 1015 struct gdma_context *gc; 1016 unsigned int msi_index; 1017 unsigned long flags; 1018 struct device *dev; 1019 int err = 0; 1020 1021 gc = gd->gdma_context; 1022 dev = gc->dev; 1023 msi_index = spec->eq.msix_index; 1024 1025 if (msi_index >= gc->num_msix_usable) { 1026 err = -ENOSPC; 1027 dev_err(dev, "Register IRQ err:%d, msi:%u nMSI:%u", 1028 err, msi_index, gc->num_msix_usable); 1029 1030 return err; 1031 } 1032 1033 queue->eq.msix_index = msi_index; 1034 /* The caller acquired a GIC reference via mana_gd_get_gic(). 1035 * That refcount prevents mana_gd_put_gic() from erasing this 1036 * irq_contexts entry concurrently. 1037 */ 1038 gic = xa_load(&gc->irq_contexts, msi_index); 1039 if (WARN_ON(!gic)) 1040 return -EINVAL; 1041 1042 spin_lock_irqsave(&gic->lock, flags); 1043 list_add_rcu(&queue->entry, &gic->eq_list); 1044 spin_unlock_irqrestore(&gic->lock, flags); 1045 1046 return 0; 1047 } 1048 1049 static void mana_gd_deregister_irq(struct gdma_queue *queue) 1050 { 1051 struct gdma_dev *gd = queue->gdma_dev; 1052 struct gdma_irq_context *gic; 1053 struct gdma_context *gc; 1054 unsigned int msix_index; 1055 unsigned long flags; 1056 struct gdma_queue *eq; 1057 1058 gc = gd->gdma_context; 1059 1060 /* At most num_online_cpus() + 1 interrupts are used. */ 1061 msix_index = queue->eq.msix_index; 1062 if (WARN_ON(msix_index >= gc->num_msix_usable)) 1063 return; 1064 1065 /* The caller releases the GIC reference via mana_gd_put_gic() 1066 * after this function returns. The refcount guarantees this 1067 * irq_contexts entry is still valid. 1068 */ 1069 gic = xa_load(&gc->irq_contexts, msix_index); 1070 if (WARN_ON(!gic)) 1071 return; 1072 1073 spin_lock_irqsave(&gic->lock, flags); 1074 list_for_each_entry_rcu(eq, &gic->eq_list, entry) { 1075 if (queue == eq) { 1076 list_del_rcu(&eq->entry); 1077 break; 1078 } 1079 } 1080 spin_unlock_irqrestore(&gic->lock, flags); 1081 1082 synchronize_rcu(); 1083 } 1084 1085 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) 1086 { 1087 struct gdma_generate_test_event_req req = {}; 1088 struct gdma_general_resp resp = {}; 1089 struct device *dev = gc->dev; 1090 int err; 1091 1092 mutex_lock(&gc->eq_test_event_mutex); 1093 1094 init_completion(&gc->eq_test_event); 1095 gc->test_event_eq_id = INVALID_QUEUE_ID; 1096 1097 mana_gd_init_req_hdr(&req.hdr, GDMA_GENERATE_TEST_EQE, 1098 sizeof(req), sizeof(resp)); 1099 1100 req.hdr.dev_id = eq->gdma_dev->dev_id; 1101 req.queue_index = eq->id; 1102 1103 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1104 if (err) { 1105 if (mana_need_log(gc, err)) 1106 dev_err(dev, "test_eq failed: %d\n", err); 1107 goto out; 1108 } 1109 1110 err = -EPROTO; 1111 1112 if (resp.hdr.status) { 1113 dev_err(dev, "test_eq failed: 0x%x\n", resp.hdr.status); 1114 goto out; 1115 } 1116 1117 if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) { 1118 dev_err(dev, "test_eq timed out on queue %d\n", eq->id); 1119 goto out; 1120 } 1121 1122 if (eq->id != gc->test_event_eq_id) { 1123 dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n", 1124 gc->test_event_eq_id, eq->id); 1125 goto out; 1126 } 1127 1128 err = 0; 1129 out: 1130 mutex_unlock(&gc->eq_test_event_mutex); 1131 return err; 1132 } 1133 1134 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets, 1135 struct gdma_queue *queue) 1136 { 1137 int err; 1138 1139 if (flush_evenets) { 1140 err = mana_gd_test_eq(gc, queue); 1141 if (err && mana_need_log(gc, err)) 1142 dev_warn(gc->dev, "Failed to flush EQ: %d\n", err); 1143 } 1144 1145 mana_gd_deregister_irq(queue); 1146 1147 if (queue->eq.disable_needed) 1148 mana_gd_disable_queue(queue); 1149 } 1150 1151 static int mana_gd_create_eq(struct gdma_dev *gd, 1152 const struct gdma_queue_spec *spec, 1153 bool create_hwq, struct gdma_queue *queue) 1154 { 1155 struct gdma_context *gc = gd->gdma_context; 1156 struct device *dev = gc->dev; 1157 u32 log2_num_entries; 1158 int err; 1159 1160 queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; 1161 queue->id = INVALID_QUEUE_ID; 1162 1163 log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); 1164 1165 if (spec->eq.log2_throttle_limit > log2_num_entries) { 1166 dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n", 1167 spec->eq.log2_throttle_limit, log2_num_entries); 1168 return -EINVAL; 1169 } 1170 1171 err = mana_gd_register_irq(queue, spec); 1172 if (err) { 1173 dev_err(dev, "Failed to register irq: %d\n", err); 1174 return err; 1175 } 1176 1177 queue->eq.callback = spec->eq.callback; 1178 queue->eq.context = spec->eq.context; 1179 queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); 1180 queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1; 1181 1182 if (create_hwq) { 1183 err = mana_gd_create_hw_eq(gc, queue); 1184 if (err) 1185 goto out; 1186 1187 err = mana_gd_test_eq(gc, queue); 1188 if (err) 1189 goto out; 1190 } 1191 1192 return 0; 1193 out: 1194 dev_err(dev, "Failed to create EQ: %d\n", err); 1195 mana_gd_destroy_eq(gc, false, queue); 1196 queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; 1197 return err; 1198 } 1199 1200 static void mana_gd_create_cq(const struct gdma_queue_spec *spec, 1201 struct gdma_queue *queue) 1202 { 1203 u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE); 1204 1205 queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); 1206 queue->cq.parent = spec->cq.parent_eq; 1207 queue->cq.context = spec->cq.context; 1208 queue->cq.callback = spec->cq.callback; 1209 } 1210 1211 static void mana_gd_destroy_cq(struct gdma_context *gc, 1212 struct gdma_queue *queue) 1213 { 1214 u32 id = queue->id; 1215 1216 if (id >= gc->max_num_cqs) 1217 return; 1218 1219 if (!gc->cq_table[id]) 1220 return; 1221 1222 gc->cq_table[id] = NULL; 1223 } 1224 1225 int mana_gd_create_hwc_queue(struct gdma_dev *gd, 1226 const struct gdma_queue_spec *spec, 1227 struct gdma_queue **queue_ptr) 1228 { 1229 struct gdma_context *gc = gd->gdma_context; 1230 struct gdma_mem_info *gmi; 1231 struct gdma_queue *queue; 1232 int err; 1233 1234 queue = kzalloc_obj(*queue); 1235 if (!queue) 1236 return -ENOMEM; 1237 1238 gmi = &queue->mem_info; 1239 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, false); 1240 if (err) { 1241 dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n", 1242 spec->type, spec->queue_size, err); 1243 goto free_q; 1244 } 1245 1246 queue->head = 0; 1247 queue->tail = 0; 1248 queue->queue_mem_ptr = gmi->virt_addr; 1249 queue->queue_size = spec->queue_size; 1250 queue->monitor_avl_buf = spec->monitor_avl_buf; 1251 queue->type = spec->type; 1252 queue->gdma_dev = gd; 1253 1254 if (spec->type == GDMA_EQ) 1255 err = mana_gd_create_eq(gd, spec, false, queue); 1256 else if (spec->type == GDMA_CQ) 1257 mana_gd_create_cq(spec, queue); 1258 1259 if (err) 1260 goto out; 1261 1262 *queue_ptr = queue; 1263 return 0; 1264 out: 1265 dev_err(gc->dev, "Failed to create queue type %d of size %u, err: %d\n", 1266 spec->type, spec->queue_size, err); 1267 mana_gd_free_memory(gmi); 1268 free_q: 1269 kfree(queue); 1270 return err; 1271 } 1272 1273 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle) 1274 { 1275 struct gdma_destroy_dma_region_req req = {}; 1276 struct gdma_general_resp resp = {}; 1277 int err; 1278 1279 if (dma_region_handle == GDMA_INVALID_DMA_REGION) 1280 return 0; 1281 1282 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_DMA_REGION, sizeof(req), 1283 sizeof(resp)); 1284 req.dma_region_handle = dma_region_handle; 1285 1286 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1287 if (err || resp.hdr.status) { 1288 if (mana_need_log(gc, err)) 1289 dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n", 1290 err, resp.hdr.status); 1291 return -EPROTO; 1292 } 1293 1294 return 0; 1295 } 1296 EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, "NET_MANA"); 1297 1298 static int mana_gd_create_dma_region(struct gdma_dev *gd, 1299 struct gdma_mem_info *gmi) 1300 { 1301 unsigned int num_page = gmi->length / MANA_PAGE_SIZE; 1302 struct gdma_create_dma_region_req *req = NULL; 1303 struct gdma_create_dma_region_resp resp = {}; 1304 struct gdma_context *gc = gd->gdma_context; 1305 struct hw_channel_context *hwc; 1306 u32 length = gmi->length; 1307 size_t req_msg_size; 1308 int err; 1309 int i; 1310 1311 if (length < MANA_PAGE_SIZE || !is_power_of_2(length)) 1312 return -EINVAL; 1313 1314 if (gmi->nr_pages == 0 && !MANA_PAGE_ALIGNED(gmi->virt_addr)) 1315 return -EINVAL; 1316 1317 hwc = gc->hwc.driver_data; 1318 req_msg_size = struct_size(req, page_addr_list, num_page); 1319 if (req_msg_size > hwc->max_req_msg_size) 1320 return -EINVAL; 1321 1322 req = kzalloc(req_msg_size, GFP_KERNEL); 1323 if (!req) 1324 return -ENOMEM; 1325 1326 mana_gd_init_req_hdr(&req->hdr, GDMA_CREATE_DMA_REGION, 1327 req_msg_size, sizeof(resp)); 1328 req->length = length; 1329 req->offset_in_page = 0; 1330 req->gdma_page_type = GDMA_PAGE_TYPE_4K; 1331 req->page_count = num_page; 1332 req->page_addr_list_len = num_page; 1333 1334 if (gmi->nr_pages > 0) { 1335 unsigned int subpages = PAGE_SIZE / MANA_PAGE_SIZE; 1336 unsigned int idx = 0; 1337 unsigned int pg, sub; 1338 1339 /* Each PAGE_SIZE chunk is physically contiguous and contains 1340 * PAGE_SIZE / MANA_PAGE_SIZE consecutive device pages. 1341 */ 1342 for (pg = 0; pg < gmi->nr_pages; pg++) 1343 for (sub = 0; sub < subpages; sub++) 1344 req->page_addr_list[idx++] = 1345 gmi->pages_dma[pg] + 1346 sub * MANA_PAGE_SIZE; 1347 } else { 1348 for (i = 0; i < num_page; i++) 1349 req->page_addr_list[i] = 1350 gmi->dma_handle + i * MANA_PAGE_SIZE; 1351 } 1352 1353 err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp); 1354 if (err) 1355 goto out; 1356 1357 if (resp.hdr.status || 1358 resp.dma_region_handle == GDMA_INVALID_DMA_REGION) { 1359 dev_err(gc->dev, "Failed to create DMA region: 0x%x\n", 1360 resp.hdr.status); 1361 err = -EPROTO; 1362 goto out; 1363 } 1364 1365 gmi->dma_region_handle = resp.dma_region_handle; 1366 dev_dbg(gc->dev, "Created DMA region handle 0x%llx\n", 1367 gmi->dma_region_handle); 1368 out: 1369 if (err) 1370 dev_dbg(gc->dev, 1371 "Failed to create DMA region of length: %u, page_type: %d, status: 0x%x, err: %d\n", 1372 length, req->gdma_page_type, resp.hdr.status, err); 1373 kfree(req); 1374 return err; 1375 } 1376 1377 int mana_gd_create_mana_eq(struct gdma_dev *gd, 1378 const struct gdma_queue_spec *spec, 1379 struct gdma_queue **queue_ptr) 1380 { 1381 struct gdma_context *gc = gd->gdma_context; 1382 struct gdma_mem_info *gmi; 1383 struct gdma_queue *queue; 1384 int err; 1385 1386 if (spec->type != GDMA_EQ) 1387 return -EINVAL; 1388 1389 queue = kzalloc_obj(*queue); 1390 if (!queue) 1391 return -ENOMEM; 1392 1393 gmi = &queue->mem_info; 1394 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true); 1395 if (err) { 1396 dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n", 1397 spec->type, spec->queue_size, err); 1398 goto free_q; 1399 } 1400 1401 err = mana_gd_create_dma_region(gd, gmi); 1402 if (err) 1403 goto out; 1404 1405 queue->head = 0; 1406 queue->tail = 0; 1407 queue->queue_mem_ptr = gmi->virt_addr; 1408 queue->queue_size = spec->queue_size; 1409 queue->monitor_avl_buf = spec->monitor_avl_buf; 1410 queue->type = spec->type; 1411 queue->gdma_dev = gd; 1412 1413 err = mana_gd_create_eq(gd, spec, true, queue); 1414 if (err) 1415 goto out; 1416 1417 *queue_ptr = queue; 1418 return 0; 1419 out: 1420 dev_err(gc->dev, "Failed to create queue type %d of size: %u, err: %d\n", 1421 spec->type, spec->queue_size, err); 1422 mana_gd_free_memory(gmi); 1423 free_q: 1424 kfree(queue); 1425 return err; 1426 } 1427 EXPORT_SYMBOL_NS(mana_gd_create_mana_eq, "NET_MANA"); 1428 1429 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, 1430 const struct gdma_queue_spec *spec, 1431 struct gdma_queue **queue_ptr) 1432 { 1433 struct gdma_context *gc = gd->gdma_context; 1434 struct gdma_mem_info *gmi; 1435 struct gdma_queue *queue; 1436 int err; 1437 1438 if (spec->type != GDMA_CQ && spec->type != GDMA_SQ && 1439 spec->type != GDMA_RQ) 1440 return -EINVAL; 1441 1442 queue = kzalloc_obj(*queue); 1443 if (!queue) 1444 return -ENOMEM; 1445 1446 queue->id = INVALID_QUEUE_ID; 1447 1448 gmi = &queue->mem_info; 1449 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true); 1450 if (err) { 1451 dev_err(gc->dev, "GDMA queue type: %d, size: %u, memory allocation err: %d\n", 1452 spec->type, spec->queue_size, err); 1453 goto free_q; 1454 } 1455 1456 err = mana_gd_create_dma_region(gd, gmi); 1457 if (err) 1458 goto out; 1459 1460 queue->head = 0; 1461 queue->tail = 0; 1462 queue->queue_mem_ptr = gmi->virt_addr; 1463 queue->queue_size = spec->queue_size; 1464 queue->monitor_avl_buf = spec->monitor_avl_buf; 1465 queue->type = spec->type; 1466 queue->gdma_dev = gd; 1467 1468 if (spec->type == GDMA_CQ) 1469 mana_gd_create_cq(spec, queue); 1470 1471 *queue_ptr = queue; 1472 return 0; 1473 out: 1474 dev_err(gc->dev, "Failed to create queue type %d of size: %u, err: %d\n", 1475 spec->type, spec->queue_size, err); 1476 mana_gd_free_memory(gmi); 1477 free_q: 1478 kfree(queue); 1479 return err; 1480 } 1481 EXPORT_SYMBOL_NS(mana_gd_create_mana_wq_cq, "NET_MANA"); 1482 1483 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) 1484 { 1485 struct gdma_mem_info *gmi = &queue->mem_info; 1486 1487 switch (queue->type) { 1488 case GDMA_EQ: 1489 mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue); 1490 break; 1491 1492 case GDMA_CQ: 1493 mana_gd_destroy_cq(gc, queue); 1494 break; 1495 1496 case GDMA_RQ: 1497 break; 1498 1499 case GDMA_SQ: 1500 break; 1501 1502 default: 1503 dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n", 1504 queue->type); 1505 return; 1506 } 1507 1508 mana_gd_destroy_dma_region(gc, gmi->dma_region_handle); 1509 mana_gd_free_memory(gmi); 1510 kfree(queue); 1511 } 1512 EXPORT_SYMBOL_NS(mana_gd_destroy_queue, "NET_MANA"); 1513 1514 int mana_gd_verify_vf_version(struct pci_dev *pdev) 1515 { 1516 struct gdma_context *gc = pci_get_drvdata(pdev); 1517 struct gdma_verify_ver_resp resp = {}; 1518 struct gdma_verify_ver_req req = {}; 1519 struct hw_channel_context *hwc; 1520 int err; 1521 1522 hwc = gc->hwc.driver_data; 1523 mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION, 1524 sizeof(req), sizeof(resp)); 1525 1526 req.protocol_ver_min = GDMA_PROTOCOL_FIRST; 1527 req.protocol_ver_max = GDMA_PROTOCOL_LAST; 1528 1529 req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; 1530 req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; 1531 req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; 1532 req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; 1533 1534 req.drv_ver = 0; /* Unused*/ 1535 req.os_type = 0x10; /* Linux */ 1536 req.os_ver_major = LINUX_VERSION_MAJOR; 1537 req.os_ver_minor = LINUX_VERSION_PATCHLEVEL; 1538 req.os_ver_build = LINUX_VERSION_SUBLEVEL; 1539 strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1)); 1540 strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2)); 1541 strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3)); 1542 1543 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1544 if (err || resp.hdr.status) { 1545 dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n", 1546 err, resp.hdr.status); 1547 return err ? err : -EPROTO; 1548 } 1549 gc->pf_cap_flags1 = resp.pf_cap_flags1; 1550 gc->gdma_protocol_ver = resp.gdma_protocol_ver; 1551 1552 debugfs_create_x64("gdma_protocol_ver", 0400, gc->mana_pci_debugfs, 1553 &gc->gdma_protocol_ver); 1554 debugfs_create_x64("pf_cap_flags1", 0400, gc->mana_pci_debugfs, 1555 &gc->pf_cap_flags1); 1556 1557 if (resp.pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG) { 1558 err = mana_gd_query_hwc_timeout(pdev, &hwc->hwc_timeout); 1559 if (err) { 1560 dev_err(gc->dev, "Failed to set the hwc timeout %d\n", err); 1561 return err; 1562 } 1563 dev_dbg(gc->dev, "set the hwc timeout to %u\n", hwc->hwc_timeout); 1564 } 1565 return 0; 1566 } 1567 1568 int mana_gd_register_device(struct gdma_dev *gd) 1569 { 1570 struct gdma_context *gc = gd->gdma_context; 1571 struct gdma_register_device_resp resp = {}; 1572 struct gdma_general_req req = {}; 1573 int err; 1574 1575 gd->pdid = INVALID_PDID; 1576 gd->doorbell = INVALID_DOORBELL; 1577 gd->gpa_mkey = INVALID_MEM_KEY; 1578 1579 mana_gd_init_req_hdr(&req.hdr, GDMA_REGISTER_DEVICE, sizeof(req), 1580 sizeof(resp)); 1581 1582 req.hdr.dev_id = gd->dev_id; 1583 1584 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1585 if (err || resp.hdr.status) { 1586 dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n", 1587 err, resp.hdr.status); 1588 return err ? err : -EPROTO; 1589 } 1590 1591 /* Validate that doorbell page for db_id is within the BAR0 region. 1592 * In mana_gd_ring_doorbell(), the address is calculated as: 1593 * addr = db_page_base + db_page_size * db_id 1594 * = (bar0_va + db_page_off) + (db_page_size * db_id) 1595 * So we need: db_page_off + db_page_size * (db_id + 1) <= bar0_size 1596 */ 1597 if (gc->db_page_off + gc->db_page_size * ((u64)resp.db_id + 1) > gc->bar0_size) { 1598 dev_err(gc->dev, "Doorbell ID %u out of range\n", resp.db_id); 1599 return -EPROTO; 1600 } 1601 1602 gd->pdid = resp.pdid; 1603 gd->gpa_mkey = resp.gpa_mkey; 1604 gd->doorbell = resp.db_id; 1605 1606 return 0; 1607 } 1608 1609 int mana_gd_deregister_device(struct gdma_dev *gd) 1610 { 1611 struct gdma_context *gc = gd->gdma_context; 1612 struct gdma_general_resp resp = {}; 1613 struct gdma_general_req req = {}; 1614 int err; 1615 1616 if (gd->pdid == INVALID_PDID) 1617 return -EINVAL; 1618 1619 mana_gd_init_req_hdr(&req.hdr, GDMA_DEREGISTER_DEVICE, sizeof(req), 1620 sizeof(resp)); 1621 1622 req.hdr.dev_id = gd->dev_id; 1623 1624 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1625 if (err || resp.hdr.status) { 1626 if (mana_need_log(gc, err)) 1627 dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n", 1628 err, resp.hdr.status); 1629 if (!err) 1630 err = -EPROTO; 1631 } 1632 1633 gd->pdid = INVALID_PDID; 1634 gd->doorbell = INVALID_DOORBELL; 1635 gd->gpa_mkey = INVALID_MEM_KEY; 1636 1637 return err; 1638 } 1639 1640 u32 mana_gd_wq_avail_space(struct gdma_queue *wq) 1641 { 1642 u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE; 1643 u32 wq_size = wq->queue_size; 1644 1645 WARN_ON_ONCE(used_space > wq_size); 1646 1647 return wq_size - used_space; 1648 } 1649 1650 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset) 1651 { 1652 u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); 1653 1654 WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size); 1655 1656 return mana_gd_ring_ptr(wq, offset); 1657 } 1658 1659 static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, 1660 enum gdma_queue_type q_type, 1661 u32 client_oob_size, u32 sgl_data_size, 1662 u8 *wqe_ptr) 1663 { 1664 bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL); 1665 bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0); 1666 struct gdma_wqe *header = (struct gdma_wqe *)wqe_ptr; 1667 u8 *ptr; 1668 1669 memset(header, 0, sizeof(struct gdma_wqe)); 1670 header->num_sge = wqe_req->num_sge; 1671 header->inline_oob_size_div4 = client_oob_size / sizeof(u32); 1672 1673 if (oob_in_sgl) { 1674 WARN_ON_ONCE(wqe_req->num_sge < 2); 1675 1676 header->client_oob_in_sgl = 1; 1677 1678 if (pad_data) 1679 header->last_vbytes = wqe_req->sgl[0].size; 1680 } 1681 1682 if (q_type == GDMA_SQ) 1683 header->client_data_unit = wqe_req->client_data_unit; 1684 1685 /* The size of gdma_wqe + client_oob_size must be less than or equal 1686 * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond 1687 * the queue memory buffer boundary. 1688 */ 1689 ptr = wqe_ptr + sizeof(header); 1690 1691 if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) { 1692 memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size); 1693 1694 if (client_oob_size > wqe_req->inline_oob_size) 1695 memset(ptr + wqe_req->inline_oob_size, 0, 1696 client_oob_size - wqe_req->inline_oob_size); 1697 } 1698 1699 return sizeof(header) + client_oob_size; 1700 } 1701 1702 static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset, 1703 const struct gdma_wqe_request *wqe_req) 1704 { 1705 u32 size_to_end = mana_gd_ring_contig_avail(wq, sgl_offset); 1706 u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge; 1707 const u8 *address = (u8 *)wqe_req->sgl; 1708 1709 if (size_to_end < sgl_size) { 1710 memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, size_to_end); 1711 1712 address += size_to_end; 1713 sgl_size -= size_to_end; 1714 sgl_offset += size_to_end; 1715 if (sgl_offset == wq->queue_size) 1716 sgl_offset = 0; 1717 } 1718 1719 memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, sgl_size); 1720 } 1721 1722 int mana_gd_post_work_request(struct gdma_queue *wq, 1723 const struct gdma_wqe_request *wqe_req, 1724 struct gdma_posted_wqe_info *wqe_info) 1725 { 1726 u32 client_oob_size = wqe_req->inline_oob_size; 1727 u32 sgl_data_size; 1728 u32 max_wqe_size; 1729 u32 wqe_offset; 1730 u32 sgl_offset; 1731 u32 wqe_size; 1732 u32 oob_len; 1733 u8 *wqe_ptr; 1734 u32 head; 1735 1736 if (wqe_req->num_sge == 0) 1737 return -EINVAL; 1738 1739 if (wq->type == GDMA_RQ) { 1740 if (client_oob_size != 0) 1741 return -EINVAL; 1742 1743 client_oob_size = INLINE_OOB_SMALL_SIZE; 1744 1745 max_wqe_size = GDMA_MAX_RQE_SIZE; 1746 } else { 1747 if (client_oob_size != INLINE_OOB_SMALL_SIZE && 1748 client_oob_size != INLINE_OOB_LARGE_SIZE) 1749 return -EINVAL; 1750 1751 max_wqe_size = GDMA_MAX_SQE_SIZE; 1752 } 1753 1754 sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge; 1755 wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size + 1756 sgl_data_size, GDMA_WQE_BU_SIZE); 1757 if (wqe_size > max_wqe_size) 1758 return -EINVAL; 1759 1760 if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) 1761 return -ENOSPC; 1762 1763 if (wqe_info) 1764 wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; 1765 1766 head = wq->head; 1767 wqe_offset = (head * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); 1768 wqe_ptr = mana_gd_get_wqe_ptr(wq, head); 1769 oob_len = mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, 1770 sgl_data_size, wqe_ptr); 1771 1772 sgl_offset = wqe_offset + oob_len; 1773 if (sgl_offset >= wq->queue_size) 1774 sgl_offset -= wq->queue_size; 1775 1776 mana_gd_write_sgl(wq, sgl_offset, wqe_req); 1777 1778 wq->head += wqe_size / GDMA_WQE_BU_SIZE; 1779 1780 return 0; 1781 } 1782 EXPORT_SYMBOL_NS(mana_gd_post_work_request, "NET_MANA"); 1783 1784 int mana_gd_post_and_ring(struct gdma_queue *queue, 1785 const struct gdma_wqe_request *wqe_req, 1786 struct gdma_posted_wqe_info *wqe_info) 1787 { 1788 struct gdma_context *gc = queue->gdma_dev->gdma_context; 1789 int err; 1790 1791 err = mana_gd_post_work_request(queue, wqe_req, wqe_info); 1792 if (err) { 1793 dev_err(gc->dev, "Failed to post work req from queue type %d of size %u (err=%d)\n", 1794 queue->type, queue->queue_size, err); 1795 return err; 1796 } 1797 1798 mana_gd_wq_ring_doorbell(gc, queue); 1799 1800 return 0; 1801 } 1802 1803 static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) 1804 { 1805 unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe); 1806 u32 owner_bits, new_bits, old_bits; 1807 struct gdma_cqe *cqe; 1808 1809 cqe = mana_gd_ring_ptr(cq, (cq->head % num_cqe) * sizeof(*cqe)); 1810 owner_bits = cqe->cqe_info.owner_bits; 1811 1812 old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK; 1813 /* Return 0 if no more entries. */ 1814 if (owner_bits == old_bits) 1815 return 0; 1816 1817 new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK; 1818 /* Return -1 if overflow detected. */ 1819 if (WARN_ON_ONCE(owner_bits != new_bits)) 1820 return -1; 1821 1822 /* Per GDMA spec, rmb is necessary after checking owner_bits, before 1823 * reading completion info 1824 */ 1825 rmb(); 1826 1827 comp->wq_num = cqe->cqe_info.wq_num; 1828 comp->is_sq = cqe->cqe_info.is_sq; 1829 memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); 1830 1831 return 1; 1832 } 1833 1834 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) 1835 { 1836 int cqe_idx; 1837 int ret; 1838 1839 for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) { 1840 ret = mana_gd_read_cqe(cq, &comp[cqe_idx]); 1841 1842 if (ret < 0) { 1843 cq->head -= cqe_idx; 1844 return ret; 1845 } 1846 1847 if (ret == 0) 1848 break; 1849 1850 cq->head++; 1851 } 1852 1853 return cqe_idx; 1854 } 1855 EXPORT_SYMBOL_NS(mana_gd_poll_cq, "NET_MANA"); 1856 1857 static irqreturn_t mana_gd_intr(int irq, void *arg) 1858 { 1859 struct gdma_irq_context *gic = arg; 1860 struct list_head *eq_list = &gic->eq_list; 1861 struct gdma_queue *eq; 1862 1863 rcu_read_lock(); 1864 list_for_each_entry_rcu(eq, eq_list, entry) { 1865 gic->handler(eq); 1866 } 1867 rcu_read_unlock(); 1868 1869 return IRQ_HANDLED; 1870 } 1871 1872 void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi) 1873 { 1874 struct pci_dev *dev = to_pci_dev(gc->dev); 1875 struct gdma_irq_context *gic; 1876 struct msi_map irq_map; 1877 int irq; 1878 1879 mutex_lock(&gc->gic_mutex); 1880 1881 gic = xa_load(&gc->irq_contexts, msi); 1882 if (WARN_ON(!gic)) { 1883 mutex_unlock(&gc->gic_mutex); 1884 return; 1885 } 1886 1887 if (use_msi_bitmap) 1888 gic->bitmap_refs--; 1889 1890 if (use_msi_bitmap && gic->bitmap_refs == 0) 1891 clear_bit(msi, gc->msi_bitmap); 1892 1893 if (!refcount_dec_and_test(&gic->refcount)) 1894 goto out; 1895 1896 irq = gic->irq; 1897 1898 irq_update_affinity_hint(irq, NULL); 1899 free_irq(irq, gic); 1900 1901 if (gic->dyn_msix) { 1902 irq_map.virq = irq; 1903 irq_map.index = msi; 1904 pci_msix_free_irq(dev, irq_map); 1905 } 1906 1907 xa_erase(&gc->irq_contexts, msi); 1908 kfree(gic); 1909 1910 out: 1911 mutex_unlock(&gc->gic_mutex); 1912 } 1913 EXPORT_SYMBOL_NS(mana_gd_put_gic, "NET_MANA"); 1914 1915 /* 1916 * Get a GIC (GDMA IRQ Context) on a MSI vector 1917 * a MSI can be shared between different EQs, this function supports setting 1918 * up separate MSIs using a bitmap, or directly using the MSI index 1919 * 1920 * @use_msi_bitmap: 1921 * True if MSI is assigned by this function on available slots from bitmap. 1922 * False if MSI is passed from *msi_requested 1923 */ 1924 struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, 1925 bool use_msi_bitmap, 1926 int *msi_requested) 1927 { 1928 struct pci_dev *dev = to_pci_dev(gc->dev); 1929 struct gdma_irq_context *gic; 1930 struct msi_map irq_map = { }; 1931 int irq; 1932 int msi; 1933 int err; 1934 1935 mutex_lock(&gc->gic_mutex); 1936 1937 if (use_msi_bitmap) { 1938 msi = find_first_zero_bit(gc->msi_bitmap, gc->num_msix_usable); 1939 if (msi >= gc->num_msix_usable) { 1940 dev_err(gc->dev, "No free MSI vectors available\n"); 1941 gic = ERR_PTR(-ENOSPC); 1942 goto out; 1943 } 1944 *msi_requested = msi; 1945 } else { 1946 msi = *msi_requested; 1947 } 1948 1949 gic = xa_load(&gc->irq_contexts, msi); 1950 if (gic) { 1951 refcount_inc(&gic->refcount); 1952 if (use_msi_bitmap) { 1953 gic->bitmap_refs++; 1954 set_bit(msi, gc->msi_bitmap); 1955 } 1956 goto out; 1957 } 1958 1959 irq = pci_irq_vector(dev, msi); 1960 if (irq == -EINVAL) { 1961 irq_map = pci_msix_alloc_irq_at(dev, msi, NULL); 1962 if (!irq_map.virq) { 1963 err = irq_map.index; 1964 dev_err(gc->dev, 1965 "Failed to alloc irq_map msi %d err %d\n", 1966 msi, err); 1967 gic = ERR_PTR(err); 1968 goto out; 1969 } 1970 irq = irq_map.virq; 1971 msi = irq_map.index; 1972 *msi_requested = msi; 1973 } 1974 1975 gic = kzalloc(sizeof(*gic), GFP_KERNEL); 1976 if (!gic) { 1977 gic = ERR_PTR(-ENOMEM); 1978 if (irq_map.virq) 1979 pci_msix_free_irq(dev, irq_map); 1980 goto out; 1981 } 1982 1983 gic->handler = mana_gd_process_eq_events; 1984 gic->msi = msi; 1985 gic->irq = irq; 1986 INIT_LIST_HEAD(&gic->eq_list); 1987 spin_lock_init(&gic->lock); 1988 1989 if (!gic->msi) 1990 snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", 1991 pci_name(dev)); 1992 else 1993 snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_msi%d@pci:%s", 1994 gic->msi, pci_name(dev)); 1995 1996 err = request_irq(irq, mana_gd_intr, 0, gic->name, gic); 1997 if (err) { 1998 dev_err(gc->dev, "Failed to request irq %d %s\n", 1999 irq, gic->name); 2000 kfree(gic); 2001 gic = ERR_PTR(err); 2002 if (irq_map.virq) 2003 pci_msix_free_irq(dev, irq_map); 2004 goto out; 2005 } 2006 2007 gic->dyn_msix = !!irq_map.virq; 2008 refcount_set(&gic->refcount, 1); 2009 gic->bitmap_refs = use_msi_bitmap ? 1 : 0; 2010 2011 err = xa_err(xa_store(&gc->irq_contexts, msi, gic, GFP_KERNEL)); 2012 if (err) { 2013 dev_err(gc->dev, "Failed to store irq context for msi %d: %d\n", 2014 msi, err); 2015 free_irq(irq, gic); 2016 kfree(gic); 2017 gic = ERR_PTR(err); 2018 if (irq_map.virq) 2019 pci_msix_free_irq(dev, irq_map); 2020 goto out; 2021 } 2022 2023 if (use_msi_bitmap) 2024 set_bit(msi, gc->msi_bitmap); 2025 2026 out: 2027 mutex_unlock(&gc->gic_mutex); 2028 return gic; 2029 } 2030 EXPORT_SYMBOL_NS(mana_gd_get_gic, "NET_MANA"); 2031 2032 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r) 2033 { 2034 r->map = bitmap_zalloc(res_avail, GFP_KERNEL); 2035 if (!r->map) 2036 return -ENOMEM; 2037 2038 r->size = res_avail; 2039 spin_lock_init(&r->lock); 2040 2041 return 0; 2042 } 2043 2044 void mana_gd_free_res_map(struct gdma_resource *r) 2045 { 2046 bitmap_free(r->map); 2047 r->map = NULL; 2048 r->size = 0; 2049 } 2050 2051 /* 2052 * Spread on CPUs with the following heuristics: 2053 * 2054 * 1. No more than one IRQ per CPU, if possible; 2055 * 2. NUMA locality is the second priority; 2056 * 3. Sibling dislocality is the last priority. 2057 * 2058 * Let's consider this topology: 2059 * 2060 * Node 0 1 2061 * Core 0 1 2 3 2062 * CPU 0 1 2 3 4 5 6 7 2063 * 2064 * The most performant IRQ distribution based on the above topology 2065 * and heuristics may look like this: 2066 * 2067 * IRQ Nodes Cores CPUs 2068 * 0 1 0 0-1 2069 * 1 1 1 2-3 2070 * 2 1 0 0-1 2071 * 3 1 1 2-3 2072 * 4 2 2 4-5 2073 * 5 2 3 6-7 2074 * 6 2 2 4-5 2075 * 7 2 3 6-7 2076 * 2077 * The heuristics is implemented as follows. 2078 * 2079 * The outer for_each() loop resets the 'weight' to the actual number 2080 * of CPUs in the hop. Then inner for_each() loop decrements it by the 2081 * number of sibling groups (cores) while assigning first set of IRQs 2082 * to each group. IRQs 0 and 1 above are distributed this way. 2083 * 2084 * Now, because NUMA locality is more important, we should walk the 2085 * same set of siblings and assign 2nd set of IRQs (2 and 3), and it's 2086 * implemented by the medium while() loop. We do like this unless the 2087 * number of IRQs assigned on this hop will not become equal to number 2088 * of CPUs in the hop (weight == 0). Then we switch to the next hop and 2089 * do the same thing. 2090 */ 2091 2092 static int mana_irq_setup_numa_aware(unsigned int *irqs, unsigned int len, 2093 int node, bool skip_first_cpu) 2094 { 2095 const struct cpumask *next, *prev = cpu_none_mask; 2096 cpumask_var_t cpus __free(free_cpumask_var); 2097 int cpu, weight; 2098 2099 if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) 2100 return -ENOMEM; 2101 2102 rcu_read_lock(); 2103 for_each_numa_hop_mask(next, node) { 2104 weight = cpumask_weight_andnot(next, prev); 2105 while (weight > 0) { 2106 cpumask_andnot(cpus, next, prev); 2107 for_each_cpu(cpu, cpus) { 2108 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); 2109 --weight; 2110 2111 if (unlikely(skip_first_cpu)) { 2112 skip_first_cpu = false; 2113 continue; 2114 } 2115 2116 if (len-- == 0) 2117 goto done; 2118 2119 irq_set_affinity_and_hint(*irqs++, topology_sibling_cpumask(cpu)); 2120 } 2121 } 2122 prev = next; 2123 } 2124 done: 2125 rcu_read_unlock(); 2126 return 0; 2127 } 2128 2129 /* must be called with cpus_read_lock() held */ 2130 static void mana_irq_setup_linear(unsigned int *irqs, unsigned int len) 2131 { 2132 int cpu; 2133 2134 for_each_online_cpu(cpu) { 2135 if (len == 0) 2136 break; 2137 2138 irq_set_affinity_and_hint(*irqs++, cpumask_of(cpu)); 2139 len--; 2140 } 2141 } 2142 2143 static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) 2144 { 2145 struct gdma_context *gc = pci_get_drvdata(pdev); 2146 struct gdma_irq_context *gic; 2147 int *irqs, err, i, msi; 2148 2149 irqs = kmalloc_objs(int, nvec); 2150 if (!irqs) 2151 return -ENOMEM; 2152 2153 /* 2154 * In this function, num_msix_usable = HWC IRQ + Queue IRQ. 2155 * nvec is only Queue IRQ (HWC already setup). 2156 * While processing the next pci irq vector, we start with index 1, 2157 * as IRQ vector at index 0 is already processed for HWC. 2158 * However, the population of irqs array starts with index 0, to be 2159 * further used in mana_irq_setup_numa_aware() 2160 */ 2161 for (i = 1; i <= nvec; i++) { 2162 msi = i; 2163 gic = mana_gd_get_gic(gc, false, &msi); 2164 if (IS_ERR(gic)) { 2165 err = PTR_ERR(gic); 2166 goto free_irq; 2167 } 2168 2169 irqs[i - 1] = gic->irq; 2170 } 2171 2172 /* 2173 * When calling mana_irq_setup_numa_aware() for dynamically added IRQs, 2174 * if number of CPUs is more than or equal to allocated MSI-X, we need to 2175 * skip the first CPU sibling group since they are already affinitized to 2176 * HWC IRQ 2177 */ 2178 cpus_read_lock(); 2179 if (gc->num_msix_usable <= num_online_cpus()) { 2180 err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, 2181 true); 2182 if (err) { 2183 cpus_read_unlock(); 2184 goto free_irq; 2185 } 2186 } else { 2187 /* 2188 * When num_msix_usable are more than num_online_cpus, our 2189 * queue IRQs should be equal to num of online vCPUs. 2190 * We try to make sure queue IRQs spread across all vCPUs. 2191 * In such a case NUMA or CPU core affinity does not matter. 2192 * Note: in this case the total mana IRQ should always be 2193 * num_online_cpus + 1. The first HWC IRQ is already handled 2194 * in HWC setup calls 2195 * However, if CPUs went offline since num_msix_usable was 2196 * computed, queue IRQs will be more than num_online_cpus(). 2197 * In such cases remaining extra IRQs will retain their default 2198 * affinity. 2199 */ 2200 int first_unassigned = num_online_cpus(); 2201 2202 if (nvec > first_unassigned) { 2203 char buf[32]; 2204 2205 if (first_unassigned == nvec - 1) 2206 snprintf(buf, sizeof(buf), "%d", 2207 first_unassigned); 2208 else 2209 snprintf(buf, sizeof(buf), "%d-%d", 2210 first_unassigned, nvec - 1); 2211 2212 dev_dbg(&pdev->dev, 2213 "MANA IRQ indices #%s will retain the default CPU affinity\n", 2214 buf); 2215 } 2216 2217 mana_irq_setup_linear(irqs, nvec); 2218 } 2219 2220 cpus_read_unlock(); 2221 kfree(irqs); 2222 return 0; 2223 2224 free_irq: 2225 for (i -= 1; i > 0; i--) 2226 mana_gd_put_gic(gc, false, i); 2227 kfree(irqs); 2228 return err; 2229 } 2230 2231 static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) 2232 { 2233 struct gdma_context *gc = pci_get_drvdata(pdev); 2234 struct gdma_irq_context *gic; 2235 int *irqs, *start_irqs; 2236 unsigned int cpu; 2237 int err, i, msi; 2238 2239 irqs = kmalloc_objs(int, nvec); 2240 if (!irqs) 2241 return -ENOMEM; 2242 2243 start_irqs = irqs; 2244 2245 for (i = 0; i < nvec; i++) { 2246 msi = i; 2247 gic = mana_gd_get_gic(gc, false, &msi); 2248 if (IS_ERR(gic)) { 2249 err = PTR_ERR(gic); 2250 goto free_irq; 2251 } 2252 2253 irqs[i] = gic->irq; 2254 } 2255 2256 /* If number of IRQ is one extra than number of online CPUs, 2257 * then we need to assign IRQ0 (hwc irq) and IRQ1 to 2258 * same CPU. 2259 * Else we will use different CPUs for IRQ0 and IRQ1. 2260 * Also we are using cpumask_local_spread instead of 2261 * cpumask_first for the node, because the node can be 2262 * mem only. 2263 */ 2264 cpus_read_lock(); 2265 if (nvec > num_online_cpus()) { 2266 cpu = cpumask_local_spread(0, gc->numa_node); 2267 irq_set_affinity_and_hint(irqs[0], cpumask_of(cpu)); 2268 irqs++; 2269 nvec -= 1; 2270 } 2271 2272 err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, false); 2273 if (err) { 2274 cpus_read_unlock(); 2275 goto free_irq; 2276 } 2277 2278 cpus_read_unlock(); 2279 kfree(start_irqs); 2280 return 0; 2281 2282 free_irq: 2283 for (i -= 1; i >= 0; i--) 2284 mana_gd_put_gic(gc, false, i); 2285 2286 kfree(start_irqs); 2287 return err; 2288 } 2289 2290 static int mana_gd_setup_hwc_irqs(struct pci_dev *pdev) 2291 { 2292 struct gdma_context *gc = pci_get_drvdata(pdev); 2293 unsigned int max_irqs, min_irqs; 2294 int nvec, err; 2295 2296 if (pci_msix_can_alloc_dyn(pdev)) { 2297 max_irqs = 1; 2298 min_irqs = 1; 2299 } else { 2300 /* Need 1 interrupt for HWC */ 2301 max_irqs = min(num_online_cpus(), MANA_MAX_NUM_QUEUES) + 1; 2302 min_irqs = 2; 2303 gc->msi_sharing = true; 2304 } 2305 2306 nvec = pci_alloc_irq_vectors(pdev, min_irqs, max_irqs, PCI_IRQ_MSIX); 2307 if (nvec < 0) 2308 return nvec; 2309 2310 err = mana_gd_setup_irqs(pdev, nvec); 2311 if (err) { 2312 pci_free_irq_vectors(pdev); 2313 return err; 2314 } 2315 2316 gc->num_msix_usable = nvec; 2317 gc->max_num_msix = nvec; 2318 2319 return 0; 2320 } 2321 2322 static int mana_gd_setup_remaining_irqs(struct pci_dev *pdev) 2323 { 2324 struct gdma_context *gc = pci_get_drvdata(pdev); 2325 struct msi_map irq_map; 2326 int max_irqs, i, err; 2327 2328 if (!pci_msix_can_alloc_dyn(pdev)) 2329 /* remain irqs are already allocated with HWC IRQ */ 2330 return 0; 2331 2332 /* allocate only remaining IRQs*/ 2333 max_irqs = gc->num_msix_usable - 1; 2334 2335 for (i = 1; i <= max_irqs; i++) { 2336 irq_map = pci_msix_alloc_irq_at(pdev, i, NULL); 2337 if (!irq_map.virq) { 2338 err = irq_map.index; 2339 /* caller will handle cleaning up all allocated 2340 * irqs, after HWC is destroyed 2341 */ 2342 return err; 2343 } 2344 } 2345 2346 err = mana_gd_setup_dyn_irqs(pdev, max_irqs); 2347 if (err) 2348 return err; 2349 2350 gc->max_num_msix = gc->max_num_msix + max_irqs; 2351 2352 return 0; 2353 } 2354 2355 static void mana_gd_remove_irqs(struct pci_dev *pdev) 2356 { 2357 struct gdma_context *gc = pci_get_drvdata(pdev); 2358 int i; 2359 2360 if (gc->max_num_msix < 1) 2361 return; 2362 2363 for (i = 0; i < gc->max_num_msix; i++) { 2364 if (!xa_load(&gc->irq_contexts, i)) 2365 continue; 2366 2367 mana_gd_put_gic(gc, false, i); 2368 } 2369 2370 WARN_ON(!xa_empty(&gc->irq_contexts)); 2371 2372 pci_free_irq_vectors(pdev); 2373 2374 bitmap_free(gc->msi_bitmap); 2375 gc->msi_bitmap = NULL; 2376 gc->max_num_msix = 0; 2377 gc->num_msix_usable = 0; 2378 } 2379 2380 static int mana_gd_setup(struct pci_dev *pdev) 2381 { 2382 struct gdma_context *gc = pci_get_drvdata(pdev); 2383 int err; 2384 2385 gc->mana_pci_debugfs = debugfs_create_dir(pci_name(pdev), 2386 mana_debugfs_root); 2387 2388 err = mana_gd_init_registers(pdev); 2389 if (err) 2390 goto remove_debugfs; 2391 2392 mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); 2393 2394 gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); 2395 if (!gc->service_wq) { 2396 err = -ENOMEM; 2397 goto remove_debugfs; 2398 } 2399 2400 err = mana_gd_setup_hwc_irqs(pdev); 2401 if (err) { 2402 dev_err(gc->dev, "Failed to setup IRQs for HWC creation: %d\n", 2403 err); 2404 goto free_workqueue; 2405 } 2406 2407 err = mana_hwc_create_channel(gc); 2408 if (err) 2409 goto remove_irq; 2410 2411 err = mana_gd_verify_vf_version(pdev); 2412 if (err) 2413 goto destroy_hwc; 2414 2415 err = mana_gd_detect_devices(pdev); 2416 if (err) 2417 goto destroy_hwc; 2418 2419 err = mana_gd_query_max_resources(pdev); 2420 if (err) 2421 goto destroy_hwc; 2422 2423 err = mana_gd_setup_remaining_irqs(pdev); 2424 if (err) { 2425 dev_err(gc->dev, "Failed to setup remaining IRQs: %d", err); 2426 goto destroy_hwc; 2427 } 2428 2429 if (!gc->msi_sharing) { 2430 gc->msi_bitmap = bitmap_zalloc(gc->num_msix_usable, GFP_KERNEL); 2431 if (!gc->msi_bitmap) { 2432 err = -ENOMEM; 2433 goto destroy_hwc; 2434 } 2435 /* Set bit for HWC */ 2436 set_bit(0, gc->msi_bitmap); 2437 } 2438 2439 dev_dbg(&pdev->dev, "mana gdma setup successful\n"); 2440 return 0; 2441 2442 destroy_hwc: 2443 mana_hwc_destroy_channel(gc); 2444 remove_irq: 2445 mana_gd_remove_irqs(pdev); 2446 free_workqueue: 2447 destroy_workqueue(gc->service_wq); 2448 gc->service_wq = NULL; 2449 remove_debugfs: 2450 debugfs_remove_recursive(gc->mana_pci_debugfs); 2451 gc->mana_pci_debugfs = NULL; 2452 dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); 2453 return err; 2454 } 2455 2456 static void mana_gd_cleanup_device(struct pci_dev *pdev) 2457 { 2458 struct gdma_context *gc = pci_get_drvdata(pdev); 2459 2460 mana_hwc_destroy_channel(gc); 2461 2462 mana_gd_remove_irqs(pdev); 2463 2464 if (gc->service_wq) { 2465 destroy_workqueue(gc->service_wq); 2466 gc->service_wq = NULL; 2467 } 2468 2469 debugfs_remove_recursive(gc->mana_pci_debugfs); 2470 gc->mana_pci_debugfs = NULL; 2471 2472 dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); 2473 } 2474 2475 static bool mana_is_pf(unsigned short dev_id) 2476 { 2477 return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; 2478 } 2479 2480 static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 2481 { 2482 struct gdma_context *gc; 2483 void __iomem *bar0_va; 2484 int bar = 0; 2485 int err; 2486 2487 /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ 2488 BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); 2489 2490 err = pci_enable_device(pdev); 2491 if (err) { 2492 dev_err(&pdev->dev, "Failed to enable pci device (err=%d)\n", err); 2493 return -ENXIO; 2494 } 2495 2496 pci_set_master(pdev); 2497 2498 err = pci_request_regions(pdev, "mana"); 2499 if (err) 2500 goto disable_dev; 2501 2502 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2503 if (err) { 2504 dev_err(&pdev->dev, "DMA set mask failed: %d\n", err); 2505 goto release_region; 2506 } 2507 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 2508 2509 err = -ENOMEM; 2510 gc = vzalloc(sizeof(*gc)); 2511 if (!gc) 2512 goto release_region; 2513 2514 mutex_init(&gc->eq_test_event_mutex); 2515 mutex_init(&gc->gic_mutex); 2516 pci_set_drvdata(pdev, gc); 2517 gc->bar0_pa = pci_resource_start(pdev, 0); 2518 gc->bar0_size = pci_resource_len(pdev, 0); 2519 2520 bar0_va = pci_iomap(pdev, bar, 0); 2521 if (!bar0_va) 2522 goto free_gc; 2523 2524 gc->numa_node = dev_to_node(&pdev->dev); 2525 gc->is_pf = mana_is_pf(pdev->device); 2526 gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); 2527 2528 gc->bar0_va = bar0_va; 2529 gc->dev = &pdev->dev; 2530 xa_init(&gc->irq_contexts); 2531 2532 err = mana_gd_setup(pdev); 2533 if (err) 2534 goto unmap_bar; 2535 2536 err = mana_probe(&gc->mana, false); 2537 if (err) 2538 goto cleanup_gd; 2539 2540 err = mana_rdma_probe(&gc->mana_ib); 2541 if (err) 2542 goto cleanup_mana; 2543 2544 /* 2545 * If a hardware reset event has occurred over HWC during probe, 2546 * rollback and perform hardware reset procedure. 2547 */ 2548 if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { 2549 err = -EPROTO; 2550 goto cleanup_mana_rdma; 2551 } 2552 2553 return 0; 2554 2555 cleanup_mana_rdma: 2556 mana_rdma_remove(&gc->mana_ib); 2557 cleanup_mana: 2558 mana_remove(&gc->mana, false); 2559 cleanup_gd: 2560 mana_gd_cleanup_device(pdev); 2561 unmap_bar: 2562 xa_destroy(&gc->irq_contexts); 2563 pci_iounmap(pdev, bar0_va); 2564 free_gc: 2565 pci_set_drvdata(pdev, NULL); 2566 vfree(gc); 2567 release_region: 2568 pci_release_regions(pdev); 2569 disable_dev: 2570 pci_disable_device(pdev); 2571 dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); 2572 2573 /* 2574 * Hardware could be in recovery mode and the HWC returns TIMEDOUT or 2575 * EPROTO from mana_gd_setup(), mana_probe() or mana_rdma_probe(), or 2576 * we received a hardware reset event over HWC interrupt. In this case, 2577 * perform the device recovery procedure after MANA_SERVICE_PERIOD 2578 * seconds. 2579 */ 2580 if (err == -ETIMEDOUT || err == -EPROTO) { 2581 struct mana_dev_recovery *dev; 2582 unsigned long flags; 2583 2584 dev_info(&pdev->dev, "Start MANA recovery mode\n"); 2585 2586 dev = kzalloc_obj(*dev); 2587 if (!dev) 2588 return err; 2589 2590 dev->pdev = pci_dev_get(pdev); 2591 dev->type = GDMA_EQE_HWC_RESET_REQUEST; 2592 2593 spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); 2594 list_add_tail(&dev->list, &mana_dev_recovery_work.dev_list); 2595 spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); 2596 2597 schedule_delayed_work(&mana_dev_recovery_work.work, 2598 secs_to_jiffies(MANA_SERVICE_PERIOD)); 2599 } 2600 2601 return err; 2602 } 2603 2604 static void mana_gd_remove(struct pci_dev *pdev) 2605 { 2606 struct gdma_context *gc = pci_get_drvdata(pdev); 2607 2608 pci_disable_sriov(pdev); 2609 2610 mana_rdma_remove(&gc->mana_ib); 2611 mana_remove(&gc->mana, false); 2612 2613 mana_gd_cleanup_device(pdev); 2614 2615 xa_destroy(&gc->irq_contexts); 2616 2617 pci_iounmap(pdev, gc->bar0_va); 2618 2619 vfree(gc); 2620 2621 pci_release_regions(pdev); 2622 pci_disable_device(pdev); 2623 2624 dev_dbg(&pdev->dev, "mana gdma remove successful\n"); 2625 } 2626 2627 /* The 'state' parameter is not used. */ 2628 int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) 2629 { 2630 struct gdma_context *gc = pci_get_drvdata(pdev); 2631 2632 mana_rdma_remove(&gc->mana_ib); 2633 mana_remove(&gc->mana, true); 2634 2635 mana_gd_cleanup_device(pdev); 2636 2637 return 0; 2638 } 2639 2640 int mana_gd_resume(struct pci_dev *pdev) 2641 { 2642 struct gdma_context *gc = pci_get_drvdata(pdev); 2643 int err; 2644 2645 err = mana_gd_setup(pdev); 2646 if (err) 2647 return err; 2648 2649 err = mana_probe(&gc->mana, true); 2650 if (err) 2651 goto cleanup_gd; 2652 2653 err = mana_rdma_probe(&gc->mana_ib); 2654 if (err) 2655 mana_rdma_remove(&gc->mana_ib); 2656 2657 return err; 2658 2659 cleanup_gd: 2660 mana_gd_cleanup_device(pdev); 2661 return err; 2662 } 2663 2664 /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ 2665 static void mana_gd_shutdown(struct pci_dev *pdev) 2666 { 2667 struct gdma_context *gc = pci_get_drvdata(pdev); 2668 2669 dev_info(&pdev->dev, "Shutdown was called\n"); 2670 2671 mana_rdma_remove(&gc->mana_ib); 2672 mana_remove(&gc->mana, true); 2673 2674 mana_gd_cleanup_device(pdev); 2675 2676 pci_disable_device(pdev); 2677 } 2678 2679 static int mana_sriov_configure(struct pci_dev *pdev, int numvfs) 2680 { 2681 int err = 0; 2682 2683 dev_info(&pdev->dev, "Requested num VFs: %d\n", numvfs); 2684 2685 if (numvfs > 0) { 2686 err = pci_enable_sriov(pdev, numvfs); 2687 } else { 2688 if (pci_vfs_assigned(pdev)) { 2689 dev_warn(&pdev->dev, 2690 "Cannot disable SR-IOV while VFs are assigned\n"); 2691 return -EPERM; 2692 } 2693 2694 pci_disable_sriov(pdev); 2695 } 2696 2697 return err ? err : numvfs; 2698 } 2699 2700 static const struct pci_device_id mana_id_table[] = { 2701 { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, 2702 { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, 2703 { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, 2704 { } 2705 }; 2706 2707 static struct pci_driver mana_driver = { 2708 .name = "mana", 2709 .id_table = mana_id_table, 2710 .probe = mana_gd_probe, 2711 .remove = mana_gd_remove, 2712 .suspend = mana_gd_suspend, 2713 .resume = mana_gd_resume, 2714 .shutdown = mana_gd_shutdown, 2715 .sriov_configure = mana_sriov_configure, 2716 }; 2717 2718 static int __init mana_driver_init(void) 2719 { 2720 int err; 2721 2722 INIT_LIST_HEAD(&mana_dev_recovery_work.dev_list); 2723 spin_lock_init(&mana_dev_recovery_work.lock); 2724 INIT_DELAYED_WORK(&mana_dev_recovery_work.work, mana_recovery_delayed_func); 2725 2726 mana_debugfs_root = debugfs_create_dir("mana", NULL); 2727 2728 err = pci_register_driver(&mana_driver); 2729 if (err) { 2730 debugfs_remove(mana_debugfs_root); 2731 mana_debugfs_root = NULL; 2732 } 2733 2734 return err; 2735 } 2736 2737 static void __exit mana_driver_exit(void) 2738 { 2739 struct mana_dev_recovery *dev; 2740 unsigned long flags; 2741 2742 disable_delayed_work_sync(&mana_dev_recovery_work.work); 2743 2744 spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); 2745 while (!list_empty(&mana_dev_recovery_work.dev_list)) { 2746 dev = list_first_entry(&mana_dev_recovery_work.dev_list, 2747 struct mana_dev_recovery, list); 2748 list_del(&dev->list); 2749 pci_dev_put(dev->pdev); 2750 kfree(dev); 2751 } 2752 spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); 2753 2754 pci_unregister_driver(&mana_driver); 2755 2756 debugfs_remove(mana_debugfs_root); 2757 2758 mana_debugfs_root = NULL; 2759 } 2760 2761 module_init(mana_driver_init); 2762 module_exit(mana_driver_exit); 2763 2764 MODULE_DEVICE_TABLE(pci, mana_id_table); 2765 2766 MODULE_LICENSE("Dual BSD/GPL"); 2767 MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); 2768