1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 #include <linux/iopoll.h> 11 12 #include "snet_vdpa.h" 13 14 /* SNET DPU device ID */ 15 #define SNET_DEVICE_ID 0x1000 16 /* SNET signature */ 17 #define SNET_SIGNATURE 0xD0D06363 18 /* Max. config version that we can work with */ 19 #define SNET_CFG_VERSION 0x2 20 /* Queue align */ 21 #define SNET_QUEUE_ALIGNMENT PAGE_SIZE 22 /* Kick value to notify that new data is available */ 23 #define SNET_KICK_VAL 0x1 24 #define SNET_CONFIG_OFF 0x0 25 /* How long we are willing to wait for a SNET device */ 26 #define SNET_DETECT_TIMEOUT 5000000 27 /* How long should we wait for the DPU to read our config */ 28 #define SNET_READ_CFG_TIMEOUT 3000000 29 /* Size of configs written to the DPU */ 30 #define SNET_GENERAL_CFG_LEN 36 31 #define SNET_GENERAL_CFG_VQ_LEN 40 32 33 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) 34 { 35 return container_of(vdpa, struct snet, vdpa); 36 } 37 38 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) 39 { 40 struct snet *snet = data; 41 /* Call callback if any */ 42 if (likely(snet->cb.callback)) 43 return snet->cb.callback(snet->cb.private); 44 45 return IRQ_HANDLED; 46 } 47 48 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) 49 { 50 struct snet_vq *vq = data; 51 /* Call callback if any */ 52 if (likely(vq->cb.callback)) 53 return vq->cb.callback(vq->cb.private); 54 55 return IRQ_HANDLED; 56 } 57 58 static void snet_free_irqs(struct snet *snet) 59 { 60 struct psnet *psnet = snet->psnet; 61 struct pci_dev *pdev; 62 u32 i; 63 64 /* Which Device allcoated the IRQs? */ 65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 66 pdev = snet->pdev->physfn; 67 else 68 pdev = snet->pdev; 69 70 /* Free config's IRQ */ 71 if (snet->cfg_irq != -1) { 72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); 73 snet->cfg_irq = -1; 74 } 75 /* Free VQ IRQs */ 76 for (i = 0; i < snet->cfg->vq_num; i++) { 77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { 78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); 79 snet->vqs[i]->irq = -1; 80 } 81 } 82 83 /* IRQ vectors are freed when the pci remove callback is called */ 84 } 85 86 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 87 u64 driver_area, u64 device_area) 88 { 89 struct snet *snet = vdpa_to_snet(vdev); 90 /* save received parameters in vqueue sturct */ 91 snet->vqs[idx]->desc_area = desc_area; 92 snet->vqs[idx]->driver_area = driver_area; 93 snet->vqs[idx]->device_area = device_area; 94 95 return 0; 96 } 97 98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 99 { 100 struct snet *snet = vdpa_to_snet(vdev); 101 /* save num in vqueue */ 102 snet->vqs[idx]->num = num; 103 } 104 105 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) 106 { 107 struct snet *snet = vdpa_to_snet(vdev); 108 /* not ready - ignore */ 109 if (unlikely(!snet->vqs[idx]->ready)) 110 return; 111 112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 113 } 114 115 static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data) 116 { 117 struct snet *snet = vdpa_to_snet(vdev); 118 u16 idx = data & 0xFFFF; 119 120 /* not ready - ignore */ 121 if (unlikely(!snet->vqs[idx]->ready)) 122 return; 123 124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 125 } 126 127 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 128 { 129 struct snet *snet = vdpa_to_snet(vdev); 130 131 snet->vqs[idx]->cb.callback = cb->callback; 132 snet->vqs[idx]->cb.private = cb->private; 133 } 134 135 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 136 { 137 struct snet *snet = vdpa_to_snet(vdev); 138 139 snet->vqs[idx]->ready = ready; 140 } 141 142 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) 143 { 144 struct snet *snet = vdpa_to_snet(vdev); 145 146 return snet->vqs[idx]->ready; 147 } 148 149 static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state) 150 { 151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { 152 const struct vdpa_vq_state_packed *p = &state->packed; 153 154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && 155 p->last_avail_idx == 0 && p->last_used_idx == 0) 156 return true; 157 } else { 158 const struct vdpa_vq_state_split *s = &state->split; 159 160 if (s->avail_index == 0) 161 return true; 162 } 163 164 return false; 165 } 166 167 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) 168 { 169 struct snet *snet = vdpa_to_snet(vdev); 170 171 /* We can set any state for config version 2+ */ 172 if (SNET_CFG_VER(snet, 2)) { 173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); 174 return 0; 175 } 176 177 /* Older config - we can't set the VQ state. 178 * Return 0 only if this is the initial state we use in the DPU. 179 */ 180 if (snet_vq_state_is_initial(snet, state)) 181 return 0; 182 183 return -EOPNOTSUPP; 184 } 185 186 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 187 { 188 struct snet *snet = vdpa_to_snet(vdev); 189 190 return snet_read_vq_state(snet, idx, state); 191 } 192 193 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) 194 { 195 struct snet *snet = vdpa_to_snet(vdev); 196 197 return snet->vqs[idx]->irq; 198 } 199 200 static u32 snet_get_vq_align(struct vdpa_device *vdev) 201 { 202 return (u32)SNET_QUEUE_ALIGNMENT; 203 } 204 205 static int snet_reset_dev(struct snet *snet) 206 { 207 struct pci_dev *pdev = snet->pdev; 208 int ret = 0; 209 u32 i; 210 211 /* If status is 0, nothing to do */ 212 if (!snet->status) 213 return 0; 214 215 /* If DPU started, destroy it */ 216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) 217 ret = snet_destroy_dev(snet); 218 219 /* Clear VQs */ 220 for (i = 0; i < snet->cfg->vq_num; i++) { 221 if (!snet->vqs[i]) 222 continue; 223 snet->vqs[i]->cb.callback = NULL; 224 snet->vqs[i]->cb.private = NULL; 225 snet->vqs[i]->desc_area = 0; 226 snet->vqs[i]->device_area = 0; 227 snet->vqs[i]->driver_area = 0; 228 snet->vqs[i]->ready = false; 229 } 230 231 /* Clear config callback */ 232 snet->cb.callback = NULL; 233 snet->cb.private = NULL; 234 /* Free IRQs */ 235 snet_free_irqs(snet); 236 /* Reset status */ 237 snet->status = 0; 238 snet->dpu_ready = false; 239 240 if (ret) 241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret); 242 else 243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); 244 245 return 0; 246 } 247 248 static int snet_reset(struct vdpa_device *vdev) 249 { 250 struct snet *snet = vdpa_to_snet(vdev); 251 252 return snet_reset_dev(snet); 253 } 254 255 static size_t snet_get_config_size(struct vdpa_device *vdev) 256 { 257 struct snet *snet = vdpa_to_snet(vdev); 258 259 return (size_t)snet->cfg->cfg_size; 260 } 261 262 static u64 snet_get_features(struct vdpa_device *vdev) 263 { 264 struct snet *snet = vdpa_to_snet(vdev); 265 266 return snet->cfg->features; 267 } 268 269 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) 270 { 271 struct snet *snet = vdpa_to_snet(vdev); 272 273 snet->negotiated_features = snet->cfg->features & features; 274 return 0; 275 } 276 277 static u64 snet_get_drv_features(struct vdpa_device *vdev) 278 { 279 struct snet *snet = vdpa_to_snet(vdev); 280 281 return snet->negotiated_features; 282 } 283 284 static u16 snet_get_vq_num_max(struct vdpa_device *vdev) 285 { 286 struct snet *snet = vdpa_to_snet(vdev); 287 288 return (u16)snet->cfg->vq_size; 289 } 290 291 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 292 { 293 struct snet *snet = vdpa_to_snet(vdev); 294 295 snet->cb.callback = cb->callback; 296 snet->cb.private = cb->private; 297 } 298 299 static u32 snet_get_device_id(struct vdpa_device *vdev) 300 { 301 struct snet *snet = vdpa_to_snet(vdev); 302 303 return snet->cfg->virtio_id; 304 } 305 306 static u32 snet_get_vendor_id(struct vdpa_device *vdev) 307 { 308 return (u32)PCI_VENDOR_ID_SOLIDRUN; 309 } 310 311 static u8 snet_get_status(struct vdpa_device *vdev) 312 { 313 struct snet *snet = vdpa_to_snet(vdev); 314 315 return snet->status; 316 } 317 318 static int snet_write_conf(struct snet *snet) 319 { 320 u32 off, i, tmp; 321 int ret; 322 323 /* No need to write the config twice */ 324 if (snet->dpu_ready) 325 return true; 326 327 /* Snet data : 328 * 329 * General data: SNET_GENERAL_CFG_LEN bytes long 330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | 332 * 333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long 334 * 0 0x4 0x8 335 * | VQ SID AND QUEUE SIZE | IRQ Index | 336 * | DESC AREA | 337 * | DEVICE AREA | 338 * | DRIVER AREA | 339 * | VQ STATE (CFG 2+) | RSVD | 340 * 341 * Magic number should be written last, this is the DPU indication that the data is ready 342 */ 343 344 /* Init offset */ 345 off = snet->psnet->cfg.host_cfg_off; 346 347 /* Ignore magic number for now */ 348 off += 4; 349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); 350 off += 4; 351 snet_write32(snet, off, snet->sid); 352 off += 4; 353 snet_write32(snet, off, snet->cfg->vq_num); 354 off += 4; 355 snet_write32(snet, off, snet->cfg_irq_idx); 356 off += 4; 357 snet_write64(snet, off, snet->negotiated_features); 358 off += 8; 359 /* Ignore reserved */ 360 off += 8; 361 /* Write VQs */ 362 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); 364 snet_write32(snet, off, tmp); 365 off += 4; 366 snet_write32(snet, off, snet->vqs[i]->irq_idx); 367 off += 4; 368 snet_write64(snet, off, snet->vqs[i]->desc_area); 369 off += 8; 370 snet_write64(snet, off, snet->vqs[i]->device_area); 371 off += 8; 372 snet_write64(snet, off, snet->vqs[i]->driver_area); 373 off += 8; 374 /* Write VQ state if config version is 2+ */ 375 if (SNET_CFG_VER(snet, 2)) 376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state); 377 off += 4; 378 379 /* Ignore reserved */ 380 off += 4; 381 } 382 383 /* Write magic number - data is ready */ 384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); 385 386 /* The DPU will ACK the config by clearing the signature */ 387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, 388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); 389 if (ret) { 390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); 391 return false; 392 } 393 394 /* set DPU flag */ 395 snet->dpu_ready = true; 396 397 return true; 398 } 399 400 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) 401 { 402 int ret, i, irq; 403 404 /* Request config IRQ */ 405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); 406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, 407 snet->cfg_irq_name, snet); 408 if (ret) { 409 SNET_ERR(pdev, "Failed to request IRQ\n"); 410 return ret; 411 } 412 snet->cfg_irq = irq; 413 414 /* Request IRQ for every VQ */ 415 for (i = 0; i < snet->cfg->vq_num; i++) { 416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); 417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, 418 snet->vqs[i]->irq_name, snet->vqs[i]); 419 if (ret) { 420 SNET_ERR(pdev, "Failed to request IRQ\n"); 421 goto err_free_irqs; 422 } 423 snet->vqs[i]->irq = irq; 424 } 425 return 0; 426 427 err_free_irqs: 428 snet_free_irqs(snet); 429 return ret; 430 } 431 432 static void snet_set_status(struct vdpa_device *vdev, u8 status) 433 { 434 struct snet *snet = vdpa_to_snet(vdev); 435 struct psnet *psnet = snet->psnet; 436 struct pci_dev *pdev = snet->pdev; 437 int ret; 438 bool pf_irqs; 439 440 if (status == snet->status) 441 return; 442 443 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 444 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 445 /* Request IRQs */ 446 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 447 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); 448 if (ret) 449 goto set_err; 450 451 /* Write config to the DPU */ 452 if (snet_write_conf(snet)) { 453 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); 454 } else { 455 snet_free_irqs(snet); 456 goto set_err; 457 } 458 } 459 460 /* Save the new status */ 461 snet->status = status; 462 return; 463 464 set_err: 465 snet->status |= VIRTIO_CONFIG_S_FAILED; 466 } 467 468 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, 469 void *buf, unsigned int len) 470 { 471 struct snet *snet = vdpa_to_snet(vdev); 472 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 473 u8 *buf_ptr = buf; 474 u32 i; 475 476 /* check for offset error */ 477 if (offset + len > snet->cfg->cfg_size) 478 return; 479 480 /* Write into buffer */ 481 for (i = 0; i < len; i++) 482 *buf_ptr++ = ioread8(cfg_ptr + i); 483 } 484 485 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, 486 const void *buf, unsigned int len) 487 { 488 struct snet *snet = vdpa_to_snet(vdev); 489 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 490 const u8 *buf_ptr = buf; 491 u32 i; 492 493 /* check for offset error */ 494 if (offset + len > snet->cfg->cfg_size) 495 return; 496 497 /* Write into PCI BAR */ 498 for (i = 0; i < len; i++) 499 iowrite8(*buf_ptr++, cfg_ptr + i); 500 } 501 502 static int snet_suspend(struct vdpa_device *vdev) 503 { 504 struct snet *snet = vdpa_to_snet(vdev); 505 int ret; 506 507 ret = snet_suspend_dev(snet); 508 if (ret) 509 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret); 510 else 511 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid); 512 513 return ret; 514 } 515 516 static int snet_resume(struct vdpa_device *vdev) 517 { 518 struct snet *snet = vdpa_to_snet(vdev); 519 int ret; 520 521 ret = snet_resume_dev(snet); 522 if (ret) 523 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); 524 else 525 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); 526 527 return ret; 528 } 529 530 static const struct vdpa_config_ops snet_config_ops = { 531 .set_vq_address = snet_set_vq_address, 532 .set_vq_num = snet_set_vq_num, 533 .kick_vq = snet_kick_vq, 534 .kick_vq_with_data = snet_kick_vq_with_data, 535 .set_vq_cb = snet_set_vq_cb, 536 .set_vq_ready = snet_set_vq_ready, 537 .get_vq_ready = snet_get_vq_ready, 538 .set_vq_state = snet_set_vq_state, 539 .get_vq_state = snet_get_vq_state, 540 .get_vq_irq = snet_get_vq_irq, 541 .get_vq_align = snet_get_vq_align, 542 .reset = snet_reset, 543 .get_config_size = snet_get_config_size, 544 .get_device_features = snet_get_features, 545 .set_driver_features = snet_set_drv_features, 546 .get_driver_features = snet_get_drv_features, 547 .get_vq_num_min = snet_get_vq_num_max, 548 .get_vq_num_max = snet_get_vq_num_max, 549 .set_config_cb = snet_set_config_cb, 550 .get_device_id = snet_get_device_id, 551 .get_vendor_id = snet_get_vendor_id, 552 .get_status = snet_get_status, 553 .set_status = snet_set_status, 554 .get_config = snet_get_config, 555 .set_config = snet_set_config, 556 .suspend = snet_suspend, 557 .resume = snet_resume, 558 }; 559 560 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) 561 { 562 char *name; 563 unsigned short i; 564 bool bars_found = false; 565 566 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev)); 567 if (!name) 568 return -ENOMEM; 569 570 /* We don't know which BAR will be used to communicate.. 571 * We will map every bar with len > 0. 572 * 573 * Later, we will discover the BAR and unmap all other BARs. 574 */ 575 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 576 void __iomem *io; 577 578 if (pci_resource_len(pdev, i) == 0) 579 continue; 580 581 io = pcim_iomap_region(pdev, i, name); 582 if (IS_ERR(io)) { 583 SNET_ERR(pdev, "Failed to request and map PCI BARs\n"); 584 return PTR_ERR(io); 585 } 586 587 psnet->bars[i] = io; 588 bars_found = true; 589 } 590 591 /* No BAR can be used.. */ 592 if (!bars_found) { 593 SNET_ERR(pdev, "Failed to find a PCI BAR\n"); 594 return -ENODEV; 595 } 596 597 return 0; 598 } 599 600 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) 601 { 602 char *name; 603 void __iomem *io; 604 605 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev)); 606 if (!name) 607 return -ENOMEM; 608 609 /* Request and map BAR */ 610 io = pcim_iomap_region(pdev, snet->psnet->cfg.vf_bar, name); 611 if (IS_ERR(io)) { 612 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n"); 613 return PTR_ERR(io); 614 } 615 616 snet->bar = io; 617 618 return 0; 619 } 620 621 static void snet_free_cfg(struct snet_cfg *cfg) 622 { 623 u32 i; 624 625 if (!cfg->devs) 626 return; 627 628 /* Free devices */ 629 for (i = 0; i < cfg->devices_num; i++) { 630 if (!cfg->devs[i]) 631 break; 632 633 kfree(cfg->devs[i]); 634 } 635 /* Free pointers to devices */ 636 kfree(cfg->devs); 637 } 638 639 /* Detect which BAR is used for communication with the device. */ 640 static int psnet_detect_bar(struct psnet *psnet, u32 off) 641 { 642 unsigned long exit_time; 643 int i; 644 645 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); 646 647 /* SNET DPU will write SNET's signature when the config is ready. */ 648 while (time_before(jiffies, exit_time)) { 649 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 650 /* Is this BAR mapped? */ 651 if (!psnet->bars[i]) 652 continue; 653 654 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) 655 return i; 656 } 657 usleep_range(1000, 10000); 658 } 659 660 return -ENODEV; 661 } 662 663 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) 664 { 665 unsigned short i; 666 667 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 668 if (psnet->bars[i] && i != psnet->barno) 669 pcim_iounmap_region(pdev, i); 670 } 671 } 672 673 /* Read SNET config from PCI BAR */ 674 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) 675 { 676 struct snet_cfg *cfg = &psnet->cfg; 677 u32 i, off; 678 int barno; 679 680 /* Move to where the config starts */ 681 off = SNET_CONFIG_OFF; 682 683 /* Find BAR used for communication */ 684 barno = psnet_detect_bar(psnet, off); 685 if (barno < 0) { 686 SNET_ERR(pdev, "SNET config is not ready.\n"); 687 return barno; 688 } 689 690 /* Save used BAR number and unmap all other BARs */ 691 psnet->barno = barno; 692 SNET_DBG(pdev, "Using BAR number %d\n", barno); 693 694 psnet_unmap_unused_bars(pdev, psnet); 695 696 /* load config from BAR */ 697 cfg->key = psnet_read32(psnet, off); 698 off += 4; 699 cfg->cfg_size = psnet_read32(psnet, off); 700 off += 4; 701 cfg->cfg_ver = psnet_read32(psnet, off); 702 off += 4; 703 /* The negotiated config version is the lower one between this driver's config 704 * and the DPU's. 705 */ 706 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); 707 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); 708 709 cfg->vf_num = psnet_read32(psnet, off); 710 off += 4; 711 cfg->vf_bar = psnet_read32(psnet, off); 712 off += 4; 713 cfg->host_cfg_off = psnet_read32(psnet, off); 714 off += 4; 715 cfg->max_size_host_cfg = psnet_read32(psnet, off); 716 off += 4; 717 cfg->virtio_cfg_off = psnet_read32(psnet, off); 718 off += 4; 719 cfg->kick_off = psnet_read32(psnet, off); 720 off += 4; 721 cfg->hwmon_off = psnet_read32(psnet, off); 722 off += 4; 723 cfg->ctrl_off = psnet_read32(psnet, off); 724 off += 4; 725 cfg->flags = psnet_read32(psnet, off); 726 off += 4; 727 /* Ignore Reserved */ 728 off += sizeof(cfg->rsvd); 729 730 cfg->devices_num = psnet_read32(psnet, off); 731 off += 4; 732 /* Allocate memory to hold pointer to the devices */ 733 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); 734 if (!cfg->devs) 735 return -ENOMEM; 736 737 /* Load device configuration from BAR */ 738 for (i = 0; i < cfg->devices_num; i++) { 739 cfg->devs[i] = kzalloc_obj(*cfg->devs[i]); 740 if (!cfg->devs[i]) { 741 snet_free_cfg(cfg); 742 return -ENOMEM; 743 } 744 /* Read device config */ 745 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); 746 off += 4; 747 cfg->devs[i]->vq_num = psnet_read32(psnet, off); 748 off += 4; 749 cfg->devs[i]->vq_size = psnet_read32(psnet, off); 750 off += 4; 751 cfg->devs[i]->vfid = psnet_read32(psnet, off); 752 off += 4; 753 cfg->devs[i]->features = psnet_read64(psnet, off); 754 off += 8; 755 /* Ignore Reserved */ 756 off += sizeof(cfg->devs[i]->rsvd); 757 758 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); 759 off += 4; 760 761 /* Is the config witten to the DPU going to be too big? */ 762 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > 763 cfg->max_size_host_cfg) { 764 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n"); 765 snet_free_cfg(cfg); 766 return -EINVAL; 767 } 768 } 769 return 0; 770 } 771 772 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) 773 { 774 int ret = 0; 775 u32 i, irq_num = 0; 776 777 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ 778 for (i = 0; i < psnet->cfg.devices_num; i++) 779 irq_num += psnet->cfg.devs[i]->vq_num + 1; 780 781 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 782 if (ret != irq_num) { 783 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 784 return ret; 785 } 786 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); 787 788 return 0; 789 } 790 791 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) 792 { 793 int ret = 0; 794 u32 irq_num; 795 796 /* We want 1 IRQ for every VQ + 1 for config change events */ 797 irq_num = snet_cfg->vq_num + 1; 798 799 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 800 if (ret <= 0) { 801 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 802 return ret; 803 } 804 805 return 0; 806 } 807 808 static void snet_free_vqs(struct snet *snet) 809 { 810 u32 i; 811 812 if (!snet->vqs) 813 return; 814 815 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 816 if (!snet->vqs[i]) 817 break; 818 819 kfree(snet->vqs[i]); 820 } 821 kfree(snet->vqs); 822 } 823 824 static int snet_build_vqs(struct snet *snet) 825 { 826 u32 i; 827 /* Allocate the VQ pointers array */ 828 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); 829 if (!snet->vqs) 830 return -ENOMEM; 831 832 /* Allocate the VQs */ 833 for (i = 0; i < snet->cfg->vq_num; i++) { 834 snet->vqs[i] = kzalloc_obj(*snet->vqs[i]); 835 if (!snet->vqs[i]) { 836 snet_free_vqs(snet); 837 return -ENOMEM; 838 } 839 /* Reset IRQ num */ 840 snet->vqs[i]->irq = -1; 841 /* VQ serial ID */ 842 snet->vqs[i]->sid = i; 843 /* Kick address - every VQ gets 4B */ 844 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + 845 snet->vqs[i]->sid * 4; 846 /* Clear kick address for this VQ */ 847 iowrite32(0, snet->vqs[i]->kick_ptr); 848 } 849 return 0; 850 } 851 852 static int psnet_get_next_irq_num(struct psnet *psnet) 853 { 854 int irq; 855 856 spin_lock(&psnet->lock); 857 irq = psnet->next_irq++; 858 spin_unlock(&psnet->lock); 859 860 return irq; 861 } 862 863 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) 864 { 865 struct psnet *psnet = snet->psnet; 866 int i; 867 868 /* one IRQ for every VQ, and one for config changes */ 869 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); 870 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", 871 pci_name(pdev), snet->cfg_irq_idx); 872 873 for (i = 0; i < snet->cfg->vq_num; i++) { 874 /* Get next free IRQ ID */ 875 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); 876 /* Write IRQ name */ 877 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", 878 pci_name(pdev), snet->vqs[i]->irq_idx); 879 } 880 } 881 882 /* Find a device config based on virtual function id */ 883 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) 884 { 885 u32 i; 886 887 for (i = 0; i < cfg->devices_num; i++) { 888 if (cfg->devs[i]->vfid == vfid) 889 return cfg->devs[i]; 890 } 891 /* Oppss.. no config found.. */ 892 return NULL; 893 } 894 895 /* Probe function for a physical PCI function */ 896 static int snet_vdpa_probe_pf(struct pci_dev *pdev) 897 { 898 struct psnet *psnet; 899 int ret = 0; 900 bool pf_irqs = false; 901 902 ret = pcim_enable_device(pdev); 903 if (ret) { 904 SNET_ERR(pdev, "Failed to enable PCI device\n"); 905 return ret; 906 } 907 908 /* Allocate a PCI physical function device */ 909 psnet = kzalloc_obj(*psnet); 910 if (!psnet) 911 return -ENOMEM; 912 913 /* Init PSNET spinlock */ 914 spin_lock_init(&psnet->lock); 915 916 pci_set_master(pdev); 917 pci_set_drvdata(pdev, psnet); 918 919 /* Open SNET MAIN BAR */ 920 ret = psnet_open_pf_bar(pdev, psnet); 921 if (ret) 922 goto free_psnet; 923 924 /* Try to read SNET's config from PCI BAR */ 925 ret = psnet_read_cfg(pdev, psnet); 926 if (ret) 927 goto free_psnet; 928 929 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use 930 * PF MSI-X vectors 931 */ 932 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 933 934 if (pf_irqs) { 935 ret = psnet_alloc_irq_vector(pdev, psnet); 936 if (ret) 937 goto free_cfg; 938 } 939 940 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); 941 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); 942 if (ret) { 943 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); 944 goto free_irq; 945 } 946 947 /* Create HW monitor device */ 948 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { 949 #if IS_ENABLED(CONFIG_HWMON) 950 psnet_create_hwmon(pdev); 951 #else 952 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n"); 953 #endif 954 } 955 956 return 0; 957 958 free_irq: 959 if (pf_irqs) 960 pci_free_irq_vectors(pdev); 961 free_cfg: 962 snet_free_cfg(&psnet->cfg); 963 free_psnet: 964 kfree(psnet); 965 return ret; 966 } 967 968 /* Probe function for a virtual PCI function */ 969 static int snet_vdpa_probe_vf(struct pci_dev *pdev) 970 { 971 struct pci_dev *pdev_pf = pdev->physfn; 972 struct psnet *psnet = pci_get_drvdata(pdev_pf); 973 struct snet_dev_cfg *dev_cfg; 974 struct snet *snet; 975 u32 vfid; 976 int ret; 977 bool pf_irqs = false; 978 979 /* Get virtual function id. 980 * (the DPU counts the VFs from 1) 981 */ 982 ret = pci_iov_vf_id(pdev); 983 if (ret < 0) { 984 SNET_ERR(pdev, "Failed to find a VF id\n"); 985 return ret; 986 } 987 vfid = ret + 1; 988 989 /* Find the snet_dev_cfg based on vfid */ 990 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); 991 if (!dev_cfg) { 992 SNET_WARN(pdev, "Failed to find a VF config..\n"); 993 return -ENODEV; 994 } 995 996 /* Which PCI device should allocate the IRQs? 997 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs 998 */ 999 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 1000 1001 ret = pcim_enable_device(pdev); 1002 if (ret) { 1003 SNET_ERR(pdev, "Failed to enable PCI VF device\n"); 1004 return ret; 1005 } 1006 1007 /* Request for MSI-X IRQs */ 1008 if (!pf_irqs) { 1009 ret = snet_alloc_irq_vector(pdev, dev_cfg); 1010 if (ret) 1011 return ret; 1012 } 1013 1014 /* Allocate vdpa device */ 1015 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1016 NULL, 1, 1, NULL, false); 1017 if (!snet) { 1018 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 1019 ret = -ENOMEM; 1020 goto free_irqs; 1021 } 1022 1023 /* Init control mutex and spinlock */ 1024 mutex_init(&snet->ctrl_lock); 1025 spin_lock_init(&snet->ctrl_spinlock); 1026 1027 /* Save pci device pointer */ 1028 snet->pdev = pdev; 1029 snet->psnet = psnet; 1030 snet->cfg = dev_cfg; 1031 snet->dpu_ready = false; 1032 snet->sid = vfid; 1033 /* Reset IRQ value */ 1034 snet->cfg_irq = -1; 1035 1036 ret = snet_open_vf_bar(pdev, snet); 1037 if (ret) 1038 goto put_device; 1039 1040 /* Create a VirtIO config pointer */ 1041 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; 1042 1043 /* Clear control registers */ 1044 snet_ctrl_clear(snet); 1045 1046 pci_set_master(pdev); 1047 pci_set_drvdata(pdev, snet); 1048 1049 ret = snet_build_vqs(snet); 1050 if (ret) 1051 goto put_device; 1052 1053 /* Reserve IRQ indexes, 1054 * The IRQs may be requested and freed multiple times, 1055 * but the indexes won't change. 1056 */ 1057 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1058 1059 /* set map metadata */ 1060 snet->vdpa.vmap.dma_dev = &pdev->dev; 1061 1062 /* Register VDPA device */ 1063 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); 1064 if (ret) { 1065 SNET_ERR(pdev, "Failed to register vdpa device\n"); 1066 goto free_vqs; 1067 } 1068 1069 return 0; 1070 1071 free_vqs: 1072 snet_free_vqs(snet); 1073 put_device: 1074 put_device(&snet->vdpa.dev); 1075 free_irqs: 1076 if (!pf_irqs) 1077 pci_free_irq_vectors(pdev); 1078 return ret; 1079 } 1080 1081 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1082 { 1083 if (pdev->is_virtfn) 1084 return snet_vdpa_probe_vf(pdev); 1085 else 1086 return snet_vdpa_probe_pf(pdev); 1087 } 1088 1089 static void snet_vdpa_remove_pf(struct pci_dev *pdev) 1090 { 1091 struct psnet *psnet = pci_get_drvdata(pdev); 1092 1093 pci_disable_sriov(pdev); 1094 /* If IRQs are allocated from the PF, we should free the IRQs */ 1095 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1096 pci_free_irq_vectors(pdev); 1097 1098 snet_free_cfg(&psnet->cfg); 1099 kfree(psnet); 1100 } 1101 1102 static void snet_vdpa_remove_vf(struct pci_dev *pdev) 1103 { 1104 struct snet *snet = pci_get_drvdata(pdev); 1105 struct psnet *psnet = snet->psnet; 1106 1107 vdpa_unregister_device(&snet->vdpa); 1108 snet_free_vqs(snet); 1109 /* If IRQs are allocated from the VF, we should free the IRQs */ 1110 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1111 pci_free_irq_vectors(pdev); 1112 } 1113 1114 static void snet_vdpa_remove(struct pci_dev *pdev) 1115 { 1116 if (pdev->is_virtfn) 1117 snet_vdpa_remove_vf(pdev); 1118 else 1119 snet_vdpa_remove_pf(pdev); 1120 } 1121 1122 static struct pci_device_id snet_driver_pci_ids[] = { 1123 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, 1124 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, 1125 { 0 }, 1126 }; 1127 1128 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); 1129 1130 static struct pci_driver snet_vdpa_driver = { 1131 .name = "snet-vdpa-driver", 1132 .id_table = snet_driver_pci_ids, 1133 .probe = snet_vdpa_probe, 1134 .remove = snet_vdpa_remove, 1135 }; 1136 1137 module_pci_driver(snet_vdpa_driver); 1138 1139 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>"); 1140 MODULE_DESCRIPTION("SolidRun vDPA driver"); 1141 MODULE_LICENSE("GPL v2"); 1142