1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 #include <linux/iopoll.h> 11 12 #include "snet_vdpa.h" 13 14 /* SNET DPU device ID */ 15 #define SNET_DEVICE_ID 0x1000 16 /* SNET signature */ 17 #define SNET_SIGNATURE 0xD0D06363 18 /* Max. config version that we can work with */ 19 #define SNET_CFG_VERSION 0x2 20 /* Queue align */ 21 #define SNET_QUEUE_ALIGNMENT PAGE_SIZE 22 /* Kick value to notify that new data is available */ 23 #define SNET_KICK_VAL 0x1 24 #define SNET_CONFIG_OFF 0x0 25 /* How long we are willing to wait for a SNET device */ 26 #define SNET_DETECT_TIMEOUT 5000000 27 /* How long should we wait for the DPU to read our config */ 28 #define SNET_READ_CFG_TIMEOUT 3000000 29 /* Size of configs written to the DPU */ 30 #define SNET_GENERAL_CFG_LEN 36 31 #define SNET_GENERAL_CFG_VQ_LEN 40 32 33 static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) 34 { 35 return container_of(vdpa, struct snet, vdpa); 36 } 37 38 static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) 39 { 40 struct snet *snet = data; 41 /* Call callback if any */ 42 if (likely(snet->cb.callback)) 43 return snet->cb.callback(snet->cb.private); 44 45 return IRQ_HANDLED; 46 } 47 48 static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) 49 { 50 struct snet_vq *vq = data; 51 /* Call callback if any */ 52 if (likely(vq->cb.callback)) 53 return vq->cb.callback(vq->cb.private); 54 55 return IRQ_HANDLED; 56 } 57 58 static void snet_free_irqs(struct snet *snet) 59 { 60 struct psnet *psnet = snet->psnet; 61 struct pci_dev *pdev; 62 u32 i; 63 64 /* Which Device allcoated the IRQs? */ 65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 66 pdev = snet->pdev->physfn; 67 else 68 pdev = snet->pdev; 69 70 /* Free config's IRQ */ 71 if (snet->cfg_irq != -1) { 72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); 73 snet->cfg_irq = -1; 74 } 75 /* Free VQ IRQs */ 76 for (i = 0; i < snet->cfg->vq_num; i++) { 77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { 78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); 79 snet->vqs[i]->irq = -1; 80 } 81 } 82 83 /* IRQ vectors are freed when the pci remove callback is called */ 84 } 85 86 static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 87 u64 driver_area, u64 device_area) 88 { 89 struct snet *snet = vdpa_to_snet(vdev); 90 /* save received parameters in vqueue sturct */ 91 snet->vqs[idx]->desc_area = desc_area; 92 snet->vqs[idx]->driver_area = driver_area; 93 snet->vqs[idx]->device_area = device_area; 94 95 return 0; 96 } 97 98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 99 { 100 struct snet *snet = vdpa_to_snet(vdev); 101 /* save num in vqueue */ 102 snet->vqs[idx]->num = num; 103 } 104 105 static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) 106 { 107 struct snet *snet = vdpa_to_snet(vdev); 108 /* not ready - ignore */ 109 if (unlikely(!snet->vqs[idx]->ready)) 110 return; 111 112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 113 } 114 115 static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data) 116 { 117 struct snet *snet = vdpa_to_snet(vdev); 118 u16 idx = data & 0xFFFF; 119 120 /* not ready - ignore */ 121 if (unlikely(!snet->vqs[idx]->ready)) 122 return; 123 124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); 125 } 126 127 static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 128 { 129 struct snet *snet = vdpa_to_snet(vdev); 130 131 snet->vqs[idx]->cb.callback = cb->callback; 132 snet->vqs[idx]->cb.private = cb->private; 133 } 134 135 static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 136 { 137 struct snet *snet = vdpa_to_snet(vdev); 138 139 snet->vqs[idx]->ready = ready; 140 } 141 142 static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) 143 { 144 struct snet *snet = vdpa_to_snet(vdev); 145 146 return snet->vqs[idx]->ready; 147 } 148 149 static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state) 150 { 151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { 152 const struct vdpa_vq_state_packed *p = &state->packed; 153 154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && 155 p->last_avail_idx == 0 && p->last_used_idx == 0) 156 return true; 157 } else { 158 const struct vdpa_vq_state_split *s = &state->split; 159 160 if (s->avail_index == 0) 161 return true; 162 } 163 164 return false; 165 } 166 167 static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) 168 { 169 struct snet *snet = vdpa_to_snet(vdev); 170 171 /* We can set any state for config version 2+ */ 172 if (SNET_CFG_VER(snet, 2)) { 173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); 174 return 0; 175 } 176 177 /* Older config - we can't set the VQ state. 178 * Return 0 only if this is the initial state we use in the DPU. 179 */ 180 if (snet_vq_state_is_initial(snet, state)) 181 return 0; 182 183 return -EOPNOTSUPP; 184 } 185 186 static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 187 { 188 struct snet *snet = vdpa_to_snet(vdev); 189 190 return snet_read_vq_state(snet, idx, state); 191 } 192 193 static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) 194 { 195 struct snet *snet = vdpa_to_snet(vdev); 196 197 return snet->vqs[idx]->irq; 198 } 199 200 static u32 snet_get_vq_align(struct vdpa_device *vdev) 201 { 202 return (u32)SNET_QUEUE_ALIGNMENT; 203 } 204 205 static int snet_reset_dev(struct snet *snet) 206 { 207 struct pci_dev *pdev = snet->pdev; 208 int ret = 0; 209 u32 i; 210 211 /* If status is 0, nothing to do */ 212 if (!snet->status) 213 return 0; 214 215 /* If DPU started, destroy it */ 216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) 217 ret = snet_destroy_dev(snet); 218 219 /* Clear VQs */ 220 for (i = 0; i < snet->cfg->vq_num; i++) { 221 if (!snet->vqs[i]) 222 continue; 223 snet->vqs[i]->cb.callback = NULL; 224 snet->vqs[i]->cb.private = NULL; 225 snet->vqs[i]->desc_area = 0; 226 snet->vqs[i]->device_area = 0; 227 snet->vqs[i]->driver_area = 0; 228 snet->vqs[i]->ready = false; 229 } 230 231 /* Clear config callback */ 232 snet->cb.callback = NULL; 233 snet->cb.private = NULL; 234 /* Free IRQs */ 235 snet_free_irqs(snet); 236 /* Reset status */ 237 snet->status = 0; 238 snet->dpu_ready = false; 239 240 if (ret) 241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret); 242 else 243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); 244 245 return 0; 246 } 247 248 static int snet_reset(struct vdpa_device *vdev) 249 { 250 struct snet *snet = vdpa_to_snet(vdev); 251 252 return snet_reset_dev(snet); 253 } 254 255 static size_t snet_get_config_size(struct vdpa_device *vdev) 256 { 257 struct snet *snet = vdpa_to_snet(vdev); 258 259 return (size_t)snet->cfg->cfg_size; 260 } 261 262 static u64 snet_get_features(struct vdpa_device *vdev) 263 { 264 struct snet *snet = vdpa_to_snet(vdev); 265 266 return snet->cfg->features; 267 } 268 269 static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) 270 { 271 struct snet *snet = vdpa_to_snet(vdev); 272 273 snet->negotiated_features = snet->cfg->features & features; 274 return 0; 275 } 276 277 static u64 snet_get_drv_features(struct vdpa_device *vdev) 278 { 279 struct snet *snet = vdpa_to_snet(vdev); 280 281 return snet->negotiated_features; 282 } 283 284 static u16 snet_get_vq_num_max(struct vdpa_device *vdev) 285 { 286 struct snet *snet = vdpa_to_snet(vdev); 287 288 return (u16)snet->cfg->vq_size; 289 } 290 291 static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 292 { 293 struct snet *snet = vdpa_to_snet(vdev); 294 295 snet->cb.callback = cb->callback; 296 snet->cb.private = cb->private; 297 } 298 299 static u32 snet_get_device_id(struct vdpa_device *vdev) 300 { 301 struct snet *snet = vdpa_to_snet(vdev); 302 303 return snet->cfg->virtio_id; 304 } 305 306 static u32 snet_get_vendor_id(struct vdpa_device *vdev) 307 { 308 return (u32)PCI_VENDOR_ID_SOLIDRUN; 309 } 310 311 static u8 snet_get_status(struct vdpa_device *vdev) 312 { 313 struct snet *snet = vdpa_to_snet(vdev); 314 315 return snet->status; 316 } 317 318 static int snet_write_conf(struct snet *snet) 319 { 320 u32 off, i, tmp; 321 int ret; 322 323 /* No need to write the config twice */ 324 if (snet->dpu_ready) 325 return true; 326 327 /* Snet data : 328 * 329 * General data: SNET_GENERAL_CFG_LEN bytes long 330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | 332 * 333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long 334 * 0 0x4 0x8 335 * | VQ SID AND QUEUE SIZE | IRQ Index | 336 * | DESC AREA | 337 * | DEVICE AREA | 338 * | DRIVER AREA | 339 * | VQ STATE (CFG 2+) | RSVD | 340 * 341 * Magic number should be written last, this is the DPU indication that the data is ready 342 */ 343 344 /* Init offset */ 345 off = snet->psnet->cfg.host_cfg_off; 346 347 /* Ignore magic number for now */ 348 off += 4; 349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); 350 off += 4; 351 snet_write32(snet, off, snet->sid); 352 off += 4; 353 snet_write32(snet, off, snet->cfg->vq_num); 354 off += 4; 355 snet_write32(snet, off, snet->cfg_irq_idx); 356 off += 4; 357 snet_write64(snet, off, snet->negotiated_features); 358 off += 8; 359 /* Ignore reserved */ 360 off += 8; 361 /* Write VQs */ 362 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); 364 snet_write32(snet, off, tmp); 365 off += 4; 366 snet_write32(snet, off, snet->vqs[i]->irq_idx); 367 off += 4; 368 snet_write64(snet, off, snet->vqs[i]->desc_area); 369 off += 8; 370 snet_write64(snet, off, snet->vqs[i]->device_area); 371 off += 8; 372 snet_write64(snet, off, snet->vqs[i]->driver_area); 373 off += 8; 374 /* Write VQ state if config version is 2+ */ 375 if (SNET_CFG_VER(snet, 2)) 376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state); 377 off += 4; 378 379 /* Ignore reserved */ 380 off += 4; 381 } 382 383 /* Write magic number - data is ready */ 384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); 385 386 /* The DPU will ACK the config by clearing the signature */ 387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, 388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); 389 if (ret) { 390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); 391 return false; 392 } 393 394 /* set DPU flag */ 395 snet->dpu_ready = true; 396 397 return true; 398 } 399 400 static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) 401 { 402 int ret, i, irq; 403 404 /* Request config IRQ */ 405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); 406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, 407 snet->cfg_irq_name, snet); 408 if (ret) { 409 SNET_ERR(pdev, "Failed to request IRQ\n"); 410 return ret; 411 } 412 snet->cfg_irq = irq; 413 414 /* Request IRQ for every VQ */ 415 for (i = 0; i < snet->cfg->vq_num; i++) { 416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); 417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, 418 snet->vqs[i]->irq_name, snet->vqs[i]); 419 if (ret) { 420 SNET_ERR(pdev, "Failed to request IRQ\n"); 421 return ret; 422 } 423 snet->vqs[i]->irq = irq; 424 } 425 return 0; 426 } 427 428 static void snet_set_status(struct vdpa_device *vdev, u8 status) 429 { 430 struct snet *snet = vdpa_to_snet(vdev); 431 struct psnet *psnet = snet->psnet; 432 struct pci_dev *pdev = snet->pdev; 433 int ret; 434 bool pf_irqs; 435 436 if (status == snet->status) 437 return; 438 439 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 441 /* Request IRQs */ 442 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); 444 if (ret) 445 goto set_err; 446 447 /* Write config to the DPU */ 448 if (snet_write_conf(snet)) { 449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); 450 } else { 451 snet_free_irqs(snet); 452 goto set_err; 453 } 454 } 455 456 /* Save the new status */ 457 snet->status = status; 458 return; 459 460 set_err: 461 snet->status |= VIRTIO_CONFIG_S_FAILED; 462 } 463 464 static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, 465 void *buf, unsigned int len) 466 { 467 struct snet *snet = vdpa_to_snet(vdev); 468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 469 u8 *buf_ptr = buf; 470 u32 i; 471 472 /* check for offset error */ 473 if (offset + len > snet->cfg->cfg_size) 474 return; 475 476 /* Write into buffer */ 477 for (i = 0; i < len; i++) 478 *buf_ptr++ = ioread8(cfg_ptr + i); 479 } 480 481 static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, 482 const void *buf, unsigned int len) 483 { 484 struct snet *snet = vdpa_to_snet(vdev); 485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; 486 const u8 *buf_ptr = buf; 487 u32 i; 488 489 /* check for offset error */ 490 if (offset + len > snet->cfg->cfg_size) 491 return; 492 493 /* Write into PCI BAR */ 494 for (i = 0; i < len; i++) 495 iowrite8(*buf_ptr++, cfg_ptr + i); 496 } 497 498 static int snet_suspend(struct vdpa_device *vdev) 499 { 500 struct snet *snet = vdpa_to_snet(vdev); 501 int ret; 502 503 ret = snet_suspend_dev(snet); 504 if (ret) 505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret); 506 else 507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid); 508 509 return ret; 510 } 511 512 static const struct vdpa_config_ops snet_config_ops = { 513 .set_vq_address = snet_set_vq_address, 514 .set_vq_num = snet_set_vq_num, 515 .kick_vq = snet_kick_vq, 516 .kick_vq_with_data = snet_kick_vq_with_data, 517 .set_vq_cb = snet_set_vq_cb, 518 .set_vq_ready = snet_set_vq_ready, 519 .get_vq_ready = snet_get_vq_ready, 520 .set_vq_state = snet_set_vq_state, 521 .get_vq_state = snet_get_vq_state, 522 .get_vq_irq = snet_get_vq_irq, 523 .get_vq_align = snet_get_vq_align, 524 .reset = snet_reset, 525 .get_config_size = snet_get_config_size, 526 .get_device_features = snet_get_features, 527 .set_driver_features = snet_set_drv_features, 528 .get_driver_features = snet_get_drv_features, 529 .get_vq_num_min = snet_get_vq_num_max, 530 .get_vq_num_max = snet_get_vq_num_max, 531 .set_config_cb = snet_set_config_cb, 532 .get_device_id = snet_get_device_id, 533 .get_vendor_id = snet_get_vendor_id, 534 .get_status = snet_get_status, 535 .set_status = snet_set_status, 536 .get_config = snet_get_config, 537 .set_config = snet_set_config, 538 .suspend = snet_suspend, 539 }; 540 541 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) 542 { 543 char name[50]; 544 int ret, i, mask = 0; 545 /* We don't know which BAR will be used to communicate.. 546 * We will map every bar with len > 0. 547 * 548 * Later, we will discover the BAR and unmap all other BARs. 549 */ 550 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 551 if (pci_resource_len(pdev, i)) 552 mask |= (1 << i); 553 } 554 555 /* No BAR can be used.. */ 556 if (!mask) { 557 SNET_ERR(pdev, "Failed to find a PCI BAR\n"); 558 return -ENODEV; 559 } 560 561 snprintf(name, sizeof(name), "psnet[%s]-bars", pci_name(pdev)); 562 ret = pcim_iomap_regions(pdev, mask, name); 563 if (ret) { 564 SNET_ERR(pdev, "Failed to request and map PCI BARs\n"); 565 return ret; 566 } 567 568 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 569 if (mask & (1 << i)) 570 psnet->bars[i] = pcim_iomap_table(pdev)[i]; 571 } 572 573 return 0; 574 } 575 576 static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) 577 { 578 char name[50]; 579 int ret; 580 581 snprintf(name, sizeof(name), "snet[%s]-bar", pci_name(pdev)); 582 /* Request and map BAR */ 583 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); 584 if (ret) { 585 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n"); 586 return ret; 587 } 588 589 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; 590 591 return 0; 592 } 593 594 static void snet_free_cfg(struct snet_cfg *cfg) 595 { 596 u32 i; 597 598 if (!cfg->devs) 599 return; 600 601 /* Free devices */ 602 for (i = 0; i < cfg->devices_num; i++) { 603 if (!cfg->devs[i]) 604 break; 605 606 kfree(cfg->devs[i]); 607 } 608 /* Free pointers to devices */ 609 kfree(cfg->devs); 610 } 611 612 /* Detect which BAR is used for communication with the device. */ 613 static int psnet_detect_bar(struct psnet *psnet, u32 off) 614 { 615 unsigned long exit_time; 616 int i; 617 618 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); 619 620 /* SNET DPU will write SNET's signature when the config is ready. */ 621 while (time_before(jiffies, exit_time)) { 622 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 623 /* Is this BAR mapped? */ 624 if (!psnet->bars[i]) 625 continue; 626 627 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) 628 return i; 629 } 630 usleep_range(1000, 10000); 631 } 632 633 return -ENODEV; 634 } 635 636 static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) 637 { 638 int i, mask = 0; 639 640 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 641 if (psnet->bars[i] && i != psnet->barno) 642 mask |= (1 << i); 643 } 644 645 if (mask) 646 pcim_iounmap_regions(pdev, mask); 647 } 648 649 /* Read SNET config from PCI BAR */ 650 static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) 651 { 652 struct snet_cfg *cfg = &psnet->cfg; 653 u32 i, off; 654 int barno; 655 656 /* Move to where the config starts */ 657 off = SNET_CONFIG_OFF; 658 659 /* Find BAR used for communication */ 660 barno = psnet_detect_bar(psnet, off); 661 if (barno < 0) { 662 SNET_ERR(pdev, "SNET config is not ready.\n"); 663 return barno; 664 } 665 666 /* Save used BAR number and unmap all other BARs */ 667 psnet->barno = barno; 668 SNET_DBG(pdev, "Using BAR number %d\n", barno); 669 670 psnet_unmap_unused_bars(pdev, psnet); 671 672 /* load config from BAR */ 673 cfg->key = psnet_read32(psnet, off); 674 off += 4; 675 cfg->cfg_size = psnet_read32(psnet, off); 676 off += 4; 677 cfg->cfg_ver = psnet_read32(psnet, off); 678 off += 4; 679 /* The negotiated config version is the lower one between this driver's config 680 * and the DPU's. 681 */ 682 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); 683 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); 684 685 cfg->vf_num = psnet_read32(psnet, off); 686 off += 4; 687 cfg->vf_bar = psnet_read32(psnet, off); 688 off += 4; 689 cfg->host_cfg_off = psnet_read32(psnet, off); 690 off += 4; 691 cfg->max_size_host_cfg = psnet_read32(psnet, off); 692 off += 4; 693 cfg->virtio_cfg_off = psnet_read32(psnet, off); 694 off += 4; 695 cfg->kick_off = psnet_read32(psnet, off); 696 off += 4; 697 cfg->hwmon_off = psnet_read32(psnet, off); 698 off += 4; 699 cfg->ctrl_off = psnet_read32(psnet, off); 700 off += 4; 701 cfg->flags = psnet_read32(psnet, off); 702 off += 4; 703 /* Ignore Reserved */ 704 off += sizeof(cfg->rsvd); 705 706 cfg->devices_num = psnet_read32(psnet, off); 707 off += 4; 708 /* Allocate memory to hold pointer to the devices */ 709 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); 710 if (!cfg->devs) 711 return -ENOMEM; 712 713 /* Load device configuration from BAR */ 714 for (i = 0; i < cfg->devices_num; i++) { 715 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL); 716 if (!cfg->devs[i]) { 717 snet_free_cfg(cfg); 718 return -ENOMEM; 719 } 720 /* Read device config */ 721 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); 722 off += 4; 723 cfg->devs[i]->vq_num = psnet_read32(psnet, off); 724 off += 4; 725 cfg->devs[i]->vq_size = psnet_read32(psnet, off); 726 off += 4; 727 cfg->devs[i]->vfid = psnet_read32(psnet, off); 728 off += 4; 729 cfg->devs[i]->features = psnet_read64(psnet, off); 730 off += 8; 731 /* Ignore Reserved */ 732 off += sizeof(cfg->devs[i]->rsvd); 733 734 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); 735 off += 4; 736 737 /* Is the config witten to the DPU going to be too big? */ 738 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > 739 cfg->max_size_host_cfg) { 740 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n"); 741 snet_free_cfg(cfg); 742 return -EINVAL; 743 } 744 } 745 return 0; 746 } 747 748 static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) 749 { 750 int ret = 0; 751 u32 i, irq_num = 0; 752 753 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ 754 for (i = 0; i < psnet->cfg.devices_num; i++) 755 irq_num += psnet->cfg.devs[i]->vq_num + 1; 756 757 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 758 if (ret != irq_num) { 759 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 760 return ret; 761 } 762 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); 763 764 return 0; 765 } 766 767 static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) 768 { 769 int ret = 0; 770 u32 irq_num; 771 772 /* We want 1 IRQ for every VQ + 1 for config change events */ 773 irq_num = snet_cfg->vq_num + 1; 774 775 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX); 776 if (ret <= 0) { 777 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); 778 return ret; 779 } 780 781 return 0; 782 } 783 784 static void snet_free_vqs(struct snet *snet) 785 { 786 u32 i; 787 788 if (!snet->vqs) 789 return; 790 791 for (i = 0 ; i < snet->cfg->vq_num ; i++) { 792 if (!snet->vqs[i]) 793 break; 794 795 kfree(snet->vqs[i]); 796 } 797 kfree(snet->vqs); 798 } 799 800 static int snet_build_vqs(struct snet *snet) 801 { 802 u32 i; 803 /* Allocate the VQ pointers array */ 804 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); 805 if (!snet->vqs) 806 return -ENOMEM; 807 808 /* Allocate the VQs */ 809 for (i = 0; i < snet->cfg->vq_num; i++) { 810 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL); 811 if (!snet->vqs[i]) { 812 snet_free_vqs(snet); 813 return -ENOMEM; 814 } 815 /* Reset IRQ num */ 816 snet->vqs[i]->irq = -1; 817 /* VQ serial ID */ 818 snet->vqs[i]->sid = i; 819 /* Kick address - every VQ gets 4B */ 820 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + 821 snet->vqs[i]->sid * 4; 822 /* Clear kick address for this VQ */ 823 iowrite32(0, snet->vqs[i]->kick_ptr); 824 } 825 return 0; 826 } 827 828 static int psnet_get_next_irq_num(struct psnet *psnet) 829 { 830 int irq; 831 832 spin_lock(&psnet->lock); 833 irq = psnet->next_irq++; 834 spin_unlock(&psnet->lock); 835 836 return irq; 837 } 838 839 static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) 840 { 841 struct psnet *psnet = snet->psnet; 842 int i; 843 844 /* one IRQ for every VQ, and one for config changes */ 845 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); 846 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", 847 pci_name(pdev), snet->cfg_irq_idx); 848 849 for (i = 0; i < snet->cfg->vq_num; i++) { 850 /* Get next free IRQ ID */ 851 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); 852 /* Write IRQ name */ 853 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", 854 pci_name(pdev), snet->vqs[i]->irq_idx); 855 } 856 } 857 858 /* Find a device config based on virtual function id */ 859 static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) 860 { 861 u32 i; 862 863 for (i = 0; i < cfg->devices_num; i++) { 864 if (cfg->devs[i]->vfid == vfid) 865 return cfg->devs[i]; 866 } 867 /* Oppss.. no config found.. */ 868 return NULL; 869 } 870 871 /* Probe function for a physical PCI function */ 872 static int snet_vdpa_probe_pf(struct pci_dev *pdev) 873 { 874 struct psnet *psnet; 875 int ret = 0; 876 bool pf_irqs = false; 877 878 ret = pcim_enable_device(pdev); 879 if (ret) { 880 SNET_ERR(pdev, "Failed to enable PCI device\n"); 881 return ret; 882 } 883 884 /* Allocate a PCI physical function device */ 885 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL); 886 if (!psnet) 887 return -ENOMEM; 888 889 /* Init PSNET spinlock */ 890 spin_lock_init(&psnet->lock); 891 892 pci_set_master(pdev); 893 pci_set_drvdata(pdev, psnet); 894 895 /* Open SNET MAIN BAR */ 896 ret = psnet_open_pf_bar(pdev, psnet); 897 if (ret) 898 goto free_psnet; 899 900 /* Try to read SNET's config from PCI BAR */ 901 ret = psnet_read_cfg(pdev, psnet); 902 if (ret) 903 goto free_psnet; 904 905 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use 906 * PF MSI-X vectors 907 */ 908 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 909 910 if (pf_irqs) { 911 ret = psnet_alloc_irq_vector(pdev, psnet); 912 if (ret) 913 goto free_cfg; 914 } 915 916 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); 917 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); 918 if (ret) { 919 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); 920 goto free_irq; 921 } 922 923 /* Create HW monitor device */ 924 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { 925 #if IS_ENABLED(CONFIG_HWMON) 926 psnet_create_hwmon(pdev); 927 #else 928 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n"); 929 #endif 930 } 931 932 return 0; 933 934 free_irq: 935 if (pf_irqs) 936 pci_free_irq_vectors(pdev); 937 free_cfg: 938 snet_free_cfg(&psnet->cfg); 939 free_psnet: 940 kfree(psnet); 941 return ret; 942 } 943 944 /* Probe function for a virtual PCI function */ 945 static int snet_vdpa_probe_vf(struct pci_dev *pdev) 946 { 947 struct pci_dev *pdev_pf = pdev->physfn; 948 struct psnet *psnet = pci_get_drvdata(pdev_pf); 949 struct snet_dev_cfg *dev_cfg; 950 struct snet *snet; 951 u32 vfid; 952 int ret; 953 bool pf_irqs = false; 954 955 /* Get virtual function id. 956 * (the DPU counts the VFs from 1) 957 */ 958 ret = pci_iov_vf_id(pdev); 959 if (ret < 0) { 960 SNET_ERR(pdev, "Failed to find a VF id\n"); 961 return ret; 962 } 963 vfid = ret + 1; 964 965 /* Find the snet_dev_cfg based on vfid */ 966 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); 967 if (!dev_cfg) { 968 SNET_WARN(pdev, "Failed to find a VF config..\n"); 969 return -ENODEV; 970 } 971 972 /* Which PCI device should allocate the IRQs? 973 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs 974 */ 975 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); 976 977 ret = pcim_enable_device(pdev); 978 if (ret) { 979 SNET_ERR(pdev, "Failed to enable PCI VF device\n"); 980 return ret; 981 } 982 983 /* Request for MSI-X IRQs */ 984 if (!pf_irqs) { 985 ret = snet_alloc_irq_vector(pdev, dev_cfg); 986 if (ret) 987 return ret; 988 } 989 990 /* Allocate vdpa device */ 991 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, 992 false); 993 if (!snet) { 994 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 995 ret = -ENOMEM; 996 goto free_irqs; 997 } 998 999 /* Init control mutex and spinlock */ 1000 mutex_init(&snet->ctrl_lock); 1001 spin_lock_init(&snet->ctrl_spinlock); 1002 1003 /* Save pci device pointer */ 1004 snet->pdev = pdev; 1005 snet->psnet = psnet; 1006 snet->cfg = dev_cfg; 1007 snet->dpu_ready = false; 1008 snet->sid = vfid; 1009 /* Reset IRQ value */ 1010 snet->cfg_irq = -1; 1011 1012 ret = snet_open_vf_bar(pdev, snet); 1013 if (ret) 1014 goto put_device; 1015 1016 /* Create a VirtIO config pointer */ 1017 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; 1018 1019 /* Clear control registers */ 1020 snet_ctrl_clear(snet); 1021 1022 pci_set_master(pdev); 1023 pci_set_drvdata(pdev, snet); 1024 1025 ret = snet_build_vqs(snet); 1026 if (ret) 1027 goto put_device; 1028 1029 /* Reserve IRQ indexes, 1030 * The IRQs may be requested and freed multiple times, 1031 * but the indexes won't change. 1032 */ 1033 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1034 1035 /*set DMA device*/ 1036 snet->vdpa.dma_dev = &pdev->dev; 1037 1038 /* Register VDPA device */ 1039 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); 1040 if (ret) { 1041 SNET_ERR(pdev, "Failed to register vdpa device\n"); 1042 goto free_vqs; 1043 } 1044 1045 return 0; 1046 1047 free_vqs: 1048 snet_free_vqs(snet); 1049 put_device: 1050 put_device(&snet->vdpa.dev); 1051 free_irqs: 1052 if (!pf_irqs) 1053 pci_free_irq_vectors(pdev); 1054 return ret; 1055 } 1056 1057 static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1058 { 1059 if (pdev->is_virtfn) 1060 return snet_vdpa_probe_vf(pdev); 1061 else 1062 return snet_vdpa_probe_pf(pdev); 1063 } 1064 1065 static void snet_vdpa_remove_pf(struct pci_dev *pdev) 1066 { 1067 struct psnet *psnet = pci_get_drvdata(pdev); 1068 1069 pci_disable_sriov(pdev); 1070 /* If IRQs are allocated from the PF, we should free the IRQs */ 1071 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1072 pci_free_irq_vectors(pdev); 1073 1074 snet_free_cfg(&psnet->cfg); 1075 kfree(psnet); 1076 } 1077 1078 static void snet_vdpa_remove_vf(struct pci_dev *pdev) 1079 { 1080 struct snet *snet = pci_get_drvdata(pdev); 1081 struct psnet *psnet = snet->psnet; 1082 1083 vdpa_unregister_device(&snet->vdpa); 1084 snet_free_vqs(snet); 1085 /* If IRQs are allocated from the VF, we should free the IRQs */ 1086 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) 1087 pci_free_irq_vectors(pdev); 1088 } 1089 1090 static void snet_vdpa_remove(struct pci_dev *pdev) 1091 { 1092 if (pdev->is_virtfn) 1093 snet_vdpa_remove_vf(pdev); 1094 else 1095 snet_vdpa_remove_pf(pdev); 1096 } 1097 1098 static struct pci_device_id snet_driver_pci_ids[] = { 1099 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, 1100 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, 1101 { 0 }, 1102 }; 1103 1104 MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); 1105 1106 static struct pci_driver snet_vdpa_driver = { 1107 .name = "snet-vdpa-driver", 1108 .id_table = snet_driver_pci_ids, 1109 .probe = snet_vdpa_probe, 1110 .remove = snet_vdpa_remove, 1111 }; 1112 1113 module_pci_driver(snet_vdpa_driver); 1114 1115 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>"); 1116 MODULE_DESCRIPTION("SolidRun vDPA driver"); 1117 MODULE_LICENSE("GPL v2"); 1118