1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel IFC VF NIC driver for virtio dataplane offloading 4 * 5 * Copyright (C) 2020 Intel Corporation. 6 * 7 * Author: Zhu Lingshan <lingshan.zhu@intel.com> 8 * 9 */ 10 11 #include <linux/interrupt.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 #include <linux/sysfs.h> 15 #include "ifcvf_base.h" 16 17 #define DRIVER_AUTHOR "Intel Corporation" 18 #define IFCVF_DRIVER_NAME "ifcvf" 19 20 static irqreturn_t ifcvf_config_changed(int irq, void *arg) 21 { 22 struct ifcvf_hw *vf = arg; 23 24 if (vf->config_cb.callback) 25 return vf->config_cb.callback(vf->config_cb.private); 26 27 return IRQ_HANDLED; 28 } 29 30 static irqreturn_t ifcvf_intr_handler(int irq, void *arg) 31 { 32 struct vring_info *vring = arg; 33 34 if (vring->cb.callback) 35 return vring->cb.callback(vring->cb.private); 36 37 return IRQ_HANDLED; 38 } 39 40 static void ifcvf_free_irq_vectors(void *data) 41 { 42 pci_free_irq_vectors(data); 43 } 44 45 static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues) 46 { 47 struct pci_dev *pdev = adapter->pdev; 48 struct ifcvf_hw *vf = &adapter->vf; 49 int i; 50 51 52 for (i = 0; i < queues; i++) { 53 devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]); 54 vf->vring[i].irq = -EINVAL; 55 } 56 57 devm_free_irq(&pdev->dev, vf->config_irq, vf); 58 ifcvf_free_irq_vectors(pdev); 59 } 60 61 static int ifcvf_request_irq(struct ifcvf_adapter *adapter) 62 { 63 struct pci_dev *pdev = adapter->pdev; 64 struct ifcvf_hw *vf = &adapter->vf; 65 int vector, i, ret, irq; 66 67 ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR, 68 IFCVF_MAX_INTR, PCI_IRQ_MSIX); 69 if (ret < 0) { 70 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n"); 71 return ret; 72 } 73 74 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n", 75 pci_name(pdev)); 76 vector = 0; 77 vf->config_irq = pci_irq_vector(pdev, vector); 78 ret = devm_request_irq(&pdev->dev, vf->config_irq, 79 ifcvf_config_changed, 0, 80 vf->config_msix_name, vf); 81 if (ret) { 82 IFCVF_ERR(pdev, "Failed to request config irq\n"); 83 return ret; 84 } 85 86 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { 87 snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", 88 pci_name(pdev), i); 89 vector = i + IFCVF_MSI_QUEUE_OFF; 90 irq = pci_irq_vector(pdev, vector); 91 ret = devm_request_irq(&pdev->dev, irq, 92 ifcvf_intr_handler, 0, 93 vf->vring[i].msix_name, 94 &vf->vring[i]); 95 if (ret) { 96 IFCVF_ERR(pdev, 97 "Failed to request irq for vq %d\n", i); 98 ifcvf_free_irq(adapter, i); 99 100 return ret; 101 } 102 103 vf->vring[i].irq = irq; 104 } 105 106 return 0; 107 } 108 109 static int ifcvf_start_datapath(void *private) 110 { 111 struct ifcvf_hw *vf = ifcvf_private_to_vf(private); 112 u8 status; 113 int ret; 114 115 vf->nr_vring = IFCVF_MAX_QUEUE_PAIRS * 2; 116 ret = ifcvf_start_hw(vf); 117 if (ret < 0) { 118 status = ifcvf_get_status(vf); 119 status |= VIRTIO_CONFIG_S_FAILED; 120 ifcvf_set_status(vf, status); 121 } 122 123 return ret; 124 } 125 126 static int ifcvf_stop_datapath(void *private) 127 { 128 struct ifcvf_hw *vf = ifcvf_private_to_vf(private); 129 int i; 130 131 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) 132 vf->vring[i].cb.callback = NULL; 133 134 ifcvf_stop_hw(vf); 135 136 return 0; 137 } 138 139 static void ifcvf_reset_vring(struct ifcvf_adapter *adapter) 140 { 141 struct ifcvf_hw *vf = ifcvf_private_to_vf(adapter); 142 int i; 143 144 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) { 145 vf->vring[i].last_avail_idx = 0; 146 vf->vring[i].desc = 0; 147 vf->vring[i].avail = 0; 148 vf->vring[i].used = 0; 149 vf->vring[i].ready = 0; 150 vf->vring[i].cb.callback = NULL; 151 vf->vring[i].cb.private = NULL; 152 } 153 154 ifcvf_reset(vf); 155 } 156 157 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev) 158 { 159 return container_of(vdpa_dev, struct ifcvf_adapter, vdpa); 160 } 161 162 static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev) 163 { 164 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); 165 166 return &adapter->vf; 167 } 168 169 static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev) 170 { 171 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); 172 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 173 struct pci_dev *pdev = adapter->pdev; 174 175 u64 features; 176 177 switch (vf->dev_type) { 178 case VIRTIO_ID_NET: 179 features = ifcvf_get_features(vf) & IFCVF_NET_SUPPORTED_FEATURES; 180 break; 181 case VIRTIO_ID_BLOCK: 182 features = ifcvf_get_features(vf); 183 break; 184 default: 185 features = 0; 186 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type); 187 } 188 189 return features; 190 } 191 192 static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features) 193 { 194 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 195 int ret; 196 197 ret = ifcvf_verify_min_features(vf, features); 198 if (ret) 199 return ret; 200 201 vf->req_features = features; 202 203 return 0; 204 } 205 206 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) 207 { 208 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 209 210 return ifcvf_get_status(vf); 211 } 212 213 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) 214 { 215 struct ifcvf_adapter *adapter; 216 struct ifcvf_hw *vf; 217 u8 status_old; 218 int ret; 219 220 vf = vdpa_to_vf(vdpa_dev); 221 adapter = dev_get_drvdata(vdpa_dev->dev.parent); 222 status_old = ifcvf_get_status(vf); 223 224 if (status_old == status) 225 return; 226 227 if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && 228 !(status & VIRTIO_CONFIG_S_DRIVER_OK)) { 229 ifcvf_stop_datapath(adapter); 230 ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2); 231 } 232 233 if (status == 0) { 234 ifcvf_reset_vring(adapter); 235 return; 236 } 237 238 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 239 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { 240 ret = ifcvf_request_irq(adapter); 241 if (ret) { 242 status = ifcvf_get_status(vf); 243 status |= VIRTIO_CONFIG_S_FAILED; 244 ifcvf_set_status(vf, status); 245 return; 246 } 247 248 if (ifcvf_start_datapath(adapter) < 0) 249 IFCVF_ERR(adapter->pdev, 250 "Failed to set ifcvf vdpa status %u\n", 251 status); 252 } 253 254 ifcvf_set_status(vf, status); 255 } 256 257 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) 258 { 259 return IFCVF_QUEUE_MAX; 260 } 261 262 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 263 struct vdpa_vq_state *state) 264 { 265 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 266 267 state->avail_index = ifcvf_get_vq_state(vf, qid); 268 return 0; 269 } 270 271 static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 272 const struct vdpa_vq_state *state) 273 { 274 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 275 276 return ifcvf_set_vq_state(vf, qid, state->avail_index); 277 } 278 279 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, 280 struct vdpa_callback *cb) 281 { 282 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 283 284 vf->vring[qid].cb = *cb; 285 } 286 287 static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, 288 u16 qid, bool ready) 289 { 290 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 291 292 vf->vring[qid].ready = ready; 293 } 294 295 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) 296 { 297 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 298 299 return vf->vring[qid].ready; 300 } 301 302 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, 303 u32 num) 304 { 305 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 306 307 vf->vring[qid].size = num; 308 } 309 310 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, 311 u64 desc_area, u64 driver_area, 312 u64 device_area) 313 { 314 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 315 316 vf->vring[qid].desc = desc_area; 317 vf->vring[qid].avail = driver_area; 318 vf->vring[qid].used = device_area; 319 320 return 0; 321 } 322 323 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) 324 { 325 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 326 327 ifcvf_notify_queue(vf, qid); 328 } 329 330 static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev) 331 { 332 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 333 334 return ioread8(&vf->common_cfg->config_generation); 335 } 336 337 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev) 338 { 339 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 340 341 return vf->dev_type; 342 } 343 344 static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) 345 { 346 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); 347 struct pci_dev *pdev = adapter->pdev; 348 349 return pdev->subsystem_vendor; 350 } 351 352 static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) 353 { 354 return IFCVF_QUEUE_ALIGNMENT; 355 } 356 357 static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev) 358 { 359 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); 360 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 361 struct pci_dev *pdev = adapter->pdev; 362 size_t size; 363 364 switch (vf->dev_type) { 365 case VIRTIO_ID_NET: 366 size = sizeof(struct virtio_net_config); 367 break; 368 case VIRTIO_ID_BLOCK: 369 size = sizeof(struct virtio_blk_config); 370 break; 371 default: 372 size = 0; 373 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type); 374 } 375 376 return size; 377 } 378 379 static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev, 380 unsigned int offset, 381 void *buf, unsigned int len) 382 { 383 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 384 385 WARN_ON(offset + len > sizeof(struct virtio_net_config)); 386 ifcvf_read_net_config(vf, offset, buf, len); 387 } 388 389 static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev, 390 unsigned int offset, const void *buf, 391 unsigned int len) 392 { 393 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 394 395 WARN_ON(offset + len > sizeof(struct virtio_net_config)); 396 ifcvf_write_net_config(vf, offset, buf, len); 397 } 398 399 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, 400 struct vdpa_callback *cb) 401 { 402 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 403 404 vf->config_cb.callback = cb->callback; 405 vf->config_cb.private = cb->private; 406 } 407 408 static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, 409 u16 qid) 410 { 411 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 412 413 return vf->vring[qid].irq; 414 } 415 416 /* 417 * IFCVF currently does't have on-chip IOMMU, so not 418 * implemented set_map()/dma_map()/dma_unmap() 419 */ 420 static const struct vdpa_config_ops ifc_vdpa_ops = { 421 .get_features = ifcvf_vdpa_get_features, 422 .set_features = ifcvf_vdpa_set_features, 423 .get_status = ifcvf_vdpa_get_status, 424 .set_status = ifcvf_vdpa_set_status, 425 .get_vq_num_max = ifcvf_vdpa_get_vq_num_max, 426 .get_vq_state = ifcvf_vdpa_get_vq_state, 427 .set_vq_state = ifcvf_vdpa_set_vq_state, 428 .set_vq_cb = ifcvf_vdpa_set_vq_cb, 429 .set_vq_ready = ifcvf_vdpa_set_vq_ready, 430 .get_vq_ready = ifcvf_vdpa_get_vq_ready, 431 .set_vq_num = ifcvf_vdpa_set_vq_num, 432 .set_vq_address = ifcvf_vdpa_set_vq_address, 433 .get_vq_irq = ifcvf_vdpa_get_vq_irq, 434 .kick_vq = ifcvf_vdpa_kick_vq, 435 .get_generation = ifcvf_vdpa_get_generation, 436 .get_device_id = ifcvf_vdpa_get_device_id, 437 .get_vendor_id = ifcvf_vdpa_get_vendor_id, 438 .get_vq_align = ifcvf_vdpa_get_vq_align, 439 .get_config_size = ifcvf_vdpa_get_config_size, 440 .get_config = ifcvf_vdpa_get_config, 441 .set_config = ifcvf_vdpa_set_config, 442 .set_config_cb = ifcvf_vdpa_set_config_cb, 443 }; 444 445 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) 446 { 447 struct device *dev = &pdev->dev; 448 struct ifcvf_adapter *adapter; 449 struct ifcvf_hw *vf; 450 int ret, i; 451 452 ret = pcim_enable_device(pdev); 453 if (ret) { 454 IFCVF_ERR(pdev, "Failed to enable device\n"); 455 return ret; 456 } 457 458 ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4), 459 IFCVF_DRIVER_NAME); 460 if (ret) { 461 IFCVF_ERR(pdev, "Failed to request MMIO region\n"); 462 return ret; 463 } 464 465 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 466 if (ret) { 467 IFCVF_ERR(pdev, "No usable DMA configuration\n"); 468 return ret; 469 } 470 471 ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev); 472 if (ret) { 473 IFCVF_ERR(pdev, 474 "Failed for adding devres for freeing irq vectors\n"); 475 return ret; 476 } 477 478 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, 479 dev, &ifc_vdpa_ops, NULL); 480 if (adapter == NULL) { 481 IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); 482 return -ENOMEM; 483 } 484 485 pci_set_master(pdev); 486 pci_set_drvdata(pdev, adapter); 487 488 vf = &adapter->vf; 489 490 /* This drirver drives both modern virtio devices and transitional 491 * devices in modern mode. 492 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM, 493 * so legacy devices and transitional devices in legacy 494 * mode will not work for vDPA, this driver will not 495 * drive devices with legacy interface. 496 */ 497 if (pdev->device < 0x1040) 498 vf->dev_type = pdev->subsystem_device; 499 else 500 vf->dev_type = pdev->device - 0x1040; 501 502 vf->base = pcim_iomap_table(pdev); 503 504 adapter->pdev = pdev; 505 adapter->vdpa.dma_dev = &pdev->dev; 506 507 ret = ifcvf_init_hw(vf, pdev); 508 if (ret) { 509 IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); 510 goto err; 511 } 512 513 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) 514 vf->vring[i].irq = -EINVAL; 515 516 vf->hw_features = ifcvf_get_hw_features(vf); 517 518 ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2); 519 if (ret) { 520 IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus"); 521 goto err; 522 } 523 524 return 0; 525 526 err: 527 put_device(&adapter->vdpa.dev); 528 return ret; 529 } 530 531 static void ifcvf_remove(struct pci_dev *pdev) 532 { 533 struct ifcvf_adapter *adapter = pci_get_drvdata(pdev); 534 535 vdpa_unregister_device(&adapter->vdpa); 536 } 537 538 static struct pci_device_id ifcvf_pci_ids[] = { 539 { PCI_DEVICE_SUB(N3000_VENDOR_ID, 540 N3000_DEVICE_ID, 541 N3000_SUBSYS_VENDOR_ID, 542 N3000_SUBSYS_DEVICE_ID) }, 543 { PCI_DEVICE_SUB(C5000X_PL_VENDOR_ID, 544 C5000X_PL_DEVICE_ID, 545 C5000X_PL_SUBSYS_VENDOR_ID, 546 C5000X_PL_SUBSYS_DEVICE_ID) }, 547 { PCI_DEVICE_SUB(C5000X_PL_BLK_VENDOR_ID, 548 C5000X_PL_BLK_DEVICE_ID, 549 C5000X_PL_BLK_SUBSYS_VENDOR_ID, 550 C5000X_PL_BLK_SUBSYS_DEVICE_ID) }, 551 552 { 0 }, 553 }; 554 MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids); 555 556 static struct pci_driver ifcvf_driver = { 557 .name = IFCVF_DRIVER_NAME, 558 .id_table = ifcvf_pci_ids, 559 .probe = ifcvf_probe, 560 .remove = ifcvf_remove, 561 }; 562 563 module_pci_driver(ifcvf_driver); 564 565 MODULE_LICENSE("GPL v2"); 566