1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CDX bus driver. 4 * 5 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. 6 */ 7 8 /* 9 * Architecture Overview 10 * ===================== 11 * CDX is a Hardware Architecture designed for AMD FPGA devices. It 12 * consists of sophisticated mechanism for interaction between FPGA, 13 * Firmware and the APUs (Application CPUs). 14 * 15 * Firmware resides on RPU (Realtime CPUs) which interacts with 16 * the FPGA program manager and the APUs. The RPU provides memory-mapped 17 * interface (RPU if) which is used to communicate with APUs. 18 * 19 * The diagram below shows an overview of the CDX architecture: 20 * 21 * +--------------------------------------+ 22 * | Application CPUs (APU) | 23 * | | 24 * | CDX device drivers| 25 * | Linux OS | | 26 * | CDX bus | 27 * | | | 28 * | CDX controller | 29 * | | | 30 * +-----------------------------|--------+ 31 * | (discover, config, 32 * | reset, rescan) 33 * | 34 * +------------------------| RPU if |----+ 35 * | | | 36 * | V | 37 * | Realtime CPUs (RPU) | 38 * | | 39 * +--------------------------------------+ 40 * | 41 * +---------------------|----------------+ 42 * | FPGA | | 43 * | +-----------------------+ | 44 * | | | | | 45 * | +-------+ +-------+ +-------+ | 46 * | | dev 1 | | dev 2 | | dev 3 | | 47 * | +-------+ +-------+ +-------+ | 48 * +--------------------------------------+ 49 * 50 * The RPU firmware extracts the device information from the loaded FPGA 51 * image and implements a mechanism that allows the APU drivers to 52 * enumerate such devices (device personality and resource details) via 53 * a dedicated communication channel. RPU mediates operations such as 54 * discover, reset and rescan of the FPGA devices for the APU. This is 55 * done using memory mapped interface provided by the RPU to APU. 56 */ 57 58 #include <linux/init.h> 59 #include <linux/kernel.h> 60 #include <linux/of_device.h> 61 #include <linux/slab.h> 62 #include <linux/mm.h> 63 #include <linux/xarray.h> 64 #include <linux/cdx/cdx_bus.h> 65 #include <linux/iommu.h> 66 #include <linux/dma-map-ops.h> 67 #include "cdx.h" 68 69 /* Default DMA mask for devices on a CDX bus */ 70 #define CDX_DEFAULT_DMA_MASK (~0ULL) 71 #define MAX_CDX_CONTROLLERS 16 72 73 /* CDX controllers registered with the CDX bus */ 74 static DEFINE_XARRAY_ALLOC(cdx_controllers); 75 76 /** 77 * cdx_dev_reset - Reset a CDX device 78 * @dev: CDX device 79 * 80 * Return: -errno on failure, 0 on success. 81 */ 82 int cdx_dev_reset(struct device *dev) 83 { 84 struct cdx_device *cdx_dev = to_cdx_device(dev); 85 struct cdx_controller *cdx = cdx_dev->cdx; 86 struct cdx_device_config dev_config = {0}; 87 struct cdx_driver *cdx_drv; 88 int ret; 89 90 cdx_drv = to_cdx_driver(dev->driver); 91 /* Notify driver that device is being reset */ 92 if (cdx_drv && cdx_drv->reset_prepare) 93 cdx_drv->reset_prepare(cdx_dev); 94 95 dev_config.type = CDX_DEV_RESET_CONF; 96 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, 97 cdx_dev->dev_num, &dev_config); 98 if (ret) 99 dev_err(dev, "cdx device reset failed\n"); 100 101 /* Notify driver that device reset is complete */ 102 if (cdx_drv && cdx_drv->reset_done) 103 cdx_drv->reset_done(cdx_dev); 104 105 return ret; 106 } 107 EXPORT_SYMBOL_GPL(cdx_dev_reset); 108 109 /** 110 * cdx_unregister_device - Unregister a CDX device 111 * @dev: CDX device 112 * @data: This is always passed as NULL, and is not used in this API, 113 * but is required here as the bus_for_each_dev() API expects 114 * the passed function (cdx_unregister_device) to have this 115 * as an argument. 116 * 117 * Return: 0 on success. 118 */ 119 static int cdx_unregister_device(struct device *dev, 120 void *data) 121 { 122 struct cdx_device *cdx_dev = to_cdx_device(dev); 123 124 kfree(cdx_dev->driver_override); 125 cdx_dev->driver_override = NULL; 126 /* 127 * Do not free cdx_dev here as it would be freed in 128 * cdx_device_release() called from within put_device(). 129 */ 130 device_del(&cdx_dev->dev); 131 put_device(&cdx_dev->dev); 132 133 return 0; 134 } 135 136 static void cdx_unregister_devices(struct bus_type *bus) 137 { 138 /* Reset all the devices attached to cdx bus */ 139 bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device); 140 } 141 142 /** 143 * cdx_match_one_device - Tell if a CDX device structure has a matching 144 * CDX device id structure 145 * @id: single CDX device id structure to match 146 * @dev: the CDX device structure to match against 147 * 148 * Return: matching cdx_device_id structure or NULL if there is no match. 149 */ 150 static inline const struct cdx_device_id * 151 cdx_match_one_device(const struct cdx_device_id *id, 152 const struct cdx_device *dev) 153 { 154 /* Use vendor ID and device ID for matching */ 155 if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) && 156 (id->device == CDX_ANY_ID || id->device == dev->device)) 157 return id; 158 return NULL; 159 } 160 161 /** 162 * cdx_match_id - See if a CDX device matches a given cdx_id table 163 * @ids: array of CDX device ID structures to search in 164 * @dev: the CDX device structure to match against. 165 * 166 * Used by a driver to check whether a CDX device is in its list of 167 * supported devices. Returns the matching cdx_device_id structure or 168 * NULL if there is no match. 169 * 170 * Return: matching cdx_device_id structure or NULL if there is no match. 171 */ 172 static inline const struct cdx_device_id * 173 cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev) 174 { 175 if (ids) { 176 while (ids->vendor || ids->device) { 177 if (cdx_match_one_device(ids, dev)) 178 return ids; 179 ids++; 180 } 181 } 182 return NULL; 183 } 184 185 int cdx_set_master(struct cdx_device *cdx_dev) 186 { 187 struct cdx_controller *cdx = cdx_dev->cdx; 188 struct cdx_device_config dev_config; 189 int ret = -EOPNOTSUPP; 190 191 dev_config.type = CDX_DEV_BUS_MASTER_CONF; 192 dev_config.bus_master_enable = true; 193 if (cdx->ops->dev_configure) 194 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, 195 cdx_dev->dev_num, &dev_config); 196 197 return ret; 198 } 199 EXPORT_SYMBOL_GPL(cdx_set_master); 200 201 int cdx_clear_master(struct cdx_device *cdx_dev) 202 { 203 struct cdx_controller *cdx = cdx_dev->cdx; 204 struct cdx_device_config dev_config; 205 int ret = -EOPNOTSUPP; 206 207 dev_config.type = CDX_DEV_BUS_MASTER_CONF; 208 dev_config.bus_master_enable = false; 209 if (cdx->ops->dev_configure) 210 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, 211 cdx_dev->dev_num, &dev_config); 212 213 return ret; 214 } 215 EXPORT_SYMBOL_GPL(cdx_clear_master); 216 217 /** 218 * cdx_bus_match - device to driver matching callback 219 * @dev: the cdx device to match against 220 * @drv: the device driver to search for matching cdx device 221 * structures 222 * 223 * Return: true on success, false otherwise. 224 */ 225 static int cdx_bus_match(struct device *dev, struct device_driver *drv) 226 { 227 struct cdx_device *cdx_dev = to_cdx_device(dev); 228 struct cdx_driver *cdx_drv = to_cdx_driver(drv); 229 const struct cdx_device_id *found_id = NULL; 230 const struct cdx_device_id *ids; 231 232 ids = cdx_drv->match_id_table; 233 234 /* When driver_override is set, only bind to the matching driver */ 235 if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name)) 236 return false; 237 238 found_id = cdx_match_id(ids, cdx_dev); 239 if (!found_id) 240 return false; 241 242 do { 243 /* 244 * In case override_only was set, enforce driver_override 245 * matching. 246 */ 247 if (!found_id->override_only) 248 return true; 249 if (cdx_dev->driver_override) 250 return true; 251 252 ids = found_id + 1; 253 found_id = cdx_match_id(ids, cdx_dev); 254 } while (found_id); 255 256 return false; 257 } 258 259 static int cdx_probe(struct device *dev) 260 { 261 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 262 struct cdx_device *cdx_dev = to_cdx_device(dev); 263 int error; 264 265 error = cdx_drv->probe(cdx_dev); 266 if (error) { 267 dev_err_probe(dev, error, "%s failed\n", __func__); 268 return error; 269 } 270 271 return 0; 272 } 273 274 static void cdx_remove(struct device *dev) 275 { 276 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 277 struct cdx_device *cdx_dev = to_cdx_device(dev); 278 279 if (cdx_drv && cdx_drv->remove) 280 cdx_drv->remove(cdx_dev); 281 } 282 283 static void cdx_shutdown(struct device *dev) 284 { 285 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 286 struct cdx_device *cdx_dev = to_cdx_device(dev); 287 288 if (cdx_drv && cdx_drv->shutdown) 289 cdx_drv->shutdown(cdx_dev); 290 } 291 292 static int cdx_dma_configure(struct device *dev) 293 { 294 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 295 struct cdx_device *cdx_dev = to_cdx_device(dev); 296 u32 input_id = cdx_dev->req_id; 297 int ret; 298 299 ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id); 300 if (ret && ret != -EPROBE_DEFER) { 301 dev_err(dev, "of_dma_configure_id() failed\n"); 302 return ret; 303 } 304 305 if (!ret && !cdx_drv->driver_managed_dma) { 306 ret = iommu_device_use_default_domain(dev); 307 if (ret) 308 arch_teardown_dma_ops(dev); 309 } 310 311 return 0; 312 } 313 314 static void cdx_dma_cleanup(struct device *dev) 315 { 316 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 317 318 if (!cdx_drv->driver_managed_dma) 319 iommu_device_unuse_default_domain(dev); 320 } 321 322 /* show configuration fields */ 323 #define cdx_config_attr(field, format_string) \ 324 static ssize_t \ 325 field##_show(struct device *dev, struct device_attribute *attr, char *buf) \ 326 { \ 327 struct cdx_device *cdx_dev = to_cdx_device(dev); \ 328 return sysfs_emit(buf, format_string, cdx_dev->field); \ 329 } \ 330 static DEVICE_ATTR_RO(field) 331 332 cdx_config_attr(vendor, "0x%04x\n"); 333 cdx_config_attr(device, "0x%04x\n"); 334 335 static ssize_t remove_store(struct device *dev, 336 struct device_attribute *attr, 337 const char *buf, size_t count) 338 { 339 bool val; 340 341 if (kstrtobool(buf, &val) < 0) 342 return -EINVAL; 343 344 if (!val) 345 return -EINVAL; 346 347 if (device_remove_file_self(dev, attr)) { 348 int ret; 349 350 ret = cdx_unregister_device(dev, NULL); 351 if (ret) 352 return ret; 353 } 354 355 return count; 356 } 357 static DEVICE_ATTR_WO(remove); 358 359 static ssize_t reset_store(struct device *dev, struct device_attribute *attr, 360 const char *buf, size_t count) 361 { 362 bool val; 363 int ret; 364 365 if (kstrtobool(buf, &val) < 0) 366 return -EINVAL; 367 368 if (!val) 369 return -EINVAL; 370 371 ret = cdx_dev_reset(dev); 372 if (ret) 373 return ret; 374 375 return count; 376 } 377 static DEVICE_ATTR_WO(reset); 378 379 static ssize_t driver_override_store(struct device *dev, 380 struct device_attribute *attr, 381 const char *buf, size_t count) 382 { 383 struct cdx_device *cdx_dev = to_cdx_device(dev); 384 int ret; 385 386 if (WARN_ON(dev->bus != &cdx_bus_type)) 387 return -EINVAL; 388 389 ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count); 390 if (ret) 391 return ret; 392 393 return count; 394 } 395 396 static ssize_t driver_override_show(struct device *dev, 397 struct device_attribute *attr, char *buf) 398 { 399 struct cdx_device *cdx_dev = to_cdx_device(dev); 400 401 return sysfs_emit(buf, "%s\n", cdx_dev->driver_override); 402 } 403 static DEVICE_ATTR_RW(driver_override); 404 405 static struct attribute *cdx_dev_attrs[] = { 406 &dev_attr_remove.attr, 407 &dev_attr_reset.attr, 408 &dev_attr_vendor.attr, 409 &dev_attr_device.attr, 410 &dev_attr_driver_override.attr, 411 NULL, 412 }; 413 ATTRIBUTE_GROUPS(cdx_dev); 414 415 static ssize_t rescan_store(const struct bus_type *bus, 416 const char *buf, size_t count) 417 { 418 struct cdx_controller *cdx; 419 unsigned long index; 420 bool val; 421 422 if (kstrtobool(buf, &val) < 0) 423 return -EINVAL; 424 425 if (!val) 426 return -EINVAL; 427 428 /* Unregister all the devices on the bus */ 429 cdx_unregister_devices(&cdx_bus_type); 430 431 /* Rescan all the devices */ 432 xa_for_each(&cdx_controllers, index, cdx) { 433 int ret; 434 435 ret = cdx->ops->scan(cdx); 436 if (ret) 437 dev_err(cdx->dev, "cdx bus scanning failed\n"); 438 } 439 440 return count; 441 } 442 static BUS_ATTR_WO(rescan); 443 444 static struct attribute *cdx_bus_attrs[] = { 445 &bus_attr_rescan.attr, 446 NULL, 447 }; 448 ATTRIBUTE_GROUPS(cdx_bus); 449 450 struct bus_type cdx_bus_type = { 451 .name = "cdx", 452 .match = cdx_bus_match, 453 .probe = cdx_probe, 454 .remove = cdx_remove, 455 .shutdown = cdx_shutdown, 456 .dma_configure = cdx_dma_configure, 457 .dma_cleanup = cdx_dma_cleanup, 458 .bus_groups = cdx_bus_groups, 459 .dev_groups = cdx_dev_groups, 460 }; 461 EXPORT_SYMBOL_GPL(cdx_bus_type); 462 463 int __cdx_driver_register(struct cdx_driver *cdx_driver, 464 struct module *owner) 465 { 466 int error; 467 468 cdx_driver->driver.owner = owner; 469 cdx_driver->driver.bus = &cdx_bus_type; 470 471 error = driver_register(&cdx_driver->driver); 472 if (error) { 473 pr_err("driver_register() failed for %s: %d\n", 474 cdx_driver->driver.name, error); 475 return error; 476 } 477 478 return 0; 479 } 480 EXPORT_SYMBOL_GPL(__cdx_driver_register); 481 482 void cdx_driver_unregister(struct cdx_driver *cdx_driver) 483 { 484 driver_unregister(&cdx_driver->driver); 485 } 486 EXPORT_SYMBOL_GPL(cdx_driver_unregister); 487 488 static void cdx_device_release(struct device *dev) 489 { 490 struct cdx_device *cdx_dev = to_cdx_device(dev); 491 492 kfree(cdx_dev); 493 } 494 495 int cdx_device_add(struct cdx_dev_params *dev_params) 496 { 497 struct cdx_controller *cdx = dev_params->cdx; 498 struct device *parent = cdx->dev; 499 struct cdx_device *cdx_dev; 500 int ret; 501 502 cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL); 503 if (!cdx_dev) 504 return -ENOMEM; 505 506 /* Populate resource */ 507 memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) * 508 dev_params->res_count); 509 cdx_dev->res_count = dev_params->res_count; 510 511 /* Populate CDX dev params */ 512 cdx_dev->req_id = dev_params->req_id; 513 cdx_dev->vendor = dev_params->vendor; 514 cdx_dev->device = dev_params->device; 515 cdx_dev->bus_num = dev_params->bus_num; 516 cdx_dev->dev_num = dev_params->dev_num; 517 cdx_dev->cdx = dev_params->cdx; 518 cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK; 519 520 /* Initialize generic device */ 521 device_initialize(&cdx_dev->dev); 522 cdx_dev->dev.parent = parent; 523 cdx_dev->dev.bus = &cdx_bus_type; 524 cdx_dev->dev.dma_mask = &cdx_dev->dma_mask; 525 cdx_dev->dev.release = cdx_device_release; 526 527 /* Set Name */ 528 dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", 529 ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)), 530 cdx_dev->dev_num); 531 532 ret = device_add(&cdx_dev->dev); 533 if (ret) { 534 dev_err(&cdx_dev->dev, 535 "cdx device add failed: %d", ret); 536 goto fail; 537 } 538 539 return 0; 540 fail: 541 /* 542 * Do not free cdx_dev here as it would be freed in 543 * cdx_device_release() called from put_device(). 544 */ 545 put_device(&cdx_dev->dev); 546 547 return ret; 548 } 549 EXPORT_SYMBOL_GPL(cdx_device_add); 550 551 int cdx_register_controller(struct cdx_controller *cdx) 552 { 553 int ret; 554 555 ret = xa_alloc(&cdx_controllers, &cdx->id, cdx, 556 XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL); 557 if (ret) { 558 dev_err(cdx->dev, 559 "No free index available. Maximum controllers already registered\n"); 560 cdx->id = (u8)MAX_CDX_CONTROLLERS; 561 return ret; 562 } 563 564 /* Scan all the devices */ 565 cdx->ops->scan(cdx); 566 567 return 0; 568 } 569 EXPORT_SYMBOL_GPL(cdx_register_controller); 570 571 void cdx_unregister_controller(struct cdx_controller *cdx) 572 { 573 if (cdx->id >= MAX_CDX_CONTROLLERS) 574 return; 575 576 device_for_each_child(cdx->dev, NULL, cdx_unregister_device); 577 xa_erase(&cdx_controllers, cdx->id); 578 } 579 EXPORT_SYMBOL_GPL(cdx_unregister_controller); 580 581 static int __init cdx_bus_init(void) 582 { 583 return bus_register(&cdx_bus_type); 584 } 585 postcore_initcall(cdx_bus_init); 586