1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * CDX bus driver.
4 *
5 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
6 */
7
8 /*
9 * Architecture Overview
10 * =====================
11 * CDX is a Hardware Architecture designed for AMD FPGA devices. It
12 * consists of sophisticated mechanism for interaction between FPGA,
13 * Firmware and the APUs (Application CPUs).
14 *
15 * Firmware resides on RPU (Realtime CPUs) which interacts with
16 * the FPGA program manager and the APUs. The RPU provides memory-mapped
17 * interface (RPU if) which is used to communicate with APUs.
18 *
19 * The diagram below shows an overview of the CDX architecture:
20 *
21 * +--------------------------------------+
22 * | Application CPUs (APU) |
23 * | |
24 * | CDX device drivers|
25 * | Linux OS | |
26 * | CDX bus |
27 * | | |
28 * | CDX controller |
29 * | | |
30 * +-----------------------------|--------+
31 * | (discover, config,
32 * | reset, rescan)
33 * |
34 * +------------------------| RPU if |----+
35 * | | |
36 * | V |
37 * | Realtime CPUs (RPU) |
38 * | |
39 * +--------------------------------------+
40 * |
41 * +---------------------|----------------+
42 * | FPGA | |
43 * | +-----------------------+ |
44 * | | | | |
45 * | +-------+ +-------+ +-------+ |
46 * | | dev 1 | | dev 2 | | dev 3 | |
47 * | +-------+ +-------+ +-------+ |
48 * +--------------------------------------+
49 *
50 * The RPU firmware extracts the device information from the loaded FPGA
51 * image and implements a mechanism that allows the APU drivers to
52 * enumerate such devices (device personality and resource details) via
53 * a dedicated communication channel. RPU mediates operations such as
54 * discover, reset and rescan of the FPGA devices for the APU. This is
55 * done using memory mapped interface provided by the RPU to APU.
56 */
57
58 #include <linux/init.h>
59 #include <linux/irqdomain.h>
60 #include <linux/kernel.h>
61 #include <linux/of.h>
62 #include <linux/of_device.h>
63 #include <linux/of_platform.h>
64 #include <linux/platform_device.h>
65 #include <linux/slab.h>
66 #include <linux/mm.h>
67 #include <linux/idr.h>
68 #include <linux/cdx/cdx_bus.h>
69 #include <linux/iommu.h>
70 #include <linux/dma-map-ops.h>
71 #include <linux/debugfs.h>
72 #include "cdx.h"
73
74 /* Default DMA mask for devices on a CDX bus */
75 #define CDX_DEFAULT_DMA_MASK (~0ULL)
76 #define MAX_CDX_CONTROLLERS 16
77
78 /* IDA for CDX controllers registered with the CDX bus */
79 static DEFINE_IDA(cdx_controller_ida);
80 /* Lock to protect controller ops */
81 static DEFINE_MUTEX(cdx_controller_lock);
82 /* Debugfs dir for cdx bus */
83 static struct dentry *cdx_debugfs_dir;
84
85 static char *compat_node_name = "xlnx,versal-net-cdx";
86
87 static void cdx_destroy_res_attr(struct cdx_device *cdx_dev, int num);
88
89 /**
90 * cdx_dev_reset - Reset a CDX device
91 * @dev: CDX device
92 *
93 * Return: -errno on failure, 0 on success.
94 */
cdx_dev_reset(struct device * dev)95 int cdx_dev_reset(struct device *dev)
96 {
97 struct cdx_device *cdx_dev = to_cdx_device(dev);
98 struct cdx_controller *cdx = cdx_dev->cdx;
99 struct cdx_device_config dev_config = {0};
100 struct cdx_driver *cdx_drv;
101 int ret;
102
103 cdx_drv = to_cdx_driver(dev->driver);
104 /* Notify driver that device is being reset */
105 if (cdx_drv && cdx_drv->reset_prepare)
106 cdx_drv->reset_prepare(cdx_dev);
107
108 dev_config.type = CDX_DEV_RESET_CONF;
109 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
110 cdx_dev->dev_num, &dev_config);
111 if (ret)
112 dev_err(dev, "cdx device reset failed\n");
113
114 /* Notify driver that device reset is complete */
115 if (cdx_drv && cdx_drv->reset_done)
116 cdx_drv->reset_done(cdx_dev);
117
118 return ret;
119 }
120 EXPORT_SYMBOL_GPL(cdx_dev_reset);
121
122 /**
123 * reset_cdx_device - Reset a CDX device
124 * @dev: CDX device
125 * @data: This is always passed as NULL, and is not used in this API,
126 * but is required here as the device_for_each_child() API expects
127 * the passed function to have this as an argument.
128 *
129 * Return: -errno on failure, 0 on success.
130 */
reset_cdx_device(struct device * dev,void * data)131 static int reset_cdx_device(struct device *dev, void *data)
132 {
133 return cdx_dev_reset(dev);
134 }
135
136 /**
137 * cdx_unregister_device - Unregister a CDX device
138 * @dev: CDX device
139 * @data: This is always passed as NULL, and is not used in this API,
140 * but is required here as the bus_for_each_dev() API expects
141 * the passed function (cdx_unregister_device) to have this
142 * as an argument.
143 *
144 * Return: 0 on success.
145 */
cdx_unregister_device(struct device * dev,void * data)146 static int cdx_unregister_device(struct device *dev,
147 void *data)
148 {
149 struct cdx_device *cdx_dev = to_cdx_device(dev);
150 struct cdx_controller *cdx = cdx_dev->cdx;
151
152 if (cdx_dev->is_bus) {
153 device_for_each_child(dev, NULL, cdx_unregister_device);
154 if (cdx_dev->enabled && cdx->ops->bus_disable)
155 cdx->ops->bus_disable(cdx, cdx_dev->bus_num);
156 } else {
157 cdx_destroy_res_attr(cdx_dev, MAX_CDX_DEV_RESOURCES);
158 debugfs_remove_recursive(cdx_dev->debugfs_dir);
159 kfree(cdx_dev->driver_override);
160 cdx_dev->driver_override = NULL;
161 }
162
163 /*
164 * Do not free cdx_dev here as it would be freed in
165 * cdx_device_release() called from within put_device().
166 */
167 device_del(&cdx_dev->dev);
168 put_device(&cdx_dev->dev);
169
170 return 0;
171 }
172
cdx_unregister_devices(const struct bus_type * bus)173 static void cdx_unregister_devices(const struct bus_type *bus)
174 {
175 /* Reset all the devices attached to cdx bus */
176 bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
177 }
178
179 /**
180 * cdx_match_one_device - Tell if a CDX device structure has a matching
181 * CDX device id structure
182 * @id: single CDX device id structure to match
183 * @dev: the CDX device structure to match against
184 *
185 * Return: matching cdx_device_id structure or NULL if there is no match.
186 */
187 static inline const struct cdx_device_id *
cdx_match_one_device(const struct cdx_device_id * id,const struct cdx_device * dev)188 cdx_match_one_device(const struct cdx_device_id *id,
189 const struct cdx_device *dev)
190 {
191 /* Use vendor ID and device ID for matching */
192 if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) &&
193 (id->device == CDX_ANY_ID || id->device == dev->device) &&
194 (id->subvendor == CDX_ANY_ID || id->subvendor == dev->subsystem_vendor) &&
195 (id->subdevice == CDX_ANY_ID || id->subdevice == dev->subsystem_device) &&
196 !((id->class ^ dev->class) & id->class_mask))
197 return id;
198 return NULL;
199 }
200
201 /**
202 * cdx_match_id - See if a CDX device matches a given cdx_id table
203 * @ids: array of CDX device ID structures to search in
204 * @dev: the CDX device structure to match against.
205 *
206 * Used by a driver to check whether a CDX device is in its list of
207 * supported devices. Returns the matching cdx_device_id structure or
208 * NULL if there is no match.
209 *
210 * Return: matching cdx_device_id structure or NULL if there is no match.
211 */
212 static inline const struct cdx_device_id *
cdx_match_id(const struct cdx_device_id * ids,struct cdx_device * dev)213 cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
214 {
215 if (ids) {
216 while (ids->vendor || ids->device) {
217 if (cdx_match_one_device(ids, dev))
218 return ids;
219 ids++;
220 }
221 }
222 return NULL;
223 }
224
cdx_set_master(struct cdx_device * cdx_dev)225 int cdx_set_master(struct cdx_device *cdx_dev)
226 {
227 struct cdx_controller *cdx = cdx_dev->cdx;
228 struct cdx_device_config dev_config;
229 int ret = -EOPNOTSUPP;
230
231 dev_config.type = CDX_DEV_BUS_MASTER_CONF;
232 dev_config.bus_master_enable = true;
233 if (cdx->ops->dev_configure)
234 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
235 cdx_dev->dev_num, &dev_config);
236
237 return ret;
238 }
239 EXPORT_SYMBOL_GPL(cdx_set_master);
240
cdx_clear_master(struct cdx_device * cdx_dev)241 int cdx_clear_master(struct cdx_device *cdx_dev)
242 {
243 struct cdx_controller *cdx = cdx_dev->cdx;
244 struct cdx_device_config dev_config;
245 int ret = -EOPNOTSUPP;
246
247 dev_config.type = CDX_DEV_BUS_MASTER_CONF;
248 dev_config.bus_master_enable = false;
249 if (cdx->ops->dev_configure)
250 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
251 cdx_dev->dev_num, &dev_config);
252
253 return ret;
254 }
255 EXPORT_SYMBOL_GPL(cdx_clear_master);
256
257 /**
258 * cdx_bus_match - device to driver matching callback
259 * @dev: the cdx device to match against
260 * @drv: the device driver to search for matching cdx device
261 * structures
262 *
263 * Return: true on success, false otherwise.
264 */
cdx_bus_match(struct device * dev,const struct device_driver * drv)265 static int cdx_bus_match(struct device *dev, const struct device_driver *drv)
266 {
267 struct cdx_device *cdx_dev = to_cdx_device(dev);
268 const struct cdx_driver *cdx_drv = to_cdx_driver(drv);
269 const struct cdx_device_id *found_id = NULL;
270 const struct cdx_device_id *ids;
271
272 if (cdx_dev->is_bus)
273 return false;
274
275 ids = cdx_drv->match_id_table;
276
277 /* When driver_override is set, only bind to the matching driver */
278 if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name))
279 return false;
280
281 found_id = cdx_match_id(ids, cdx_dev);
282 if (!found_id)
283 return false;
284
285 do {
286 /*
287 * In case override_only was set, enforce driver_override
288 * matching.
289 */
290 if (!found_id->override_only)
291 return true;
292 if (cdx_dev->driver_override)
293 return true;
294
295 ids = found_id + 1;
296 found_id = cdx_match_id(ids, cdx_dev);
297 } while (found_id);
298
299 return false;
300 }
301
cdx_probe(struct device * dev)302 static int cdx_probe(struct device *dev)
303 {
304 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
305 struct cdx_device *cdx_dev = to_cdx_device(dev);
306 struct cdx_controller *cdx = cdx_dev->cdx;
307 int error;
308
309 /*
310 * Setup MSI device data so that generic MSI alloc/free can
311 * be used by the device driver.
312 */
313 if (IS_ENABLED(CONFIG_GENERIC_MSI_IRQ) && cdx->msi_domain) {
314 error = msi_setup_device_data(&cdx_dev->dev);
315 if (error)
316 return error;
317 }
318
319 error = cdx_drv->probe(cdx_dev);
320 if (error) {
321 dev_err_probe(dev, error, "%s failed\n", __func__);
322 return error;
323 }
324
325 return 0;
326 }
327
cdx_remove(struct device * dev)328 static void cdx_remove(struct device *dev)
329 {
330 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
331 struct cdx_device *cdx_dev = to_cdx_device(dev);
332
333 if (cdx_drv && cdx_drv->remove)
334 cdx_drv->remove(cdx_dev);
335 }
336
cdx_shutdown(struct device * dev)337 static void cdx_shutdown(struct device *dev)
338 {
339 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
340 struct cdx_device *cdx_dev = to_cdx_device(dev);
341 struct cdx_controller *cdx = cdx_dev->cdx;
342
343 if (cdx_dev->is_bus && cdx_dev->enabled && cdx->ops->bus_disable)
344 cdx->ops->bus_disable(cdx, cdx_dev->bus_num);
345 if (cdx_drv && cdx_drv->shutdown)
346 cdx_drv->shutdown(cdx_dev);
347 }
348
cdx_dma_configure(struct device * dev)349 static int cdx_dma_configure(struct device *dev)
350 {
351 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
352 struct cdx_device *cdx_dev = to_cdx_device(dev);
353 struct cdx_controller *cdx = cdx_dev->cdx;
354 u32 input_id = cdx_dev->req_id;
355 int ret;
356
357 ret = of_dma_configure_id(dev, cdx->dev->of_node, 0, &input_id);
358 if (ret && ret != -EPROBE_DEFER) {
359 dev_err(dev, "of_dma_configure_id() failed\n");
360 return ret;
361 }
362
363 /* @cdx_drv may not be valid when we're called from the IOMMU layer */
364 if (!ret && dev->driver && !cdx_drv->driver_managed_dma) {
365 ret = iommu_device_use_default_domain(dev);
366 if (ret)
367 arch_teardown_dma_ops(dev);
368 }
369
370 return 0;
371 }
372
cdx_dma_cleanup(struct device * dev)373 static void cdx_dma_cleanup(struct device *dev)
374 {
375 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
376
377 if (!cdx_drv->driver_managed_dma)
378 iommu_device_unuse_default_domain(dev);
379 }
380
381 /* show configuration fields */
382 #define cdx_config_attr(field, format_string) \
383 static ssize_t \
384 field##_show(struct device *dev, struct device_attribute *attr, char *buf) \
385 { \
386 struct cdx_device *cdx_dev = to_cdx_device(dev); \
387 return sysfs_emit(buf, format_string, cdx_dev->field); \
388 } \
389 static DEVICE_ATTR_RO(field)
390
391 cdx_config_attr(vendor, "0x%04x\n");
392 cdx_config_attr(device, "0x%04x\n");
393 cdx_config_attr(subsystem_vendor, "0x%04x\n");
394 cdx_config_attr(subsystem_device, "0x%04x\n");
395 cdx_config_attr(revision, "0x%02x\n");
396 cdx_config_attr(class, "0x%06x\n");
397
remove_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)398 static ssize_t remove_store(struct device *dev,
399 struct device_attribute *attr,
400 const char *buf, size_t count)
401 {
402 bool val;
403
404 if (kstrtobool(buf, &val) < 0)
405 return -EINVAL;
406
407 if (!val)
408 return -EINVAL;
409
410 if (device_remove_file_self(dev, attr)) {
411 int ret;
412
413 ret = cdx_unregister_device(dev, NULL);
414 if (ret)
415 return ret;
416 }
417
418 return count;
419 }
420 static DEVICE_ATTR_WO(remove);
421
reset_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)422 static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
423 const char *buf, size_t count)
424 {
425 struct cdx_device *cdx_dev = to_cdx_device(dev);
426 bool val;
427 int ret;
428
429 if (kstrtobool(buf, &val) < 0)
430 return -EINVAL;
431
432 if (!val)
433 return -EINVAL;
434
435 if (cdx_dev->is_bus)
436 /* Reset all the devices attached to cdx bus */
437 ret = device_for_each_child(dev, NULL, reset_cdx_device);
438 else
439 ret = cdx_dev_reset(dev);
440
441 return ret < 0 ? ret : count;
442 }
443 static DEVICE_ATTR_WO(reset);
444
modalias_show(struct device * dev,struct device_attribute * attr,char * buf)445 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
446 char *buf)
447 {
448 struct cdx_device *cdx_dev = to_cdx_device(dev);
449
450 return sprintf(buf, "cdx:v%04Xd%04Xsv%04Xsd%04Xc%06X\n", cdx_dev->vendor,
451 cdx_dev->device, cdx_dev->subsystem_vendor, cdx_dev->subsystem_device,
452 cdx_dev->class);
453 }
454 static DEVICE_ATTR_RO(modalias);
455
driver_override_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)456 static ssize_t driver_override_store(struct device *dev,
457 struct device_attribute *attr,
458 const char *buf, size_t count)
459 {
460 struct cdx_device *cdx_dev = to_cdx_device(dev);
461 int ret;
462
463 if (WARN_ON(dev->bus != &cdx_bus_type))
464 return -EINVAL;
465
466 ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count);
467 if (ret)
468 return ret;
469
470 return count;
471 }
472
driver_override_show(struct device * dev,struct device_attribute * attr,char * buf)473 static ssize_t driver_override_show(struct device *dev,
474 struct device_attribute *attr, char *buf)
475 {
476 struct cdx_device *cdx_dev = to_cdx_device(dev);
477 ssize_t len;
478
479 device_lock(dev);
480 len = sysfs_emit(buf, "%s\n", cdx_dev->driver_override);
481 device_unlock(dev);
482 return len;
483 }
484 static DEVICE_ATTR_RW(driver_override);
485
enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)486 static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
487 const char *buf, size_t count)
488 {
489 struct cdx_device *cdx_dev = to_cdx_device(dev);
490 struct cdx_controller *cdx = cdx_dev->cdx;
491 bool enable;
492 int ret;
493
494 if (kstrtobool(buf, &enable) < 0)
495 return -EINVAL;
496
497 if (enable == cdx_dev->enabled)
498 return count;
499
500 if (enable && cdx->ops->bus_enable)
501 ret = cdx->ops->bus_enable(cdx, cdx_dev->bus_num);
502 else if (!enable && cdx->ops->bus_disable)
503 ret = cdx->ops->bus_disable(cdx, cdx_dev->bus_num);
504 else
505 ret = -EOPNOTSUPP;
506
507 if (!ret)
508 cdx_dev->enabled = enable;
509
510 return ret < 0 ? ret : count;
511 }
512
enable_show(struct device * dev,struct device_attribute * attr,char * buf)513 static ssize_t enable_show(struct device *dev, struct device_attribute *attr, char *buf)
514 {
515 struct cdx_device *cdx_dev = to_cdx_device(dev);
516
517 return sysfs_emit(buf, "%u\n", cdx_dev->enabled);
518 }
519 static DEVICE_ATTR_RW(enable);
520
cdx_dev_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)521 static umode_t cdx_dev_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n)
522 {
523 struct device *dev = kobj_to_dev(kobj);
524 struct cdx_device *cdx_dev;
525
526 cdx_dev = to_cdx_device(dev);
527 if (!cdx_dev->is_bus)
528 return a->mode;
529
530 return 0;
531 }
532
cdx_bus_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)533 static umode_t cdx_bus_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n)
534 {
535 struct device *dev = kobj_to_dev(kobj);
536 struct cdx_device *cdx_dev;
537
538 cdx_dev = to_cdx_device(dev);
539 if (cdx_dev->is_bus)
540 return a->mode;
541
542 return 0;
543 }
544
545 static struct attribute *cdx_dev_attrs[] = {
546 &dev_attr_remove.attr,
547 &dev_attr_reset.attr,
548 &dev_attr_vendor.attr,
549 &dev_attr_device.attr,
550 &dev_attr_subsystem_vendor.attr,
551 &dev_attr_subsystem_device.attr,
552 &dev_attr_class.attr,
553 &dev_attr_revision.attr,
554 &dev_attr_modalias.attr,
555 &dev_attr_driver_override.attr,
556 NULL,
557 };
558
559 static const struct attribute_group cdx_dev_group = {
560 .attrs = cdx_dev_attrs,
561 .is_visible = cdx_dev_attrs_are_visible,
562 };
563
564 static struct attribute *cdx_bus_dev_attrs[] = {
565 &dev_attr_enable.attr,
566 &dev_attr_reset.attr,
567 NULL,
568 };
569
570 static const struct attribute_group cdx_bus_dev_group = {
571 .attrs = cdx_bus_dev_attrs,
572 .is_visible = cdx_bus_attrs_are_visible,
573 };
574
575 static const struct attribute_group *cdx_dev_groups[] = {
576 &cdx_dev_group,
577 &cdx_bus_dev_group,
578 NULL,
579 };
580
cdx_debug_resource_show(struct seq_file * s,void * data)581 static int cdx_debug_resource_show(struct seq_file *s, void *data)
582 {
583 struct cdx_device *cdx_dev = s->private;
584 int i;
585
586 for (i = 0; i < MAX_CDX_DEV_RESOURCES; i++) {
587 struct resource *res = &cdx_dev->res[i];
588
589 seq_printf(s, "%pr\n", res);
590 }
591
592 return 0;
593 }
594 DEFINE_SHOW_ATTRIBUTE(cdx_debug_resource);
595
cdx_device_debugfs_init(struct cdx_device * cdx_dev)596 static void cdx_device_debugfs_init(struct cdx_device *cdx_dev)
597 {
598 cdx_dev->debugfs_dir = debugfs_create_dir(dev_name(&cdx_dev->dev), cdx_debugfs_dir);
599 if (IS_ERR(cdx_dev->debugfs_dir))
600 return;
601
602 debugfs_create_file("resource", 0444, cdx_dev->debugfs_dir, cdx_dev,
603 &cdx_debug_resource_fops);
604 }
605
rescan_store(const struct bus_type * bus,const char * buf,size_t count)606 static ssize_t rescan_store(const struct bus_type *bus,
607 const char *buf, size_t count)
608 {
609 struct cdx_controller *cdx;
610 struct platform_device *pd;
611 bool val;
612
613 if (kstrtobool(buf, &val) < 0)
614 return -EINVAL;
615
616 if (!val)
617 return -EINVAL;
618
619 guard(mutex)(&cdx_controller_lock);
620
621 /* Unregister all the devices on the bus */
622 cdx_unregister_devices(&cdx_bus_type);
623
624 /* Rescan all the devices */
625 for_each_compatible_node_scoped(np, NULL, compat_node_name) {
626 pd = of_find_device_by_node(np);
627 if (!pd)
628 return -EINVAL;
629
630 cdx = platform_get_drvdata(pd);
631 if (cdx && cdx->controller_registered && cdx->ops->scan)
632 cdx->ops->scan(cdx);
633
634 put_device(&pd->dev);
635 }
636
637 return count;
638 }
639 static BUS_ATTR_WO(rescan);
640
641 static struct attribute *cdx_bus_attrs[] = {
642 &bus_attr_rescan.attr,
643 NULL,
644 };
645 ATTRIBUTE_GROUPS(cdx_bus);
646
647 const struct bus_type cdx_bus_type = {
648 .name = "cdx",
649 .match = cdx_bus_match,
650 .probe = cdx_probe,
651 .remove = cdx_remove,
652 .shutdown = cdx_shutdown,
653 .dma_configure = cdx_dma_configure,
654 .dma_cleanup = cdx_dma_cleanup,
655 .bus_groups = cdx_bus_groups,
656 .dev_groups = cdx_dev_groups,
657 };
658 EXPORT_SYMBOL_GPL(cdx_bus_type);
659
__cdx_driver_register(struct cdx_driver * cdx_driver,struct module * owner)660 int __cdx_driver_register(struct cdx_driver *cdx_driver,
661 struct module *owner)
662 {
663 int error;
664
665 cdx_driver->driver.owner = owner;
666 cdx_driver->driver.bus = &cdx_bus_type;
667
668 error = driver_register(&cdx_driver->driver);
669 if (error) {
670 pr_err("driver_register() failed for %s: %d\n",
671 cdx_driver->driver.name, error);
672 return error;
673 }
674
675 return 0;
676 }
677 EXPORT_SYMBOL_GPL(__cdx_driver_register);
678
cdx_driver_unregister(struct cdx_driver * cdx_driver)679 void cdx_driver_unregister(struct cdx_driver *cdx_driver)
680 {
681 driver_unregister(&cdx_driver->driver);
682 }
683 EXPORT_SYMBOL_GPL(cdx_driver_unregister);
684
cdx_device_release(struct device * dev)685 static void cdx_device_release(struct device *dev)
686 {
687 struct cdx_device *cdx_dev = to_cdx_device(dev);
688
689 kfree(cdx_dev);
690 }
691
692 static const struct vm_operations_struct cdx_phys_vm_ops = {
693 #ifdef CONFIG_HAVE_IOREMAP_PROT
694 .access = generic_access_phys,
695 #endif
696 };
697
698 /**
699 * cdx_mmap_resource - map a CDX resource into user memory space
700 * @fp: File pointer. Not used in this function, but required where
701 * this API is registered as a callback.
702 * @kobj: kobject for mapping
703 * @attr: struct bin_attribute for the file being mapped
704 * @vma: struct vm_area_struct passed into the mmap
705 *
706 * Use the regular CDX mapping routines to map a CDX resource into userspace.
707 *
708 * Return: true on success, false otherwise.
709 */
cdx_mmap_resource(struct file * fp,struct kobject * kobj,const struct bin_attribute * attr,struct vm_area_struct * vma)710 static int cdx_mmap_resource(struct file *fp, struct kobject *kobj,
711 const struct bin_attribute *attr,
712 struct vm_area_struct *vma)
713 {
714 struct cdx_device *cdx_dev = to_cdx_device(kobj_to_dev(kobj));
715 int num = (unsigned long)attr->private;
716 struct resource *res;
717 unsigned long size;
718
719 res = &cdx_dev->res[num];
720 if (iomem_is_exclusive(res->start))
721 return -EINVAL;
722
723 /* Make sure the caller is mapping a valid resource for this device */
724 size = ((cdx_resource_len(cdx_dev, num) - 1) >> PAGE_SHIFT) + 1;
725 if (vma->vm_pgoff + vma_pages(vma) > size)
726 return -EINVAL;
727
728 /*
729 * Map memory region and vm->vm_pgoff is expected to be an
730 * offset within that region.
731 */
732 vma->vm_page_prot = pgprot_device(vma->vm_page_prot);
733 vma->vm_pgoff += (cdx_resource_start(cdx_dev, num) >> PAGE_SHIFT);
734 vma->vm_ops = &cdx_phys_vm_ops;
735 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
736 vma->vm_end - vma->vm_start,
737 vma->vm_page_prot);
738 }
739
cdx_destroy_res_attr(struct cdx_device * cdx_dev,int num)740 static void cdx_destroy_res_attr(struct cdx_device *cdx_dev, int num)
741 {
742 int i;
743
744 /* removing the bin attributes */
745 for (i = 0; i < num; i++) {
746 struct bin_attribute *res_attr;
747
748 res_attr = cdx_dev->res_attr[i];
749 if (res_attr) {
750 sysfs_remove_bin_file(&cdx_dev->dev.kobj, res_attr);
751 kfree(res_attr);
752 }
753 }
754 }
755
756 #define CDX_RES_ATTR_NAME_LEN 10
cdx_create_res_attr(struct cdx_device * cdx_dev,int num)757 static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num)
758 {
759 struct bin_attribute *res_attr;
760 char *res_attr_name;
761 int ret;
762
763 res_attr = kzalloc(sizeof(*res_attr) + CDX_RES_ATTR_NAME_LEN, GFP_ATOMIC);
764 if (!res_attr)
765 return -ENOMEM;
766
767 res_attr_name = (char *)(res_attr + 1);
768
769 sysfs_bin_attr_init(res_attr);
770
771 cdx_dev->res_attr[num] = res_attr;
772 sprintf(res_attr_name, "resource%d", num);
773
774 res_attr->mmap = cdx_mmap_resource;
775 res_attr->attr.name = res_attr_name;
776 res_attr->attr.mode = 0600;
777 res_attr->size = cdx_resource_len(cdx_dev, num);
778 res_attr->private = (void *)(unsigned long)num;
779 ret = sysfs_create_bin_file(&cdx_dev->dev.kobj, res_attr);
780 if (ret)
781 kfree(res_attr);
782
783 return ret;
784 }
785
cdx_device_add(struct cdx_dev_params * dev_params)786 int cdx_device_add(struct cdx_dev_params *dev_params)
787 {
788 struct cdx_controller *cdx = dev_params->cdx;
789 struct cdx_device *cdx_dev;
790 int ret, i;
791
792 cdx_dev = kzalloc_obj(*cdx_dev);
793 if (!cdx_dev)
794 return -ENOMEM;
795
796 /* Populate resource */
797 memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
798 dev_params->res_count);
799 cdx_dev->res_count = dev_params->res_count;
800
801 /* Populate CDX dev params */
802 cdx_dev->req_id = dev_params->req_id;
803 cdx_dev->msi_dev_id = dev_params->msi_dev_id;
804 cdx_dev->vendor = dev_params->vendor;
805 cdx_dev->device = dev_params->device;
806 cdx_dev->subsystem_vendor = dev_params->subsys_vendor;
807 cdx_dev->subsystem_device = dev_params->subsys_device;
808 cdx_dev->class = dev_params->class;
809 cdx_dev->revision = dev_params->revision;
810 cdx_dev->bus_num = dev_params->bus_num;
811 cdx_dev->dev_num = dev_params->dev_num;
812 cdx_dev->cdx = dev_params->cdx;
813 cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
814
815 /* Initialize generic device */
816 device_initialize(&cdx_dev->dev);
817 cdx_dev->dev.parent = dev_params->parent;
818 cdx_dev->dev.bus = &cdx_bus_type;
819 cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
820 cdx_dev->dev.release = cdx_device_release;
821 cdx_dev->msi_write_pending = false;
822 mutex_init(&cdx_dev->irqchip_lock);
823
824 /* Set Name */
825 dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
826 ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
827 cdx_dev->dev_num);
828
829 if (IS_ENABLED(CONFIG_GENERIC_MSI_IRQ) && cdx->msi_domain) {
830 cdx_dev->num_msi = dev_params->num_msi;
831 dev_set_msi_domain(&cdx_dev->dev, cdx->msi_domain);
832 }
833
834 ret = device_add(&cdx_dev->dev);
835 if (ret) {
836 dev_err(&cdx_dev->dev,
837 "cdx device add failed: %d", ret);
838 goto fail;
839 }
840
841 /* Create resource<N> attributes */
842 for (i = 0; i < MAX_CDX_DEV_RESOURCES; i++) {
843 if (cdx_resource_flags(cdx_dev, i) & IORESOURCE_MEM) {
844 /* skip empty resources */
845 if (!cdx_resource_len(cdx_dev, i))
846 continue;
847
848 ret = cdx_create_res_attr(cdx_dev, i);
849 if (ret != 0) {
850 dev_err(&cdx_dev->dev,
851 "cdx device resource<%d> file creation failed: %d", i, ret);
852 goto resource_create_fail;
853 }
854 }
855 }
856
857 cdx_device_debugfs_init(cdx_dev);
858
859 return 0;
860 resource_create_fail:
861 cdx_destroy_res_attr(cdx_dev, i);
862 device_del(&cdx_dev->dev);
863 fail:
864 /*
865 * Do not free cdx_dev here as it would be freed in
866 * cdx_device_release() called from put_device().
867 */
868 put_device(&cdx_dev->dev);
869
870 return ret;
871 }
872 EXPORT_SYMBOL_NS_GPL(cdx_device_add, "CDX_BUS_CONTROLLER");
873
cdx_bus_add(struct cdx_controller * cdx,u8 bus_num)874 struct device *cdx_bus_add(struct cdx_controller *cdx, u8 bus_num)
875 {
876 struct cdx_device *cdx_dev;
877 int ret;
878
879 cdx_dev = kzalloc_obj(*cdx_dev);
880 if (!cdx_dev)
881 return NULL;
882
883 device_initialize(&cdx_dev->dev);
884 cdx_dev->cdx = cdx;
885
886 cdx_dev->dev.parent = cdx->dev;
887 cdx_dev->dev.bus = &cdx_bus_type;
888 cdx_dev->dev.release = cdx_device_release;
889 cdx_dev->is_bus = true;
890 cdx_dev->bus_num = bus_num;
891
892 dev_set_name(&cdx_dev->dev, "cdx-%02x",
893 ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (bus_num & CDX_BUS_NUM_MASK)));
894
895 ret = device_add(&cdx_dev->dev);
896 if (ret) {
897 dev_err(&cdx_dev->dev, "cdx bus device add failed: %d\n", ret);
898 goto device_add_fail;
899 }
900
901 if (cdx->ops->bus_enable) {
902 ret = cdx->ops->bus_enable(cdx, bus_num);
903 if (ret && ret != -EALREADY) {
904 dev_err(cdx->dev, "cdx bus enable failed: %d\n", ret);
905 goto bus_enable_fail;
906 }
907 }
908
909 cdx_dev->enabled = true;
910 return &cdx_dev->dev;
911
912 bus_enable_fail:
913 device_del(&cdx_dev->dev);
914 device_add_fail:
915 put_device(&cdx_dev->dev);
916
917 return NULL;
918 }
919 EXPORT_SYMBOL_NS_GPL(cdx_bus_add, "CDX_BUS_CONTROLLER");
920
cdx_register_controller(struct cdx_controller * cdx)921 int cdx_register_controller(struct cdx_controller *cdx)
922 {
923 int ret;
924
925 ret = ida_alloc_range(&cdx_controller_ida, 0, MAX_CDX_CONTROLLERS - 1, GFP_KERNEL);
926 if (ret < 0) {
927 dev_err(cdx->dev,
928 "No free index available. Maximum controllers already registered\n");
929 cdx->id = (u8)MAX_CDX_CONTROLLERS;
930 return ret;
931 }
932
933 mutex_lock(&cdx_controller_lock);
934 cdx->id = ret;
935
936 /* Scan all the devices */
937 if (cdx->ops->scan)
938 cdx->ops->scan(cdx);
939 cdx->controller_registered = true;
940 mutex_unlock(&cdx_controller_lock);
941
942 return 0;
943 }
944 EXPORT_SYMBOL_NS_GPL(cdx_register_controller, "CDX_BUS_CONTROLLER");
945
cdx_unregister_controller(struct cdx_controller * cdx)946 void cdx_unregister_controller(struct cdx_controller *cdx)
947 {
948 if (cdx->id >= MAX_CDX_CONTROLLERS)
949 return;
950
951 mutex_lock(&cdx_controller_lock);
952
953 cdx->controller_registered = false;
954 device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
955 ida_free(&cdx_controller_ida, cdx->id);
956
957 mutex_unlock(&cdx_controller_lock);
958 }
959 EXPORT_SYMBOL_NS_GPL(cdx_unregister_controller, "CDX_BUS_CONTROLLER");
960
cdx_bus_init(void)961 static int __init cdx_bus_init(void)
962 {
963 int ret;
964
965 ret = bus_register(&cdx_bus_type);
966 if (!ret)
967 cdx_debugfs_dir = debugfs_create_dir(cdx_bus_type.name, NULL);
968
969 return ret;
970 }
971 postcore_initcall(cdx_bus_init);
972