1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/platform_device.h>
4 #include <linux/memregion.h>
5 #include <linux/workqueue.h>
6 #include <linux/debugfs.h>
7 #include <linux/device.h>
8 #include <linux/module.h>
9 #include <linux/pci.h>
10 #include <linux/slab.h>
11 #include <linux/idr.h>
12 #include <linux/node.h>
13 #include <cxl/einj.h>
14 #include <cxlmem.h>
15 #include <cxlpci.h>
16 #include <cxl.h>
17 #include "core.h"
18
19 /**
20 * DOC: cxl core
21 *
22 * The CXL core provides a set of interfaces that can be consumed by CXL aware
23 * drivers. The interfaces allow for creation, modification, and destruction of
24 * regions, memory devices, ports, and decoders. CXL aware drivers must register
25 * with the CXL core via these interfaces in order to be able to participate in
26 * cross-device interleave coordination. The CXL core also establishes and
27 * maintains the bridge to the nvdimm subsystem.
28 *
29 * CXL core introduces sysfs hierarchy to control the devices that are
30 * instantiated by the core.
31 */
32
33 static DEFINE_IDA(cxl_port_ida);
34 static DEFINE_XARRAY(cxl_root_buses);
35
36 /*
37 * The terminal device in PCI is NULL and @platform_bus
38 * for platform devices (for cxl_test)
39 */
is_cxl_host_bridge(struct device * dev)40 static bool is_cxl_host_bridge(struct device *dev)
41 {
42 return (!dev || dev == &platform_bus);
43 }
44
cxl_num_decoders_committed(struct cxl_port * port)45 int cxl_num_decoders_committed(struct cxl_port *port)
46 {
47 lockdep_assert_held(&cxl_rwsem.region);
48
49 return port->commit_end + 1;
50 }
51
devtype_show(struct device * dev,struct device_attribute * attr,char * buf)52 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
53 char *buf)
54 {
55 return sysfs_emit(buf, "%s\n", dev->type->name);
56 }
57 static DEVICE_ATTR_RO(devtype);
58
cxl_device_id(const struct device * dev)59 static int cxl_device_id(const struct device *dev)
60 {
61 if (dev->type == &cxl_nvdimm_bridge_type)
62 return CXL_DEVICE_NVDIMM_BRIDGE;
63 if (dev->type == &cxl_nvdimm_type)
64 return CXL_DEVICE_NVDIMM;
65 if (dev->type == CXL_PMEM_REGION_TYPE())
66 return CXL_DEVICE_PMEM_REGION;
67 if (dev->type == CXL_DAX_REGION_TYPE())
68 return CXL_DEVICE_DAX_REGION;
69 if (is_cxl_port(dev)) {
70 if (is_cxl_root(to_cxl_port(dev)))
71 return CXL_DEVICE_ROOT;
72 return CXL_DEVICE_PORT;
73 }
74 if (is_cxl_memdev(dev))
75 return CXL_DEVICE_MEMORY_EXPANDER;
76 if (dev->type == CXL_REGION_TYPE())
77 return CXL_DEVICE_REGION;
78 if (dev->type == &cxl_pmu_type)
79 return CXL_DEVICE_PMU;
80 return 0;
81 }
82
modalias_show(struct device * dev,struct device_attribute * attr,char * buf)83 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
84 char *buf)
85 {
86 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
87 }
88 static DEVICE_ATTR_RO(modalias);
89
90 static struct attribute *cxl_base_attributes[] = {
91 &dev_attr_devtype.attr,
92 &dev_attr_modalias.attr,
93 NULL,
94 };
95
96 struct attribute_group cxl_base_attribute_group = {
97 .attrs = cxl_base_attributes,
98 };
99
start_show(struct device * dev,struct device_attribute * attr,char * buf)100 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
101 char *buf)
102 {
103 struct cxl_decoder *cxld = to_cxl_decoder(dev);
104
105 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
106 }
107 static DEVICE_ATTR_ADMIN_RO(start);
108
size_show(struct device * dev,struct device_attribute * attr,char * buf)109 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
110 char *buf)
111 {
112 struct cxl_decoder *cxld = to_cxl_decoder(dev);
113
114 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
115 }
116 static DEVICE_ATTR_RO(size);
117
118 #define CXL_DECODER_FLAG_ATTR(name, flag) \
119 static ssize_t name##_show(struct device *dev, \
120 struct device_attribute *attr, char *buf) \
121 { \
122 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
123 \
124 return sysfs_emit(buf, "%s\n", \
125 (cxld->flags & (flag)) ? "1" : "0"); \
126 } \
127 static DEVICE_ATTR_RO(name)
128
129 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
130 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
131 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
132 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
133 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
134
target_type_show(struct device * dev,struct device_attribute * attr,char * buf)135 static ssize_t target_type_show(struct device *dev,
136 struct device_attribute *attr, char *buf)
137 {
138 struct cxl_decoder *cxld = to_cxl_decoder(dev);
139
140 switch (cxld->target_type) {
141 case CXL_DECODER_DEVMEM:
142 return sysfs_emit(buf, "accelerator\n");
143 case CXL_DECODER_HOSTONLYMEM:
144 return sysfs_emit(buf, "expander\n");
145 }
146 return -ENXIO;
147 }
148 static DEVICE_ATTR_RO(target_type);
149
emit_target_list(struct cxl_switch_decoder * cxlsd,char * buf)150 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
151 {
152 struct cxl_decoder *cxld = &cxlsd->cxld;
153 ssize_t offset = 0;
154 int i, rc = 0;
155
156 for (i = 0; i < cxld->interleave_ways; i++) {
157 struct cxl_dport *dport = cxlsd->target[i];
158 struct cxl_dport *next = NULL;
159
160 if (!dport)
161 break;
162
163 if (i + 1 < cxld->interleave_ways)
164 next = cxlsd->target[i + 1];
165 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
166 next ? "," : "");
167 if (rc < 0)
168 return rc;
169 offset += rc;
170 }
171
172 return offset;
173 }
174
target_list_show(struct device * dev,struct device_attribute * attr,char * buf)175 static ssize_t target_list_show(struct device *dev,
176 struct device_attribute *attr, char *buf)
177 {
178 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
179 ssize_t offset;
180 int rc;
181
182 guard(rwsem_read)(&cxl_rwsem.region);
183 rc = emit_target_list(cxlsd, buf);
184 if (rc < 0)
185 return rc;
186 offset = rc;
187
188 rc = sysfs_emit_at(buf, offset, "\n");
189 if (rc < 0)
190 return rc;
191
192 return offset + rc;
193 }
194 static DEVICE_ATTR_RO(target_list);
195
mode_show(struct device * dev,struct device_attribute * attr,char * buf)196 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
197 char *buf)
198 {
199 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
200 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
201 struct cxl_dev_state *cxlds = cxlmd->cxlds;
202 /* without @cxl_rwsem.dpa, make sure @part is not reloaded */
203 int part = READ_ONCE(cxled->part);
204 const char *desc;
205
206 if (part < 0)
207 desc = "none";
208 else
209 desc = cxlds->part[part].res.name;
210
211 return sysfs_emit(buf, "%s\n", desc);
212 }
213
mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)214 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
215 const char *buf, size_t len)
216 {
217 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
218 enum cxl_partition_mode mode;
219 ssize_t rc;
220
221 if (sysfs_streq(buf, "pmem"))
222 mode = CXL_PARTMODE_PMEM;
223 else if (sysfs_streq(buf, "ram"))
224 mode = CXL_PARTMODE_RAM;
225 else
226 return -EINVAL;
227
228 rc = cxl_dpa_set_part(cxled, mode);
229 if (rc)
230 return rc;
231
232 return len;
233 }
234 static DEVICE_ATTR_RW(mode);
235
dpa_resource_show(struct device * dev,struct device_attribute * attr,char * buf)236 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
237 char *buf)
238 {
239 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
240
241 guard(rwsem_read)(&cxl_rwsem.dpa);
242 return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
243 }
244 static DEVICE_ATTR_RO(dpa_resource);
245
dpa_size_show(struct device * dev,struct device_attribute * attr,char * buf)246 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
247 char *buf)
248 {
249 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
250 resource_size_t size = cxl_dpa_size(cxled);
251
252 return sysfs_emit(buf, "%pa\n", &size);
253 }
254
dpa_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)255 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
256 const char *buf, size_t len)
257 {
258 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
259 unsigned long long size;
260 ssize_t rc;
261
262 rc = kstrtoull(buf, 0, &size);
263 if (rc)
264 return rc;
265
266 if (!IS_ALIGNED(size, SZ_256M))
267 return -EINVAL;
268
269 rc = cxl_dpa_free(cxled);
270 if (rc)
271 return rc;
272
273 if (size == 0)
274 return len;
275
276 rc = cxl_dpa_alloc(cxled, size);
277 if (rc)
278 return rc;
279
280 return len;
281 }
282 static DEVICE_ATTR_RW(dpa_size);
283
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)284 static ssize_t interleave_granularity_show(struct device *dev,
285 struct device_attribute *attr,
286 char *buf)
287 {
288 struct cxl_decoder *cxld = to_cxl_decoder(dev);
289
290 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
291 }
292
293 static DEVICE_ATTR_RO(interleave_granularity);
294
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)295 static ssize_t interleave_ways_show(struct device *dev,
296 struct device_attribute *attr, char *buf)
297 {
298 struct cxl_decoder *cxld = to_cxl_decoder(dev);
299
300 return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
301 }
302
303 static DEVICE_ATTR_RO(interleave_ways);
304
qos_class_show(struct device * dev,struct device_attribute * attr,char * buf)305 static ssize_t qos_class_show(struct device *dev,
306 struct device_attribute *attr, char *buf)
307 {
308 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
309
310 return sysfs_emit(buf, "%d\n", cxlrd->qos_class);
311 }
312 static DEVICE_ATTR_RO(qos_class);
313
314 static struct attribute *cxl_decoder_base_attrs[] = {
315 &dev_attr_start.attr,
316 &dev_attr_size.attr,
317 &dev_attr_locked.attr,
318 &dev_attr_interleave_granularity.attr,
319 &dev_attr_interleave_ways.attr,
320 NULL,
321 };
322
323 static struct attribute_group cxl_decoder_base_attribute_group = {
324 .attrs = cxl_decoder_base_attrs,
325 };
326
327 static struct attribute *cxl_decoder_root_attrs[] = {
328 &dev_attr_cap_pmem.attr,
329 &dev_attr_cap_ram.attr,
330 &dev_attr_cap_type2.attr,
331 &dev_attr_cap_type3.attr,
332 &dev_attr_target_list.attr,
333 &dev_attr_qos_class.attr,
334 SET_CXL_REGION_ATTR(create_pmem_region)
335 SET_CXL_REGION_ATTR(create_ram_region)
336 SET_CXL_REGION_ATTR(delete_region)
337 NULL,
338 };
339
can_create_pmem(struct cxl_root_decoder * cxlrd)340 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
341 {
342 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
343
344 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
345 }
346
can_create_ram(struct cxl_root_decoder * cxlrd)347 static bool can_create_ram(struct cxl_root_decoder *cxlrd)
348 {
349 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
350
351 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
352 }
353
cxl_root_decoder_visible(struct kobject * kobj,struct attribute * a,int n)354 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
355 {
356 struct device *dev = kobj_to_dev(kobj);
357 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
358
359 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
360 return 0;
361
362 if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
363 return 0;
364
365 if (a == CXL_REGION_ATTR(delete_region) &&
366 !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
367 return 0;
368
369 return a->mode;
370 }
371
372 static struct attribute_group cxl_decoder_root_attribute_group = {
373 .attrs = cxl_decoder_root_attrs,
374 .is_visible = cxl_root_decoder_visible,
375 };
376
377 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
378 &cxl_decoder_root_attribute_group,
379 &cxl_decoder_base_attribute_group,
380 &cxl_base_attribute_group,
381 NULL,
382 };
383
384 static struct attribute *cxl_decoder_switch_attrs[] = {
385 &dev_attr_target_type.attr,
386 &dev_attr_target_list.attr,
387 SET_CXL_REGION_ATTR(region)
388 NULL,
389 };
390
391 static struct attribute_group cxl_decoder_switch_attribute_group = {
392 .attrs = cxl_decoder_switch_attrs,
393 };
394
395 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
396 &cxl_decoder_switch_attribute_group,
397 &cxl_decoder_base_attribute_group,
398 &cxl_base_attribute_group,
399 NULL,
400 };
401
402 static struct attribute *cxl_decoder_endpoint_attrs[] = {
403 &dev_attr_target_type.attr,
404 &dev_attr_mode.attr,
405 &dev_attr_dpa_size.attr,
406 &dev_attr_dpa_resource.attr,
407 SET_CXL_REGION_ATTR(region)
408 NULL,
409 };
410
411 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
412 .attrs = cxl_decoder_endpoint_attrs,
413 };
414
415 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
416 &cxl_decoder_base_attribute_group,
417 &cxl_decoder_endpoint_attribute_group,
418 &cxl_base_attribute_group,
419 NULL,
420 };
421
__cxl_decoder_release(struct cxl_decoder * cxld)422 static void __cxl_decoder_release(struct cxl_decoder *cxld)
423 {
424 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
425
426 ida_free(&port->decoder_ida, cxld->id);
427 put_device(&port->dev);
428 }
429
cxl_endpoint_decoder_release(struct device * dev)430 static void cxl_endpoint_decoder_release(struct device *dev)
431 {
432 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
433
434 __cxl_decoder_release(&cxled->cxld);
435 kfree(cxled);
436 }
437
cxl_switch_decoder_release(struct device * dev)438 static void cxl_switch_decoder_release(struct device *dev)
439 {
440 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
441
442 __cxl_decoder_release(&cxlsd->cxld);
443 kfree(cxlsd);
444 }
445
to_cxl_root_decoder(struct device * dev)446 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
447 {
448 if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
449 "not a cxl_root_decoder device\n"))
450 return NULL;
451 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
452 }
453 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL");
454
cxl_root_decoder_release(struct device * dev)455 static void cxl_root_decoder_release(struct device *dev)
456 {
457 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
458
459 if (atomic_read(&cxlrd->region_id) >= 0)
460 memregion_free(atomic_read(&cxlrd->region_id));
461 __cxl_decoder_release(&cxlrd->cxlsd.cxld);
462 kfree(cxlrd->ops);
463 kfree(cxlrd);
464 }
465
466 static const struct device_type cxl_decoder_endpoint_type = {
467 .name = "cxl_decoder_endpoint",
468 .release = cxl_endpoint_decoder_release,
469 .groups = cxl_decoder_endpoint_attribute_groups,
470 };
471
472 static const struct device_type cxl_decoder_switch_type = {
473 .name = "cxl_decoder_switch",
474 .release = cxl_switch_decoder_release,
475 .groups = cxl_decoder_switch_attribute_groups,
476 };
477
478 static const struct device_type cxl_decoder_root_type = {
479 .name = "cxl_decoder_root",
480 .release = cxl_root_decoder_release,
481 .groups = cxl_decoder_root_attribute_groups,
482 };
483
is_endpoint_decoder(struct device * dev)484 bool is_endpoint_decoder(struct device *dev)
485 {
486 return dev->type == &cxl_decoder_endpoint_type;
487 }
488 EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL");
489
is_root_decoder(struct device * dev)490 bool is_root_decoder(struct device *dev)
491 {
492 return dev->type == &cxl_decoder_root_type;
493 }
494 EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL");
495
is_switch_decoder(struct device * dev)496 bool is_switch_decoder(struct device *dev)
497 {
498 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
499 }
500 EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL");
501
to_cxl_decoder(struct device * dev)502 struct cxl_decoder *to_cxl_decoder(struct device *dev)
503 {
504 if (dev_WARN_ONCE(dev,
505 !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
506 "not a cxl_decoder device\n"))
507 return NULL;
508 return container_of(dev, struct cxl_decoder, dev);
509 }
510 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL");
511
to_cxl_endpoint_decoder(struct device * dev)512 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
513 {
514 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
515 "not a cxl_endpoint_decoder device\n"))
516 return NULL;
517 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
518 }
519 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL");
520
to_cxl_switch_decoder(struct device * dev)521 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
522 {
523 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
524 "not a cxl_switch_decoder device\n"))
525 return NULL;
526 return container_of(dev, struct cxl_switch_decoder, cxld.dev);
527 }
528 EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
529
cxl_ep_release(struct cxl_ep * ep)530 static void cxl_ep_release(struct cxl_ep *ep)
531 {
532 put_device(ep->ep);
533 kfree(ep);
534 }
535
cxl_ep_remove(struct cxl_port * port,struct cxl_ep * ep)536 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
537 {
538 if (!ep)
539 return;
540 xa_erase(&port->endpoints, (unsigned long) ep->ep);
541 cxl_ep_release(ep);
542 }
543
cxl_port_release(struct device * dev)544 static void cxl_port_release(struct device *dev)
545 {
546 struct cxl_port *port = to_cxl_port(dev);
547 unsigned long index;
548 struct cxl_ep *ep;
549
550 xa_for_each(&port->endpoints, index, ep)
551 cxl_ep_remove(port, ep);
552 xa_destroy(&port->endpoints);
553 xa_destroy(&port->dports);
554 xa_destroy(&port->regions);
555 ida_free(&cxl_port_ida, port->id);
556 if (is_cxl_root(port))
557 kfree(to_cxl_root(port));
558 else
559 kfree(port);
560 }
561
decoders_committed_show(struct device * dev,struct device_attribute * attr,char * buf)562 static ssize_t decoders_committed_show(struct device *dev,
563 struct device_attribute *attr, char *buf)
564 {
565 struct cxl_port *port = to_cxl_port(dev);
566
567 guard(rwsem_read)(&cxl_rwsem.region);
568 return sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
569 }
570
571 static DEVICE_ATTR_RO(decoders_committed);
572
573 static struct attribute *cxl_port_attrs[] = {
574 &dev_attr_decoders_committed.attr,
575 NULL,
576 };
577
578 static struct attribute_group cxl_port_attribute_group = {
579 .attrs = cxl_port_attrs,
580 };
581
582 static const struct attribute_group *cxl_port_attribute_groups[] = {
583 &cxl_base_attribute_group,
584 &cxl_port_attribute_group,
585 NULL,
586 };
587
588 static const struct device_type cxl_port_type = {
589 .name = "cxl_port",
590 .release = cxl_port_release,
591 .groups = cxl_port_attribute_groups,
592 };
593
is_cxl_port(const struct device * dev)594 bool is_cxl_port(const struct device *dev)
595 {
596 return dev->type == &cxl_port_type;
597 }
598 EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL");
599
to_cxl_port(const struct device * dev)600 struct cxl_port *to_cxl_port(const struct device *dev)
601 {
602 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
603 "not a cxl_port device\n"))
604 return NULL;
605 return container_of(dev, struct cxl_port, dev);
606 }
607 EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL");
608
parent_port_of(struct cxl_port * port)609 struct cxl_port *parent_port_of(struct cxl_port *port)
610 {
611 if (!port || !port->parent_dport)
612 return NULL;
613 return port->parent_dport->port;
614 }
615
unregister_port(void * _port)616 static void unregister_port(void *_port)
617 {
618 struct cxl_port *port = _port;
619 struct cxl_port *parent = parent_port_of(port);
620 struct device *lock_dev;
621
622 /*
623 * CXL root port's and the first level of ports are unregistered
624 * under the platform firmware device lock, all other ports are
625 * unregistered while holding their parent port lock.
626 */
627 if (!parent)
628 lock_dev = port->uport_dev;
629 else if (is_cxl_root(parent))
630 lock_dev = parent->uport_dev;
631 else
632 lock_dev = &parent->dev;
633
634 device_lock_assert(lock_dev);
635 port->dead = true;
636 device_unregister(&port->dev);
637 }
638
cxl_unlink_uport(void * _port)639 static void cxl_unlink_uport(void *_port)
640 {
641 struct cxl_port *port = _port;
642
643 sysfs_remove_link(&port->dev.kobj, "uport");
644 }
645
devm_cxl_link_uport(struct device * host,struct cxl_port * port)646 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
647 {
648 int rc;
649
650 rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
651 "uport");
652 if (rc)
653 return rc;
654 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
655 }
656
cxl_unlink_parent_dport(void * _port)657 static void cxl_unlink_parent_dport(void *_port)
658 {
659 struct cxl_port *port = _port;
660
661 sysfs_remove_link(&port->dev.kobj, "parent_dport");
662 }
663
devm_cxl_link_parent_dport(struct device * host,struct cxl_port * port,struct cxl_dport * parent_dport)664 static int devm_cxl_link_parent_dport(struct device *host,
665 struct cxl_port *port,
666 struct cxl_dport *parent_dport)
667 {
668 int rc;
669
670 if (!parent_dport)
671 return 0;
672
673 rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
674 "parent_dport");
675 if (rc)
676 return rc;
677 return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
678 }
679
680 static struct lock_class_key cxl_port_key;
681
cxl_port_alloc(struct device * uport_dev,struct cxl_dport * parent_dport)682 static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
683 struct cxl_dport *parent_dport)
684 {
685 struct cxl_root *cxl_root __free(kfree) = NULL;
686 struct cxl_port *port, *_port __free(kfree) = NULL;
687 struct device *dev;
688 int rc;
689
690 /* No parent_dport, root cxl_port */
691 if (!parent_dport) {
692 cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
693 if (!cxl_root)
694 return ERR_PTR(-ENOMEM);
695 } else {
696 _port = kzalloc(sizeof(*port), GFP_KERNEL);
697 if (!_port)
698 return ERR_PTR(-ENOMEM);
699 }
700
701 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
702 if (rc < 0)
703 return ERR_PTR(rc);
704
705 if (cxl_root)
706 port = &no_free_ptr(cxl_root)->port;
707 else
708 port = no_free_ptr(_port);
709
710 port->id = rc;
711 port->uport_dev = uport_dev;
712
713 /*
714 * The top-level cxl_port "cxl_root" does not have a cxl_port as
715 * its parent and it does not have any corresponding component
716 * registers as its decode is described by a fixed platform
717 * description.
718 */
719 dev = &port->dev;
720 if (parent_dport) {
721 struct cxl_port *parent_port = parent_dport->port;
722 struct cxl_port *iter;
723
724 dev->parent = &parent_port->dev;
725 port->depth = parent_port->depth + 1;
726 port->parent_dport = parent_dport;
727
728 /*
729 * walk to the host bridge, or the first ancestor that knows
730 * the host bridge
731 */
732 iter = port;
733 while (!iter->host_bridge &&
734 !is_cxl_root(to_cxl_port(iter->dev.parent)))
735 iter = to_cxl_port(iter->dev.parent);
736 if (iter->host_bridge)
737 port->host_bridge = iter->host_bridge;
738 else if (parent_dport->rch)
739 port->host_bridge = parent_dport->dport_dev;
740 else
741 port->host_bridge = iter->uport_dev;
742 dev_dbg(uport_dev, "host-bridge: %s\n",
743 dev_name(port->host_bridge));
744 } else
745 dev->parent = uport_dev;
746
747 ida_init(&port->decoder_ida);
748 port->hdm_end = -1;
749 port->commit_end = -1;
750 xa_init(&port->dports);
751 xa_init(&port->endpoints);
752 xa_init(&port->regions);
753 port->component_reg_phys = CXL_RESOURCE_NONE;
754
755 device_initialize(dev);
756 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
757 device_set_pm_not_required(dev);
758 dev->bus = &cxl_bus_type;
759 dev->type = &cxl_port_type;
760
761 return port;
762 }
763
cxl_setup_comp_regs(struct device * host,struct cxl_register_map * map,resource_size_t component_reg_phys)764 static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
765 resource_size_t component_reg_phys)
766 {
767 *map = (struct cxl_register_map) {
768 .host = host,
769 .reg_type = CXL_REGLOC_RBI_EMPTY,
770 .resource = component_reg_phys,
771 };
772
773 if (component_reg_phys == CXL_RESOURCE_NONE)
774 return 0;
775
776 map->reg_type = CXL_REGLOC_RBI_COMPONENT;
777 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
778
779 return cxl_setup_regs(map);
780 }
781
cxl_port_setup_regs(struct cxl_port * port,resource_size_t component_reg_phys)782 static int cxl_port_setup_regs(struct cxl_port *port,
783 resource_size_t component_reg_phys)
784 {
785 if (dev_is_platform(port->uport_dev))
786 return 0;
787 return cxl_setup_comp_regs(&port->dev, &port->reg_map,
788 component_reg_phys);
789 }
790
cxl_dport_setup_regs(struct device * host,struct cxl_dport * dport,resource_size_t component_reg_phys)791 static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
792 resource_size_t component_reg_phys)
793 {
794 int rc;
795
796 if (dev_is_platform(dport->dport_dev))
797 return 0;
798
799 /*
800 * use @dport->dport_dev for the context for error messages during
801 * register probing, and fixup @host after the fact, since @host may be
802 * NULL.
803 */
804 rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
805 component_reg_phys);
806 dport->reg_map.host = host;
807 return rc;
808 }
809
810 DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
811
cxl_einj_inject(void * data,u64 type)812 static int cxl_einj_inject(void *data, u64 type)
813 {
814 struct cxl_dport *dport = data;
815
816 if (dport->rch)
817 return einj_cxl_inject_rch_error(dport->rcrb.base, type);
818
819 return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type);
820 }
821 DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
822 "0x%llx\n");
823
cxl_debugfs_create_dport_dir(struct cxl_dport * dport)824 static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
825 {
826 struct dentry *dir;
827
828 if (!einj_cxl_is_initialized())
829 return;
830
831 /*
832 * dport_dev needs to be a PCIe port for CXL 2.0+ ports because
833 * EINJ expects a dport SBDF to be specified for 2.0 error injection.
834 */
835 if (!dport->rch && !dev_is_pci(dport->dport_dev))
836 return;
837
838 dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
839
840 debugfs_create_file("einj_inject", 0200, dir, dport,
841 &cxl_einj_inject_fops);
842 }
843
cxl_port_add(struct cxl_port * port,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)844 static int cxl_port_add(struct cxl_port *port,
845 resource_size_t component_reg_phys,
846 struct cxl_dport *parent_dport)
847 {
848 struct device *dev __free(put_device) = &port->dev;
849 int rc;
850
851 if (is_cxl_memdev(port->uport_dev)) {
852 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
853 struct cxl_dev_state *cxlds = cxlmd->cxlds;
854
855 rc = dev_set_name(dev, "endpoint%d", port->id);
856 if (rc)
857 return rc;
858
859 /*
860 * The endpoint driver already enumerated the component and RAS
861 * registers. Reuse that enumeration while prepping them to be
862 * mapped by the cxl_port driver.
863 */
864 port->reg_map = cxlds->reg_map;
865 port->reg_map.host = &port->dev;
866 cxlmd->endpoint = port;
867 } else if (parent_dport) {
868 rc = dev_set_name(dev, "port%d", port->id);
869 if (rc)
870 return rc;
871
872 port->component_reg_phys = component_reg_phys;
873 } else {
874 rc = dev_set_name(dev, "root%d", port->id);
875 if (rc)
876 return rc;
877 }
878
879 rc = device_add(dev);
880 if (rc)
881 return rc;
882
883 /* Inhibit the cleanup function invoked */
884 dev = NULL;
885 return 0;
886 }
887
__devm_cxl_add_port(struct device * host,struct device * uport_dev,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)888 static struct cxl_port *__devm_cxl_add_port(struct device *host,
889 struct device *uport_dev,
890 resource_size_t component_reg_phys,
891 struct cxl_dport *parent_dport)
892 {
893 struct cxl_port *port;
894 int rc;
895
896 port = cxl_port_alloc(uport_dev, parent_dport);
897 if (IS_ERR(port))
898 return port;
899
900 rc = cxl_port_add(port, component_reg_phys, parent_dport);
901 if (rc)
902 return ERR_PTR(rc);
903
904 rc = devm_add_action_or_reset(host, unregister_port, port);
905 if (rc)
906 return ERR_PTR(rc);
907
908 rc = devm_cxl_link_uport(host, port);
909 if (rc)
910 return ERR_PTR(rc);
911
912 rc = devm_cxl_link_parent_dport(host, port, parent_dport);
913 if (rc)
914 return ERR_PTR(rc);
915
916 if (parent_dport && dev_is_pci(uport_dev))
917 port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
918
919 return port;
920 }
921
922 /**
923 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
924 * @host: host device for devm operations
925 * @uport_dev: "physical" device implementing this upstream port
926 * @component_reg_phys: (optional) for configurable cxl_port instances
927 * @parent_dport: next hop up in the CXL memory decode hierarchy
928 */
devm_cxl_add_port(struct device * host,struct device * uport_dev,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)929 struct cxl_port *devm_cxl_add_port(struct device *host,
930 struct device *uport_dev,
931 resource_size_t component_reg_phys,
932 struct cxl_dport *parent_dport)
933 {
934 struct cxl_port *port, *parent_port;
935
936 port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
937 parent_dport);
938
939 parent_port = parent_dport ? parent_dport->port : NULL;
940 if (IS_ERR(port)) {
941 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
942 parent_port ? " port to " : "",
943 parent_port ? dev_name(&parent_port->dev) : "",
944 parent_port ? "" : " root port",
945 PTR_ERR(port));
946 } else {
947 dev_dbg(uport_dev, "%s added%s%s%s\n",
948 dev_name(&port->dev),
949 parent_port ? " to " : "",
950 parent_port ? dev_name(&parent_port->dev) : "",
951 parent_port ? "" : " (root port)");
952 }
953
954 return port;
955 }
956 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL");
957
devm_cxl_add_root(struct device * host,const struct cxl_root_ops * ops)958 struct cxl_root *devm_cxl_add_root(struct device *host,
959 const struct cxl_root_ops *ops)
960 {
961 struct cxl_root *cxl_root;
962 struct cxl_port *port;
963
964 port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
965 if (IS_ERR(port))
966 return ERR_CAST(port);
967
968 cxl_root = to_cxl_root(port);
969 cxl_root->ops = ops;
970 return cxl_root;
971 }
972 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL");
973
cxl_port_to_pci_bus(struct cxl_port * port)974 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
975 {
976 /* There is no pci_bus associated with a CXL platform-root port */
977 if (is_cxl_root(port))
978 return NULL;
979
980 if (dev_is_pci(port->uport_dev)) {
981 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
982
983 return pdev->subordinate;
984 }
985
986 return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
987 }
988 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL");
989
unregister_pci_bus(void * uport_dev)990 static void unregister_pci_bus(void *uport_dev)
991 {
992 xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
993 }
994
devm_cxl_register_pci_bus(struct device * host,struct device * uport_dev,struct pci_bus * bus)995 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
996 struct pci_bus *bus)
997 {
998 int rc;
999
1000 if (dev_is_pci(uport_dev))
1001 return -EINVAL;
1002
1003 rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
1004 GFP_KERNEL);
1005 if (rc)
1006 return rc;
1007 return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
1008 }
1009 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL");
1010
dev_is_cxl_root_child(struct device * dev)1011 static bool dev_is_cxl_root_child(struct device *dev)
1012 {
1013 struct cxl_port *port, *parent;
1014
1015 if (!is_cxl_port(dev))
1016 return false;
1017
1018 port = to_cxl_port(dev);
1019 if (is_cxl_root(port))
1020 return false;
1021
1022 parent = to_cxl_port(port->dev.parent);
1023 if (is_cxl_root(parent))
1024 return true;
1025
1026 return false;
1027 }
1028
find_cxl_root(struct cxl_port * port)1029 struct cxl_root *find_cxl_root(struct cxl_port *port)
1030 {
1031 struct cxl_port *iter = port;
1032
1033 while (iter && !is_cxl_root(iter))
1034 iter = to_cxl_port(iter->dev.parent);
1035
1036 if (!iter)
1037 return NULL;
1038 get_device(&iter->dev);
1039 return to_cxl_root(iter);
1040 }
1041 EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL");
1042
find_dport(struct cxl_port * port,int id)1043 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
1044 {
1045 struct cxl_dport *dport;
1046 unsigned long index;
1047
1048 device_lock_assert(&port->dev);
1049 xa_for_each(&port->dports, index, dport)
1050 if (dport->port_id == id)
1051 return dport;
1052 return NULL;
1053 }
1054
add_dport(struct cxl_port * port,struct cxl_dport * dport)1055 static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1056 {
1057 struct cxl_dport *dup;
1058 int rc;
1059
1060 device_lock_assert(&port->dev);
1061 dup = find_dport(port, dport->port_id);
1062 if (dup) {
1063 dev_err(&port->dev,
1064 "unable to add dport%d-%s non-unique port id (%s)\n",
1065 dport->port_id, dev_name(dport->dport_dev),
1066 dev_name(dup->dport_dev));
1067 return -EBUSY;
1068 }
1069
1070 rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
1071 GFP_KERNEL);
1072 if (rc)
1073 return rc;
1074
1075 port->nr_dports++;
1076 return 0;
1077 }
1078
1079 /*
1080 * Since root-level CXL dports cannot be enumerated by PCI they are not
1081 * enumerated by the common port driver that acquires the port lock over
1082 * dport add/remove. Instead, root dports are manually added by a
1083 * platform driver and cond_cxl_root_lock() is used to take the missing
1084 * port lock in that case.
1085 */
cond_cxl_root_lock(struct cxl_port * port)1086 static void cond_cxl_root_lock(struct cxl_port *port)
1087 {
1088 if (is_cxl_root(port))
1089 device_lock(&port->dev);
1090 }
1091
cond_cxl_root_unlock(struct cxl_port * port)1092 static void cond_cxl_root_unlock(struct cxl_port *port)
1093 {
1094 if (is_cxl_root(port))
1095 device_unlock(&port->dev);
1096 }
1097
cxl_dport_remove(void * data)1098 static void cxl_dport_remove(void *data)
1099 {
1100 struct cxl_dport *dport = data;
1101 struct cxl_port *port = dport->port;
1102
1103 xa_erase(&port->dports, (unsigned long) dport->dport_dev);
1104 put_device(dport->dport_dev);
1105 }
1106
cxl_dport_unlink(void * data)1107 static void cxl_dport_unlink(void *data)
1108 {
1109 struct cxl_dport *dport = data;
1110 struct cxl_port *port = dport->port;
1111 char link_name[CXL_TARGET_STRLEN];
1112
1113 sprintf(link_name, "dport%d", dport->port_id);
1114 sysfs_remove_link(&port->dev.kobj, link_name);
1115 }
1116
1117 static struct cxl_dport *
__devm_cxl_add_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t component_reg_phys,resource_size_t rcrb)1118 __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1119 int port_id, resource_size_t component_reg_phys,
1120 resource_size_t rcrb)
1121 {
1122 char link_name[CXL_TARGET_STRLEN];
1123 struct cxl_dport *dport;
1124 struct device *host;
1125 int rc;
1126
1127 if (is_cxl_root(port))
1128 host = port->uport_dev;
1129 else
1130 host = &port->dev;
1131
1132 if (!host->driver) {
1133 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1134 dev_name(dport_dev));
1135 return ERR_PTR(-ENXIO);
1136 }
1137
1138 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
1139 CXL_TARGET_STRLEN)
1140 return ERR_PTR(-EINVAL);
1141
1142 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1143 if (!dport)
1144 return ERR_PTR(-ENOMEM);
1145
1146 dport->dport_dev = dport_dev;
1147 dport->port_id = port_id;
1148 dport->port = port;
1149
1150 if (rcrb == CXL_RESOURCE_NONE) {
1151 rc = cxl_dport_setup_regs(&port->dev, dport,
1152 component_reg_phys);
1153 if (rc)
1154 return ERR_PTR(rc);
1155 } else {
1156 dport->rcrb.base = rcrb;
1157 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1158 CXL_RCRB_DOWNSTREAM);
1159 if (component_reg_phys == CXL_RESOURCE_NONE) {
1160 dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1161 return ERR_PTR(-ENXIO);
1162 }
1163
1164 /*
1165 * RCH @dport is not ready to map until associated with its
1166 * memdev
1167 */
1168 rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1169 if (rc)
1170 return ERR_PTR(rc);
1171
1172 dport->rch = true;
1173 }
1174
1175 if (component_reg_phys != CXL_RESOURCE_NONE)
1176 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1177 &component_reg_phys);
1178
1179 cond_cxl_root_lock(port);
1180 rc = add_dport(port, dport);
1181 cond_cxl_root_unlock(port);
1182 if (rc)
1183 return ERR_PTR(rc);
1184
1185 /*
1186 * Setup port register if this is the first dport showed up. Having
1187 * a dport also means that there is at least 1 active link.
1188 */
1189 if (port->nr_dports == 1 &&
1190 port->component_reg_phys != CXL_RESOURCE_NONE) {
1191 rc = cxl_port_setup_regs(port, port->component_reg_phys);
1192 if (rc) {
1193 xa_erase(&port->dports, (unsigned long)dport->dport_dev);
1194 return ERR_PTR(rc);
1195 }
1196 port->component_reg_phys = CXL_RESOURCE_NONE;
1197 }
1198
1199 get_device(dport_dev);
1200 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1201 if (rc)
1202 return ERR_PTR(rc);
1203
1204 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1205 if (rc)
1206 return ERR_PTR(rc);
1207
1208 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1209 if (rc)
1210 return ERR_PTR(rc);
1211
1212 if (dev_is_pci(dport_dev))
1213 dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1214
1215 cxl_debugfs_create_dport_dir(dport);
1216
1217 return dport;
1218 }
1219
1220 /**
1221 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1222 * @port: the cxl_port that references this dport
1223 * @dport_dev: firmware or PCI device representing the dport
1224 * @port_id: identifier for this dport in a decoder's target list
1225 * @component_reg_phys: optional location of CXL component registers
1226 *
1227 * Note that dports are appended to the devm release action's of the
1228 * either the port's host (for root ports), or the port itself (for
1229 * switch ports)
1230 */
devm_cxl_add_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t component_reg_phys)1231 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1232 struct device *dport_dev, int port_id,
1233 resource_size_t component_reg_phys)
1234 {
1235 struct cxl_dport *dport;
1236
1237 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1238 component_reg_phys, CXL_RESOURCE_NONE);
1239 if (IS_ERR(dport)) {
1240 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1241 dev_name(&port->dev), PTR_ERR(dport));
1242 } else {
1243 dev_dbg(dport_dev, "dport added to %s\n",
1244 dev_name(&port->dev));
1245 }
1246
1247 return dport;
1248 }
1249 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL");
1250
1251 /**
1252 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1253 * @port: the cxl_port that references this dport
1254 * @dport_dev: firmware or PCI device representing the dport
1255 * @port_id: identifier for this dport in a decoder's target list
1256 * @rcrb: mandatory location of a Root Complex Register Block
1257 *
1258 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1259 */
devm_cxl_add_rch_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t rcrb)1260 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1261 struct device *dport_dev, int port_id,
1262 resource_size_t rcrb)
1263 {
1264 struct cxl_dport *dport;
1265
1266 if (rcrb == CXL_RESOURCE_NONE) {
1267 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1268 return ERR_PTR(-EINVAL);
1269 }
1270
1271 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1272 CXL_RESOURCE_NONE, rcrb);
1273 if (IS_ERR(dport)) {
1274 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1275 dev_name(&port->dev), PTR_ERR(dport));
1276 } else {
1277 dev_dbg(dport_dev, "RCH dport added to %s\n",
1278 dev_name(&port->dev));
1279 }
1280
1281 return dport;
1282 }
1283 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL");
1284
add_ep(struct cxl_ep * new)1285 static int add_ep(struct cxl_ep *new)
1286 {
1287 struct cxl_port *port = new->dport->port;
1288
1289 guard(device)(&port->dev);
1290 if (port->dead)
1291 return -ENXIO;
1292
1293 return xa_insert(&port->endpoints, (unsigned long)new->ep,
1294 new, GFP_KERNEL);
1295 }
1296
1297 /**
1298 * cxl_add_ep - register an endpoint's interest in a port
1299 * @dport: the dport that routes to @ep_dev
1300 * @ep_dev: device representing the endpoint
1301 *
1302 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1303 * When those endpoints depart the port can be destroyed once all
1304 * endpoints that care about that port have been removed.
1305 */
cxl_add_ep(struct cxl_dport * dport,struct device * ep_dev)1306 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1307 {
1308 struct cxl_ep *ep;
1309 int rc;
1310
1311 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1312 if (!ep)
1313 return -ENOMEM;
1314
1315 ep->ep = get_device(ep_dev);
1316 ep->dport = dport;
1317
1318 rc = add_ep(ep);
1319 if (rc)
1320 cxl_ep_release(ep);
1321 return rc;
1322 }
1323
1324 struct cxl_find_port_ctx {
1325 const struct device *dport_dev;
1326 const struct cxl_port *parent_port;
1327 struct cxl_dport **dport;
1328 };
1329
match_port_by_dport(struct device * dev,const void * data)1330 static int match_port_by_dport(struct device *dev, const void *data)
1331 {
1332 const struct cxl_find_port_ctx *ctx = data;
1333 struct cxl_dport *dport;
1334 struct cxl_port *port;
1335
1336 if (!is_cxl_port(dev))
1337 return 0;
1338 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1339 return 0;
1340
1341 port = to_cxl_port(dev);
1342 dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1343 if (ctx->dport)
1344 *ctx->dport = dport;
1345 return dport != NULL;
1346 }
1347
__find_cxl_port(struct cxl_find_port_ctx * ctx)1348 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1349 {
1350 struct device *dev;
1351
1352 if (!ctx->dport_dev)
1353 return NULL;
1354
1355 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1356 if (dev)
1357 return to_cxl_port(dev);
1358 return NULL;
1359 }
1360
find_cxl_port(struct device * dport_dev,struct cxl_dport ** dport)1361 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1362 struct cxl_dport **dport)
1363 {
1364 struct cxl_find_port_ctx ctx = {
1365 .dport_dev = dport_dev,
1366 .dport = dport,
1367 };
1368 struct cxl_port *port;
1369
1370 port = __find_cxl_port(&ctx);
1371 return port;
1372 }
1373
1374 /*
1375 * All users of grandparent() are using it to walk PCIe-like switch port
1376 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1377 * upstream switch port and N bridges representing downstream switch ports. When
1378 * bridges stack the grand-parent of a downstream switch port is another
1379 * downstream switch port in the immediate ancestor switch.
1380 */
grandparent(struct device * dev)1381 static struct device *grandparent(struct device *dev)
1382 {
1383 if (dev && dev->parent)
1384 return dev->parent->parent;
1385 return NULL;
1386 }
1387
endpoint_host(struct cxl_port * endpoint)1388 static struct device *endpoint_host(struct cxl_port *endpoint)
1389 {
1390 struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1391
1392 if (is_cxl_root(port))
1393 return port->uport_dev;
1394 return &port->dev;
1395 }
1396
delete_endpoint(void * data)1397 static void delete_endpoint(void *data)
1398 {
1399 struct cxl_memdev *cxlmd = data;
1400 struct cxl_port *endpoint = cxlmd->endpoint;
1401 struct device *host = endpoint_host(endpoint);
1402
1403 scoped_guard(device, host) {
1404 if (host->driver && !endpoint->dead) {
1405 devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1406 devm_release_action(host, cxl_unlink_uport, endpoint);
1407 devm_release_action(host, unregister_port, endpoint);
1408 }
1409 cxlmd->endpoint = NULL;
1410 }
1411 put_device(&endpoint->dev);
1412 put_device(host);
1413 }
1414
cxl_endpoint_autoremove(struct cxl_memdev * cxlmd,struct cxl_port * endpoint)1415 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1416 {
1417 struct device *host = endpoint_host(endpoint);
1418 struct device *dev = &cxlmd->dev;
1419
1420 get_device(host);
1421 get_device(&endpoint->dev);
1422 cxlmd->depth = endpoint->depth;
1423 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1424 }
1425 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
1426
1427 /*
1428 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1429 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1430 * for a port to be unregistered is when all memdevs beneath that port have gone
1431 * through ->remove(). This "bottom-up" removal selectively removes individual
1432 * child ports manually. This depends on devm_cxl_add_port() to not change is
1433 * devm action registration order, and for dports to have already been
1434 * destroyed by del_dports().
1435 */
delete_switch_port(struct cxl_port * port)1436 static void delete_switch_port(struct cxl_port *port)
1437 {
1438 devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1439 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1440 devm_release_action(port->dev.parent, unregister_port, port);
1441 }
1442
del_dport(struct cxl_dport * dport)1443 static void del_dport(struct cxl_dport *dport)
1444 {
1445 struct cxl_port *port = dport->port;
1446
1447 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1448 devm_release_action(&port->dev, cxl_dport_remove, dport);
1449 devm_kfree(&port->dev, dport);
1450 }
1451
del_dports(struct cxl_port * port)1452 static void del_dports(struct cxl_port *port)
1453 {
1454 struct cxl_dport *dport;
1455 unsigned long index;
1456
1457 device_lock_assert(&port->dev);
1458
1459 xa_for_each(&port->dports, index, dport)
1460 del_dport(dport);
1461 }
1462
1463 struct detach_ctx {
1464 struct cxl_memdev *cxlmd;
1465 int depth;
1466 };
1467
port_has_memdev(struct device * dev,const void * data)1468 static int port_has_memdev(struct device *dev, const void *data)
1469 {
1470 const struct detach_ctx *ctx = data;
1471 struct cxl_port *port;
1472
1473 if (!is_cxl_port(dev))
1474 return 0;
1475
1476 port = to_cxl_port(dev);
1477 if (port->depth != ctx->depth)
1478 return 0;
1479
1480 return !!cxl_ep_load(port, ctx->cxlmd);
1481 }
1482
cxl_detach_ep(void * data)1483 static void cxl_detach_ep(void *data)
1484 {
1485 struct cxl_memdev *cxlmd = data;
1486
1487 for (int i = cxlmd->depth - 1; i >= 1; i--) {
1488 struct cxl_port *port, *parent_port;
1489 struct detach_ctx ctx = {
1490 .cxlmd = cxlmd,
1491 .depth = i,
1492 };
1493 struct cxl_ep *ep;
1494 bool died = false;
1495
1496 struct device *dev __free(put_device) =
1497 bus_find_device(&cxl_bus_type, NULL, &ctx, port_has_memdev);
1498 if (!dev)
1499 continue;
1500 port = to_cxl_port(dev);
1501
1502 parent_port = to_cxl_port(port->dev.parent);
1503 device_lock(&parent_port->dev);
1504 device_lock(&port->dev);
1505 ep = cxl_ep_load(port, cxlmd);
1506 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1507 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1508 cxl_ep_remove(port, ep);
1509 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1510 !is_cxl_root(parent_port) && parent_port->dev.driver) {
1511 /*
1512 * This was the last ep attached to a dynamically
1513 * enumerated port. Block new cxl_add_ep() and garbage
1514 * collect the port.
1515 */
1516 died = true;
1517 port->dead = true;
1518 del_dports(port);
1519 }
1520 device_unlock(&port->dev);
1521
1522 if (died) {
1523 dev_dbg(&cxlmd->dev, "delete %s\n",
1524 dev_name(&port->dev));
1525 delete_switch_port(port);
1526 }
1527 device_unlock(&parent_port->dev);
1528 }
1529 }
1530
find_component_registers(struct device * dev)1531 static resource_size_t find_component_registers(struct device *dev)
1532 {
1533 struct cxl_register_map map;
1534 struct pci_dev *pdev;
1535
1536 /*
1537 * Theoretically, CXL component registers can be hosted on a
1538 * non-PCI device, in practice, only cxl_test hits this case.
1539 */
1540 if (!dev_is_pci(dev))
1541 return CXL_RESOURCE_NONE;
1542
1543 pdev = to_pci_dev(dev);
1544
1545 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1546 return map.resource;
1547 }
1548
match_port_by_uport(struct device * dev,const void * data)1549 static int match_port_by_uport(struct device *dev, const void *data)
1550 {
1551 const struct device *uport_dev = data;
1552 struct cxl_port *port;
1553
1554 if (!is_cxl_port(dev))
1555 return 0;
1556
1557 port = to_cxl_port(dev);
1558 return uport_dev == port->uport_dev;
1559 }
1560
1561 /*
1562 * Function takes a device reference on the port device. Caller should do a
1563 * put_device() when done.
1564 */
find_cxl_port_by_uport(struct device * uport_dev)1565 static struct cxl_port *find_cxl_port_by_uport(struct device *uport_dev)
1566 {
1567 struct device *dev;
1568
1569 dev = bus_find_device(&cxl_bus_type, NULL, uport_dev, match_port_by_uport);
1570 if (dev)
1571 return to_cxl_port(dev);
1572 return NULL;
1573 }
1574
update_decoder_targets(struct device * dev,void * data)1575 static int update_decoder_targets(struct device *dev, void *data)
1576 {
1577 struct cxl_dport *dport = data;
1578 struct cxl_switch_decoder *cxlsd;
1579 struct cxl_decoder *cxld;
1580 int i;
1581
1582 if (!is_switch_decoder(dev))
1583 return 0;
1584
1585 cxlsd = to_cxl_switch_decoder(dev);
1586 cxld = &cxlsd->cxld;
1587 guard(rwsem_write)(&cxl_rwsem.region);
1588
1589 for (i = 0; i < cxld->interleave_ways; i++) {
1590 if (cxld->target_map[i] == dport->port_id) {
1591 cxlsd->target[i] = dport;
1592 dev_dbg(dev, "dport%d found in target list, index %d\n",
1593 dport->port_id, i);
1594 return 1;
1595 }
1596 }
1597
1598 return 0;
1599 }
1600
1601 DEFINE_FREE(del_cxl_dport, struct cxl_dport *, if (!IS_ERR_OR_NULL(_T)) del_dport(_T))
cxl_port_add_dport(struct cxl_port * port,struct device * dport_dev)1602 static struct cxl_dport *cxl_port_add_dport(struct cxl_port *port,
1603 struct device *dport_dev)
1604 {
1605 struct cxl_dport *dport;
1606 int rc;
1607
1608 device_lock_assert(&port->dev);
1609 if (!port->dev.driver)
1610 return ERR_PTR(-ENXIO);
1611
1612 dport = cxl_find_dport_by_dev(port, dport_dev);
1613 if (dport) {
1614 dev_dbg(&port->dev, "dport%d:%s already exists\n",
1615 dport->port_id, dev_name(dport_dev));
1616 return ERR_PTR(-EBUSY);
1617 }
1618
1619 struct cxl_dport *new_dport __free(del_cxl_dport) =
1620 devm_cxl_add_dport_by_dev(port, dport_dev);
1621 if (IS_ERR(new_dport))
1622 return new_dport;
1623
1624 cxl_switch_parse_cdat(new_dport);
1625
1626 if (ida_is_empty(&port->decoder_ida)) {
1627 rc = devm_cxl_switch_port_decoders_setup(port);
1628 if (rc)
1629 return ERR_PTR(rc);
1630 dev_dbg(&port->dev, "first dport%d:%s added with decoders\n",
1631 new_dport->port_id, dev_name(dport_dev));
1632 return no_free_ptr(new_dport);
1633 }
1634
1635 /* New dport added, update the decoder targets */
1636 device_for_each_child(&port->dev, new_dport, update_decoder_targets);
1637
1638 dev_dbg(&port->dev, "dport%d:%s added\n", new_dport->port_id,
1639 dev_name(dport_dev));
1640
1641 return no_free_ptr(new_dport);
1642 }
1643
devm_cxl_create_port(struct device * ep_dev,struct cxl_port * parent_port,struct cxl_dport * parent_dport,struct device * uport_dev,struct device * dport_dev)1644 static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
1645 struct cxl_port *parent_port,
1646 struct cxl_dport *parent_dport,
1647 struct device *uport_dev,
1648 struct device *dport_dev)
1649 {
1650 resource_size_t component_reg_phys;
1651
1652 device_lock_assert(&parent_port->dev);
1653 if (!parent_port->dev.driver) {
1654 dev_warn(ep_dev,
1655 "port %s:%s:%s disabled, failed to enumerate CXL.mem\n",
1656 dev_name(&parent_port->dev), dev_name(uport_dev),
1657 dev_name(dport_dev));
1658 }
1659
1660 struct cxl_port *port __free(put_cxl_port) =
1661 find_cxl_port_by_uport(uport_dev);
1662 if (!port) {
1663 component_reg_phys = find_component_registers(uport_dev);
1664 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1665 component_reg_phys, parent_dport);
1666 if (IS_ERR(port))
1667 return ERR_CAST(port);
1668
1669 /*
1670 * retry to make sure a port is found. a port device
1671 * reference is taken.
1672 */
1673 port = find_cxl_port_by_uport(uport_dev);
1674 if (!port)
1675 return ERR_PTR(-ENODEV);
1676
1677 dev_dbg(ep_dev, "created port %s:%s\n",
1678 dev_name(&port->dev), dev_name(port->uport_dev));
1679 } else {
1680 /*
1681 * Port was created before right before this function is
1682 * called. Signal the caller to deal with it.
1683 */
1684 return ERR_PTR(-EAGAIN);
1685 }
1686
1687 guard(device)(&port->dev);
1688 return cxl_port_add_dport(port, dport_dev);
1689 }
1690
add_port_attach_ep(struct cxl_memdev * cxlmd,struct device * uport_dev,struct device * dport_dev)1691 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1692 struct device *uport_dev,
1693 struct device *dport_dev)
1694 {
1695 struct device *dparent = grandparent(dport_dev);
1696 struct cxl_dport *dport, *parent_dport;
1697 int rc;
1698
1699 if (is_cxl_host_bridge(dparent)) {
1700 /*
1701 * The iteration reached the topology root without finding the
1702 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1703 * be re-probed after platform driver attaches.
1704 */
1705 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1706 dev_name(dport_dev));
1707 return -ENXIO;
1708 }
1709
1710 struct cxl_port *parent_port __free(put_cxl_port) =
1711 find_cxl_port_by_uport(dparent->parent);
1712 if (!parent_port) {
1713 /* iterate to create this parent_port */
1714 return -EAGAIN;
1715 }
1716
1717 scoped_guard(device, &parent_port->dev) {
1718 parent_dport = cxl_find_dport_by_dev(parent_port, dparent);
1719 if (!parent_dport) {
1720 parent_dport = cxl_port_add_dport(parent_port, dparent);
1721 if (IS_ERR(parent_dport))
1722 return PTR_ERR(parent_dport);
1723 }
1724
1725 dport = devm_cxl_create_port(&cxlmd->dev, parent_port,
1726 parent_dport, uport_dev,
1727 dport_dev);
1728 if (IS_ERR(dport)) {
1729 /* Port already exists, restart iteration */
1730 if (PTR_ERR(dport) == -EAGAIN)
1731 return 0;
1732 return PTR_ERR(dport);
1733 }
1734 }
1735
1736 rc = cxl_add_ep(dport, &cxlmd->dev);
1737 if (rc == -EBUSY) {
1738 /*
1739 * "can't" happen, but this error code means
1740 * something to the caller, so translate it.
1741 */
1742 rc = -ENXIO;
1743 }
1744
1745 return rc;
1746 }
1747
find_or_add_dport(struct cxl_port * port,struct device * dport_dev)1748 static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
1749 struct device *dport_dev)
1750 {
1751 struct cxl_dport *dport;
1752
1753 device_lock_assert(&port->dev);
1754 dport = cxl_find_dport_by_dev(port, dport_dev);
1755 if (!dport) {
1756 dport = cxl_port_add_dport(port, dport_dev);
1757 if (IS_ERR(dport))
1758 return dport;
1759
1760 /* New dport added, restart iteration */
1761 return ERR_PTR(-EAGAIN);
1762 }
1763
1764 return dport;
1765 }
1766
devm_cxl_enumerate_ports(struct cxl_memdev * cxlmd)1767 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1768 {
1769 struct device *dev = &cxlmd->dev;
1770 struct device *iter;
1771 int rc;
1772
1773 /*
1774 * Skip intermediate port enumeration in the RCH case, there
1775 * are no ports in between a host bridge and an endpoint.
1776 */
1777 if (cxlmd->cxlds->rcd)
1778 return 0;
1779
1780 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1781 if (rc)
1782 return rc;
1783
1784 /*
1785 * Scan for and add all cxl_ports in this device's ancestry.
1786 * Repeat until no more ports are added. Abort if a port add
1787 * attempt fails.
1788 */
1789 retry:
1790 for (iter = dev; iter; iter = grandparent(iter)) {
1791 struct device *dport_dev = grandparent(iter);
1792 struct device *uport_dev;
1793 struct cxl_dport *dport;
1794
1795 if (is_cxl_host_bridge(dport_dev))
1796 return 0;
1797
1798 uport_dev = dport_dev->parent;
1799 if (!uport_dev) {
1800 dev_warn(dev, "at %s no parent for dport: %s\n",
1801 dev_name(iter), dev_name(dport_dev));
1802 return -ENXIO;
1803 }
1804
1805 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1806 dev_name(iter), dev_name(dport_dev),
1807 dev_name(uport_dev));
1808 struct cxl_port *port __free(put_cxl_port) =
1809 find_cxl_port_by_uport(uport_dev);
1810 if (port) {
1811 dev_dbg(&cxlmd->dev,
1812 "found already registered port %s:%s\n",
1813 dev_name(&port->dev),
1814 dev_name(port->uport_dev));
1815
1816 /*
1817 * RP port enumerated by cxl_acpi without dport will
1818 * have the dport added here.
1819 */
1820 scoped_guard(device, &port->dev) {
1821 dport = find_or_add_dport(port, dport_dev);
1822 if (IS_ERR(dport)) {
1823 if (PTR_ERR(dport) == -EAGAIN)
1824 goto retry;
1825 return PTR_ERR(dport);
1826 }
1827 }
1828
1829 rc = cxl_add_ep(dport, &cxlmd->dev);
1830
1831 /*
1832 * If the endpoint already exists in the port's list,
1833 * that's ok, it was added on a previous pass.
1834 * Otherwise, retry in add_port_attach_ep() after taking
1835 * the parent_port lock as the current port may be being
1836 * reaped.
1837 */
1838 if (rc && rc != -EBUSY)
1839 return rc;
1840
1841 cxl_gpf_port_setup(dport);
1842
1843 /* Any more ports to add between this one and the root? */
1844 if (!dev_is_cxl_root_child(&port->dev))
1845 continue;
1846
1847 return 0;
1848 }
1849
1850 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1851 /* port missing, try to add parent */
1852 if (rc == -EAGAIN)
1853 continue;
1854 /* failed to add ep or port */
1855 if (rc)
1856 return rc;
1857 /* port added, new descendants possible, start over */
1858 goto retry;
1859 }
1860
1861 return 0;
1862 }
1863 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
1864
cxl_pci_find_port(struct pci_dev * pdev,struct cxl_dport ** dport)1865 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1866 struct cxl_dport **dport)
1867 {
1868 return find_cxl_port(pdev->dev.parent, dport);
1869 }
1870 EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
1871
cxl_mem_find_port(struct cxl_memdev * cxlmd,struct cxl_dport ** dport)1872 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1873 struct cxl_dport **dport)
1874 {
1875 return find_cxl_port(grandparent(&cxlmd->dev), dport);
1876 }
1877 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
1878
decoder_populate_targets(struct cxl_switch_decoder * cxlsd,struct cxl_port * port)1879 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1880 struct cxl_port *port)
1881 {
1882 struct cxl_decoder *cxld = &cxlsd->cxld;
1883 int i;
1884
1885 device_lock_assert(&port->dev);
1886
1887 if (xa_empty(&port->dports))
1888 return 0;
1889
1890 guard(rwsem_write)(&cxl_rwsem.region);
1891 for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1892 struct cxl_dport *dport = find_dport(port, cxld->target_map[i]);
1893
1894 if (!dport) {
1895 /* dport may be activated later */
1896 continue;
1897 }
1898 cxlsd->target[i] = dport;
1899 }
1900
1901 return 0;
1902 }
1903
1904 static struct lock_class_key cxl_decoder_key;
1905
1906 /**
1907 * cxl_decoder_init - Common decoder setup / initialization
1908 * @port: owning port of this decoder
1909 * @cxld: common decoder properties to initialize
1910 *
1911 * A port may contain one or more decoders. Each of those decoders
1912 * enable some address space for CXL.mem utilization. A decoder is
1913 * expected to be configured by the caller before registering via
1914 * cxl_decoder_add()
1915 */
cxl_decoder_init(struct cxl_port * port,struct cxl_decoder * cxld)1916 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1917 {
1918 struct device *dev;
1919 int rc;
1920
1921 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1922 if (rc < 0)
1923 return rc;
1924
1925 /* need parent to stick around to release the id */
1926 get_device(&port->dev);
1927 cxld->id = rc;
1928
1929 dev = &cxld->dev;
1930 device_initialize(dev);
1931 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1932 device_set_pm_not_required(dev);
1933 dev->parent = &port->dev;
1934 dev->bus = &cxl_bus_type;
1935
1936 /* Pre initialize an "empty" decoder */
1937 cxld->interleave_ways = 1;
1938 cxld->interleave_granularity = PAGE_SIZE;
1939 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1940 cxld->hpa_range = (struct range) {
1941 .start = 0,
1942 .end = -1,
1943 };
1944
1945 return 0;
1946 }
1947
cxl_switch_decoder_init(struct cxl_port * port,struct cxl_switch_decoder * cxlsd,int nr_targets)1948 static int cxl_switch_decoder_init(struct cxl_port *port,
1949 struct cxl_switch_decoder *cxlsd,
1950 int nr_targets)
1951 {
1952 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1953 return -EINVAL;
1954
1955 cxlsd->nr_targets = nr_targets;
1956 return cxl_decoder_init(port, &cxlsd->cxld);
1957 }
1958
1959 /**
1960 * cxl_root_decoder_alloc - Allocate a root level decoder
1961 * @port: owning CXL root of this decoder
1962 * @nr_targets: static number of downstream targets
1963 *
1964 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1965 * 'CXL root' decoder is one that decodes from a top-level / static platform
1966 * firmware description of CXL resources into a CXL standard decode
1967 * topology.
1968 */
cxl_root_decoder_alloc(struct cxl_port * port,unsigned int nr_targets)1969 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1970 unsigned int nr_targets)
1971 {
1972 struct cxl_root_decoder *cxlrd;
1973 struct cxl_switch_decoder *cxlsd;
1974 struct cxl_decoder *cxld;
1975 int rc;
1976
1977 if (!is_cxl_root(port))
1978 return ERR_PTR(-EINVAL);
1979
1980 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1981 GFP_KERNEL);
1982 if (!cxlrd)
1983 return ERR_PTR(-ENOMEM);
1984
1985 cxlsd = &cxlrd->cxlsd;
1986 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1987 if (rc) {
1988 kfree(cxlrd);
1989 return ERR_PTR(rc);
1990 }
1991
1992 mutex_init(&cxlrd->range_lock);
1993
1994 cxld = &cxlsd->cxld;
1995 cxld->dev.type = &cxl_decoder_root_type;
1996 /*
1997 * cxl_root_decoder_release() special cases negative ids to
1998 * detect memregion_alloc() failures.
1999 */
2000 atomic_set(&cxlrd->region_id, -1);
2001 rc = memregion_alloc(GFP_KERNEL);
2002 if (rc < 0) {
2003 put_device(&cxld->dev);
2004 return ERR_PTR(rc);
2005 }
2006
2007 atomic_set(&cxlrd->region_id, rc);
2008 cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
2009 return cxlrd;
2010 }
2011 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL");
2012
2013 /**
2014 * cxl_switch_decoder_alloc - Allocate a switch level decoder
2015 * @port: owning CXL switch port of this decoder
2016 * @nr_targets: max number of dynamically addressable downstream targets
2017 *
2018 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
2019 * 'switch' decoder is any decoder that can be enumerated by PCIe
2020 * topology and the HDM Decoder Capability. This includes the decoders
2021 * that sit between Switch Upstream Ports / Switch Downstream Ports and
2022 * Host Bridges / Root Ports.
2023 */
cxl_switch_decoder_alloc(struct cxl_port * port,unsigned int nr_targets)2024 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
2025 unsigned int nr_targets)
2026 {
2027 struct cxl_switch_decoder *cxlsd;
2028 struct cxl_decoder *cxld;
2029 int rc;
2030
2031 if (is_cxl_root(port) || is_cxl_endpoint(port))
2032 return ERR_PTR(-EINVAL);
2033
2034 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
2035 if (!cxlsd)
2036 return ERR_PTR(-ENOMEM);
2037
2038 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
2039 if (rc) {
2040 kfree(cxlsd);
2041 return ERR_PTR(rc);
2042 }
2043
2044 cxld = &cxlsd->cxld;
2045 cxld->dev.type = &cxl_decoder_switch_type;
2046 return cxlsd;
2047 }
2048 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL");
2049
2050 /**
2051 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
2052 * @port: owning port of this decoder
2053 *
2054 * Return: A new cxl decoder to be registered by cxl_decoder_add()
2055 */
cxl_endpoint_decoder_alloc(struct cxl_port * port)2056 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
2057 {
2058 struct cxl_endpoint_decoder *cxled;
2059 struct cxl_decoder *cxld;
2060 int rc;
2061
2062 if (!is_cxl_endpoint(port))
2063 return ERR_PTR(-EINVAL);
2064
2065 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
2066 if (!cxled)
2067 return ERR_PTR(-ENOMEM);
2068
2069 cxled->pos = -1;
2070 cxled->part = -1;
2071 cxld = &cxled->cxld;
2072 rc = cxl_decoder_init(port, cxld);
2073 if (rc) {
2074 kfree(cxled);
2075 return ERR_PTR(rc);
2076 }
2077
2078 cxld->dev.type = &cxl_decoder_endpoint_type;
2079 return cxled;
2080 }
2081 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL");
2082
2083 /**
2084 * cxl_decoder_add_locked - Add a decoder with targets
2085 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2086 *
2087 * Certain types of decoders may not have any targets. The main example of this
2088 * is an endpoint device. A more awkward example is a hostbridge whose root
2089 * ports get hot added (technically possible, though unlikely).
2090 *
2091 * This is the locked variant of cxl_decoder_add().
2092 *
2093 * Context: Process context. Expects the device lock of the port that owns the
2094 * @cxld to be held.
2095 *
2096 * Return: Negative error code if the decoder wasn't properly configured; else
2097 * returns 0.
2098 */
cxl_decoder_add_locked(struct cxl_decoder * cxld)2099 int cxl_decoder_add_locked(struct cxl_decoder *cxld)
2100 {
2101 struct cxl_port *port;
2102 struct device *dev;
2103 int rc;
2104
2105 if (WARN_ON_ONCE(!cxld))
2106 return -EINVAL;
2107
2108 if (WARN_ON_ONCE(IS_ERR(cxld)))
2109 return PTR_ERR(cxld);
2110
2111 if (cxld->interleave_ways < 1)
2112 return -EINVAL;
2113
2114 dev = &cxld->dev;
2115
2116 port = to_cxl_port(cxld->dev.parent);
2117 if (!is_endpoint_decoder(dev)) {
2118 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
2119
2120 rc = decoder_populate_targets(cxlsd, port);
2121 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
2122 dev_err(&port->dev,
2123 "Failed to populate active decoder targets\n");
2124 return rc;
2125 }
2126 }
2127
2128 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
2129 if (rc)
2130 return rc;
2131
2132 return device_add(dev);
2133 }
2134 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL");
2135
2136 /**
2137 * cxl_decoder_add - Add a decoder with targets
2138 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2139 *
2140 * This is the unlocked variant of cxl_decoder_add_locked().
2141 * See cxl_decoder_add_locked().
2142 *
2143 * Context: Process context. Takes and releases the device lock of the port that
2144 * owns the @cxld.
2145 */
cxl_decoder_add(struct cxl_decoder * cxld)2146 int cxl_decoder_add(struct cxl_decoder *cxld)
2147 {
2148 struct cxl_port *port;
2149
2150 if (WARN_ON_ONCE(!cxld))
2151 return -EINVAL;
2152
2153 if (WARN_ON_ONCE(IS_ERR(cxld)))
2154 return PTR_ERR(cxld);
2155
2156 port = to_cxl_port(cxld->dev.parent);
2157
2158 guard(device)(&port->dev);
2159 return cxl_decoder_add_locked(cxld);
2160 }
2161 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
2162
cxld_unregister(void * dev)2163 static void cxld_unregister(void *dev)
2164 {
2165 if (is_endpoint_decoder(dev))
2166 cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
2167 DETACH_INVALIDATE);
2168
2169 device_unregister(dev);
2170 }
2171
cxl_decoder_autoremove(struct device * host,struct cxl_decoder * cxld)2172 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
2173 {
2174 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2175 }
2176 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
2177
2178 /**
2179 * __cxl_driver_register - register a driver for the cxl bus
2180 * @cxl_drv: cxl driver structure to attach
2181 * @owner: owning module/driver
2182 * @modname: KBUILD_MODNAME for parent driver
2183 */
__cxl_driver_register(struct cxl_driver * cxl_drv,struct module * owner,const char * modname)2184 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2185 const char *modname)
2186 {
2187 if (!cxl_drv->probe) {
2188 pr_debug("%s ->probe() must be specified\n", modname);
2189 return -EINVAL;
2190 }
2191
2192 if (!cxl_drv->name) {
2193 pr_debug("%s ->name must be specified\n", modname);
2194 return -EINVAL;
2195 }
2196
2197 if (!cxl_drv->id) {
2198 pr_debug("%s ->id must be specified\n", modname);
2199 return -EINVAL;
2200 }
2201
2202 cxl_drv->drv.bus = &cxl_bus_type;
2203 cxl_drv->drv.owner = owner;
2204 cxl_drv->drv.mod_name = modname;
2205 cxl_drv->drv.name = cxl_drv->name;
2206
2207 return driver_register(&cxl_drv->drv);
2208 }
2209 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL");
2210
cxl_driver_unregister(struct cxl_driver * cxl_drv)2211 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2212 {
2213 driver_unregister(&cxl_drv->drv);
2214 }
2215 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL");
2216
cxl_bus_uevent(const struct device * dev,struct kobj_uevent_env * env)2217 static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2218 {
2219 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
2220 cxl_device_id(dev));
2221 }
2222
cxl_bus_match(struct device * dev,const struct device_driver * drv)2223 static int cxl_bus_match(struct device *dev, const struct device_driver *drv)
2224 {
2225 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2226 }
2227
cxl_bus_probe(struct device * dev)2228 static int cxl_bus_probe(struct device *dev)
2229 {
2230 int rc;
2231
2232 rc = to_cxl_drv(dev->driver)->probe(dev);
2233 dev_dbg(dev, "probe: %d\n", rc);
2234 return rc;
2235 }
2236
cxl_bus_remove(struct device * dev)2237 static void cxl_bus_remove(struct device *dev)
2238 {
2239 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2240
2241 if (cxl_drv->remove)
2242 cxl_drv->remove(dev);
2243 }
2244
2245 static struct workqueue_struct *cxl_bus_wq;
2246
cxl_rescan_attach(struct device * dev,void * data)2247 static int cxl_rescan_attach(struct device *dev, void *data)
2248 {
2249 int rc = device_attach(dev);
2250
2251 dev_vdbg(dev, "rescan: %s\n", rc ? "attach" : "detached");
2252
2253 return 0;
2254 }
2255
cxl_bus_rescan_queue(struct work_struct * w)2256 static void cxl_bus_rescan_queue(struct work_struct *w)
2257 {
2258 bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_rescan_attach);
2259 }
2260
cxl_bus_rescan(void)2261 void cxl_bus_rescan(void)
2262 {
2263 static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2264
2265 queue_work(cxl_bus_wq, &rescan_work);
2266 }
2267 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL");
2268
cxl_bus_drain(void)2269 void cxl_bus_drain(void)
2270 {
2271 drain_workqueue(cxl_bus_wq);
2272 }
2273 EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
2274
schedule_cxl_memdev_detach(struct cxl_memdev * cxlmd)2275 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2276 {
2277 return queue_work(cxl_bus_wq, &cxlmd->detach_work);
2278 }
2279 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
2280
add_latency(struct access_coordinate * c,long latency)2281 static void add_latency(struct access_coordinate *c, long latency)
2282 {
2283 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2284 c[i].write_latency += latency;
2285 c[i].read_latency += latency;
2286 }
2287 }
2288
coordinates_valid(struct access_coordinate * c)2289 static bool coordinates_valid(struct access_coordinate *c)
2290 {
2291 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2292 if (c[i].read_bandwidth && c[i].write_bandwidth &&
2293 c[i].read_latency && c[i].write_latency)
2294 continue;
2295 return false;
2296 }
2297
2298 return true;
2299 }
2300
set_min_bandwidth(struct access_coordinate * c,unsigned int bw)2301 static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw)
2302 {
2303 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2304 c[i].write_bandwidth = min(c[i].write_bandwidth, bw);
2305 c[i].read_bandwidth = min(c[i].read_bandwidth, bw);
2306 }
2307 }
2308
set_access_coordinates(struct access_coordinate * out,struct access_coordinate * in)2309 static void set_access_coordinates(struct access_coordinate *out,
2310 struct access_coordinate *in)
2311 {
2312 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
2313 out[i] = in[i];
2314 }
2315
parent_port_is_cxl_root(struct cxl_port * port)2316 static bool parent_port_is_cxl_root(struct cxl_port *port)
2317 {
2318 return is_cxl_root(to_cxl_port(port->dev.parent));
2319 }
2320
2321 /**
2322 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2323 * of CXL path
2324 * @port: endpoint cxl_port
2325 * @coord: output performance data
2326 *
2327 * Return: errno on failure, 0 on success.
2328 */
cxl_endpoint_get_perf_coordinates(struct cxl_port * port,struct access_coordinate * coord)2329 int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2330 struct access_coordinate *coord)
2331 {
2332 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
2333 struct access_coordinate c[] = {
2334 {
2335 .read_bandwidth = UINT_MAX,
2336 .write_bandwidth = UINT_MAX,
2337 },
2338 {
2339 .read_bandwidth = UINT_MAX,
2340 .write_bandwidth = UINT_MAX,
2341 },
2342 };
2343 struct cxl_port *iter = port;
2344 struct cxl_dport *dport;
2345 struct pci_dev *pdev;
2346 struct device *dev;
2347 unsigned int bw;
2348 bool is_cxl_root;
2349
2350 if (!is_cxl_endpoint(port))
2351 return -EINVAL;
2352
2353 /*
2354 * Skip calculation for RCD. Expectation is HMAT already covers RCD case
2355 * since RCH does not support hotplug.
2356 */
2357 if (cxlmd->cxlds->rcd)
2358 return 0;
2359
2360 /*
2361 * Exit the loop when the parent port of the current iter port is cxl
2362 * root. The iterative loop starts at the endpoint and gathers the
2363 * latency of the CXL link from the current device/port to the connected
2364 * downstream port each iteration.
2365 */
2366 do {
2367 dport = iter->parent_dport;
2368 iter = to_cxl_port(iter->dev.parent);
2369 is_cxl_root = parent_port_is_cxl_root(iter);
2370
2371 /*
2372 * There's no valid access_coordinate for a root port since RPs do not
2373 * have CDAT and therefore needs to be skipped.
2374 */
2375 if (!is_cxl_root) {
2376 if (!coordinates_valid(dport->coord))
2377 return -EINVAL;
2378 cxl_coordinates_combine(c, c, dport->coord);
2379 }
2380 add_latency(c, dport->link_latency);
2381 } while (!is_cxl_root);
2382
2383 dport = iter->parent_dport;
2384 /* Retrieve HB coords */
2385 if (!coordinates_valid(dport->coord))
2386 return -EINVAL;
2387 cxl_coordinates_combine(c, c, dport->coord);
2388
2389 dev = port->uport_dev->parent;
2390 if (!dev_is_pci(dev))
2391 return -ENODEV;
2392
2393 /* Get the calculated PCI paths bandwidth */
2394 pdev = to_pci_dev(dev);
2395 bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL);
2396 if (bw == 0)
2397 return -ENXIO;
2398 bw /= BITS_PER_BYTE;
2399
2400 set_min_bandwidth(c, bw);
2401 set_access_coordinates(coord, c);
2402
2403 return 0;
2404 }
2405 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL");
2406
cxl_port_get_switch_dport_bandwidth(struct cxl_port * port,struct access_coordinate * c)2407 int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
2408 struct access_coordinate *c)
2409 {
2410 struct cxl_dport *dport = port->parent_dport;
2411
2412 /* Check this port is connected to a switch DSP and not an RP */
2413 if (parent_port_is_cxl_root(to_cxl_port(port->dev.parent)))
2414 return -ENODEV;
2415
2416 if (!coordinates_valid(dport->coord))
2417 return -EINVAL;
2418
2419 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2420 c[i].read_bandwidth = dport->coord[i].read_bandwidth;
2421 c[i].write_bandwidth = dport->coord[i].write_bandwidth;
2422 }
2423
2424 return 0;
2425 }
2426
2427 /* for user tooling to ensure port disable work has completed */
flush_store(const struct bus_type * bus,const char * buf,size_t count)2428 static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2429 {
2430 if (sysfs_streq(buf, "1")) {
2431 flush_workqueue(cxl_bus_wq);
2432 return count;
2433 }
2434
2435 return -EINVAL;
2436 }
2437
2438 static BUS_ATTR_WO(flush);
2439
2440 static struct attribute *cxl_bus_attributes[] = {
2441 &bus_attr_flush.attr,
2442 NULL,
2443 };
2444
2445 static struct attribute_group cxl_bus_attribute_group = {
2446 .attrs = cxl_bus_attributes,
2447 };
2448
2449 static const struct attribute_group *cxl_bus_attribute_groups[] = {
2450 &cxl_bus_attribute_group,
2451 NULL,
2452 };
2453
2454 const struct bus_type cxl_bus_type = {
2455 .name = "cxl",
2456 .uevent = cxl_bus_uevent,
2457 .match = cxl_bus_match,
2458 .probe = cxl_bus_probe,
2459 .remove = cxl_bus_remove,
2460 .bus_groups = cxl_bus_attribute_groups,
2461 };
2462 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL");
2463
2464 static struct dentry *cxl_debugfs;
2465
cxl_debugfs_create_dir(const char * dir)2466 struct dentry *cxl_debugfs_create_dir(const char *dir)
2467 {
2468 return debugfs_create_dir(dir, cxl_debugfs);
2469 }
2470 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL");
2471
cxl_core_init(void)2472 static __init int cxl_core_init(void)
2473 {
2474 int rc;
2475
2476 cxl_debugfs = debugfs_create_dir("cxl", NULL);
2477
2478 if (einj_cxl_is_initialized())
2479 debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL,
2480 &einj_cxl_available_error_type_fops);
2481
2482 cxl_mbox_init();
2483
2484 rc = cxl_memdev_init();
2485 if (rc)
2486 return rc;
2487
2488 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2489 if (!cxl_bus_wq) {
2490 rc = -ENOMEM;
2491 goto err_wq;
2492 }
2493
2494 rc = bus_register(&cxl_bus_type);
2495 if (rc)
2496 goto err_bus;
2497
2498 rc = cxl_region_init();
2499 if (rc)
2500 goto err_region;
2501
2502 rc = cxl_ras_init();
2503 if (rc)
2504 goto err_ras;
2505
2506 return 0;
2507
2508 err_ras:
2509 cxl_region_exit();
2510 err_region:
2511 bus_unregister(&cxl_bus_type);
2512 err_bus:
2513 destroy_workqueue(cxl_bus_wq);
2514 err_wq:
2515 cxl_memdev_exit();
2516 return rc;
2517 }
2518
cxl_core_exit(void)2519 static void cxl_core_exit(void)
2520 {
2521 cxl_ras_exit();
2522 cxl_region_exit();
2523 bus_unregister(&cxl_bus_type);
2524 destroy_workqueue(cxl_bus_wq);
2525 cxl_memdev_exit();
2526 debugfs_remove_recursive(cxl_debugfs);
2527 }
2528
2529 subsys_initcall(cxl_core_init);
2530 module_exit(cxl_core_exit);
2531 MODULE_DESCRIPTION("CXL: Core Compute Express Link support");
2532 MODULE_LICENSE("GPL v2");
2533 MODULE_IMPORT_NS("CXL");
2534