1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/debugfs.h>
6 #include <linux/device.h>
7 #include <linux/module.h>
8 #include <linux/memory.h>
9 #include <linux/slab.h>
10 #include <linux/uuid.h>
11 #include <linux/sort.h>
12 #include <linux/idr.h>
13 #include <linux/memory-tiers.h>
14 #include <linux/string_choices.h>
15 #include <cxlmem.h>
16 #include <cxl.h>
17 #include "core.h"
18
19 /**
20 * DOC: cxl core region
21 *
22 * CXL Regions represent mapped memory capacity in system physical address
23 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
24 * Memory ranges, Regions represent the active mapped capacity by the HDM
25 * Decoder Capability structures throughout the Host Bridges, Switches, and
26 * Endpoints in the topology.
27 *
28 * Region configuration has ordering constraints. UUID may be set at any time
29 * but is only visible for persistent regions.
30 * 1. Interleave granularity
31 * 2. Interleave size
32 * 3. Decoder targets
33 */
34
35 /*
36 * nodemask that sets per node when the access_coordinates for the node has
37 * been updated by the CXL memory hotplug notifier.
38 */
39 static nodemask_t nodemask_region_seen = NODE_MASK_NONE;
40
41 static struct cxl_region *to_cxl_region(struct device *dev);
42
43 #define __ACCESS_ATTR_RO(_level, _name) { \
44 .attr = { .name = __stringify(_name), .mode = 0444 }, \
45 .show = _name##_access##_level##_show, \
46 }
47
48 #define ACCESS_DEVICE_ATTR_RO(level, name) \
49 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
50
51 #define ACCESS_ATTR_RO(level, attrib) \
52 static ssize_t attrib##_access##level##_show(struct device *dev, \
53 struct device_attribute *attr, \
54 char *buf) \
55 { \
56 struct cxl_region *cxlr = to_cxl_region(dev); \
57 \
58 if (cxlr->coord[level].attrib == 0) \
59 return -ENOENT; \
60 \
61 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
62 } \
63 static ACCESS_DEVICE_ATTR_RO(level, attrib)
64
65 ACCESS_ATTR_RO(0, read_bandwidth);
66 ACCESS_ATTR_RO(0, read_latency);
67 ACCESS_ATTR_RO(0, write_bandwidth);
68 ACCESS_ATTR_RO(0, write_latency);
69
70 #define ACCESS_ATTR_DECLARE(level, attrib) \
71 (&dev_attr_access##level##_##attrib.attr)
72
73 static struct attribute *access0_coordinate_attrs[] = {
74 ACCESS_ATTR_DECLARE(0, read_bandwidth),
75 ACCESS_ATTR_DECLARE(0, write_bandwidth),
76 ACCESS_ATTR_DECLARE(0, read_latency),
77 ACCESS_ATTR_DECLARE(0, write_latency),
78 NULL
79 };
80
81 ACCESS_ATTR_RO(1, read_bandwidth);
82 ACCESS_ATTR_RO(1, read_latency);
83 ACCESS_ATTR_RO(1, write_bandwidth);
84 ACCESS_ATTR_RO(1, write_latency);
85
86 static struct attribute *access1_coordinate_attrs[] = {
87 ACCESS_ATTR_DECLARE(1, read_bandwidth),
88 ACCESS_ATTR_DECLARE(1, write_bandwidth),
89 ACCESS_ATTR_DECLARE(1, read_latency),
90 ACCESS_ATTR_DECLARE(1, write_latency),
91 NULL
92 };
93
94 #define ACCESS_VISIBLE(level) \
95 static umode_t cxl_region_access##level##_coordinate_visible( \
96 struct kobject *kobj, struct attribute *a, int n) \
97 { \
98 struct device *dev = kobj_to_dev(kobj); \
99 struct cxl_region *cxlr = to_cxl_region(dev); \
100 \
101 if (a == &dev_attr_access##level##_read_latency.attr && \
102 cxlr->coord[level].read_latency == 0) \
103 return 0; \
104 \
105 if (a == &dev_attr_access##level##_write_latency.attr && \
106 cxlr->coord[level].write_latency == 0) \
107 return 0; \
108 \
109 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
110 cxlr->coord[level].read_bandwidth == 0) \
111 return 0; \
112 \
113 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
114 cxlr->coord[level].write_bandwidth == 0) \
115 return 0; \
116 \
117 return a->mode; \
118 }
119
120 ACCESS_VISIBLE(0);
121 ACCESS_VISIBLE(1);
122
123 static const struct attribute_group cxl_region_access0_coordinate_group = {
124 .name = "access0",
125 .attrs = access0_coordinate_attrs,
126 .is_visible = cxl_region_access0_coordinate_visible,
127 };
128
get_cxl_region_access0_group(void)129 static const struct attribute_group *get_cxl_region_access0_group(void)
130 {
131 return &cxl_region_access0_coordinate_group;
132 }
133
134 static const struct attribute_group cxl_region_access1_coordinate_group = {
135 .name = "access1",
136 .attrs = access1_coordinate_attrs,
137 .is_visible = cxl_region_access1_coordinate_visible,
138 };
139
get_cxl_region_access1_group(void)140 static const struct attribute_group *get_cxl_region_access1_group(void)
141 {
142 return &cxl_region_access1_coordinate_group;
143 }
144
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)145 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
146 char *buf)
147 {
148 struct cxl_region *cxlr = to_cxl_region(dev);
149 struct cxl_region_params *p = &cxlr->params;
150 ssize_t rc;
151
152 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
153 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem)))
154 return rc;
155 if (cxlr->mode != CXL_PARTMODE_PMEM)
156 return sysfs_emit(buf, "\n");
157 return sysfs_emit(buf, "%pUb\n", &p->uuid);
158 }
159
is_dup(struct device * match,void * data)160 static int is_dup(struct device *match, void *data)
161 {
162 struct cxl_region_params *p;
163 struct cxl_region *cxlr;
164 uuid_t *uuid = data;
165
166 if (!is_cxl_region(match))
167 return 0;
168
169 lockdep_assert_held(&cxl_rwsem.region);
170 cxlr = to_cxl_region(match);
171 p = &cxlr->params;
172
173 if (uuid_equal(&p->uuid, uuid)) {
174 dev_dbg(match, "already has uuid: %pUb\n", uuid);
175 return -EBUSY;
176 }
177
178 return 0;
179 }
180
uuid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)181 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
182 const char *buf, size_t len)
183 {
184 struct cxl_region *cxlr = to_cxl_region(dev);
185 struct cxl_region_params *p = &cxlr->params;
186 uuid_t temp;
187 ssize_t rc;
188
189 if (len != UUID_STRING_LEN + 1)
190 return -EINVAL;
191
192 rc = uuid_parse(buf, &temp);
193 if (rc)
194 return rc;
195
196 if (uuid_is_null(&temp))
197 return -EINVAL;
198
199 ACQUIRE(rwsem_write_kill, region_rwsem)(&cxl_rwsem.region);
200 if ((rc = ACQUIRE_ERR(rwsem_write_kill, ®ion_rwsem)))
201 return rc;
202
203 if (uuid_equal(&p->uuid, &temp))
204 return len;
205
206 if (p->state >= CXL_CONFIG_ACTIVE)
207 return -EBUSY;
208
209 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
210 if (rc < 0)
211 return rc;
212
213 uuid_copy(&p->uuid, &temp);
214
215 return len;
216 }
217 static DEVICE_ATTR_RW(uuid);
218
cxl_rr_load(struct cxl_port * port,struct cxl_region * cxlr)219 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
220 struct cxl_region *cxlr)
221 {
222 return xa_load(&port->regions, (unsigned long)cxlr);
223 }
224
cxl_region_invalidate_memregion(struct cxl_region * cxlr)225 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
226 {
227 if (!cpu_cache_has_invalidate_memregion()) {
228 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
229 dev_info_once(
230 &cxlr->dev,
231 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
232 return 0;
233 }
234 dev_WARN(&cxlr->dev,
235 "Failed to synchronize CPU cache state\n");
236 return -ENXIO;
237 }
238
239 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
240 return 0;
241 }
242
cxl_region_decode_reset(struct cxl_region * cxlr,int count)243 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
244 {
245 struct cxl_region_params *p = &cxlr->params;
246 int i;
247
248 /*
249 * Before region teardown attempt to flush, evict any data cached for
250 * this region, or scream loudly about missing arch / platform support
251 * for CXL teardown.
252 */
253 cxl_region_invalidate_memregion(cxlr);
254
255 for (i = count - 1; i >= 0; i--) {
256 struct cxl_endpoint_decoder *cxled = p->targets[i];
257 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
258 struct cxl_port *iter = cxled_to_port(cxled);
259 struct cxl_dev_state *cxlds = cxlmd->cxlds;
260 struct cxl_ep *ep;
261
262 if (cxlds->rcd)
263 goto endpoint_reset;
264
265 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
266 iter = to_cxl_port(iter->dev.parent);
267
268 for (ep = cxl_ep_load(iter, cxlmd); iter;
269 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
270 struct cxl_region_ref *cxl_rr;
271 struct cxl_decoder *cxld;
272
273 cxl_rr = cxl_rr_load(iter, cxlr);
274 cxld = cxl_rr->decoder;
275 if (cxld->reset)
276 cxld->reset(cxld);
277 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
278 }
279
280 endpoint_reset:
281 cxled->cxld.reset(&cxled->cxld);
282 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
283 }
284
285 /* all decoders associated with this region have been torn down */
286 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
287 }
288
commit_decoder(struct cxl_decoder * cxld)289 static int commit_decoder(struct cxl_decoder *cxld)
290 {
291 struct cxl_switch_decoder *cxlsd = NULL;
292
293 if (cxld->commit)
294 return cxld->commit(cxld);
295
296 if (is_switch_decoder(&cxld->dev))
297 cxlsd = to_cxl_switch_decoder(&cxld->dev);
298
299 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
300 "->commit() is required\n"))
301 return -ENXIO;
302 return 0;
303 }
304
cxl_region_decode_commit(struct cxl_region * cxlr)305 static int cxl_region_decode_commit(struct cxl_region *cxlr)
306 {
307 struct cxl_region_params *p = &cxlr->params;
308 int i, rc = 0;
309
310 for (i = 0; i < p->nr_targets; i++) {
311 struct cxl_endpoint_decoder *cxled = p->targets[i];
312 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
313 struct cxl_region_ref *cxl_rr;
314 struct cxl_decoder *cxld;
315 struct cxl_port *iter;
316 struct cxl_ep *ep;
317
318 /* commit bottom up */
319 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
320 iter = to_cxl_port(iter->dev.parent)) {
321 cxl_rr = cxl_rr_load(iter, cxlr);
322 cxld = cxl_rr->decoder;
323 rc = commit_decoder(cxld);
324 if (rc)
325 break;
326 }
327
328 if (rc) {
329 /* programming @iter failed, teardown */
330 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
331 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
332 cxl_rr = cxl_rr_load(iter, cxlr);
333 cxld = cxl_rr->decoder;
334 if (cxld->reset)
335 cxld->reset(cxld);
336 }
337
338 cxled->cxld.reset(&cxled->cxld);
339 goto err;
340 }
341 }
342
343 return 0;
344
345 err:
346 /* undo the targets that were successfully committed */
347 cxl_region_decode_reset(cxlr, i);
348 return rc;
349 }
350
queue_reset(struct cxl_region * cxlr)351 static int queue_reset(struct cxl_region *cxlr)
352 {
353 struct cxl_region_params *p = &cxlr->params;
354 int rc;
355
356 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
357 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
358 return rc;
359
360 /* Already in the requested state? */
361 if (p->state < CXL_CONFIG_COMMIT)
362 return 0;
363
364 p->state = CXL_CONFIG_RESET_PENDING;
365
366 return 0;
367 }
368
__commit(struct cxl_region * cxlr)369 static int __commit(struct cxl_region *cxlr)
370 {
371 struct cxl_region_params *p = &cxlr->params;
372 int rc;
373
374 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
375 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
376 return rc;
377
378 /* Already in the requested state? */
379 if (p->state >= CXL_CONFIG_COMMIT)
380 return 0;
381
382 /* Not ready to commit? */
383 if (p->state < CXL_CONFIG_ACTIVE)
384 return -ENXIO;
385
386 /*
387 * Invalidate caches before region setup to drop any speculative
388 * consumption of this address space
389 */
390 rc = cxl_region_invalidate_memregion(cxlr);
391 if (rc)
392 return rc;
393
394 rc = cxl_region_decode_commit(cxlr);
395 if (rc)
396 return rc;
397
398 p->state = CXL_CONFIG_COMMIT;
399
400 return 0;
401 }
402
commit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)403 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
404 const char *buf, size_t len)
405 {
406 struct cxl_region *cxlr = to_cxl_region(dev);
407 struct cxl_region_params *p = &cxlr->params;
408 bool commit;
409 ssize_t rc;
410
411 rc = kstrtobool(buf, &commit);
412 if (rc)
413 return rc;
414
415 if (commit) {
416 rc = __commit(cxlr);
417 if (rc)
418 return rc;
419 return len;
420 }
421
422 rc = queue_reset(cxlr);
423 if (rc)
424 return rc;
425
426 /*
427 * Unmap the region and depend the reset-pending state to ensure
428 * it does not go active again until post reset
429 */
430 device_release_driver(&cxlr->dev);
431
432 /*
433 * With the reset pending take cxl_rwsem.region unconditionally
434 * to ensure the reset gets handled before returning.
435 */
436 guard(rwsem_write)(&cxl_rwsem.region);
437
438 /*
439 * Revalidate that the reset is still pending in case another
440 * thread already handled this reset.
441 */
442 if (p->state == CXL_CONFIG_RESET_PENDING) {
443 cxl_region_decode_reset(cxlr, p->interleave_ways);
444 p->state = CXL_CONFIG_ACTIVE;
445 }
446
447 return len;
448 }
449
commit_show(struct device * dev,struct device_attribute * attr,char * buf)450 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
451 char *buf)
452 {
453 struct cxl_region *cxlr = to_cxl_region(dev);
454 struct cxl_region_params *p = &cxlr->params;
455 ssize_t rc;
456
457 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
458 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
459 return rc;
460 return sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
461 }
462 static DEVICE_ATTR_RW(commit);
463
cxl_region_visible(struct kobject * kobj,struct attribute * a,int n)464 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
465 int n)
466 {
467 struct device *dev = kobj_to_dev(kobj);
468 struct cxl_region *cxlr = to_cxl_region(dev);
469
470 /*
471 * Support tooling that expects to find a 'uuid' attribute for all
472 * regions regardless of mode.
473 */
474 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_PARTMODE_PMEM)
475 return 0444;
476 return a->mode;
477 }
478
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)479 static ssize_t interleave_ways_show(struct device *dev,
480 struct device_attribute *attr, char *buf)
481 {
482 struct cxl_region *cxlr = to_cxl_region(dev);
483 struct cxl_region_params *p = &cxlr->params;
484 int rc;
485
486 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
487 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
488 return rc;
489 return sysfs_emit(buf, "%d\n", p->interleave_ways);
490 }
491
492 static const struct attribute_group *get_cxl_region_target_group(void);
493
interleave_ways_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)494 static ssize_t interleave_ways_store(struct device *dev,
495 struct device_attribute *attr,
496 const char *buf, size_t len)
497 {
498 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
499 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
500 struct cxl_region *cxlr = to_cxl_region(dev);
501 struct cxl_region_params *p = &cxlr->params;
502 unsigned int val, save;
503 int rc;
504 u8 iw;
505
506 rc = kstrtouint(buf, 0, &val);
507 if (rc)
508 return rc;
509
510 rc = ways_to_eiw(val, &iw);
511 if (rc)
512 return rc;
513
514 /*
515 * Even for x3, x6, and x12 interleaves the region interleave must be a
516 * power of 2 multiple of the host bridge interleave.
517 */
518 if (!is_power_of_2(val / cxld->interleave_ways) ||
519 (val % cxld->interleave_ways)) {
520 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
521 return -EINVAL;
522 }
523
524 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
525 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
526 return rc;
527
528 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
529 return -EBUSY;
530
531 save = p->interleave_ways;
532 p->interleave_ways = val;
533 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
534 if (rc) {
535 p->interleave_ways = save;
536 return rc;
537 }
538
539 return len;
540 }
541 static DEVICE_ATTR_RW(interleave_ways);
542
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)543 static ssize_t interleave_granularity_show(struct device *dev,
544 struct device_attribute *attr,
545 char *buf)
546 {
547 struct cxl_region *cxlr = to_cxl_region(dev);
548 struct cxl_region_params *p = &cxlr->params;
549 int rc;
550
551 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
552 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
553 return rc;
554 return sysfs_emit(buf, "%d\n", p->interleave_granularity);
555 }
556
interleave_granularity_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)557 static ssize_t interleave_granularity_store(struct device *dev,
558 struct device_attribute *attr,
559 const char *buf, size_t len)
560 {
561 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
562 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
563 struct cxl_region *cxlr = to_cxl_region(dev);
564 struct cxl_region_params *p = &cxlr->params;
565 int rc, val;
566 u16 ig;
567
568 rc = kstrtoint(buf, 0, &val);
569 if (rc)
570 return rc;
571
572 rc = granularity_to_eig(val, &ig);
573 if (rc)
574 return rc;
575
576 /*
577 * When the host-bridge is interleaved, disallow region granularity !=
578 * root granularity. Regions with a granularity less than the root
579 * interleave result in needing multiple endpoints to support a single
580 * slot in the interleave (possible to support in the future). Regions
581 * with a granularity greater than the root interleave result in invalid
582 * DPA translations (invalid to support).
583 */
584 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
585 return -EINVAL;
586
587 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
588 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
589 return rc;
590
591 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
592 return -EBUSY;
593
594 p->interleave_granularity = val;
595
596 return len;
597 }
598 static DEVICE_ATTR_RW(interleave_granularity);
599
resource_show(struct device * dev,struct device_attribute * attr,char * buf)600 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
601 char *buf)
602 {
603 struct cxl_region *cxlr = to_cxl_region(dev);
604 struct cxl_region_params *p = &cxlr->params;
605 u64 resource = -1ULL;
606 int rc;
607
608 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
609 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
610 return rc;
611
612 if (p->res)
613 resource = p->res->start;
614 return sysfs_emit(buf, "%#llx\n", resource);
615 }
616 static DEVICE_ATTR_RO(resource);
617
mode_show(struct device * dev,struct device_attribute * attr,char * buf)618 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
619 char *buf)
620 {
621 struct cxl_region *cxlr = to_cxl_region(dev);
622 const char *desc;
623
624 if (cxlr->mode == CXL_PARTMODE_RAM)
625 desc = "ram";
626 else if (cxlr->mode == CXL_PARTMODE_PMEM)
627 desc = "pmem";
628 else
629 desc = "";
630
631 return sysfs_emit(buf, "%s\n", desc);
632 }
633 static DEVICE_ATTR_RO(mode);
634
alloc_hpa(struct cxl_region * cxlr,resource_size_t size)635 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
636 {
637 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
638 struct cxl_region_params *p = &cxlr->params;
639 struct resource *res;
640 u64 remainder = 0;
641
642 lockdep_assert_held_write(&cxl_rwsem.region);
643
644 /* Nothing to do... */
645 if (p->res && resource_size(p->res) == size)
646 return 0;
647
648 /* To change size the old size must be freed first */
649 if (p->res)
650 return -EBUSY;
651
652 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
653 return -EBUSY;
654
655 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
656 if (!p->interleave_ways || !p->interleave_granularity ||
657 (cxlr->mode == CXL_PARTMODE_PMEM && uuid_is_null(&p->uuid)))
658 return -ENXIO;
659
660 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
661 if (remainder)
662 return -EINVAL;
663
664 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
665 dev_name(&cxlr->dev));
666 if (IS_ERR(res)) {
667 dev_dbg(&cxlr->dev,
668 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
669 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
670 return PTR_ERR(res);
671 }
672
673 p->res = res;
674 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
675
676 return 0;
677 }
678
cxl_region_iomem_release(struct cxl_region * cxlr)679 static void cxl_region_iomem_release(struct cxl_region *cxlr)
680 {
681 struct cxl_region_params *p = &cxlr->params;
682
683 if (device_is_registered(&cxlr->dev))
684 lockdep_assert_held_write(&cxl_rwsem.region);
685 if (p->res) {
686 /*
687 * Autodiscovered regions may not have been able to insert their
688 * resource.
689 */
690 if (p->res->parent)
691 remove_resource(p->res);
692 kfree(p->res);
693 p->res = NULL;
694 }
695 }
696
free_hpa(struct cxl_region * cxlr)697 static int free_hpa(struct cxl_region *cxlr)
698 {
699 struct cxl_region_params *p = &cxlr->params;
700
701 lockdep_assert_held_write(&cxl_rwsem.region);
702
703 if (!p->res)
704 return 0;
705
706 if (p->state >= CXL_CONFIG_ACTIVE)
707 return -EBUSY;
708
709 cxl_region_iomem_release(cxlr);
710 p->state = CXL_CONFIG_IDLE;
711 return 0;
712 }
713
size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)714 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
715 const char *buf, size_t len)
716 {
717 struct cxl_region *cxlr = to_cxl_region(dev);
718 u64 val;
719 int rc;
720
721 rc = kstrtou64(buf, 0, &val);
722 if (rc)
723 return rc;
724
725 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
726 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
727 return rc;
728
729 if (val)
730 rc = alloc_hpa(cxlr, val);
731 else
732 rc = free_hpa(cxlr);
733
734 if (rc)
735 return rc;
736
737 return len;
738 }
739
size_show(struct device * dev,struct device_attribute * attr,char * buf)740 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
741 char *buf)
742 {
743 struct cxl_region *cxlr = to_cxl_region(dev);
744 struct cxl_region_params *p = &cxlr->params;
745 u64 size = 0;
746 ssize_t rc;
747
748 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
749 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
750 return rc;
751 if (p->res)
752 size = resource_size(p->res);
753 return sysfs_emit(buf, "%#llx\n", size);
754 }
755 static DEVICE_ATTR_RW(size);
756
757 static struct attribute *cxl_region_attrs[] = {
758 &dev_attr_uuid.attr,
759 &dev_attr_commit.attr,
760 &dev_attr_interleave_ways.attr,
761 &dev_attr_interleave_granularity.attr,
762 &dev_attr_resource.attr,
763 &dev_attr_size.attr,
764 &dev_attr_mode.attr,
765 NULL,
766 };
767
768 static const struct attribute_group cxl_region_group = {
769 .attrs = cxl_region_attrs,
770 .is_visible = cxl_region_visible,
771 };
772
show_targetN(struct cxl_region * cxlr,char * buf,int pos)773 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
774 {
775 struct cxl_region_params *p = &cxlr->params;
776 struct cxl_endpoint_decoder *cxled;
777 int rc;
778
779 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
780 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
781 return rc;
782
783 if (pos >= p->interleave_ways) {
784 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
785 p->interleave_ways);
786 return -ENXIO;
787 }
788
789 cxled = p->targets[pos];
790 if (!cxled)
791 return sysfs_emit(buf, "\n");
792 return sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
793 }
794
check_commit_order(struct device * dev,void * data)795 static int check_commit_order(struct device *dev, void *data)
796 {
797 struct cxl_decoder *cxld = to_cxl_decoder(dev);
798
799 /*
800 * if port->commit_end is not the only free decoder, then out of
801 * order shutdown has occurred, block further allocations until
802 * that is resolved
803 */
804 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
805 return -EBUSY;
806 return 0;
807 }
808
match_free_decoder(struct device * dev,const void * data)809 static int match_free_decoder(struct device *dev, const void *data)
810 {
811 struct cxl_port *port = to_cxl_port(dev->parent);
812 struct cxl_decoder *cxld;
813 int rc;
814
815 if (!is_switch_decoder(dev))
816 return 0;
817
818 cxld = to_cxl_decoder(dev);
819
820 if (cxld->id != port->commit_end + 1)
821 return 0;
822
823 if (cxld->region) {
824 dev_dbg(dev->parent,
825 "next decoder to commit (%s) is already reserved (%s)\n",
826 dev_name(dev), dev_name(&cxld->region->dev));
827 return 0;
828 }
829
830 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL,
831 check_commit_order);
832 if (rc) {
833 dev_dbg(dev->parent,
834 "unable to allocate %s due to out of order shutdown\n",
835 dev_name(dev));
836 return 0;
837 }
838 return 1;
839 }
840
region_res_match_cxl_range(const struct cxl_region_params * p,const struct range * range)841 static bool region_res_match_cxl_range(const struct cxl_region_params *p,
842 const struct range *range)
843 {
844 if (!p->res)
845 return false;
846
847 /*
848 * If an extended linear cache region then the CXL range is assumed
849 * to be fronted by the DRAM range in current known implementation.
850 * This assumption will be made until a variant implementation exists.
851 */
852 return p->res->start + p->cache_size == range->start &&
853 p->res->end == range->end;
854 }
855
match_auto_decoder(struct device * dev,const void * data)856 static int match_auto_decoder(struct device *dev, const void *data)
857 {
858 const struct cxl_region_params *p = data;
859 struct cxl_decoder *cxld;
860 struct range *r;
861
862 if (!is_switch_decoder(dev))
863 return 0;
864
865 cxld = to_cxl_decoder(dev);
866 r = &cxld->hpa_range;
867
868 if (region_res_match_cxl_range(p, r))
869 return 1;
870
871 return 0;
872 }
873
874 /**
875 * cxl_port_pick_region_decoder() - assign or lookup a decoder for a region
876 * @port: a port in the ancestry of the endpoint implied by @cxled
877 * @cxled: endpoint decoder to be, or currently, mapped by @port
878 * @cxlr: region to establish, or validate, decode @port
879 *
880 * In the region creation path cxl_port_pick_region_decoder() is an
881 * allocator to find a free port. In the region assembly path, it is
882 * recalling the decoder that platform firmware picked for validation
883 * purposes.
884 *
885 * The result is recorded in a 'struct cxl_region_ref' in @port.
886 */
887 static struct cxl_decoder *
cxl_port_pick_region_decoder(struct cxl_port * port,struct cxl_endpoint_decoder * cxled,struct cxl_region * cxlr)888 cxl_port_pick_region_decoder(struct cxl_port *port,
889 struct cxl_endpoint_decoder *cxled,
890 struct cxl_region *cxlr)
891 {
892 struct device *dev;
893
894 if (port == cxled_to_port(cxled))
895 return &cxled->cxld;
896
897 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
898 dev = device_find_child(&port->dev, &cxlr->params,
899 match_auto_decoder);
900 else
901 dev = device_find_child(&port->dev, NULL, match_free_decoder);
902 if (!dev)
903 return NULL;
904 /*
905 * This decoder is pinned registered as long as the endpoint decoder is
906 * registered, and endpoint decoder unregistration holds the
907 * cxl_rwsem.region over unregister events, so no need to hold on to
908 * this extra reference.
909 */
910 put_device(dev);
911 return to_cxl_decoder(dev);
912 }
913
auto_order_ok(struct cxl_port * port,struct cxl_region * cxlr_iter,struct cxl_decoder * cxld)914 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
915 struct cxl_decoder *cxld)
916 {
917 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
918 struct cxl_decoder *cxld_iter = rr->decoder;
919
920 /*
921 * Allow the out of order assembly of auto-discovered regions.
922 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
923 * in HPA order. Confirm that the decoder with the lesser HPA
924 * starting address has the lesser id.
925 */
926 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
927 dev_name(&cxld->dev), cxld->id,
928 dev_name(&cxld_iter->dev), cxld_iter->id);
929
930 if (cxld_iter->id > cxld->id)
931 return true;
932
933 return false;
934 }
935
936 static struct cxl_region_ref *
alloc_region_ref(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_decoder * cxld)937 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
938 struct cxl_endpoint_decoder *cxled,
939 struct cxl_decoder *cxld)
940 {
941 struct cxl_region_params *p = &cxlr->params;
942 struct cxl_region_ref *cxl_rr, *iter;
943 unsigned long index;
944 int rc;
945
946 xa_for_each(&port->regions, index, iter) {
947 struct cxl_region_params *ip = &iter->region->params;
948
949 if (!ip->res || ip->res->start < p->res->start)
950 continue;
951
952 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
953 if (auto_order_ok(port, iter->region, cxld))
954 continue;
955 }
956 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
957 dev_name(&port->dev),
958 dev_name(&iter->region->dev), ip->res, p->res);
959
960 return ERR_PTR(-EBUSY);
961 }
962
963 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
964 if (!cxl_rr)
965 return ERR_PTR(-ENOMEM);
966 cxl_rr->port = port;
967 cxl_rr->region = cxlr;
968 cxl_rr->nr_targets = 1;
969 xa_init(&cxl_rr->endpoints);
970
971 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
972 if (rc) {
973 dev_dbg(&cxlr->dev,
974 "%s: failed to track region reference: %d\n",
975 dev_name(&port->dev), rc);
976 kfree(cxl_rr);
977 return ERR_PTR(rc);
978 }
979
980 return cxl_rr;
981 }
982
cxl_rr_free_decoder(struct cxl_region_ref * cxl_rr)983 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
984 {
985 struct cxl_region *cxlr = cxl_rr->region;
986 struct cxl_decoder *cxld = cxl_rr->decoder;
987
988 if (!cxld)
989 return;
990
991 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
992 if (cxld->region == cxlr) {
993 cxld->region = NULL;
994 put_device(&cxlr->dev);
995 }
996 }
997
free_region_ref(struct cxl_region_ref * cxl_rr)998 static void free_region_ref(struct cxl_region_ref *cxl_rr)
999 {
1000 struct cxl_port *port = cxl_rr->port;
1001 struct cxl_region *cxlr = cxl_rr->region;
1002
1003 cxl_rr_free_decoder(cxl_rr);
1004 xa_erase(&port->regions, (unsigned long)cxlr);
1005 xa_destroy(&cxl_rr->endpoints);
1006 kfree(cxl_rr);
1007 }
1008
cxl_rr_ep_add(struct cxl_region_ref * cxl_rr,struct cxl_endpoint_decoder * cxled)1009 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
1010 struct cxl_endpoint_decoder *cxled)
1011 {
1012 int rc;
1013 struct cxl_port *port = cxl_rr->port;
1014 struct cxl_region *cxlr = cxl_rr->region;
1015 struct cxl_decoder *cxld = cxl_rr->decoder;
1016 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
1017
1018 if (ep) {
1019 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
1020 GFP_KERNEL);
1021 if (rc)
1022 return rc;
1023 }
1024 cxl_rr->nr_eps++;
1025
1026 if (!cxld->region) {
1027 cxld->region = cxlr;
1028 get_device(&cxlr->dev);
1029 }
1030
1031 return 0;
1032 }
1033
cxl_rr_assign_decoder(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_region_ref * cxl_rr,struct cxl_decoder * cxld)1034 static int cxl_rr_assign_decoder(struct cxl_port *port, struct cxl_region *cxlr,
1035 struct cxl_endpoint_decoder *cxled,
1036 struct cxl_region_ref *cxl_rr,
1037 struct cxl_decoder *cxld)
1038 {
1039 if (cxld->region) {
1040 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1041 dev_name(&port->dev), dev_name(&cxld->dev),
1042 dev_name(&cxld->region->dev));
1043 return -EBUSY;
1044 }
1045
1046 /*
1047 * Endpoints should already match the region type, but backstop that
1048 * assumption with an assertion. Switch-decoders change mapping-type
1049 * based on what is mapped when they are assigned to a region.
1050 */
1051 dev_WARN_ONCE(&cxlr->dev,
1052 port == cxled_to_port(cxled) &&
1053 cxld->target_type != cxlr->type,
1054 "%s:%s mismatch decoder type %d -> %d\n",
1055 dev_name(&cxled_to_memdev(cxled)->dev),
1056 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1057 cxld->target_type = cxlr->type;
1058 cxl_rr->decoder = cxld;
1059 return 0;
1060 }
1061
1062 /**
1063 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1064 * @port: port to add a new region reference 'struct cxl_region_ref'
1065 * @cxlr: region to attach to @port
1066 * @cxled: endpoint decoder used to create or further pin a region reference
1067 * @pos: interleave position of @cxled in @cxlr
1068 *
1069 * The attach event is an opportunity to validate CXL decode setup
1070 * constraints and record metadata needed for programming HDM decoders,
1071 * in particular decoder target lists.
1072 *
1073 * The steps are:
1074 *
1075 * - validate that there are no other regions with a higher HPA already
1076 * associated with @port
1077 * - establish a region reference if one is not already present
1078 *
1079 * - additionally allocate a decoder instance that will host @cxlr on
1080 * @port
1081 *
1082 * - pin the region reference by the endpoint
1083 * - account for how many entries in @port's target list are needed to
1084 * cover all of the added endpoints.
1085 */
cxl_port_attach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1086 static int cxl_port_attach_region(struct cxl_port *port,
1087 struct cxl_region *cxlr,
1088 struct cxl_endpoint_decoder *cxled, int pos)
1089 {
1090 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1091 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1092 struct cxl_region_ref *cxl_rr;
1093 bool nr_targets_inc = false;
1094 struct cxl_decoder *cxld;
1095 unsigned long index;
1096 int rc = -EBUSY;
1097
1098 lockdep_assert_held_write(&cxl_rwsem.region);
1099
1100 cxl_rr = cxl_rr_load(port, cxlr);
1101 if (cxl_rr) {
1102 struct cxl_ep *ep_iter;
1103 int found = 0;
1104
1105 /*
1106 * Walk the existing endpoints that have been attached to
1107 * @cxlr at @port and see if they share the same 'next' port
1108 * in the downstream direction. I.e. endpoints that share common
1109 * upstream switch.
1110 */
1111 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1112 if (ep_iter == ep)
1113 continue;
1114 if (ep_iter->next == ep->next) {
1115 found++;
1116 break;
1117 }
1118 }
1119
1120 /*
1121 * New target port, or @port is an endpoint port that always
1122 * accounts its own local decode as a target.
1123 */
1124 if (!found || !ep->next) {
1125 cxl_rr->nr_targets++;
1126 nr_targets_inc = true;
1127 }
1128 } else {
1129 struct cxl_decoder *cxld;
1130
1131 cxld = cxl_port_pick_region_decoder(port, cxled, cxlr);
1132 if (!cxld) {
1133 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1134 dev_name(&port->dev));
1135 return -EBUSY;
1136 }
1137
1138 cxl_rr = alloc_region_ref(port, cxlr, cxled, cxld);
1139 if (IS_ERR(cxl_rr)) {
1140 dev_dbg(&cxlr->dev,
1141 "%s: failed to allocate region reference\n",
1142 dev_name(&port->dev));
1143 return PTR_ERR(cxl_rr);
1144 }
1145 nr_targets_inc = true;
1146
1147 rc = cxl_rr_assign_decoder(port, cxlr, cxled, cxl_rr, cxld);
1148 if (rc)
1149 goto out_erase;
1150 }
1151 cxld = cxl_rr->decoder;
1152
1153 /*
1154 * the number of targets should not exceed the target_count
1155 * of the decoder
1156 */
1157 if (is_switch_decoder(&cxld->dev)) {
1158 struct cxl_switch_decoder *cxlsd;
1159
1160 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1161 if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1162 dev_dbg(&cxlr->dev,
1163 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1164 dev_name(port->uport_dev), dev_name(&port->dev),
1165 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1166 dev_name(&cxled->cxld.dev), pos,
1167 cxlsd->nr_targets);
1168 rc = -ENXIO;
1169 goto out_erase;
1170 }
1171 }
1172
1173 rc = cxl_rr_ep_add(cxl_rr, cxled);
1174 if (rc) {
1175 dev_dbg(&cxlr->dev,
1176 "%s: failed to track endpoint %s:%s reference\n",
1177 dev_name(&port->dev), dev_name(&cxlmd->dev),
1178 dev_name(&cxld->dev));
1179 goto out_erase;
1180 }
1181
1182 dev_dbg(&cxlr->dev,
1183 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1184 dev_name(port->uport_dev), dev_name(&port->dev),
1185 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1186 dev_name(&cxled->cxld.dev), pos,
1187 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1188 dev_name(&cxlmd->dev) :
1189 "none",
1190 cxl_rr->nr_eps, cxl_rr->nr_targets);
1191
1192 return 0;
1193 out_erase:
1194 if (nr_targets_inc)
1195 cxl_rr->nr_targets--;
1196 if (cxl_rr->nr_eps == 0)
1197 free_region_ref(cxl_rr);
1198 return rc;
1199 }
1200
cxl_port_detach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1201 static void cxl_port_detach_region(struct cxl_port *port,
1202 struct cxl_region *cxlr,
1203 struct cxl_endpoint_decoder *cxled)
1204 {
1205 struct cxl_region_ref *cxl_rr;
1206 struct cxl_ep *ep = NULL;
1207
1208 lockdep_assert_held_write(&cxl_rwsem.region);
1209
1210 cxl_rr = cxl_rr_load(port, cxlr);
1211 if (!cxl_rr)
1212 return;
1213
1214 /*
1215 * Endpoint ports do not carry cxl_ep references, and they
1216 * never target more than one endpoint by definition
1217 */
1218 if (cxl_rr->decoder == &cxled->cxld)
1219 cxl_rr->nr_eps--;
1220 else
1221 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1222 if (ep) {
1223 struct cxl_ep *ep_iter;
1224 unsigned long index;
1225 int found = 0;
1226
1227 cxl_rr->nr_eps--;
1228 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1229 if (ep_iter->next == ep->next) {
1230 found++;
1231 break;
1232 }
1233 }
1234 if (!found)
1235 cxl_rr->nr_targets--;
1236 }
1237
1238 if (cxl_rr->nr_eps == 0)
1239 free_region_ref(cxl_rr);
1240 }
1241
check_last_peer(struct cxl_endpoint_decoder * cxled,struct cxl_ep * ep,struct cxl_region_ref * cxl_rr,int distance)1242 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1243 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1244 int distance)
1245 {
1246 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1247 struct cxl_region *cxlr = cxl_rr->region;
1248 struct cxl_region_params *p = &cxlr->params;
1249 struct cxl_endpoint_decoder *cxled_peer;
1250 struct cxl_port *port = cxl_rr->port;
1251 struct cxl_memdev *cxlmd_peer;
1252 struct cxl_ep *ep_peer;
1253 int pos = cxled->pos;
1254
1255 /*
1256 * If this position wants to share a dport with the last endpoint mapped
1257 * then that endpoint, at index 'position - distance', must also be
1258 * mapped by this dport.
1259 */
1260 if (pos < distance) {
1261 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1262 dev_name(port->uport_dev), dev_name(&port->dev),
1263 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1264 return -ENXIO;
1265 }
1266 cxled_peer = p->targets[pos - distance];
1267 cxlmd_peer = cxled_to_memdev(cxled_peer);
1268 ep_peer = cxl_ep_load(port, cxlmd_peer);
1269 if (ep->dport != ep_peer->dport) {
1270 dev_dbg(&cxlr->dev,
1271 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1272 dev_name(port->uport_dev), dev_name(&port->dev),
1273 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1274 dev_name(&cxlmd_peer->dev),
1275 dev_name(&cxled_peer->cxld.dev));
1276 return -ENXIO;
1277 }
1278
1279 return 0;
1280 }
1281
check_interleave_cap(struct cxl_decoder * cxld,int iw,int ig)1282 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1283 {
1284 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1285 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1286 unsigned int interleave_mask;
1287 u8 eiw;
1288 u16 eig;
1289 int high_pos, low_pos;
1290
1291 if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1292 return -ENXIO;
1293 /*
1294 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1295 * if eiw < 8:
1296 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1297 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1298 *
1299 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1300 * interleave bits are none.
1301 *
1302 * if eiw >= 8:
1303 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1304 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1305 *
1306 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1307 * interleave bits are none.
1308 */
1309 ways_to_eiw(iw, &eiw);
1310 if (eiw == 0 || eiw == 8)
1311 return 0;
1312
1313 granularity_to_eig(ig, &eig);
1314 if (eiw > 8)
1315 high_pos = eiw + eig - 1;
1316 else
1317 high_pos = eiw + eig + 7;
1318 low_pos = eig + 8;
1319 interleave_mask = GENMASK(high_pos, low_pos);
1320 if (interleave_mask & ~cxlhdm->interleave_mask)
1321 return -ENXIO;
1322
1323 return 0;
1324 }
1325
cxl_port_setup_targets(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1326 static int cxl_port_setup_targets(struct cxl_port *port,
1327 struct cxl_region *cxlr,
1328 struct cxl_endpoint_decoder *cxled)
1329 {
1330 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1331 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1332 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1333 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1334 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1335 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1336 struct cxl_region_params *p = &cxlr->params;
1337 struct cxl_decoder *cxld = cxl_rr->decoder;
1338 struct cxl_switch_decoder *cxlsd;
1339 struct cxl_port *iter = port;
1340 u16 eig, peig;
1341 u8 eiw, peiw;
1342
1343 /*
1344 * While root level decoders support x3, x6, x12, switch level
1345 * decoders only support powers of 2 up to x16.
1346 */
1347 if (!is_power_of_2(cxl_rr->nr_targets)) {
1348 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1349 dev_name(port->uport_dev), dev_name(&port->dev),
1350 cxl_rr->nr_targets);
1351 return -EINVAL;
1352 }
1353
1354 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1355 if (cxl_rr->nr_targets_set) {
1356 int i, distance = 1;
1357 struct cxl_region_ref *cxl_rr_iter;
1358
1359 /*
1360 * The "distance" between peer downstream ports represents which
1361 * endpoint positions in the region interleave a given port can
1362 * host.
1363 *
1364 * For example, at the root of a hierarchy the distance is
1365 * always 1 as every index targets a different host-bridge. At
1366 * each subsequent switch level those ports map every Nth region
1367 * position where N is the width of the switch == distance.
1368 */
1369 do {
1370 cxl_rr_iter = cxl_rr_load(iter, cxlr);
1371 distance *= cxl_rr_iter->nr_targets;
1372 iter = to_cxl_port(iter->dev.parent);
1373 } while (!is_cxl_root(iter));
1374 distance *= cxlrd->cxlsd.cxld.interleave_ways;
1375
1376 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1377 if (ep->dport == cxlsd->target[i]) {
1378 rc = check_last_peer(cxled, ep, cxl_rr,
1379 distance);
1380 if (rc)
1381 return rc;
1382 goto out_target_set;
1383 }
1384 goto add_target;
1385 }
1386
1387 if (is_cxl_root(parent_port)) {
1388 /*
1389 * Root decoder IG is always set to value in CFMWS which
1390 * may be different than this region's IG. We can use the
1391 * region's IG here since interleave_granularity_store()
1392 * does not allow interleaved host-bridges with
1393 * root IG != region IG.
1394 */
1395 parent_ig = p->interleave_granularity;
1396 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1397 /*
1398 * For purposes of address bit routing, use power-of-2 math for
1399 * switch ports.
1400 */
1401 if (!is_power_of_2(parent_iw))
1402 parent_iw /= 3;
1403 } else {
1404 struct cxl_region_ref *parent_rr;
1405 struct cxl_decoder *parent_cxld;
1406
1407 parent_rr = cxl_rr_load(parent_port, cxlr);
1408 parent_cxld = parent_rr->decoder;
1409 parent_ig = parent_cxld->interleave_granularity;
1410 parent_iw = parent_cxld->interleave_ways;
1411 }
1412
1413 rc = granularity_to_eig(parent_ig, &peig);
1414 if (rc) {
1415 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1416 dev_name(parent_port->uport_dev),
1417 dev_name(&parent_port->dev), parent_ig);
1418 return rc;
1419 }
1420
1421 rc = ways_to_eiw(parent_iw, &peiw);
1422 if (rc) {
1423 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1424 dev_name(parent_port->uport_dev),
1425 dev_name(&parent_port->dev), parent_iw);
1426 return rc;
1427 }
1428
1429 iw = cxl_rr->nr_targets;
1430 rc = ways_to_eiw(iw, &eiw);
1431 if (rc) {
1432 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1433 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1434 return rc;
1435 }
1436
1437 /*
1438 * Interleave granularity is a multiple of @parent_port granularity.
1439 * Multiplier is the parent port interleave ways.
1440 */
1441 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1442 if (rc) {
1443 dev_dbg(&cxlr->dev,
1444 "%s: invalid granularity calculation (%d * %d)\n",
1445 dev_name(&parent_port->dev), parent_ig, parent_iw);
1446 return rc;
1447 }
1448
1449 rc = eig_to_granularity(eig, &ig);
1450 if (rc) {
1451 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1452 dev_name(port->uport_dev), dev_name(&port->dev),
1453 256 << eig);
1454 return rc;
1455 }
1456
1457 if (iw > 8 || iw > cxlsd->nr_targets) {
1458 dev_dbg(&cxlr->dev,
1459 "%s:%s:%s: ways: %d overflows targets: %d\n",
1460 dev_name(port->uport_dev), dev_name(&port->dev),
1461 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1462 return -ENXIO;
1463 }
1464
1465 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1466 if (cxld->interleave_ways != iw ||
1467 (iw > 1 && cxld->interleave_granularity != ig) ||
1468 !region_res_match_cxl_range(p, &cxld->hpa_range) ||
1469 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1470 dev_err(&cxlr->dev,
1471 "%s:%s %s expected iw: %d ig: %d %pr\n",
1472 dev_name(port->uport_dev), dev_name(&port->dev),
1473 __func__, iw, ig, p->res);
1474 dev_err(&cxlr->dev,
1475 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1476 dev_name(port->uport_dev), dev_name(&port->dev),
1477 __func__, cxld->interleave_ways,
1478 cxld->interleave_granularity,
1479 str_enabled_disabled(cxld->flags & CXL_DECODER_F_ENABLE),
1480 cxld->hpa_range.start, cxld->hpa_range.end);
1481 return -ENXIO;
1482 }
1483 } else {
1484 rc = check_interleave_cap(cxld, iw, ig);
1485 if (rc) {
1486 dev_dbg(&cxlr->dev,
1487 "%s:%s iw: %d ig: %d is not supported\n",
1488 dev_name(port->uport_dev),
1489 dev_name(&port->dev), iw, ig);
1490 return rc;
1491 }
1492
1493 cxld->interleave_ways = iw;
1494 cxld->interleave_granularity = ig;
1495 cxld->hpa_range = (struct range) {
1496 .start = p->res->start,
1497 .end = p->res->end,
1498 };
1499 }
1500 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1501 dev_name(&port->dev), iw, ig);
1502 add_target:
1503 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1504 dev_dbg(&cxlr->dev,
1505 "%s:%s: targets full trying to add %s:%s at %d\n",
1506 dev_name(port->uport_dev), dev_name(&port->dev),
1507 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1508 return -ENXIO;
1509 }
1510 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1511 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1512 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1513 dev_name(port->uport_dev), dev_name(&port->dev),
1514 dev_name(&cxlsd->cxld.dev),
1515 dev_name(ep->dport->dport_dev),
1516 cxl_rr->nr_targets_set);
1517 return -ENXIO;
1518 }
1519 } else {
1520 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1521 cxlsd->cxld.target_map[cxl_rr->nr_targets_set] = ep->dport->port_id;
1522 }
1523 inc = 1;
1524 out_target_set:
1525 cxl_rr->nr_targets_set += inc;
1526 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1527 dev_name(port->uport_dev), dev_name(&port->dev),
1528 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1529 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1530
1531 return 0;
1532 }
1533
cxl_port_reset_targets(struct cxl_port * port,struct cxl_region * cxlr)1534 static void cxl_port_reset_targets(struct cxl_port *port,
1535 struct cxl_region *cxlr)
1536 {
1537 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1538 struct cxl_decoder *cxld;
1539
1540 /*
1541 * After the last endpoint has been detached the entire cxl_rr may now
1542 * be gone.
1543 */
1544 if (!cxl_rr)
1545 return;
1546 cxl_rr->nr_targets_set = 0;
1547
1548 cxld = cxl_rr->decoder;
1549 cxld->hpa_range = (struct range) {
1550 .start = 0,
1551 .end = -1,
1552 };
1553 }
1554
cxl_region_teardown_targets(struct cxl_region * cxlr)1555 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1556 {
1557 struct cxl_region_params *p = &cxlr->params;
1558 struct cxl_endpoint_decoder *cxled;
1559 struct cxl_dev_state *cxlds;
1560 struct cxl_memdev *cxlmd;
1561 struct cxl_port *iter;
1562 struct cxl_ep *ep;
1563 int i;
1564
1565 /*
1566 * In the auto-discovery case skip automatic teardown since the
1567 * address space is already active
1568 */
1569 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1570 return;
1571
1572 for (i = 0; i < p->nr_targets; i++) {
1573 cxled = p->targets[i];
1574 cxlmd = cxled_to_memdev(cxled);
1575 cxlds = cxlmd->cxlds;
1576
1577 if (cxlds->rcd)
1578 continue;
1579
1580 iter = cxled_to_port(cxled);
1581 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1582 iter = to_cxl_port(iter->dev.parent);
1583
1584 for (ep = cxl_ep_load(iter, cxlmd); iter;
1585 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1586 cxl_port_reset_targets(iter, cxlr);
1587 }
1588 }
1589
cxl_region_setup_targets(struct cxl_region * cxlr)1590 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1591 {
1592 struct cxl_region_params *p = &cxlr->params;
1593 struct cxl_endpoint_decoder *cxled;
1594 struct cxl_dev_state *cxlds;
1595 int i, rc, rch = 0, vh = 0;
1596 struct cxl_memdev *cxlmd;
1597 struct cxl_port *iter;
1598 struct cxl_ep *ep;
1599
1600 for (i = 0; i < p->nr_targets; i++) {
1601 cxled = p->targets[i];
1602 cxlmd = cxled_to_memdev(cxled);
1603 cxlds = cxlmd->cxlds;
1604
1605 /* validate that all targets agree on topology */
1606 if (!cxlds->rcd) {
1607 vh++;
1608 } else {
1609 rch++;
1610 continue;
1611 }
1612
1613 iter = cxled_to_port(cxled);
1614 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1615 iter = to_cxl_port(iter->dev.parent);
1616
1617 /*
1618 * Descend the topology tree programming / validating
1619 * targets while looking for conflicts.
1620 */
1621 for (ep = cxl_ep_load(iter, cxlmd); iter;
1622 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1623 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1624 if (rc) {
1625 cxl_region_teardown_targets(cxlr);
1626 return rc;
1627 }
1628 }
1629 }
1630
1631 if (rch && vh) {
1632 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1633 cxl_region_teardown_targets(cxlr);
1634 return -ENXIO;
1635 }
1636
1637 return 0;
1638 }
1639
cxl_region_validate_position(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1640 static int cxl_region_validate_position(struct cxl_region *cxlr,
1641 struct cxl_endpoint_decoder *cxled,
1642 int pos)
1643 {
1644 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1645 struct cxl_region_params *p = &cxlr->params;
1646 int i;
1647
1648 if (pos < 0 || pos >= p->interleave_ways) {
1649 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1650 p->interleave_ways);
1651 return -ENXIO;
1652 }
1653
1654 if (p->targets[pos] == cxled)
1655 return 0;
1656
1657 if (p->targets[pos]) {
1658 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1659 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1660
1661 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1662 pos, dev_name(&cxlmd_target->dev),
1663 dev_name(&cxled_target->cxld.dev));
1664 return -EBUSY;
1665 }
1666
1667 for (i = 0; i < p->interleave_ways; i++) {
1668 struct cxl_endpoint_decoder *cxled_target;
1669 struct cxl_memdev *cxlmd_target;
1670
1671 cxled_target = p->targets[i];
1672 if (!cxled_target)
1673 continue;
1674
1675 cxlmd_target = cxled_to_memdev(cxled_target);
1676 if (cxlmd_target == cxlmd) {
1677 dev_dbg(&cxlr->dev,
1678 "%s already specified at position %d via: %s\n",
1679 dev_name(&cxlmd->dev), pos,
1680 dev_name(&cxled_target->cxld.dev));
1681 return -EBUSY;
1682 }
1683 }
1684
1685 return 0;
1686 }
1687
cxl_region_attach_position(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled,const struct cxl_dport * dport,int pos)1688 static int cxl_region_attach_position(struct cxl_region *cxlr,
1689 struct cxl_root_decoder *cxlrd,
1690 struct cxl_endpoint_decoder *cxled,
1691 const struct cxl_dport *dport, int pos)
1692 {
1693 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1694 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1695 struct cxl_decoder *cxld = &cxlsd->cxld;
1696 int iw = cxld->interleave_ways;
1697 struct cxl_port *iter;
1698 int rc;
1699
1700 if (dport != cxlrd->cxlsd.target[pos % iw]) {
1701 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1702 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1703 dev_name(&cxlrd->cxlsd.cxld.dev));
1704 return -ENXIO;
1705 }
1706
1707 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1708 iter = to_cxl_port(iter->dev.parent)) {
1709 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1710 if (rc)
1711 goto err;
1712 }
1713
1714 return 0;
1715
1716 err:
1717 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1718 iter = to_cxl_port(iter->dev.parent))
1719 cxl_port_detach_region(iter, cxlr, cxled);
1720 return rc;
1721 }
1722
cxl_region_attach_auto(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1723 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1724 struct cxl_endpoint_decoder *cxled, int pos)
1725 {
1726 struct cxl_region_params *p = &cxlr->params;
1727
1728 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1729 dev_err(&cxlr->dev,
1730 "%s: unable to add decoder to autodetected region\n",
1731 dev_name(&cxled->cxld.dev));
1732 return -EINVAL;
1733 }
1734
1735 if (pos >= 0) {
1736 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1737 dev_name(&cxled->cxld.dev), pos);
1738 return -EINVAL;
1739 }
1740
1741 if (p->nr_targets >= p->interleave_ways) {
1742 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1743 dev_name(&cxled->cxld.dev));
1744 return -ENXIO;
1745 }
1746
1747 /*
1748 * Temporarily record the endpoint decoder into the target array. Yes,
1749 * this means that userspace can view devices in the wrong position
1750 * before the region activates, and must be careful to understand when
1751 * it might be racing region autodiscovery.
1752 */
1753 pos = p->nr_targets;
1754 p->targets[pos] = cxled;
1755 cxled->pos = pos;
1756 p->nr_targets++;
1757
1758 return 0;
1759 }
1760
cmp_interleave_pos(const void * a,const void * b)1761 static int cmp_interleave_pos(const void *a, const void *b)
1762 {
1763 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1764 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1765
1766 return cxled_a->pos - cxled_b->pos;
1767 }
1768
match_switch_decoder_by_range(struct device * dev,const void * data)1769 static int match_switch_decoder_by_range(struct device *dev,
1770 const void *data)
1771 {
1772 struct cxl_switch_decoder *cxlsd;
1773 const struct range *r1, *r2 = data;
1774
1775
1776 if (!is_switch_decoder(dev))
1777 return 0;
1778
1779 cxlsd = to_cxl_switch_decoder(dev);
1780 r1 = &cxlsd->cxld.hpa_range;
1781
1782 if (is_root_decoder(dev))
1783 return range_contains(r1, r2);
1784 return (r1->start == r2->start && r1->end == r2->end);
1785 }
1786
find_pos_and_ways(struct cxl_port * port,struct range * range,int * pos,int * ways)1787 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1788 int *pos, int *ways)
1789 {
1790 struct cxl_switch_decoder *cxlsd;
1791 struct cxl_port *parent;
1792 struct device *dev;
1793 int rc = -ENXIO;
1794
1795 parent = parent_port_of(port);
1796 if (!parent)
1797 return rc;
1798
1799 dev = device_find_child(&parent->dev, range,
1800 match_switch_decoder_by_range);
1801 if (!dev) {
1802 dev_err(port->uport_dev,
1803 "failed to find decoder mapping %#llx-%#llx\n",
1804 range->start, range->end);
1805 return rc;
1806 }
1807 cxlsd = to_cxl_switch_decoder(dev);
1808 *ways = cxlsd->cxld.interleave_ways;
1809
1810 for (int i = 0; i < *ways; i++) {
1811 if (cxlsd->target[i] == port->parent_dport) {
1812 *pos = i;
1813 rc = 0;
1814 break;
1815 }
1816 }
1817 put_device(dev);
1818
1819 if (rc)
1820 dev_err(port->uport_dev,
1821 "failed to find %s:%s in target list of %s\n",
1822 dev_name(&port->dev),
1823 dev_name(port->parent_dport->dport_dev),
1824 dev_name(&cxlsd->cxld.dev));
1825
1826 return rc;
1827 }
1828
1829 /**
1830 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1831 * @cxled: endpoint decoder member of given region
1832 *
1833 * The endpoint position is calculated by traversing the topology from
1834 * the endpoint to the root decoder and iteratively applying this
1835 * calculation:
1836 *
1837 * position = position * parent_ways + parent_pos;
1838 *
1839 * ...where @position is inferred from switch and root decoder target lists.
1840 *
1841 * Return: position >= 0 on success
1842 * -ENXIO on failure
1843 */
cxl_calc_interleave_pos(struct cxl_endpoint_decoder * cxled)1844 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1845 {
1846 struct cxl_port *iter, *port = cxled_to_port(cxled);
1847 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1848 struct range *range = &cxled->cxld.hpa_range;
1849 int parent_ways = 0, parent_pos = 0, pos = 0;
1850 int rc;
1851
1852 /*
1853 * Example: the expected interleave order of the 4-way region shown
1854 * below is: mem0, mem2, mem1, mem3
1855 *
1856 * root_port
1857 * / \
1858 * host_bridge_0 host_bridge_1
1859 * | | | |
1860 * mem0 mem1 mem2 mem3
1861 *
1862 * In the example the calculator will iterate twice. The first iteration
1863 * uses the mem position in the host-bridge and the ways of the host-
1864 * bridge to generate the first, or local, position. The second
1865 * iteration uses the host-bridge position in the root_port and the ways
1866 * of the root_port to refine the position.
1867 *
1868 * A trace of the calculation per endpoint looks like this:
1869 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1870 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1871 * pos: 0 pos: 1
1872 *
1873 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1874 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1875 * pos: 2 pos = 3
1876 *
1877 * Note that while this example is simple, the method applies to more
1878 * complex topologies, including those with switches.
1879 */
1880
1881 /* Iterate from endpoint to root_port refining the position */
1882 for (iter = port; iter; iter = parent_port_of(iter)) {
1883 if (is_cxl_root(iter))
1884 break;
1885
1886 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1887 if (rc)
1888 return rc;
1889
1890 pos = pos * parent_ways + parent_pos;
1891 }
1892
1893 dev_dbg(&cxlmd->dev,
1894 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1895 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1896 dev_name(&port->dev), range->start, range->end, pos);
1897
1898 return pos;
1899 }
1900
cxl_region_sort_targets(struct cxl_region * cxlr)1901 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1902 {
1903 struct cxl_region_params *p = &cxlr->params;
1904 int i, rc = 0;
1905
1906 for (i = 0; i < p->nr_targets; i++) {
1907 struct cxl_endpoint_decoder *cxled = p->targets[i];
1908
1909 cxled->pos = cxl_calc_interleave_pos(cxled);
1910 /*
1911 * Record that sorting failed, but still continue to calc
1912 * cxled->pos so that follow-on code paths can reliably
1913 * do p->targets[cxled->pos] to self-reference their entry.
1914 */
1915 if (cxled->pos < 0)
1916 rc = -ENXIO;
1917 }
1918 /* Keep the cxlr target list in interleave position order */
1919 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1920 cmp_interleave_pos, NULL);
1921
1922 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1923 return rc;
1924 }
1925
cxl_region_attach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1926 static int cxl_region_attach(struct cxl_region *cxlr,
1927 struct cxl_endpoint_decoder *cxled, int pos)
1928 {
1929 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1930 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1931 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1932 struct cxl_region_params *p = &cxlr->params;
1933 struct cxl_port *ep_port, *root_port;
1934 struct cxl_dport *dport;
1935 int rc = -ENXIO;
1936
1937 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1938 p->interleave_granularity);
1939 if (rc) {
1940 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1941 dev_name(&cxled->cxld.dev), p->interleave_ways,
1942 p->interleave_granularity);
1943 return rc;
1944 }
1945
1946 if (cxled->part < 0) {
1947 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1948 return -ENODEV;
1949 }
1950
1951 if (cxlds->part[cxled->part].mode != cxlr->mode) {
1952 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch\n",
1953 dev_name(&cxled->cxld.dev), cxlr->mode);
1954 return -EINVAL;
1955 }
1956
1957 /* all full of members, or interleave config not established? */
1958 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1959 dev_dbg(&cxlr->dev, "region already active\n");
1960 return -EBUSY;
1961 }
1962
1963 if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1964 dev_dbg(&cxlr->dev, "interleave config missing\n");
1965 return -ENXIO;
1966 }
1967
1968 if (p->nr_targets >= p->interleave_ways) {
1969 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1970 p->nr_targets);
1971 return -EINVAL;
1972 }
1973
1974 ep_port = cxled_to_port(cxled);
1975 root_port = cxlrd_to_port(cxlrd);
1976 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1977 if (!dport) {
1978 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1979 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1980 dev_name(cxlr->dev.parent));
1981 return -ENXIO;
1982 }
1983
1984 if (cxled->cxld.target_type != cxlr->type) {
1985 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1986 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1987 cxled->cxld.target_type, cxlr->type);
1988 return -ENXIO;
1989 }
1990
1991 if (!cxled->dpa_res) {
1992 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1993 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1994 return -ENXIO;
1995 }
1996
1997 if (resource_size(cxled->dpa_res) * p->interleave_ways + p->cache_size !=
1998 resource_size(p->res)) {
1999 dev_dbg(&cxlr->dev,
2000 "%s:%s-size-%#llx * ways-%d + cache-%#llx != region-size-%#llx\n",
2001 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2002 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
2003 (u64)p->cache_size, (u64)resource_size(p->res));
2004 return -EINVAL;
2005 }
2006
2007 cxl_region_perf_data_calculate(cxlr, cxled);
2008
2009 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
2010 int i;
2011
2012 rc = cxl_region_attach_auto(cxlr, cxled, pos);
2013 if (rc)
2014 return rc;
2015
2016 /* await more targets to arrive... */
2017 if (p->nr_targets < p->interleave_ways)
2018 return 0;
2019
2020 /*
2021 * All targets are here, which implies all PCI enumeration that
2022 * affects this region has been completed. Walk the topology to
2023 * sort the devices into their relative region decode position.
2024 */
2025 rc = cxl_region_sort_targets(cxlr);
2026 if (rc)
2027 return rc;
2028
2029 for (i = 0; i < p->nr_targets; i++) {
2030 cxled = p->targets[i];
2031 ep_port = cxled_to_port(cxled);
2032 dport = cxl_find_dport_by_dev(root_port,
2033 ep_port->host_bridge);
2034 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
2035 dport, i);
2036 if (rc)
2037 return rc;
2038 }
2039
2040 rc = cxl_region_setup_targets(cxlr);
2041 if (rc)
2042 return rc;
2043
2044 /*
2045 * If target setup succeeds in the autodiscovery case
2046 * then the region is already committed.
2047 */
2048 p->state = CXL_CONFIG_COMMIT;
2049 cxl_region_shared_upstream_bandwidth_update(cxlr);
2050
2051 return 0;
2052 }
2053
2054 rc = cxl_region_validate_position(cxlr, cxled, pos);
2055 if (rc)
2056 return rc;
2057
2058 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2059 if (rc)
2060 return rc;
2061
2062 p->targets[pos] = cxled;
2063 cxled->pos = pos;
2064 p->nr_targets++;
2065
2066 if (p->nr_targets == p->interleave_ways) {
2067 rc = cxl_region_setup_targets(cxlr);
2068 if (rc)
2069 return rc;
2070 p->state = CXL_CONFIG_ACTIVE;
2071 cxl_region_shared_upstream_bandwidth_update(cxlr);
2072 }
2073
2074 cxled->cxld.interleave_ways = p->interleave_ways;
2075 cxled->cxld.interleave_granularity = p->interleave_granularity;
2076 cxled->cxld.hpa_range = (struct range) {
2077 .start = p->res->start,
2078 .end = p->res->end,
2079 };
2080
2081 if (p->nr_targets != p->interleave_ways)
2082 return 0;
2083
2084 /*
2085 * Test the auto-discovery position calculator function
2086 * against this successfully created user-defined region.
2087 * A fail message here means that this interleave config
2088 * will fail when presented as CXL_REGION_F_AUTO.
2089 */
2090 for (int i = 0; i < p->nr_targets; i++) {
2091 struct cxl_endpoint_decoder *cxled = p->targets[i];
2092 int test_pos;
2093
2094 test_pos = cxl_calc_interleave_pos(cxled);
2095 dev_dbg(&cxled->cxld.dev,
2096 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2097 (test_pos == cxled->pos) ? "success" : "fail",
2098 test_pos, cxled->pos);
2099 }
2100
2101 return 0;
2102 }
2103
2104 static struct cxl_region *
__cxl_decoder_detach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,enum cxl_detach_mode mode)2105 __cxl_decoder_detach(struct cxl_region *cxlr,
2106 struct cxl_endpoint_decoder *cxled, int pos,
2107 enum cxl_detach_mode mode)
2108 {
2109 struct cxl_region_params *p;
2110
2111 lockdep_assert_held_write(&cxl_rwsem.region);
2112
2113 if (!cxled) {
2114 p = &cxlr->params;
2115
2116 if (pos >= p->interleave_ways) {
2117 dev_dbg(&cxlr->dev, "position %d out of range %d\n",
2118 pos, p->interleave_ways);
2119 return NULL;
2120 }
2121
2122 if (!p->targets[pos])
2123 return NULL;
2124 cxled = p->targets[pos];
2125 } else {
2126 cxlr = cxled->cxld.region;
2127 if (!cxlr)
2128 return NULL;
2129 p = &cxlr->params;
2130 }
2131
2132 if (mode == DETACH_INVALIDATE)
2133 cxled->part = -1;
2134
2135 if (p->state > CXL_CONFIG_ACTIVE) {
2136 cxl_region_decode_reset(cxlr, p->interleave_ways);
2137 p->state = CXL_CONFIG_ACTIVE;
2138 }
2139
2140 for (struct cxl_port *iter = cxled_to_port(cxled); !is_cxl_root(iter);
2141 iter = to_cxl_port(iter->dev.parent))
2142 cxl_port_detach_region(iter, cxlr, cxled);
2143
2144 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2145 p->targets[cxled->pos] != cxled) {
2146 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2147
2148 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2149 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2150 cxled->pos);
2151 return NULL;
2152 }
2153
2154 if (p->state == CXL_CONFIG_ACTIVE) {
2155 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2156 cxl_region_teardown_targets(cxlr);
2157 }
2158 p->targets[cxled->pos] = NULL;
2159 p->nr_targets--;
2160 cxled->cxld.hpa_range = (struct range) {
2161 .start = 0,
2162 .end = -1,
2163 };
2164
2165 get_device(&cxlr->dev);
2166 return cxlr;
2167 }
2168
2169 /*
2170 * Cleanup a decoder's interest in a region. There are 2 cases to
2171 * handle, removing an unknown @cxled from a known position in a region
2172 * (detach_target()) or removing a known @cxled from an unknown @cxlr
2173 * (cxld_unregister())
2174 *
2175 * When the detachment finds a region release the region driver.
2176 */
cxl_decoder_detach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,enum cxl_detach_mode mode)2177 int cxl_decoder_detach(struct cxl_region *cxlr,
2178 struct cxl_endpoint_decoder *cxled, int pos,
2179 enum cxl_detach_mode mode)
2180 {
2181 struct cxl_region *detach;
2182
2183 /* when the decoder is being destroyed lock unconditionally */
2184 if (mode == DETACH_INVALIDATE) {
2185 guard(rwsem_write)(&cxl_rwsem.region);
2186 detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
2187 } else {
2188 int rc;
2189
2190 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
2191 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
2192 return rc;
2193 detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
2194 }
2195
2196 if (detach) {
2197 device_release_driver(&detach->dev);
2198 put_device(&detach->dev);
2199 }
2200 return 0;
2201 }
2202
__attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)2203 static int __attach_target(struct cxl_region *cxlr,
2204 struct cxl_endpoint_decoder *cxled, int pos,
2205 unsigned int state)
2206 {
2207 int rc;
2208
2209 if (state == TASK_INTERRUPTIBLE) {
2210 ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
2211 if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
2212 return rc;
2213 guard(rwsem_read)(&cxl_rwsem.dpa);
2214 return cxl_region_attach(cxlr, cxled, pos);
2215 }
2216 guard(rwsem_write)(&cxl_rwsem.region);
2217 guard(rwsem_read)(&cxl_rwsem.dpa);
2218 return cxl_region_attach(cxlr, cxled, pos);
2219 }
2220
attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)2221 static int attach_target(struct cxl_region *cxlr,
2222 struct cxl_endpoint_decoder *cxled, int pos,
2223 unsigned int state)
2224 {
2225 int rc = __attach_target(cxlr, cxled, pos, state);
2226
2227 if (rc == 0)
2228 return 0;
2229
2230 dev_warn(cxled->cxld.dev.parent, "failed to attach %s to %s: %d\n",
2231 dev_name(&cxled->cxld.dev), dev_name(&cxlr->dev), rc);
2232 return rc;
2233 }
2234
detach_target(struct cxl_region * cxlr,int pos)2235 static int detach_target(struct cxl_region *cxlr, int pos)
2236 {
2237 return cxl_decoder_detach(cxlr, NULL, pos, DETACH_ONLY);
2238 }
2239
store_targetN(struct cxl_region * cxlr,const char * buf,int pos,size_t len)2240 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2241 size_t len)
2242 {
2243 int rc;
2244
2245 if (sysfs_streq(buf, "\n"))
2246 rc = detach_target(cxlr, pos);
2247 else {
2248 struct device *dev;
2249
2250 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2251 if (!dev)
2252 return -ENODEV;
2253
2254 if (!is_endpoint_decoder(dev)) {
2255 rc = -EINVAL;
2256 goto out;
2257 }
2258
2259 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2260 TASK_INTERRUPTIBLE);
2261 out:
2262 put_device(dev);
2263 }
2264
2265 if (rc < 0)
2266 return rc;
2267 return len;
2268 }
2269
2270 #define TARGET_ATTR_RW(n) \
2271 static ssize_t target##n##_show( \
2272 struct device *dev, struct device_attribute *attr, char *buf) \
2273 { \
2274 return show_targetN(to_cxl_region(dev), buf, (n)); \
2275 } \
2276 static ssize_t target##n##_store(struct device *dev, \
2277 struct device_attribute *attr, \
2278 const char *buf, size_t len) \
2279 { \
2280 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2281 } \
2282 static DEVICE_ATTR_RW(target##n)
2283
2284 TARGET_ATTR_RW(0);
2285 TARGET_ATTR_RW(1);
2286 TARGET_ATTR_RW(2);
2287 TARGET_ATTR_RW(3);
2288 TARGET_ATTR_RW(4);
2289 TARGET_ATTR_RW(5);
2290 TARGET_ATTR_RW(6);
2291 TARGET_ATTR_RW(7);
2292 TARGET_ATTR_RW(8);
2293 TARGET_ATTR_RW(9);
2294 TARGET_ATTR_RW(10);
2295 TARGET_ATTR_RW(11);
2296 TARGET_ATTR_RW(12);
2297 TARGET_ATTR_RW(13);
2298 TARGET_ATTR_RW(14);
2299 TARGET_ATTR_RW(15);
2300
2301 static struct attribute *target_attrs[] = {
2302 &dev_attr_target0.attr,
2303 &dev_attr_target1.attr,
2304 &dev_attr_target2.attr,
2305 &dev_attr_target3.attr,
2306 &dev_attr_target4.attr,
2307 &dev_attr_target5.attr,
2308 &dev_attr_target6.attr,
2309 &dev_attr_target7.attr,
2310 &dev_attr_target8.attr,
2311 &dev_attr_target9.attr,
2312 &dev_attr_target10.attr,
2313 &dev_attr_target11.attr,
2314 &dev_attr_target12.attr,
2315 &dev_attr_target13.attr,
2316 &dev_attr_target14.attr,
2317 &dev_attr_target15.attr,
2318 NULL,
2319 };
2320
cxl_region_target_visible(struct kobject * kobj,struct attribute * a,int n)2321 static umode_t cxl_region_target_visible(struct kobject *kobj,
2322 struct attribute *a, int n)
2323 {
2324 struct device *dev = kobj_to_dev(kobj);
2325 struct cxl_region *cxlr = to_cxl_region(dev);
2326 struct cxl_region_params *p = &cxlr->params;
2327
2328 if (n < p->interleave_ways)
2329 return a->mode;
2330 return 0;
2331 }
2332
2333 static const struct attribute_group cxl_region_target_group = {
2334 .attrs = target_attrs,
2335 .is_visible = cxl_region_target_visible,
2336 };
2337
get_cxl_region_target_group(void)2338 static const struct attribute_group *get_cxl_region_target_group(void)
2339 {
2340 return &cxl_region_target_group;
2341 }
2342
2343 static const struct attribute_group *region_groups[] = {
2344 &cxl_base_attribute_group,
2345 &cxl_region_group,
2346 &cxl_region_target_group,
2347 &cxl_region_access0_coordinate_group,
2348 &cxl_region_access1_coordinate_group,
2349 NULL,
2350 };
2351
cxl_region_release(struct device * dev)2352 static void cxl_region_release(struct device *dev)
2353 {
2354 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2355 struct cxl_region *cxlr = to_cxl_region(dev);
2356 int id = atomic_read(&cxlrd->region_id);
2357
2358 /*
2359 * Try to reuse the recently idled id rather than the cached
2360 * next id to prevent the region id space from increasing
2361 * unnecessarily.
2362 */
2363 if (cxlr->id < id)
2364 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2365 memregion_free(id);
2366 goto out;
2367 }
2368
2369 memregion_free(cxlr->id);
2370 out:
2371 put_device(dev->parent);
2372 kfree(cxlr);
2373 }
2374
2375 const struct device_type cxl_region_type = {
2376 .name = "cxl_region",
2377 .release = cxl_region_release,
2378 .groups = region_groups
2379 };
2380
is_cxl_region(struct device * dev)2381 bool is_cxl_region(struct device *dev)
2382 {
2383 return dev->type == &cxl_region_type;
2384 }
2385 EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2386
to_cxl_region(struct device * dev)2387 static struct cxl_region *to_cxl_region(struct device *dev)
2388 {
2389 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2390 "not a cxl_region device\n"))
2391 return NULL;
2392
2393 return container_of(dev, struct cxl_region, dev);
2394 }
2395
unregister_region(void * _cxlr)2396 static void unregister_region(void *_cxlr)
2397 {
2398 struct cxl_region *cxlr = _cxlr;
2399 struct cxl_region_params *p = &cxlr->params;
2400 int i;
2401
2402 device_del(&cxlr->dev);
2403
2404 /*
2405 * Now that region sysfs is shutdown, the parameter block is now
2406 * read-only, so no need to hold the region rwsem to access the
2407 * region parameters.
2408 */
2409 for (i = 0; i < p->interleave_ways; i++)
2410 detach_target(cxlr, i);
2411
2412 cxl_region_iomem_release(cxlr);
2413 put_device(&cxlr->dev);
2414 }
2415
2416 static struct lock_class_key cxl_region_key;
2417
cxl_region_alloc(struct cxl_root_decoder * cxlrd,int id)2418 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2419 {
2420 struct cxl_region *cxlr;
2421 struct device *dev;
2422
2423 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2424 if (!cxlr) {
2425 memregion_free(id);
2426 return ERR_PTR(-ENOMEM);
2427 }
2428
2429 dev = &cxlr->dev;
2430 device_initialize(dev);
2431 lockdep_set_class(&dev->mutex, &cxl_region_key);
2432 dev->parent = &cxlrd->cxlsd.cxld.dev;
2433 /*
2434 * Keep root decoder pinned through cxl_region_release to fixup
2435 * region id allocations
2436 */
2437 get_device(dev->parent);
2438 device_set_pm_not_required(dev);
2439 dev->bus = &cxl_bus_type;
2440 dev->type = &cxl_region_type;
2441 cxlr->id = id;
2442
2443 return cxlr;
2444 }
2445
cxl_region_update_coordinates(struct cxl_region * cxlr,int nid)2446 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2447 {
2448 int cset = 0;
2449 int rc;
2450
2451 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2452 if (cxlr->coord[i].read_bandwidth) {
2453 node_update_perf_attrs(nid, &cxlr->coord[i], i);
2454 cset++;
2455 }
2456 }
2457
2458 if (!cset)
2459 return false;
2460
2461 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2462 if (rc)
2463 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2464
2465 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2466 if (rc)
2467 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2468
2469 return true;
2470 }
2471
cxl_region_perf_attrs_callback(struct notifier_block * nb,unsigned long action,void * arg)2472 static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2473 unsigned long action, void *arg)
2474 {
2475 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2476 node_notifier);
2477 struct node_notify *nn = arg;
2478 int nid = nn->nid;
2479 int region_nid;
2480
2481 if (action != NODE_ADDED_FIRST_MEMORY)
2482 return NOTIFY_DONE;
2483
2484 /*
2485 * No need to hold cxl_rwsem.region; region parameters are stable
2486 * within the cxl_region driver.
2487 */
2488 region_nid = phys_to_target_node(cxlr->params.res->start);
2489 if (nid != region_nid)
2490 return NOTIFY_DONE;
2491
2492 /* No action needed if node bit already set */
2493 if (node_test_and_set(nid, nodemask_region_seen))
2494 return NOTIFY_DONE;
2495
2496 if (!cxl_region_update_coordinates(cxlr, nid))
2497 return NOTIFY_DONE;
2498
2499 return NOTIFY_OK;
2500 }
2501
cxl_region_calculate_adistance(struct notifier_block * nb,unsigned long nid,void * data)2502 static int cxl_region_calculate_adistance(struct notifier_block *nb,
2503 unsigned long nid, void *data)
2504 {
2505 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2506 adist_notifier);
2507 struct access_coordinate *perf;
2508 int *adist = data;
2509 int region_nid;
2510
2511 /*
2512 * No need to hold cxl_rwsem.region; region parameters are stable
2513 * within the cxl_region driver.
2514 */
2515 region_nid = phys_to_target_node(cxlr->params.res->start);
2516 if (nid != region_nid)
2517 return NOTIFY_OK;
2518
2519 perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2520
2521 if (mt_perf_to_adistance(perf, adist))
2522 return NOTIFY_OK;
2523
2524 return NOTIFY_STOP;
2525 }
2526
2527 /**
2528 * devm_cxl_add_region - Adds a region to a decoder
2529 * @cxlrd: root decoder
2530 * @id: memregion id to create, or memregion_free() on failure
2531 * @mode: mode for the endpoint decoders of this region
2532 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2533 *
2534 * This is the second step of region initialization. Regions exist within an
2535 * address space which is mapped by a @cxlrd.
2536 *
2537 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2538 * code. The region will be named "regionZ" where Z is the unique region number.
2539 */
devm_cxl_add_region(struct cxl_root_decoder * cxlrd,int id,enum cxl_partition_mode mode,enum cxl_decoder_type type)2540 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2541 int id,
2542 enum cxl_partition_mode mode,
2543 enum cxl_decoder_type type)
2544 {
2545 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2546 struct cxl_region *cxlr;
2547 struct device *dev;
2548 int rc;
2549
2550 cxlr = cxl_region_alloc(cxlrd, id);
2551 if (IS_ERR(cxlr))
2552 return cxlr;
2553 cxlr->mode = mode;
2554 cxlr->type = type;
2555
2556 dev = &cxlr->dev;
2557 rc = dev_set_name(dev, "region%d", id);
2558 if (rc)
2559 goto err;
2560
2561 rc = device_add(dev);
2562 if (rc)
2563 goto err;
2564
2565 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2566 if (rc)
2567 return ERR_PTR(rc);
2568
2569 dev_dbg(port->uport_dev, "%s: created %s\n",
2570 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2571 return cxlr;
2572
2573 err:
2574 put_device(dev);
2575 return ERR_PTR(rc);
2576 }
2577
__create_region_show(struct cxl_root_decoder * cxlrd,char * buf)2578 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2579 {
2580 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2581 }
2582
create_pmem_region_show(struct device * dev,struct device_attribute * attr,char * buf)2583 static ssize_t create_pmem_region_show(struct device *dev,
2584 struct device_attribute *attr, char *buf)
2585 {
2586 return __create_region_show(to_cxl_root_decoder(dev), buf);
2587 }
2588
create_ram_region_show(struct device * dev,struct device_attribute * attr,char * buf)2589 static ssize_t create_ram_region_show(struct device *dev,
2590 struct device_attribute *attr, char *buf)
2591 {
2592 return __create_region_show(to_cxl_root_decoder(dev), buf);
2593 }
2594
__create_region(struct cxl_root_decoder * cxlrd,enum cxl_partition_mode mode,int id)2595 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2596 enum cxl_partition_mode mode, int id)
2597 {
2598 int rc;
2599
2600 switch (mode) {
2601 case CXL_PARTMODE_RAM:
2602 case CXL_PARTMODE_PMEM:
2603 break;
2604 default:
2605 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2606 return ERR_PTR(-EINVAL);
2607 }
2608
2609 rc = memregion_alloc(GFP_KERNEL);
2610 if (rc < 0)
2611 return ERR_PTR(rc);
2612
2613 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2614 memregion_free(rc);
2615 return ERR_PTR(-EBUSY);
2616 }
2617
2618 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2619 }
2620
create_region_store(struct device * dev,const char * buf,size_t len,enum cxl_partition_mode mode)2621 static ssize_t create_region_store(struct device *dev, const char *buf,
2622 size_t len, enum cxl_partition_mode mode)
2623 {
2624 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2625 struct cxl_region *cxlr;
2626 int rc, id;
2627
2628 rc = sscanf(buf, "region%d\n", &id);
2629 if (rc != 1)
2630 return -EINVAL;
2631
2632 cxlr = __create_region(cxlrd, mode, id);
2633 if (IS_ERR(cxlr))
2634 return PTR_ERR(cxlr);
2635
2636 return len;
2637 }
2638
create_pmem_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2639 static ssize_t create_pmem_region_store(struct device *dev,
2640 struct device_attribute *attr,
2641 const char *buf, size_t len)
2642 {
2643 return create_region_store(dev, buf, len, CXL_PARTMODE_PMEM);
2644 }
2645 DEVICE_ATTR_RW(create_pmem_region);
2646
create_ram_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2647 static ssize_t create_ram_region_store(struct device *dev,
2648 struct device_attribute *attr,
2649 const char *buf, size_t len)
2650 {
2651 return create_region_store(dev, buf, len, CXL_PARTMODE_RAM);
2652 }
2653 DEVICE_ATTR_RW(create_ram_region);
2654
region_show(struct device * dev,struct device_attribute * attr,char * buf)2655 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2656 char *buf)
2657 {
2658 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2659 ssize_t rc;
2660
2661 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
2662 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
2663 return rc;
2664
2665 if (cxld->region)
2666 return sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2667 return sysfs_emit(buf, "\n");
2668 }
2669 DEVICE_ATTR_RO(region);
2670
2671 static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder * cxlrd,const char * name)2672 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2673 {
2674 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2675 struct device *region_dev;
2676
2677 region_dev = device_find_child_by_name(&cxld->dev, name);
2678 if (!region_dev)
2679 return ERR_PTR(-ENODEV);
2680
2681 return to_cxl_region(region_dev);
2682 }
2683
delete_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2684 static ssize_t delete_region_store(struct device *dev,
2685 struct device_attribute *attr,
2686 const char *buf, size_t len)
2687 {
2688 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2689 struct cxl_port *port = to_cxl_port(dev->parent);
2690 struct cxl_region *cxlr;
2691
2692 cxlr = cxl_find_region_by_name(cxlrd, buf);
2693 if (IS_ERR(cxlr))
2694 return PTR_ERR(cxlr);
2695
2696 devm_release_action(port->uport_dev, unregister_region, cxlr);
2697 put_device(&cxlr->dev);
2698
2699 return len;
2700 }
2701 DEVICE_ATTR_WO(delete_region);
2702
cxl_pmem_region_release(struct device * dev)2703 static void cxl_pmem_region_release(struct device *dev)
2704 {
2705 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2706 int i;
2707
2708 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2709 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2710
2711 put_device(&cxlmd->dev);
2712 }
2713
2714 kfree(cxlr_pmem);
2715 }
2716
2717 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2718 &cxl_base_attribute_group,
2719 NULL,
2720 };
2721
2722 const struct device_type cxl_pmem_region_type = {
2723 .name = "cxl_pmem_region",
2724 .release = cxl_pmem_region_release,
2725 .groups = cxl_pmem_region_attribute_groups,
2726 };
2727
is_cxl_pmem_region(struct device * dev)2728 bool is_cxl_pmem_region(struct device *dev)
2729 {
2730 return dev->type == &cxl_pmem_region_type;
2731 }
2732 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2733
to_cxl_pmem_region(struct device * dev)2734 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2735 {
2736 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2737 "not a cxl_pmem_region device\n"))
2738 return NULL;
2739 return container_of(dev, struct cxl_pmem_region, dev);
2740 }
2741 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2742
2743 struct cxl_poison_context {
2744 struct cxl_port *port;
2745 int part;
2746 u64 offset;
2747 };
2748
cxl_get_poison_unmapped(struct cxl_memdev * cxlmd,struct cxl_poison_context * ctx)2749 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2750 struct cxl_poison_context *ctx)
2751 {
2752 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2753 const struct resource *res;
2754 struct resource *p, *last;
2755 u64 offset, length;
2756 int rc = 0;
2757
2758 if (ctx->part < 0)
2759 return 0;
2760
2761 /*
2762 * Collect poison for the remaining unmapped resources after
2763 * poison is collected by committed endpoints decoders.
2764 */
2765 for (int i = ctx->part; i < cxlds->nr_partitions; i++) {
2766 res = &cxlds->part[i].res;
2767 for (p = res->child, last = NULL; p; p = p->sibling)
2768 last = p;
2769 if (last)
2770 offset = last->end + 1;
2771 else
2772 offset = res->start;
2773 length = res->end - offset + 1;
2774 if (!length)
2775 break;
2776 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2777 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
2778 continue;
2779 if (rc)
2780 break;
2781 }
2782
2783 return rc;
2784 }
2785
poison_by_decoder(struct device * dev,void * arg)2786 static int poison_by_decoder(struct device *dev, void *arg)
2787 {
2788 struct cxl_poison_context *ctx = arg;
2789 struct cxl_endpoint_decoder *cxled;
2790 enum cxl_partition_mode mode;
2791 struct cxl_dev_state *cxlds;
2792 struct cxl_memdev *cxlmd;
2793 u64 offset, length;
2794 int rc = 0;
2795
2796 if (!is_endpoint_decoder(dev))
2797 return rc;
2798
2799 cxled = to_cxl_endpoint_decoder(dev);
2800 if (!cxled->dpa_res)
2801 return rc;
2802
2803 cxlmd = cxled_to_memdev(cxled);
2804 cxlds = cxlmd->cxlds;
2805 mode = cxlds->part[cxled->part].mode;
2806
2807 if (cxled->skip) {
2808 offset = cxled->dpa_res->start - cxled->skip;
2809 length = cxled->skip;
2810 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2811 if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
2812 rc = 0;
2813 if (rc)
2814 return rc;
2815 }
2816
2817 offset = cxled->dpa_res->start;
2818 length = cxled->dpa_res->end - offset + 1;
2819 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2820 if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
2821 rc = 0;
2822 if (rc)
2823 return rc;
2824
2825 /* Iterate until commit_end is reached */
2826 if (cxled->cxld.id == ctx->port->commit_end) {
2827 ctx->offset = cxled->dpa_res->end + 1;
2828 ctx->part = cxled->part;
2829 return 1;
2830 }
2831
2832 return 0;
2833 }
2834
cxl_get_poison_by_endpoint(struct cxl_port * port)2835 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2836 {
2837 struct cxl_poison_context ctx;
2838 int rc = 0;
2839
2840 ctx = (struct cxl_poison_context) {
2841 .port = port,
2842 .part = -1,
2843 };
2844
2845 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2846 if (rc == 1)
2847 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2848 &ctx);
2849
2850 return rc;
2851 }
2852
2853 struct cxl_dpa_to_region_context {
2854 struct cxl_region *cxlr;
2855 u64 dpa;
2856 };
2857
__cxl_dpa_to_region(struct device * dev,void * arg)2858 static int __cxl_dpa_to_region(struct device *dev, void *arg)
2859 {
2860 struct cxl_dpa_to_region_context *ctx = arg;
2861 struct cxl_endpoint_decoder *cxled;
2862 struct cxl_region *cxlr;
2863 u64 dpa = ctx->dpa;
2864
2865 if (!is_endpoint_decoder(dev))
2866 return 0;
2867
2868 cxled = to_cxl_endpoint_decoder(dev);
2869 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2870 return 0;
2871
2872 if (!cxl_resource_contains_addr(cxled->dpa_res, dpa))
2873 return 0;
2874
2875 /*
2876 * Stop the region search (return 1) when an endpoint mapping is
2877 * found. The region may not be fully constructed so offering
2878 * the cxlr in the context structure is not guaranteed.
2879 */
2880 cxlr = cxled->cxld.region;
2881 if (cxlr)
2882 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2883 dev_name(&cxlr->dev));
2884 else
2885 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2886 dev_name(dev));
2887
2888 ctx->cxlr = cxlr;
2889
2890 return 1;
2891 }
2892
cxl_dpa_to_region(const struct cxl_memdev * cxlmd,u64 dpa)2893 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2894 {
2895 struct cxl_dpa_to_region_context ctx;
2896 struct cxl_port *port;
2897
2898 ctx = (struct cxl_dpa_to_region_context) {
2899 .dpa = dpa,
2900 };
2901 port = cxlmd->endpoint;
2902 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2903 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2904
2905 return ctx.cxlr;
2906 }
2907
cxl_is_hpa_in_chunk(u64 hpa,struct cxl_region * cxlr,int pos)2908 static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2909 {
2910 struct cxl_region_params *p = &cxlr->params;
2911 int gran = p->interleave_granularity;
2912 int ways = p->interleave_ways;
2913 u64 offset;
2914
2915 /* Is the hpa in an expected chunk for its pos(-ition) */
2916 offset = hpa - p->res->start;
2917 offset = do_div(offset, gran * ways);
2918 if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2919 return true;
2920
2921 dev_dbg(&cxlr->dev,
2922 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2923
2924 return false;
2925 }
2926
has_hpa_to_spa(struct cxl_root_decoder * cxlrd)2927 static bool has_hpa_to_spa(struct cxl_root_decoder *cxlrd)
2928 {
2929 return cxlrd->ops && cxlrd->ops->hpa_to_spa;
2930 }
2931
has_spa_to_hpa(struct cxl_root_decoder * cxlrd)2932 static bool has_spa_to_hpa(struct cxl_root_decoder *cxlrd)
2933 {
2934 return cxlrd->ops && cxlrd->ops->spa_to_hpa;
2935 }
2936
cxl_dpa_to_hpa(struct cxl_region * cxlr,const struct cxl_memdev * cxlmd,u64 dpa)2937 u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2938 u64 dpa)
2939 {
2940 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
2941 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2942 struct cxl_region_params *p = &cxlr->params;
2943 struct cxl_endpoint_decoder *cxled = NULL;
2944 u16 eig = 0;
2945 u8 eiw = 0;
2946 int pos;
2947
2948 for (int i = 0; i < p->nr_targets; i++) {
2949 cxled = p->targets[i];
2950 if (cxlmd == cxled_to_memdev(cxled))
2951 break;
2952 }
2953 if (!cxled || cxlmd != cxled_to_memdev(cxled))
2954 return ULLONG_MAX;
2955
2956 pos = cxled->pos;
2957 ways_to_eiw(p->interleave_ways, &eiw);
2958 granularity_to_eig(p->interleave_granularity, &eig);
2959
2960 /*
2961 * The device position in the region interleave set was removed
2962 * from the offset at HPA->DPA translation. To reconstruct the
2963 * HPA, place the 'pos' in the offset.
2964 *
2965 * The placement of 'pos' in the HPA is determined by interleave
2966 * ways and granularity and is defined in the CXL Spec 3.0 Section
2967 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2968 */
2969
2970 /* Remove the dpa base */
2971 dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2972
2973 mask_upper = GENMASK_ULL(51, eig + 8);
2974
2975 if (eiw < 8) {
2976 hpa_offset = (dpa_offset & mask_upper) << eiw;
2977 hpa_offset |= pos << (eig + 8);
2978 } else {
2979 bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2980 bits_upper = bits_upper * 3;
2981 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2982 }
2983
2984 /* The lower bits remain unchanged */
2985 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2986
2987 /* Apply the hpa_offset to the region base address */
2988 hpa = hpa_offset + p->res->start + p->cache_size;
2989
2990 /* Root decoder translation overrides typical modulo decode */
2991 if (has_hpa_to_spa(cxlrd))
2992 hpa = cxlrd->ops->hpa_to_spa(cxlrd, hpa);
2993
2994 if (!cxl_resource_contains_addr(p->res, hpa)) {
2995 dev_dbg(&cxlr->dev,
2996 "Addr trans fail: hpa 0x%llx not in region\n", hpa);
2997 return ULLONG_MAX;
2998 }
2999
3000 /* Simple chunk check, by pos & gran, only applies to modulo decodes */
3001 if (!has_hpa_to_spa(cxlrd) && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos)))
3002 return ULLONG_MAX;
3003
3004 return hpa;
3005 }
3006
3007 struct dpa_result {
3008 struct cxl_memdev *cxlmd;
3009 u64 dpa;
3010 };
3011
region_offset_to_dpa_result(struct cxl_region * cxlr,u64 offset,struct dpa_result * result)3012 static int region_offset_to_dpa_result(struct cxl_region *cxlr, u64 offset,
3013 struct dpa_result *result)
3014 {
3015 struct cxl_region_params *p = &cxlr->params;
3016 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
3017 struct cxl_endpoint_decoder *cxled;
3018 u64 hpa, hpa_offset, dpa_offset;
3019 u64 bits_upper, bits_lower;
3020 u64 shifted, rem, temp;
3021 u16 eig = 0;
3022 u8 eiw = 0;
3023 int pos;
3024
3025 lockdep_assert_held(&cxl_rwsem.region);
3026 lockdep_assert_held(&cxl_rwsem.dpa);
3027
3028 /* Input validation ensures valid ways and gran */
3029 granularity_to_eig(p->interleave_granularity, &eig);
3030 ways_to_eiw(p->interleave_ways, &eiw);
3031
3032 /*
3033 * If the root decoder has SPA to CXL HPA callback, use it. Otherwise
3034 * CXL HPA is assumed to equal SPA.
3035 */
3036 if (has_spa_to_hpa(cxlrd)) {
3037 hpa = cxlrd->ops->spa_to_hpa(cxlrd, p->res->start + offset);
3038 hpa_offset = hpa - p->res->start;
3039 } else {
3040 hpa_offset = offset;
3041 }
3042 /*
3043 * Interleave position: CXL Spec 3.2 Section 8.2.4.20.13
3044 * eiw < 8
3045 * Position is in the IW bits at HPA_OFFSET[IG+8+IW-1:IG+8].
3046 * Per spec "remove IW bits starting with bit position IG+8"
3047 * eiw >= 8
3048 * Position is not explicitly stored in HPA_OFFSET bits. It is
3049 * derived from the modulo operation of the upper bits using
3050 * the total number of interleave ways.
3051 */
3052 if (eiw < 8) {
3053 pos = (hpa_offset >> (eig + 8)) & GENMASK(eiw - 1, 0);
3054 } else {
3055 shifted = hpa_offset >> (eig + 8);
3056 div64_u64_rem(shifted, p->interleave_ways, &rem);
3057 pos = rem;
3058 }
3059 if (pos < 0 || pos >= p->nr_targets) {
3060 dev_dbg(&cxlr->dev, "Invalid position %d for %d targets\n",
3061 pos, p->nr_targets);
3062 return -ENXIO;
3063 }
3064
3065 /*
3066 * DPA offset: CXL Spec 3.2 Section 8.2.4.20.13
3067 * Lower bits [IG+7:0] pass through unchanged
3068 * (eiw < 8)
3069 * Per spec: DPAOffset[51:IG+8] = (HPAOffset[51:IG+IW+8] >> IW)
3070 * Clear the position bits to isolate upper section, then
3071 * reverse the left shift by eiw that occurred during DPA->HPA
3072 * (eiw >= 8)
3073 * Per spec: DPAOffset[51:IG+8] = HPAOffset[51:IG+IW] / 3
3074 * Extract upper bits from the correct bit range and divide by 3
3075 * to recover the original DPA upper bits
3076 */
3077 bits_lower = hpa_offset & GENMASK_ULL(eig + 7, 0);
3078 if (eiw < 8) {
3079 temp = hpa_offset &= ~((u64)GENMASK(eig + eiw + 8 - 1, 0));
3080 dpa_offset = temp >> eiw;
3081 } else {
3082 bits_upper = div64_u64(hpa_offset >> (eig + eiw), 3);
3083 dpa_offset = bits_upper << (eig + 8);
3084 }
3085 dpa_offset |= bits_lower;
3086
3087 /* Look-up and return the result: a memdev and a DPA */
3088 for (int i = 0; i < p->nr_targets; i++) {
3089 cxled = p->targets[i];
3090 if (cxled->pos != pos)
3091 continue;
3092 result->cxlmd = cxled_to_memdev(cxled);
3093 result->dpa = cxl_dpa_resource_start(cxled) + dpa_offset;
3094
3095 return 0;
3096 }
3097 dev_err(&cxlr->dev, "No device found for position %d\n", pos);
3098
3099 return -ENXIO;
3100 }
3101
3102 static struct lock_class_key cxl_pmem_region_key;
3103
cxl_pmem_region_alloc(struct cxl_region * cxlr)3104 static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
3105 {
3106 struct cxl_region_params *p = &cxlr->params;
3107 struct cxl_nvdimm_bridge *cxl_nvb;
3108 struct device *dev;
3109 int i;
3110
3111 guard(rwsem_read)(&cxl_rwsem.region);
3112 if (p->state != CXL_CONFIG_COMMIT)
3113 return -ENXIO;
3114
3115 struct cxl_pmem_region *cxlr_pmem __free(kfree) =
3116 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
3117 if (!cxlr_pmem)
3118 return -ENOMEM;
3119
3120 cxlr_pmem->hpa_range.start = p->res->start;
3121 cxlr_pmem->hpa_range.end = p->res->end;
3122
3123 /* Snapshot the region configuration underneath the cxl_rwsem.region */
3124 cxlr_pmem->nr_mappings = p->nr_targets;
3125 for (i = 0; i < p->nr_targets; i++) {
3126 struct cxl_endpoint_decoder *cxled = p->targets[i];
3127 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3128 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
3129
3130 /*
3131 * Regions never span CXL root devices, so by definition the
3132 * bridge for one device is the same for all.
3133 */
3134 if (i == 0) {
3135 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint);
3136 if (!cxl_nvb)
3137 return -ENODEV;
3138 cxlr->cxl_nvb = cxl_nvb;
3139 }
3140 m->cxlmd = cxlmd;
3141 get_device(&cxlmd->dev);
3142 m->start = cxled->dpa_res->start;
3143 m->size = resource_size(cxled->dpa_res);
3144 m->position = i;
3145 }
3146
3147 dev = &cxlr_pmem->dev;
3148 device_initialize(dev);
3149 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
3150 device_set_pm_not_required(dev);
3151 dev->parent = &cxlr->dev;
3152 dev->bus = &cxl_bus_type;
3153 dev->type = &cxl_pmem_region_type;
3154 cxlr_pmem->cxlr = cxlr;
3155 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
3156
3157 return 0;
3158 }
3159
cxl_dax_region_release(struct device * dev)3160 static void cxl_dax_region_release(struct device *dev)
3161 {
3162 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3163
3164 kfree(cxlr_dax);
3165 }
3166
3167 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3168 &cxl_base_attribute_group,
3169 NULL,
3170 };
3171
3172 const struct device_type cxl_dax_region_type = {
3173 .name = "cxl_dax_region",
3174 .release = cxl_dax_region_release,
3175 .groups = cxl_dax_region_attribute_groups,
3176 };
3177
is_cxl_dax_region(struct device * dev)3178 static bool is_cxl_dax_region(struct device *dev)
3179 {
3180 return dev->type == &cxl_dax_region_type;
3181 }
3182
to_cxl_dax_region(struct device * dev)3183 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3184 {
3185 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3186 "not a cxl_dax_region device\n"))
3187 return NULL;
3188 return container_of(dev, struct cxl_dax_region, dev);
3189 }
3190 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3191
3192 static struct lock_class_key cxl_dax_region_key;
3193
cxl_dax_region_alloc(struct cxl_region * cxlr)3194 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3195 {
3196 struct cxl_region_params *p = &cxlr->params;
3197 struct cxl_dax_region *cxlr_dax;
3198 struct device *dev;
3199
3200 guard(rwsem_read)(&cxl_rwsem.region);
3201 if (p->state != CXL_CONFIG_COMMIT)
3202 return ERR_PTR(-ENXIO);
3203
3204 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3205 if (!cxlr_dax)
3206 return ERR_PTR(-ENOMEM);
3207
3208 cxlr_dax->hpa_range.start = p->res->start;
3209 cxlr_dax->hpa_range.end = p->res->end;
3210
3211 dev = &cxlr_dax->dev;
3212 cxlr_dax->cxlr = cxlr;
3213 device_initialize(dev);
3214 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3215 device_set_pm_not_required(dev);
3216 dev->parent = &cxlr->dev;
3217 dev->bus = &cxl_bus_type;
3218 dev->type = &cxl_dax_region_type;
3219
3220 return cxlr_dax;
3221 }
3222
cxlr_pmem_unregister(void * _cxlr_pmem)3223 static void cxlr_pmem_unregister(void *_cxlr_pmem)
3224 {
3225 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3226 struct cxl_region *cxlr = cxlr_pmem->cxlr;
3227 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3228
3229 /*
3230 * Either the bridge is in ->remove() context under the device_lock(),
3231 * or cxlr_release_nvdimm() is cancelling the bridge's release action
3232 * for @cxlr_pmem and doing it itself (while manually holding the bridge
3233 * lock).
3234 */
3235 device_lock_assert(&cxl_nvb->dev);
3236 cxlr->cxlr_pmem = NULL;
3237 cxlr_pmem->cxlr = NULL;
3238 device_unregister(&cxlr_pmem->dev);
3239 }
3240
cxlr_release_nvdimm(void * _cxlr)3241 static void cxlr_release_nvdimm(void *_cxlr)
3242 {
3243 struct cxl_region *cxlr = _cxlr;
3244 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3245
3246 scoped_guard(device, &cxl_nvb->dev) {
3247 if (cxlr->cxlr_pmem)
3248 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
3249 cxlr->cxlr_pmem);
3250 }
3251 cxlr->cxl_nvb = NULL;
3252 put_device(&cxl_nvb->dev);
3253 }
3254
3255 /**
3256 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3257 * @cxlr: parent CXL region for this pmem region bridge device
3258 *
3259 * Return: 0 on success negative error code on failure.
3260 */
devm_cxl_add_pmem_region(struct cxl_region * cxlr)3261 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3262 {
3263 struct cxl_pmem_region *cxlr_pmem;
3264 struct cxl_nvdimm_bridge *cxl_nvb;
3265 struct device *dev;
3266 int rc;
3267
3268 rc = cxl_pmem_region_alloc(cxlr);
3269 if (rc)
3270 return rc;
3271 cxlr_pmem = cxlr->cxlr_pmem;
3272 cxl_nvb = cxlr->cxl_nvb;
3273
3274 dev = &cxlr_pmem->dev;
3275 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
3276 if (rc)
3277 goto err;
3278
3279 rc = device_add(dev);
3280 if (rc)
3281 goto err;
3282
3283 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3284 dev_name(dev));
3285
3286 scoped_guard(device, &cxl_nvb->dev) {
3287 if (cxl_nvb->dev.driver)
3288 rc = devm_add_action_or_reset(&cxl_nvb->dev,
3289 cxlr_pmem_unregister,
3290 cxlr_pmem);
3291 else
3292 rc = -ENXIO;
3293 }
3294
3295 if (rc)
3296 goto err_bridge;
3297
3298 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3299 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3300
3301 err:
3302 put_device(dev);
3303 err_bridge:
3304 put_device(&cxl_nvb->dev);
3305 cxlr->cxl_nvb = NULL;
3306 return rc;
3307 }
3308
cxlr_dax_unregister(void * _cxlr_dax)3309 static void cxlr_dax_unregister(void *_cxlr_dax)
3310 {
3311 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3312
3313 device_unregister(&cxlr_dax->dev);
3314 }
3315
devm_cxl_add_dax_region(struct cxl_region * cxlr)3316 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3317 {
3318 struct cxl_dax_region *cxlr_dax;
3319 struct device *dev;
3320 int rc;
3321
3322 cxlr_dax = cxl_dax_region_alloc(cxlr);
3323 if (IS_ERR(cxlr_dax))
3324 return PTR_ERR(cxlr_dax);
3325
3326 dev = &cxlr_dax->dev;
3327 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3328 if (rc)
3329 goto err;
3330
3331 rc = device_add(dev);
3332 if (rc)
3333 goto err;
3334
3335 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3336 dev_name(dev));
3337
3338 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3339 cxlr_dax);
3340 err:
3341 put_device(dev);
3342 return rc;
3343 }
3344
match_decoder_by_range(struct device * dev,const void * data)3345 static int match_decoder_by_range(struct device *dev, const void *data)
3346 {
3347 const struct range *r1, *r2 = data;
3348 struct cxl_decoder *cxld;
3349
3350 if (!is_switch_decoder(dev))
3351 return 0;
3352
3353 cxld = to_cxl_decoder(dev);
3354 r1 = &cxld->hpa_range;
3355 return range_contains(r1, r2);
3356 }
3357
3358 static struct cxl_decoder *
cxl_port_find_switch_decoder(struct cxl_port * port,struct range * hpa)3359 cxl_port_find_switch_decoder(struct cxl_port *port, struct range *hpa)
3360 {
3361 struct device *cxld_dev = device_find_child(&port->dev, hpa,
3362 match_decoder_by_range);
3363
3364 return cxld_dev ? to_cxl_decoder(cxld_dev) : NULL;
3365 }
3366
3367 static struct cxl_root_decoder *
cxl_find_root_decoder(struct cxl_endpoint_decoder * cxled)3368 cxl_find_root_decoder(struct cxl_endpoint_decoder *cxled)
3369 {
3370 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3371 struct cxl_port *port = cxled_to_port(cxled);
3372 struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);
3373 struct cxl_decoder *root, *cxld = &cxled->cxld;
3374 struct range *hpa = &cxld->hpa_range;
3375
3376 root = cxl_port_find_switch_decoder(&cxl_root->port, hpa);
3377 if (!root) {
3378 dev_err(cxlmd->dev.parent,
3379 "%s:%s no CXL window for range %#llx:%#llx\n",
3380 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3381 cxld->hpa_range.start, cxld->hpa_range.end);
3382 return NULL;
3383 }
3384
3385 return to_cxl_root_decoder(&root->dev);
3386 }
3387
match_region_by_range(struct device * dev,const void * data)3388 static int match_region_by_range(struct device *dev, const void *data)
3389 {
3390 struct cxl_region_params *p;
3391 struct cxl_region *cxlr;
3392 const struct range *r = data;
3393
3394 if (!is_cxl_region(dev))
3395 return 0;
3396
3397 cxlr = to_cxl_region(dev);
3398 p = &cxlr->params;
3399
3400 guard(rwsem_read)(&cxl_rwsem.region);
3401 return region_res_match_cxl_range(p, r);
3402 }
3403
cxl_extended_linear_cache_resize(struct cxl_region * cxlr,struct resource * res)3404 static int cxl_extended_linear_cache_resize(struct cxl_region *cxlr,
3405 struct resource *res)
3406 {
3407 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
3408 struct cxl_region_params *p = &cxlr->params;
3409 resource_size_t size = resource_size(res);
3410 resource_size_t cache_size, start;
3411
3412 cache_size = cxlrd->cache_size;
3413 if (!cache_size)
3414 return 0;
3415
3416 if (size != cache_size) {
3417 dev_warn(&cxlr->dev,
3418 "Extended Linear Cache size %pa != CXL size %pa. No Support!",
3419 &cache_size, &size);
3420 return -ENXIO;
3421 }
3422
3423 /*
3424 * Move the start of the range to where the cache range starts. The
3425 * implementation assumes that the cache range is in front of the
3426 * CXL range. This is not dictated by the HMAT spec but is how the
3427 * current known implementation is configured.
3428 *
3429 * The cache range is expected to be within the CFMWS. The adjusted
3430 * res->start should not be less than cxlrd->res->start.
3431 */
3432 start = res->start - cache_size;
3433 if (start < cxlrd->res->start)
3434 return -ENXIO;
3435
3436 res->start = start;
3437 p->cache_size = cache_size;
3438
3439 return 0;
3440 }
3441
__construct_region(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)3442 static int __construct_region(struct cxl_region *cxlr,
3443 struct cxl_root_decoder *cxlrd,
3444 struct cxl_endpoint_decoder *cxled)
3445 {
3446 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3447 struct range *hpa = &cxled->cxld.hpa_range;
3448 struct cxl_region_params *p;
3449 struct resource *res;
3450 int rc;
3451
3452 guard(rwsem_write)(&cxl_rwsem.region);
3453 p = &cxlr->params;
3454 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3455 dev_err(cxlmd->dev.parent,
3456 "%s:%s: %s autodiscovery interrupted\n",
3457 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3458 __func__);
3459 return -EBUSY;
3460 }
3461
3462 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3463
3464 res = kmalloc(sizeof(*res), GFP_KERNEL);
3465 if (!res)
3466 return -ENOMEM;
3467
3468 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3469 dev_name(&cxlr->dev));
3470
3471 rc = cxl_extended_linear_cache_resize(cxlr, res);
3472 if (rc && rc != -EOPNOTSUPP) {
3473 /*
3474 * Failing to support extended linear cache region resize does not
3475 * prevent the region from functioning. Only causes cxl list showing
3476 * incorrect region size.
3477 */
3478 dev_warn(cxlmd->dev.parent,
3479 "Extended linear cache calculation failed rc:%d\n", rc);
3480 }
3481
3482 rc = insert_resource(cxlrd->res, res);
3483 if (rc) {
3484 /*
3485 * Platform-firmware may not have split resources like "System
3486 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3487 */
3488 dev_warn(cxlmd->dev.parent,
3489 "%s:%s: %s %s cannot insert resource\n",
3490 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3491 __func__, dev_name(&cxlr->dev));
3492 }
3493
3494 p->res = res;
3495 p->interleave_ways = cxled->cxld.interleave_ways;
3496 p->interleave_granularity = cxled->cxld.interleave_granularity;
3497 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3498
3499 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3500 if (rc)
3501 return rc;
3502
3503 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3504 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3505 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3506 p->interleave_granularity);
3507
3508 /* ...to match put_device() in cxl_add_to_region() */
3509 get_device(&cxlr->dev);
3510
3511 return 0;
3512 }
3513
3514 /* Establish an empty region covering the given HPA range */
construct_region(struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)3515 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3516 struct cxl_endpoint_decoder *cxled)
3517 {
3518 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3519 struct cxl_port *port = cxlrd_to_port(cxlrd);
3520 struct cxl_dev_state *cxlds = cxlmd->cxlds;
3521 int rc, part = READ_ONCE(cxled->part);
3522 struct cxl_region *cxlr;
3523
3524 do {
3525 cxlr = __create_region(cxlrd, cxlds->part[part].mode,
3526 atomic_read(&cxlrd->region_id));
3527 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3528
3529 if (IS_ERR(cxlr)) {
3530 dev_err(cxlmd->dev.parent,
3531 "%s:%s: %s failed assign region: %ld\n",
3532 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3533 __func__, PTR_ERR(cxlr));
3534 return cxlr;
3535 }
3536
3537 rc = __construct_region(cxlr, cxlrd, cxled);
3538 if (rc) {
3539 devm_release_action(port->uport_dev, unregister_region, cxlr);
3540 return ERR_PTR(rc);
3541 }
3542
3543 return cxlr;
3544 }
3545
3546 static struct cxl_region *
cxl_find_region_by_range(struct cxl_root_decoder * cxlrd,struct range * hpa)3547 cxl_find_region_by_range(struct cxl_root_decoder *cxlrd, struct range *hpa)
3548 {
3549 struct device *region_dev;
3550
3551 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3552 match_region_by_range);
3553 if (!region_dev)
3554 return NULL;
3555
3556 return to_cxl_region(region_dev);
3557 }
3558
cxl_add_to_region(struct cxl_endpoint_decoder * cxled)3559 int cxl_add_to_region(struct cxl_endpoint_decoder *cxled)
3560 {
3561 struct range *hpa = &cxled->cxld.hpa_range;
3562 struct cxl_region_params *p;
3563 bool attach = false;
3564 int rc;
3565
3566 struct cxl_root_decoder *cxlrd __free(put_cxl_root_decoder) =
3567 cxl_find_root_decoder(cxled);
3568 if (!cxlrd)
3569 return -ENXIO;
3570
3571 /*
3572 * Ensure that if multiple threads race to construct_region() for @hpa
3573 * one does the construction and the others add to that.
3574 */
3575 mutex_lock(&cxlrd->range_lock);
3576 struct cxl_region *cxlr __free(put_cxl_region) =
3577 cxl_find_region_by_range(cxlrd, hpa);
3578 if (!cxlr)
3579 cxlr = construct_region(cxlrd, cxled);
3580 mutex_unlock(&cxlrd->range_lock);
3581
3582 rc = PTR_ERR_OR_ZERO(cxlr);
3583 if (rc)
3584 return rc;
3585
3586 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3587
3588 scoped_guard(rwsem_read, &cxl_rwsem.region) {
3589 p = &cxlr->params;
3590 attach = p->state == CXL_CONFIG_COMMIT;
3591 }
3592
3593 if (attach) {
3594 /*
3595 * If device_attach() fails the range may still be active via
3596 * the platform-firmware memory map, otherwise the driver for
3597 * regions is local to this file, so driver matching can't fail.
3598 */
3599 if (device_attach(&cxlr->dev) < 0)
3600 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3601 p->res);
3602 }
3603
3604 return rc;
3605 }
3606 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3607
cxl_port_get_spa_cache_alias(struct cxl_port * endpoint,u64 spa)3608 u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa)
3609 {
3610 struct cxl_region_ref *iter;
3611 unsigned long index;
3612
3613 if (!endpoint)
3614 return ~0ULL;
3615
3616 guard(rwsem_write)(&cxl_rwsem.region);
3617
3618 xa_for_each(&endpoint->regions, index, iter) {
3619 struct cxl_region_params *p = &iter->region->params;
3620
3621 if (cxl_resource_contains_addr(p->res, spa)) {
3622 if (!p->cache_size)
3623 return ~0ULL;
3624
3625 if (spa >= p->res->start + p->cache_size)
3626 return spa - p->cache_size;
3627
3628 return spa + p->cache_size;
3629 }
3630 }
3631
3632 return ~0ULL;
3633 }
3634 EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL");
3635
is_system_ram(struct resource * res,void * arg)3636 static int is_system_ram(struct resource *res, void *arg)
3637 {
3638 struct cxl_region *cxlr = arg;
3639 struct cxl_region_params *p = &cxlr->params;
3640
3641 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3642 return 1;
3643 }
3644
shutdown_notifiers(void * _cxlr)3645 static void shutdown_notifiers(void *_cxlr)
3646 {
3647 struct cxl_region *cxlr = _cxlr;
3648
3649 unregister_node_notifier(&cxlr->node_notifier);
3650 unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
3651 }
3652
remove_debugfs(void * dentry)3653 static void remove_debugfs(void *dentry)
3654 {
3655 debugfs_remove_recursive(dentry);
3656 }
3657
validate_region_offset(struct cxl_region * cxlr,u64 offset)3658 static int validate_region_offset(struct cxl_region *cxlr, u64 offset)
3659 {
3660 struct cxl_region_params *p = &cxlr->params;
3661 resource_size_t region_size;
3662 u64 hpa;
3663
3664 if (offset < p->cache_size) {
3665 dev_err(&cxlr->dev,
3666 "Offset %#llx is within extended linear cache %pa\n",
3667 offset, &p->cache_size);
3668 return -EINVAL;
3669 }
3670
3671 region_size = resource_size(p->res);
3672 if (offset >= region_size) {
3673 dev_err(&cxlr->dev, "Offset %#llx exceeds region size %pa\n",
3674 offset, ®ion_size);
3675 return -EINVAL;
3676 }
3677
3678 hpa = p->res->start + offset;
3679 if (hpa < p->res->start || hpa > p->res->end) {
3680 dev_err(&cxlr->dev, "HPA %#llx not in region %pr\n", hpa,
3681 p->res);
3682 return -EINVAL;
3683 }
3684
3685 return 0;
3686 }
3687
cxl_region_debugfs_poison_inject(void * data,u64 offset)3688 static int cxl_region_debugfs_poison_inject(void *data, u64 offset)
3689 {
3690 struct dpa_result result = { .dpa = ULLONG_MAX, .cxlmd = NULL };
3691 struct cxl_region *cxlr = data;
3692 int rc;
3693
3694 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
3695 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem)))
3696 return rc;
3697
3698 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
3699 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
3700 return rc;
3701
3702 if (validate_region_offset(cxlr, offset))
3703 return -EINVAL;
3704
3705 rc = region_offset_to_dpa_result(cxlr, offset, &result);
3706 if (rc || !result.cxlmd || result.dpa == ULLONG_MAX) {
3707 dev_dbg(&cxlr->dev,
3708 "Failed to resolve DPA for region offset %#llx rc %d\n",
3709 offset, rc);
3710
3711 return rc ? rc : -EINVAL;
3712 }
3713
3714 return cxl_inject_poison_locked(result.cxlmd, result.dpa);
3715 }
3716
3717 DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_inject_fops, NULL,
3718 cxl_region_debugfs_poison_inject, "%llx\n");
3719
cxl_region_debugfs_poison_clear(void * data,u64 offset)3720 static int cxl_region_debugfs_poison_clear(void *data, u64 offset)
3721 {
3722 struct dpa_result result = { .dpa = ULLONG_MAX, .cxlmd = NULL };
3723 struct cxl_region *cxlr = data;
3724 int rc;
3725
3726 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
3727 if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem)))
3728 return rc;
3729
3730 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
3731 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
3732 return rc;
3733
3734 if (validate_region_offset(cxlr, offset))
3735 return -EINVAL;
3736
3737 rc = region_offset_to_dpa_result(cxlr, offset, &result);
3738 if (rc || !result.cxlmd || result.dpa == ULLONG_MAX) {
3739 dev_dbg(&cxlr->dev,
3740 "Failed to resolve DPA for region offset %#llx rc %d\n",
3741 offset, rc);
3742
3743 return rc ? rc : -EINVAL;
3744 }
3745
3746 return cxl_clear_poison_locked(result.cxlmd, result.dpa);
3747 }
3748
3749 DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_clear_fops, NULL,
3750 cxl_region_debugfs_poison_clear, "%llx\n");
3751
cxl_region_can_probe(struct cxl_region * cxlr)3752 static int cxl_region_can_probe(struct cxl_region *cxlr)
3753 {
3754 struct cxl_region_params *p = &cxlr->params;
3755 int rc;
3756
3757 ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
3758 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem))) {
3759 dev_dbg(&cxlr->dev, "probe interrupted\n");
3760 return rc;
3761 }
3762
3763 if (p->state < CXL_CONFIG_COMMIT) {
3764 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3765 return -ENXIO;
3766 }
3767
3768 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3769 dev_err(&cxlr->dev,
3770 "failed to activate, re-commit region and retry\n");
3771 return -ENXIO;
3772 }
3773
3774 return 0;
3775 }
3776
cxl_region_probe(struct device * dev)3777 static int cxl_region_probe(struct device *dev)
3778 {
3779 struct cxl_region *cxlr = to_cxl_region(dev);
3780 struct cxl_region_params *p = &cxlr->params;
3781 bool poison_supported = true;
3782 int rc;
3783
3784 rc = cxl_region_can_probe(cxlr);
3785 if (rc)
3786 return rc;
3787
3788 /*
3789 * From this point on any path that changes the region's state away from
3790 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3791 */
3792
3793 cxlr->node_notifier.notifier_call = cxl_region_perf_attrs_callback;
3794 cxlr->node_notifier.priority = CXL_CALLBACK_PRI;
3795 register_node_notifier(&cxlr->node_notifier);
3796
3797 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3798 cxlr->adist_notifier.priority = 100;
3799 register_mt_adistance_algorithm(&cxlr->adist_notifier);
3800
3801 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3802 if (rc)
3803 return rc;
3804
3805 /* Create poison attributes if all memdevs support the capabilities */
3806 for (int i = 0; i < p->nr_targets; i++) {
3807 struct cxl_endpoint_decoder *cxled = p->targets[i];
3808 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3809
3810 if (!cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_INJECT) ||
3811 !cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_CLEAR)) {
3812 poison_supported = false;
3813 break;
3814 }
3815 }
3816
3817 if (poison_supported) {
3818 struct dentry *dentry;
3819
3820 dentry = cxl_debugfs_create_dir(dev_name(dev));
3821 debugfs_create_file("inject_poison", 0200, dentry, cxlr,
3822 &cxl_poison_inject_fops);
3823 debugfs_create_file("clear_poison", 0200, dentry, cxlr,
3824 &cxl_poison_clear_fops);
3825 rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
3826 if (rc)
3827 return rc;
3828 }
3829
3830 switch (cxlr->mode) {
3831 case CXL_PARTMODE_PMEM:
3832 rc = devm_cxl_region_edac_register(cxlr);
3833 if (rc)
3834 dev_dbg(&cxlr->dev, "CXL EDAC registration for region_id=%d failed\n",
3835 cxlr->id);
3836
3837 return devm_cxl_add_pmem_region(cxlr);
3838 case CXL_PARTMODE_RAM:
3839 rc = devm_cxl_region_edac_register(cxlr);
3840 if (rc)
3841 dev_dbg(&cxlr->dev, "CXL EDAC registration for region_id=%d failed\n",
3842 cxlr->id);
3843
3844 /*
3845 * The region can not be manged by CXL if any portion of
3846 * it is already online as 'System RAM'
3847 */
3848 if (walk_iomem_res_desc(IORES_DESC_NONE,
3849 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3850 p->res->start, p->res->end, cxlr,
3851 is_system_ram) > 0)
3852 return 0;
3853 return devm_cxl_add_dax_region(cxlr);
3854 default:
3855 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3856 cxlr->mode);
3857 return -ENXIO;
3858 }
3859 }
3860
3861 static struct cxl_driver cxl_region_driver = {
3862 .name = "cxl_region",
3863 .probe = cxl_region_probe,
3864 .id = CXL_DEVICE_REGION,
3865 };
3866
cxl_region_init(void)3867 int cxl_region_init(void)
3868 {
3869 return cxl_driver_register(&cxl_region_driver);
3870 }
3871
cxl_region_exit(void)3872 void cxl_region_exit(void)
3873 {
3874 cxl_driver_unregister(&cxl_region_driver);
3875 }
3876
3877 MODULE_IMPORT_NS("CXL");
3878 MODULE_IMPORT_NS("DEVMEM");
3879 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
3880