xref: /linux/drivers/cxl/core/region.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/memory.h>
8 #include <linux/slab.h>
9 #include <linux/uuid.h>
10 #include <linux/sort.h>
11 #include <linux/idr.h>
12 #include <linux/memory-tiers.h>
13 #include <cxlmem.h>
14 #include <cxl.h>
15 #include "core.h"
16 
17 /**
18  * DOC: cxl core region
19  *
20  * CXL Regions represent mapped memory capacity in system physical address
21  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
22  * Memory ranges, Regions represent the active mapped capacity by the HDM
23  * Decoder Capability structures throughout the Host Bridges, Switches, and
24  * Endpoints in the topology.
25  *
26  * Region configuration has ordering constraints. UUID may be set at any time
27  * but is only visible for persistent regions.
28  * 1. Interleave granularity
29  * 2. Interleave size
30  * 3. Decoder targets
31  */
32 
33 static struct cxl_region *to_cxl_region(struct device *dev);
34 
35 #define __ACCESS_ATTR_RO(_level, _name) {				\
36 	.attr	= { .name = __stringify(_name), .mode = 0444 },		\
37 	.show	= _name##_access##_level##_show,			\
38 }
39 
40 #define ACCESS_DEVICE_ATTR_RO(level, name)	\
41 	struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
42 
43 #define ACCESS_ATTR_RO(level, attrib)					      \
44 static ssize_t attrib##_access##level##_show(struct device *dev,	      \
45 					  struct device_attribute *attr,      \
46 					  char *buf)			      \
47 {									      \
48 	struct cxl_region *cxlr = to_cxl_region(dev);			      \
49 									      \
50 	if (cxlr->coord[level].attrib == 0)				      \
51 		return -ENOENT;						      \
52 									      \
53 	return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib);	      \
54 }									      \
55 static ACCESS_DEVICE_ATTR_RO(level, attrib)
56 
57 ACCESS_ATTR_RO(0, read_bandwidth);
58 ACCESS_ATTR_RO(0, read_latency);
59 ACCESS_ATTR_RO(0, write_bandwidth);
60 ACCESS_ATTR_RO(0, write_latency);
61 
62 #define ACCESS_ATTR_DECLARE(level, attrib)	\
63 	(&dev_attr_access##level##_##attrib.attr)
64 
65 static struct attribute *access0_coordinate_attrs[] = {
66 	ACCESS_ATTR_DECLARE(0, read_bandwidth),
67 	ACCESS_ATTR_DECLARE(0, write_bandwidth),
68 	ACCESS_ATTR_DECLARE(0, read_latency),
69 	ACCESS_ATTR_DECLARE(0, write_latency),
70 	NULL
71 };
72 
73 ACCESS_ATTR_RO(1, read_bandwidth);
74 ACCESS_ATTR_RO(1, read_latency);
75 ACCESS_ATTR_RO(1, write_bandwidth);
76 ACCESS_ATTR_RO(1, write_latency);
77 
78 static struct attribute *access1_coordinate_attrs[] = {
79 	ACCESS_ATTR_DECLARE(1, read_bandwidth),
80 	ACCESS_ATTR_DECLARE(1, write_bandwidth),
81 	ACCESS_ATTR_DECLARE(1, read_latency),
82 	ACCESS_ATTR_DECLARE(1, write_latency),
83 	NULL
84 };
85 
86 #define ACCESS_VISIBLE(level)						\
87 static umode_t cxl_region_access##level##_coordinate_visible(		\
88 		struct kobject *kobj, struct attribute *a, int n)	\
89 {									\
90 	struct device *dev = kobj_to_dev(kobj);				\
91 	struct cxl_region *cxlr = to_cxl_region(dev);			\
92 									\
93 	if (a == &dev_attr_access##level##_read_latency.attr &&		\
94 	    cxlr->coord[level].read_latency == 0)			\
95 		return 0;						\
96 									\
97 	if (a == &dev_attr_access##level##_write_latency.attr &&	\
98 	    cxlr->coord[level].write_latency == 0)			\
99 		return 0;						\
100 									\
101 	if (a == &dev_attr_access##level##_read_bandwidth.attr &&	\
102 	    cxlr->coord[level].read_bandwidth == 0)			\
103 		return 0;						\
104 									\
105 	if (a == &dev_attr_access##level##_write_bandwidth.attr &&	\
106 	    cxlr->coord[level].write_bandwidth == 0)			\
107 		return 0;						\
108 									\
109 	return a->mode;							\
110 }
111 
112 ACCESS_VISIBLE(0);
113 ACCESS_VISIBLE(1);
114 
115 static const struct attribute_group cxl_region_access0_coordinate_group = {
116 	.name = "access0",
117 	.attrs = access0_coordinate_attrs,
118 	.is_visible = cxl_region_access0_coordinate_visible,
119 };
120 
get_cxl_region_access0_group(void)121 static const struct attribute_group *get_cxl_region_access0_group(void)
122 {
123 	return &cxl_region_access0_coordinate_group;
124 }
125 
126 static const struct attribute_group cxl_region_access1_coordinate_group = {
127 	.name = "access1",
128 	.attrs = access1_coordinate_attrs,
129 	.is_visible = cxl_region_access1_coordinate_visible,
130 };
131 
get_cxl_region_access1_group(void)132 static const struct attribute_group *get_cxl_region_access1_group(void)
133 {
134 	return &cxl_region_access1_coordinate_group;
135 }
136 
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)137 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
138 			 char *buf)
139 {
140 	struct cxl_region *cxlr = to_cxl_region(dev);
141 	struct cxl_region_params *p = &cxlr->params;
142 	ssize_t rc;
143 
144 	rc = down_read_interruptible(&cxl_region_rwsem);
145 	if (rc)
146 		return rc;
147 	if (cxlr->mode != CXL_DECODER_PMEM)
148 		rc = sysfs_emit(buf, "\n");
149 	else
150 		rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
151 	up_read(&cxl_region_rwsem);
152 
153 	return rc;
154 }
155 
is_dup(struct device * match,void * data)156 static int is_dup(struct device *match, void *data)
157 {
158 	struct cxl_region_params *p;
159 	struct cxl_region *cxlr;
160 	uuid_t *uuid = data;
161 
162 	if (!is_cxl_region(match))
163 		return 0;
164 
165 	lockdep_assert_held(&cxl_region_rwsem);
166 	cxlr = to_cxl_region(match);
167 	p = &cxlr->params;
168 
169 	if (uuid_equal(&p->uuid, uuid)) {
170 		dev_dbg(match, "already has uuid: %pUb\n", uuid);
171 		return -EBUSY;
172 	}
173 
174 	return 0;
175 }
176 
uuid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)177 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
178 			  const char *buf, size_t len)
179 {
180 	struct cxl_region *cxlr = to_cxl_region(dev);
181 	struct cxl_region_params *p = &cxlr->params;
182 	uuid_t temp;
183 	ssize_t rc;
184 
185 	if (len != UUID_STRING_LEN + 1)
186 		return -EINVAL;
187 
188 	rc = uuid_parse(buf, &temp);
189 	if (rc)
190 		return rc;
191 
192 	if (uuid_is_null(&temp))
193 		return -EINVAL;
194 
195 	rc = down_write_killable(&cxl_region_rwsem);
196 	if (rc)
197 		return rc;
198 
199 	if (uuid_equal(&p->uuid, &temp))
200 		goto out;
201 
202 	rc = -EBUSY;
203 	if (p->state >= CXL_CONFIG_ACTIVE)
204 		goto out;
205 
206 	rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
207 	if (rc < 0)
208 		goto out;
209 
210 	uuid_copy(&p->uuid, &temp);
211 out:
212 	up_write(&cxl_region_rwsem);
213 
214 	if (rc)
215 		return rc;
216 	return len;
217 }
218 static DEVICE_ATTR_RW(uuid);
219 
cxl_rr_load(struct cxl_port * port,struct cxl_region * cxlr)220 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
221 					  struct cxl_region *cxlr)
222 {
223 	return xa_load(&port->regions, (unsigned long)cxlr);
224 }
225 
cxl_region_invalidate_memregion(struct cxl_region * cxlr)226 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
227 {
228 	if (!cpu_cache_has_invalidate_memregion()) {
229 		if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
230 			dev_info_once(
231 				&cxlr->dev,
232 				"Bypassing cpu_cache_invalidate_memregion() for testing!\n");
233 			return 0;
234 		} else {
235 			dev_WARN(&cxlr->dev,
236 				 "Failed to synchronize CPU cache state\n");
237 			return -ENXIO;
238 		}
239 	}
240 
241 	cpu_cache_invalidate_memregion(IORES_DESC_CXL);
242 	return 0;
243 }
244 
cxl_region_decode_reset(struct cxl_region * cxlr,int count)245 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
246 {
247 	struct cxl_region_params *p = &cxlr->params;
248 	int i;
249 
250 	/*
251 	 * Before region teardown attempt to flush, evict any data cached for
252 	 * this region, or scream loudly about missing arch / platform support
253 	 * for CXL teardown.
254 	 */
255 	cxl_region_invalidate_memregion(cxlr);
256 
257 	for (i = count - 1; i >= 0; i--) {
258 		struct cxl_endpoint_decoder *cxled = p->targets[i];
259 		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
260 		struct cxl_port *iter = cxled_to_port(cxled);
261 		struct cxl_dev_state *cxlds = cxlmd->cxlds;
262 		struct cxl_ep *ep;
263 
264 		if (cxlds->rcd)
265 			goto endpoint_reset;
266 
267 		while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
268 			iter = to_cxl_port(iter->dev.parent);
269 
270 		for (ep = cxl_ep_load(iter, cxlmd); iter;
271 		     iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
272 			struct cxl_region_ref *cxl_rr;
273 			struct cxl_decoder *cxld;
274 
275 			cxl_rr = cxl_rr_load(iter, cxlr);
276 			cxld = cxl_rr->decoder;
277 			if (cxld->reset)
278 				cxld->reset(cxld);
279 			set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
280 		}
281 
282 endpoint_reset:
283 		cxled->cxld.reset(&cxled->cxld);
284 		set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
285 	}
286 
287 	/* all decoders associated with this region have been torn down */
288 	clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
289 }
290 
commit_decoder(struct cxl_decoder * cxld)291 static int commit_decoder(struct cxl_decoder *cxld)
292 {
293 	struct cxl_switch_decoder *cxlsd = NULL;
294 
295 	if (cxld->commit)
296 		return cxld->commit(cxld);
297 
298 	if (is_switch_decoder(&cxld->dev))
299 		cxlsd = to_cxl_switch_decoder(&cxld->dev);
300 
301 	if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
302 			  "->commit() is required\n"))
303 		return -ENXIO;
304 	return 0;
305 }
306 
cxl_region_decode_commit(struct cxl_region * cxlr)307 static int cxl_region_decode_commit(struct cxl_region *cxlr)
308 {
309 	struct cxl_region_params *p = &cxlr->params;
310 	int i, rc = 0;
311 
312 	for (i = 0; i < p->nr_targets; i++) {
313 		struct cxl_endpoint_decoder *cxled = p->targets[i];
314 		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
315 		struct cxl_region_ref *cxl_rr;
316 		struct cxl_decoder *cxld;
317 		struct cxl_port *iter;
318 		struct cxl_ep *ep;
319 
320 		/* commit bottom up */
321 		for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
322 		     iter = to_cxl_port(iter->dev.parent)) {
323 			cxl_rr = cxl_rr_load(iter, cxlr);
324 			cxld = cxl_rr->decoder;
325 			rc = commit_decoder(cxld);
326 			if (rc)
327 				break;
328 		}
329 
330 		if (rc) {
331 			/* programming @iter failed, teardown */
332 			for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
333 			     iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
334 				cxl_rr = cxl_rr_load(iter, cxlr);
335 				cxld = cxl_rr->decoder;
336 				if (cxld->reset)
337 					cxld->reset(cxld);
338 			}
339 
340 			cxled->cxld.reset(&cxled->cxld);
341 			goto err;
342 		}
343 	}
344 
345 	return 0;
346 
347 err:
348 	/* undo the targets that were successfully committed */
349 	cxl_region_decode_reset(cxlr, i);
350 	return rc;
351 }
352 
commit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)353 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
354 			    const char *buf, size_t len)
355 {
356 	struct cxl_region *cxlr = to_cxl_region(dev);
357 	struct cxl_region_params *p = &cxlr->params;
358 	bool commit;
359 	ssize_t rc;
360 
361 	rc = kstrtobool(buf, &commit);
362 	if (rc)
363 		return rc;
364 
365 	rc = down_write_killable(&cxl_region_rwsem);
366 	if (rc)
367 		return rc;
368 
369 	/* Already in the requested state? */
370 	if (commit && p->state >= CXL_CONFIG_COMMIT)
371 		goto out;
372 	if (!commit && p->state < CXL_CONFIG_COMMIT)
373 		goto out;
374 
375 	/* Not ready to commit? */
376 	if (commit && p->state < CXL_CONFIG_ACTIVE) {
377 		rc = -ENXIO;
378 		goto out;
379 	}
380 
381 	/*
382 	 * Invalidate caches before region setup to drop any speculative
383 	 * consumption of this address space
384 	 */
385 	rc = cxl_region_invalidate_memregion(cxlr);
386 	if (rc)
387 		goto out;
388 
389 	if (commit) {
390 		rc = cxl_region_decode_commit(cxlr);
391 		if (rc == 0)
392 			p->state = CXL_CONFIG_COMMIT;
393 	} else {
394 		p->state = CXL_CONFIG_RESET_PENDING;
395 		up_write(&cxl_region_rwsem);
396 		device_release_driver(&cxlr->dev);
397 		down_write(&cxl_region_rwsem);
398 
399 		/*
400 		 * The lock was dropped, so need to revalidate that the reset is
401 		 * still pending.
402 		 */
403 		if (p->state == CXL_CONFIG_RESET_PENDING) {
404 			cxl_region_decode_reset(cxlr, p->interleave_ways);
405 			p->state = CXL_CONFIG_ACTIVE;
406 		}
407 	}
408 
409 out:
410 	up_write(&cxl_region_rwsem);
411 
412 	if (rc)
413 		return rc;
414 	return len;
415 }
416 
commit_show(struct device * dev,struct device_attribute * attr,char * buf)417 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
418 			   char *buf)
419 {
420 	struct cxl_region *cxlr = to_cxl_region(dev);
421 	struct cxl_region_params *p = &cxlr->params;
422 	ssize_t rc;
423 
424 	rc = down_read_interruptible(&cxl_region_rwsem);
425 	if (rc)
426 		return rc;
427 	rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
428 	up_read(&cxl_region_rwsem);
429 
430 	return rc;
431 }
432 static DEVICE_ATTR_RW(commit);
433 
cxl_region_visible(struct kobject * kobj,struct attribute * a,int n)434 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
435 				  int n)
436 {
437 	struct device *dev = kobj_to_dev(kobj);
438 	struct cxl_region *cxlr = to_cxl_region(dev);
439 
440 	/*
441 	 * Support tooling that expects to find a 'uuid' attribute for all
442 	 * regions regardless of mode.
443 	 */
444 	if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
445 		return 0444;
446 	return a->mode;
447 }
448 
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)449 static ssize_t interleave_ways_show(struct device *dev,
450 				    struct device_attribute *attr, char *buf)
451 {
452 	struct cxl_region *cxlr = to_cxl_region(dev);
453 	struct cxl_region_params *p = &cxlr->params;
454 	ssize_t rc;
455 
456 	rc = down_read_interruptible(&cxl_region_rwsem);
457 	if (rc)
458 		return rc;
459 	rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
460 	up_read(&cxl_region_rwsem);
461 
462 	return rc;
463 }
464 
465 static const struct attribute_group *get_cxl_region_target_group(void);
466 
interleave_ways_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)467 static ssize_t interleave_ways_store(struct device *dev,
468 				     struct device_attribute *attr,
469 				     const char *buf, size_t len)
470 {
471 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
472 	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
473 	struct cxl_region *cxlr = to_cxl_region(dev);
474 	struct cxl_region_params *p = &cxlr->params;
475 	unsigned int val, save;
476 	int rc;
477 	u8 iw;
478 
479 	rc = kstrtouint(buf, 0, &val);
480 	if (rc)
481 		return rc;
482 
483 	rc = ways_to_eiw(val, &iw);
484 	if (rc)
485 		return rc;
486 
487 	/*
488 	 * Even for x3, x6, and x12 interleaves the region interleave must be a
489 	 * power of 2 multiple of the host bridge interleave.
490 	 */
491 	if (!is_power_of_2(val / cxld->interleave_ways) ||
492 	    (val % cxld->interleave_ways)) {
493 		dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
494 		return -EINVAL;
495 	}
496 
497 	rc = down_write_killable(&cxl_region_rwsem);
498 	if (rc)
499 		return rc;
500 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
501 		rc = -EBUSY;
502 		goto out;
503 	}
504 
505 	save = p->interleave_ways;
506 	p->interleave_ways = val;
507 	rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
508 	if (rc)
509 		p->interleave_ways = save;
510 out:
511 	up_write(&cxl_region_rwsem);
512 	if (rc)
513 		return rc;
514 	return len;
515 }
516 static DEVICE_ATTR_RW(interleave_ways);
517 
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)518 static ssize_t interleave_granularity_show(struct device *dev,
519 					   struct device_attribute *attr,
520 					   char *buf)
521 {
522 	struct cxl_region *cxlr = to_cxl_region(dev);
523 	struct cxl_region_params *p = &cxlr->params;
524 	ssize_t rc;
525 
526 	rc = down_read_interruptible(&cxl_region_rwsem);
527 	if (rc)
528 		return rc;
529 	rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
530 	up_read(&cxl_region_rwsem);
531 
532 	return rc;
533 }
534 
interleave_granularity_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)535 static ssize_t interleave_granularity_store(struct device *dev,
536 					    struct device_attribute *attr,
537 					    const char *buf, size_t len)
538 {
539 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
540 	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
541 	struct cxl_region *cxlr = to_cxl_region(dev);
542 	struct cxl_region_params *p = &cxlr->params;
543 	int rc, val;
544 	u16 ig;
545 
546 	rc = kstrtoint(buf, 0, &val);
547 	if (rc)
548 		return rc;
549 
550 	rc = granularity_to_eig(val, &ig);
551 	if (rc)
552 		return rc;
553 
554 	/*
555 	 * When the host-bridge is interleaved, disallow region granularity !=
556 	 * root granularity. Regions with a granularity less than the root
557 	 * interleave result in needing multiple endpoints to support a single
558 	 * slot in the interleave (possible to support in the future). Regions
559 	 * with a granularity greater than the root interleave result in invalid
560 	 * DPA translations (invalid to support).
561 	 */
562 	if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
563 		return -EINVAL;
564 
565 	rc = down_write_killable(&cxl_region_rwsem);
566 	if (rc)
567 		return rc;
568 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
569 		rc = -EBUSY;
570 		goto out;
571 	}
572 
573 	p->interleave_granularity = val;
574 out:
575 	up_write(&cxl_region_rwsem);
576 	if (rc)
577 		return rc;
578 	return len;
579 }
580 static DEVICE_ATTR_RW(interleave_granularity);
581 
resource_show(struct device * dev,struct device_attribute * attr,char * buf)582 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
583 			     char *buf)
584 {
585 	struct cxl_region *cxlr = to_cxl_region(dev);
586 	struct cxl_region_params *p = &cxlr->params;
587 	u64 resource = -1ULL;
588 	ssize_t rc;
589 
590 	rc = down_read_interruptible(&cxl_region_rwsem);
591 	if (rc)
592 		return rc;
593 	if (p->res)
594 		resource = p->res->start;
595 	rc = sysfs_emit(buf, "%#llx\n", resource);
596 	up_read(&cxl_region_rwsem);
597 
598 	return rc;
599 }
600 static DEVICE_ATTR_RO(resource);
601 
mode_show(struct device * dev,struct device_attribute * attr,char * buf)602 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
603 			 char *buf)
604 {
605 	struct cxl_region *cxlr = to_cxl_region(dev);
606 
607 	return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
608 }
609 static DEVICE_ATTR_RO(mode);
610 
alloc_hpa(struct cxl_region * cxlr,resource_size_t size)611 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
612 {
613 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
614 	struct cxl_region_params *p = &cxlr->params;
615 	struct resource *res;
616 	u64 remainder = 0;
617 
618 	lockdep_assert_held_write(&cxl_region_rwsem);
619 
620 	/* Nothing to do... */
621 	if (p->res && resource_size(p->res) == size)
622 		return 0;
623 
624 	/* To change size the old size must be freed first */
625 	if (p->res)
626 		return -EBUSY;
627 
628 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
629 		return -EBUSY;
630 
631 	/* ways, granularity and uuid (if PMEM) need to be set before HPA */
632 	if (!p->interleave_ways || !p->interleave_granularity ||
633 	    (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
634 		return -ENXIO;
635 
636 	div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
637 	if (remainder)
638 		return -EINVAL;
639 
640 	res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
641 				    dev_name(&cxlr->dev));
642 	if (IS_ERR(res)) {
643 		dev_dbg(&cxlr->dev,
644 			"HPA allocation error (%ld) for size:%pap in %s %pr\n",
645 			PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
646 		return PTR_ERR(res);
647 	}
648 
649 	p->res = res;
650 	p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
651 
652 	return 0;
653 }
654 
cxl_region_iomem_release(struct cxl_region * cxlr)655 static void cxl_region_iomem_release(struct cxl_region *cxlr)
656 {
657 	struct cxl_region_params *p = &cxlr->params;
658 
659 	if (device_is_registered(&cxlr->dev))
660 		lockdep_assert_held_write(&cxl_region_rwsem);
661 	if (p->res) {
662 		/*
663 		 * Autodiscovered regions may not have been able to insert their
664 		 * resource.
665 		 */
666 		if (p->res->parent)
667 			remove_resource(p->res);
668 		kfree(p->res);
669 		p->res = NULL;
670 	}
671 }
672 
free_hpa(struct cxl_region * cxlr)673 static int free_hpa(struct cxl_region *cxlr)
674 {
675 	struct cxl_region_params *p = &cxlr->params;
676 
677 	lockdep_assert_held_write(&cxl_region_rwsem);
678 
679 	if (!p->res)
680 		return 0;
681 
682 	if (p->state >= CXL_CONFIG_ACTIVE)
683 		return -EBUSY;
684 
685 	cxl_region_iomem_release(cxlr);
686 	p->state = CXL_CONFIG_IDLE;
687 	return 0;
688 }
689 
size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)690 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
691 			  const char *buf, size_t len)
692 {
693 	struct cxl_region *cxlr = to_cxl_region(dev);
694 	u64 val;
695 	int rc;
696 
697 	rc = kstrtou64(buf, 0, &val);
698 	if (rc)
699 		return rc;
700 
701 	rc = down_write_killable(&cxl_region_rwsem);
702 	if (rc)
703 		return rc;
704 
705 	if (val)
706 		rc = alloc_hpa(cxlr, val);
707 	else
708 		rc = free_hpa(cxlr);
709 	up_write(&cxl_region_rwsem);
710 
711 	if (rc)
712 		return rc;
713 
714 	return len;
715 }
716 
size_show(struct device * dev,struct device_attribute * attr,char * buf)717 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
718 			 char *buf)
719 {
720 	struct cxl_region *cxlr = to_cxl_region(dev);
721 	struct cxl_region_params *p = &cxlr->params;
722 	u64 size = 0;
723 	ssize_t rc;
724 
725 	rc = down_read_interruptible(&cxl_region_rwsem);
726 	if (rc)
727 		return rc;
728 	if (p->res)
729 		size = resource_size(p->res);
730 	rc = sysfs_emit(buf, "%#llx\n", size);
731 	up_read(&cxl_region_rwsem);
732 
733 	return rc;
734 }
735 static DEVICE_ATTR_RW(size);
736 
737 static struct attribute *cxl_region_attrs[] = {
738 	&dev_attr_uuid.attr,
739 	&dev_attr_commit.attr,
740 	&dev_attr_interleave_ways.attr,
741 	&dev_attr_interleave_granularity.attr,
742 	&dev_attr_resource.attr,
743 	&dev_attr_size.attr,
744 	&dev_attr_mode.attr,
745 	NULL,
746 };
747 
748 static const struct attribute_group cxl_region_group = {
749 	.attrs = cxl_region_attrs,
750 	.is_visible = cxl_region_visible,
751 };
752 
show_targetN(struct cxl_region * cxlr,char * buf,int pos)753 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
754 {
755 	struct cxl_region_params *p = &cxlr->params;
756 	struct cxl_endpoint_decoder *cxled;
757 	int rc;
758 
759 	rc = down_read_interruptible(&cxl_region_rwsem);
760 	if (rc)
761 		return rc;
762 
763 	if (pos >= p->interleave_ways) {
764 		dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
765 			p->interleave_ways);
766 		rc = -ENXIO;
767 		goto out;
768 	}
769 
770 	cxled = p->targets[pos];
771 	if (!cxled)
772 		rc = sysfs_emit(buf, "\n");
773 	else
774 		rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
775 out:
776 	up_read(&cxl_region_rwsem);
777 
778 	return rc;
779 }
780 
check_commit_order(struct device * dev,void * data)781 static int check_commit_order(struct device *dev, void *data)
782 {
783 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
784 
785 	/*
786 	 * if port->commit_end is not the only free decoder, then out of
787 	 * order shutdown has occurred, block further allocations until
788 	 * that is resolved
789 	 */
790 	if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
791 		return -EBUSY;
792 	return 0;
793 }
794 
match_free_decoder(struct device * dev,const void * data)795 static int match_free_decoder(struct device *dev, const void *data)
796 {
797 	struct cxl_port *port = to_cxl_port(dev->parent);
798 	struct cxl_decoder *cxld;
799 	int rc;
800 
801 	if (!is_switch_decoder(dev))
802 		return 0;
803 
804 	cxld = to_cxl_decoder(dev);
805 
806 	if (cxld->id != port->commit_end + 1)
807 		return 0;
808 
809 	if (cxld->region) {
810 		dev_dbg(dev->parent,
811 			"next decoder to commit (%s) is already reserved (%s)\n",
812 			dev_name(dev), dev_name(&cxld->region->dev));
813 		return 0;
814 	}
815 
816 	rc = device_for_each_child_reverse_from(dev->parent, dev, NULL,
817 						check_commit_order);
818 	if (rc) {
819 		dev_dbg(dev->parent,
820 			"unable to allocate %s due to out of order shutdown\n",
821 			dev_name(dev));
822 		return 0;
823 	}
824 	return 1;
825 }
826 
match_auto_decoder(struct device * dev,const void * data)827 static int match_auto_decoder(struct device *dev, const void *data)
828 {
829 	const struct cxl_region_params *p = data;
830 	struct cxl_decoder *cxld;
831 	struct range *r;
832 
833 	if (!is_switch_decoder(dev))
834 		return 0;
835 
836 	cxld = to_cxl_decoder(dev);
837 	r = &cxld->hpa_range;
838 
839 	if (p->res && p->res->start == r->start && p->res->end == r->end)
840 		return 1;
841 
842 	return 0;
843 }
844 
845 static struct cxl_decoder *
cxl_region_find_decoder(struct cxl_port * port,struct cxl_endpoint_decoder * cxled,struct cxl_region * cxlr)846 cxl_region_find_decoder(struct cxl_port *port,
847 			struct cxl_endpoint_decoder *cxled,
848 			struct cxl_region *cxlr)
849 {
850 	struct device *dev;
851 
852 	if (port == cxled_to_port(cxled))
853 		return &cxled->cxld;
854 
855 	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
856 		dev = device_find_child(&port->dev, &cxlr->params,
857 					match_auto_decoder);
858 	else
859 		dev = device_find_child(&port->dev, NULL, match_free_decoder);
860 	if (!dev)
861 		return NULL;
862 	/*
863 	 * This decoder is pinned registered as long as the endpoint decoder is
864 	 * registered, and endpoint decoder unregistration holds the
865 	 * cxl_region_rwsem over unregister events, so no need to hold on to
866 	 * this extra reference.
867 	 */
868 	put_device(dev);
869 	return to_cxl_decoder(dev);
870 }
871 
auto_order_ok(struct cxl_port * port,struct cxl_region * cxlr_iter,struct cxl_decoder * cxld)872 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
873 			  struct cxl_decoder *cxld)
874 {
875 	struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
876 	struct cxl_decoder *cxld_iter = rr->decoder;
877 
878 	/*
879 	 * Allow the out of order assembly of auto-discovered regions.
880 	 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
881 	 * in HPA order. Confirm that the decoder with the lesser HPA
882 	 * starting address has the lesser id.
883 	 */
884 	dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
885 		dev_name(&cxld->dev), cxld->id,
886 		dev_name(&cxld_iter->dev), cxld_iter->id);
887 
888 	if (cxld_iter->id > cxld->id)
889 		return true;
890 
891 	return false;
892 }
893 
894 static struct cxl_region_ref *
alloc_region_ref(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)895 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
896 		 struct cxl_endpoint_decoder *cxled)
897 {
898 	struct cxl_region_params *p = &cxlr->params;
899 	struct cxl_region_ref *cxl_rr, *iter;
900 	unsigned long index;
901 	int rc;
902 
903 	xa_for_each(&port->regions, index, iter) {
904 		struct cxl_region_params *ip = &iter->region->params;
905 
906 		if (!ip->res || ip->res->start < p->res->start)
907 			continue;
908 
909 		if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
910 			struct cxl_decoder *cxld;
911 
912 			cxld = cxl_region_find_decoder(port, cxled, cxlr);
913 			if (auto_order_ok(port, iter->region, cxld))
914 				continue;
915 		}
916 		dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
917 			dev_name(&port->dev),
918 			dev_name(&iter->region->dev), ip->res, p->res);
919 
920 		return ERR_PTR(-EBUSY);
921 	}
922 
923 	cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
924 	if (!cxl_rr)
925 		return ERR_PTR(-ENOMEM);
926 	cxl_rr->port = port;
927 	cxl_rr->region = cxlr;
928 	cxl_rr->nr_targets = 1;
929 	xa_init(&cxl_rr->endpoints);
930 
931 	rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
932 	if (rc) {
933 		dev_dbg(&cxlr->dev,
934 			"%s: failed to track region reference: %d\n",
935 			dev_name(&port->dev), rc);
936 		kfree(cxl_rr);
937 		return ERR_PTR(rc);
938 	}
939 
940 	return cxl_rr;
941 }
942 
cxl_rr_free_decoder(struct cxl_region_ref * cxl_rr)943 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
944 {
945 	struct cxl_region *cxlr = cxl_rr->region;
946 	struct cxl_decoder *cxld = cxl_rr->decoder;
947 
948 	if (!cxld)
949 		return;
950 
951 	dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
952 	if (cxld->region == cxlr) {
953 		cxld->region = NULL;
954 		put_device(&cxlr->dev);
955 	}
956 }
957 
free_region_ref(struct cxl_region_ref * cxl_rr)958 static void free_region_ref(struct cxl_region_ref *cxl_rr)
959 {
960 	struct cxl_port *port = cxl_rr->port;
961 	struct cxl_region *cxlr = cxl_rr->region;
962 
963 	cxl_rr_free_decoder(cxl_rr);
964 	xa_erase(&port->regions, (unsigned long)cxlr);
965 	xa_destroy(&cxl_rr->endpoints);
966 	kfree(cxl_rr);
967 }
968 
cxl_rr_ep_add(struct cxl_region_ref * cxl_rr,struct cxl_endpoint_decoder * cxled)969 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
970 			 struct cxl_endpoint_decoder *cxled)
971 {
972 	int rc;
973 	struct cxl_port *port = cxl_rr->port;
974 	struct cxl_region *cxlr = cxl_rr->region;
975 	struct cxl_decoder *cxld = cxl_rr->decoder;
976 	struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
977 
978 	if (ep) {
979 		rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
980 			       GFP_KERNEL);
981 		if (rc)
982 			return rc;
983 	}
984 	cxl_rr->nr_eps++;
985 
986 	if (!cxld->region) {
987 		cxld->region = cxlr;
988 		get_device(&cxlr->dev);
989 	}
990 
991 	return 0;
992 }
993 
cxl_rr_alloc_decoder(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_region_ref * cxl_rr)994 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
995 				struct cxl_endpoint_decoder *cxled,
996 				struct cxl_region_ref *cxl_rr)
997 {
998 	struct cxl_decoder *cxld;
999 
1000 	cxld = cxl_region_find_decoder(port, cxled, cxlr);
1001 	if (!cxld) {
1002 		dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1003 			dev_name(&port->dev));
1004 		return -EBUSY;
1005 	}
1006 
1007 	if (cxld->region) {
1008 		dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1009 			dev_name(&port->dev), dev_name(&cxld->dev),
1010 			dev_name(&cxld->region->dev));
1011 		return -EBUSY;
1012 	}
1013 
1014 	/*
1015 	 * Endpoints should already match the region type, but backstop that
1016 	 * assumption with an assertion. Switch-decoders change mapping-type
1017 	 * based on what is mapped when they are assigned to a region.
1018 	 */
1019 	dev_WARN_ONCE(&cxlr->dev,
1020 		      port == cxled_to_port(cxled) &&
1021 			      cxld->target_type != cxlr->type,
1022 		      "%s:%s mismatch decoder type %d -> %d\n",
1023 		      dev_name(&cxled_to_memdev(cxled)->dev),
1024 		      dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1025 	cxld->target_type = cxlr->type;
1026 	cxl_rr->decoder = cxld;
1027 	return 0;
1028 }
1029 
1030 /**
1031  * cxl_port_attach_region() - track a region's interest in a port by endpoint
1032  * @port: port to add a new region reference 'struct cxl_region_ref'
1033  * @cxlr: region to attach to @port
1034  * @cxled: endpoint decoder used to create or further pin a region reference
1035  * @pos: interleave position of @cxled in @cxlr
1036  *
1037  * The attach event is an opportunity to validate CXL decode setup
1038  * constraints and record metadata needed for programming HDM decoders,
1039  * in particular decoder target lists.
1040  *
1041  * The steps are:
1042  *
1043  * - validate that there are no other regions with a higher HPA already
1044  *   associated with @port
1045  * - establish a region reference if one is not already present
1046  *
1047  *   - additionally allocate a decoder instance that will host @cxlr on
1048  *     @port
1049  *
1050  * - pin the region reference by the endpoint
1051  * - account for how many entries in @port's target list are needed to
1052  *   cover all of the added endpoints.
1053  */
cxl_port_attach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1054 static int cxl_port_attach_region(struct cxl_port *port,
1055 				  struct cxl_region *cxlr,
1056 				  struct cxl_endpoint_decoder *cxled, int pos)
1057 {
1058 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1059 	struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1060 	struct cxl_region_ref *cxl_rr;
1061 	bool nr_targets_inc = false;
1062 	struct cxl_decoder *cxld;
1063 	unsigned long index;
1064 	int rc = -EBUSY;
1065 
1066 	lockdep_assert_held_write(&cxl_region_rwsem);
1067 
1068 	cxl_rr = cxl_rr_load(port, cxlr);
1069 	if (cxl_rr) {
1070 		struct cxl_ep *ep_iter;
1071 		int found = 0;
1072 
1073 		/*
1074 		 * Walk the existing endpoints that have been attached to
1075 		 * @cxlr at @port and see if they share the same 'next' port
1076 		 * in the downstream direction. I.e. endpoints that share common
1077 		 * upstream switch.
1078 		 */
1079 		xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1080 			if (ep_iter == ep)
1081 				continue;
1082 			if (ep_iter->next == ep->next) {
1083 				found++;
1084 				break;
1085 			}
1086 		}
1087 
1088 		/*
1089 		 * New target port, or @port is an endpoint port that always
1090 		 * accounts its own local decode as a target.
1091 		 */
1092 		if (!found || !ep->next) {
1093 			cxl_rr->nr_targets++;
1094 			nr_targets_inc = true;
1095 		}
1096 	} else {
1097 		cxl_rr = alloc_region_ref(port, cxlr, cxled);
1098 		if (IS_ERR(cxl_rr)) {
1099 			dev_dbg(&cxlr->dev,
1100 				"%s: failed to allocate region reference\n",
1101 				dev_name(&port->dev));
1102 			return PTR_ERR(cxl_rr);
1103 		}
1104 		nr_targets_inc = true;
1105 
1106 		rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1107 		if (rc)
1108 			goto out_erase;
1109 	}
1110 	cxld = cxl_rr->decoder;
1111 
1112 	/*
1113 	 * the number of targets should not exceed the target_count
1114 	 * of the decoder
1115 	 */
1116 	if (is_switch_decoder(&cxld->dev)) {
1117 		struct cxl_switch_decoder *cxlsd;
1118 
1119 		cxlsd = to_cxl_switch_decoder(&cxld->dev);
1120 		if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1121 			dev_dbg(&cxlr->dev,
1122 				"%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1123 				dev_name(port->uport_dev), dev_name(&port->dev),
1124 				dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1125 				dev_name(&cxled->cxld.dev), pos,
1126 				cxlsd->nr_targets);
1127 			rc = -ENXIO;
1128 			goto out_erase;
1129 		}
1130 	}
1131 
1132 	rc = cxl_rr_ep_add(cxl_rr, cxled);
1133 	if (rc) {
1134 		dev_dbg(&cxlr->dev,
1135 			"%s: failed to track endpoint %s:%s reference\n",
1136 			dev_name(&port->dev), dev_name(&cxlmd->dev),
1137 			dev_name(&cxld->dev));
1138 		goto out_erase;
1139 	}
1140 
1141 	dev_dbg(&cxlr->dev,
1142 		"%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1143 		dev_name(port->uport_dev), dev_name(&port->dev),
1144 		dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1145 		dev_name(&cxled->cxld.dev), pos,
1146 		ep ? ep->next ? dev_name(ep->next->uport_dev) :
1147 				      dev_name(&cxlmd->dev) :
1148 			   "none",
1149 		cxl_rr->nr_eps, cxl_rr->nr_targets);
1150 
1151 	return 0;
1152 out_erase:
1153 	if (nr_targets_inc)
1154 		cxl_rr->nr_targets--;
1155 	if (cxl_rr->nr_eps == 0)
1156 		free_region_ref(cxl_rr);
1157 	return rc;
1158 }
1159 
cxl_port_detach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1160 static void cxl_port_detach_region(struct cxl_port *port,
1161 				   struct cxl_region *cxlr,
1162 				   struct cxl_endpoint_decoder *cxled)
1163 {
1164 	struct cxl_region_ref *cxl_rr;
1165 	struct cxl_ep *ep = NULL;
1166 
1167 	lockdep_assert_held_write(&cxl_region_rwsem);
1168 
1169 	cxl_rr = cxl_rr_load(port, cxlr);
1170 	if (!cxl_rr)
1171 		return;
1172 
1173 	/*
1174 	 * Endpoint ports do not carry cxl_ep references, and they
1175 	 * never target more than one endpoint by definition
1176 	 */
1177 	if (cxl_rr->decoder == &cxled->cxld)
1178 		cxl_rr->nr_eps--;
1179 	else
1180 		ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1181 	if (ep) {
1182 		struct cxl_ep *ep_iter;
1183 		unsigned long index;
1184 		int found = 0;
1185 
1186 		cxl_rr->nr_eps--;
1187 		xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1188 			if (ep_iter->next == ep->next) {
1189 				found++;
1190 				break;
1191 			}
1192 		}
1193 		if (!found)
1194 			cxl_rr->nr_targets--;
1195 	}
1196 
1197 	if (cxl_rr->nr_eps == 0)
1198 		free_region_ref(cxl_rr);
1199 }
1200 
check_last_peer(struct cxl_endpoint_decoder * cxled,struct cxl_ep * ep,struct cxl_region_ref * cxl_rr,int distance)1201 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1202 			   struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1203 			   int distance)
1204 {
1205 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1206 	struct cxl_region *cxlr = cxl_rr->region;
1207 	struct cxl_region_params *p = &cxlr->params;
1208 	struct cxl_endpoint_decoder *cxled_peer;
1209 	struct cxl_port *port = cxl_rr->port;
1210 	struct cxl_memdev *cxlmd_peer;
1211 	struct cxl_ep *ep_peer;
1212 	int pos = cxled->pos;
1213 
1214 	/*
1215 	 * If this position wants to share a dport with the last endpoint mapped
1216 	 * then that endpoint, at index 'position - distance', must also be
1217 	 * mapped by this dport.
1218 	 */
1219 	if (pos < distance) {
1220 		dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1221 			dev_name(port->uport_dev), dev_name(&port->dev),
1222 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1223 		return -ENXIO;
1224 	}
1225 	cxled_peer = p->targets[pos - distance];
1226 	cxlmd_peer = cxled_to_memdev(cxled_peer);
1227 	ep_peer = cxl_ep_load(port, cxlmd_peer);
1228 	if (ep->dport != ep_peer->dport) {
1229 		dev_dbg(&cxlr->dev,
1230 			"%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1231 			dev_name(port->uport_dev), dev_name(&port->dev),
1232 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1233 			dev_name(&cxlmd_peer->dev),
1234 			dev_name(&cxled_peer->cxld.dev));
1235 		return -ENXIO;
1236 	}
1237 
1238 	return 0;
1239 }
1240 
check_interleave_cap(struct cxl_decoder * cxld,int iw,int ig)1241 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1242 {
1243 	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1244 	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1245 	unsigned int interleave_mask;
1246 	u8 eiw;
1247 	u16 eig;
1248 	int high_pos, low_pos;
1249 
1250 	if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1251 		return -ENXIO;
1252 	/*
1253 	 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1254 	 * if eiw < 8:
1255 	 *   DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1256 	 *   DPAOFFSET[eig + 7: 0]  = HPAOFFSET[eig + 7: 0]
1257 	 *
1258 	 *   when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1259 	 *   interleave bits are none.
1260 	 *
1261 	 * if eiw >= 8:
1262 	 *   DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1263 	 *   DPAOFFSET[eig + 7: 0]  = HPAOFFSET[eig + 7: 0]
1264 	 *
1265 	 *   when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1266 	 *   interleave bits are none.
1267 	 */
1268 	ways_to_eiw(iw, &eiw);
1269 	if (eiw == 0 || eiw == 8)
1270 		return 0;
1271 
1272 	granularity_to_eig(ig, &eig);
1273 	if (eiw > 8)
1274 		high_pos = eiw + eig - 1;
1275 	else
1276 		high_pos = eiw + eig + 7;
1277 	low_pos = eig + 8;
1278 	interleave_mask = GENMASK(high_pos, low_pos);
1279 	if (interleave_mask & ~cxlhdm->interleave_mask)
1280 		return -ENXIO;
1281 
1282 	return 0;
1283 }
1284 
cxl_port_setup_targets(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1285 static int cxl_port_setup_targets(struct cxl_port *port,
1286 				  struct cxl_region *cxlr,
1287 				  struct cxl_endpoint_decoder *cxled)
1288 {
1289 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1290 	int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1291 	struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1292 	struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1293 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1294 	struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1295 	struct cxl_region_params *p = &cxlr->params;
1296 	struct cxl_decoder *cxld = cxl_rr->decoder;
1297 	struct cxl_switch_decoder *cxlsd;
1298 	struct cxl_port *iter = port;
1299 	u16 eig, peig;
1300 	u8 eiw, peiw;
1301 
1302 	/*
1303 	 * While root level decoders support x3, x6, x12, switch level
1304 	 * decoders only support powers of 2 up to x16.
1305 	 */
1306 	if (!is_power_of_2(cxl_rr->nr_targets)) {
1307 		dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1308 			dev_name(port->uport_dev), dev_name(&port->dev),
1309 			cxl_rr->nr_targets);
1310 		return -EINVAL;
1311 	}
1312 
1313 	cxlsd = to_cxl_switch_decoder(&cxld->dev);
1314 	if (cxl_rr->nr_targets_set) {
1315 		int i, distance = 1;
1316 		struct cxl_region_ref *cxl_rr_iter;
1317 
1318 		/*
1319 		 * The "distance" between peer downstream ports represents which
1320 		 * endpoint positions in the region interleave a given port can
1321 		 * host.
1322 		 *
1323 		 * For example, at the root of a hierarchy the distance is
1324 		 * always 1 as every index targets a different host-bridge. At
1325 		 * each subsequent switch level those ports map every Nth region
1326 		 * position where N is the width of the switch == distance.
1327 		 */
1328 		do {
1329 			cxl_rr_iter = cxl_rr_load(iter, cxlr);
1330 			distance *= cxl_rr_iter->nr_targets;
1331 			iter = to_cxl_port(iter->dev.parent);
1332 		} while (!is_cxl_root(iter));
1333 		distance *= cxlrd->cxlsd.cxld.interleave_ways;
1334 
1335 		for (i = 0; i < cxl_rr->nr_targets_set; i++)
1336 			if (ep->dport == cxlsd->target[i]) {
1337 				rc = check_last_peer(cxled, ep, cxl_rr,
1338 						     distance);
1339 				if (rc)
1340 					return rc;
1341 				goto out_target_set;
1342 			}
1343 		goto add_target;
1344 	}
1345 
1346 	if (is_cxl_root(parent_port)) {
1347 		/*
1348 		 * Root decoder IG is always set to value in CFMWS which
1349 		 * may be different than this region's IG.  We can use the
1350 		 * region's IG here since interleave_granularity_store()
1351 		 * does not allow interleaved host-bridges with
1352 		 * root IG != region IG.
1353 		 */
1354 		parent_ig = p->interleave_granularity;
1355 		parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1356 		/*
1357 		 * For purposes of address bit routing, use power-of-2 math for
1358 		 * switch ports.
1359 		 */
1360 		if (!is_power_of_2(parent_iw))
1361 			parent_iw /= 3;
1362 	} else {
1363 		struct cxl_region_ref *parent_rr;
1364 		struct cxl_decoder *parent_cxld;
1365 
1366 		parent_rr = cxl_rr_load(parent_port, cxlr);
1367 		parent_cxld = parent_rr->decoder;
1368 		parent_ig = parent_cxld->interleave_granularity;
1369 		parent_iw = parent_cxld->interleave_ways;
1370 	}
1371 
1372 	rc = granularity_to_eig(parent_ig, &peig);
1373 	if (rc) {
1374 		dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1375 			dev_name(parent_port->uport_dev),
1376 			dev_name(&parent_port->dev), parent_ig);
1377 		return rc;
1378 	}
1379 
1380 	rc = ways_to_eiw(parent_iw, &peiw);
1381 	if (rc) {
1382 		dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1383 			dev_name(parent_port->uport_dev),
1384 			dev_name(&parent_port->dev), parent_iw);
1385 		return rc;
1386 	}
1387 
1388 	iw = cxl_rr->nr_targets;
1389 	rc = ways_to_eiw(iw, &eiw);
1390 	if (rc) {
1391 		dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1392 			dev_name(port->uport_dev), dev_name(&port->dev), iw);
1393 		return rc;
1394 	}
1395 
1396 	/*
1397 	 * Interleave granularity is a multiple of @parent_port granularity.
1398 	 * Multiplier is the parent port interleave ways.
1399 	 */
1400 	rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1401 	if (rc) {
1402 		dev_dbg(&cxlr->dev,
1403 			"%s: invalid granularity calculation (%d * %d)\n",
1404 			dev_name(&parent_port->dev), parent_ig, parent_iw);
1405 		return rc;
1406 	}
1407 
1408 	rc = eig_to_granularity(eig, &ig);
1409 	if (rc) {
1410 		dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1411 			dev_name(port->uport_dev), dev_name(&port->dev),
1412 			256 << eig);
1413 		return rc;
1414 	}
1415 
1416 	if (iw > 8 || iw > cxlsd->nr_targets) {
1417 		dev_dbg(&cxlr->dev,
1418 			"%s:%s:%s: ways: %d overflows targets: %d\n",
1419 			dev_name(port->uport_dev), dev_name(&port->dev),
1420 			dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1421 		return -ENXIO;
1422 	}
1423 
1424 	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1425 		if (cxld->interleave_ways != iw ||
1426 		    cxld->interleave_granularity != ig ||
1427 		    cxld->hpa_range.start != p->res->start ||
1428 		    cxld->hpa_range.end != p->res->end ||
1429 		    ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1430 			dev_err(&cxlr->dev,
1431 				"%s:%s %s expected iw: %d ig: %d %pr\n",
1432 				dev_name(port->uport_dev), dev_name(&port->dev),
1433 				__func__, iw, ig, p->res);
1434 			dev_err(&cxlr->dev,
1435 				"%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1436 				dev_name(port->uport_dev), dev_name(&port->dev),
1437 				__func__, cxld->interleave_ways,
1438 				cxld->interleave_granularity,
1439 				(cxld->flags & CXL_DECODER_F_ENABLE) ?
1440 					"enabled" :
1441 					"disabled",
1442 				cxld->hpa_range.start, cxld->hpa_range.end);
1443 			return -ENXIO;
1444 		}
1445 	} else {
1446 		rc = check_interleave_cap(cxld, iw, ig);
1447 		if (rc) {
1448 			dev_dbg(&cxlr->dev,
1449 				"%s:%s iw: %d ig: %d is not supported\n",
1450 				dev_name(port->uport_dev),
1451 				dev_name(&port->dev), iw, ig);
1452 			return rc;
1453 		}
1454 
1455 		cxld->interleave_ways = iw;
1456 		cxld->interleave_granularity = ig;
1457 		cxld->hpa_range = (struct range) {
1458 			.start = p->res->start,
1459 			.end = p->res->end,
1460 		};
1461 	}
1462 	dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1463 		dev_name(&port->dev), iw, ig);
1464 add_target:
1465 	if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1466 		dev_dbg(&cxlr->dev,
1467 			"%s:%s: targets full trying to add %s:%s at %d\n",
1468 			dev_name(port->uport_dev), dev_name(&port->dev),
1469 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1470 		return -ENXIO;
1471 	}
1472 	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1473 		if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1474 			dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1475 				dev_name(port->uport_dev), dev_name(&port->dev),
1476 				dev_name(&cxlsd->cxld.dev),
1477 				dev_name(ep->dport->dport_dev),
1478 				cxl_rr->nr_targets_set);
1479 			return -ENXIO;
1480 		}
1481 	} else
1482 		cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1483 	inc = 1;
1484 out_target_set:
1485 	cxl_rr->nr_targets_set += inc;
1486 	dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1487 		dev_name(port->uport_dev), dev_name(&port->dev),
1488 		cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1489 		dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1490 
1491 	return 0;
1492 }
1493 
cxl_port_reset_targets(struct cxl_port * port,struct cxl_region * cxlr)1494 static void cxl_port_reset_targets(struct cxl_port *port,
1495 				   struct cxl_region *cxlr)
1496 {
1497 	struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1498 	struct cxl_decoder *cxld;
1499 
1500 	/*
1501 	 * After the last endpoint has been detached the entire cxl_rr may now
1502 	 * be gone.
1503 	 */
1504 	if (!cxl_rr)
1505 		return;
1506 	cxl_rr->nr_targets_set = 0;
1507 
1508 	cxld = cxl_rr->decoder;
1509 	cxld->hpa_range = (struct range) {
1510 		.start = 0,
1511 		.end = -1,
1512 	};
1513 }
1514 
cxl_region_teardown_targets(struct cxl_region * cxlr)1515 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1516 {
1517 	struct cxl_region_params *p = &cxlr->params;
1518 	struct cxl_endpoint_decoder *cxled;
1519 	struct cxl_dev_state *cxlds;
1520 	struct cxl_memdev *cxlmd;
1521 	struct cxl_port *iter;
1522 	struct cxl_ep *ep;
1523 	int i;
1524 
1525 	/*
1526 	 * In the auto-discovery case skip automatic teardown since the
1527 	 * address space is already active
1528 	 */
1529 	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1530 		return;
1531 
1532 	for (i = 0; i < p->nr_targets; i++) {
1533 		cxled = p->targets[i];
1534 		cxlmd = cxled_to_memdev(cxled);
1535 		cxlds = cxlmd->cxlds;
1536 
1537 		if (cxlds->rcd)
1538 			continue;
1539 
1540 		iter = cxled_to_port(cxled);
1541 		while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1542 			iter = to_cxl_port(iter->dev.parent);
1543 
1544 		for (ep = cxl_ep_load(iter, cxlmd); iter;
1545 		     iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1546 			cxl_port_reset_targets(iter, cxlr);
1547 	}
1548 }
1549 
cxl_region_setup_targets(struct cxl_region * cxlr)1550 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1551 {
1552 	struct cxl_region_params *p = &cxlr->params;
1553 	struct cxl_endpoint_decoder *cxled;
1554 	struct cxl_dev_state *cxlds;
1555 	int i, rc, rch = 0, vh = 0;
1556 	struct cxl_memdev *cxlmd;
1557 	struct cxl_port *iter;
1558 	struct cxl_ep *ep;
1559 
1560 	for (i = 0; i < p->nr_targets; i++) {
1561 		cxled = p->targets[i];
1562 		cxlmd = cxled_to_memdev(cxled);
1563 		cxlds = cxlmd->cxlds;
1564 
1565 		/* validate that all targets agree on topology */
1566 		if (!cxlds->rcd) {
1567 			vh++;
1568 		} else {
1569 			rch++;
1570 			continue;
1571 		}
1572 
1573 		iter = cxled_to_port(cxled);
1574 		while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1575 			iter = to_cxl_port(iter->dev.parent);
1576 
1577 		/*
1578 		 * Descend the topology tree programming / validating
1579 		 * targets while looking for conflicts.
1580 		 */
1581 		for (ep = cxl_ep_load(iter, cxlmd); iter;
1582 		     iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1583 			rc = cxl_port_setup_targets(iter, cxlr, cxled);
1584 			if (rc) {
1585 				cxl_region_teardown_targets(cxlr);
1586 				return rc;
1587 			}
1588 		}
1589 	}
1590 
1591 	if (rch && vh) {
1592 		dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1593 		cxl_region_teardown_targets(cxlr);
1594 		return -ENXIO;
1595 	}
1596 
1597 	return 0;
1598 }
1599 
cxl_region_validate_position(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1600 static int cxl_region_validate_position(struct cxl_region *cxlr,
1601 					struct cxl_endpoint_decoder *cxled,
1602 					int pos)
1603 {
1604 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1605 	struct cxl_region_params *p = &cxlr->params;
1606 	int i;
1607 
1608 	if (pos < 0 || pos >= p->interleave_ways) {
1609 		dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1610 			p->interleave_ways);
1611 		return -ENXIO;
1612 	}
1613 
1614 	if (p->targets[pos] == cxled)
1615 		return 0;
1616 
1617 	if (p->targets[pos]) {
1618 		struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1619 		struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1620 
1621 		dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1622 			pos, dev_name(&cxlmd_target->dev),
1623 			dev_name(&cxled_target->cxld.dev));
1624 		return -EBUSY;
1625 	}
1626 
1627 	for (i = 0; i < p->interleave_ways; i++) {
1628 		struct cxl_endpoint_decoder *cxled_target;
1629 		struct cxl_memdev *cxlmd_target;
1630 
1631 		cxled_target = p->targets[i];
1632 		if (!cxled_target)
1633 			continue;
1634 
1635 		cxlmd_target = cxled_to_memdev(cxled_target);
1636 		if (cxlmd_target == cxlmd) {
1637 			dev_dbg(&cxlr->dev,
1638 				"%s already specified at position %d via: %s\n",
1639 				dev_name(&cxlmd->dev), pos,
1640 				dev_name(&cxled_target->cxld.dev));
1641 			return -EBUSY;
1642 		}
1643 	}
1644 
1645 	return 0;
1646 }
1647 
cxl_region_attach_position(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled,const struct cxl_dport * dport,int pos)1648 static int cxl_region_attach_position(struct cxl_region *cxlr,
1649 				      struct cxl_root_decoder *cxlrd,
1650 				      struct cxl_endpoint_decoder *cxled,
1651 				      const struct cxl_dport *dport, int pos)
1652 {
1653 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1654 	struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1655 	struct cxl_decoder *cxld = &cxlsd->cxld;
1656 	int iw = cxld->interleave_ways;
1657 	struct cxl_port *iter;
1658 	int rc;
1659 
1660 	if (dport != cxlrd->cxlsd.target[pos % iw]) {
1661 		dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1662 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1663 			dev_name(&cxlrd->cxlsd.cxld.dev));
1664 		return -ENXIO;
1665 	}
1666 
1667 	for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1668 	     iter = to_cxl_port(iter->dev.parent)) {
1669 		rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1670 		if (rc)
1671 			goto err;
1672 	}
1673 
1674 	return 0;
1675 
1676 err:
1677 	for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1678 	     iter = to_cxl_port(iter->dev.parent))
1679 		cxl_port_detach_region(iter, cxlr, cxled);
1680 	return rc;
1681 }
1682 
cxl_region_attach_auto(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1683 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1684 				  struct cxl_endpoint_decoder *cxled, int pos)
1685 {
1686 	struct cxl_region_params *p = &cxlr->params;
1687 
1688 	if (cxled->state != CXL_DECODER_STATE_AUTO) {
1689 		dev_err(&cxlr->dev,
1690 			"%s: unable to add decoder to autodetected region\n",
1691 			dev_name(&cxled->cxld.dev));
1692 		return -EINVAL;
1693 	}
1694 
1695 	if (pos >= 0) {
1696 		dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1697 			dev_name(&cxled->cxld.dev), pos);
1698 		return -EINVAL;
1699 	}
1700 
1701 	if (p->nr_targets >= p->interleave_ways) {
1702 		dev_err(&cxlr->dev, "%s: no more target slots available\n",
1703 			dev_name(&cxled->cxld.dev));
1704 		return -ENXIO;
1705 	}
1706 
1707 	/*
1708 	 * Temporarily record the endpoint decoder into the target array. Yes,
1709 	 * this means that userspace can view devices in the wrong position
1710 	 * before the region activates, and must be careful to understand when
1711 	 * it might be racing region autodiscovery.
1712 	 */
1713 	pos = p->nr_targets;
1714 	p->targets[pos] = cxled;
1715 	cxled->pos = pos;
1716 	p->nr_targets++;
1717 
1718 	return 0;
1719 }
1720 
cmp_interleave_pos(const void * a,const void * b)1721 static int cmp_interleave_pos(const void *a, const void *b)
1722 {
1723 	struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1724 	struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1725 
1726 	return cxled_a->pos - cxled_b->pos;
1727 }
1728 
next_port(struct cxl_port * port)1729 static struct cxl_port *next_port(struct cxl_port *port)
1730 {
1731 	if (!port->parent_dport)
1732 		return NULL;
1733 	return port->parent_dport->port;
1734 }
1735 
match_switch_decoder_by_range(struct device * dev,const void * data)1736 static int match_switch_decoder_by_range(struct device *dev,
1737 					 const void *data)
1738 {
1739 	struct cxl_switch_decoder *cxlsd;
1740 	const struct range *r1, *r2 = data;
1741 
1742 
1743 	if (!is_switch_decoder(dev))
1744 		return 0;
1745 
1746 	cxlsd = to_cxl_switch_decoder(dev);
1747 	r1 = &cxlsd->cxld.hpa_range;
1748 
1749 	if (is_root_decoder(dev))
1750 		return range_contains(r1, r2);
1751 	return (r1->start == r2->start && r1->end == r2->end);
1752 }
1753 
find_pos_and_ways(struct cxl_port * port,struct range * range,int * pos,int * ways)1754 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1755 			     int *pos, int *ways)
1756 {
1757 	struct cxl_switch_decoder *cxlsd;
1758 	struct cxl_port *parent;
1759 	struct device *dev;
1760 	int rc = -ENXIO;
1761 
1762 	parent = next_port(port);
1763 	if (!parent)
1764 		return rc;
1765 
1766 	dev = device_find_child(&parent->dev, range,
1767 				match_switch_decoder_by_range);
1768 	if (!dev) {
1769 		dev_err(port->uport_dev,
1770 			"failed to find decoder mapping %#llx-%#llx\n",
1771 			range->start, range->end);
1772 		return rc;
1773 	}
1774 	cxlsd = to_cxl_switch_decoder(dev);
1775 	*ways = cxlsd->cxld.interleave_ways;
1776 
1777 	for (int i = 0; i < *ways; i++) {
1778 		if (cxlsd->target[i] == port->parent_dport) {
1779 			*pos = i;
1780 			rc = 0;
1781 			break;
1782 		}
1783 	}
1784 	put_device(dev);
1785 
1786 	return rc;
1787 }
1788 
1789 /**
1790  * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1791  * @cxled: endpoint decoder member of given region
1792  *
1793  * The endpoint position is calculated by traversing the topology from
1794  * the endpoint to the root decoder and iteratively applying this
1795  * calculation:
1796  *
1797  *    position = position * parent_ways + parent_pos;
1798  *
1799  * ...where @position is inferred from switch and root decoder target lists.
1800  *
1801  * Return: position >= 0 on success
1802  *	   -ENXIO on failure
1803  */
cxl_calc_interleave_pos(struct cxl_endpoint_decoder * cxled)1804 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1805 {
1806 	struct cxl_port *iter, *port = cxled_to_port(cxled);
1807 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1808 	struct range *range = &cxled->cxld.hpa_range;
1809 	int parent_ways = 0, parent_pos = 0, pos = 0;
1810 	int rc;
1811 
1812 	/*
1813 	 * Example: the expected interleave order of the 4-way region shown
1814 	 * below is: mem0, mem2, mem1, mem3
1815 	 *
1816 	 *		  root_port
1817 	 *                 /      \
1818 	 *      host_bridge_0    host_bridge_1
1819 	 *        |    |           |    |
1820 	 *       mem0 mem1        mem2 mem3
1821 	 *
1822 	 * In the example the calculator will iterate twice. The first iteration
1823 	 * uses the mem position in the host-bridge and the ways of the host-
1824 	 * bridge to generate the first, or local, position. The second
1825 	 * iteration uses the host-bridge position in the root_port and the ways
1826 	 * of the root_port to refine the position.
1827 	 *
1828 	 * A trace of the calculation per endpoint looks like this:
1829 	 * mem0: pos = 0 * 2 + 0    mem2: pos = 0 * 2 + 0
1830 	 *       pos = 0 * 2 + 0          pos = 0 * 2 + 1
1831 	 *       pos: 0                   pos: 1
1832 	 *
1833 	 * mem1: pos = 0 * 2 + 1    mem3: pos = 0 * 2 + 1
1834 	 *       pos = 1 * 2 + 0          pos = 1 * 2 + 1
1835 	 *       pos: 2                   pos = 3
1836 	 *
1837 	 * Note that while this example is simple, the method applies to more
1838 	 * complex topologies, including those with switches.
1839 	 */
1840 
1841 	/* Iterate from endpoint to root_port refining the position */
1842 	for (iter = port; iter; iter = next_port(iter)) {
1843 		if (is_cxl_root(iter))
1844 			break;
1845 
1846 		rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1847 		if (rc)
1848 			return rc;
1849 
1850 		pos = pos * parent_ways + parent_pos;
1851 	}
1852 
1853 	dev_dbg(&cxlmd->dev,
1854 		"decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1855 		dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1856 		dev_name(&port->dev), range->start, range->end, pos);
1857 
1858 	return pos;
1859 }
1860 
cxl_region_sort_targets(struct cxl_region * cxlr)1861 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1862 {
1863 	struct cxl_region_params *p = &cxlr->params;
1864 	int i, rc = 0;
1865 
1866 	for (i = 0; i < p->nr_targets; i++) {
1867 		struct cxl_endpoint_decoder *cxled = p->targets[i];
1868 
1869 		cxled->pos = cxl_calc_interleave_pos(cxled);
1870 		/*
1871 		 * Record that sorting failed, but still continue to calc
1872 		 * cxled->pos so that follow-on code paths can reliably
1873 		 * do p->targets[cxled->pos] to self-reference their entry.
1874 		 */
1875 		if (cxled->pos < 0)
1876 			rc = -ENXIO;
1877 	}
1878 	/* Keep the cxlr target list in interleave position order */
1879 	sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1880 	     cmp_interleave_pos, NULL);
1881 
1882 	dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1883 	return rc;
1884 }
1885 
cxl_region_attach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1886 static int cxl_region_attach(struct cxl_region *cxlr,
1887 			     struct cxl_endpoint_decoder *cxled, int pos)
1888 {
1889 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1890 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1891 	struct cxl_region_params *p = &cxlr->params;
1892 	struct cxl_port *ep_port, *root_port;
1893 	struct cxl_dport *dport;
1894 	int rc = -ENXIO;
1895 
1896 	rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1897 				  p->interleave_granularity);
1898 	if (rc) {
1899 		dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1900 			dev_name(&cxled->cxld.dev), p->interleave_ways,
1901 			p->interleave_granularity);
1902 		return rc;
1903 	}
1904 
1905 	if (cxled->mode != cxlr->mode) {
1906 		dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1907 			dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1908 		return -EINVAL;
1909 	}
1910 
1911 	if (cxled->mode == CXL_DECODER_DEAD) {
1912 		dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1913 		return -ENODEV;
1914 	}
1915 
1916 	/* all full of members, or interleave config not established? */
1917 	if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1918 		dev_dbg(&cxlr->dev, "region already active\n");
1919 		return -EBUSY;
1920 	} else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1921 		dev_dbg(&cxlr->dev, "interleave config missing\n");
1922 		return -ENXIO;
1923 	}
1924 
1925 	if (p->nr_targets >= p->interleave_ways) {
1926 		dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1927 			p->nr_targets);
1928 		return -EINVAL;
1929 	}
1930 
1931 	ep_port = cxled_to_port(cxled);
1932 	root_port = cxlrd_to_port(cxlrd);
1933 	dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1934 	if (!dport) {
1935 		dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1936 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1937 			dev_name(cxlr->dev.parent));
1938 		return -ENXIO;
1939 	}
1940 
1941 	if (cxled->cxld.target_type != cxlr->type) {
1942 		dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1943 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1944 			cxled->cxld.target_type, cxlr->type);
1945 		return -ENXIO;
1946 	}
1947 
1948 	if (!cxled->dpa_res) {
1949 		dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1950 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1951 		return -ENXIO;
1952 	}
1953 
1954 	if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1955 	    resource_size(p->res)) {
1956 		dev_dbg(&cxlr->dev,
1957 			"%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1958 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1959 			(u64)resource_size(cxled->dpa_res), p->interleave_ways,
1960 			(u64)resource_size(p->res));
1961 		return -EINVAL;
1962 	}
1963 
1964 	cxl_region_perf_data_calculate(cxlr, cxled);
1965 
1966 	if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1967 		int i;
1968 
1969 		rc = cxl_region_attach_auto(cxlr, cxled, pos);
1970 		if (rc)
1971 			return rc;
1972 
1973 		/* await more targets to arrive... */
1974 		if (p->nr_targets < p->interleave_ways)
1975 			return 0;
1976 
1977 		/*
1978 		 * All targets are here, which implies all PCI enumeration that
1979 		 * affects this region has been completed. Walk the topology to
1980 		 * sort the devices into their relative region decode position.
1981 		 */
1982 		rc = cxl_region_sort_targets(cxlr);
1983 		if (rc)
1984 			return rc;
1985 
1986 		for (i = 0; i < p->nr_targets; i++) {
1987 			cxled = p->targets[i];
1988 			ep_port = cxled_to_port(cxled);
1989 			dport = cxl_find_dport_by_dev(root_port,
1990 						      ep_port->host_bridge);
1991 			rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1992 							dport, i);
1993 			if (rc)
1994 				return rc;
1995 		}
1996 
1997 		rc = cxl_region_setup_targets(cxlr);
1998 		if (rc)
1999 			return rc;
2000 
2001 		/*
2002 		 * If target setup succeeds in the autodiscovery case
2003 		 * then the region is already committed.
2004 		 */
2005 		p->state = CXL_CONFIG_COMMIT;
2006 		cxl_region_shared_upstream_bandwidth_update(cxlr);
2007 
2008 		return 0;
2009 	}
2010 
2011 	rc = cxl_region_validate_position(cxlr, cxled, pos);
2012 	if (rc)
2013 		return rc;
2014 
2015 	rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2016 	if (rc)
2017 		return rc;
2018 
2019 	p->targets[pos] = cxled;
2020 	cxled->pos = pos;
2021 	p->nr_targets++;
2022 
2023 	if (p->nr_targets == p->interleave_ways) {
2024 		rc = cxl_region_setup_targets(cxlr);
2025 		if (rc)
2026 			return rc;
2027 		p->state = CXL_CONFIG_ACTIVE;
2028 		cxl_region_shared_upstream_bandwidth_update(cxlr);
2029 	}
2030 
2031 	cxled->cxld.interleave_ways = p->interleave_ways;
2032 	cxled->cxld.interleave_granularity = p->interleave_granularity;
2033 	cxled->cxld.hpa_range = (struct range) {
2034 		.start = p->res->start,
2035 		.end = p->res->end,
2036 	};
2037 
2038 	if (p->nr_targets != p->interleave_ways)
2039 		return 0;
2040 
2041 	/*
2042 	 * Test the auto-discovery position calculator function
2043 	 * against this successfully created user-defined region.
2044 	 * A fail message here means that this interleave config
2045 	 * will fail when presented as CXL_REGION_F_AUTO.
2046 	 */
2047 	for (int i = 0; i < p->nr_targets; i++) {
2048 		struct cxl_endpoint_decoder *cxled = p->targets[i];
2049 		int test_pos;
2050 
2051 		test_pos = cxl_calc_interleave_pos(cxled);
2052 		dev_dbg(&cxled->cxld.dev,
2053 			"Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2054 			(test_pos == cxled->pos) ? "success" : "fail",
2055 			test_pos, cxled->pos);
2056 	}
2057 
2058 	return 0;
2059 }
2060 
cxl_region_detach(struct cxl_endpoint_decoder * cxled)2061 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
2062 {
2063 	struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
2064 	struct cxl_region *cxlr = cxled->cxld.region;
2065 	struct cxl_region_params *p;
2066 	int rc = 0;
2067 
2068 	lockdep_assert_held_write(&cxl_region_rwsem);
2069 
2070 	if (!cxlr)
2071 		return 0;
2072 
2073 	p = &cxlr->params;
2074 	get_device(&cxlr->dev);
2075 
2076 	if (p->state > CXL_CONFIG_ACTIVE) {
2077 		cxl_region_decode_reset(cxlr, p->interleave_ways);
2078 		p->state = CXL_CONFIG_ACTIVE;
2079 	}
2080 
2081 	for (iter = ep_port; !is_cxl_root(iter);
2082 	     iter = to_cxl_port(iter->dev.parent))
2083 		cxl_port_detach_region(iter, cxlr, cxled);
2084 
2085 	if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2086 	    p->targets[cxled->pos] != cxled) {
2087 		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2088 
2089 		dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2090 			      dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2091 			      cxled->pos);
2092 		goto out;
2093 	}
2094 
2095 	if (p->state == CXL_CONFIG_ACTIVE) {
2096 		p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2097 		cxl_region_teardown_targets(cxlr);
2098 	}
2099 	p->targets[cxled->pos] = NULL;
2100 	p->nr_targets--;
2101 	cxled->cxld.hpa_range = (struct range) {
2102 		.start = 0,
2103 		.end = -1,
2104 	};
2105 
2106 	/* notify the region driver that one of its targets has departed */
2107 	up_write(&cxl_region_rwsem);
2108 	device_release_driver(&cxlr->dev);
2109 	down_write(&cxl_region_rwsem);
2110 out:
2111 	put_device(&cxlr->dev);
2112 	return rc;
2113 }
2114 
cxl_decoder_kill_region(struct cxl_endpoint_decoder * cxled)2115 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2116 {
2117 	down_write(&cxl_region_rwsem);
2118 	cxled->mode = CXL_DECODER_DEAD;
2119 	cxl_region_detach(cxled);
2120 	up_write(&cxl_region_rwsem);
2121 }
2122 
attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)2123 static int attach_target(struct cxl_region *cxlr,
2124 			 struct cxl_endpoint_decoder *cxled, int pos,
2125 			 unsigned int state)
2126 {
2127 	int rc = 0;
2128 
2129 	if (state == TASK_INTERRUPTIBLE)
2130 		rc = down_write_killable(&cxl_region_rwsem);
2131 	else
2132 		down_write(&cxl_region_rwsem);
2133 	if (rc)
2134 		return rc;
2135 
2136 	down_read(&cxl_dpa_rwsem);
2137 	rc = cxl_region_attach(cxlr, cxled, pos);
2138 	up_read(&cxl_dpa_rwsem);
2139 	up_write(&cxl_region_rwsem);
2140 	return rc;
2141 }
2142 
detach_target(struct cxl_region * cxlr,int pos)2143 static int detach_target(struct cxl_region *cxlr, int pos)
2144 {
2145 	struct cxl_region_params *p = &cxlr->params;
2146 	int rc;
2147 
2148 	rc = down_write_killable(&cxl_region_rwsem);
2149 	if (rc)
2150 		return rc;
2151 
2152 	if (pos >= p->interleave_ways) {
2153 		dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2154 			p->interleave_ways);
2155 		rc = -ENXIO;
2156 		goto out;
2157 	}
2158 
2159 	if (!p->targets[pos]) {
2160 		rc = 0;
2161 		goto out;
2162 	}
2163 
2164 	rc = cxl_region_detach(p->targets[pos]);
2165 out:
2166 	up_write(&cxl_region_rwsem);
2167 	return rc;
2168 }
2169 
store_targetN(struct cxl_region * cxlr,const char * buf,int pos,size_t len)2170 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2171 			    size_t len)
2172 {
2173 	int rc;
2174 
2175 	if (sysfs_streq(buf, "\n"))
2176 		rc = detach_target(cxlr, pos);
2177 	else {
2178 		struct device *dev;
2179 
2180 		dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2181 		if (!dev)
2182 			return -ENODEV;
2183 
2184 		if (!is_endpoint_decoder(dev)) {
2185 			rc = -EINVAL;
2186 			goto out;
2187 		}
2188 
2189 		rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2190 				   TASK_INTERRUPTIBLE);
2191 out:
2192 		put_device(dev);
2193 	}
2194 
2195 	if (rc < 0)
2196 		return rc;
2197 	return len;
2198 }
2199 
2200 #define TARGET_ATTR_RW(n)                                              \
2201 static ssize_t target##n##_show(                                       \
2202 	struct device *dev, struct device_attribute *attr, char *buf)  \
2203 {                                                                      \
2204 	return show_targetN(to_cxl_region(dev), buf, (n));             \
2205 }                                                                      \
2206 static ssize_t target##n##_store(struct device *dev,                   \
2207 				 struct device_attribute *attr,        \
2208 				 const char *buf, size_t len)          \
2209 {                                                                      \
2210 	return store_targetN(to_cxl_region(dev), buf, (n), len);       \
2211 }                                                                      \
2212 static DEVICE_ATTR_RW(target##n)
2213 
2214 TARGET_ATTR_RW(0);
2215 TARGET_ATTR_RW(1);
2216 TARGET_ATTR_RW(2);
2217 TARGET_ATTR_RW(3);
2218 TARGET_ATTR_RW(4);
2219 TARGET_ATTR_RW(5);
2220 TARGET_ATTR_RW(6);
2221 TARGET_ATTR_RW(7);
2222 TARGET_ATTR_RW(8);
2223 TARGET_ATTR_RW(9);
2224 TARGET_ATTR_RW(10);
2225 TARGET_ATTR_RW(11);
2226 TARGET_ATTR_RW(12);
2227 TARGET_ATTR_RW(13);
2228 TARGET_ATTR_RW(14);
2229 TARGET_ATTR_RW(15);
2230 
2231 static struct attribute *target_attrs[] = {
2232 	&dev_attr_target0.attr,
2233 	&dev_attr_target1.attr,
2234 	&dev_attr_target2.attr,
2235 	&dev_attr_target3.attr,
2236 	&dev_attr_target4.attr,
2237 	&dev_attr_target5.attr,
2238 	&dev_attr_target6.attr,
2239 	&dev_attr_target7.attr,
2240 	&dev_attr_target8.attr,
2241 	&dev_attr_target9.attr,
2242 	&dev_attr_target10.attr,
2243 	&dev_attr_target11.attr,
2244 	&dev_attr_target12.attr,
2245 	&dev_attr_target13.attr,
2246 	&dev_attr_target14.attr,
2247 	&dev_attr_target15.attr,
2248 	NULL,
2249 };
2250 
cxl_region_target_visible(struct kobject * kobj,struct attribute * a,int n)2251 static umode_t cxl_region_target_visible(struct kobject *kobj,
2252 					 struct attribute *a, int n)
2253 {
2254 	struct device *dev = kobj_to_dev(kobj);
2255 	struct cxl_region *cxlr = to_cxl_region(dev);
2256 	struct cxl_region_params *p = &cxlr->params;
2257 
2258 	if (n < p->interleave_ways)
2259 		return a->mode;
2260 	return 0;
2261 }
2262 
2263 static const struct attribute_group cxl_region_target_group = {
2264 	.attrs = target_attrs,
2265 	.is_visible = cxl_region_target_visible,
2266 };
2267 
get_cxl_region_target_group(void)2268 static const struct attribute_group *get_cxl_region_target_group(void)
2269 {
2270 	return &cxl_region_target_group;
2271 }
2272 
2273 static const struct attribute_group *region_groups[] = {
2274 	&cxl_base_attribute_group,
2275 	&cxl_region_group,
2276 	&cxl_region_target_group,
2277 	&cxl_region_access0_coordinate_group,
2278 	&cxl_region_access1_coordinate_group,
2279 	NULL,
2280 };
2281 
cxl_region_release(struct device * dev)2282 static void cxl_region_release(struct device *dev)
2283 {
2284 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2285 	struct cxl_region *cxlr = to_cxl_region(dev);
2286 	int id = atomic_read(&cxlrd->region_id);
2287 
2288 	/*
2289 	 * Try to reuse the recently idled id rather than the cached
2290 	 * next id to prevent the region id space from increasing
2291 	 * unnecessarily.
2292 	 */
2293 	if (cxlr->id < id)
2294 		if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2295 			memregion_free(id);
2296 			goto out;
2297 		}
2298 
2299 	memregion_free(cxlr->id);
2300 out:
2301 	put_device(dev->parent);
2302 	kfree(cxlr);
2303 }
2304 
2305 const struct device_type cxl_region_type = {
2306 	.name = "cxl_region",
2307 	.release = cxl_region_release,
2308 	.groups = region_groups
2309 };
2310 
is_cxl_region(struct device * dev)2311 bool is_cxl_region(struct device *dev)
2312 {
2313 	return dev->type == &cxl_region_type;
2314 }
2315 EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2316 
to_cxl_region(struct device * dev)2317 static struct cxl_region *to_cxl_region(struct device *dev)
2318 {
2319 	if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2320 			  "not a cxl_region device\n"))
2321 		return NULL;
2322 
2323 	return container_of(dev, struct cxl_region, dev);
2324 }
2325 
unregister_region(void * _cxlr)2326 static void unregister_region(void *_cxlr)
2327 {
2328 	struct cxl_region *cxlr = _cxlr;
2329 	struct cxl_region_params *p = &cxlr->params;
2330 	int i;
2331 
2332 	device_del(&cxlr->dev);
2333 
2334 	/*
2335 	 * Now that region sysfs is shutdown, the parameter block is now
2336 	 * read-only, so no need to hold the region rwsem to access the
2337 	 * region parameters.
2338 	 */
2339 	for (i = 0; i < p->interleave_ways; i++)
2340 		detach_target(cxlr, i);
2341 
2342 	cxl_region_iomem_release(cxlr);
2343 	put_device(&cxlr->dev);
2344 }
2345 
2346 static struct lock_class_key cxl_region_key;
2347 
cxl_region_alloc(struct cxl_root_decoder * cxlrd,int id)2348 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2349 {
2350 	struct cxl_region *cxlr;
2351 	struct device *dev;
2352 
2353 	cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2354 	if (!cxlr) {
2355 		memregion_free(id);
2356 		return ERR_PTR(-ENOMEM);
2357 	}
2358 
2359 	dev = &cxlr->dev;
2360 	device_initialize(dev);
2361 	lockdep_set_class(&dev->mutex, &cxl_region_key);
2362 	dev->parent = &cxlrd->cxlsd.cxld.dev;
2363 	/*
2364 	 * Keep root decoder pinned through cxl_region_release to fixup
2365 	 * region id allocations
2366 	 */
2367 	get_device(dev->parent);
2368 	device_set_pm_not_required(dev);
2369 	dev->bus = &cxl_bus_type;
2370 	dev->type = &cxl_region_type;
2371 	cxlr->id = id;
2372 
2373 	return cxlr;
2374 }
2375 
cxl_region_update_coordinates(struct cxl_region * cxlr,int nid)2376 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2377 {
2378 	int cset = 0;
2379 	int rc;
2380 
2381 	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2382 		if (cxlr->coord[i].read_bandwidth) {
2383 			rc = 0;
2384 			if (cxl_need_node_perf_attrs_update(nid))
2385 				node_set_perf_attrs(nid, &cxlr->coord[i], i);
2386 			else
2387 				rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2388 
2389 			if (rc == 0)
2390 				cset++;
2391 		}
2392 	}
2393 
2394 	if (!cset)
2395 		return false;
2396 
2397 	rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2398 	if (rc)
2399 		dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2400 
2401 	rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2402 	if (rc)
2403 		dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2404 
2405 	return true;
2406 }
2407 
cxl_region_perf_attrs_callback(struct notifier_block * nb,unsigned long action,void * arg)2408 static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2409 					  unsigned long action, void *arg)
2410 {
2411 	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2412 					       memory_notifier);
2413 	struct memory_notify *mnb = arg;
2414 	int nid = mnb->status_change_nid;
2415 	int region_nid;
2416 
2417 	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2418 		return NOTIFY_DONE;
2419 
2420 	/*
2421 	 * No need to hold cxl_region_rwsem; region parameters are stable
2422 	 * within the cxl_region driver.
2423 	 */
2424 	region_nid = phys_to_target_node(cxlr->params.res->start);
2425 	if (nid != region_nid)
2426 		return NOTIFY_DONE;
2427 
2428 	if (!cxl_region_update_coordinates(cxlr, nid))
2429 		return NOTIFY_DONE;
2430 
2431 	return NOTIFY_OK;
2432 }
2433 
cxl_region_calculate_adistance(struct notifier_block * nb,unsigned long nid,void * data)2434 static int cxl_region_calculate_adistance(struct notifier_block *nb,
2435 					  unsigned long nid, void *data)
2436 {
2437 	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2438 					       adist_notifier);
2439 	struct access_coordinate *perf;
2440 	int *adist = data;
2441 	int region_nid;
2442 
2443 	/*
2444 	 * No need to hold cxl_region_rwsem; region parameters are stable
2445 	 * within the cxl_region driver.
2446 	 */
2447 	region_nid = phys_to_target_node(cxlr->params.res->start);
2448 	if (nid != region_nid)
2449 		return NOTIFY_OK;
2450 
2451 	perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2452 
2453 	if (mt_perf_to_adistance(perf, adist))
2454 		return NOTIFY_OK;
2455 
2456 	return NOTIFY_STOP;
2457 }
2458 
2459 /**
2460  * devm_cxl_add_region - Adds a region to a decoder
2461  * @cxlrd: root decoder
2462  * @id: memregion id to create, or memregion_free() on failure
2463  * @mode: mode for the endpoint decoders of this region
2464  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2465  *
2466  * This is the second step of region initialization. Regions exist within an
2467  * address space which is mapped by a @cxlrd.
2468  *
2469  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2470  * code. The region will be named "regionZ" where Z is the unique region number.
2471  */
devm_cxl_add_region(struct cxl_root_decoder * cxlrd,int id,enum cxl_decoder_mode mode,enum cxl_decoder_type type)2472 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2473 					      int id,
2474 					      enum cxl_decoder_mode mode,
2475 					      enum cxl_decoder_type type)
2476 {
2477 	struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2478 	struct cxl_region *cxlr;
2479 	struct device *dev;
2480 	int rc;
2481 
2482 	cxlr = cxl_region_alloc(cxlrd, id);
2483 	if (IS_ERR(cxlr))
2484 		return cxlr;
2485 	cxlr->mode = mode;
2486 	cxlr->type = type;
2487 
2488 	dev = &cxlr->dev;
2489 	rc = dev_set_name(dev, "region%d", id);
2490 	if (rc)
2491 		goto err;
2492 
2493 	rc = device_add(dev);
2494 	if (rc)
2495 		goto err;
2496 
2497 	rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2498 	if (rc)
2499 		return ERR_PTR(rc);
2500 
2501 	dev_dbg(port->uport_dev, "%s: created %s\n",
2502 		dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2503 	return cxlr;
2504 
2505 err:
2506 	put_device(dev);
2507 	return ERR_PTR(rc);
2508 }
2509 
__create_region_show(struct cxl_root_decoder * cxlrd,char * buf)2510 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2511 {
2512 	return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2513 }
2514 
create_pmem_region_show(struct device * dev,struct device_attribute * attr,char * buf)2515 static ssize_t create_pmem_region_show(struct device *dev,
2516 				       struct device_attribute *attr, char *buf)
2517 {
2518 	return __create_region_show(to_cxl_root_decoder(dev), buf);
2519 }
2520 
create_ram_region_show(struct device * dev,struct device_attribute * attr,char * buf)2521 static ssize_t create_ram_region_show(struct device *dev,
2522 				      struct device_attribute *attr, char *buf)
2523 {
2524 	return __create_region_show(to_cxl_root_decoder(dev), buf);
2525 }
2526 
__create_region(struct cxl_root_decoder * cxlrd,enum cxl_decoder_mode mode,int id)2527 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2528 					  enum cxl_decoder_mode mode, int id)
2529 {
2530 	int rc;
2531 
2532 	switch (mode) {
2533 	case CXL_DECODER_RAM:
2534 	case CXL_DECODER_PMEM:
2535 		break;
2536 	default:
2537 		dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2538 		return ERR_PTR(-EINVAL);
2539 	}
2540 
2541 	rc = memregion_alloc(GFP_KERNEL);
2542 	if (rc < 0)
2543 		return ERR_PTR(rc);
2544 
2545 	if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2546 		memregion_free(rc);
2547 		return ERR_PTR(-EBUSY);
2548 	}
2549 
2550 	return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2551 }
2552 
create_region_store(struct device * dev,const char * buf,size_t len,enum cxl_decoder_mode mode)2553 static ssize_t create_region_store(struct device *dev, const char *buf,
2554 				   size_t len, enum cxl_decoder_mode mode)
2555 {
2556 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2557 	struct cxl_region *cxlr;
2558 	int rc, id;
2559 
2560 	rc = sscanf(buf, "region%d\n", &id);
2561 	if (rc != 1)
2562 		return -EINVAL;
2563 
2564 	cxlr = __create_region(cxlrd, mode, id);
2565 	if (IS_ERR(cxlr))
2566 		return PTR_ERR(cxlr);
2567 
2568 	return len;
2569 }
2570 
create_pmem_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2571 static ssize_t create_pmem_region_store(struct device *dev,
2572 					struct device_attribute *attr,
2573 					const char *buf, size_t len)
2574 {
2575 	return create_region_store(dev, buf, len, CXL_DECODER_PMEM);
2576 }
2577 DEVICE_ATTR_RW(create_pmem_region);
2578 
create_ram_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2579 static ssize_t create_ram_region_store(struct device *dev,
2580 				       struct device_attribute *attr,
2581 				       const char *buf, size_t len)
2582 {
2583 	return create_region_store(dev, buf, len, CXL_DECODER_RAM);
2584 }
2585 DEVICE_ATTR_RW(create_ram_region);
2586 
region_show(struct device * dev,struct device_attribute * attr,char * buf)2587 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2588 			   char *buf)
2589 {
2590 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
2591 	ssize_t rc;
2592 
2593 	rc = down_read_interruptible(&cxl_region_rwsem);
2594 	if (rc)
2595 		return rc;
2596 
2597 	if (cxld->region)
2598 		rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2599 	else
2600 		rc = sysfs_emit(buf, "\n");
2601 	up_read(&cxl_region_rwsem);
2602 
2603 	return rc;
2604 }
2605 DEVICE_ATTR_RO(region);
2606 
2607 static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder * cxlrd,const char * name)2608 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2609 {
2610 	struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2611 	struct device *region_dev;
2612 
2613 	region_dev = device_find_child_by_name(&cxld->dev, name);
2614 	if (!region_dev)
2615 		return ERR_PTR(-ENODEV);
2616 
2617 	return to_cxl_region(region_dev);
2618 }
2619 
delete_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2620 static ssize_t delete_region_store(struct device *dev,
2621 				   struct device_attribute *attr,
2622 				   const char *buf, size_t len)
2623 {
2624 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2625 	struct cxl_port *port = to_cxl_port(dev->parent);
2626 	struct cxl_region *cxlr;
2627 
2628 	cxlr = cxl_find_region_by_name(cxlrd, buf);
2629 	if (IS_ERR(cxlr))
2630 		return PTR_ERR(cxlr);
2631 
2632 	devm_release_action(port->uport_dev, unregister_region, cxlr);
2633 	put_device(&cxlr->dev);
2634 
2635 	return len;
2636 }
2637 DEVICE_ATTR_WO(delete_region);
2638 
cxl_pmem_region_release(struct device * dev)2639 static void cxl_pmem_region_release(struct device *dev)
2640 {
2641 	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2642 	int i;
2643 
2644 	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2645 		struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2646 
2647 		put_device(&cxlmd->dev);
2648 	}
2649 
2650 	kfree(cxlr_pmem);
2651 }
2652 
2653 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2654 	&cxl_base_attribute_group,
2655 	NULL,
2656 };
2657 
2658 const struct device_type cxl_pmem_region_type = {
2659 	.name = "cxl_pmem_region",
2660 	.release = cxl_pmem_region_release,
2661 	.groups = cxl_pmem_region_attribute_groups,
2662 };
2663 
is_cxl_pmem_region(struct device * dev)2664 bool is_cxl_pmem_region(struct device *dev)
2665 {
2666 	return dev->type == &cxl_pmem_region_type;
2667 }
2668 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2669 
to_cxl_pmem_region(struct device * dev)2670 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2671 {
2672 	if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2673 			  "not a cxl_pmem_region device\n"))
2674 		return NULL;
2675 	return container_of(dev, struct cxl_pmem_region, dev);
2676 }
2677 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2678 
2679 struct cxl_poison_context {
2680 	struct cxl_port *port;
2681 	enum cxl_decoder_mode mode;
2682 	u64 offset;
2683 };
2684 
cxl_get_poison_unmapped(struct cxl_memdev * cxlmd,struct cxl_poison_context * ctx)2685 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2686 				   struct cxl_poison_context *ctx)
2687 {
2688 	struct cxl_dev_state *cxlds = cxlmd->cxlds;
2689 	u64 offset, length;
2690 	int rc = 0;
2691 
2692 	/*
2693 	 * Collect poison for the remaining unmapped resources
2694 	 * after poison is collected by committed endpoints.
2695 	 *
2696 	 * Knowing that PMEM must always follow RAM, get poison
2697 	 * for unmapped resources based on the last decoder's mode:
2698 	 *	ram: scan remains of ram range, then any pmem range
2699 	 *	pmem: scan remains of pmem range
2700 	 */
2701 
2702 	if (ctx->mode == CXL_DECODER_RAM) {
2703 		offset = ctx->offset;
2704 		length = resource_size(&cxlds->ram_res) - offset;
2705 		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2706 		if (rc == -EFAULT)
2707 			rc = 0;
2708 		if (rc)
2709 			return rc;
2710 	}
2711 	if (ctx->mode == CXL_DECODER_PMEM) {
2712 		offset = ctx->offset;
2713 		length = resource_size(&cxlds->dpa_res) - offset;
2714 		if (!length)
2715 			return 0;
2716 	} else if (resource_size(&cxlds->pmem_res)) {
2717 		offset = cxlds->pmem_res.start;
2718 		length = resource_size(&cxlds->pmem_res);
2719 	} else {
2720 		return 0;
2721 	}
2722 
2723 	return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2724 }
2725 
poison_by_decoder(struct device * dev,void * arg)2726 static int poison_by_decoder(struct device *dev, void *arg)
2727 {
2728 	struct cxl_poison_context *ctx = arg;
2729 	struct cxl_endpoint_decoder *cxled;
2730 	struct cxl_memdev *cxlmd;
2731 	u64 offset, length;
2732 	int rc = 0;
2733 
2734 	if (!is_endpoint_decoder(dev))
2735 		return rc;
2736 
2737 	cxled = to_cxl_endpoint_decoder(dev);
2738 	if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2739 		return rc;
2740 
2741 	/*
2742 	 * Regions are only created with single mode decoders: pmem or ram.
2743 	 * Linux does not support mixed mode decoders. This means that
2744 	 * reading poison per endpoint decoder adheres to the requirement
2745 	 * that poison reads of pmem and ram must be separated.
2746 	 * CXL 3.0 Spec 8.2.9.8.4.1
2747 	 */
2748 	if (cxled->mode == CXL_DECODER_MIXED) {
2749 		dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2750 		return rc;
2751 	}
2752 
2753 	cxlmd = cxled_to_memdev(cxled);
2754 	if (cxled->skip) {
2755 		offset = cxled->dpa_res->start - cxled->skip;
2756 		length = cxled->skip;
2757 		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2758 		if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2759 			rc = 0;
2760 		if (rc)
2761 			return rc;
2762 	}
2763 
2764 	offset = cxled->dpa_res->start;
2765 	length = cxled->dpa_res->end - offset + 1;
2766 	rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2767 	if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2768 		rc = 0;
2769 	if (rc)
2770 		return rc;
2771 
2772 	/* Iterate until commit_end is reached */
2773 	if (cxled->cxld.id == ctx->port->commit_end) {
2774 		ctx->offset = cxled->dpa_res->end + 1;
2775 		ctx->mode = cxled->mode;
2776 		return 1;
2777 	}
2778 
2779 	return 0;
2780 }
2781 
cxl_get_poison_by_endpoint(struct cxl_port * port)2782 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2783 {
2784 	struct cxl_poison_context ctx;
2785 	int rc = 0;
2786 
2787 	ctx = (struct cxl_poison_context) {
2788 		.port = port
2789 	};
2790 
2791 	rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2792 	if (rc == 1)
2793 		rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2794 					     &ctx);
2795 
2796 	return rc;
2797 }
2798 
2799 struct cxl_dpa_to_region_context {
2800 	struct cxl_region *cxlr;
2801 	u64 dpa;
2802 };
2803 
__cxl_dpa_to_region(struct device * dev,void * arg)2804 static int __cxl_dpa_to_region(struct device *dev, void *arg)
2805 {
2806 	struct cxl_dpa_to_region_context *ctx = arg;
2807 	struct cxl_endpoint_decoder *cxled;
2808 	struct cxl_region *cxlr;
2809 	u64 dpa = ctx->dpa;
2810 
2811 	if (!is_endpoint_decoder(dev))
2812 		return 0;
2813 
2814 	cxled = to_cxl_endpoint_decoder(dev);
2815 	if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2816 		return 0;
2817 
2818 	if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2819 		return 0;
2820 
2821 	/*
2822 	 * Stop the region search (return 1) when an endpoint mapping is
2823 	 * found. The region may not be fully constructed so offering
2824 	 * the cxlr in the context structure is not guaranteed.
2825 	 */
2826 	cxlr = cxled->cxld.region;
2827 	if (cxlr)
2828 		dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2829 			dev_name(&cxlr->dev));
2830 	else
2831 		dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2832 			dev_name(dev));
2833 
2834 	ctx->cxlr = cxlr;
2835 
2836 	return 1;
2837 }
2838 
cxl_dpa_to_region(const struct cxl_memdev * cxlmd,u64 dpa)2839 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2840 {
2841 	struct cxl_dpa_to_region_context ctx;
2842 	struct cxl_port *port;
2843 
2844 	ctx = (struct cxl_dpa_to_region_context) {
2845 		.dpa = dpa,
2846 	};
2847 	port = cxlmd->endpoint;
2848 	if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2849 		device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2850 
2851 	return ctx.cxlr;
2852 }
2853 
cxl_is_hpa_in_chunk(u64 hpa,struct cxl_region * cxlr,int pos)2854 static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2855 {
2856 	struct cxl_region_params *p = &cxlr->params;
2857 	int gran = p->interleave_granularity;
2858 	int ways = p->interleave_ways;
2859 	u64 offset;
2860 
2861 	/* Is the hpa in an expected chunk for its pos(-ition) */
2862 	offset = hpa - p->res->start;
2863 	offset = do_div(offset, gran * ways);
2864 	if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2865 		return true;
2866 
2867 	dev_dbg(&cxlr->dev,
2868 		"Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2869 
2870 	return false;
2871 }
2872 
cxl_dpa_to_hpa(struct cxl_region * cxlr,const struct cxl_memdev * cxlmd,u64 dpa)2873 u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2874 		   u64 dpa)
2875 {
2876 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
2877 	u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2878 	struct cxl_region_params *p = &cxlr->params;
2879 	struct cxl_endpoint_decoder *cxled = NULL;
2880 	u16 eig = 0;
2881 	u8 eiw = 0;
2882 	int pos;
2883 
2884 	for (int i = 0; i < p->nr_targets; i++) {
2885 		cxled = p->targets[i];
2886 		if (cxlmd == cxled_to_memdev(cxled))
2887 			break;
2888 	}
2889 	if (!cxled || cxlmd != cxled_to_memdev(cxled))
2890 		return ULLONG_MAX;
2891 
2892 	pos = cxled->pos;
2893 	ways_to_eiw(p->interleave_ways, &eiw);
2894 	granularity_to_eig(p->interleave_granularity, &eig);
2895 
2896 	/*
2897 	 * The device position in the region interleave set was removed
2898 	 * from the offset at HPA->DPA translation. To reconstruct the
2899 	 * HPA, place the 'pos' in the offset.
2900 	 *
2901 	 * The placement of 'pos' in the HPA is determined by interleave
2902 	 * ways and granularity and is defined in the CXL Spec 3.0 Section
2903 	 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2904 	 */
2905 
2906 	/* Remove the dpa base */
2907 	dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2908 
2909 	mask_upper = GENMASK_ULL(51, eig + 8);
2910 
2911 	if (eiw < 8) {
2912 		hpa_offset = (dpa_offset & mask_upper) << eiw;
2913 		hpa_offset |= pos << (eig + 8);
2914 	} else {
2915 		bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2916 		bits_upper = bits_upper * 3;
2917 		hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2918 	}
2919 
2920 	/* The lower bits remain unchanged */
2921 	hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2922 
2923 	/* Apply the hpa_offset to the region base address */
2924 	hpa = hpa_offset + p->res->start;
2925 
2926 	/* Root decoder translation overrides typical modulo decode */
2927 	if (cxlrd->hpa_to_spa)
2928 		hpa = cxlrd->hpa_to_spa(cxlrd, hpa);
2929 
2930 	if (hpa < p->res->start || hpa > p->res->end) {
2931 		dev_dbg(&cxlr->dev,
2932 			"Addr trans fail: hpa 0x%llx not in region\n", hpa);
2933 		return ULLONG_MAX;
2934 	}
2935 
2936 	/* Simple chunk check, by pos & gran, only applies to modulo decodes */
2937 	if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos)))
2938 		return ULLONG_MAX;
2939 
2940 	return hpa;
2941 }
2942 
2943 static struct lock_class_key cxl_pmem_region_key;
2944 
cxl_pmem_region_alloc(struct cxl_region * cxlr)2945 static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
2946 {
2947 	struct cxl_region_params *p = &cxlr->params;
2948 	struct cxl_nvdimm_bridge *cxl_nvb;
2949 	struct device *dev;
2950 	int i;
2951 
2952 	guard(rwsem_read)(&cxl_region_rwsem);
2953 	if (p->state != CXL_CONFIG_COMMIT)
2954 		return -ENXIO;
2955 
2956 	struct cxl_pmem_region *cxlr_pmem __free(kfree) =
2957 		kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
2958 	if (!cxlr_pmem)
2959 		return -ENOMEM;
2960 
2961 	cxlr_pmem->hpa_range.start = p->res->start;
2962 	cxlr_pmem->hpa_range.end = p->res->end;
2963 
2964 	/* Snapshot the region configuration underneath the cxl_region_rwsem */
2965 	cxlr_pmem->nr_mappings = p->nr_targets;
2966 	for (i = 0; i < p->nr_targets; i++) {
2967 		struct cxl_endpoint_decoder *cxled = p->targets[i];
2968 		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2969 		struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2970 
2971 		/*
2972 		 * Regions never span CXL root devices, so by definition the
2973 		 * bridge for one device is the same for all.
2974 		 */
2975 		if (i == 0) {
2976 			cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint);
2977 			if (!cxl_nvb)
2978 				return -ENODEV;
2979 			cxlr->cxl_nvb = cxl_nvb;
2980 		}
2981 		m->cxlmd = cxlmd;
2982 		get_device(&cxlmd->dev);
2983 		m->start = cxled->dpa_res->start;
2984 		m->size = resource_size(cxled->dpa_res);
2985 		m->position = i;
2986 	}
2987 
2988 	dev = &cxlr_pmem->dev;
2989 	device_initialize(dev);
2990 	lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2991 	device_set_pm_not_required(dev);
2992 	dev->parent = &cxlr->dev;
2993 	dev->bus = &cxl_bus_type;
2994 	dev->type = &cxl_pmem_region_type;
2995 	cxlr_pmem->cxlr = cxlr;
2996 	cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
2997 
2998 	return 0;
2999 }
3000 
cxl_dax_region_release(struct device * dev)3001 static void cxl_dax_region_release(struct device *dev)
3002 {
3003 	struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3004 
3005 	kfree(cxlr_dax);
3006 }
3007 
3008 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3009 	&cxl_base_attribute_group,
3010 	NULL,
3011 };
3012 
3013 const struct device_type cxl_dax_region_type = {
3014 	.name = "cxl_dax_region",
3015 	.release = cxl_dax_region_release,
3016 	.groups = cxl_dax_region_attribute_groups,
3017 };
3018 
is_cxl_dax_region(struct device * dev)3019 static bool is_cxl_dax_region(struct device *dev)
3020 {
3021 	return dev->type == &cxl_dax_region_type;
3022 }
3023 
to_cxl_dax_region(struct device * dev)3024 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3025 {
3026 	if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3027 			  "not a cxl_dax_region device\n"))
3028 		return NULL;
3029 	return container_of(dev, struct cxl_dax_region, dev);
3030 }
3031 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3032 
3033 static struct lock_class_key cxl_dax_region_key;
3034 
cxl_dax_region_alloc(struct cxl_region * cxlr)3035 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3036 {
3037 	struct cxl_region_params *p = &cxlr->params;
3038 	struct cxl_dax_region *cxlr_dax;
3039 	struct device *dev;
3040 
3041 	down_read(&cxl_region_rwsem);
3042 	if (p->state != CXL_CONFIG_COMMIT) {
3043 		cxlr_dax = ERR_PTR(-ENXIO);
3044 		goto out;
3045 	}
3046 
3047 	cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3048 	if (!cxlr_dax) {
3049 		cxlr_dax = ERR_PTR(-ENOMEM);
3050 		goto out;
3051 	}
3052 
3053 	cxlr_dax->hpa_range.start = p->res->start;
3054 	cxlr_dax->hpa_range.end = p->res->end;
3055 
3056 	dev = &cxlr_dax->dev;
3057 	cxlr_dax->cxlr = cxlr;
3058 	device_initialize(dev);
3059 	lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3060 	device_set_pm_not_required(dev);
3061 	dev->parent = &cxlr->dev;
3062 	dev->bus = &cxl_bus_type;
3063 	dev->type = &cxl_dax_region_type;
3064 out:
3065 	up_read(&cxl_region_rwsem);
3066 
3067 	return cxlr_dax;
3068 }
3069 
cxlr_pmem_unregister(void * _cxlr_pmem)3070 static void cxlr_pmem_unregister(void *_cxlr_pmem)
3071 {
3072 	struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3073 	struct cxl_region *cxlr = cxlr_pmem->cxlr;
3074 	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3075 
3076 	/*
3077 	 * Either the bridge is in ->remove() context under the device_lock(),
3078 	 * or cxlr_release_nvdimm() is cancelling the bridge's release action
3079 	 * for @cxlr_pmem and doing it itself (while manually holding the bridge
3080 	 * lock).
3081 	 */
3082 	device_lock_assert(&cxl_nvb->dev);
3083 	cxlr->cxlr_pmem = NULL;
3084 	cxlr_pmem->cxlr = NULL;
3085 	device_unregister(&cxlr_pmem->dev);
3086 }
3087 
cxlr_release_nvdimm(void * _cxlr)3088 static void cxlr_release_nvdimm(void *_cxlr)
3089 {
3090 	struct cxl_region *cxlr = _cxlr;
3091 	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3092 
3093 	scoped_guard(device, &cxl_nvb->dev) {
3094 		if (cxlr->cxlr_pmem)
3095 			devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
3096 					    cxlr->cxlr_pmem);
3097 	}
3098 	cxlr->cxl_nvb = NULL;
3099 	put_device(&cxl_nvb->dev);
3100 }
3101 
3102 /**
3103  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3104  * @cxlr: parent CXL region for this pmem region bridge device
3105  *
3106  * Return: 0 on success negative error code on failure.
3107  */
devm_cxl_add_pmem_region(struct cxl_region * cxlr)3108 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3109 {
3110 	struct cxl_pmem_region *cxlr_pmem;
3111 	struct cxl_nvdimm_bridge *cxl_nvb;
3112 	struct device *dev;
3113 	int rc;
3114 
3115 	rc = cxl_pmem_region_alloc(cxlr);
3116 	if (rc)
3117 		return rc;
3118 	cxlr_pmem = cxlr->cxlr_pmem;
3119 	cxl_nvb = cxlr->cxl_nvb;
3120 
3121 	dev = &cxlr_pmem->dev;
3122 	rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
3123 	if (rc)
3124 		goto err;
3125 
3126 	rc = device_add(dev);
3127 	if (rc)
3128 		goto err;
3129 
3130 	dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3131 		dev_name(dev));
3132 
3133 	scoped_guard(device, &cxl_nvb->dev) {
3134 		if (cxl_nvb->dev.driver)
3135 			rc = devm_add_action_or_reset(&cxl_nvb->dev,
3136 						      cxlr_pmem_unregister,
3137 						      cxlr_pmem);
3138 		else
3139 			rc = -ENXIO;
3140 	}
3141 
3142 	if (rc)
3143 		goto err_bridge;
3144 
3145 	/* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3146 	return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3147 
3148 err:
3149 	put_device(dev);
3150 err_bridge:
3151 	put_device(&cxl_nvb->dev);
3152 	cxlr->cxl_nvb = NULL;
3153 	return rc;
3154 }
3155 
cxlr_dax_unregister(void * _cxlr_dax)3156 static void cxlr_dax_unregister(void *_cxlr_dax)
3157 {
3158 	struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3159 
3160 	device_unregister(&cxlr_dax->dev);
3161 }
3162 
devm_cxl_add_dax_region(struct cxl_region * cxlr)3163 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3164 {
3165 	struct cxl_dax_region *cxlr_dax;
3166 	struct device *dev;
3167 	int rc;
3168 
3169 	cxlr_dax = cxl_dax_region_alloc(cxlr);
3170 	if (IS_ERR(cxlr_dax))
3171 		return PTR_ERR(cxlr_dax);
3172 
3173 	dev = &cxlr_dax->dev;
3174 	rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3175 	if (rc)
3176 		goto err;
3177 
3178 	rc = device_add(dev);
3179 	if (rc)
3180 		goto err;
3181 
3182 	dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3183 		dev_name(dev));
3184 
3185 	return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3186 					cxlr_dax);
3187 err:
3188 	put_device(dev);
3189 	return rc;
3190 }
3191 
match_root_decoder_by_range(struct device * dev,const void * data)3192 static int match_root_decoder_by_range(struct device *dev,
3193 				       const void *data)
3194 {
3195 	const struct range *r1, *r2 = data;
3196 	struct cxl_root_decoder *cxlrd;
3197 
3198 	if (!is_root_decoder(dev))
3199 		return 0;
3200 
3201 	cxlrd = to_cxl_root_decoder(dev);
3202 	r1 = &cxlrd->cxlsd.cxld.hpa_range;
3203 	return range_contains(r1, r2);
3204 }
3205 
match_region_by_range(struct device * dev,const void * data)3206 static int match_region_by_range(struct device *dev, const void *data)
3207 {
3208 	struct cxl_region_params *p;
3209 	struct cxl_region *cxlr;
3210 	const struct range *r = data;
3211 	int rc = 0;
3212 
3213 	if (!is_cxl_region(dev))
3214 		return 0;
3215 
3216 	cxlr = to_cxl_region(dev);
3217 	p = &cxlr->params;
3218 
3219 	down_read(&cxl_region_rwsem);
3220 	if (p->res && p->res->start == r->start && p->res->end == r->end)
3221 		rc = 1;
3222 	up_read(&cxl_region_rwsem);
3223 
3224 	return rc;
3225 }
3226 
3227 /* Establish an empty region covering the given HPA range */
construct_region(struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)3228 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3229 					   struct cxl_endpoint_decoder *cxled)
3230 {
3231 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3232 	struct cxl_port *port = cxlrd_to_port(cxlrd);
3233 	struct range *hpa = &cxled->cxld.hpa_range;
3234 	struct cxl_region_params *p;
3235 	struct cxl_region *cxlr;
3236 	struct resource *res;
3237 	int rc;
3238 
3239 	do {
3240 		cxlr = __create_region(cxlrd, cxled->mode,
3241 				       atomic_read(&cxlrd->region_id));
3242 	} while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3243 
3244 	if (IS_ERR(cxlr)) {
3245 		dev_err(cxlmd->dev.parent,
3246 			"%s:%s: %s failed assign region: %ld\n",
3247 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3248 			__func__, PTR_ERR(cxlr));
3249 		return cxlr;
3250 	}
3251 
3252 	down_write(&cxl_region_rwsem);
3253 	p = &cxlr->params;
3254 	if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3255 		dev_err(cxlmd->dev.parent,
3256 			"%s:%s: %s autodiscovery interrupted\n",
3257 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3258 			__func__);
3259 		rc = -EBUSY;
3260 		goto err;
3261 	}
3262 
3263 	set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3264 
3265 	res = kmalloc(sizeof(*res), GFP_KERNEL);
3266 	if (!res) {
3267 		rc = -ENOMEM;
3268 		goto err;
3269 	}
3270 
3271 	*res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3272 				    dev_name(&cxlr->dev));
3273 	rc = insert_resource(cxlrd->res, res);
3274 	if (rc) {
3275 		/*
3276 		 * Platform-firmware may not have split resources like "System
3277 		 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3278 		 */
3279 		dev_warn(cxlmd->dev.parent,
3280 			 "%s:%s: %s %s cannot insert resource\n",
3281 			 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3282 			 __func__, dev_name(&cxlr->dev));
3283 	}
3284 
3285 	p->res = res;
3286 	p->interleave_ways = cxled->cxld.interleave_ways;
3287 	p->interleave_granularity = cxled->cxld.interleave_granularity;
3288 	p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3289 
3290 	rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3291 	if (rc)
3292 		goto err;
3293 
3294 	dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3295 		dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3296 		dev_name(&cxlr->dev), p->res, p->interleave_ways,
3297 		p->interleave_granularity);
3298 
3299 	/* ...to match put_device() in cxl_add_to_region() */
3300 	get_device(&cxlr->dev);
3301 	up_write(&cxl_region_rwsem);
3302 
3303 	return cxlr;
3304 
3305 err:
3306 	up_write(&cxl_region_rwsem);
3307 	devm_release_action(port->uport_dev, unregister_region, cxlr);
3308 	return ERR_PTR(rc);
3309 }
3310 
cxl_add_to_region(struct cxl_port * root,struct cxl_endpoint_decoder * cxled)3311 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3312 {
3313 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3314 	struct range *hpa = &cxled->cxld.hpa_range;
3315 	struct cxl_decoder *cxld = &cxled->cxld;
3316 	struct device *cxlrd_dev, *region_dev;
3317 	struct cxl_root_decoder *cxlrd;
3318 	struct cxl_region_params *p;
3319 	struct cxl_region *cxlr;
3320 	bool attach = false;
3321 	int rc;
3322 
3323 	cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3324 				      match_root_decoder_by_range);
3325 	if (!cxlrd_dev) {
3326 		dev_err(cxlmd->dev.parent,
3327 			"%s:%s no CXL window for range %#llx:%#llx\n",
3328 			dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3329 			cxld->hpa_range.start, cxld->hpa_range.end);
3330 		return -ENXIO;
3331 	}
3332 
3333 	cxlrd = to_cxl_root_decoder(cxlrd_dev);
3334 
3335 	/*
3336 	 * Ensure that if multiple threads race to construct_region() for @hpa
3337 	 * one does the construction and the others add to that.
3338 	 */
3339 	mutex_lock(&cxlrd->range_lock);
3340 	region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3341 				       match_region_by_range);
3342 	if (!region_dev) {
3343 		cxlr = construct_region(cxlrd, cxled);
3344 		region_dev = &cxlr->dev;
3345 	} else
3346 		cxlr = to_cxl_region(region_dev);
3347 	mutex_unlock(&cxlrd->range_lock);
3348 
3349 	rc = PTR_ERR_OR_ZERO(cxlr);
3350 	if (rc)
3351 		goto out;
3352 
3353 	attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3354 
3355 	down_read(&cxl_region_rwsem);
3356 	p = &cxlr->params;
3357 	attach = p->state == CXL_CONFIG_COMMIT;
3358 	up_read(&cxl_region_rwsem);
3359 
3360 	if (attach) {
3361 		/*
3362 		 * If device_attach() fails the range may still be active via
3363 		 * the platform-firmware memory map, otherwise the driver for
3364 		 * regions is local to this file, so driver matching can't fail.
3365 		 */
3366 		if (device_attach(&cxlr->dev) < 0)
3367 			dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3368 				p->res);
3369 	}
3370 
3371 	put_device(region_dev);
3372 out:
3373 	put_device(cxlrd_dev);
3374 	return rc;
3375 }
3376 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3377 
is_system_ram(struct resource * res,void * arg)3378 static int is_system_ram(struct resource *res, void *arg)
3379 {
3380 	struct cxl_region *cxlr = arg;
3381 	struct cxl_region_params *p = &cxlr->params;
3382 
3383 	dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3384 	return 1;
3385 }
3386 
shutdown_notifiers(void * _cxlr)3387 static void shutdown_notifiers(void *_cxlr)
3388 {
3389 	struct cxl_region *cxlr = _cxlr;
3390 
3391 	unregister_memory_notifier(&cxlr->memory_notifier);
3392 	unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
3393 }
3394 
cxl_region_probe(struct device * dev)3395 static int cxl_region_probe(struct device *dev)
3396 {
3397 	struct cxl_region *cxlr = to_cxl_region(dev);
3398 	struct cxl_region_params *p = &cxlr->params;
3399 	int rc;
3400 
3401 	rc = down_read_interruptible(&cxl_region_rwsem);
3402 	if (rc) {
3403 		dev_dbg(&cxlr->dev, "probe interrupted\n");
3404 		return rc;
3405 	}
3406 
3407 	if (p->state < CXL_CONFIG_COMMIT) {
3408 		dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3409 		rc = -ENXIO;
3410 		goto out;
3411 	}
3412 
3413 	if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3414 		dev_err(&cxlr->dev,
3415 			"failed to activate, re-commit region and retry\n");
3416 		rc = -ENXIO;
3417 		goto out;
3418 	}
3419 
3420 	/*
3421 	 * From this point on any path that changes the region's state away from
3422 	 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3423 	 */
3424 out:
3425 	up_read(&cxl_region_rwsem);
3426 
3427 	if (rc)
3428 		return rc;
3429 
3430 	cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
3431 	cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
3432 	register_memory_notifier(&cxlr->memory_notifier);
3433 
3434 	cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3435 	cxlr->adist_notifier.priority = 100;
3436 	register_mt_adistance_algorithm(&cxlr->adist_notifier);
3437 
3438 	rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3439 	if (rc)
3440 		return rc;
3441 
3442 	switch (cxlr->mode) {
3443 	case CXL_DECODER_PMEM:
3444 		return devm_cxl_add_pmem_region(cxlr);
3445 	case CXL_DECODER_RAM:
3446 		/*
3447 		 * The region can not be manged by CXL if any portion of
3448 		 * it is already online as 'System RAM'
3449 		 */
3450 		if (walk_iomem_res_desc(IORES_DESC_NONE,
3451 					IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3452 					p->res->start, p->res->end, cxlr,
3453 					is_system_ram) > 0)
3454 			return 0;
3455 		return devm_cxl_add_dax_region(cxlr);
3456 	default:
3457 		dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3458 			cxlr->mode);
3459 		return -ENXIO;
3460 	}
3461 }
3462 
3463 static struct cxl_driver cxl_region_driver = {
3464 	.name = "cxl_region",
3465 	.probe = cxl_region_probe,
3466 	.id = CXL_DEVICE_REGION,
3467 };
3468 
cxl_region_init(void)3469 int cxl_region_init(void)
3470 {
3471 	return cxl_driver_register(&cxl_region_driver);
3472 }
3473 
cxl_region_exit(void)3474 void cxl_region_exit(void)
3475 {
3476 	cxl_driver_unregister(&cxl_region_driver);
3477 }
3478 
3479 MODULE_IMPORT_NS("CXL");
3480 MODULE_IMPORT_NS("DEVMEM");
3481 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
3482