1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/seq_file.h>
4 #include <linux/device.h>
5 #include <linux/delay.h>
6
7 #include "cxlmem.h"
8 #include "core.h"
9
10 /**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19 DECLARE_RWSEM(cxl_dpa_rwsem);
20
add_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map)21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
22 int *target_map)
23 {
24 int rc;
25
26 rc = cxl_decoder_add_locked(cxld, target_map);
27 if (rc) {
28 put_device(&cxld->dev);
29 dev_err(&port->dev, "Failed to add decoder\n");
30 return rc;
31 }
32
33 rc = cxl_decoder_autoremove(&port->dev, cxld);
34 if (rc)
35 return rc;
36
37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
38
39 return 0;
40 }
41
42 /*
43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
44 * single ported host-bridges need not publish a decoder capability when a
45 * passthrough decode can be assumed, i.e. all transactions that the uport sees
46 * are claimed and passed to the single dport. Disable the range until the first
47 * CXL region is enumerated / activated.
48 */
devm_cxl_add_passthrough_decoder(struct cxl_port * port)49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
50 {
51 struct cxl_switch_decoder *cxlsd;
52 struct cxl_dport *dport = NULL;
53 int single_port_map[1];
54 unsigned long index;
55 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
56
57 /*
58 * Capability checks are moot for passthrough decoders, support
59 * any and all possibilities.
60 */
61 cxlhdm->interleave_mask = ~0U;
62 cxlhdm->iw_cap_mask = ~0UL;
63
64 cxlsd = cxl_switch_decoder_alloc(port, 1);
65 if (IS_ERR(cxlsd))
66 return PTR_ERR(cxlsd);
67
68 device_lock_assert(&port->dev);
69
70 xa_for_each(&port->dports, index, dport)
71 break;
72 single_port_map[0] = dport->port_id;
73
74 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
75 }
76 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, "CXL");
77
parse_hdm_decoder_caps(struct cxl_hdm * cxlhdm)78 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
79 {
80 u32 hdm_cap;
81
82 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
83 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
84 cxlhdm->target_count =
85 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
86 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
87 cxlhdm->interleave_mask |= GENMASK(11, 8);
88 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
89 cxlhdm->interleave_mask |= GENMASK(14, 12);
90 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);
91 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))
92 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);
93 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))
94 cxlhdm->iw_cap_mask |= BIT(16);
95 }
96
should_emulate_decoders(struct cxl_endpoint_dvsec_info * info)97 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
98 {
99 struct cxl_hdm *cxlhdm;
100 void __iomem *hdm;
101 u32 ctrl;
102 int i;
103
104 if (!info)
105 return false;
106
107 cxlhdm = dev_get_drvdata(&info->port->dev);
108 hdm = cxlhdm->regs.hdm_decoder;
109
110 if (!hdm)
111 return true;
112
113 /*
114 * If HDM decoders are present and the driver is in control of
115 * Mem_Enable skip DVSEC based emulation
116 */
117 if (!info->mem_enabled)
118 return false;
119
120 /*
121 * If any decoders are committed already, there should not be any
122 * emulated DVSEC decoders.
123 */
124 for (i = 0; i < cxlhdm->decoder_count; i++) {
125 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
126 dev_dbg(&info->port->dev,
127 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
128 info->port->id, i,
129 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
130 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
131 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
132 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
133 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
134 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
135 return false;
136 }
137
138 return true;
139 }
140
141 /**
142 * devm_cxl_setup_hdm - map HDM decoder component registers
143 * @port: cxl_port to map
144 * @info: cached DVSEC range register info
145 */
devm_cxl_setup_hdm(struct cxl_port * port,struct cxl_endpoint_dvsec_info * info)146 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
147 struct cxl_endpoint_dvsec_info *info)
148 {
149 struct cxl_register_map *reg_map = &port->reg_map;
150 struct device *dev = &port->dev;
151 struct cxl_hdm *cxlhdm;
152 int rc;
153
154 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
155 if (!cxlhdm)
156 return ERR_PTR(-ENOMEM);
157 cxlhdm->port = port;
158 dev_set_drvdata(dev, cxlhdm);
159
160 /* Memory devices can configure device HDM using DVSEC range regs. */
161 if (reg_map->resource == CXL_RESOURCE_NONE) {
162 if (!info || !info->mem_enabled) {
163 dev_err(dev, "No component registers mapped\n");
164 return ERR_PTR(-ENXIO);
165 }
166
167 cxlhdm->decoder_count = info->ranges;
168 return cxlhdm;
169 }
170
171 if (!reg_map->component_map.hdm_decoder.valid) {
172 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
173 /* unique error code to indicate no HDM decoder capability */
174 return ERR_PTR(-ENODEV);
175 }
176
177 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,
178 BIT(CXL_CM_CAP_CAP_ID_HDM));
179 if (rc) {
180 dev_err(dev, "Failed to map HDM capability.\n");
181 return ERR_PTR(rc);
182 }
183
184 parse_hdm_decoder_caps(cxlhdm);
185 if (cxlhdm->decoder_count == 0) {
186 dev_err(dev, "Spec violation. Caps invalid\n");
187 return ERR_PTR(-ENXIO);
188 }
189
190 /*
191 * Now that the hdm capability is parsed, decide if range
192 * register emulation is needed and fixup cxlhdm accordingly.
193 */
194 if (should_emulate_decoders(info)) {
195 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
196 info->ranges > 1 ? "s" : "");
197 cxlhdm->decoder_count = info->ranges;
198 }
199
200 return cxlhdm;
201 }
202 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, "CXL");
203
__cxl_dpa_debug(struct seq_file * file,struct resource * r,int depth)204 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
205 {
206 unsigned long long start = r->start, end = r->end;
207
208 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
209 r->name);
210 }
211
cxl_dpa_debug(struct seq_file * file,struct cxl_dev_state * cxlds)212 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
213 {
214 struct resource *p1, *p2;
215
216 guard(rwsem_read)(&cxl_dpa_rwsem);
217 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
218 __cxl_dpa_debug(file, p1, 0);
219 for (p2 = p1->child; p2; p2 = p2->sibling)
220 __cxl_dpa_debug(file, p2, 1);
221 }
222 }
223 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL");
224
225 /* See request_skip() kernel-doc */
__adjust_skip(struct cxl_dev_state * cxlds,const resource_size_t skip_base,const resource_size_t skip_len,const char * requester)226 static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds,
227 const resource_size_t skip_base,
228 const resource_size_t skip_len,
229 const char *requester)
230 {
231 const resource_size_t skip_end = skip_base + skip_len - 1;
232
233 for (int i = 0; i < cxlds->nr_partitions; i++) {
234 const struct resource *part_res = &cxlds->part[i].res;
235 resource_size_t adjust_start, adjust_end, size;
236
237 adjust_start = max(skip_base, part_res->start);
238 adjust_end = min(skip_end, part_res->end);
239
240 if (adjust_end < adjust_start)
241 continue;
242
243 size = adjust_end - adjust_start + 1;
244
245 if (!requester)
246 __release_region(&cxlds->dpa_res, adjust_start, size);
247 else if (!__request_region(&cxlds->dpa_res, adjust_start, size,
248 requester, 0))
249 return adjust_start - skip_base;
250 }
251
252 return skip_len;
253 }
254 #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL)
255
256 /*
257 * Must be called in a context that synchronizes against this decoder's
258 * port ->remove() callback (like an endpoint decoder sysfs attribute)
259 */
__cxl_dpa_release(struct cxl_endpoint_decoder * cxled)260 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
261 {
262 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
263 struct cxl_port *port = cxled_to_port(cxled);
264 struct cxl_dev_state *cxlds = cxlmd->cxlds;
265 struct resource *res = cxled->dpa_res;
266 resource_size_t skip_start;
267
268 lockdep_assert_held_write(&cxl_dpa_rwsem);
269
270 /* save @skip_start, before @res is released */
271 skip_start = res->start - cxled->skip;
272 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
273 if (cxled->skip)
274 release_skip(cxlds, skip_start, cxled->skip);
275 cxled->skip = 0;
276 cxled->dpa_res = NULL;
277 put_device(&cxled->cxld.dev);
278 port->hdm_end--;
279 }
280
cxl_dpa_release(void * cxled)281 static void cxl_dpa_release(void *cxled)
282 {
283 guard(rwsem_write)(&cxl_dpa_rwsem);
284 __cxl_dpa_release(cxled);
285 }
286
287 /*
288 * Must be called from context that will not race port device
289 * unregistration, like decoder sysfs attribute methods
290 */
devm_cxl_dpa_release(struct cxl_endpoint_decoder * cxled)291 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
292 {
293 struct cxl_port *port = cxled_to_port(cxled);
294
295 lockdep_assert_held_write(&cxl_dpa_rwsem);
296 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
297 __cxl_dpa_release(cxled);
298 }
299
300 /**
301 * request_skip() - Track DPA 'skip' in @cxlds->dpa_res resource tree
302 * @cxlds: CXL.mem device context that parents @cxled
303 * @cxled: Endpoint decoder establishing new allocation that skips lower DPA
304 * @skip_base: DPA < start of new DPA allocation (DPAnew)
305 * @skip_len: @skip_base + @skip_len == DPAnew
306 *
307 * DPA 'skip' arises from out-of-sequence DPA allocation events relative
308 * to free capacity across multiple partitions. It is a wasteful event
309 * as usable DPA gets thrown away, but if a deployment has, for example,
310 * a dual RAM+PMEM device, wants to use PMEM, and has unallocated RAM
311 * DPA, the free RAM DPA must be sacrificed to start allocating PMEM.
312 * See third "Implementation Note" in CXL 3.1 8.2.4.19.13 "Decoder
313 * Protection" for more details.
314 *
315 * A 'skip' always covers the last allocated DPA in a previous partition
316 * to the start of the current partition to allocate. Allocations never
317 * start in the middle of a partition, and allocations are always
318 * de-allocated in reverse order (see cxl_dpa_free(), or natural devm
319 * unwind order from forced in-order allocation).
320 *
321 * If @cxlds->nr_partitions was guaranteed to be <= 2 then the 'skip'
322 * would always be contained to a single partition. Given
323 * @cxlds->nr_partitions may be > 2 it results in cases where the 'skip'
324 * might span "tail capacity of partition[0], all of partition[1], ...,
325 * all of partition[N-1]" to support allocating from partition[N]. That
326 * in turn interacts with the partition 'struct resource' boundaries
327 * within @cxlds->dpa_res whereby 'skip' requests need to be divided by
328 * partition. I.e. this is a quirk of using a 'struct resource' tree to
329 * detect range conflicts while also tracking partition boundaries in
330 * @cxlds->dpa_res.
331 */
request_skip(struct cxl_dev_state * cxlds,struct cxl_endpoint_decoder * cxled,const resource_size_t skip_base,const resource_size_t skip_len)332 static int request_skip(struct cxl_dev_state *cxlds,
333 struct cxl_endpoint_decoder *cxled,
334 const resource_size_t skip_base,
335 const resource_size_t skip_len)
336 {
337 resource_size_t skipped = __adjust_skip(cxlds, skip_base, skip_len,
338 dev_name(&cxled->cxld.dev));
339
340 if (skipped == skip_len)
341 return 0;
342
343 dev_dbg(cxlds->dev,
344 "%s: failed to reserve skipped space (%pa %pa %pa)\n",
345 dev_name(&cxled->cxld.dev), &skip_base, &skip_len, &skipped);
346
347 release_skip(cxlds, skip_base, skipped);
348
349 return -EBUSY;
350 }
351
__cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)352 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
353 resource_size_t base, resource_size_t len,
354 resource_size_t skipped)
355 {
356 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
357 struct cxl_port *port = cxled_to_port(cxled);
358 struct cxl_dev_state *cxlds = cxlmd->cxlds;
359 struct device *dev = &port->dev;
360 struct resource *res;
361 int rc;
362
363 lockdep_assert_held_write(&cxl_dpa_rwsem);
364
365 if (!len) {
366 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
367 port->id, cxled->cxld.id);
368 return -EINVAL;
369 }
370
371 if (cxled->dpa_res) {
372 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
373 port->id, cxled->cxld.id, cxled->dpa_res);
374 return -EBUSY;
375 }
376
377 if (port->hdm_end + 1 != cxled->cxld.id) {
378 /*
379 * Assumes alloc and commit order is always in hardware instance
380 * order per expectations from 8.2.5.12.20 Committing Decoder
381 * Programming that enforce decoder[m] committed before
382 * decoder[m+1] commit start.
383 */
384 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
385 cxled->cxld.id, port->id, port->hdm_end + 1);
386 return -EBUSY;
387 }
388
389 if (skipped) {
390 rc = request_skip(cxlds, cxled, base - skipped, skipped);
391 if (rc)
392 return rc;
393 }
394 res = __request_region(&cxlds->dpa_res, base, len,
395 dev_name(&cxled->cxld.dev), 0);
396 if (!res) {
397 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
398 port->id, cxled->cxld.id);
399 if (skipped)
400 release_skip(cxlds, base - skipped, skipped);
401 return -EBUSY;
402 }
403 cxled->dpa_res = res;
404 cxled->skip = skipped;
405
406 /*
407 * When allocating new capacity, ->part is already set, when
408 * discovering decoder settings at initial enumeration, ->part
409 * is not set.
410 */
411 if (cxled->part < 0)
412 for (int i = 0; cxlds->nr_partitions; i++)
413 if (resource_contains(&cxlds->part[i].res, res)) {
414 cxled->part = i;
415 break;
416 }
417
418 if (cxled->part < 0)
419 dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n",
420 port->id, cxled->cxld.id, res);
421
422 port->hdm_end++;
423 get_device(&cxled->cxld.dev);
424 return 0;
425 }
426
add_dpa_res(struct device * dev,struct resource * parent,struct resource * res,resource_size_t start,resource_size_t size,const char * type)427 static int add_dpa_res(struct device *dev, struct resource *parent,
428 struct resource *res, resource_size_t start,
429 resource_size_t size, const char *type)
430 {
431 int rc;
432
433 *res = (struct resource) {
434 .name = type,
435 .start = start,
436 .end = start + size - 1,
437 .flags = IORESOURCE_MEM,
438 };
439 if (resource_size(res) == 0) {
440 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
441 return 0;
442 }
443 rc = request_resource(parent, res);
444 if (rc) {
445 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
446 res, rc);
447 return rc;
448 }
449
450 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
451
452 return 0;
453 }
454
cxl_mode_name(enum cxl_partition_mode mode)455 static const char *cxl_mode_name(enum cxl_partition_mode mode)
456 {
457 switch (mode) {
458 case CXL_PARTMODE_RAM:
459 return "ram";
460 case CXL_PARTMODE_PMEM:
461 return "pmem";
462 default:
463 return "";
464 };
465 }
466
467 /* if this fails the caller must destroy @cxlds, there is no recovery */
cxl_dpa_setup(struct cxl_dev_state * cxlds,const struct cxl_dpa_info * info)468 int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
469 {
470 struct device *dev = cxlds->dev;
471
472 guard(rwsem_write)(&cxl_dpa_rwsem);
473
474 if (cxlds->nr_partitions)
475 return -EBUSY;
476
477 if (!info->size || !info->nr_partitions) {
478 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
479 cxlds->nr_partitions = 0;
480 return 0;
481 }
482
483 cxlds->dpa_res = DEFINE_RES_MEM(0, info->size);
484
485 for (int i = 0; i < info->nr_partitions; i++) {
486 const struct cxl_dpa_part_info *part = &info->part[i];
487 int rc;
488
489 cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID;
490 cxlds->part[i].mode = part->mode;
491
492 /* Require ordered + contiguous partitions */
493 if (i) {
494 const struct cxl_dpa_part_info *prev = &info->part[i - 1];
495
496 if (prev->range.end + 1 != part->range.start)
497 return -EINVAL;
498 }
499 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res,
500 part->range.start, range_len(&part->range),
501 cxl_mode_name(part->mode));
502 if (rc)
503 return rc;
504 cxlds->nr_partitions++;
505 }
506
507 return 0;
508 }
509 EXPORT_SYMBOL_GPL(cxl_dpa_setup);
510
devm_cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)511 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
512 resource_size_t base, resource_size_t len,
513 resource_size_t skipped)
514 {
515 struct cxl_port *port = cxled_to_port(cxled);
516 int rc;
517
518 down_write(&cxl_dpa_rwsem);
519 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
520 up_write(&cxl_dpa_rwsem);
521
522 if (rc)
523 return rc;
524
525 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
526 }
527 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL");
528
cxl_dpa_size(struct cxl_endpoint_decoder * cxled)529 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
530 {
531 guard(rwsem_read)(&cxl_dpa_rwsem);
532 if (cxled->dpa_res)
533 return resource_size(cxled->dpa_res);
534
535 return 0;
536 }
537
cxl_dpa_resource_start(struct cxl_endpoint_decoder * cxled)538 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
539 {
540 resource_size_t base = -1;
541
542 lockdep_assert_held(&cxl_dpa_rwsem);
543 if (cxled->dpa_res)
544 base = cxled->dpa_res->start;
545
546 return base;
547 }
548
cxl_dpa_free(struct cxl_endpoint_decoder * cxled)549 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
550 {
551 struct cxl_port *port = cxled_to_port(cxled);
552 struct device *dev = &cxled->cxld.dev;
553
554 guard(rwsem_write)(&cxl_dpa_rwsem);
555 if (!cxled->dpa_res)
556 return 0;
557 if (cxled->cxld.region) {
558 dev_dbg(dev, "decoder assigned to: %s\n",
559 dev_name(&cxled->cxld.region->dev));
560 return -EBUSY;
561 }
562 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
563 dev_dbg(dev, "decoder enabled\n");
564 return -EBUSY;
565 }
566 if (cxled->cxld.id != port->hdm_end) {
567 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
568 port->hdm_end);
569 return -EBUSY;
570 }
571
572 devm_cxl_dpa_release(cxled);
573 return 0;
574 }
575
cxl_dpa_set_part(struct cxl_endpoint_decoder * cxled,enum cxl_partition_mode mode)576 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
577 enum cxl_partition_mode mode)
578 {
579 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
580 struct cxl_dev_state *cxlds = cxlmd->cxlds;
581 struct device *dev = &cxled->cxld.dev;
582 int part;
583
584 guard(rwsem_write)(&cxl_dpa_rwsem);
585 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)
586 return -EBUSY;
587
588 for (part = 0; part < cxlds->nr_partitions; part++)
589 if (cxlds->part[part].mode == mode)
590 break;
591
592 if (part >= cxlds->nr_partitions) {
593 dev_dbg(dev, "unsupported mode: %d\n", mode);
594 return -EINVAL;
595 }
596
597 if (!resource_size(&cxlds->part[part].res)) {
598 dev_dbg(dev, "no available capacity for mode: %d\n", mode);
599 return -ENXIO;
600 }
601
602 cxled->part = part;
603 return 0;
604 }
605
__cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,unsigned long long size)606 static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
607 {
608 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
609 struct cxl_dev_state *cxlds = cxlmd->cxlds;
610 struct device *dev = &cxled->cxld.dev;
611 struct resource *res, *prev = NULL;
612 resource_size_t start, avail, skip, skip_start;
613 struct resource *p, *last;
614 int part;
615
616 guard(rwsem_write)(&cxl_dpa_rwsem);
617 if (cxled->cxld.region) {
618 dev_dbg(dev, "decoder attached to %s\n",
619 dev_name(&cxled->cxld.region->dev));
620 return -EBUSY;
621 }
622
623 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
624 dev_dbg(dev, "decoder enabled\n");
625 return -EBUSY;
626 }
627
628 part = cxled->part;
629 if (part < 0) {
630 dev_dbg(dev, "partition not set\n");
631 return -EBUSY;
632 }
633
634 res = &cxlds->part[part].res;
635 for (p = res->child, last = NULL; p; p = p->sibling)
636 last = p;
637 if (last)
638 start = last->end + 1;
639 else
640 start = res->start;
641
642 /*
643 * To allocate at partition N, a skip needs to be calculated for all
644 * unallocated space at lower partitions indices.
645 *
646 * If a partition has any allocations, the search can end because a
647 * previous cxl_dpa_alloc() invocation is assumed to have accounted for
648 * all previous partitions.
649 */
650 skip_start = CXL_RESOURCE_NONE;
651 for (int i = part; i; i--) {
652 prev = &cxlds->part[i - 1].res;
653 for (p = prev->child, last = NULL; p; p = p->sibling)
654 last = p;
655 if (last) {
656 skip_start = last->end + 1;
657 break;
658 }
659 skip_start = prev->start;
660 }
661
662 avail = res->end - start + 1;
663 if (skip_start == CXL_RESOURCE_NONE)
664 skip = 0;
665 else
666 skip = res->start - skip_start;
667
668 if (size > avail) {
669 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
670 res->name, &avail);
671 return -ENOSPC;
672 }
673
674 return __cxl_dpa_reserve(cxled, start, size, skip);
675 }
676
cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,unsigned long long size)677 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
678 {
679 struct cxl_port *port = cxled_to_port(cxled);
680 int rc;
681
682 rc = __cxl_dpa_alloc(cxled, size);
683 if (rc)
684 return rc;
685
686 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
687 }
688
cxld_set_interleave(struct cxl_decoder * cxld,u32 * ctrl)689 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
690 {
691 u16 eig;
692 u8 eiw;
693
694 /*
695 * Input validation ensures these warns never fire, but otherwise
696 * suppress unititalized variable usage warnings.
697 */
698 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
699 "invalid interleave_ways: %d\n", cxld->interleave_ways))
700 return;
701 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
702 "invalid interleave_granularity: %d\n",
703 cxld->interleave_granularity))
704 return;
705
706 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
707 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
708 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
709 }
710
cxld_set_type(struct cxl_decoder * cxld,u32 * ctrl)711 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
712 {
713 u32p_replace_bits(ctrl,
714 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
715 CXL_HDM_DECODER0_CTRL_HOSTONLY);
716 }
717
cxlsd_set_targets(struct cxl_switch_decoder * cxlsd,u64 * tgt)718 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
719 {
720 struct cxl_dport **t = &cxlsd->target[0];
721 int ways = cxlsd->cxld.interleave_ways;
722
723 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
724 if (ways > 1)
725 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
726 if (ways > 2)
727 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
728 if (ways > 3)
729 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
730 if (ways > 4)
731 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
732 if (ways > 5)
733 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
734 if (ways > 6)
735 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
736 if (ways > 7)
737 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
738 }
739
740 /*
741 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
742 * committed or error within 10ms, but just be generous with 20ms to account for
743 * clock skew and other marginal behavior
744 */
745 #define COMMIT_TIMEOUT_MS 20
cxld_await_commit(void __iomem * hdm,int id)746 static int cxld_await_commit(void __iomem *hdm, int id)
747 {
748 u32 ctrl;
749 int i;
750
751 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
752 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
753 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
754 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
755 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
756 return -EIO;
757 }
758 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
759 return 0;
760 fsleep(1000);
761 }
762
763 return -ETIMEDOUT;
764 }
765
cxl_decoder_commit(struct cxl_decoder * cxld)766 static int cxl_decoder_commit(struct cxl_decoder *cxld)
767 {
768 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
769 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
770 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
771 int id = cxld->id, rc;
772 u64 base, size;
773 u32 ctrl;
774
775 if (cxld->flags & CXL_DECODER_F_ENABLE)
776 return 0;
777
778 if (cxl_num_decoders_committed(port) != id) {
779 dev_dbg(&port->dev,
780 "%s: out of order commit, expected decoder%d.%d\n",
781 dev_name(&cxld->dev), port->id,
782 cxl_num_decoders_committed(port));
783 return -EBUSY;
784 }
785
786 /*
787 * For endpoint decoders hosted on CXL memory devices that
788 * support the sanitize operation, make sure sanitize is not in-flight.
789 */
790 if (is_endpoint_decoder(&cxld->dev)) {
791 struct cxl_endpoint_decoder *cxled =
792 to_cxl_endpoint_decoder(&cxld->dev);
793 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
794 struct cxl_memdev_state *mds =
795 to_cxl_memdev_state(cxlmd->cxlds);
796
797 if (mds && mds->security.sanitize_active) {
798 dev_dbg(&cxlmd->dev,
799 "attempted to commit %s during sanitize\n",
800 dev_name(&cxld->dev));
801 return -EBUSY;
802 }
803 }
804
805 down_read(&cxl_dpa_rwsem);
806 /* common decoder settings */
807 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
808 cxld_set_interleave(cxld, &ctrl);
809 cxld_set_type(cxld, &ctrl);
810 base = cxld->hpa_range.start;
811 size = range_len(&cxld->hpa_range);
812
813 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
814 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
815 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
816 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
817
818 if (is_switch_decoder(&cxld->dev)) {
819 struct cxl_switch_decoder *cxlsd =
820 to_cxl_switch_decoder(&cxld->dev);
821 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
822 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
823 u64 targets;
824
825 cxlsd_set_targets(cxlsd, &targets);
826 writel(upper_32_bits(targets), tl_hi);
827 writel(lower_32_bits(targets), tl_lo);
828 } else {
829 struct cxl_endpoint_decoder *cxled =
830 to_cxl_endpoint_decoder(&cxld->dev);
831 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
832 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
833
834 writel(upper_32_bits(cxled->skip), sk_hi);
835 writel(lower_32_bits(cxled->skip), sk_lo);
836 }
837
838 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
839 up_read(&cxl_dpa_rwsem);
840
841 port->commit_end++;
842 rc = cxld_await_commit(hdm, cxld->id);
843 if (rc) {
844 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
845 dev_name(&cxld->dev), rc);
846 cxld->reset(cxld);
847 return rc;
848 }
849 cxld->flags |= CXL_DECODER_F_ENABLE;
850
851 return 0;
852 }
853
commit_reap(struct device * dev,void * data)854 static int commit_reap(struct device *dev, void *data)
855 {
856 struct cxl_port *port = to_cxl_port(dev->parent);
857 struct cxl_decoder *cxld;
858
859 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev))
860 return 0;
861
862 cxld = to_cxl_decoder(dev);
863 if (port->commit_end == cxld->id &&
864 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
865 port->commit_end--;
866 dev_dbg(&port->dev, "reap: %s commit_end: %d\n",
867 dev_name(&cxld->dev), port->commit_end);
868 }
869
870 return 0;
871 }
872
cxl_port_commit_reap(struct cxl_decoder * cxld)873 void cxl_port_commit_reap(struct cxl_decoder *cxld)
874 {
875 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
876
877 lockdep_assert_held_write(&cxl_region_rwsem);
878
879 /*
880 * Once the highest committed decoder is disabled, free any other
881 * decoders that were pinned allocated by out-of-order release.
882 */
883 port->commit_end--;
884 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev),
885 port->commit_end);
886 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL,
887 commit_reap);
888 }
889 EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL");
890
cxl_decoder_reset(struct cxl_decoder * cxld)891 static void cxl_decoder_reset(struct cxl_decoder *cxld)
892 {
893 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
894 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
895 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
896 int id = cxld->id;
897 u32 ctrl;
898
899 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
900 return;
901
902 if (port->commit_end == id)
903 cxl_port_commit_reap(cxld);
904 else
905 dev_dbg(&port->dev,
906 "%s: out of order reset, expected decoder%d.%d\n",
907 dev_name(&cxld->dev), port->id, port->commit_end);
908
909 down_read(&cxl_dpa_rwsem);
910 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
911 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
912 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
913
914 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
915 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
916 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
917 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
918 up_read(&cxl_dpa_rwsem);
919
920 cxld->flags &= ~CXL_DECODER_F_ENABLE;
921
922 /* Userspace is now responsible for reconfiguring this decoder */
923 if (is_endpoint_decoder(&cxld->dev)) {
924 struct cxl_endpoint_decoder *cxled;
925
926 cxled = to_cxl_endpoint_decoder(&cxld->dev);
927 cxled->state = CXL_DECODER_STATE_MANUAL;
928 }
929 }
930
cxl_setup_hdm_decoder_from_dvsec(struct cxl_port * port,struct cxl_decoder * cxld,u64 * dpa_base,int which,struct cxl_endpoint_dvsec_info * info)931 static int cxl_setup_hdm_decoder_from_dvsec(
932 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
933 int which, struct cxl_endpoint_dvsec_info *info)
934 {
935 struct cxl_endpoint_decoder *cxled;
936 u64 len;
937 int rc;
938
939 if (!is_cxl_endpoint(port))
940 return -EOPNOTSUPP;
941
942 cxled = to_cxl_endpoint_decoder(&cxld->dev);
943 len = range_len(&info->dvsec_range[which]);
944 if (!len)
945 return -ENOENT;
946
947 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
948 cxld->commit = NULL;
949 cxld->reset = NULL;
950 cxld->hpa_range = info->dvsec_range[which];
951
952 /*
953 * Set the emulated decoder as locked pending additional support to
954 * change the range registers at run time.
955 */
956 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
957 port->commit_end = cxld->id;
958
959 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
960 if (rc) {
961 dev_err(&port->dev,
962 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
963 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
964 return rc;
965 }
966 *dpa_base += len;
967 cxled->state = CXL_DECODER_STATE_AUTO;
968
969 return 0;
970 }
971
init_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map,void __iomem * hdm,int which,u64 * dpa_base,struct cxl_endpoint_dvsec_info * info)972 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
973 int *target_map, void __iomem *hdm, int which,
974 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
975 {
976 struct cxl_endpoint_decoder *cxled = NULL;
977 u64 size, base, skip, dpa_size, lo, hi;
978 bool committed;
979 u32 remainder;
980 int i, rc;
981 u32 ctrl;
982 union {
983 u64 value;
984 unsigned char target_id[8];
985 } target_list;
986
987 if (should_emulate_decoders(info))
988 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
989 which, info);
990
991 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
992 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
993 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
994 base = (hi << 32) + lo;
995 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
996 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
997 size = (hi << 32) + lo;
998 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
999 cxld->commit = cxl_decoder_commit;
1000 cxld->reset = cxl_decoder_reset;
1001
1002 if (!committed)
1003 size = 0;
1004 if (base == U64_MAX || size == U64_MAX) {
1005 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
1006 port->id, cxld->id);
1007 return -ENXIO;
1008 }
1009
1010 if (info)
1011 cxled = to_cxl_endpoint_decoder(&cxld->dev);
1012 cxld->hpa_range = (struct range) {
1013 .start = base,
1014 .end = base + size - 1,
1015 };
1016
1017 /* decoders are enabled if committed */
1018 if (committed) {
1019 cxld->flags |= CXL_DECODER_F_ENABLE;
1020 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
1021 cxld->flags |= CXL_DECODER_F_LOCK;
1022 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
1023 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1024 else
1025 cxld->target_type = CXL_DECODER_DEVMEM;
1026
1027 guard(rwsem_write)(&cxl_region_rwsem);
1028 if (cxld->id != cxl_num_decoders_committed(port)) {
1029 dev_warn(&port->dev,
1030 "decoder%d.%d: Committed out of order\n",
1031 port->id, cxld->id);
1032 return -ENXIO;
1033 }
1034
1035 if (size == 0) {
1036 dev_warn(&port->dev,
1037 "decoder%d.%d: Committed with zero size\n",
1038 port->id, cxld->id);
1039 return -ENXIO;
1040 }
1041 port->commit_end = cxld->id;
1042 } else {
1043 if (cxled) {
1044 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1045 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1046
1047 /*
1048 * Default by devtype until a device arrives that needs
1049 * more precision.
1050 */
1051 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
1052 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1053 else
1054 cxld->target_type = CXL_DECODER_DEVMEM;
1055 } else {
1056 /* To be overridden by region type at commit time */
1057 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1058 }
1059
1060 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
1061 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
1062 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
1063 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
1064 }
1065 }
1066 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
1067 &cxld->interleave_ways);
1068 if (rc) {
1069 dev_warn(&port->dev,
1070 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
1071 port->id, cxld->id, ctrl);
1072 return rc;
1073 }
1074 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
1075 &cxld->interleave_granularity);
1076 if (rc) {
1077 dev_warn(&port->dev,
1078 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",
1079 port->id, cxld->id, ctrl);
1080 return rc;
1081 }
1082
1083 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
1084 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
1085 cxld->interleave_ways, cxld->interleave_granularity);
1086
1087 if (!cxled) {
1088 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
1089 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
1090 target_list.value = (hi << 32) + lo;
1091 for (i = 0; i < cxld->interleave_ways; i++)
1092 target_map[i] = target_list.target_id[i];
1093
1094 return 0;
1095 }
1096
1097 if (!committed)
1098 return 0;
1099
1100 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
1101 if (remainder) {
1102 dev_err(&port->dev,
1103 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
1104 port->id, cxld->id, size, cxld->interleave_ways);
1105 return -ENXIO;
1106 }
1107 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
1108 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
1109 skip = (hi << 32) + lo;
1110 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
1111 if (rc) {
1112 dev_err(&port->dev,
1113 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
1114 port->id, cxld->id, *dpa_base,
1115 *dpa_base + dpa_size + skip - 1, rc);
1116 return rc;
1117 }
1118 *dpa_base += dpa_size + skip;
1119
1120 cxled->state = CXL_DECODER_STATE_AUTO;
1121
1122 return 0;
1123 }
1124
cxl_settle_decoders(struct cxl_hdm * cxlhdm)1125 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
1126 {
1127 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1128 int committed, i;
1129 u32 ctrl;
1130
1131 if (!hdm)
1132 return;
1133
1134 /*
1135 * Since the register resource was recently claimed via request_region()
1136 * be careful about trusting the "not-committed" status until the commit
1137 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
1138 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
1139 * host and target.
1140 */
1141 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
1142 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
1143 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
1144 committed++;
1145 }
1146
1147 /* ensure that future checks of committed can be trusted */
1148 if (committed != cxlhdm->decoder_count)
1149 msleep(20);
1150 }
1151
1152 /**
1153 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
1154 * @cxlhdm: Structure to populate with HDM capabilities
1155 * @info: cached DVSEC range register info
1156 */
devm_cxl_enumerate_decoders(struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)1157 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
1158 struct cxl_endpoint_dvsec_info *info)
1159 {
1160 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1161 struct cxl_port *port = cxlhdm->port;
1162 int i;
1163 u64 dpa_base = 0;
1164
1165 cxl_settle_decoders(cxlhdm);
1166
1167 for (i = 0; i < cxlhdm->decoder_count; i++) {
1168 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
1169 int rc, target_count = cxlhdm->target_count;
1170 struct cxl_decoder *cxld;
1171
1172 if (is_cxl_endpoint(port)) {
1173 struct cxl_endpoint_decoder *cxled;
1174
1175 cxled = cxl_endpoint_decoder_alloc(port);
1176 if (IS_ERR(cxled)) {
1177 dev_warn(&port->dev,
1178 "Failed to allocate decoder%d.%d\n",
1179 port->id, i);
1180 return PTR_ERR(cxled);
1181 }
1182 cxld = &cxled->cxld;
1183 } else {
1184 struct cxl_switch_decoder *cxlsd;
1185
1186 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1187 if (IS_ERR(cxlsd)) {
1188 dev_warn(&port->dev,
1189 "Failed to allocate decoder%d.%d\n",
1190 port->id, i);
1191 return PTR_ERR(cxlsd);
1192 }
1193 cxld = &cxlsd->cxld;
1194 }
1195
1196 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1197 &dpa_base, info);
1198 if (rc) {
1199 dev_warn(&port->dev,
1200 "Failed to initialize decoder%d.%d\n",
1201 port->id, i);
1202 put_device(&cxld->dev);
1203 return rc;
1204 }
1205 rc = add_hdm_decoder(port, cxld, target_map);
1206 if (rc) {
1207 dev_warn(&port->dev,
1208 "Failed to add decoder%d.%d\n", port->id, i);
1209 return rc;
1210 }
1211 }
1212
1213 return 0;
1214 }
1215 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, "CXL");
1216