1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/seq_file.h>
4 #include <linux/device.h>
5 #include <linux/delay.h>
6
7 #include "cxlmem.h"
8 #include "core.h"
9
10 /**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19 DECLARE_RWSEM(cxl_dpa_rwsem);
20
add_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map)21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
22 int *target_map)
23 {
24 int rc;
25
26 rc = cxl_decoder_add_locked(cxld, target_map);
27 if (rc) {
28 put_device(&cxld->dev);
29 dev_err(&port->dev, "Failed to add decoder\n");
30 return rc;
31 }
32
33 rc = cxl_decoder_autoremove(&port->dev, cxld);
34 if (rc)
35 return rc;
36
37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
38
39 return 0;
40 }
41
42 /*
43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
44 * single ported host-bridges need not publish a decoder capability when a
45 * passthrough decode can be assumed, i.e. all transactions that the uport sees
46 * are claimed and passed to the single dport. Disable the range until the first
47 * CXL region is enumerated / activated.
48 */
devm_cxl_add_passthrough_decoder(struct cxl_port * port)49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
50 {
51 struct cxl_switch_decoder *cxlsd;
52 struct cxl_dport *dport = NULL;
53 int single_port_map[1];
54 unsigned long index;
55 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
56
57 /*
58 * Capability checks are moot for passthrough decoders, support
59 * any and all possibilities.
60 */
61 cxlhdm->interleave_mask = ~0U;
62 cxlhdm->iw_cap_mask = ~0UL;
63
64 cxlsd = cxl_switch_decoder_alloc(port, 1);
65 if (IS_ERR(cxlsd))
66 return PTR_ERR(cxlsd);
67
68 device_lock_assert(&port->dev);
69
70 xa_for_each(&port->dports, index, dport)
71 break;
72 single_port_map[0] = dport->port_id;
73
74 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
75 }
76 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
77
parse_hdm_decoder_caps(struct cxl_hdm * cxlhdm)78 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
79 {
80 u32 hdm_cap;
81
82 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
83 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
84 cxlhdm->target_count =
85 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
86 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
87 cxlhdm->interleave_mask |= GENMASK(11, 8);
88 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
89 cxlhdm->interleave_mask |= GENMASK(14, 12);
90 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);
91 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))
92 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);
93 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))
94 cxlhdm->iw_cap_mask |= BIT(16);
95 }
96
should_emulate_decoders(struct cxl_endpoint_dvsec_info * info)97 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
98 {
99 struct cxl_hdm *cxlhdm;
100 void __iomem *hdm;
101 u32 ctrl;
102 int i;
103
104 if (!info)
105 return false;
106
107 cxlhdm = dev_get_drvdata(&info->port->dev);
108 hdm = cxlhdm->regs.hdm_decoder;
109
110 if (!hdm)
111 return true;
112
113 /*
114 * If HDM decoders are present and the driver is in control of
115 * Mem_Enable skip DVSEC based emulation
116 */
117 if (!info->mem_enabled)
118 return false;
119
120 /*
121 * If any decoders are committed already, there should not be any
122 * emulated DVSEC decoders.
123 */
124 for (i = 0; i < cxlhdm->decoder_count; i++) {
125 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
126 dev_dbg(&info->port->dev,
127 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
128 info->port->id, i,
129 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
130 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
131 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
132 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
133 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
134 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
135 return false;
136 }
137
138 return true;
139 }
140
141 /**
142 * devm_cxl_setup_hdm - map HDM decoder component registers
143 * @port: cxl_port to map
144 * @info: cached DVSEC range register info
145 */
devm_cxl_setup_hdm(struct cxl_port * port,struct cxl_endpoint_dvsec_info * info)146 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
147 struct cxl_endpoint_dvsec_info *info)
148 {
149 struct cxl_register_map *reg_map = &port->reg_map;
150 struct device *dev = &port->dev;
151 struct cxl_hdm *cxlhdm;
152 int rc;
153
154 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
155 if (!cxlhdm)
156 return ERR_PTR(-ENOMEM);
157 cxlhdm->port = port;
158 dev_set_drvdata(dev, cxlhdm);
159
160 /* Memory devices can configure device HDM using DVSEC range regs. */
161 if (reg_map->resource == CXL_RESOURCE_NONE) {
162 if (!info || !info->mem_enabled) {
163 dev_err(dev, "No component registers mapped\n");
164 return ERR_PTR(-ENXIO);
165 }
166
167 cxlhdm->decoder_count = info->ranges;
168 return cxlhdm;
169 }
170
171 if (!reg_map->component_map.hdm_decoder.valid) {
172 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
173 /* unique error code to indicate no HDM decoder capability */
174 return ERR_PTR(-ENODEV);
175 }
176
177 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,
178 BIT(CXL_CM_CAP_CAP_ID_HDM));
179 if (rc) {
180 dev_err(dev, "Failed to map HDM capability.\n");
181 return ERR_PTR(rc);
182 }
183
184 parse_hdm_decoder_caps(cxlhdm);
185 if (cxlhdm->decoder_count == 0) {
186 dev_err(dev, "Spec violation. Caps invalid\n");
187 return ERR_PTR(-ENXIO);
188 }
189
190 /*
191 * Now that the hdm capability is parsed, decide if range
192 * register emulation is needed and fixup cxlhdm accordingly.
193 */
194 if (should_emulate_decoders(info)) {
195 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
196 info->ranges > 1 ? "s" : "");
197 cxlhdm->decoder_count = info->ranges;
198 }
199
200 return cxlhdm;
201 }
202 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
203
__cxl_dpa_debug(struct seq_file * file,struct resource * r,int depth)204 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
205 {
206 unsigned long long start = r->start, end = r->end;
207
208 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
209 r->name);
210 }
211
cxl_dpa_debug(struct seq_file * file,struct cxl_dev_state * cxlds)212 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
213 {
214 struct resource *p1, *p2;
215
216 down_read(&cxl_dpa_rwsem);
217 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
218 __cxl_dpa_debug(file, p1, 0);
219 for (p2 = p1->child; p2; p2 = p2->sibling)
220 __cxl_dpa_debug(file, p2, 1);
221 }
222 up_read(&cxl_dpa_rwsem);
223 }
224 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
225
226 /*
227 * Must be called in a context that synchronizes against this decoder's
228 * port ->remove() callback (like an endpoint decoder sysfs attribute)
229 */
__cxl_dpa_release(struct cxl_endpoint_decoder * cxled)230 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
231 {
232 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
233 struct cxl_port *port = cxled_to_port(cxled);
234 struct cxl_dev_state *cxlds = cxlmd->cxlds;
235 struct resource *res = cxled->dpa_res;
236 resource_size_t skip_start;
237
238 lockdep_assert_held_write(&cxl_dpa_rwsem);
239
240 /* save @skip_start, before @res is released */
241 skip_start = res->start - cxled->skip;
242 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
243 if (cxled->skip)
244 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
245 cxled->skip = 0;
246 cxled->dpa_res = NULL;
247 put_device(&cxled->cxld.dev);
248 port->hdm_end--;
249 }
250
cxl_dpa_release(void * cxled)251 static void cxl_dpa_release(void *cxled)
252 {
253 down_write(&cxl_dpa_rwsem);
254 __cxl_dpa_release(cxled);
255 up_write(&cxl_dpa_rwsem);
256 }
257
258 /*
259 * Must be called from context that will not race port device
260 * unregistration, like decoder sysfs attribute methods
261 */
devm_cxl_dpa_release(struct cxl_endpoint_decoder * cxled)262 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
263 {
264 struct cxl_port *port = cxled_to_port(cxled);
265
266 lockdep_assert_held_write(&cxl_dpa_rwsem);
267 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
268 __cxl_dpa_release(cxled);
269 }
270
__cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)271 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
272 resource_size_t base, resource_size_t len,
273 resource_size_t skipped)
274 {
275 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
276 struct cxl_port *port = cxled_to_port(cxled);
277 struct cxl_dev_state *cxlds = cxlmd->cxlds;
278 struct device *dev = &port->dev;
279 struct resource *res;
280
281 lockdep_assert_held_write(&cxl_dpa_rwsem);
282
283 if (!len) {
284 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
285 port->id, cxled->cxld.id);
286 return -EINVAL;
287 }
288
289 if (cxled->dpa_res) {
290 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
291 port->id, cxled->cxld.id, cxled->dpa_res);
292 return -EBUSY;
293 }
294
295 if (port->hdm_end + 1 != cxled->cxld.id) {
296 /*
297 * Assumes alloc and commit order is always in hardware instance
298 * order per expectations from 8.2.5.12.20 Committing Decoder
299 * Programming that enforce decoder[m] committed before
300 * decoder[m+1] commit start.
301 */
302 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
303 cxled->cxld.id, port->id, port->hdm_end + 1);
304 return -EBUSY;
305 }
306
307 if (skipped) {
308 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
309 dev_name(&cxled->cxld.dev), 0);
310 if (!res) {
311 dev_dbg(dev,
312 "decoder%d.%d: failed to reserve skipped space\n",
313 port->id, cxled->cxld.id);
314 return -EBUSY;
315 }
316 }
317 res = __request_region(&cxlds->dpa_res, base, len,
318 dev_name(&cxled->cxld.dev), 0);
319 if (!res) {
320 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
321 port->id, cxled->cxld.id);
322 if (skipped)
323 __release_region(&cxlds->dpa_res, base - skipped,
324 skipped);
325 return -EBUSY;
326 }
327 cxled->dpa_res = res;
328 cxled->skip = skipped;
329
330 if (resource_contains(&cxlds->pmem_res, res))
331 cxled->mode = CXL_DECODER_PMEM;
332 else if (resource_contains(&cxlds->ram_res, res))
333 cxled->mode = CXL_DECODER_RAM;
334 else {
335 dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",
336 port->id, cxled->cxld.id, cxled->dpa_res);
337 cxled->mode = CXL_DECODER_MIXED;
338 }
339
340 port->hdm_end++;
341 get_device(&cxled->cxld.dev);
342 return 0;
343 }
344
devm_cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)345 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
346 resource_size_t base, resource_size_t len,
347 resource_size_t skipped)
348 {
349 struct cxl_port *port = cxled_to_port(cxled);
350 int rc;
351
352 down_write(&cxl_dpa_rwsem);
353 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
354 up_write(&cxl_dpa_rwsem);
355
356 if (rc)
357 return rc;
358
359 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
360 }
361 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
362
cxl_dpa_size(struct cxl_endpoint_decoder * cxled)363 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
364 {
365 resource_size_t size = 0;
366
367 down_read(&cxl_dpa_rwsem);
368 if (cxled->dpa_res)
369 size = resource_size(cxled->dpa_res);
370 up_read(&cxl_dpa_rwsem);
371
372 return size;
373 }
374
cxl_dpa_resource_start(struct cxl_endpoint_decoder * cxled)375 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
376 {
377 resource_size_t base = -1;
378
379 lockdep_assert_held(&cxl_dpa_rwsem);
380 if (cxled->dpa_res)
381 base = cxled->dpa_res->start;
382
383 return base;
384 }
385
cxl_dpa_free(struct cxl_endpoint_decoder * cxled)386 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
387 {
388 struct cxl_port *port = cxled_to_port(cxled);
389 struct device *dev = &cxled->cxld.dev;
390 int rc;
391
392 down_write(&cxl_dpa_rwsem);
393 if (!cxled->dpa_res) {
394 rc = 0;
395 goto out;
396 }
397 if (cxled->cxld.region) {
398 dev_dbg(dev, "decoder assigned to: %s\n",
399 dev_name(&cxled->cxld.region->dev));
400 rc = -EBUSY;
401 goto out;
402 }
403 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
404 dev_dbg(dev, "decoder enabled\n");
405 rc = -EBUSY;
406 goto out;
407 }
408 if (cxled->cxld.id != port->hdm_end) {
409 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
410 port->hdm_end);
411 rc = -EBUSY;
412 goto out;
413 }
414 devm_cxl_dpa_release(cxled);
415 rc = 0;
416 out:
417 up_write(&cxl_dpa_rwsem);
418 return rc;
419 }
420
cxl_dpa_set_mode(struct cxl_endpoint_decoder * cxled,enum cxl_decoder_mode mode)421 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
422 enum cxl_decoder_mode mode)
423 {
424 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
425 struct cxl_dev_state *cxlds = cxlmd->cxlds;
426 struct device *dev = &cxled->cxld.dev;
427 int rc;
428
429 switch (mode) {
430 case CXL_DECODER_RAM:
431 case CXL_DECODER_PMEM:
432 break;
433 default:
434 dev_dbg(dev, "unsupported mode: %d\n", mode);
435 return -EINVAL;
436 }
437
438 down_write(&cxl_dpa_rwsem);
439 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
440 rc = -EBUSY;
441 goto out;
442 }
443
444 /*
445 * Only allow modes that are supported by the current partition
446 * configuration
447 */
448 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
449 dev_dbg(dev, "no available pmem capacity\n");
450 rc = -ENXIO;
451 goto out;
452 }
453 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
454 dev_dbg(dev, "no available ram capacity\n");
455 rc = -ENXIO;
456 goto out;
457 }
458
459 cxled->mode = mode;
460 rc = 0;
461 out:
462 up_write(&cxl_dpa_rwsem);
463
464 return rc;
465 }
466
cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,unsigned long long size)467 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
468 {
469 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
470 resource_size_t free_ram_start, free_pmem_start;
471 struct cxl_port *port = cxled_to_port(cxled);
472 struct cxl_dev_state *cxlds = cxlmd->cxlds;
473 struct device *dev = &cxled->cxld.dev;
474 resource_size_t start, avail, skip;
475 struct resource *p, *last;
476 int rc;
477
478 down_write(&cxl_dpa_rwsem);
479 if (cxled->cxld.region) {
480 dev_dbg(dev, "decoder attached to %s\n",
481 dev_name(&cxled->cxld.region->dev));
482 rc = -EBUSY;
483 goto out;
484 }
485
486 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
487 dev_dbg(dev, "decoder enabled\n");
488 rc = -EBUSY;
489 goto out;
490 }
491
492 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
493 last = p;
494 if (last)
495 free_ram_start = last->end + 1;
496 else
497 free_ram_start = cxlds->ram_res.start;
498
499 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
500 last = p;
501 if (last)
502 free_pmem_start = last->end + 1;
503 else
504 free_pmem_start = cxlds->pmem_res.start;
505
506 if (cxled->mode == CXL_DECODER_RAM) {
507 start = free_ram_start;
508 avail = cxlds->ram_res.end - start + 1;
509 skip = 0;
510 } else if (cxled->mode == CXL_DECODER_PMEM) {
511 resource_size_t skip_start, skip_end;
512
513 start = free_pmem_start;
514 avail = cxlds->pmem_res.end - start + 1;
515 skip_start = free_ram_start;
516
517 /*
518 * If some pmem is already allocated, then that allocation
519 * already handled the skip.
520 */
521 if (cxlds->pmem_res.child &&
522 skip_start == cxlds->pmem_res.child->start)
523 skip_end = skip_start - 1;
524 else
525 skip_end = start - 1;
526 skip = skip_end - skip_start + 1;
527 } else {
528 dev_dbg(dev, "mode not set\n");
529 rc = -EINVAL;
530 goto out;
531 }
532
533 if (size > avail) {
534 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
535 cxl_decoder_mode_name(cxled->mode), &avail);
536 rc = -ENOSPC;
537 goto out;
538 }
539
540 rc = __cxl_dpa_reserve(cxled, start, size, skip);
541 out:
542 up_write(&cxl_dpa_rwsem);
543
544 if (rc)
545 return rc;
546
547 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
548 }
549
cxld_set_interleave(struct cxl_decoder * cxld,u32 * ctrl)550 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
551 {
552 u16 eig;
553 u8 eiw;
554
555 /*
556 * Input validation ensures these warns never fire, but otherwise
557 * suppress unititalized variable usage warnings.
558 */
559 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
560 "invalid interleave_ways: %d\n", cxld->interleave_ways))
561 return;
562 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
563 "invalid interleave_granularity: %d\n",
564 cxld->interleave_granularity))
565 return;
566
567 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
568 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
569 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
570 }
571
cxld_set_type(struct cxl_decoder * cxld,u32 * ctrl)572 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
573 {
574 u32p_replace_bits(ctrl,
575 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
576 CXL_HDM_DECODER0_CTRL_HOSTONLY);
577 }
578
cxlsd_set_targets(struct cxl_switch_decoder * cxlsd,u64 * tgt)579 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
580 {
581 struct cxl_dport **t = &cxlsd->target[0];
582 int ways = cxlsd->cxld.interleave_ways;
583
584 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
585 if (ways > 1)
586 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
587 if (ways > 2)
588 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
589 if (ways > 3)
590 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
591 if (ways > 4)
592 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
593 if (ways > 5)
594 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
595 if (ways > 6)
596 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
597 if (ways > 7)
598 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
599 }
600
601 /*
602 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
603 * committed or error within 10ms, but just be generous with 20ms to account for
604 * clock skew and other marginal behavior
605 */
606 #define COMMIT_TIMEOUT_MS 20
cxld_await_commit(void __iomem * hdm,int id)607 static int cxld_await_commit(void __iomem *hdm, int id)
608 {
609 u32 ctrl;
610 int i;
611
612 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
613 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
614 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
615 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
616 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
617 return -EIO;
618 }
619 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
620 return 0;
621 fsleep(1000);
622 }
623
624 return -ETIMEDOUT;
625 }
626
cxl_decoder_commit(struct cxl_decoder * cxld)627 static int cxl_decoder_commit(struct cxl_decoder *cxld)
628 {
629 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
630 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
631 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
632 int id = cxld->id, rc;
633 u64 base, size;
634 u32 ctrl;
635
636 if (cxld->flags & CXL_DECODER_F_ENABLE)
637 return 0;
638
639 if (cxl_num_decoders_committed(port) != id) {
640 dev_dbg(&port->dev,
641 "%s: out of order commit, expected decoder%d.%d\n",
642 dev_name(&cxld->dev), port->id,
643 cxl_num_decoders_committed(port));
644 return -EBUSY;
645 }
646
647 /*
648 * For endpoint decoders hosted on CXL memory devices that
649 * support the sanitize operation, make sure sanitize is not in-flight.
650 */
651 if (is_endpoint_decoder(&cxld->dev)) {
652 struct cxl_endpoint_decoder *cxled =
653 to_cxl_endpoint_decoder(&cxld->dev);
654 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
655 struct cxl_memdev_state *mds =
656 to_cxl_memdev_state(cxlmd->cxlds);
657
658 if (mds && mds->security.sanitize_active) {
659 dev_dbg(&cxlmd->dev,
660 "attempted to commit %s during sanitize\n",
661 dev_name(&cxld->dev));
662 return -EBUSY;
663 }
664 }
665
666 down_read(&cxl_dpa_rwsem);
667 /* common decoder settings */
668 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
669 cxld_set_interleave(cxld, &ctrl);
670 cxld_set_type(cxld, &ctrl);
671 base = cxld->hpa_range.start;
672 size = range_len(&cxld->hpa_range);
673
674 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
675 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
676 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
677 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
678
679 if (is_switch_decoder(&cxld->dev)) {
680 struct cxl_switch_decoder *cxlsd =
681 to_cxl_switch_decoder(&cxld->dev);
682 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
683 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
684 u64 targets;
685
686 cxlsd_set_targets(cxlsd, &targets);
687 writel(upper_32_bits(targets), tl_hi);
688 writel(lower_32_bits(targets), tl_lo);
689 } else {
690 struct cxl_endpoint_decoder *cxled =
691 to_cxl_endpoint_decoder(&cxld->dev);
692 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
693 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
694
695 writel(upper_32_bits(cxled->skip), sk_hi);
696 writel(lower_32_bits(cxled->skip), sk_lo);
697 }
698
699 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
700 up_read(&cxl_dpa_rwsem);
701
702 port->commit_end++;
703 rc = cxld_await_commit(hdm, cxld->id);
704 if (rc) {
705 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
706 dev_name(&cxld->dev), rc);
707 cxld->reset(cxld);
708 return rc;
709 }
710 cxld->flags |= CXL_DECODER_F_ENABLE;
711
712 return 0;
713 }
714
cxl_decoder_reset(struct cxl_decoder * cxld)715 static int cxl_decoder_reset(struct cxl_decoder *cxld)
716 {
717 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
718 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
719 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
720 int id = cxld->id;
721 u32 ctrl;
722
723 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
724 return 0;
725
726 if (port->commit_end != id) {
727 dev_dbg(&port->dev,
728 "%s: out of order reset, expected decoder%d.%d\n",
729 dev_name(&cxld->dev), port->id, port->commit_end);
730 return -EBUSY;
731 }
732
733 down_read(&cxl_dpa_rwsem);
734 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
735 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
736 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
737
738 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
739 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
740 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
741 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
742 up_read(&cxl_dpa_rwsem);
743
744 port->commit_end--;
745 cxld->flags &= ~CXL_DECODER_F_ENABLE;
746
747 /* Userspace is now responsible for reconfiguring this decoder */
748 if (is_endpoint_decoder(&cxld->dev)) {
749 struct cxl_endpoint_decoder *cxled;
750
751 cxled = to_cxl_endpoint_decoder(&cxld->dev);
752 cxled->state = CXL_DECODER_STATE_MANUAL;
753 }
754
755 return 0;
756 }
757
cxl_setup_hdm_decoder_from_dvsec(struct cxl_port * port,struct cxl_decoder * cxld,u64 * dpa_base,int which,struct cxl_endpoint_dvsec_info * info)758 static int cxl_setup_hdm_decoder_from_dvsec(
759 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
760 int which, struct cxl_endpoint_dvsec_info *info)
761 {
762 struct cxl_endpoint_decoder *cxled;
763 u64 len;
764 int rc;
765
766 if (!is_cxl_endpoint(port))
767 return -EOPNOTSUPP;
768
769 cxled = to_cxl_endpoint_decoder(&cxld->dev);
770 len = range_len(&info->dvsec_range[which]);
771 if (!len)
772 return -ENOENT;
773
774 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
775 cxld->commit = NULL;
776 cxld->reset = NULL;
777 cxld->hpa_range = info->dvsec_range[which];
778
779 /*
780 * Set the emulated decoder as locked pending additional support to
781 * change the range registers at run time.
782 */
783 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
784 port->commit_end = cxld->id;
785
786 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
787 if (rc) {
788 dev_err(&port->dev,
789 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
790 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
791 return rc;
792 }
793 *dpa_base += len;
794 cxled->state = CXL_DECODER_STATE_AUTO;
795
796 return 0;
797 }
798
init_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map,void __iomem * hdm,int which,u64 * dpa_base,struct cxl_endpoint_dvsec_info * info)799 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
800 int *target_map, void __iomem *hdm, int which,
801 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
802 {
803 struct cxl_endpoint_decoder *cxled = NULL;
804 u64 size, base, skip, dpa_size, lo, hi;
805 bool committed;
806 u32 remainder;
807 int i, rc;
808 u32 ctrl;
809 union {
810 u64 value;
811 unsigned char target_id[8];
812 } target_list;
813
814 if (should_emulate_decoders(info))
815 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
816 which, info);
817
818 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
819 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
820 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
821 base = (hi << 32) + lo;
822 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
823 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
824 size = (hi << 32) + lo;
825 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
826 cxld->commit = cxl_decoder_commit;
827 cxld->reset = cxl_decoder_reset;
828
829 if (!committed)
830 size = 0;
831 if (base == U64_MAX || size == U64_MAX) {
832 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
833 port->id, cxld->id);
834 return -ENXIO;
835 }
836
837 if (info)
838 cxled = to_cxl_endpoint_decoder(&cxld->dev);
839 cxld->hpa_range = (struct range) {
840 .start = base,
841 .end = base + size - 1,
842 };
843
844 /* decoders are enabled if committed */
845 if (committed) {
846 cxld->flags |= CXL_DECODER_F_ENABLE;
847 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
848 cxld->flags |= CXL_DECODER_F_LOCK;
849 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
850 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
851 else
852 cxld->target_type = CXL_DECODER_DEVMEM;
853
854 guard(rwsem_write)(&cxl_region_rwsem);
855 if (cxld->id != cxl_num_decoders_committed(port)) {
856 dev_warn(&port->dev,
857 "decoder%d.%d: Committed out of order\n",
858 port->id, cxld->id);
859 return -ENXIO;
860 }
861
862 if (size == 0) {
863 dev_warn(&port->dev,
864 "decoder%d.%d: Committed with zero size\n",
865 port->id, cxld->id);
866 return -ENXIO;
867 }
868 port->commit_end = cxld->id;
869 } else {
870 if (cxled) {
871 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
872 struct cxl_dev_state *cxlds = cxlmd->cxlds;
873
874 /*
875 * Default by devtype until a device arrives that needs
876 * more precision.
877 */
878 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
879 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
880 else
881 cxld->target_type = CXL_DECODER_DEVMEM;
882 } else {
883 /* To be overridden by region type at commit time */
884 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
885 }
886
887 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
888 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
889 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
890 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
891 }
892 }
893 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
894 &cxld->interleave_ways);
895 if (rc) {
896 dev_warn(&port->dev,
897 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
898 port->id, cxld->id, ctrl);
899 return rc;
900 }
901 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
902 &cxld->interleave_granularity);
903 if (rc) {
904 dev_warn(&port->dev,
905 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",
906 port->id, cxld->id, ctrl);
907 return rc;
908 }
909
910 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
911 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
912 cxld->interleave_ways, cxld->interleave_granularity);
913
914 if (!cxled) {
915 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
916 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
917 target_list.value = (hi << 32) + lo;
918 for (i = 0; i < cxld->interleave_ways; i++)
919 target_map[i] = target_list.target_id[i];
920
921 return 0;
922 }
923
924 if (!committed)
925 return 0;
926
927 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
928 if (remainder) {
929 dev_err(&port->dev,
930 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
931 port->id, cxld->id, size, cxld->interleave_ways);
932 return -ENXIO;
933 }
934 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
935 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
936 skip = (hi << 32) + lo;
937 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
938 if (rc) {
939 dev_err(&port->dev,
940 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
941 port->id, cxld->id, *dpa_base,
942 *dpa_base + dpa_size + skip - 1, rc);
943 return rc;
944 }
945 *dpa_base += dpa_size + skip;
946
947 cxled->state = CXL_DECODER_STATE_AUTO;
948
949 return 0;
950 }
951
cxl_settle_decoders(struct cxl_hdm * cxlhdm)952 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
953 {
954 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
955 int committed, i;
956 u32 ctrl;
957
958 if (!hdm)
959 return;
960
961 /*
962 * Since the register resource was recently claimed via request_region()
963 * be careful about trusting the "not-committed" status until the commit
964 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
965 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
966 * host and target.
967 */
968 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
969 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
970 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
971 committed++;
972 }
973
974 /* ensure that future checks of committed can be trusted */
975 if (committed != cxlhdm->decoder_count)
976 msleep(20);
977 }
978
979 /**
980 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
981 * @cxlhdm: Structure to populate with HDM capabilities
982 * @info: cached DVSEC range register info
983 */
devm_cxl_enumerate_decoders(struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)984 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
985 struct cxl_endpoint_dvsec_info *info)
986 {
987 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
988 struct cxl_port *port = cxlhdm->port;
989 int i;
990 u64 dpa_base = 0;
991
992 cxl_settle_decoders(cxlhdm);
993
994 for (i = 0; i < cxlhdm->decoder_count; i++) {
995 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
996 int rc, target_count = cxlhdm->target_count;
997 struct cxl_decoder *cxld;
998
999 if (is_cxl_endpoint(port)) {
1000 struct cxl_endpoint_decoder *cxled;
1001
1002 cxled = cxl_endpoint_decoder_alloc(port);
1003 if (IS_ERR(cxled)) {
1004 dev_warn(&port->dev,
1005 "Failed to allocate decoder%d.%d\n",
1006 port->id, i);
1007 return PTR_ERR(cxled);
1008 }
1009 cxld = &cxled->cxld;
1010 } else {
1011 struct cxl_switch_decoder *cxlsd;
1012
1013 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1014 if (IS_ERR(cxlsd)) {
1015 dev_warn(&port->dev,
1016 "Failed to allocate decoder%d.%d\n",
1017 port->id, i);
1018 return PTR_ERR(cxlsd);
1019 }
1020 cxld = &cxlsd->cxld;
1021 }
1022
1023 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1024 &dpa_base, info);
1025 if (rc) {
1026 dev_warn(&port->dev,
1027 "Failed to initialize decoder%d.%d\n",
1028 port->id, i);
1029 put_device(&cxld->dev);
1030 return rc;
1031 }
1032 rc = add_hdm_decoder(port, cxld, target_map);
1033 if (rc) {
1034 dev_warn(&port->dev,
1035 "Failed to add decoder%d.%d\n", port->id, i);
1036 return rc;
1037 }
1038 }
1039
1040 return 0;
1041 }
1042 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);
1043