1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3 #include <linux/units.h>
4 #include <linux/io-64-nonatomic-lo-hi.h>
5 #include <linux/device.h>
6 #include <linux/delay.h>
7 #include <linux/pci.h>
8 #include <linux/pci-doe.h>
9 #include <cxl/pci.h>
10 #include <linux/aer.h>
11 #include <cxlpci.h>
12 #include <cxlmem.h>
13 #include <cxl.h>
14 #include "core.h"
15 #include "trace.h"
16
17 /**
18 * DOC: cxl core pci
19 *
20 * Compute Express Link protocols are layered on top of PCIe. CXL core provides
21 * a set of helpers for CXL interactions which occur via PCIe.
22 */
23
24 static unsigned short media_ready_timeout = 60;
25 module_param(media_ready_timeout, ushort, 0644);
26 MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready");
27
pci_get_port_num(struct pci_dev * pdev)28 static int pci_get_port_num(struct pci_dev *pdev)
29 {
30 u32 lnkcap;
31 int type;
32
33 type = pci_pcie_type(pdev);
34 if (type != PCI_EXP_TYPE_DOWNSTREAM && type != PCI_EXP_TYPE_ROOT_PORT)
35 return -EINVAL;
36
37 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
38 &lnkcap))
39 return -ENXIO;
40
41 return FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
42 }
43
44 /**
45 * devm_cxl_add_dport_by_dev - allocate a dport by dport device
46 * @port: cxl_port that hosts the dport
47 * @dport_dev: 'struct device' of the dport
48 *
49 * Returns the allocated dport on success or ERR_PTR() of -errno on error
50 */
devm_cxl_add_dport_by_dev(struct cxl_port * port,struct device * dport_dev)51 struct cxl_dport *devm_cxl_add_dport_by_dev(struct cxl_port *port,
52 struct device *dport_dev)
53 {
54 struct cxl_register_map map;
55 struct pci_dev *pdev;
56 int port_num, rc;
57
58 if (!dev_is_pci(dport_dev))
59 return ERR_PTR(-EINVAL);
60
61 pdev = to_pci_dev(dport_dev);
62 port_num = pci_get_port_num(pdev);
63 if (port_num < 0)
64 return ERR_PTR(port_num);
65
66 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
67 if (rc)
68 return ERR_PTR(rc);
69
70 device_lock_assert(&port->dev);
71 return devm_cxl_add_dport(port, dport_dev, port_num, map.resource);
72 }
73 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport_by_dev, "CXL");
74
cxl_dvsec_mem_range_valid(struct cxl_dev_state * cxlds,int id)75 static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
76 {
77 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
78 int d = cxlds->cxl_dvsec;
79 bool valid = false;
80 int rc, i;
81 u32 temp;
82
83 if (id > CXL_DVSEC_RANGE_MAX)
84 return -EINVAL;
85
86 /* Check MEM INFO VALID bit first, give up after 1s */
87 i = 1;
88 do {
89 rc = pci_read_config_dword(pdev,
90 d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id),
91 &temp);
92 if (rc)
93 return pcibios_err_to_errno(rc);
94
95 valid = FIELD_GET(PCI_DVSEC_CXL_MEM_INFO_VALID, temp);
96 if (valid)
97 break;
98 msleep(1000);
99 } while (i--);
100
101 if (!valid) {
102 dev_err(&pdev->dev,
103 "Timeout awaiting memory range %d valid after 1s.\n",
104 id);
105 return -ETIMEDOUT;
106 }
107
108 return 0;
109 }
110
cxl_dvsec_mem_range_active(struct cxl_dev_state * cxlds,int id)111 static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
112 {
113 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
114 int d = cxlds->cxl_dvsec;
115 bool active = false;
116 int rc, i;
117 u32 temp;
118
119 if (id > CXL_DVSEC_RANGE_MAX)
120 return -EINVAL;
121
122 /* Check MEM ACTIVE bit, up to 60s timeout by default */
123 for (i = media_ready_timeout; i; i--) {
124 rc = pci_read_config_dword(
125 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), &temp);
126 if (rc)
127 return pcibios_err_to_errno(rc);
128
129 active = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE, temp);
130 if (active)
131 break;
132 msleep(1000);
133 }
134
135 if (!active) {
136 dev_err(&pdev->dev,
137 "timeout awaiting memory active after %d seconds\n",
138 media_ready_timeout);
139 return -ETIMEDOUT;
140 }
141
142 return 0;
143 }
144
145 /*
146 * Wait up to @media_ready_timeout for the device to report memory
147 * active.
148 */
cxl_await_media_ready(struct cxl_dev_state * cxlds)149 int cxl_await_media_ready(struct cxl_dev_state *cxlds)
150 {
151 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
152 int d = cxlds->cxl_dvsec;
153 int rc, i, hdm_count;
154 u64 md_status;
155 u16 cap;
156
157 rc = pci_read_config_word(pdev,
158 d + PCI_DVSEC_CXL_CAP, &cap);
159 if (rc)
160 return pcibios_err_to_errno(rc);
161
162 hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap);
163 for (i = 0; i < hdm_count; i++) {
164 rc = cxl_dvsec_mem_range_valid(cxlds, i);
165 if (rc)
166 return rc;
167 }
168
169 for (i = 0; i < hdm_count; i++) {
170 rc = cxl_dvsec_mem_range_active(cxlds, i);
171 if (rc)
172 return rc;
173 }
174
175 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
176 if (!CXLMDEV_READY(md_status))
177 return -EIO;
178
179 return 0;
180 }
181 EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, "CXL");
182
cxl_set_mem_enable(struct cxl_dev_state * cxlds,u16 val)183 static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val)
184 {
185 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
186 int d = cxlds->cxl_dvsec;
187 u16 ctrl;
188 int rc;
189
190 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl);
191 if (rc)
192 return pcibios_err_to_errno(rc);
193
194 if ((ctrl & PCI_DVSEC_CXL_MEM_ENABLE) == val)
195 return 1;
196 ctrl &= ~PCI_DVSEC_CXL_MEM_ENABLE;
197 ctrl |= val;
198
199 rc = pci_write_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, ctrl);
200 if (rc)
201 return pcibios_err_to_errno(rc);
202
203 return 0;
204 }
205
clear_mem_enable(void * cxlds)206 static void clear_mem_enable(void *cxlds)
207 {
208 cxl_set_mem_enable(cxlds, 0);
209 }
210
devm_cxl_enable_mem(struct device * host,struct cxl_dev_state * cxlds)211 static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds)
212 {
213 int rc;
214
215 rc = cxl_set_mem_enable(cxlds, PCI_DVSEC_CXL_MEM_ENABLE);
216 if (rc < 0)
217 return rc;
218 if (rc > 0)
219 return 0;
220 return devm_add_action_or_reset(host, clear_mem_enable, cxlds);
221 }
222
223 /* require dvsec ranges to be covered by a locked platform window */
dvsec_range_allowed(struct device * dev,const void * arg)224 static int dvsec_range_allowed(struct device *dev, const void *arg)
225 {
226 const struct range *dev_range = arg;
227 struct cxl_decoder *cxld;
228
229 if (!is_root_decoder(dev))
230 return 0;
231
232 cxld = to_cxl_decoder(dev);
233
234 if (!(cxld->flags & CXL_DECODER_F_RAM))
235 return 0;
236
237 return range_contains(&cxld->hpa_range, dev_range);
238 }
239
disable_hdm(void * _cxlhdm)240 static void disable_hdm(void *_cxlhdm)
241 {
242 u32 global_ctrl;
243 struct cxl_hdm *cxlhdm = _cxlhdm;
244 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
245
246 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
247 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE,
248 hdm + CXL_HDM_DECODER_CTRL_OFFSET);
249 }
250
devm_cxl_enable_hdm(struct device * host,struct cxl_hdm * cxlhdm)251 static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm)
252 {
253 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
254 u32 global_ctrl;
255
256 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
257 writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
258 hdm + CXL_HDM_DECODER_CTRL_OFFSET);
259
260 return devm_add_action_or_reset(host, disable_hdm, cxlhdm);
261 }
262
cxl_dvsec_rr_decode(struct cxl_dev_state * cxlds,struct cxl_endpoint_dvsec_info * info)263 int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
264 struct cxl_endpoint_dvsec_info *info)
265 {
266 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
267 struct device *dev = cxlds->dev;
268 int hdm_count, rc, i, ranges = 0;
269 int d = cxlds->cxl_dvsec;
270 u16 cap, ctrl;
271
272 if (!d) {
273 dev_dbg(dev, "No DVSEC Capability\n");
274 return -ENXIO;
275 }
276
277 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CAP, &cap);
278 if (rc)
279 return pcibios_err_to_errno(rc);
280
281 if (!(cap & PCI_DVSEC_CXL_MEM_CAPABLE)) {
282 dev_dbg(dev, "Not MEM Capable\n");
283 return -ENXIO;
284 }
285
286 /*
287 * It is not allowed by spec for MEM.capable to be set and have 0 legacy
288 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
289 * driver is for a spec defined class code which must be CXL.mem
290 * capable, there is no point in continuing to enable CXL.mem.
291 */
292 hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap);
293 if (!hdm_count || hdm_count > 2)
294 return -EINVAL;
295
296 /*
297 * The current DVSEC values are moot if the memory capability is
298 * disabled, and they will remain moot after the HDM Decoder
299 * capability is enabled.
300 */
301 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl);
302 if (rc)
303 return pcibios_err_to_errno(rc);
304
305 info->mem_enabled = FIELD_GET(PCI_DVSEC_CXL_MEM_ENABLE, ctrl);
306 if (!info->mem_enabled)
307 return 0;
308
309 for (i = 0; i < hdm_count; i++) {
310 u64 base, size;
311 u32 temp;
312
313 rc = cxl_dvsec_mem_range_valid(cxlds, i);
314 if (rc)
315 return rc;
316
317 rc = pci_read_config_dword(
318 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_HIGH(i), &temp);
319 if (rc)
320 return pcibios_err_to_errno(rc);
321
322 size = (u64)temp << 32;
323
324 rc = pci_read_config_dword(
325 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(i), &temp);
326 if (rc)
327 return pcibios_err_to_errno(rc);
328
329 size |= temp & PCI_DVSEC_CXL_MEM_SIZE_LOW;
330 if (!size) {
331 continue;
332 }
333
334 rc = pci_read_config_dword(
335 pdev, d + PCI_DVSEC_CXL_RANGE_BASE_HIGH(i), &temp);
336 if (rc)
337 return pcibios_err_to_errno(rc);
338
339 base = (u64)temp << 32;
340
341 rc = pci_read_config_dword(
342 pdev, d + PCI_DVSEC_CXL_RANGE_BASE_LOW(i), &temp);
343 if (rc)
344 return pcibios_err_to_errno(rc);
345
346 base |= temp & PCI_DVSEC_CXL_MEM_BASE_LOW;
347
348 info->dvsec_range[ranges++] = (struct range) {
349 .start = base,
350 .end = base + size - 1
351 };
352 }
353
354 info->ranges = ranges;
355
356 return 0;
357 }
358 EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, "CXL");
359
360 /**
361 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint
362 * @cxlds: Device state
363 * @cxlhdm: Mapped HDM decoder Capability
364 * @info: Cached DVSEC range registers info
365 *
366 * Try to enable the endpoint's HDM Decoder Capability
367 */
cxl_hdm_decode_init(struct cxl_dev_state * cxlds,struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)368 int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
369 struct cxl_endpoint_dvsec_info *info)
370 {
371 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
372 struct cxl_port *port = cxlhdm->port;
373 struct device *dev = cxlds->dev;
374 struct cxl_port *root;
375 int i, rc, allowed;
376 u32 global_ctrl = 0;
377
378 if (hdm)
379 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
380
381 /*
382 * If the HDM Decoder Capability is already enabled then assume
383 * that some other agent like platform firmware set it up.
384 */
385 if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled))
386 return devm_cxl_enable_mem(&port->dev, cxlds);
387
388 /*
389 * If the HDM Decoder Capability does not exist and DVSEC was
390 * not setup, the DVSEC based emulation cannot be used.
391 */
392 if (!hdm)
393 return -ENODEV;
394
395 /* The HDM Decoder Capability exists but is globally disabled. */
396
397 /*
398 * If the DVSEC CXL Range registers are not enabled, just
399 * enable and use the HDM Decoder Capability registers.
400 */
401 if (!info->mem_enabled) {
402 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);
403 if (rc)
404 return rc;
405
406 return devm_cxl_enable_mem(&port->dev, cxlds);
407 }
408
409 /*
410 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base
411 * [High,Low] when HDM operation is enabled the range register values
412 * are ignored by the device, but the spec also recommends matching the
413 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges
414 * are expected even though Linux does not require or maintain that
415 * match. Check if at least one DVSEC range is enabled and allowed by
416 * the platform. That is, the DVSEC range must be covered by a locked
417 * platform window (CFMWS). Fail otherwise as the endpoint's decoders
418 * cannot be used.
419 */
420
421 root = to_cxl_port(port->dev.parent);
422 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent))
423 root = to_cxl_port(root->dev.parent);
424 if (!is_cxl_root(root)) {
425 dev_err(dev, "Failed to acquire root port for HDM enable\n");
426 return -ENODEV;
427 }
428
429 for (i = 0, allowed = 0; i < info->ranges; i++) {
430 struct device *cxld_dev;
431
432 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i],
433 dvsec_range_allowed);
434 if (!cxld_dev) {
435 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);
436 continue;
437 }
438 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);
439 put_device(cxld_dev);
440 allowed++;
441 }
442
443 if (!allowed) {
444 dev_err(dev, "Range register decodes outside platform defined CXL ranges.\n");
445 return -ENXIO;
446 }
447
448 return 0;
449 }
450 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, "CXL");
451
452 #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
453 #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
454 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
455 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0
456 #define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000
457 #define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff
458 #define CXL_DOE_PROTOCOL_TABLE_ACCESS 2
459
460 #define CDAT_DOE_REQ(entry_handle) cpu_to_le32 \
461 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \
462 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \
463 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \
464 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \
465 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
466
cxl_cdat_get_length(struct device * dev,struct pci_doe_mb * doe_mb,size_t * length)467 static int cxl_cdat_get_length(struct device *dev,
468 struct pci_doe_mb *doe_mb,
469 size_t *length)
470 {
471 __le32 request = CDAT_DOE_REQ(0);
472 __le32 response[2];
473 int rc;
474
475 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL,
476 CXL_DOE_PROTOCOL_TABLE_ACCESS,
477 &request, sizeof(request),
478 &response, sizeof(response));
479 if (rc < 0) {
480 dev_err(dev, "DOE failed: %d", rc);
481 return rc;
482 }
483 if (rc < sizeof(response))
484 return -EIO;
485
486 *length = le32_to_cpu(response[1]);
487 dev_dbg(dev, "CDAT length %zu\n", *length);
488
489 return 0;
490 }
491
cxl_cdat_read_table(struct device * dev,struct pci_doe_mb * doe_mb,struct cdat_doe_rsp * rsp,size_t * length)492 static int cxl_cdat_read_table(struct device *dev,
493 struct pci_doe_mb *doe_mb,
494 struct cdat_doe_rsp *rsp, size_t *length)
495 {
496 size_t received, remaining = *length;
497 unsigned int entry_handle = 0;
498 union cdat_data *data;
499 __le32 saved_dw = 0;
500
501 do {
502 __le32 request = CDAT_DOE_REQ(entry_handle);
503 int rc;
504
505 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL,
506 CXL_DOE_PROTOCOL_TABLE_ACCESS,
507 &request, sizeof(request),
508 rsp, sizeof(*rsp) + remaining);
509 if (rc < 0) {
510 dev_err(dev, "DOE failed: %d", rc);
511 return rc;
512 }
513
514 if (rc < sizeof(*rsp))
515 return -EIO;
516
517 data = (union cdat_data *)rsp->data;
518 received = rc - sizeof(*rsp);
519
520 if (entry_handle == 0) {
521 if (received != sizeof(data->header))
522 return -EIO;
523 } else {
524 if (received < sizeof(data->entry) ||
525 received != le16_to_cpu(data->entry.length))
526 return -EIO;
527 }
528
529 /* Get the CXL table access header entry handle */
530 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE,
531 le32_to_cpu(rsp->doe_header));
532
533 /*
534 * Table Access Response Header overwrote the last DW of
535 * previous entry, so restore that DW
536 */
537 rsp->doe_header = saved_dw;
538 remaining -= received;
539 rsp = (void *)rsp + received;
540 saved_dw = rsp->doe_header;
541 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY);
542
543 /* Length in CDAT header may exceed concatenation of CDAT entries */
544 *length -= remaining;
545
546 return 0;
547 }
548
cdat_checksum(void * buf,size_t size)549 static unsigned char cdat_checksum(void *buf, size_t size)
550 {
551 unsigned char sum, *data = buf;
552 size_t i;
553
554 for (sum = 0, i = 0; i < size; i++)
555 sum += data[i];
556 return sum;
557 }
558
559 /**
560 * read_cdat_data - Read the CDAT data on this port
561 * @port: Port to read data from
562 *
563 * This call will sleep waiting for responses from the DOE mailbox.
564 */
read_cdat_data(struct cxl_port * port)565 void read_cdat_data(struct cxl_port *port)
566 {
567 struct device *uport = port->uport_dev;
568 struct device *dev = &port->dev;
569 struct pci_doe_mb *doe_mb;
570 struct pci_dev *pdev = NULL;
571 struct cxl_memdev *cxlmd;
572 struct cdat_doe_rsp *buf;
573 size_t table_length, length;
574 int rc;
575
576 if (is_cxl_memdev(uport)) {
577 struct device *host;
578
579 cxlmd = to_cxl_memdev(uport);
580 host = cxlmd->dev.parent;
581 if (dev_is_pci(host))
582 pdev = to_pci_dev(host);
583 } else if (dev_is_pci(uport)) {
584 pdev = to_pci_dev(uport);
585 }
586
587 if (!pdev)
588 return;
589
590 doe_mb = pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_CXL,
591 CXL_DOE_PROTOCOL_TABLE_ACCESS);
592 if (!doe_mb) {
593 dev_dbg(dev, "No CDAT mailbox\n");
594 return;
595 }
596
597 port->cdat_available = true;
598
599 if (cxl_cdat_get_length(dev, doe_mb, &length)) {
600 dev_dbg(dev, "No CDAT length\n");
601 return;
602 }
603
604 /*
605 * The begin of the CDAT buffer needs space for additional 4
606 * bytes for the DOE header. Table data starts afterwards.
607 */
608 buf = devm_kzalloc(dev, sizeof(*buf) + length, GFP_KERNEL);
609 if (!buf)
610 goto err;
611
612 table_length = length;
613
614 rc = cxl_cdat_read_table(dev, doe_mb, buf, &length);
615 if (rc)
616 goto err;
617
618 if (table_length != length)
619 dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n",
620 table_length, length);
621
622 if (cdat_checksum(buf->data, length))
623 goto err;
624
625 port->cdat.table = buf->data;
626 port->cdat.length = length;
627
628 return;
629 err:
630 /* Don't leave table data allocated on error */
631 devm_kfree(dev, buf);
632 dev_err(dev, "Failed to read/validate CDAT.\n");
633 }
634 EXPORT_SYMBOL_NS_GPL(read_cdat_data, "CXL");
635
cxl_flit_size(struct pci_dev * pdev)636 static int cxl_flit_size(struct pci_dev *pdev)
637 {
638 if (cxl_pci_flit_256(pdev))
639 return 256;
640
641 return 68;
642 }
643
644 /**
645 * cxl_pci_get_latency - calculate the link latency for the PCIe link
646 * @pdev: PCI device
647 *
648 * return: calculated latency or 0 for no latency
649 *
650 * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation
651 * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency
652 * LinkProgationLatency is negligible, so 0 will be used
653 * RetimerLatency is assumed to be negligible and 0 will be used
654 * FlitLatency = FlitSize / LinkBandwidth
655 * FlitSize is defined by spec. CXL rev3.0 4.2.1.
656 * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used.
657 * The FlitLatency is converted to picoseconds.
658 */
cxl_pci_get_latency(struct pci_dev * pdev)659 long cxl_pci_get_latency(struct pci_dev *pdev)
660 {
661 long bw;
662
663 bw = pcie_link_speed_mbps(pdev);
664 if (bw < 0)
665 return 0;
666 bw /= BITS_PER_BYTE;
667
668 return cxl_flit_size(pdev) * MEGA / bw;
669 }
670
__cxl_endpoint_decoder_reset_detected(struct device * dev,void * data)671 static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data)
672 {
673 struct cxl_port *port = data;
674 struct cxl_decoder *cxld;
675 struct cxl_hdm *cxlhdm;
676 void __iomem *hdm;
677 u32 ctrl;
678
679 if (!is_endpoint_decoder(dev))
680 return 0;
681
682 cxld = to_cxl_decoder(dev);
683 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
684 return 0;
685
686 cxlhdm = dev_get_drvdata(&port->dev);
687 hdm = cxlhdm->regs.hdm_decoder;
688 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
689
690 return !FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl);
691 }
692
cxl_endpoint_decoder_reset_detected(struct cxl_port * port)693 bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
694 {
695 return device_for_each_child(&port->dev, port,
696 __cxl_endpoint_decoder_reset_detected);
697 }
698 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, "CXL");
699
cxl_rcrb_get_comp_regs(struct pci_dev * pdev,struct cxl_register_map * map,struct cxl_dport * dport)700 static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
701 struct cxl_register_map *map,
702 struct cxl_dport *dport)
703 {
704 resource_size_t component_reg_phys;
705
706 *map = (struct cxl_register_map) {
707 .host = &pdev->dev,
708 .resource = CXL_RESOURCE_NONE,
709 };
710
711 component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
712 if (component_reg_phys == CXL_RESOURCE_NONE)
713 return -ENXIO;
714
715 map->resource = component_reg_phys;
716 map->reg_type = CXL_REGLOC_RBI_COMPONENT;
717 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
718
719 return 0;
720 }
721
cxl_pci_setup_regs(struct pci_dev * pdev,enum cxl_regloc_type type,struct cxl_register_map * map)722 int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
723 struct cxl_register_map *map)
724 {
725 int rc;
726
727 rc = cxl_find_regblock(pdev, type, map);
728
729 /*
730 * If the Register Locator DVSEC does not exist, check if it
731 * is an RCH and try to extract the Component Registers from
732 * an RCRB.
733 */
734 if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) {
735 struct cxl_dport *dport;
736 struct cxl_port *port __free(put_cxl_port) =
737 cxl_pci_find_port(pdev, &dport);
738 if (!port)
739 return -EPROBE_DEFER;
740
741 rc = cxl_rcrb_get_comp_regs(pdev, map, dport);
742 if (rc)
743 return rc;
744
745 rc = cxl_dport_map_rcd_linkcap(pdev, dport);
746 if (rc)
747 return rc;
748
749 } else if (rc) {
750 return rc;
751 }
752
753 return cxl_setup_regs(map);
754 }
755 EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, "CXL");
756
cxl_pci_get_bandwidth(struct pci_dev * pdev,struct access_coordinate * c)757 int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c)
758 {
759 int speed, bw;
760 u16 lnksta;
761 u32 width;
762
763 speed = pcie_link_speed_mbps(pdev);
764 if (speed < 0)
765 return speed;
766 speed /= BITS_PER_BYTE;
767
768 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
769 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
770 bw = speed * width;
771
772 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
773 c[i].read_bandwidth = bw;
774 c[i].write_bandwidth = bw;
775 }
776
777 return 0;
778 }
779
780 /*
781 * Set max timeout such that platforms will optimize GPF flow to avoid
782 * the implied worst-case scenario delays. On a sane platform, all
783 * devices should always complete GPF within the energy budget of
784 * the GPF flow. The kernel does not have enough information to pick
785 * anything better than "maximize timeouts and hope it works".
786 *
787 * A misbehaving device could block forward progress of GPF for all
788 * the other devices, exhausting the energy budget of the platform.
789 * However, the spec seems to assume that moving on from slow to respond
790 * devices is a virtue. It is not possible to know that, in actuality,
791 * the slow to respond device is *the* most critical device in the
792 * system to wait.
793 */
794 #define GPF_TIMEOUT_BASE_MAX 2
795 #define GPF_TIMEOUT_SCALE_MAX 7 /* 10 seconds */
796
cxl_gpf_get_dvsec(struct device * dev)797 u16 cxl_gpf_get_dvsec(struct device *dev)
798 {
799 struct pci_dev *pdev;
800 bool is_port = true;
801 u16 dvsec;
802
803 if (!dev_is_pci(dev))
804 return 0;
805
806 pdev = to_pci_dev(dev);
807 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT)
808 is_port = false;
809
810 dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
811 is_port ? PCI_DVSEC_CXL_PORT_GPF : PCI_DVSEC_CXL_DEVICE_GPF);
812 if (!dvsec)
813 dev_warn(dev, "%s GPF DVSEC not present\n",
814 is_port ? "Port" : "Device");
815 return dvsec;
816 }
817 EXPORT_SYMBOL_NS_GPL(cxl_gpf_get_dvsec, "CXL");
818
update_gpf_port_dvsec(struct pci_dev * pdev,int dvsec,int phase)819 static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase)
820 {
821 u64 base, scale;
822 int rc, offset;
823 u16 ctrl;
824
825 switch (phase) {
826 case 1:
827 offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_CONTROL;
828 base = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_BASE;
829 scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_SCALE;
830 break;
831 case 2:
832 offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_CONTROL;
833 base = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_BASE;
834 scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_SCALE;
835 break;
836 default:
837 return -EINVAL;
838 }
839
840 rc = pci_read_config_word(pdev, dvsec + offset, &ctrl);
841 if (rc)
842 return rc;
843
844 if (FIELD_GET(base, ctrl) == GPF_TIMEOUT_BASE_MAX &&
845 FIELD_GET(scale, ctrl) == GPF_TIMEOUT_SCALE_MAX)
846 return 0;
847
848 ctrl = FIELD_PREP(base, GPF_TIMEOUT_BASE_MAX);
849 ctrl |= FIELD_PREP(scale, GPF_TIMEOUT_SCALE_MAX);
850
851 rc = pci_write_config_word(pdev, dvsec + offset, ctrl);
852 if (!rc)
853 pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n",
854 phase, GPF_TIMEOUT_BASE_MAX);
855
856 return rc;
857 }
858
cxl_gpf_port_setup(struct cxl_dport * dport)859 int cxl_gpf_port_setup(struct cxl_dport *dport)
860 {
861 if (!dport)
862 return -EINVAL;
863
864 if (!dport->gpf_dvsec) {
865 struct pci_dev *pdev;
866 int dvsec;
867
868 dvsec = cxl_gpf_get_dvsec(dport->dport_dev);
869 if (!dvsec)
870 return -EINVAL;
871
872 dport->gpf_dvsec = dvsec;
873 pdev = to_pci_dev(dport->dport_dev);
874 update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 1);
875 update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 2);
876 }
877
878 return 0;
879 }
880
881 struct cxl_walk_context {
882 struct pci_bus *bus;
883 struct cxl_port *port;
884 int type;
885 int error;
886 int count;
887 };
888
count_dports(struct pci_dev * pdev,void * data)889 static int count_dports(struct pci_dev *pdev, void *data)
890 {
891 struct cxl_walk_context *ctx = data;
892 int type = pci_pcie_type(pdev);
893
894 if (pdev->bus != ctx->bus)
895 return 0;
896 if (!pci_is_pcie(pdev))
897 return 0;
898 if (type != ctx->type)
899 return 0;
900
901 ctx->count++;
902 return 0;
903 }
904
cxl_port_get_possible_dports(struct cxl_port * port)905 int cxl_port_get_possible_dports(struct cxl_port *port)
906 {
907 struct pci_bus *bus = cxl_port_to_pci_bus(port);
908 struct cxl_walk_context ctx;
909 int type;
910
911 if (!bus) {
912 dev_err(&port->dev, "No PCI bus found for port %s\n",
913 dev_name(&port->dev));
914 return -ENXIO;
915 }
916
917 if (pci_is_root_bus(bus))
918 type = PCI_EXP_TYPE_ROOT_PORT;
919 else
920 type = PCI_EXP_TYPE_DOWNSTREAM;
921
922 ctx = (struct cxl_walk_context) {
923 .bus = bus,
924 .type = type,
925 };
926 pci_walk_bus(bus, count_dports, &ctx);
927
928 return ctx.count;
929 }
930