1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/unaligned.h>
4 #include <linux/io-64-nonatomic-lo-hi.h>
5 #include <linux/moduleparam.h>
6 #include <linux/module.h>
7 #include <linux/delay.h>
8 #include <linux/sizes.h>
9 #include <linux/mutex.h>
10 #include <linux/list.h>
11 #include <linux/pci.h>
12 #include <linux/aer.h>
13 #include <linux/io.h>
14 #include <cxl/pci.h>
15 #include <cxl/mailbox.h>
16 #include "cxlmem.h"
17 #include "cxlpci.h"
18 #include "cxl.h"
19 #include "pmu.h"
20
21 /**
22 * DOC: cxl pci
23 *
24 * This implements the PCI exclusive functionality for a CXL device as it is
25 * defined by the Compute Express Link specification. CXL devices may surface
26 * certain functionality even if it isn't CXL enabled. While this driver is
27 * focused around the PCI specific aspects of a CXL device, it binds to the
28 * specific CXL memory device class code, and therefore the implementation of
29 * cxl_pci is focused around CXL memory devices.
30 *
31 * The driver has several responsibilities, mainly:
32 * - Create the memX device and register on the CXL bus.
33 * - Enumerate device's register interface and map them.
34 * - Registers nvdimm bridge device with cxl_core.
35 * - Registers a CXL mailbox with cxl_core.
36 */
37
38 #define cxl_doorbell_busy(cxlds) \
39 (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \
40 CXLDEV_MBOX_CTRL_DOORBELL)
41
42 /* CXL 2.0 - 8.2.8.4 */
43 #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
44
45 /*
46 * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
47 * dictate how long to wait for the mailbox to become ready. The new
48 * field allows the device to tell software the amount of time to wait
49 * before mailbox ready. This field per the spec theoretically allows
50 * for up to 255 seconds. 255 seconds is unreasonably long, its longer
51 * than the maximum SATA port link recovery wait. Default to 60 seconds
52 * until someone builds a CXL device that needs more time in practice.
53 */
54 static unsigned short mbox_ready_timeout = 60;
55 module_param(mbox_ready_timeout, ushort, 0644);
56 MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready");
57
cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state * cxlds)58 static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
59 {
60 const unsigned long start = jiffies;
61 unsigned long end = start;
62
63 while (cxl_doorbell_busy(cxlds)) {
64 end = jiffies;
65
66 if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
67 /* Check again in case preempted before timeout test */
68 if (!cxl_doorbell_busy(cxlds))
69 break;
70 return -ETIMEDOUT;
71 }
72 cpu_relax();
73 }
74
75 dev_dbg(cxlds->dev, "Doorbell wait took %dms",
76 jiffies_to_msecs(end) - jiffies_to_msecs(start));
77 return 0;
78 }
79
80 #define cxl_err(dev, status, msg) \
81 dev_err_ratelimited(dev, msg ", device state %s%s\n", \
82 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
83 status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
84
85 #define cxl_cmd_err(dev, cmd, status, msg) \
86 dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \
87 (cmd)->opcode, \
88 status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
89 status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
90
91 /*
92 * Threaded irq dev_id's must be globally unique. cxl_dev_id provides a unique
93 * wrapper object for each irq within the same cxlds.
94 */
95 struct cxl_dev_id {
96 struct cxl_dev_state *cxlds;
97 };
98
cxl_request_irq(struct cxl_dev_state * cxlds,int irq,irq_handler_t thread_fn)99 static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq,
100 irq_handler_t thread_fn)
101 {
102 struct device *dev = cxlds->dev;
103 struct cxl_dev_id *dev_id;
104
105 dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL);
106 if (!dev_id)
107 return -ENOMEM;
108 dev_id->cxlds = cxlds;
109
110 return devm_request_threaded_irq(dev, irq, NULL, thread_fn,
111 IRQF_SHARED | IRQF_ONESHOT, NULL,
112 dev_id);
113 }
114
cxl_mbox_background_complete(struct cxl_dev_state * cxlds)115 static bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds)
116 {
117 u64 reg;
118
119 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
120 return FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg) == 100;
121 }
122
cxl_pci_mbox_irq(int irq,void * id)123 static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
124 {
125 u64 reg;
126 u16 opcode;
127 struct cxl_dev_id *dev_id = id;
128 struct cxl_dev_state *cxlds = dev_id->cxlds;
129 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
130 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
131
132 if (!cxl_mbox_background_complete(cxlds))
133 return IRQ_NONE;
134
135 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
136 opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
137 if (opcode == CXL_MBOX_OP_SANITIZE) {
138 mutex_lock(&cxl_mbox->mbox_mutex);
139 if (mds->security.sanitize_node)
140 mod_delayed_work(system_percpu_wq, &mds->security.poll_dwork, 0);
141 mutex_unlock(&cxl_mbox->mbox_mutex);
142 } else {
143 /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */
144 rcuwait_wake_up(&cxl_mbox->mbox_wait);
145 }
146
147 return IRQ_HANDLED;
148 }
149
150 /*
151 * Sanitization operation polling mode.
152 */
cxl_mbox_sanitize_work(struct work_struct * work)153 static void cxl_mbox_sanitize_work(struct work_struct *work)
154 {
155 struct cxl_memdev_state *mds =
156 container_of(work, typeof(*mds), security.poll_dwork.work);
157 struct cxl_dev_state *cxlds = &mds->cxlds;
158 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
159
160 mutex_lock(&cxl_mbox->mbox_mutex);
161 if (cxl_mbox_background_complete(cxlds)) {
162 mds->security.poll_tmo_secs = 0;
163 if (mds->security.sanitize_node)
164 sysfs_notify_dirent(mds->security.sanitize_node);
165 mds->security.sanitize_active = false;
166
167 dev_dbg(cxlds->dev, "Sanitization operation ended\n");
168 } else {
169 int timeout = mds->security.poll_tmo_secs + 10;
170
171 mds->security.poll_tmo_secs = min(15 * 60, timeout);
172 schedule_delayed_work(&mds->security.poll_dwork, timeout * HZ);
173 }
174 mutex_unlock(&cxl_mbox->mbox_mutex);
175 }
176
177 /**
178 * __cxl_pci_mbox_send_cmd() - Execute a mailbox command
179 * @cxl_mbox: CXL mailbox context
180 * @mbox_cmd: Command to send to the memory device.
181 *
182 * Context: Any context. Expects mbox_mutex to be held.
183 * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
184 * Caller should check the return code in @mbox_cmd to make sure it
185 * succeeded.
186 *
187 * This is a generic form of the CXL mailbox send command thus only using the
188 * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
189 * devices, and perhaps other types of CXL devices may have further information
190 * available upon error conditions. Driver facilities wishing to send mailbox
191 * commands should use the wrapper command.
192 *
193 * The CXL spec allows for up to two mailboxes. The intention is for the primary
194 * mailbox to be OS controlled and the secondary mailbox to be used by system
195 * firmware. This allows the OS and firmware to communicate with the device and
196 * not need to coordinate with each other. The driver only uses the primary
197 * mailbox.
198 */
__cxl_pci_mbox_send_cmd(struct cxl_mailbox * cxl_mbox,struct cxl_mbox_cmd * mbox_cmd)199 static int __cxl_pci_mbox_send_cmd(struct cxl_mailbox *cxl_mbox,
200 struct cxl_mbox_cmd *mbox_cmd)
201 {
202 struct cxl_dev_state *cxlds = mbox_to_cxlds(cxl_mbox);
203 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
204 void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
205 struct device *dev = cxlds->dev;
206 u64 cmd_reg, status_reg;
207 size_t out_len;
208 int rc;
209
210 lockdep_assert_held(&cxl_mbox->mbox_mutex);
211
212 /*
213 * Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
214 * 1. Caller reads MB Control Register to verify doorbell is clear
215 * 2. Caller writes Command Register
216 * 3. Caller writes Command Payload Registers if input payload is non-empty
217 * 4. Caller writes MB Control Register to set doorbell
218 * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured
219 * 6. Caller reads MB Status Register to fetch Return code
220 * 7. If command successful, Caller reads Command Register to get Payload Length
221 * 8. If output payload is non-empty, host reads Command Payload Registers
222 *
223 * Hardware is free to do whatever it wants before the doorbell is rung,
224 * and isn't allowed to change anything after it clears the doorbell. As
225 * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can
226 * also happen in any order (though some orders might not make sense).
227 */
228
229 /* #1 */
230 if (cxl_doorbell_busy(cxlds)) {
231 u64 md_status =
232 readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
233
234 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,
235 "mailbox queue busy");
236 return -EBUSY;
237 }
238
239 /*
240 * With sanitize polling, hardware might be done and the poller still
241 * not be in sync. Ensure no new command comes in until so. Keep the
242 * hardware semantics and only allow device health status.
243 */
244 if (mds->security.poll_tmo_secs > 0) {
245 if (mbox_cmd->opcode != CXL_MBOX_OP_GET_HEALTH_INFO)
246 return -EBUSY;
247 }
248
249 cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
250 mbox_cmd->opcode);
251 if (mbox_cmd->size_in) {
252 if (WARN_ON(!mbox_cmd->payload_in))
253 return -EINVAL;
254
255 cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK,
256 mbox_cmd->size_in);
257 memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in);
258 }
259
260 /* #2, #3 */
261 writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
262
263 /* #4 */
264 dev_dbg(dev, "Sending command: 0x%04x\n", mbox_cmd->opcode);
265 writel(CXLDEV_MBOX_CTRL_DOORBELL,
266 cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
267
268 /* #5 */
269 rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
270 if (rc == -ETIMEDOUT) {
271 u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
272
273 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");
274 return rc;
275 }
276
277 /* #6 */
278 status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);
279 mbox_cmd->return_code =
280 FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
281
282 /*
283 * Handle the background command in a synchronous manner.
284 *
285 * All other mailbox commands will serialize/queue on the mbox_mutex,
286 * which we currently hold. Furthermore this also guarantees that
287 * cxl_mbox_background_complete() checks are safe amongst each other,
288 * in that no new bg operation can occur in between.
289 *
290 * Background operations are timesliced in accordance with the nature
291 * of the command. In the event of timeout, the mailbox state is
292 * indeterminate until the next successful command submission and the
293 * driver can get back in sync with the hardware state.
294 */
295 if (mbox_cmd->return_code == CXL_MBOX_CMD_RC_BACKGROUND) {
296 u64 bg_status_reg;
297 int i, timeout;
298
299 /*
300 * Sanitization is a special case which monopolizes the device
301 * and cannot be timesliced. Handle asynchronously instead,
302 * and allow userspace to poll(2) for completion.
303 */
304 if (mbox_cmd->opcode == CXL_MBOX_OP_SANITIZE) {
305 if (mds->security.sanitize_active)
306 return -EBUSY;
307
308 /* give first timeout a second */
309 timeout = 1;
310 mds->security.poll_tmo_secs = timeout;
311 mds->security.sanitize_active = true;
312 schedule_delayed_work(&mds->security.poll_dwork,
313 timeout * HZ);
314 dev_dbg(dev, "Sanitization operation started\n");
315 goto success;
316 }
317
318 dev_dbg(dev, "Mailbox background operation (0x%04x) started\n",
319 mbox_cmd->opcode);
320
321 timeout = mbox_cmd->poll_interval_ms;
322 for (i = 0; i < mbox_cmd->poll_count; i++) {
323 if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait,
324 cxl_mbox_background_complete(cxlds),
325 TASK_UNINTERRUPTIBLE,
326 msecs_to_jiffies(timeout)) > 0)
327 break;
328 }
329
330 if (!cxl_mbox_background_complete(cxlds)) {
331 dev_err(dev, "timeout waiting for background (%d ms)\n",
332 timeout * mbox_cmd->poll_count);
333 return -ETIMEDOUT;
334 }
335
336 bg_status_reg = readq(cxlds->regs.mbox +
337 CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
338 mbox_cmd->return_code =
339 FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK,
340 bg_status_reg);
341 dev_dbg(dev,
342 "Mailbox background operation (0x%04x) completed\n",
343 mbox_cmd->opcode);
344 }
345
346 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) {
347 dev_dbg(dev, "Mailbox operation had an error: %s\n",
348 cxl_mbox_cmd_rc2str(mbox_cmd));
349 return 0; /* completed but caller must check return_code */
350 }
351
352 success:
353 /* #7 */
354 cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
355 out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);
356
357 /* #8 */
358 if (out_len && mbox_cmd->payload_out) {
359 /*
360 * Sanitize the copy. If hardware misbehaves, out_len per the
361 * spec can actually be greater than the max allowed size (21
362 * bits available but spec defined 1M max). The caller also may
363 * have requested less data than the hardware supplied even
364 * within spec.
365 */
366 size_t n;
367
368 n = min3(mbox_cmd->size_out, cxl_mbox->payload_size, out_len);
369 memcpy_fromio(mbox_cmd->payload_out, payload, n);
370 mbox_cmd->size_out = n;
371 } else {
372 mbox_cmd->size_out = 0;
373 }
374
375 return 0;
376 }
377
cxl_pci_mbox_send(struct cxl_mailbox * cxl_mbox,struct cxl_mbox_cmd * cmd)378 static int cxl_pci_mbox_send(struct cxl_mailbox *cxl_mbox,
379 struct cxl_mbox_cmd *cmd)
380 {
381 int rc;
382
383 mutex_lock(&cxl_mbox->mbox_mutex);
384 rc = __cxl_pci_mbox_send_cmd(cxl_mbox, cmd);
385 mutex_unlock(&cxl_mbox->mbox_mutex);
386
387 return rc;
388 }
389
cxl_pci_setup_mailbox(struct cxl_memdev_state * mds,bool irq_avail)390 static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
391 {
392 struct cxl_dev_state *cxlds = &mds->cxlds;
393 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
394 const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
395 struct device *dev = cxlds->dev;
396 unsigned long timeout;
397 int irq, msgnum;
398 u64 md_status;
399 u32 ctrl;
400
401 timeout = jiffies + mbox_ready_timeout * HZ;
402 do {
403 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
404 if (md_status & CXLMDEV_MBOX_IF_READY)
405 break;
406 if (msleep_interruptible(100))
407 break;
408 } while (!time_after(jiffies, timeout));
409
410 if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
411 cxl_err(dev, md_status, "timeout awaiting mailbox ready");
412 return -ETIMEDOUT;
413 }
414
415 /*
416 * A command may be in flight from a previous driver instance,
417 * think kexec, do one doorbell wait so that
418 * __cxl_pci_mbox_send_cmd() can assume that it is the only
419 * source for future doorbell busy events.
420 */
421 if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
422 cxl_err(dev, md_status, "timeout awaiting mailbox idle");
423 return -ETIMEDOUT;
424 }
425
426 cxl_mbox->mbox_send = cxl_pci_mbox_send;
427 cxl_mbox->payload_size =
428 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
429
430 /*
431 * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register
432 *
433 * If the size is too small, mandatory commands will not work and so
434 * there's no point in going forward. If the size is too large, there's
435 * no harm is soft limiting it.
436 */
437 cxl_mbox->payload_size = min_t(size_t, cxl_mbox->payload_size, SZ_1M);
438 if (cxl_mbox->payload_size < 256) {
439 dev_err(dev, "Mailbox is too small (%zub)",
440 cxl_mbox->payload_size);
441 return -ENXIO;
442 }
443
444 dev_dbg(dev, "Mailbox payload sized %zu", cxl_mbox->payload_size);
445
446 INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work);
447
448 /* background command interrupts are optional */
449 if (!(cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ) || !irq_avail)
450 return 0;
451
452 msgnum = FIELD_GET(CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK, cap);
453 irq = pci_irq_vector(to_pci_dev(cxlds->dev), msgnum);
454 if (irq < 0)
455 return 0;
456
457 if (cxl_request_irq(cxlds, irq, cxl_pci_mbox_irq))
458 return 0;
459
460 dev_dbg(cxlds->dev, "Mailbox interrupts enabled\n");
461 /* enable background command mbox irq support */
462 ctrl = readl(cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
463 ctrl |= CXLDEV_MBOX_CTRL_BG_CMD_IRQ;
464 writel(ctrl, cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
465
466 return 0;
467 }
468
free_event_buf(void * buf)469 static void free_event_buf(void *buf)
470 {
471 kvfree(buf);
472 }
473
474 /*
475 * There is a single buffer for reading event logs from the mailbox. All logs
476 * share this buffer protected by the mds->event_log_lock.
477 */
cxl_mem_alloc_event_buf(struct cxl_memdev_state * mds)478 static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds)
479 {
480 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
481 struct cxl_get_event_payload *buf;
482
483 buf = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
484 if (!buf)
485 return -ENOMEM;
486 mds->event.buf = buf;
487
488 return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf);
489 }
490
cxl_alloc_irq_vectors(struct pci_dev * pdev)491 static bool cxl_alloc_irq_vectors(struct pci_dev *pdev)
492 {
493 int nvecs;
494
495 /*
496 * Per CXL 3.0 3.1.1 CXL.io Endpoint a function on a CXL device must
497 * not generate INTx messages if that function participates in
498 * CXL.cache or CXL.mem.
499 *
500 * Additionally pci_alloc_irq_vectors() handles calling
501 * pci_free_irq_vectors() automatically despite not being called
502 * pcim_*. See pci_setup_msi_context().
503 */
504 nvecs = pci_alloc_irq_vectors(pdev, 1, CXL_PCI_DEFAULT_MAX_VECTORS,
505 PCI_IRQ_MSIX | PCI_IRQ_MSI);
506 if (nvecs < 1) {
507 dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs);
508 return false;
509 }
510 return true;
511 }
512
cxl_event_thread(int irq,void * id)513 static irqreturn_t cxl_event_thread(int irq, void *id)
514 {
515 struct cxl_dev_id *dev_id = id;
516 struct cxl_dev_state *cxlds = dev_id->cxlds;
517 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
518 u32 status;
519
520 do {
521 /*
522 * CXL 3.0 8.2.8.3.1: The lower 32 bits are the status;
523 * ignore the reserved upper 32 bits
524 */
525 status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET);
526 /* Ignore logs unknown to the driver */
527 status &= CXLDEV_EVENT_STATUS_ALL;
528 if (!status)
529 break;
530 cxl_mem_get_event_records(mds, status);
531 cond_resched();
532 } while (status);
533
534 return IRQ_HANDLED;
535 }
536
cxl_event_req_irq(struct cxl_dev_state * cxlds,u8 setting)537 static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
538 {
539 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
540 int irq;
541
542 if (FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting) != CXL_INT_MSI_MSIX)
543 return -ENXIO;
544
545 irq = pci_irq_vector(pdev,
546 FIELD_GET(CXLDEV_EVENT_INT_MSGNUM_MASK, setting));
547 if (irq < 0)
548 return irq;
549
550 return cxl_request_irq(cxlds, irq, cxl_event_thread);
551 }
552
cxl_event_get_int_policy(struct cxl_memdev_state * mds,struct cxl_event_interrupt_policy * policy)553 static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
554 struct cxl_event_interrupt_policy *policy)
555 {
556 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
557 struct cxl_mbox_cmd mbox_cmd = {
558 .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY,
559 .payload_out = policy,
560 .size_out = sizeof(*policy),
561 };
562 int rc;
563
564 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
565 if (rc < 0)
566 dev_err(mds->cxlds.dev,
567 "Failed to get event interrupt policy : %d", rc);
568
569 return rc;
570 }
571
cxl_event_config_msgnums(struct cxl_memdev_state * mds,struct cxl_event_interrupt_policy * policy)572 static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
573 struct cxl_event_interrupt_policy *policy)
574 {
575 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
576 struct cxl_mbox_cmd mbox_cmd;
577 int rc;
578
579 *policy = (struct cxl_event_interrupt_policy) {
580 .info_settings = CXL_INT_MSI_MSIX,
581 .warn_settings = CXL_INT_MSI_MSIX,
582 .failure_settings = CXL_INT_MSI_MSIX,
583 .fatal_settings = CXL_INT_MSI_MSIX,
584 };
585
586 mbox_cmd = (struct cxl_mbox_cmd) {
587 .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY,
588 .payload_in = policy,
589 .size_in = sizeof(*policy),
590 };
591
592 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
593 if (rc < 0) {
594 dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d",
595 rc);
596 return rc;
597 }
598
599 /* Retrieve final interrupt settings */
600 return cxl_event_get_int_policy(mds, policy);
601 }
602
cxl_event_irqsetup(struct cxl_memdev_state * mds)603 static int cxl_event_irqsetup(struct cxl_memdev_state *mds)
604 {
605 struct cxl_dev_state *cxlds = &mds->cxlds;
606 struct cxl_event_interrupt_policy policy;
607 int rc;
608
609 rc = cxl_event_config_msgnums(mds, &policy);
610 if (rc)
611 return rc;
612
613 rc = cxl_event_req_irq(cxlds, policy.info_settings);
614 if (rc) {
615 dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n");
616 return rc;
617 }
618
619 rc = cxl_event_req_irq(cxlds, policy.warn_settings);
620 if (rc) {
621 dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n");
622 return rc;
623 }
624
625 rc = cxl_event_req_irq(cxlds, policy.failure_settings);
626 if (rc) {
627 dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n");
628 return rc;
629 }
630
631 rc = cxl_event_req_irq(cxlds, policy.fatal_settings);
632 if (rc) {
633 dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n");
634 return rc;
635 }
636
637 return 0;
638 }
639
cxl_event_int_is_fw(u8 setting)640 static bool cxl_event_int_is_fw(u8 setting)
641 {
642 u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting);
643
644 return mode == CXL_INT_FW;
645 }
646
cxl_event_config(struct pci_host_bridge * host_bridge,struct cxl_memdev_state * mds,bool irq_avail)647 static int cxl_event_config(struct pci_host_bridge *host_bridge,
648 struct cxl_memdev_state *mds, bool irq_avail)
649 {
650 struct cxl_event_interrupt_policy policy;
651 int rc;
652
653 /*
654 * When BIOS maintains CXL error reporting control, it will process
655 * event records. Only one agent can do so.
656 */
657 if (!host_bridge->native_cxl_error)
658 return 0;
659
660 if (!irq_avail) {
661 dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n");
662 return 0;
663 }
664
665 rc = cxl_event_get_int_policy(mds, &policy);
666 if (rc)
667 return rc;
668
669 if (cxl_event_int_is_fw(policy.info_settings) ||
670 cxl_event_int_is_fw(policy.warn_settings) ||
671 cxl_event_int_is_fw(policy.failure_settings) ||
672 cxl_event_int_is_fw(policy.fatal_settings)) {
673 dev_err(mds->cxlds.dev,
674 "FW still in control of Event Logs despite _OSC settings\n");
675 return -EBUSY;
676 }
677
678 rc = cxl_mem_alloc_event_buf(mds);
679 if (rc)
680 return rc;
681
682 rc = cxl_event_irqsetup(mds);
683 if (rc)
684 return rc;
685
686 cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
687
688 return 0;
689 }
690
cxl_pci_type3_init_mailbox(struct cxl_dev_state * cxlds)691 static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds)
692 {
693 int rc;
694
695 rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev);
696 if (rc)
697 return rc;
698
699 return 0;
700 }
701
rcd_pcie_cap_emit(struct device * dev,u16 offset,char * buf,size_t width)702 static ssize_t rcd_pcie_cap_emit(struct device *dev, u16 offset, char *buf, size_t width)
703 {
704 struct cxl_dev_state *cxlds = dev_get_drvdata(dev);
705 struct cxl_memdev *cxlmd = cxlds->cxlmd;
706 struct device *root_dev;
707 struct cxl_dport *dport;
708 struct cxl_port *root __free(put_cxl_port) =
709 cxl_mem_find_port(cxlmd, &dport);
710
711 if (!root)
712 return -ENXIO;
713
714 root_dev = root->uport_dev;
715 if (!root_dev)
716 return -ENXIO;
717
718 if (!dport->regs.rcd_pcie_cap)
719 return -ENXIO;
720
721 guard(device)(root_dev);
722 if (!root_dev->driver)
723 return -ENXIO;
724
725 switch (width) {
726 case 2:
727 return sysfs_emit(buf, "%#x\n",
728 readw(dport->regs.rcd_pcie_cap + offset));
729 case 4:
730 return sysfs_emit(buf, "%#x\n",
731 readl(dport->regs.rcd_pcie_cap + offset));
732 default:
733 return -EINVAL;
734 }
735 }
736
rcd_link_cap_show(struct device * dev,struct device_attribute * attr,char * buf)737 static ssize_t rcd_link_cap_show(struct device *dev,
738 struct device_attribute *attr, char *buf)
739 {
740 return rcd_pcie_cap_emit(dev, PCI_EXP_LNKCAP, buf, sizeof(u32));
741 }
742 static DEVICE_ATTR_RO(rcd_link_cap);
743
rcd_link_ctrl_show(struct device * dev,struct device_attribute * attr,char * buf)744 static ssize_t rcd_link_ctrl_show(struct device *dev,
745 struct device_attribute *attr, char *buf)
746 {
747 return rcd_pcie_cap_emit(dev, PCI_EXP_LNKCTL, buf, sizeof(u16));
748 }
749 static DEVICE_ATTR_RO(rcd_link_ctrl);
750
rcd_link_status_show(struct device * dev,struct device_attribute * attr,char * buf)751 static ssize_t rcd_link_status_show(struct device *dev,
752 struct device_attribute *attr, char *buf)
753 {
754 return rcd_pcie_cap_emit(dev, PCI_EXP_LNKSTA, buf, sizeof(u16));
755 }
756 static DEVICE_ATTR_RO(rcd_link_status);
757
758 static struct attribute *cxl_rcd_attrs[] = {
759 &dev_attr_rcd_link_cap.attr,
760 &dev_attr_rcd_link_ctrl.attr,
761 &dev_attr_rcd_link_status.attr,
762 NULL
763 };
764
cxl_rcd_visible(struct kobject * kobj,struct attribute * a,int n)765 static umode_t cxl_rcd_visible(struct kobject *kobj, struct attribute *a, int n)
766 {
767 struct device *dev = kobj_to_dev(kobj);
768 struct pci_dev *pdev = to_pci_dev(dev);
769
770 if (is_cxl_restricted(pdev))
771 return a->mode;
772
773 return 0;
774 }
775
776 static struct attribute_group cxl_rcd_group = {
777 .attrs = cxl_rcd_attrs,
778 .is_visible = cxl_rcd_visible,
779 };
780 __ATTRIBUTE_GROUPS(cxl_rcd);
781
cxl_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)782 static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
783 {
784 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus);
785 struct cxl_dpa_info range_info = { 0 };
786 struct cxl_memdev_state *mds;
787 struct cxl_dev_state *cxlds;
788 struct cxl_register_map map;
789 struct cxl_memdev *cxlmd;
790 int rc, pmu_count;
791 unsigned int i;
792 bool irq_avail;
793 u16 dvsec;
794
795 rc = pcim_enable_device(pdev);
796 if (rc)
797 return rc;
798 pci_set_master(pdev);
799
800 dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
801 PCI_DVSEC_CXL_DEVICE);
802 if (!dvsec)
803 pci_warn(pdev, "Device DVSEC not present, skip CXL.mem init\n");
804
805 mds = cxl_memdev_state_create(&pdev->dev, pci_get_dsn(pdev), dvsec);
806 if (IS_ERR(mds))
807 return PTR_ERR(mds);
808 cxlds = &mds->cxlds;
809 pci_set_drvdata(pdev, cxlds);
810
811 cxlds->rcd = is_cxl_restricted(pdev);
812
813 rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
814 if (rc)
815 return rc;
816
817 rc = cxl_map_device_regs(&map, &cxlds->regs);
818 if (rc)
819 return rc;
820
821 /*
822 * If the component registers can't be found, the cxl_pci driver may
823 * still be useful for management functions so don't return an error.
824 */
825 rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
826 &cxlds->reg_map);
827 if (rc) {
828 if (rc == -EPROBE_DEFER)
829 return rc;
830 dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
831 } else if (!cxlds->reg_map.component_map.ras.valid) {
832 dev_dbg(&pdev->dev, "RAS registers not found\n");
833 }
834
835 rc = cxl_pci_type3_init_mailbox(cxlds);
836 if (rc)
837 return rc;
838
839 rc = cxl_await_media_ready(cxlds);
840 if (rc == 0)
841 cxlds->media_ready = true;
842 else
843 dev_warn(&pdev->dev, "Media not active (%d)\n", rc);
844
845 irq_avail = cxl_alloc_irq_vectors(pdev);
846
847 rc = cxl_pci_setup_mailbox(mds, irq_avail);
848 if (rc)
849 return rc;
850
851 rc = cxl_enumerate_cmds(mds);
852 if (rc)
853 return rc;
854
855 rc = cxl_set_timestamp(mds);
856 if (rc)
857 return rc;
858
859 rc = cxl_poison_state_init(mds);
860 if (rc)
861 return rc;
862
863 rc = cxl_dev_state_identify(mds);
864 if (rc)
865 return rc;
866
867 rc = cxl_mem_dpa_fetch(mds, &range_info);
868 if (rc)
869 return rc;
870
871 rc = cxl_dpa_setup(cxlds, &range_info);
872 if (rc)
873 return rc;
874
875 rc = devm_cxl_setup_features(cxlds);
876 if (rc)
877 dev_dbg(&pdev->dev, "No CXL Features discovered\n");
878
879 cxlmd = devm_cxl_add_classdev(cxlds);
880 if (IS_ERR(cxlmd))
881 return PTR_ERR(cxlmd);
882
883 rc = devm_cxl_setup_fw_upload(&pdev->dev, mds);
884 if (rc)
885 return rc;
886
887 rc = devm_cxl_sanitize_setup_notifier(&pdev->dev, cxlmd);
888 if (rc)
889 return rc;
890
891 rc = devm_cxl_setup_fwctl(&pdev->dev, cxlmd);
892 if (rc)
893 dev_dbg(&pdev->dev, "No CXL FWCTL setup\n");
894
895 pmu_count = cxl_count_regblock(pdev, CXL_REGLOC_RBI_PMU);
896 if (pmu_count < 0)
897 return pmu_count;
898
899 for (i = 0; i < pmu_count; i++) {
900 struct cxl_pmu_regs pmu_regs;
901
902 rc = cxl_find_regblock_instance(pdev, CXL_REGLOC_RBI_PMU, &map, i);
903 if (rc) {
904 dev_dbg(&pdev->dev, "Could not find PMU regblock\n");
905 break;
906 }
907
908 rc = cxl_map_pmu_regs(&map, &pmu_regs);
909 if (rc) {
910 dev_dbg(&pdev->dev, "Could not map PMU regs\n");
911 break;
912 }
913
914 rc = devm_cxl_pmu_add(cxlds->dev, &pmu_regs, cxlmd->id, i, CXL_PMU_MEMDEV);
915 if (rc) {
916 dev_dbg(&pdev->dev, "Could not add PMU instance\n");
917 break;
918 }
919 }
920
921 rc = cxl_event_config(host_bridge, mds, irq_avail);
922 if (rc)
923 return rc;
924
925 pci_save_state(pdev);
926
927 return rc;
928 }
929
930 static const struct pci_device_id cxl_mem_pci_tbl[] = {
931 /* PCI class code for CXL.mem Type-3 Devices */
932 { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)},
933 { /* terminate list */ },
934 };
935 MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
936
cxl_slot_reset(struct pci_dev * pdev)937 static pci_ers_result_t cxl_slot_reset(struct pci_dev *pdev)
938 {
939 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
940 struct cxl_memdev *cxlmd = cxlds->cxlmd;
941 struct device *dev = &cxlmd->dev;
942
943 dev_info(&pdev->dev, "%s: restart CXL.mem after slot reset\n",
944 dev_name(dev));
945 pci_restore_state(pdev);
946 if (device_attach(dev) <= 0)
947 return PCI_ERS_RESULT_DISCONNECT;
948 return PCI_ERS_RESULT_RECOVERED;
949 }
950
cxl_error_resume(struct pci_dev * pdev)951 static void cxl_error_resume(struct pci_dev *pdev)
952 {
953 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
954 struct cxl_memdev *cxlmd = cxlds->cxlmd;
955 struct device *dev = &cxlmd->dev;
956
957 dev_info(&pdev->dev, "%s: error resume %s\n", dev_name(dev),
958 dev->driver ? "successful" : "failed");
959 }
960
cxl_endpoint_decoder_clear_reset_flags(struct device * dev,void * data)961 static int cxl_endpoint_decoder_clear_reset_flags(struct device *dev, void *data)
962 {
963 struct cxl_endpoint_decoder *cxled;
964
965 if (!is_endpoint_decoder(dev))
966 return 0;
967
968 cxled = to_cxl_endpoint_decoder(dev);
969 cxled->cxld.flags &= ~CXL_DECODER_F_RESET_MASK;
970
971 return 0;
972 }
973
cxl_reset_done(struct pci_dev * pdev)974 static void cxl_reset_done(struct pci_dev *pdev)
975 {
976 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
977 struct cxl_memdev *cxlmd = cxlds->cxlmd;
978 struct device *dev = &pdev->dev;
979
980 /*
981 * FLR does not expect to touch the HDM decoders and related
982 * registers. SBR, however, will wipe all device configurations.
983 * Issue a warning if there was an active decoder before the reset
984 * that no longer exists.
985 */
986 guard(device)(&cxlmd->dev);
987 if (!cxlmd->dev.driver)
988 return;
989
990 if (cxlmd->endpoint &&
991 cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) {
992 device_for_each_child(&cxlmd->endpoint->dev, NULL,
993 cxl_endpoint_decoder_clear_reset_flags);
994
995 dev_crit(dev, "SBR happened without memory regions removal.\n");
996 dev_crit(dev, "System may be unstable if regions hosted system memory.\n");
997 add_taint(TAINT_USER, LOCKDEP_STILL_OK);
998 }
999 }
1000
1001 static const struct pci_error_handlers cxl_error_handlers = {
1002 .error_detected = cxl_error_detected,
1003 .slot_reset = cxl_slot_reset,
1004 .resume = cxl_error_resume,
1005 .cor_error_detected = cxl_cor_error_detected,
1006 .reset_done = cxl_reset_done,
1007 };
1008
1009 static struct pci_driver cxl_pci_driver = {
1010 .name = KBUILD_MODNAME,
1011 .id_table = cxl_mem_pci_tbl,
1012 .probe = cxl_pci_probe,
1013 .err_handler = &cxl_error_handlers,
1014 .dev_groups = cxl_rcd_groups,
1015 .driver = {
1016 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
1017 },
1018 };
1019
1020 #define CXL_EVENT_HDR_FLAGS_REC_SEVERITY GENMASK(1, 0)
cxl_handle_cper_event(enum cxl_event_type ev_type,struct cxl_cper_event_rec * rec)1021 static void cxl_handle_cper_event(enum cxl_event_type ev_type,
1022 struct cxl_cper_event_rec *rec)
1023 {
1024 struct cper_cxl_event_devid *device_id = &rec->hdr.device_id;
1025 struct pci_dev *pdev __free(pci_dev_put) = NULL;
1026 enum cxl_event_log_type log_type;
1027 struct cxl_dev_state *cxlds;
1028 unsigned int devfn;
1029 u32 hdr_flags;
1030
1031 pr_debug("CPER event %d for device %u:%u:%u.%u\n", ev_type,
1032 device_id->segment_num, device_id->bus_num,
1033 device_id->device_num, device_id->func_num);
1034
1035 devfn = PCI_DEVFN(device_id->device_num, device_id->func_num);
1036 pdev = pci_get_domain_bus_and_slot(device_id->segment_num,
1037 device_id->bus_num, devfn);
1038 if (!pdev)
1039 return;
1040
1041 guard(device)(&pdev->dev);
1042 if (pdev->driver != &cxl_pci_driver)
1043 return;
1044
1045 cxlds = pci_get_drvdata(pdev);
1046 if (!cxlds)
1047 return;
1048
1049 /* Fabricate a log type */
1050 hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags);
1051 log_type = FIELD_GET(CXL_EVENT_HDR_FLAGS_REC_SEVERITY, hdr_flags);
1052
1053 cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type,
1054 &uuid_null, &rec->event);
1055 }
1056
cxl_cper_work_fn(struct work_struct * work)1057 static void cxl_cper_work_fn(struct work_struct *work)
1058 {
1059 struct cxl_cper_work_data wd;
1060
1061 while (cxl_cper_kfifo_get(&wd))
1062 cxl_handle_cper_event(wd.event_type, &wd.rec);
1063 }
1064 static DECLARE_WORK(cxl_cper_work, cxl_cper_work_fn);
1065
cxl_pci_driver_init(void)1066 static int __init cxl_pci_driver_init(void)
1067 {
1068 int rc;
1069
1070 rc = pci_register_driver(&cxl_pci_driver);
1071 if (rc)
1072 return rc;
1073
1074 rc = cxl_cper_register_work(&cxl_cper_work);
1075 if (rc)
1076 pci_unregister_driver(&cxl_pci_driver);
1077
1078 return rc;
1079 }
1080
cxl_pci_driver_exit(void)1081 static void __exit cxl_pci_driver_exit(void)
1082 {
1083 cxl_cper_unregister_work(&cxl_cper_work);
1084 pci_unregister_driver(&cxl_pci_driver);
1085 }
1086
1087 module_init(cxl_pci_driver_init);
1088 module_exit(cxl_pci_driver_exit);
1089 MODULE_DESCRIPTION("CXL: PCI manageability");
1090 MODULE_LICENSE("GPL v2");
1091 MODULE_IMPORT_NS("CXL");
1092