Lines Matching +full:touch +full:- +full:hold +full:- +full:ms
1 // SPDX-License-Identifier: GPL-2.0-only
4 #include <linux/io-64-nonatomic-lo-hi.h>
31 * - Create the memX device and register on the CXL bus.
32 * - Enumerate device's register interface and map them.
33 * - Registers nvdimm bridge device with cxl_core.
34 * - Registers a CXL mailbox with cxl_core.
38 (readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \
41 /* CXL 2.0 - 8.2.8.4 */
69 return -ETIMEDOUT; in cxl_pci_mbox_wait_for_doorbell()
74 dev_dbg(cxlds->dev, "Doorbell wait took %dms", in cxl_pci_mbox_wait_for_doorbell()
75 jiffies_to_msecs(end) - jiffies_to_msecs(start)); in cxl_pci_mbox_wait_for_doorbell()
82 status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
86 (cmd)->opcode, \
88 status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
101 struct device *dev = cxlds->dev; in cxl_request_irq()
106 return -ENOMEM; in cxl_request_irq()
107 dev_id->cxlds = cxlds; in cxl_request_irq()
118 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET); in cxl_mbox_background_complete()
127 struct cxl_dev_state *cxlds = dev_id->cxlds; in cxl_pci_mbox_irq()
128 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; in cxl_pci_mbox_irq()
134 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET); in cxl_pci_mbox_irq()
137 mutex_lock(&cxl_mbox->mbox_mutex); in cxl_pci_mbox_irq()
138 if (mds->security.sanitize_node) in cxl_pci_mbox_irq()
139 mod_delayed_work(system_wq, &mds->security.poll_dwork, 0); in cxl_pci_mbox_irq()
140 mutex_unlock(&cxl_mbox->mbox_mutex); in cxl_pci_mbox_irq()
142 /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */ in cxl_pci_mbox_irq()
143 rcuwait_wake_up(&cxl_mbox->mbox_wait); in cxl_pci_mbox_irq()
156 struct cxl_dev_state *cxlds = &mds->cxlds; in cxl_mbox_sanitize_work()
157 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; in cxl_mbox_sanitize_work()
159 mutex_lock(&cxl_mbox->mbox_mutex); in cxl_mbox_sanitize_work()
161 mds->security.poll_tmo_secs = 0; in cxl_mbox_sanitize_work()
162 if (mds->security.sanitize_node) in cxl_mbox_sanitize_work()
163 sysfs_notify_dirent(mds->security.sanitize_node); in cxl_mbox_sanitize_work()
164 mds->security.sanitize_active = false; in cxl_mbox_sanitize_work()
166 dev_dbg(cxlds->dev, "Sanitization operation ended\n"); in cxl_mbox_sanitize_work()
168 int timeout = mds->security.poll_tmo_secs + 10; in cxl_mbox_sanitize_work()
170 mds->security.poll_tmo_secs = min(15 * 60, timeout); in cxl_mbox_sanitize_work()
171 schedule_delayed_work(&mds->security.poll_dwork, timeout * HZ); in cxl_mbox_sanitize_work()
173 mutex_unlock(&cxl_mbox->mbox_mutex); in cxl_mbox_sanitize_work()
177 * __cxl_pci_mbox_send_cmd() - Execute a mailbox command
182 * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
187 * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
203 void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; in __cxl_pci_mbox_send_cmd()
204 struct device *dev = cxlds->dev; in __cxl_pci_mbox_send_cmd()
209 lockdep_assert_held(&cxl_mbox->mbox_mutex); in __cxl_pci_mbox_send_cmd()
215 * 3. Caller writes Command Payload Registers if input payload is non-empty in __cxl_pci_mbox_send_cmd()
220 * 8. If output payload is non-empty, host reads Command Payload Registers in __cxl_pci_mbox_send_cmd()
231 readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); in __cxl_pci_mbox_send_cmd()
233 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, in __cxl_pci_mbox_send_cmd()
235 return -EBUSY; in __cxl_pci_mbox_send_cmd()
243 if (mds->security.poll_tmo_secs > 0) { in __cxl_pci_mbox_send_cmd()
244 if (mbox_cmd->opcode != CXL_MBOX_OP_GET_HEALTH_INFO) in __cxl_pci_mbox_send_cmd()
245 return -EBUSY; in __cxl_pci_mbox_send_cmd()
249 mbox_cmd->opcode); in __cxl_pci_mbox_send_cmd()
250 if (mbox_cmd->size_in) { in __cxl_pci_mbox_send_cmd()
251 if (WARN_ON(!mbox_cmd->payload_in)) in __cxl_pci_mbox_send_cmd()
252 return -EINVAL; in __cxl_pci_mbox_send_cmd()
255 mbox_cmd->size_in); in __cxl_pci_mbox_send_cmd()
256 memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); in __cxl_pci_mbox_send_cmd()
260 writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); in __cxl_pci_mbox_send_cmd()
263 dev_dbg(dev, "Sending command: 0x%04x\n", mbox_cmd->opcode); in __cxl_pci_mbox_send_cmd()
265 cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); in __cxl_pci_mbox_send_cmd()
269 if (rc == -ETIMEDOUT) { in __cxl_pci_mbox_send_cmd()
270 u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); in __cxl_pci_mbox_send_cmd()
272 cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout"); in __cxl_pci_mbox_send_cmd()
277 status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET); in __cxl_pci_mbox_send_cmd()
278 mbox_cmd->return_code = in __cxl_pci_mbox_send_cmd()
285 * which we currently hold. Furthermore this also guarantees that in __cxl_pci_mbox_send_cmd()
294 if (mbox_cmd->return_code == CXL_MBOX_CMD_RC_BACKGROUND) { in __cxl_pci_mbox_send_cmd()
303 if (mbox_cmd->opcode == CXL_MBOX_OP_SANITIZE) { in __cxl_pci_mbox_send_cmd()
304 if (mds->security.sanitize_active) in __cxl_pci_mbox_send_cmd()
305 return -EBUSY; in __cxl_pci_mbox_send_cmd()
309 mds->security.poll_tmo_secs = timeout; in __cxl_pci_mbox_send_cmd()
310 mds->security.sanitize_active = true; in __cxl_pci_mbox_send_cmd()
311 schedule_delayed_work(&mds->security.poll_dwork, in __cxl_pci_mbox_send_cmd()
318 mbox_cmd->opcode); in __cxl_pci_mbox_send_cmd()
320 timeout = mbox_cmd->poll_interval_ms; in __cxl_pci_mbox_send_cmd()
321 for (i = 0; i < mbox_cmd->poll_count; i++) { in __cxl_pci_mbox_send_cmd()
322 if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait, in __cxl_pci_mbox_send_cmd()
330 dev_err(dev, "timeout waiting for background (%d ms)\n", in __cxl_pci_mbox_send_cmd()
331 timeout * mbox_cmd->poll_count); in __cxl_pci_mbox_send_cmd()
332 return -ETIMEDOUT; in __cxl_pci_mbox_send_cmd()
335 bg_status_reg = readq(cxlds->regs.mbox + in __cxl_pci_mbox_send_cmd()
337 mbox_cmd->return_code = in __cxl_pci_mbox_send_cmd()
342 mbox_cmd->opcode); in __cxl_pci_mbox_send_cmd()
345 if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) { in __cxl_pci_mbox_send_cmd()
353 cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET); in __cxl_pci_mbox_send_cmd()
357 if (out_len && mbox_cmd->payload_out) { in __cxl_pci_mbox_send_cmd()
367 n = min3(mbox_cmd->size_out, cxl_mbox->payload_size, out_len); in __cxl_pci_mbox_send_cmd()
368 memcpy_fromio(mbox_cmd->payload_out, payload, n); in __cxl_pci_mbox_send_cmd()
369 mbox_cmd->size_out = n; in __cxl_pci_mbox_send_cmd()
371 mbox_cmd->size_out = 0; in __cxl_pci_mbox_send_cmd()
382 mutex_lock_io(&cxl_mbox->mbox_mutex); in cxl_pci_mbox_send()
384 mutex_unlock(&cxl_mbox->mbox_mutex); in cxl_pci_mbox_send()
391 struct cxl_dev_state *cxlds = &mds->cxlds; in cxl_pci_setup_mailbox()
392 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; in cxl_pci_setup_mailbox()
393 const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); in cxl_pci_setup_mailbox()
394 struct device *dev = cxlds->dev; in cxl_pci_setup_mailbox()
402 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); in cxl_pci_setup_mailbox()
411 return -ETIMEDOUT; in cxl_pci_setup_mailbox()
422 return -ETIMEDOUT; in cxl_pci_setup_mailbox()
425 cxl_mbox->mbox_send = cxl_pci_mbox_send; in cxl_pci_setup_mailbox()
426 cxl_mbox->payload_size = in cxl_pci_setup_mailbox()
436 cxl_mbox->payload_size = min_t(size_t, cxl_mbox->payload_size, SZ_1M); in cxl_pci_setup_mailbox()
437 if (cxl_mbox->payload_size < 256) { in cxl_pci_setup_mailbox()
439 cxl_mbox->payload_size); in cxl_pci_setup_mailbox()
440 return -ENXIO; in cxl_pci_setup_mailbox()
443 dev_dbg(dev, "Mailbox payload sized %zu", cxl_mbox->payload_size); in cxl_pci_setup_mailbox()
445 INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work); in cxl_pci_setup_mailbox()
452 irq = pci_irq_vector(to_pci_dev(cxlds->dev), msgnum); in cxl_pci_setup_mailbox()
459 dev_dbg(cxlds->dev, "Mailbox interrupts enabled\n"); in cxl_pci_setup_mailbox()
461 ctrl = readl(cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); in cxl_pci_setup_mailbox()
463 writel(ctrl, cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET); in cxl_pci_setup_mailbox()
484 .host = &pdev->dev, in cxl_rcrb_get_comp_regs()
491 return -EPROBE_DEFER; in cxl_rcrb_get_comp_regs()
493 component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport); in cxl_rcrb_get_comp_regs()
495 return -ENXIO; in cxl_rcrb_get_comp_regs()
497 map->resource = component_reg_phys; in cxl_rcrb_get_comp_regs()
498 map->reg_type = CXL_REGLOC_RBI_COMPONENT; in cxl_rcrb_get_comp_regs()
499 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE; in cxl_rcrb_get_comp_regs()
521 return -EPROBE_DEFER; in cxl_pci_setup_regs()
546 if (!cxlds->regs.ras) { in cxl_pci_ras_unmask()
547 dev_dbg(&pdev->dev, "No RAS registers.\n"); in cxl_pci_ras_unmask()
560 addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_MASK_OFFSET; in cxl_pci_ras_unmask()
567 dev_dbg(&pdev->dev, in cxl_pci_ras_unmask()
568 "Uncorrectable RAS Errors Mask: %#x -> %#x\n", in cxl_pci_ras_unmask()
573 addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_MASK_OFFSET; in cxl_pci_ras_unmask()
577 dev_dbg(&pdev->dev, "Correctable RAS Errors Mask: %#x -> %#x\n", in cxl_pci_ras_unmask()
591 * share this buffer protected by the mds->event_log_lock.
595 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; in cxl_mem_alloc_event_buf()
598 buf = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL); in cxl_mem_alloc_event_buf()
600 return -ENOMEM; in cxl_mem_alloc_event_buf()
601 mds->event.buf = buf; in cxl_mem_alloc_event_buf()
603 return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf); in cxl_mem_alloc_event_buf()
622 dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs); in cxl_alloc_irq_vectors()
631 struct cxl_dev_state *cxlds = dev_id->cxlds; in cxl_event_thread()
640 status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET); in cxl_event_thread()
654 struct pci_dev *pdev = to_pci_dev(cxlds->dev); in cxl_event_req_irq()
658 return -ENXIO; in cxl_event_req_irq()
671 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; in cxl_event_get_int_policy()
681 dev_err(mds->cxlds.dev, in cxl_event_get_int_policy()
690 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; in cxl_event_config_msgnums()
709 dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", in cxl_event_config_msgnums()
720 struct cxl_dev_state *cxlds = &mds->cxlds; in cxl_event_irqsetup()
730 dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n"); in cxl_event_irqsetup()
736 dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n"); in cxl_event_irqsetup()
742 dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n"); in cxl_event_irqsetup()
748 dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n"); in cxl_event_irqsetup()
772 if (!host_bridge->native_cxl_error) in cxl_event_config()
776 dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n"); in cxl_event_config()
788 dev_err(mds->cxlds.dev, in cxl_event_config()
790 return -EBUSY; in cxl_event_config()
813 if (!cxlds->reg_map.device_map.mbox.valid) in cxl_pci_type3_init_mailbox()
814 return -ENODEV; in cxl_pci_type3_init_mailbox()
816 rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); in cxl_pci_type3_init_mailbox()
826 struct cxl_memdev *cxlmd = cxlds->cxlmd; in rcd_pcie_cap_emit()
833 return -ENXIO; in rcd_pcie_cap_emit()
835 root_dev = root->uport_dev; in rcd_pcie_cap_emit()
837 return -ENXIO; in rcd_pcie_cap_emit()
839 if (!dport->regs.rcd_pcie_cap) in rcd_pcie_cap_emit()
840 return -ENXIO; in rcd_pcie_cap_emit()
843 if (!root_dev->driver) in rcd_pcie_cap_emit()
844 return -ENXIO; in rcd_pcie_cap_emit()
849 readw(dport->regs.rcd_pcie_cap + offset)); in rcd_pcie_cap_emit()
852 readl(dport->regs.rcd_pcie_cap + offset)); in rcd_pcie_cap_emit()
854 return -EINVAL; in rcd_pcie_cap_emit()
892 return a->mode; in cxl_rcd_visible()
905 struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); in cxl_pci_probe()
925 mds = cxl_memdev_state_create(&pdev->dev); in cxl_pci_probe()
928 cxlds = &mds->cxlds; in cxl_pci_probe()
931 cxlds->rcd = is_cxl_restricted(pdev); in cxl_pci_probe()
932 cxlds->serial = pci_get_dsn(pdev); in cxl_pci_probe()
933 cxlds->cxl_dvsec = pci_find_dvsec_capability( in cxl_pci_probe()
935 if (!cxlds->cxl_dvsec) in cxl_pci_probe()
936 dev_warn(&pdev->dev, in cxl_pci_probe()
943 rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs); in cxl_pci_probe()
952 &cxlds->reg_map); in cxl_pci_probe()
954 dev_warn(&pdev->dev, "No component registers (%d)\n", rc); in cxl_pci_probe()
955 else if (!cxlds->reg_map.component_map.ras.valid) in cxl_pci_probe()
956 dev_dbg(&pdev->dev, "RAS registers not found\n"); in cxl_pci_probe()
958 rc = cxl_map_component_regs(&cxlds->reg_map, &cxlds->regs.component, in cxl_pci_probe()
961 dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); in cxl_pci_probe()
969 cxlds->media_ready = true; in cxl_pci_probe()
971 dev_warn(&pdev->dev, "Media not active (%d)\n", rc); in cxl_pci_probe()
999 cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlds); in cxl_pci_probe()
1003 rc = devm_cxl_setup_fw_upload(&pdev->dev, mds); in cxl_pci_probe()
1007 rc = devm_cxl_sanitize_setup_notifier(&pdev->dev, cxlmd); in cxl_pci_probe()
1017 dev_dbg(&pdev->dev, "Could not find PMU regblock\n"); in cxl_pci_probe()
1023 dev_dbg(&pdev->dev, "Could not map PMU regs\n"); in cxl_pci_probe()
1027 rc = devm_cxl_pmu_add(cxlds->dev, &pmu_regs, cxlmd->id, i, CXL_PMU_MEMDEV); in cxl_pci_probe()
1029 dev_dbg(&pdev->dev, "Could not add PMU instance\n"); in cxl_pci_probe()
1039 dev_dbg(&pdev->dev, "No RAS reporting unmasked\n"); in cxl_pci_probe()
1047 /* PCI class code for CXL.mem Type-3 Devices */
1056 struct cxl_memdev *cxlmd = cxlds->cxlmd; in cxl_slot_reset()
1057 struct device *dev = &cxlmd->dev; in cxl_slot_reset()
1059 dev_info(&pdev->dev, "%s: restart CXL.mem after slot reset\n", in cxl_slot_reset()
1070 struct cxl_memdev *cxlmd = cxlds->cxlmd; in cxl_error_resume()
1071 struct device *dev = &cxlmd->dev; in cxl_error_resume()
1073 dev_info(&pdev->dev, "%s: error resume %s\n", dev_name(dev), in cxl_error_resume()
1074 dev->driver ? "successful" : "failed"); in cxl_error_resume()
1080 struct cxl_memdev *cxlmd = cxlds->cxlmd; in cxl_reset_done()
1081 struct device *dev = &pdev->dev; in cxl_reset_done()
1084 * FLR does not expect to touch the HDM decoders and related in cxl_reset_done()
1089 guard(device)(&cxlmd->dev); in cxl_reset_done()
1090 if (cxlmd->endpoint && in cxl_reset_done()
1091 cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) { in cxl_reset_done()
1121 struct cper_cxl_event_devid *device_id = &rec->hdr.device_id; in cxl_handle_cper_event()
1129 device_id->segment_num, device_id->bus_num, in cxl_handle_cper_event()
1130 device_id->device_num, device_id->func_num); in cxl_handle_cper_event()
1132 devfn = PCI_DEVFN(device_id->device_num, device_id->func_num); in cxl_handle_cper_event()
1133 pdev = pci_get_domain_bus_and_slot(device_id->segment_num, in cxl_handle_cper_event()
1134 device_id->bus_num, devfn); in cxl_handle_cper_event()
1138 guard(device)(&pdev->dev); in cxl_handle_cper_event()
1139 if (pdev->driver != &cxl_pci_driver) in cxl_handle_cper_event()
1147 hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags); in cxl_handle_cper_event()
1150 cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type, in cxl_handle_cper_event()
1151 &uuid_null, &rec->event); in cxl_handle_cper_event()