1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2023 Advanced Micro Devices, Inc.
4 */
5
6 #define pr_fmt(fmt) "AMD-Vi: " fmt
7 #define dev_fmt(fmt) pr_fmt(fmt)
8
9 #include <linux/amd-iommu.h>
10 #include <linux/delay.h>
11 #include <linux/mmu_notifier.h>
12
13 #include <asm/iommu.h>
14
15 #include "amd_iommu.h"
16 #include "amd_iommu_types.h"
17
18 #include "../iommu-pages.h"
19
amd_iommu_alloc_ppr_log(struct amd_iommu * iommu)20 int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)
21 {
22 iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
23 amd_iommu_pprlog_size);
24 return iommu->ppr_log ? 0 : -ENOMEM;
25 }
26
amd_iommu_enable_ppr_log(struct amd_iommu * iommu)27 void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)
28 {
29 u64 entry;
30
31 if (iommu->ppr_log == NULL)
32 return;
33
34 iommu_feature_enable(iommu, CONTROL_PPR_EN);
35
36 entry = iommu_virt_to_phys(iommu->ppr_log);
37 entry |= (amd_iommu_pprlog_size == PPRLOG_SIZE_DEF) ?
38 PPRLOG_LEN_MASK_DEF : PPRLOG_LEN_MASK_MAX;
39
40 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
41 &entry, sizeof(entry));
42
43 /* set head and tail to zero manually */
44 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
45 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
46
47 iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
48 iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
49 }
50
amd_iommu_free_ppr_log(struct amd_iommu * iommu)51 void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu)
52 {
53 iommu_free_pages(iommu->ppr_log);
54 }
55
56 /*
57 * This function restarts ppr logging in case the IOMMU experienced
58 * PPR log overflow.
59 */
amd_iommu_restart_ppr_log(struct amd_iommu * iommu)60 void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)
61 {
62 amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,
63 CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK,
64 MMIO_STATUS_PPR_OVERFLOW_MASK);
65 }
66
ppr_flag_to_fault_perm(u16 flag)67 static inline u32 ppr_flag_to_fault_perm(u16 flag)
68 {
69 int perm = 0;
70
71 if (flag & PPR_FLAG_READ)
72 perm |= IOMMU_FAULT_PERM_READ;
73 if (flag & PPR_FLAG_WRITE)
74 perm |= IOMMU_FAULT_PERM_WRITE;
75 if (flag & PPR_FLAG_EXEC)
76 perm |= IOMMU_FAULT_PERM_EXEC;
77 if (!(flag & PPR_FLAG_US))
78 perm |= IOMMU_FAULT_PERM_PRIV;
79
80 return perm;
81 }
82
ppr_is_valid(struct amd_iommu * iommu,u64 * raw)83 static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw)
84 {
85 struct device *dev = iommu->iommu.dev;
86 u16 devid = PPR_DEVID(raw[0]);
87
88 if (!(PPR_FLAGS(raw[0]) & PPR_FLAG_GN)) {
89 dev_dbg(dev, "PPR logged [Request ignored due to GN=0 (device=%04x:%02x:%02x.%x "
90 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n",
91 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
92 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0]));
93 return false;
94 }
95
96 if (PPR_FLAGS(raw[0]) & PPR_FLAG_RVSD) {
97 dev_dbg(dev, "PPR logged [Invalid request format (device=%04x:%02x:%02x.%x "
98 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n",
99 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
100 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0]));
101 return false;
102 }
103
104 return true;
105 }
106
iommu_call_iopf_notifier(struct amd_iommu * iommu,u64 * raw)107 static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
108 {
109 struct iommu_dev_data *dev_data;
110 struct iopf_fault event;
111 struct pci_dev *pdev;
112 u16 devid = PPR_DEVID(raw[0]);
113
114 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
115 pr_info_ratelimited("Unknown PPR request received\n");
116 return;
117 }
118
119 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
120 PCI_BUS_NUM(devid), devid & 0xff);
121 if (!pdev)
122 return;
123
124 if (!ppr_is_valid(iommu, raw))
125 goto out;
126
127 memset(&event, 0, sizeof(struct iopf_fault));
128
129 event.fault.type = IOMMU_FAULT_PAGE_REQ;
130 event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0]));
131 event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK);
132 event.fault.prm.pasid = PPR_PASID(raw[0]);
133 event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF;
134
135 /*
136 * PASID zero is used for requests from the I/O device without
137 * a PASID
138 */
139 dev_data = dev_iommu_priv_get(&pdev->dev);
140 if (event.fault.prm.pasid == 0 ||
141 event.fault.prm.pasid >= dev_data->max_pasids) {
142 pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n",
143 event.fault.prm.pasid, pdev->dev.id);
144 goto out;
145 }
146
147 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
148 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
149 if (PPR_TAG(raw[0]) & 0x200)
150 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
151
152 /* Submit event */
153 iommu_report_device_fault(&pdev->dev, &event);
154
155 return;
156
157 out:
158 /* Nobody cared, abort */
159 amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]),
160 IOMMU_PAGE_RESP_FAILURE,
161 PPR_TAG(raw[0]) & 0x1FF);
162 }
163
amd_iommu_poll_ppr_log(struct amd_iommu * iommu)164 void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
165 {
166 u32 head, tail;
167
168 if (iommu->ppr_log == NULL)
169 return;
170
171 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
172 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
173
174 while (head != tail) {
175 volatile u64 *raw;
176 u64 entry[2];
177 int i;
178
179 raw = (u64 *)(iommu->ppr_log + head);
180
181 /*
182 * Hardware bug: Interrupt may arrive before the entry is
183 * written to memory. If this happens we need to wait for the
184 * entry to arrive.
185 */
186 for (i = 0; i < LOOP_TIMEOUT; ++i) {
187 if (PPR_REQ_TYPE(raw[0]) != 0)
188 break;
189 udelay(1);
190 }
191
192 /* Avoid memcpy function-call overhead */
193 entry[0] = raw[0];
194 entry[1] = raw[1];
195
196 /*
197 * To detect the hardware errata 733 we need to clear the
198 * entry back to zero. This issue does not exist on SNP
199 * enabled system. Also this buffer is not writeable on
200 * SNP enabled system.
201 */
202 if (!amd_iommu_snp_en)
203 raw[0] = raw[1] = 0UL;
204
205 /* Update head pointer of hardware ring-buffer */
206 head = (head + PPRLOG_ENTRY_SIZE) % amd_iommu_pprlog_size;
207 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
208
209 /* Handle PPR entry */
210 iommu_call_iopf_notifier(iommu, entry);
211 }
212 }
213
214 /**************************************************************
215 *
216 * IOPF handling stuff
217 */
218
219 /* Setup per-IOMMU IOPF queue if not exist. */
amd_iommu_iopf_init(struct amd_iommu * iommu)220 int amd_iommu_iopf_init(struct amd_iommu *iommu)
221 {
222 int ret = 0;
223
224 if (iommu->iopf_queue)
225 return ret;
226
227 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), "amdvi-%#x",
228 PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid));
229
230 iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name);
231 if (!iommu->iopf_queue)
232 ret = -ENOMEM;
233
234 return ret;
235 }
236
237 /* Destroy per-IOMMU IOPF queue if no longer needed. */
amd_iommu_iopf_uninit(struct amd_iommu * iommu)238 void amd_iommu_iopf_uninit(struct amd_iommu *iommu)
239 {
240 iopf_queue_free(iommu->iopf_queue);
241 iommu->iopf_queue = NULL;
242 }
243
amd_iommu_page_response(struct device * dev,struct iopf_fault * evt,struct iommu_page_response * resp)244 void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt,
245 struct iommu_page_response *resp)
246 {
247 amd_iommu_complete_ppr(dev, resp->pasid, resp->code, resp->grpid);
248 }
249
amd_iommu_iopf_add_device(struct amd_iommu * iommu,struct iommu_dev_data * dev_data)250 int amd_iommu_iopf_add_device(struct amd_iommu *iommu,
251 struct iommu_dev_data *dev_data)
252 {
253 int ret = 0;
254
255 if (!dev_data->pri_enabled)
256 return ret;
257
258 if (!iommu->iopf_queue)
259 return -EINVAL;
260
261 ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev);
262 if (ret)
263 return ret;
264
265 dev_data->ppr = true;
266 return 0;
267 }
268
269 /* Its assumed that caller has verified that device was added to iopf queue */
amd_iommu_iopf_remove_device(struct amd_iommu * iommu,struct iommu_dev_data * dev_data)270 void amd_iommu_iopf_remove_device(struct amd_iommu *iommu,
271 struct iommu_dev_data *dev_data)
272 {
273 iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev);
274 dev_data->ppr = false;
275 }
276