1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2025 Advanced Micro Devices, Inc.
4 */
5
6 #define dev_fmt(fmt) "AMD-Vi: " fmt
7
8 #include <linux/iommu.h>
9 #include <linux/refcount.h>
10 #include <uapi/linux/iommufd.h>
11
12 #include "amd_iommu.h"
13
14 static const struct iommu_domain_ops nested_domain_ops;
15
to_ndomain(struct iommu_domain * dom)16 static inline struct nested_domain *to_ndomain(struct iommu_domain *dom)
17 {
18 return container_of(dom, struct nested_domain, domain);
19 }
20
21 /*
22 * Validate guest DTE to make sure that configuration for host (v1)
23 * and guest (v2) page tables are valid when allocating nested domain.
24 */
validate_gdte_nested(struct iommu_hwpt_amd_guest * gdte)25 static int validate_gdte_nested(struct iommu_hwpt_amd_guest *gdte)
26 {
27 u32 gpt_level = FIELD_GET(DTE_GPT_LEVEL_MASK, gdte->dte[2]);
28
29 /* Must be zero: Mode, Host-TPR */
30 if (FIELD_GET(DTE_MODE_MASK, gdte->dte[0]) != 0 ||
31 FIELD_GET(DTE_HOST_TRP, gdte->dte[0]) != 0)
32 return -EINVAL;
33
34 /* GCR3 TRP must be non-zero if V, GV is set */
35 if (FIELD_GET(DTE_FLAG_V, gdte->dte[0]) == 1 &&
36 FIELD_GET(DTE_FLAG_GV, gdte->dte[0]) == 1 &&
37 FIELD_GET(DTE_GCR3_14_12, gdte->dte[0]) == 0 &&
38 FIELD_GET(DTE_GCR3_30_15, gdte->dte[1]) == 0 &&
39 FIELD_GET(DTE_GCR3_51_31, gdte->dte[1]) == 0)
40 return -EINVAL;
41
42 /* Valid Guest Paging Mode values are 0 and 1 */
43 if (gpt_level != GUEST_PGTABLE_4_LEVEL &&
44 gpt_level != GUEST_PGTABLE_5_LEVEL)
45 return -EINVAL;
46
47 /* GLX = 3 is reserved */
48 if (FIELD_GET(DTE_GLX, gdte->dte[0]) == 3)
49 return -EINVAL;
50
51 /*
52 * We need to check host capability before setting
53 * the Guest Paging Mode
54 */
55 if (gpt_level == GUEST_PGTABLE_5_LEVEL &&
56 amd_iommu_gpt_level < PAGE_MODE_5_LEVEL)
57 return -EOPNOTSUPP;
58
59 return 0;
60 }
61
gdom_info_load_or_alloc_locked(struct xarray * xa,unsigned long index,unsigned long * flags)62 static void *gdom_info_load_or_alloc_locked(struct xarray *xa,
63 unsigned long index,
64 unsigned long *flags)
65 {
66 struct guest_domain_mapping_info *elm, *res;
67
68 elm = xa_load(xa, index);
69 if (elm)
70 return elm;
71
72 xa_unlock_irqrestore(xa, *flags);
73 elm = kzalloc_obj(struct guest_domain_mapping_info);
74 xa_lock_irqsave(xa, *flags);
75 if (!elm)
76 return ERR_PTR(-ENOMEM);
77
78 res = __xa_cmpxchg(xa, index, NULL, elm, GFP_ATOMIC);
79 if (xa_is_err(res))
80 res = ERR_PTR(xa_err(res));
81
82 if (res) {
83 kfree(elm);
84 return res;
85 }
86
87 refcount_set(&elm->users, 0);
88 return elm;
89 }
90
91 /*
92 * This function is assigned to struct iommufd_viommu_ops.alloc_domain_nested()
93 * during the call to struct iommu_ops.viommu_init().
94 */
95 struct iommu_domain *
amd_iommu_alloc_domain_nested(struct iommufd_viommu * viommu,u32 flags,const struct iommu_user_data * user_data)96 amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
97 const struct iommu_user_data *user_data)
98 {
99 int ret;
100 unsigned long irqflags;
101 struct nested_domain *ndom;
102 struct guest_domain_mapping_info *gdom_info;
103 struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core);
104
105 if (user_data->type != IOMMU_HWPT_DATA_AMD_GUEST)
106 return ERR_PTR(-EOPNOTSUPP);
107
108 ndom = kzalloc_obj(*ndom);
109 if (!ndom)
110 return ERR_PTR(-ENOMEM);
111
112 ret = iommu_copy_struct_from_user(&ndom->gdte, user_data,
113 IOMMU_HWPT_DATA_AMD_GUEST,
114 dte);
115 if (ret)
116 goto out_err;
117
118 ret = validate_gdte_nested(&ndom->gdte);
119 if (ret)
120 goto out_err;
121
122 ndom->gdom_id = FIELD_GET(DTE_DOMID_MASK, ndom->gdte.dte[1]);
123 ndom->domain.ops = &nested_domain_ops;
124 ndom->domain.type = IOMMU_DOMAIN_NESTED;
125 ndom->viommu = aviommu;
126
127 /*
128 * Normally, when a guest has multiple pass-through devices,
129 * the IOMMU driver setup DTEs with the same stage-2 table and
130 * use the same host domain ID (hDomId). In case of nested translation,
131 * if the guest setup different stage-1 tables with same PASID,
132 * IOMMU would use the same TLB tag. This will results in TLB
133 * aliasing issue.
134 *
135 * The guest is assigning gDomIDs based on its own algorithm for managing
136 * cache tags of (DomID, PASID). Within a single viommu, the nest parent domain
137 * (w/ S2 table) is used by all DTEs. But we need to consistently map the gDomID
138 * to a single hDomID. This is done using an xarray in the vIOMMU to
139 * keep track of the gDomID mapping. When the S2 is changed, the INVALIDATE_IOMMU_PAGES
140 * command must be issued for each hDomID in the xarray.
141 */
142 xa_lock_irqsave(&aviommu->gdomid_array, irqflags);
143
144 gdom_info = gdom_info_load_or_alloc_locked(&aviommu->gdomid_array,
145 ndom->gdom_id, &irqflags);
146 if (IS_ERR(gdom_info)) {
147 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
148 ret = PTR_ERR(gdom_info);
149 goto out_err;
150 }
151
152 /* Check if gDomID exist */
153 if (refcount_inc_not_zero(&gdom_info->users)) {
154 ndom->gdom_info = gdom_info;
155 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
156
157 pr_debug("%s: Found gdom_id=%#x, hdom_id=%#x\n",
158 __func__, ndom->gdom_id, gdom_info->hdom_id);
159
160 return &ndom->domain;
161 }
162
163 /* The gDomID does not exist. We allocate new hdom_id */
164 gdom_info->hdom_id = amd_iommu_pdom_id_alloc();
165 if (gdom_info->hdom_id <= 0) {
166 __xa_cmpxchg(&aviommu->gdomid_array,
167 ndom->gdom_id, gdom_info, NULL, GFP_ATOMIC);
168 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
169 ret = -ENOSPC;
170 goto out_err_gdom_info;
171 }
172
173 ndom->gdom_info = gdom_info;
174 refcount_set(&gdom_info->users, 1);
175
176 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
177
178 pr_debug("%s: Allocate gdom_id=%#x, hdom_id=%#x\n",
179 __func__, ndom->gdom_id, gdom_info->hdom_id);
180
181 return &ndom->domain;
182
183 out_err_gdom_info:
184 kfree(gdom_info);
185 out_err:
186 kfree(ndom);
187 return ERR_PTR(ret);
188 }
189
set_dte_nested(struct amd_iommu * iommu,struct iommu_domain * dom,struct iommu_dev_data * dev_data,struct dev_table_entry * new)190 static void set_dte_nested(struct amd_iommu *iommu, struct iommu_domain *dom,
191 struct iommu_dev_data *dev_data, struct dev_table_entry *new)
192 {
193 struct protection_domain *parent;
194 struct nested_domain *ndom = to_ndomain(dom);
195 struct iommu_hwpt_amd_guest *gdte = &ndom->gdte;
196 struct pt_iommu_amdv1_hw_info pt_info;
197
198 /*
199 * The nest parent domain is attached during the call to the
200 * struct iommu_ops.viommu_init(), which will be stored as part
201 * of the struct amd_iommu_viommu.parent.
202 */
203 if (WARN_ON(!ndom->viommu || !ndom->viommu->parent))
204 return;
205
206 parent = ndom->viommu->parent;
207 amd_iommu_make_clear_dte(dev_data, new);
208
209 /* Retrieve the current pagetable info via the IOMMU PT API. */
210 pt_iommu_amdv1_hw_info(&parent->amdv1, &pt_info);
211
212 /*
213 * Use domain ID from nested domain to program DTE.
214 * See amd_iommu_alloc_domain_nested().
215 */
216 amd_iommu_set_dte_v1(dev_data, parent, ndom->gdom_info->hdom_id,
217 &pt_info, new);
218
219 /* GV is required for nested page table */
220 new->data[0] |= DTE_FLAG_GV;
221
222 /* Guest PPR */
223 new->data[0] |= gdte->dte[0] & DTE_FLAG_PPR;
224
225 /* Guest translation stuff */
226 new->data[0] |= gdte->dte[0] & (DTE_GLX | DTE_FLAG_GIOV);
227
228 /* GCR3 table */
229 new->data[0] |= gdte->dte[0] & DTE_GCR3_14_12;
230 new->data[1] |= gdte->dte[1] & (DTE_GCR3_30_15 | DTE_GCR3_51_31);
231
232 /* Guest paging mode */
233 new->data[2] |= gdte->dte[2] & DTE_GPT_LEVEL_MASK;
234 }
235
nested_attach_device(struct iommu_domain * dom,struct device * dev,struct iommu_domain * old)236 static int nested_attach_device(struct iommu_domain *dom, struct device *dev,
237 struct iommu_domain *old)
238 {
239 struct dev_table_entry new = {0};
240 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
241 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
242 int ret = 0;
243
244 /*
245 * Needs to make sure PASID is not enabled
246 * for this attach path.
247 */
248 if (WARN_ON(dev_data->pasid_enabled))
249 return -EINVAL;
250
251 mutex_lock(&dev_data->mutex);
252
253 set_dte_nested(iommu, dom, dev_data, &new);
254
255 amd_iommu_update_dte(iommu, dev_data, &new);
256
257 mutex_unlock(&dev_data->mutex);
258
259 return ret;
260 }
261
nested_domain_free(struct iommu_domain * dom)262 static void nested_domain_free(struct iommu_domain *dom)
263 {
264 unsigned long irqflags;
265 struct guest_domain_mapping_info *curr;
266 struct nested_domain *ndom __free(kfree) = to_ndomain(dom);
267 struct amd_iommu_viommu *aviommu = ndom->viommu;
268
269 xa_lock_irqsave(&aviommu->gdomid_array, irqflags);
270
271 if (!refcount_dec_and_test(&ndom->gdom_info->users)) {
272 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
273 return;
274 }
275
276 /*
277 * The refcount for the gdom_id to hdom_id mapping is zero.
278 * It is now safe to remove the mapping.
279 */
280 curr = __xa_cmpxchg(&aviommu->gdomid_array, ndom->gdom_id,
281 ndom->gdom_info, NULL, GFP_ATOMIC);
282
283 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags);
284 if (WARN_ON(!curr || xa_err(curr)))
285 return;
286
287 /* success */
288 pr_debug("%s: Free gdom_id=%#x, hdom_id=%#x\n",
289 __func__, ndom->gdom_id, curr->hdom_id);
290
291 amd_iommu_pdom_id_free(ndom->gdom_info->hdom_id);
292 kfree(curr);
293 }
294
295 static const struct iommu_domain_ops nested_domain_ops = {
296 .attach_dev = nested_attach_device,
297 .free = nested_domain_free,
298 };
299