1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2025 Advanced Micro Devices, Inc. 4 */ 5 6 #define dev_fmt(fmt) "AMD-Vi: " fmt 7 8 #include <linux/iommu.h> 9 #include <linux/refcount.h> 10 #include <uapi/linux/iommufd.h> 11 12 #include "amd_iommu.h" 13 14 static const struct iommu_domain_ops nested_domain_ops; 15 16 static inline struct nested_domain *to_ndomain(struct iommu_domain *dom) 17 { 18 return container_of(dom, struct nested_domain, domain); 19 } 20 21 /* 22 * Validate guest DTE to make sure that configuration for host (v1) 23 * and guest (v2) page tables are valid when allocating nested domain. 24 */ 25 static int validate_gdte_nested(struct iommu_hwpt_amd_guest *gdte) 26 { 27 u32 gpt_level = FIELD_GET(DTE_GPT_LEVEL_MASK, gdte->dte[2]); 28 29 /* Must be zero: Mode, Host-TPR */ 30 if (FIELD_GET(DTE_MODE_MASK, gdte->dte[0]) != 0 || 31 FIELD_GET(DTE_HOST_TRP, gdte->dte[0]) != 0) 32 return -EINVAL; 33 34 /* GCR3 TRP must be non-zero if V, GV is set */ 35 if (FIELD_GET(DTE_FLAG_V, gdte->dte[0]) == 1 && 36 FIELD_GET(DTE_FLAG_GV, gdte->dte[0]) == 1 && 37 FIELD_GET(DTE_GCR3_14_12, gdte->dte[0]) == 0 && 38 FIELD_GET(DTE_GCR3_30_15, gdte->dte[1]) == 0 && 39 FIELD_GET(DTE_GCR3_51_31, gdte->dte[1]) == 0) 40 return -EINVAL; 41 42 /* Valid Guest Paging Mode values are 0 and 1 */ 43 if (gpt_level != GUEST_PGTABLE_4_LEVEL && 44 gpt_level != GUEST_PGTABLE_5_LEVEL) 45 return -EINVAL; 46 47 /* GLX = 3 is reserved */ 48 if (FIELD_GET(DTE_GLX, gdte->dte[0]) == 3) 49 return -EINVAL; 50 51 /* 52 * We need to check host capability before setting 53 * the Guest Paging Mode 54 */ 55 if (gpt_level == GUEST_PGTABLE_5_LEVEL && 56 amd_iommu_gpt_level < PAGE_MODE_5_LEVEL) 57 return -EOPNOTSUPP; 58 59 return 0; 60 } 61 62 static void *gdom_info_load_or_alloc_locked(struct xarray *xa, 63 unsigned long index, 64 unsigned long *flags) 65 { 66 struct guest_domain_mapping_info *elm, *res; 67 68 elm = xa_load(xa, index); 69 if (elm) 70 return elm; 71 72 xa_unlock_irqrestore(xa, *flags); 73 elm = kzalloc_obj(struct guest_domain_mapping_info); 74 xa_lock_irqsave(xa, *flags); 75 if (!elm) 76 return ERR_PTR(-ENOMEM); 77 78 res = __xa_cmpxchg(xa, index, NULL, elm, GFP_ATOMIC); 79 if (xa_is_err(res)) 80 res = ERR_PTR(xa_err(res)); 81 82 if (res) { 83 kfree(elm); 84 return res; 85 } 86 87 refcount_set(&elm->users, 0); 88 return elm; 89 } 90 91 /* 92 * This function is assigned to struct iommufd_viommu_ops.alloc_domain_nested() 93 * during the call to struct iommu_ops.viommu_init(). 94 */ 95 struct iommu_domain * 96 amd_iommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags, 97 const struct iommu_user_data *user_data) 98 { 99 int ret, hdom_id; 100 unsigned long irqflags; 101 struct nested_domain *ndom; 102 struct guest_domain_mapping_info *gdom_info; 103 struct amd_iommu_viommu *aviommu = container_of(viommu, struct amd_iommu_viommu, core); 104 105 if (user_data->type != IOMMU_HWPT_DATA_AMD_GUEST) 106 return ERR_PTR(-EOPNOTSUPP); 107 108 ndom = kzalloc_obj(*ndom); 109 if (!ndom) 110 return ERR_PTR(-ENOMEM); 111 112 ret = iommu_copy_struct_from_user(&ndom->gdte, user_data, 113 IOMMU_HWPT_DATA_AMD_GUEST, 114 dte); 115 if (ret) 116 goto out_err; 117 118 ret = validate_gdte_nested(&ndom->gdte); 119 if (ret) 120 goto out_err; 121 122 ndom->gdom_id = FIELD_GET(DTE_DOMID_MASK, ndom->gdte.dte[1]); 123 ndom->domain.ops = &nested_domain_ops; 124 ndom->domain.type = IOMMU_DOMAIN_NESTED; 125 ndom->viommu = aviommu; 126 127 /* 128 * Normally, when a guest has multiple pass-through devices, 129 * the IOMMU driver setup DTEs with the same stage-2 table and 130 * use the same host domain ID (hDomId). In case of nested translation, 131 * if the guest setup different stage-1 tables with same PASID, 132 * IOMMU would use the same TLB tag. This will results in TLB 133 * aliasing issue. 134 * 135 * The guest is assigning gDomIDs based on its own algorithm for managing 136 * cache tags of (DomID, PASID). Within a single viommu, the nest parent domain 137 * (w/ S2 table) is used by all DTEs. But we need to consistently map the gDomID 138 * to a single hDomID. This is done using an xarray in the vIOMMU to 139 * keep track of the gDomID mapping. When the S2 is changed, the INVALIDATE_IOMMU_PAGES 140 * command must be issued for each hDomID in the xarray. 141 */ 142 xa_lock_irqsave(&aviommu->gdomid_array, irqflags); 143 144 gdom_info = gdom_info_load_or_alloc_locked(&aviommu->gdomid_array, 145 ndom->gdom_id, &irqflags); 146 if (IS_ERR(gdom_info)) { 147 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 148 ret = PTR_ERR(gdom_info); 149 goto out_err; 150 } 151 152 /* Check if gDomID exist */ 153 if (refcount_inc_not_zero(&gdom_info->users)) { 154 ndom->gdom_info = gdom_info; 155 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 156 157 pr_debug("%s: Found gdom_id=%#x, hdom_id=%#x\n", 158 __func__, ndom->gdom_id, gdom_info->hdom_id); 159 160 return &ndom->domain; 161 } 162 163 /* The gDomID does not exist. We allocate new hdom_id */ 164 hdom_id = amd_iommu_pdom_id_alloc(); 165 if (hdom_id <= 0) { 166 __xa_cmpxchg(&aviommu->gdomid_array, 167 ndom->gdom_id, gdom_info, NULL, GFP_ATOMIC); 168 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 169 ret = -ENOSPC; 170 goto out_err_gdom_info; 171 } 172 173 gdom_info->hdom_id = hdom_id; 174 ndom->gdom_info = gdom_info; 175 refcount_set(&gdom_info->users, 1); 176 177 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 178 179 pr_debug("%s: Allocate gdom_id=%#x, hdom_id=%#x\n", 180 __func__, ndom->gdom_id, gdom_info->hdom_id); 181 182 return &ndom->domain; 183 184 out_err_gdom_info: 185 kfree(gdom_info); 186 out_err: 187 kfree(ndom); 188 return ERR_PTR(ret); 189 } 190 191 static void set_dte_nested(struct amd_iommu *iommu, struct iommu_domain *dom, 192 struct iommu_dev_data *dev_data, struct dev_table_entry *new) 193 { 194 struct protection_domain *parent; 195 struct nested_domain *ndom = to_ndomain(dom); 196 struct iommu_hwpt_amd_guest *gdte = &ndom->gdte; 197 struct pt_iommu_amdv1_hw_info pt_info; 198 199 /* 200 * The nest parent domain is attached during the call to the 201 * struct iommu_ops.viommu_init(), which will be stored as part 202 * of the struct amd_iommu_viommu.parent. 203 */ 204 if (WARN_ON(!ndom->viommu || !ndom->viommu->parent)) 205 return; 206 207 parent = ndom->viommu->parent; 208 amd_iommu_make_clear_dte(dev_data, new); 209 210 /* Retrieve the current pagetable info via the IOMMU PT API. */ 211 pt_iommu_amdv1_hw_info(&parent->amdv1, &pt_info); 212 213 /* 214 * Use domain ID from nested domain to program DTE. 215 * See amd_iommu_alloc_domain_nested(). 216 */ 217 amd_iommu_set_dte_v1(dev_data, parent, ndom->gdom_info->hdom_id, 218 &pt_info, new); 219 220 /* GV is required for nested page table */ 221 new->data[0] |= DTE_FLAG_GV; 222 223 /* Guest PPR */ 224 new->data[0] |= gdte->dte[0] & DTE_FLAG_PPR; 225 226 /* Guest translation stuff */ 227 new->data[0] |= gdte->dte[0] & (DTE_GLX | DTE_FLAG_GIOV); 228 229 /* GCR3 table */ 230 new->data[0] |= gdte->dte[0] & DTE_GCR3_14_12; 231 new->data[1] |= gdte->dte[1] & (DTE_GCR3_30_15 | DTE_GCR3_51_31); 232 233 /* Guest paging mode */ 234 new->data[2] |= gdte->dte[2] & DTE_GPT_LEVEL_MASK; 235 } 236 237 static int nested_attach_device(struct iommu_domain *dom, struct device *dev, 238 struct iommu_domain *old) 239 { 240 struct dev_table_entry new = {0}; 241 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 242 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 243 int ret = 0; 244 245 /* 246 * Needs to make sure PASID is not enabled 247 * for this attach path. 248 */ 249 if (WARN_ON(dev_data->pasid_enabled)) 250 return -EINVAL; 251 252 mutex_lock(&dev_data->mutex); 253 254 set_dte_nested(iommu, dom, dev_data, &new); 255 256 amd_iommu_update_dte(iommu, dev_data, &new); 257 258 mutex_unlock(&dev_data->mutex); 259 260 return ret; 261 } 262 263 static void nested_domain_free(struct iommu_domain *dom) 264 { 265 unsigned long irqflags; 266 struct guest_domain_mapping_info *curr; 267 struct nested_domain *ndom __free(kfree) = to_ndomain(dom); 268 struct amd_iommu_viommu *aviommu = ndom->viommu; 269 270 xa_lock_irqsave(&aviommu->gdomid_array, irqflags); 271 272 if (!refcount_dec_and_test(&ndom->gdom_info->users)) { 273 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 274 return; 275 } 276 277 /* 278 * The refcount for the gdom_id to hdom_id mapping is zero. 279 * It is now safe to remove the mapping. 280 */ 281 curr = __xa_cmpxchg(&aviommu->gdomid_array, ndom->gdom_id, 282 ndom->gdom_info, NULL, GFP_ATOMIC); 283 284 xa_unlock_irqrestore(&aviommu->gdomid_array, irqflags); 285 if (WARN_ON(!curr || xa_err(curr))) 286 return; 287 288 /* success */ 289 pr_debug("%s: Free gdom_id=%#x, hdom_id=%#x\n", 290 __func__, ndom->gdom_id, curr->hdom_id); 291 292 amd_iommu_pdom_id_free(ndom->gdom_info->hdom_id); 293 kfree(curr); 294 } 295 296 static const struct iommu_domain_ops nested_domain_ops = { 297 .attach_dev = nested_attach_device, 298 .free = nested_domain_free, 299 }; 300