1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU-agnostic AMD IO page table v2 allocator. 4 * 5 * Copyright (C) 2022, 2023 Advanced Micro Devices, Inc. 6 * Author: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> 7 * Author: Vasant Hegde <vasant.hegde@amd.com> 8 */ 9 10 #define pr_fmt(fmt) "AMD-Vi: " fmt 11 #define dev_fmt(fmt) pr_fmt(fmt) 12 13 #include <linux/bitops.h> 14 #include <linux/io-pgtable.h> 15 #include <linux/kernel.h> 16 17 #include <asm/barrier.h> 18 19 #include "amd_iommu_types.h" 20 #include "amd_iommu.h" 21 #include "../iommu-pages.h" 22 23 #define IOMMU_PAGE_PRESENT BIT_ULL(0) /* Is present */ 24 #define IOMMU_PAGE_RW BIT_ULL(1) /* Writeable */ 25 #define IOMMU_PAGE_USER BIT_ULL(2) /* Userspace addressable */ 26 #define IOMMU_PAGE_PWT BIT_ULL(3) /* Page write through */ 27 #define IOMMU_PAGE_PCD BIT_ULL(4) /* Page cache disabled */ 28 #define IOMMU_PAGE_ACCESS BIT_ULL(5) /* Was accessed (updated by IOMMU) */ 29 #define IOMMU_PAGE_DIRTY BIT_ULL(6) /* Was written to (updated by IOMMU) */ 30 #define IOMMU_PAGE_PSE BIT_ULL(7) /* Page Size Extensions */ 31 #define IOMMU_PAGE_NX BIT_ULL(63) /* No execute */ 32 33 #define MAX_PTRS_PER_PAGE 512 34 35 #define IOMMU_PAGE_SIZE_2M BIT_ULL(21) 36 #define IOMMU_PAGE_SIZE_1G BIT_ULL(30) 37 38 39 static inline int get_pgtable_level(void) 40 { 41 return amd_iommu_gpt_level; 42 } 43 44 static inline bool is_large_pte(u64 pte) 45 { 46 return (pte & IOMMU_PAGE_PSE); 47 } 48 49 static inline u64 set_pgtable_attr(u64 *page) 50 { 51 u64 prot; 52 53 prot = IOMMU_PAGE_PRESENT | IOMMU_PAGE_RW | IOMMU_PAGE_USER; 54 prot |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; 55 56 return (iommu_virt_to_phys(page) | prot); 57 } 58 59 static inline void *get_pgtable_pte(u64 pte) 60 { 61 return iommu_phys_to_virt(pte & PM_ADDR_MASK); 62 } 63 64 static u64 set_pte_attr(u64 paddr, u64 pg_size, int prot) 65 { 66 u64 pte; 67 68 pte = __sme_set(paddr & PM_ADDR_MASK); 69 pte |= IOMMU_PAGE_PRESENT | IOMMU_PAGE_USER; 70 pte |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; 71 72 if (prot & IOMMU_PROT_IW) 73 pte |= IOMMU_PAGE_RW; 74 75 /* Large page */ 76 if (pg_size == IOMMU_PAGE_SIZE_1G || pg_size == IOMMU_PAGE_SIZE_2M) 77 pte |= IOMMU_PAGE_PSE; 78 79 return pte; 80 } 81 82 static inline u64 get_alloc_page_size(u64 size) 83 { 84 if (size >= IOMMU_PAGE_SIZE_1G) 85 return IOMMU_PAGE_SIZE_1G; 86 87 if (size >= IOMMU_PAGE_SIZE_2M) 88 return IOMMU_PAGE_SIZE_2M; 89 90 return PAGE_SIZE; 91 } 92 93 static inline int page_size_to_level(u64 pg_size) 94 { 95 if (pg_size == IOMMU_PAGE_SIZE_1G) 96 return PAGE_MODE_3_LEVEL; 97 if (pg_size == IOMMU_PAGE_SIZE_2M) 98 return PAGE_MODE_2_LEVEL; 99 100 return PAGE_MODE_1_LEVEL; 101 } 102 103 static void free_pgtable(u64 *pt, int level) 104 { 105 u64 *p; 106 int i; 107 108 for (i = 0; i < MAX_PTRS_PER_PAGE; i++) { 109 /* PTE present? */ 110 if (!IOMMU_PTE_PRESENT(pt[i])) 111 continue; 112 113 if (is_large_pte(pt[i])) 114 continue; 115 116 /* 117 * Free the next level. No need to look at l1 tables here since 118 * they can only contain leaf PTEs; just free them directly. 119 */ 120 p = get_pgtable_pte(pt[i]); 121 if (level > 2) 122 free_pgtable(p, level - 1); 123 else 124 iommu_free_page(p); 125 } 126 127 iommu_free_page(pt); 128 } 129 130 /* Allocate page table */ 131 static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, 132 unsigned long pg_size, gfp_t gfp, bool *updated) 133 { 134 u64 *pte, *page; 135 int level, end_level; 136 137 level = get_pgtable_level() - 1; 138 end_level = page_size_to_level(pg_size); 139 pte = &pgd[PM_LEVEL_INDEX(level, iova)]; 140 iova = PAGE_SIZE_ALIGN(iova, PAGE_SIZE); 141 142 while (level >= end_level) { 143 u64 __pte, __npte; 144 145 __pte = *pte; 146 147 if (IOMMU_PTE_PRESENT(__pte) && is_large_pte(__pte)) { 148 /* Unmap large pte */ 149 cmpxchg64(pte, *pte, 0ULL); 150 *updated = true; 151 continue; 152 } 153 154 if (!IOMMU_PTE_PRESENT(__pte)) { 155 page = iommu_alloc_page_node(nid, gfp); 156 if (!page) 157 return NULL; 158 159 __npte = set_pgtable_attr(page); 160 /* pte could have been changed somewhere. */ 161 if (!try_cmpxchg64(pte, &__pte, __npte)) 162 iommu_free_page(page); 163 else if (IOMMU_PTE_PRESENT(__pte)) 164 *updated = true; 165 166 continue; 167 } 168 169 level -= 1; 170 pte = get_pgtable_pte(__pte); 171 pte = &pte[PM_LEVEL_INDEX(level, iova)]; 172 } 173 174 /* Tear down existing pte entries */ 175 if (IOMMU_PTE_PRESENT(*pte)) { 176 u64 *__pte; 177 178 *updated = true; 179 __pte = get_pgtable_pte(*pte); 180 cmpxchg64(pte, *pte, 0ULL); 181 if (pg_size == IOMMU_PAGE_SIZE_1G) 182 free_pgtable(__pte, end_level - 1); 183 else if (pg_size == IOMMU_PAGE_SIZE_2M) 184 iommu_free_page(__pte); 185 } 186 187 return pte; 188 } 189 190 /* 191 * This function checks if there is a PTE for a given dma address. 192 * If there is one, it returns the pointer to it. 193 */ 194 static u64 *fetch_pte(struct amd_io_pgtable *pgtable, 195 unsigned long iova, unsigned long *page_size) 196 { 197 u64 *pte; 198 int level; 199 200 level = get_pgtable_level() - 1; 201 pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)]; 202 /* Default page size is 4K */ 203 *page_size = PAGE_SIZE; 204 205 while (level) { 206 /* Not present */ 207 if (!IOMMU_PTE_PRESENT(*pte)) 208 return NULL; 209 210 /* Walk to the next level */ 211 pte = get_pgtable_pte(*pte); 212 pte = &pte[PM_LEVEL_INDEX(level - 1, iova)]; 213 214 /* Large page */ 215 if (is_large_pte(*pte)) { 216 if (level == PAGE_MODE_3_LEVEL) 217 *page_size = IOMMU_PAGE_SIZE_1G; 218 else if (level == PAGE_MODE_2_LEVEL) 219 *page_size = IOMMU_PAGE_SIZE_2M; 220 else 221 return NULL; /* Wrongly set PSE bit in PTE */ 222 223 break; 224 } 225 226 level -= 1; 227 } 228 229 return pte; 230 } 231 232 static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, 233 phys_addr_t paddr, size_t pgsize, size_t pgcount, 234 int prot, gfp_t gfp, size_t *mapped) 235 { 236 struct protection_domain *pdom = io_pgtable_ops_to_domain(ops); 237 struct io_pgtable_cfg *cfg = &pdom->iop.iop.cfg; 238 u64 *pte; 239 unsigned long map_size; 240 unsigned long mapped_size = 0; 241 unsigned long o_iova = iova; 242 size_t size = pgcount << __ffs(pgsize); 243 int ret = 0; 244 bool updated = false; 245 246 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize) || !pgcount) 247 return -EINVAL; 248 249 if (!(prot & IOMMU_PROT_MASK)) 250 return -EINVAL; 251 252 while (mapped_size < size) { 253 map_size = get_alloc_page_size(pgsize); 254 pte = v2_alloc_pte(pdom->nid, pdom->iop.pgd, 255 iova, map_size, gfp, &updated); 256 if (!pte) { 257 ret = -EINVAL; 258 goto out; 259 } 260 261 *pte = set_pte_attr(paddr, map_size, prot); 262 263 iova += map_size; 264 paddr += map_size; 265 mapped_size += map_size; 266 } 267 268 out: 269 if (updated) 270 amd_iommu_domain_flush_pages(pdom, o_iova, size); 271 272 if (mapped) 273 *mapped += mapped_size; 274 275 return ret; 276 } 277 278 static unsigned long iommu_v2_unmap_pages(struct io_pgtable_ops *ops, 279 unsigned long iova, 280 size_t pgsize, size_t pgcount, 281 struct iommu_iotlb_gather *gather) 282 { 283 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); 284 struct io_pgtable_cfg *cfg = &pgtable->iop.cfg; 285 unsigned long unmap_size; 286 unsigned long unmapped = 0; 287 size_t size = pgcount << __ffs(pgsize); 288 u64 *pte; 289 290 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) 291 return 0; 292 293 while (unmapped < size) { 294 pte = fetch_pte(pgtable, iova, &unmap_size); 295 if (!pte) 296 return unmapped; 297 298 *pte = 0ULL; 299 300 iova = (iova & ~(unmap_size - 1)) + unmap_size; 301 unmapped += unmap_size; 302 } 303 304 return unmapped; 305 } 306 307 static phys_addr_t iommu_v2_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) 308 { 309 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); 310 unsigned long offset_mask, pte_pgsize; 311 u64 *pte, __pte; 312 313 pte = fetch_pte(pgtable, iova, &pte_pgsize); 314 if (!pte || !IOMMU_PTE_PRESENT(*pte)) 315 return 0; 316 317 offset_mask = pte_pgsize - 1; 318 __pte = __sme_clr(*pte & PM_ADDR_MASK); 319 320 return (__pte & ~offset_mask) | (iova & offset_mask); 321 } 322 323 /* 324 * ---------------------------------------------------- 325 */ 326 static void v2_tlb_flush_all(void *cookie) 327 { 328 } 329 330 static void v2_tlb_flush_walk(unsigned long iova, size_t size, 331 size_t granule, void *cookie) 332 { 333 } 334 335 static void v2_tlb_add_page(struct iommu_iotlb_gather *gather, 336 unsigned long iova, size_t granule, 337 void *cookie) 338 { 339 } 340 341 static const struct iommu_flush_ops v2_flush_ops = { 342 .tlb_flush_all = v2_tlb_flush_all, 343 .tlb_flush_walk = v2_tlb_flush_walk, 344 .tlb_add_page = v2_tlb_add_page, 345 }; 346 347 static void v2_free_pgtable(struct io_pgtable *iop) 348 { 349 struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); 350 351 if (!pgtable || !pgtable->pgd) 352 return; 353 354 /* Free page table */ 355 free_pgtable(pgtable->pgd, get_pgtable_level()); 356 pgtable->pgd = NULL; 357 } 358 359 static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) 360 { 361 struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); 362 struct protection_domain *pdom = (struct protection_domain *)cookie; 363 int ias = IOMMU_IN_ADDR_BIT_SIZE; 364 365 pgtable->pgd = iommu_alloc_page_node(pdom->nid, GFP_ATOMIC); 366 if (!pgtable->pgd) 367 return NULL; 368 369 if (get_pgtable_level() == PAGE_MODE_5_LEVEL) 370 ias = 57; 371 372 pgtable->iop.ops.map_pages = iommu_v2_map_pages; 373 pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages; 374 pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; 375 376 cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, 377 cfg->ias = ias, 378 cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, 379 cfg->tlb = &v2_flush_ops; 380 381 return &pgtable->iop; 382 } 383 384 struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { 385 .alloc = v2_alloc_pgtable, 386 .free = v2_free_pgtable, 387 }; 388