1 /* SPDX-License-Identifier: MIT 2 * 3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. 4 */ 5 #include "vmm.h" 6 7 #include <subdev/fb.h> 8 9 #include <nvhw/drf.h> 10 #include <nvhw/ref/gh100/dev_mmu.h> 11 12 static inline void 13 gh100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, 14 struct nvkm_vmm_map *map, u64 addr) 15 { 16 u64 data = addr | map->type; 17 18 while (ptes--) { 19 VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data); 20 data += map->next; 21 } 22 } 23 24 static void 25 gh100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, 26 struct nvkm_vmm_map *map) 27 { 28 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte); 29 } 30 31 static void 32 gh100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, 33 struct nvkm_vmm_map *map) 34 { 35 if (map->page->shift == PAGE_SHIFT) { 36 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); 37 38 nvkm_kmap(pt->memory); 39 while (ptes--) { 40 const u64 data = *map->dma++ | map->type; 41 42 VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data); 43 } 44 nvkm_done(pt->memory); 45 return; 46 } 47 48 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte); 49 } 50 51 static void 52 gh100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes, 53 struct nvkm_vmm_map *map) 54 { 55 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte); 56 } 57 58 static void 59 gh100_vmm_pgt_sparse(struct nvkm_vmm *vmm, 60 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 61 { 62 const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, SPARSE); 63 64 VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes); 65 } 66 67 static const struct nvkm_vmm_desc_func 68 gh100_vmm_desc_spt = { 69 .unmap = gf100_vmm_pgt_unmap, 70 .sparse = gh100_vmm_pgt_sparse, 71 .mem = gh100_vmm_pgt_mem, 72 .dma = gh100_vmm_pgt_dma, 73 .sgl = gh100_vmm_pgt_sgl, 74 }; 75 76 static void 77 gh100_vmm_lpt_invalid(struct nvkm_vmm *vmm, 78 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 79 { 80 const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, NO_VALID_4KB_PAGE); 81 82 VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes); 83 } 84 85 static const struct nvkm_vmm_desc_func 86 gh100_vmm_desc_lpt = { 87 .invalid = gh100_vmm_lpt_invalid, 88 .unmap = gf100_vmm_pgt_unmap, 89 .sparse = gh100_vmm_pgt_sparse, 90 .mem = gh100_vmm_pgt_mem, 91 }; 92 93 static inline void 94 gh100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 95 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 96 { 97 u64 data = addr | map->type; 98 99 while (ptes--) { 100 VMM_WO128(pt, vmm, ptei++ * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL); 101 data += map->next; 102 } 103 } 104 105 static void 106 gh100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 107 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 108 { 109 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pd0_pte); 110 } 111 112 static inline bool 113 gh100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data) 114 { 115 switch (nvkm_memory_target(pt->memory)) { 116 case NVKM_MEM_TARGET_VRAM: 117 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, VIDEO_MEMORY); 118 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_NOT_ALLOWED); 119 break; 120 case NVKM_MEM_TARGET_HOST: 121 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_COHERENT_MEMORY); 122 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_UNCACHED_ATS_ALLOWED); 123 break; 124 case NVKM_MEM_TARGET_NCOH: 125 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_NON_COHERENT_MEMORY); 126 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_ALLOWED); 127 break; 128 default: 129 WARN_ON(1); 130 return false; 131 } 132 133 *data |= pt->addr; 134 return true; 135 } 136 137 static void 138 gh100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 139 { 140 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 141 struct nvkm_mmu_pt *pd = pgd->pt[0]; 142 u64 data[2] = {}; 143 144 if (pgt->pt[0] && !gh100_vmm_pde(pgt->pt[0], &data[0])) 145 return; 146 if (pgt->pt[1] && !gh100_vmm_pde(pgt->pt[1], &data[1])) 147 return; 148 149 nvkm_kmap(pd->memory); 150 VMM_WO128(pd, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data[0], data[1]); 151 nvkm_done(pd->memory); 152 } 153 154 static void 155 gh100_vmm_pd0_sparse(struct nvkm_vmm *vmm, 156 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 157 { 158 const u64 data = NVDEF(NV_MMU, VER3_DUAL_PDE, PCF_BIG, SPARSE_ATS_ALLOWED); 159 160 VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL, pdes); 161 } 162 163 static void 164 gh100_vmm_pd0_unmap(struct nvkm_vmm *vmm, 165 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes) 166 { 167 VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, 0ULL, 0ULL, pdes); 168 } 169 170 static const struct nvkm_vmm_desc_func 171 gh100_vmm_desc_pd0 = { 172 .unmap = gh100_vmm_pd0_unmap, 173 .sparse = gh100_vmm_pd0_sparse, 174 .pde = gh100_vmm_pd0_pde, 175 .mem = gh100_vmm_pd0_mem, 176 }; 177 178 static void 179 gh100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 180 { 181 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 182 struct nvkm_mmu_pt *pd = pgd->pt[0]; 183 u64 data = 0; 184 185 if (!gh100_vmm_pde(pgt->pt[0], &data)) 186 return; 187 188 nvkm_kmap(pd->memory); 189 VMM_WO064(pd, vmm, pdei * NV_MMU_VER3_PDE__SIZE, data); 190 nvkm_done(pd->memory); 191 } 192 193 static const struct nvkm_vmm_desc_func 194 gh100_vmm_desc_pd1 = { 195 .unmap = gf100_vmm_pgt_unmap, 196 .sparse = gh100_vmm_pgt_sparse, 197 .pde = gh100_vmm_pd1_pde, 198 }; 199 200 static const struct nvkm_vmm_desc 201 gh100_vmm_desc_16[] = { 202 { LPT, 5, 8, 0x0100, &gh100_vmm_desc_lpt }, 203 { PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 }, 204 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 205 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 206 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 207 { PGD, 1, 8, 0x1000, &gh100_vmm_desc_pd1 }, 208 {} 209 }; 210 211 static const struct nvkm_vmm_desc 212 gh100_vmm_desc_12[] = { 213 { SPT, 9, 8, 0x1000, &gh100_vmm_desc_spt }, 214 { PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 }, 215 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 216 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 217 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 }, 218 { PGD, 1, 8, 0x1000, &gh100_vmm_desc_pd1 }, 219 {} 220 }; 221 222 static int 223 gh100_vmm_valid(struct nvkm_vmm *vmm, bool ro, bool priv, u8 kind, u8 comp, 224 struct nvkm_vmm_map *map) 225 { 226 const enum nvkm_memory_target target = nvkm_memory_target(map->memory); 227 const bool vol = target == NVKM_MEM_TARGET_HOST; 228 const struct nvkm_vmm_page *page = map->page; 229 u8 kind_inv, pcf; 230 int kindn, aper; 231 const u8 *kindm; 232 233 map->next = 1ULL << page->shift; 234 map->type = 0; 235 236 aper = vmm->func->aper(target); 237 if (WARN_ON(aper < 0)) 238 return aper; 239 240 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv); 241 if (kind >= kindn || kindm[kind] == kind_inv) { 242 VMM_DEBUG(vmm, "kind %02x", kind); 243 return -EINVAL; 244 } 245 246 if (priv) { 247 if (ro) { 248 if (vol) 249 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_UNCACHED_ACD; 250 else 251 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_CACHED_ACD; 252 } else { 253 if (vol) 254 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_UNCACHED_ACD; 255 else 256 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_CACHED_ACD; 257 } 258 } else { 259 if (ro) { 260 if (vol) 261 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_UNCACHED_ACD; 262 else 263 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_CACHED_ACD; 264 } else { 265 if (vol) 266 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACD; 267 else 268 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACD; 269 } 270 } 271 272 map->type |= NVDEF(NV_MMU, VER3_PTE, VALID, TRUE); 273 map->type |= NVVAL(NV_MMU, VER3_PTE, APERTURE, aper); 274 map->type |= NVVAL(NV_MMU, VER3_PTE, PCF, pcf); 275 map->type |= NVVAL(NV_MMU, VER3_PTE, KIND, kind); 276 return 0; 277 } 278 279 static const struct nvkm_vmm_func 280 gh100_vmm = { 281 .join = gv100_vmm_join, 282 .part = gf100_vmm_part, 283 .aper = gf100_vmm_aper, 284 .valid = gp100_vmm_valid, 285 .valid2 = gh100_vmm_valid, 286 .flush = tu102_vmm_flush, 287 .page = { 288 { 56, &gh100_vmm_desc_16[5], NVKM_VMM_PAGE_Sxxx }, 289 { 47, &gh100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx }, 290 { 38, &gh100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx }, 291 { 29, &gh100_vmm_desc_16[2], NVKM_VMM_PAGE_SVxC }, 292 { 21, &gh100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC }, 293 { 16, &gh100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC }, 294 { 12, &gh100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx }, 295 {} 296 } 297 }; 298 299 int 300 gh100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 301 void *argv, u32 argc, struct lock_class_key *key, 302 const char *name, struct nvkm_vmm **pvmm) 303 { 304 return gp100_vmm_new_(&gh100_vmm, mmu, managed, addr, size, 305 argv, argc, key, name, pvmm); 306 } 307