1 /* 2 * Copyright 2017 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "vmm.h" 23 24 #include <subdev/fb.h> 25 #include <subdev/ltc.h> 26 #include <subdev/timer.h> 27 28 #include <nvif/if900d.h> 29 #include <nvif/unpack.h> 30 31 static inline void 32 gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 33 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr) 34 { 35 u64 base = (addr >> 8) | map->type; 36 u64 data = base; 37 38 if (map->ctag && !(map->next & (1ULL << 44))) { 39 while (ptes--) { 40 data = base | ((map->ctag >> 1) << 44); 41 if (!(map->ctag++ & 1)) 42 data |= BIT_ULL(60); 43 44 VMM_WO064(pt, vmm, ptei++ * 8, data); 45 base += map->next; 46 } 47 } else { 48 map->type += ptes * map->ctag; 49 50 while (ptes--) { 51 VMM_WO064(pt, vmm, ptei++ * 8, data); 52 data += map->next; 53 } 54 } 55 } 56 57 void 58 gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 59 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 60 { 61 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 62 } 63 64 void 65 gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 66 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 67 { 68 if (map->page->shift == PAGE_SHIFT) { 69 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes); 70 nvkm_kmap(pt->memory); 71 while (ptes--) { 72 const u64 data = (*map->dma++ >> 8) | map->type; 73 VMM_WO064(pt, vmm, ptei++ * 8, data); 74 map->type += map->ctag; 75 } 76 nvkm_done(pt->memory); 77 return; 78 } 79 80 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 81 } 82 83 void 84 gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, 85 u32 ptei, u32 ptes, struct nvkm_vmm_map *map) 86 { 87 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte); 88 } 89 90 void 91 gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm, 92 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes) 93 { 94 VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes); 95 } 96 97 const struct nvkm_vmm_desc_func 98 gf100_vmm_pgt = { 99 .unmap = gf100_vmm_pgt_unmap, 100 .mem = gf100_vmm_pgt_mem, 101 .dma = gf100_vmm_pgt_dma, 102 .sgl = gf100_vmm_pgt_sgl, 103 }; 104 105 void 106 gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei) 107 { 108 struct nvkm_vmm_pt *pgt = pgd->pde[pdei]; 109 struct nvkm_mmu_pt *pd = pgd->pt[0]; 110 struct nvkm_mmu_pt *pt; 111 u64 data = 0; 112 113 if ((pt = pgt->pt[0])) { 114 switch (nvkm_memory_target(pt->memory)) { 115 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break; 116 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0; 117 data |= BIT_ULL(35); /* VOL */ 118 break; 119 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break; 120 default: 121 WARN_ON(1); 122 return; 123 } 124 data |= pt->addr >> 8; 125 } 126 127 if ((pt = pgt->pt[1])) { 128 switch (nvkm_memory_target(pt->memory)) { 129 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break; 130 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32; 131 data |= BIT_ULL(34); /* VOL */ 132 break; 133 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break; 134 default: 135 WARN_ON(1); 136 return; 137 } 138 data |= pt->addr << 24; 139 } 140 141 nvkm_kmap(pd->memory); 142 VMM_WO064(pd, vmm, pdei * 8, data); 143 nvkm_done(pd->memory); 144 } 145 146 const struct nvkm_vmm_desc_func 147 gf100_vmm_pgd = { 148 .unmap = gf100_vmm_pgt_unmap, 149 .pde = gf100_vmm_pgd_pde, 150 }; 151 152 static const struct nvkm_vmm_desc 153 gf100_vmm_desc_17_12[] = { 154 { SPT, 15, 8, 0x1000, &gf100_vmm_pgt }, 155 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, 156 {} 157 }; 158 159 static const struct nvkm_vmm_desc 160 gf100_vmm_desc_17_17[] = { 161 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, 162 { PGD, 13, 8, 0x1000, &gf100_vmm_pgd }, 163 {} 164 }; 165 166 static const struct nvkm_vmm_desc 167 gf100_vmm_desc_16_12[] = { 168 { SPT, 14, 8, 0x1000, &gf100_vmm_pgt }, 169 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, 170 {} 171 }; 172 173 static const struct nvkm_vmm_desc 174 gf100_vmm_desc_16_16[] = { 175 { LPT, 10, 8, 0x1000, &gf100_vmm_pgt }, 176 { PGD, 14, 8, 0x1000, &gf100_vmm_pgd }, 177 {} 178 }; 179 180 void 181 gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr) 182 { 183 struct nvkm_device *device = vmm->mmu->subdev.device; 184 nvkm_wr32(device, 0x100cb8, addr); 185 } 186 187 void 188 gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type) 189 { 190 struct nvkm_device *device = vmm->mmu->subdev.device; 191 struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; 192 u64 addr = 0; 193 194 mutex_lock(&vmm->mmu->mutex); 195 /* Looks like maybe a "free flush slots" counter, the 196 * faster you write to 0x100cbc to more it decreases. 197 */ 198 nvkm_msec(device, 2000, 199 if (nvkm_rd32(device, 0x100c80) & 0x00ff0000) 200 break; 201 ); 202 203 if (!(type & 0x00000002) /* ALL_PDB. */) { 204 switch (nvkm_memory_target(pd->memory)) { 205 case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break; 206 case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break; 207 case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break; 208 default: 209 WARN_ON(1); 210 break; 211 } 212 addr |= (vmm->pd->pt[0]->addr >> 12) << 4; 213 214 vmm->func->invalidate_pdb(vmm, addr); 215 } 216 217 nvkm_wr32(device, 0x100cbc, 0x80000000 | type); 218 219 /* Wait for flush to be queued? */ 220 nvkm_msec(device, 2000, 221 if (nvkm_rd32(device, 0x100c80) & 0x00008000) 222 break; 223 ); 224 mutex_unlock(&vmm->mmu->mutex); 225 } 226 227 void 228 gf100_vmm_flush(struct nvkm_vmm *vmm, int depth) 229 { 230 u32 type = 0x00000001; /* PAGE_ALL */ 231 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR])) 232 type |= 0x00000004; /* HUB_ONLY */ 233 gf100_vmm_invalidate(vmm, type); 234 } 235 236 int 237 gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc, 238 struct nvkm_vmm_map *map) 239 { 240 const enum nvkm_memory_target target = nvkm_memory_target(map->memory); 241 const struct nvkm_vmm_page *page = map->page; 242 const bool gm20x = page->desc->func->sparse != NULL; 243 union { 244 struct gf100_vmm_map_vn vn; 245 struct gf100_vmm_map_v0 v0; 246 } *args = argv; 247 struct nvkm_device *device = vmm->mmu->subdev.device; 248 struct nvkm_memory *memory = map->memory; 249 u8 kind, kind_inv, priv, ro, vol; 250 int kindn, aper, ret = -ENOSYS; 251 const u8 *kindm; 252 253 map->next = (1 << page->shift) >> 8; 254 map->type = map->ctag = 0; 255 256 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 257 vol = !!args->v0.vol; 258 ro = !!args->v0.ro; 259 priv = !!args->v0.priv; 260 kind = args->v0.kind; 261 } else 262 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) { 263 vol = target == NVKM_MEM_TARGET_HOST; 264 ro = 0; 265 priv = 0; 266 kind = 0x00; 267 } else { 268 VMM_DEBUG(vmm, "args"); 269 return ret; 270 } 271 272 aper = vmm->func->aper(target); 273 if (WARN_ON(aper < 0)) 274 return aper; 275 276 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv); 277 if (kind >= kindn || kindm[kind] == kind_inv) { 278 VMM_DEBUG(vmm, "kind %02x", kind); 279 return -EINVAL; 280 } 281 282 if (kindm[kind] != kind) { 283 u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17; 284 u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp; 285 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) { 286 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type); 287 return -EINVAL; 288 } 289 290 if (!map->no_comp) { 291 ret = nvkm_memory_tags_get(memory, device, tags, 292 nvkm_ltc_tags_clear, 293 &map->tags); 294 if (ret) { 295 VMM_DEBUG(vmm, "comp %d", ret); 296 return ret; 297 } 298 } 299 300 if (!map->no_comp && map->tags->mn) { 301 u64 tags = map->tags->mn->offset + (map->offset >> 17); 302 if (page->shift == 17 || !gm20x) { 303 map->type |= tags << 44; 304 map->ctag |= 1ULL << 44; 305 map->next |= 1ULL << 44; 306 } else { 307 map->ctag |= tags << 1 | 1; 308 } 309 } else { 310 kind = kindm[kind]; 311 } 312 } 313 314 map->type |= BIT(0); 315 map->type |= (u64)priv << 1; 316 map->type |= (u64) ro << 2; 317 map->type |= (u64) vol << 32; 318 map->type |= (u64)aper << 33; 319 map->type |= (u64)kind << 36; 320 return 0; 321 } 322 323 int 324 gf100_vmm_aper(enum nvkm_memory_target target) 325 { 326 switch (target) { 327 case NVKM_MEM_TARGET_VRAM: return 0; 328 case NVKM_MEM_TARGET_HOST: return 2; 329 case NVKM_MEM_TARGET_NCOH: return 3; 330 default: 331 return -EINVAL; 332 } 333 } 334 335 void 336 gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 337 { 338 nvkm_fo64(inst, 0x0200, 0x00000000, 2); 339 } 340 341 int 342 gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base) 343 { 344 struct nvkm_mmu_pt *pd = vmm->pd->pt[0]; 345 346 switch (nvkm_memory_target(pd->memory)) { 347 case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break; 348 case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0; 349 base |= BIT_ULL(2) /* VOL. */; 350 break; 351 case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break; 352 default: 353 WARN_ON(1); 354 return -EINVAL; 355 } 356 base |= pd->addr; 357 358 nvkm_kmap(inst); 359 nvkm_wo64(inst, 0x0200, base); 360 nvkm_wo64(inst, 0x0208, vmm->limit - 1); 361 nvkm_done(inst); 362 return 0; 363 } 364 365 int 366 gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst) 367 { 368 return gf100_vmm_join_(vmm, inst, 0); 369 } 370 371 static const struct nvkm_vmm_func 372 gf100_vmm_17 = { 373 .join = gf100_vmm_join, 374 .part = gf100_vmm_part, 375 .aper = gf100_vmm_aper, 376 .valid = gf100_vmm_valid, 377 .flush = gf100_vmm_flush, 378 .invalidate_pdb = gf100_vmm_invalidate_pdb, 379 .page = { 380 { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC }, 381 { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx }, 382 {} 383 } 384 }; 385 386 static const struct nvkm_vmm_func 387 gf100_vmm_16 = { 388 .join = gf100_vmm_join, 389 .part = gf100_vmm_part, 390 .aper = gf100_vmm_aper, 391 .valid = gf100_vmm_valid, 392 .flush = gf100_vmm_flush, 393 .invalidate_pdb = gf100_vmm_invalidate_pdb, 394 .page = { 395 { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC }, 396 { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx }, 397 {} 398 } 399 }; 400 401 int 402 gf100_vmm_new_(const struct nvkm_vmm_func *func_16, 403 const struct nvkm_vmm_func *func_17, 404 struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 405 void *argv, u32 argc, struct lock_class_key *key, 406 const char *name, struct nvkm_vmm **pvmm) 407 { 408 switch (mmu->subdev.device->fb->page) { 409 case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size, 410 argv, argc, key, name, pvmm); 411 case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size, 412 argv, argc, key, name, pvmm); 413 default: 414 WARN_ON(1); 415 return -EINVAL; 416 } 417 } 418 419 int 420 gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size, 421 void *argv, u32 argc, struct lock_class_key *key, 422 const char *name, struct nvkm_vmm **pvmm) 423 { 424 return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr, 425 size, argv, argc, key, name, pvmm); 426 } 427