1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/bitfield.h> 7 #include <linux/highmem.h> 8 9 #include "ivpu_drv.h" 10 #include "ivpu_hw.h" 11 #include "ivpu_mmu.h" 12 #include "ivpu_mmu_context.h" 13 14 #define IVPU_MMU_PGD_INDEX_MASK GENMASK(38, 30) 15 #define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21) 16 #define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12) 17 #define IVPU_MMU_ENTRY_FLAGS_MASK GENMASK(11, 0) 18 #define IVPU_MMU_ENTRY_FLAG_NG BIT(11) 19 #define IVPU_MMU_ENTRY_FLAG_AF BIT(10) 20 #define IVPU_MMU_ENTRY_FLAG_USER BIT(6) 21 #define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2) 22 #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1) 23 #define IVPU_MMU_ENTRY_FLAG_VALID BIT(0) 24 25 #define IVPU_MMU_PAGE_SIZE SZ_4K 26 #define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) 27 #define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) 28 #define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) 29 30 #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000 31 #define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID) 32 #define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK) 33 #define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \ 34 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID) 35 36 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) 37 { 38 dma_addr_t pgd_dma; 39 u64 *pgd; 40 41 pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL); 42 if (!pgd) 43 return -ENOMEM; 44 45 pgtable->pgd = pgd; 46 pgtable->pgd_dma = pgd_dma; 47 48 return 0; 49 } 50 51 static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) 52 { 53 int pgd_index, pmd_index; 54 55 for (pgd_index = 0; pgd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_index) { 56 u64 **pmd_entries = pgtable->pgd_cpu_entries[pgd_index]; 57 u64 *pmd = pgtable->pgd_entries[pgd_index]; 58 59 if (!pmd_entries) 60 continue; 61 62 for (pmd_index = 0; pmd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_index) { 63 if (pmd_entries[pmd_index]) 64 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, 65 pmd_entries[pmd_index], 66 pmd[pmd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK); 67 } 68 69 kfree(pmd_entries); 70 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_index], 71 pgtable->pgd[pgd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK); 72 } 73 74 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd, 75 pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK); 76 } 77 78 static u64* 79 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, u64 pgd_index) 80 { 81 u64 **pmd_entries; 82 dma_addr_t pmd_dma; 83 u64 *pmd; 84 85 if (pgtable->pgd_entries[pgd_index]) 86 return pgtable->pgd_entries[pgd_index]; 87 88 pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL); 89 if (!pmd) 90 return NULL; 91 92 pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); 93 if (!pmd_entries) 94 goto err_free_pgd; 95 96 pgtable->pgd_entries[pgd_index] = pmd; 97 pgtable->pgd_cpu_entries[pgd_index] = pmd_entries; 98 pgtable->pgd[pgd_index] = pmd_dma | IVPU_MMU_ENTRY_VALID; 99 100 return pmd; 101 102 err_free_pgd: 103 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma); 104 return NULL; 105 } 106 107 static u64* 108 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, 109 int pgd_index, int pmd_index) 110 { 111 dma_addr_t pte_dma; 112 u64 *pte; 113 114 if (pgtable->pgd_cpu_entries[pgd_index][pmd_index]) 115 return pgtable->pgd_cpu_entries[pgd_index][pmd_index]; 116 117 pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL); 118 if (!pte) 119 return NULL; 120 121 pgtable->pgd_cpu_entries[pgd_index][pmd_index] = pte; 122 pgtable->pgd_entries[pgd_index][pmd_index] = pte_dma | IVPU_MMU_ENTRY_VALID; 123 124 return pte; 125 } 126 127 static int 128 ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 129 u64 vpu_addr, dma_addr_t dma_addr, int prot) 130 { 131 u64 *pte; 132 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 133 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 134 int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); 135 136 /* Allocate PMD - second level page table if needed */ 137 if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_index)) 138 return -ENOMEM; 139 140 /* Allocate PTE - third level page table if needed */ 141 pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_index, pmd_index); 142 if (!pte) 143 return -ENOMEM; 144 145 /* Update PTE - third level page table with DMA address */ 146 pte[pte_index] = dma_addr | prot; 147 148 return 0; 149 } 150 151 static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr) 152 { 153 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 154 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 155 int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); 156 157 /* Update PTE with dummy physical address and clear flags */ 158 ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index][pte_index] = IVPU_MMU_ENTRY_INVALID; 159 } 160 161 static void 162 ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) 163 { 164 u64 end_addr = vpu_addr + size; 165 u64 *pgd = ctx->pgtable.pgd; 166 167 /* Align to PMD entry (2 MB) */ 168 vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1); 169 170 while (vpu_addr < end_addr) { 171 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 172 u64 pmd_end = (pgd_index + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE; 173 u64 *pmd = ctx->pgtable.pgd_entries[pgd_index]; 174 175 while (vpu_addr < end_addr && vpu_addr < pmd_end) { 176 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 177 u64 *pte = ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index]; 178 179 clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE); 180 vpu_addr += IVPU_MMU_PTE_MAP_SIZE; 181 } 182 clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE); 183 } 184 clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE); 185 } 186 187 static int 188 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 189 u64 vpu_addr, dma_addr_t dma_addr, size_t size, int prot) 190 { 191 while (size) { 192 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); 193 194 if (ret) 195 return ret; 196 197 vpu_addr += IVPU_MMU_PAGE_SIZE; 198 dma_addr += IVPU_MMU_PAGE_SIZE; 199 size -= IVPU_MMU_PAGE_SIZE; 200 } 201 202 return 0; 203 } 204 205 static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) 206 { 207 while (size) { 208 ivpu_mmu_context_unmap_page(ctx, vpu_addr); 209 vpu_addr += IVPU_MMU_PAGE_SIZE; 210 size -= IVPU_MMU_PAGE_SIZE; 211 } 212 } 213 214 int 215 ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 216 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent) 217 { 218 struct scatterlist *sg; 219 int prot; 220 int ret; 221 u64 i; 222 223 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) 224 return -EINVAL; 225 /* 226 * VPU is only 32 bit, but DMA engine is 38 bit 227 * Ranges < 2 GB are reserved for VPU internal registers 228 * Limit range to 8 GB 229 */ 230 if (vpu_addr < SZ_2G || vpu_addr > SZ_8G) 231 return -EINVAL; 232 233 prot = IVPU_MMU_ENTRY_MAPPED; 234 if (llc_coherent) 235 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT; 236 237 mutex_lock(&ctx->lock); 238 239 for_each_sgtable_dma_sg(sgt, sg, i) { 240 u64 dma_addr = sg_dma_address(sg) - sg->offset; 241 size_t size = sg_dma_len(sg) + sg->offset; 242 243 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot); 244 if (ret) { 245 ivpu_err(vdev, "Failed to map context pages\n"); 246 mutex_unlock(&ctx->lock); 247 return ret; 248 } 249 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size); 250 vpu_addr += size; 251 } 252 253 mutex_unlock(&ctx->lock); 254 255 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); 256 if (ret) 257 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); 258 return ret; 259 } 260 261 void 262 ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 263 u64 vpu_addr, struct sg_table *sgt) 264 { 265 struct scatterlist *sg; 266 int ret; 267 u64 i; 268 269 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) 270 ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr); 271 272 mutex_lock(&ctx->lock); 273 274 for_each_sgtable_dma_sg(sgt, sg, i) { 275 size_t size = sg_dma_len(sg) + sg->offset; 276 277 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size); 278 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size); 279 vpu_addr += size; 280 } 281 282 mutex_unlock(&ctx->lock); 283 284 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); 285 if (ret) 286 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); 287 } 288 289 int 290 ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx, 291 const struct ivpu_addr_range *range, 292 u64 size, struct drm_mm_node *node) 293 { 294 lockdep_assert_held(&ctx->lock); 295 296 return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 297 0, range->start, range->end, DRM_MM_INSERT_BEST); 298 } 299 300 void 301 ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node) 302 { 303 lockdep_assert_held(&ctx->lock); 304 305 drm_mm_remove_node(node); 306 } 307 308 static int 309 ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id) 310 { 311 u64 start, end; 312 int ret; 313 314 mutex_init(&ctx->lock); 315 INIT_LIST_HEAD(&ctx->bo_list); 316 317 ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable); 318 if (ret) 319 return ret; 320 321 if (!context_id) { 322 start = vdev->hw->ranges.global_low.start; 323 end = vdev->hw->ranges.global_high.end; 324 } else { 325 start = vdev->hw->ranges.user_low.start; 326 end = vdev->hw->ranges.user_high.end; 327 } 328 329 drm_mm_init(&ctx->mm, start, end - start); 330 ctx->id = context_id; 331 332 return 0; 333 } 334 335 static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 336 { 337 drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd); 338 339 mutex_destroy(&ctx->lock); 340 ivpu_mmu_pgtable_free(vdev, &ctx->pgtable); 341 drm_mm_takedown(&ctx->mm); 342 } 343 344 int ivpu_mmu_global_context_init(struct ivpu_device *vdev) 345 { 346 return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID); 347 } 348 349 void ivpu_mmu_global_context_fini(struct ivpu_device *vdev) 350 { 351 return ivpu_mmu_context_fini(vdev, &vdev->gctx); 352 } 353 354 void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid) 355 { 356 struct ivpu_file_priv *file_priv; 357 358 xa_lock(&vdev->context_xa); 359 360 file_priv = xa_load(&vdev->context_xa, ssid); 361 if (file_priv) 362 file_priv->has_mmu_faults = true; 363 364 xa_unlock(&vdev->context_xa); 365 } 366 367 int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id) 368 { 369 int ret; 370 371 drm_WARN_ON(&vdev->drm, !ctx_id); 372 373 ret = ivpu_mmu_context_init(vdev, ctx, ctx_id); 374 if (ret) { 375 ivpu_err(vdev, "Failed to initialize context: %d\n", ret); 376 return ret; 377 } 378 379 ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable); 380 if (ret) { 381 ivpu_err(vdev, "Failed to set page table: %d\n", ret); 382 goto err_context_fini; 383 } 384 385 return 0; 386 387 err_context_fini: 388 ivpu_mmu_context_fini(vdev, ctx); 389 return ret; 390 } 391 392 void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 393 { 394 drm_WARN_ON(&vdev->drm, !ctx->id); 395 396 ivpu_mmu_clear_pgtable(vdev, ctx->id); 397 ivpu_mmu_context_fini(vdev, ctx); 398 } 399