1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2016-2018 Etnaviv Project 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/platform_device.h> 9 #include <linux/sizes.h> 10 #include <linux/slab.h> 11 #include <linux/vmalloc.h> 12 13 #include "etnaviv_cmdbuf.h" 14 #include "etnaviv_gpu.h" 15 #include "etnaviv_mmu.h" 16 #include "state.xml.h" 17 #include "state_hi.xml.h" 18 19 #define MMUv2_PTE_PRESENT BIT(0) 20 #define MMUv2_PTE_EXCEPTION BIT(1) 21 #define MMUv2_PTE_WRITEABLE BIT(2) 22 23 #define MMUv2_MTLB_MASK 0xffc00000 24 #define MMUv2_MTLB_SHIFT 22 25 #define MMUv2_STLB_MASK 0x003ff000 26 #define MMUv2_STLB_SHIFT 12 27 28 #define MMUv2_MAX_STLB_ENTRIES 1024 29 30 struct etnaviv_iommuv2_context { 31 struct etnaviv_iommu_context base; 32 unsigned short id; 33 /* M(aster) TLB aka first level pagetable */ 34 u32 *mtlb_cpu; 35 dma_addr_t mtlb_dma; 36 /* S(lave) TLB aka second level pagetable */ 37 u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES]; 38 dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES]; 39 }; 40 41 static struct etnaviv_iommuv2_context * 42 to_v2_context(struct etnaviv_iommu_context *context) 43 { 44 return container_of(context, struct etnaviv_iommuv2_context, base); 45 } 46 47 static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context) 48 { 49 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 50 int i; 51 52 drm_mm_takedown(&context->mm); 53 54 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { 55 if (v2_context->stlb_cpu[i]) 56 dma_free_wc(context->global->dev, SZ_4K, 57 v2_context->stlb_cpu[i], 58 v2_context->stlb_dma[i]); 59 } 60 61 dma_free_wc(context->global->dev, SZ_4K, v2_context->mtlb_cpu, 62 v2_context->mtlb_dma); 63 64 clear_bit(v2_context->id, context->global->v2.pta_alloc); 65 66 vfree(v2_context); 67 } 68 static int 69 etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context, 70 int stlb) 71 { 72 if (v2_context->stlb_cpu[stlb]) 73 return 0; 74 75 v2_context->stlb_cpu[stlb] = 76 dma_alloc_wc(v2_context->base.global->dev, SZ_4K, 77 &v2_context->stlb_dma[stlb], 78 GFP_KERNEL); 79 80 if (!v2_context->stlb_cpu[stlb]) 81 return -ENOMEM; 82 83 memset32(v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION, 84 SZ_4K / sizeof(u32)); 85 86 v2_context->mtlb_cpu[stlb] = 87 v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT; 88 89 return 0; 90 } 91 92 static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context, 93 unsigned long iova, phys_addr_t paddr, 94 size_t size, int prot) 95 { 96 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 97 int mtlb_entry, stlb_entry, ret; 98 u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT; 99 100 if (size != SZ_4K) 101 return -EINVAL; 102 103 if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 104 entry |= (upper_32_bits(paddr) & 0xff) << 4; 105 106 if (prot & ETNAVIV_PROT_WRITE) 107 entry |= MMUv2_PTE_WRITEABLE; 108 109 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 110 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 111 112 ret = etnaviv_iommuv2_ensure_stlb(v2_context, mtlb_entry); 113 if (ret) 114 return ret; 115 116 v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry; 117 118 return 0; 119 } 120 121 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context, 122 unsigned long iova, size_t size) 123 { 124 struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context); 125 int mtlb_entry, stlb_entry; 126 127 if (size != SZ_4K) 128 return -EINVAL; 129 130 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; 131 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; 132 133 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION; 134 135 return SZ_4K; 136 } 137 138 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context) 139 { 140 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 141 size_t dump_size = SZ_4K; 142 int i; 143 144 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 145 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) 146 dump_size += SZ_4K; 147 148 return dump_size; 149 } 150 151 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf) 152 { 153 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 154 int i; 155 156 memcpy(buf, v2_context->mtlb_cpu, SZ_4K); 157 buf += SZ_4K; 158 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) 159 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) { 160 memcpy(buf, v2_context->stlb_cpu[i], SZ_4K); 161 buf += SZ_4K; 162 } 163 } 164 165 static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu, 166 struct etnaviv_iommu_context *context) 167 { 168 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 169 u16 prefetch; 170 171 /* If the MMU is already enabled the state is still there. */ 172 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE) 173 return; 174 175 if (gpu->mmu_context) 176 etnaviv_iommu_context_put(gpu->mmu_context); 177 gpu->mmu_context = etnaviv_iommu_context_get(context); 178 179 prefetch = etnaviv_buffer_config_mmuv2(gpu, 180 (u32)v2_context->mtlb_dma, 181 (u32)context->global->bad_page_dma); 182 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), 183 prefetch); 184 etnaviv_gpu_wait_idle(gpu, 100); 185 186 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); 187 } 188 189 static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu, 190 struct etnaviv_iommu_context *context) 191 { 192 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 193 u16 prefetch; 194 195 /* If the MMU is already enabled the state is still there. */ 196 if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE) 197 return; 198 199 if (gpu->mmu_context) 200 etnaviv_iommu_context_put(gpu->mmu_context); 201 gpu->mmu_context = etnaviv_iommu_context_get(context); 202 203 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW, 204 lower_32_bits(context->global->v2.pta_dma)); 205 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH, 206 upper_32_bits(context->global->v2.pta_dma)); 207 gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE); 208 209 gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW, 210 lower_32_bits(context->global->bad_page_dma)); 211 gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW, 212 lower_32_bits(context->global->bad_page_dma)); 213 gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG, 214 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH( 215 upper_32_bits(context->global->bad_page_dma)) | 216 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH( 217 upper_32_bits(context->global->bad_page_dma))); 218 219 context->global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma | 220 VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; 221 222 /* trigger a PTA load through the FE */ 223 prefetch = etnaviv_buffer_config_pta(gpu, v2_context->id); 224 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), 225 prefetch); 226 etnaviv_gpu_wait_idle(gpu, 100); 227 228 gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE); 229 } 230 231 u32 etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context *context) 232 { 233 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 234 235 return v2_context->mtlb_dma; 236 } 237 238 unsigned short etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context *context) 239 { 240 struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); 241 242 return v2_context->id; 243 } 244 static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu, 245 struct etnaviv_iommu_context *context) 246 { 247 switch (gpu->sec_mode) { 248 case ETNA_SEC_NONE: 249 etnaviv_iommuv2_restore_nonsec(gpu, context); 250 break; 251 case ETNA_SEC_KERNEL: 252 etnaviv_iommuv2_restore_sec(gpu, context); 253 break; 254 default: 255 WARN(1, "unhandled GPU security mode\n"); 256 break; 257 } 258 } 259 260 const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = { 261 .free = etnaviv_iommuv2_free, 262 .map = etnaviv_iommuv2_map, 263 .unmap = etnaviv_iommuv2_unmap, 264 .dump_size = etnaviv_iommuv2_dump_size, 265 .dump = etnaviv_iommuv2_dump, 266 .restore = etnaviv_iommuv2_restore, 267 }; 268 269 struct etnaviv_iommu_context * 270 etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global) 271 { 272 struct etnaviv_iommuv2_context *v2_context; 273 struct etnaviv_iommu_context *context; 274 275 v2_context = vzalloc(sizeof(*v2_context)); 276 if (!v2_context) 277 return NULL; 278 279 mutex_lock(&global->lock); 280 v2_context->id = find_first_zero_bit(global->v2.pta_alloc, 281 ETNAVIV_PTA_ENTRIES); 282 if (v2_context->id < ETNAVIV_PTA_ENTRIES) { 283 set_bit(v2_context->id, global->v2.pta_alloc); 284 } else { 285 mutex_unlock(&global->lock); 286 goto out_free; 287 } 288 mutex_unlock(&global->lock); 289 290 v2_context->mtlb_cpu = dma_alloc_wc(global->dev, SZ_4K, 291 &v2_context->mtlb_dma, GFP_KERNEL); 292 if (!v2_context->mtlb_cpu) 293 goto out_free_id; 294 295 memset32(v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION, 296 MMUv2_MAX_STLB_ENTRIES); 297 298 global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma; 299 300 context = &v2_context->base; 301 context->global = global; 302 kref_init(&context->refcount); 303 mutex_init(&context->lock); 304 INIT_LIST_HEAD(&context->mappings); 305 drm_mm_init(&context->mm, SZ_4K, (u64)SZ_1G * 4 - SZ_4K); 306 307 return context; 308 309 out_free_id: 310 clear_bit(v2_context->id, global->v2.pta_alloc); 311 out_free: 312 vfree(v2_context); 313 return NULL; 314 } 315