1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2017-2018 Broadcom */ 3 4 /** 5 * DOC: Broadcom V3D MMU 6 * 7 * The V3D 3.x hardware (compared to VC4) now includes an MMU. It has 8 * a single level of page tables for the V3D's 4GB address space to 9 * map to AXI bus addresses, thus it could need up to 4MB of 10 * physically contiguous memory to store the PTEs. 11 * 12 * Because the 4MB of contiguous memory for page tables is precious, 13 * and switching between them is expensive, we load all BOs into the 14 * same 4GB address space. 15 * 16 * To protect clients from each other, we should use the GMP to 17 * quickly mask out (at 128kb granularity) what pages are available to 18 * each client. This is not yet implemented. 19 */ 20 21 #include <drm/drm_print.h> 22 23 #include "v3d_drv.h" 24 #include "v3d_regs.h" 25 26 /* Note: All PTEs for the 64KB bigpage or 1MB superpage must be filled 27 * with the bigpage/superpage bit set. 28 */ 29 #define V3D_PTE_SUPERPAGE BIT(31) 30 #define V3D_PTE_BIGPAGE BIT(30) 31 #define V3D_PTE_WRITEABLE BIT(29) 32 #define V3D_PTE_VALID BIT(28) 33 34 static bool v3d_mmu_is_aligned(u32 page, u32 page_address, size_t alignment) 35 { 36 return IS_ALIGNED(page, alignment >> V3D_MMU_PAGE_SHIFT) && 37 IS_ALIGNED(page_address, alignment >> V3D_MMU_PAGE_SHIFT); 38 } 39 40 /* 41 * Issue the MMUC flush and TLB clear unconditionally. The caller must 42 * already know that V3D is reachable. In particular, this is used from 43 * the runtime resume callback. 44 */ 45 static int v3d_mmu_flush_all_locked(struct v3d_dev *v3d) 46 { 47 int ret; 48 49 V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_FLUSH | 50 V3D_MMUC_CONTROL_ENABLE); 51 52 ret = wait_for(!(V3D_READ(V3D_MMUC_CONTROL) & 53 V3D_MMUC_CONTROL_FLUSHING), 100); 54 if (ret) { 55 dev_err(v3d->drm.dev, "MMUC flush wait idle failed\n"); 56 return ret; 57 } 58 59 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL) | 60 V3D_MMU_CTL_TLB_CLEAR); 61 62 ret = wait_for(!(V3D_READ(V3D_MMU_CTL) & 63 V3D_MMU_CTL_TLB_CLEARING), 100); 64 if (ret) 65 dev_err(v3d->drm.dev, "MMU TLB clear wait idle failed\n"); 66 67 return ret; 68 } 69 70 int v3d_mmu_flush_all(struct v3d_dev *v3d) 71 { 72 int ret; 73 74 /* Flush the PTs only if we're already awake */ 75 if (!pm_runtime_get_if_active(v3d->drm.dev)) 76 return 0; 77 78 ret = v3d_mmu_flush_all_locked(v3d); 79 80 v3d_pm_runtime_put(v3d); 81 return ret; 82 } 83 84 int v3d_mmu_set_page_table(struct v3d_dev *v3d) 85 { 86 V3D_WRITE(V3D_MMU_PT_PA_BASE, v3d->pt_paddr >> V3D_MMU_PAGE_SHIFT); 87 V3D_WRITE(V3D_MMU_CTL, 88 V3D_MMU_CTL_ENABLE | 89 V3D_MMU_CTL_PT_INVALID_ENABLE | 90 V3D_MMU_CTL_PT_INVALID_ABORT | 91 V3D_MMU_CTL_PT_INVALID_INT | 92 V3D_MMU_CTL_WRITE_VIOLATION_ABORT | 93 V3D_MMU_CTL_WRITE_VIOLATION_INT | 94 V3D_MMU_CTL_CAP_EXCEEDED_ABORT | 95 V3D_MMU_CTL_CAP_EXCEEDED_INT); 96 V3D_WRITE(V3D_MMU_ILLEGAL_ADDR, 97 (v3d->mmu_scratch_paddr >> V3D_MMU_PAGE_SHIFT) | 98 V3D_MMU_ILLEGAL_ADDR_ENABLE); 99 V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_ENABLE); 100 101 return v3d_mmu_flush_all_locked(v3d); 102 } 103 104 void v3d_mmu_insert_ptes(struct v3d_bo *bo) 105 { 106 struct drm_gem_shmem_object *shmem_obj = &bo->base; 107 struct v3d_dev *v3d = to_v3d_dev(shmem_obj->base.dev); 108 u32 page = bo->node.start; 109 struct scatterlist *sgl; 110 unsigned int count; 111 112 for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, count) { 113 dma_addr_t dma_addr = sg_dma_address(sgl); 114 u32 pfn = dma_addr >> V3D_MMU_PAGE_SHIFT; 115 unsigned int len = sg_dma_len(sgl); 116 117 while (len > 0) { 118 u32 page_prot = V3D_PTE_WRITEABLE | V3D_PTE_VALID; 119 u32 page_address = page_prot | pfn; 120 unsigned int i, page_size; 121 122 BUG_ON(pfn + V3D_PAGE_FACTOR >= BIT(24)); 123 124 if (len >= SZ_1M && 125 v3d_mmu_is_aligned(page, page_address, SZ_1M)) { 126 page_size = SZ_1M; 127 page_address |= V3D_PTE_SUPERPAGE; 128 } else if (len >= SZ_64K && 129 v3d_mmu_is_aligned(page, page_address, SZ_64K)) { 130 page_size = SZ_64K; 131 page_address |= V3D_PTE_BIGPAGE; 132 } else { 133 page_size = SZ_4K; 134 } 135 136 for (i = 0; i < page_size >> V3D_MMU_PAGE_SHIFT; i++) { 137 v3d->pt[page++] = page_address + i; 138 pfn++; 139 } 140 141 len -= page_size; 142 } 143 } 144 145 WARN_ON_ONCE(page - bo->node.start != 146 shmem_obj->base.size >> V3D_MMU_PAGE_SHIFT); 147 148 if (v3d_mmu_flush_all(v3d)) 149 drm_err(&v3d->drm, "MMU flush timeout\n"); 150 } 151 152 void v3d_mmu_remove_ptes(struct v3d_bo *bo) 153 { 154 struct v3d_dev *v3d = to_v3d_dev(bo->base.base.dev); 155 u32 npages = bo->base.base.size >> V3D_MMU_PAGE_SHIFT; 156 u32 page; 157 158 for (page = bo->node.start; page < bo->node.start + npages; page++) 159 v3d->pt[page] = 0; 160 161 if (v3d_mmu_flush_all(v3d)) 162 drm_err(&v3d->drm, "MMU flush timeout\n"); 163 } 164