1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/drm_print.h> 7 #include <drm/intel/display_parent_interface.h> 8 9 #include "display/intel_display_core.h" 10 #include "gem/i915_gem_domain.h" 11 #include "gem/i915_gem_internal.h" 12 #include "gem/i915_gem_lmem.h" 13 #include "gt/gen8_ppgtt.h" 14 15 #include "i915_dpt.h" 16 #include "i915_drv.h" 17 18 struct intel_dpt { 19 struct i915_address_space vm; 20 21 struct drm_i915_gem_object *obj; 22 struct i915_vma *vma; 23 void __iomem *iomem; 24 }; 25 26 #define i915_is_dpt(vm) ((vm)->is_dpt) 27 28 static inline struct intel_dpt * 29 i915_vm_to_dpt(struct i915_address_space *vm) 30 { 31 BUILD_BUG_ON(offsetof(struct intel_dpt, vm)); 32 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm)); 33 return container_of(vm, struct intel_dpt, vm); 34 } 35 36 struct i915_address_space *i915_dpt_to_vm(struct intel_dpt *dpt) 37 { 38 return &dpt->vm; 39 } 40 41 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 42 { 43 writeq(pte, addr); 44 } 45 46 static void dpt_insert_page(struct i915_address_space *vm, 47 dma_addr_t addr, 48 u64 offset, 49 unsigned int pat_index, 50 u32 flags) 51 { 52 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 53 gen8_pte_t __iomem *base = dpt->iomem; 54 55 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, 56 vm->pte_encode(addr, pat_index, flags)); 57 } 58 59 static void dpt_insert_entries(struct i915_address_space *vm, 60 struct i915_vma_resource *vma_res, 61 unsigned int pat_index, 62 u32 flags) 63 { 64 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 65 gen8_pte_t __iomem *base = dpt->iomem; 66 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); 67 struct sgt_iter sgt_iter; 68 dma_addr_t addr; 69 int i; 70 71 /* 72 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 73 * not to allow the user to override access to a read only page. 74 */ 75 76 i = vma_res->start / I915_GTT_PAGE_SIZE; 77 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) 78 gen8_set_pte(&base[i++], pte_encode | addr); 79 } 80 81 static void dpt_clear_range(struct i915_address_space *vm, 82 u64 start, u64 length) 83 { 84 } 85 86 static void dpt_bind_vma(struct i915_address_space *vm, 87 struct i915_vm_pt_stash *stash, 88 struct i915_vma_resource *vma_res, 89 unsigned int pat_index, 90 u32 flags) 91 { 92 u32 pte_flags; 93 94 if (vma_res->bound_flags) 95 return; 96 97 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 98 pte_flags = 0; 99 if (vm->has_read_only && vma_res->bi.readonly) 100 pte_flags |= PTE_READ_ONLY; 101 if (vma_res->bi.lmem) 102 pte_flags |= PTE_LM; 103 104 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 105 106 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 107 108 /* 109 * Without aliasing PPGTT there's no difference between 110 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 111 * upgrade to both bound if we bind either to avoid double-binding. 112 */ 113 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 114 } 115 116 static void dpt_unbind_vma(struct i915_address_space *vm, 117 struct i915_vma_resource *vma_res) 118 { 119 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 120 } 121 122 static void dpt_cleanup(struct i915_address_space *vm) 123 { 124 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 125 126 i915_gem_object_put(dpt->obj); 127 } 128 129 struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignment) 130 { 131 struct drm_i915_private *i915 = dpt->vm.i915; 132 struct ref_tracker *wakeref; 133 struct i915_vma *vma; 134 void __iomem *iomem; 135 struct i915_gem_ww_ctx ww; 136 u64 pin_flags = 0; 137 int err; 138 139 if (i915_gem_object_is_stolen(dpt->obj)) 140 pin_flags |= PIN_MAPPABLE; 141 142 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 143 atomic_inc(&i915->pending_fb_pin); 144 145 for_i915_gem_ww(&ww, err, true) { 146 err = i915_gem_object_lock(dpt->obj, &ww); 147 if (err) 148 continue; 149 150 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 151 alignment, pin_flags); 152 if (IS_ERR(vma)) { 153 err = PTR_ERR(vma); 154 continue; 155 } 156 157 iomem = i915_vma_pin_iomap(vma); 158 i915_vma_unpin(vma); 159 160 if (IS_ERR(iomem)) { 161 err = PTR_ERR(iomem); 162 continue; 163 } 164 165 dpt->vma = vma; 166 dpt->iomem = iomem; 167 168 i915_vma_get(vma); 169 } 170 171 dpt->obj->mm.dirty = true; 172 173 atomic_dec(&i915->pending_fb_pin); 174 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 175 176 return err ? ERR_PTR(err) : vma; 177 } 178 179 void i915_dpt_unpin_from_ggtt(struct intel_dpt *dpt) 180 { 181 i915_vma_unpin_iomap(dpt->vma); 182 i915_vma_put(dpt->vma); 183 } 184 185 static struct intel_dpt *i915_dpt_create(struct drm_gem_object *obj, size_t size) 186 { 187 struct drm_i915_private *i915 = to_i915(obj->dev); 188 struct drm_i915_gem_object *dpt_obj; 189 struct i915_address_space *vm; 190 struct intel_dpt *dpt; 191 int ret; 192 193 if (!size) 194 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); 195 196 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); 197 198 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); 199 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 200 dpt_obj = i915_gem_object_create_stolen(i915, size); 201 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) { 202 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n"); 203 dpt_obj = i915_gem_object_create_shmem(i915, size); 204 } 205 if (IS_ERR(dpt_obj)) 206 return ERR_CAST(dpt_obj); 207 208 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); 209 if (!ret) { 210 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); 211 i915_gem_object_unlock(dpt_obj); 212 } 213 if (ret) { 214 i915_gem_object_put(dpt_obj); 215 return ERR_PTR(ret); 216 } 217 218 dpt = kzalloc_obj(*dpt); 219 if (!dpt) { 220 i915_gem_object_put(dpt_obj); 221 return ERR_PTR(-ENOMEM); 222 } 223 224 vm = &dpt->vm; 225 226 vm->gt = to_gt(i915); 227 vm->i915 = i915; 228 vm->dma = i915->drm.dev; 229 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 230 vm->is_dpt = true; 231 232 i915_address_space_init(vm, VM_CLASS_DPT); 233 234 vm->insert_page = dpt_insert_page; 235 vm->clear_range = dpt_clear_range; 236 vm->insert_entries = dpt_insert_entries; 237 vm->cleanup = dpt_cleanup; 238 239 vm->vma_ops.bind_vma = dpt_bind_vma; 240 vm->vma_ops.unbind_vma = dpt_unbind_vma; 241 242 vm->pte_encode = vm->gt->ggtt->vm.pte_encode; 243 244 dpt->obj = dpt_obj; 245 dpt->obj->is_dpt = true; 246 247 return dpt; 248 } 249 250 static void i915_dpt_destroy(struct intel_dpt *dpt) 251 { 252 dpt->obj->is_dpt = false; 253 i915_vm_put(&dpt->vm); 254 } 255 256 static void i915_dpt_suspend(struct intel_dpt *dpt) 257 { 258 i915_ggtt_suspend_vm(&dpt->vm, true); 259 } 260 261 static void i915_dpt_resume(struct intel_dpt *dpt) 262 { 263 i915_ggtt_resume_vm(&dpt->vm, true); 264 } 265 266 u64 i915_dpt_offset(struct i915_vma *dpt_vma) 267 { 268 return i915_vma_offset(dpt_vma); 269 } 270 271 const struct intel_display_dpt_interface i915_display_dpt_interface = { 272 .create = i915_dpt_create, 273 .destroy = i915_dpt_destroy, 274 .suspend = i915_dpt_suspend, 275 .resume = i915_dpt_resume, 276 }; 277