1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/drm_print.h> 7 #include <drm/intel/display_parent_interface.h> 8 9 #include "display/intel_display_core.h" 10 #include "gem/i915_gem_domain.h" 11 #include "gem/i915_gem_internal.h" 12 #include "gem/i915_gem_lmem.h" 13 #include "gt/gen8_ppgtt.h" 14 15 #include "i915_dpt.h" 16 #include "i915_drv.h" 17 18 struct intel_dpt { 19 struct i915_address_space vm; 20 21 struct drm_i915_gem_object *obj; 22 struct i915_vma *vma; 23 void __iomem *iomem; 24 }; 25 26 #define i915_is_dpt(vm) ((vm)->is_dpt) 27 28 static inline struct intel_dpt * 29 i915_vm_to_dpt(struct i915_address_space *vm) 30 { 31 BUILD_BUG_ON(offsetof(struct intel_dpt, vm)); 32 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm)); 33 return container_of(vm, struct intel_dpt, vm); 34 } 35 36 struct i915_address_space *i915_dpt_to_vm(struct intel_dpt *dpt) 37 { 38 return &dpt->vm; 39 } 40 41 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 42 { 43 writeq(pte, addr); 44 } 45 46 static void dpt_insert_page(struct i915_address_space *vm, 47 dma_addr_t addr, 48 u64 offset, 49 unsigned int pat_index, 50 u32 flags) 51 { 52 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 53 gen8_pte_t __iomem *base = dpt->iomem; 54 55 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, 56 vm->pte_encode(addr, pat_index, flags)); 57 } 58 59 static void dpt_insert_entries(struct i915_address_space *vm, 60 struct i915_vma_resource *vma_res, 61 unsigned int pat_index, 62 u32 flags) 63 { 64 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 65 gen8_pte_t __iomem *base = dpt->iomem; 66 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); 67 struct sgt_iter sgt_iter; 68 dma_addr_t addr; 69 int i; 70 71 /* 72 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 73 * not to allow the user to override access to a read only page. 74 */ 75 76 i = vma_res->start / I915_GTT_PAGE_SIZE; 77 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) 78 gen8_set_pte(&base[i++], pte_encode | addr); 79 } 80 81 static void dpt_clear_range(struct i915_address_space *vm, 82 u64 start, u64 length) 83 { 84 } 85 86 static void dpt_bind_vma(struct i915_address_space *vm, 87 struct i915_vm_pt_stash *stash, 88 struct i915_vma_resource *vma_res, 89 unsigned int pat_index, 90 u32 flags) 91 { 92 u32 pte_flags; 93 94 if (vma_res->bound_flags) 95 return; 96 97 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 98 pte_flags = 0; 99 if (vm->has_read_only && vma_res->bi.readonly) 100 pte_flags |= PTE_READ_ONLY; 101 if (vma_res->bi.lmem) 102 pte_flags |= PTE_LM; 103 104 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 105 106 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 107 108 /* 109 * Without aliasing PPGTT there's no difference between 110 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 111 * upgrade to both bound if we bind either to avoid double-binding. 112 */ 113 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 114 } 115 116 static void dpt_unbind_vma(struct i915_address_space *vm, 117 struct i915_vma_resource *vma_res) 118 { 119 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 120 } 121 122 static void dpt_cleanup(struct i915_address_space *vm) 123 { 124 struct intel_dpt *dpt = i915_vm_to_dpt(vm); 125 126 i915_gem_object_put(dpt->obj); 127 } 128 129 struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignment) 130 { 131 struct drm_i915_private *i915 = dpt->vm.i915; 132 struct intel_display *display = i915->display; 133 struct ref_tracker *wakeref; 134 struct i915_vma *vma; 135 void __iomem *iomem; 136 struct i915_gem_ww_ctx ww; 137 u64 pin_flags = 0; 138 int err; 139 140 if (i915_gem_object_is_stolen(dpt->obj)) 141 pin_flags |= PIN_MAPPABLE; 142 143 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 144 atomic_inc(&display->restore.pending_fb_pin); 145 146 for_i915_gem_ww(&ww, err, true) { 147 err = i915_gem_object_lock(dpt->obj, &ww); 148 if (err) 149 continue; 150 151 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 152 alignment, pin_flags); 153 if (IS_ERR(vma)) { 154 err = PTR_ERR(vma); 155 continue; 156 } 157 158 iomem = i915_vma_pin_iomap(vma); 159 i915_vma_unpin(vma); 160 161 if (IS_ERR(iomem)) { 162 err = PTR_ERR(iomem); 163 continue; 164 } 165 166 dpt->vma = vma; 167 dpt->iomem = iomem; 168 169 i915_vma_get(vma); 170 } 171 172 dpt->obj->mm.dirty = true; 173 174 atomic_dec(&display->restore.pending_fb_pin); 175 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 176 177 return err ? ERR_PTR(err) : vma; 178 } 179 180 void i915_dpt_unpin_from_ggtt(struct intel_dpt *dpt) 181 { 182 i915_vma_unpin_iomap(dpt->vma); 183 i915_vma_put(dpt->vma); 184 } 185 186 static struct intel_dpt *i915_dpt_create(struct drm_gem_object *obj, size_t size) 187 { 188 struct drm_i915_private *i915 = to_i915(obj->dev); 189 struct drm_i915_gem_object *dpt_obj; 190 struct i915_address_space *vm; 191 struct intel_dpt *dpt; 192 int ret; 193 194 if (!size) 195 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); 196 197 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); 198 199 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); 200 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 201 dpt_obj = i915_gem_object_create_stolen(i915, size); 202 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) { 203 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n"); 204 dpt_obj = i915_gem_object_create_shmem(i915, size); 205 } 206 if (IS_ERR(dpt_obj)) 207 return ERR_CAST(dpt_obj); 208 209 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); 210 if (!ret) { 211 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); 212 i915_gem_object_unlock(dpt_obj); 213 } 214 if (ret) { 215 i915_gem_object_put(dpt_obj); 216 return ERR_PTR(ret); 217 } 218 219 dpt = kzalloc_obj(*dpt); 220 if (!dpt) { 221 i915_gem_object_put(dpt_obj); 222 return ERR_PTR(-ENOMEM); 223 } 224 225 vm = &dpt->vm; 226 227 vm->gt = to_gt(i915); 228 vm->i915 = i915; 229 vm->dma = i915->drm.dev; 230 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 231 vm->is_dpt = true; 232 233 i915_address_space_init(vm, VM_CLASS_DPT); 234 235 vm->insert_page = dpt_insert_page; 236 vm->clear_range = dpt_clear_range; 237 vm->insert_entries = dpt_insert_entries; 238 vm->cleanup = dpt_cleanup; 239 240 vm->vma_ops.bind_vma = dpt_bind_vma; 241 vm->vma_ops.unbind_vma = dpt_unbind_vma; 242 243 vm->pte_encode = vm->gt->ggtt->vm.pte_encode; 244 245 dpt->obj = dpt_obj; 246 dpt->obj->is_dpt = true; 247 248 return dpt; 249 } 250 251 static void i915_dpt_destroy(struct intel_dpt *dpt) 252 { 253 dpt->obj->is_dpt = false; 254 i915_vm_put(&dpt->vm); 255 } 256 257 static void i915_dpt_suspend(struct intel_dpt *dpt) 258 { 259 i915_ggtt_suspend_vm(&dpt->vm, true); 260 } 261 262 static void i915_dpt_resume(struct intel_dpt *dpt) 263 { 264 i915_ggtt_resume_vm(&dpt->vm, true); 265 } 266 267 u64 i915_dpt_offset(struct i915_vma *dpt_vma) 268 { 269 return i915_vma_offset(dpt_vma); 270 } 271 272 const struct intel_display_dpt_interface i915_display_dpt_interface = { 273 .create = i915_dpt_create, 274 .destroy = i915_dpt_destroy, 275 .suspend = i915_dpt_suspend, 276 .resume = i915_dpt_resume, 277 }; 278