1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_domain.h" 7 #include "gem/i915_gem_internal.h" 8 #include "gem/i915_gem_lmem.h" 9 #include "gt/gen8_ppgtt.h" 10 11 #include "i915_drv.h" 12 #include "intel_display_core.h" 13 #include "intel_display_rpm.h" 14 #include "intel_display_types.h" 15 #include "intel_dpt.h" 16 #include "intel_fb.h" 17 18 struct i915_dpt { 19 struct i915_address_space vm; 20 21 struct drm_i915_gem_object *obj; 22 struct i915_vma *vma; 23 void __iomem *iomem; 24 }; 25 26 #define i915_is_dpt(vm) ((vm)->is_dpt) 27 28 static inline struct i915_dpt * 29 i915_vm_to_dpt(struct i915_address_space *vm) 30 { 31 BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); 32 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm)); 33 return container_of(vm, struct i915_dpt, vm); 34 } 35 36 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) 37 38 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 39 { 40 writeq(pte, addr); 41 } 42 43 static void dpt_insert_page(struct i915_address_space *vm, 44 dma_addr_t addr, 45 u64 offset, 46 unsigned int pat_index, 47 u32 flags) 48 { 49 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 50 gen8_pte_t __iomem *base = dpt->iomem; 51 52 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, 53 vm->pte_encode(addr, pat_index, flags)); 54 } 55 56 static void dpt_insert_entries(struct i915_address_space *vm, 57 struct i915_vma_resource *vma_res, 58 unsigned int pat_index, 59 u32 flags) 60 { 61 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 62 gen8_pte_t __iomem *base = dpt->iomem; 63 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); 64 struct sgt_iter sgt_iter; 65 dma_addr_t addr; 66 int i; 67 68 /* 69 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 70 * not to allow the user to override access to a read only page. 71 */ 72 73 i = vma_res->start / I915_GTT_PAGE_SIZE; 74 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) 75 gen8_set_pte(&base[i++], pte_encode | addr); 76 } 77 78 static void dpt_clear_range(struct i915_address_space *vm, 79 u64 start, u64 length) 80 { 81 } 82 83 static void dpt_bind_vma(struct i915_address_space *vm, 84 struct i915_vm_pt_stash *stash, 85 struct i915_vma_resource *vma_res, 86 unsigned int pat_index, 87 u32 flags) 88 { 89 u32 pte_flags; 90 91 if (vma_res->bound_flags) 92 return; 93 94 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 95 pte_flags = 0; 96 if (vm->has_read_only && vma_res->bi.readonly) 97 pte_flags |= PTE_READ_ONLY; 98 if (vma_res->bi.lmem) 99 pte_flags |= PTE_LM; 100 101 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 102 103 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 104 105 /* 106 * Without aliasing PPGTT there's no difference between 107 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 108 * upgrade to both bound if we bind either to avoid double-binding. 109 */ 110 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 111 } 112 113 static void dpt_unbind_vma(struct i915_address_space *vm, 114 struct i915_vma_resource *vma_res) 115 { 116 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 117 } 118 119 static void dpt_cleanup(struct i915_address_space *vm) 120 { 121 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 122 123 i915_gem_object_put(dpt->obj); 124 } 125 126 struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm, 127 unsigned int alignment) 128 { 129 struct drm_i915_private *i915 = vm->i915; 130 struct intel_display *display = i915->display; 131 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 132 struct ref_tracker *wakeref; 133 struct i915_vma *vma; 134 void __iomem *iomem; 135 struct i915_gem_ww_ctx ww; 136 u64 pin_flags = 0; 137 int err; 138 139 if (i915_gem_object_is_stolen(dpt->obj)) 140 pin_flags |= PIN_MAPPABLE; 141 142 wakeref = intel_display_rpm_get(display); 143 atomic_inc(&display->restore.pending_fb_pin); 144 145 for_i915_gem_ww(&ww, err, true) { 146 err = i915_gem_object_lock(dpt->obj, &ww); 147 if (err) 148 continue; 149 150 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 151 alignment, pin_flags); 152 if (IS_ERR(vma)) { 153 err = PTR_ERR(vma); 154 continue; 155 } 156 157 iomem = i915_vma_pin_iomap(vma); 158 i915_vma_unpin(vma); 159 160 if (IS_ERR(iomem)) { 161 err = PTR_ERR(iomem); 162 continue; 163 } 164 165 dpt->vma = vma; 166 dpt->iomem = iomem; 167 168 i915_vma_get(vma); 169 } 170 171 dpt->obj->mm.dirty = true; 172 173 atomic_dec(&display->restore.pending_fb_pin); 174 intel_display_rpm_put(display, wakeref); 175 176 return err ? ERR_PTR(err) : vma; 177 } 178 179 void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm) 180 { 181 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 182 183 i915_vma_unpin_iomap(dpt->vma); 184 i915_vma_put(dpt->vma); 185 } 186 187 /** 188 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume 189 * @display: display device instance 190 * 191 * Restore the memory mapping during system resume for all framebuffers which 192 * are mapped to HW via a GGTT->DPT page table. The content of these page 193 * tables are not stored in the hibernation image during S4 and S3RST->S4 194 * transitions, so here we reprogram the PTE entries in those tables. 195 * 196 * This function must be called after the mappings in GGTT have been restored calling 197 * i915_ggtt_resume(). 198 */ 199 void intel_dpt_resume(struct intel_display *display) 200 { 201 struct drm_framebuffer *drm_fb; 202 203 if (!HAS_DISPLAY(display)) 204 return; 205 206 mutex_lock(&display->drm->mode_config.fb_lock); 207 drm_for_each_fb(drm_fb, display->drm) { 208 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 209 210 if (fb->dpt_vm) 211 i915_ggtt_resume_vm(fb->dpt_vm, true); 212 } 213 mutex_unlock(&display->drm->mode_config.fb_lock); 214 } 215 216 /** 217 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend 218 * @display: display device instance 219 * 220 * Suspend the memory mapping during system suspend for all framebuffers which 221 * are mapped to HW via a GGTT->DPT page table. 222 * 223 * This function must be called before the mappings in GGTT are suspended calling 224 * i915_ggtt_suspend(). 225 */ 226 void intel_dpt_suspend(struct intel_display *display) 227 { 228 struct drm_framebuffer *drm_fb; 229 230 if (!HAS_DISPLAY(display)) 231 return; 232 233 mutex_lock(&display->drm->mode_config.fb_lock); 234 235 drm_for_each_fb(drm_fb, display->drm) { 236 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 237 238 if (fb->dpt_vm) 239 i915_ggtt_suspend_vm(fb->dpt_vm, true); 240 } 241 242 mutex_unlock(&display->drm->mode_config.fb_lock); 243 } 244 245 struct i915_address_space * 246 intel_dpt_create(struct intel_framebuffer *fb) 247 { 248 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 249 struct drm_i915_private *i915 = to_i915(obj->dev); 250 struct drm_i915_gem_object *dpt_obj; 251 struct i915_address_space *vm; 252 struct i915_dpt *dpt; 253 size_t size; 254 int ret; 255 256 if (intel_fb_needs_pot_stride_remap(fb)) 257 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped); 258 else 259 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); 260 261 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); 262 263 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); 264 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 265 dpt_obj = i915_gem_object_create_stolen(i915, size); 266 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) { 267 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n"); 268 dpt_obj = i915_gem_object_create_shmem(i915, size); 269 } 270 if (IS_ERR(dpt_obj)) 271 return ERR_CAST(dpt_obj); 272 273 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); 274 if (!ret) { 275 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); 276 i915_gem_object_unlock(dpt_obj); 277 } 278 if (ret) { 279 i915_gem_object_put(dpt_obj); 280 return ERR_PTR(ret); 281 } 282 283 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL); 284 if (!dpt) { 285 i915_gem_object_put(dpt_obj); 286 return ERR_PTR(-ENOMEM); 287 } 288 289 vm = &dpt->vm; 290 291 vm->gt = to_gt(i915); 292 vm->i915 = i915; 293 vm->dma = i915->drm.dev; 294 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 295 vm->is_dpt = true; 296 297 i915_address_space_init(vm, VM_CLASS_DPT); 298 299 vm->insert_page = dpt_insert_page; 300 vm->clear_range = dpt_clear_range; 301 vm->insert_entries = dpt_insert_entries; 302 vm->cleanup = dpt_cleanup; 303 304 vm->vma_ops.bind_vma = dpt_bind_vma; 305 vm->vma_ops.unbind_vma = dpt_unbind_vma; 306 307 vm->pte_encode = vm->gt->ggtt->vm.pte_encode; 308 309 dpt->obj = dpt_obj; 310 dpt->obj->is_dpt = true; 311 312 return &dpt->vm; 313 } 314 315 void intel_dpt_destroy(struct i915_address_space *vm) 316 { 317 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 318 319 dpt->obj->is_dpt = false; 320 i915_vm_put(&dpt->vm); 321 } 322 323 u64 intel_dpt_offset(struct i915_vma *dpt_vma) 324 { 325 return dpt_vma->node.start; 326 } 327