1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_domain.h" 7 #include "gt/gen8_ppgtt.h" 8 9 #include "i915_drv.h" 10 #include "intel_display_types.h" 11 #include "intel_dpt.h" 12 #include "intel_fb.h" 13 14 struct i915_dpt { 15 struct i915_address_space vm; 16 17 struct drm_i915_gem_object *obj; 18 struct i915_vma *vma; 19 void __iomem *iomem; 20 }; 21 22 #define i915_is_dpt(vm) ((vm)->is_dpt) 23 24 static inline struct i915_dpt * 25 i915_vm_to_dpt(struct i915_address_space *vm) 26 { 27 BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); 28 GEM_BUG_ON(!i915_is_dpt(vm)); 29 return container_of(vm, struct i915_dpt, vm); 30 } 31 32 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) 33 34 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 35 { 36 writeq(pte, addr); 37 } 38 39 static void dpt_insert_page(struct i915_address_space *vm, 40 dma_addr_t addr, 41 u64 offset, 42 enum i915_cache_level level, 43 u32 flags) 44 { 45 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 46 gen8_pte_t __iomem *base = dpt->iomem; 47 48 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, 49 vm->pte_encode(addr, level, flags)); 50 } 51 52 static void dpt_insert_entries(struct i915_address_space *vm, 53 struct i915_vma_resource *vma_res, 54 enum i915_cache_level level, 55 u32 flags) 56 { 57 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 58 gen8_pte_t __iomem *base = dpt->iomem; 59 const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags); 60 struct sgt_iter sgt_iter; 61 dma_addr_t addr; 62 int i; 63 64 /* 65 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 66 * not to allow the user to override access to a read only page. 67 */ 68 69 i = vma_res->start / I915_GTT_PAGE_SIZE; 70 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) 71 gen8_set_pte(&base[i++], pte_encode | addr); 72 } 73 74 static void dpt_clear_range(struct i915_address_space *vm, 75 u64 start, u64 length) 76 { 77 } 78 79 static void dpt_bind_vma(struct i915_address_space *vm, 80 struct i915_vm_pt_stash *stash, 81 struct i915_vma_resource *vma_res, 82 enum i915_cache_level cache_level, 83 u32 flags) 84 { 85 u32 pte_flags; 86 87 if (vma_res->bound_flags) 88 return; 89 90 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 91 pte_flags = 0; 92 if (vm->has_read_only && vma_res->bi.readonly) 93 pte_flags |= PTE_READ_ONLY; 94 if (vma_res->bi.lmem) 95 pte_flags |= PTE_LM; 96 97 vm->insert_entries(vm, vma_res, cache_level, pte_flags); 98 99 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 100 101 /* 102 * Without aliasing PPGTT there's no difference between 103 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 104 * upgrade to both bound if we bind either to avoid double-binding. 105 */ 106 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 107 } 108 109 static void dpt_unbind_vma(struct i915_address_space *vm, 110 struct i915_vma_resource *vma_res) 111 { 112 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 113 } 114 115 static void dpt_cleanup(struct i915_address_space *vm) 116 { 117 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 118 119 i915_gem_object_put(dpt->obj); 120 } 121 122 struct i915_vma *intel_dpt_pin(struct i915_address_space *vm) 123 { 124 struct drm_i915_private *i915 = vm->i915; 125 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 126 intel_wakeref_t wakeref; 127 struct i915_vma *vma; 128 void __iomem *iomem; 129 struct i915_gem_ww_ctx ww; 130 int err; 131 132 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 133 atomic_inc(&i915->gpu_error.pending_fb_pin); 134 135 for_i915_gem_ww(&ww, err, true) { 136 err = i915_gem_object_lock(dpt->obj, &ww); 137 if (err) 138 continue; 139 140 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096, 141 HAS_LMEM(i915) ? 0 : PIN_MAPPABLE); 142 if (IS_ERR(vma)) { 143 err = PTR_ERR(vma); 144 continue; 145 } 146 147 iomem = i915_vma_pin_iomap(vma); 148 i915_vma_unpin(vma); 149 150 if (IS_ERR(iomem)) { 151 err = PTR_ERR(iomem); 152 continue; 153 } 154 155 dpt->vma = vma; 156 dpt->iomem = iomem; 157 158 i915_vma_get(vma); 159 } 160 161 atomic_dec(&i915->gpu_error.pending_fb_pin); 162 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 163 164 return err ? ERR_PTR(err) : vma; 165 } 166 167 void intel_dpt_unpin(struct i915_address_space *vm) 168 { 169 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 170 171 i915_vma_unpin_iomap(dpt->vma); 172 i915_vma_put(dpt->vma); 173 } 174 175 /** 176 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume 177 * @i915: device instance 178 * 179 * Restore the memory mapping during system resume for all framebuffers which 180 * are mapped to HW via a GGTT->DPT page table. The content of these page 181 * tables are not stored in the hibernation image during S4 and S3RST->S4 182 * transitions, so here we reprogram the PTE entries in those tables. 183 * 184 * This function must be called after the mappings in GGTT have been restored calling 185 * i915_ggtt_resume(). 186 */ 187 void intel_dpt_resume(struct drm_i915_private *i915) 188 { 189 struct drm_framebuffer *drm_fb; 190 191 if (!HAS_DISPLAY(i915)) 192 return; 193 194 mutex_lock(&i915->drm.mode_config.fb_lock); 195 drm_for_each_fb(drm_fb, &i915->drm) { 196 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 197 198 if (fb->dpt_vm) 199 i915_ggtt_resume_vm(fb->dpt_vm); 200 } 201 mutex_unlock(&i915->drm.mode_config.fb_lock); 202 } 203 204 /** 205 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend 206 * @i915: device instance 207 * 208 * Suspend the memory mapping during system suspend for all framebuffers which 209 * are mapped to HW via a GGTT->DPT page table. 210 * 211 * This function must be called before the mappings in GGTT are suspended calling 212 * i915_ggtt_suspend(). 213 */ 214 void intel_dpt_suspend(struct drm_i915_private *i915) 215 { 216 struct drm_framebuffer *drm_fb; 217 218 if (!HAS_DISPLAY(i915)) 219 return; 220 221 mutex_lock(&i915->drm.mode_config.fb_lock); 222 223 drm_for_each_fb(drm_fb, &i915->drm) { 224 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 225 226 if (fb->dpt_vm) 227 i915_ggtt_suspend_vm(fb->dpt_vm); 228 } 229 230 mutex_unlock(&i915->drm.mode_config.fb_lock); 231 } 232 233 struct i915_address_space * 234 intel_dpt_create(struct intel_framebuffer *fb) 235 { 236 struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base; 237 struct drm_i915_private *i915 = to_i915(obj->dev); 238 struct drm_i915_gem_object *dpt_obj; 239 struct i915_address_space *vm; 240 struct i915_dpt *dpt; 241 size_t size; 242 int ret; 243 244 if (intel_fb_needs_pot_stride_remap(fb)) 245 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped); 246 else 247 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); 248 249 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); 250 251 if (HAS_LMEM(i915)) 252 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); 253 else 254 dpt_obj = i915_gem_object_create_stolen(i915, size); 255 if (IS_ERR(dpt_obj)) 256 return ERR_CAST(dpt_obj); 257 258 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); 259 if (!ret) { 260 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); 261 i915_gem_object_unlock(dpt_obj); 262 } 263 if (ret) { 264 i915_gem_object_put(dpt_obj); 265 return ERR_PTR(ret); 266 } 267 268 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL); 269 if (!dpt) { 270 i915_gem_object_put(dpt_obj); 271 return ERR_PTR(-ENOMEM); 272 } 273 274 vm = &dpt->vm; 275 276 vm->gt = to_gt(i915); 277 vm->i915 = i915; 278 vm->dma = i915->drm.dev; 279 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 280 vm->is_dpt = true; 281 282 i915_address_space_init(vm, VM_CLASS_DPT); 283 284 vm->insert_page = dpt_insert_page; 285 vm->clear_range = dpt_clear_range; 286 vm->insert_entries = dpt_insert_entries; 287 vm->cleanup = dpt_cleanup; 288 289 vm->vma_ops.bind_vma = dpt_bind_vma; 290 vm->vma_ops.unbind_vma = dpt_unbind_vma; 291 292 vm->pte_encode = gen8_ggtt_pte_encode; 293 294 dpt->obj = dpt_obj; 295 296 return &dpt->vm; 297 } 298 299 void intel_dpt_destroy(struct i915_address_space *vm) 300 { 301 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 302 303 i915_vm_put(&dpt->vm); 304 } 305