1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/slab.h> 7 8 #include "i915_trace.h" 9 #include "intel_gtt.h" 10 #include "gen6_ppgtt.h" 11 #include "gen8_ppgtt.h" 12 13 struct i915_page_table *alloc_pt(struct i915_address_space *vm) 14 { 15 struct i915_page_table *pt; 16 17 pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL); 18 if (unlikely(!pt)) 19 return ERR_PTR(-ENOMEM); 20 21 pt->base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); 22 if (IS_ERR(pt->base)) { 23 kfree(pt); 24 return ERR_PTR(-ENOMEM); 25 } 26 27 atomic_set(&pt->used, 0); 28 return pt; 29 } 30 31 struct i915_page_directory *__alloc_pd(int count) 32 { 33 struct i915_page_directory *pd; 34 35 pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); 36 if (unlikely(!pd)) 37 return NULL; 38 39 pd->entry = kcalloc(count, sizeof(*pd->entry), I915_GFP_ALLOW_FAIL); 40 if (unlikely(!pd->entry)) { 41 kfree(pd); 42 return NULL; 43 } 44 45 spin_lock_init(&pd->lock); 46 return pd; 47 } 48 49 struct i915_page_directory *alloc_pd(struct i915_address_space *vm) 50 { 51 struct i915_page_directory *pd; 52 53 pd = __alloc_pd(I915_PDES); 54 if (unlikely(!pd)) 55 return ERR_PTR(-ENOMEM); 56 57 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); 58 if (IS_ERR(pd->pt.base)) { 59 kfree(pd->entry); 60 kfree(pd); 61 return ERR_PTR(-ENOMEM); 62 } 63 64 return pd; 65 } 66 67 void free_px(struct i915_address_space *vm, struct i915_page_table *pt, int lvl) 68 { 69 BUILD_BUG_ON(offsetof(struct i915_page_directory, pt)); 70 71 if (lvl) { 72 struct i915_page_directory *pd = 73 container_of(pt, typeof(*pd), pt); 74 kfree(pd->entry); 75 } 76 77 if (pt->base) 78 i915_gem_object_put(pt->base); 79 80 kfree(pt); 81 } 82 83 static inline void 84 write_dma_entry(struct drm_i915_gem_object * const pdma, 85 const unsigned short idx, 86 const u64 encoded_entry) 87 { 88 u64 * const vaddr = kmap_atomic(__px_page(pdma)); 89 90 vaddr[idx] = encoded_entry; 91 clflush_cache_range(&vaddr[idx], sizeof(u64)); 92 kunmap_atomic(vaddr); 93 } 94 95 void 96 __set_pd_entry(struct i915_page_directory * const pd, 97 const unsigned short idx, 98 struct i915_page_table * const to, 99 u64 (*encode)(const dma_addr_t, const enum i915_cache_level)) 100 { 101 /* Each thread pre-pins the pd, and we may have a thread per pde. */ 102 GEM_BUG_ON(atomic_read(px_used(pd)) > NALLOC * I915_PDES); 103 104 atomic_inc(px_used(pd)); 105 pd->entry[idx] = to; 106 write_dma_entry(px_base(pd), idx, encode(px_dma(to), I915_CACHE_LLC)); 107 } 108 109 void 110 clear_pd_entry(struct i915_page_directory * const pd, 111 const unsigned short idx, 112 const struct drm_i915_gem_object * const scratch) 113 { 114 GEM_BUG_ON(atomic_read(px_used(pd)) == 0); 115 116 write_dma_entry(px_base(pd), idx, scratch->encode); 117 pd->entry[idx] = NULL; 118 atomic_dec(px_used(pd)); 119 } 120 121 bool 122 release_pd_entry(struct i915_page_directory * const pd, 123 const unsigned short idx, 124 struct i915_page_table * const pt, 125 const struct drm_i915_gem_object * const scratch) 126 { 127 bool free = false; 128 129 if (atomic_add_unless(&pt->used, -1, 1)) 130 return false; 131 132 spin_lock(&pd->lock); 133 if (atomic_dec_and_test(&pt->used)) { 134 clear_pd_entry(pd, idx, scratch); 135 free = true; 136 } 137 spin_unlock(&pd->lock); 138 139 return free; 140 } 141 142 int i915_ppgtt_init_hw(struct intel_gt *gt) 143 { 144 struct drm_i915_private *i915 = gt->i915; 145 146 gtt_write_workarounds(gt); 147 148 if (IS_GEN(i915, 6)) 149 gen6_ppgtt_enable(gt); 150 else if (IS_GEN(i915, 7)) 151 gen7_ppgtt_enable(gt); 152 153 return 0; 154 } 155 156 static struct i915_ppgtt * 157 __ppgtt_create(struct intel_gt *gt) 158 { 159 if (INTEL_GEN(gt->i915) < 8) 160 return gen6_ppgtt_create(gt); 161 else 162 return gen8_ppgtt_create(gt); 163 } 164 165 struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt) 166 { 167 struct i915_ppgtt *ppgtt; 168 169 ppgtt = __ppgtt_create(gt); 170 if (IS_ERR(ppgtt)) 171 return ppgtt; 172 173 trace_i915_ppgtt_create(&ppgtt->vm); 174 175 return ppgtt; 176 } 177 178 void ppgtt_bind_vma(struct i915_address_space *vm, 179 struct i915_vm_pt_stash *stash, 180 struct i915_vma *vma, 181 enum i915_cache_level cache_level, 182 u32 flags) 183 { 184 u32 pte_flags; 185 186 if (!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) { 187 vm->allocate_va_range(vm, stash, vma->node.start, vma->size); 188 set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)); 189 } 190 191 /* Applicable to VLV, and gen8+ */ 192 pte_flags = 0; 193 if (i915_gem_object_is_readonly(vma->obj)) 194 pte_flags |= PTE_READ_ONLY; 195 196 vm->insert_entries(vm, vma, cache_level, pte_flags); 197 wmb(); 198 } 199 200 void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) 201 { 202 if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) 203 vm->clear_range(vm, vma->node.start, vma->size); 204 } 205 206 static unsigned long pd_count(u64 size, int shift) 207 { 208 /* Beware later misalignment */ 209 return (size + 2 * (BIT_ULL(shift) - 1)) >> shift; 210 } 211 212 int i915_vm_alloc_pt_stash(struct i915_address_space *vm, 213 struct i915_vm_pt_stash *stash, 214 u64 size) 215 { 216 unsigned long count; 217 int shift, n; 218 219 shift = vm->pd_shift; 220 if (!shift) 221 return 0; 222 223 count = pd_count(size, shift); 224 while (count--) { 225 struct i915_page_table *pt; 226 227 pt = alloc_pt(vm); 228 if (IS_ERR(pt)) { 229 i915_vm_free_pt_stash(vm, stash); 230 return PTR_ERR(pt); 231 } 232 233 pt->stash = stash->pt[0]; 234 stash->pt[0] = pt; 235 } 236 237 for (n = 1; n < vm->top; n++) { 238 shift += ilog2(I915_PDES); /* Each PD holds 512 entries */ 239 count = pd_count(size, shift); 240 while (count--) { 241 struct i915_page_directory *pd; 242 243 pd = alloc_pd(vm); 244 if (IS_ERR(pd)) { 245 i915_vm_free_pt_stash(vm, stash); 246 return PTR_ERR(pd); 247 } 248 249 pd->pt.stash = stash->pt[1]; 250 stash->pt[1] = &pd->pt; 251 } 252 } 253 254 return 0; 255 } 256 257 int i915_vm_pin_pt_stash(struct i915_address_space *vm, 258 struct i915_vm_pt_stash *stash) 259 { 260 struct i915_page_table *pt; 261 int n, err; 262 263 for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { 264 for (pt = stash->pt[n]; pt; pt = pt->stash) { 265 err = pin_pt_dma(vm, pt->base); 266 if (err) 267 return err; 268 } 269 } 270 271 return 0; 272 } 273 274 void i915_vm_free_pt_stash(struct i915_address_space *vm, 275 struct i915_vm_pt_stash *stash) 276 { 277 struct i915_page_table *pt; 278 int n; 279 280 for (n = 0; n < ARRAY_SIZE(stash->pt); n++) { 281 while ((pt = stash->pt[n])) { 282 stash->pt[n] = pt->stash; 283 free_px(vm, pt, n); 284 } 285 } 286 } 287 288 int ppgtt_set_pages(struct i915_vma *vma) 289 { 290 GEM_BUG_ON(vma->pages); 291 292 vma->pages = vma->obj->mm.pages; 293 vma->page_sizes = vma->obj->mm.page_sizes; 294 295 return 0; 296 } 297 298 void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt) 299 { 300 struct drm_i915_private *i915 = gt->i915; 301 302 ppgtt->vm.gt = gt; 303 ppgtt->vm.i915 = i915; 304 ppgtt->vm.dma = &i915->drm.pdev->dev; 305 ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size); 306 307 i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); 308 309 ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma; 310 ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma; 311 ppgtt->vm.vma_ops.set_pages = ppgtt_set_pages; 312 ppgtt->vm.vma_ops.clear_pages = clear_pages; 313 } 314