xref: /linux/drivers/gpu/drm/i915/gt/gen6_ppgtt.c (revision ca853314e78b0a65c20b6a889a23c31f918d4aa2) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include <linux/log2.h>
7 
8 #include "gen6_ppgtt.h"
9 #include "i915_scatterlist.h"
10 #include "i915_trace.h"
11 #include "i915_vgpu.h"
12 #include "intel_gt.h"
13 
14 /* Write pde (index) from the page directory @pd to the page table @pt */
15 static void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
16 			   const unsigned int pde,
17 			   const struct i915_page_table *pt)
18 {
19 	dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);
20 
21 	/* Caller needs to make sure the write completes if necessary */
22 	iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID,
23 		  ppgtt->pd_addr + pde);
24 }
25 
26 void gen7_ppgtt_enable(struct intel_gt *gt)
27 {
28 	struct drm_i915_private *i915 = gt->i915;
29 	struct intel_uncore *uncore = gt->uncore;
30 	u32 ecochk;
31 
32 	intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);
33 
34 	ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
35 	if (IS_HASWELL(i915)) {
36 		ecochk |= ECOCHK_PPGTT_WB_HSW;
37 	} else {
38 		ecochk |= ECOCHK_PPGTT_LLC_IVB;
39 		ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
40 	}
41 	intel_uncore_write(uncore, GAM_ECOCHK, ecochk);
42 }
43 
44 void gen6_ppgtt_enable(struct intel_gt *gt)
45 {
46 	struct intel_uncore *uncore = gt->uncore;
47 
48 	intel_uncore_rmw(uncore,
49 			 GAC_ECO_BITS,
50 			 0,
51 			 ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B);
52 
53 	intel_uncore_rmw(uncore,
54 			 GAB_CTL,
55 			 0,
56 			 GAB_CTL_CONT_AFTER_PAGEFAULT);
57 
58 	intel_uncore_rmw(uncore,
59 			 GAM_ECOCHK,
60 			 0,
61 			 ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
62 
63 	if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
64 		intel_uncore_write(uncore,
65 				   GFX_MODE,
66 				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
67 }
68 
69 /* PPGTT support for Sandybdrige/Gen6 and later */
70 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
71 				   u64 start, u64 length)
72 {
73 	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
74 	const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
75 	const gen6_pte_t scratch_pte = vm->scratch[0]->encode;
76 	unsigned int pde = first_entry / GEN6_PTES;
77 	unsigned int pte = first_entry % GEN6_PTES;
78 	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
79 
80 	while (num_entries) {
81 		struct i915_page_table * const pt =
82 			i915_pt_entry(ppgtt->base.pd, pde++);
83 		const unsigned int count = min(num_entries, GEN6_PTES - pte);
84 		gen6_pte_t *vaddr;
85 
86 		num_entries -= count;
87 
88 		GEM_BUG_ON(count > atomic_read(&pt->used));
89 		if (!atomic_sub_return(count, &pt->used))
90 			ppgtt->scan_for_unused_pt = true;
91 
92 		/*
93 		 * Note that the hw doesn't support removing PDE on the fly
94 		 * (they are cached inside the context with no means to
95 		 * invalidate the cache), so we can only reset the PTE
96 		 * entries back to scratch.
97 		 */
98 
99 		vaddr = kmap_atomic_px(pt);
100 		memset32(vaddr + pte, scratch_pte, count);
101 		kunmap_atomic(vaddr);
102 
103 		pte = 0;
104 	}
105 }
106 
107 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
108 				      struct i915_vma *vma,
109 				      enum i915_cache_level cache_level,
110 				      u32 flags)
111 {
112 	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
113 	struct i915_page_directory * const pd = ppgtt->pd;
114 	unsigned int first_entry = vma->node.start / I915_GTT_PAGE_SIZE;
115 	unsigned int act_pt = first_entry / GEN6_PTES;
116 	unsigned int act_pte = first_entry % GEN6_PTES;
117 	const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
118 	struct sgt_dma iter = sgt_dma(vma);
119 	gen6_pte_t *vaddr;
120 
121 	GEM_BUG_ON(!pd->entry[act_pt]);
122 
123 	vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
124 	do {
125 		GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);
126 		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
127 
128 		iter.dma += I915_GTT_PAGE_SIZE;
129 		if (iter.dma == iter.max) {
130 			iter.sg = __sg_next(iter.sg);
131 			if (!iter.sg || sg_dma_len(iter.sg) == 0)
132 				break;
133 
134 			iter.dma = sg_dma_address(iter.sg);
135 			iter.max = iter.dma + sg_dma_len(iter.sg);
136 		}
137 
138 		if (++act_pte == GEN6_PTES) {
139 			kunmap_atomic(vaddr);
140 			vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));
141 			act_pte = 0;
142 		}
143 	} while (1);
144 	kunmap_atomic(vaddr);
145 
146 	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
147 }
148 
149 static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end)
150 {
151 	struct i915_page_directory * const pd = ppgtt->base.pd;
152 	struct i915_page_table *pt;
153 	unsigned int pde;
154 
155 	start = round_down(start, SZ_64K);
156 	end = round_up(end, SZ_64K) - start;
157 
158 	mutex_lock(&ppgtt->flush);
159 
160 	gen6_for_each_pde(pt, pd, start, end, pde)
161 		gen6_write_pde(ppgtt, pde, pt);
162 
163 	mb();
164 	ioread32(ppgtt->pd_addr + pde - 1);
165 	gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);
166 	mb();
167 
168 	mutex_unlock(&ppgtt->flush);
169 }
170 
171 static void gen6_alloc_va_range(struct i915_address_space *vm,
172 				struct i915_vm_pt_stash *stash,
173 				u64 start, u64 length)
174 {
175 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
176 	struct i915_page_directory * const pd = ppgtt->base.pd;
177 	struct i915_page_table *pt;
178 	bool flush = false;
179 	u64 from = start;
180 	unsigned int pde;
181 
182 	spin_lock(&pd->lock);
183 	gen6_for_each_pde(pt, pd, start, length, pde) {
184 		const unsigned int count = gen6_pte_count(start, length);
185 
186 		if (!pt) {
187 			spin_unlock(&pd->lock);
188 
189 			pt = stash->pt[0];
190 			__i915_gem_object_pin_pages(pt->base);
191 			i915_gem_object_make_unshrinkable(pt->base);
192 
193 			fill32_px(pt, vm->scratch[0]->encode);
194 
195 			spin_lock(&pd->lock);
196 			if (!pd->entry[pde]) {
197 				stash->pt[0] = pt->stash;
198 				atomic_set(&pt->used, 0);
199 				pd->entry[pde] = pt;
200 			} else {
201 				pt = pd->entry[pde];
202 			}
203 
204 			flush = true;
205 		}
206 
207 		atomic_add(count, &pt->used);
208 	}
209 	spin_unlock(&pd->lock);
210 
211 	if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {
212 		intel_wakeref_t wakeref;
213 
214 		with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)
215 			gen6_flush_pd(ppgtt, from, start);
216 	}
217 }
218 
219 static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
220 {
221 	struct i915_address_space * const vm = &ppgtt->base.vm;
222 	int ret;
223 
224 	ret = setup_scratch_page(vm);
225 	if (ret)
226 		return ret;
227 
228 	vm->scratch[0]->encode =
229 		vm->pte_encode(px_dma(vm->scratch[0]),
230 			       I915_CACHE_NONE, PTE_READ_ONLY);
231 
232 	vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
233 	if (IS_ERR(vm->scratch[1])) {
234 		ret = PTR_ERR(vm->scratch[1]);
235 		goto err_scratch0;
236 	}
237 
238 	ret = pin_pt_dma(vm, vm->scratch[1]);
239 	if (ret)
240 		goto err_scratch1;
241 
242 	fill32_px(vm->scratch[1], vm->scratch[0]->encode);
243 
244 	return 0;
245 
246 err_scratch1:
247 	i915_gem_object_put(vm->scratch[1]);
248 err_scratch0:
249 	i915_gem_object_put(vm->scratch[0]);
250 	return ret;
251 }
252 
253 static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
254 {
255 	struct i915_page_directory * const pd = ppgtt->base.pd;
256 	struct i915_page_table *pt;
257 	u32 pde;
258 
259 	gen6_for_all_pdes(pt, pd, pde)
260 		if (pt)
261 			free_pt(&ppgtt->base.vm, pt);
262 }
263 
264 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
265 {
266 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
267 
268 	__i915_vma_put(ppgtt->vma);
269 
270 	gen6_ppgtt_free_pd(ppgtt);
271 	free_scratch(vm);
272 
273 	mutex_destroy(&ppgtt->flush);
274 	mutex_destroy(&ppgtt->pin_mutex);
275 
276 	free_pd(&ppgtt->base.vm, ppgtt->base.pd);
277 }
278 
279 static int pd_vma_set_pages(struct i915_vma *vma)
280 {
281 	vma->pages = ERR_PTR(-ENODEV);
282 	return 0;
283 }
284 
285 static void pd_vma_clear_pages(struct i915_vma *vma)
286 {
287 	GEM_BUG_ON(!vma->pages);
288 
289 	vma->pages = NULL;
290 }
291 
292 static void pd_vma_bind(struct i915_address_space *vm,
293 			struct i915_vm_pt_stash *stash,
294 			struct i915_vma *vma,
295 			enum i915_cache_level cache_level,
296 			u32 unused)
297 {
298 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
299 	struct gen6_ppgtt *ppgtt = vma->private;
300 	u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE;
301 
302 	ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10;
303 	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
304 
305 	gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);
306 }
307 
308 static void pd_vma_unbind(struct i915_address_space *vm, struct i915_vma *vma)
309 {
310 	struct gen6_ppgtt *ppgtt = vma->private;
311 	struct i915_page_directory * const pd = ppgtt->base.pd;
312 	struct i915_page_table *pt;
313 	unsigned int pde;
314 
315 	if (!ppgtt->scan_for_unused_pt)
316 		return;
317 
318 	/* Free all no longer used page tables */
319 	gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
320 		if (!pt || atomic_read(&pt->used))
321 			continue;
322 
323 		free_pt(&ppgtt->base.vm, pt);
324 		pd->entry[pde] = NULL;
325 	}
326 
327 	ppgtt->scan_for_unused_pt = false;
328 }
329 
330 static const struct i915_vma_ops pd_vma_ops = {
331 	.set_pages = pd_vma_set_pages,
332 	.clear_pages = pd_vma_clear_pages,
333 	.bind_vma = pd_vma_bind,
334 	.unbind_vma = pd_vma_unbind,
335 };
336 
337 static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
338 {
339 	struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt;
340 	struct i915_vma *vma;
341 
342 	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
343 	GEM_BUG_ON(size > ggtt->vm.total);
344 
345 	vma = i915_vma_alloc();
346 	if (!vma)
347 		return ERR_PTR(-ENOMEM);
348 
349 	i915_active_init(&vma->active, NULL, NULL);
350 
351 	kref_init(&vma->ref);
352 	mutex_init(&vma->pages_mutex);
353 	vma->vm = i915_vm_get(&ggtt->vm);
354 	vma->ops = &pd_vma_ops;
355 	vma->private = ppgtt;
356 
357 	vma->size = size;
358 	vma->fence_size = size;
359 	atomic_set(&vma->flags, I915_VMA_GGTT);
360 	vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */
361 
362 	INIT_LIST_HEAD(&vma->obj_link);
363 	INIT_LIST_HEAD(&vma->closed_link);
364 
365 	return vma;
366 }
367 
368 int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww)
369 {
370 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
371 	int err;
372 
373 	GEM_BUG_ON(!atomic_read(&ppgtt->base.vm.open));
374 
375 	/*
376 	 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
377 	 * which will be pinned into every active context.
378 	 * (When vma->pin_count becomes atomic, I expect we will naturally
379 	 * need a larger, unpacked, type and kill this redundancy.)
380 	 */
381 	if (atomic_add_unless(&ppgtt->pin_count, 1, 0))
382 		return 0;
383 
384 	if (mutex_lock_interruptible(&ppgtt->pin_mutex))
385 		return -EINTR;
386 
387 	/*
388 	 * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
389 	 * allocator works in address space sizes, so it's multiplied by page
390 	 * size. We allocate at the top of the GTT to avoid fragmentation.
391 	 */
392 	err = 0;
393 	if (!atomic_read(&ppgtt->pin_count))
394 		err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH);
395 	if (!err)
396 		atomic_inc(&ppgtt->pin_count);
397 	mutex_unlock(&ppgtt->pin_mutex);
398 
399 	return err;
400 }
401 
402 void gen6_ppgtt_unpin(struct i915_ppgtt *base)
403 {
404 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
405 
406 	GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));
407 	if (atomic_dec_and_test(&ppgtt->pin_count))
408 		i915_vma_unpin(ppgtt->vma);
409 }
410 
411 void gen6_ppgtt_unpin_all(struct i915_ppgtt *base)
412 {
413 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
414 
415 	if (!atomic_read(&ppgtt->pin_count))
416 		return;
417 
418 	i915_vma_unpin(ppgtt->vma);
419 	atomic_set(&ppgtt->pin_count, 0);
420 }
421 
422 struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt)
423 {
424 	struct i915_ggtt * const ggtt = gt->ggtt;
425 	struct gen6_ppgtt *ppgtt;
426 	int err;
427 
428 	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
429 	if (!ppgtt)
430 		return ERR_PTR(-ENOMEM);
431 
432 	mutex_init(&ppgtt->flush);
433 	mutex_init(&ppgtt->pin_mutex);
434 
435 	ppgtt_init(&ppgtt->base, gt);
436 	ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));
437 	ppgtt->base.vm.top = 1;
438 
439 	ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;
440 	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
441 	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
442 	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
443 	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
444 
445 	ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;
446 	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
447 
448 	ppgtt->base.pd = __alloc_pd(I915_PDES);
449 	if (!ppgtt->base.pd) {
450 		err = -ENOMEM;
451 		goto err_free;
452 	}
453 
454 	err = gen6_ppgtt_init_scratch(ppgtt);
455 	if (err)
456 		goto err_pd;
457 
458 	ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
459 	if (IS_ERR(ppgtt->vma)) {
460 		err = PTR_ERR(ppgtt->vma);
461 		goto err_scratch;
462 	}
463 
464 	return &ppgtt->base;
465 
466 err_scratch:
467 	free_scratch(&ppgtt->base.vm);
468 err_pd:
469 	free_pd(&ppgtt->base.vm, ppgtt->base.pd);
470 err_free:
471 	mutex_destroy(&ppgtt->pin_mutex);
472 	kfree(ppgtt);
473 	return ERR_PTR(err);
474 }
475