1a1d2a633SQiang Yu // SPDX-License-Identifier: GPL-2.0 OR MIT
2a1d2a633SQiang Yu /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3a1d2a633SQiang Yu
4a1d2a633SQiang Yu #include <linux/slab.h>
5a1d2a633SQiang Yu #include <linux/dma-mapping.h>
6a1d2a633SQiang Yu
7a1d2a633SQiang Yu #include "lima_device.h"
8a1d2a633SQiang Yu #include "lima_vm.h"
9d61dd248SQiang Yu #include "lima_gem.h"
10a1d2a633SQiang Yu #include "lima_regs.h"
11a1d2a633SQiang Yu
12a1d2a633SQiang Yu struct lima_bo_va {
13a1d2a633SQiang Yu struct list_head list;
14a1d2a633SQiang Yu unsigned int ref_count;
15a1d2a633SQiang Yu
16a1d2a633SQiang Yu struct drm_mm_node node;
17a1d2a633SQiang Yu
18a1d2a633SQiang Yu struct lima_vm *vm;
19a1d2a633SQiang Yu };
20a1d2a633SQiang Yu
21a1d2a633SQiang Yu #define LIMA_VM_PD_SHIFT 22
22a1d2a633SQiang Yu #define LIMA_VM_PT_SHIFT 12
23a1d2a633SQiang Yu #define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)
24a1d2a633SQiang Yu #define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT
25a1d2a633SQiang Yu
26a1d2a633SQiang Yu #define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
27a1d2a633SQiang Yu #define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
28a1d2a633SQiang Yu
29a1d2a633SQiang Yu #define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)
30a1d2a633SQiang Yu #define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)
31a1d2a633SQiang Yu #define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)
32a1d2a633SQiang Yu #define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)
33a1d2a633SQiang Yu
34a1d2a633SQiang Yu
lima_vm_unmap_range(struct lima_vm * vm,u32 start,u32 end)35d61dd248SQiang Yu static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end)
36a1d2a633SQiang Yu {
37a1d2a633SQiang Yu u32 addr;
38a1d2a633SQiang Yu
39a1d2a633SQiang Yu for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
40a1d2a633SQiang Yu u32 pbe = LIMA_PBE(addr);
41a1d2a633SQiang Yu u32 bte = LIMA_BTE(addr);
42a1d2a633SQiang Yu
43a1d2a633SQiang Yu vm->bts[pbe].cpu[bte] = 0;
44a1d2a633SQiang Yu }
45a1d2a633SQiang Yu }
46a1d2a633SQiang Yu
lima_vm_map_page(struct lima_vm * vm,dma_addr_t pa,u32 va)47d61dd248SQiang Yu static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va)
48a1d2a633SQiang Yu {
49d61dd248SQiang Yu u32 pbe = LIMA_PBE(va);
50d61dd248SQiang Yu u32 bte = LIMA_BTE(va);
51a1d2a633SQiang Yu
52a1d2a633SQiang Yu if (!vm->bts[pbe].cpu) {
53a1d2a633SQiang Yu dma_addr_t pts;
54a1d2a633SQiang Yu u32 *pd;
55a1d2a633SQiang Yu int j;
56a1d2a633SQiang Yu
57a1d2a633SQiang Yu vm->bts[pbe].cpu = dma_alloc_wc(
58a1d2a633SQiang Yu vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
59e30b38b7SQiang Yu &vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
60d61dd248SQiang Yu if (!vm->bts[pbe].cpu)
61a1d2a633SQiang Yu return -ENOMEM;
62a1d2a633SQiang Yu
63a1d2a633SQiang Yu pts = vm->bts[pbe].dma;
64a1d2a633SQiang Yu pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);
65a1d2a633SQiang Yu for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
66a1d2a633SQiang Yu pd[j] = pts | LIMA_VM_FLAG_PRESENT;
67a1d2a633SQiang Yu pts += LIMA_PAGE_SIZE;
68a1d2a633SQiang Yu }
69a1d2a633SQiang Yu }
70a1d2a633SQiang Yu
71d61dd248SQiang Yu vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE;
72a1d2a633SQiang Yu
73a1d2a633SQiang Yu return 0;
74a1d2a633SQiang Yu }
75a1d2a633SQiang Yu
76a1d2a633SQiang Yu static struct lima_bo_va *
lima_vm_bo_find(struct lima_vm * vm,struct lima_bo * bo)77a1d2a633SQiang Yu lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)
78a1d2a633SQiang Yu {
79a1d2a633SQiang Yu struct lima_bo_va *bo_va, *ret = NULL;
80a1d2a633SQiang Yu
81a1d2a633SQiang Yu list_for_each_entry(bo_va, &bo->va, list) {
82a1d2a633SQiang Yu if (bo_va->vm == vm) {
83a1d2a633SQiang Yu ret = bo_va;
84a1d2a633SQiang Yu break;
85a1d2a633SQiang Yu }
86a1d2a633SQiang Yu }
87a1d2a633SQiang Yu
88a1d2a633SQiang Yu return ret;
89a1d2a633SQiang Yu }
90a1d2a633SQiang Yu
lima_vm_bo_add(struct lima_vm * vm,struct lima_bo * bo,bool create)91a1d2a633SQiang Yu int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)
92a1d2a633SQiang Yu {
93a1d2a633SQiang Yu struct lima_bo_va *bo_va;
94d61dd248SQiang Yu struct sg_dma_page_iter sg_iter;
95d61dd248SQiang Yu int offset = 0, err;
96a1d2a633SQiang Yu
97a1d2a633SQiang Yu mutex_lock(&bo->lock);
98a1d2a633SQiang Yu
99a1d2a633SQiang Yu bo_va = lima_vm_bo_find(vm, bo);
100a1d2a633SQiang Yu if (bo_va) {
101a1d2a633SQiang Yu bo_va->ref_count++;
102a1d2a633SQiang Yu mutex_unlock(&bo->lock);
103a1d2a633SQiang Yu return 0;
104a1d2a633SQiang Yu }
105a1d2a633SQiang Yu
106a1d2a633SQiang Yu /* should not create new bo_va if not asked by caller */
107a1d2a633SQiang Yu if (!create) {
108a1d2a633SQiang Yu mutex_unlock(&bo->lock);
109a1d2a633SQiang Yu return -ENOENT;
110a1d2a633SQiang Yu }
111a1d2a633SQiang Yu
112a1d2a633SQiang Yu bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);
113a1d2a633SQiang Yu if (!bo_va) {
114a1d2a633SQiang Yu err = -ENOMEM;
115a1d2a633SQiang Yu goto err_out0;
116a1d2a633SQiang Yu }
117a1d2a633SQiang Yu
118a1d2a633SQiang Yu bo_va->vm = vm;
119a1d2a633SQiang Yu bo_va->ref_count = 1;
120a1d2a633SQiang Yu
121a1d2a633SQiang Yu mutex_lock(&vm->lock);
122a1d2a633SQiang Yu
123d61dd248SQiang Yu err = drm_mm_insert_node(&vm->mm, &bo_va->node, lima_bo_size(bo));
124a1d2a633SQiang Yu if (err)
125a1d2a633SQiang Yu goto err_out1;
126a1d2a633SQiang Yu
127*c3d9c17fSMarek Szyprowski for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) {
128d61dd248SQiang Yu err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),
129d61dd248SQiang Yu bo_va->node.start + offset);
130a1d2a633SQiang Yu if (err)
131a1d2a633SQiang Yu goto err_out2;
132a1d2a633SQiang Yu
133d61dd248SQiang Yu offset += PAGE_SIZE;
134d61dd248SQiang Yu }
135d61dd248SQiang Yu
136a1d2a633SQiang Yu mutex_unlock(&vm->lock);
137a1d2a633SQiang Yu
138a1d2a633SQiang Yu list_add_tail(&bo_va->list, &bo->va);
139a1d2a633SQiang Yu
140a1d2a633SQiang Yu mutex_unlock(&bo->lock);
141a1d2a633SQiang Yu return 0;
142a1d2a633SQiang Yu
143a1d2a633SQiang Yu err_out2:
144d61dd248SQiang Yu if (offset)
145d61dd248SQiang Yu lima_vm_unmap_range(vm, bo_va->node.start, bo_va->node.start + offset - 1);
146a1d2a633SQiang Yu drm_mm_remove_node(&bo_va->node);
147a1d2a633SQiang Yu err_out1:
148a1d2a633SQiang Yu mutex_unlock(&vm->lock);
149a1d2a633SQiang Yu kfree(bo_va);
150a1d2a633SQiang Yu err_out0:
151a1d2a633SQiang Yu mutex_unlock(&bo->lock);
152a1d2a633SQiang Yu return err;
153a1d2a633SQiang Yu }
154a1d2a633SQiang Yu
lima_vm_bo_del(struct lima_vm * vm,struct lima_bo * bo)155a1d2a633SQiang Yu void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)
156a1d2a633SQiang Yu {
157a1d2a633SQiang Yu struct lima_bo_va *bo_va;
1586aebc51dSQiang Yu u32 size;
159a1d2a633SQiang Yu
160a1d2a633SQiang Yu mutex_lock(&bo->lock);
161a1d2a633SQiang Yu
162a1d2a633SQiang Yu bo_va = lima_vm_bo_find(vm, bo);
163a1d2a633SQiang Yu if (--bo_va->ref_count > 0) {
164a1d2a633SQiang Yu mutex_unlock(&bo->lock);
165a1d2a633SQiang Yu return;
166a1d2a633SQiang Yu }
167a1d2a633SQiang Yu
168a1d2a633SQiang Yu mutex_lock(&vm->lock);
169a1d2a633SQiang Yu
1706aebc51dSQiang Yu size = bo->heap_size ? bo->heap_size : bo_va->node.size;
171d61dd248SQiang Yu lima_vm_unmap_range(vm, bo_va->node.start,
1726aebc51dSQiang Yu bo_va->node.start + size - 1);
173a1d2a633SQiang Yu
174a1d2a633SQiang Yu drm_mm_remove_node(&bo_va->node);
175a1d2a633SQiang Yu
176a1d2a633SQiang Yu mutex_unlock(&vm->lock);
177a1d2a633SQiang Yu
178a1d2a633SQiang Yu list_del(&bo_va->list);
179a1d2a633SQiang Yu
180a1d2a633SQiang Yu mutex_unlock(&bo->lock);
181a1d2a633SQiang Yu
182a1d2a633SQiang Yu kfree(bo_va);
183a1d2a633SQiang Yu }
184a1d2a633SQiang Yu
lima_vm_get_va(struct lima_vm * vm,struct lima_bo * bo)185a1d2a633SQiang Yu u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)
186a1d2a633SQiang Yu {
187a1d2a633SQiang Yu struct lima_bo_va *bo_va;
188a1d2a633SQiang Yu u32 ret;
189a1d2a633SQiang Yu
190a1d2a633SQiang Yu mutex_lock(&bo->lock);
191a1d2a633SQiang Yu
192a1d2a633SQiang Yu bo_va = lima_vm_bo_find(vm, bo);
193a1d2a633SQiang Yu ret = bo_va->node.start;
194a1d2a633SQiang Yu
195a1d2a633SQiang Yu mutex_unlock(&bo->lock);
196a1d2a633SQiang Yu
197a1d2a633SQiang Yu return ret;
198a1d2a633SQiang Yu }
199a1d2a633SQiang Yu
lima_vm_create(struct lima_device * dev)200a1d2a633SQiang Yu struct lima_vm *lima_vm_create(struct lima_device *dev)
201a1d2a633SQiang Yu {
202a1d2a633SQiang Yu struct lima_vm *vm;
203a1d2a633SQiang Yu
204a1d2a633SQiang Yu vm = kzalloc(sizeof(*vm), GFP_KERNEL);
205a1d2a633SQiang Yu if (!vm)
206a1d2a633SQiang Yu return NULL;
207a1d2a633SQiang Yu
208a1d2a633SQiang Yu vm->dev = dev;
209a1d2a633SQiang Yu mutex_init(&vm->lock);
210a1d2a633SQiang Yu kref_init(&vm->refcount);
211a1d2a633SQiang Yu
212a1d2a633SQiang Yu vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,
213e30b38b7SQiang Yu GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
214a1d2a633SQiang Yu if (!vm->pd.cpu)
215a1d2a633SQiang Yu goto err_out0;
216a1d2a633SQiang Yu
217a1d2a633SQiang Yu if (dev->dlbu_cpu) {
218d61dd248SQiang Yu int err = lima_vm_map_page(
219d61dd248SQiang Yu vm, dev->dlbu_dma, LIMA_VA_RESERVE_DLBU);
220a1d2a633SQiang Yu if (err)
221a1d2a633SQiang Yu goto err_out1;
222a1d2a633SQiang Yu }
223a1d2a633SQiang Yu
224a1d2a633SQiang Yu drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);
225a1d2a633SQiang Yu
226a1d2a633SQiang Yu return vm;
227a1d2a633SQiang Yu
228a1d2a633SQiang Yu err_out1:
229a1d2a633SQiang Yu dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
230a1d2a633SQiang Yu err_out0:
231a1d2a633SQiang Yu kfree(vm);
232a1d2a633SQiang Yu return NULL;
233a1d2a633SQiang Yu }
234a1d2a633SQiang Yu
lima_vm_release(struct kref * kref)235a1d2a633SQiang Yu void lima_vm_release(struct kref *kref)
236a1d2a633SQiang Yu {
237a1d2a633SQiang Yu struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);
238a1d2a633SQiang Yu int i;
239a1d2a633SQiang Yu
240a1d2a633SQiang Yu drm_mm_takedown(&vm->mm);
241a1d2a633SQiang Yu
242a1d2a633SQiang Yu for (i = 0; i < LIMA_VM_NUM_BT; i++) {
243a1d2a633SQiang Yu if (vm->bts[i].cpu)
244a1d2a633SQiang Yu dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
245a1d2a633SQiang Yu vm->bts[i].cpu, vm->bts[i].dma);
246a1d2a633SQiang Yu }
247a1d2a633SQiang Yu
248a1d2a633SQiang Yu if (vm->pd.cpu)
249a1d2a633SQiang Yu dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
250a1d2a633SQiang Yu
251a1d2a633SQiang Yu kfree(vm);
252a1d2a633SQiang Yu }
253a1d2a633SQiang Yu
lima_vm_print(struct lima_vm * vm)254a1d2a633SQiang Yu void lima_vm_print(struct lima_vm *vm)
255a1d2a633SQiang Yu {
256a1d2a633SQiang Yu int i, j, k;
257a1d2a633SQiang Yu u32 *pd, *pt;
258a1d2a633SQiang Yu
259a1d2a633SQiang Yu if (!vm->pd.cpu)
260a1d2a633SQiang Yu return;
261a1d2a633SQiang Yu
262a1d2a633SQiang Yu pd = vm->pd.cpu;
263a1d2a633SQiang Yu for (i = 0; i < LIMA_VM_NUM_BT; i++) {
264a1d2a633SQiang Yu if (!vm->bts[i].cpu)
265a1d2a633SQiang Yu continue;
266a1d2a633SQiang Yu
267a1d2a633SQiang Yu pt = vm->bts[i].cpu;
268a1d2a633SQiang Yu for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
269a1d2a633SQiang Yu int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;
270a1d2a633SQiang Yu
271a1d2a633SQiang Yu printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);
272a1d2a633SQiang Yu
273a1d2a633SQiang Yu for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {
274a1d2a633SQiang Yu u32 pte = *pt++;
275a1d2a633SQiang Yu
276a1d2a633SQiang Yu if (pte)
277a1d2a633SQiang Yu printk(KERN_INFO " pt %03x:%08x\n", k, pte);
278a1d2a633SQiang Yu }
279a1d2a633SQiang Yu }
280a1d2a633SQiang Yu }
281a1d2a633SQiang Yu }
282dc76cb7aSQiang Yu
lima_vm_map_bo(struct lima_vm * vm,struct lima_bo * bo,int pageoff)283dc76cb7aSQiang Yu int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff)
284dc76cb7aSQiang Yu {
285dc76cb7aSQiang Yu struct lima_bo_va *bo_va;
286dc76cb7aSQiang Yu struct sg_dma_page_iter sg_iter;
287dc76cb7aSQiang Yu int offset = 0, err;
288dc76cb7aSQiang Yu u32 base;
289dc76cb7aSQiang Yu
290dc76cb7aSQiang Yu mutex_lock(&bo->lock);
291dc76cb7aSQiang Yu
292dc76cb7aSQiang Yu bo_va = lima_vm_bo_find(vm, bo);
293dc76cb7aSQiang Yu if (!bo_va) {
294dc76cb7aSQiang Yu err = -ENOENT;
295dc76cb7aSQiang Yu goto err_out0;
296dc76cb7aSQiang Yu }
297dc76cb7aSQiang Yu
298dc76cb7aSQiang Yu mutex_lock(&vm->lock);
299dc76cb7aSQiang Yu
300dc76cb7aSQiang Yu base = bo_va->node.start + (pageoff << PAGE_SHIFT);
301*c3d9c17fSMarek Szyprowski for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) {
302dc76cb7aSQiang Yu err = lima_vm_map_page(vm, sg_page_iter_dma_address(&sg_iter),
303dc76cb7aSQiang Yu base + offset);
304dc76cb7aSQiang Yu if (err)
305dc76cb7aSQiang Yu goto err_out1;
306dc76cb7aSQiang Yu
307dc76cb7aSQiang Yu offset += PAGE_SIZE;
308dc76cb7aSQiang Yu }
309dc76cb7aSQiang Yu
310dc76cb7aSQiang Yu mutex_unlock(&vm->lock);
311dc76cb7aSQiang Yu
312dc76cb7aSQiang Yu mutex_unlock(&bo->lock);
313dc76cb7aSQiang Yu return 0;
314dc76cb7aSQiang Yu
315dc76cb7aSQiang Yu err_out1:
316dc76cb7aSQiang Yu if (offset)
317dc76cb7aSQiang Yu lima_vm_unmap_range(vm, base, base + offset - 1);
318dc76cb7aSQiang Yu mutex_unlock(&vm->lock);
319dc76cb7aSQiang Yu err_out0:
320dc76cb7aSQiang Yu mutex_unlock(&bo->lock);
321dc76cb7aSQiang Yu return err;
322dc76cb7aSQiang Yu }
323