xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c (revision e6f2a617ac53bc0753b885ffb94379ff48b2e2df)
1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include "amdgpu_vm.h"
24 #include "amdgpu_job.h"
25 #include "amdgpu_object.h"
26 #include "amdgpu_trace.h"
27 
28 #define AMDGPU_VM_SDMA_MIN_NUM_DW	256u
29 #define AMDGPU_VM_SDMA_MAX_NUM_DW	(16u * 1024u)
30 
31 /**
32  * amdgpu_vm_sdma_map_table - make sure new PDs/PTs are GTT mapped
33  *
34  * @table: newly allocated or validated PD/PT
35  */
36 static int amdgpu_vm_sdma_map_table(struct amdgpu_bo *table)
37 {
38 	int r;
39 
40 	r = amdgpu_ttm_alloc_gart(&table->tbo);
41 	if (r)
42 		return r;
43 
44 	if (table->shadow)
45 		r = amdgpu_ttm_alloc_gart(&table->shadow->tbo);
46 
47 	return r;
48 }
49 
50 /**
51  * amdgpu_vm_sdma_prepare - prepare SDMA command submission
52  *
53  * @p: see amdgpu_vm_update_params definition
54  * @owner: owner we need to sync to
55  * @exclusive: exclusive move fence we need to sync to
56  *
57  * Returns:
58  * Negativ errno, 0 for success.
59  */
60 static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,
61 				  void *owner, struct dma_fence *exclusive)
62 {
63 	struct amdgpu_bo *root = p->vm->root.base.bo;
64 	unsigned int ndw = AMDGPU_VM_SDMA_MIN_NUM_DW;
65 	int r;
66 
67 	r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job);
68 	if (r)
69 		return r;
70 
71 	p->num_dw_left = ndw;
72 
73 	/* Wait for moves to be completed */
74 	r = amdgpu_sync_fence(p->adev, &p->job->sync, exclusive, false);
75 	if (r)
76 		return r;
77 
78 	/* Don't wait for any submissions during page fault handling */
79 	if (p->direct)
80 		return 0;
81 
82 	return amdgpu_sync_resv(p->adev, &p->job->sync, root->tbo.base.resv,
83 				owner, false);
84 }
85 
86 /**
87  * amdgpu_vm_sdma_commit - commit SDMA command submission
88  *
89  * @p: see amdgpu_vm_update_params definition
90  * @fence: resulting fence
91  *
92  * Returns:
93  * Negativ errno, 0 for success.
94  */
95 static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
96 				 struct dma_fence **fence)
97 {
98 	struct amdgpu_bo *root = p->vm->root.base.bo;
99 	struct amdgpu_ib *ib = p->job->ibs;
100 	struct drm_sched_entity *entity;
101 	struct amdgpu_ring *ring;
102 	struct dma_fence *f;
103 	int r;
104 
105 	entity = p->direct ? &p->vm->direct : &p->vm->delayed;
106 	ring = container_of(entity->rq->sched, struct amdgpu_ring, sched);
107 
108 	WARN_ON(ib->length_dw == 0);
109 	amdgpu_ring_pad_ib(ring, ib);
110 	WARN_ON(ib->length_dw > p->num_dw_left);
111 	r = amdgpu_job_submit(p->job, entity, AMDGPU_FENCE_OWNER_VM, &f);
112 	if (r)
113 		goto error;
114 
115 	amdgpu_bo_fence(root, f, true);
116 	if (fence && !p->direct)
117 		swap(*fence, f);
118 	dma_fence_put(f);
119 	return 0;
120 
121 error:
122 	amdgpu_job_free(p->job);
123 	return r;
124 }
125 
126 /**
127  * amdgpu_vm_sdma_copy_ptes - copy the PTEs from mapping
128  *
129  * @p: see amdgpu_vm_update_params definition
130  * @bo: PD/PT to update
131  * @pe: addr of the page entry
132  * @count: number of page entries to copy
133  *
134  * Traces the parameters and calls the DMA function to copy the PTEs.
135  */
136 static void amdgpu_vm_sdma_copy_ptes(struct amdgpu_vm_update_params *p,
137 				     struct amdgpu_bo *bo, uint64_t pe,
138 				     unsigned count)
139 {
140 	struct amdgpu_ib *ib = p->job->ibs;
141 	uint64_t src = ib->gpu_addr;
142 
143 	src += p->num_dw_left * 4;
144 
145 	pe += amdgpu_bo_gpu_offset(bo);
146 	trace_amdgpu_vm_copy_ptes(pe, src, count, p->direct);
147 
148 	amdgpu_vm_copy_pte(p->adev, ib, pe, src, count);
149 }
150 
151 /**
152  * amdgpu_vm_sdma_set_ptes - helper to call the right asic function
153  *
154  * @p: see amdgpu_vm_update_params definition
155  * @bo: PD/PT to update
156  * @pe: addr of the page entry
157  * @addr: dst addr to write into pe
158  * @count: number of page entries to update
159  * @incr: increase next addr by incr bytes
160  * @flags: hw access flags
161  *
162  * Traces the parameters and calls the right asic functions
163  * to setup the page table using the DMA.
164  */
165 static void amdgpu_vm_sdma_set_ptes(struct amdgpu_vm_update_params *p,
166 				    struct amdgpu_bo *bo, uint64_t pe,
167 				    uint64_t addr, unsigned count,
168 				    uint32_t incr, uint64_t flags)
169 {
170 	struct amdgpu_ib *ib = p->job->ibs;
171 
172 	pe += amdgpu_bo_gpu_offset(bo);
173 	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->direct);
174 	if (count < 3) {
175 		amdgpu_vm_write_pte(p->adev, ib, pe, addr | flags,
176 				    count, incr);
177 	} else {
178 		amdgpu_vm_set_pte_pde(p->adev, ib, pe, addr,
179 				      count, incr, flags);
180 	}
181 }
182 
183 /**
184  * amdgpu_vm_sdma_update - execute VM update
185  *
186  * @p: see amdgpu_vm_update_params definition
187  * @bo: PD/PT to update
188  * @pe: addr of the page entry
189  * @addr: dst addr to write into pe
190  * @count: number of page entries to update
191  * @incr: increase next addr by incr bytes
192  * @flags: hw access flags
193  *
194  * Reserve space in the IB, setup mapping buffer on demand and write commands to
195  * the IB.
196  */
197 static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
198 				 struct amdgpu_bo *bo, uint64_t pe,
199 				 uint64_t addr, unsigned count, uint32_t incr,
200 				 uint64_t flags)
201 {
202 	unsigned int i, ndw, nptes;
203 	uint64_t *pte;
204 	int r;
205 
206 	do {
207 		ndw = p->num_dw_left;
208 		ndw -= p->job->ibs->length_dw;
209 
210 		if (ndw < 32) {
211 			r = amdgpu_vm_sdma_commit(p, NULL);
212 			if (r)
213 				return r;
214 
215 			/* estimate how many dw we need */
216 			ndw = 32;
217 			if (p->pages_addr)
218 				ndw += count * 2;
219 			ndw = max(ndw, AMDGPU_VM_SDMA_MIN_NUM_DW);
220 			ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW);
221 
222 			r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job);
223 			if (r)
224 				return r;
225 
226 			p->num_dw_left = ndw;
227 		}
228 
229 		if (!p->pages_addr) {
230 			/* set page commands needed */
231 			if (bo->shadow)
232 				amdgpu_vm_sdma_set_ptes(p, bo->shadow, pe, addr,
233 							count, incr, flags);
234 			amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count,
235 						incr, flags);
236 			return 0;
237 		}
238 
239 		/* copy commands needed */
240 		ndw -= p->adev->vm_manager.vm_pte_funcs->copy_pte_num_dw *
241 			(bo->shadow ? 2 : 1);
242 
243 		/* for padding */
244 		ndw -= 7;
245 
246 		nptes = min(count, ndw / 2);
247 
248 		/* Put the PTEs at the end of the IB. */
249 		p->num_dw_left -= nptes * 2;
250 		pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]);
251 		for (i = 0; i < nptes; ++i, addr += incr) {
252 			pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr);
253 			pte[i] |= flags;
254 		}
255 
256 		if (bo->shadow)
257 			amdgpu_vm_sdma_copy_ptes(p, bo->shadow, pe, nptes);
258 		amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes);
259 
260 		pe += nptes * 8;
261 		count -= nptes;
262 	} while (count);
263 
264 	return 0;
265 }
266 
267 const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs = {
268 	.map_table = amdgpu_vm_sdma_map_table,
269 	.prepare = amdgpu_vm_sdma_prepare,
270 	.update = amdgpu_vm_sdma_update,
271 	.commit = amdgpu_vm_sdma_commit
272 };
273