xref: /linux/drivers/gpu/drm/amd/amdkfd/kfd_queue.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/slab.h>
26 #include <linux/overflow.h>
27 #include "kfd_priv.h"
28 #include "kfd_topology.h"
29 #include "kfd_svm.h"
30 
31 void print_queue_properties(struct queue_properties *q)
32 {
33 	if (!q)
34 		return;
35 
36 	pr_debug("Printing queue properties:\n");
37 	pr_debug("Queue Type: %u\n", q->type);
38 	pr_debug("Queue Size: %llu\n", q->queue_size);
39 	pr_debug("Queue percent: %u\n", q->queue_percent);
40 	pr_debug("Queue Address: 0x%llX\n", q->queue_address);
41 	pr_debug("Queue Id: %u\n", q->queue_id);
42 	pr_debug("Queue Process Vmid: %u\n", q->vmid);
43 	pr_debug("Queue Read Pointer: 0x%px\n", q->read_ptr);
44 	pr_debug("Queue Write Pointer: 0x%px\n", q->write_ptr);
45 	pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr);
46 	pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off);
47 }
48 
49 void print_queue(struct queue *q)
50 {
51 	if (!q)
52 		return;
53 	pr_debug("Printing queue:\n");
54 	pr_debug("Queue Type: %u\n", q->properties.type);
55 	pr_debug("Queue Size: %llu\n", q->properties.queue_size);
56 	pr_debug("Queue percent: %u\n", q->properties.queue_percent);
57 	pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address);
58 	pr_debug("Queue Id: %u\n", q->properties.queue_id);
59 	pr_debug("Queue Process Vmid: %u\n", q->properties.vmid);
60 	pr_debug("Queue Read Pointer: 0x%px\n", q->properties.read_ptr);
61 	pr_debug("Queue Write Pointer: 0x%px\n", q->properties.write_ptr);
62 	pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr);
63 	pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off);
64 	pr_debug("Queue MQD Address: 0x%p\n", q->mqd);
65 	pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr);
66 	pr_debug("Queue Process Address: 0x%p\n", q->process);
67 	pr_debug("Queue Device Address: 0x%p\n", q->device);
68 }
69 
70 int init_queue(struct queue **q, const struct queue_properties *properties)
71 {
72 	struct queue *tmp_q;
73 
74 	tmp_q = kzalloc_obj(*tmp_q);
75 	if (!tmp_q)
76 		return -ENOMEM;
77 
78 	memcpy(&tmp_q->properties, properties, sizeof(*properties));
79 
80 	*q = tmp_q;
81 	return 0;
82 }
83 
84 void uninit_queue(struct queue *q)
85 {
86 	kfree(q);
87 }
88 
89 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
90 
91 static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size)
92 {
93 	struct kfd_process *p = pdd->process;
94 	struct list_head update_list;
95 	struct svm_range *prange;
96 	int ret = -EINVAL;
97 
98 	INIT_LIST_HEAD(&update_list);
99 	addr >>= PAGE_SHIFT;
100 	size >>= PAGE_SHIFT;
101 
102 	mutex_lock(&p->svms.lock);
103 
104 	/*
105 	 * range may split to multiple svm pranges aligned to granularity boundary.
106 	 */
107 	while (size) {
108 		uint32_t gpuid, gpuidx;
109 		int r;
110 
111 		prange = svm_range_from_addr(&p->svms, addr, NULL);
112 		if (!prange)
113 			break;
114 
115 		r = kfd_process_gpuid_from_node(p, pdd->dev, &gpuid, &gpuidx);
116 		if (r < 0)
117 			break;
118 		if (!test_bit(gpuidx, prange->bitmap_mapped))
119 			break;
120 		if (!test_bit(gpuidx, prange->bitmap_access) &&
121 		    !test_bit(gpuidx, prange->bitmap_aip))
122 			break;
123 
124 		if (!(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED))
125 			break;
126 
127 		list_add(&prange->update_list, &update_list);
128 
129 		if (prange->last - prange->start + 1 >= size) {
130 			size = 0;
131 			break;
132 		}
133 
134 		size -= prange->last - prange->start + 1;
135 		addr += prange->last - prange->start + 1;
136 	}
137 	if (size) {
138 		pr_debug("[0x%llx 0x%llx] not registered\n", addr, addr + size - 1);
139 		goto out_unlock;
140 	}
141 
142 	list_for_each_entry(prange, &update_list, update_list)
143 		atomic_inc(&prange->queue_refcount);
144 	ret = 0;
145 
146 out_unlock:
147 	mutex_unlock(&p->svms.lock);
148 	return ret;
149 }
150 
151 static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size)
152 {
153 	struct kfd_process *p = pdd->process;
154 	struct svm_range *prange, *pchild;
155 	struct interval_tree_node *node;
156 	unsigned long last;
157 
158 	addr >>= PAGE_SHIFT;
159 	last = addr + (size >> PAGE_SHIFT) - 1;
160 
161 	mutex_lock(&p->svms.lock);
162 
163 	node = interval_tree_iter_first(&p->svms.objects, addr, last);
164 	while (node) {
165 		struct interval_tree_node *next_node;
166 		unsigned long next_start;
167 
168 		prange = container_of(node, struct svm_range, it_node);
169 		next_node = interval_tree_iter_next(node, addr, last);
170 		next_start = min(node->last, last) + 1;
171 
172 		if (atomic_add_unless(&prange->queue_refcount, -1, 0)) {
173 			list_for_each_entry(pchild, &prange->child_list, child_list)
174 				atomic_add_unless(&pchild->queue_refcount, -1, 0);
175 		}
176 
177 		node = next_node;
178 		addr = next_start;
179 	}
180 
181 	mutex_unlock(&p->svms.lock);
182 }
183 #else
184 
185 static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size)
186 {
187 	return -EINVAL;
188 }
189 
190 static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size)
191 {
192 }
193 
194 #endif
195 
196 int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_bo **pbo,
197 			 u64 expected_size)
198 {
199 	struct amdgpu_bo_va_mapping *mapping;
200 	u64 user_pfn;
201 	u64 size;
202 
203 	size = expected_size >> AMDGPU_GPU_PAGE_SHIFT;
204 
205 	mapping = amdgpu_vm_bo_lookup_mapping(vm, (u64)(uintptr_t)addr);
206 	if (!mapping)
207 		goto out_err;
208 
209 	user_pfn = (u64)(uintptr_t)addr >> AMDGPU_GPU_PAGE_SHIFT;
210 	if (user_pfn != mapping->start ||
211 	    (size != 0 && user_pfn + size - 1 != mapping->last)) {
212 		pr_debug("expected size 0x%llx not equal to mapping addr 0x%llx size 0x%llx\n",
213 			expected_size, mapping->start << AMDGPU_GPU_PAGE_SHIFT,
214 			(mapping->last - mapping->start + 1) << AMDGPU_GPU_PAGE_SHIFT);
215 		goto out_err;
216 	}
217 
218 	*pbo = amdgpu_bo_ref(mapping->bo_va->base.bo);
219 	mapping->bo_va->queue_refcount++;
220 	return 0;
221 
222 out_err:
223 	*pbo = NULL;
224 	return -EINVAL;
225 }
226 
227 /* FIXME: remove this function, just call amdgpu_bo_unref directly */
228 void kfd_queue_buffer_put(struct amdgpu_bo **bo)
229 {
230 	amdgpu_bo_unref(bo);
231 }
232 
233 static void kfd_queue_put_bos(struct queue_properties *properties)
234 {
235 	kfd_queue_buffer_put(&properties->wptr_bo);
236 	kfd_queue_buffer_put(&properties->rptr_bo);
237 	kfd_queue_buffer_put(&properties->ring_bo);
238 	kfd_queue_buffer_put(&properties->eop_buf_bo);
239 	kfd_queue_buffer_put(&properties->cwsr_bo);
240 }
241 
242 int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
243 {
244 	struct kfd_topology_device *topo_dev;
245 	u64 expected_queue_size;
246 	struct amdgpu_vm *vm;
247 	u64 total_cwsr_size;
248 	int err;
249 
250 	topo_dev = kfd_topology_device_by_id(pdd->dev->id);
251 	if (!topo_dev)
252 		return -EINVAL;
253 
254 	/* AQL queues on GFX7 and GFX8 appear twice their actual size */
255 	if (properties->type == KFD_QUEUE_TYPE_COMPUTE &&
256 	    properties->format == KFD_QUEUE_FORMAT_AQL &&
257 	    topo_dev->node_props.gfx_target_version >= 70000 &&
258 	    topo_dev->node_props.gfx_target_version < 90000)
259 		/* metadata_queue_size not supported on GFX7/GFX8 */
260 		expected_queue_size =
261 			PAGE_ALIGN(properties->queue_size / 2);
262 	else
263 		expected_queue_size =
264 			PAGE_ALIGN(properties->queue_size + properties->metadata_queue_size);
265 
266 	vm = drm_priv_to_vm(pdd->drm_priv);
267 	err = amdgpu_bo_reserve(vm->root.bo, false);
268 	if (err)
269 		return err;
270 
271 	err = kfd_queue_buffer_get(vm, properties->write_ptr, &properties->wptr_bo, PAGE_SIZE);
272 	if (err)
273 		goto out_err_unreserve;
274 
275 	err = kfd_queue_buffer_get(vm, properties->read_ptr, &properties->rptr_bo, PAGE_SIZE);
276 	if (err)
277 		goto out_err_unreserve;
278 
279 	err = kfd_queue_buffer_get(vm, (void *)properties->queue_address,
280 				   &properties->ring_bo, expected_queue_size);
281 	if (err)
282 		goto out_err_unreserve;
283 
284 	/* only compute queue requires EOP buffer and CWSR area */
285 	if (properties->type != KFD_QUEUE_TYPE_COMPUTE)
286 		goto out_unreserve;
287 
288 	/* EOP buffer is not required for all ASICs */
289 	if (properties->eop_ring_buffer_address) {
290 		if (properties->eop_ring_buffer_size < topo_dev->node_props.eop_buffer_size) {
291 			pr_debug("queue eop bo size 0x%x is less than node eop buf size 0x%x\n",
292 				properties->eop_ring_buffer_size,
293 				topo_dev->node_props.eop_buffer_size);
294 			err = -EINVAL;
295 			goto out_err_unreserve;
296 		}
297 		err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address,
298 					   &properties->eop_buf_bo,
299 					   ALIGN((u64)properties->eop_ring_buffer_size, PAGE_SIZE));
300 		if (err)
301 			goto out_err_unreserve;
302 	}
303 
304 	if (properties->ctl_stack_size != topo_dev->node_props.ctl_stack_size) {
305 		pr_debug("queue ctl stack size 0x%x not equal to node ctl stack size 0x%x\n",
306 			properties->ctl_stack_size,
307 			topo_dev->node_props.ctl_stack_size);
308 		err = -EINVAL;
309 		goto out_err_unreserve;
310 	}
311 
312 	if (properties->ctx_save_restore_area_size < topo_dev->node_props.cwsr_size) {
313 		pr_debug("queue cwsr size 0x%x not sufficient for node cwsr size 0x%x\n",
314 			properties->ctx_save_restore_area_size,
315 			topo_dev->node_props.cwsr_size);
316 		err = -EINVAL;
317 		goto out_err_unreserve;
318 	}
319 
320 	total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
321 			  topo_dev->node_props.debug_memory_size;
322 	if (check_mul_overflow(total_cwsr_size,
323 			       NUM_XCC(pdd->dev->xcc_mask),
324 			       &total_cwsr_size)) {
325 		err = -EINVAL;
326 		goto out_err_unreserve;
327 	}
328 	total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
329 
330 	err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address,
331 				   &properties->cwsr_bo, total_cwsr_size);
332 	if (!err)
333 		goto out_unreserve;
334 
335 	amdgpu_bo_unreserve(vm->root.bo);
336 
337 	err = kfd_queue_buffer_svm_get(pdd, properties->ctx_save_restore_area_address,
338 				       total_cwsr_size);
339 	if (err)
340 		goto out_err_release;
341 
342 	return 0;
343 
344 out_unreserve:
345 	amdgpu_bo_unreserve(vm->root.bo);
346 	return 0;
347 
348 out_err_unreserve:
349 	amdgpu_bo_unreserve(vm->root.bo);
350 out_err_release:
351 	/* FIXME: make a _locked version of this that can be called before
352 	 * dropping the VM reservation.
353 	 */
354 	kfd_queue_unref_bo_vas(pdd, properties);
355 	/*
356 	 * The SVM CWSR range is the last thing acquired and a failed
357 	 * kfd_queue_buffer_svm_get() takes nothing, so only the BOs are held
358 	 * here. Dropping the SVM reference would drop the one of another queue
359 	 * using the same CWSR address.
360 	 */
361 	kfd_queue_put_bos(properties);
362 	return err;
363 }
364 
365 int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
366 {
367 	struct kfd_topology_device *topo_dev;
368 	u64 total_cwsr_size;
369 
370 	kfd_queue_put_bos(properties);
371 
372 	topo_dev = kfd_topology_device_by_id(pdd->dev->id);
373 	if (!topo_dev)
374 		return -EINVAL;
375 	total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
376 			  topo_dev->node_props.debug_memory_size;
377 	if (check_mul_overflow(total_cwsr_size,
378 			       NUM_XCC(pdd->dev->xcc_mask),
379 			       &total_cwsr_size))
380 		return -EINVAL;
381 	total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
382 
383 	kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size);
384 	return 0;
385 }
386 
387 void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo)
388 {
389 	if (*bo) {
390 		struct amdgpu_bo_va *bo_va;
391 
392 		bo_va = amdgpu_vm_bo_find(vm, *bo);
393 		if (bo_va && bo_va->queue_refcount)
394 			bo_va->queue_refcount--;
395 	}
396 }
397 
398 int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd,
399 			   struct queue_properties *properties)
400 {
401 	struct amdgpu_vm *vm;
402 	int err;
403 
404 	vm = drm_priv_to_vm(pdd->drm_priv);
405 	err = amdgpu_bo_reserve(vm->root.bo, false);
406 	if (err)
407 		return err;
408 
409 	kfd_queue_unref_bo_va(vm, &properties->wptr_bo);
410 	kfd_queue_unref_bo_va(vm, &properties->rptr_bo);
411 	kfd_queue_unref_bo_va(vm, &properties->ring_bo);
412 	kfd_queue_unref_bo_va(vm, &properties->eop_buf_bo);
413 	kfd_queue_unref_bo_va(vm, &properties->cwsr_bo);
414 
415 	amdgpu_bo_unreserve(vm->root.bo);
416 	return 0;
417 }
418 
419 #define DEBUGGER_BYTES_ALIGN	64
420 #define DEBUGGER_BYTES_PER_WAVE	32
421 
422 static u32 kfd_get_sgpr_size_per_cu(u32 gfxv)
423 {
424 	u32 sgpr_size = 0x4000;
425 
426 	if (gfxv == 120500 ||
427 	    gfxv == 120501)
428 		sgpr_size = 0x8000;
429 
430 	return sgpr_size;
431 }
432 
433 static u32 kfd_get_vgpr_size_per_cu(u32 gfxv)
434 {
435 	u32 vgpr_size = 0x40000;
436 
437 	if (gfxv == 90402 ||			/* GFX_VERSION_AQUA_VANJARAM */
438 	    gfxv == 90010 ||			/* GFX_VERSION_ALDEBARAN */
439 	    gfxv == 90008 ||			/* GFX_VERSION_ARCTURUS */
440 	    gfxv == 90500)
441 		vgpr_size = 0x80000;
442 	else if (gfxv == 110000 ||		/* GFX_VERSION_PLUM_BONITO */
443 		 gfxv == 110001 ||		/* GFX_VERSION_WHEAT_NAS */
444 		 gfxv == 110501 ||		/* GFX_VERSION_GFX1151 */
445 		 gfxv == 120000 ||		/* GFX_VERSION_GFX1200 */
446 		 gfxv == 120001)		/* GFX_VERSION_GFX1201 */
447 		vgpr_size = 0x60000;
448 	else if (gfxv == 120500 ||		/* GFX_VERSION_GFX1250 */
449 		 gfxv == 120501)		/* GFX_VERSION_GFX1251 */
450 		vgpr_size = 0x80000;
451 
452 	return vgpr_size;
453 }
454 
455 static u32 kfd_get_hwreg_size_per_cu(u32 gfxv)
456 {
457 	u32 hwreg_size = 0x1000;
458 
459 	if (gfxv == 120500 || gfxv == 120501)
460 		hwreg_size = 0x8000;
461 
462 	return hwreg_size;
463 }
464 
465 static u32 kfd_get_lds_size_per_cu(u32 gfxv, struct kfd_node_properties *props)
466 {
467 	u32 lds_size = 0x10000;
468 
469 	if (gfxv == 90500 || gfxv == 120500 || gfxv == 120501)
470 		lds_size = props->lds_size_in_kb << 10;
471 
472 	return lds_size;
473 }
474 
475 static u32 get_num_waves(struct kfd_node_properties *props, u32 gfxv, u32 cu_num)
476 {
477 	u32 wave_num = 0;
478 
479 	if (gfxv < 100100)
480 		wave_num = min(cu_num * 40,
481 				props->array_count / props->simd_arrays_per_engine * 512);
482 	else if (gfxv < 120500)
483 		wave_num = cu_num * 32;
484 	else if (gfxv <= 120501)
485 		wave_num = cu_num * 64;
486 
487 	WARN_ON(wave_num == 0);
488 
489 	return wave_num;
490 }
491 
492 #define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props)	\
493 	(kfd_get_vgpr_size_per_cu(gfxv) + kfd_get_sgpr_size_per_cu(gfxv) +\
494 	 kfd_get_lds_size_per_cu(gfxv, props) + kfd_get_hwreg_size_per_cu(gfxv))
495 
496 #define CNTL_STACK_BYTES_PER_WAVE(gfxv)	\
497 	((gfxv) >= 100100 ? 12 : 8)	/* GFX_VERSION_NAVI10*/
498 
499 #define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40
500 
501 void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev)
502 {
503 	struct kfd_node_properties *props = &dev->node_props;
504 	u32 gfxv = props->gfx_target_version;
505 	u32 ctl_stack_size;
506 	u32 wg_data_size;
507 	u32 wave_num;
508 	u32 cu_num;
509 
510 	if (gfxv < 80001)	/* GFX_VERSION_CARRIZO */
511 		return;
512 
513 	cu_num = props->simd_count / props->simd_per_cu / NUM_XCC(dev->gpu->xcc_mask);
514 	wave_num = get_num_waves(props, gfxv, cu_num);
515 
516 	wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props),
517 				AMDGPU_GPU_PAGE_SIZE);
518 	ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8;
519 	ctl_stack_size = ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,
520 			       AMDGPU_GPU_PAGE_SIZE);
521 
522 	if ((gfxv / 10000 * 10000) == 100000) {
523 		/* HW design limits control stack size to 0x7000.
524 		 * This is insufficient for theoretical PM4 cases
525 		 * but sufficient for AQL, limited by SPI events.
526 		 */
527 		ctl_stack_size = min(ctl_stack_size, 0x7000);
528 	}
529 
530 	props->ctl_stack_size = ctl_stack_size;
531 	props->debug_memory_size = ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);
532 	props->cwsr_size = ALIGN(ctl_stack_size + wg_data_size, PAGE_SIZE);
533 
534 	if (gfxv == 80002)	/* GFX_VERSION_TONGA */
535 		props->eop_buffer_size = 0x8000;
536 	else if (gfxv == 90402)	/* GFX_VERSION_AQUA_VANJARAM */
537 		props->eop_buffer_size = 4096;
538 	else if (gfxv >= 80000)
539 		props->eop_buffer_size = 4096;
540 }
541