1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/slab.h> 26 #include <linux/overflow.h> 27 #include "kfd_priv.h" 28 #include "kfd_topology.h" 29 #include "kfd_svm.h" 30 31 void print_queue_properties(struct queue_properties *q) 32 { 33 if (!q) 34 return; 35 36 pr_debug("Printing queue properties:\n"); 37 pr_debug("Queue Type: %u\n", q->type); 38 pr_debug("Queue Size: %llu\n", q->queue_size); 39 pr_debug("Queue percent: %u\n", q->queue_percent); 40 pr_debug("Queue Address: 0x%llX\n", q->queue_address); 41 pr_debug("Queue Id: %u\n", q->queue_id); 42 pr_debug("Queue Process Vmid: %u\n", q->vmid); 43 pr_debug("Queue Read Pointer: 0x%px\n", q->read_ptr); 44 pr_debug("Queue Write Pointer: 0x%px\n", q->write_ptr); 45 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr); 46 pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off); 47 } 48 49 void print_queue(struct queue *q) 50 { 51 if (!q) 52 return; 53 pr_debug("Printing queue:\n"); 54 pr_debug("Queue Type: %u\n", q->properties.type); 55 pr_debug("Queue Size: %llu\n", q->properties.queue_size); 56 pr_debug("Queue percent: %u\n", q->properties.queue_percent); 57 pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address); 58 pr_debug("Queue Id: %u\n", q->properties.queue_id); 59 pr_debug("Queue Process Vmid: %u\n", q->properties.vmid); 60 pr_debug("Queue Read Pointer: 0x%px\n", q->properties.read_ptr); 61 pr_debug("Queue Write Pointer: 0x%px\n", q->properties.write_ptr); 62 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr); 63 pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off); 64 pr_debug("Queue MQD Address: 0x%p\n", q->mqd); 65 pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr); 66 pr_debug("Queue Process Address: 0x%p\n", q->process); 67 pr_debug("Queue Device Address: 0x%p\n", q->device); 68 } 69 70 int init_queue(struct queue **q, const struct queue_properties *properties) 71 { 72 struct queue *tmp_q; 73 74 tmp_q = kzalloc_obj(*tmp_q); 75 if (!tmp_q) 76 return -ENOMEM; 77 78 memcpy(&tmp_q->properties, properties, sizeof(*properties)); 79 80 *q = tmp_q; 81 return 0; 82 } 83 84 void uninit_queue(struct queue *q) 85 { 86 kfree(q); 87 } 88 89 #if IS_ENABLED(CONFIG_HSA_AMD_SVM) 90 91 static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) 92 { 93 struct kfd_process *p = pdd->process; 94 struct list_head update_list; 95 struct svm_range *prange; 96 int ret = -EINVAL; 97 98 INIT_LIST_HEAD(&update_list); 99 addr >>= PAGE_SHIFT; 100 size >>= PAGE_SHIFT; 101 102 mutex_lock(&p->svms.lock); 103 104 /* 105 * range may split to multiple svm pranges aligned to granularity boundaery. 106 */ 107 while (size) { 108 uint32_t gpuid, gpuidx; 109 int r; 110 111 prange = svm_range_from_addr(&p->svms, addr, NULL); 112 if (!prange) 113 break; 114 115 if (!prange->mapped_to_gpu) 116 break; 117 118 r = kfd_process_gpuid_from_node(p, pdd->dev, &gpuid, &gpuidx); 119 if (r < 0) 120 break; 121 if (!test_bit(gpuidx, prange->bitmap_access) && 122 !test_bit(gpuidx, prange->bitmap_aip)) 123 break; 124 125 if (!(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) 126 break; 127 128 list_add(&prange->update_list, &update_list); 129 130 if (prange->last - prange->start + 1 >= size) { 131 size = 0; 132 break; 133 } 134 135 size -= prange->last - prange->start + 1; 136 addr += prange->last - prange->start + 1; 137 } 138 if (size) { 139 pr_debug("[0x%llx 0x%llx] not registered\n", addr, addr + size - 1); 140 goto out_unlock; 141 } 142 143 list_for_each_entry(prange, &update_list, update_list) 144 atomic_inc(&prange->queue_refcount); 145 ret = 0; 146 147 out_unlock: 148 mutex_unlock(&p->svms.lock); 149 return ret; 150 } 151 152 static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size) 153 { 154 struct kfd_process *p = pdd->process; 155 struct svm_range *prange, *pchild; 156 struct interval_tree_node *node; 157 unsigned long last; 158 159 addr >>= PAGE_SHIFT; 160 last = addr + (size >> PAGE_SHIFT) - 1; 161 162 mutex_lock(&p->svms.lock); 163 164 node = interval_tree_iter_first(&p->svms.objects, addr, last); 165 while (node) { 166 struct interval_tree_node *next_node; 167 unsigned long next_start; 168 169 prange = container_of(node, struct svm_range, it_node); 170 next_node = interval_tree_iter_next(node, addr, last); 171 next_start = min(node->last, last) + 1; 172 173 if (atomic_add_unless(&prange->queue_refcount, -1, 0)) { 174 list_for_each_entry(pchild, &prange->child_list, child_list) 175 atomic_add_unless(&pchild->queue_refcount, -1, 0); 176 } 177 178 node = next_node; 179 addr = next_start; 180 } 181 182 mutex_unlock(&p->svms.lock); 183 } 184 #else 185 186 static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size) 187 { 188 return -EINVAL; 189 } 190 191 static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size) 192 { 193 } 194 195 #endif 196 197 int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_bo **pbo, 198 u64 expected_size) 199 { 200 struct amdgpu_bo_va_mapping *mapping; 201 u64 user_addr; 202 u64 size; 203 204 user_addr = (u64)addr >> AMDGPU_GPU_PAGE_SHIFT; 205 size = expected_size >> AMDGPU_GPU_PAGE_SHIFT; 206 207 mapping = amdgpu_vm_bo_lookup_mapping(vm, user_addr); 208 if (!mapping) 209 goto out_err; 210 211 if (user_addr != mapping->start || 212 (size != 0 && user_addr + size - 1 != mapping->last)) { 213 pr_debug("expected size 0x%llx not equal to mapping addr 0x%llx size 0x%llx\n", 214 expected_size, mapping->start << AMDGPU_GPU_PAGE_SHIFT, 215 (mapping->last - mapping->start + 1) << AMDGPU_GPU_PAGE_SHIFT); 216 goto out_err; 217 } 218 219 *pbo = amdgpu_bo_ref(mapping->bo_va->base.bo); 220 mapping->bo_va->queue_refcount++; 221 return 0; 222 223 out_err: 224 *pbo = NULL; 225 return -EINVAL; 226 } 227 228 /* FIXME: remove this function, just call amdgpu_bo_unref directly */ 229 void kfd_queue_buffer_put(struct amdgpu_bo **bo) 230 { 231 amdgpu_bo_unref(bo); 232 } 233 234 int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties) 235 { 236 struct kfd_topology_device *topo_dev; 237 u64 expected_queue_size; 238 struct amdgpu_vm *vm; 239 u64 total_cwsr_size; 240 int err; 241 242 topo_dev = kfd_topology_device_by_id(pdd->dev->id); 243 if (!topo_dev) 244 return -EINVAL; 245 246 /* AQL queues on GFX7 and GFX8 appear twice their actual size */ 247 if (properties->type == KFD_QUEUE_TYPE_COMPUTE && 248 properties->format == KFD_QUEUE_FORMAT_AQL && 249 topo_dev->node_props.gfx_target_version >= 70000 && 250 topo_dev->node_props.gfx_target_version < 90000) 251 /* metadata_queue_size not supported on GFX7/GFX8 */ 252 expected_queue_size = 253 PAGE_ALIGN(properties->queue_size / 2); 254 else 255 expected_queue_size = 256 PAGE_ALIGN(properties->queue_size + properties->metadata_queue_size); 257 258 vm = drm_priv_to_vm(pdd->drm_priv); 259 err = amdgpu_bo_reserve(vm->root.bo, false); 260 if (err) 261 return err; 262 263 err = kfd_queue_buffer_get(vm, properties->write_ptr, &properties->wptr_bo, PAGE_SIZE); 264 if (err) 265 goto out_err_unreserve; 266 267 err = kfd_queue_buffer_get(vm, properties->read_ptr, &properties->rptr_bo, PAGE_SIZE); 268 if (err) 269 goto out_err_unreserve; 270 271 err = kfd_queue_buffer_get(vm, (void *)properties->queue_address, 272 &properties->ring_bo, expected_queue_size); 273 if (err) 274 goto out_err_unreserve; 275 276 /* only compute queue requires EOP buffer and CWSR area */ 277 if (properties->type != KFD_QUEUE_TYPE_COMPUTE) 278 goto out_unreserve; 279 280 /* EOP buffer is not required for all ASICs */ 281 if (properties->eop_ring_buffer_address) { 282 if (properties->eop_ring_buffer_size < topo_dev->node_props.eop_buffer_size) { 283 pr_debug("queue eop bo size 0x%x is less than node eop buf size 0x%x\n", 284 properties->eop_ring_buffer_size, 285 topo_dev->node_props.eop_buffer_size); 286 err = -EINVAL; 287 goto out_err_unreserve; 288 } 289 err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address, 290 &properties->eop_buf_bo, 291 ALIGN((u64)properties->eop_ring_buffer_size, PAGE_SIZE)); 292 if (err) 293 goto out_err_unreserve; 294 } 295 296 if (properties->ctl_stack_size != topo_dev->node_props.ctl_stack_size) { 297 pr_debug("queue ctl stack size 0x%x not equal to node ctl stack size 0x%x\n", 298 properties->ctl_stack_size, 299 topo_dev->node_props.ctl_stack_size); 300 err = -EINVAL; 301 goto out_err_unreserve; 302 } 303 304 if (properties->ctx_save_restore_area_size < topo_dev->node_props.cwsr_size) { 305 pr_debug("queue cwsr size 0x%x not sufficient for node cwsr size 0x%x\n", 306 properties->ctx_save_restore_area_size, 307 topo_dev->node_props.cwsr_size); 308 err = -EINVAL; 309 goto out_err_unreserve; 310 } 311 312 total_cwsr_size = (u64)properties->ctx_save_restore_area_size + 313 topo_dev->node_props.debug_memory_size; 314 if (check_mul_overflow(total_cwsr_size, 315 NUM_XCC(pdd->dev->xcc_mask), 316 &total_cwsr_size)) { 317 err = -EINVAL; 318 goto out_err_unreserve; 319 } 320 total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); 321 322 err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address, 323 &properties->cwsr_bo, total_cwsr_size); 324 if (!err) 325 goto out_unreserve; 326 327 amdgpu_bo_unreserve(vm->root.bo); 328 329 err = kfd_queue_buffer_svm_get(pdd, properties->ctx_save_restore_area_address, 330 total_cwsr_size); 331 if (err) 332 goto out_err_release; 333 334 return 0; 335 336 out_unreserve: 337 amdgpu_bo_unreserve(vm->root.bo); 338 return 0; 339 340 out_err_unreserve: 341 amdgpu_bo_unreserve(vm->root.bo); 342 out_err_release: 343 /* FIXME: make a _locked version of this that can be called before 344 * dropping the VM reservation. 345 */ 346 kfd_queue_unref_bo_vas(pdd, properties); 347 kfd_queue_release_buffers(pdd, properties); 348 return err; 349 } 350 351 int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties) 352 { 353 struct kfd_topology_device *topo_dev; 354 u64 total_cwsr_size; 355 356 kfd_queue_buffer_put(&properties->wptr_bo); 357 kfd_queue_buffer_put(&properties->rptr_bo); 358 kfd_queue_buffer_put(&properties->ring_bo); 359 kfd_queue_buffer_put(&properties->eop_buf_bo); 360 kfd_queue_buffer_put(&properties->cwsr_bo); 361 362 topo_dev = kfd_topology_device_by_id(pdd->dev->id); 363 if (!topo_dev) 364 return -EINVAL; 365 total_cwsr_size = (u64)properties->ctx_save_restore_area_size + 366 topo_dev->node_props.debug_memory_size; 367 if (check_mul_overflow(total_cwsr_size, 368 NUM_XCC(pdd->dev->xcc_mask), 369 &total_cwsr_size)) 370 return -EINVAL; 371 total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); 372 373 kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size); 374 return 0; 375 } 376 377 void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo) 378 { 379 if (*bo) { 380 struct amdgpu_bo_va *bo_va; 381 382 bo_va = amdgpu_vm_bo_find(vm, *bo); 383 if (bo_va && bo_va->queue_refcount) 384 bo_va->queue_refcount--; 385 } 386 } 387 388 int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd, 389 struct queue_properties *properties) 390 { 391 struct amdgpu_vm *vm; 392 int err; 393 394 vm = drm_priv_to_vm(pdd->drm_priv); 395 err = amdgpu_bo_reserve(vm->root.bo, false); 396 if (err) 397 return err; 398 399 kfd_queue_unref_bo_va(vm, &properties->wptr_bo); 400 kfd_queue_unref_bo_va(vm, &properties->rptr_bo); 401 kfd_queue_unref_bo_va(vm, &properties->ring_bo); 402 kfd_queue_unref_bo_va(vm, &properties->eop_buf_bo); 403 kfd_queue_unref_bo_va(vm, &properties->cwsr_bo); 404 405 amdgpu_bo_unreserve(vm->root.bo); 406 return 0; 407 } 408 409 #define DEBUGGER_BYTES_ALIGN 64 410 #define DEBUGGER_BYTES_PER_WAVE 32 411 412 static u32 kfd_get_sgpr_size_per_cu(u32 gfxv) 413 { 414 u32 sgpr_size = 0x4000; 415 416 if (gfxv == 120500 || 417 gfxv == 120501) 418 sgpr_size = 0x8000; 419 420 return sgpr_size; 421 } 422 423 static u32 kfd_get_vgpr_size_per_cu(u32 gfxv) 424 { 425 u32 vgpr_size = 0x40000; 426 427 if (gfxv == 90402 || /* GFX_VERSION_AQUA_VANJARAM */ 428 gfxv == 90010 || /* GFX_VERSION_ALDEBARAN */ 429 gfxv == 90008 || /* GFX_VERSION_ARCTURUS */ 430 gfxv == 90500) 431 vgpr_size = 0x80000; 432 else if (gfxv == 110000 || /* GFX_VERSION_PLUM_BONITO */ 433 gfxv == 110001 || /* GFX_VERSION_WHEAT_NAS */ 434 gfxv == 110501 || /* GFX_VERSION_GFX1151 */ 435 gfxv == 120000 || /* GFX_VERSION_GFX1200 */ 436 gfxv == 120001) /* GFX_VERSION_GFX1201 */ 437 vgpr_size = 0x60000; 438 else if (gfxv == 120500 || /* GFX_VERSION_GFX1250 */ 439 gfxv == 120501) /* GFX_VERSION_GFX1251 */ 440 vgpr_size = 0x80000; 441 442 return vgpr_size; 443 } 444 445 static u32 kfd_get_hwreg_size_per_cu(u32 gfxv) 446 { 447 u32 hwreg_size = 0x1000; 448 449 if (gfxv == 120500 || gfxv == 120501) 450 hwreg_size = 0x8000; 451 452 return hwreg_size; 453 } 454 455 static u32 kfd_get_lds_size_per_cu(u32 gfxv, struct kfd_node_properties *props) 456 { 457 u32 lds_size = 0x10000; 458 459 if (gfxv == 90500 || gfxv == 120500 || gfxv == 120501) 460 lds_size = props->lds_size_in_kb << 10; 461 462 return lds_size; 463 } 464 465 static u32 get_num_waves(struct kfd_node_properties *props, u32 gfxv, u32 cu_num) 466 { 467 u32 wave_num = 0; 468 469 if (gfxv < 100100) 470 wave_num = min(cu_num * 40, 471 props->array_count / props->simd_arrays_per_engine * 512); 472 else if (gfxv < 120500) 473 wave_num = cu_num * 32; 474 else if (gfxv <= 120501) 475 wave_num = cu_num * 64; 476 477 WARN_ON(wave_num == 0); 478 479 return wave_num; 480 } 481 482 #define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props) \ 483 (kfd_get_vgpr_size_per_cu(gfxv) + kfd_get_sgpr_size_per_cu(gfxv) +\ 484 kfd_get_lds_size_per_cu(gfxv, props) + kfd_get_hwreg_size_per_cu(gfxv)) 485 486 #define CNTL_STACK_BYTES_PER_WAVE(gfxv) \ 487 ((gfxv) >= 100100 ? 12 : 8) /* GFX_VERSION_NAVI10*/ 488 489 #define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40 490 491 void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev) 492 { 493 struct kfd_node_properties *props = &dev->node_props; 494 u32 gfxv = props->gfx_target_version; 495 u32 ctl_stack_size; 496 u32 wg_data_size; 497 u32 wave_num; 498 u32 cu_num; 499 500 if (gfxv < 80001) /* GFX_VERSION_CARRIZO */ 501 return; 502 503 cu_num = props->simd_count / props->simd_per_cu / NUM_XCC(dev->gpu->xcc_mask); 504 wave_num = get_num_waves(props, gfxv, cu_num); 505 506 wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), 507 AMDGPU_GPU_PAGE_SIZE); 508 ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8; 509 ctl_stack_size = ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size, 510 AMDGPU_GPU_PAGE_SIZE); 511 512 if ((gfxv / 10000 * 10000) == 100000) { 513 /* HW design limits control stack size to 0x7000. 514 * This is insufficient for theoretical PM4 cases 515 * but sufficient for AQL, limited by SPI events. 516 */ 517 ctl_stack_size = min(ctl_stack_size, 0x7000); 518 } 519 520 props->ctl_stack_size = ctl_stack_size; 521 props->debug_memory_size = ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); 522 props->cwsr_size = ALIGN(ctl_stack_size + wg_data_size, PAGE_SIZE); 523 524 if (gfxv == 80002) /* GFX_VERSION_TONGA */ 525 props->eop_buffer_size = 0x8000; 526 else if (gfxv == 90402) /* GFX_VERSION_AQUA_VANJARAM */ 527 props->eop_buffer_size = 4096; 528 else if (gfxv >= 80000) 529 props->eop_buffer_size = 4096; 530 } 531