1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "amdgpu_sdma.h" 40 #include "mes_v11_api_def.h" 41 #include "kfd_debug.h" 42 43 /* Size of the per-pipe EOP queue */ 44 #define CIK_HPD_EOP_BYTES_LOG2 11 45 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 46 /* See unmap_queues_cpsch() */ 47 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff 48 49 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 50 u32 pasid, unsigned int vmid); 51 52 static int execute_queues_cpsch(struct device_queue_manager *dqm, 53 enum kfd_unmap_queues_filter filter, 54 uint32_t filter_param, 55 uint32_t grace_period); 56 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 57 enum kfd_unmap_queues_filter filter, 58 uint32_t filter_param, 59 uint32_t grace_period, 60 bool reset); 61 62 static int map_queues_cpsch(struct device_queue_manager *dqm); 63 64 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 65 struct queue *q); 66 67 static inline void deallocate_hqd(struct device_queue_manager *dqm, 68 struct queue *q); 69 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 70 static int allocate_sdma_queue(struct device_queue_manager *dqm, 71 struct queue *q, const uint32_t *restore_sdma_id); 72 73 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); 74 static int resume_all_queues_mes(struct device_queue_manager *dqm); 75 static int suspend_all_queues_mes(struct device_queue_manager *dqm); 76 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, 77 u32 doorbell_offset); 78 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 79 struct qcm_process_device *qpd); 80 81 static inline 82 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 83 { 84 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 85 return KFD_MQD_TYPE_SDMA; 86 return KFD_MQD_TYPE_CP; 87 } 88 89 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 90 { 91 int i; 92 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 93 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 94 95 /* queue is available for KFD usage if bit is 1 */ 96 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 97 if (test_bit(pipe_offset + i, 98 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 99 return true; 100 return false; 101 } 102 103 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 104 { 105 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 106 AMDGPU_MAX_QUEUES); 107 } 108 109 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 110 { 111 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 112 } 113 114 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 115 { 116 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 117 } 118 119 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 120 { 121 return kfd_get_num_sdma_engines(dqm->dev) + 122 kfd_get_num_xgmi_sdma_engines(dqm->dev); 123 } 124 125 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 126 { 127 return kfd_get_num_sdma_engines(dqm->dev) * 128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 129 } 130 131 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 132 { 133 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 134 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 135 } 136 137 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 138 { 139 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 140 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 141 142 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 143 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 144 145 /* Mask out the reserved queues */ 146 bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) * 147 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 148 bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) * 149 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 150 } 151 152 void program_sh_mem_settings(struct device_queue_manager *dqm, 153 struct qcm_process_device *qpd) 154 { 155 uint32_t xcc_mask = dqm->dev->xcc_mask; 156 int xcc_id; 157 158 for_each_inst(xcc_id, xcc_mask) 159 dqm->dev->kfd2kgd->program_sh_mem_settings( 160 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 161 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 162 qpd->sh_mem_bases, xcc_id); 163 } 164 165 static void kfd_hws_hang(struct device_queue_manager *dqm) 166 { 167 struct device_process_node *cur; 168 struct qcm_process_device *qpd; 169 struct queue *q; 170 171 /* Mark all device queues as reset. */ 172 list_for_each_entry(cur, &dqm->queues, list) { 173 qpd = cur->qpd; 174 list_for_each_entry(q, &qpd->queues_list, list) { 175 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 176 177 pdd->has_reset_queue = true; 178 } 179 } 180 181 /* 182 * Issue a GPU reset if HWS is unresponsive 183 */ 184 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 185 } 186 187 static int convert_to_mes_queue_type(int queue_type) 188 { 189 int mes_queue_type; 190 191 switch (queue_type) { 192 case KFD_QUEUE_TYPE_COMPUTE: 193 mes_queue_type = MES_QUEUE_TYPE_COMPUTE; 194 break; 195 case KFD_QUEUE_TYPE_SDMA: 196 mes_queue_type = MES_QUEUE_TYPE_SDMA; 197 break; 198 default: 199 WARN(1, "Invalid queue type %d", queue_type); 200 mes_queue_type = -EINVAL; 201 break; 202 } 203 204 return mes_queue_type; 205 } 206 207 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 208 struct qcm_process_device *qpd) 209 { 210 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 211 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 212 struct mes_add_queue_input queue_input; 213 int r, queue_type; 214 uint64_t wptr_addr_off; 215 216 if (!dqm->sched_running || dqm->sched_halt) 217 return 0; 218 if (!down_read_trylock(&adev->reset_domain->sem)) 219 return -EIO; 220 221 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 222 queue_input.process_id = pdd->pasid; 223 queue_input.page_table_base_addr = qpd->page_table_base; 224 queue_input.process_va_start = 0; 225 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 226 /* MES unit for quantum is 100ns */ 227 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 228 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 229 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 230 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 231 queue_input.inprocess_gang_priority = q->properties.priority; 232 queue_input.gang_global_priority_level = 233 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 234 queue_input.doorbell_offset = q->properties.doorbell_off; 235 queue_input.mqd_addr = q->gart_mqd_addr; 236 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 237 238 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 239 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 240 241 queue_input.is_kfd_process = 1; 242 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 243 queue_input.queue_size = q->properties.queue_size >> 2; 244 245 queue_input.paging = false; 246 queue_input.tba_addr = qpd->tba_addr; 247 queue_input.tma_addr = qpd->tma_addr; 248 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 249 queue_input.skip_process_ctx_clear = 250 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 251 (qpd->pqm->process->debug_trap_enabled || 252 kfd_dbg_has_ttmps_always_setup(q->device)); 253 254 queue_type = convert_to_mes_queue_type(q->properties.type); 255 if (queue_type < 0) { 256 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 257 q->properties.type); 258 up_read(&adev->reset_domain->sem); 259 return -EINVAL; 260 } 261 queue_input.queue_type = (uint32_t)queue_type; 262 263 queue_input.exclusively_scheduled = q->properties.is_gws; 264 queue_input.sh_mem_config_data = qpd->sh_mem_config; 265 queue_input.vm_cntx_cntl = qpd->vm_cntx_cntl; 266 queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 267 268 amdgpu_mes_lock(&adev->mes); 269 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 270 amdgpu_mes_unlock(&adev->mes); 271 up_read(&adev->reset_domain->sem); 272 if (r) { 273 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 274 q->properties.doorbell_off); 275 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 276 kfd_hws_hang(dqm); 277 } 278 279 return r; 280 } 281 282 static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q, 283 struct qcm_process_device *qpd, 284 bool is_for_reset, 285 bool flush_mes_queue) 286 { 287 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 288 int r; 289 struct mes_remove_queue_input queue_input; 290 291 /* queue was already removed during reset */ 292 if (q->properties.is_reset) 293 return 0; 294 295 if (!dqm->sched_running || dqm->sched_halt) 296 return 0; 297 if (!down_read_trylock(&adev->reset_domain->sem)) 298 return -EIO; 299 300 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 301 queue_input.doorbell_offset = q->properties.doorbell_off; 302 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 303 queue_input.remove_queue_after_reset = flush_mes_queue; 304 queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 305 306 amdgpu_mes_lock(&adev->mes); 307 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 308 amdgpu_mes_unlock(&adev->mes); 309 up_read(&adev->reset_domain->sem); 310 311 if (is_for_reset) 312 return r; 313 314 if (r) { 315 if (!suspend_all_queues_mes(dqm)) 316 return resume_all_queues_mes(dqm); 317 318 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", 319 q->properties.doorbell_off); 320 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 321 kfd_hws_hang(dqm); 322 } 323 324 return r; 325 } 326 327 static void set_perfcount(struct device_queue_manager *dqm, int enable) 328 { 329 struct device_process_node *cur; 330 struct qcm_process_device *qpd; 331 struct queue *q; 332 struct mqd_update_info minfo = { 0 }; 333 334 if (!dqm) 335 return; 336 337 minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE : 338 UPDATE_FLAG_PERFCOUNT_DISABLE); 339 dqm_lock(dqm); 340 list_for_each_entry(cur, &dqm->queues, list) { 341 qpd = cur->qpd; 342 list_for_each_entry(q, &qpd->queues_list, list) { 343 pqm_update_mqd(qpd->pqm, q->properties.queue_id, 344 &minfo); 345 } 346 } 347 dqm_unlock(dqm); 348 } 349 350 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 351 struct qcm_process_device *qpd) 352 { 353 return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false); 354 } 355 356 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) 357 { 358 struct device_process_node *cur; 359 struct device *dev = dqm->dev->adev->dev; 360 struct qcm_process_device *qpd; 361 struct queue *q; 362 int retval = 0; 363 364 list_for_each_entry(cur, &dqm->queues, list) { 365 qpd = cur->qpd; 366 list_for_each_entry(q, &qpd->queues_list, list) { 367 if (q->properties.is_active) { 368 retval = remove_queue_mes(dqm, q, qpd); 369 if (retval) { 370 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 371 __func__, 372 q->properties.queue_id, 373 dqm->dev->id); 374 return retval; 375 } 376 } 377 } 378 } 379 380 return retval; 381 } 382 383 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) 384 { 385 struct device_process_node *cur; 386 struct device *dev = dqm->dev->adev->dev; 387 struct qcm_process_device *qpd; 388 struct queue *q; 389 int retval = 0; 390 391 list_for_each_entry(cur, &dqm->queues, list) { 392 qpd = cur->qpd; 393 list_for_each_entry(q, &qpd->queues_list, list) { 394 if (!q->properties.is_active) 395 continue; 396 retval = add_queue_mes(dqm, q, qpd); 397 if (retval) { 398 dev_err(dev, "%s: Failed to add queue %d for dev %d", 399 __func__, 400 q->properties.queue_id, 401 dqm->dev->id); 402 return retval; 403 } 404 } 405 } 406 407 return retval; 408 } 409 410 static int reset_queues_mes(struct device_queue_manager *dqm) 411 { 412 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 413 int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; 414 int num_hung = 0, r = 0, i, pipe, queue, queue_type; 415 u32 *hung_array = dqm->hung_db_array; 416 struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info; 417 struct kfd_process_device *pdd; 418 struct queue *q; 419 420 if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) { 421 r = -ENOTRECOVERABLE; 422 goto fail; 423 } 424 425 /* reset should be used only in dqm locked queue reset */ 426 if (WARN_ON(dqm->detect_hang_count > 0)) 427 return 0; 428 429 if (!amdgpu_gpu_recovery) { 430 r = -ENOTRECOVERABLE; 431 goto fail; 432 } 433 434 if (!hung_array || !hqd_info) { 435 r = -ENOMEM; 436 goto fail; 437 } 438 439 memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info)); 440 441 /* 442 * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called 443 * post suspend_all as reset & detect will return all hung queue types. 444 * 445 * Passed parameter is for targeting queues not scheduled by MES add_queue. 446 */ 447 r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE, 448 false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1); 449 450 if (!num_hung || r) { 451 r = -ENOTRECOVERABLE; 452 goto fail; 453 } 454 455 /* MES resets queue/pipe and cleans up internally */ 456 for (i = 0; i < num_hung; i++) { 457 hqd_info[i].bit0_31 = hung_array[i + hqd_info_size]; 458 pipe = hqd_info[i].pipe_index; 459 queue = hqd_info[i].queue_index; 460 queue_type = hqd_info[i].queue_type; 461 462 if (queue_type != MES_QUEUE_TYPE_COMPUTE && 463 queue_type != MES_QUEUE_TYPE_SDMA) { 464 pr_warn("Unsupported hung queue reset type: %d\n", queue_type); 465 hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET; 466 continue; 467 } 468 469 q = find_queue_by_doorbell_offset(dqm, hung_array[i]); 470 if (!q) { 471 r = -ENOTRECOVERABLE; 472 goto fail; 473 } 474 475 pdd = kfd_get_process_device_data(q->device, q->process); 476 if (!pdd) { 477 r = -ENODEV; 478 goto fail; 479 } 480 481 pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n", 482 hung_array[i], pipe, queue, queue_type); 483 /* Proceed remove_queue with reset=true */ 484 remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false); 485 set_queue_as_reset(dqm, q, &pdd->qpd); 486 } 487 488 dqm->detect_hang_count = num_hung; 489 kfd_signal_reset_event(dqm->dev); 490 491 fail: 492 dqm->detect_hang_count = 0; 493 return r; 494 } 495 496 static int suspend_all_queues_mes(struct device_queue_manager *dqm) 497 { 498 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 499 int r = 0; 500 501 if (!down_read_trylock(&adev->reset_domain->sem)) 502 return -EIO; 503 504 r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1); 505 up_read(&adev->reset_domain->sem); 506 507 if (r) { 508 if (!reset_queues_mes(dqm)) 509 return 0; 510 511 dev_err(adev->dev, "failed to suspend gangs from MES\n"); 512 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 513 kfd_hws_hang(dqm); 514 } 515 516 return r; 517 } 518 519 static int resume_all_queues_mes(struct device_queue_manager *dqm) 520 { 521 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 522 int r = 0; 523 524 if (!down_read_trylock(&adev->reset_domain->sem)) 525 return -EIO; 526 527 r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1); 528 up_read(&adev->reset_domain->sem); 529 530 if (r) { 531 dev_err(adev->dev, "failed to resume gangs from MES\n"); 532 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 533 kfd_hws_hang(dqm); 534 } 535 536 return r; 537 } 538 539 static void increment_queue_count(struct device_queue_manager *dqm, 540 struct qcm_process_device *qpd, 541 struct queue *q) 542 { 543 dqm->active_queue_count++; 544 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 545 dqm->active_cp_queue_count++; 546 547 if (q->properties.is_gws) { 548 dqm->gws_queue_count++; 549 qpd->mapped_gws_queue = true; 550 } 551 } 552 553 static void decrement_queue_count(struct device_queue_manager *dqm, 554 struct qcm_process_device *qpd, 555 struct queue *q) 556 { 557 dqm->active_queue_count--; 558 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 559 dqm->active_cp_queue_count--; 560 561 if (q->properties.is_gws) { 562 dqm->gws_queue_count--; 563 qpd->mapped_gws_queue = false; 564 } 565 } 566 567 /* 568 * Allocate a doorbell ID to this queue. 569 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 570 */ 571 static int allocate_doorbell(struct qcm_process_device *qpd, 572 struct queue *q, 573 uint32_t const *restore_id) 574 { 575 struct kfd_node *dev = qpd->dqm->dev; 576 577 if (!KFD_IS_SOC15(dev)) { 578 /* On pre-SOC15 chips we need to use the queue ID to 579 * preserve the user mode ABI. 580 */ 581 582 if (restore_id && *restore_id != q->properties.queue_id) 583 return -EINVAL; 584 585 q->doorbell_id = q->properties.queue_id; 586 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 587 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 588 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 589 * doorbell assignments based on the engine and queue id. 590 * The doobell index distance between RLC (2*i) and (2*i+1) 591 * for a SDMA engine is 512. 592 */ 593 594 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 595 596 /* 597 * q->properties.sdma_engine_id corresponds to the virtual 598 * sdma engine number. However, for doorbell allocation, 599 * we need the physical sdma engine id in order to get the 600 * correct doorbell offset. 601 */ 602 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * 603 get_num_all_sdma_engines(qpd->dqm) + 604 q->properties.sdma_engine_id] 605 + (q->properties.sdma_queue_id & 1) 606 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET 607 + (q->properties.sdma_queue_id >> 1); 608 609 if (restore_id && *restore_id != valid_id) 610 return -EINVAL; 611 q->doorbell_id = valid_id; 612 } else { 613 /* For CP queues on SOC15 */ 614 if (restore_id) { 615 if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 616 return -EINVAL; 617 618 /* make sure that ID is free */ 619 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 620 return -EINVAL; 621 622 q->doorbell_id = *restore_id; 623 } else { 624 /* or reserve a free doorbell ID */ 625 unsigned int found; 626 627 found = find_first_zero_bit(qpd->doorbell_bitmap, 628 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 629 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 630 pr_debug("No doorbells available"); 631 return -EBUSY; 632 } 633 set_bit(found, qpd->doorbell_bitmap); 634 q->doorbell_id = found; 635 } 636 } 637 638 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 639 qpd->proc_doorbells, 640 q->doorbell_id, 641 dev->kfd->device_info.doorbell_size); 642 return 0; 643 } 644 645 static void deallocate_doorbell(struct qcm_process_device *qpd, 646 struct queue *q) 647 { 648 unsigned int old; 649 struct kfd_node *dev = qpd->dqm->dev; 650 651 if (!KFD_IS_SOC15(dev) || 652 q->properties.type == KFD_QUEUE_TYPE_SDMA || 653 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 654 return; 655 656 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 657 WARN_ON(!old); 658 } 659 660 static void program_trap_handler_settings(struct device_queue_manager *dqm, 661 struct qcm_process_device *qpd) 662 { 663 uint32_t xcc_mask = dqm->dev->xcc_mask; 664 int xcc_id; 665 666 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 667 for_each_inst(xcc_id, xcc_mask) 668 dqm->dev->kfd2kgd->program_trap_handler_settings( 669 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 670 qpd->tma_addr, xcc_id); 671 } 672 673 static int allocate_vmid(struct device_queue_manager *dqm, 674 struct qcm_process_device *qpd, 675 struct queue *q) 676 { 677 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 678 struct device *dev = dqm->dev->adev->dev; 679 int allocated_vmid = -1, i; 680 681 for (i = dqm->dev->vm_info.first_vmid_kfd; 682 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 683 if (!dqm->vmid_pasid[i]) { 684 allocated_vmid = i; 685 break; 686 } 687 } 688 689 if (allocated_vmid < 0) { 690 dev_err(dev, "no more vmid to allocate\n"); 691 return -ENOSPC; 692 } 693 694 pr_debug("vmid allocated: %d\n", allocated_vmid); 695 696 dqm->vmid_pasid[allocated_vmid] = pdd->pasid; 697 698 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid); 699 700 qpd->vmid = allocated_vmid; 701 q->properties.vmid = allocated_vmid; 702 703 program_sh_mem_settings(dqm, qpd); 704 705 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 706 program_trap_handler_settings(dqm, qpd); 707 708 /* qpd->page_table_base is set earlier when register_process() 709 * is called, i.e. when the first queue is created. 710 */ 711 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 712 qpd->vmid, 713 qpd->page_table_base); 714 /* invalidate the VM context after pasid and vmid mapping is set up */ 715 kfd_flush_tlb(qpd_to_pdd(qpd)); 716 717 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 718 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 719 qpd->sh_hidden_private_base, qpd->vmid); 720 721 return 0; 722 } 723 724 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 725 struct qcm_process_device *qpd) 726 { 727 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 728 int ret; 729 730 if (!qpd->ib_kaddr) 731 return -ENOMEM; 732 733 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 734 if (ret) 735 return ret; 736 737 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 738 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 739 pmf->release_mem_size / sizeof(uint32_t)); 740 } 741 742 static void deallocate_vmid(struct device_queue_manager *dqm, 743 struct qcm_process_device *qpd, 744 struct queue *q) 745 { 746 struct device *dev = dqm->dev->adev->dev; 747 748 /* On GFX v7, CP doesn't flush TC at dequeue */ 749 if (q->device->adev->asic_type == CHIP_HAWAII) 750 if (flush_texture_cache_nocpsch(q->device, qpd)) 751 dev_err(dev, "Failed to flush TC\n"); 752 753 kfd_flush_tlb(qpd_to_pdd(qpd)); 754 755 /* Release the vmid mapping */ 756 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 757 dqm->vmid_pasid[qpd->vmid] = 0; 758 759 qpd->vmid = 0; 760 q->properties.vmid = 0; 761 } 762 763 static int create_queue_nocpsch(struct device_queue_manager *dqm, 764 struct queue *q, 765 struct qcm_process_device *qpd, 766 const struct kfd_criu_queue_priv_data *qd, 767 const void *restore_mqd, const void *restore_ctl_stack) 768 { 769 struct mqd_manager *mqd_mgr; 770 int retval; 771 772 dqm_lock(dqm); 773 774 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 775 pr_warn("Can't create new usermode queue because %d queues were already created\n", 776 dqm->total_queue_count); 777 retval = -EPERM; 778 goto out_unlock; 779 } 780 781 if (list_empty(&qpd->queues_list)) { 782 retval = allocate_vmid(dqm, qpd, q); 783 if (retval) 784 goto out_unlock; 785 } 786 q->properties.vmid = qpd->vmid; 787 /* 788 * Eviction state logic: mark all queues as evicted, even ones 789 * not currently active. Restoring inactive queues later only 790 * updates the is_evicted flag but is a no-op otherwise. 791 */ 792 q->properties.is_evicted = !!qpd->evicted; 793 794 q->properties.tba_addr = qpd->tba_addr; 795 q->properties.tma_addr = qpd->tma_addr; 796 797 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 798 q->properties.type)]; 799 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 800 retval = allocate_hqd(dqm, q); 801 if (retval) 802 goto deallocate_vmid; 803 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 804 q->pipe, q->queue); 805 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 806 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 807 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 808 if (retval) 809 goto deallocate_vmid; 810 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 811 } 812 813 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 814 if (retval) 815 goto out_deallocate_hqd; 816 817 /* Temporarily release dqm lock to avoid a circular lock dependency */ 818 dqm_unlock(dqm); 819 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties); 820 dqm_lock(dqm); 821 822 if (!q->mqd_mem_obj) { 823 retval = -ENOMEM; 824 goto out_deallocate_doorbell; 825 } 826 827 if (qd) 828 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 829 &q->properties, restore_mqd, restore_ctl_stack, 830 qd->ctl_stack_size); 831 else 832 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 833 &q->gart_mqd_addr, &q->properties); 834 835 if (q->properties.is_active) { 836 if (!dqm->sched_running) { 837 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 838 goto add_queue_to_list; 839 } 840 841 if (WARN(q->process->mm != current->mm, 842 "should only run in user thread")) 843 retval = -EFAULT; 844 else 845 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 846 q->queue, &q->properties, current->mm); 847 if (retval) 848 goto out_free_mqd; 849 } 850 851 add_queue_to_list: 852 list_add(&q->list, &qpd->queues_list); 853 qpd->queue_count++; 854 if (q->properties.is_active) 855 increment_queue_count(dqm, qpd, q); 856 857 /* 858 * Unconditionally increment this counter, regardless of the queue's 859 * type or whether the queue is active. 860 */ 861 dqm->total_queue_count++; 862 pr_debug("Total of %d queues are accountable so far\n", 863 dqm->total_queue_count); 864 goto out_unlock; 865 866 out_free_mqd: 867 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 868 out_deallocate_doorbell: 869 deallocate_doorbell(qpd, q); 870 out_deallocate_hqd: 871 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 872 deallocate_hqd(dqm, q); 873 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 874 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 875 deallocate_sdma_queue(dqm, q); 876 deallocate_vmid: 877 if (list_empty(&qpd->queues_list)) 878 deallocate_vmid(dqm, qpd, q); 879 out_unlock: 880 dqm_unlock(dqm); 881 return retval; 882 } 883 884 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 885 { 886 bool set; 887 int pipe, bit, i; 888 889 set = false; 890 891 for (pipe = dqm->next_pipe_to_allocate, i = 0; 892 i < get_pipes_per_mec(dqm); 893 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 894 895 if (!is_pipe_enabled(dqm, 0, pipe)) 896 continue; 897 898 if (dqm->allocated_queues[pipe] != 0) { 899 bit = ffs(dqm->allocated_queues[pipe]) - 1; 900 dqm->allocated_queues[pipe] &= ~(1 << bit); 901 q->pipe = pipe; 902 q->queue = bit; 903 set = true; 904 break; 905 } 906 } 907 908 if (!set) 909 return -EBUSY; 910 911 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 912 /* horizontal hqd allocation */ 913 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 914 915 return 0; 916 } 917 918 static inline void deallocate_hqd(struct device_queue_manager *dqm, 919 struct queue *q) 920 { 921 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 922 } 923 924 #define SQ_IND_CMD_CMD_KILL 0x00000003 925 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 926 927 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 928 { 929 int status = 0; 930 unsigned int vmid; 931 uint16_t queried_pasid; 932 union SQ_CMD_BITS reg_sq_cmd; 933 union GRBM_GFX_INDEX_BITS reg_gfx_index; 934 struct kfd_process_device *pdd; 935 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 936 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 937 uint32_t xcc_mask = dev->xcc_mask; 938 int xcc_id; 939 940 reg_sq_cmd.u32All = 0; 941 reg_gfx_index.u32All = 0; 942 943 pr_debug("Killing all process wavefronts\n"); 944 945 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 946 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 947 return -EOPNOTSUPP; 948 } 949 950 /* taking the VMID for that process on the safe way using PDD */ 951 pdd = kfd_get_process_device_data(dev, p); 952 if (!pdd) 953 return -EFAULT; 954 955 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 956 * ATC_VMID15_PASID_MAPPING 957 * to check which VMID the current process is mapped to. 958 */ 959 960 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 961 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 962 (dev->adev, vmid, &queried_pasid); 963 964 if (status && queried_pasid == pdd->pasid) { 965 pr_debug("Killing wave fronts of vmid %d and process pid %d\n", 966 vmid, p->lead_thread->pid); 967 break; 968 } 969 } 970 971 if (vmid > last_vmid_to_scan) { 972 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n", 973 p->lead_thread->pid); 974 return -EFAULT; 975 } 976 977 reg_gfx_index.bits.sh_broadcast_writes = 1; 978 reg_gfx_index.bits.se_broadcast_writes = 1; 979 reg_gfx_index.bits.instance_broadcast_writes = 1; 980 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 981 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 982 reg_sq_cmd.bits.vm_id = vmid; 983 984 for_each_inst(xcc_id, xcc_mask) 985 dev->kfd2kgd->wave_control_execute( 986 dev->adev, reg_gfx_index.u32All, 987 reg_sq_cmd.u32All, xcc_id); 988 989 return 0; 990 } 991 992 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 993 * to avoid asynchronized access 994 */ 995 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 996 struct qcm_process_device *qpd, 997 struct queue *q) 998 { 999 int retval; 1000 struct mqd_manager *mqd_mgr; 1001 1002 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 1003 1004 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 1005 deallocate_hqd(dqm, q); 1006 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1007 deallocate_sdma_queue(dqm, q); 1008 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 1009 deallocate_sdma_queue(dqm, q); 1010 else { 1011 pr_debug("q->properties.type %d is invalid\n", 1012 q->properties.type); 1013 return -EINVAL; 1014 } 1015 dqm->total_queue_count--; 1016 1017 deallocate_doorbell(qpd, q); 1018 1019 if (!dqm->sched_running) { 1020 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 1021 return 0; 1022 } 1023 1024 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1025 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 1026 KFD_UNMAP_LATENCY_MS, 1027 q->pipe, q->queue); 1028 if (retval == -ETIME) 1029 qpd->reset_wavefronts = true; 1030 1031 list_del(&q->list); 1032 if (list_empty(&qpd->queues_list)) { 1033 if (qpd->reset_wavefronts) { 1034 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 1035 dqm->dev); 1036 /* dbgdev_wave_reset_wavefronts has to be called before 1037 * deallocate_vmid(), i.e. when vmid is still in use. 1038 */ 1039 dbgdev_wave_reset_wavefronts(dqm->dev, 1040 qpd->pqm->process); 1041 qpd->reset_wavefronts = false; 1042 } 1043 1044 deallocate_vmid(dqm, qpd, q); 1045 } 1046 qpd->queue_count--; 1047 if (q->properties.is_active) 1048 decrement_queue_count(dqm, qpd, q); 1049 1050 return retval; 1051 } 1052 1053 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 1054 struct qcm_process_device *qpd, 1055 struct queue *q) 1056 { 1057 int retval; 1058 uint64_t sdma_val = 0; 1059 struct device *dev = dqm->dev->adev->dev; 1060 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 1061 struct mqd_manager *mqd_mgr = 1062 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 1063 1064 /* Get the SDMA queue stats */ 1065 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 1066 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1067 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 1068 &sdma_val); 1069 if (retval) 1070 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 1071 q->properties.queue_id); 1072 } 1073 1074 dqm_lock(dqm); 1075 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 1076 if (!retval) 1077 pdd->sdma_past_activity_counter += sdma_val; 1078 dqm_unlock(dqm); 1079 1080 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 1081 1082 return retval; 1083 } 1084 1085 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 1086 struct mqd_update_info *minfo) 1087 { 1088 int retval = 0; 1089 struct device *dev = dqm->dev->adev->dev; 1090 struct mqd_manager *mqd_mgr; 1091 struct kfd_process_device *pdd; 1092 bool prev_active = false; 1093 1094 dqm_lock(dqm); 1095 pdd = kfd_get_process_device_data(q->device, q->process); 1096 if (!pdd) { 1097 retval = -ENODEV; 1098 goto out_unlock; 1099 } 1100 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1101 q->properties.type)]; 1102 1103 /* Save previous activity state for counters */ 1104 prev_active = q->properties.is_active; 1105 1106 /* Make sure the queue is unmapped before updating the MQD */ 1107 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1108 if (!dqm->dev->kfd->shared_resources.enable_mes) 1109 retval = unmap_queues_cpsch(dqm, 1110 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 1111 else if (prev_active) 1112 retval = remove_queue_mes(dqm, q, &pdd->qpd); 1113 1114 /* queue is reset so inaccessable */ 1115 if (pdd->has_reset_queue) { 1116 retval = -EACCES; 1117 goto out_unlock; 1118 } 1119 1120 if (retval) { 1121 dev_err(dev, "unmap queue failed\n"); 1122 goto out_unlock; 1123 } 1124 } else if (prev_active && 1125 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1126 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1127 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1128 1129 if (!dqm->sched_running) { 1130 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 1131 goto out_unlock; 1132 } 1133 1134 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1135 (dqm->dev->kfd->cwsr_enabled ? 1136 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1137 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1138 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1139 if (retval) { 1140 dev_err(dev, "destroy mqd failed\n"); 1141 goto out_unlock; 1142 } 1143 } 1144 1145 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 1146 1147 /* 1148 * check active state vs. the previous state and modify 1149 * counter accordingly. map_queues_cpsch uses the 1150 * dqm->active_queue_count to determine whether a new runlist must be 1151 * uploaded. 1152 */ 1153 if (q->properties.is_active && !prev_active) { 1154 increment_queue_count(dqm, &pdd->qpd, q); 1155 } else if (!q->properties.is_active && prev_active) { 1156 decrement_queue_count(dqm, &pdd->qpd, q); 1157 } else if (q->gws && !q->properties.is_gws) { 1158 if (q->properties.is_active) { 1159 dqm->gws_queue_count++; 1160 pdd->qpd.mapped_gws_queue = true; 1161 } 1162 q->properties.is_gws = true; 1163 } else if (!q->gws && q->properties.is_gws) { 1164 if (q->properties.is_active) { 1165 dqm->gws_queue_count--; 1166 pdd->qpd.mapped_gws_queue = false; 1167 } 1168 q->properties.is_gws = false; 1169 } 1170 1171 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1172 if (!dqm->dev->kfd->shared_resources.enable_mes) 1173 retval = map_queues_cpsch(dqm); 1174 else if (q->properties.is_active) 1175 retval = add_queue_mes(dqm, q, &pdd->qpd); 1176 } else if (q->properties.is_active && 1177 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1178 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1179 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1180 if (WARN(q->process->mm != current->mm, 1181 "should only run in user thread")) 1182 retval = -EFAULT; 1183 else 1184 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 1185 q->pipe, q->queue, 1186 &q->properties, current->mm); 1187 } 1188 1189 out_unlock: 1190 dqm_unlock(dqm); 1191 return retval; 1192 } 1193 1194 /* suspend_single_queue does not lock the dqm like the 1195 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 1196 * lock the dqm before calling, and unlock after calling. 1197 * 1198 * The reason we don't lock the dqm is because this function may be 1199 * called on multiple queues in a loop, so rather than locking/unlocking 1200 * multiple times, we will just keep the dqm locked for all of the calls. 1201 */ 1202 static int suspend_single_queue(struct device_queue_manager *dqm, 1203 struct kfd_process_device *pdd, 1204 struct queue *q) 1205 { 1206 bool is_new; 1207 1208 if (q->properties.is_suspended) 1209 return 0; 1210 1211 pr_debug("Suspending process pid %d queue [%i]\n", 1212 pdd->process->lead_thread->pid, 1213 q->properties.queue_id); 1214 1215 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 1216 1217 if (is_new || q->properties.is_being_destroyed) { 1218 pr_debug("Suspend: skip %s queue id %i\n", 1219 is_new ? "new" : "destroyed", 1220 q->properties.queue_id); 1221 return -EBUSY; 1222 } 1223 1224 q->properties.is_suspended = true; 1225 if (q->properties.is_active) { 1226 if (dqm->dev->kfd->shared_resources.enable_mes) { 1227 int r = remove_queue_mes(dqm, q, &pdd->qpd); 1228 1229 if (r) 1230 return r; 1231 } 1232 1233 decrement_queue_count(dqm, &pdd->qpd, q); 1234 q->properties.is_active = false; 1235 } 1236 1237 return 0; 1238 } 1239 1240 /* resume_single_queue does not lock the dqm like the functions 1241 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1242 * lock the dqm before calling, and unlock after calling. 1243 * 1244 * The reason we don't lock the dqm is because this function may be 1245 * called on multiple queues in a loop, so rather than locking/unlocking 1246 * multiple times, we will just keep the dqm locked for all of the calls. 1247 */ 1248 static int resume_single_queue(struct device_queue_manager *dqm, 1249 struct qcm_process_device *qpd, 1250 struct queue *q) 1251 { 1252 struct kfd_process_device *pdd; 1253 1254 if (!q->properties.is_suspended) 1255 return 0; 1256 1257 pdd = qpd_to_pdd(qpd); 1258 1259 pr_debug("Restoring from suspend process pid %d queue [%i]\n", 1260 pdd->process->lead_thread->pid, 1261 q->properties.queue_id); 1262 1263 q->properties.is_suspended = false; 1264 1265 if (QUEUE_IS_ACTIVE(q->properties)) { 1266 if (dqm->dev->kfd->shared_resources.enable_mes) { 1267 int r = add_queue_mes(dqm, q, &pdd->qpd); 1268 1269 if (r) 1270 return r; 1271 } 1272 1273 q->properties.is_active = true; 1274 increment_queue_count(dqm, qpd, q); 1275 } 1276 1277 return 0; 1278 } 1279 1280 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1281 struct qcm_process_device *qpd) 1282 { 1283 struct queue *q; 1284 struct mqd_manager *mqd_mgr; 1285 struct kfd_process_device *pdd; 1286 int retval, ret = 0; 1287 1288 dqm_lock(dqm); 1289 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1290 goto out; 1291 1292 pdd = qpd_to_pdd(qpd); 1293 pr_debug_ratelimited("Evicting process pid %d queues\n", 1294 pdd->process->lead_thread->pid); 1295 1296 pdd->last_evict_timestamp = get_jiffies_64(); 1297 /* Mark all queues as evicted. Deactivate all active queues on 1298 * the qpd. 1299 */ 1300 list_for_each_entry(q, &qpd->queues_list, list) { 1301 q->properties.is_evicted = true; 1302 if (!q->properties.is_active) 1303 continue; 1304 1305 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1306 q->properties.type)]; 1307 q->properties.is_active = false; 1308 decrement_queue_count(dqm, qpd, q); 1309 1310 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1311 continue; 1312 1313 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1314 (dqm->dev->kfd->cwsr_enabled ? 1315 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1316 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1317 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1318 if (retval && !ret) 1319 /* Return the first error, but keep going to 1320 * maintain a consistent eviction state 1321 */ 1322 ret = retval; 1323 } 1324 1325 out: 1326 dqm_unlock(dqm); 1327 return ret; 1328 } 1329 1330 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1331 struct qcm_process_device *qpd) 1332 { 1333 struct queue *q; 1334 struct device *dev = dqm->dev->adev->dev; 1335 struct kfd_process_device *pdd; 1336 int retval = 0; 1337 1338 dqm_lock(dqm); 1339 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1340 goto out; 1341 1342 pdd = qpd_to_pdd(qpd); 1343 1344 /* The debugger creates processes that temporarily have not acquired 1345 * all VMs for all devices and has no VMs itself. 1346 * Skip queue eviction on process eviction. 1347 */ 1348 if (!pdd->drm_priv) 1349 goto out; 1350 1351 pr_debug_ratelimited("Evicting process pid %d queues\n", 1352 pdd->process->lead_thread->pid); 1353 1354 if (dqm->dev->kfd->shared_resources.enable_mes) 1355 pdd->last_evict_timestamp = get_jiffies_64(); 1356 1357 /* Mark all queues as evicted. Deactivate all active queues on 1358 * the qpd. 1359 */ 1360 list_for_each_entry(q, &qpd->queues_list, list) { 1361 q->properties.is_evicted = true; 1362 if (!q->properties.is_active) 1363 continue; 1364 1365 q->properties.is_active = false; 1366 decrement_queue_count(dqm, qpd, q); 1367 1368 if (dqm->dev->kfd->shared_resources.enable_mes) { 1369 retval = remove_queue_mes(dqm, q, qpd); 1370 if (retval) { 1371 dev_err(dev, "Failed to evict queue %d\n", 1372 q->properties.queue_id); 1373 goto out; 1374 } 1375 } 1376 } 1377 1378 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1379 pdd->last_evict_timestamp = get_jiffies_64(); 1380 retval = execute_queues_cpsch(dqm, 1381 qpd->is_debug ? 1382 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1383 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1384 USE_DEFAULT_GRACE_PERIOD); 1385 } 1386 1387 out: 1388 dqm_unlock(dqm); 1389 return retval; 1390 } 1391 1392 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1393 struct qcm_process_device *qpd) 1394 { 1395 struct mm_struct *mm = NULL; 1396 struct queue *q; 1397 struct mqd_manager *mqd_mgr; 1398 struct kfd_process_device *pdd; 1399 uint64_t pd_base; 1400 uint64_t eviction_duration; 1401 int retval, ret = 0; 1402 1403 pdd = qpd_to_pdd(qpd); 1404 /* Retrieve PD base */ 1405 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1406 1407 dqm_lock(dqm); 1408 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1409 goto out; 1410 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1411 qpd->evicted--; 1412 goto out; 1413 } 1414 1415 pr_debug_ratelimited("Restoring process pid %d queues\n", 1416 pdd->process->lead_thread->pid); 1417 1418 /* Update PD Base in QPD */ 1419 qpd->page_table_base = pd_base; 1420 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1421 1422 if (!list_empty(&qpd->queues_list)) { 1423 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1424 dqm->dev->adev, 1425 qpd->vmid, 1426 qpd->page_table_base); 1427 kfd_flush_tlb(pdd); 1428 } 1429 1430 /* Take a safe reference to the mm_struct, which may otherwise 1431 * disappear even while the kfd_process is still referenced. 1432 */ 1433 mm = get_task_mm(pdd->process->lead_thread); 1434 if (!mm) { 1435 ret = -EFAULT; 1436 goto out; 1437 } 1438 1439 /* Remove the eviction flags. Activate queues that are not 1440 * inactive for other reasons. 1441 */ 1442 list_for_each_entry(q, &qpd->queues_list, list) { 1443 q->properties.is_evicted = false; 1444 if (!QUEUE_IS_ACTIVE(q->properties)) 1445 continue; 1446 1447 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1448 q->properties.type)]; 1449 q->properties.is_active = true; 1450 increment_queue_count(dqm, qpd, q); 1451 1452 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1453 continue; 1454 1455 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1456 q->queue, &q->properties, mm); 1457 if (retval && !ret) 1458 /* Return the first error, but keep going to 1459 * maintain a consistent eviction state 1460 */ 1461 ret = retval; 1462 } 1463 qpd->evicted = 0; 1464 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1465 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1466 out: 1467 if (mm) 1468 mmput(mm); 1469 dqm_unlock(dqm); 1470 return ret; 1471 } 1472 1473 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1474 struct qcm_process_device *qpd) 1475 { 1476 struct queue *q; 1477 struct device *dev = dqm->dev->adev->dev; 1478 struct kfd_process_device *pdd; 1479 uint64_t eviction_duration; 1480 int retval = 0; 1481 1482 pdd = qpd_to_pdd(qpd); 1483 1484 dqm_lock(dqm); 1485 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1486 goto out; 1487 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1488 qpd->evicted--; 1489 goto out; 1490 } 1491 1492 /* The debugger creates processes that temporarily have not acquired 1493 * all VMs for all devices and has no VMs itself. 1494 * Skip queue restore on process restore. 1495 */ 1496 if (!pdd->drm_priv) 1497 goto vm_not_acquired; 1498 1499 pr_debug_ratelimited("Restoring process pid %d queues\n", 1500 pdd->process->lead_thread->pid); 1501 1502 /* Update PD Base in QPD */ 1503 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1504 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1505 1506 /* activate all active queues on the qpd */ 1507 list_for_each_entry(q, &qpd->queues_list, list) { 1508 q->properties.is_evicted = false; 1509 if (!QUEUE_IS_ACTIVE(q->properties)) 1510 continue; 1511 1512 q->properties.is_active = true; 1513 increment_queue_count(dqm, &pdd->qpd, q); 1514 1515 if (dqm->dev->kfd->shared_resources.enable_mes) { 1516 retval = add_queue_mes(dqm, q, qpd); 1517 if (retval) { 1518 dev_err(dev, "Failed to restore queue %d\n", 1519 q->properties.queue_id); 1520 goto out; 1521 } 1522 } 1523 } 1524 if (!dqm->dev->kfd->shared_resources.enable_mes) 1525 retval = execute_queues_cpsch(dqm, 1526 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1527 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1528 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1529 vm_not_acquired: 1530 qpd->evicted = 0; 1531 out: 1532 dqm_unlock(dqm); 1533 return retval; 1534 } 1535 1536 static int register_process(struct device_queue_manager *dqm, 1537 struct qcm_process_device *qpd) 1538 { 1539 struct device_process_node *n; 1540 struct kfd_process_device *pdd; 1541 uint64_t pd_base; 1542 int retval; 1543 1544 n = kzalloc_obj(*n); 1545 if (!n) 1546 return -ENOMEM; 1547 1548 n->qpd = qpd; 1549 1550 pdd = qpd_to_pdd(qpd); 1551 /* Retrieve PD base */ 1552 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1553 1554 dqm_lock(dqm); 1555 list_add(&n->list, &dqm->queues); 1556 1557 /* Update PD Base in QPD */ 1558 qpd->page_table_base = pd_base; 1559 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1560 1561 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1562 1563 dqm->processes_count++; 1564 1565 dqm_unlock(dqm); 1566 1567 /* Outside the DQM lock because under the DQM lock we can't do 1568 * reclaim or take other locks that others hold while reclaiming. 1569 */ 1570 kfd_inc_compute_active(dqm->dev); 1571 1572 return retval; 1573 } 1574 1575 static int unregister_process(struct device_queue_manager *dqm, 1576 struct qcm_process_device *qpd) 1577 { 1578 int retval = 0; 1579 struct device_process_node *cur, *next; 1580 1581 pr_debug("qpd->queues_list is %s\n", 1582 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1583 1584 dqm_lock(dqm); 1585 1586 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1587 if (qpd == cur->qpd) { 1588 list_del(&cur->list); 1589 kfree(cur); 1590 dqm->processes_count--; 1591 goto out; 1592 } 1593 } 1594 /* qpd not found in dqm list */ 1595 retval = 1; 1596 out: 1597 dqm_unlock(dqm); 1598 1599 /* Outside the DQM lock because under the DQM lock we can't do 1600 * reclaim or take other locks that others hold while reclaiming. 1601 */ 1602 if (!retval) 1603 kfd_dec_compute_active(dqm->dev); 1604 1605 return retval; 1606 } 1607 1608 static int 1609 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1610 unsigned int vmid) 1611 { 1612 uint32_t xcc_mask = dqm->dev->xcc_mask; 1613 int xcc_id, ret = 0; 1614 1615 for_each_inst(xcc_id, xcc_mask) { 1616 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1617 dqm->dev->adev, pasid, vmid, xcc_id); 1618 if (ret) 1619 break; 1620 } 1621 1622 return ret; 1623 } 1624 1625 static void init_interrupts(struct device_queue_manager *dqm) 1626 { 1627 uint32_t xcc_mask = dqm->dev->xcc_mask; 1628 unsigned int i, xcc_id; 1629 1630 for_each_inst(xcc_id, xcc_mask) { 1631 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1632 if (is_pipe_enabled(dqm, 0, i)) { 1633 dqm->dev->kfd2kgd->init_interrupts( 1634 dqm->dev->adev, i, xcc_id); 1635 } 1636 } 1637 } 1638 } 1639 1640 static int initialize_nocpsch(struct device_queue_manager *dqm) 1641 { 1642 int pipe, queue; 1643 1644 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1645 1646 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), 1647 sizeof(unsigned int), GFP_KERNEL); 1648 if (!dqm->allocated_queues) 1649 return -ENOMEM; 1650 1651 mutex_init(&dqm->lock_hidden); 1652 INIT_LIST_HEAD(&dqm->queues); 1653 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1654 dqm->active_cp_queue_count = 0; 1655 dqm->gws_queue_count = 0; 1656 1657 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1658 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1659 1660 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1661 if (test_bit(pipe_offset + queue, 1662 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1663 dqm->allocated_queues[pipe] |= 1 << queue; 1664 } 1665 1666 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1667 1668 init_sdma_bitmaps(dqm); 1669 1670 return 0; 1671 } 1672 1673 static void uninitialize(struct device_queue_manager *dqm) 1674 { 1675 int i; 1676 1677 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1678 1679 kfree(dqm->allocated_queues); 1680 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1681 kfree(dqm->mqd_mgrs[i]); 1682 mutex_destroy(&dqm->lock_hidden); 1683 } 1684 1685 static int start_nocpsch(struct device_queue_manager *dqm) 1686 { 1687 int r = 0; 1688 1689 pr_info("SW scheduler is used"); 1690 init_interrupts(dqm); 1691 1692 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1693 r = pm_init(&dqm->packet_mgr, dqm); 1694 if (!r) 1695 dqm->sched_running = true; 1696 1697 return r; 1698 } 1699 1700 static int stop_nocpsch(struct device_queue_manager *dqm) 1701 { 1702 dqm_lock(dqm); 1703 if (!dqm->sched_running) { 1704 dqm_unlock(dqm); 1705 return 0; 1706 } 1707 1708 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1709 pm_uninit(&dqm->packet_mgr); 1710 dqm->sched_running = false; 1711 dqm_unlock(dqm); 1712 1713 return 0; 1714 } 1715 1716 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1717 struct queue *q, const uint32_t *restore_sdma_id) 1718 { 1719 struct device *dev = dqm->dev->adev->dev; 1720 int bit; 1721 1722 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1723 if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) { 1724 dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n", 1725 get_num_sdma_queues(dqm)); 1726 return -ENOMEM; 1727 } 1728 1729 if (restore_sdma_id) { 1730 if (*restore_sdma_id >= get_num_sdma_queues(dqm)) 1731 return -EINVAL; 1732 1733 /* Re-use existing sdma_id */ 1734 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1735 dev_err(dev, "SDMA queue already in use\n"); 1736 return -EBUSY; 1737 } 1738 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1739 q->sdma_id = *restore_sdma_id; 1740 } else { 1741 /* Find first available sdma_id */ 1742 bit = find_first_bit(dqm->sdma_bitmap, 1743 get_num_sdma_queues(dqm)); 1744 clear_bit(bit, dqm->sdma_bitmap); 1745 q->sdma_id = bit; 1746 } 1747 1748 q->properties.sdma_engine_id = 1749 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1750 q->properties.sdma_queue_id = q->sdma_id / 1751 kfd_get_num_sdma_engines(dqm->dev); 1752 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1753 if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) { 1754 dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n", 1755 get_num_xgmi_sdma_queues(dqm)); 1756 return -ENOMEM; 1757 } 1758 if (restore_sdma_id) { 1759 if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1760 return -EINVAL; 1761 1762 /* Re-use existing sdma_id */ 1763 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1764 dev_err(dev, "SDMA queue already in use\n"); 1765 return -EBUSY; 1766 } 1767 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1768 q->sdma_id = *restore_sdma_id; 1769 } else { 1770 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1771 get_num_xgmi_sdma_queues(dqm)); 1772 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1773 q->sdma_id = bit; 1774 } 1775 /* sdma_engine_id is sdma id including 1776 * both PCIe-optimized SDMAs and XGMI- 1777 * optimized SDMAs. The calculation below 1778 * assumes the first N engines are always 1779 * PCIe-optimized ones 1780 */ 1781 q->properties.sdma_engine_id = 1782 kfd_get_num_sdma_engines(dqm->dev) + 1783 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1784 q->properties.sdma_queue_id = q->sdma_id / 1785 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1786 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1787 int i, num_queues, num_engines, eng_offset = 0, start_engine; 1788 bool free_bit_found = false, is_xgmi = false; 1789 1790 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { 1791 num_queues = get_num_sdma_queues(dqm); 1792 num_engines = kfd_get_num_sdma_engines(dqm->dev); 1793 q->properties.type = KFD_QUEUE_TYPE_SDMA; 1794 } else { 1795 num_queues = get_num_xgmi_sdma_queues(dqm); 1796 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); 1797 eng_offset = kfd_get_num_sdma_engines(dqm->dev); 1798 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI; 1799 is_xgmi = true; 1800 } 1801 1802 /* Scan available bit based on target engine ID. */ 1803 start_engine = q->properties.sdma_engine_id - eng_offset; 1804 for (i = start_engine; i < num_queues; i += num_engines) { 1805 1806 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) 1807 continue; 1808 1809 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); 1810 q->sdma_id = i; 1811 q->properties.sdma_queue_id = q->sdma_id / num_engines; 1812 free_bit_found = true; 1813 break; 1814 } 1815 1816 if (!free_bit_found) { 1817 dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n", 1818 q->properties.sdma_engine_id, num_queues); 1819 return -ENOMEM; 1820 } 1821 } 1822 1823 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 1824 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 1825 1826 return 0; 1827 } 1828 1829 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 1830 struct queue *q) 1831 { 1832 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1833 if (q->sdma_id >= get_num_sdma_queues(dqm)) 1834 return; 1835 set_bit(q->sdma_id, dqm->sdma_bitmap); 1836 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1837 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1838 return; 1839 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 1840 } 1841 } 1842 1843 /* 1844 * Device Queue Manager implementation for cp scheduler 1845 */ 1846 1847 static int set_sched_resources(struct device_queue_manager *dqm) 1848 { 1849 int i, mec; 1850 struct scheduling_resources res; 1851 struct device *dev = dqm->dev->adev->dev; 1852 1853 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 1854 1855 res.queue_mask = 0; 1856 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 1857 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 1858 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 1859 1860 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1861 continue; 1862 1863 /* only acquire queues from the first MEC */ 1864 if (mec > 0) 1865 continue; 1866 1867 /* This situation may be hit in the future if a new HW 1868 * generation exposes more than 64 queues. If so, the 1869 * definition of res.queue_mask needs updating 1870 */ 1871 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 1872 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 1873 break; 1874 } 1875 1876 res.queue_mask |= 1ull 1877 << amdgpu_queue_mask_bit_to_set_resource_bit( 1878 dqm->dev->adev, i); 1879 } 1880 res.gws_mask = ~0ull; 1881 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 1882 1883 pr_debug("Scheduling resources:\n" 1884 "vmid mask: 0x%8X\n" 1885 "queue mask: 0x%8llX\n", 1886 res.vmid_mask, res.queue_mask); 1887 1888 return pm_send_set_resources(&dqm->packet_mgr, &res); 1889 } 1890 1891 static int initialize_cpsch(struct device_queue_manager *dqm) 1892 { 1893 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1894 1895 mutex_init(&dqm->lock_hidden); 1896 INIT_LIST_HEAD(&dqm->queues); 1897 dqm->active_queue_count = dqm->processes_count = 0; 1898 dqm->active_cp_queue_count = 0; 1899 dqm->gws_queue_count = 0; 1900 dqm->active_runlist = false; 1901 dqm->trap_debug_vmid = 0; 1902 1903 init_sdma_bitmaps(dqm); 1904 1905 update_dqm_wait_times(dqm); 1906 return 0; 1907 } 1908 1909 /* halt_cpsch: 1910 * Unmap queues so the schedule doesn't continue remaining jobs in the queue. 1911 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch 1912 * is called. 1913 */ 1914 static int halt_cpsch(struct device_queue_manager *dqm) 1915 { 1916 int ret = 0; 1917 1918 dqm_lock(dqm); 1919 if (!dqm->sched_running) { 1920 dqm_unlock(dqm); 1921 return 0; 1922 } 1923 1924 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); 1925 1926 if (!dqm->is_hws_hang) { 1927 if (!dqm->dev->kfd->shared_resources.enable_mes) 1928 ret = unmap_queues_cpsch(dqm, 1929 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1930 USE_DEFAULT_GRACE_PERIOD, false); 1931 else 1932 ret = remove_all_kfd_queues_mes(dqm); 1933 } 1934 dqm->sched_halt = true; 1935 dqm_unlock(dqm); 1936 1937 return ret; 1938 } 1939 1940 /* unhalt_cpsch 1941 * Unset dqm->sched_halt and map queues back to runlist 1942 */ 1943 static int unhalt_cpsch(struct device_queue_manager *dqm) 1944 { 1945 int ret = 0; 1946 struct amdgpu_device *adev = dqm->dev->adev; 1947 1948 dqm_lock(dqm); 1949 if (!dqm->sched_running || !dqm->sched_halt) { 1950 dev_dbg(adev->dev, "Scheduling is not on halt.\n"); 1951 dqm_unlock(dqm); 1952 return 0; 1953 } 1954 dqm->sched_halt = false; 1955 if (!dqm->dev->kfd->shared_resources.enable_mes) 1956 ret = execute_queues_cpsch(dqm, 1957 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 1958 0, USE_DEFAULT_GRACE_PERIOD); 1959 else 1960 ret = add_all_kfd_queues_mes(dqm); 1961 1962 dqm_unlock(dqm); 1963 1964 return ret; 1965 } 1966 1967 static int start_cpsch(struct device_queue_manager *dqm) 1968 { 1969 struct device *dev = dqm->dev->adev->dev; 1970 int retval, num_hw_queue_slots; 1971 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 1972 int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); 1973 int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; 1974 1975 dqm_lock(dqm); 1976 1977 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1978 retval = pm_init(&dqm->packet_mgr, dqm); 1979 if (retval) 1980 goto fail_packet_manager_init; 1981 1982 retval = set_sched_resources(dqm); 1983 if (retval) 1984 goto fail_set_sched_resources; 1985 } 1986 pr_debug("Allocating fence memory\n"); 1987 1988 /* allocate fence memory on the gart */ 1989 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 1990 &dqm->fence_mem); 1991 1992 if (retval) 1993 goto fail_allocate_vidmem; 1994 1995 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 1996 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 1997 1998 init_interrupts(dqm); 1999 2000 /* clear hang status when driver try to start the hw scheduler */ 2001 dqm->sched_running = true; 2002 2003 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2004 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2005 KFD_DEQUEUE_WAIT_INIT, 0 /* unused */)) 2006 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); 2007 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2008 } 2009 2010 /* setup per-queue reset detection buffer */ 2011 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * 2012 dqm->dev->kfd->shared_resources.num_pipe_per_mec * 2013 NUM_XCC(dqm->dev->xcc_mask); 2014 2015 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); 2016 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); 2017 2018 if (!dqm->detect_hang_info) { 2019 retval = -ENOMEM; 2020 goto fail_detect_hang_buffer; 2021 } 2022 2023 dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL); 2024 dqm->hqd_info = kzalloc( 2025 hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info), 2026 GFP_KERNEL); 2027 2028 dqm_unlock(dqm); 2029 2030 return 0; 2031 fail_detect_hang_buffer: 2032 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 2033 fail_allocate_vidmem: 2034 fail_set_sched_resources: 2035 if (!dqm->dev->kfd->shared_resources.enable_mes) 2036 pm_uninit(&dqm->packet_mgr); 2037 fail_packet_manager_init: 2038 dqm_unlock(dqm); 2039 return retval; 2040 } 2041 2042 static int stop_cpsch(struct device_queue_manager *dqm) 2043 { 2044 int ret = 0; 2045 2046 dqm_lock(dqm); 2047 if (!dqm->sched_running) { 2048 dqm_unlock(dqm); 2049 return 0; 2050 } 2051 2052 if (!dqm->dev->kfd->shared_resources.enable_mes) 2053 ret = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 2054 0, USE_DEFAULT_GRACE_PERIOD, false); 2055 else 2056 ret = remove_all_kfd_queues_mes(dqm); 2057 2058 dqm->sched_running = false; 2059 2060 if (!dqm->dev->kfd->shared_resources.enable_mes) 2061 pm_release_ib(&dqm->packet_mgr); 2062 2063 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 2064 if (!dqm->dev->kfd->shared_resources.enable_mes) 2065 pm_uninit(&dqm->packet_mgr); 2066 kfree(dqm->detect_hang_info); 2067 dqm->detect_hang_info = NULL; 2068 kfree(dqm->hung_db_array); 2069 kfree(dqm->hqd_info); 2070 2071 dqm_unlock(dqm); 2072 2073 return ret; 2074 } 2075 2076 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 2077 struct kernel_queue *kq, 2078 struct qcm_process_device *qpd) 2079 { 2080 dqm_lock(dqm); 2081 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 2082 pr_warn("Can't create new kernel queue because %d queues were already created\n", 2083 dqm->total_queue_count); 2084 dqm_unlock(dqm); 2085 return -EPERM; 2086 } 2087 2088 /* 2089 * Unconditionally increment this counter, regardless of the queue's 2090 * type or whether the queue is active. 2091 */ 2092 dqm->total_queue_count++; 2093 pr_debug("Total of %d queues are accountable so far\n", 2094 dqm->total_queue_count); 2095 2096 list_add(&kq->list, &qpd->priv_queue_list); 2097 increment_queue_count(dqm, qpd, kq->queue); 2098 qpd->is_debug = true; 2099 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2100 USE_DEFAULT_GRACE_PERIOD); 2101 dqm_unlock(dqm); 2102 2103 return 0; 2104 } 2105 2106 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 2107 struct kernel_queue *kq, 2108 struct qcm_process_device *qpd) 2109 { 2110 dqm_lock(dqm); 2111 list_del(&kq->list); 2112 decrement_queue_count(dqm, qpd, kq->queue); 2113 qpd->is_debug = false; 2114 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 2115 USE_DEFAULT_GRACE_PERIOD); 2116 /* 2117 * Unconditionally decrement this counter, regardless of the queue's 2118 * type. 2119 */ 2120 dqm->total_queue_count--; 2121 pr_debug("Total of %d queues are accountable so far\n", 2122 dqm->total_queue_count); 2123 dqm_unlock(dqm); 2124 } 2125 2126 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 2127 struct qcm_process_device *qpd, 2128 const struct kfd_criu_queue_priv_data *qd, 2129 const void *restore_mqd, const void *restore_ctl_stack) 2130 { 2131 int retval; 2132 struct mqd_manager *mqd_mgr; 2133 2134 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 2135 pr_warn("Can't create new usermode queue because %d queues were already created\n", 2136 dqm->total_queue_count); 2137 retval = -EPERM; 2138 goto out; 2139 } 2140 2141 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2142 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI || 2143 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 2144 dqm_lock(dqm); 2145 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 2146 dqm_unlock(dqm); 2147 if (retval) 2148 goto out; 2149 } 2150 2151 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 2152 if (retval) 2153 goto out_deallocate_sdma_queue; 2154 2155 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2156 q->properties.type)]; 2157 2158 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2159 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2160 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 2161 q->properties.tba_addr = qpd->tba_addr; 2162 q->properties.tma_addr = qpd->tma_addr; 2163 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties); 2164 if (!q->mqd_mem_obj) { 2165 retval = -ENOMEM; 2166 goto out_deallocate_doorbell; 2167 } 2168 2169 dqm_lock(dqm); 2170 /* 2171 * Eviction state logic: mark all queues as evicted, even ones 2172 * not currently active. Restoring inactive queues later only 2173 * updates the is_evicted flag but is a no-op otherwise. 2174 */ 2175 q->properties.is_evicted = !!qpd->evicted; 2176 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 2177 kfd_dbg_has_cwsr_workaround(q->device); 2178 2179 if (qd) 2180 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 2181 &q->properties, restore_mqd, restore_ctl_stack, 2182 qd->ctl_stack_size); 2183 else 2184 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 2185 &q->gart_mqd_addr, &q->properties); 2186 2187 list_add(&q->list, &qpd->queues_list); 2188 qpd->queue_count++; 2189 2190 if (q->properties.is_active) { 2191 increment_queue_count(dqm, qpd, q); 2192 2193 if (!dqm->dev->kfd->shared_resources.enable_mes) 2194 retval = execute_queues_cpsch(dqm, 2195 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2196 else 2197 retval = add_queue_mes(dqm, q, qpd); 2198 if (retval) 2199 goto cleanup_queue; 2200 } 2201 2202 /* 2203 * Unconditionally increment this counter, regardless of the queue's 2204 * type or whether the queue is active. 2205 */ 2206 dqm->total_queue_count++; 2207 2208 pr_debug("Total of %d queues are accountable so far\n", 2209 dqm->total_queue_count); 2210 2211 dqm_unlock(dqm); 2212 return retval; 2213 2214 cleanup_queue: 2215 qpd->queue_count--; 2216 list_del(&q->list); 2217 if (q->properties.is_active) 2218 decrement_queue_count(dqm, qpd, q); 2219 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2220 dqm_unlock(dqm); 2221 out_deallocate_doorbell: 2222 deallocate_doorbell(qpd, q); 2223 out_deallocate_sdma_queue: 2224 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2225 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2226 dqm_lock(dqm); 2227 deallocate_sdma_queue(dqm, q); 2228 dqm_unlock(dqm); 2229 } 2230 out: 2231 return retval; 2232 } 2233 2234 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 2235 uint64_t fence_value, 2236 unsigned int timeout_ms) 2237 { 2238 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 2239 struct device *dev = dqm->dev->adev->dev; 2240 uint64_t *fence_addr = dqm->fence_addr; 2241 2242 while (*fence_addr != fence_value) { 2243 /* Fatal err detected, this response won't come */ 2244 if (amdgpu_amdkfd_is_fed(dqm->dev->adev) || 2245 amdgpu_in_reset(dqm->dev->adev)) 2246 return -EIO; 2247 2248 if (time_after(jiffies, end_jiffies)) { 2249 dev_err(dev, "qcm fence wait loop timeout expired\n"); 2250 /* In HWS case, this is used to halt the driver thread 2251 * in order not to mess up CP states before doing 2252 * scandumps for FW debugging. 2253 */ 2254 while (halt_if_hws_hang) 2255 schedule(); 2256 2257 return -ETIME; 2258 } 2259 schedule(); 2260 } 2261 2262 return 0; 2263 } 2264 2265 /* dqm->lock mutex has to be locked before calling this function */ 2266 static int map_queues_cpsch(struct device_queue_manager *dqm) 2267 { 2268 struct device *dev = dqm->dev->adev->dev; 2269 int retval; 2270 2271 if (!dqm->sched_running || dqm->sched_halt) 2272 return 0; 2273 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 2274 return 0; 2275 if (dqm->active_runlist) 2276 return 0; 2277 2278 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 2279 pr_debug("%s sent runlist\n", __func__); 2280 if (retval) { 2281 dev_err(dev, "failed to execute runlist\n"); 2282 return retval; 2283 } 2284 dqm->active_runlist = true; 2285 2286 return retval; 2287 } 2288 2289 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 2290 struct qcm_process_device *qpd) 2291 { 2292 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2293 2294 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n", 2295 q->properties.queue_id, pdd->process->lead_thread->pid); 2296 2297 pdd->has_reset_queue = true; 2298 q->properties.is_reset = true; 2299 if (q->properties.is_active) { 2300 q->properties.is_active = false; 2301 decrement_queue_count(dqm, qpd, q); 2302 } 2303 } 2304 2305 static int detect_queue_hang(struct device_queue_manager *dqm) 2306 { 2307 int i; 2308 2309 /* detect should be used only in dqm locked queue reset */ 2310 if (WARN_ON(dqm->detect_hang_count > 0)) 2311 return 0; 2312 2313 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); 2314 2315 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2316 uint32_t mec, pipe, queue; 2317 int xcc_id; 2318 2319 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2320 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2321 2322 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2323 continue; 2324 2325 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); 2326 2327 for_each_inst(xcc_id, dqm->dev->xcc_mask) { 2328 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( 2329 dqm->dev->adev, pipe, queue, xcc_id); 2330 struct dqm_detect_hang_info hang_info; 2331 2332 if (!queue_addr) 2333 continue; 2334 2335 hang_info.pipe_id = pipe; 2336 hang_info.queue_id = queue; 2337 hang_info.xcc_id = xcc_id; 2338 hang_info.queue_address = queue_addr; 2339 2340 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; 2341 dqm->detect_hang_count++; 2342 } 2343 } 2344 2345 return dqm->detect_hang_count; 2346 } 2347 2348 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) 2349 { 2350 struct device_process_node *cur; 2351 struct qcm_process_device *qpd; 2352 struct queue *q; 2353 2354 list_for_each_entry(cur, &dqm->queues, list) { 2355 qpd = cur->qpd; 2356 list_for_each_entry(q, &qpd->queues_list, list) { 2357 if (queue_address == q->properties.queue_address) 2358 return q; 2359 } 2360 } 2361 2362 return NULL; 2363 } 2364 2365 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset) 2366 { 2367 struct device_process_node *cur; 2368 struct qcm_process_device *qpd; 2369 struct queue *q; 2370 2371 list_for_each_entry(cur, &dqm->queues, list) { 2372 qpd = cur->qpd; 2373 list_for_each_entry(q, &qpd->queues_list, list) { 2374 if (doorbell_offset == q->properties.doorbell_off) 2375 return q; 2376 } 2377 } 2378 2379 return NULL; 2380 } 2381 2382 static int reset_hung_queues(struct device_queue_manager *dqm) 2383 { 2384 int r = 0, reset_count = 0, i; 2385 2386 if (!dqm->detect_hang_info || dqm->is_hws_hang) 2387 return -EIO; 2388 2389 /* assume dqm locked. */ 2390 if (!detect_queue_hang(dqm)) 2391 return -ENOTRECOVERABLE; 2392 2393 for (i = 0; i < dqm->detect_hang_count; i++) { 2394 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; 2395 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); 2396 struct kfd_process_device *pdd; 2397 uint64_t queue_addr = 0; 2398 2399 if (!q) { 2400 r = -ENOTRECOVERABLE; 2401 goto reset_fail; 2402 } 2403 2404 pdd = kfd_get_process_device_data(dqm->dev, q->process); 2405 if (!pdd) { 2406 r = -ENOTRECOVERABLE; 2407 goto reset_fail; 2408 } 2409 2410 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, 2411 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id, 2412 KFD_UNMAP_LATENCY_MS); 2413 2414 /* either reset failed or we reset an unexpected queue. */ 2415 if (queue_addr != q->properties.queue_address) { 2416 r = -ENOTRECOVERABLE; 2417 goto reset_fail; 2418 } 2419 2420 set_queue_as_reset(dqm, q, &pdd->qpd); 2421 reset_count++; 2422 } 2423 2424 if (reset_count == dqm->detect_hang_count) 2425 kfd_signal_reset_event(dqm->dev); 2426 else 2427 r = -ENOTRECOVERABLE; 2428 2429 reset_fail: 2430 dqm->detect_hang_count = 0; 2431 2432 return r; 2433 } 2434 2435 static bool sdma_has_hang(struct device_queue_manager *dqm) 2436 { 2437 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2438 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2439 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2440 int i, j; 2441 2442 for (i = engine_start; i < engine_end; i++) { 2443 for (j = 0; j < num_queues_per_eng; j++) { 2444 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j)) 2445 continue; 2446 2447 return true; 2448 } 2449 } 2450 2451 return false; 2452 } 2453 2454 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, 2455 uint32_t doorbell_off) 2456 { 2457 struct device_process_node *cur; 2458 struct qcm_process_device *qpd; 2459 struct queue *q; 2460 2461 list_for_each_entry(cur, &dqm->queues, list) { 2462 qpd = cur->qpd; 2463 list_for_each_entry(q, &qpd->queues_list, list) { 2464 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || 2465 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) && 2466 q->properties.doorbell_off == doorbell_off) { 2467 set_queue_as_reset(dqm, q, qpd); 2468 return true; 2469 } 2470 } 2471 } 2472 2473 return false; 2474 } 2475 2476 static int reset_hung_queues_sdma(struct device_queue_manager *dqm) 2477 { 2478 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2479 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2480 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2481 int r = 0, i, j; 2482 2483 if (dqm->is_hws_hang) 2484 return -EIO; 2485 2486 /* Scan for hung HW queues and reset engine. */ 2487 dqm->detect_hang_count = 0; 2488 for (i = engine_start; i < engine_end; i++) { 2489 for (j = 0; j < num_queues_per_eng; j++) { 2490 uint32_t doorbell_off = 2491 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j); 2492 2493 if (!doorbell_off) 2494 continue; 2495 2496 /* Reset engine and check. */ 2497 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i, false) || 2498 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) || 2499 !set_sdma_queue_as_reset(dqm, doorbell_off)) { 2500 r = -ENOTRECOVERABLE; 2501 goto reset_fail; 2502 } 2503 2504 /* Should only expect one queue active per engine */ 2505 dqm->detect_hang_count++; 2506 break; 2507 } 2508 } 2509 2510 /* Signal process reset */ 2511 if (dqm->detect_hang_count) 2512 kfd_signal_reset_event(dqm->dev); 2513 else 2514 r = -ENOTRECOVERABLE; 2515 2516 reset_fail: 2517 dqm->detect_hang_count = 0; 2518 2519 return r; 2520 } 2521 2522 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma) 2523 { 2524 struct amdgpu_device *adev = dqm->dev->adev; 2525 2526 while (halt_if_hws_hang) 2527 schedule(); 2528 2529 if (adev->debug_disable_gpu_ring_reset) { 2530 dev_info_once(adev->dev, 2531 "%s queue hung, but ring reset disabled", 2532 is_sdma ? "sdma" : "compute"); 2533 2534 return -EPERM; 2535 } 2536 if (!amdgpu_gpu_recovery) 2537 return -ENOTRECOVERABLE; 2538 2539 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm); 2540 } 2541 2542 /* dqm->lock mutex has to be locked before calling this function 2543 * 2544 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time 2545 * for context switch latency. Lower values are used by debugger 2546 * since context switching are triggered at high frequency. 2547 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE 2548 * 2549 */ 2550 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 2551 enum kfd_unmap_queues_filter filter, 2552 uint32_t filter_param, 2553 uint32_t grace_period, 2554 bool reset) 2555 { 2556 struct device *dev = dqm->dev->adev->dev; 2557 struct mqd_manager *mqd_mgr; 2558 int retval; 2559 2560 if (!dqm->sched_running) 2561 return 0; 2562 if (!dqm->active_runlist) 2563 return 0; 2564 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2565 return -EIO; 2566 2567 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2568 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2569 KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period); 2570 if (retval) 2571 goto out; 2572 } 2573 2574 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 2575 if (retval) 2576 goto out; 2577 2578 *dqm->fence_addr = KFD_FENCE_INIT; 2579 mb(); 2580 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 2581 KFD_FENCE_COMPLETED); 2582 /* should be timed out */ 2583 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 2584 queue_preemption_timeout_ms); 2585 if (retval) { 2586 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 2587 kfd_hws_hang(dqm); 2588 goto out; 2589 } 2590 2591 /* In the current MEC firmware implementation, if compute queue 2592 * doesn't response to the preemption request in time, HIQ will 2593 * abandon the unmap request without returning any timeout error 2594 * to driver. Instead, MEC firmware will log the doorbell of the 2595 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 2596 * To make sure the queue unmap was successful, driver need to 2597 * check those fields 2598 */ 2599 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 2600 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) && 2601 reset_queues_on_hws_hang(dqm, false)) 2602 goto reset_fail; 2603 2604 /* Check for SDMA hang and attempt SDMA reset */ 2605 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true)) 2606 goto reset_fail; 2607 2608 /* We need to reset the grace period value for this device */ 2609 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2610 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2611 KFD_DEQUEUE_WAIT_RESET, 0 /* unused */)) 2612 dev_err(dev, "Failed to reset grace period\n"); 2613 } 2614 2615 pm_release_ib(&dqm->packet_mgr); 2616 dqm->active_runlist = false; 2617 out: 2618 up_read(&dqm->dev->adev->reset_domain->sem); 2619 return retval; 2620 2621 reset_fail: 2622 dqm->is_hws_hang = true; 2623 kfd_hws_hang(dqm); 2624 up_read(&dqm->dev->adev->reset_domain->sem); 2625 return -ETIME; 2626 } 2627 2628 /* only for compute queue */ 2629 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) 2630 { 2631 int retval; 2632 2633 dqm_lock(dqm); 2634 2635 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2636 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2637 2638 dqm_unlock(dqm); 2639 return retval; 2640 } 2641 2642 /* dqm->lock mutex has to be locked before calling this function */ 2643 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2644 enum kfd_unmap_queues_filter filter, 2645 uint32_t filter_param, 2646 uint32_t grace_period) 2647 { 2648 int retval; 2649 2650 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2651 return -EIO; 2652 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2653 if (!retval) 2654 retval = map_queues_cpsch(dqm); 2655 up_read(&dqm->dev->adev->reset_domain->sem); 2656 return retval; 2657 } 2658 2659 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2660 struct queue *q) 2661 { 2662 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2663 q->process); 2664 int ret = 0; 2665 2666 if (WARN_ON(!pdd)) 2667 return ret; 2668 2669 if (pdd->qpd.is_debug) 2670 return ret; 2671 2672 if (q->properties.is_being_destroyed) 2673 return -EBUSY; 2674 2675 q->properties.is_being_destroyed = true; 2676 2677 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2678 dqm_unlock(dqm); 2679 mutex_unlock(&q->process->mutex); 2680 ret = wait_event_interruptible(dqm->destroy_wait, 2681 !q->properties.is_suspended); 2682 2683 mutex_lock(&q->process->mutex); 2684 dqm_lock(dqm); 2685 } 2686 2687 if (ret) 2688 q->properties.is_being_destroyed = false; 2689 2690 return ret; 2691 } 2692 2693 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2694 struct qcm_process_device *qpd, 2695 struct queue *q) 2696 { 2697 int retval; 2698 struct mqd_manager *mqd_mgr; 2699 uint64_t sdma_val = 0; 2700 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2701 struct device *dev = dqm->dev->adev->dev; 2702 2703 /* Get the SDMA queue stats */ 2704 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2705 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2706 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 2707 &sdma_val); 2708 if (retval) 2709 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2710 q->properties.queue_id); 2711 } 2712 2713 /* remove queue from list to prevent rescheduling after preemption */ 2714 dqm_lock(dqm); 2715 2716 retval = wait_on_destroy_queue(dqm, q); 2717 2718 if (retval) { 2719 dqm_unlock(dqm); 2720 return retval; 2721 } 2722 2723 if (qpd->is_debug) { 2724 /* 2725 * error, currently we do not allow to destroy a queue 2726 * of a currently debugged process 2727 */ 2728 retval = -EBUSY; 2729 goto failed_try_destroy_debugged_queue; 2730 2731 } 2732 2733 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2734 q->properties.type)]; 2735 2736 deallocate_doorbell(qpd, q); 2737 2738 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2739 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2740 deallocate_sdma_queue(dqm, q); 2741 pdd->sdma_past_activity_counter += sdma_val; 2742 } 2743 2744 if (q->properties.is_active) { 2745 decrement_queue_count(dqm, qpd, q); 2746 q->properties.is_active = false; 2747 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2748 retval = execute_queues_cpsch(dqm, 2749 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2750 USE_DEFAULT_GRACE_PERIOD); 2751 if (retval == -ETIME) 2752 qpd->reset_wavefronts = true; 2753 } else { 2754 retval = remove_queue_mes(dqm, q, qpd); 2755 } 2756 } 2757 list_del(&q->list); 2758 qpd->queue_count--; 2759 2760 /* 2761 * Unconditionally decrement this counter, regardless of the queue's 2762 * type 2763 */ 2764 dqm->total_queue_count--; 2765 pr_debug("Total of %d queues are accountable so far\n", 2766 dqm->total_queue_count); 2767 2768 dqm_unlock(dqm); 2769 2770 /* 2771 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 2772 * circular locking 2773 */ 2774 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 2775 qpd->pqm->process, q->device, 2776 -1, false, NULL, 0); 2777 2778 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2779 2780 return retval; 2781 2782 failed_try_destroy_debugged_queue: 2783 q->properties.is_being_destroyed = false; 2784 dqm_unlock(dqm); 2785 return retval; 2786 } 2787 2788 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 2789 struct qcm_process_device *qpd, 2790 enum cache_policy default_policy, 2791 enum cache_policy alternate_policy, 2792 void __user *alternate_aperture_base, 2793 uint64_t alternate_aperture_size, 2794 u32 misc_process_properties) 2795 { 2796 bool retval = true; 2797 2798 if (!dqm->asic_ops.set_cache_memory_policy) 2799 return retval; 2800 2801 dqm_lock(dqm); 2802 2803 retval = dqm->asic_ops.set_cache_memory_policy( 2804 dqm, 2805 qpd, 2806 default_policy, 2807 alternate_policy, 2808 alternate_aperture_base, 2809 alternate_aperture_size, 2810 misc_process_properties); 2811 2812 if (retval) 2813 goto out; 2814 2815 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 2816 program_sh_mem_settings(dqm, qpd); 2817 2818 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 2819 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 2820 qpd->sh_mem_ape1_limit); 2821 2822 out: 2823 dqm_unlock(dqm); 2824 return retval; 2825 } 2826 2827 static int process_termination_nocpsch(struct device_queue_manager *dqm, 2828 struct qcm_process_device *qpd) 2829 { 2830 struct queue *q; 2831 struct device_process_node *cur, *next_dpn; 2832 int retval = 0; 2833 bool found = false; 2834 2835 dqm_lock(dqm); 2836 2837 /* Clear all user mode queues */ 2838 while (!list_empty(&qpd->queues_list)) { 2839 struct mqd_manager *mqd_mgr; 2840 int ret; 2841 2842 q = list_first_entry(&qpd->queues_list, struct queue, list); 2843 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2844 q->properties.type)]; 2845 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 2846 if (ret) 2847 retval = ret; 2848 dqm_unlock(dqm); 2849 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2850 dqm_lock(dqm); 2851 } 2852 2853 /* Unregister process */ 2854 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2855 if (qpd == cur->qpd) { 2856 list_del(&cur->list); 2857 kfree(cur); 2858 dqm->processes_count--; 2859 found = true; 2860 break; 2861 } 2862 } 2863 2864 dqm_unlock(dqm); 2865 2866 /* Outside the DQM lock because under the DQM lock we can't do 2867 * reclaim or take other locks that others hold while reclaiming. 2868 */ 2869 if (found) 2870 kfd_dec_compute_active(dqm->dev); 2871 2872 return retval; 2873 } 2874 2875 static int get_wave_state(struct device_queue_manager *dqm, 2876 struct queue *q, 2877 void __user *ctl_stack, 2878 u32 *ctl_stack_used_size, 2879 u32 *save_area_used_size) 2880 { 2881 struct mqd_manager *mqd_mgr; 2882 2883 dqm_lock(dqm); 2884 2885 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2886 2887 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 2888 q->properties.is_active || !q->device->kfd->cwsr_enabled || 2889 !mqd_mgr->get_wave_state) { 2890 dqm_unlock(dqm); 2891 return -EINVAL; 2892 } 2893 2894 dqm_unlock(dqm); 2895 2896 /* 2897 * get_wave_state is outside the dqm lock to prevent circular locking 2898 * and the queue should be protected against destruction by the process 2899 * lock. 2900 */ 2901 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 2902 ctl_stack, ctl_stack_used_size, save_area_used_size); 2903 } 2904 2905 static int get_queue_checkpoint_info(struct device_queue_manager *dqm, 2906 const struct queue *q, 2907 u32 *mqd_size, 2908 u32 *ctl_stack_size) 2909 { 2910 struct mqd_manager *mqd_mgr; 2911 enum KFD_MQD_TYPE mqd_type = 2912 get_mqd_type_from_queue_type(q->properties.type); 2913 int ret = 0; 2914 2915 dqm_lock(dqm); 2916 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2917 *mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask); 2918 *ctl_stack_size = 0; 2919 2920 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 2921 ret = mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 2922 2923 dqm_unlock(dqm); 2924 2925 return ret; 2926 } 2927 2928 static int checkpoint_mqd(struct device_queue_manager *dqm, 2929 const struct queue *q, 2930 void *mqd, 2931 void *ctl_stack) 2932 { 2933 struct mqd_manager *mqd_mgr; 2934 int r = 0; 2935 enum KFD_MQD_TYPE mqd_type = 2936 get_mqd_type_from_queue_type(q->properties.type); 2937 2938 dqm_lock(dqm); 2939 2940 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 2941 r = -EINVAL; 2942 goto dqm_unlock; 2943 } 2944 2945 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2946 if (!mqd_mgr->checkpoint_mqd) { 2947 r = -EOPNOTSUPP; 2948 goto dqm_unlock; 2949 } 2950 2951 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 2952 2953 dqm_unlock: 2954 dqm_unlock(dqm); 2955 return r; 2956 } 2957 2958 static int process_termination_cpsch(struct device_queue_manager *dqm, 2959 struct qcm_process_device *qpd) 2960 { 2961 int retval = 0; 2962 struct queue *q; 2963 struct device *dev = dqm->dev->adev->dev; 2964 struct kernel_queue *kq, *kq_next; 2965 struct mqd_manager *mqd_mgr; 2966 struct device_process_node *cur, *next_dpn; 2967 enum kfd_unmap_queues_filter filter = 2968 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 2969 bool found = false; 2970 2971 dqm_lock(dqm); 2972 2973 /* Clean all kernel queues */ 2974 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 2975 list_del(&kq->list); 2976 decrement_queue_count(dqm, qpd, kq->queue); 2977 qpd->is_debug = false; 2978 dqm->total_queue_count--; 2979 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 2980 } 2981 2982 /* Clear all user mode queues */ 2983 list_for_each_entry(q, &qpd->queues_list, list) { 2984 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 2985 deallocate_sdma_queue(dqm, q); 2986 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2987 deallocate_sdma_queue(dqm, q); 2988 2989 if (q->properties.is_active) { 2990 decrement_queue_count(dqm, qpd, q); 2991 2992 if (dqm->dev->kfd->shared_resources.enable_mes) { 2993 retval = remove_queue_mes(dqm, q, qpd); 2994 if (retval) 2995 dev_err(dev, "Failed to remove queue %d\n", 2996 q->properties.queue_id); 2997 } 2998 } 2999 3000 dqm->total_queue_count--; 3001 } 3002 3003 /* Unregister process */ 3004 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 3005 if (qpd == cur->qpd) { 3006 list_del(&cur->list); 3007 kfree(cur); 3008 dqm->processes_count--; 3009 found = true; 3010 break; 3011 } 3012 } 3013 3014 if (!dqm->dev->kfd->shared_resources.enable_mes) 3015 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 3016 3017 if ((retval || qpd->reset_wavefronts) && 3018 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 3019 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 3020 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 3021 qpd->reset_wavefronts = false; 3022 up_read(&dqm->dev->adev->reset_domain->sem); 3023 } 3024 3025 /* Lastly, free mqd resources. 3026 * Do free_mqd() after dqm_unlock to avoid circular locking. 3027 */ 3028 while (!list_empty(&qpd->queues_list)) { 3029 q = list_first_entry(&qpd->queues_list, struct queue, list); 3030 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 3031 q->properties.type)]; 3032 list_del(&q->list); 3033 qpd->queue_count--; 3034 dqm_unlock(dqm); 3035 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 3036 dqm_lock(dqm); 3037 } 3038 dqm_unlock(dqm); 3039 3040 /* Outside the DQM lock because under the DQM lock we can't do 3041 * reclaim or take other locks that others hold while reclaiming. 3042 */ 3043 if (found) 3044 kfd_dec_compute_active(dqm->dev); 3045 3046 return retval; 3047 } 3048 3049 static int init_mqd_managers(struct device_queue_manager *dqm) 3050 { 3051 int i, j; 3052 struct device *dev = dqm->dev->adev->dev; 3053 struct mqd_manager *mqd_mgr; 3054 3055 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 3056 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 3057 if (!mqd_mgr) { 3058 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 3059 goto out_free; 3060 } 3061 dqm->mqd_mgrs[i] = mqd_mgr; 3062 } 3063 3064 return 0; 3065 3066 out_free: 3067 for (j = 0; j < i; j++) { 3068 kfree(dqm->mqd_mgrs[j]); 3069 dqm->mqd_mgrs[j] = NULL; 3070 } 3071 3072 return -ENOMEM; 3073 } 3074 3075 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 3076 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 3077 { 3078 int retval; 3079 struct kfd_node *dev = dqm->dev; 3080 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 3081 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 3082 get_num_all_sdma_engines(dqm) * 3083 dev->kfd->device_info.num_sdma_queues_per_engine + 3084 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 3085 NUM_XCC(dqm->dev->xcc_mask)); 3086 3087 retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, size, 3088 AMDGPU_GEM_DOMAIN_GTT, 3089 &(mem_obj->mem), &(mem_obj->gpu_addr), 3090 (void *)&(mem_obj->cpu_ptr), false); 3091 3092 return retval; 3093 } 3094 3095 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 3096 struct kfd_mem_obj *mqd) 3097 { 3098 WARN(!mqd, "No hiq sdma mqd trunk to free"); 3099 3100 amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem); 3101 } 3102 3103 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 3104 { 3105 struct device_queue_manager *dqm; 3106 3107 pr_debug("Loading device queue manager\n"); 3108 3109 dqm = kzalloc_obj(*dqm); 3110 if (!dqm) 3111 return NULL; 3112 3113 switch (dev->adev->asic_type) { 3114 /* HWS is not available on Hawaii. */ 3115 case CHIP_HAWAII: 3116 /* HWS depends on CWSR for timely dequeue. CWSR is not 3117 * available on Tonga. 3118 * 3119 * FIXME: This argument also applies to Kaveri. 3120 */ 3121 case CHIP_TONGA: 3122 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 3123 break; 3124 default: 3125 dqm->sched_policy = sched_policy; 3126 break; 3127 } 3128 3129 dqm->dev = dev; 3130 switch (dqm->sched_policy) { 3131 case KFD_SCHED_POLICY_HWS: 3132 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 3133 /* initialize dqm for cp scheduling */ 3134 dqm->ops.create_queue = create_queue_cpsch; 3135 dqm->ops.initialize = initialize_cpsch; 3136 dqm->ops.start = start_cpsch; 3137 dqm->ops.stop = stop_cpsch; 3138 dqm->ops.halt = halt_cpsch; 3139 dqm->ops.unhalt = unhalt_cpsch; 3140 dqm->ops.destroy_queue = destroy_queue_cpsch; 3141 dqm->ops.update_queue = update_queue; 3142 dqm->ops.register_process = register_process; 3143 dqm->ops.unregister_process = unregister_process; 3144 dqm->ops.uninitialize = uninitialize; 3145 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 3146 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 3147 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 3148 dqm->ops.process_termination = process_termination_cpsch; 3149 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 3150 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 3151 dqm->ops.get_wave_state = get_wave_state; 3152 dqm->ops.reset_queues = reset_queues_cpsch; 3153 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 3154 dqm->ops.checkpoint_mqd = checkpoint_mqd; 3155 dqm->ops.set_perfcount = set_perfcount; 3156 break; 3157 case KFD_SCHED_POLICY_NO_HWS: 3158 /* initialize dqm for no cp scheduling */ 3159 dqm->ops.start = start_nocpsch; 3160 dqm->ops.stop = stop_nocpsch; 3161 dqm->ops.create_queue = create_queue_nocpsch; 3162 dqm->ops.destroy_queue = destroy_queue_nocpsch; 3163 dqm->ops.update_queue = update_queue; 3164 dqm->ops.register_process = register_process; 3165 dqm->ops.unregister_process = unregister_process; 3166 dqm->ops.initialize = initialize_nocpsch; 3167 dqm->ops.uninitialize = uninitialize; 3168 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 3169 dqm->ops.process_termination = process_termination_nocpsch; 3170 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 3171 dqm->ops.restore_process_queues = 3172 restore_process_queues_nocpsch; 3173 dqm->ops.get_wave_state = get_wave_state; 3174 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 3175 dqm->ops.checkpoint_mqd = checkpoint_mqd; 3176 dqm->ops.set_perfcount = set_perfcount; 3177 break; 3178 default: 3179 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 3180 goto out_free; 3181 } 3182 3183 switch (dev->adev->asic_type) { 3184 case CHIP_KAVERI: 3185 case CHIP_HAWAII: 3186 device_queue_manager_init_cik(&dqm->asic_ops); 3187 break; 3188 3189 case CHIP_CARRIZO: 3190 case CHIP_TONGA: 3191 case CHIP_FIJI: 3192 case CHIP_POLARIS10: 3193 case CHIP_POLARIS11: 3194 case CHIP_POLARIS12: 3195 case CHIP_VEGAM: 3196 device_queue_manager_init_vi(&dqm->asic_ops); 3197 break; 3198 3199 default: 3200 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0)) 3201 device_queue_manager_init_v12_1(&dqm->asic_ops); 3202 else if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 3203 device_queue_manager_init_v12(&dqm->asic_ops); 3204 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 3205 device_queue_manager_init_v11(&dqm->asic_ops); 3206 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 3207 device_queue_manager_init_v10(&dqm->asic_ops); 3208 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 3209 device_queue_manager_init_v9(&dqm->asic_ops); 3210 else { 3211 WARN(1, "Unexpected ASIC family %u", 3212 dev->adev->asic_type); 3213 goto out_free; 3214 } 3215 } 3216 3217 if (init_mqd_managers(dqm)) 3218 goto out_free; 3219 3220 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 3221 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 3222 goto out_free; 3223 } 3224 3225 if (!dqm->ops.initialize(dqm)) { 3226 init_waitqueue_head(&dqm->destroy_wait); 3227 return dqm; 3228 } 3229 3230 if (!dev->kfd->shared_resources.enable_mes) 3231 deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd); 3232 3233 out_free: 3234 kfree(dqm); 3235 return NULL; 3236 } 3237 3238 void device_queue_manager_uninit(struct device_queue_manager *dqm) 3239 { 3240 dqm->ops.stop(dqm); 3241 dqm->ops.uninitialize(dqm); 3242 if (!dqm->dev->kfd->shared_resources.enable_mes) 3243 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 3244 kfree(dqm); 3245 } 3246 3247 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) 3248 { 3249 struct kfd_process_device *pdd = NULL; 3250 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); 3251 struct device_queue_manager *dqm = knode->dqm; 3252 struct device *dev = dqm->dev->adev->dev; 3253 struct qcm_process_device *qpd; 3254 struct queue *q = NULL; 3255 int ret = 0; 3256 3257 if (!pdd) 3258 return -EINVAL; 3259 3260 dqm_lock(dqm); 3261 3262 if (pdd) { 3263 qpd = &pdd->qpd; 3264 3265 list_for_each_entry(q, &qpd->queues_list, list) { 3266 if (q->doorbell_id == doorbell_id && q->properties.is_active) { 3267 /* suspend all queues will save any good queues and mark the rest as bad */ 3268 suspend_all_queues_mes(dqm); 3269 3270 q->properties.is_evicted = true; 3271 q->properties.is_active = false; 3272 decrement_queue_count(dqm, qpd, q); 3273 3274 /* this will remove the bad queue and sched a GPU reset if needed */ 3275 ret = remove_queue_mes(dqm, q, qpd); 3276 if (ret) 3277 dev_err(dev, "Removing bad queue failed"); 3278 /* resume the good queues */ 3279 resume_all_queues_mes(dqm); 3280 break; 3281 } 3282 } 3283 } 3284 3285 dqm_unlock(dqm); 3286 kfd_unref_process(p); 3287 return ret; 3288 } 3289 3290 int kfd_evict_process_device(struct kfd_process_device *pdd) 3291 { 3292 struct device_queue_manager *dqm; 3293 struct kfd_process *p; 3294 3295 p = pdd->process; 3296 dqm = pdd->dev->dqm; 3297 3298 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 3299 3300 return dqm->ops.evict_process_queues(dqm, &pdd->qpd); 3301 } 3302 3303 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 3304 struct qcm_process_device *qpd) 3305 { 3306 int r; 3307 struct device *dev = dqm->dev->adev->dev; 3308 int updated_vmid_mask; 3309 3310 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3311 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3312 return -EINVAL; 3313 } 3314 3315 dqm_lock(dqm); 3316 3317 if (dqm->trap_debug_vmid != 0) { 3318 dev_err(dev, "Trap debug id already reserved\n"); 3319 r = -EBUSY; 3320 goto out_unlock; 3321 } 3322 3323 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3324 USE_DEFAULT_GRACE_PERIOD, false); 3325 if (r) 3326 goto out_unlock; 3327 3328 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3329 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 3330 3331 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3332 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 3333 r = set_sched_resources(dqm); 3334 if (r) 3335 goto out_unlock; 3336 3337 r = map_queues_cpsch(dqm); 3338 if (r) 3339 goto out_unlock; 3340 3341 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 3342 3343 out_unlock: 3344 dqm_unlock(dqm); 3345 return r; 3346 } 3347 3348 /* 3349 * Releases vmid for the trap debugger 3350 */ 3351 int release_debug_trap_vmid(struct device_queue_manager *dqm, 3352 struct qcm_process_device *qpd) 3353 { 3354 struct device *dev = dqm->dev->adev->dev; 3355 int r; 3356 int updated_vmid_mask; 3357 uint32_t trap_debug_vmid; 3358 3359 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3360 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3361 return -EINVAL; 3362 } 3363 3364 dqm_lock(dqm); 3365 trap_debug_vmid = dqm->trap_debug_vmid; 3366 if (dqm->trap_debug_vmid == 0) { 3367 dev_err(dev, "Trap debug id is not reserved\n"); 3368 r = -EINVAL; 3369 goto out_unlock; 3370 } 3371 3372 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3373 USE_DEFAULT_GRACE_PERIOD, false); 3374 if (r) 3375 goto out_unlock; 3376 3377 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3378 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 3379 3380 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3381 dqm->trap_debug_vmid = 0; 3382 r = set_sched_resources(dqm); 3383 if (r) 3384 goto out_unlock; 3385 3386 r = map_queues_cpsch(dqm); 3387 if (r) 3388 goto out_unlock; 3389 3390 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 3391 3392 out_unlock: 3393 dqm_unlock(dqm); 3394 return r; 3395 } 3396 3397 #define QUEUE_NOT_FOUND -1 3398 /* invalidate queue operation in array */ 3399 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 3400 { 3401 int i; 3402 3403 for (i = 0; i < num_queues; i++) 3404 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 3405 } 3406 3407 /* find queue index in array */ 3408 static int q_array_get_index(unsigned int queue_id, 3409 uint32_t num_queues, 3410 uint32_t *queue_ids) 3411 { 3412 int i; 3413 3414 for (i = 0; i < num_queues; i++) 3415 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 3416 return i; 3417 3418 return QUEUE_NOT_FOUND; 3419 } 3420 3421 struct copy_context_work_handler_workarea { 3422 struct work_struct copy_context_work; 3423 struct kfd_process *p; 3424 }; 3425 3426 static void copy_context_work_handler(struct work_struct *work) 3427 { 3428 struct copy_context_work_handler_workarea *workarea; 3429 struct mqd_manager *mqd_mgr; 3430 struct queue *q; 3431 struct mm_struct *mm; 3432 struct kfd_process *p; 3433 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 3434 int i; 3435 3436 workarea = container_of(work, 3437 struct copy_context_work_handler_workarea, 3438 copy_context_work); 3439 3440 p = workarea->p; 3441 mm = get_task_mm(p->lead_thread); 3442 3443 if (!mm) 3444 return; 3445 3446 kthread_use_mm(mm); 3447 for (i = 0; i < p->n_pdds; i++) { 3448 struct kfd_process_device *pdd = p->pdds[i]; 3449 struct device_queue_manager *dqm = pdd->dev->dqm; 3450 struct qcm_process_device *qpd = &pdd->qpd; 3451 3452 list_for_each_entry(q, &qpd->queues_list, list) { 3453 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) 3454 continue; 3455 3456 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3457 3458 /* We ignore the return value from get_wave_state 3459 * because 3460 * i) right now, it always returns 0, and 3461 * ii) if we hit an error, we would continue to the 3462 * next queue anyway. 3463 */ 3464 mqd_mgr->get_wave_state(mqd_mgr, 3465 q->mqd, 3466 &q->properties, 3467 (void __user *) q->properties.ctx_save_restore_area_address, 3468 &tmp_ctl_stack_used_size, 3469 &tmp_save_area_used_size); 3470 } 3471 } 3472 kthread_unuse_mm(mm); 3473 mmput(mm); 3474 } 3475 3476 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 3477 { 3478 if (!usr_queue_id_array) 3479 return num_queues ? ERR_PTR(-EINVAL) : NULL; 3480 3481 if (num_queues > KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 3482 return ERR_PTR(-EINVAL); 3483 3484 return memdup_user(usr_queue_id_array, 3485 array_size(num_queues, sizeof(uint32_t))); 3486 } 3487 3488 int resume_queues(struct kfd_process *p, 3489 uint32_t num_queues, 3490 uint32_t *usr_queue_id_array) 3491 { 3492 uint32_t *queue_ids = NULL; 3493 int total_resumed = 0; 3494 int i; 3495 3496 if (usr_queue_id_array) { 3497 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3498 3499 if (IS_ERR(queue_ids)) 3500 return PTR_ERR(queue_ids); 3501 3502 /* mask all queues as invalid. unmask per successful request */ 3503 q_array_invalidate(num_queues, queue_ids); 3504 } 3505 3506 for (i = 0; i < p->n_pdds; i++) { 3507 struct kfd_process_device *pdd = p->pdds[i]; 3508 struct device_queue_manager *dqm = pdd->dev->dqm; 3509 struct device *dev = dqm->dev->adev->dev; 3510 struct qcm_process_device *qpd = &pdd->qpd; 3511 struct queue *q; 3512 int r, per_device_resumed = 0; 3513 3514 dqm_lock(dqm); 3515 3516 /* unmask queues that resume or already resumed as valid */ 3517 list_for_each_entry(q, &qpd->queues_list, list) { 3518 int q_idx = QUEUE_NOT_FOUND; 3519 3520 if (queue_ids) 3521 q_idx = q_array_get_index( 3522 q->properties.queue_id, 3523 num_queues, 3524 queue_ids); 3525 3526 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 3527 int err = resume_single_queue(dqm, &pdd->qpd, q); 3528 3529 if (queue_ids) { 3530 if (!err) { 3531 queue_ids[q_idx] &= 3532 ~KFD_DBG_QUEUE_INVALID_MASK; 3533 } else { 3534 queue_ids[q_idx] |= 3535 KFD_DBG_QUEUE_ERROR_MASK; 3536 break; 3537 } 3538 } 3539 3540 if (dqm->dev->kfd->shared_resources.enable_mes) { 3541 wake_up_all(&dqm->destroy_wait); 3542 if (!err) 3543 total_resumed++; 3544 } else { 3545 per_device_resumed++; 3546 } 3547 } 3548 } 3549 3550 if (!per_device_resumed) { 3551 dqm_unlock(dqm); 3552 continue; 3553 } 3554 3555 r = execute_queues_cpsch(dqm, 3556 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 3557 0, 3558 USE_DEFAULT_GRACE_PERIOD); 3559 if (r) { 3560 dev_err(dev, "Failed to resume process queues\n"); 3561 if (queue_ids) { 3562 list_for_each_entry(q, &qpd->queues_list, list) { 3563 int q_idx = q_array_get_index( 3564 q->properties.queue_id, 3565 num_queues, 3566 queue_ids); 3567 3568 /* mask queue as error on resume fail */ 3569 if (q_idx != QUEUE_NOT_FOUND) 3570 queue_ids[q_idx] |= 3571 KFD_DBG_QUEUE_ERROR_MASK; 3572 } 3573 } 3574 } else { 3575 wake_up_all(&dqm->destroy_wait); 3576 total_resumed += per_device_resumed; 3577 } 3578 3579 dqm_unlock(dqm); 3580 } 3581 3582 if (queue_ids) { 3583 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3584 num_queues * sizeof(uint32_t))) 3585 pr_err("copy_to_user failed on queue resume\n"); 3586 3587 kfree(queue_ids); 3588 } 3589 3590 return total_resumed; 3591 } 3592 3593 int suspend_queues(struct kfd_process *p, 3594 uint32_t num_queues, 3595 uint32_t grace_period, 3596 uint64_t exception_clear_mask, 3597 uint32_t *usr_queue_id_array) 3598 { 3599 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3600 int total_suspended = 0; 3601 int i; 3602 3603 if (IS_ERR(queue_ids)) 3604 return PTR_ERR(queue_ids); 3605 3606 /* mask all queues as invalid. umask on successful request */ 3607 q_array_invalidate(num_queues, queue_ids); 3608 3609 for (i = 0; i < p->n_pdds; i++) { 3610 struct kfd_process_device *pdd = p->pdds[i]; 3611 struct device_queue_manager *dqm = pdd->dev->dqm; 3612 struct device *dev = dqm->dev->adev->dev; 3613 struct qcm_process_device *qpd = &pdd->qpd; 3614 struct queue *q; 3615 int r, per_device_suspended = 0; 3616 3617 mutex_lock(&p->event_mutex); 3618 dqm_lock(dqm); 3619 3620 /* unmask queues that suspend or already suspended */ 3621 list_for_each_entry(q, &qpd->queues_list, list) { 3622 int q_idx = q_array_get_index(q->properties.queue_id, 3623 num_queues, 3624 queue_ids); 3625 3626 if (q_idx != QUEUE_NOT_FOUND) { 3627 int err = suspend_single_queue(dqm, pdd, q); 3628 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 3629 3630 if (!err) { 3631 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 3632 if (exception_clear_mask && is_mes) 3633 q->properties.exception_status &= 3634 ~exception_clear_mask; 3635 3636 if (is_mes) 3637 total_suspended++; 3638 else 3639 per_device_suspended++; 3640 } else if (err != -EBUSY) { 3641 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3642 break; 3643 } 3644 } 3645 } 3646 3647 if (!per_device_suspended) { 3648 dqm_unlock(dqm); 3649 mutex_unlock(&p->event_mutex); 3650 if (total_suspended) 3651 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3652 continue; 3653 } 3654 3655 r = execute_queues_cpsch(dqm, 3656 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3657 grace_period); 3658 3659 if (r) 3660 dev_err(dev, "Failed to suspend process queues.\n"); 3661 else 3662 total_suspended += per_device_suspended; 3663 3664 list_for_each_entry(q, &qpd->queues_list, list) { 3665 int q_idx = q_array_get_index(q->properties.queue_id, 3666 num_queues, queue_ids); 3667 3668 if (q_idx == QUEUE_NOT_FOUND) 3669 continue; 3670 3671 /* mask queue as error on suspend fail */ 3672 if (r) 3673 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3674 else if (exception_clear_mask) 3675 q->properties.exception_status &= 3676 ~exception_clear_mask; 3677 } 3678 3679 dqm_unlock(dqm); 3680 mutex_unlock(&p->event_mutex); 3681 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3682 } 3683 3684 if (total_suspended) { 3685 struct copy_context_work_handler_workarea copy_context_worker; 3686 3687 INIT_WORK_ONSTACK( 3688 ©_context_worker.copy_context_work, 3689 copy_context_work_handler); 3690 3691 copy_context_worker.p = p; 3692 3693 schedule_work(©_context_worker.copy_context_work); 3694 3695 3696 flush_work(©_context_worker.copy_context_work); 3697 destroy_work_on_stack(©_context_worker.copy_context_work); 3698 } 3699 3700 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3701 num_queues * sizeof(uint32_t))) 3702 pr_err("copy_to_user failed on queue suspend\n"); 3703 3704 kfree(queue_ids); 3705 3706 return total_suspended; 3707 } 3708 3709 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 3710 { 3711 switch (q_props->type) { 3712 case KFD_QUEUE_TYPE_COMPUTE: 3713 return q_props->format == KFD_QUEUE_FORMAT_PM4 3714 ? KFD_IOC_QUEUE_TYPE_COMPUTE 3715 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 3716 case KFD_QUEUE_TYPE_SDMA: 3717 return KFD_IOC_QUEUE_TYPE_SDMA; 3718 case KFD_QUEUE_TYPE_SDMA_XGMI: 3719 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 3720 default: 3721 WARN_ONCE(true, "queue type not recognized!"); 3722 return 0xffffffff; 3723 }; 3724 } 3725 3726 void set_queue_snapshot_entry(struct queue *q, 3727 uint64_t exception_clear_mask, 3728 struct kfd_queue_snapshot_entry *qss_entry) 3729 { 3730 qss_entry->ring_base_address = q->properties.queue_address; 3731 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 3732 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 3733 qss_entry->ctx_save_restore_address = 3734 q->properties.ctx_save_restore_area_address; 3735 qss_entry->ctx_save_restore_area_size = 3736 q->properties.ctx_save_restore_area_size; 3737 qss_entry->exception_status = q->properties.exception_status; 3738 qss_entry->queue_id = q->properties.queue_id; 3739 qss_entry->gpu_id = q->device->id; 3740 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 3741 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 3742 q->properties.exception_status &= ~exception_clear_mask; 3743 } 3744 3745 int debug_lock_and_unmap(struct device_queue_manager *dqm) 3746 { 3747 struct device *dev = dqm->dev->adev->dev; 3748 int r; 3749 3750 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3751 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3752 return -EINVAL; 3753 } 3754 3755 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3756 return 0; 3757 3758 dqm_lock(dqm); 3759 3760 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 3761 if (r) 3762 dqm_unlock(dqm); 3763 3764 return r; 3765 } 3766 3767 int debug_map_and_unlock(struct device_queue_manager *dqm) 3768 { 3769 struct device *dev = dqm->dev->adev->dev; 3770 int r; 3771 3772 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3773 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3774 return -EINVAL; 3775 } 3776 3777 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3778 return 0; 3779 3780 r = map_queues_cpsch(dqm); 3781 3782 dqm_unlock(dqm); 3783 3784 return r; 3785 } 3786 3787 int debug_refresh_runlist(struct device_queue_manager *dqm) 3788 { 3789 int r = debug_lock_and_unmap(dqm); 3790 3791 if (r) 3792 return r; 3793 3794 return debug_map_and_unlock(dqm); 3795 } 3796 3797 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, 3798 struct qcm_process_device *qpd, 3799 int doorbell_off, u32 *queue_format) 3800 { 3801 struct queue *q; 3802 bool r = false; 3803 3804 if (!queue_format) 3805 return r; 3806 3807 dqm_lock(dqm); 3808 3809 list_for_each_entry(q, &qpd->queues_list, list) { 3810 if (q->properties.doorbell_off == doorbell_off) { 3811 *queue_format = q->properties.format; 3812 r = true; 3813 goto out; 3814 } 3815 } 3816 3817 out: 3818 dqm_unlock(dqm); 3819 return r; 3820 } 3821 #if defined(CONFIG_DEBUG_FS) 3822 3823 static void seq_reg_dump(struct seq_file *m, 3824 uint32_t (*dump)[2], uint32_t n_regs) 3825 { 3826 uint32_t i, count; 3827 3828 for (i = 0, count = 0; i < n_regs; i++) { 3829 if (count == 0 || 3830 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 3831 seq_printf(m, "%s %08x: %08x", 3832 i ? "\n" : "", 3833 dump[i][0], dump[i][1]); 3834 count = 7; 3835 } else { 3836 seq_printf(m, " %08x", dump[i][1]); 3837 count--; 3838 } 3839 } 3840 3841 seq_puts(m, "\n"); 3842 } 3843 3844 int dqm_debugfs_hqds(struct seq_file *m, void *data) 3845 { 3846 struct device_queue_manager *dqm = data; 3847 uint32_t xcc_mask = dqm->dev->xcc_mask; 3848 uint32_t (*dump)[2], n_regs; 3849 int pipe, queue; 3850 int r = 0, xcc_id; 3851 uint32_t sdma_engine_start; 3852 3853 if (!dqm->sched_running) { 3854 seq_puts(m, " Device is stopped\n"); 3855 return 0; 3856 } 3857 3858 for_each_inst(xcc_id, xcc_mask) { 3859 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3860 KFD_CIK_HIQ_PIPE, 3861 KFD_CIK_HIQ_QUEUE, &dump, 3862 &n_regs, xcc_id); 3863 if (!r) { 3864 seq_printf( 3865 m, 3866 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 3867 xcc_id, 3868 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 3869 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 3870 KFD_CIK_HIQ_QUEUE); 3871 seq_reg_dump(m, dump, n_regs); 3872 3873 kfree(dump); 3874 } 3875 3876 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 3877 int pipe_offset = pipe * get_queues_per_pipe(dqm); 3878 3879 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 3880 if (!test_bit(pipe_offset + queue, 3881 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 3882 continue; 3883 3884 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3885 pipe, queue, 3886 &dump, &n_regs, 3887 xcc_id); 3888 if (r) 3889 break; 3890 3891 seq_printf(m, 3892 " Inst %d, CP Pipe %d, Queue %d\n", 3893 xcc_id, pipe, queue); 3894 seq_reg_dump(m, dump, n_regs); 3895 3896 kfree(dump); 3897 } 3898 } 3899 } 3900 3901 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 3902 for (pipe = sdma_engine_start; 3903 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 3904 pipe++) { 3905 for (queue = 0; 3906 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 3907 queue++) { 3908 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 3909 dqm->dev->adev, pipe, queue, &dump, &n_regs); 3910 if (r) 3911 break; 3912 3913 seq_printf(m, " SDMA Engine %d, RLC %d\n", 3914 pipe, queue); 3915 seq_reg_dump(m, dump, n_regs); 3916 3917 kfree(dump); 3918 } 3919 } 3920 3921 return r; 3922 } 3923 3924 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 3925 { 3926 int r = 0; 3927 3928 dqm_lock(dqm); 3929 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 3930 if (r) { 3931 dqm_unlock(dqm); 3932 return r; 3933 } 3934 dqm->active_runlist = true; 3935 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 3936 0, USE_DEFAULT_GRACE_PERIOD); 3937 dqm_unlock(dqm); 3938 3939 return r; 3940 } 3941 3942 #endif 3943