1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "amdgpu_sdma.h" 40 #include "mes_v11_api_def.h" 41 #include "kfd_debug.h" 42 43 /* Size of the per-pipe EOP queue */ 44 #define CIK_HPD_EOP_BYTES_LOG2 11 45 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 46 /* See unmap_queues_cpsch() */ 47 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff 48 49 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 50 u32 pasid, unsigned int vmid); 51 52 static int execute_queues_cpsch(struct device_queue_manager *dqm, 53 enum kfd_unmap_queues_filter filter, 54 uint32_t filter_param, 55 uint32_t grace_period); 56 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 57 enum kfd_unmap_queues_filter filter, 58 uint32_t filter_param, 59 uint32_t grace_period, 60 bool reset); 61 62 static int map_queues_cpsch(struct device_queue_manager *dqm); 63 64 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 65 struct queue *q); 66 67 static inline void deallocate_hqd(struct device_queue_manager *dqm, 68 struct queue *q); 69 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 70 static int allocate_sdma_queue(struct device_queue_manager *dqm, 71 struct queue *q, const uint32_t *restore_sdma_id); 72 73 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); 74 75 static inline 76 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 77 { 78 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 79 return KFD_MQD_TYPE_SDMA; 80 return KFD_MQD_TYPE_CP; 81 } 82 83 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 84 { 85 int i; 86 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 87 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 88 89 /* queue is available for KFD usage if bit is 1 */ 90 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 91 if (test_bit(pipe_offset + i, 92 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 93 return true; 94 return false; 95 } 96 97 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 98 { 99 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 100 AMDGPU_MAX_QUEUES); 101 } 102 103 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 104 { 105 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 106 } 107 108 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 109 { 110 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 111 } 112 113 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 114 { 115 return kfd_get_num_sdma_engines(dqm->dev) + 116 kfd_get_num_xgmi_sdma_engines(dqm->dev); 117 } 118 119 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 120 { 121 return kfd_get_num_sdma_engines(dqm->dev) * 122 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 123 } 124 125 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 126 { 127 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 129 } 130 131 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 132 { 133 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 134 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 135 136 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 137 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 138 139 /* Mask out the reserved queues */ 140 bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) * 141 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 142 bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) * 143 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 144 } 145 146 void program_sh_mem_settings(struct device_queue_manager *dqm, 147 struct qcm_process_device *qpd) 148 { 149 uint32_t xcc_mask = dqm->dev->xcc_mask; 150 int xcc_id; 151 152 for_each_inst(xcc_id, xcc_mask) 153 dqm->dev->kfd2kgd->program_sh_mem_settings( 154 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 155 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 156 qpd->sh_mem_bases, xcc_id); 157 } 158 159 static void kfd_hws_hang(struct device_queue_manager *dqm) 160 { 161 struct device_process_node *cur; 162 struct qcm_process_device *qpd; 163 struct queue *q; 164 165 /* Mark all device queues as reset. */ 166 list_for_each_entry(cur, &dqm->queues, list) { 167 qpd = cur->qpd; 168 list_for_each_entry(q, &qpd->queues_list, list) { 169 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 170 171 pdd->has_reset_queue = true; 172 } 173 } 174 175 /* 176 * Issue a GPU reset if HWS is unresponsive 177 */ 178 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 179 } 180 181 static int convert_to_mes_queue_type(int queue_type) 182 { 183 int mes_queue_type; 184 185 switch (queue_type) { 186 case KFD_QUEUE_TYPE_COMPUTE: 187 mes_queue_type = MES_QUEUE_TYPE_COMPUTE; 188 break; 189 case KFD_QUEUE_TYPE_SDMA: 190 mes_queue_type = MES_QUEUE_TYPE_SDMA; 191 break; 192 default: 193 WARN(1, "Invalid queue type %d", queue_type); 194 mes_queue_type = -EINVAL; 195 break; 196 } 197 198 return mes_queue_type; 199 } 200 201 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 202 struct qcm_process_device *qpd) 203 { 204 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 205 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 206 struct mes_add_queue_input queue_input; 207 int r, queue_type; 208 uint64_t wptr_addr_off; 209 210 if (!dqm->sched_running || dqm->sched_halt) 211 return 0; 212 if (!down_read_trylock(&adev->reset_domain->sem)) 213 return -EIO; 214 215 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 216 queue_input.process_id = pdd->pasid; 217 queue_input.page_table_base_addr = qpd->page_table_base; 218 queue_input.process_va_start = 0; 219 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 220 /* MES unit for quantum is 100ns */ 221 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 222 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 223 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 224 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 225 queue_input.inprocess_gang_priority = q->properties.priority; 226 queue_input.gang_global_priority_level = 227 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 228 queue_input.doorbell_offset = q->properties.doorbell_off; 229 queue_input.mqd_addr = q->gart_mqd_addr; 230 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 231 232 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 233 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 234 235 queue_input.is_kfd_process = 1; 236 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 237 queue_input.queue_size = q->properties.queue_size >> 2; 238 239 queue_input.paging = false; 240 queue_input.tba_addr = qpd->tba_addr; 241 queue_input.tma_addr = qpd->tma_addr; 242 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 243 queue_input.skip_process_ctx_clear = 244 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 245 (qpd->pqm->process->debug_trap_enabled || 246 kfd_dbg_has_ttmps_always_setup(q->device)); 247 248 queue_type = convert_to_mes_queue_type(q->properties.type); 249 if (queue_type < 0) { 250 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 251 q->properties.type); 252 up_read(&adev->reset_domain->sem); 253 return -EINVAL; 254 } 255 queue_input.queue_type = (uint32_t)queue_type; 256 257 queue_input.exclusively_scheduled = q->properties.is_gws; 258 259 amdgpu_mes_lock(&adev->mes); 260 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 261 amdgpu_mes_unlock(&adev->mes); 262 up_read(&adev->reset_domain->sem); 263 if (r) { 264 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 265 q->properties.doorbell_off); 266 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 267 kfd_hws_hang(dqm); 268 } 269 270 return r; 271 } 272 273 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 274 struct qcm_process_device *qpd) 275 { 276 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 277 int r; 278 struct mes_remove_queue_input queue_input; 279 280 if (!dqm->sched_running || dqm->sched_halt) 281 return 0; 282 if (!down_read_trylock(&adev->reset_domain->sem)) 283 return -EIO; 284 285 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 286 queue_input.doorbell_offset = q->properties.doorbell_off; 287 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 288 289 amdgpu_mes_lock(&adev->mes); 290 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 291 amdgpu_mes_unlock(&adev->mes); 292 up_read(&adev->reset_domain->sem); 293 294 if (r) { 295 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", 296 q->properties.doorbell_off); 297 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 298 kfd_hws_hang(dqm); 299 } 300 301 return r; 302 } 303 304 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) 305 { 306 struct device_process_node *cur; 307 struct device *dev = dqm->dev->adev->dev; 308 struct qcm_process_device *qpd; 309 struct queue *q; 310 int retval = 0; 311 312 list_for_each_entry(cur, &dqm->queues, list) { 313 qpd = cur->qpd; 314 list_for_each_entry(q, &qpd->queues_list, list) { 315 if (q->properties.is_active) { 316 retval = remove_queue_mes(dqm, q, qpd); 317 if (retval) { 318 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 319 __func__, 320 q->properties.queue_id, 321 dqm->dev->id); 322 return retval; 323 } 324 } 325 } 326 } 327 328 return retval; 329 } 330 331 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) 332 { 333 struct device_process_node *cur; 334 struct device *dev = dqm->dev->adev->dev; 335 struct qcm_process_device *qpd; 336 struct queue *q; 337 int retval = 0; 338 339 list_for_each_entry(cur, &dqm->queues, list) { 340 qpd = cur->qpd; 341 list_for_each_entry(q, &qpd->queues_list, list) { 342 if (!q->properties.is_active) 343 continue; 344 retval = add_queue_mes(dqm, q, qpd); 345 if (retval) { 346 dev_err(dev, "%s: Failed to add queue %d for dev %d", 347 __func__, 348 q->properties.queue_id, 349 dqm->dev->id); 350 return retval; 351 } 352 } 353 } 354 355 return retval; 356 } 357 358 static int suspend_all_queues_mes(struct device_queue_manager *dqm) 359 { 360 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 361 int r = 0; 362 363 if (!down_read_trylock(&adev->reset_domain->sem)) 364 return -EIO; 365 366 r = amdgpu_mes_suspend(adev); 367 up_read(&adev->reset_domain->sem); 368 369 if (r) { 370 dev_err(adev->dev, "failed to suspend gangs from MES\n"); 371 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 372 kfd_hws_hang(dqm); 373 } 374 375 return r; 376 } 377 378 static int resume_all_queues_mes(struct device_queue_manager *dqm) 379 { 380 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 381 int r = 0; 382 383 if (!down_read_trylock(&adev->reset_domain->sem)) 384 return -EIO; 385 386 r = amdgpu_mes_resume(adev); 387 up_read(&adev->reset_domain->sem); 388 389 if (r) { 390 dev_err(adev->dev, "failed to resume gangs from MES\n"); 391 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 392 kfd_hws_hang(dqm); 393 } 394 395 return r; 396 } 397 398 static void increment_queue_count(struct device_queue_manager *dqm, 399 struct qcm_process_device *qpd, 400 struct queue *q) 401 { 402 dqm->active_queue_count++; 403 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 404 dqm->active_cp_queue_count++; 405 406 if (q->properties.is_gws) { 407 dqm->gws_queue_count++; 408 qpd->mapped_gws_queue = true; 409 } 410 } 411 412 static void decrement_queue_count(struct device_queue_manager *dqm, 413 struct qcm_process_device *qpd, 414 struct queue *q) 415 { 416 dqm->active_queue_count--; 417 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 418 dqm->active_cp_queue_count--; 419 420 if (q->properties.is_gws) { 421 dqm->gws_queue_count--; 422 qpd->mapped_gws_queue = false; 423 } 424 } 425 426 /* 427 * Allocate a doorbell ID to this queue. 428 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 429 */ 430 static int allocate_doorbell(struct qcm_process_device *qpd, 431 struct queue *q, 432 uint32_t const *restore_id) 433 { 434 struct kfd_node *dev = qpd->dqm->dev; 435 436 if (!KFD_IS_SOC15(dev)) { 437 /* On pre-SOC15 chips we need to use the queue ID to 438 * preserve the user mode ABI. 439 */ 440 441 if (restore_id && *restore_id != q->properties.queue_id) 442 return -EINVAL; 443 444 q->doorbell_id = q->properties.queue_id; 445 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 446 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 447 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 448 * doorbell assignments based on the engine and queue id. 449 * The doobell index distance between RLC (2*i) and (2*i+1) 450 * for a SDMA engine is 512. 451 */ 452 453 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 454 455 /* 456 * q->properties.sdma_engine_id corresponds to the virtual 457 * sdma engine number. However, for doorbell allocation, 458 * we need the physical sdma engine id in order to get the 459 * correct doorbell offset. 460 */ 461 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * 462 get_num_all_sdma_engines(qpd->dqm) + 463 q->properties.sdma_engine_id] 464 + (q->properties.sdma_queue_id & 1) 465 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET 466 + (q->properties.sdma_queue_id >> 1); 467 468 if (restore_id && *restore_id != valid_id) 469 return -EINVAL; 470 q->doorbell_id = valid_id; 471 } else { 472 /* For CP queues on SOC15 */ 473 if (restore_id) { 474 /* make sure that ID is free */ 475 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 476 return -EINVAL; 477 478 q->doorbell_id = *restore_id; 479 } else { 480 /* or reserve a free doorbell ID */ 481 unsigned int found; 482 483 found = find_first_zero_bit(qpd->doorbell_bitmap, 484 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 485 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 486 pr_debug("No doorbells available"); 487 return -EBUSY; 488 } 489 set_bit(found, qpd->doorbell_bitmap); 490 q->doorbell_id = found; 491 } 492 } 493 494 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 495 qpd->proc_doorbells, 496 q->doorbell_id, 497 dev->kfd->device_info.doorbell_size); 498 return 0; 499 } 500 501 static void deallocate_doorbell(struct qcm_process_device *qpd, 502 struct queue *q) 503 { 504 unsigned int old; 505 struct kfd_node *dev = qpd->dqm->dev; 506 507 if (!KFD_IS_SOC15(dev) || 508 q->properties.type == KFD_QUEUE_TYPE_SDMA || 509 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 510 return; 511 512 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 513 WARN_ON(!old); 514 } 515 516 static void program_trap_handler_settings(struct device_queue_manager *dqm, 517 struct qcm_process_device *qpd) 518 { 519 uint32_t xcc_mask = dqm->dev->xcc_mask; 520 int xcc_id; 521 522 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 523 for_each_inst(xcc_id, xcc_mask) 524 dqm->dev->kfd2kgd->program_trap_handler_settings( 525 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 526 qpd->tma_addr, xcc_id); 527 } 528 529 static int allocate_vmid(struct device_queue_manager *dqm, 530 struct qcm_process_device *qpd, 531 struct queue *q) 532 { 533 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 534 struct device *dev = dqm->dev->adev->dev; 535 int allocated_vmid = -1, i; 536 537 for (i = dqm->dev->vm_info.first_vmid_kfd; 538 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 539 if (!dqm->vmid_pasid[i]) { 540 allocated_vmid = i; 541 break; 542 } 543 } 544 545 if (allocated_vmid < 0) { 546 dev_err(dev, "no more vmid to allocate\n"); 547 return -ENOSPC; 548 } 549 550 pr_debug("vmid allocated: %d\n", allocated_vmid); 551 552 dqm->vmid_pasid[allocated_vmid] = pdd->pasid; 553 554 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid); 555 556 qpd->vmid = allocated_vmid; 557 q->properties.vmid = allocated_vmid; 558 559 program_sh_mem_settings(dqm, qpd); 560 561 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 562 program_trap_handler_settings(dqm, qpd); 563 564 /* qpd->page_table_base is set earlier when register_process() 565 * is called, i.e. when the first queue is created. 566 */ 567 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 568 qpd->vmid, 569 qpd->page_table_base); 570 /* invalidate the VM context after pasid and vmid mapping is set up */ 571 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 572 573 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 574 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 575 qpd->sh_hidden_private_base, qpd->vmid); 576 577 return 0; 578 } 579 580 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 581 struct qcm_process_device *qpd) 582 { 583 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 584 int ret; 585 586 if (!qpd->ib_kaddr) 587 return -ENOMEM; 588 589 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 590 if (ret) 591 return ret; 592 593 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 594 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 595 pmf->release_mem_size / sizeof(uint32_t)); 596 } 597 598 static void deallocate_vmid(struct device_queue_manager *dqm, 599 struct qcm_process_device *qpd, 600 struct queue *q) 601 { 602 struct device *dev = dqm->dev->adev->dev; 603 604 /* On GFX v7, CP doesn't flush TC at dequeue */ 605 if (q->device->adev->asic_type == CHIP_HAWAII) 606 if (flush_texture_cache_nocpsch(q->device, qpd)) 607 dev_err(dev, "Failed to flush TC\n"); 608 609 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 610 611 /* Release the vmid mapping */ 612 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 613 dqm->vmid_pasid[qpd->vmid] = 0; 614 615 qpd->vmid = 0; 616 q->properties.vmid = 0; 617 } 618 619 static int create_queue_nocpsch(struct device_queue_manager *dqm, 620 struct queue *q, 621 struct qcm_process_device *qpd, 622 const struct kfd_criu_queue_priv_data *qd, 623 const void *restore_mqd, const void *restore_ctl_stack) 624 { 625 struct mqd_manager *mqd_mgr; 626 int retval; 627 628 dqm_lock(dqm); 629 630 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 631 pr_warn("Can't create new usermode queue because %d queues were already created\n", 632 dqm->total_queue_count); 633 retval = -EPERM; 634 goto out_unlock; 635 } 636 637 if (list_empty(&qpd->queues_list)) { 638 retval = allocate_vmid(dqm, qpd, q); 639 if (retval) 640 goto out_unlock; 641 } 642 q->properties.vmid = qpd->vmid; 643 /* 644 * Eviction state logic: mark all queues as evicted, even ones 645 * not currently active. Restoring inactive queues later only 646 * updates the is_evicted flag but is a no-op otherwise. 647 */ 648 q->properties.is_evicted = !!qpd->evicted; 649 650 q->properties.tba_addr = qpd->tba_addr; 651 q->properties.tma_addr = qpd->tma_addr; 652 653 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 654 q->properties.type)]; 655 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 656 retval = allocate_hqd(dqm, q); 657 if (retval) 658 goto deallocate_vmid; 659 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 660 q->pipe, q->queue); 661 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 662 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 663 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 664 if (retval) 665 goto deallocate_vmid; 666 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 667 } 668 669 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 670 if (retval) 671 goto out_deallocate_hqd; 672 673 /* Temporarily release dqm lock to avoid a circular lock dependency */ 674 dqm_unlock(dqm); 675 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 676 dqm_lock(dqm); 677 678 if (!q->mqd_mem_obj) { 679 retval = -ENOMEM; 680 goto out_deallocate_doorbell; 681 } 682 683 if (qd) 684 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 685 &q->properties, restore_mqd, restore_ctl_stack, 686 qd->ctl_stack_size); 687 else 688 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 689 &q->gart_mqd_addr, &q->properties); 690 691 if (q->properties.is_active) { 692 if (!dqm->sched_running) { 693 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 694 goto add_queue_to_list; 695 } 696 697 if (WARN(q->process->mm != current->mm, 698 "should only run in user thread")) 699 retval = -EFAULT; 700 else 701 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 702 q->queue, &q->properties, current->mm); 703 if (retval) 704 goto out_free_mqd; 705 } 706 707 add_queue_to_list: 708 list_add(&q->list, &qpd->queues_list); 709 qpd->queue_count++; 710 if (q->properties.is_active) 711 increment_queue_count(dqm, qpd, q); 712 713 /* 714 * Unconditionally increment this counter, regardless of the queue's 715 * type or whether the queue is active. 716 */ 717 dqm->total_queue_count++; 718 pr_debug("Total of %d queues are accountable so far\n", 719 dqm->total_queue_count); 720 goto out_unlock; 721 722 out_free_mqd: 723 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 724 out_deallocate_doorbell: 725 deallocate_doorbell(qpd, q); 726 out_deallocate_hqd: 727 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 728 deallocate_hqd(dqm, q); 729 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 730 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 731 deallocate_sdma_queue(dqm, q); 732 deallocate_vmid: 733 if (list_empty(&qpd->queues_list)) 734 deallocate_vmid(dqm, qpd, q); 735 out_unlock: 736 dqm_unlock(dqm); 737 return retval; 738 } 739 740 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 741 { 742 bool set; 743 int pipe, bit, i; 744 745 set = false; 746 747 for (pipe = dqm->next_pipe_to_allocate, i = 0; 748 i < get_pipes_per_mec(dqm); 749 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 750 751 if (!is_pipe_enabled(dqm, 0, pipe)) 752 continue; 753 754 if (dqm->allocated_queues[pipe] != 0) { 755 bit = ffs(dqm->allocated_queues[pipe]) - 1; 756 dqm->allocated_queues[pipe] &= ~(1 << bit); 757 q->pipe = pipe; 758 q->queue = bit; 759 set = true; 760 break; 761 } 762 } 763 764 if (!set) 765 return -EBUSY; 766 767 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 768 /* horizontal hqd allocation */ 769 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 770 771 return 0; 772 } 773 774 static inline void deallocate_hqd(struct device_queue_manager *dqm, 775 struct queue *q) 776 { 777 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 778 } 779 780 #define SQ_IND_CMD_CMD_KILL 0x00000003 781 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 782 783 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 784 { 785 int status = 0; 786 unsigned int vmid; 787 uint16_t queried_pasid; 788 union SQ_CMD_BITS reg_sq_cmd; 789 union GRBM_GFX_INDEX_BITS reg_gfx_index; 790 struct kfd_process_device *pdd; 791 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 792 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 793 uint32_t xcc_mask = dev->xcc_mask; 794 int xcc_id; 795 796 reg_sq_cmd.u32All = 0; 797 reg_gfx_index.u32All = 0; 798 799 pr_debug("Killing all process wavefronts\n"); 800 801 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 802 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 803 return -EOPNOTSUPP; 804 } 805 806 /* taking the VMID for that process on the safe way using PDD */ 807 pdd = kfd_get_process_device_data(dev, p); 808 if (!pdd) 809 return -EFAULT; 810 811 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 812 * ATC_VMID15_PASID_MAPPING 813 * to check which VMID the current process is mapped to. 814 */ 815 816 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 817 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 818 (dev->adev, vmid, &queried_pasid); 819 820 if (status && queried_pasid == pdd->pasid) { 821 pr_debug("Killing wave fronts of vmid %d and process pid %d\n", 822 vmid, p->lead_thread->pid); 823 break; 824 } 825 } 826 827 if (vmid > last_vmid_to_scan) { 828 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n", 829 p->lead_thread->pid); 830 return -EFAULT; 831 } 832 833 reg_gfx_index.bits.sh_broadcast_writes = 1; 834 reg_gfx_index.bits.se_broadcast_writes = 1; 835 reg_gfx_index.bits.instance_broadcast_writes = 1; 836 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 837 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 838 reg_sq_cmd.bits.vm_id = vmid; 839 840 for_each_inst(xcc_id, xcc_mask) 841 dev->kfd2kgd->wave_control_execute( 842 dev->adev, reg_gfx_index.u32All, 843 reg_sq_cmd.u32All, xcc_id); 844 845 return 0; 846 } 847 848 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 849 * to avoid asynchronized access 850 */ 851 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 852 struct qcm_process_device *qpd, 853 struct queue *q) 854 { 855 int retval; 856 struct mqd_manager *mqd_mgr; 857 858 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 859 860 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 861 deallocate_hqd(dqm, q); 862 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 863 deallocate_sdma_queue(dqm, q); 864 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 865 deallocate_sdma_queue(dqm, q); 866 else { 867 pr_debug("q->properties.type %d is invalid\n", 868 q->properties.type); 869 return -EINVAL; 870 } 871 dqm->total_queue_count--; 872 873 deallocate_doorbell(qpd, q); 874 875 if (!dqm->sched_running) { 876 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 877 return 0; 878 } 879 880 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 881 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 882 KFD_UNMAP_LATENCY_MS, 883 q->pipe, q->queue); 884 if (retval == -ETIME) 885 qpd->reset_wavefronts = true; 886 887 list_del(&q->list); 888 if (list_empty(&qpd->queues_list)) { 889 if (qpd->reset_wavefronts) { 890 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 891 dqm->dev); 892 /* dbgdev_wave_reset_wavefronts has to be called before 893 * deallocate_vmid(), i.e. when vmid is still in use. 894 */ 895 dbgdev_wave_reset_wavefronts(dqm->dev, 896 qpd->pqm->process); 897 qpd->reset_wavefronts = false; 898 } 899 900 deallocate_vmid(dqm, qpd, q); 901 } 902 qpd->queue_count--; 903 if (q->properties.is_active) 904 decrement_queue_count(dqm, qpd, q); 905 906 return retval; 907 } 908 909 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 910 struct qcm_process_device *qpd, 911 struct queue *q) 912 { 913 int retval; 914 uint64_t sdma_val = 0; 915 struct device *dev = dqm->dev->adev->dev; 916 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 917 struct mqd_manager *mqd_mgr = 918 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 919 920 /* Get the SDMA queue stats */ 921 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 922 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 923 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 924 &sdma_val); 925 if (retval) 926 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 927 q->properties.queue_id); 928 } 929 930 dqm_lock(dqm); 931 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 932 if (!retval) 933 pdd->sdma_past_activity_counter += sdma_val; 934 dqm_unlock(dqm); 935 936 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 937 938 return retval; 939 } 940 941 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 942 struct mqd_update_info *minfo) 943 { 944 int retval = 0; 945 struct device *dev = dqm->dev->adev->dev; 946 struct mqd_manager *mqd_mgr; 947 struct kfd_process_device *pdd; 948 bool prev_active = false; 949 950 dqm_lock(dqm); 951 pdd = kfd_get_process_device_data(q->device, q->process); 952 if (!pdd) { 953 retval = -ENODEV; 954 goto out_unlock; 955 } 956 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 957 q->properties.type)]; 958 959 /* Save previous activity state for counters */ 960 prev_active = q->properties.is_active; 961 962 /* Make sure the queue is unmapped before updating the MQD */ 963 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 964 if (!dqm->dev->kfd->shared_resources.enable_mes) 965 retval = unmap_queues_cpsch(dqm, 966 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 967 else if (prev_active) 968 retval = remove_queue_mes(dqm, q, &pdd->qpd); 969 970 /* queue is reset so inaccessable */ 971 if (pdd->has_reset_queue) { 972 retval = -EACCES; 973 goto out_unlock; 974 } 975 976 if (retval) { 977 dev_err(dev, "unmap queue failed\n"); 978 goto out_unlock; 979 } 980 } else if (prev_active && 981 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 982 q->properties.type == KFD_QUEUE_TYPE_SDMA || 983 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 984 985 if (!dqm->sched_running) { 986 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 987 goto out_unlock; 988 } 989 990 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 991 (dqm->dev->kfd->cwsr_enabled ? 992 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 993 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 994 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 995 if (retval) { 996 dev_err(dev, "destroy mqd failed\n"); 997 goto out_unlock; 998 } 999 } 1000 1001 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 1002 1003 /* 1004 * check active state vs. the previous state and modify 1005 * counter accordingly. map_queues_cpsch uses the 1006 * dqm->active_queue_count to determine whether a new runlist must be 1007 * uploaded. 1008 */ 1009 if (q->properties.is_active && !prev_active) { 1010 increment_queue_count(dqm, &pdd->qpd, q); 1011 } else if (!q->properties.is_active && prev_active) { 1012 decrement_queue_count(dqm, &pdd->qpd, q); 1013 } else if (q->gws && !q->properties.is_gws) { 1014 if (q->properties.is_active) { 1015 dqm->gws_queue_count++; 1016 pdd->qpd.mapped_gws_queue = true; 1017 } 1018 q->properties.is_gws = true; 1019 } else if (!q->gws && q->properties.is_gws) { 1020 if (q->properties.is_active) { 1021 dqm->gws_queue_count--; 1022 pdd->qpd.mapped_gws_queue = false; 1023 } 1024 q->properties.is_gws = false; 1025 } 1026 1027 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1028 if (!dqm->dev->kfd->shared_resources.enable_mes) 1029 retval = map_queues_cpsch(dqm); 1030 else if (q->properties.is_active) 1031 retval = add_queue_mes(dqm, q, &pdd->qpd); 1032 } else if (q->properties.is_active && 1033 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1034 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1035 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1036 if (WARN(q->process->mm != current->mm, 1037 "should only run in user thread")) 1038 retval = -EFAULT; 1039 else 1040 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 1041 q->pipe, q->queue, 1042 &q->properties, current->mm); 1043 } 1044 1045 out_unlock: 1046 dqm_unlock(dqm); 1047 return retval; 1048 } 1049 1050 /* suspend_single_queue does not lock the dqm like the 1051 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 1052 * lock the dqm before calling, and unlock after calling. 1053 * 1054 * The reason we don't lock the dqm is because this function may be 1055 * called on multiple queues in a loop, so rather than locking/unlocking 1056 * multiple times, we will just keep the dqm locked for all of the calls. 1057 */ 1058 static int suspend_single_queue(struct device_queue_manager *dqm, 1059 struct kfd_process_device *pdd, 1060 struct queue *q) 1061 { 1062 bool is_new; 1063 1064 if (q->properties.is_suspended) 1065 return 0; 1066 1067 pr_debug("Suspending process pid %d queue [%i]\n", 1068 pdd->process->lead_thread->pid, 1069 q->properties.queue_id); 1070 1071 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 1072 1073 if (is_new || q->properties.is_being_destroyed) { 1074 pr_debug("Suspend: skip %s queue id %i\n", 1075 is_new ? "new" : "destroyed", 1076 q->properties.queue_id); 1077 return -EBUSY; 1078 } 1079 1080 q->properties.is_suspended = true; 1081 if (q->properties.is_active) { 1082 if (dqm->dev->kfd->shared_resources.enable_mes) { 1083 int r = remove_queue_mes(dqm, q, &pdd->qpd); 1084 1085 if (r) 1086 return r; 1087 } 1088 1089 decrement_queue_count(dqm, &pdd->qpd, q); 1090 q->properties.is_active = false; 1091 } 1092 1093 return 0; 1094 } 1095 1096 /* resume_single_queue does not lock the dqm like the functions 1097 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1098 * lock the dqm before calling, and unlock after calling. 1099 * 1100 * The reason we don't lock the dqm is because this function may be 1101 * called on multiple queues in a loop, so rather than locking/unlocking 1102 * multiple times, we will just keep the dqm locked for all of the calls. 1103 */ 1104 static int resume_single_queue(struct device_queue_manager *dqm, 1105 struct qcm_process_device *qpd, 1106 struct queue *q) 1107 { 1108 struct kfd_process_device *pdd; 1109 1110 if (!q->properties.is_suspended) 1111 return 0; 1112 1113 pdd = qpd_to_pdd(qpd); 1114 1115 pr_debug("Restoring from suspend process pid %d queue [%i]\n", 1116 pdd->process->lead_thread->pid, 1117 q->properties.queue_id); 1118 1119 q->properties.is_suspended = false; 1120 1121 if (QUEUE_IS_ACTIVE(q->properties)) { 1122 if (dqm->dev->kfd->shared_resources.enable_mes) { 1123 int r = add_queue_mes(dqm, q, &pdd->qpd); 1124 1125 if (r) 1126 return r; 1127 } 1128 1129 q->properties.is_active = true; 1130 increment_queue_count(dqm, qpd, q); 1131 } 1132 1133 return 0; 1134 } 1135 1136 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1137 struct qcm_process_device *qpd) 1138 { 1139 struct queue *q; 1140 struct mqd_manager *mqd_mgr; 1141 struct kfd_process_device *pdd; 1142 int retval, ret = 0; 1143 1144 dqm_lock(dqm); 1145 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1146 goto out; 1147 1148 pdd = qpd_to_pdd(qpd); 1149 pr_debug_ratelimited("Evicting process pid %d queues\n", 1150 pdd->process->lead_thread->pid); 1151 1152 pdd->last_evict_timestamp = get_jiffies_64(); 1153 /* Mark all queues as evicted. Deactivate all active queues on 1154 * the qpd. 1155 */ 1156 list_for_each_entry(q, &qpd->queues_list, list) { 1157 q->properties.is_evicted = true; 1158 if (!q->properties.is_active) 1159 continue; 1160 1161 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1162 q->properties.type)]; 1163 q->properties.is_active = false; 1164 decrement_queue_count(dqm, qpd, q); 1165 1166 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1167 continue; 1168 1169 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1170 (dqm->dev->kfd->cwsr_enabled ? 1171 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1172 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1173 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1174 if (retval && !ret) 1175 /* Return the first error, but keep going to 1176 * maintain a consistent eviction state 1177 */ 1178 ret = retval; 1179 } 1180 1181 out: 1182 dqm_unlock(dqm); 1183 return ret; 1184 } 1185 1186 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1187 struct qcm_process_device *qpd) 1188 { 1189 struct queue *q; 1190 struct device *dev = dqm->dev->adev->dev; 1191 struct kfd_process_device *pdd; 1192 int retval = 0; 1193 1194 dqm_lock(dqm); 1195 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1196 goto out; 1197 1198 pdd = qpd_to_pdd(qpd); 1199 1200 /* The debugger creates processes that temporarily have not acquired 1201 * all VMs for all devices and has no VMs itself. 1202 * Skip queue eviction on process eviction. 1203 */ 1204 if (!pdd->drm_priv) 1205 goto out; 1206 1207 pr_debug_ratelimited("Evicting process pid %d queues\n", 1208 pdd->process->lead_thread->pid); 1209 1210 if (dqm->dev->kfd->shared_resources.enable_mes) { 1211 pdd->last_evict_timestamp = get_jiffies_64(); 1212 retval = suspend_all_queues_mes(dqm); 1213 if (retval) { 1214 dev_err(dev, "Suspending all queues failed"); 1215 goto out; 1216 } 1217 } 1218 1219 /* Mark all queues as evicted. Deactivate all active queues on 1220 * the qpd. 1221 */ 1222 list_for_each_entry(q, &qpd->queues_list, list) { 1223 q->properties.is_evicted = true; 1224 if (!q->properties.is_active) 1225 continue; 1226 1227 q->properties.is_active = false; 1228 decrement_queue_count(dqm, qpd, q); 1229 1230 if (dqm->dev->kfd->shared_resources.enable_mes) { 1231 retval = remove_queue_mes(dqm, q, qpd); 1232 if (retval) { 1233 dev_err(dev, "Failed to evict queue %d\n", 1234 q->properties.queue_id); 1235 goto out; 1236 } 1237 } 1238 } 1239 1240 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1241 pdd->last_evict_timestamp = get_jiffies_64(); 1242 retval = execute_queues_cpsch(dqm, 1243 qpd->is_debug ? 1244 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1245 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1246 USE_DEFAULT_GRACE_PERIOD); 1247 } else { 1248 retval = resume_all_queues_mes(dqm); 1249 if (retval) 1250 dev_err(dev, "Resuming all queues failed"); 1251 } 1252 1253 out: 1254 dqm_unlock(dqm); 1255 return retval; 1256 } 1257 1258 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1259 struct qcm_process_device *qpd) 1260 { 1261 struct mm_struct *mm = NULL; 1262 struct queue *q; 1263 struct mqd_manager *mqd_mgr; 1264 struct kfd_process_device *pdd; 1265 uint64_t pd_base; 1266 uint64_t eviction_duration; 1267 int retval, ret = 0; 1268 1269 pdd = qpd_to_pdd(qpd); 1270 /* Retrieve PD base */ 1271 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1272 1273 dqm_lock(dqm); 1274 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1275 goto out; 1276 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1277 qpd->evicted--; 1278 goto out; 1279 } 1280 1281 pr_debug_ratelimited("Restoring process pid %d queues\n", 1282 pdd->process->lead_thread->pid); 1283 1284 /* Update PD Base in QPD */ 1285 qpd->page_table_base = pd_base; 1286 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1287 1288 if (!list_empty(&qpd->queues_list)) { 1289 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1290 dqm->dev->adev, 1291 qpd->vmid, 1292 qpd->page_table_base); 1293 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); 1294 } 1295 1296 /* Take a safe reference to the mm_struct, which may otherwise 1297 * disappear even while the kfd_process is still referenced. 1298 */ 1299 mm = get_task_mm(pdd->process->lead_thread); 1300 if (!mm) { 1301 ret = -EFAULT; 1302 goto out; 1303 } 1304 1305 /* Remove the eviction flags. Activate queues that are not 1306 * inactive for other reasons. 1307 */ 1308 list_for_each_entry(q, &qpd->queues_list, list) { 1309 q->properties.is_evicted = false; 1310 if (!QUEUE_IS_ACTIVE(q->properties)) 1311 continue; 1312 1313 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1314 q->properties.type)]; 1315 q->properties.is_active = true; 1316 increment_queue_count(dqm, qpd, q); 1317 1318 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1319 continue; 1320 1321 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1322 q->queue, &q->properties, mm); 1323 if (retval && !ret) 1324 /* Return the first error, but keep going to 1325 * maintain a consistent eviction state 1326 */ 1327 ret = retval; 1328 } 1329 qpd->evicted = 0; 1330 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1331 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1332 out: 1333 if (mm) 1334 mmput(mm); 1335 dqm_unlock(dqm); 1336 return ret; 1337 } 1338 1339 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1340 struct qcm_process_device *qpd) 1341 { 1342 struct queue *q; 1343 struct device *dev = dqm->dev->adev->dev; 1344 struct kfd_process_device *pdd; 1345 uint64_t eviction_duration; 1346 int retval = 0; 1347 1348 pdd = qpd_to_pdd(qpd); 1349 1350 dqm_lock(dqm); 1351 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1352 goto out; 1353 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1354 qpd->evicted--; 1355 goto out; 1356 } 1357 1358 /* The debugger creates processes that temporarily have not acquired 1359 * all VMs for all devices and has no VMs itself. 1360 * Skip queue restore on process restore. 1361 */ 1362 if (!pdd->drm_priv) 1363 goto vm_not_acquired; 1364 1365 pr_debug_ratelimited("Restoring process pid %d queues\n", 1366 pdd->process->lead_thread->pid); 1367 1368 /* Update PD Base in QPD */ 1369 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1370 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1371 1372 /* activate all active queues on the qpd */ 1373 list_for_each_entry(q, &qpd->queues_list, list) { 1374 q->properties.is_evicted = false; 1375 if (!QUEUE_IS_ACTIVE(q->properties)) 1376 continue; 1377 1378 q->properties.is_active = true; 1379 increment_queue_count(dqm, &pdd->qpd, q); 1380 1381 if (dqm->dev->kfd->shared_resources.enable_mes) { 1382 retval = add_queue_mes(dqm, q, qpd); 1383 if (retval) { 1384 dev_err(dev, "Failed to restore queue %d\n", 1385 q->properties.queue_id); 1386 goto out; 1387 } 1388 } 1389 } 1390 if (!dqm->dev->kfd->shared_resources.enable_mes) 1391 retval = execute_queues_cpsch(dqm, 1392 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1393 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1394 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1395 vm_not_acquired: 1396 qpd->evicted = 0; 1397 out: 1398 dqm_unlock(dqm); 1399 return retval; 1400 } 1401 1402 static int register_process(struct device_queue_manager *dqm, 1403 struct qcm_process_device *qpd) 1404 { 1405 struct device_process_node *n; 1406 struct kfd_process_device *pdd; 1407 uint64_t pd_base; 1408 int retval; 1409 1410 n = kzalloc(sizeof(*n), GFP_KERNEL); 1411 if (!n) 1412 return -ENOMEM; 1413 1414 n->qpd = qpd; 1415 1416 pdd = qpd_to_pdd(qpd); 1417 /* Retrieve PD base */ 1418 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1419 1420 dqm_lock(dqm); 1421 list_add(&n->list, &dqm->queues); 1422 1423 /* Update PD Base in QPD */ 1424 qpd->page_table_base = pd_base; 1425 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1426 1427 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1428 1429 dqm->processes_count++; 1430 1431 dqm_unlock(dqm); 1432 1433 /* Outside the DQM lock because under the DQM lock we can't do 1434 * reclaim or take other locks that others hold while reclaiming. 1435 */ 1436 kfd_inc_compute_active(dqm->dev); 1437 1438 return retval; 1439 } 1440 1441 static int unregister_process(struct device_queue_manager *dqm, 1442 struct qcm_process_device *qpd) 1443 { 1444 int retval = 0; 1445 struct device_process_node *cur, *next; 1446 1447 pr_debug("qpd->queues_list is %s\n", 1448 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1449 1450 dqm_lock(dqm); 1451 1452 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1453 if (qpd == cur->qpd) { 1454 list_del(&cur->list); 1455 kfree(cur); 1456 dqm->processes_count--; 1457 goto out; 1458 } 1459 } 1460 /* qpd not found in dqm list */ 1461 retval = 1; 1462 out: 1463 dqm_unlock(dqm); 1464 1465 /* Outside the DQM lock because under the DQM lock we can't do 1466 * reclaim or take other locks that others hold while reclaiming. 1467 */ 1468 if (!retval) 1469 kfd_dec_compute_active(dqm->dev); 1470 1471 return retval; 1472 } 1473 1474 static int 1475 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1476 unsigned int vmid) 1477 { 1478 uint32_t xcc_mask = dqm->dev->xcc_mask; 1479 int xcc_id, ret = 0; 1480 1481 for_each_inst(xcc_id, xcc_mask) { 1482 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1483 dqm->dev->adev, pasid, vmid, xcc_id); 1484 if (ret) 1485 break; 1486 } 1487 1488 return ret; 1489 } 1490 1491 static void init_interrupts(struct device_queue_manager *dqm) 1492 { 1493 uint32_t xcc_mask = dqm->dev->xcc_mask; 1494 unsigned int i, xcc_id; 1495 1496 for_each_inst(xcc_id, xcc_mask) { 1497 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1498 if (is_pipe_enabled(dqm, 0, i)) { 1499 dqm->dev->kfd2kgd->init_interrupts( 1500 dqm->dev->adev, i, xcc_id); 1501 } 1502 } 1503 } 1504 } 1505 1506 static int initialize_nocpsch(struct device_queue_manager *dqm) 1507 { 1508 int pipe, queue; 1509 1510 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1511 1512 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), 1513 sizeof(unsigned int), GFP_KERNEL); 1514 if (!dqm->allocated_queues) 1515 return -ENOMEM; 1516 1517 mutex_init(&dqm->lock_hidden); 1518 INIT_LIST_HEAD(&dqm->queues); 1519 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1520 dqm->active_cp_queue_count = 0; 1521 dqm->gws_queue_count = 0; 1522 1523 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1524 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1525 1526 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1527 if (test_bit(pipe_offset + queue, 1528 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1529 dqm->allocated_queues[pipe] |= 1 << queue; 1530 } 1531 1532 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1533 1534 init_sdma_bitmaps(dqm); 1535 1536 return 0; 1537 } 1538 1539 static void uninitialize(struct device_queue_manager *dqm) 1540 { 1541 int i; 1542 1543 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1544 1545 kfree(dqm->allocated_queues); 1546 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1547 kfree(dqm->mqd_mgrs[i]); 1548 mutex_destroy(&dqm->lock_hidden); 1549 } 1550 1551 static int start_nocpsch(struct device_queue_manager *dqm) 1552 { 1553 int r = 0; 1554 1555 pr_info("SW scheduler is used"); 1556 init_interrupts(dqm); 1557 1558 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1559 r = pm_init(&dqm->packet_mgr, dqm); 1560 if (!r) 1561 dqm->sched_running = true; 1562 1563 return r; 1564 } 1565 1566 static int stop_nocpsch(struct device_queue_manager *dqm) 1567 { 1568 dqm_lock(dqm); 1569 if (!dqm->sched_running) { 1570 dqm_unlock(dqm); 1571 return 0; 1572 } 1573 1574 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1575 pm_uninit(&dqm->packet_mgr); 1576 dqm->sched_running = false; 1577 dqm_unlock(dqm); 1578 1579 return 0; 1580 } 1581 1582 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1583 struct queue *q, const uint32_t *restore_sdma_id) 1584 { 1585 struct device *dev = dqm->dev->adev->dev; 1586 int bit; 1587 1588 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1589 if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) { 1590 dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n", 1591 get_num_sdma_queues(dqm)); 1592 return -ENOMEM; 1593 } 1594 1595 if (restore_sdma_id) { 1596 /* Re-use existing sdma_id */ 1597 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1598 dev_err(dev, "SDMA queue already in use\n"); 1599 return -EBUSY; 1600 } 1601 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1602 q->sdma_id = *restore_sdma_id; 1603 } else { 1604 /* Find first available sdma_id */ 1605 bit = find_first_bit(dqm->sdma_bitmap, 1606 get_num_sdma_queues(dqm)); 1607 clear_bit(bit, dqm->sdma_bitmap); 1608 q->sdma_id = bit; 1609 } 1610 1611 q->properties.sdma_engine_id = 1612 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1613 q->properties.sdma_queue_id = q->sdma_id / 1614 kfd_get_num_sdma_engines(dqm->dev); 1615 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1616 if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) { 1617 dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n", 1618 get_num_xgmi_sdma_queues(dqm)); 1619 return -ENOMEM; 1620 } 1621 if (restore_sdma_id) { 1622 /* Re-use existing sdma_id */ 1623 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1624 dev_err(dev, "SDMA queue already in use\n"); 1625 return -EBUSY; 1626 } 1627 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1628 q->sdma_id = *restore_sdma_id; 1629 } else { 1630 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1631 get_num_xgmi_sdma_queues(dqm)); 1632 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1633 q->sdma_id = bit; 1634 } 1635 /* sdma_engine_id is sdma id including 1636 * both PCIe-optimized SDMAs and XGMI- 1637 * optimized SDMAs. The calculation below 1638 * assumes the first N engines are always 1639 * PCIe-optimized ones 1640 */ 1641 q->properties.sdma_engine_id = 1642 kfd_get_num_sdma_engines(dqm->dev) + 1643 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1644 q->properties.sdma_queue_id = q->sdma_id / 1645 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1646 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1647 int i, num_queues, num_engines, eng_offset = 0, start_engine; 1648 bool free_bit_found = false, is_xgmi = false; 1649 1650 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { 1651 num_queues = get_num_sdma_queues(dqm); 1652 num_engines = kfd_get_num_sdma_engines(dqm->dev); 1653 q->properties.type = KFD_QUEUE_TYPE_SDMA; 1654 } else { 1655 num_queues = get_num_xgmi_sdma_queues(dqm); 1656 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); 1657 eng_offset = kfd_get_num_sdma_engines(dqm->dev); 1658 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI; 1659 is_xgmi = true; 1660 } 1661 1662 /* Scan available bit based on target engine ID. */ 1663 start_engine = q->properties.sdma_engine_id - eng_offset; 1664 for (i = start_engine; i < num_queues; i += num_engines) { 1665 1666 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) 1667 continue; 1668 1669 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); 1670 q->sdma_id = i; 1671 q->properties.sdma_queue_id = q->sdma_id / num_engines; 1672 free_bit_found = true; 1673 break; 1674 } 1675 1676 if (!free_bit_found) { 1677 dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n", 1678 q->properties.sdma_engine_id, num_queues); 1679 return -ENOMEM; 1680 } 1681 } 1682 1683 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 1684 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 1685 1686 return 0; 1687 } 1688 1689 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 1690 struct queue *q) 1691 { 1692 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1693 if (q->sdma_id >= get_num_sdma_queues(dqm)) 1694 return; 1695 set_bit(q->sdma_id, dqm->sdma_bitmap); 1696 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1697 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1698 return; 1699 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 1700 } 1701 } 1702 1703 /* 1704 * Device Queue Manager implementation for cp scheduler 1705 */ 1706 1707 static int set_sched_resources(struct device_queue_manager *dqm) 1708 { 1709 int i, mec; 1710 struct scheduling_resources res; 1711 struct device *dev = dqm->dev->adev->dev; 1712 1713 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 1714 1715 res.queue_mask = 0; 1716 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 1717 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 1718 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 1719 1720 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1721 continue; 1722 1723 /* only acquire queues from the first MEC */ 1724 if (mec > 0) 1725 continue; 1726 1727 /* This situation may be hit in the future if a new HW 1728 * generation exposes more than 64 queues. If so, the 1729 * definition of res.queue_mask needs updating 1730 */ 1731 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 1732 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 1733 break; 1734 } 1735 1736 res.queue_mask |= 1ull 1737 << amdgpu_queue_mask_bit_to_set_resource_bit( 1738 dqm->dev->adev, i); 1739 } 1740 res.gws_mask = ~0ull; 1741 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 1742 1743 pr_debug("Scheduling resources:\n" 1744 "vmid mask: 0x%8X\n" 1745 "queue mask: 0x%8llX\n", 1746 res.vmid_mask, res.queue_mask); 1747 1748 return pm_send_set_resources(&dqm->packet_mgr, &res); 1749 } 1750 1751 static int initialize_cpsch(struct device_queue_manager *dqm) 1752 { 1753 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1754 1755 mutex_init(&dqm->lock_hidden); 1756 INIT_LIST_HEAD(&dqm->queues); 1757 dqm->active_queue_count = dqm->processes_count = 0; 1758 dqm->active_cp_queue_count = 0; 1759 dqm->gws_queue_count = 0; 1760 dqm->active_runlist = false; 1761 dqm->trap_debug_vmid = 0; 1762 1763 init_sdma_bitmaps(dqm); 1764 1765 update_dqm_wait_times(dqm); 1766 return 0; 1767 } 1768 1769 /* halt_cpsch: 1770 * Unmap queues so the schedule doesn't continue remaining jobs in the queue. 1771 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch 1772 * is called. 1773 */ 1774 static int halt_cpsch(struct device_queue_manager *dqm) 1775 { 1776 int ret = 0; 1777 1778 dqm_lock(dqm); 1779 if (!dqm->sched_running) { 1780 dqm_unlock(dqm); 1781 return 0; 1782 } 1783 1784 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); 1785 1786 if (!dqm->is_hws_hang) { 1787 if (!dqm->dev->kfd->shared_resources.enable_mes) 1788 ret = unmap_queues_cpsch(dqm, 1789 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1790 USE_DEFAULT_GRACE_PERIOD, false); 1791 else 1792 ret = remove_all_kfd_queues_mes(dqm); 1793 } 1794 dqm->sched_halt = true; 1795 dqm_unlock(dqm); 1796 1797 return ret; 1798 } 1799 1800 /* unhalt_cpsch 1801 * Unset dqm->sched_halt and map queues back to runlist 1802 */ 1803 static int unhalt_cpsch(struct device_queue_manager *dqm) 1804 { 1805 int ret = 0; 1806 1807 dqm_lock(dqm); 1808 if (!dqm->sched_running || !dqm->sched_halt) { 1809 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n"); 1810 dqm_unlock(dqm); 1811 return 0; 1812 } 1813 dqm->sched_halt = false; 1814 if (!dqm->dev->kfd->shared_resources.enable_mes) 1815 ret = execute_queues_cpsch(dqm, 1816 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 1817 0, USE_DEFAULT_GRACE_PERIOD); 1818 else 1819 ret = add_all_kfd_queues_mes(dqm); 1820 1821 dqm_unlock(dqm); 1822 1823 return ret; 1824 } 1825 1826 static int start_cpsch(struct device_queue_manager *dqm) 1827 { 1828 struct device *dev = dqm->dev->adev->dev; 1829 int retval, num_hw_queue_slots; 1830 1831 dqm_lock(dqm); 1832 1833 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1834 retval = pm_init(&dqm->packet_mgr, dqm); 1835 if (retval) 1836 goto fail_packet_manager_init; 1837 1838 retval = set_sched_resources(dqm); 1839 if (retval) 1840 goto fail_set_sched_resources; 1841 } 1842 pr_debug("Allocating fence memory\n"); 1843 1844 /* allocate fence memory on the gart */ 1845 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 1846 &dqm->fence_mem); 1847 1848 if (retval) 1849 goto fail_allocate_vidmem; 1850 1851 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 1852 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 1853 1854 init_interrupts(dqm); 1855 1856 /* clear hang status when driver try to start the hw scheduler */ 1857 dqm->sched_running = true; 1858 1859 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1860 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 1861 KFD_DEQUEUE_WAIT_INIT, 0 /* unused */)) 1862 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); 1863 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1864 } 1865 1866 /* setup per-queue reset detection buffer */ 1867 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * 1868 dqm->dev->kfd->shared_resources.num_pipe_per_mec * 1869 NUM_XCC(dqm->dev->xcc_mask); 1870 1871 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); 1872 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); 1873 1874 if (!dqm->detect_hang_info) { 1875 retval = -ENOMEM; 1876 goto fail_detect_hang_buffer; 1877 } 1878 1879 dqm_unlock(dqm); 1880 1881 return 0; 1882 fail_detect_hang_buffer: 1883 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1884 fail_allocate_vidmem: 1885 fail_set_sched_resources: 1886 if (!dqm->dev->kfd->shared_resources.enable_mes) 1887 pm_uninit(&dqm->packet_mgr); 1888 fail_packet_manager_init: 1889 dqm_unlock(dqm); 1890 return retval; 1891 } 1892 1893 static int stop_cpsch(struct device_queue_manager *dqm) 1894 { 1895 int ret = 0; 1896 1897 dqm_lock(dqm); 1898 if (!dqm->sched_running) { 1899 dqm_unlock(dqm); 1900 return 0; 1901 } 1902 1903 if (!dqm->dev->kfd->shared_resources.enable_mes) 1904 ret = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 1905 0, USE_DEFAULT_GRACE_PERIOD, false); 1906 else 1907 ret = remove_all_kfd_queues_mes(dqm); 1908 1909 dqm->sched_running = false; 1910 1911 if (!dqm->dev->kfd->shared_resources.enable_mes) 1912 pm_release_ib(&dqm->packet_mgr); 1913 1914 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1915 if (!dqm->dev->kfd->shared_resources.enable_mes) 1916 pm_uninit(&dqm->packet_mgr); 1917 kfree(dqm->detect_hang_info); 1918 dqm->detect_hang_info = NULL; 1919 dqm_unlock(dqm); 1920 1921 return ret; 1922 } 1923 1924 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 1925 struct kernel_queue *kq, 1926 struct qcm_process_device *qpd) 1927 { 1928 dqm_lock(dqm); 1929 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1930 pr_warn("Can't create new kernel queue because %d queues were already created\n", 1931 dqm->total_queue_count); 1932 dqm_unlock(dqm); 1933 return -EPERM; 1934 } 1935 1936 /* 1937 * Unconditionally increment this counter, regardless of the queue's 1938 * type or whether the queue is active. 1939 */ 1940 dqm->total_queue_count++; 1941 pr_debug("Total of %d queues are accountable so far\n", 1942 dqm->total_queue_count); 1943 1944 list_add(&kq->list, &qpd->priv_queue_list); 1945 increment_queue_count(dqm, qpd, kq->queue); 1946 qpd->is_debug = true; 1947 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1948 USE_DEFAULT_GRACE_PERIOD); 1949 dqm_unlock(dqm); 1950 1951 return 0; 1952 } 1953 1954 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 1955 struct kernel_queue *kq, 1956 struct qcm_process_device *qpd) 1957 { 1958 dqm_lock(dqm); 1959 list_del(&kq->list); 1960 decrement_queue_count(dqm, qpd, kq->queue); 1961 qpd->is_debug = false; 1962 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1963 USE_DEFAULT_GRACE_PERIOD); 1964 /* 1965 * Unconditionally decrement this counter, regardless of the queue's 1966 * type. 1967 */ 1968 dqm->total_queue_count--; 1969 pr_debug("Total of %d queues are accountable so far\n", 1970 dqm->total_queue_count); 1971 dqm_unlock(dqm); 1972 } 1973 1974 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 1975 struct qcm_process_device *qpd, 1976 const struct kfd_criu_queue_priv_data *qd, 1977 const void *restore_mqd, const void *restore_ctl_stack) 1978 { 1979 int retval; 1980 struct mqd_manager *mqd_mgr; 1981 1982 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1983 pr_warn("Can't create new usermode queue because %d queues were already created\n", 1984 dqm->total_queue_count); 1985 retval = -EPERM; 1986 goto out; 1987 } 1988 1989 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1990 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI || 1991 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1992 dqm_lock(dqm); 1993 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 1994 dqm_unlock(dqm); 1995 if (retval) 1996 goto out; 1997 } 1998 1999 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 2000 if (retval) 2001 goto out_deallocate_sdma_queue; 2002 2003 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2004 q->properties.type)]; 2005 2006 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2007 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2008 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 2009 q->properties.tba_addr = qpd->tba_addr; 2010 q->properties.tma_addr = qpd->tma_addr; 2011 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 2012 if (!q->mqd_mem_obj) { 2013 retval = -ENOMEM; 2014 goto out_deallocate_doorbell; 2015 } 2016 2017 dqm_lock(dqm); 2018 /* 2019 * Eviction state logic: mark all queues as evicted, even ones 2020 * not currently active. Restoring inactive queues later only 2021 * updates the is_evicted flag but is a no-op otherwise. 2022 */ 2023 q->properties.is_evicted = !!qpd->evicted; 2024 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 2025 kfd_dbg_has_cwsr_workaround(q->device); 2026 2027 if (qd) 2028 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 2029 &q->properties, restore_mqd, restore_ctl_stack, 2030 qd->ctl_stack_size); 2031 else 2032 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 2033 &q->gart_mqd_addr, &q->properties); 2034 2035 list_add(&q->list, &qpd->queues_list); 2036 qpd->queue_count++; 2037 2038 if (q->properties.is_active) { 2039 increment_queue_count(dqm, qpd, q); 2040 2041 if (!dqm->dev->kfd->shared_resources.enable_mes) 2042 retval = execute_queues_cpsch(dqm, 2043 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2044 else 2045 retval = add_queue_mes(dqm, q, qpd); 2046 if (retval) 2047 goto cleanup_queue; 2048 } 2049 2050 /* 2051 * Unconditionally increment this counter, regardless of the queue's 2052 * type or whether the queue is active. 2053 */ 2054 dqm->total_queue_count++; 2055 2056 pr_debug("Total of %d queues are accountable so far\n", 2057 dqm->total_queue_count); 2058 2059 dqm_unlock(dqm); 2060 return retval; 2061 2062 cleanup_queue: 2063 qpd->queue_count--; 2064 list_del(&q->list); 2065 if (q->properties.is_active) 2066 decrement_queue_count(dqm, qpd, q); 2067 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2068 dqm_unlock(dqm); 2069 out_deallocate_doorbell: 2070 deallocate_doorbell(qpd, q); 2071 out_deallocate_sdma_queue: 2072 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2073 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2074 dqm_lock(dqm); 2075 deallocate_sdma_queue(dqm, q); 2076 dqm_unlock(dqm); 2077 } 2078 out: 2079 return retval; 2080 } 2081 2082 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 2083 uint64_t fence_value, 2084 unsigned int timeout_ms) 2085 { 2086 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 2087 struct device *dev = dqm->dev->adev->dev; 2088 uint64_t *fence_addr = dqm->fence_addr; 2089 2090 while (*fence_addr != fence_value) { 2091 /* Fatal err detected, this response won't come */ 2092 if (amdgpu_amdkfd_is_fed(dqm->dev->adev) || 2093 amdgpu_in_reset(dqm->dev->adev)) 2094 return -EIO; 2095 2096 if (time_after(jiffies, end_jiffies)) { 2097 dev_err(dev, "qcm fence wait loop timeout expired\n"); 2098 /* In HWS case, this is used to halt the driver thread 2099 * in order not to mess up CP states before doing 2100 * scandumps for FW debugging. 2101 */ 2102 while (halt_if_hws_hang) 2103 schedule(); 2104 2105 return -ETIME; 2106 } 2107 schedule(); 2108 } 2109 2110 return 0; 2111 } 2112 2113 /* dqm->lock mutex has to be locked before calling this function */ 2114 static int map_queues_cpsch(struct device_queue_manager *dqm) 2115 { 2116 struct device *dev = dqm->dev->adev->dev; 2117 int retval; 2118 2119 if (!dqm->sched_running || dqm->sched_halt) 2120 return 0; 2121 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 2122 return 0; 2123 if (dqm->active_runlist) 2124 return 0; 2125 2126 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 2127 pr_debug("%s sent runlist\n", __func__); 2128 if (retval) { 2129 dev_err(dev, "failed to execute runlist\n"); 2130 return retval; 2131 } 2132 dqm->active_runlist = true; 2133 2134 return retval; 2135 } 2136 2137 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 2138 struct qcm_process_device *qpd) 2139 { 2140 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2141 2142 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n", 2143 q->properties.queue_id, pdd->process->lead_thread->pid); 2144 2145 pdd->has_reset_queue = true; 2146 if (q->properties.is_active) { 2147 q->properties.is_active = false; 2148 decrement_queue_count(dqm, qpd, q); 2149 } 2150 } 2151 2152 static int detect_queue_hang(struct device_queue_manager *dqm) 2153 { 2154 int i; 2155 2156 /* detect should be used only in dqm locked queue reset */ 2157 if (WARN_ON(dqm->detect_hang_count > 0)) 2158 return 0; 2159 2160 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); 2161 2162 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2163 uint32_t mec, pipe, queue; 2164 int xcc_id; 2165 2166 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2167 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2168 2169 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2170 continue; 2171 2172 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); 2173 2174 for_each_inst(xcc_id, dqm->dev->xcc_mask) { 2175 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( 2176 dqm->dev->adev, pipe, queue, xcc_id); 2177 struct dqm_detect_hang_info hang_info; 2178 2179 if (!queue_addr) 2180 continue; 2181 2182 hang_info.pipe_id = pipe; 2183 hang_info.queue_id = queue; 2184 hang_info.xcc_id = xcc_id; 2185 hang_info.queue_address = queue_addr; 2186 2187 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; 2188 dqm->detect_hang_count++; 2189 } 2190 } 2191 2192 return dqm->detect_hang_count; 2193 } 2194 2195 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) 2196 { 2197 struct device_process_node *cur; 2198 struct qcm_process_device *qpd; 2199 struct queue *q; 2200 2201 list_for_each_entry(cur, &dqm->queues, list) { 2202 qpd = cur->qpd; 2203 list_for_each_entry(q, &qpd->queues_list, list) { 2204 if (queue_address == q->properties.queue_address) 2205 return q; 2206 } 2207 } 2208 2209 return NULL; 2210 } 2211 2212 static int reset_hung_queues(struct device_queue_manager *dqm) 2213 { 2214 int r = 0, reset_count = 0, i; 2215 2216 if (!dqm->detect_hang_info || dqm->is_hws_hang) 2217 return -EIO; 2218 2219 /* assume dqm locked. */ 2220 if (!detect_queue_hang(dqm)) 2221 return -ENOTRECOVERABLE; 2222 2223 for (i = 0; i < dqm->detect_hang_count; i++) { 2224 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; 2225 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); 2226 struct kfd_process_device *pdd; 2227 uint64_t queue_addr = 0; 2228 2229 if (!q) { 2230 r = -ENOTRECOVERABLE; 2231 goto reset_fail; 2232 } 2233 2234 pdd = kfd_get_process_device_data(dqm->dev, q->process); 2235 if (!pdd) { 2236 r = -ENOTRECOVERABLE; 2237 goto reset_fail; 2238 } 2239 2240 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, 2241 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id, 2242 KFD_UNMAP_LATENCY_MS); 2243 2244 /* either reset failed or we reset an unexpected queue. */ 2245 if (queue_addr != q->properties.queue_address) { 2246 r = -ENOTRECOVERABLE; 2247 goto reset_fail; 2248 } 2249 2250 set_queue_as_reset(dqm, q, &pdd->qpd); 2251 reset_count++; 2252 } 2253 2254 if (reset_count == dqm->detect_hang_count) 2255 kfd_signal_reset_event(dqm->dev); 2256 else 2257 r = -ENOTRECOVERABLE; 2258 2259 reset_fail: 2260 dqm->detect_hang_count = 0; 2261 2262 return r; 2263 } 2264 2265 static bool sdma_has_hang(struct device_queue_manager *dqm) 2266 { 2267 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2268 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2269 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2270 int i, j; 2271 2272 for (i = engine_start; i < engine_end; i++) { 2273 for (j = 0; j < num_queues_per_eng; j++) { 2274 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j)) 2275 continue; 2276 2277 return true; 2278 } 2279 } 2280 2281 return false; 2282 } 2283 2284 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, 2285 uint32_t doorbell_off) 2286 { 2287 struct device_process_node *cur; 2288 struct qcm_process_device *qpd; 2289 struct queue *q; 2290 2291 list_for_each_entry(cur, &dqm->queues, list) { 2292 qpd = cur->qpd; 2293 list_for_each_entry(q, &qpd->queues_list, list) { 2294 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || 2295 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) && 2296 q->properties.doorbell_off == doorbell_off) { 2297 set_queue_as_reset(dqm, q, qpd); 2298 return true; 2299 } 2300 } 2301 } 2302 2303 return false; 2304 } 2305 2306 static int reset_hung_queues_sdma(struct device_queue_manager *dqm) 2307 { 2308 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2309 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2310 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2311 int r = 0, i, j; 2312 2313 if (dqm->is_hws_hang) 2314 return -EIO; 2315 2316 /* Scan for hung HW queues and reset engine. */ 2317 dqm->detect_hang_count = 0; 2318 for (i = engine_start; i < engine_end; i++) { 2319 for (j = 0; j < num_queues_per_eng; j++) { 2320 uint32_t doorbell_off = 2321 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j); 2322 2323 if (!doorbell_off) 2324 continue; 2325 2326 /* Reset engine and check. */ 2327 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i, false) || 2328 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) || 2329 !set_sdma_queue_as_reset(dqm, doorbell_off)) { 2330 r = -ENOTRECOVERABLE; 2331 goto reset_fail; 2332 } 2333 2334 /* Should only expect one queue active per engine */ 2335 dqm->detect_hang_count++; 2336 break; 2337 } 2338 } 2339 2340 /* Signal process reset */ 2341 if (dqm->detect_hang_count) 2342 kfd_signal_reset_event(dqm->dev); 2343 else 2344 r = -ENOTRECOVERABLE; 2345 2346 reset_fail: 2347 dqm->detect_hang_count = 0; 2348 2349 return r; 2350 } 2351 2352 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma) 2353 { 2354 struct amdgpu_device *adev = dqm->dev->adev; 2355 2356 while (halt_if_hws_hang) 2357 schedule(); 2358 2359 if (adev->debug_disable_gpu_ring_reset) { 2360 dev_info_once(adev->dev, 2361 "%s queue hung, but ring reset disabled", 2362 is_sdma ? "sdma" : "compute"); 2363 2364 return -EPERM; 2365 } 2366 if (!amdgpu_gpu_recovery) 2367 return -ENOTRECOVERABLE; 2368 2369 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm); 2370 } 2371 2372 /* dqm->lock mutex has to be locked before calling this function 2373 * 2374 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time 2375 * for context switch latency. Lower values are used by debugger 2376 * since context switching are triggered at high frequency. 2377 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE 2378 * 2379 */ 2380 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 2381 enum kfd_unmap_queues_filter filter, 2382 uint32_t filter_param, 2383 uint32_t grace_period, 2384 bool reset) 2385 { 2386 struct device *dev = dqm->dev->adev->dev; 2387 struct mqd_manager *mqd_mgr; 2388 int retval; 2389 2390 if (!dqm->sched_running) 2391 return 0; 2392 if (!dqm->active_runlist) 2393 return 0; 2394 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2395 return -EIO; 2396 2397 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2398 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2399 KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period); 2400 if (retval) 2401 goto out; 2402 } 2403 2404 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 2405 if (retval) 2406 goto out; 2407 2408 *dqm->fence_addr = KFD_FENCE_INIT; 2409 mb(); 2410 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 2411 KFD_FENCE_COMPLETED); 2412 /* should be timed out */ 2413 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 2414 queue_preemption_timeout_ms); 2415 if (retval) { 2416 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 2417 kfd_hws_hang(dqm); 2418 goto out; 2419 } 2420 2421 /* In the current MEC firmware implementation, if compute queue 2422 * doesn't response to the preemption request in time, HIQ will 2423 * abandon the unmap request without returning any timeout error 2424 * to driver. Instead, MEC firmware will log the doorbell of the 2425 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 2426 * To make sure the queue unmap was successful, driver need to 2427 * check those fields 2428 */ 2429 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 2430 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) && 2431 reset_queues_on_hws_hang(dqm, false)) 2432 goto reset_fail; 2433 2434 /* Check for SDMA hang and attempt SDMA reset */ 2435 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true)) 2436 goto reset_fail; 2437 2438 /* We need to reset the grace period value for this device */ 2439 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2440 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2441 KFD_DEQUEUE_WAIT_RESET, 0 /* unused */)) 2442 dev_err(dev, "Failed to reset grace period\n"); 2443 } 2444 2445 pm_release_ib(&dqm->packet_mgr); 2446 dqm->active_runlist = false; 2447 out: 2448 up_read(&dqm->dev->adev->reset_domain->sem); 2449 return retval; 2450 2451 reset_fail: 2452 dqm->is_hws_hang = true; 2453 kfd_hws_hang(dqm); 2454 up_read(&dqm->dev->adev->reset_domain->sem); 2455 return -ETIME; 2456 } 2457 2458 /* only for compute queue */ 2459 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) 2460 { 2461 int retval; 2462 2463 dqm_lock(dqm); 2464 2465 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2466 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2467 2468 dqm_unlock(dqm); 2469 return retval; 2470 } 2471 2472 /* dqm->lock mutex has to be locked before calling this function */ 2473 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2474 enum kfd_unmap_queues_filter filter, 2475 uint32_t filter_param, 2476 uint32_t grace_period) 2477 { 2478 int retval; 2479 2480 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2481 return -EIO; 2482 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2483 if (!retval) 2484 retval = map_queues_cpsch(dqm); 2485 up_read(&dqm->dev->adev->reset_domain->sem); 2486 return retval; 2487 } 2488 2489 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2490 struct queue *q) 2491 { 2492 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2493 q->process); 2494 int ret = 0; 2495 2496 if (WARN_ON(!pdd)) 2497 return ret; 2498 2499 if (pdd->qpd.is_debug) 2500 return ret; 2501 2502 q->properties.is_being_destroyed = true; 2503 2504 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2505 dqm_unlock(dqm); 2506 mutex_unlock(&q->process->mutex); 2507 ret = wait_event_interruptible(dqm->destroy_wait, 2508 !q->properties.is_suspended); 2509 2510 mutex_lock(&q->process->mutex); 2511 dqm_lock(dqm); 2512 } 2513 2514 return ret; 2515 } 2516 2517 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2518 struct qcm_process_device *qpd, 2519 struct queue *q) 2520 { 2521 int retval; 2522 struct mqd_manager *mqd_mgr; 2523 uint64_t sdma_val = 0; 2524 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2525 struct device *dev = dqm->dev->adev->dev; 2526 2527 /* Get the SDMA queue stats */ 2528 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2529 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2530 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 2531 &sdma_val); 2532 if (retval) 2533 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2534 q->properties.queue_id); 2535 } 2536 2537 /* remove queue from list to prevent rescheduling after preemption */ 2538 dqm_lock(dqm); 2539 2540 retval = wait_on_destroy_queue(dqm, q); 2541 2542 if (retval) { 2543 dqm_unlock(dqm); 2544 return retval; 2545 } 2546 2547 if (qpd->is_debug) { 2548 /* 2549 * error, currently we do not allow to destroy a queue 2550 * of a currently debugged process 2551 */ 2552 retval = -EBUSY; 2553 goto failed_try_destroy_debugged_queue; 2554 2555 } 2556 2557 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2558 q->properties.type)]; 2559 2560 deallocate_doorbell(qpd, q); 2561 2562 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2563 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2564 deallocate_sdma_queue(dqm, q); 2565 pdd->sdma_past_activity_counter += sdma_val; 2566 } 2567 2568 if (q->properties.is_active) { 2569 decrement_queue_count(dqm, qpd, q); 2570 q->properties.is_active = false; 2571 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2572 retval = execute_queues_cpsch(dqm, 2573 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2574 USE_DEFAULT_GRACE_PERIOD); 2575 if (retval == -ETIME) 2576 qpd->reset_wavefronts = true; 2577 } else { 2578 retval = remove_queue_mes(dqm, q, qpd); 2579 } 2580 } 2581 list_del(&q->list); 2582 qpd->queue_count--; 2583 2584 /* 2585 * Unconditionally decrement this counter, regardless of the queue's 2586 * type 2587 */ 2588 dqm->total_queue_count--; 2589 pr_debug("Total of %d queues are accountable so far\n", 2590 dqm->total_queue_count); 2591 2592 dqm_unlock(dqm); 2593 2594 /* 2595 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 2596 * circular locking 2597 */ 2598 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 2599 qpd->pqm->process, q->device, 2600 -1, false, NULL, 0); 2601 2602 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2603 2604 return retval; 2605 2606 failed_try_destroy_debugged_queue: 2607 2608 dqm_unlock(dqm); 2609 return retval; 2610 } 2611 2612 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 2613 struct qcm_process_device *qpd, 2614 enum cache_policy default_policy, 2615 enum cache_policy alternate_policy, 2616 void __user *alternate_aperture_base, 2617 uint64_t alternate_aperture_size, 2618 u32 misc_process_properties) 2619 { 2620 bool retval = true; 2621 2622 if (!dqm->asic_ops.set_cache_memory_policy) 2623 return retval; 2624 2625 dqm_lock(dqm); 2626 2627 retval = dqm->asic_ops.set_cache_memory_policy( 2628 dqm, 2629 qpd, 2630 default_policy, 2631 alternate_policy, 2632 alternate_aperture_base, 2633 alternate_aperture_size, 2634 misc_process_properties); 2635 2636 if (retval) 2637 goto out; 2638 2639 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 2640 program_sh_mem_settings(dqm, qpd); 2641 2642 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 2643 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 2644 qpd->sh_mem_ape1_limit); 2645 2646 out: 2647 dqm_unlock(dqm); 2648 return retval; 2649 } 2650 2651 static int process_termination_nocpsch(struct device_queue_manager *dqm, 2652 struct qcm_process_device *qpd) 2653 { 2654 struct queue *q; 2655 struct device_process_node *cur, *next_dpn; 2656 int retval = 0; 2657 bool found = false; 2658 2659 dqm_lock(dqm); 2660 2661 /* Clear all user mode queues */ 2662 while (!list_empty(&qpd->queues_list)) { 2663 struct mqd_manager *mqd_mgr; 2664 int ret; 2665 2666 q = list_first_entry(&qpd->queues_list, struct queue, list); 2667 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2668 q->properties.type)]; 2669 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 2670 if (ret) 2671 retval = ret; 2672 dqm_unlock(dqm); 2673 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2674 dqm_lock(dqm); 2675 } 2676 2677 /* Unregister process */ 2678 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2679 if (qpd == cur->qpd) { 2680 list_del(&cur->list); 2681 kfree(cur); 2682 dqm->processes_count--; 2683 found = true; 2684 break; 2685 } 2686 } 2687 2688 dqm_unlock(dqm); 2689 2690 /* Outside the DQM lock because under the DQM lock we can't do 2691 * reclaim or take other locks that others hold while reclaiming. 2692 */ 2693 if (found) 2694 kfd_dec_compute_active(dqm->dev); 2695 2696 return retval; 2697 } 2698 2699 static int get_wave_state(struct device_queue_manager *dqm, 2700 struct queue *q, 2701 void __user *ctl_stack, 2702 u32 *ctl_stack_used_size, 2703 u32 *save_area_used_size) 2704 { 2705 struct mqd_manager *mqd_mgr; 2706 2707 dqm_lock(dqm); 2708 2709 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2710 2711 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 2712 q->properties.is_active || !q->device->kfd->cwsr_enabled || 2713 !mqd_mgr->get_wave_state) { 2714 dqm_unlock(dqm); 2715 return -EINVAL; 2716 } 2717 2718 dqm_unlock(dqm); 2719 2720 /* 2721 * get_wave_state is outside the dqm lock to prevent circular locking 2722 * and the queue should be protected against destruction by the process 2723 * lock. 2724 */ 2725 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 2726 ctl_stack, ctl_stack_used_size, save_area_used_size); 2727 } 2728 2729 static void get_queue_checkpoint_info(struct device_queue_manager *dqm, 2730 const struct queue *q, 2731 u32 *mqd_size, 2732 u32 *ctl_stack_size) 2733 { 2734 struct mqd_manager *mqd_mgr; 2735 enum KFD_MQD_TYPE mqd_type = 2736 get_mqd_type_from_queue_type(q->properties.type); 2737 2738 dqm_lock(dqm); 2739 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2740 *mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask); 2741 *ctl_stack_size = 0; 2742 2743 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 2744 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 2745 2746 dqm_unlock(dqm); 2747 } 2748 2749 static int checkpoint_mqd(struct device_queue_manager *dqm, 2750 const struct queue *q, 2751 void *mqd, 2752 void *ctl_stack) 2753 { 2754 struct mqd_manager *mqd_mgr; 2755 int r = 0; 2756 enum KFD_MQD_TYPE mqd_type = 2757 get_mqd_type_from_queue_type(q->properties.type); 2758 2759 dqm_lock(dqm); 2760 2761 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 2762 r = -EINVAL; 2763 goto dqm_unlock; 2764 } 2765 2766 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2767 if (!mqd_mgr->checkpoint_mqd) { 2768 r = -EOPNOTSUPP; 2769 goto dqm_unlock; 2770 } 2771 2772 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 2773 2774 dqm_unlock: 2775 dqm_unlock(dqm); 2776 return r; 2777 } 2778 2779 static int process_termination_cpsch(struct device_queue_manager *dqm, 2780 struct qcm_process_device *qpd) 2781 { 2782 int retval = 0; 2783 struct queue *q; 2784 struct device *dev = dqm->dev->adev->dev; 2785 struct kernel_queue *kq, *kq_next; 2786 struct mqd_manager *mqd_mgr; 2787 struct device_process_node *cur, *next_dpn; 2788 enum kfd_unmap_queues_filter filter = 2789 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 2790 bool found = false; 2791 2792 dqm_lock(dqm); 2793 2794 /* Clean all kernel queues */ 2795 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 2796 list_del(&kq->list); 2797 decrement_queue_count(dqm, qpd, kq->queue); 2798 qpd->is_debug = false; 2799 dqm->total_queue_count--; 2800 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 2801 } 2802 2803 /* Clear all user mode queues */ 2804 list_for_each_entry(q, &qpd->queues_list, list) { 2805 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 2806 deallocate_sdma_queue(dqm, q); 2807 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2808 deallocate_sdma_queue(dqm, q); 2809 2810 if (q->properties.is_active) { 2811 decrement_queue_count(dqm, qpd, q); 2812 2813 if (dqm->dev->kfd->shared_resources.enable_mes) { 2814 retval = remove_queue_mes(dqm, q, qpd); 2815 if (retval) 2816 dev_err(dev, "Failed to remove queue %d\n", 2817 q->properties.queue_id); 2818 } 2819 } 2820 2821 dqm->total_queue_count--; 2822 } 2823 2824 /* Unregister process */ 2825 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2826 if (qpd == cur->qpd) { 2827 list_del(&cur->list); 2828 kfree(cur); 2829 dqm->processes_count--; 2830 found = true; 2831 break; 2832 } 2833 } 2834 2835 if (!dqm->dev->kfd->shared_resources.enable_mes) 2836 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 2837 2838 if ((retval || qpd->reset_wavefronts) && 2839 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 2840 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 2841 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 2842 qpd->reset_wavefronts = false; 2843 up_read(&dqm->dev->adev->reset_domain->sem); 2844 } 2845 2846 /* Lastly, free mqd resources. 2847 * Do free_mqd() after dqm_unlock to avoid circular locking. 2848 */ 2849 while (!list_empty(&qpd->queues_list)) { 2850 q = list_first_entry(&qpd->queues_list, struct queue, list); 2851 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2852 q->properties.type)]; 2853 list_del(&q->list); 2854 qpd->queue_count--; 2855 dqm_unlock(dqm); 2856 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2857 dqm_lock(dqm); 2858 } 2859 dqm_unlock(dqm); 2860 2861 /* Outside the DQM lock because under the DQM lock we can't do 2862 * reclaim or take other locks that others hold while reclaiming. 2863 */ 2864 if (found) 2865 kfd_dec_compute_active(dqm->dev); 2866 2867 return retval; 2868 } 2869 2870 static int init_mqd_managers(struct device_queue_manager *dqm) 2871 { 2872 int i, j; 2873 struct device *dev = dqm->dev->adev->dev; 2874 struct mqd_manager *mqd_mgr; 2875 2876 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 2877 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 2878 if (!mqd_mgr) { 2879 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 2880 goto out_free; 2881 } 2882 dqm->mqd_mgrs[i] = mqd_mgr; 2883 } 2884 2885 return 0; 2886 2887 out_free: 2888 for (j = 0; j < i; j++) { 2889 kfree(dqm->mqd_mgrs[j]); 2890 dqm->mqd_mgrs[j] = NULL; 2891 } 2892 2893 return -ENOMEM; 2894 } 2895 2896 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 2897 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 2898 { 2899 int retval; 2900 struct kfd_node *dev = dqm->dev; 2901 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 2902 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 2903 get_num_all_sdma_engines(dqm) * 2904 dev->kfd->device_info.num_sdma_queues_per_engine + 2905 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 2906 NUM_XCC(dqm->dev->xcc_mask)); 2907 2908 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size, 2909 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr), 2910 (void *)&(mem_obj->cpu_ptr), false); 2911 2912 return retval; 2913 } 2914 2915 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 2916 { 2917 struct device_queue_manager *dqm; 2918 2919 pr_debug("Loading device queue manager\n"); 2920 2921 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); 2922 if (!dqm) 2923 return NULL; 2924 2925 switch (dev->adev->asic_type) { 2926 /* HWS is not available on Hawaii. */ 2927 case CHIP_HAWAII: 2928 /* HWS depends on CWSR for timely dequeue. CWSR is not 2929 * available on Tonga. 2930 * 2931 * FIXME: This argument also applies to Kaveri. 2932 */ 2933 case CHIP_TONGA: 2934 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 2935 break; 2936 default: 2937 dqm->sched_policy = sched_policy; 2938 break; 2939 } 2940 2941 dqm->dev = dev; 2942 switch (dqm->sched_policy) { 2943 case KFD_SCHED_POLICY_HWS: 2944 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 2945 /* initialize dqm for cp scheduling */ 2946 dqm->ops.create_queue = create_queue_cpsch; 2947 dqm->ops.initialize = initialize_cpsch; 2948 dqm->ops.start = start_cpsch; 2949 dqm->ops.stop = stop_cpsch; 2950 dqm->ops.halt = halt_cpsch; 2951 dqm->ops.unhalt = unhalt_cpsch; 2952 dqm->ops.destroy_queue = destroy_queue_cpsch; 2953 dqm->ops.update_queue = update_queue; 2954 dqm->ops.register_process = register_process; 2955 dqm->ops.unregister_process = unregister_process; 2956 dqm->ops.uninitialize = uninitialize; 2957 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 2958 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 2959 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2960 dqm->ops.process_termination = process_termination_cpsch; 2961 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 2962 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 2963 dqm->ops.get_wave_state = get_wave_state; 2964 dqm->ops.reset_queues = reset_queues_cpsch; 2965 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2966 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2967 break; 2968 case KFD_SCHED_POLICY_NO_HWS: 2969 /* initialize dqm for no cp scheduling */ 2970 dqm->ops.start = start_nocpsch; 2971 dqm->ops.stop = stop_nocpsch; 2972 dqm->ops.create_queue = create_queue_nocpsch; 2973 dqm->ops.destroy_queue = destroy_queue_nocpsch; 2974 dqm->ops.update_queue = update_queue; 2975 dqm->ops.register_process = register_process; 2976 dqm->ops.unregister_process = unregister_process; 2977 dqm->ops.initialize = initialize_nocpsch; 2978 dqm->ops.uninitialize = uninitialize; 2979 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2980 dqm->ops.process_termination = process_termination_nocpsch; 2981 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 2982 dqm->ops.restore_process_queues = 2983 restore_process_queues_nocpsch; 2984 dqm->ops.get_wave_state = get_wave_state; 2985 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2986 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2987 break; 2988 default: 2989 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 2990 goto out_free; 2991 } 2992 2993 switch (dev->adev->asic_type) { 2994 case CHIP_KAVERI: 2995 case CHIP_HAWAII: 2996 device_queue_manager_init_cik(&dqm->asic_ops); 2997 break; 2998 2999 case CHIP_CARRIZO: 3000 case CHIP_TONGA: 3001 case CHIP_FIJI: 3002 case CHIP_POLARIS10: 3003 case CHIP_POLARIS11: 3004 case CHIP_POLARIS12: 3005 case CHIP_VEGAM: 3006 device_queue_manager_init_vi(&dqm->asic_ops); 3007 break; 3008 3009 default: 3010 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0)) 3011 device_queue_manager_init_v12_1(&dqm->asic_ops); 3012 else if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 3013 device_queue_manager_init_v12(&dqm->asic_ops); 3014 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 3015 device_queue_manager_init_v11(&dqm->asic_ops); 3016 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 3017 device_queue_manager_init_v10(&dqm->asic_ops); 3018 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 3019 device_queue_manager_init_v9(&dqm->asic_ops); 3020 else { 3021 WARN(1, "Unexpected ASIC family %u", 3022 dev->adev->asic_type); 3023 goto out_free; 3024 } 3025 } 3026 3027 if (init_mqd_managers(dqm)) 3028 goto out_free; 3029 3030 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 3031 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 3032 goto out_free; 3033 } 3034 3035 if (!dqm->ops.initialize(dqm)) { 3036 init_waitqueue_head(&dqm->destroy_wait); 3037 return dqm; 3038 } 3039 3040 out_free: 3041 kfree(dqm); 3042 return NULL; 3043 } 3044 3045 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 3046 struct kfd_mem_obj *mqd) 3047 { 3048 WARN(!mqd, "No hiq sdma mqd trunk to free"); 3049 3050 amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); 3051 } 3052 3053 void device_queue_manager_uninit(struct device_queue_manager *dqm) 3054 { 3055 dqm->ops.stop(dqm); 3056 dqm->ops.uninitialize(dqm); 3057 if (!dqm->dev->kfd->shared_resources.enable_mes) 3058 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 3059 kfree(dqm); 3060 } 3061 3062 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) 3063 { 3064 struct kfd_process_device *pdd = NULL; 3065 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); 3066 struct device_queue_manager *dqm = knode->dqm; 3067 struct device *dev = dqm->dev->adev->dev; 3068 struct qcm_process_device *qpd; 3069 struct queue *q = NULL; 3070 int ret = 0; 3071 3072 if (!pdd) 3073 return -EINVAL; 3074 3075 dqm_lock(dqm); 3076 3077 if (pdd) { 3078 qpd = &pdd->qpd; 3079 3080 list_for_each_entry(q, &qpd->queues_list, list) { 3081 if (q->doorbell_id == doorbell_id && q->properties.is_active) { 3082 ret = suspend_all_queues_mes(dqm); 3083 if (ret) { 3084 dev_err(dev, "Suspending all queues failed"); 3085 goto out; 3086 } 3087 3088 q->properties.is_evicted = true; 3089 q->properties.is_active = false; 3090 decrement_queue_count(dqm, qpd, q); 3091 3092 ret = remove_queue_mes(dqm, q, qpd); 3093 if (ret) { 3094 dev_err(dev, "Removing bad queue failed"); 3095 goto out; 3096 } 3097 3098 ret = resume_all_queues_mes(dqm); 3099 if (ret) 3100 dev_err(dev, "Resuming all queues failed"); 3101 3102 break; 3103 } 3104 } 3105 } 3106 3107 out: 3108 dqm_unlock(dqm); 3109 kfd_unref_process(p); 3110 return ret; 3111 } 3112 3113 int kfd_evict_process_device(struct kfd_process_device *pdd) 3114 { 3115 struct device_queue_manager *dqm; 3116 struct kfd_process *p; 3117 3118 p = pdd->process; 3119 dqm = pdd->dev->dqm; 3120 3121 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 3122 3123 return dqm->ops.evict_process_queues(dqm, &pdd->qpd); 3124 } 3125 3126 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 3127 struct qcm_process_device *qpd) 3128 { 3129 int r; 3130 struct device *dev = dqm->dev->adev->dev; 3131 int updated_vmid_mask; 3132 3133 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3134 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3135 return -EINVAL; 3136 } 3137 3138 dqm_lock(dqm); 3139 3140 if (dqm->trap_debug_vmid != 0) { 3141 dev_err(dev, "Trap debug id already reserved\n"); 3142 r = -EBUSY; 3143 goto out_unlock; 3144 } 3145 3146 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3147 USE_DEFAULT_GRACE_PERIOD, false); 3148 if (r) 3149 goto out_unlock; 3150 3151 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3152 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 3153 3154 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3155 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 3156 r = set_sched_resources(dqm); 3157 if (r) 3158 goto out_unlock; 3159 3160 r = map_queues_cpsch(dqm); 3161 if (r) 3162 goto out_unlock; 3163 3164 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 3165 3166 out_unlock: 3167 dqm_unlock(dqm); 3168 return r; 3169 } 3170 3171 /* 3172 * Releases vmid for the trap debugger 3173 */ 3174 int release_debug_trap_vmid(struct device_queue_manager *dqm, 3175 struct qcm_process_device *qpd) 3176 { 3177 struct device *dev = dqm->dev->adev->dev; 3178 int r; 3179 int updated_vmid_mask; 3180 uint32_t trap_debug_vmid; 3181 3182 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3183 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3184 return -EINVAL; 3185 } 3186 3187 dqm_lock(dqm); 3188 trap_debug_vmid = dqm->trap_debug_vmid; 3189 if (dqm->trap_debug_vmid == 0) { 3190 dev_err(dev, "Trap debug id is not reserved\n"); 3191 r = -EINVAL; 3192 goto out_unlock; 3193 } 3194 3195 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3196 USE_DEFAULT_GRACE_PERIOD, false); 3197 if (r) 3198 goto out_unlock; 3199 3200 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3201 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 3202 3203 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3204 dqm->trap_debug_vmid = 0; 3205 r = set_sched_resources(dqm); 3206 if (r) 3207 goto out_unlock; 3208 3209 r = map_queues_cpsch(dqm); 3210 if (r) 3211 goto out_unlock; 3212 3213 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 3214 3215 out_unlock: 3216 dqm_unlock(dqm); 3217 return r; 3218 } 3219 3220 #define QUEUE_NOT_FOUND -1 3221 /* invalidate queue operation in array */ 3222 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 3223 { 3224 int i; 3225 3226 for (i = 0; i < num_queues; i++) 3227 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 3228 } 3229 3230 /* find queue index in array */ 3231 static int q_array_get_index(unsigned int queue_id, 3232 uint32_t num_queues, 3233 uint32_t *queue_ids) 3234 { 3235 int i; 3236 3237 for (i = 0; i < num_queues; i++) 3238 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 3239 return i; 3240 3241 return QUEUE_NOT_FOUND; 3242 } 3243 3244 struct copy_context_work_handler_workarea { 3245 struct work_struct copy_context_work; 3246 struct kfd_process *p; 3247 }; 3248 3249 static void copy_context_work_handler(struct work_struct *work) 3250 { 3251 struct copy_context_work_handler_workarea *workarea; 3252 struct mqd_manager *mqd_mgr; 3253 struct queue *q; 3254 struct mm_struct *mm; 3255 struct kfd_process *p; 3256 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 3257 int i; 3258 3259 workarea = container_of(work, 3260 struct copy_context_work_handler_workarea, 3261 copy_context_work); 3262 3263 p = workarea->p; 3264 mm = get_task_mm(p->lead_thread); 3265 3266 if (!mm) 3267 return; 3268 3269 kthread_use_mm(mm); 3270 for (i = 0; i < p->n_pdds; i++) { 3271 struct kfd_process_device *pdd = p->pdds[i]; 3272 struct device_queue_manager *dqm = pdd->dev->dqm; 3273 struct qcm_process_device *qpd = &pdd->qpd; 3274 3275 list_for_each_entry(q, &qpd->queues_list, list) { 3276 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) 3277 continue; 3278 3279 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3280 3281 /* We ignore the return value from get_wave_state 3282 * because 3283 * i) right now, it always returns 0, and 3284 * ii) if we hit an error, we would continue to the 3285 * next queue anyway. 3286 */ 3287 mqd_mgr->get_wave_state(mqd_mgr, 3288 q->mqd, 3289 &q->properties, 3290 (void __user *) q->properties.ctx_save_restore_area_address, 3291 &tmp_ctl_stack_used_size, 3292 &tmp_save_area_used_size); 3293 } 3294 } 3295 kthread_unuse_mm(mm); 3296 mmput(mm); 3297 } 3298 3299 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 3300 { 3301 size_t array_size = num_queues * sizeof(uint32_t); 3302 3303 if (!usr_queue_id_array) 3304 return NULL; 3305 3306 return memdup_user(usr_queue_id_array, array_size); 3307 } 3308 3309 int resume_queues(struct kfd_process *p, 3310 uint32_t num_queues, 3311 uint32_t *usr_queue_id_array) 3312 { 3313 uint32_t *queue_ids = NULL; 3314 int total_resumed = 0; 3315 int i; 3316 3317 if (usr_queue_id_array) { 3318 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3319 3320 if (IS_ERR(queue_ids)) 3321 return PTR_ERR(queue_ids); 3322 3323 /* mask all queues as invalid. unmask per successful request */ 3324 q_array_invalidate(num_queues, queue_ids); 3325 } 3326 3327 for (i = 0; i < p->n_pdds; i++) { 3328 struct kfd_process_device *pdd = p->pdds[i]; 3329 struct device_queue_manager *dqm = pdd->dev->dqm; 3330 struct device *dev = dqm->dev->adev->dev; 3331 struct qcm_process_device *qpd = &pdd->qpd; 3332 struct queue *q; 3333 int r, per_device_resumed = 0; 3334 3335 dqm_lock(dqm); 3336 3337 /* unmask queues that resume or already resumed as valid */ 3338 list_for_each_entry(q, &qpd->queues_list, list) { 3339 int q_idx = QUEUE_NOT_FOUND; 3340 3341 if (queue_ids) 3342 q_idx = q_array_get_index( 3343 q->properties.queue_id, 3344 num_queues, 3345 queue_ids); 3346 3347 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 3348 int err = resume_single_queue(dqm, &pdd->qpd, q); 3349 3350 if (queue_ids) { 3351 if (!err) { 3352 queue_ids[q_idx] &= 3353 ~KFD_DBG_QUEUE_INVALID_MASK; 3354 } else { 3355 queue_ids[q_idx] |= 3356 KFD_DBG_QUEUE_ERROR_MASK; 3357 break; 3358 } 3359 } 3360 3361 if (dqm->dev->kfd->shared_resources.enable_mes) { 3362 wake_up_all(&dqm->destroy_wait); 3363 if (!err) 3364 total_resumed++; 3365 } else { 3366 per_device_resumed++; 3367 } 3368 } 3369 } 3370 3371 if (!per_device_resumed) { 3372 dqm_unlock(dqm); 3373 continue; 3374 } 3375 3376 r = execute_queues_cpsch(dqm, 3377 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 3378 0, 3379 USE_DEFAULT_GRACE_PERIOD); 3380 if (r) { 3381 dev_err(dev, "Failed to resume process queues\n"); 3382 if (queue_ids) { 3383 list_for_each_entry(q, &qpd->queues_list, list) { 3384 int q_idx = q_array_get_index( 3385 q->properties.queue_id, 3386 num_queues, 3387 queue_ids); 3388 3389 /* mask queue as error on resume fail */ 3390 if (q_idx != QUEUE_NOT_FOUND) 3391 queue_ids[q_idx] |= 3392 KFD_DBG_QUEUE_ERROR_MASK; 3393 } 3394 } 3395 } else { 3396 wake_up_all(&dqm->destroy_wait); 3397 total_resumed += per_device_resumed; 3398 } 3399 3400 dqm_unlock(dqm); 3401 } 3402 3403 if (queue_ids) { 3404 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3405 num_queues * sizeof(uint32_t))) 3406 pr_err("copy_to_user failed on queue resume\n"); 3407 3408 kfree(queue_ids); 3409 } 3410 3411 return total_resumed; 3412 } 3413 3414 int suspend_queues(struct kfd_process *p, 3415 uint32_t num_queues, 3416 uint32_t grace_period, 3417 uint64_t exception_clear_mask, 3418 uint32_t *usr_queue_id_array) 3419 { 3420 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3421 int total_suspended = 0; 3422 int i; 3423 3424 if (IS_ERR(queue_ids)) 3425 return PTR_ERR(queue_ids); 3426 3427 /* mask all queues as invalid. umask on successful request */ 3428 q_array_invalidate(num_queues, queue_ids); 3429 3430 for (i = 0; i < p->n_pdds; i++) { 3431 struct kfd_process_device *pdd = p->pdds[i]; 3432 struct device_queue_manager *dqm = pdd->dev->dqm; 3433 struct device *dev = dqm->dev->adev->dev; 3434 struct qcm_process_device *qpd = &pdd->qpd; 3435 struct queue *q; 3436 int r, per_device_suspended = 0; 3437 3438 mutex_lock(&p->event_mutex); 3439 dqm_lock(dqm); 3440 3441 /* unmask queues that suspend or already suspended */ 3442 list_for_each_entry(q, &qpd->queues_list, list) { 3443 int q_idx = q_array_get_index(q->properties.queue_id, 3444 num_queues, 3445 queue_ids); 3446 3447 if (q_idx != QUEUE_NOT_FOUND) { 3448 int err = suspend_single_queue(dqm, pdd, q); 3449 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 3450 3451 if (!err) { 3452 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 3453 if (exception_clear_mask && is_mes) 3454 q->properties.exception_status &= 3455 ~exception_clear_mask; 3456 3457 if (is_mes) 3458 total_suspended++; 3459 else 3460 per_device_suspended++; 3461 } else if (err != -EBUSY) { 3462 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3463 break; 3464 } 3465 } 3466 } 3467 3468 if (!per_device_suspended) { 3469 dqm_unlock(dqm); 3470 mutex_unlock(&p->event_mutex); 3471 if (total_suspended) 3472 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3473 continue; 3474 } 3475 3476 r = execute_queues_cpsch(dqm, 3477 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3478 grace_period); 3479 3480 if (r) 3481 dev_err(dev, "Failed to suspend process queues.\n"); 3482 else 3483 total_suspended += per_device_suspended; 3484 3485 list_for_each_entry(q, &qpd->queues_list, list) { 3486 int q_idx = q_array_get_index(q->properties.queue_id, 3487 num_queues, queue_ids); 3488 3489 if (q_idx == QUEUE_NOT_FOUND) 3490 continue; 3491 3492 /* mask queue as error on suspend fail */ 3493 if (r) 3494 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3495 else if (exception_clear_mask) 3496 q->properties.exception_status &= 3497 ~exception_clear_mask; 3498 } 3499 3500 dqm_unlock(dqm); 3501 mutex_unlock(&p->event_mutex); 3502 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3503 } 3504 3505 if (total_suspended) { 3506 struct copy_context_work_handler_workarea copy_context_worker; 3507 3508 INIT_WORK_ONSTACK( 3509 ©_context_worker.copy_context_work, 3510 copy_context_work_handler); 3511 3512 copy_context_worker.p = p; 3513 3514 schedule_work(©_context_worker.copy_context_work); 3515 3516 3517 flush_work(©_context_worker.copy_context_work); 3518 destroy_work_on_stack(©_context_worker.copy_context_work); 3519 } 3520 3521 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3522 num_queues * sizeof(uint32_t))) 3523 pr_err("copy_to_user failed on queue suspend\n"); 3524 3525 kfree(queue_ids); 3526 3527 return total_suspended; 3528 } 3529 3530 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 3531 { 3532 switch (q_props->type) { 3533 case KFD_QUEUE_TYPE_COMPUTE: 3534 return q_props->format == KFD_QUEUE_FORMAT_PM4 3535 ? KFD_IOC_QUEUE_TYPE_COMPUTE 3536 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 3537 case KFD_QUEUE_TYPE_SDMA: 3538 return KFD_IOC_QUEUE_TYPE_SDMA; 3539 case KFD_QUEUE_TYPE_SDMA_XGMI: 3540 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 3541 default: 3542 WARN_ONCE(true, "queue type not recognized!"); 3543 return 0xffffffff; 3544 }; 3545 } 3546 3547 void set_queue_snapshot_entry(struct queue *q, 3548 uint64_t exception_clear_mask, 3549 struct kfd_queue_snapshot_entry *qss_entry) 3550 { 3551 qss_entry->ring_base_address = q->properties.queue_address; 3552 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 3553 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 3554 qss_entry->ctx_save_restore_address = 3555 q->properties.ctx_save_restore_area_address; 3556 qss_entry->ctx_save_restore_area_size = 3557 q->properties.ctx_save_restore_area_size; 3558 qss_entry->exception_status = q->properties.exception_status; 3559 qss_entry->queue_id = q->properties.queue_id; 3560 qss_entry->gpu_id = q->device->id; 3561 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 3562 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 3563 q->properties.exception_status &= ~exception_clear_mask; 3564 } 3565 3566 int debug_lock_and_unmap(struct device_queue_manager *dqm) 3567 { 3568 struct device *dev = dqm->dev->adev->dev; 3569 int r; 3570 3571 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3572 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3573 return -EINVAL; 3574 } 3575 3576 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3577 return 0; 3578 3579 dqm_lock(dqm); 3580 3581 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 3582 if (r) 3583 dqm_unlock(dqm); 3584 3585 return r; 3586 } 3587 3588 int debug_map_and_unlock(struct device_queue_manager *dqm) 3589 { 3590 struct device *dev = dqm->dev->adev->dev; 3591 int r; 3592 3593 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3594 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3595 return -EINVAL; 3596 } 3597 3598 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3599 return 0; 3600 3601 r = map_queues_cpsch(dqm); 3602 3603 dqm_unlock(dqm); 3604 3605 return r; 3606 } 3607 3608 int debug_refresh_runlist(struct device_queue_manager *dqm) 3609 { 3610 int r = debug_lock_and_unmap(dqm); 3611 3612 if (r) 3613 return r; 3614 3615 return debug_map_and_unlock(dqm); 3616 } 3617 3618 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, 3619 struct qcm_process_device *qpd, 3620 int doorbell_off, u32 *queue_format) 3621 { 3622 struct queue *q; 3623 bool r = false; 3624 3625 if (!queue_format) 3626 return r; 3627 3628 dqm_lock(dqm); 3629 3630 list_for_each_entry(q, &qpd->queues_list, list) { 3631 if (q->properties.doorbell_off == doorbell_off) { 3632 *queue_format = q->properties.format; 3633 r = true; 3634 goto out; 3635 } 3636 } 3637 3638 out: 3639 dqm_unlock(dqm); 3640 return r; 3641 } 3642 #if defined(CONFIG_DEBUG_FS) 3643 3644 static void seq_reg_dump(struct seq_file *m, 3645 uint32_t (*dump)[2], uint32_t n_regs) 3646 { 3647 uint32_t i, count; 3648 3649 for (i = 0, count = 0; i < n_regs; i++) { 3650 if (count == 0 || 3651 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 3652 seq_printf(m, "%s %08x: %08x", 3653 i ? "\n" : "", 3654 dump[i][0], dump[i][1]); 3655 count = 7; 3656 } else { 3657 seq_printf(m, " %08x", dump[i][1]); 3658 count--; 3659 } 3660 } 3661 3662 seq_puts(m, "\n"); 3663 } 3664 3665 int dqm_debugfs_hqds(struct seq_file *m, void *data) 3666 { 3667 struct device_queue_manager *dqm = data; 3668 uint32_t xcc_mask = dqm->dev->xcc_mask; 3669 uint32_t (*dump)[2], n_regs; 3670 int pipe, queue; 3671 int r = 0, xcc_id; 3672 uint32_t sdma_engine_start; 3673 3674 if (!dqm->sched_running) { 3675 seq_puts(m, " Device is stopped\n"); 3676 return 0; 3677 } 3678 3679 for_each_inst(xcc_id, xcc_mask) { 3680 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3681 KFD_CIK_HIQ_PIPE, 3682 KFD_CIK_HIQ_QUEUE, &dump, 3683 &n_regs, xcc_id); 3684 if (!r) { 3685 seq_printf( 3686 m, 3687 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 3688 xcc_id, 3689 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 3690 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 3691 KFD_CIK_HIQ_QUEUE); 3692 seq_reg_dump(m, dump, n_regs); 3693 3694 kfree(dump); 3695 } 3696 3697 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 3698 int pipe_offset = pipe * get_queues_per_pipe(dqm); 3699 3700 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 3701 if (!test_bit(pipe_offset + queue, 3702 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 3703 continue; 3704 3705 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3706 pipe, queue, 3707 &dump, &n_regs, 3708 xcc_id); 3709 if (r) 3710 break; 3711 3712 seq_printf(m, 3713 " Inst %d, CP Pipe %d, Queue %d\n", 3714 xcc_id, pipe, queue); 3715 seq_reg_dump(m, dump, n_regs); 3716 3717 kfree(dump); 3718 } 3719 } 3720 } 3721 3722 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 3723 for (pipe = sdma_engine_start; 3724 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 3725 pipe++) { 3726 for (queue = 0; 3727 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 3728 queue++) { 3729 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 3730 dqm->dev->adev, pipe, queue, &dump, &n_regs); 3731 if (r) 3732 break; 3733 3734 seq_printf(m, " SDMA Engine %d, RLC %d\n", 3735 pipe, queue); 3736 seq_reg_dump(m, dump, n_regs); 3737 3738 kfree(dump); 3739 } 3740 } 3741 3742 return r; 3743 } 3744 3745 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 3746 { 3747 int r = 0; 3748 3749 dqm_lock(dqm); 3750 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 3751 if (r) { 3752 dqm_unlock(dqm); 3753 return r; 3754 } 3755 dqm->active_runlist = true; 3756 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 3757 0, USE_DEFAULT_GRACE_PERIOD); 3758 dqm_unlock(dqm); 3759 3760 return r; 3761 } 3762 3763 #endif 3764