1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "amdgpu_sdma.h" 40 #include "mes_v11_api_def.h" 41 #include "kfd_debug.h" 42 43 /* Size of the per-pipe EOP queue */ 44 #define CIK_HPD_EOP_BYTES_LOG2 11 45 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 46 /* See unmap_queues_cpsch() */ 47 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff 48 49 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 50 u32 pasid, unsigned int vmid); 51 52 static int execute_queues_cpsch(struct device_queue_manager *dqm, 53 enum kfd_unmap_queues_filter filter, 54 uint32_t filter_param, 55 uint32_t grace_period); 56 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 57 enum kfd_unmap_queues_filter filter, 58 uint32_t filter_param, 59 uint32_t grace_period, 60 bool reset); 61 62 static int map_queues_cpsch(struct device_queue_manager *dqm); 63 64 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 65 struct queue *q); 66 67 static inline void deallocate_hqd(struct device_queue_manager *dqm, 68 struct queue *q); 69 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 70 static int allocate_sdma_queue(struct device_queue_manager *dqm, 71 struct queue *q, const uint32_t *restore_sdma_id); 72 73 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); 74 75 static inline 76 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 77 { 78 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 79 return KFD_MQD_TYPE_SDMA; 80 return KFD_MQD_TYPE_CP; 81 } 82 83 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 84 { 85 int i; 86 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 87 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 88 89 /* queue is available for KFD usage if bit is 1 */ 90 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 91 if (test_bit(pipe_offset + i, 92 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 93 return true; 94 return false; 95 } 96 97 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 98 { 99 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 100 AMDGPU_MAX_QUEUES); 101 } 102 103 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 104 { 105 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 106 } 107 108 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 109 { 110 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 111 } 112 113 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 114 { 115 return kfd_get_num_sdma_engines(dqm->dev) + 116 kfd_get_num_xgmi_sdma_engines(dqm->dev); 117 } 118 119 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 120 { 121 return kfd_get_num_sdma_engines(dqm->dev) * 122 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 123 } 124 125 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 126 { 127 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 129 } 130 131 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 132 { 133 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 134 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 135 136 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 137 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 138 139 /* Mask out the reserved queues */ 140 bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap, 141 dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap, 142 KFD_MAX_SDMA_QUEUES); 143 } 144 145 void program_sh_mem_settings(struct device_queue_manager *dqm, 146 struct qcm_process_device *qpd) 147 { 148 uint32_t xcc_mask = dqm->dev->xcc_mask; 149 int xcc_id; 150 151 for_each_inst(xcc_id, xcc_mask) 152 dqm->dev->kfd2kgd->program_sh_mem_settings( 153 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 154 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 155 qpd->sh_mem_bases, xcc_id); 156 } 157 158 static void kfd_hws_hang(struct device_queue_manager *dqm) 159 { 160 struct device_process_node *cur; 161 struct qcm_process_device *qpd; 162 struct queue *q; 163 164 /* Mark all device queues as reset. */ 165 list_for_each_entry(cur, &dqm->queues, list) { 166 qpd = cur->qpd; 167 list_for_each_entry(q, &qpd->queues_list, list) { 168 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 169 170 pdd->has_reset_queue = true; 171 } 172 } 173 174 /* 175 * Issue a GPU reset if HWS is unresponsive 176 */ 177 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 178 } 179 180 static int convert_to_mes_queue_type(int queue_type) 181 { 182 int mes_queue_type; 183 184 switch (queue_type) { 185 case KFD_QUEUE_TYPE_COMPUTE: 186 mes_queue_type = MES_QUEUE_TYPE_COMPUTE; 187 break; 188 case KFD_QUEUE_TYPE_SDMA: 189 mes_queue_type = MES_QUEUE_TYPE_SDMA; 190 break; 191 default: 192 WARN(1, "Invalid queue type %d", queue_type); 193 mes_queue_type = -EINVAL; 194 break; 195 } 196 197 return mes_queue_type; 198 } 199 200 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 201 struct qcm_process_device *qpd) 202 { 203 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 204 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 205 struct mes_add_queue_input queue_input; 206 int r, queue_type; 207 uint64_t wptr_addr_off; 208 209 if (!dqm->sched_running || dqm->sched_halt) 210 return 0; 211 if (!down_read_trylock(&adev->reset_domain->sem)) 212 return -EIO; 213 214 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 215 queue_input.process_id = pdd->pasid; 216 queue_input.page_table_base_addr = qpd->page_table_base; 217 queue_input.process_va_start = 0; 218 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 219 /* MES unit for quantum is 100ns */ 220 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 221 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 222 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 223 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 224 queue_input.inprocess_gang_priority = q->properties.priority; 225 queue_input.gang_global_priority_level = 226 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 227 queue_input.doorbell_offset = q->properties.doorbell_off; 228 queue_input.mqd_addr = q->gart_mqd_addr; 229 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 230 231 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 232 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 233 234 queue_input.is_kfd_process = 1; 235 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 236 queue_input.queue_size = q->properties.queue_size >> 2; 237 238 queue_input.paging = false; 239 queue_input.tba_addr = qpd->tba_addr; 240 queue_input.tma_addr = qpd->tma_addr; 241 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 242 queue_input.skip_process_ctx_clear = 243 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 244 (qpd->pqm->process->debug_trap_enabled || 245 kfd_dbg_has_ttmps_always_setup(q->device)); 246 247 queue_type = convert_to_mes_queue_type(q->properties.type); 248 if (queue_type < 0) { 249 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 250 q->properties.type); 251 up_read(&adev->reset_domain->sem); 252 return -EINVAL; 253 } 254 queue_input.queue_type = (uint32_t)queue_type; 255 256 queue_input.exclusively_scheduled = q->properties.is_gws; 257 258 amdgpu_mes_lock(&adev->mes); 259 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 260 amdgpu_mes_unlock(&adev->mes); 261 up_read(&adev->reset_domain->sem); 262 if (r) { 263 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 264 q->properties.doorbell_off); 265 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 266 kfd_hws_hang(dqm); 267 } 268 269 return r; 270 } 271 272 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 273 struct qcm_process_device *qpd) 274 { 275 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 276 int r; 277 struct mes_remove_queue_input queue_input; 278 279 if (!dqm->sched_running || dqm->sched_halt) 280 return 0; 281 if (!down_read_trylock(&adev->reset_domain->sem)) 282 return -EIO; 283 284 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 285 queue_input.doorbell_offset = q->properties.doorbell_off; 286 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 287 288 amdgpu_mes_lock(&adev->mes); 289 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 290 amdgpu_mes_unlock(&adev->mes); 291 up_read(&adev->reset_domain->sem); 292 293 if (r) { 294 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", 295 q->properties.doorbell_off); 296 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 297 kfd_hws_hang(dqm); 298 } 299 300 return r; 301 } 302 303 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) 304 { 305 struct device_process_node *cur; 306 struct device *dev = dqm->dev->adev->dev; 307 struct qcm_process_device *qpd; 308 struct queue *q; 309 int retval = 0; 310 311 list_for_each_entry(cur, &dqm->queues, list) { 312 qpd = cur->qpd; 313 list_for_each_entry(q, &qpd->queues_list, list) { 314 if (q->properties.is_active) { 315 retval = remove_queue_mes(dqm, q, qpd); 316 if (retval) { 317 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 318 __func__, 319 q->properties.queue_id, 320 dqm->dev->id); 321 return retval; 322 } 323 } 324 } 325 } 326 327 return retval; 328 } 329 330 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) 331 { 332 struct device_process_node *cur; 333 struct device *dev = dqm->dev->adev->dev; 334 struct qcm_process_device *qpd; 335 struct queue *q; 336 int retval = 0; 337 338 list_for_each_entry(cur, &dqm->queues, list) { 339 qpd = cur->qpd; 340 list_for_each_entry(q, &qpd->queues_list, list) { 341 if (!q->properties.is_active) 342 continue; 343 retval = add_queue_mes(dqm, q, qpd); 344 if (retval) { 345 dev_err(dev, "%s: Failed to add queue %d for dev %d", 346 __func__, 347 q->properties.queue_id, 348 dqm->dev->id); 349 return retval; 350 } 351 } 352 } 353 354 return retval; 355 } 356 357 static int suspend_all_queues_mes(struct device_queue_manager *dqm) 358 { 359 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 360 int r = 0; 361 362 if (!down_read_trylock(&adev->reset_domain->sem)) 363 return -EIO; 364 365 r = amdgpu_mes_suspend(adev); 366 up_read(&adev->reset_domain->sem); 367 368 if (r) { 369 dev_err(adev->dev, "failed to suspend gangs from MES\n"); 370 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 371 kfd_hws_hang(dqm); 372 } 373 374 return r; 375 } 376 377 static int resume_all_queues_mes(struct device_queue_manager *dqm) 378 { 379 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 380 int r = 0; 381 382 if (!down_read_trylock(&adev->reset_domain->sem)) 383 return -EIO; 384 385 r = amdgpu_mes_resume(adev); 386 up_read(&adev->reset_domain->sem); 387 388 if (r) { 389 dev_err(adev->dev, "failed to resume gangs from MES\n"); 390 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 391 kfd_hws_hang(dqm); 392 } 393 394 return r; 395 } 396 397 static void increment_queue_count(struct device_queue_manager *dqm, 398 struct qcm_process_device *qpd, 399 struct queue *q) 400 { 401 dqm->active_queue_count++; 402 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 403 q->properties.type == KFD_QUEUE_TYPE_DIQ) 404 dqm->active_cp_queue_count++; 405 406 if (q->properties.is_gws) { 407 dqm->gws_queue_count++; 408 qpd->mapped_gws_queue = true; 409 } 410 } 411 412 static void decrement_queue_count(struct device_queue_manager *dqm, 413 struct qcm_process_device *qpd, 414 struct queue *q) 415 { 416 dqm->active_queue_count--; 417 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 418 q->properties.type == KFD_QUEUE_TYPE_DIQ) 419 dqm->active_cp_queue_count--; 420 421 if (q->properties.is_gws) { 422 dqm->gws_queue_count--; 423 qpd->mapped_gws_queue = false; 424 } 425 } 426 427 /* 428 * Allocate a doorbell ID to this queue. 429 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 430 */ 431 static int allocate_doorbell(struct qcm_process_device *qpd, 432 struct queue *q, 433 uint32_t const *restore_id) 434 { 435 struct kfd_node *dev = qpd->dqm->dev; 436 437 if (!KFD_IS_SOC15(dev)) { 438 /* On pre-SOC15 chips we need to use the queue ID to 439 * preserve the user mode ABI. 440 */ 441 442 if (restore_id && *restore_id != q->properties.queue_id) 443 return -EINVAL; 444 445 q->doorbell_id = q->properties.queue_id; 446 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 447 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 448 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 449 * doorbell assignments based on the engine and queue id. 450 * The doobell index distance between RLC (2*i) and (2*i+1) 451 * for a SDMA engine is 512. 452 */ 453 454 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 455 456 /* 457 * q->properties.sdma_engine_id corresponds to the virtual 458 * sdma engine number. However, for doorbell allocation, 459 * we need the physical sdma engine id in order to get the 460 * correct doorbell offset. 461 */ 462 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * 463 get_num_all_sdma_engines(qpd->dqm) + 464 q->properties.sdma_engine_id] 465 + (q->properties.sdma_queue_id & 1) 466 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET 467 + (q->properties.sdma_queue_id >> 1); 468 469 if (restore_id && *restore_id != valid_id) 470 return -EINVAL; 471 q->doorbell_id = valid_id; 472 } else { 473 /* For CP queues on SOC15 */ 474 if (restore_id) { 475 /* make sure that ID is free */ 476 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 477 return -EINVAL; 478 479 q->doorbell_id = *restore_id; 480 } else { 481 /* or reserve a free doorbell ID */ 482 unsigned int found; 483 484 found = find_first_zero_bit(qpd->doorbell_bitmap, 485 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 486 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 487 pr_debug("No doorbells available"); 488 return -EBUSY; 489 } 490 set_bit(found, qpd->doorbell_bitmap); 491 q->doorbell_id = found; 492 } 493 } 494 495 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 496 qpd->proc_doorbells, 497 q->doorbell_id, 498 dev->kfd->device_info.doorbell_size); 499 return 0; 500 } 501 502 static void deallocate_doorbell(struct qcm_process_device *qpd, 503 struct queue *q) 504 { 505 unsigned int old; 506 struct kfd_node *dev = qpd->dqm->dev; 507 508 if (!KFD_IS_SOC15(dev) || 509 q->properties.type == KFD_QUEUE_TYPE_SDMA || 510 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 511 return; 512 513 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 514 WARN_ON(!old); 515 } 516 517 static void program_trap_handler_settings(struct device_queue_manager *dqm, 518 struct qcm_process_device *qpd) 519 { 520 uint32_t xcc_mask = dqm->dev->xcc_mask; 521 int xcc_id; 522 523 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 524 for_each_inst(xcc_id, xcc_mask) 525 dqm->dev->kfd2kgd->program_trap_handler_settings( 526 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 527 qpd->tma_addr, xcc_id); 528 } 529 530 static int allocate_vmid(struct device_queue_manager *dqm, 531 struct qcm_process_device *qpd, 532 struct queue *q) 533 { 534 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 535 struct device *dev = dqm->dev->adev->dev; 536 int allocated_vmid = -1, i; 537 538 for (i = dqm->dev->vm_info.first_vmid_kfd; 539 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 540 if (!dqm->vmid_pasid[i]) { 541 allocated_vmid = i; 542 break; 543 } 544 } 545 546 if (allocated_vmid < 0) { 547 dev_err(dev, "no more vmid to allocate\n"); 548 return -ENOSPC; 549 } 550 551 pr_debug("vmid allocated: %d\n", allocated_vmid); 552 553 dqm->vmid_pasid[allocated_vmid] = pdd->pasid; 554 555 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid); 556 557 qpd->vmid = allocated_vmid; 558 q->properties.vmid = allocated_vmid; 559 560 program_sh_mem_settings(dqm, qpd); 561 562 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 563 program_trap_handler_settings(dqm, qpd); 564 565 /* qpd->page_table_base is set earlier when register_process() 566 * is called, i.e. when the first queue is created. 567 */ 568 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 569 qpd->vmid, 570 qpd->page_table_base); 571 /* invalidate the VM context after pasid and vmid mapping is set up */ 572 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 573 574 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 575 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 576 qpd->sh_hidden_private_base, qpd->vmid); 577 578 return 0; 579 } 580 581 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 582 struct qcm_process_device *qpd) 583 { 584 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 585 int ret; 586 587 if (!qpd->ib_kaddr) 588 return -ENOMEM; 589 590 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 591 if (ret) 592 return ret; 593 594 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 595 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 596 pmf->release_mem_size / sizeof(uint32_t)); 597 } 598 599 static void deallocate_vmid(struct device_queue_manager *dqm, 600 struct qcm_process_device *qpd, 601 struct queue *q) 602 { 603 struct device *dev = dqm->dev->adev->dev; 604 605 /* On GFX v7, CP doesn't flush TC at dequeue */ 606 if (q->device->adev->asic_type == CHIP_HAWAII) 607 if (flush_texture_cache_nocpsch(q->device, qpd)) 608 dev_err(dev, "Failed to flush TC\n"); 609 610 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 611 612 /* Release the vmid mapping */ 613 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 614 dqm->vmid_pasid[qpd->vmid] = 0; 615 616 qpd->vmid = 0; 617 q->properties.vmid = 0; 618 } 619 620 static int create_queue_nocpsch(struct device_queue_manager *dqm, 621 struct queue *q, 622 struct qcm_process_device *qpd, 623 const struct kfd_criu_queue_priv_data *qd, 624 const void *restore_mqd, const void *restore_ctl_stack) 625 { 626 struct mqd_manager *mqd_mgr; 627 int retval; 628 629 dqm_lock(dqm); 630 631 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 632 pr_warn("Can't create new usermode queue because %d queues were already created\n", 633 dqm->total_queue_count); 634 retval = -EPERM; 635 goto out_unlock; 636 } 637 638 if (list_empty(&qpd->queues_list)) { 639 retval = allocate_vmid(dqm, qpd, q); 640 if (retval) 641 goto out_unlock; 642 } 643 q->properties.vmid = qpd->vmid; 644 /* 645 * Eviction state logic: mark all queues as evicted, even ones 646 * not currently active. Restoring inactive queues later only 647 * updates the is_evicted flag but is a no-op otherwise. 648 */ 649 q->properties.is_evicted = !!qpd->evicted; 650 651 q->properties.tba_addr = qpd->tba_addr; 652 q->properties.tma_addr = qpd->tma_addr; 653 654 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 655 q->properties.type)]; 656 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 657 retval = allocate_hqd(dqm, q); 658 if (retval) 659 goto deallocate_vmid; 660 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 661 q->pipe, q->queue); 662 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 663 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 664 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 665 if (retval) 666 goto deallocate_vmid; 667 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 668 } 669 670 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 671 if (retval) 672 goto out_deallocate_hqd; 673 674 /* Temporarily release dqm lock to avoid a circular lock dependency */ 675 dqm_unlock(dqm); 676 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 677 dqm_lock(dqm); 678 679 if (!q->mqd_mem_obj) { 680 retval = -ENOMEM; 681 goto out_deallocate_doorbell; 682 } 683 684 if (qd) 685 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 686 &q->properties, restore_mqd, restore_ctl_stack, 687 qd->ctl_stack_size); 688 else 689 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 690 &q->gart_mqd_addr, &q->properties); 691 692 if (q->properties.is_active) { 693 if (!dqm->sched_running) { 694 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 695 goto add_queue_to_list; 696 } 697 698 if (WARN(q->process->mm != current->mm, 699 "should only run in user thread")) 700 retval = -EFAULT; 701 else 702 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 703 q->queue, &q->properties, current->mm); 704 if (retval) 705 goto out_free_mqd; 706 } 707 708 add_queue_to_list: 709 list_add(&q->list, &qpd->queues_list); 710 qpd->queue_count++; 711 if (q->properties.is_active) 712 increment_queue_count(dqm, qpd, q); 713 714 /* 715 * Unconditionally increment this counter, regardless of the queue's 716 * type or whether the queue is active. 717 */ 718 dqm->total_queue_count++; 719 pr_debug("Total of %d queues are accountable so far\n", 720 dqm->total_queue_count); 721 goto out_unlock; 722 723 out_free_mqd: 724 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 725 out_deallocate_doorbell: 726 deallocate_doorbell(qpd, q); 727 out_deallocate_hqd: 728 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 729 deallocate_hqd(dqm, q); 730 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 731 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 732 deallocate_sdma_queue(dqm, q); 733 deallocate_vmid: 734 if (list_empty(&qpd->queues_list)) 735 deallocate_vmid(dqm, qpd, q); 736 out_unlock: 737 dqm_unlock(dqm); 738 return retval; 739 } 740 741 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 742 { 743 bool set; 744 int pipe, bit, i; 745 746 set = false; 747 748 for (pipe = dqm->next_pipe_to_allocate, i = 0; 749 i < get_pipes_per_mec(dqm); 750 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 751 752 if (!is_pipe_enabled(dqm, 0, pipe)) 753 continue; 754 755 if (dqm->allocated_queues[pipe] != 0) { 756 bit = ffs(dqm->allocated_queues[pipe]) - 1; 757 dqm->allocated_queues[pipe] &= ~(1 << bit); 758 q->pipe = pipe; 759 q->queue = bit; 760 set = true; 761 break; 762 } 763 } 764 765 if (!set) 766 return -EBUSY; 767 768 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 769 /* horizontal hqd allocation */ 770 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 771 772 return 0; 773 } 774 775 static inline void deallocate_hqd(struct device_queue_manager *dqm, 776 struct queue *q) 777 { 778 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 779 } 780 781 #define SQ_IND_CMD_CMD_KILL 0x00000003 782 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 783 784 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 785 { 786 int status = 0; 787 unsigned int vmid; 788 uint16_t queried_pasid; 789 union SQ_CMD_BITS reg_sq_cmd; 790 union GRBM_GFX_INDEX_BITS reg_gfx_index; 791 struct kfd_process_device *pdd; 792 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 793 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 794 uint32_t xcc_mask = dev->xcc_mask; 795 int xcc_id; 796 797 reg_sq_cmd.u32All = 0; 798 reg_gfx_index.u32All = 0; 799 800 pr_debug("Killing all process wavefronts\n"); 801 802 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 803 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 804 return -EOPNOTSUPP; 805 } 806 807 /* taking the VMID for that process on the safe way using PDD */ 808 pdd = kfd_get_process_device_data(dev, p); 809 if (!pdd) 810 return -EFAULT; 811 812 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 813 * ATC_VMID15_PASID_MAPPING 814 * to check which VMID the current process is mapped to. 815 */ 816 817 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 818 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 819 (dev->adev, vmid, &queried_pasid); 820 821 if (status && queried_pasid == pdd->pasid) { 822 pr_debug("Killing wave fronts of vmid %d and process pid %d\n", 823 vmid, p->lead_thread->pid); 824 break; 825 } 826 } 827 828 if (vmid > last_vmid_to_scan) { 829 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n", 830 p->lead_thread->pid); 831 return -EFAULT; 832 } 833 834 reg_gfx_index.bits.sh_broadcast_writes = 1; 835 reg_gfx_index.bits.se_broadcast_writes = 1; 836 reg_gfx_index.bits.instance_broadcast_writes = 1; 837 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 838 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 839 reg_sq_cmd.bits.vm_id = vmid; 840 841 for_each_inst(xcc_id, xcc_mask) 842 dev->kfd2kgd->wave_control_execute( 843 dev->adev, reg_gfx_index.u32All, 844 reg_sq_cmd.u32All, xcc_id); 845 846 return 0; 847 } 848 849 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 850 * to avoid asynchronized access 851 */ 852 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 853 struct qcm_process_device *qpd, 854 struct queue *q) 855 { 856 int retval; 857 struct mqd_manager *mqd_mgr; 858 859 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 860 q->properties.type)]; 861 862 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 863 deallocate_hqd(dqm, q); 864 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 865 deallocate_sdma_queue(dqm, q); 866 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 867 deallocate_sdma_queue(dqm, q); 868 else { 869 pr_debug("q->properties.type %d is invalid\n", 870 q->properties.type); 871 return -EINVAL; 872 } 873 dqm->total_queue_count--; 874 875 deallocate_doorbell(qpd, q); 876 877 if (!dqm->sched_running) { 878 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 879 return 0; 880 } 881 882 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 883 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 884 KFD_UNMAP_LATENCY_MS, 885 q->pipe, q->queue); 886 if (retval == -ETIME) 887 qpd->reset_wavefronts = true; 888 889 list_del(&q->list); 890 if (list_empty(&qpd->queues_list)) { 891 if (qpd->reset_wavefronts) { 892 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 893 dqm->dev); 894 /* dbgdev_wave_reset_wavefronts has to be called before 895 * deallocate_vmid(), i.e. when vmid is still in use. 896 */ 897 dbgdev_wave_reset_wavefronts(dqm->dev, 898 qpd->pqm->process); 899 qpd->reset_wavefronts = false; 900 } 901 902 deallocate_vmid(dqm, qpd, q); 903 } 904 qpd->queue_count--; 905 if (q->properties.is_active) 906 decrement_queue_count(dqm, qpd, q); 907 908 return retval; 909 } 910 911 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 912 struct qcm_process_device *qpd, 913 struct queue *q) 914 { 915 int retval; 916 uint64_t sdma_val = 0; 917 struct device *dev = dqm->dev->adev->dev; 918 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 919 struct mqd_manager *mqd_mgr = 920 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 921 922 /* Get the SDMA queue stats */ 923 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 924 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 925 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 926 &sdma_val); 927 if (retval) 928 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 929 q->properties.queue_id); 930 } 931 932 dqm_lock(dqm); 933 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 934 if (!retval) 935 pdd->sdma_past_activity_counter += sdma_val; 936 dqm_unlock(dqm); 937 938 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 939 940 return retval; 941 } 942 943 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 944 struct mqd_update_info *minfo) 945 { 946 int retval = 0; 947 struct device *dev = dqm->dev->adev->dev; 948 struct mqd_manager *mqd_mgr; 949 struct kfd_process_device *pdd; 950 bool prev_active = false; 951 952 dqm_lock(dqm); 953 pdd = kfd_get_process_device_data(q->device, q->process); 954 if (!pdd) { 955 retval = -ENODEV; 956 goto out_unlock; 957 } 958 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 959 q->properties.type)]; 960 961 /* Save previous activity state for counters */ 962 prev_active = q->properties.is_active; 963 964 /* Make sure the queue is unmapped before updating the MQD */ 965 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 966 if (!dqm->dev->kfd->shared_resources.enable_mes) 967 retval = unmap_queues_cpsch(dqm, 968 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 969 else if (prev_active) 970 retval = remove_queue_mes(dqm, q, &pdd->qpd); 971 972 /* queue is reset so inaccessable */ 973 if (pdd->has_reset_queue) { 974 retval = -EACCES; 975 goto out_unlock; 976 } 977 978 if (retval) { 979 dev_err(dev, "unmap queue failed\n"); 980 goto out_unlock; 981 } 982 } else if (prev_active && 983 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 984 q->properties.type == KFD_QUEUE_TYPE_SDMA || 985 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 986 987 if (!dqm->sched_running) { 988 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 989 goto out_unlock; 990 } 991 992 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 993 (dqm->dev->kfd->cwsr_enabled ? 994 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 995 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 996 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 997 if (retval) { 998 dev_err(dev, "destroy mqd failed\n"); 999 goto out_unlock; 1000 } 1001 } 1002 1003 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 1004 1005 /* 1006 * check active state vs. the previous state and modify 1007 * counter accordingly. map_queues_cpsch uses the 1008 * dqm->active_queue_count to determine whether a new runlist must be 1009 * uploaded. 1010 */ 1011 if (q->properties.is_active && !prev_active) { 1012 increment_queue_count(dqm, &pdd->qpd, q); 1013 } else if (!q->properties.is_active && prev_active) { 1014 decrement_queue_count(dqm, &pdd->qpd, q); 1015 } else if (q->gws && !q->properties.is_gws) { 1016 if (q->properties.is_active) { 1017 dqm->gws_queue_count++; 1018 pdd->qpd.mapped_gws_queue = true; 1019 } 1020 q->properties.is_gws = true; 1021 } else if (!q->gws && q->properties.is_gws) { 1022 if (q->properties.is_active) { 1023 dqm->gws_queue_count--; 1024 pdd->qpd.mapped_gws_queue = false; 1025 } 1026 q->properties.is_gws = false; 1027 } 1028 1029 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1030 if (!dqm->dev->kfd->shared_resources.enable_mes) 1031 retval = map_queues_cpsch(dqm); 1032 else if (q->properties.is_active) 1033 retval = add_queue_mes(dqm, q, &pdd->qpd); 1034 } else if (q->properties.is_active && 1035 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1036 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1037 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1038 if (WARN(q->process->mm != current->mm, 1039 "should only run in user thread")) 1040 retval = -EFAULT; 1041 else 1042 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 1043 q->pipe, q->queue, 1044 &q->properties, current->mm); 1045 } 1046 1047 out_unlock: 1048 dqm_unlock(dqm); 1049 return retval; 1050 } 1051 1052 /* suspend_single_queue does not lock the dqm like the 1053 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 1054 * lock the dqm before calling, and unlock after calling. 1055 * 1056 * The reason we don't lock the dqm is because this function may be 1057 * called on multiple queues in a loop, so rather than locking/unlocking 1058 * multiple times, we will just keep the dqm locked for all of the calls. 1059 */ 1060 static int suspend_single_queue(struct device_queue_manager *dqm, 1061 struct kfd_process_device *pdd, 1062 struct queue *q) 1063 { 1064 bool is_new; 1065 1066 if (q->properties.is_suspended) 1067 return 0; 1068 1069 pr_debug("Suspending process pid %d queue [%i]\n", 1070 pdd->process->lead_thread->pid, 1071 q->properties.queue_id); 1072 1073 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 1074 1075 if (is_new || q->properties.is_being_destroyed) { 1076 pr_debug("Suspend: skip %s queue id %i\n", 1077 is_new ? "new" : "destroyed", 1078 q->properties.queue_id); 1079 return -EBUSY; 1080 } 1081 1082 q->properties.is_suspended = true; 1083 if (q->properties.is_active) { 1084 if (dqm->dev->kfd->shared_resources.enable_mes) { 1085 int r = remove_queue_mes(dqm, q, &pdd->qpd); 1086 1087 if (r) 1088 return r; 1089 } 1090 1091 decrement_queue_count(dqm, &pdd->qpd, q); 1092 q->properties.is_active = false; 1093 } 1094 1095 return 0; 1096 } 1097 1098 /* resume_single_queue does not lock the dqm like the functions 1099 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1100 * lock the dqm before calling, and unlock after calling. 1101 * 1102 * The reason we don't lock the dqm is because this function may be 1103 * called on multiple queues in a loop, so rather than locking/unlocking 1104 * multiple times, we will just keep the dqm locked for all of the calls. 1105 */ 1106 static int resume_single_queue(struct device_queue_manager *dqm, 1107 struct qcm_process_device *qpd, 1108 struct queue *q) 1109 { 1110 struct kfd_process_device *pdd; 1111 1112 if (!q->properties.is_suspended) 1113 return 0; 1114 1115 pdd = qpd_to_pdd(qpd); 1116 1117 pr_debug("Restoring from suspend process pid %d queue [%i]\n", 1118 pdd->process->lead_thread->pid, 1119 q->properties.queue_id); 1120 1121 q->properties.is_suspended = false; 1122 1123 if (QUEUE_IS_ACTIVE(q->properties)) { 1124 if (dqm->dev->kfd->shared_resources.enable_mes) { 1125 int r = add_queue_mes(dqm, q, &pdd->qpd); 1126 1127 if (r) 1128 return r; 1129 } 1130 1131 q->properties.is_active = true; 1132 increment_queue_count(dqm, qpd, q); 1133 } 1134 1135 return 0; 1136 } 1137 1138 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1139 struct qcm_process_device *qpd) 1140 { 1141 struct queue *q; 1142 struct mqd_manager *mqd_mgr; 1143 struct kfd_process_device *pdd; 1144 int retval, ret = 0; 1145 1146 dqm_lock(dqm); 1147 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1148 goto out; 1149 1150 pdd = qpd_to_pdd(qpd); 1151 pr_debug_ratelimited("Evicting process pid %d queues\n", 1152 pdd->process->lead_thread->pid); 1153 1154 pdd->last_evict_timestamp = get_jiffies_64(); 1155 /* Mark all queues as evicted. Deactivate all active queues on 1156 * the qpd. 1157 */ 1158 list_for_each_entry(q, &qpd->queues_list, list) { 1159 q->properties.is_evicted = true; 1160 if (!q->properties.is_active) 1161 continue; 1162 1163 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1164 q->properties.type)]; 1165 q->properties.is_active = false; 1166 decrement_queue_count(dqm, qpd, q); 1167 1168 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1169 continue; 1170 1171 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1172 (dqm->dev->kfd->cwsr_enabled ? 1173 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1174 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1175 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1176 if (retval && !ret) 1177 /* Return the first error, but keep going to 1178 * maintain a consistent eviction state 1179 */ 1180 ret = retval; 1181 } 1182 1183 out: 1184 dqm_unlock(dqm); 1185 return ret; 1186 } 1187 1188 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1189 struct qcm_process_device *qpd) 1190 { 1191 struct queue *q; 1192 struct device *dev = dqm->dev->adev->dev; 1193 struct kfd_process_device *pdd; 1194 int retval = 0; 1195 1196 dqm_lock(dqm); 1197 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1198 goto out; 1199 1200 pdd = qpd_to_pdd(qpd); 1201 1202 /* The debugger creates processes that temporarily have not acquired 1203 * all VMs for all devices and has no VMs itself. 1204 * Skip queue eviction on process eviction. 1205 */ 1206 if (!pdd->drm_priv) 1207 goto out; 1208 1209 pr_debug_ratelimited("Evicting process pid %d queues\n", 1210 pdd->process->lead_thread->pid); 1211 1212 /* Mark all queues as evicted. Deactivate all active queues on 1213 * the qpd. 1214 */ 1215 list_for_each_entry(q, &qpd->queues_list, list) { 1216 q->properties.is_evicted = true; 1217 if (!q->properties.is_active) 1218 continue; 1219 1220 q->properties.is_active = false; 1221 decrement_queue_count(dqm, qpd, q); 1222 1223 if (dqm->dev->kfd->shared_resources.enable_mes) { 1224 int err; 1225 1226 err = remove_queue_mes(dqm, q, qpd); 1227 if (err) { 1228 dev_err(dev, "Failed to evict queue %d\n", 1229 q->properties.queue_id); 1230 retval = err; 1231 } 1232 } 1233 } 1234 pdd->last_evict_timestamp = get_jiffies_64(); 1235 if (!dqm->dev->kfd->shared_resources.enable_mes) 1236 retval = execute_queues_cpsch(dqm, 1237 qpd->is_debug ? 1238 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1239 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1240 USE_DEFAULT_GRACE_PERIOD); 1241 1242 out: 1243 dqm_unlock(dqm); 1244 return retval; 1245 } 1246 1247 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1248 struct qcm_process_device *qpd) 1249 { 1250 struct mm_struct *mm = NULL; 1251 struct queue *q; 1252 struct mqd_manager *mqd_mgr; 1253 struct kfd_process_device *pdd; 1254 uint64_t pd_base; 1255 uint64_t eviction_duration; 1256 int retval, ret = 0; 1257 1258 pdd = qpd_to_pdd(qpd); 1259 /* Retrieve PD base */ 1260 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1261 1262 dqm_lock(dqm); 1263 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1264 goto out; 1265 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1266 qpd->evicted--; 1267 goto out; 1268 } 1269 1270 pr_debug_ratelimited("Restoring process pid %d queues\n", 1271 pdd->process->lead_thread->pid); 1272 1273 /* Update PD Base in QPD */ 1274 qpd->page_table_base = pd_base; 1275 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1276 1277 if (!list_empty(&qpd->queues_list)) { 1278 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1279 dqm->dev->adev, 1280 qpd->vmid, 1281 qpd->page_table_base); 1282 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); 1283 } 1284 1285 /* Take a safe reference to the mm_struct, which may otherwise 1286 * disappear even while the kfd_process is still referenced. 1287 */ 1288 mm = get_task_mm(pdd->process->lead_thread); 1289 if (!mm) { 1290 ret = -EFAULT; 1291 goto out; 1292 } 1293 1294 /* Remove the eviction flags. Activate queues that are not 1295 * inactive for other reasons. 1296 */ 1297 list_for_each_entry(q, &qpd->queues_list, list) { 1298 q->properties.is_evicted = false; 1299 if (!QUEUE_IS_ACTIVE(q->properties)) 1300 continue; 1301 1302 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1303 q->properties.type)]; 1304 q->properties.is_active = true; 1305 increment_queue_count(dqm, qpd, q); 1306 1307 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1308 continue; 1309 1310 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1311 q->queue, &q->properties, mm); 1312 if (retval && !ret) 1313 /* Return the first error, but keep going to 1314 * maintain a consistent eviction state 1315 */ 1316 ret = retval; 1317 } 1318 qpd->evicted = 0; 1319 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1320 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1321 out: 1322 if (mm) 1323 mmput(mm); 1324 dqm_unlock(dqm); 1325 return ret; 1326 } 1327 1328 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1329 struct qcm_process_device *qpd) 1330 { 1331 struct queue *q; 1332 struct device *dev = dqm->dev->adev->dev; 1333 struct kfd_process_device *pdd; 1334 uint64_t eviction_duration; 1335 int retval = 0; 1336 1337 pdd = qpd_to_pdd(qpd); 1338 1339 dqm_lock(dqm); 1340 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1341 goto out; 1342 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1343 qpd->evicted--; 1344 goto out; 1345 } 1346 1347 /* The debugger creates processes that temporarily have not acquired 1348 * all VMs for all devices and has no VMs itself. 1349 * Skip queue restore on process restore. 1350 */ 1351 if (!pdd->drm_priv) 1352 goto vm_not_acquired; 1353 1354 pr_debug_ratelimited("Restoring process pid %d queues\n", 1355 pdd->process->lead_thread->pid); 1356 1357 /* Update PD Base in QPD */ 1358 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1359 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1360 1361 /* activate all active queues on the qpd */ 1362 list_for_each_entry(q, &qpd->queues_list, list) { 1363 q->properties.is_evicted = false; 1364 if (!QUEUE_IS_ACTIVE(q->properties)) 1365 continue; 1366 1367 q->properties.is_active = true; 1368 increment_queue_count(dqm, &pdd->qpd, q); 1369 1370 if (dqm->dev->kfd->shared_resources.enable_mes) { 1371 retval = add_queue_mes(dqm, q, qpd); 1372 if (retval) { 1373 dev_err(dev, "Failed to restore queue %d\n", 1374 q->properties.queue_id); 1375 goto out; 1376 } 1377 } 1378 } 1379 if (!dqm->dev->kfd->shared_resources.enable_mes) 1380 retval = execute_queues_cpsch(dqm, 1381 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1382 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1383 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1384 vm_not_acquired: 1385 qpd->evicted = 0; 1386 out: 1387 dqm_unlock(dqm); 1388 return retval; 1389 } 1390 1391 static int register_process(struct device_queue_manager *dqm, 1392 struct qcm_process_device *qpd) 1393 { 1394 struct device_process_node *n; 1395 struct kfd_process_device *pdd; 1396 uint64_t pd_base; 1397 int retval; 1398 1399 n = kzalloc(sizeof(*n), GFP_KERNEL); 1400 if (!n) 1401 return -ENOMEM; 1402 1403 n->qpd = qpd; 1404 1405 pdd = qpd_to_pdd(qpd); 1406 /* Retrieve PD base */ 1407 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1408 1409 dqm_lock(dqm); 1410 list_add(&n->list, &dqm->queues); 1411 1412 /* Update PD Base in QPD */ 1413 qpd->page_table_base = pd_base; 1414 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1415 1416 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1417 1418 dqm->processes_count++; 1419 1420 dqm_unlock(dqm); 1421 1422 /* Outside the DQM lock because under the DQM lock we can't do 1423 * reclaim or take other locks that others hold while reclaiming. 1424 */ 1425 kfd_inc_compute_active(dqm->dev); 1426 1427 return retval; 1428 } 1429 1430 static int unregister_process(struct device_queue_manager *dqm, 1431 struct qcm_process_device *qpd) 1432 { 1433 int retval; 1434 struct device_process_node *cur, *next; 1435 1436 pr_debug("qpd->queues_list is %s\n", 1437 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1438 1439 retval = 0; 1440 dqm_lock(dqm); 1441 1442 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1443 if (qpd == cur->qpd) { 1444 list_del(&cur->list); 1445 kfree(cur); 1446 dqm->processes_count--; 1447 goto out; 1448 } 1449 } 1450 /* qpd not found in dqm list */ 1451 retval = 1; 1452 out: 1453 dqm_unlock(dqm); 1454 1455 /* Outside the DQM lock because under the DQM lock we can't do 1456 * reclaim or take other locks that others hold while reclaiming. 1457 */ 1458 if (!retval) 1459 kfd_dec_compute_active(dqm->dev); 1460 1461 return retval; 1462 } 1463 1464 static int 1465 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1466 unsigned int vmid) 1467 { 1468 uint32_t xcc_mask = dqm->dev->xcc_mask; 1469 int xcc_id, ret; 1470 1471 for_each_inst(xcc_id, xcc_mask) { 1472 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1473 dqm->dev->adev, pasid, vmid, xcc_id); 1474 if (ret) 1475 break; 1476 } 1477 1478 return ret; 1479 } 1480 1481 static void init_interrupts(struct device_queue_manager *dqm) 1482 { 1483 uint32_t xcc_mask = dqm->dev->xcc_mask; 1484 unsigned int i, xcc_id; 1485 1486 for_each_inst(xcc_id, xcc_mask) { 1487 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1488 if (is_pipe_enabled(dqm, 0, i)) { 1489 dqm->dev->kfd2kgd->init_interrupts( 1490 dqm->dev->adev, i, xcc_id); 1491 } 1492 } 1493 } 1494 } 1495 1496 static int initialize_nocpsch(struct device_queue_manager *dqm) 1497 { 1498 int pipe, queue; 1499 1500 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1501 1502 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), 1503 sizeof(unsigned int), GFP_KERNEL); 1504 if (!dqm->allocated_queues) 1505 return -ENOMEM; 1506 1507 mutex_init(&dqm->lock_hidden); 1508 INIT_LIST_HEAD(&dqm->queues); 1509 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1510 dqm->active_cp_queue_count = 0; 1511 dqm->gws_queue_count = 0; 1512 1513 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1514 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1515 1516 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1517 if (test_bit(pipe_offset + queue, 1518 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1519 dqm->allocated_queues[pipe] |= 1 << queue; 1520 } 1521 1522 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1523 1524 init_sdma_bitmaps(dqm); 1525 1526 return 0; 1527 } 1528 1529 static void uninitialize(struct device_queue_manager *dqm) 1530 { 1531 int i; 1532 1533 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1534 1535 kfree(dqm->allocated_queues); 1536 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1537 kfree(dqm->mqd_mgrs[i]); 1538 mutex_destroy(&dqm->lock_hidden); 1539 } 1540 1541 static int start_nocpsch(struct device_queue_manager *dqm) 1542 { 1543 int r = 0; 1544 1545 pr_info("SW scheduler is used"); 1546 init_interrupts(dqm); 1547 1548 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1549 r = pm_init(&dqm->packet_mgr, dqm); 1550 if (!r) 1551 dqm->sched_running = true; 1552 1553 return r; 1554 } 1555 1556 static int stop_nocpsch(struct device_queue_manager *dqm) 1557 { 1558 dqm_lock(dqm); 1559 if (!dqm->sched_running) { 1560 dqm_unlock(dqm); 1561 return 0; 1562 } 1563 1564 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1565 pm_uninit(&dqm->packet_mgr); 1566 dqm->sched_running = false; 1567 dqm_unlock(dqm); 1568 1569 return 0; 1570 } 1571 1572 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1573 struct queue *q, const uint32_t *restore_sdma_id) 1574 { 1575 struct device *dev = dqm->dev->adev->dev; 1576 int bit; 1577 1578 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1579 if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) { 1580 dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n", 1581 get_num_sdma_queues(dqm)); 1582 return -ENOMEM; 1583 } 1584 1585 if (restore_sdma_id) { 1586 /* Re-use existing sdma_id */ 1587 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1588 dev_err(dev, "SDMA queue already in use\n"); 1589 return -EBUSY; 1590 } 1591 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1592 q->sdma_id = *restore_sdma_id; 1593 } else { 1594 /* Find first available sdma_id */ 1595 bit = find_first_bit(dqm->sdma_bitmap, 1596 get_num_sdma_queues(dqm)); 1597 clear_bit(bit, dqm->sdma_bitmap); 1598 q->sdma_id = bit; 1599 } 1600 1601 q->properties.sdma_engine_id = 1602 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1603 q->properties.sdma_queue_id = q->sdma_id / 1604 kfd_get_num_sdma_engines(dqm->dev); 1605 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1606 if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) { 1607 dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n", 1608 get_num_xgmi_sdma_queues(dqm)); 1609 return -ENOMEM; 1610 } 1611 if (restore_sdma_id) { 1612 /* Re-use existing sdma_id */ 1613 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1614 dev_err(dev, "SDMA queue already in use\n"); 1615 return -EBUSY; 1616 } 1617 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1618 q->sdma_id = *restore_sdma_id; 1619 } else { 1620 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1621 get_num_xgmi_sdma_queues(dqm)); 1622 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1623 q->sdma_id = bit; 1624 } 1625 /* sdma_engine_id is sdma id including 1626 * both PCIe-optimized SDMAs and XGMI- 1627 * optimized SDMAs. The calculation below 1628 * assumes the first N engines are always 1629 * PCIe-optimized ones 1630 */ 1631 q->properties.sdma_engine_id = 1632 kfd_get_num_sdma_engines(dqm->dev) + 1633 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1634 q->properties.sdma_queue_id = q->sdma_id / 1635 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1636 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1637 int i, num_queues, num_engines, eng_offset = 0, start_engine; 1638 bool free_bit_found = false, is_xgmi = false; 1639 1640 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { 1641 num_queues = get_num_sdma_queues(dqm); 1642 num_engines = kfd_get_num_sdma_engines(dqm->dev); 1643 q->properties.type = KFD_QUEUE_TYPE_SDMA; 1644 } else { 1645 num_queues = get_num_xgmi_sdma_queues(dqm); 1646 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); 1647 eng_offset = kfd_get_num_sdma_engines(dqm->dev); 1648 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI; 1649 is_xgmi = true; 1650 } 1651 1652 /* Scan available bit based on target engine ID. */ 1653 start_engine = q->properties.sdma_engine_id - eng_offset; 1654 for (i = start_engine; i < num_queues; i += num_engines) { 1655 1656 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) 1657 continue; 1658 1659 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); 1660 q->sdma_id = i; 1661 q->properties.sdma_queue_id = q->sdma_id / num_engines; 1662 free_bit_found = true; 1663 break; 1664 } 1665 1666 if (!free_bit_found) { 1667 dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n", 1668 q->properties.sdma_engine_id, num_queues); 1669 return -ENOMEM; 1670 } 1671 } 1672 1673 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 1674 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 1675 1676 return 0; 1677 } 1678 1679 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 1680 struct queue *q) 1681 { 1682 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1683 if (q->sdma_id >= get_num_sdma_queues(dqm)) 1684 return; 1685 set_bit(q->sdma_id, dqm->sdma_bitmap); 1686 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1687 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1688 return; 1689 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 1690 } 1691 } 1692 1693 /* 1694 * Device Queue Manager implementation for cp scheduler 1695 */ 1696 1697 static int set_sched_resources(struct device_queue_manager *dqm) 1698 { 1699 int i, mec; 1700 struct scheduling_resources res; 1701 struct device *dev = dqm->dev->adev->dev; 1702 1703 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 1704 1705 res.queue_mask = 0; 1706 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 1707 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 1708 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 1709 1710 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1711 continue; 1712 1713 /* only acquire queues from the first MEC */ 1714 if (mec > 0) 1715 continue; 1716 1717 /* This situation may be hit in the future if a new HW 1718 * generation exposes more than 64 queues. If so, the 1719 * definition of res.queue_mask needs updating 1720 */ 1721 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 1722 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 1723 break; 1724 } 1725 1726 res.queue_mask |= 1ull 1727 << amdgpu_queue_mask_bit_to_set_resource_bit( 1728 dqm->dev->adev, i); 1729 } 1730 res.gws_mask = ~0ull; 1731 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 1732 1733 pr_debug("Scheduling resources:\n" 1734 "vmid mask: 0x%8X\n" 1735 "queue mask: 0x%8llX\n", 1736 res.vmid_mask, res.queue_mask); 1737 1738 return pm_send_set_resources(&dqm->packet_mgr, &res); 1739 } 1740 1741 static int initialize_cpsch(struct device_queue_manager *dqm) 1742 { 1743 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1744 1745 mutex_init(&dqm->lock_hidden); 1746 INIT_LIST_HEAD(&dqm->queues); 1747 dqm->active_queue_count = dqm->processes_count = 0; 1748 dqm->active_cp_queue_count = 0; 1749 dqm->gws_queue_count = 0; 1750 dqm->active_runlist = false; 1751 dqm->trap_debug_vmid = 0; 1752 1753 init_sdma_bitmaps(dqm); 1754 1755 update_dqm_wait_times(dqm); 1756 return 0; 1757 } 1758 1759 /* halt_cpsch: 1760 * Unmap queues so the schedule doesn't continue remaining jobs in the queue. 1761 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch 1762 * is called. 1763 */ 1764 static int halt_cpsch(struct device_queue_manager *dqm) 1765 { 1766 int ret = 0; 1767 1768 dqm_lock(dqm); 1769 if (!dqm->sched_running) { 1770 dqm_unlock(dqm); 1771 return 0; 1772 } 1773 1774 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); 1775 1776 if (!dqm->is_hws_hang) { 1777 if (!dqm->dev->kfd->shared_resources.enable_mes) 1778 ret = unmap_queues_cpsch(dqm, 1779 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1780 USE_DEFAULT_GRACE_PERIOD, false); 1781 else 1782 ret = remove_all_kfd_queues_mes(dqm); 1783 } 1784 dqm->sched_halt = true; 1785 dqm_unlock(dqm); 1786 1787 return ret; 1788 } 1789 1790 /* unhalt_cpsch 1791 * Unset dqm->sched_halt and map queues back to runlist 1792 */ 1793 static int unhalt_cpsch(struct device_queue_manager *dqm) 1794 { 1795 int ret = 0; 1796 1797 dqm_lock(dqm); 1798 if (!dqm->sched_running || !dqm->sched_halt) { 1799 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n"); 1800 dqm_unlock(dqm); 1801 return 0; 1802 } 1803 dqm->sched_halt = false; 1804 if (!dqm->dev->kfd->shared_resources.enable_mes) 1805 ret = execute_queues_cpsch(dqm, 1806 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 1807 0, USE_DEFAULT_GRACE_PERIOD); 1808 else 1809 ret = add_all_kfd_queues_mes(dqm); 1810 1811 dqm_unlock(dqm); 1812 1813 return ret; 1814 } 1815 1816 static int start_cpsch(struct device_queue_manager *dqm) 1817 { 1818 struct device *dev = dqm->dev->adev->dev; 1819 int retval, num_hw_queue_slots; 1820 1821 retval = 0; 1822 1823 dqm_lock(dqm); 1824 1825 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1826 retval = pm_init(&dqm->packet_mgr, dqm); 1827 if (retval) 1828 goto fail_packet_manager_init; 1829 1830 retval = set_sched_resources(dqm); 1831 if (retval) 1832 goto fail_set_sched_resources; 1833 } 1834 pr_debug("Allocating fence memory\n"); 1835 1836 /* allocate fence memory on the gart */ 1837 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 1838 &dqm->fence_mem); 1839 1840 if (retval) 1841 goto fail_allocate_vidmem; 1842 1843 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 1844 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 1845 1846 init_interrupts(dqm); 1847 1848 /* clear hang status when driver try to start the hw scheduler */ 1849 dqm->sched_running = true; 1850 1851 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1852 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 1853 KFD_DEQUEUE_WAIT_INIT, 0 /* unused */)) 1854 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); 1855 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1856 } 1857 1858 /* setup per-queue reset detection buffer */ 1859 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * 1860 dqm->dev->kfd->shared_resources.num_pipe_per_mec * 1861 NUM_XCC(dqm->dev->xcc_mask); 1862 1863 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); 1864 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); 1865 1866 if (!dqm->detect_hang_info) { 1867 retval = -ENOMEM; 1868 goto fail_detect_hang_buffer; 1869 } 1870 1871 dqm_unlock(dqm); 1872 1873 return 0; 1874 fail_detect_hang_buffer: 1875 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1876 fail_allocate_vidmem: 1877 fail_set_sched_resources: 1878 if (!dqm->dev->kfd->shared_resources.enable_mes) 1879 pm_uninit(&dqm->packet_mgr); 1880 fail_packet_manager_init: 1881 dqm_unlock(dqm); 1882 return retval; 1883 } 1884 1885 static int stop_cpsch(struct device_queue_manager *dqm) 1886 { 1887 dqm_lock(dqm); 1888 if (!dqm->sched_running) { 1889 dqm_unlock(dqm); 1890 return 0; 1891 } 1892 1893 if (!dqm->dev->kfd->shared_resources.enable_mes) 1894 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 1895 else 1896 remove_all_kfd_queues_mes(dqm); 1897 1898 dqm->sched_running = false; 1899 1900 if (!dqm->dev->kfd->shared_resources.enable_mes) 1901 pm_release_ib(&dqm->packet_mgr); 1902 1903 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1904 if (!dqm->dev->kfd->shared_resources.enable_mes) 1905 pm_uninit(&dqm->packet_mgr); 1906 kfree(dqm->detect_hang_info); 1907 dqm->detect_hang_info = NULL; 1908 dqm_unlock(dqm); 1909 1910 return 0; 1911 } 1912 1913 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 1914 struct kernel_queue *kq, 1915 struct qcm_process_device *qpd) 1916 { 1917 dqm_lock(dqm); 1918 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1919 pr_warn("Can't create new kernel queue because %d queues were already created\n", 1920 dqm->total_queue_count); 1921 dqm_unlock(dqm); 1922 return -EPERM; 1923 } 1924 1925 /* 1926 * Unconditionally increment this counter, regardless of the queue's 1927 * type or whether the queue is active. 1928 */ 1929 dqm->total_queue_count++; 1930 pr_debug("Total of %d queues are accountable so far\n", 1931 dqm->total_queue_count); 1932 1933 list_add(&kq->list, &qpd->priv_queue_list); 1934 increment_queue_count(dqm, qpd, kq->queue); 1935 qpd->is_debug = true; 1936 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1937 USE_DEFAULT_GRACE_PERIOD); 1938 dqm_unlock(dqm); 1939 1940 return 0; 1941 } 1942 1943 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 1944 struct kernel_queue *kq, 1945 struct qcm_process_device *qpd) 1946 { 1947 dqm_lock(dqm); 1948 list_del(&kq->list); 1949 decrement_queue_count(dqm, qpd, kq->queue); 1950 qpd->is_debug = false; 1951 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1952 USE_DEFAULT_GRACE_PERIOD); 1953 /* 1954 * Unconditionally decrement this counter, regardless of the queue's 1955 * type. 1956 */ 1957 dqm->total_queue_count--; 1958 pr_debug("Total of %d queues are accountable so far\n", 1959 dqm->total_queue_count); 1960 dqm_unlock(dqm); 1961 } 1962 1963 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 1964 struct qcm_process_device *qpd, 1965 const struct kfd_criu_queue_priv_data *qd, 1966 const void *restore_mqd, const void *restore_ctl_stack) 1967 { 1968 int retval; 1969 struct mqd_manager *mqd_mgr; 1970 1971 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1972 pr_warn("Can't create new usermode queue because %d queues were already created\n", 1973 dqm->total_queue_count); 1974 retval = -EPERM; 1975 goto out; 1976 } 1977 1978 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1979 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI || 1980 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1981 dqm_lock(dqm); 1982 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 1983 dqm_unlock(dqm); 1984 if (retval) 1985 goto out; 1986 } 1987 1988 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 1989 if (retval) 1990 goto out_deallocate_sdma_queue; 1991 1992 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1993 q->properties.type)]; 1994 1995 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1996 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 1997 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 1998 q->properties.tba_addr = qpd->tba_addr; 1999 q->properties.tma_addr = qpd->tma_addr; 2000 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 2001 if (!q->mqd_mem_obj) { 2002 retval = -ENOMEM; 2003 goto out_deallocate_doorbell; 2004 } 2005 2006 dqm_lock(dqm); 2007 /* 2008 * Eviction state logic: mark all queues as evicted, even ones 2009 * not currently active. Restoring inactive queues later only 2010 * updates the is_evicted flag but is a no-op otherwise. 2011 */ 2012 q->properties.is_evicted = !!qpd->evicted; 2013 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 2014 kfd_dbg_has_cwsr_workaround(q->device); 2015 2016 if (qd) 2017 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 2018 &q->properties, restore_mqd, restore_ctl_stack, 2019 qd->ctl_stack_size); 2020 else 2021 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 2022 &q->gart_mqd_addr, &q->properties); 2023 2024 list_add(&q->list, &qpd->queues_list); 2025 qpd->queue_count++; 2026 2027 if (q->properties.is_active) { 2028 increment_queue_count(dqm, qpd, q); 2029 2030 if (!dqm->dev->kfd->shared_resources.enable_mes) 2031 retval = execute_queues_cpsch(dqm, 2032 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2033 else 2034 retval = add_queue_mes(dqm, q, qpd); 2035 if (retval) 2036 goto cleanup_queue; 2037 } 2038 2039 /* 2040 * Unconditionally increment this counter, regardless of the queue's 2041 * type or whether the queue is active. 2042 */ 2043 dqm->total_queue_count++; 2044 2045 pr_debug("Total of %d queues are accountable so far\n", 2046 dqm->total_queue_count); 2047 2048 dqm_unlock(dqm); 2049 return retval; 2050 2051 cleanup_queue: 2052 qpd->queue_count--; 2053 list_del(&q->list); 2054 if (q->properties.is_active) 2055 decrement_queue_count(dqm, qpd, q); 2056 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2057 dqm_unlock(dqm); 2058 out_deallocate_doorbell: 2059 deallocate_doorbell(qpd, q); 2060 out_deallocate_sdma_queue: 2061 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2062 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2063 dqm_lock(dqm); 2064 deallocate_sdma_queue(dqm, q); 2065 dqm_unlock(dqm); 2066 } 2067 out: 2068 return retval; 2069 } 2070 2071 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 2072 uint64_t fence_value, 2073 unsigned int timeout_ms) 2074 { 2075 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 2076 struct device *dev = dqm->dev->adev->dev; 2077 uint64_t *fence_addr = dqm->fence_addr; 2078 2079 while (*fence_addr != fence_value) { 2080 /* Fatal err detected, this response won't come */ 2081 if (amdgpu_amdkfd_is_fed(dqm->dev->adev)) 2082 return -EIO; 2083 2084 if (time_after(jiffies, end_jiffies)) { 2085 dev_err(dev, "qcm fence wait loop timeout expired\n"); 2086 /* In HWS case, this is used to halt the driver thread 2087 * in order not to mess up CP states before doing 2088 * scandumps for FW debugging. 2089 */ 2090 while (halt_if_hws_hang) 2091 schedule(); 2092 2093 return -ETIME; 2094 } 2095 schedule(); 2096 } 2097 2098 return 0; 2099 } 2100 2101 /* dqm->lock mutex has to be locked before calling this function */ 2102 static int map_queues_cpsch(struct device_queue_manager *dqm) 2103 { 2104 struct device *dev = dqm->dev->adev->dev; 2105 int retval; 2106 2107 if (!dqm->sched_running || dqm->sched_halt) 2108 return 0; 2109 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 2110 return 0; 2111 if (dqm->active_runlist) 2112 return 0; 2113 2114 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 2115 pr_debug("%s sent runlist\n", __func__); 2116 if (retval) { 2117 dev_err(dev, "failed to execute runlist\n"); 2118 return retval; 2119 } 2120 dqm->active_runlist = true; 2121 2122 return retval; 2123 } 2124 2125 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 2126 struct qcm_process_device *qpd) 2127 { 2128 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2129 2130 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n", 2131 q->properties.queue_id, pdd->process->lead_thread->pid); 2132 2133 pdd->has_reset_queue = true; 2134 if (q->properties.is_active) { 2135 q->properties.is_active = false; 2136 decrement_queue_count(dqm, qpd, q); 2137 } 2138 } 2139 2140 static int detect_queue_hang(struct device_queue_manager *dqm) 2141 { 2142 int i; 2143 2144 /* detect should be used only in dqm locked queue reset */ 2145 if (WARN_ON(dqm->detect_hang_count > 0)) 2146 return 0; 2147 2148 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); 2149 2150 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2151 uint32_t mec, pipe, queue; 2152 int xcc_id; 2153 2154 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2155 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2156 2157 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2158 continue; 2159 2160 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); 2161 2162 for_each_inst(xcc_id, dqm->dev->xcc_mask) { 2163 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( 2164 dqm->dev->adev, pipe, queue, xcc_id); 2165 struct dqm_detect_hang_info hang_info; 2166 2167 if (!queue_addr) 2168 continue; 2169 2170 hang_info.pipe_id = pipe; 2171 hang_info.queue_id = queue; 2172 hang_info.xcc_id = xcc_id; 2173 hang_info.queue_address = queue_addr; 2174 2175 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; 2176 dqm->detect_hang_count++; 2177 } 2178 } 2179 2180 return dqm->detect_hang_count; 2181 } 2182 2183 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) 2184 { 2185 struct device_process_node *cur; 2186 struct qcm_process_device *qpd; 2187 struct queue *q; 2188 2189 list_for_each_entry(cur, &dqm->queues, list) { 2190 qpd = cur->qpd; 2191 list_for_each_entry(q, &qpd->queues_list, list) { 2192 if (queue_address == q->properties.queue_address) 2193 return q; 2194 } 2195 } 2196 2197 return NULL; 2198 } 2199 2200 static int reset_hung_queues(struct device_queue_manager *dqm) 2201 { 2202 int r = 0, reset_count = 0, i; 2203 2204 if (!dqm->detect_hang_info || dqm->is_hws_hang) 2205 return -EIO; 2206 2207 /* assume dqm locked. */ 2208 if (!detect_queue_hang(dqm)) 2209 return -ENOTRECOVERABLE; 2210 2211 for (i = 0; i < dqm->detect_hang_count; i++) { 2212 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; 2213 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); 2214 struct kfd_process_device *pdd; 2215 uint64_t queue_addr = 0; 2216 2217 if (!q) { 2218 r = -ENOTRECOVERABLE; 2219 goto reset_fail; 2220 } 2221 2222 pdd = kfd_get_process_device_data(dqm->dev, q->process); 2223 if (!pdd) { 2224 r = -ENOTRECOVERABLE; 2225 goto reset_fail; 2226 } 2227 2228 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, 2229 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id, 2230 KFD_UNMAP_LATENCY_MS); 2231 2232 /* either reset failed or we reset an unexpected queue. */ 2233 if (queue_addr != q->properties.queue_address) { 2234 r = -ENOTRECOVERABLE; 2235 goto reset_fail; 2236 } 2237 2238 set_queue_as_reset(dqm, q, &pdd->qpd); 2239 reset_count++; 2240 } 2241 2242 if (reset_count == dqm->detect_hang_count) 2243 kfd_signal_reset_event(dqm->dev); 2244 else 2245 r = -ENOTRECOVERABLE; 2246 2247 reset_fail: 2248 dqm->detect_hang_count = 0; 2249 2250 return r; 2251 } 2252 2253 static bool sdma_has_hang(struct device_queue_manager *dqm) 2254 { 2255 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2256 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2257 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2258 int i, j; 2259 2260 for (i = engine_start; i < engine_end; i++) { 2261 for (j = 0; j < num_queues_per_eng; j++) { 2262 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j)) 2263 continue; 2264 2265 return true; 2266 } 2267 } 2268 2269 return false; 2270 } 2271 2272 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, 2273 uint32_t doorbell_off) 2274 { 2275 struct device_process_node *cur; 2276 struct qcm_process_device *qpd; 2277 struct queue *q; 2278 2279 list_for_each_entry(cur, &dqm->queues, list) { 2280 qpd = cur->qpd; 2281 list_for_each_entry(q, &qpd->queues_list, list) { 2282 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || 2283 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) && 2284 q->properties.doorbell_off == doorbell_off) { 2285 set_queue_as_reset(dqm, q, qpd); 2286 return true; 2287 } 2288 } 2289 } 2290 2291 return false; 2292 } 2293 2294 static int reset_hung_queues_sdma(struct device_queue_manager *dqm) 2295 { 2296 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2297 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2298 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2299 int r = 0, i, j; 2300 2301 if (dqm->is_hws_hang) 2302 return -EIO; 2303 2304 /* Scan for hung HW queues and reset engine. */ 2305 dqm->detect_hang_count = 0; 2306 for (i = engine_start; i < engine_end; i++) { 2307 for (j = 0; j < num_queues_per_eng; j++) { 2308 uint32_t doorbell_off = 2309 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j); 2310 2311 if (!doorbell_off) 2312 continue; 2313 2314 /* Reset engine and check. */ 2315 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i, false) || 2316 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) || 2317 !set_sdma_queue_as_reset(dqm, doorbell_off)) { 2318 r = -ENOTRECOVERABLE; 2319 goto reset_fail; 2320 } 2321 2322 /* Should only expect one queue active per engine */ 2323 dqm->detect_hang_count++; 2324 break; 2325 } 2326 } 2327 2328 /* Signal process reset */ 2329 if (dqm->detect_hang_count) 2330 kfd_signal_reset_event(dqm->dev); 2331 else 2332 r = -ENOTRECOVERABLE; 2333 2334 reset_fail: 2335 dqm->detect_hang_count = 0; 2336 2337 return r; 2338 } 2339 2340 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma) 2341 { 2342 struct amdgpu_device *adev = dqm->dev->adev; 2343 2344 while (halt_if_hws_hang) 2345 schedule(); 2346 2347 if (adev->debug_disable_gpu_ring_reset) { 2348 dev_info_once(adev->dev, 2349 "%s queue hung, but ring reset disabled", 2350 is_sdma ? "sdma" : "compute"); 2351 2352 return -EPERM; 2353 } 2354 if (!amdgpu_gpu_recovery) 2355 return -ENOTRECOVERABLE; 2356 2357 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm); 2358 } 2359 2360 /* dqm->lock mutex has to be locked before calling this function 2361 * 2362 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time 2363 * for context switch latency. Lower values are used by debugger 2364 * since context switching are triggered at high frequency. 2365 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE 2366 * 2367 */ 2368 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 2369 enum kfd_unmap_queues_filter filter, 2370 uint32_t filter_param, 2371 uint32_t grace_period, 2372 bool reset) 2373 { 2374 struct device *dev = dqm->dev->adev->dev; 2375 struct mqd_manager *mqd_mgr; 2376 int retval; 2377 2378 if (!dqm->sched_running) 2379 return 0; 2380 if (!dqm->active_runlist) 2381 return 0; 2382 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2383 return -EIO; 2384 2385 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2386 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2387 KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period); 2388 if (retval) 2389 goto out; 2390 } 2391 2392 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 2393 if (retval) 2394 goto out; 2395 2396 *dqm->fence_addr = KFD_FENCE_INIT; 2397 mb(); 2398 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 2399 KFD_FENCE_COMPLETED); 2400 /* should be timed out */ 2401 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 2402 queue_preemption_timeout_ms); 2403 if (retval) { 2404 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 2405 kfd_hws_hang(dqm); 2406 goto out; 2407 } 2408 2409 /* In the current MEC firmware implementation, if compute queue 2410 * doesn't response to the preemption request in time, HIQ will 2411 * abandon the unmap request without returning any timeout error 2412 * to driver. Instead, MEC firmware will log the doorbell of the 2413 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 2414 * To make sure the queue unmap was successful, driver need to 2415 * check those fields 2416 */ 2417 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 2418 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) && 2419 reset_queues_on_hws_hang(dqm, false)) 2420 goto reset_fail; 2421 2422 /* Check for SDMA hang and attempt SDMA reset */ 2423 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true)) 2424 goto reset_fail; 2425 2426 /* We need to reset the grace period value for this device */ 2427 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2428 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2429 KFD_DEQUEUE_WAIT_RESET, 0 /* unused */)) 2430 dev_err(dev, "Failed to reset grace period\n"); 2431 } 2432 2433 pm_release_ib(&dqm->packet_mgr); 2434 dqm->active_runlist = false; 2435 out: 2436 up_read(&dqm->dev->adev->reset_domain->sem); 2437 return retval; 2438 2439 reset_fail: 2440 dqm->is_hws_hang = true; 2441 kfd_hws_hang(dqm); 2442 up_read(&dqm->dev->adev->reset_domain->sem); 2443 return -ETIME; 2444 } 2445 2446 /* only for compute queue */ 2447 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) 2448 { 2449 int retval; 2450 2451 dqm_lock(dqm); 2452 2453 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2454 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2455 2456 dqm_unlock(dqm); 2457 return retval; 2458 } 2459 2460 /* dqm->lock mutex has to be locked before calling this function */ 2461 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2462 enum kfd_unmap_queues_filter filter, 2463 uint32_t filter_param, 2464 uint32_t grace_period) 2465 { 2466 int retval; 2467 2468 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2469 return -EIO; 2470 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2471 if (!retval) 2472 retval = map_queues_cpsch(dqm); 2473 up_read(&dqm->dev->adev->reset_domain->sem); 2474 return retval; 2475 } 2476 2477 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2478 struct queue *q) 2479 { 2480 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2481 q->process); 2482 int ret = 0; 2483 2484 if (WARN_ON(!pdd)) 2485 return ret; 2486 2487 if (pdd->qpd.is_debug) 2488 return ret; 2489 2490 q->properties.is_being_destroyed = true; 2491 2492 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2493 dqm_unlock(dqm); 2494 mutex_unlock(&q->process->mutex); 2495 ret = wait_event_interruptible(dqm->destroy_wait, 2496 !q->properties.is_suspended); 2497 2498 mutex_lock(&q->process->mutex); 2499 dqm_lock(dqm); 2500 } 2501 2502 return ret; 2503 } 2504 2505 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2506 struct qcm_process_device *qpd, 2507 struct queue *q) 2508 { 2509 int retval; 2510 struct mqd_manager *mqd_mgr; 2511 uint64_t sdma_val = 0; 2512 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2513 struct device *dev = dqm->dev->adev->dev; 2514 2515 /* Get the SDMA queue stats */ 2516 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2517 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2518 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 2519 &sdma_val); 2520 if (retval) 2521 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2522 q->properties.queue_id); 2523 } 2524 2525 /* remove queue from list to prevent rescheduling after preemption */ 2526 dqm_lock(dqm); 2527 2528 retval = wait_on_destroy_queue(dqm, q); 2529 2530 if (retval) { 2531 dqm_unlock(dqm); 2532 return retval; 2533 } 2534 2535 if (qpd->is_debug) { 2536 /* 2537 * error, currently we do not allow to destroy a queue 2538 * of a currently debugged process 2539 */ 2540 retval = -EBUSY; 2541 goto failed_try_destroy_debugged_queue; 2542 2543 } 2544 2545 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2546 q->properties.type)]; 2547 2548 deallocate_doorbell(qpd, q); 2549 2550 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2551 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2552 deallocate_sdma_queue(dqm, q); 2553 pdd->sdma_past_activity_counter += sdma_val; 2554 } 2555 2556 if (q->properties.is_active) { 2557 decrement_queue_count(dqm, qpd, q); 2558 q->properties.is_active = false; 2559 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2560 retval = execute_queues_cpsch(dqm, 2561 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2562 USE_DEFAULT_GRACE_PERIOD); 2563 if (retval == -ETIME) 2564 qpd->reset_wavefronts = true; 2565 } else { 2566 retval = remove_queue_mes(dqm, q, qpd); 2567 } 2568 } 2569 list_del(&q->list); 2570 qpd->queue_count--; 2571 2572 /* 2573 * Unconditionally decrement this counter, regardless of the queue's 2574 * type 2575 */ 2576 dqm->total_queue_count--; 2577 pr_debug("Total of %d queues are accountable so far\n", 2578 dqm->total_queue_count); 2579 2580 dqm_unlock(dqm); 2581 2582 /* 2583 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 2584 * circular locking 2585 */ 2586 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 2587 qpd->pqm->process, q->device, 2588 -1, false, NULL, 0); 2589 2590 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2591 2592 return retval; 2593 2594 failed_try_destroy_debugged_queue: 2595 2596 dqm_unlock(dqm); 2597 return retval; 2598 } 2599 2600 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 2601 struct qcm_process_device *qpd, 2602 enum cache_policy default_policy, 2603 enum cache_policy alternate_policy, 2604 void __user *alternate_aperture_base, 2605 uint64_t alternate_aperture_size, 2606 u32 misc_process_properties) 2607 { 2608 bool retval = true; 2609 2610 if (!dqm->asic_ops.set_cache_memory_policy) 2611 return retval; 2612 2613 dqm_lock(dqm); 2614 2615 retval = dqm->asic_ops.set_cache_memory_policy( 2616 dqm, 2617 qpd, 2618 default_policy, 2619 alternate_policy, 2620 alternate_aperture_base, 2621 alternate_aperture_size, 2622 misc_process_properties); 2623 2624 if (retval) 2625 goto out; 2626 2627 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 2628 program_sh_mem_settings(dqm, qpd); 2629 2630 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 2631 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 2632 qpd->sh_mem_ape1_limit); 2633 2634 out: 2635 dqm_unlock(dqm); 2636 return retval; 2637 } 2638 2639 static int process_termination_nocpsch(struct device_queue_manager *dqm, 2640 struct qcm_process_device *qpd) 2641 { 2642 struct queue *q; 2643 struct device_process_node *cur, *next_dpn; 2644 int retval = 0; 2645 bool found = false; 2646 2647 dqm_lock(dqm); 2648 2649 /* Clear all user mode queues */ 2650 while (!list_empty(&qpd->queues_list)) { 2651 struct mqd_manager *mqd_mgr; 2652 int ret; 2653 2654 q = list_first_entry(&qpd->queues_list, struct queue, list); 2655 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2656 q->properties.type)]; 2657 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 2658 if (ret) 2659 retval = ret; 2660 dqm_unlock(dqm); 2661 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2662 dqm_lock(dqm); 2663 } 2664 2665 /* Unregister process */ 2666 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2667 if (qpd == cur->qpd) { 2668 list_del(&cur->list); 2669 kfree(cur); 2670 dqm->processes_count--; 2671 found = true; 2672 break; 2673 } 2674 } 2675 2676 dqm_unlock(dqm); 2677 2678 /* Outside the DQM lock because under the DQM lock we can't do 2679 * reclaim or take other locks that others hold while reclaiming. 2680 */ 2681 if (found) 2682 kfd_dec_compute_active(dqm->dev); 2683 2684 return retval; 2685 } 2686 2687 static int get_wave_state(struct device_queue_manager *dqm, 2688 struct queue *q, 2689 void __user *ctl_stack, 2690 u32 *ctl_stack_used_size, 2691 u32 *save_area_used_size) 2692 { 2693 struct mqd_manager *mqd_mgr; 2694 2695 dqm_lock(dqm); 2696 2697 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2698 2699 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 2700 q->properties.is_active || !q->device->kfd->cwsr_enabled || 2701 !mqd_mgr->get_wave_state) { 2702 dqm_unlock(dqm); 2703 return -EINVAL; 2704 } 2705 2706 dqm_unlock(dqm); 2707 2708 /* 2709 * get_wave_state is outside the dqm lock to prevent circular locking 2710 * and the queue should be protected against destruction by the process 2711 * lock. 2712 */ 2713 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 2714 ctl_stack, ctl_stack_used_size, save_area_used_size); 2715 } 2716 2717 static void get_queue_checkpoint_info(struct device_queue_manager *dqm, 2718 const struct queue *q, 2719 u32 *mqd_size, 2720 u32 *ctl_stack_size) 2721 { 2722 struct mqd_manager *mqd_mgr; 2723 enum KFD_MQD_TYPE mqd_type = 2724 get_mqd_type_from_queue_type(q->properties.type); 2725 2726 dqm_lock(dqm); 2727 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2728 *mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask); 2729 *ctl_stack_size = 0; 2730 2731 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 2732 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 2733 2734 dqm_unlock(dqm); 2735 } 2736 2737 static int checkpoint_mqd(struct device_queue_manager *dqm, 2738 const struct queue *q, 2739 void *mqd, 2740 void *ctl_stack) 2741 { 2742 struct mqd_manager *mqd_mgr; 2743 int r = 0; 2744 enum KFD_MQD_TYPE mqd_type = 2745 get_mqd_type_from_queue_type(q->properties.type); 2746 2747 dqm_lock(dqm); 2748 2749 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 2750 r = -EINVAL; 2751 goto dqm_unlock; 2752 } 2753 2754 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2755 if (!mqd_mgr->checkpoint_mqd) { 2756 r = -EOPNOTSUPP; 2757 goto dqm_unlock; 2758 } 2759 2760 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 2761 2762 dqm_unlock: 2763 dqm_unlock(dqm); 2764 return r; 2765 } 2766 2767 static int process_termination_cpsch(struct device_queue_manager *dqm, 2768 struct qcm_process_device *qpd) 2769 { 2770 int retval; 2771 struct queue *q; 2772 struct device *dev = dqm->dev->adev->dev; 2773 struct kernel_queue *kq, *kq_next; 2774 struct mqd_manager *mqd_mgr; 2775 struct device_process_node *cur, *next_dpn; 2776 enum kfd_unmap_queues_filter filter = 2777 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 2778 bool found = false; 2779 2780 retval = 0; 2781 2782 dqm_lock(dqm); 2783 2784 /* Clean all kernel queues */ 2785 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 2786 list_del(&kq->list); 2787 decrement_queue_count(dqm, qpd, kq->queue); 2788 qpd->is_debug = false; 2789 dqm->total_queue_count--; 2790 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 2791 } 2792 2793 /* Clear all user mode queues */ 2794 list_for_each_entry(q, &qpd->queues_list, list) { 2795 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 2796 deallocate_sdma_queue(dqm, q); 2797 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2798 deallocate_sdma_queue(dqm, q); 2799 2800 if (q->properties.is_active) { 2801 decrement_queue_count(dqm, qpd, q); 2802 2803 if (dqm->dev->kfd->shared_resources.enable_mes) { 2804 retval = remove_queue_mes(dqm, q, qpd); 2805 if (retval) 2806 dev_err(dev, "Failed to remove queue %d\n", 2807 q->properties.queue_id); 2808 } 2809 } 2810 2811 dqm->total_queue_count--; 2812 } 2813 2814 /* Unregister process */ 2815 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2816 if (qpd == cur->qpd) { 2817 list_del(&cur->list); 2818 kfree(cur); 2819 dqm->processes_count--; 2820 found = true; 2821 break; 2822 } 2823 } 2824 2825 if (!dqm->dev->kfd->shared_resources.enable_mes) 2826 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 2827 2828 if ((retval || qpd->reset_wavefronts) && 2829 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 2830 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 2831 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 2832 qpd->reset_wavefronts = false; 2833 up_read(&dqm->dev->adev->reset_domain->sem); 2834 } 2835 2836 /* Lastly, free mqd resources. 2837 * Do free_mqd() after dqm_unlock to avoid circular locking. 2838 */ 2839 while (!list_empty(&qpd->queues_list)) { 2840 q = list_first_entry(&qpd->queues_list, struct queue, list); 2841 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2842 q->properties.type)]; 2843 list_del(&q->list); 2844 qpd->queue_count--; 2845 dqm_unlock(dqm); 2846 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2847 dqm_lock(dqm); 2848 } 2849 dqm_unlock(dqm); 2850 2851 /* Outside the DQM lock because under the DQM lock we can't do 2852 * reclaim or take other locks that others hold while reclaiming. 2853 */ 2854 if (found) 2855 kfd_dec_compute_active(dqm->dev); 2856 2857 return retval; 2858 } 2859 2860 static int init_mqd_managers(struct device_queue_manager *dqm) 2861 { 2862 int i, j; 2863 struct device *dev = dqm->dev->adev->dev; 2864 struct mqd_manager *mqd_mgr; 2865 2866 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 2867 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 2868 if (!mqd_mgr) { 2869 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 2870 goto out_free; 2871 } 2872 dqm->mqd_mgrs[i] = mqd_mgr; 2873 } 2874 2875 return 0; 2876 2877 out_free: 2878 for (j = 0; j < i; j++) { 2879 kfree(dqm->mqd_mgrs[j]); 2880 dqm->mqd_mgrs[j] = NULL; 2881 } 2882 2883 return -ENOMEM; 2884 } 2885 2886 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 2887 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 2888 { 2889 int retval; 2890 struct kfd_node *dev = dqm->dev; 2891 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 2892 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 2893 get_num_all_sdma_engines(dqm) * 2894 dev->kfd->device_info.num_sdma_queues_per_engine + 2895 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 2896 NUM_XCC(dqm->dev->xcc_mask)); 2897 2898 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size, 2899 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr), 2900 (void *)&(mem_obj->cpu_ptr), false); 2901 2902 return retval; 2903 } 2904 2905 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 2906 { 2907 struct device_queue_manager *dqm; 2908 2909 pr_debug("Loading device queue manager\n"); 2910 2911 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); 2912 if (!dqm) 2913 return NULL; 2914 2915 switch (dev->adev->asic_type) { 2916 /* HWS is not available on Hawaii. */ 2917 case CHIP_HAWAII: 2918 /* HWS depends on CWSR for timely dequeue. CWSR is not 2919 * available on Tonga. 2920 * 2921 * FIXME: This argument also applies to Kaveri. 2922 */ 2923 case CHIP_TONGA: 2924 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 2925 break; 2926 default: 2927 dqm->sched_policy = sched_policy; 2928 break; 2929 } 2930 2931 dqm->dev = dev; 2932 switch (dqm->sched_policy) { 2933 case KFD_SCHED_POLICY_HWS: 2934 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 2935 /* initialize dqm for cp scheduling */ 2936 dqm->ops.create_queue = create_queue_cpsch; 2937 dqm->ops.initialize = initialize_cpsch; 2938 dqm->ops.start = start_cpsch; 2939 dqm->ops.stop = stop_cpsch; 2940 dqm->ops.halt = halt_cpsch; 2941 dqm->ops.unhalt = unhalt_cpsch; 2942 dqm->ops.destroy_queue = destroy_queue_cpsch; 2943 dqm->ops.update_queue = update_queue; 2944 dqm->ops.register_process = register_process; 2945 dqm->ops.unregister_process = unregister_process; 2946 dqm->ops.uninitialize = uninitialize; 2947 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 2948 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 2949 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2950 dqm->ops.process_termination = process_termination_cpsch; 2951 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 2952 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 2953 dqm->ops.get_wave_state = get_wave_state; 2954 dqm->ops.reset_queues = reset_queues_cpsch; 2955 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2956 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2957 break; 2958 case KFD_SCHED_POLICY_NO_HWS: 2959 /* initialize dqm for no cp scheduling */ 2960 dqm->ops.start = start_nocpsch; 2961 dqm->ops.stop = stop_nocpsch; 2962 dqm->ops.create_queue = create_queue_nocpsch; 2963 dqm->ops.destroy_queue = destroy_queue_nocpsch; 2964 dqm->ops.update_queue = update_queue; 2965 dqm->ops.register_process = register_process; 2966 dqm->ops.unregister_process = unregister_process; 2967 dqm->ops.initialize = initialize_nocpsch; 2968 dqm->ops.uninitialize = uninitialize; 2969 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2970 dqm->ops.process_termination = process_termination_nocpsch; 2971 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 2972 dqm->ops.restore_process_queues = 2973 restore_process_queues_nocpsch; 2974 dqm->ops.get_wave_state = get_wave_state; 2975 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2976 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2977 break; 2978 default: 2979 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 2980 goto out_free; 2981 } 2982 2983 switch (dev->adev->asic_type) { 2984 case CHIP_KAVERI: 2985 case CHIP_HAWAII: 2986 device_queue_manager_init_cik(&dqm->asic_ops); 2987 break; 2988 2989 case CHIP_CARRIZO: 2990 case CHIP_TONGA: 2991 case CHIP_FIJI: 2992 case CHIP_POLARIS10: 2993 case CHIP_POLARIS11: 2994 case CHIP_POLARIS12: 2995 case CHIP_VEGAM: 2996 device_queue_manager_init_vi(&dqm->asic_ops); 2997 break; 2998 2999 default: 3000 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 3001 device_queue_manager_init_v12(&dqm->asic_ops); 3002 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 3003 device_queue_manager_init_v11(&dqm->asic_ops); 3004 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 3005 device_queue_manager_init_v10(&dqm->asic_ops); 3006 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 3007 device_queue_manager_init_v9(&dqm->asic_ops); 3008 else { 3009 WARN(1, "Unexpected ASIC family %u", 3010 dev->adev->asic_type); 3011 goto out_free; 3012 } 3013 } 3014 3015 if (init_mqd_managers(dqm)) 3016 goto out_free; 3017 3018 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 3019 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 3020 goto out_free; 3021 } 3022 3023 if (!dqm->ops.initialize(dqm)) { 3024 init_waitqueue_head(&dqm->destroy_wait); 3025 return dqm; 3026 } 3027 3028 out_free: 3029 kfree(dqm); 3030 return NULL; 3031 } 3032 3033 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 3034 struct kfd_mem_obj *mqd) 3035 { 3036 WARN(!mqd, "No hiq sdma mqd trunk to free"); 3037 3038 amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); 3039 } 3040 3041 void device_queue_manager_uninit(struct device_queue_manager *dqm) 3042 { 3043 dqm->ops.stop(dqm); 3044 dqm->ops.uninitialize(dqm); 3045 if (!dqm->dev->kfd->shared_resources.enable_mes) 3046 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 3047 kfree(dqm); 3048 } 3049 3050 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) 3051 { 3052 struct kfd_process_device *pdd = NULL; 3053 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); 3054 struct device_queue_manager *dqm = knode->dqm; 3055 struct device *dev = dqm->dev->adev->dev; 3056 struct qcm_process_device *qpd; 3057 struct queue *q = NULL; 3058 int ret = 0; 3059 3060 if (!pdd) 3061 return -EINVAL; 3062 3063 dqm_lock(dqm); 3064 3065 if (pdd) { 3066 qpd = &pdd->qpd; 3067 3068 list_for_each_entry(q, &qpd->queues_list, list) { 3069 if (q->doorbell_id == doorbell_id && q->properties.is_active) { 3070 ret = suspend_all_queues_mes(dqm); 3071 if (ret) { 3072 dev_err(dev, "Suspending all queues failed"); 3073 goto out; 3074 } 3075 3076 q->properties.is_evicted = true; 3077 q->properties.is_active = false; 3078 decrement_queue_count(dqm, qpd, q); 3079 3080 ret = remove_queue_mes(dqm, q, qpd); 3081 if (ret) { 3082 dev_err(dev, "Removing bad queue failed"); 3083 goto out; 3084 } 3085 3086 ret = resume_all_queues_mes(dqm); 3087 if (ret) 3088 dev_err(dev, "Resuming all queues failed"); 3089 3090 break; 3091 } 3092 } 3093 } 3094 3095 out: 3096 dqm_unlock(dqm); 3097 kfd_unref_process(p); 3098 return ret; 3099 } 3100 3101 static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm, 3102 struct qcm_process_device *qpd) 3103 { 3104 struct device *dev = dqm->dev->adev->dev; 3105 int ret = 0; 3106 3107 /* Check if process is already evicted */ 3108 dqm_lock(dqm); 3109 if (qpd->evicted) { 3110 /* Increment the evicted count to make sure the 3111 * process stays evicted before its terminated. 3112 */ 3113 qpd->evicted++; 3114 dqm_unlock(dqm); 3115 goto out; 3116 } 3117 dqm_unlock(dqm); 3118 3119 ret = suspend_all_queues_mes(dqm); 3120 if (ret) { 3121 dev_err(dev, "Suspending all queues failed"); 3122 goto out; 3123 } 3124 3125 ret = dqm->ops.evict_process_queues(dqm, qpd); 3126 if (ret) { 3127 dev_err(dev, "Evicting process queues failed"); 3128 goto out; 3129 } 3130 3131 ret = resume_all_queues_mes(dqm); 3132 if (ret) 3133 dev_err(dev, "Resuming all queues failed"); 3134 3135 out: 3136 return ret; 3137 } 3138 3139 int kfd_evict_process_device(struct kfd_process_device *pdd) 3140 { 3141 struct device_queue_manager *dqm; 3142 struct kfd_process *p; 3143 int ret = 0; 3144 3145 p = pdd->process; 3146 dqm = pdd->dev->dqm; 3147 3148 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 3149 3150 if (dqm->dev->kfd->shared_resources.enable_mes) 3151 ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd); 3152 else 3153 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); 3154 3155 return ret; 3156 } 3157 3158 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 3159 struct qcm_process_device *qpd) 3160 { 3161 int r; 3162 struct device *dev = dqm->dev->adev->dev; 3163 int updated_vmid_mask; 3164 3165 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3166 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3167 return -EINVAL; 3168 } 3169 3170 dqm_lock(dqm); 3171 3172 if (dqm->trap_debug_vmid != 0) { 3173 dev_err(dev, "Trap debug id already reserved\n"); 3174 r = -EBUSY; 3175 goto out_unlock; 3176 } 3177 3178 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3179 USE_DEFAULT_GRACE_PERIOD, false); 3180 if (r) 3181 goto out_unlock; 3182 3183 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3184 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 3185 3186 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3187 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 3188 r = set_sched_resources(dqm); 3189 if (r) 3190 goto out_unlock; 3191 3192 r = map_queues_cpsch(dqm); 3193 if (r) 3194 goto out_unlock; 3195 3196 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 3197 3198 out_unlock: 3199 dqm_unlock(dqm); 3200 return r; 3201 } 3202 3203 /* 3204 * Releases vmid for the trap debugger 3205 */ 3206 int release_debug_trap_vmid(struct device_queue_manager *dqm, 3207 struct qcm_process_device *qpd) 3208 { 3209 struct device *dev = dqm->dev->adev->dev; 3210 int r; 3211 int updated_vmid_mask; 3212 uint32_t trap_debug_vmid; 3213 3214 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3215 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3216 return -EINVAL; 3217 } 3218 3219 dqm_lock(dqm); 3220 trap_debug_vmid = dqm->trap_debug_vmid; 3221 if (dqm->trap_debug_vmid == 0) { 3222 dev_err(dev, "Trap debug id is not reserved\n"); 3223 r = -EINVAL; 3224 goto out_unlock; 3225 } 3226 3227 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3228 USE_DEFAULT_GRACE_PERIOD, false); 3229 if (r) 3230 goto out_unlock; 3231 3232 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3233 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 3234 3235 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3236 dqm->trap_debug_vmid = 0; 3237 r = set_sched_resources(dqm); 3238 if (r) 3239 goto out_unlock; 3240 3241 r = map_queues_cpsch(dqm); 3242 if (r) 3243 goto out_unlock; 3244 3245 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 3246 3247 out_unlock: 3248 dqm_unlock(dqm); 3249 return r; 3250 } 3251 3252 #define QUEUE_NOT_FOUND -1 3253 /* invalidate queue operation in array */ 3254 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 3255 { 3256 int i; 3257 3258 for (i = 0; i < num_queues; i++) 3259 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 3260 } 3261 3262 /* find queue index in array */ 3263 static int q_array_get_index(unsigned int queue_id, 3264 uint32_t num_queues, 3265 uint32_t *queue_ids) 3266 { 3267 int i; 3268 3269 for (i = 0; i < num_queues; i++) 3270 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 3271 return i; 3272 3273 return QUEUE_NOT_FOUND; 3274 } 3275 3276 struct copy_context_work_handler_workarea { 3277 struct work_struct copy_context_work; 3278 struct kfd_process *p; 3279 }; 3280 3281 static void copy_context_work_handler(struct work_struct *work) 3282 { 3283 struct copy_context_work_handler_workarea *workarea; 3284 struct mqd_manager *mqd_mgr; 3285 struct queue *q; 3286 struct mm_struct *mm; 3287 struct kfd_process *p; 3288 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 3289 int i; 3290 3291 workarea = container_of(work, 3292 struct copy_context_work_handler_workarea, 3293 copy_context_work); 3294 3295 p = workarea->p; 3296 mm = get_task_mm(p->lead_thread); 3297 3298 if (!mm) 3299 return; 3300 3301 kthread_use_mm(mm); 3302 for (i = 0; i < p->n_pdds; i++) { 3303 struct kfd_process_device *pdd = p->pdds[i]; 3304 struct device_queue_manager *dqm = pdd->dev->dqm; 3305 struct qcm_process_device *qpd = &pdd->qpd; 3306 3307 list_for_each_entry(q, &qpd->queues_list, list) { 3308 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) 3309 continue; 3310 3311 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3312 3313 /* We ignore the return value from get_wave_state 3314 * because 3315 * i) right now, it always returns 0, and 3316 * ii) if we hit an error, we would continue to the 3317 * next queue anyway. 3318 */ 3319 mqd_mgr->get_wave_state(mqd_mgr, 3320 q->mqd, 3321 &q->properties, 3322 (void __user *) q->properties.ctx_save_restore_area_address, 3323 &tmp_ctl_stack_used_size, 3324 &tmp_save_area_used_size); 3325 } 3326 } 3327 kthread_unuse_mm(mm); 3328 mmput(mm); 3329 } 3330 3331 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 3332 { 3333 size_t array_size = num_queues * sizeof(uint32_t); 3334 3335 if (!usr_queue_id_array) 3336 return NULL; 3337 3338 return memdup_user(usr_queue_id_array, array_size); 3339 } 3340 3341 int resume_queues(struct kfd_process *p, 3342 uint32_t num_queues, 3343 uint32_t *usr_queue_id_array) 3344 { 3345 uint32_t *queue_ids = NULL; 3346 int total_resumed = 0; 3347 int i; 3348 3349 if (usr_queue_id_array) { 3350 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3351 3352 if (IS_ERR(queue_ids)) 3353 return PTR_ERR(queue_ids); 3354 3355 /* mask all queues as invalid. unmask per successful request */ 3356 q_array_invalidate(num_queues, queue_ids); 3357 } 3358 3359 for (i = 0; i < p->n_pdds; i++) { 3360 struct kfd_process_device *pdd = p->pdds[i]; 3361 struct device_queue_manager *dqm = pdd->dev->dqm; 3362 struct device *dev = dqm->dev->adev->dev; 3363 struct qcm_process_device *qpd = &pdd->qpd; 3364 struct queue *q; 3365 int r, per_device_resumed = 0; 3366 3367 dqm_lock(dqm); 3368 3369 /* unmask queues that resume or already resumed as valid */ 3370 list_for_each_entry(q, &qpd->queues_list, list) { 3371 int q_idx = QUEUE_NOT_FOUND; 3372 3373 if (queue_ids) 3374 q_idx = q_array_get_index( 3375 q->properties.queue_id, 3376 num_queues, 3377 queue_ids); 3378 3379 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 3380 int err = resume_single_queue(dqm, &pdd->qpd, q); 3381 3382 if (queue_ids) { 3383 if (!err) { 3384 queue_ids[q_idx] &= 3385 ~KFD_DBG_QUEUE_INVALID_MASK; 3386 } else { 3387 queue_ids[q_idx] |= 3388 KFD_DBG_QUEUE_ERROR_MASK; 3389 break; 3390 } 3391 } 3392 3393 if (dqm->dev->kfd->shared_resources.enable_mes) { 3394 wake_up_all(&dqm->destroy_wait); 3395 if (!err) 3396 total_resumed++; 3397 } else { 3398 per_device_resumed++; 3399 } 3400 } 3401 } 3402 3403 if (!per_device_resumed) { 3404 dqm_unlock(dqm); 3405 continue; 3406 } 3407 3408 r = execute_queues_cpsch(dqm, 3409 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 3410 0, 3411 USE_DEFAULT_GRACE_PERIOD); 3412 if (r) { 3413 dev_err(dev, "Failed to resume process queues\n"); 3414 if (queue_ids) { 3415 list_for_each_entry(q, &qpd->queues_list, list) { 3416 int q_idx = q_array_get_index( 3417 q->properties.queue_id, 3418 num_queues, 3419 queue_ids); 3420 3421 /* mask queue as error on resume fail */ 3422 if (q_idx != QUEUE_NOT_FOUND) 3423 queue_ids[q_idx] |= 3424 KFD_DBG_QUEUE_ERROR_MASK; 3425 } 3426 } 3427 } else { 3428 wake_up_all(&dqm->destroy_wait); 3429 total_resumed += per_device_resumed; 3430 } 3431 3432 dqm_unlock(dqm); 3433 } 3434 3435 if (queue_ids) { 3436 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3437 num_queues * sizeof(uint32_t))) 3438 pr_err("copy_to_user failed on queue resume\n"); 3439 3440 kfree(queue_ids); 3441 } 3442 3443 return total_resumed; 3444 } 3445 3446 int suspend_queues(struct kfd_process *p, 3447 uint32_t num_queues, 3448 uint32_t grace_period, 3449 uint64_t exception_clear_mask, 3450 uint32_t *usr_queue_id_array) 3451 { 3452 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3453 int total_suspended = 0; 3454 int i; 3455 3456 if (IS_ERR(queue_ids)) 3457 return PTR_ERR(queue_ids); 3458 3459 /* mask all queues as invalid. umask on successful request */ 3460 q_array_invalidate(num_queues, queue_ids); 3461 3462 for (i = 0; i < p->n_pdds; i++) { 3463 struct kfd_process_device *pdd = p->pdds[i]; 3464 struct device_queue_manager *dqm = pdd->dev->dqm; 3465 struct device *dev = dqm->dev->adev->dev; 3466 struct qcm_process_device *qpd = &pdd->qpd; 3467 struct queue *q; 3468 int r, per_device_suspended = 0; 3469 3470 mutex_lock(&p->event_mutex); 3471 dqm_lock(dqm); 3472 3473 /* unmask queues that suspend or already suspended */ 3474 list_for_each_entry(q, &qpd->queues_list, list) { 3475 int q_idx = q_array_get_index(q->properties.queue_id, 3476 num_queues, 3477 queue_ids); 3478 3479 if (q_idx != QUEUE_NOT_FOUND) { 3480 int err = suspend_single_queue(dqm, pdd, q); 3481 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 3482 3483 if (!err) { 3484 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 3485 if (exception_clear_mask && is_mes) 3486 q->properties.exception_status &= 3487 ~exception_clear_mask; 3488 3489 if (is_mes) 3490 total_suspended++; 3491 else 3492 per_device_suspended++; 3493 } else if (err != -EBUSY) { 3494 r = err; 3495 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3496 break; 3497 } 3498 } 3499 } 3500 3501 if (!per_device_suspended) { 3502 dqm_unlock(dqm); 3503 mutex_unlock(&p->event_mutex); 3504 if (total_suspended) 3505 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3506 continue; 3507 } 3508 3509 r = execute_queues_cpsch(dqm, 3510 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3511 grace_period); 3512 3513 if (r) 3514 dev_err(dev, "Failed to suspend process queues.\n"); 3515 else 3516 total_suspended += per_device_suspended; 3517 3518 list_for_each_entry(q, &qpd->queues_list, list) { 3519 int q_idx = q_array_get_index(q->properties.queue_id, 3520 num_queues, queue_ids); 3521 3522 if (q_idx == QUEUE_NOT_FOUND) 3523 continue; 3524 3525 /* mask queue as error on suspend fail */ 3526 if (r) 3527 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3528 else if (exception_clear_mask) 3529 q->properties.exception_status &= 3530 ~exception_clear_mask; 3531 } 3532 3533 dqm_unlock(dqm); 3534 mutex_unlock(&p->event_mutex); 3535 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3536 } 3537 3538 if (total_suspended) { 3539 struct copy_context_work_handler_workarea copy_context_worker; 3540 3541 INIT_WORK_ONSTACK( 3542 ©_context_worker.copy_context_work, 3543 copy_context_work_handler); 3544 3545 copy_context_worker.p = p; 3546 3547 schedule_work(©_context_worker.copy_context_work); 3548 3549 3550 flush_work(©_context_worker.copy_context_work); 3551 destroy_work_on_stack(©_context_worker.copy_context_work); 3552 } 3553 3554 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3555 num_queues * sizeof(uint32_t))) 3556 pr_err("copy_to_user failed on queue suspend\n"); 3557 3558 kfree(queue_ids); 3559 3560 return total_suspended; 3561 } 3562 3563 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 3564 { 3565 switch (q_props->type) { 3566 case KFD_QUEUE_TYPE_COMPUTE: 3567 return q_props->format == KFD_QUEUE_FORMAT_PM4 3568 ? KFD_IOC_QUEUE_TYPE_COMPUTE 3569 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 3570 case KFD_QUEUE_TYPE_SDMA: 3571 return KFD_IOC_QUEUE_TYPE_SDMA; 3572 case KFD_QUEUE_TYPE_SDMA_XGMI: 3573 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 3574 default: 3575 WARN_ONCE(true, "queue type not recognized!"); 3576 return 0xffffffff; 3577 }; 3578 } 3579 3580 void set_queue_snapshot_entry(struct queue *q, 3581 uint64_t exception_clear_mask, 3582 struct kfd_queue_snapshot_entry *qss_entry) 3583 { 3584 qss_entry->ring_base_address = q->properties.queue_address; 3585 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 3586 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 3587 qss_entry->ctx_save_restore_address = 3588 q->properties.ctx_save_restore_area_address; 3589 qss_entry->ctx_save_restore_area_size = 3590 q->properties.ctx_save_restore_area_size; 3591 qss_entry->exception_status = q->properties.exception_status; 3592 qss_entry->queue_id = q->properties.queue_id; 3593 qss_entry->gpu_id = q->device->id; 3594 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 3595 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 3596 q->properties.exception_status &= ~exception_clear_mask; 3597 } 3598 3599 int debug_lock_and_unmap(struct device_queue_manager *dqm) 3600 { 3601 struct device *dev = dqm->dev->adev->dev; 3602 int r; 3603 3604 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3605 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3606 return -EINVAL; 3607 } 3608 3609 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3610 return 0; 3611 3612 dqm_lock(dqm); 3613 3614 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 3615 if (r) 3616 dqm_unlock(dqm); 3617 3618 return r; 3619 } 3620 3621 int debug_map_and_unlock(struct device_queue_manager *dqm) 3622 { 3623 struct device *dev = dqm->dev->adev->dev; 3624 int r; 3625 3626 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3627 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3628 return -EINVAL; 3629 } 3630 3631 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3632 return 0; 3633 3634 r = map_queues_cpsch(dqm); 3635 3636 dqm_unlock(dqm); 3637 3638 return r; 3639 } 3640 3641 int debug_refresh_runlist(struct device_queue_manager *dqm) 3642 { 3643 int r = debug_lock_and_unmap(dqm); 3644 3645 if (r) 3646 return r; 3647 3648 return debug_map_and_unlock(dqm); 3649 } 3650 3651 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, 3652 struct qcm_process_device *qpd, 3653 int doorbell_off, u32 *queue_format) 3654 { 3655 struct queue *q; 3656 bool r = false; 3657 3658 if (!queue_format) 3659 return r; 3660 3661 dqm_lock(dqm); 3662 3663 list_for_each_entry(q, &qpd->queues_list, list) { 3664 if (q->properties.doorbell_off == doorbell_off) { 3665 *queue_format = q->properties.format; 3666 r = true; 3667 goto out; 3668 } 3669 } 3670 3671 out: 3672 dqm_unlock(dqm); 3673 return r; 3674 } 3675 #if defined(CONFIG_DEBUG_FS) 3676 3677 static void seq_reg_dump(struct seq_file *m, 3678 uint32_t (*dump)[2], uint32_t n_regs) 3679 { 3680 uint32_t i, count; 3681 3682 for (i = 0, count = 0; i < n_regs; i++) { 3683 if (count == 0 || 3684 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 3685 seq_printf(m, "%s %08x: %08x", 3686 i ? "\n" : "", 3687 dump[i][0], dump[i][1]); 3688 count = 7; 3689 } else { 3690 seq_printf(m, " %08x", dump[i][1]); 3691 count--; 3692 } 3693 } 3694 3695 seq_puts(m, "\n"); 3696 } 3697 3698 int dqm_debugfs_hqds(struct seq_file *m, void *data) 3699 { 3700 struct device_queue_manager *dqm = data; 3701 uint32_t xcc_mask = dqm->dev->xcc_mask; 3702 uint32_t (*dump)[2], n_regs; 3703 int pipe, queue; 3704 int r = 0, xcc_id; 3705 uint32_t sdma_engine_start; 3706 3707 if (!dqm->sched_running) { 3708 seq_puts(m, " Device is stopped\n"); 3709 return 0; 3710 } 3711 3712 for_each_inst(xcc_id, xcc_mask) { 3713 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3714 KFD_CIK_HIQ_PIPE, 3715 KFD_CIK_HIQ_QUEUE, &dump, 3716 &n_regs, xcc_id); 3717 if (!r) { 3718 seq_printf( 3719 m, 3720 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 3721 xcc_id, 3722 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 3723 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 3724 KFD_CIK_HIQ_QUEUE); 3725 seq_reg_dump(m, dump, n_regs); 3726 3727 kfree(dump); 3728 } 3729 3730 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 3731 int pipe_offset = pipe * get_queues_per_pipe(dqm); 3732 3733 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 3734 if (!test_bit(pipe_offset + queue, 3735 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 3736 continue; 3737 3738 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3739 pipe, queue, 3740 &dump, &n_regs, 3741 xcc_id); 3742 if (r) 3743 break; 3744 3745 seq_printf(m, 3746 " Inst %d, CP Pipe %d, Queue %d\n", 3747 xcc_id, pipe, queue); 3748 seq_reg_dump(m, dump, n_regs); 3749 3750 kfree(dump); 3751 } 3752 } 3753 } 3754 3755 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 3756 for (pipe = sdma_engine_start; 3757 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 3758 pipe++) { 3759 for (queue = 0; 3760 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 3761 queue++) { 3762 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 3763 dqm->dev->adev, pipe, queue, &dump, &n_regs); 3764 if (r) 3765 break; 3766 3767 seq_printf(m, " SDMA Engine %d, RLC %d\n", 3768 pipe, queue); 3769 seq_reg_dump(m, dump, n_regs); 3770 3771 kfree(dump); 3772 } 3773 } 3774 3775 return r; 3776 } 3777 3778 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 3779 { 3780 int r = 0; 3781 3782 dqm_lock(dqm); 3783 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 3784 if (r) { 3785 dqm_unlock(dqm); 3786 return r; 3787 } 3788 dqm->active_runlist = true; 3789 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 3790 0, USE_DEFAULT_GRACE_PERIOD); 3791 dqm_unlock(dqm); 3792 3793 return r; 3794 } 3795 3796 #endif 3797