1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "mes_v11_api_def.h" 40 #include "kfd_debug.h" 41 42 /* Size of the per-pipe EOP queue */ 43 #define CIK_HPD_EOP_BYTES_LOG2 11 44 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 45 46 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 47 u32 pasid, unsigned int vmid); 48 49 static int execute_queues_cpsch(struct device_queue_manager *dqm, 50 enum kfd_unmap_queues_filter filter, 51 uint32_t filter_param, 52 uint32_t grace_period); 53 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 54 enum kfd_unmap_queues_filter filter, 55 uint32_t filter_param, 56 uint32_t grace_period, 57 bool reset); 58 59 static int map_queues_cpsch(struct device_queue_manager *dqm); 60 61 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 62 struct queue *q); 63 64 static inline void deallocate_hqd(struct device_queue_manager *dqm, 65 struct queue *q); 66 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 67 static int allocate_sdma_queue(struct device_queue_manager *dqm, 68 struct queue *q, const uint32_t *restore_sdma_id); 69 static void kfd_process_hw_exception(struct work_struct *work); 70 71 static inline 72 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 73 { 74 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 75 return KFD_MQD_TYPE_SDMA; 76 return KFD_MQD_TYPE_CP; 77 } 78 79 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 80 { 81 int i; 82 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 83 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 84 85 /* queue is available for KFD usage if bit is 1 */ 86 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 87 if (test_bit(pipe_offset + i, 88 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 89 return true; 90 return false; 91 } 92 93 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 94 { 95 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 96 AMDGPU_MAX_QUEUES); 97 } 98 99 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 100 { 101 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 102 } 103 104 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 105 { 106 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 107 } 108 109 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 110 { 111 return kfd_get_num_sdma_engines(dqm->dev) + 112 kfd_get_num_xgmi_sdma_engines(dqm->dev); 113 } 114 115 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 116 { 117 return kfd_get_num_sdma_engines(dqm->dev) * 118 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 119 } 120 121 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 122 { 123 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 124 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 125 } 126 127 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 128 { 129 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 130 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 131 132 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 133 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 134 135 /* Mask out the reserved queues */ 136 bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap, 137 dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap, 138 KFD_MAX_SDMA_QUEUES); 139 } 140 141 void program_sh_mem_settings(struct device_queue_manager *dqm, 142 struct qcm_process_device *qpd) 143 { 144 uint32_t xcc_mask = dqm->dev->xcc_mask; 145 int xcc_id; 146 147 for_each_inst(xcc_id, xcc_mask) 148 dqm->dev->kfd2kgd->program_sh_mem_settings( 149 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 150 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 151 qpd->sh_mem_bases, xcc_id); 152 } 153 154 static void kfd_hws_hang(struct device_queue_manager *dqm) 155 { 156 struct device_process_node *cur; 157 struct qcm_process_device *qpd; 158 struct queue *q; 159 160 /* Mark all device queues as reset. */ 161 list_for_each_entry(cur, &dqm->queues, list) { 162 qpd = cur->qpd; 163 list_for_each_entry(q, &qpd->queues_list, list) { 164 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 165 166 pdd->has_reset_queue = true; 167 } 168 } 169 170 /* 171 * Issue a GPU reset if HWS is unresponsive 172 */ 173 schedule_work(&dqm->hw_exception_work); 174 } 175 176 static int convert_to_mes_queue_type(int queue_type) 177 { 178 int mes_queue_type; 179 180 switch (queue_type) { 181 case KFD_QUEUE_TYPE_COMPUTE: 182 mes_queue_type = MES_QUEUE_TYPE_COMPUTE; 183 break; 184 case KFD_QUEUE_TYPE_SDMA: 185 mes_queue_type = MES_QUEUE_TYPE_SDMA; 186 break; 187 default: 188 WARN(1, "Invalid queue type %d", queue_type); 189 mes_queue_type = -EINVAL; 190 break; 191 } 192 193 return mes_queue_type; 194 } 195 196 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 197 struct qcm_process_device *qpd) 198 { 199 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 200 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 201 struct mes_add_queue_input queue_input; 202 int r, queue_type; 203 uint64_t wptr_addr_off; 204 205 if (!down_read_trylock(&adev->reset_domain->sem)) 206 return -EIO; 207 208 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 209 queue_input.process_id = qpd->pqm->process->pasid; 210 queue_input.page_table_base_addr = qpd->page_table_base; 211 queue_input.process_va_start = 0; 212 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 213 /* MES unit for quantum is 100ns */ 214 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 215 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 216 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 217 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 218 queue_input.inprocess_gang_priority = q->properties.priority; 219 queue_input.gang_global_priority_level = 220 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 221 queue_input.doorbell_offset = q->properties.doorbell_off; 222 queue_input.mqd_addr = q->gart_mqd_addr; 223 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 224 225 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 226 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 227 228 queue_input.is_kfd_process = 1; 229 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 230 queue_input.queue_size = q->properties.queue_size >> 2; 231 232 queue_input.paging = false; 233 queue_input.tba_addr = qpd->tba_addr; 234 queue_input.tma_addr = qpd->tma_addr; 235 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 236 queue_input.skip_process_ctx_clear = 237 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 238 (qpd->pqm->process->debug_trap_enabled || 239 kfd_dbg_has_ttmps_always_setup(q->device)); 240 241 queue_type = convert_to_mes_queue_type(q->properties.type); 242 if (queue_type < 0) { 243 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 244 q->properties.type); 245 up_read(&adev->reset_domain->sem); 246 return -EINVAL; 247 } 248 queue_input.queue_type = (uint32_t)queue_type; 249 250 queue_input.exclusively_scheduled = q->properties.is_gws; 251 252 amdgpu_mes_lock(&adev->mes); 253 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 254 amdgpu_mes_unlock(&adev->mes); 255 up_read(&adev->reset_domain->sem); 256 if (r) { 257 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 258 q->properties.doorbell_off); 259 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 260 kfd_hws_hang(dqm); 261 } 262 263 return r; 264 } 265 266 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 267 struct qcm_process_device *qpd) 268 { 269 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 270 int r; 271 struct mes_remove_queue_input queue_input; 272 273 if (!down_read_trylock(&adev->reset_domain->sem)) 274 return -EIO; 275 276 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 277 queue_input.doorbell_offset = q->properties.doorbell_off; 278 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 279 280 amdgpu_mes_lock(&adev->mes); 281 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 282 amdgpu_mes_unlock(&adev->mes); 283 up_read(&adev->reset_domain->sem); 284 285 if (r) { 286 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", 287 q->properties.doorbell_off); 288 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 289 kfd_hws_hang(dqm); 290 } 291 292 return r; 293 } 294 295 static int remove_all_queues_mes(struct device_queue_manager *dqm) 296 { 297 struct device_process_node *cur; 298 struct device *dev = dqm->dev->adev->dev; 299 struct qcm_process_device *qpd; 300 struct queue *q; 301 int retval = 0; 302 303 list_for_each_entry(cur, &dqm->queues, list) { 304 qpd = cur->qpd; 305 list_for_each_entry(q, &qpd->queues_list, list) { 306 if (q->properties.is_active) { 307 retval = remove_queue_mes(dqm, q, qpd); 308 if (retval) { 309 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 310 __func__, 311 q->properties.queue_id, 312 dqm->dev->id); 313 return retval; 314 } 315 } 316 } 317 } 318 319 return retval; 320 } 321 322 static int suspend_all_queues_mes(struct device_queue_manager *dqm) 323 { 324 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 325 int r = 0; 326 327 if (!down_read_trylock(&adev->reset_domain->sem)) 328 return -EIO; 329 330 r = amdgpu_mes_suspend(adev); 331 up_read(&adev->reset_domain->sem); 332 333 if (r) { 334 dev_err(adev->dev, "failed to suspend gangs from MES\n"); 335 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 336 kfd_hws_hang(dqm); 337 } 338 339 return r; 340 } 341 342 static int resume_all_queues_mes(struct device_queue_manager *dqm) 343 { 344 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 345 int r = 0; 346 347 if (!down_read_trylock(&adev->reset_domain->sem)) 348 return -EIO; 349 350 r = amdgpu_mes_resume(adev); 351 up_read(&adev->reset_domain->sem); 352 353 if (r) { 354 dev_err(adev->dev, "failed to resume gangs from MES\n"); 355 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 356 kfd_hws_hang(dqm); 357 } 358 359 return r; 360 } 361 362 static void increment_queue_count(struct device_queue_manager *dqm, 363 struct qcm_process_device *qpd, 364 struct queue *q) 365 { 366 dqm->active_queue_count++; 367 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 368 q->properties.type == KFD_QUEUE_TYPE_DIQ) 369 dqm->active_cp_queue_count++; 370 371 if (q->properties.is_gws) { 372 dqm->gws_queue_count++; 373 qpd->mapped_gws_queue = true; 374 } 375 } 376 377 static void decrement_queue_count(struct device_queue_manager *dqm, 378 struct qcm_process_device *qpd, 379 struct queue *q) 380 { 381 dqm->active_queue_count--; 382 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 383 q->properties.type == KFD_QUEUE_TYPE_DIQ) 384 dqm->active_cp_queue_count--; 385 386 if (q->properties.is_gws) { 387 dqm->gws_queue_count--; 388 qpd->mapped_gws_queue = false; 389 } 390 } 391 392 /* 393 * Allocate a doorbell ID to this queue. 394 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 395 */ 396 static int allocate_doorbell(struct qcm_process_device *qpd, 397 struct queue *q, 398 uint32_t const *restore_id) 399 { 400 struct kfd_node *dev = qpd->dqm->dev; 401 402 if (!KFD_IS_SOC15(dev)) { 403 /* On pre-SOC15 chips we need to use the queue ID to 404 * preserve the user mode ABI. 405 */ 406 407 if (restore_id && *restore_id != q->properties.queue_id) 408 return -EINVAL; 409 410 q->doorbell_id = q->properties.queue_id; 411 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 412 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 413 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 414 * doorbell assignments based on the engine and queue id. 415 * The doobell index distance between RLC (2*i) and (2*i+1) 416 * for a SDMA engine is 512. 417 */ 418 419 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 420 421 /* 422 * q->properties.sdma_engine_id corresponds to the virtual 423 * sdma engine number. However, for doorbell allocation, 424 * we need the physical sdma engine id in order to get the 425 * correct doorbell offset. 426 */ 427 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * 428 get_num_all_sdma_engines(qpd->dqm) + 429 q->properties.sdma_engine_id] 430 + (q->properties.sdma_queue_id & 1) 431 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET 432 + (q->properties.sdma_queue_id >> 1); 433 434 if (restore_id && *restore_id != valid_id) 435 return -EINVAL; 436 q->doorbell_id = valid_id; 437 } else { 438 /* For CP queues on SOC15 */ 439 if (restore_id) { 440 /* make sure that ID is free */ 441 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 442 return -EINVAL; 443 444 q->doorbell_id = *restore_id; 445 } else { 446 /* or reserve a free doorbell ID */ 447 unsigned int found; 448 449 found = find_first_zero_bit(qpd->doorbell_bitmap, 450 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 451 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 452 pr_debug("No doorbells available"); 453 return -EBUSY; 454 } 455 set_bit(found, qpd->doorbell_bitmap); 456 q->doorbell_id = found; 457 } 458 } 459 460 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 461 qpd->proc_doorbells, 462 q->doorbell_id, 463 dev->kfd->device_info.doorbell_size); 464 return 0; 465 } 466 467 static void deallocate_doorbell(struct qcm_process_device *qpd, 468 struct queue *q) 469 { 470 unsigned int old; 471 struct kfd_node *dev = qpd->dqm->dev; 472 473 if (!KFD_IS_SOC15(dev) || 474 q->properties.type == KFD_QUEUE_TYPE_SDMA || 475 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 476 return; 477 478 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 479 WARN_ON(!old); 480 } 481 482 static void program_trap_handler_settings(struct device_queue_manager *dqm, 483 struct qcm_process_device *qpd) 484 { 485 uint32_t xcc_mask = dqm->dev->xcc_mask; 486 int xcc_id; 487 488 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 489 for_each_inst(xcc_id, xcc_mask) 490 dqm->dev->kfd2kgd->program_trap_handler_settings( 491 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 492 qpd->tma_addr, xcc_id); 493 } 494 495 static int allocate_vmid(struct device_queue_manager *dqm, 496 struct qcm_process_device *qpd, 497 struct queue *q) 498 { 499 struct device *dev = dqm->dev->adev->dev; 500 int allocated_vmid = -1, i; 501 502 for (i = dqm->dev->vm_info.first_vmid_kfd; 503 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 504 if (!dqm->vmid_pasid[i]) { 505 allocated_vmid = i; 506 break; 507 } 508 } 509 510 if (allocated_vmid < 0) { 511 dev_err(dev, "no more vmid to allocate\n"); 512 return -ENOSPC; 513 } 514 515 pr_debug("vmid allocated: %d\n", allocated_vmid); 516 517 dqm->vmid_pasid[allocated_vmid] = q->process->pasid; 518 519 set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid); 520 521 qpd->vmid = allocated_vmid; 522 q->properties.vmid = allocated_vmid; 523 524 program_sh_mem_settings(dqm, qpd); 525 526 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 527 program_trap_handler_settings(dqm, qpd); 528 529 /* qpd->page_table_base is set earlier when register_process() 530 * is called, i.e. when the first queue is created. 531 */ 532 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 533 qpd->vmid, 534 qpd->page_table_base); 535 /* invalidate the VM context after pasid and vmid mapping is set up */ 536 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 537 538 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 539 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 540 qpd->sh_hidden_private_base, qpd->vmid); 541 542 return 0; 543 } 544 545 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 546 struct qcm_process_device *qpd) 547 { 548 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 549 int ret; 550 551 if (!qpd->ib_kaddr) 552 return -ENOMEM; 553 554 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 555 if (ret) 556 return ret; 557 558 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 559 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 560 pmf->release_mem_size / sizeof(uint32_t)); 561 } 562 563 static void deallocate_vmid(struct device_queue_manager *dqm, 564 struct qcm_process_device *qpd, 565 struct queue *q) 566 { 567 struct device *dev = dqm->dev->adev->dev; 568 569 /* On GFX v7, CP doesn't flush TC at dequeue */ 570 if (q->device->adev->asic_type == CHIP_HAWAII) 571 if (flush_texture_cache_nocpsch(q->device, qpd)) 572 dev_err(dev, "Failed to flush TC\n"); 573 574 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); 575 576 /* Release the vmid mapping */ 577 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 578 dqm->vmid_pasid[qpd->vmid] = 0; 579 580 qpd->vmid = 0; 581 q->properties.vmid = 0; 582 } 583 584 static int create_queue_nocpsch(struct device_queue_manager *dqm, 585 struct queue *q, 586 struct qcm_process_device *qpd, 587 const struct kfd_criu_queue_priv_data *qd, 588 const void *restore_mqd, const void *restore_ctl_stack) 589 { 590 struct mqd_manager *mqd_mgr; 591 int retval; 592 593 dqm_lock(dqm); 594 595 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 596 pr_warn("Can't create new usermode queue because %d queues were already created\n", 597 dqm->total_queue_count); 598 retval = -EPERM; 599 goto out_unlock; 600 } 601 602 if (list_empty(&qpd->queues_list)) { 603 retval = allocate_vmid(dqm, qpd, q); 604 if (retval) 605 goto out_unlock; 606 } 607 q->properties.vmid = qpd->vmid; 608 /* 609 * Eviction state logic: mark all queues as evicted, even ones 610 * not currently active. Restoring inactive queues later only 611 * updates the is_evicted flag but is a no-op otherwise. 612 */ 613 q->properties.is_evicted = !!qpd->evicted; 614 615 q->properties.tba_addr = qpd->tba_addr; 616 q->properties.tma_addr = qpd->tma_addr; 617 618 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 619 q->properties.type)]; 620 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 621 retval = allocate_hqd(dqm, q); 622 if (retval) 623 goto deallocate_vmid; 624 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 625 q->pipe, q->queue); 626 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 627 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 628 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 629 if (retval) 630 goto deallocate_vmid; 631 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 632 } 633 634 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 635 if (retval) 636 goto out_deallocate_hqd; 637 638 /* Temporarily release dqm lock to avoid a circular lock dependency */ 639 dqm_unlock(dqm); 640 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 641 dqm_lock(dqm); 642 643 if (!q->mqd_mem_obj) { 644 retval = -ENOMEM; 645 goto out_deallocate_doorbell; 646 } 647 648 if (qd) 649 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 650 &q->properties, restore_mqd, restore_ctl_stack, 651 qd->ctl_stack_size); 652 else 653 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 654 &q->gart_mqd_addr, &q->properties); 655 656 if (q->properties.is_active) { 657 if (!dqm->sched_running) { 658 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 659 goto add_queue_to_list; 660 } 661 662 if (WARN(q->process->mm != current->mm, 663 "should only run in user thread")) 664 retval = -EFAULT; 665 else 666 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 667 q->queue, &q->properties, current->mm); 668 if (retval) 669 goto out_free_mqd; 670 } 671 672 add_queue_to_list: 673 list_add(&q->list, &qpd->queues_list); 674 qpd->queue_count++; 675 if (q->properties.is_active) 676 increment_queue_count(dqm, qpd, q); 677 678 /* 679 * Unconditionally increment this counter, regardless of the queue's 680 * type or whether the queue is active. 681 */ 682 dqm->total_queue_count++; 683 pr_debug("Total of %d queues are accountable so far\n", 684 dqm->total_queue_count); 685 goto out_unlock; 686 687 out_free_mqd: 688 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 689 out_deallocate_doorbell: 690 deallocate_doorbell(qpd, q); 691 out_deallocate_hqd: 692 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 693 deallocate_hqd(dqm, q); 694 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 695 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 696 deallocate_sdma_queue(dqm, q); 697 deallocate_vmid: 698 if (list_empty(&qpd->queues_list)) 699 deallocate_vmid(dqm, qpd, q); 700 out_unlock: 701 dqm_unlock(dqm); 702 return retval; 703 } 704 705 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 706 { 707 bool set; 708 int pipe, bit, i; 709 710 set = false; 711 712 for (pipe = dqm->next_pipe_to_allocate, i = 0; 713 i < get_pipes_per_mec(dqm); 714 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 715 716 if (!is_pipe_enabled(dqm, 0, pipe)) 717 continue; 718 719 if (dqm->allocated_queues[pipe] != 0) { 720 bit = ffs(dqm->allocated_queues[pipe]) - 1; 721 dqm->allocated_queues[pipe] &= ~(1 << bit); 722 q->pipe = pipe; 723 q->queue = bit; 724 set = true; 725 break; 726 } 727 } 728 729 if (!set) 730 return -EBUSY; 731 732 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 733 /* horizontal hqd allocation */ 734 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 735 736 return 0; 737 } 738 739 static inline void deallocate_hqd(struct device_queue_manager *dqm, 740 struct queue *q) 741 { 742 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 743 } 744 745 #define SQ_IND_CMD_CMD_KILL 0x00000003 746 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 747 748 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 749 { 750 int status = 0; 751 unsigned int vmid; 752 uint16_t queried_pasid; 753 union SQ_CMD_BITS reg_sq_cmd; 754 union GRBM_GFX_INDEX_BITS reg_gfx_index; 755 struct kfd_process_device *pdd; 756 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 757 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 758 uint32_t xcc_mask = dev->xcc_mask; 759 int xcc_id; 760 761 reg_sq_cmd.u32All = 0; 762 reg_gfx_index.u32All = 0; 763 764 pr_debug("Killing all process wavefronts\n"); 765 766 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 767 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 768 return -EOPNOTSUPP; 769 } 770 771 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 772 * ATC_VMID15_PASID_MAPPING 773 * to check which VMID the current process is mapped to. 774 */ 775 776 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 777 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 778 (dev->adev, vmid, &queried_pasid); 779 780 if (status && queried_pasid == p->pasid) { 781 pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n", 782 vmid, p->pasid); 783 break; 784 } 785 } 786 787 if (vmid > last_vmid_to_scan) { 788 dev_err(dev->adev->dev, "Didn't find vmid for pasid 0x%x\n", p->pasid); 789 return -EFAULT; 790 } 791 792 /* taking the VMID for that process on the safe way using PDD */ 793 pdd = kfd_get_process_device_data(dev, p); 794 if (!pdd) 795 return -EFAULT; 796 797 reg_gfx_index.bits.sh_broadcast_writes = 1; 798 reg_gfx_index.bits.se_broadcast_writes = 1; 799 reg_gfx_index.bits.instance_broadcast_writes = 1; 800 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 801 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 802 reg_sq_cmd.bits.vm_id = vmid; 803 804 for_each_inst(xcc_id, xcc_mask) 805 dev->kfd2kgd->wave_control_execute( 806 dev->adev, reg_gfx_index.u32All, 807 reg_sq_cmd.u32All, xcc_id); 808 809 return 0; 810 } 811 812 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 813 * to avoid asynchronized access 814 */ 815 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 816 struct qcm_process_device *qpd, 817 struct queue *q) 818 { 819 int retval; 820 struct mqd_manager *mqd_mgr; 821 822 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 823 q->properties.type)]; 824 825 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 826 deallocate_hqd(dqm, q); 827 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 828 deallocate_sdma_queue(dqm, q); 829 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 830 deallocate_sdma_queue(dqm, q); 831 else { 832 pr_debug("q->properties.type %d is invalid\n", 833 q->properties.type); 834 return -EINVAL; 835 } 836 dqm->total_queue_count--; 837 838 deallocate_doorbell(qpd, q); 839 840 if (!dqm->sched_running) { 841 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 842 return 0; 843 } 844 845 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 846 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 847 KFD_UNMAP_LATENCY_MS, 848 q->pipe, q->queue); 849 if (retval == -ETIME) 850 qpd->reset_wavefronts = true; 851 852 list_del(&q->list); 853 if (list_empty(&qpd->queues_list)) { 854 if (qpd->reset_wavefronts) { 855 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 856 dqm->dev); 857 /* dbgdev_wave_reset_wavefronts has to be called before 858 * deallocate_vmid(), i.e. when vmid is still in use. 859 */ 860 dbgdev_wave_reset_wavefronts(dqm->dev, 861 qpd->pqm->process); 862 qpd->reset_wavefronts = false; 863 } 864 865 deallocate_vmid(dqm, qpd, q); 866 } 867 qpd->queue_count--; 868 if (q->properties.is_active) 869 decrement_queue_count(dqm, qpd, q); 870 871 return retval; 872 } 873 874 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 875 struct qcm_process_device *qpd, 876 struct queue *q) 877 { 878 int retval; 879 uint64_t sdma_val = 0; 880 struct device *dev = dqm->dev->adev->dev; 881 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 882 struct mqd_manager *mqd_mgr = 883 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 884 885 /* Get the SDMA queue stats */ 886 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 887 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 888 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 889 &sdma_val); 890 if (retval) 891 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 892 q->properties.queue_id); 893 } 894 895 dqm_lock(dqm); 896 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 897 if (!retval) 898 pdd->sdma_past_activity_counter += sdma_val; 899 dqm_unlock(dqm); 900 901 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 902 903 return retval; 904 } 905 906 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 907 struct mqd_update_info *minfo) 908 { 909 int retval = 0; 910 struct device *dev = dqm->dev->adev->dev; 911 struct mqd_manager *mqd_mgr; 912 struct kfd_process_device *pdd; 913 bool prev_active = false; 914 915 dqm_lock(dqm); 916 pdd = kfd_get_process_device_data(q->device, q->process); 917 if (!pdd) { 918 retval = -ENODEV; 919 goto out_unlock; 920 } 921 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 922 q->properties.type)]; 923 924 /* Save previous activity state for counters */ 925 prev_active = q->properties.is_active; 926 927 /* Make sure the queue is unmapped before updating the MQD */ 928 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 929 if (!dqm->dev->kfd->shared_resources.enable_mes) 930 retval = unmap_queues_cpsch(dqm, 931 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 932 else if (prev_active) 933 retval = remove_queue_mes(dqm, q, &pdd->qpd); 934 935 /* queue is reset so inaccessable */ 936 if (pdd->has_reset_queue) { 937 retval = -EACCES; 938 goto out_unlock; 939 } 940 941 if (retval) { 942 dev_err(dev, "unmap queue failed\n"); 943 goto out_unlock; 944 } 945 } else if (prev_active && 946 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 947 q->properties.type == KFD_QUEUE_TYPE_SDMA || 948 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 949 950 if (!dqm->sched_running) { 951 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 952 goto out_unlock; 953 } 954 955 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 956 (dqm->dev->kfd->cwsr_enabled ? 957 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 958 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 959 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 960 if (retval) { 961 dev_err(dev, "destroy mqd failed\n"); 962 goto out_unlock; 963 } 964 } 965 966 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 967 968 /* 969 * check active state vs. the previous state and modify 970 * counter accordingly. map_queues_cpsch uses the 971 * dqm->active_queue_count to determine whether a new runlist must be 972 * uploaded. 973 */ 974 if (q->properties.is_active && !prev_active) { 975 increment_queue_count(dqm, &pdd->qpd, q); 976 } else if (!q->properties.is_active && prev_active) { 977 decrement_queue_count(dqm, &pdd->qpd, q); 978 } else if (q->gws && !q->properties.is_gws) { 979 if (q->properties.is_active) { 980 dqm->gws_queue_count++; 981 pdd->qpd.mapped_gws_queue = true; 982 } 983 q->properties.is_gws = true; 984 } else if (!q->gws && q->properties.is_gws) { 985 if (q->properties.is_active) { 986 dqm->gws_queue_count--; 987 pdd->qpd.mapped_gws_queue = false; 988 } 989 q->properties.is_gws = false; 990 } 991 992 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 993 if (!dqm->dev->kfd->shared_resources.enable_mes) 994 retval = map_queues_cpsch(dqm); 995 else if (q->properties.is_active) 996 retval = add_queue_mes(dqm, q, &pdd->qpd); 997 } else if (q->properties.is_active && 998 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 999 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1000 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1001 if (WARN(q->process->mm != current->mm, 1002 "should only run in user thread")) 1003 retval = -EFAULT; 1004 else 1005 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 1006 q->pipe, q->queue, 1007 &q->properties, current->mm); 1008 } 1009 1010 out_unlock: 1011 dqm_unlock(dqm); 1012 return retval; 1013 } 1014 1015 /* suspend_single_queue does not lock the dqm like the 1016 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 1017 * lock the dqm before calling, and unlock after calling. 1018 * 1019 * The reason we don't lock the dqm is because this function may be 1020 * called on multiple queues in a loop, so rather than locking/unlocking 1021 * multiple times, we will just keep the dqm locked for all of the calls. 1022 */ 1023 static int suspend_single_queue(struct device_queue_manager *dqm, 1024 struct kfd_process_device *pdd, 1025 struct queue *q) 1026 { 1027 bool is_new; 1028 1029 if (q->properties.is_suspended) 1030 return 0; 1031 1032 pr_debug("Suspending PASID %u queue [%i]\n", 1033 pdd->process->pasid, 1034 q->properties.queue_id); 1035 1036 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 1037 1038 if (is_new || q->properties.is_being_destroyed) { 1039 pr_debug("Suspend: skip %s queue id %i\n", 1040 is_new ? "new" : "destroyed", 1041 q->properties.queue_id); 1042 return -EBUSY; 1043 } 1044 1045 q->properties.is_suspended = true; 1046 if (q->properties.is_active) { 1047 if (dqm->dev->kfd->shared_resources.enable_mes) { 1048 int r = remove_queue_mes(dqm, q, &pdd->qpd); 1049 1050 if (r) 1051 return r; 1052 } 1053 1054 decrement_queue_count(dqm, &pdd->qpd, q); 1055 q->properties.is_active = false; 1056 } 1057 1058 return 0; 1059 } 1060 1061 /* resume_single_queue does not lock the dqm like the functions 1062 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1063 * lock the dqm before calling, and unlock after calling. 1064 * 1065 * The reason we don't lock the dqm is because this function may be 1066 * called on multiple queues in a loop, so rather than locking/unlocking 1067 * multiple times, we will just keep the dqm locked for all of the calls. 1068 */ 1069 static int resume_single_queue(struct device_queue_manager *dqm, 1070 struct qcm_process_device *qpd, 1071 struct queue *q) 1072 { 1073 struct kfd_process_device *pdd; 1074 1075 if (!q->properties.is_suspended) 1076 return 0; 1077 1078 pdd = qpd_to_pdd(qpd); 1079 1080 pr_debug("Restoring from suspend PASID %u queue [%i]\n", 1081 pdd->process->pasid, 1082 q->properties.queue_id); 1083 1084 q->properties.is_suspended = false; 1085 1086 if (QUEUE_IS_ACTIVE(q->properties)) { 1087 if (dqm->dev->kfd->shared_resources.enable_mes) { 1088 int r = add_queue_mes(dqm, q, &pdd->qpd); 1089 1090 if (r) 1091 return r; 1092 } 1093 1094 q->properties.is_active = true; 1095 increment_queue_count(dqm, qpd, q); 1096 } 1097 1098 return 0; 1099 } 1100 1101 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1102 struct qcm_process_device *qpd) 1103 { 1104 struct queue *q; 1105 struct mqd_manager *mqd_mgr; 1106 struct kfd_process_device *pdd; 1107 int retval, ret = 0; 1108 1109 dqm_lock(dqm); 1110 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1111 goto out; 1112 1113 pdd = qpd_to_pdd(qpd); 1114 pr_debug_ratelimited("Evicting PASID 0x%x queues\n", 1115 pdd->process->pasid); 1116 1117 pdd->last_evict_timestamp = get_jiffies_64(); 1118 /* Mark all queues as evicted. Deactivate all active queues on 1119 * the qpd. 1120 */ 1121 list_for_each_entry(q, &qpd->queues_list, list) { 1122 q->properties.is_evicted = true; 1123 if (!q->properties.is_active) 1124 continue; 1125 1126 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1127 q->properties.type)]; 1128 q->properties.is_active = false; 1129 decrement_queue_count(dqm, qpd, q); 1130 1131 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1132 continue; 1133 1134 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1135 (dqm->dev->kfd->cwsr_enabled ? 1136 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1137 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1138 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1139 if (retval && !ret) 1140 /* Return the first error, but keep going to 1141 * maintain a consistent eviction state 1142 */ 1143 ret = retval; 1144 } 1145 1146 out: 1147 dqm_unlock(dqm); 1148 return ret; 1149 } 1150 1151 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1152 struct qcm_process_device *qpd) 1153 { 1154 struct queue *q; 1155 struct device *dev = dqm->dev->adev->dev; 1156 struct kfd_process_device *pdd; 1157 int retval = 0; 1158 1159 dqm_lock(dqm); 1160 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1161 goto out; 1162 1163 pdd = qpd_to_pdd(qpd); 1164 1165 /* The debugger creates processes that temporarily have not acquired 1166 * all VMs for all devices and has no VMs itself. 1167 * Skip queue eviction on process eviction. 1168 */ 1169 if (!pdd->drm_priv) 1170 goto out; 1171 1172 pr_debug_ratelimited("Evicting PASID 0x%x queues\n", 1173 pdd->process->pasid); 1174 1175 /* Mark all queues as evicted. Deactivate all active queues on 1176 * the qpd. 1177 */ 1178 list_for_each_entry(q, &qpd->queues_list, list) { 1179 q->properties.is_evicted = true; 1180 if (!q->properties.is_active) 1181 continue; 1182 1183 q->properties.is_active = false; 1184 decrement_queue_count(dqm, qpd, q); 1185 1186 if (dqm->dev->kfd->shared_resources.enable_mes) { 1187 retval = remove_queue_mes(dqm, q, qpd); 1188 if (retval) { 1189 dev_err(dev, "Failed to evict queue %d\n", 1190 q->properties.queue_id); 1191 goto out; 1192 } 1193 } 1194 } 1195 pdd->last_evict_timestamp = get_jiffies_64(); 1196 if (!dqm->dev->kfd->shared_resources.enable_mes) 1197 retval = execute_queues_cpsch(dqm, 1198 qpd->is_debug ? 1199 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1200 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1201 USE_DEFAULT_GRACE_PERIOD); 1202 1203 out: 1204 dqm_unlock(dqm); 1205 return retval; 1206 } 1207 1208 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1209 struct qcm_process_device *qpd) 1210 { 1211 struct mm_struct *mm = NULL; 1212 struct queue *q; 1213 struct mqd_manager *mqd_mgr; 1214 struct kfd_process_device *pdd; 1215 uint64_t pd_base; 1216 uint64_t eviction_duration; 1217 int retval, ret = 0; 1218 1219 pdd = qpd_to_pdd(qpd); 1220 /* Retrieve PD base */ 1221 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1222 1223 dqm_lock(dqm); 1224 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1225 goto out; 1226 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1227 qpd->evicted--; 1228 goto out; 1229 } 1230 1231 pr_debug_ratelimited("Restoring PASID 0x%x queues\n", 1232 pdd->process->pasid); 1233 1234 /* Update PD Base in QPD */ 1235 qpd->page_table_base = pd_base; 1236 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1237 1238 if (!list_empty(&qpd->queues_list)) { 1239 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1240 dqm->dev->adev, 1241 qpd->vmid, 1242 qpd->page_table_base); 1243 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); 1244 } 1245 1246 /* Take a safe reference to the mm_struct, which may otherwise 1247 * disappear even while the kfd_process is still referenced. 1248 */ 1249 mm = get_task_mm(pdd->process->lead_thread); 1250 if (!mm) { 1251 ret = -EFAULT; 1252 goto out; 1253 } 1254 1255 /* Remove the eviction flags. Activate queues that are not 1256 * inactive for other reasons. 1257 */ 1258 list_for_each_entry(q, &qpd->queues_list, list) { 1259 q->properties.is_evicted = false; 1260 if (!QUEUE_IS_ACTIVE(q->properties)) 1261 continue; 1262 1263 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1264 q->properties.type)]; 1265 q->properties.is_active = true; 1266 increment_queue_count(dqm, qpd, q); 1267 1268 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1269 continue; 1270 1271 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1272 q->queue, &q->properties, mm); 1273 if (retval && !ret) 1274 /* Return the first error, but keep going to 1275 * maintain a consistent eviction state 1276 */ 1277 ret = retval; 1278 } 1279 qpd->evicted = 0; 1280 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1281 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1282 out: 1283 if (mm) 1284 mmput(mm); 1285 dqm_unlock(dqm); 1286 return ret; 1287 } 1288 1289 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1290 struct qcm_process_device *qpd) 1291 { 1292 struct queue *q; 1293 struct device *dev = dqm->dev->adev->dev; 1294 struct kfd_process_device *pdd; 1295 uint64_t eviction_duration; 1296 int retval = 0; 1297 1298 pdd = qpd_to_pdd(qpd); 1299 1300 dqm_lock(dqm); 1301 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1302 goto out; 1303 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1304 qpd->evicted--; 1305 goto out; 1306 } 1307 1308 /* The debugger creates processes that temporarily have not acquired 1309 * all VMs for all devices and has no VMs itself. 1310 * Skip queue restore on process restore. 1311 */ 1312 if (!pdd->drm_priv) 1313 goto vm_not_acquired; 1314 1315 pr_debug_ratelimited("Restoring PASID 0x%x queues\n", 1316 pdd->process->pasid); 1317 1318 /* Update PD Base in QPD */ 1319 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1320 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1321 1322 /* activate all active queues on the qpd */ 1323 list_for_each_entry(q, &qpd->queues_list, list) { 1324 q->properties.is_evicted = false; 1325 if (!QUEUE_IS_ACTIVE(q->properties)) 1326 continue; 1327 1328 q->properties.is_active = true; 1329 increment_queue_count(dqm, &pdd->qpd, q); 1330 1331 if (dqm->dev->kfd->shared_resources.enable_mes) { 1332 retval = add_queue_mes(dqm, q, qpd); 1333 if (retval) { 1334 dev_err(dev, "Failed to restore queue %d\n", 1335 q->properties.queue_id); 1336 goto out; 1337 } 1338 } 1339 } 1340 if (!dqm->dev->kfd->shared_resources.enable_mes) 1341 retval = execute_queues_cpsch(dqm, 1342 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1343 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1344 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1345 vm_not_acquired: 1346 qpd->evicted = 0; 1347 out: 1348 dqm_unlock(dqm); 1349 return retval; 1350 } 1351 1352 static int register_process(struct device_queue_manager *dqm, 1353 struct qcm_process_device *qpd) 1354 { 1355 struct device_process_node *n; 1356 struct kfd_process_device *pdd; 1357 uint64_t pd_base; 1358 int retval; 1359 1360 n = kzalloc(sizeof(*n), GFP_KERNEL); 1361 if (!n) 1362 return -ENOMEM; 1363 1364 n->qpd = qpd; 1365 1366 pdd = qpd_to_pdd(qpd); 1367 /* Retrieve PD base */ 1368 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1369 1370 dqm_lock(dqm); 1371 list_add(&n->list, &dqm->queues); 1372 1373 /* Update PD Base in QPD */ 1374 qpd->page_table_base = pd_base; 1375 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1376 1377 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1378 1379 dqm->processes_count++; 1380 1381 dqm_unlock(dqm); 1382 1383 /* Outside the DQM lock because under the DQM lock we can't do 1384 * reclaim or take other locks that others hold while reclaiming. 1385 */ 1386 kfd_inc_compute_active(dqm->dev); 1387 1388 return retval; 1389 } 1390 1391 static int unregister_process(struct device_queue_manager *dqm, 1392 struct qcm_process_device *qpd) 1393 { 1394 int retval; 1395 struct device_process_node *cur, *next; 1396 1397 pr_debug("qpd->queues_list is %s\n", 1398 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1399 1400 retval = 0; 1401 dqm_lock(dqm); 1402 1403 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1404 if (qpd == cur->qpd) { 1405 list_del(&cur->list); 1406 kfree(cur); 1407 dqm->processes_count--; 1408 goto out; 1409 } 1410 } 1411 /* qpd not found in dqm list */ 1412 retval = 1; 1413 out: 1414 dqm_unlock(dqm); 1415 1416 /* Outside the DQM lock because under the DQM lock we can't do 1417 * reclaim or take other locks that others hold while reclaiming. 1418 */ 1419 if (!retval) 1420 kfd_dec_compute_active(dqm->dev); 1421 1422 return retval; 1423 } 1424 1425 static int 1426 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1427 unsigned int vmid) 1428 { 1429 uint32_t xcc_mask = dqm->dev->xcc_mask; 1430 int xcc_id, ret; 1431 1432 for_each_inst(xcc_id, xcc_mask) { 1433 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1434 dqm->dev->adev, pasid, vmid, xcc_id); 1435 if (ret) 1436 break; 1437 } 1438 1439 return ret; 1440 } 1441 1442 static void init_interrupts(struct device_queue_manager *dqm) 1443 { 1444 uint32_t xcc_mask = dqm->dev->xcc_mask; 1445 unsigned int i, xcc_id; 1446 1447 for_each_inst(xcc_id, xcc_mask) { 1448 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1449 if (is_pipe_enabled(dqm, 0, i)) { 1450 dqm->dev->kfd2kgd->init_interrupts( 1451 dqm->dev->adev, i, xcc_id); 1452 } 1453 } 1454 } 1455 } 1456 1457 static int initialize_nocpsch(struct device_queue_manager *dqm) 1458 { 1459 int pipe, queue; 1460 1461 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1462 1463 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), 1464 sizeof(unsigned int), GFP_KERNEL); 1465 if (!dqm->allocated_queues) 1466 return -ENOMEM; 1467 1468 mutex_init(&dqm->lock_hidden); 1469 INIT_LIST_HEAD(&dqm->queues); 1470 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1471 dqm->active_cp_queue_count = 0; 1472 dqm->gws_queue_count = 0; 1473 1474 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1475 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1476 1477 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1478 if (test_bit(pipe_offset + queue, 1479 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1480 dqm->allocated_queues[pipe] |= 1 << queue; 1481 } 1482 1483 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1484 1485 init_sdma_bitmaps(dqm); 1486 1487 return 0; 1488 } 1489 1490 static void uninitialize(struct device_queue_manager *dqm) 1491 { 1492 int i; 1493 1494 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1495 1496 kfree(dqm->allocated_queues); 1497 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1498 kfree(dqm->mqd_mgrs[i]); 1499 mutex_destroy(&dqm->lock_hidden); 1500 } 1501 1502 static int start_nocpsch(struct device_queue_manager *dqm) 1503 { 1504 int r = 0; 1505 1506 pr_info("SW scheduler is used"); 1507 init_interrupts(dqm); 1508 1509 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1510 r = pm_init(&dqm->packet_mgr, dqm); 1511 if (!r) 1512 dqm->sched_running = true; 1513 1514 return r; 1515 } 1516 1517 static int stop_nocpsch(struct device_queue_manager *dqm) 1518 { 1519 dqm_lock(dqm); 1520 if (!dqm->sched_running) { 1521 dqm_unlock(dqm); 1522 return 0; 1523 } 1524 1525 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1526 pm_uninit(&dqm->packet_mgr); 1527 dqm->sched_running = false; 1528 dqm_unlock(dqm); 1529 1530 return 0; 1531 } 1532 1533 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1534 struct queue *q, const uint32_t *restore_sdma_id) 1535 { 1536 struct device *dev = dqm->dev->adev->dev; 1537 int bit; 1538 1539 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1540 if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { 1541 dev_err(dev, "No more SDMA queue to allocate\n"); 1542 return -ENOMEM; 1543 } 1544 1545 if (restore_sdma_id) { 1546 /* Re-use existing sdma_id */ 1547 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1548 dev_err(dev, "SDMA queue already in use\n"); 1549 return -EBUSY; 1550 } 1551 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1552 q->sdma_id = *restore_sdma_id; 1553 } else { 1554 /* Find first available sdma_id */ 1555 bit = find_first_bit(dqm->sdma_bitmap, 1556 get_num_sdma_queues(dqm)); 1557 clear_bit(bit, dqm->sdma_bitmap); 1558 q->sdma_id = bit; 1559 } 1560 1561 q->properties.sdma_engine_id = 1562 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1563 q->properties.sdma_queue_id = q->sdma_id / 1564 kfd_get_num_sdma_engines(dqm->dev); 1565 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1566 if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) { 1567 dev_err(dev, "No more XGMI SDMA queue to allocate\n"); 1568 return -ENOMEM; 1569 } 1570 if (restore_sdma_id) { 1571 /* Re-use existing sdma_id */ 1572 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1573 dev_err(dev, "SDMA queue already in use\n"); 1574 return -EBUSY; 1575 } 1576 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1577 q->sdma_id = *restore_sdma_id; 1578 } else { 1579 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1580 get_num_xgmi_sdma_queues(dqm)); 1581 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1582 q->sdma_id = bit; 1583 } 1584 /* sdma_engine_id is sdma id including 1585 * both PCIe-optimized SDMAs and XGMI- 1586 * optimized SDMAs. The calculation below 1587 * assumes the first N engines are always 1588 * PCIe-optimized ones 1589 */ 1590 q->properties.sdma_engine_id = 1591 kfd_get_num_sdma_engines(dqm->dev) + 1592 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1593 q->properties.sdma_queue_id = q->sdma_id / 1594 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1595 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1596 int i, num_queues, num_engines, eng_offset = 0, start_engine; 1597 bool free_bit_found = false, is_xgmi = false; 1598 1599 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { 1600 num_queues = get_num_sdma_queues(dqm); 1601 num_engines = kfd_get_num_sdma_engines(dqm->dev); 1602 q->properties.type = KFD_QUEUE_TYPE_SDMA; 1603 } else { 1604 num_queues = get_num_xgmi_sdma_queues(dqm); 1605 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); 1606 eng_offset = kfd_get_num_sdma_engines(dqm->dev); 1607 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI; 1608 is_xgmi = true; 1609 } 1610 1611 /* Scan available bit based on target engine ID. */ 1612 start_engine = q->properties.sdma_engine_id - eng_offset; 1613 for (i = start_engine; i < num_queues; i += num_engines) { 1614 1615 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) 1616 continue; 1617 1618 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); 1619 q->sdma_id = i; 1620 q->properties.sdma_queue_id = q->sdma_id / num_engines; 1621 free_bit_found = true; 1622 break; 1623 } 1624 1625 if (!free_bit_found) { 1626 dev_err(dev, "No more SDMA queue to allocate for target ID %i\n", 1627 q->properties.sdma_engine_id); 1628 return -ENOMEM; 1629 } 1630 } 1631 1632 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 1633 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 1634 1635 return 0; 1636 } 1637 1638 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 1639 struct queue *q) 1640 { 1641 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1642 if (q->sdma_id >= get_num_sdma_queues(dqm)) 1643 return; 1644 set_bit(q->sdma_id, dqm->sdma_bitmap); 1645 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1646 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1647 return; 1648 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 1649 } 1650 } 1651 1652 /* 1653 * Device Queue Manager implementation for cp scheduler 1654 */ 1655 1656 static int set_sched_resources(struct device_queue_manager *dqm) 1657 { 1658 int i, mec; 1659 struct scheduling_resources res; 1660 struct device *dev = dqm->dev->adev->dev; 1661 1662 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 1663 1664 res.queue_mask = 0; 1665 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 1666 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 1667 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 1668 1669 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1670 continue; 1671 1672 /* only acquire queues from the first MEC */ 1673 if (mec > 0) 1674 continue; 1675 1676 /* This situation may be hit in the future if a new HW 1677 * generation exposes more than 64 queues. If so, the 1678 * definition of res.queue_mask needs updating 1679 */ 1680 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 1681 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 1682 break; 1683 } 1684 1685 res.queue_mask |= 1ull 1686 << amdgpu_queue_mask_bit_to_set_resource_bit( 1687 dqm->dev->adev, i); 1688 } 1689 res.gws_mask = ~0ull; 1690 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 1691 1692 pr_debug("Scheduling resources:\n" 1693 "vmid mask: 0x%8X\n" 1694 "queue mask: 0x%8llX\n", 1695 res.vmid_mask, res.queue_mask); 1696 1697 return pm_send_set_resources(&dqm->packet_mgr, &res); 1698 } 1699 1700 static int initialize_cpsch(struct device_queue_manager *dqm) 1701 { 1702 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1703 1704 mutex_init(&dqm->lock_hidden); 1705 INIT_LIST_HEAD(&dqm->queues); 1706 dqm->active_queue_count = dqm->processes_count = 0; 1707 dqm->active_cp_queue_count = 0; 1708 dqm->gws_queue_count = 0; 1709 dqm->active_runlist = false; 1710 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); 1711 dqm->trap_debug_vmid = 0; 1712 1713 init_sdma_bitmaps(dqm); 1714 1715 if (dqm->dev->kfd2kgd->get_iq_wait_times) 1716 dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev, 1717 &dqm->wait_times, 1718 ffs(dqm->dev->xcc_mask) - 1); 1719 return 0; 1720 } 1721 1722 /* halt_cpsch: 1723 * Unmap queues so the schedule doesn't continue remaining jobs in the queue. 1724 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch 1725 * is called. 1726 */ 1727 static int halt_cpsch(struct device_queue_manager *dqm) 1728 { 1729 int ret = 0; 1730 1731 dqm_lock(dqm); 1732 if (!dqm->sched_running) { 1733 dqm_unlock(dqm); 1734 return 0; 1735 } 1736 1737 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); 1738 1739 if (!dqm->is_hws_hang) { 1740 if (!dqm->dev->kfd->shared_resources.enable_mes) 1741 ret = unmap_queues_cpsch(dqm, 1742 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1743 USE_DEFAULT_GRACE_PERIOD, false); 1744 else 1745 ret = remove_all_queues_mes(dqm); 1746 } 1747 dqm->sched_halt = true; 1748 dqm_unlock(dqm); 1749 1750 return ret; 1751 } 1752 1753 /* unhalt_cpsch 1754 * Unset dqm->sched_halt and map queues back to runlist 1755 */ 1756 static int unhalt_cpsch(struct device_queue_manager *dqm) 1757 { 1758 int ret = 0; 1759 1760 dqm_lock(dqm); 1761 if (!dqm->sched_running || !dqm->sched_halt) { 1762 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n"); 1763 dqm_unlock(dqm); 1764 return 0; 1765 } 1766 dqm->sched_halt = false; 1767 if (!dqm->dev->kfd->shared_resources.enable_mes) 1768 ret = execute_queues_cpsch(dqm, 1769 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 1770 0, USE_DEFAULT_GRACE_PERIOD); 1771 dqm_unlock(dqm); 1772 1773 return ret; 1774 } 1775 1776 static int start_cpsch(struct device_queue_manager *dqm) 1777 { 1778 struct device *dev = dqm->dev->adev->dev; 1779 int retval, num_hw_queue_slots; 1780 1781 retval = 0; 1782 1783 dqm_lock(dqm); 1784 1785 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1786 retval = pm_init(&dqm->packet_mgr, dqm); 1787 if (retval) 1788 goto fail_packet_manager_init; 1789 1790 retval = set_sched_resources(dqm); 1791 if (retval) 1792 goto fail_set_sched_resources; 1793 } 1794 pr_debug("Allocating fence memory\n"); 1795 1796 /* allocate fence memory on the gart */ 1797 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 1798 &dqm->fence_mem); 1799 1800 if (retval) 1801 goto fail_allocate_vidmem; 1802 1803 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 1804 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 1805 1806 init_interrupts(dqm); 1807 1808 /* clear hang status when driver try to start the hw scheduler */ 1809 dqm->sched_running = true; 1810 1811 if (!dqm->dev->kfd->shared_resources.enable_mes) 1812 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1813 1814 /* Set CWSR grace period to 1x1000 cycle for GFX9.4.3 APU */ 1815 if (amdgpu_emu_mode == 0 && dqm->dev->adev->gmc.is_app_apu && 1816 (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3))) { 1817 uint32_t reg_offset = 0; 1818 uint32_t grace_period = 1; 1819 1820 retval = pm_update_grace_period(&dqm->packet_mgr, 1821 grace_period); 1822 if (retval) 1823 dev_err(dev, "Setting grace timeout failed\n"); 1824 else if (dqm->dev->kfd2kgd->build_grace_period_packet_info) 1825 /* Update dqm->wait_times maintained in software */ 1826 dqm->dev->kfd2kgd->build_grace_period_packet_info( 1827 dqm->dev->adev, dqm->wait_times, 1828 grace_period, ®_offset, 1829 &dqm->wait_times); 1830 } 1831 1832 /* setup per-queue reset detection buffer */ 1833 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * 1834 dqm->dev->kfd->shared_resources.num_pipe_per_mec * 1835 NUM_XCC(dqm->dev->xcc_mask); 1836 1837 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); 1838 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); 1839 1840 if (!dqm->detect_hang_info) { 1841 retval = -ENOMEM; 1842 goto fail_detect_hang_buffer; 1843 } 1844 1845 dqm_unlock(dqm); 1846 1847 return 0; 1848 fail_detect_hang_buffer: 1849 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1850 fail_allocate_vidmem: 1851 fail_set_sched_resources: 1852 if (!dqm->dev->kfd->shared_resources.enable_mes) 1853 pm_uninit(&dqm->packet_mgr); 1854 fail_packet_manager_init: 1855 dqm_unlock(dqm); 1856 return retval; 1857 } 1858 1859 static int stop_cpsch(struct device_queue_manager *dqm) 1860 { 1861 dqm_lock(dqm); 1862 if (!dqm->sched_running) { 1863 dqm_unlock(dqm); 1864 return 0; 1865 } 1866 1867 if (!dqm->dev->kfd->shared_resources.enable_mes) 1868 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 1869 else 1870 remove_all_queues_mes(dqm); 1871 1872 dqm->sched_running = false; 1873 1874 if (!dqm->dev->kfd->shared_resources.enable_mes) 1875 pm_release_ib(&dqm->packet_mgr); 1876 1877 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 1878 if (!dqm->dev->kfd->shared_resources.enable_mes) 1879 pm_uninit(&dqm->packet_mgr); 1880 kfree(dqm->detect_hang_info); 1881 dqm->detect_hang_info = NULL; 1882 dqm_unlock(dqm); 1883 1884 return 0; 1885 } 1886 1887 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 1888 struct kernel_queue *kq, 1889 struct qcm_process_device *qpd) 1890 { 1891 dqm_lock(dqm); 1892 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1893 pr_warn("Can't create new kernel queue because %d queues were already created\n", 1894 dqm->total_queue_count); 1895 dqm_unlock(dqm); 1896 return -EPERM; 1897 } 1898 1899 /* 1900 * Unconditionally increment this counter, regardless of the queue's 1901 * type or whether the queue is active. 1902 */ 1903 dqm->total_queue_count++; 1904 pr_debug("Total of %d queues are accountable so far\n", 1905 dqm->total_queue_count); 1906 1907 list_add(&kq->list, &qpd->priv_queue_list); 1908 increment_queue_count(dqm, qpd, kq->queue); 1909 qpd->is_debug = true; 1910 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1911 USE_DEFAULT_GRACE_PERIOD); 1912 dqm_unlock(dqm); 1913 1914 return 0; 1915 } 1916 1917 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 1918 struct kernel_queue *kq, 1919 struct qcm_process_device *qpd) 1920 { 1921 dqm_lock(dqm); 1922 list_del(&kq->list); 1923 decrement_queue_count(dqm, qpd, kq->queue); 1924 qpd->is_debug = false; 1925 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 1926 USE_DEFAULT_GRACE_PERIOD); 1927 /* 1928 * Unconditionally decrement this counter, regardless of the queue's 1929 * type. 1930 */ 1931 dqm->total_queue_count--; 1932 pr_debug("Total of %d queues are accountable so far\n", 1933 dqm->total_queue_count); 1934 dqm_unlock(dqm); 1935 } 1936 1937 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 1938 struct qcm_process_device *qpd, 1939 const struct kfd_criu_queue_priv_data *qd, 1940 const void *restore_mqd, const void *restore_ctl_stack) 1941 { 1942 int retval; 1943 struct mqd_manager *mqd_mgr; 1944 1945 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 1946 pr_warn("Can't create new usermode queue because %d queues were already created\n", 1947 dqm->total_queue_count); 1948 retval = -EPERM; 1949 goto out; 1950 } 1951 1952 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1953 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI || 1954 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1955 dqm_lock(dqm); 1956 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 1957 dqm_unlock(dqm); 1958 if (retval) 1959 goto out; 1960 } 1961 1962 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 1963 if (retval) 1964 goto out_deallocate_sdma_queue; 1965 1966 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1967 q->properties.type)]; 1968 1969 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 1970 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 1971 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 1972 q->properties.tba_addr = qpd->tba_addr; 1973 q->properties.tma_addr = qpd->tma_addr; 1974 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties); 1975 if (!q->mqd_mem_obj) { 1976 retval = -ENOMEM; 1977 goto out_deallocate_doorbell; 1978 } 1979 1980 dqm_lock(dqm); 1981 /* 1982 * Eviction state logic: mark all queues as evicted, even ones 1983 * not currently active. Restoring inactive queues later only 1984 * updates the is_evicted flag but is a no-op otherwise. 1985 */ 1986 q->properties.is_evicted = !!qpd->evicted; 1987 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 1988 kfd_dbg_has_cwsr_workaround(q->device); 1989 1990 if (qd) 1991 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 1992 &q->properties, restore_mqd, restore_ctl_stack, 1993 qd->ctl_stack_size); 1994 else 1995 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 1996 &q->gart_mqd_addr, &q->properties); 1997 1998 list_add(&q->list, &qpd->queues_list); 1999 qpd->queue_count++; 2000 2001 if (q->properties.is_active) { 2002 increment_queue_count(dqm, qpd, q); 2003 2004 if (!dqm->dev->kfd->shared_resources.enable_mes) 2005 retval = execute_queues_cpsch(dqm, 2006 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2007 else 2008 retval = add_queue_mes(dqm, q, qpd); 2009 if (retval) 2010 goto cleanup_queue; 2011 } 2012 2013 /* 2014 * Unconditionally increment this counter, regardless of the queue's 2015 * type or whether the queue is active. 2016 */ 2017 dqm->total_queue_count++; 2018 2019 pr_debug("Total of %d queues are accountable so far\n", 2020 dqm->total_queue_count); 2021 2022 dqm_unlock(dqm); 2023 return retval; 2024 2025 cleanup_queue: 2026 qpd->queue_count--; 2027 list_del(&q->list); 2028 if (q->properties.is_active) 2029 decrement_queue_count(dqm, qpd, q); 2030 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2031 dqm_unlock(dqm); 2032 out_deallocate_doorbell: 2033 deallocate_doorbell(qpd, q); 2034 out_deallocate_sdma_queue: 2035 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2036 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2037 dqm_lock(dqm); 2038 deallocate_sdma_queue(dqm, q); 2039 dqm_unlock(dqm); 2040 } 2041 out: 2042 return retval; 2043 } 2044 2045 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 2046 uint64_t fence_value, 2047 unsigned int timeout_ms) 2048 { 2049 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 2050 struct device *dev = dqm->dev->adev->dev; 2051 uint64_t *fence_addr = dqm->fence_addr; 2052 2053 while (*fence_addr != fence_value) { 2054 /* Fatal err detected, this response won't come */ 2055 if (amdgpu_amdkfd_is_fed(dqm->dev->adev)) 2056 return -EIO; 2057 2058 if (time_after(jiffies, end_jiffies)) { 2059 dev_err(dev, "qcm fence wait loop timeout expired\n"); 2060 /* In HWS case, this is used to halt the driver thread 2061 * in order not to mess up CP states before doing 2062 * scandumps for FW debugging. 2063 */ 2064 while (halt_if_hws_hang) 2065 schedule(); 2066 2067 return -ETIME; 2068 } 2069 schedule(); 2070 } 2071 2072 return 0; 2073 } 2074 2075 /* dqm->lock mutex has to be locked before calling this function */ 2076 static int map_queues_cpsch(struct device_queue_manager *dqm) 2077 { 2078 struct device *dev = dqm->dev->adev->dev; 2079 int retval; 2080 2081 if (!dqm->sched_running || dqm->sched_halt) 2082 return 0; 2083 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 2084 return 0; 2085 if (dqm->active_runlist) 2086 return 0; 2087 2088 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 2089 pr_debug("%s sent runlist\n", __func__); 2090 if (retval) { 2091 dev_err(dev, "failed to execute runlist\n"); 2092 return retval; 2093 } 2094 dqm->active_runlist = true; 2095 2096 return retval; 2097 } 2098 2099 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 2100 struct qcm_process_device *qpd) 2101 { 2102 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2103 2104 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid 0x%0x is reset\n", 2105 q->properties.queue_id, q->process->pasid); 2106 2107 pdd->has_reset_queue = true; 2108 if (q->properties.is_active) { 2109 q->properties.is_active = false; 2110 decrement_queue_count(dqm, qpd, q); 2111 } 2112 } 2113 2114 static int detect_queue_hang(struct device_queue_manager *dqm) 2115 { 2116 int i; 2117 2118 /* detect should be used only in dqm locked queue reset */ 2119 if (WARN_ON(dqm->detect_hang_count > 0)) 2120 return 0; 2121 2122 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); 2123 2124 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2125 uint32_t mec, pipe, queue; 2126 int xcc_id; 2127 2128 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2129 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2130 2131 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2132 continue; 2133 2134 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); 2135 2136 for_each_inst(xcc_id, dqm->dev->xcc_mask) { 2137 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( 2138 dqm->dev->adev, pipe, queue, xcc_id); 2139 struct dqm_detect_hang_info hang_info; 2140 2141 if (!queue_addr) 2142 continue; 2143 2144 hang_info.pipe_id = pipe; 2145 hang_info.queue_id = queue; 2146 hang_info.xcc_id = xcc_id; 2147 hang_info.queue_address = queue_addr; 2148 2149 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; 2150 dqm->detect_hang_count++; 2151 } 2152 } 2153 2154 return dqm->detect_hang_count; 2155 } 2156 2157 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) 2158 { 2159 struct device_process_node *cur; 2160 struct qcm_process_device *qpd; 2161 struct queue *q; 2162 2163 list_for_each_entry(cur, &dqm->queues, list) { 2164 qpd = cur->qpd; 2165 list_for_each_entry(q, &qpd->queues_list, list) { 2166 if (queue_address == q->properties.queue_address) 2167 return q; 2168 } 2169 } 2170 2171 return NULL; 2172 } 2173 2174 /* only for compute queue */ 2175 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm) 2176 { 2177 int r = 0, reset_count = 0, i; 2178 2179 if (!dqm->detect_hang_info || dqm->is_hws_hang) 2180 return -EIO; 2181 2182 /* assume dqm locked. */ 2183 if (!detect_queue_hang(dqm)) 2184 return -ENOTRECOVERABLE; 2185 2186 for (i = 0; i < dqm->detect_hang_count; i++) { 2187 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; 2188 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); 2189 struct kfd_process_device *pdd; 2190 uint64_t queue_addr = 0; 2191 2192 if (!q) { 2193 r = -ENOTRECOVERABLE; 2194 goto reset_fail; 2195 } 2196 2197 pdd = kfd_get_process_device_data(dqm->dev, q->process); 2198 if (!pdd) { 2199 r = -ENOTRECOVERABLE; 2200 goto reset_fail; 2201 } 2202 2203 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, 2204 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id, 2205 KFD_UNMAP_LATENCY_MS); 2206 2207 /* either reset failed or we reset an unexpected queue. */ 2208 if (queue_addr != q->properties.queue_address) { 2209 r = -ENOTRECOVERABLE; 2210 goto reset_fail; 2211 } 2212 2213 set_queue_as_reset(dqm, q, &pdd->qpd); 2214 reset_count++; 2215 } 2216 2217 if (reset_count == dqm->detect_hang_count) 2218 kfd_signal_reset_event(dqm->dev); 2219 else 2220 r = -ENOTRECOVERABLE; 2221 2222 reset_fail: 2223 dqm->detect_hang_count = 0; 2224 2225 return r; 2226 } 2227 2228 /* dqm->lock mutex has to be locked before calling this function */ 2229 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 2230 enum kfd_unmap_queues_filter filter, 2231 uint32_t filter_param, 2232 uint32_t grace_period, 2233 bool reset) 2234 { 2235 struct device *dev = dqm->dev->adev->dev; 2236 struct mqd_manager *mqd_mgr; 2237 int retval; 2238 2239 if (!dqm->sched_running) 2240 return 0; 2241 if (!dqm->active_runlist) 2242 return 0; 2243 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2244 return -EIO; 2245 2246 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2247 retval = pm_update_grace_period(&dqm->packet_mgr, grace_period); 2248 if (retval) 2249 goto out; 2250 } 2251 2252 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 2253 if (retval) 2254 goto out; 2255 2256 *dqm->fence_addr = KFD_FENCE_INIT; 2257 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 2258 KFD_FENCE_COMPLETED); 2259 /* should be timed out */ 2260 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 2261 queue_preemption_timeout_ms); 2262 if (retval) { 2263 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 2264 kfd_hws_hang(dqm); 2265 goto out; 2266 } 2267 2268 /* In the current MEC firmware implementation, if compute queue 2269 * doesn't response to the preemption request in time, HIQ will 2270 * abandon the unmap request without returning any timeout error 2271 * to driver. Instead, MEC firmware will log the doorbell of the 2272 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 2273 * To make sure the queue unmap was successful, driver need to 2274 * check those fields 2275 */ 2276 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 2277 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) { 2278 if (reset_queues_on_hws_hang(dqm)) { 2279 while (halt_if_hws_hang) 2280 schedule(); 2281 dqm->is_hws_hang = true; 2282 kfd_hws_hang(dqm); 2283 retval = -ETIME; 2284 goto out; 2285 } 2286 } 2287 2288 /* We need to reset the grace period value for this device */ 2289 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2290 if (pm_update_grace_period(&dqm->packet_mgr, 2291 USE_DEFAULT_GRACE_PERIOD)) 2292 dev_err(dev, "Failed to reset grace period\n"); 2293 } 2294 2295 pm_release_ib(&dqm->packet_mgr); 2296 dqm->active_runlist = false; 2297 2298 out: 2299 up_read(&dqm->dev->adev->reset_domain->sem); 2300 return retval; 2301 } 2302 2303 /* only for compute queue */ 2304 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) 2305 { 2306 int retval; 2307 2308 dqm_lock(dqm); 2309 2310 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2311 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2312 2313 dqm_unlock(dqm); 2314 return retval; 2315 } 2316 2317 /* dqm->lock mutex has to be locked before calling this function */ 2318 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2319 enum kfd_unmap_queues_filter filter, 2320 uint32_t filter_param, 2321 uint32_t grace_period) 2322 { 2323 int retval; 2324 2325 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2326 return -EIO; 2327 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2328 if (!retval) 2329 retval = map_queues_cpsch(dqm); 2330 up_read(&dqm->dev->adev->reset_domain->sem); 2331 return retval; 2332 } 2333 2334 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2335 struct queue *q) 2336 { 2337 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2338 q->process); 2339 int ret = 0; 2340 2341 if (pdd->qpd.is_debug) 2342 return ret; 2343 2344 q->properties.is_being_destroyed = true; 2345 2346 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2347 dqm_unlock(dqm); 2348 mutex_unlock(&q->process->mutex); 2349 ret = wait_event_interruptible(dqm->destroy_wait, 2350 !q->properties.is_suspended); 2351 2352 mutex_lock(&q->process->mutex); 2353 dqm_lock(dqm); 2354 } 2355 2356 return ret; 2357 } 2358 2359 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2360 struct qcm_process_device *qpd, 2361 struct queue *q) 2362 { 2363 int retval; 2364 struct mqd_manager *mqd_mgr; 2365 uint64_t sdma_val = 0; 2366 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2367 struct device *dev = dqm->dev->adev->dev; 2368 2369 /* Get the SDMA queue stats */ 2370 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2371 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2372 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, 2373 &sdma_val); 2374 if (retval) 2375 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2376 q->properties.queue_id); 2377 } 2378 2379 /* remove queue from list to prevent rescheduling after preemption */ 2380 dqm_lock(dqm); 2381 2382 retval = wait_on_destroy_queue(dqm, q); 2383 2384 if (retval) { 2385 dqm_unlock(dqm); 2386 return retval; 2387 } 2388 2389 if (qpd->is_debug) { 2390 /* 2391 * error, currently we do not allow to destroy a queue 2392 * of a currently debugged process 2393 */ 2394 retval = -EBUSY; 2395 goto failed_try_destroy_debugged_queue; 2396 2397 } 2398 2399 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2400 q->properties.type)]; 2401 2402 deallocate_doorbell(qpd, q); 2403 2404 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2405 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2406 deallocate_sdma_queue(dqm, q); 2407 pdd->sdma_past_activity_counter += sdma_val; 2408 } 2409 2410 list_del(&q->list); 2411 qpd->queue_count--; 2412 if (q->properties.is_active) { 2413 decrement_queue_count(dqm, qpd, q); 2414 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2415 retval = execute_queues_cpsch(dqm, 2416 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2417 USE_DEFAULT_GRACE_PERIOD); 2418 if (retval == -ETIME) 2419 qpd->reset_wavefronts = true; 2420 } else { 2421 retval = remove_queue_mes(dqm, q, qpd); 2422 } 2423 } 2424 2425 /* 2426 * Unconditionally decrement this counter, regardless of the queue's 2427 * type 2428 */ 2429 dqm->total_queue_count--; 2430 pr_debug("Total of %d queues are accountable so far\n", 2431 dqm->total_queue_count); 2432 2433 dqm_unlock(dqm); 2434 2435 /* 2436 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 2437 * circular locking 2438 */ 2439 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 2440 qpd->pqm->process, q->device, 2441 -1, false, NULL, 0); 2442 2443 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2444 2445 return retval; 2446 2447 failed_try_destroy_debugged_queue: 2448 2449 dqm_unlock(dqm); 2450 return retval; 2451 } 2452 2453 /* 2454 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 2455 * stay in user mode. 2456 */ 2457 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 2458 /* APE1 limit is inclusive and 64K aligned. */ 2459 #define APE1_LIMIT_ALIGNMENT 0xFFFF 2460 2461 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 2462 struct qcm_process_device *qpd, 2463 enum cache_policy default_policy, 2464 enum cache_policy alternate_policy, 2465 void __user *alternate_aperture_base, 2466 uint64_t alternate_aperture_size) 2467 { 2468 bool retval = true; 2469 2470 if (!dqm->asic_ops.set_cache_memory_policy) 2471 return retval; 2472 2473 dqm_lock(dqm); 2474 2475 if (alternate_aperture_size == 0) { 2476 /* base > limit disables APE1 */ 2477 qpd->sh_mem_ape1_base = 1; 2478 qpd->sh_mem_ape1_limit = 0; 2479 } else { 2480 /* 2481 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 2482 * SH_MEM_APE1_BASE[31:0], 0x0000 } 2483 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 2484 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 2485 * Verify that the base and size parameters can be 2486 * represented in this format and convert them. 2487 * Additionally restrict APE1 to user-mode addresses. 2488 */ 2489 2490 uint64_t base = (uintptr_t)alternate_aperture_base; 2491 uint64_t limit = base + alternate_aperture_size - 1; 2492 2493 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 || 2494 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) { 2495 retval = false; 2496 goto out; 2497 } 2498 2499 qpd->sh_mem_ape1_base = base >> 16; 2500 qpd->sh_mem_ape1_limit = limit >> 16; 2501 } 2502 2503 retval = dqm->asic_ops.set_cache_memory_policy( 2504 dqm, 2505 qpd, 2506 default_policy, 2507 alternate_policy, 2508 alternate_aperture_base, 2509 alternate_aperture_size); 2510 2511 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 2512 program_sh_mem_settings(dqm, qpd); 2513 2514 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 2515 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 2516 qpd->sh_mem_ape1_limit); 2517 2518 out: 2519 dqm_unlock(dqm); 2520 return retval; 2521 } 2522 2523 static int process_termination_nocpsch(struct device_queue_manager *dqm, 2524 struct qcm_process_device *qpd) 2525 { 2526 struct queue *q; 2527 struct device_process_node *cur, *next_dpn; 2528 int retval = 0; 2529 bool found = false; 2530 2531 dqm_lock(dqm); 2532 2533 /* Clear all user mode queues */ 2534 while (!list_empty(&qpd->queues_list)) { 2535 struct mqd_manager *mqd_mgr; 2536 int ret; 2537 2538 q = list_first_entry(&qpd->queues_list, struct queue, list); 2539 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2540 q->properties.type)]; 2541 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 2542 if (ret) 2543 retval = ret; 2544 dqm_unlock(dqm); 2545 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2546 dqm_lock(dqm); 2547 } 2548 2549 /* Unregister process */ 2550 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2551 if (qpd == cur->qpd) { 2552 list_del(&cur->list); 2553 kfree(cur); 2554 dqm->processes_count--; 2555 found = true; 2556 break; 2557 } 2558 } 2559 2560 dqm_unlock(dqm); 2561 2562 /* Outside the DQM lock because under the DQM lock we can't do 2563 * reclaim or take other locks that others hold while reclaiming. 2564 */ 2565 if (found) 2566 kfd_dec_compute_active(dqm->dev); 2567 2568 return retval; 2569 } 2570 2571 static int get_wave_state(struct device_queue_manager *dqm, 2572 struct queue *q, 2573 void __user *ctl_stack, 2574 u32 *ctl_stack_used_size, 2575 u32 *save_area_used_size) 2576 { 2577 struct mqd_manager *mqd_mgr; 2578 2579 dqm_lock(dqm); 2580 2581 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 2582 2583 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 2584 q->properties.is_active || !q->device->kfd->cwsr_enabled || 2585 !mqd_mgr->get_wave_state) { 2586 dqm_unlock(dqm); 2587 return -EINVAL; 2588 } 2589 2590 dqm_unlock(dqm); 2591 2592 /* 2593 * get_wave_state is outside the dqm lock to prevent circular locking 2594 * and the queue should be protected against destruction by the process 2595 * lock. 2596 */ 2597 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 2598 ctl_stack, ctl_stack_used_size, save_area_used_size); 2599 } 2600 2601 static void get_queue_checkpoint_info(struct device_queue_manager *dqm, 2602 const struct queue *q, 2603 u32 *mqd_size, 2604 u32 *ctl_stack_size) 2605 { 2606 struct mqd_manager *mqd_mgr; 2607 enum KFD_MQD_TYPE mqd_type = 2608 get_mqd_type_from_queue_type(q->properties.type); 2609 2610 dqm_lock(dqm); 2611 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2612 *mqd_size = mqd_mgr->mqd_size; 2613 *ctl_stack_size = 0; 2614 2615 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 2616 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 2617 2618 dqm_unlock(dqm); 2619 } 2620 2621 static int checkpoint_mqd(struct device_queue_manager *dqm, 2622 const struct queue *q, 2623 void *mqd, 2624 void *ctl_stack) 2625 { 2626 struct mqd_manager *mqd_mgr; 2627 int r = 0; 2628 enum KFD_MQD_TYPE mqd_type = 2629 get_mqd_type_from_queue_type(q->properties.type); 2630 2631 dqm_lock(dqm); 2632 2633 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 2634 r = -EINVAL; 2635 goto dqm_unlock; 2636 } 2637 2638 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 2639 if (!mqd_mgr->checkpoint_mqd) { 2640 r = -EOPNOTSUPP; 2641 goto dqm_unlock; 2642 } 2643 2644 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 2645 2646 dqm_unlock: 2647 dqm_unlock(dqm); 2648 return r; 2649 } 2650 2651 static int process_termination_cpsch(struct device_queue_manager *dqm, 2652 struct qcm_process_device *qpd) 2653 { 2654 int retval; 2655 struct queue *q; 2656 struct device *dev = dqm->dev->adev->dev; 2657 struct kernel_queue *kq, *kq_next; 2658 struct mqd_manager *mqd_mgr; 2659 struct device_process_node *cur, *next_dpn; 2660 enum kfd_unmap_queues_filter filter = 2661 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 2662 bool found = false; 2663 2664 retval = 0; 2665 2666 dqm_lock(dqm); 2667 2668 /* Clean all kernel queues */ 2669 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 2670 list_del(&kq->list); 2671 decrement_queue_count(dqm, qpd, kq->queue); 2672 qpd->is_debug = false; 2673 dqm->total_queue_count--; 2674 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 2675 } 2676 2677 /* Clear all user mode queues */ 2678 list_for_each_entry(q, &qpd->queues_list, list) { 2679 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 2680 deallocate_sdma_queue(dqm, q); 2681 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2682 deallocate_sdma_queue(dqm, q); 2683 2684 if (q->properties.is_active) { 2685 decrement_queue_count(dqm, qpd, q); 2686 2687 if (dqm->dev->kfd->shared_resources.enable_mes) { 2688 retval = remove_queue_mes(dqm, q, qpd); 2689 if (retval) 2690 dev_err(dev, "Failed to remove queue %d\n", 2691 q->properties.queue_id); 2692 } 2693 } 2694 2695 dqm->total_queue_count--; 2696 } 2697 2698 /* Unregister process */ 2699 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 2700 if (qpd == cur->qpd) { 2701 list_del(&cur->list); 2702 kfree(cur); 2703 dqm->processes_count--; 2704 found = true; 2705 break; 2706 } 2707 } 2708 2709 if (!dqm->dev->kfd->shared_resources.enable_mes) 2710 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 2711 2712 if ((retval || qpd->reset_wavefronts) && 2713 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 2714 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 2715 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 2716 qpd->reset_wavefronts = false; 2717 up_read(&dqm->dev->adev->reset_domain->sem); 2718 } 2719 2720 /* Lastly, free mqd resources. 2721 * Do free_mqd() after dqm_unlock to avoid circular locking. 2722 */ 2723 while (!list_empty(&qpd->queues_list)) { 2724 q = list_first_entry(&qpd->queues_list, struct queue, list); 2725 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2726 q->properties.type)]; 2727 list_del(&q->list); 2728 qpd->queue_count--; 2729 dqm_unlock(dqm); 2730 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2731 dqm_lock(dqm); 2732 } 2733 dqm_unlock(dqm); 2734 2735 /* Outside the DQM lock because under the DQM lock we can't do 2736 * reclaim or take other locks that others hold while reclaiming. 2737 */ 2738 if (found) 2739 kfd_dec_compute_active(dqm->dev); 2740 2741 return retval; 2742 } 2743 2744 static int init_mqd_managers(struct device_queue_manager *dqm) 2745 { 2746 int i, j; 2747 struct device *dev = dqm->dev->adev->dev; 2748 struct mqd_manager *mqd_mgr; 2749 2750 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 2751 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 2752 if (!mqd_mgr) { 2753 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 2754 goto out_free; 2755 } 2756 dqm->mqd_mgrs[i] = mqd_mgr; 2757 } 2758 2759 return 0; 2760 2761 out_free: 2762 for (j = 0; j < i; j++) { 2763 kfree(dqm->mqd_mgrs[j]); 2764 dqm->mqd_mgrs[j] = NULL; 2765 } 2766 2767 return -ENOMEM; 2768 } 2769 2770 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 2771 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 2772 { 2773 int retval; 2774 struct kfd_node *dev = dqm->dev; 2775 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 2776 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 2777 get_num_all_sdma_engines(dqm) * 2778 dev->kfd->device_info.num_sdma_queues_per_engine + 2779 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 2780 NUM_XCC(dqm->dev->xcc_mask)); 2781 2782 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size, 2783 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr), 2784 (void *)&(mem_obj->cpu_ptr), false); 2785 2786 return retval; 2787 } 2788 2789 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 2790 { 2791 struct device_queue_manager *dqm; 2792 2793 pr_debug("Loading device queue manager\n"); 2794 2795 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); 2796 if (!dqm) 2797 return NULL; 2798 2799 switch (dev->adev->asic_type) { 2800 /* HWS is not available on Hawaii. */ 2801 case CHIP_HAWAII: 2802 /* HWS depends on CWSR for timely dequeue. CWSR is not 2803 * available on Tonga. 2804 * 2805 * FIXME: This argument also applies to Kaveri. 2806 */ 2807 case CHIP_TONGA: 2808 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 2809 break; 2810 default: 2811 dqm->sched_policy = sched_policy; 2812 break; 2813 } 2814 2815 dqm->dev = dev; 2816 switch (dqm->sched_policy) { 2817 case KFD_SCHED_POLICY_HWS: 2818 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 2819 /* initialize dqm for cp scheduling */ 2820 dqm->ops.create_queue = create_queue_cpsch; 2821 dqm->ops.initialize = initialize_cpsch; 2822 dqm->ops.start = start_cpsch; 2823 dqm->ops.stop = stop_cpsch; 2824 dqm->ops.halt = halt_cpsch; 2825 dqm->ops.unhalt = unhalt_cpsch; 2826 dqm->ops.destroy_queue = destroy_queue_cpsch; 2827 dqm->ops.update_queue = update_queue; 2828 dqm->ops.register_process = register_process; 2829 dqm->ops.unregister_process = unregister_process; 2830 dqm->ops.uninitialize = uninitialize; 2831 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 2832 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 2833 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2834 dqm->ops.process_termination = process_termination_cpsch; 2835 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 2836 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 2837 dqm->ops.get_wave_state = get_wave_state; 2838 dqm->ops.reset_queues = reset_queues_cpsch; 2839 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2840 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2841 break; 2842 case KFD_SCHED_POLICY_NO_HWS: 2843 /* initialize dqm for no cp scheduling */ 2844 dqm->ops.start = start_nocpsch; 2845 dqm->ops.stop = stop_nocpsch; 2846 dqm->ops.create_queue = create_queue_nocpsch; 2847 dqm->ops.destroy_queue = destroy_queue_nocpsch; 2848 dqm->ops.update_queue = update_queue; 2849 dqm->ops.register_process = register_process; 2850 dqm->ops.unregister_process = unregister_process; 2851 dqm->ops.initialize = initialize_nocpsch; 2852 dqm->ops.uninitialize = uninitialize; 2853 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 2854 dqm->ops.process_termination = process_termination_nocpsch; 2855 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 2856 dqm->ops.restore_process_queues = 2857 restore_process_queues_nocpsch; 2858 dqm->ops.get_wave_state = get_wave_state; 2859 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 2860 dqm->ops.checkpoint_mqd = checkpoint_mqd; 2861 break; 2862 default: 2863 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 2864 goto out_free; 2865 } 2866 2867 switch (dev->adev->asic_type) { 2868 case CHIP_KAVERI: 2869 case CHIP_HAWAII: 2870 device_queue_manager_init_cik(&dqm->asic_ops); 2871 break; 2872 2873 case CHIP_CARRIZO: 2874 case CHIP_TONGA: 2875 case CHIP_FIJI: 2876 case CHIP_POLARIS10: 2877 case CHIP_POLARIS11: 2878 case CHIP_POLARIS12: 2879 case CHIP_VEGAM: 2880 device_queue_manager_init_vi(&dqm->asic_ops); 2881 break; 2882 2883 default: 2884 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 2885 device_queue_manager_init_v12(&dqm->asic_ops); 2886 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 2887 device_queue_manager_init_v11(&dqm->asic_ops); 2888 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 2889 device_queue_manager_init_v10(&dqm->asic_ops); 2890 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 2891 device_queue_manager_init_v9(&dqm->asic_ops); 2892 else { 2893 WARN(1, "Unexpected ASIC family %u", 2894 dev->adev->asic_type); 2895 goto out_free; 2896 } 2897 } 2898 2899 if (init_mqd_managers(dqm)) 2900 goto out_free; 2901 2902 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 2903 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 2904 goto out_free; 2905 } 2906 2907 if (!dqm->ops.initialize(dqm)) { 2908 init_waitqueue_head(&dqm->destroy_wait); 2909 return dqm; 2910 } 2911 2912 out_free: 2913 kfree(dqm); 2914 return NULL; 2915 } 2916 2917 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 2918 struct kfd_mem_obj *mqd) 2919 { 2920 WARN(!mqd, "No hiq sdma mqd trunk to free"); 2921 2922 amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); 2923 } 2924 2925 void device_queue_manager_uninit(struct device_queue_manager *dqm) 2926 { 2927 dqm->ops.stop(dqm); 2928 dqm->ops.uninitialize(dqm); 2929 if (!dqm->dev->kfd->shared_resources.enable_mes) 2930 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 2931 kfree(dqm); 2932 } 2933 2934 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) 2935 { 2936 struct kfd_process_device *pdd; 2937 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 2938 struct device_queue_manager *dqm = knode->dqm; 2939 struct device *dev = dqm->dev->adev->dev; 2940 struct qcm_process_device *qpd; 2941 struct queue *q = NULL; 2942 int ret = 0; 2943 2944 if (!p) 2945 return -EINVAL; 2946 2947 dqm_lock(dqm); 2948 2949 pdd = kfd_get_process_device_data(dqm->dev, p); 2950 if (pdd) { 2951 qpd = &pdd->qpd; 2952 2953 list_for_each_entry(q, &qpd->queues_list, list) { 2954 if (q->doorbell_id == doorbell_id && q->properties.is_active) { 2955 ret = suspend_all_queues_mes(dqm); 2956 if (ret) { 2957 dev_err(dev, "Suspending all queues failed"); 2958 goto out; 2959 } 2960 2961 q->properties.is_evicted = true; 2962 q->properties.is_active = false; 2963 decrement_queue_count(dqm, qpd, q); 2964 2965 ret = remove_queue_mes(dqm, q, qpd); 2966 if (ret) { 2967 dev_err(dev, "Removing bad queue failed"); 2968 goto out; 2969 } 2970 2971 ret = resume_all_queues_mes(dqm); 2972 if (ret) 2973 dev_err(dev, "Resuming all queues failed"); 2974 2975 break; 2976 } 2977 } 2978 } 2979 2980 out: 2981 dqm_unlock(dqm); 2982 return ret; 2983 } 2984 2985 static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm, 2986 struct qcm_process_device *qpd) 2987 { 2988 struct device *dev = dqm->dev->adev->dev; 2989 int ret = 0; 2990 2991 /* Check if process is already evicted */ 2992 dqm_lock(dqm); 2993 if (qpd->evicted) { 2994 /* Increment the evicted count to make sure the 2995 * process stays evicted before its terminated. 2996 */ 2997 qpd->evicted++; 2998 dqm_unlock(dqm); 2999 goto out; 3000 } 3001 dqm_unlock(dqm); 3002 3003 ret = suspend_all_queues_mes(dqm); 3004 if (ret) { 3005 dev_err(dev, "Suspending all queues failed"); 3006 goto out; 3007 } 3008 3009 ret = dqm->ops.evict_process_queues(dqm, qpd); 3010 if (ret) { 3011 dev_err(dev, "Evicting process queues failed"); 3012 goto out; 3013 } 3014 3015 ret = resume_all_queues_mes(dqm); 3016 if (ret) 3017 dev_err(dev, "Resuming all queues failed"); 3018 3019 out: 3020 return ret; 3021 } 3022 3023 int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid) 3024 { 3025 struct kfd_process_device *pdd; 3026 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 3027 int ret = 0; 3028 3029 if (!p) 3030 return -EINVAL; 3031 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 3032 pdd = kfd_get_process_device_data(dqm->dev, p); 3033 if (pdd) { 3034 if (dqm->dev->kfd->shared_resources.enable_mes) 3035 ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd); 3036 else 3037 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); 3038 } 3039 3040 kfd_unref_process(p); 3041 3042 return ret; 3043 } 3044 3045 static void kfd_process_hw_exception(struct work_struct *work) 3046 { 3047 struct device_queue_manager *dqm = container_of(work, 3048 struct device_queue_manager, hw_exception_work); 3049 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 3050 } 3051 3052 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 3053 struct qcm_process_device *qpd) 3054 { 3055 int r; 3056 struct device *dev = dqm->dev->adev->dev; 3057 int updated_vmid_mask; 3058 3059 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3060 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3061 return -EINVAL; 3062 } 3063 3064 dqm_lock(dqm); 3065 3066 if (dqm->trap_debug_vmid != 0) { 3067 dev_err(dev, "Trap debug id already reserved\n"); 3068 r = -EBUSY; 3069 goto out_unlock; 3070 } 3071 3072 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3073 USE_DEFAULT_GRACE_PERIOD, false); 3074 if (r) 3075 goto out_unlock; 3076 3077 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3078 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 3079 3080 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3081 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 3082 r = set_sched_resources(dqm); 3083 if (r) 3084 goto out_unlock; 3085 3086 r = map_queues_cpsch(dqm); 3087 if (r) 3088 goto out_unlock; 3089 3090 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 3091 3092 out_unlock: 3093 dqm_unlock(dqm); 3094 return r; 3095 } 3096 3097 /* 3098 * Releases vmid for the trap debugger 3099 */ 3100 int release_debug_trap_vmid(struct device_queue_manager *dqm, 3101 struct qcm_process_device *qpd) 3102 { 3103 struct device *dev = dqm->dev->adev->dev; 3104 int r; 3105 int updated_vmid_mask; 3106 uint32_t trap_debug_vmid; 3107 3108 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3109 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3110 return -EINVAL; 3111 } 3112 3113 dqm_lock(dqm); 3114 trap_debug_vmid = dqm->trap_debug_vmid; 3115 if (dqm->trap_debug_vmid == 0) { 3116 dev_err(dev, "Trap debug id is not reserved\n"); 3117 r = -EINVAL; 3118 goto out_unlock; 3119 } 3120 3121 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3122 USE_DEFAULT_GRACE_PERIOD, false); 3123 if (r) 3124 goto out_unlock; 3125 3126 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3127 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 3128 3129 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3130 dqm->trap_debug_vmid = 0; 3131 r = set_sched_resources(dqm); 3132 if (r) 3133 goto out_unlock; 3134 3135 r = map_queues_cpsch(dqm); 3136 if (r) 3137 goto out_unlock; 3138 3139 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 3140 3141 out_unlock: 3142 dqm_unlock(dqm); 3143 return r; 3144 } 3145 3146 #define QUEUE_NOT_FOUND -1 3147 /* invalidate queue operation in array */ 3148 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 3149 { 3150 int i; 3151 3152 for (i = 0; i < num_queues; i++) 3153 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 3154 } 3155 3156 /* find queue index in array */ 3157 static int q_array_get_index(unsigned int queue_id, 3158 uint32_t num_queues, 3159 uint32_t *queue_ids) 3160 { 3161 int i; 3162 3163 for (i = 0; i < num_queues; i++) 3164 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 3165 return i; 3166 3167 return QUEUE_NOT_FOUND; 3168 } 3169 3170 struct copy_context_work_handler_workarea { 3171 struct work_struct copy_context_work; 3172 struct kfd_process *p; 3173 }; 3174 3175 static void copy_context_work_handler (struct work_struct *work) 3176 { 3177 struct copy_context_work_handler_workarea *workarea; 3178 struct mqd_manager *mqd_mgr; 3179 struct queue *q; 3180 struct mm_struct *mm; 3181 struct kfd_process *p; 3182 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 3183 int i; 3184 3185 workarea = container_of(work, 3186 struct copy_context_work_handler_workarea, 3187 copy_context_work); 3188 3189 p = workarea->p; 3190 mm = get_task_mm(p->lead_thread); 3191 3192 if (!mm) 3193 return; 3194 3195 kthread_use_mm(mm); 3196 for (i = 0; i < p->n_pdds; i++) { 3197 struct kfd_process_device *pdd = p->pdds[i]; 3198 struct device_queue_manager *dqm = pdd->dev->dqm; 3199 struct qcm_process_device *qpd = &pdd->qpd; 3200 3201 list_for_each_entry(q, &qpd->queues_list, list) { 3202 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3203 3204 /* We ignore the return value from get_wave_state 3205 * because 3206 * i) right now, it always returns 0, and 3207 * ii) if we hit an error, we would continue to the 3208 * next queue anyway. 3209 */ 3210 mqd_mgr->get_wave_state(mqd_mgr, 3211 q->mqd, 3212 &q->properties, 3213 (void __user *) q->properties.ctx_save_restore_area_address, 3214 &tmp_ctl_stack_used_size, 3215 &tmp_save_area_used_size); 3216 } 3217 } 3218 kthread_unuse_mm(mm); 3219 mmput(mm); 3220 } 3221 3222 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 3223 { 3224 size_t array_size = num_queues * sizeof(uint32_t); 3225 3226 if (!usr_queue_id_array) 3227 return NULL; 3228 3229 return memdup_user(usr_queue_id_array, array_size); 3230 } 3231 3232 int resume_queues(struct kfd_process *p, 3233 uint32_t num_queues, 3234 uint32_t *usr_queue_id_array) 3235 { 3236 uint32_t *queue_ids = NULL; 3237 int total_resumed = 0; 3238 int i; 3239 3240 if (usr_queue_id_array) { 3241 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3242 3243 if (IS_ERR(queue_ids)) 3244 return PTR_ERR(queue_ids); 3245 3246 /* mask all queues as invalid. unmask per successful request */ 3247 q_array_invalidate(num_queues, queue_ids); 3248 } 3249 3250 for (i = 0; i < p->n_pdds; i++) { 3251 struct kfd_process_device *pdd = p->pdds[i]; 3252 struct device_queue_manager *dqm = pdd->dev->dqm; 3253 struct device *dev = dqm->dev->adev->dev; 3254 struct qcm_process_device *qpd = &pdd->qpd; 3255 struct queue *q; 3256 int r, per_device_resumed = 0; 3257 3258 dqm_lock(dqm); 3259 3260 /* unmask queues that resume or already resumed as valid */ 3261 list_for_each_entry(q, &qpd->queues_list, list) { 3262 int q_idx = QUEUE_NOT_FOUND; 3263 3264 if (queue_ids) 3265 q_idx = q_array_get_index( 3266 q->properties.queue_id, 3267 num_queues, 3268 queue_ids); 3269 3270 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 3271 int err = resume_single_queue(dqm, &pdd->qpd, q); 3272 3273 if (queue_ids) { 3274 if (!err) { 3275 queue_ids[q_idx] &= 3276 ~KFD_DBG_QUEUE_INVALID_MASK; 3277 } else { 3278 queue_ids[q_idx] |= 3279 KFD_DBG_QUEUE_ERROR_MASK; 3280 break; 3281 } 3282 } 3283 3284 if (dqm->dev->kfd->shared_resources.enable_mes) { 3285 wake_up_all(&dqm->destroy_wait); 3286 if (!err) 3287 total_resumed++; 3288 } else { 3289 per_device_resumed++; 3290 } 3291 } 3292 } 3293 3294 if (!per_device_resumed) { 3295 dqm_unlock(dqm); 3296 continue; 3297 } 3298 3299 r = execute_queues_cpsch(dqm, 3300 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 3301 0, 3302 USE_DEFAULT_GRACE_PERIOD); 3303 if (r) { 3304 dev_err(dev, "Failed to resume process queues\n"); 3305 if (queue_ids) { 3306 list_for_each_entry(q, &qpd->queues_list, list) { 3307 int q_idx = q_array_get_index( 3308 q->properties.queue_id, 3309 num_queues, 3310 queue_ids); 3311 3312 /* mask queue as error on resume fail */ 3313 if (q_idx != QUEUE_NOT_FOUND) 3314 queue_ids[q_idx] |= 3315 KFD_DBG_QUEUE_ERROR_MASK; 3316 } 3317 } 3318 } else { 3319 wake_up_all(&dqm->destroy_wait); 3320 total_resumed += per_device_resumed; 3321 } 3322 3323 dqm_unlock(dqm); 3324 } 3325 3326 if (queue_ids) { 3327 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3328 num_queues * sizeof(uint32_t))) 3329 pr_err("copy_to_user failed on queue resume\n"); 3330 3331 kfree(queue_ids); 3332 } 3333 3334 return total_resumed; 3335 } 3336 3337 int suspend_queues(struct kfd_process *p, 3338 uint32_t num_queues, 3339 uint32_t grace_period, 3340 uint64_t exception_clear_mask, 3341 uint32_t *usr_queue_id_array) 3342 { 3343 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3344 int total_suspended = 0; 3345 int i; 3346 3347 if (IS_ERR(queue_ids)) 3348 return PTR_ERR(queue_ids); 3349 3350 /* mask all queues as invalid. umask on successful request */ 3351 q_array_invalidate(num_queues, queue_ids); 3352 3353 for (i = 0; i < p->n_pdds; i++) { 3354 struct kfd_process_device *pdd = p->pdds[i]; 3355 struct device_queue_manager *dqm = pdd->dev->dqm; 3356 struct device *dev = dqm->dev->adev->dev; 3357 struct qcm_process_device *qpd = &pdd->qpd; 3358 struct queue *q; 3359 int r, per_device_suspended = 0; 3360 3361 mutex_lock(&p->event_mutex); 3362 dqm_lock(dqm); 3363 3364 /* unmask queues that suspend or already suspended */ 3365 list_for_each_entry(q, &qpd->queues_list, list) { 3366 int q_idx = q_array_get_index(q->properties.queue_id, 3367 num_queues, 3368 queue_ids); 3369 3370 if (q_idx != QUEUE_NOT_FOUND) { 3371 int err = suspend_single_queue(dqm, pdd, q); 3372 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 3373 3374 if (!err) { 3375 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 3376 if (exception_clear_mask && is_mes) 3377 q->properties.exception_status &= 3378 ~exception_clear_mask; 3379 3380 if (is_mes) 3381 total_suspended++; 3382 else 3383 per_device_suspended++; 3384 } else if (err != -EBUSY) { 3385 r = err; 3386 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3387 break; 3388 } 3389 } 3390 } 3391 3392 if (!per_device_suspended) { 3393 dqm_unlock(dqm); 3394 mutex_unlock(&p->event_mutex); 3395 if (total_suspended) 3396 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3397 continue; 3398 } 3399 3400 r = execute_queues_cpsch(dqm, 3401 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3402 grace_period); 3403 3404 if (r) 3405 dev_err(dev, "Failed to suspend process queues.\n"); 3406 else 3407 total_suspended += per_device_suspended; 3408 3409 list_for_each_entry(q, &qpd->queues_list, list) { 3410 int q_idx = q_array_get_index(q->properties.queue_id, 3411 num_queues, queue_ids); 3412 3413 if (q_idx == QUEUE_NOT_FOUND) 3414 continue; 3415 3416 /* mask queue as error on suspend fail */ 3417 if (r) 3418 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3419 else if (exception_clear_mask) 3420 q->properties.exception_status &= 3421 ~exception_clear_mask; 3422 } 3423 3424 dqm_unlock(dqm); 3425 mutex_unlock(&p->event_mutex); 3426 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3427 } 3428 3429 if (total_suspended) { 3430 struct copy_context_work_handler_workarea copy_context_worker; 3431 3432 INIT_WORK_ONSTACK( 3433 ©_context_worker.copy_context_work, 3434 copy_context_work_handler); 3435 3436 copy_context_worker.p = p; 3437 3438 schedule_work(©_context_worker.copy_context_work); 3439 3440 3441 flush_work(©_context_worker.copy_context_work); 3442 destroy_work_on_stack(©_context_worker.copy_context_work); 3443 } 3444 3445 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3446 num_queues * sizeof(uint32_t))) 3447 pr_err("copy_to_user failed on queue suspend\n"); 3448 3449 kfree(queue_ids); 3450 3451 return total_suspended; 3452 } 3453 3454 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 3455 { 3456 switch (q_props->type) { 3457 case KFD_QUEUE_TYPE_COMPUTE: 3458 return q_props->format == KFD_QUEUE_FORMAT_PM4 3459 ? KFD_IOC_QUEUE_TYPE_COMPUTE 3460 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 3461 case KFD_QUEUE_TYPE_SDMA: 3462 return KFD_IOC_QUEUE_TYPE_SDMA; 3463 case KFD_QUEUE_TYPE_SDMA_XGMI: 3464 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 3465 default: 3466 WARN_ONCE(true, "queue type not recognized!"); 3467 return 0xffffffff; 3468 }; 3469 } 3470 3471 void set_queue_snapshot_entry(struct queue *q, 3472 uint64_t exception_clear_mask, 3473 struct kfd_queue_snapshot_entry *qss_entry) 3474 { 3475 qss_entry->ring_base_address = q->properties.queue_address; 3476 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 3477 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 3478 qss_entry->ctx_save_restore_address = 3479 q->properties.ctx_save_restore_area_address; 3480 qss_entry->ctx_save_restore_area_size = 3481 q->properties.ctx_save_restore_area_size; 3482 qss_entry->exception_status = q->properties.exception_status; 3483 qss_entry->queue_id = q->properties.queue_id; 3484 qss_entry->gpu_id = q->device->id; 3485 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 3486 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 3487 q->properties.exception_status &= ~exception_clear_mask; 3488 } 3489 3490 int debug_lock_and_unmap(struct device_queue_manager *dqm) 3491 { 3492 struct device *dev = dqm->dev->adev->dev; 3493 int r; 3494 3495 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3496 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3497 return -EINVAL; 3498 } 3499 3500 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3501 return 0; 3502 3503 dqm_lock(dqm); 3504 3505 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 3506 if (r) 3507 dqm_unlock(dqm); 3508 3509 return r; 3510 } 3511 3512 int debug_map_and_unlock(struct device_queue_manager *dqm) 3513 { 3514 struct device *dev = dqm->dev->adev->dev; 3515 int r; 3516 3517 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3518 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3519 return -EINVAL; 3520 } 3521 3522 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 3523 return 0; 3524 3525 r = map_queues_cpsch(dqm); 3526 3527 dqm_unlock(dqm); 3528 3529 return r; 3530 } 3531 3532 int debug_refresh_runlist(struct device_queue_manager *dqm) 3533 { 3534 int r = debug_lock_and_unmap(dqm); 3535 3536 if (r) 3537 return r; 3538 3539 return debug_map_and_unlock(dqm); 3540 } 3541 3542 #if defined(CONFIG_DEBUG_FS) 3543 3544 static void seq_reg_dump(struct seq_file *m, 3545 uint32_t (*dump)[2], uint32_t n_regs) 3546 { 3547 uint32_t i, count; 3548 3549 for (i = 0, count = 0; i < n_regs; i++) { 3550 if (count == 0 || 3551 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 3552 seq_printf(m, "%s %08x: %08x", 3553 i ? "\n" : "", 3554 dump[i][0], dump[i][1]); 3555 count = 7; 3556 } else { 3557 seq_printf(m, " %08x", dump[i][1]); 3558 count--; 3559 } 3560 } 3561 3562 seq_puts(m, "\n"); 3563 } 3564 3565 int dqm_debugfs_hqds(struct seq_file *m, void *data) 3566 { 3567 struct device_queue_manager *dqm = data; 3568 uint32_t xcc_mask = dqm->dev->xcc_mask; 3569 uint32_t (*dump)[2], n_regs; 3570 int pipe, queue; 3571 int r = 0, xcc_id; 3572 uint32_t sdma_engine_start; 3573 3574 if (!dqm->sched_running) { 3575 seq_puts(m, " Device is stopped\n"); 3576 return 0; 3577 } 3578 3579 for_each_inst(xcc_id, xcc_mask) { 3580 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3581 KFD_CIK_HIQ_PIPE, 3582 KFD_CIK_HIQ_QUEUE, &dump, 3583 &n_regs, xcc_id); 3584 if (!r) { 3585 seq_printf( 3586 m, 3587 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 3588 xcc_id, 3589 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 3590 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 3591 KFD_CIK_HIQ_QUEUE); 3592 seq_reg_dump(m, dump, n_regs); 3593 3594 kfree(dump); 3595 } 3596 3597 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 3598 int pipe_offset = pipe * get_queues_per_pipe(dqm); 3599 3600 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 3601 if (!test_bit(pipe_offset + queue, 3602 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 3603 continue; 3604 3605 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 3606 pipe, queue, 3607 &dump, &n_regs, 3608 xcc_id); 3609 if (r) 3610 break; 3611 3612 seq_printf(m, 3613 " Inst %d, CP Pipe %d, Queue %d\n", 3614 xcc_id, pipe, queue); 3615 seq_reg_dump(m, dump, n_regs); 3616 3617 kfree(dump); 3618 } 3619 } 3620 } 3621 3622 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 3623 for (pipe = sdma_engine_start; 3624 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 3625 pipe++) { 3626 for (queue = 0; 3627 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 3628 queue++) { 3629 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 3630 dqm->dev->adev, pipe, queue, &dump, &n_regs); 3631 if (r) 3632 break; 3633 3634 seq_printf(m, " SDMA Engine %d, RLC %d\n", 3635 pipe, queue); 3636 seq_reg_dump(m, dump, n_regs); 3637 3638 kfree(dump); 3639 } 3640 } 3641 3642 return r; 3643 } 3644 3645 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 3646 { 3647 int r = 0; 3648 3649 dqm_lock(dqm); 3650 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 3651 if (r) { 3652 dqm_unlock(dqm); 3653 return r; 3654 } 3655 dqm->active_runlist = true; 3656 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 3657 0, USE_DEFAULT_GRACE_PERIOD); 3658 dqm_unlock(dqm); 3659 3660 return r; 3661 } 3662 3663 #endif 3664