1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/ratelimit.h> 26 #include <linux/printk.h> 27 #include <linux/slab.h> 28 #include <linux/list.h> 29 #include <linux/types.h> 30 #include <linux/bitops.h> 31 #include <linux/sched.h> 32 #include "kfd_priv.h" 33 #include "kfd_device_queue_manager.h" 34 #include "kfd_mqd_manager.h" 35 #include "cik_regs.h" 36 #include "kfd_kernel_queue.h" 37 #include "amdgpu_amdkfd.h" 38 #include "amdgpu_reset.h" 39 #include "amdgpu_sdma.h" 40 #include "amdgpu_ring.h" 41 #include "amdgpu_mes.h" 42 #include "kfd_debug.h" 43 44 /* Size of the per-pipe EOP queue */ 45 #define CIK_HPD_EOP_BYTES_LOG2 11 46 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 47 /* See unmap_queues_cpsch() */ 48 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff 49 50 /* Interval for notifying MES of work on unmapped queues during oversubscription */ 51 #define DQM_MES_UNMAP_NOTIFY_DELAY_US 50 52 53 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 54 u32 pasid, unsigned int vmid); 55 56 static int execute_queues_cpsch(struct device_queue_manager *dqm, 57 enum kfd_unmap_queues_filter filter, 58 uint32_t filter_param, 59 uint32_t grace_period); 60 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 61 enum kfd_unmap_queues_filter filter, 62 uint32_t filter_param, 63 uint32_t grace_period, 64 bool reset); 65 66 static int map_queues_cpsch(struct device_queue_manager *dqm); 67 68 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 69 struct queue *q); 70 71 static inline void deallocate_hqd(struct device_queue_manager *dqm, 72 struct queue *q); 73 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); 74 static int allocate_sdma_queue(struct device_queue_manager *dqm, 75 struct queue *q, const uint32_t *restore_sdma_id); 76 77 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); 78 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, 79 u32 doorbell_offset); 80 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 81 struct qcm_process_device *qpd, bool verbose); 82 static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q); 83 84 static inline 85 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 86 { 87 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) 88 return KFD_MQD_TYPE_SDMA; 89 return KFD_MQD_TYPE_CP; 90 } 91 92 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) 93 { 94 int i; 95 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec 96 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 97 98 /* queue is available for KFD usage if bit is 1 */ 99 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i) 100 if (test_bit(pipe_offset + i, 101 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 102 return true; 103 return false; 104 } 105 106 unsigned int get_cp_queues_num(struct device_queue_manager *dqm) 107 { 108 return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap, 109 AMDGPU_MAX_QUEUES); 110 } 111 112 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) 113 { 114 return dqm->dev->kfd->shared_resources.num_queue_per_pipe; 115 } 116 117 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) 118 { 119 return dqm->dev->kfd->shared_resources.num_pipe_per_mec; 120 } 121 122 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm) 123 { 124 return kfd_get_num_sdma_engines(dqm->dev) + 125 kfd_get_num_xgmi_sdma_engines(dqm->dev); 126 } 127 128 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) 129 { 130 return kfd_get_num_sdma_engines(dqm->dev) * 131 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 132 } 133 134 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) 135 { 136 return kfd_get_num_xgmi_sdma_engines(dqm->dev) * 137 dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 138 } 139 140 static void init_sdma_bitmaps(struct device_queue_manager *dqm) 141 { 142 bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES); 143 bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 144 145 bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 146 bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 147 148 /* Mask out the reserved queues */ 149 bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) * 150 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 151 bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) * 152 dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine); 153 } 154 155 void program_sh_mem_settings(struct device_queue_manager *dqm, 156 struct qcm_process_device *qpd) 157 { 158 uint32_t xcc_mask = dqm->dev->xcc_mask; 159 int xcc_id; 160 161 for_each_inst(xcc_id, xcc_mask) 162 dqm->dev->kfd2kgd->program_sh_mem_settings( 163 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, 164 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, 165 qpd->sh_mem_bases, xcc_id); 166 } 167 168 static void kfd_hws_hang(struct device_queue_manager *dqm) 169 { 170 struct device_process_node *cur; 171 struct qcm_process_device *qpd; 172 struct queue *q; 173 174 /* Mark all device queues as reset. */ 175 list_for_each_entry(cur, &dqm->queues, list) { 176 qpd = cur->qpd; 177 list_for_each_entry(q, &qpd->queues_list, list) { 178 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 179 180 pdd->has_reset_queue = true; 181 } 182 } 183 184 /* 185 * Issue a GPU reset if HWS is unresponsive 186 */ 187 amdgpu_amdkfd_gpu_reset(dqm->dev->adev); 188 } 189 190 static int convert_to_amdgpu_ring_type(int queue_type) 191 { 192 int amdgpu_ring_type; 193 194 switch (queue_type) { 195 case KFD_QUEUE_TYPE_COMPUTE: 196 amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE; 197 break; 198 case KFD_QUEUE_TYPE_SDMA: 199 amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA; 200 break; 201 default: 202 WARN(1, "Invalid queue type %d", queue_type); 203 amdgpu_ring_type = -EINVAL; 204 break; 205 } 206 207 return amdgpu_ring_type; 208 } 209 210 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, 211 struct qcm_process_device *qpd) 212 { 213 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 214 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 215 struct mes_add_queue_input queue_input; 216 int r, queue_type; 217 uint64_t wptr_addr_off; 218 219 if (!dqm->sched_running || dqm->sched_halt) 220 return 0; 221 if (!down_read_trylock(&adev->reset_domain->sem)) 222 return -EIO; 223 224 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 225 queue_input.process_id = pdd->pasid; 226 queue_input.page_table_base_addr = qpd->page_table_base; 227 queue_input.process_va_start = 0; 228 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 229 /* MES unit for quantum is 100ns */ 230 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */ 231 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr; 232 queue_input.process_context_array_index = pdd->proc_ctx_array_index; 233 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */ 234 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 235 queue_input.gang_context_array_index = q->gang_ctx_array_index; 236 queue_input.inprocess_gang_priority = q->properties.priority; 237 queue_input.gang_global_priority_level = 238 AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 239 queue_input.doorbell_offset = q->properties.doorbell_off; 240 queue_input.mqd_addr = q->gart_mqd_addr; 241 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 242 243 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1); 244 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 245 246 queue_input.is_kfd_process = 1; 247 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); 248 queue_input.queue_size = q->properties.queue_size >> 2; 249 250 queue_input.paging = false; 251 queue_input.tba_addr = qpd->tba_addr; 252 queue_input.tma_addr = qpd->tma_addr; 253 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device); 254 queue_input.skip_process_ctx_clear = 255 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && 256 (qpd->pqm->process->debug_trap_enabled || 257 kfd_dbg_has_ttmps_always_setup(q->device)); 258 259 queue_type = convert_to_amdgpu_ring_type(q->properties.type); 260 if (queue_type < 0) { 261 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", 262 q->properties.type); 263 up_read(&adev->reset_domain->sem); 264 return -EINVAL; 265 } 266 queue_input.queue_type = (uint32_t)queue_type; 267 268 queue_input.exclusively_scheduled = q->properties.is_gws; 269 queue_input.sh_mem_config_data = qpd->sh_mem_config; 270 queue_input.vm_cntx_cntl = qpd->vm_cntx_cntl; 271 queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 272 273 amdgpu_mes_lock(&adev->mes); 274 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 275 amdgpu_mes_unlock(&adev->mes); 276 up_read(&adev->reset_domain->sem); 277 if (r) { 278 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n", 279 q->properties.doorbell_off); 280 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 281 kfd_hws_hang(dqm); 282 return r; 283 } 284 285 /* 286 * GFX11: start notify timer only once when oversubscription begins. 287 * Skip under SR-IOV: MES round-trips are far slower there, and this 288 * notify call shares mes->mutex_hidden with add/remove_hw_queue, so 289 * a slow notify can stall remapping queues. SR-IOV guests keep MES's 290 * own firmware oversubscription timer instead. 291 */ 292 if (!amdgpu_sriov_vf(adev) && 293 KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) && 294 KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) && 295 dqm->active_cp_queue_count > get_cp_queues_num(dqm)) 296 queue_delayed_work(system_wq, &dqm->notify_unmap_work, 297 usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US)); 298 299 return r; 300 } 301 302 static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q, 303 struct qcm_process_device *qpd, 304 bool is_for_reset, 305 bool flush_mes_queue) 306 { 307 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 308 int r; 309 struct mes_remove_queue_input queue_input; 310 311 /* queue was already removed during reset */ 312 if (q->properties.is_reset) 313 return 0; 314 315 if (!dqm->sched_running || dqm->sched_halt) 316 return 0; 317 if (!down_read_trylock(&adev->reset_domain->sem)) 318 return -EIO; 319 320 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 321 queue_input.doorbell_offset = q->properties.doorbell_off; 322 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 323 queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); 324 queue_input.remove_queue_after_reset = flush_mes_queue; 325 queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 326 queue_input.gang_context_array_index = q->gang_ctx_array_index; 327 328 amdgpu_mes_lock(&adev->mes); 329 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 330 amdgpu_mes_unlock(&adev->mes); 331 up_read(&adev->reset_domain->sem); 332 333 /* If is_for_reset set, it is a mes internal cleanup */ 334 if (!r || is_for_reset) 335 return r; 336 337 /* remove_hw_queue failure indicates a queue hang. reset the queue */ 338 r = reset_queues_mes(dqm, q); 339 if (r && amdgpu_gpu_recovery) { 340 dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n", 341 q->properties.doorbell_off); 342 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); 343 kfd_hws_hang(dqm); 344 } 345 346 return r; 347 } 348 349 static void set_perfcount(struct device_queue_manager *dqm, int enable) 350 { 351 struct device_process_node *cur; 352 struct qcm_process_device *qpd; 353 struct queue *q; 354 struct mqd_update_info minfo = { 0 }; 355 356 if (!dqm) 357 return; 358 359 minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE : 360 UPDATE_FLAG_PERFCOUNT_DISABLE); 361 dqm_lock(dqm); 362 list_for_each_entry(cur, &dqm->queues, list) { 363 qpd = cur->qpd; 364 list_for_each_entry(q, &qpd->queues_list, list) { 365 pqm_update_mqd(qpd->pqm, q->properties.queue_id, 366 &minfo); 367 } 368 } 369 dqm_unlock(dqm); 370 } 371 372 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q, 373 struct qcm_process_device *qpd) 374 { 375 int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false); 376 377 /* GFX11: stop notify timer when oversubscription clears */ 378 if (!r && 379 KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) && 380 KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) && 381 dqm->active_cp_queue_count <= get_cp_queues_num(dqm)) 382 cancel_delayed_work(&dqm->notify_unmap_work); 383 384 return r; 385 } 386 387 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm) 388 { 389 struct device_process_node *cur; 390 struct device *dev = dqm->dev->adev->dev; 391 struct qcm_process_device *qpd; 392 struct queue *q; 393 int retval = 0; 394 395 list_for_each_entry(cur, &dqm->queues, list) { 396 qpd = cur->qpd; 397 list_for_each_entry(q, &qpd->queues_list, list) { 398 if (q->properties.is_active) { 399 retval = remove_queue_mes(dqm, q, qpd); 400 if (retval) { 401 dev_err(dev, "%s: Failed to remove queue %d for dev %d", 402 __func__, 403 q->properties.queue_id, 404 dqm->dev->id); 405 return retval; 406 } 407 } 408 } 409 } 410 411 return retval; 412 } 413 414 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) 415 { 416 struct device_process_node *cur; 417 struct device *dev = dqm->dev->adev->dev; 418 struct qcm_process_device *qpd; 419 struct queue *q; 420 int retval = 0; 421 422 list_for_each_entry(cur, &dqm->queues, list) { 423 qpd = cur->qpd; 424 list_for_each_entry(q, &qpd->queues_list, list) { 425 if (!q->properties.is_active) 426 continue; 427 retval = add_queue_mes(dqm, q, qpd); 428 if (retval) { 429 dev_err(dev, "%s: Failed to add queue %d for dev %d", 430 __func__, 431 q->properties.queue_id, 432 dqm->dev->id); 433 return retval; 434 } 435 } 436 } 437 438 return retval; 439 } 440 441 static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q, 442 int queue_type, int pipe, int queue, unsigned int db) 443 { 444 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 445 struct kfd_process_device *pdd; 446 bool use_mmio = adev->gfx.mec.use_mmio_for_reset; 447 int r; 448 449 pdd = kfd_get_process_device_data(q->device, q->process); 450 if (!pdd) 451 return -ENODEV; 452 453 if (use_mmio) 454 r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 455 ffs(dqm->dev->xcc_mask) - 1); 456 else 457 r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 458 ffs(dqm->dev->xcc_mask) - 1); 459 if (r) 460 return r; 461 /* Proceed remove_queue with reset=true */ 462 remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true); 463 set_queue_as_reset(dqm, q, &pdd->qpd, true); 464 return 0; 465 } 466 467 int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, 468 int pipe, int queue, unsigned int db) 469 { 470 struct queue *q; 471 472 q = find_queue_by_doorbell_offset(dqm, db); 473 if (!q) 474 return 0; 475 return reset_queue_mes(dqm, q, queue_type, pipe, queue, db); 476 } 477 478 static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q) 479 { 480 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 481 struct drm_wedge_task_info *info = NULL; 482 struct amdgpu_task_info *ti = NULL; 483 struct kfd_process_device *pdd; 484 unsigned int num_hung = 0; 485 int r = 0; 486 struct mes_remove_queue_input queue_input; 487 488 if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) { 489 r = -ENOTRECOVERABLE; 490 goto fail; 491 } 492 493 /* reset should be used only in dqm locked queue reset */ 494 if (WARN_ON(dqm->detect_hang_count > 0)) 495 return 0; 496 497 if (!amdgpu_gpu_recovery) { 498 r = -ENOTRECOVERABLE; 499 goto fail; 500 } 501 502 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 503 queue_input.doorbell_offset = q->properties.doorbell_off; 504 queue_input.gang_context_addr = q->gang_ctx_gpu_addr; 505 queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); 506 queue_input.remove_queue_after_reset = false; 507 queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 508 /* pass the known bad queue info to the reset function */ 509 r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input); 510 if (r) 511 goto fail; 512 pdd = kfd_get_process_device_data(q->device, q->process); 513 if (pdd) { 514 ti = amdgpu_vm_get_task_info_pasid(adev, pdd->pasid); 515 if (ti) { 516 amdgpu_vm_print_task_info(adev, ti); 517 info = &ti->task; 518 } 519 } 520 521 dqm->detect_hang_count = num_hung; 522 /* When MES doesn't detect any queue hang, no reset happens. Don't signal reset 523 * event. 524 */ 525 if (dqm->detect_hang_count) { 526 kfd_signal_reset_event(dqm->dev); 527 if (pdd && pdd->has_reset_queue) { 528 atomic_inc(&adev->gpu_reset_counter); 529 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info); 530 } 531 } 532 amdgpu_vm_put_task_info(ti); 533 534 fail: 535 dqm->detect_hang_count = 0; 536 return r; 537 } 538 539 static void increment_queue_count(struct device_queue_manager *dqm, 540 struct qcm_process_device *qpd, 541 struct queue *q) 542 { 543 dqm->active_queue_count++; 544 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 545 dqm->active_cp_queue_count++; 546 547 if (q->properties.is_gws) { 548 dqm->gws_queue_count++; 549 qpd->mapped_gws_queue = true; 550 } 551 } 552 553 static void decrement_queue_count(struct device_queue_manager *dqm, 554 struct qcm_process_device *qpd, 555 struct queue *q) 556 { 557 dqm->active_queue_count--; 558 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 559 dqm->active_cp_queue_count--; 560 561 if (q->properties.is_gws) { 562 dqm->gws_queue_count--; 563 qpd->mapped_gws_queue = false; 564 } 565 } 566 567 /* 568 * Allocate a doorbell ID to this queue. 569 * If doorbell_id is passed in, make sure requested ID is valid then allocate it. 570 */ 571 static int allocate_doorbell(struct qcm_process_device *qpd, 572 struct queue *q, 573 uint32_t const *restore_id) 574 { 575 struct kfd_node *dev = qpd->dqm->dev; 576 577 if (!KFD_IS_SOC15(dev)) { 578 /* On pre-SOC15 chips we need to use the queue ID to 579 * preserve the user mode ABI. 580 */ 581 582 if (restore_id && *restore_id != q->properties.queue_id) 583 return -EINVAL; 584 585 q->doorbell_id = q->properties.queue_id; 586 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 587 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 588 /* For SDMA queues on SOC15 with 8-byte doorbell, use static 589 * doorbell assignments based on the engine and queue id. 590 * The doobell index distance between RLC (2*i) and (2*i+1) 591 * for a SDMA engine is 512. 592 */ 593 594 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx; 595 596 /* 597 * q->properties.sdma_engine_id corresponds to the virtual 598 * sdma engine number. However, for doorbell allocation, 599 * we need the physical sdma engine id in order to get the 600 * correct doorbell offset. 601 */ 602 u32 engine_base = idx_offset[dev->node_id * 603 get_num_all_sdma_engines(qpd->dqm) + 604 q->properties.sdma_engine_id]; 605 u32 valid_id; 606 607 /* 608 * With the aqua_vanjaram doorbell layout, a shader doorbell 609 * write whose 32-byte block starts in the previous engine's 610 * range rings that engine instead. Engine ranges are 10 611 * indices apart (sdma_doorbell_range = 20 dwords, halved for 612 * 8-byte doorbells, in aqua_vanjaram_doorbell_index_init()), 613 * which puts odd engines 2 indices into a shared block. 614 * Aligning costs at most 2 indices, and 8 queues per engine use 615 * only 4 of the 10, so the aligned base stays in range. The 616 * 512-index mirror offset keeps the alignment. 617 */ 618 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) || 619 KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) || 620 KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) 621 engine_base = ALIGN(engine_base, 622 KFD_SDMA_SHADER_DOORBELL_GRANULARITY); 623 624 valid_id = engine_base + 625 (q->properties.sdma_queue_id & 1) * 626 KFD_QUEUE_DOORBELL_MIRROR_OFFSET + 627 (q->properties.sdma_queue_id >> 1); 628 629 if (restore_id && *restore_id != valid_id) 630 return -EINVAL; 631 q->doorbell_id = valid_id; 632 } else { 633 /* For CP queues on SOC15 */ 634 if (restore_id) { 635 if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 636 return -EINVAL; 637 638 /* make sure that ID is free */ 639 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) 640 return -EINVAL; 641 642 q->doorbell_id = *restore_id; 643 } else { 644 /* or reserve a free doorbell ID */ 645 unsigned int found; 646 647 found = find_first_zero_bit(qpd->doorbell_bitmap, 648 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); 649 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { 650 pr_debug("No doorbells available"); 651 return -EBUSY; 652 } 653 set_bit(found, qpd->doorbell_bitmap); 654 q->doorbell_id = found; 655 } 656 } 657 658 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev, 659 qpd->proc_doorbells, 660 q->doorbell_id, 661 dev->kfd->device_info.doorbell_size); 662 return 0; 663 } 664 665 static void deallocate_doorbell(struct qcm_process_device *qpd, 666 struct queue *q) 667 { 668 unsigned int old; 669 struct kfd_node *dev = qpd->dqm->dev; 670 671 if (!KFD_IS_SOC15(dev) || 672 q->properties.type == KFD_QUEUE_TYPE_SDMA || 673 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 674 return; 675 676 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); 677 WARN_ON(!old); 678 } 679 680 static void program_trap_handler_settings(struct device_queue_manager *dqm, 681 struct qcm_process_device *qpd) 682 { 683 uint32_t xcc_mask = dqm->dev->xcc_mask; 684 int xcc_id; 685 686 if (dqm->dev->kfd2kgd->program_trap_handler_settings) 687 for_each_inst(xcc_id, xcc_mask) 688 dqm->dev->kfd2kgd->program_trap_handler_settings( 689 dqm->dev->adev, qpd->vmid, qpd->tba_addr, 690 qpd->tma_addr, xcc_id); 691 } 692 693 static int allocate_vmid(struct device_queue_manager *dqm, 694 struct qcm_process_device *qpd, 695 struct queue *q) 696 { 697 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 698 struct device *dev = dqm->dev->adev->dev; 699 int allocated_vmid = -1, i; 700 701 for (i = dqm->dev->vm_info.first_vmid_kfd; 702 i <= dqm->dev->vm_info.last_vmid_kfd; i++) { 703 if (!dqm->vmid_pasid[i]) { 704 allocated_vmid = i; 705 break; 706 } 707 } 708 709 if (allocated_vmid < 0) { 710 dev_err(dev, "no more vmid to allocate\n"); 711 return -ENOSPC; 712 } 713 714 pr_debug("vmid allocated: %d\n", allocated_vmid); 715 716 dqm->vmid_pasid[allocated_vmid] = pdd->pasid; 717 718 set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid); 719 720 qpd->vmid = allocated_vmid; 721 q->properties.vmid = allocated_vmid; 722 723 program_sh_mem_settings(dqm, qpd); 724 725 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled) 726 program_trap_handler_settings(dqm, qpd); 727 728 /* qpd->page_table_base is set earlier when register_process() 729 * is called, i.e. when the first queue is created. 730 */ 731 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev, 732 qpd->vmid, 733 qpd->page_table_base); 734 /* invalidate the VM context after pasid and vmid mapping is set up */ 735 kfd_flush_tlb(qpd_to_pdd(qpd)); 736 737 if (dqm->dev->kfd2kgd->set_scratch_backing_va) 738 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, 739 qpd->sh_hidden_private_base, qpd->vmid); 740 741 return 0; 742 } 743 744 static int flush_texture_cache_nocpsch(struct kfd_node *kdev, 745 struct qcm_process_device *qpd) 746 { 747 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; 748 int ret; 749 750 if (!qpd->ib_kaddr) 751 return -ENOMEM; 752 753 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); 754 if (ret) 755 return ret; 756 757 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, 758 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, 759 pmf->release_mem_size / sizeof(uint32_t)); 760 } 761 762 static void deallocate_vmid(struct device_queue_manager *dqm, 763 struct qcm_process_device *qpd, 764 struct queue *q) 765 { 766 struct device *dev = dqm->dev->adev->dev; 767 768 /* On GFX v7, CP doesn't flush TC at dequeue */ 769 if (q->device->adev->asic_type == CHIP_HAWAII) 770 if (flush_texture_cache_nocpsch(q->device, qpd)) 771 dev_err(dev, "Failed to flush TC\n"); 772 773 kfd_flush_tlb(qpd_to_pdd(qpd)); 774 775 /* Release the vmid mapping */ 776 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 777 dqm->vmid_pasid[qpd->vmid] = 0; 778 779 qpd->vmid = 0; 780 q->properties.vmid = 0; 781 } 782 783 static int create_queue_nocpsch(struct device_queue_manager *dqm, 784 struct queue *q, 785 struct qcm_process_device *qpd, 786 const struct kfd_criu_queue_priv_data *qd, 787 const void *restore_mqd, const void *restore_ctl_stack) 788 { 789 struct mqd_manager *mqd_mgr; 790 int retval; 791 792 dqm_lock(dqm); 793 794 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 795 pr_warn("Can't create new usermode queue because %d queues were already created\n", 796 dqm->total_queue_count); 797 retval = -EPERM; 798 goto out_unlock; 799 } 800 801 if (list_empty(&qpd->queues_list)) { 802 retval = allocate_vmid(dqm, qpd, q); 803 if (retval) 804 goto out_unlock; 805 } 806 q->properties.vmid = qpd->vmid; 807 /* 808 * Eviction state logic: mark all queues as evicted, even ones 809 * not currently active. Restoring inactive queues later only 810 * updates the is_evicted flag but is a no-op otherwise. 811 */ 812 q->properties.is_evicted = !!qpd->evicted; 813 814 q->properties.tba_addr = qpd->tba_addr; 815 q->properties.tma_addr = qpd->tma_addr; 816 817 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 818 q->properties.type)]; 819 if (qd && !mqd_mgr->restore_mqd) { 820 pr_debug("restore_mqd not implemented for queue type %d\n", 821 q->properties.type); 822 retval = -EOPNOTSUPP; 823 goto deallocate_vmid; 824 } 825 826 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 827 retval = allocate_hqd(dqm, q); 828 if (retval) 829 goto deallocate_vmid; 830 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n", 831 q->pipe, q->queue); 832 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 833 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 834 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 835 if (retval) 836 goto deallocate_vmid; 837 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 838 } 839 840 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 841 if (retval) 842 goto out_deallocate_hqd; 843 844 /* Temporarily release dqm lock to avoid a circular lock dependency */ 845 dqm_unlock(dqm); 846 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties); 847 dqm_lock(dqm); 848 849 if (!q->mqd_mem_obj) { 850 retval = -ENOMEM; 851 goto out_deallocate_doorbell; 852 } 853 854 if (qd) 855 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 856 &q->properties, restore_mqd, restore_ctl_stack, 857 qd->ctl_stack_size); 858 else 859 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 860 &q->gart_mqd_addr, &q->properties); 861 862 if (q->properties.is_active) { 863 if (!dqm->sched_running) { 864 WARN_ONCE(1, "Load non-HWS mqd while stopped\n"); 865 goto add_queue_to_list; 866 } 867 868 if (WARN(q->process->mm != current->mm, 869 "should only run in user thread")) 870 retval = -EFAULT; 871 else 872 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 873 q->queue, &q->properties, current->mm); 874 if (retval) 875 goto out_free_mqd; 876 } 877 878 add_queue_to_list: 879 list_add(&q->list, &qpd->queues_list); 880 qpd->queue_count++; 881 if (q->properties.is_active) 882 increment_queue_count(dqm, qpd, q); 883 884 /* 885 * Unconditionally increment this counter, regardless of the queue's 886 * type or whether the queue is active. 887 */ 888 dqm->total_queue_count++; 889 pr_debug("Total of %d queues are accountable so far\n", 890 dqm->total_queue_count); 891 goto out_unlock; 892 893 out_free_mqd: 894 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 895 out_deallocate_doorbell: 896 deallocate_doorbell(qpd, q); 897 out_deallocate_hqd: 898 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 899 deallocate_hqd(dqm, q); 900 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 901 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 902 deallocate_sdma_queue(dqm, q); 903 deallocate_vmid: 904 if (list_empty(&qpd->queues_list)) 905 deallocate_vmid(dqm, qpd, q); 906 out_unlock: 907 dqm_unlock(dqm); 908 return retval; 909 } 910 911 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 912 { 913 bool set; 914 int pipe, bit, i; 915 916 set = false; 917 918 for (pipe = dqm->next_pipe_to_allocate, i = 0; 919 i < get_pipes_per_mec(dqm); 920 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { 921 922 if (!is_pipe_enabled(dqm, 0, pipe)) 923 continue; 924 925 if (dqm->allocated_queues[pipe] != 0) { 926 bit = ffs(dqm->allocated_queues[pipe]) - 1; 927 dqm->allocated_queues[pipe] &= ~(1 << bit); 928 q->pipe = pipe; 929 q->queue = bit; 930 set = true; 931 break; 932 } 933 } 934 935 if (!set) 936 return -EBUSY; 937 938 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue); 939 /* horizontal hqd allocation */ 940 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); 941 942 return 0; 943 } 944 945 static inline void deallocate_hqd(struct device_queue_manager *dqm, 946 struct queue *q) 947 { 948 dqm->allocated_queues[q->pipe] |= (1 << q->queue); 949 } 950 951 #define SQ_IND_CMD_CMD_KILL 0x00000003 952 #define SQ_IND_CMD_MODE_BROADCAST 0x00000001 953 954 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p) 955 { 956 int status = 0; 957 unsigned int vmid; 958 uint16_t queried_pasid; 959 union SQ_CMD_BITS reg_sq_cmd; 960 union GRBM_GFX_INDEX_BITS reg_gfx_index; 961 struct kfd_process_device *pdd; 962 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd; 963 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd; 964 uint32_t xcc_mask = dev->xcc_mask; 965 int xcc_id; 966 967 reg_sq_cmd.u32All = 0; 968 reg_gfx_index.u32All = 0; 969 970 pr_debug("Killing all process wavefronts\n"); 971 972 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) { 973 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n"); 974 return -EOPNOTSUPP; 975 } 976 977 /* taking the VMID for that process on the safe way using PDD */ 978 pdd = kfd_get_process_device_data(dev, p); 979 if (!pdd) 980 return -EFAULT; 981 982 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING .. 983 * ATC_VMID15_PASID_MAPPING 984 * to check which VMID the current process is mapped to. 985 */ 986 987 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) { 988 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info 989 (dev->adev, vmid, &queried_pasid); 990 991 if (status && queried_pasid == pdd->pasid) { 992 pr_debug("Killing wave fronts of vmid %d and process pid %d\n", 993 vmid, p->lead_thread->pid); 994 break; 995 } 996 } 997 998 if (vmid > last_vmid_to_scan) { 999 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n", 1000 p->lead_thread->pid); 1001 return -EFAULT; 1002 } 1003 1004 reg_gfx_index.bits.sh_broadcast_writes = 1; 1005 reg_gfx_index.bits.se_broadcast_writes = 1; 1006 reg_gfx_index.bits.instance_broadcast_writes = 1; 1007 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST; 1008 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL; 1009 reg_sq_cmd.bits.vm_id = vmid; 1010 1011 for_each_inst(xcc_id, xcc_mask) 1012 dev->kfd2kgd->wave_control_execute( 1013 dev->adev, reg_gfx_index.u32All, 1014 reg_sq_cmd.u32All, xcc_id); 1015 1016 return 0; 1017 } 1018 1019 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked 1020 * to avoid asynchronized access 1021 */ 1022 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, 1023 struct qcm_process_device *qpd, 1024 struct queue *q) 1025 { 1026 int retval; 1027 struct mqd_manager *mqd_mgr; 1028 1029 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 1030 1031 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 1032 deallocate_hqd(dqm, q); 1033 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1034 deallocate_sdma_queue(dqm, q); 1035 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 1036 deallocate_sdma_queue(dqm, q); 1037 else { 1038 pr_debug("q->properties.type %d is invalid\n", 1039 q->properties.type); 1040 return -EINVAL; 1041 } 1042 dqm->total_queue_count--; 1043 1044 deallocate_doorbell(qpd, q); 1045 1046 if (!dqm->sched_running) { 1047 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n"); 1048 return 0; 1049 } 1050 1051 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1052 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 1053 KFD_UNMAP_LATENCY_MS, 1054 q->pipe, q->queue); 1055 if (retval == -ETIME) 1056 qpd->reset_wavefronts = true; 1057 1058 list_del(&q->list); 1059 if (list_empty(&qpd->queues_list)) { 1060 if (qpd->reset_wavefronts) { 1061 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n", 1062 dqm->dev); 1063 /* dbgdev_wave_reset_wavefronts has to be called before 1064 * deallocate_vmid(), i.e. when vmid is still in use. 1065 */ 1066 dbgdev_wave_reset_wavefronts(dqm->dev, 1067 qpd->pqm->process); 1068 qpd->reset_wavefronts = false; 1069 } 1070 1071 deallocate_vmid(dqm, qpd, q); 1072 } 1073 qpd->queue_count--; 1074 if (q->properties.is_active) 1075 decrement_queue_count(dqm, qpd, q); 1076 1077 return retval; 1078 } 1079 1080 static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 1081 struct qcm_process_device *qpd, 1082 struct queue *q) 1083 { 1084 int retval; 1085 uint64_t sdma_val = 0; 1086 struct device *dev = dqm->dev->adev->dev; 1087 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 1088 struct mqd_manager *mqd_mgr = 1089 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 1090 1091 /* Get the SDMA queue stats */ 1092 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 1093 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1094 if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) 1095 retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( 1096 dqm->dev->adev, q->mqd, 1097 dqm->dev->kfd->device_info.num_sdma_queues_per_engine, 1098 &sdma_val); 1099 else 1100 retval = read_sdma_queue_counter( 1101 (uint64_t __user *)q->properties.read_ptr, 1102 &sdma_val); 1103 if (retval) 1104 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 1105 q->properties.queue_id); 1106 } 1107 1108 dqm_lock(dqm); 1109 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); 1110 if (!retval) 1111 pdd->sdma_past_activity_counter += sdma_val; 1112 dqm_unlock(dqm); 1113 1114 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 1115 1116 return retval; 1117 } 1118 1119 static int update_queue(struct device_queue_manager *dqm, struct queue *q, 1120 struct mqd_update_info *minfo) 1121 { 1122 int retval = 0; 1123 struct device *dev = dqm->dev->adev->dev; 1124 struct mqd_manager *mqd_mgr; 1125 struct kfd_process_device *pdd; 1126 bool prev_active = false; 1127 1128 dqm_lock(dqm); 1129 pdd = kfd_get_process_device_data(q->device, q->process); 1130 if (!pdd) { 1131 retval = -ENODEV; 1132 goto out_unlock; 1133 } 1134 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1135 q->properties.type)]; 1136 1137 /* Save previous activity state for counters */ 1138 prev_active = q->properties.is_active; 1139 1140 /* Make sure the queue is unmapped before updating the MQD */ 1141 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1142 if (!dqm->dev->kfd->shared_resources.enable_mes) 1143 retval = unmap_queues_cpsch(dqm, 1144 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false); 1145 else if (prev_active) 1146 retval = remove_queue_mes(dqm, q, &pdd->qpd); 1147 1148 /* queue is reset so inaccessable */ 1149 if (pdd->has_reset_queue) { 1150 retval = -EACCES; 1151 goto out_unlock; 1152 } 1153 1154 if (retval) { 1155 dev_err(dev, "unmap queue failed\n"); 1156 goto out_unlock; 1157 } 1158 } else if (prev_active && 1159 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1160 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1161 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1162 1163 if (!dqm->sched_running) { 1164 WARN_ONCE(1, "Update non-HWS queue while stopped\n"); 1165 goto out_unlock; 1166 } 1167 1168 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1169 (dqm->dev->kfd->cwsr_enabled ? 1170 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1171 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1172 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1173 if (retval) { 1174 dev_err(dev, "destroy mqd failed\n"); 1175 goto out_unlock; 1176 } 1177 } 1178 1179 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo); 1180 1181 /* 1182 * check active state vs. the previous state and modify 1183 * counter accordingly. map_queues_cpsch uses the 1184 * dqm->active_queue_count to determine whether a new runlist must be 1185 * uploaded. 1186 */ 1187 if (q->properties.is_active && !prev_active) { 1188 increment_queue_count(dqm, &pdd->qpd, q); 1189 } else if (!q->properties.is_active && prev_active) { 1190 decrement_queue_count(dqm, &pdd->qpd, q); 1191 } else if (q->gws && !q->properties.is_gws) { 1192 if (q->properties.is_active) { 1193 dqm->gws_queue_count++; 1194 pdd->qpd.mapped_gws_queue = true; 1195 } 1196 q->properties.is_gws = true; 1197 } else if (!q->gws && q->properties.is_gws) { 1198 if (q->properties.is_active) { 1199 dqm->gws_queue_count--; 1200 pdd->qpd.mapped_gws_queue = false; 1201 } 1202 q->properties.is_gws = false; 1203 } 1204 1205 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { 1206 if (!dqm->dev->kfd->shared_resources.enable_mes) 1207 retval = map_queues_cpsch(dqm); 1208 else if (q->properties.is_active) 1209 retval = add_queue_mes(dqm, q, &pdd->qpd); 1210 } else if (q->properties.is_active && 1211 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || 1212 q->properties.type == KFD_QUEUE_TYPE_SDMA || 1213 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 1214 if (WARN(q->process->mm != current->mm, 1215 "should only run in user thread")) 1216 retval = -EFAULT; 1217 else 1218 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 1219 q->pipe, q->queue, 1220 &q->properties, current->mm); 1221 } 1222 1223 out_unlock: 1224 dqm_unlock(dqm); 1225 return retval; 1226 } 1227 1228 /* suspend_single_queue does not lock the dqm like the 1229 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should 1230 * lock the dqm before calling, and unlock after calling. 1231 * 1232 * The reason we don't lock the dqm is because this function may be 1233 * called on multiple queues in a loop, so rather than locking/unlocking 1234 * multiple times, we will just keep the dqm locked for all of the calls. 1235 */ 1236 static int suspend_single_queue(struct device_queue_manager *dqm, 1237 struct kfd_process_device *pdd, 1238 struct queue *q) 1239 { 1240 bool is_new; 1241 1242 if (q->properties.is_suspended) 1243 return 0; 1244 1245 pr_debug("Suspending process pid %d queue [%i]\n", 1246 pdd->process->lead_thread->pid, 1247 q->properties.queue_id); 1248 1249 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW); 1250 1251 if (is_new || q->properties.is_being_destroyed) { 1252 pr_debug("Suspend: skip %s queue id %i\n", 1253 is_new ? "new" : "destroyed", 1254 q->properties.queue_id); 1255 return -EBUSY; 1256 } 1257 1258 q->properties.is_suspended = true; 1259 if (q->properties.is_active) { 1260 if (dqm->dev->kfd->shared_resources.enable_mes) { 1261 int r = remove_queue_mes(dqm, q, &pdd->qpd); 1262 1263 if (r) 1264 return r; 1265 } 1266 1267 decrement_queue_count(dqm, &pdd->qpd, q); 1268 q->properties.is_active = false; 1269 } 1270 1271 return 0; 1272 } 1273 1274 /* resume_single_queue does not lock the dqm like the functions 1275 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should 1276 * lock the dqm before calling, and unlock after calling. 1277 * 1278 * The reason we don't lock the dqm is because this function may be 1279 * called on multiple queues in a loop, so rather than locking/unlocking 1280 * multiple times, we will just keep the dqm locked for all of the calls. 1281 */ 1282 static int resume_single_queue(struct device_queue_manager *dqm, 1283 struct qcm_process_device *qpd, 1284 struct queue *q) 1285 { 1286 struct kfd_process_device *pdd; 1287 1288 if (!q->properties.is_suspended) 1289 return 0; 1290 1291 pdd = qpd_to_pdd(qpd); 1292 1293 pr_debug("Restoring from suspend process pid %d queue [%i]\n", 1294 pdd->process->lead_thread->pid, 1295 q->properties.queue_id); 1296 1297 q->properties.is_suspended = false; 1298 1299 if (QUEUE_IS_ACTIVE(q->properties)) { 1300 if (dqm->dev->kfd->shared_resources.enable_mes) { 1301 int r = add_queue_mes(dqm, q, &pdd->qpd); 1302 1303 if (r) 1304 return r; 1305 } 1306 1307 q->properties.is_active = true; 1308 increment_queue_count(dqm, qpd, q); 1309 } 1310 1311 return 0; 1312 } 1313 1314 /* Unpin the MQD BO at S4 suspend so it is evicted into the hibernation image; 1315 * dqm_repin_mqd_bo() pins it back on resume. Gated on adev->in_s4 so runtime 1316 * eviction is untouched. 1317 */ 1318 static void dqm_evict_mqd_bo(struct device_queue_manager *dqm, struct queue *q) 1319 { 1320 struct mqd_manager *mqd_mgr; 1321 struct amdgpu_bo *bo; 1322 1323 if (!dqm->dev->adev->in_s4) 1324 return; 1325 if (!mqd_on_vram(dqm->dev->adev)) 1326 return; 1327 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) 1328 return; 1329 if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem) 1330 return; 1331 1332 /* Without update_mqd_gpu_addr() the MQD self-address cannot be fixed up 1333 * after a repin, so skip eviction (with a warning) instead of faulting. 1334 */ 1335 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; 1336 if (!mqd_mgr->update_mqd_gpu_addr) { 1337 dev_warn_once(dqm->dev->adev->dev, 1338 "MQD is in VRAM but update_mqd_gpu_addr is not implemented; skipping hibernation eviction\n"); 1339 return; 1340 } 1341 1342 bo = q->mqd_mem_obj->mem; 1343 if (amdgpu_bo_reserve(bo, false)) 1344 return; 1345 1346 amdgpu_bo_unpin(bo); 1347 amdgpu_bo_unreserve(bo); 1348 q->mqd = NULL; 1349 q->needs_mqd_repin = true; 1350 } 1351 1352 /* Repin the MQD BO to VRAM and refresh the cached mapping and GPU addresses. 1353 * Used both on resume and when a queue is destroyed before resume has repinned 1354 * it. A no-op unless a repin is owed (needs_mqd_repin set). 1355 */ 1356 static int dqm_repin_mqd_bo(struct device_queue_manager *dqm, struct queue *q) 1357 { 1358 struct mqd_manager *mqd_mgr; 1359 struct amdgpu_bo *bo; 1360 void *cpu_ptr; 1361 int r; 1362 1363 if (!q->needs_mqd_repin) 1364 return 0; 1365 if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem) 1366 return 0; 1367 1368 bo = q->mqd_mem_obj->mem; 1369 r = amdgpu_bo_reserve(bo, false); 1370 if (r) 1371 return r; 1372 r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM); 1373 if (r) { 1374 amdgpu_bo_unreserve(bo); 1375 dev_err(dqm->dev->adev->dev, 1376 "Failed to repin MQD of queue %d to VRAM: %d\n", 1377 q->properties.queue_id, r); 1378 return r; 1379 } 1380 /* The BO may have moved; refresh the kernel mapping and gpu address. */ 1381 amdgpu_bo_kunmap(bo); 1382 r = amdgpu_bo_kmap(bo, &cpu_ptr); 1383 amdgpu_bo_unreserve(bo); 1384 if (r) { 1385 dev_err(dqm->dev->adev->dev, 1386 "Failed to remap MQD of queue %d: %d\n", 1387 q->properties.queue_id, r); 1388 return r; 1389 } 1390 1391 q->mqd_mem_obj->cpu_ptr = cpu_ptr; 1392 q->mqd_mem_obj->gpu_addr = amdgpu_bo_gpu_offset(bo); 1393 q->gart_mqd_addr = q->mqd_mem_obj->gpu_addr; 1394 q->mqd = cpu_ptr; 1395 1396 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1397 q->properties.type)]; 1398 if (mqd_mgr->update_mqd_gpu_addr) 1399 mqd_mgr->update_mqd_gpu_addr(mqd_mgr, q->mqd, 1400 q->mqd_mem_obj, 1401 &q->properties); 1402 1403 q->needs_mqd_repin = false; 1404 return 0; 1405 } 1406 1407 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, 1408 struct qcm_process_device *qpd) 1409 { 1410 struct queue *q; 1411 struct mqd_manager *mqd_mgr; 1412 struct kfd_process_device *pdd; 1413 int retval, ret = 0; 1414 1415 dqm_lock(dqm); 1416 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1417 goto out; 1418 1419 pdd = qpd_to_pdd(qpd); 1420 pr_debug_ratelimited("Evicting process pid %d queues\n", 1421 pdd->process->lead_thread->pid); 1422 1423 pdd->last_evict_timestamp = get_jiffies_64(); 1424 /* Mark all queues as evicted. Deactivate all active queues on 1425 * the qpd. 1426 */ 1427 list_for_each_entry(q, &qpd->queues_list, list) { 1428 q->properties.is_evicted = true; 1429 if (!q->properties.is_active) 1430 continue; 1431 1432 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1433 q->properties.type)]; 1434 q->properties.is_active = false; 1435 decrement_queue_count(dqm, qpd, q); 1436 1437 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n")) 1438 continue; 1439 1440 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, 1441 (dqm->dev->kfd->cwsr_enabled ? 1442 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE : 1443 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN), 1444 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); 1445 if (retval && !ret) 1446 /* Return the first error, but keep going to 1447 * maintain a consistent eviction state 1448 */ 1449 ret = retval; 1450 } 1451 1452 out: 1453 dqm_unlock(dqm); 1454 return ret; 1455 } 1456 1457 static int evict_process_queues_cpsch(struct device_queue_manager *dqm, 1458 struct qcm_process_device *qpd) 1459 { 1460 struct queue *q; 1461 struct device *dev = dqm->dev->adev->dev; 1462 struct kfd_process_device *pdd; 1463 int retval = 0; 1464 1465 dqm_lock(dqm); 1466 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1467 goto out; 1468 1469 pdd = qpd_to_pdd(qpd); 1470 1471 /* The debugger creates processes that temporarily have not acquired 1472 * all VMs for all devices and has no VMs itself. 1473 * Skip queue eviction on process eviction. 1474 */ 1475 if (!pdd->drm_priv) 1476 goto out; 1477 1478 pr_debug_ratelimited("Evicting process pid %d queues\n", 1479 pdd->process->lead_thread->pid); 1480 1481 if (dqm->dev->kfd->shared_resources.enable_mes) 1482 pdd->last_evict_timestamp = get_jiffies_64(); 1483 1484 /* Mark all queues as evicted. Deactivate all active queues on 1485 * the qpd. 1486 */ 1487 list_for_each_entry(q, &qpd->queues_list, list) { 1488 q->properties.is_evicted = true; 1489 if (!q->properties.is_active) 1490 continue; 1491 1492 q->properties.is_active = false; 1493 decrement_queue_count(dqm, qpd, q); 1494 1495 if (dqm->dev->kfd->shared_resources.enable_mes) { 1496 retval = remove_queue_mes(dqm, q, qpd); 1497 if (retval) { 1498 dev_err(dev, "Failed to evict queue %d\n", 1499 q->properties.queue_id); 1500 goto out; 1501 } 1502 } 1503 1504 dqm_evict_mqd_bo(dqm, q); 1505 } 1506 1507 /* 1508 * Heavy-weight TLB flush after MES removes queues to ensure 1509 * in-flight memory accesses complete before memory is freed/migrated. 1510 * HWS does this automatically, MES does not. 1511 */ 1512 if (dqm->dev->kfd->shared_resources.enable_mes) 1513 kfd_flush_tlb(pdd); 1514 1515 if (!dqm->dev->kfd->shared_resources.enable_mes) { 1516 pdd->last_evict_timestamp = get_jiffies_64(); 1517 retval = execute_queues_cpsch(dqm, 1518 qpd->is_debug ? 1519 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : 1520 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 1521 USE_DEFAULT_GRACE_PERIOD); 1522 } 1523 1524 out: 1525 dqm_unlock(dqm); 1526 return retval; 1527 } 1528 1529 static int clean_process_queues_cpsch(struct device_queue_manager *dqm, 1530 struct qcm_process_device *qpd) 1531 { 1532 struct queue *q; 1533 struct kfd_process_device *pdd; 1534 int retval = 0; 1535 1536 dqm_lock(dqm); 1537 if (qpd->evicted++ > 0) /* already evicted, do nothing */ 1538 goto out; 1539 1540 pdd = qpd_to_pdd(qpd); 1541 1542 /* The debugger creates processes that temporarily have not acquired 1543 * all VMs for all devices and has no VMs itself. 1544 * Skip queue eviction on process eviction. 1545 */ 1546 if (!pdd->drm_priv) 1547 goto out; 1548 1549 pr_debug_ratelimited("Evicting process pid %d queues\n", 1550 pdd->process->lead_thread->pid); 1551 1552 /* Mark all queues as evicted. Deactivate all active queues on 1553 * the qpd. 1554 */ 1555 list_for_each_entry(q, &qpd->queues_list, list) { 1556 q->properties.is_evicted = true; 1557 if (!q->properties.is_active) 1558 continue; 1559 1560 q->properties.is_active = false; 1561 decrement_queue_count(dqm, qpd, q); 1562 1563 dqm_evict_mqd_bo(dqm, q); 1564 } 1565 1566 pdd->last_evict_timestamp = get_jiffies_64(); 1567 1568 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 1569 retval = -EIO; 1570 goto out; 1571 } 1572 1573 retval = unmap_queues_cpsch(dqm, 1574 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 1575 0, 1576 USE_DEFAULT_GRACE_PERIOD, 1577 false); 1578 1579 if (!retval) { 1580 dqm->dev->kfd2kgd->hqd_gfx_clean_fault(dqm->dev->adev); 1581 retval = map_queues_cpsch(dqm); 1582 } 1583 up_read(&dqm->dev->adev->reset_domain->sem); 1584 1585 out: 1586 dqm_unlock(dqm); 1587 return retval; 1588 } 1589 1590 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, 1591 struct qcm_process_device *qpd) 1592 { 1593 struct mm_struct *mm = NULL; 1594 struct queue *q; 1595 struct mqd_manager *mqd_mgr; 1596 struct kfd_process_device *pdd; 1597 uint64_t pd_base; 1598 uint64_t eviction_duration; 1599 int retval, ret = 0; 1600 1601 pdd = qpd_to_pdd(qpd); 1602 /* Retrieve PD base */ 1603 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1604 1605 dqm_lock(dqm); 1606 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1607 goto out; 1608 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1609 qpd->evicted--; 1610 goto out; 1611 } 1612 1613 pr_debug_ratelimited("Restoring process pid %d queues\n", 1614 pdd->process->lead_thread->pid); 1615 1616 /* Update PD Base in QPD */ 1617 qpd->page_table_base = pd_base; 1618 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1619 1620 if (!list_empty(&qpd->queues_list)) { 1621 dqm->dev->kfd2kgd->set_vm_context_page_table_base( 1622 dqm->dev->adev, 1623 qpd->vmid, 1624 qpd->page_table_base); 1625 kfd_flush_tlb(pdd); 1626 } 1627 1628 /* Take a safe reference to the mm_struct, which may otherwise 1629 * disappear even while the kfd_process is still referenced. 1630 */ 1631 mm = get_task_mm(pdd->process->lead_thread); 1632 if (!mm) { 1633 ret = -EFAULT; 1634 goto out; 1635 } 1636 1637 /* Remove the eviction flags. Activate queues that are not 1638 * inactive for other reasons. 1639 */ 1640 list_for_each_entry(q, &qpd->queues_list, list) { 1641 q->properties.is_evicted = false; 1642 if (!QUEUE_IS_ACTIVE(q->properties)) 1643 continue; 1644 1645 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 1646 q->properties.type)]; 1647 q->properties.is_active = true; 1648 increment_queue_count(dqm, qpd, q); 1649 1650 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n")) 1651 continue; 1652 1653 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, 1654 q->queue, &q->properties, mm); 1655 if (retval && !ret) 1656 /* Return the first error, but keep going to 1657 * maintain a consistent eviction state 1658 */ 1659 ret = retval; 1660 } 1661 qpd->evicted = 0; 1662 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1663 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1664 out: 1665 if (mm) 1666 mmput(mm); 1667 dqm_unlock(dqm); 1668 return ret; 1669 } 1670 1671 static int restore_process_queues_cpsch(struct device_queue_manager *dqm, 1672 struct qcm_process_device *qpd) 1673 { 1674 struct queue *q; 1675 struct device *dev = dqm->dev->adev->dev; 1676 struct kfd_process_device *pdd; 1677 uint64_t eviction_duration; 1678 int retval = 0; 1679 1680 pdd = qpd_to_pdd(qpd); 1681 1682 dqm_lock(dqm); 1683 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ 1684 goto out; 1685 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ 1686 qpd->evicted--; 1687 goto out; 1688 } 1689 1690 /* The debugger creates processes that temporarily have not acquired 1691 * all VMs for all devices and has no VMs itself. 1692 * Skip queue restore on process restore. 1693 */ 1694 if (!pdd->drm_priv) 1695 goto vm_not_acquired; 1696 1697 pr_debug_ratelimited("Restoring process pid %d queues\n", 1698 pdd->process->lead_thread->pid); 1699 1700 /* Update PD Base in QPD */ 1701 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1702 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); 1703 1704 /* activate all active queues on the qpd */ 1705 list_for_each_entry(q, &qpd->queues_list, list) { 1706 q->properties.is_evicted = false; 1707 if (!QUEUE_IS_ACTIVE(q->properties)) 1708 continue; 1709 1710 q->properties.is_active = true; 1711 increment_queue_count(dqm, &pdd->qpd, q); 1712 1713 retval = dqm_repin_mqd_bo(dqm, q); 1714 if (retval) { 1715 dev_err(dev, "Failed to repin MQD for queue %d\n", 1716 q->properties.queue_id); 1717 goto out; 1718 } 1719 1720 if (dqm->dev->kfd->shared_resources.enable_mes) { 1721 retval = add_queue_mes(dqm, q, qpd); 1722 if (retval) { 1723 dev_err(dev, "Failed to restore queue %d\n", 1724 q->properties.queue_id); 1725 goto out; 1726 } 1727 } 1728 } 1729 if (!dqm->dev->kfd->shared_resources.enable_mes) 1730 retval = execute_queues_cpsch(dqm, 1731 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 1732 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp; 1733 atomic64_add(eviction_duration, &pdd->evict_duration_counter); 1734 vm_not_acquired: 1735 qpd->evicted = 0; 1736 out: 1737 dqm_unlock(dqm); 1738 return retval; 1739 } 1740 1741 static int register_process(struct device_queue_manager *dqm, 1742 struct qcm_process_device *qpd) 1743 { 1744 struct device_process_node *n; 1745 struct kfd_process_device *pdd; 1746 uint64_t pd_base; 1747 int retval; 1748 1749 n = kzalloc_obj(*n); 1750 if (!n) 1751 return -ENOMEM; 1752 1753 n->qpd = qpd; 1754 1755 pdd = qpd_to_pdd(qpd); 1756 /* Retrieve PD base */ 1757 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); 1758 1759 dqm_lock(dqm); 1760 list_add(&n->list, &dqm->queues); 1761 1762 /* Update PD Base in QPD */ 1763 qpd->page_table_base = pd_base; 1764 pr_debug("Updated PD address to 0x%llx\n", pd_base); 1765 1766 retval = dqm->asic_ops.update_qpd(dqm, qpd); 1767 1768 dqm->processes_count++; 1769 1770 dqm_unlock(dqm); 1771 1772 /* Outside the DQM lock because under the DQM lock we can't do 1773 * reclaim or take other locks that others hold while reclaiming. 1774 */ 1775 kfd_inc_compute_active(dqm->dev); 1776 1777 return retval; 1778 } 1779 1780 static int unregister_process(struct device_queue_manager *dqm, 1781 struct qcm_process_device *qpd) 1782 { 1783 int retval = 0; 1784 struct device_process_node *cur, *next; 1785 1786 pr_debug("qpd->queues_list is %s\n", 1787 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 1788 1789 dqm_lock(dqm); 1790 1791 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 1792 if (qpd == cur->qpd) { 1793 list_del(&cur->list); 1794 kfree(cur); 1795 dqm->processes_count--; 1796 goto out; 1797 } 1798 } 1799 /* qpd not found in dqm list */ 1800 retval = 1; 1801 out: 1802 dqm_unlock(dqm); 1803 1804 /* Outside the DQM lock because under the DQM lock we can't do 1805 * reclaim or take other locks that others hold while reclaiming. 1806 */ 1807 if (!retval) 1808 kfd_dec_compute_active(dqm->dev); 1809 1810 return retval; 1811 } 1812 1813 static int 1814 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid, 1815 unsigned int vmid) 1816 { 1817 uint32_t xcc_mask = dqm->dev->xcc_mask; 1818 int xcc_id, ret = 0; 1819 1820 for_each_inst(xcc_id, xcc_mask) { 1821 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 1822 dqm->dev->adev, pasid, vmid, xcc_id); 1823 if (ret) 1824 break; 1825 } 1826 1827 return ret; 1828 } 1829 1830 static void init_interrupts(struct device_queue_manager *dqm) 1831 { 1832 uint32_t xcc_mask = dqm->dev->xcc_mask; 1833 unsigned int i, xcc_id; 1834 1835 for_each_inst(xcc_id, xcc_mask) { 1836 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) { 1837 if (is_pipe_enabled(dqm, 0, i)) { 1838 dqm->dev->kfd2kgd->init_interrupts( 1839 dqm->dev->adev, i, xcc_id); 1840 } 1841 } 1842 } 1843 } 1844 1845 static int initialize_nocpsch(struct device_queue_manager *dqm) 1846 { 1847 int pipe, queue; 1848 1849 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 1850 1851 dqm->allocated_queues = kzalloc_objs(unsigned int, 1852 get_pipes_per_mec(dqm)); 1853 if (!dqm->allocated_queues) 1854 return -ENOMEM; 1855 1856 mutex_init(&dqm->lock_hidden); 1857 INIT_LIST_HEAD(&dqm->queues); 1858 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0; 1859 dqm->active_cp_queue_count = 0; 1860 dqm->gws_queue_count = 0; 1861 1862 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 1863 int pipe_offset = pipe * get_queues_per_pipe(dqm); 1864 1865 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) 1866 if (test_bit(pipe_offset + queue, 1867 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 1868 dqm->allocated_queues[pipe] |= 1 << queue; 1869 } 1870 1871 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); 1872 1873 init_sdma_bitmaps(dqm); 1874 1875 return 0; 1876 } 1877 1878 static void uninitialize(struct device_queue_manager *dqm) 1879 { 1880 int i; 1881 1882 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0); 1883 1884 kfree(dqm->allocated_queues); 1885 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 1886 kfree(dqm->mqd_mgrs[i]); 1887 mutex_destroy(&dqm->lock_hidden); 1888 } 1889 1890 static int start_nocpsch(struct device_queue_manager *dqm) 1891 { 1892 int r = 0; 1893 1894 pr_info("SW scheduler is used"); 1895 init_interrupts(dqm); 1896 1897 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1898 r = pm_init(&dqm->packet_mgr, dqm); 1899 if (!r) 1900 dqm->sched_running = true; 1901 1902 return r; 1903 } 1904 1905 static int stop_nocpsch(struct device_queue_manager *dqm) 1906 { 1907 dqm_lock(dqm); 1908 if (!dqm->sched_running) { 1909 dqm_unlock(dqm); 1910 return 0; 1911 } 1912 1913 if (dqm->dev->adev->asic_type == CHIP_HAWAII) 1914 pm_uninit(&dqm->packet_mgr); 1915 dqm->sched_running = false; 1916 dqm_unlock(dqm); 1917 1918 return 0; 1919 } 1920 1921 static int allocate_sdma_queue(struct device_queue_manager *dqm, 1922 struct queue *q, const uint32_t *restore_sdma_id) 1923 { 1924 struct device *dev = dqm->dev->adev->dev; 1925 int bit; 1926 1927 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 1928 if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) { 1929 dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n", 1930 get_num_sdma_queues(dqm)); 1931 return -ENOMEM; 1932 } 1933 1934 if (restore_sdma_id) { 1935 if (*restore_sdma_id >= get_num_sdma_queues(dqm)) 1936 return -EINVAL; 1937 1938 /* Re-use existing sdma_id */ 1939 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { 1940 dev_err(dev, "SDMA queue already in use\n"); 1941 return -EBUSY; 1942 } 1943 clear_bit(*restore_sdma_id, dqm->sdma_bitmap); 1944 q->sdma_id = *restore_sdma_id; 1945 } else { 1946 /* Find first available sdma_id */ 1947 bit = find_first_bit(dqm->sdma_bitmap, 1948 get_num_sdma_queues(dqm)); 1949 clear_bit(bit, dqm->sdma_bitmap); 1950 q->sdma_id = bit; 1951 } 1952 1953 q->properties.sdma_engine_id = 1954 q->sdma_id % kfd_get_num_sdma_engines(dqm->dev); 1955 q->properties.sdma_queue_id = q->sdma_id / 1956 kfd_get_num_sdma_engines(dqm->dev); 1957 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 1958 if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) { 1959 dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n", 1960 get_num_xgmi_sdma_queues(dqm)); 1961 return -ENOMEM; 1962 } 1963 if (restore_sdma_id) { 1964 if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm)) 1965 return -EINVAL; 1966 1967 /* Re-use existing sdma_id */ 1968 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { 1969 dev_err(dev, "SDMA queue already in use\n"); 1970 return -EBUSY; 1971 } 1972 clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap); 1973 q->sdma_id = *restore_sdma_id; 1974 } else { 1975 bit = find_first_bit(dqm->xgmi_sdma_bitmap, 1976 get_num_xgmi_sdma_queues(dqm)); 1977 clear_bit(bit, dqm->xgmi_sdma_bitmap); 1978 q->sdma_id = bit; 1979 } 1980 /* sdma_engine_id is sdma id including 1981 * both PCIe-optimized SDMAs and XGMI- 1982 * optimized SDMAs. The calculation below 1983 * assumes the first N engines are always 1984 * PCIe-optimized ones 1985 */ 1986 q->properties.sdma_engine_id = 1987 kfd_get_num_sdma_engines(dqm->dev) + 1988 q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev); 1989 q->properties.sdma_queue_id = q->sdma_id / 1990 kfd_get_num_xgmi_sdma_engines(dqm->dev); 1991 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 1992 int i, num_queues, num_engines, eng_offset = 0, start_engine; 1993 bool free_bit_found = false, is_xgmi = false; 1994 1995 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) { 1996 num_queues = get_num_sdma_queues(dqm); 1997 num_engines = kfd_get_num_sdma_engines(dqm->dev); 1998 q->properties.type = KFD_QUEUE_TYPE_SDMA; 1999 } else { 2000 num_queues = get_num_xgmi_sdma_queues(dqm); 2001 num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev); 2002 eng_offset = kfd_get_num_sdma_engines(dqm->dev); 2003 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI; 2004 is_xgmi = true; 2005 } 2006 2007 /* Scan available bit based on target engine ID. */ 2008 start_engine = q->properties.sdma_engine_id - eng_offset; 2009 for (i = start_engine; i < num_queues; i += num_engines) { 2010 2011 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap)) 2012 continue; 2013 2014 clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap); 2015 q->sdma_id = i; 2016 q->properties.sdma_queue_id = q->sdma_id / num_engines; 2017 free_bit_found = true; 2018 break; 2019 } 2020 2021 if (!free_bit_found) { 2022 dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n", 2023 q->properties.sdma_engine_id, num_queues); 2024 return -ENOMEM; 2025 } 2026 } 2027 2028 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); 2029 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); 2030 2031 return 0; 2032 } 2033 2034 static void deallocate_sdma_queue(struct device_queue_manager *dqm, 2035 struct queue *q) 2036 { 2037 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 2038 if (q->sdma_id >= get_num_sdma_queues(dqm)) 2039 return; 2040 set_bit(q->sdma_id, dqm->sdma_bitmap); 2041 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2042 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) 2043 return; 2044 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap); 2045 } 2046 } 2047 2048 /* 2049 * Device Queue Manager implementation for cp scheduler 2050 */ 2051 2052 static int set_sched_resources(struct device_queue_manager *dqm) 2053 { 2054 int i, mec; 2055 struct scheduling_resources res; 2056 struct device *dev = dqm->dev->adev->dev; 2057 2058 res.vmid_mask = dqm->dev->compute_vmid_bitmap; 2059 2060 res.queue_mask = 0; 2061 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2062 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2063 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2064 2065 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2066 continue; 2067 2068 /* only acquire queues from the first MEC */ 2069 if (mec > 0) 2070 continue; 2071 2072 /* This situation may be hit in the future if a new HW 2073 * generation exposes more than 64 queues. If so, the 2074 * definition of res.queue_mask needs updating 2075 */ 2076 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) { 2077 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i); 2078 break; 2079 } 2080 2081 res.queue_mask |= 1ull 2082 << amdgpu_queue_mask_bit_to_set_resource_bit( 2083 dqm->dev->adev, i); 2084 } 2085 res.gws_mask = ~0ull; 2086 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0; 2087 2088 pr_debug("Scheduling resources:\n" 2089 "vmid mask: 0x%8X\n" 2090 "queue mask: 0x%8llX\n", 2091 res.vmid_mask, res.queue_mask); 2092 2093 return pm_send_set_resources(&dqm->packet_mgr, &res); 2094 } 2095 2096 static int initialize_cpsch(struct device_queue_manager *dqm) 2097 { 2098 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); 2099 2100 mutex_init(&dqm->lock_hidden); 2101 INIT_LIST_HEAD(&dqm->queues); 2102 dqm->active_queue_count = dqm->processes_count = 0; 2103 dqm->active_cp_queue_count = 0; 2104 dqm->gws_queue_count = 0; 2105 dqm->active_runlist = false; 2106 dqm->trap_debug_vmid = 0; 2107 2108 init_sdma_bitmaps(dqm); 2109 2110 update_dqm_wait_times(dqm); 2111 return 0; 2112 } 2113 2114 /* halt_cpsch: 2115 * Unmap queues so the schedule doesn't continue remaining jobs in the queue. 2116 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch 2117 * is called. 2118 */ 2119 static int halt_cpsch(struct device_queue_manager *dqm) 2120 { 2121 int ret = 0; 2122 2123 dqm_lock(dqm); 2124 if (!dqm->sched_running) { 2125 dqm_unlock(dqm); 2126 return 0; 2127 } 2128 2129 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n"); 2130 2131 if (!dqm->is_hws_hang) { 2132 if (!dqm->dev->kfd->shared_resources.enable_mes) 2133 ret = unmap_queues_cpsch(dqm, 2134 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 2135 USE_DEFAULT_GRACE_PERIOD, false); 2136 else 2137 ret = remove_all_kfd_queues_mes(dqm); 2138 } 2139 dqm->sched_halt = true; 2140 dqm_unlock(dqm); 2141 2142 return ret; 2143 } 2144 2145 /* unhalt_cpsch 2146 * Unset dqm->sched_halt and map queues back to runlist 2147 */ 2148 static int unhalt_cpsch(struct device_queue_manager *dqm) 2149 { 2150 int ret = 0; 2151 struct amdgpu_device *adev = dqm->dev->adev; 2152 2153 dqm_lock(dqm); 2154 if (!dqm->sched_running || !dqm->sched_halt) { 2155 dev_dbg(adev->dev, "Scheduling is not on halt.\n"); 2156 dqm_unlock(dqm); 2157 return 0; 2158 } 2159 dqm->sched_halt = false; 2160 if (!dqm->dev->kfd->shared_resources.enable_mes) 2161 ret = execute_queues_cpsch(dqm, 2162 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 2163 0, USE_DEFAULT_GRACE_PERIOD); 2164 else 2165 ret = add_all_kfd_queues_mes(dqm); 2166 2167 dqm_unlock(dqm); 2168 2169 return ret; 2170 } 2171 2172 /* Program CP_IQ_WAIT_TIME2 via the MES WRITE_REG op (MES has no HIQ). */ 2173 static int init_dequeue_wait_counts_mes(struct device_queue_manager *dqm) 2174 { 2175 struct kfd_node *dev = dqm->dev; 2176 struct amdgpu_device *adev = dev->adev; 2177 uint32_t sch_wave = 0, que_sleep = 1; 2178 int inst, ret; 2179 2180 if (!dev->kfd2kgd->build_dequeue_wait_counts_packet_info) 2181 return 0; 2182 2183 if (KFD_GC_VERSION(dev) != IP_VERSION(12, 1, 0)) 2184 return 0; 2185 2186 /* CP_IQ_WAIT_TIME2 is per-XCC; the offset must encode the target 2187 * XCC so MES routes the write to that XCC's local register. 2188 */ 2189 for_each_inst(inst, dev->xcc_mask) { 2190 uint32_t reg_offset = 0, reg_data = 0; 2191 2192 dev->kfd2kgd->build_dequeue_wait_counts_packet_info( 2193 adev, dqm->wait_times, sch_wave, que_sleep, 2194 ®_offset, ®_data, inst); 2195 2196 ret = amdgpu_mes_wreg(adev, reg_offset, reg_data, inst); 2197 if (ret) { 2198 dev_err(adev->dev, 2199 "Failed to set optimized dequeue wait via MES on xcc %d\n", 2200 inst); 2201 return ret; 2202 } 2203 } 2204 2205 update_dqm_wait_times(dqm); 2206 return 0; 2207 } 2208 2209 static int start_cpsch(struct device_queue_manager *dqm) 2210 { 2211 struct device *dev = dqm->dev->adev->dev; 2212 int retval, num_hw_queue_slots; 2213 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; 2214 int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); 2215 int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; 2216 2217 dqm_lock(dqm); 2218 2219 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2220 retval = pm_init(&dqm->packet_mgr, dqm); 2221 if (retval) 2222 goto fail_packet_manager_init; 2223 2224 retval = set_sched_resources(dqm); 2225 if (retval) 2226 goto fail_set_sched_resources; 2227 } 2228 pr_debug("Allocating fence memory\n"); 2229 2230 /* allocate fence memory on the gart */ 2231 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 2232 &dqm->fence_mem); 2233 2234 if (retval) 2235 goto fail_allocate_vidmem; 2236 2237 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr; 2238 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 2239 2240 init_interrupts(dqm); 2241 2242 /* clear hang status when driver try to start the hw scheduler */ 2243 dqm->sched_running = true; 2244 2245 if (!dqm->dev->kfd->shared_resources.enable_mes) { 2246 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2247 KFD_DEQUEUE_WAIT_INIT, 0 /* unused */)) 2248 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); 2249 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2250 } else { 2251 if (init_dequeue_wait_counts_mes(dqm)) 2252 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n"); 2253 } 2254 2255 /* setup per-queue reset detection buffer */ 2256 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * 2257 dqm->dev->kfd->shared_resources.num_pipe_per_mec * 2258 NUM_XCC(dqm->dev->xcc_mask); 2259 2260 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info); 2261 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL); 2262 2263 if (!dqm->detect_hang_info) { 2264 retval = -ENOMEM; 2265 goto fail_detect_hang_buffer; 2266 } 2267 2268 dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL); 2269 dqm->hqd_info = kzalloc( 2270 hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info), 2271 GFP_KERNEL); 2272 2273 dqm_unlock(dqm); 2274 2275 return 0; 2276 fail_detect_hang_buffer: 2277 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 2278 fail_allocate_vidmem: 2279 fail_set_sched_resources: 2280 if (!dqm->dev->kfd->shared_resources.enable_mes) 2281 pm_uninit(&dqm->packet_mgr); 2282 fail_packet_manager_init: 2283 dqm_unlock(dqm); 2284 return retval; 2285 } 2286 2287 static int stop_cpsch(struct device_queue_manager *dqm) 2288 { 2289 int ret = 0; 2290 2291 dqm_lock(dqm); 2292 if (!dqm->sched_running) { 2293 dqm_unlock(dqm); 2294 return 0; 2295 } 2296 2297 if (!dqm->dev->kfd->shared_resources.enable_mes) 2298 ret = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 2299 0, USE_DEFAULT_GRACE_PERIOD, false); 2300 else 2301 ret = remove_all_kfd_queues_mes(dqm); 2302 2303 dqm->sched_running = false; 2304 2305 if (!dqm->dev->kfd->shared_resources.enable_mes) 2306 pm_release_ib(&dqm->packet_mgr); 2307 2308 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 2309 if (!dqm->dev->kfd->shared_resources.enable_mes) 2310 pm_uninit(&dqm->packet_mgr); 2311 kfree(dqm->detect_hang_info); 2312 dqm->detect_hang_info = NULL; 2313 kfree(dqm->hung_db_array); 2314 kfree(dqm->hqd_info); 2315 2316 dqm_unlock(dqm); 2317 2318 return ret; 2319 } 2320 2321 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 2322 struct kernel_queue *kq, 2323 struct qcm_process_device *qpd) 2324 { 2325 dqm_lock(dqm); 2326 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 2327 pr_warn("Can't create new kernel queue because %d queues were already created\n", 2328 dqm->total_queue_count); 2329 dqm_unlock(dqm); 2330 return -EPERM; 2331 } 2332 2333 /* 2334 * Unconditionally increment this counter, regardless of the queue's 2335 * type or whether the queue is active. 2336 */ 2337 dqm->total_queue_count++; 2338 pr_debug("Total of %d queues are accountable so far\n", 2339 dqm->total_queue_count); 2340 2341 list_add(&kq->list, &qpd->priv_queue_list); 2342 increment_queue_count(dqm, qpd, kq->queue); 2343 qpd->is_debug = true; 2344 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 2345 USE_DEFAULT_GRACE_PERIOD); 2346 dqm_unlock(dqm); 2347 2348 return 0; 2349 } 2350 2351 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 2352 struct kernel_queue *kq, 2353 struct qcm_process_device *qpd) 2354 { 2355 dqm_lock(dqm); 2356 list_del(&kq->list); 2357 decrement_queue_count(dqm, qpd, kq->queue); 2358 qpd->is_debug = false; 2359 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 2360 USE_DEFAULT_GRACE_PERIOD); 2361 /* 2362 * Unconditionally decrement this counter, regardless of the queue's 2363 * type. 2364 */ 2365 dqm->total_queue_count--; 2366 pr_debug("Total of %d queues are accountable so far\n", 2367 dqm->total_queue_count); 2368 dqm_unlock(dqm); 2369 } 2370 2371 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 2372 struct qcm_process_device *qpd, 2373 const struct kfd_criu_queue_priv_data *qd, 2374 const void *restore_mqd, const void *restore_ctl_stack) 2375 { 2376 int retval; 2377 struct mqd_manager *mqd_mgr; 2378 2379 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 2380 pr_warn("Can't create new usermode queue because %d queues were already created\n", 2381 dqm->total_queue_count); 2382 retval = -EPERM; 2383 goto out; 2384 } 2385 2386 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2387 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI || 2388 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) { 2389 dqm_lock(dqm); 2390 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL); 2391 dqm_unlock(dqm); 2392 if (retval) 2393 goto out; 2394 } 2395 2396 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); 2397 if (retval) 2398 goto out_deallocate_sdma_queue; 2399 2400 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 2401 q->properties.type)]; 2402 if (qd && !mqd_mgr->restore_mqd) { 2403 pr_debug("restore_mqd not implemented for queue type %d\n", 2404 q->properties.type); 2405 retval = -EOPNOTSUPP; 2406 goto out_deallocate_doorbell; 2407 } 2408 2409 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2410 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 2411 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); 2412 q->properties.tba_addr = qpd->tba_addr; 2413 q->properties.tma_addr = qpd->tma_addr; 2414 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties); 2415 if (!q->mqd_mem_obj) { 2416 retval = -ENOMEM; 2417 goto out_deallocate_doorbell; 2418 } 2419 2420 dqm_lock(dqm); 2421 /* 2422 * Eviction state logic: mark all queues as evicted, even ones 2423 * not currently active. Restoring inactive queues later only 2424 * updates the is_evicted flag but is a no-op otherwise. 2425 */ 2426 q->properties.is_evicted = !!qpd->evicted; 2427 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && 2428 kfd_dbg_has_cwsr_workaround(q->device); 2429 2430 if (qd) 2431 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr, 2432 &q->properties, restore_mqd, restore_ctl_stack, 2433 qd->ctl_stack_size); 2434 else 2435 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, 2436 &q->gart_mqd_addr, &q->properties); 2437 2438 list_add(&q->list, &qpd->queues_list); 2439 qpd->queue_count++; 2440 2441 if (q->properties.is_active) { 2442 increment_queue_count(dqm, qpd, q); 2443 2444 if (!dqm->dev->kfd->shared_resources.enable_mes) 2445 retval = execute_queues_cpsch(dqm, 2446 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); 2447 else 2448 retval = add_queue_mes(dqm, q, qpd); 2449 if (retval) 2450 goto cleanup_queue; 2451 } 2452 2453 /* 2454 * Unconditionally increment this counter, regardless of the queue's 2455 * type or whether the queue is active. 2456 */ 2457 dqm->total_queue_count++; 2458 2459 pr_debug("Total of %d queues are accountable so far\n", 2460 dqm->total_queue_count); 2461 2462 dqm_unlock(dqm); 2463 return retval; 2464 2465 cleanup_queue: 2466 qpd->queue_count--; 2467 list_del(&q->list); 2468 if (q->properties.is_active) 2469 decrement_queue_count(dqm, qpd, q); 2470 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 2471 dqm_unlock(dqm); 2472 out_deallocate_doorbell: 2473 deallocate_doorbell(qpd, q); 2474 out_deallocate_sdma_queue: 2475 if (q->properties.type == KFD_QUEUE_TYPE_SDMA || 2476 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { 2477 dqm_lock(dqm); 2478 deallocate_sdma_queue(dqm, q); 2479 dqm_unlock(dqm); 2480 } 2481 out: 2482 return retval; 2483 } 2484 2485 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm, 2486 uint64_t fence_value, 2487 unsigned int timeout_ms) 2488 { 2489 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies; 2490 struct device *dev = dqm->dev->adev->dev; 2491 uint64_t *fence_addr = dqm->fence_addr; 2492 2493 while (*fence_addr != fence_value) { 2494 /* Fatal err detected, this response won't come */ 2495 if (amdgpu_amdkfd_is_fed(dqm->dev->adev) || 2496 amdgpu_in_reset(dqm->dev->adev)) 2497 return -EIO; 2498 2499 if (time_after(jiffies, end_jiffies)) { 2500 dev_err(dev, "qcm fence wait loop timeout expired\n"); 2501 /* In HWS case, this is used to halt the driver thread 2502 * in order not to mess up CP states before doing 2503 * scandumps for FW debugging. 2504 */ 2505 while (halt_if_hws_hang) 2506 schedule(); 2507 2508 return -ETIME; 2509 } 2510 schedule(); 2511 } 2512 2513 return 0; 2514 } 2515 2516 /* dqm->lock mutex has to be locked before calling this function */ 2517 static int map_queues_cpsch(struct device_queue_manager *dqm) 2518 { 2519 struct device *dev = dqm->dev->adev->dev; 2520 int retval; 2521 2522 if (!dqm->sched_running || dqm->sched_halt) 2523 return 0; 2524 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0) 2525 return 0; 2526 if (dqm->active_runlist) 2527 return 0; 2528 2529 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues); 2530 pr_debug("%s sent runlist\n", __func__); 2531 if (retval) { 2532 dev_err(dev, "failed to execute runlist\n"); 2533 return retval; 2534 } 2535 dqm->active_runlist = true; 2536 2537 return retval; 2538 } 2539 2540 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, 2541 struct qcm_process_device *qpd, bool verbose) 2542 { 2543 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2544 2545 if (verbose) 2546 dev_err(dqm->dev->adev->dev, 2547 "queue id 0x%0x at pasid %d is reset\n", 2548 q->properties.queue_id, pdd->process->lead_thread->pid); 2549 2550 pdd->has_reset_queue = true; 2551 q->properties.is_reset = true; 2552 if (q->properties.is_active) { 2553 q->properties.is_active = false; 2554 decrement_queue_count(dqm, qpd, q); 2555 } 2556 } 2557 2558 void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm) 2559 { 2560 struct device_process_node *cur; 2561 struct qcm_process_device *qpd; 2562 struct queue *q; 2563 2564 dqm_lock(dqm); 2565 if (dqm->processes_count) 2566 dev_warn(dqm->dev->adev->dev, 2567 "GPU reset: queues of %d process(es) are no longer usable\n", 2568 dqm->processes_count); 2569 2570 list_for_each_entry(cur, &dqm->queues, list) { 2571 qpd = cur->qpd; 2572 list_for_each_entry(q, &qpd->queues_list, list) 2573 set_queue_as_reset(dqm, q, qpd, false); 2574 } 2575 dqm_unlock(dqm); 2576 } 2577 2578 static int detect_queue_hang(struct device_queue_manager *dqm) 2579 { 2580 int i; 2581 2582 /* detect should be used only in dqm locked queue reset */ 2583 if (WARN_ON(dqm->detect_hang_count > 0)) 2584 return 0; 2585 2586 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size); 2587 2588 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) { 2589 uint32_t mec, pipe, queue; 2590 int xcc_id; 2591 2592 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) 2593 / dqm->dev->kfd->shared_resources.num_pipe_per_mec; 2594 2595 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 2596 continue; 2597 2598 amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue); 2599 2600 for_each_inst(xcc_id, dqm->dev->xcc_mask) { 2601 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr( 2602 dqm->dev->adev, pipe, queue, xcc_id); 2603 struct dqm_detect_hang_info hang_info; 2604 2605 if (!queue_addr) 2606 continue; 2607 2608 hang_info.pipe_id = pipe; 2609 hang_info.queue_id = queue; 2610 hang_info.xcc_id = xcc_id; 2611 hang_info.queue_address = queue_addr; 2612 2613 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; 2614 dqm->detect_hang_count++; 2615 } 2616 } 2617 2618 return dqm->detect_hang_count; 2619 } 2620 2621 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address) 2622 { 2623 struct device_process_node *cur; 2624 struct qcm_process_device *qpd; 2625 struct queue *q; 2626 2627 list_for_each_entry(cur, &dqm->queues, list) { 2628 qpd = cur->qpd; 2629 list_for_each_entry(q, &qpd->queues_list, list) { 2630 if (queue_address == q->properties.queue_address) 2631 return q; 2632 } 2633 } 2634 2635 return NULL; 2636 } 2637 2638 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset) 2639 { 2640 struct device_process_node *cur; 2641 struct qcm_process_device *qpd; 2642 struct queue *q; 2643 2644 list_for_each_entry(cur, &dqm->queues, list) { 2645 qpd = cur->qpd; 2646 list_for_each_entry(q, &qpd->queues_list, list) { 2647 if (doorbell_offset == q->properties.doorbell_off) 2648 return q; 2649 } 2650 } 2651 2652 return NULL; 2653 } 2654 2655 static int reset_hung_queues(struct device_queue_manager *dqm) 2656 { 2657 int r = 0, reset_count = 0, i; 2658 2659 if (!dqm->detect_hang_info || dqm->is_hws_hang) 2660 return -EIO; 2661 2662 /* assume dqm locked. */ 2663 if (!detect_queue_hang(dqm)) 2664 return -ENOTRECOVERABLE; 2665 2666 for (i = 0; i < dqm->detect_hang_count; i++) { 2667 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; 2668 struct queue *q = find_queue_by_address(dqm, hang_info.queue_address); 2669 struct kfd_process_device *pdd; 2670 uint64_t queue_addr = 0; 2671 2672 if (!q) { 2673 r = -ENOTRECOVERABLE; 2674 goto reset_fail; 2675 } 2676 2677 pdd = kfd_get_process_device_data(dqm->dev, q->process); 2678 if (!pdd) { 2679 r = -ENOTRECOVERABLE; 2680 goto reset_fail; 2681 } 2682 2683 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev, 2684 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id, 2685 KFD_UNMAP_LATENCY_MS); 2686 2687 /* either reset failed or we reset an unexpected queue. */ 2688 if (queue_addr != q->properties.queue_address) { 2689 r = -ENOTRECOVERABLE; 2690 goto reset_fail; 2691 } 2692 2693 set_queue_as_reset(dqm, q, &pdd->qpd, true); 2694 reset_count++; 2695 } 2696 2697 if (reset_count == dqm->detect_hang_count) 2698 kfd_signal_reset_event(dqm->dev); 2699 else 2700 r = -ENOTRECOVERABLE; 2701 2702 reset_fail: 2703 dqm->detect_hang_count = 0; 2704 2705 return r; 2706 } 2707 2708 static bool sdma_has_hang(struct device_queue_manager *dqm) 2709 { 2710 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2711 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2712 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2713 int i, j; 2714 2715 for (i = engine_start; i < engine_end; i++) { 2716 for (j = 0; j < num_queues_per_eng; j++) { 2717 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j)) 2718 continue; 2719 2720 return true; 2721 } 2722 } 2723 2724 return false; 2725 } 2726 2727 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm, 2728 uint32_t doorbell_off) 2729 { 2730 struct device_process_node *cur; 2731 struct qcm_process_device *qpd; 2732 struct queue *q; 2733 2734 list_for_each_entry(cur, &dqm->queues, list) { 2735 qpd = cur->qpd; 2736 list_for_each_entry(q, &qpd->queues_list, list) { 2737 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA || 2738 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) && 2739 q->properties.doorbell_off == doorbell_off) { 2740 set_queue_as_reset(dqm, q, qpd, true); 2741 return true; 2742 } 2743 } 2744 } 2745 2746 return false; 2747 } 2748 2749 static int reset_hung_queues_sdma(struct device_queue_manager *dqm) 2750 { 2751 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 2752 int engine_end = engine_start + get_num_all_sdma_engines(dqm); 2753 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 2754 int r = 0, i, j; 2755 2756 if (dqm->is_hws_hang) 2757 return -EIO; 2758 2759 /* Scan for hung HW queues and reset engine. */ 2760 dqm->detect_hang_count = 0; 2761 for (i = engine_start; i < engine_end; i++) { 2762 for (j = 0; j < num_queues_per_eng; j++) { 2763 uint32_t doorbell_off = 2764 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j); 2765 2766 if (!doorbell_off) 2767 continue; 2768 2769 /* Reset engine and check. */ 2770 if (amdgpu_sdma_reset_engine(dqm->dev->adev, i) || 2771 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) || 2772 !set_sdma_queue_as_reset(dqm, doorbell_off)) { 2773 r = -ENOTRECOVERABLE; 2774 goto reset_fail; 2775 } 2776 2777 /* Should only expect one queue active per engine */ 2778 dqm->detect_hang_count++; 2779 break; 2780 } 2781 } 2782 2783 /* Signal process reset */ 2784 if (dqm->detect_hang_count) 2785 kfd_signal_reset_event(dqm->dev); 2786 else 2787 r = -ENOTRECOVERABLE; 2788 2789 reset_fail: 2790 dqm->detect_hang_count = 0; 2791 2792 return r; 2793 } 2794 2795 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma) 2796 { 2797 struct amdgpu_device *adev = dqm->dev->adev; 2798 2799 while (halt_if_hws_hang) 2800 schedule(); 2801 2802 if (adev->debug_disable_gpu_ring_reset) { 2803 dev_info_once(adev->dev, 2804 "%s queue hung, but ring reset disabled", 2805 is_sdma ? "sdma" : "compute"); 2806 2807 return -EPERM; 2808 } 2809 if (!amdgpu_gpu_recovery) 2810 return -ENOTRECOVERABLE; 2811 2812 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm); 2813 } 2814 2815 /* dqm->lock mutex has to be locked before calling this function 2816 * 2817 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time 2818 * for context switch latency. Lower values are used by debugger 2819 * since context switching are triggered at high frequency. 2820 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE 2821 * 2822 */ 2823 static int unmap_queues_cpsch(struct device_queue_manager *dqm, 2824 enum kfd_unmap_queues_filter filter, 2825 uint32_t filter_param, 2826 uint32_t grace_period, 2827 bool reset) 2828 { 2829 struct device *dev = dqm->dev->adev->dev; 2830 struct mqd_manager *mqd_mgr; 2831 int retval; 2832 2833 if (!dqm->sched_running) 2834 return 0; 2835 if (!dqm->active_runlist) 2836 return 0; 2837 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2838 return -EIO; 2839 2840 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2841 retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2842 KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period); 2843 if (retval) 2844 goto out; 2845 } 2846 2847 retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset); 2848 if (retval) 2849 goto out; 2850 2851 *dqm->fence_addr = KFD_FENCE_INIT; 2852 mb(); 2853 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr, 2854 KFD_FENCE_COMPLETED); 2855 /* should be timed out */ 2856 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED, 2857 queue_preemption_timeout_ms); 2858 if (retval) { 2859 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n"); 2860 kfd_hws_hang(dqm); 2861 goto out; 2862 } 2863 2864 /* In the current MEC firmware implementation, if compute queue 2865 * doesn't response to the preemption request in time, HIQ will 2866 * abandon the unmap request without returning any timeout error 2867 * to driver. Instead, MEC firmware will log the doorbell of the 2868 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields. 2869 * To make sure the queue unmap was successful, driver need to 2870 * check those fields 2871 */ 2872 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; 2873 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) && 2874 reset_queues_on_hws_hang(dqm, false)) 2875 goto reset_fail; 2876 2877 /* Check for SDMA hang and attempt SDMA reset */ 2878 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true)) 2879 goto reset_fail; 2880 2881 /* We need to reset the grace period value for this device */ 2882 if (grace_period != USE_DEFAULT_GRACE_PERIOD) { 2883 if (pm_config_dequeue_wait_counts(&dqm->packet_mgr, 2884 KFD_DEQUEUE_WAIT_RESET, 0 /* unused */)) 2885 dev_err(dev, "Failed to reset grace period\n"); 2886 } 2887 2888 pm_release_ib(&dqm->packet_mgr); 2889 dqm->active_runlist = false; 2890 out: 2891 up_read(&dqm->dev->adev->reset_domain->sem); 2892 return retval; 2893 2894 reset_fail: 2895 dqm->is_hws_hang = true; 2896 kfd_hws_hang(dqm); 2897 up_read(&dqm->dev->adev->reset_domain->sem); 2898 return -ETIME; 2899 } 2900 2901 /* only for compute queue */ 2902 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid) 2903 { 2904 int retval; 2905 2906 dqm_lock(dqm); 2907 2908 retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID, 2909 pasid, USE_DEFAULT_GRACE_PERIOD, true); 2910 2911 dqm_unlock(dqm); 2912 return retval; 2913 } 2914 2915 /* dqm->lock mutex has to be locked before calling this function */ 2916 static int execute_queues_cpsch(struct device_queue_manager *dqm, 2917 enum kfd_unmap_queues_filter filter, 2918 uint32_t filter_param, 2919 uint32_t grace_period) 2920 { 2921 int retval; 2922 2923 if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) 2924 return -EIO; 2925 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false); 2926 if (!retval) 2927 retval = map_queues_cpsch(dqm); 2928 up_read(&dqm->dev->adev->reset_domain->sem); 2929 return retval; 2930 } 2931 2932 static int wait_on_destroy_queue(struct device_queue_manager *dqm, 2933 struct queue *q) 2934 { 2935 struct kfd_process_device *pdd = kfd_get_process_device_data(q->device, 2936 q->process); 2937 int ret = 0; 2938 2939 if (WARN_ON(!pdd)) 2940 return ret; 2941 2942 if (pdd->qpd.is_debug) 2943 return ret; 2944 2945 if (q->properties.is_being_destroyed) 2946 return -EBUSY; 2947 2948 q->properties.is_being_destroyed = true; 2949 2950 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { 2951 dqm_unlock(dqm); 2952 mutex_unlock(&q->process->mutex); 2953 ret = wait_event_interruptible(dqm->destroy_wait, 2954 !q->properties.is_suspended); 2955 2956 mutex_lock(&q->process->mutex); 2957 dqm_lock(dqm); 2958 } 2959 2960 if (ret) 2961 q->properties.is_being_destroyed = false; 2962 2963 return ret; 2964 } 2965 2966 static int destroy_queue_cpsch(struct device_queue_manager *dqm, 2967 struct qcm_process_device *qpd, 2968 struct queue *q) 2969 { 2970 int retval; 2971 struct mqd_manager *mqd_mgr; 2972 uint64_t sdma_val = 0; 2973 struct kfd_process_device *pdd = qpd_to_pdd(qpd); 2974 struct device *dev = dqm->dev->adev->dev; 2975 2976 /* Get the SDMA queue stats */ 2977 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 2978 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 2979 if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) 2980 retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( 2981 dqm->dev->adev, q->mqd, 2982 dqm->dev->kfd->device_info.num_sdma_queues_per_engine, 2983 &sdma_val); 2984 else 2985 retval = read_sdma_queue_counter( 2986 (uint64_t __user *)q->properties.read_ptr, 2987 &sdma_val); 2988 2989 if (retval) 2990 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", 2991 q->properties.queue_id); 2992 } 2993 2994 /* remove queue from list to prevent rescheduling after preemption */ 2995 dqm_lock(dqm); 2996 2997 retval = wait_on_destroy_queue(dqm, q); 2998 2999 if (retval) { 3000 dqm_unlock(dqm); 3001 return retval; 3002 } 3003 3004 if (qpd->is_debug) { 3005 /* 3006 * error, currently we do not allow to destroy a queue 3007 * of a currently debugged process 3008 */ 3009 retval = -EBUSY; 3010 goto failed_try_destroy_debugged_queue; 3011 3012 } 3013 3014 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 3015 q->properties.type)]; 3016 3017 deallocate_doorbell(qpd, q); 3018 3019 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || 3020 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { 3021 deallocate_sdma_queue(dqm, q); 3022 pdd->sdma_past_activity_counter += sdma_val; 3023 } 3024 3025 if (q->properties.is_active) { 3026 decrement_queue_count(dqm, qpd, q); 3027 q->properties.is_active = false; 3028 if (!dqm->dev->kfd->shared_resources.enable_mes) { 3029 retval = execute_queues_cpsch(dqm, 3030 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3031 USE_DEFAULT_GRACE_PERIOD); 3032 if (retval == -ETIME) 3033 qpd->reset_wavefronts = true; 3034 } else { 3035 retval = remove_queue_mes(dqm, q, qpd); 3036 } 3037 } 3038 list_del(&q->list); 3039 qpd->queue_count--; 3040 3041 /* 3042 * Unconditionally decrement this counter, regardless of the queue's 3043 * type 3044 */ 3045 dqm->total_queue_count--; 3046 pr_debug("Total of %d queues are accountable so far\n", 3047 dqm->total_queue_count); 3048 3049 dqm_unlock(dqm); 3050 3051 /* 3052 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid 3053 * circular locking 3054 */ 3055 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE), 3056 qpd->pqm->process, q->device, 3057 -1, false, NULL, 0); 3058 3059 /* Repin the MQD BO if still evicted for hibernation, before it is freed. */ 3060 dqm_repin_mqd_bo(dqm, q); 3061 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 3062 3063 return retval; 3064 3065 failed_try_destroy_debugged_queue: 3066 q->properties.is_being_destroyed = false; 3067 dqm_unlock(dqm); 3068 return retval; 3069 } 3070 3071 static bool set_cache_memory_policy(struct device_queue_manager *dqm, 3072 struct qcm_process_device *qpd, 3073 enum cache_policy default_policy, 3074 enum cache_policy alternate_policy, 3075 void __user *alternate_aperture_base, 3076 uint64_t alternate_aperture_size, 3077 u32 misc_process_properties) 3078 { 3079 bool retval = true; 3080 3081 if (!dqm->asic_ops.set_cache_memory_policy) 3082 return retval; 3083 3084 dqm_lock(dqm); 3085 3086 retval = dqm->asic_ops.set_cache_memory_policy( 3087 dqm, 3088 qpd, 3089 default_policy, 3090 alternate_policy, 3091 alternate_aperture_base, 3092 alternate_aperture_size, 3093 misc_process_properties); 3094 3095 if (retval) 3096 goto out; 3097 3098 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 3099 program_sh_mem_settings(dqm, qpd); 3100 3101 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 3102 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 3103 qpd->sh_mem_ape1_limit); 3104 3105 out: 3106 dqm_unlock(dqm); 3107 return retval; 3108 } 3109 3110 static int process_termination_nocpsch(struct device_queue_manager *dqm, 3111 struct qcm_process_device *qpd) 3112 { 3113 struct queue *q; 3114 struct device_process_node *cur, *next_dpn; 3115 int retval = 0; 3116 bool found = false; 3117 3118 dqm_lock(dqm); 3119 3120 /* Clear all user mode queues */ 3121 while (!list_empty(&qpd->queues_list)) { 3122 struct mqd_manager *mqd_mgr; 3123 int ret; 3124 3125 q = list_first_entry(&qpd->queues_list, struct queue, list); 3126 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 3127 q->properties.type)]; 3128 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); 3129 if (ret) 3130 retval = ret; 3131 dqm_unlock(dqm); 3132 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 3133 dqm_lock(dqm); 3134 } 3135 3136 /* Unregister process */ 3137 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 3138 if (qpd == cur->qpd) { 3139 list_del(&cur->list); 3140 kfree(cur); 3141 dqm->processes_count--; 3142 found = true; 3143 break; 3144 } 3145 } 3146 3147 dqm_unlock(dqm); 3148 3149 /* Outside the DQM lock because under the DQM lock we can't do 3150 * reclaim or take other locks that others hold while reclaiming. 3151 */ 3152 if (found) 3153 kfd_dec_compute_active(dqm->dev); 3154 3155 return retval; 3156 } 3157 3158 static int get_wave_state(struct device_queue_manager *dqm, 3159 struct queue *q, 3160 void __user *ctl_stack, 3161 u32 *ctl_stack_used_size, 3162 u32 *save_area_used_size) 3163 { 3164 struct mqd_manager *mqd_mgr; 3165 3166 dqm_lock(dqm); 3167 3168 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3169 3170 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE || 3171 q->properties.is_active || !q->device->kfd->cwsr_enabled || 3172 !mqd_mgr->get_wave_state) { 3173 dqm_unlock(dqm); 3174 return -EINVAL; 3175 } 3176 3177 dqm_unlock(dqm); 3178 3179 /* 3180 * get_wave_state is outside the dqm lock to prevent circular locking 3181 * and the queue should be protected against destruction by the process 3182 * lock. 3183 */ 3184 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties, 3185 ctl_stack, ctl_stack_used_size, save_area_used_size); 3186 } 3187 3188 static int get_queue_checkpoint_info(struct device_queue_manager *dqm, 3189 const struct queue *q, 3190 u32 *mqd_size, 3191 u32 *ctl_stack_size) 3192 { 3193 struct mqd_manager *mqd_mgr; 3194 enum KFD_MQD_TYPE mqd_type = 3195 get_mqd_type_from_queue_type(q->properties.type); 3196 int ret = 0; 3197 3198 dqm_lock(dqm); 3199 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 3200 *mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask); 3201 *ctl_stack_size = 0; 3202 3203 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) 3204 ret = mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size); 3205 3206 dqm_unlock(dqm); 3207 3208 return ret; 3209 } 3210 3211 static int checkpoint_mqd(struct device_queue_manager *dqm, 3212 const struct queue *q, 3213 void *mqd, 3214 void *ctl_stack) 3215 { 3216 struct mqd_manager *mqd_mgr; 3217 int r = 0; 3218 enum KFD_MQD_TYPE mqd_type = 3219 get_mqd_type_from_queue_type(q->properties.type); 3220 3221 dqm_lock(dqm); 3222 3223 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) { 3224 r = -EINVAL; 3225 goto dqm_unlock; 3226 } 3227 3228 mqd_mgr = dqm->mqd_mgrs[mqd_type]; 3229 if (!mqd_mgr->checkpoint_mqd) { 3230 r = -EOPNOTSUPP; 3231 goto dqm_unlock; 3232 } 3233 3234 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack); 3235 3236 dqm_unlock: 3237 dqm_unlock(dqm); 3238 return r; 3239 } 3240 3241 static int process_termination_cpsch(struct device_queue_manager *dqm, 3242 struct qcm_process_device *qpd) 3243 { 3244 int retval = 0; 3245 struct queue *q; 3246 struct device *dev = dqm->dev->adev->dev; 3247 struct kernel_queue *kq, *kq_next; 3248 struct mqd_manager *mqd_mgr; 3249 struct device_process_node *cur, *next_dpn; 3250 enum kfd_unmap_queues_filter filter = 3251 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES; 3252 bool found = false; 3253 3254 dqm_lock(dqm); 3255 3256 /* Clean all kernel queues */ 3257 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { 3258 list_del(&kq->list); 3259 decrement_queue_count(dqm, qpd, kq->queue); 3260 qpd->is_debug = false; 3261 dqm->total_queue_count--; 3262 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES; 3263 } 3264 3265 /* Clear all user mode queues */ 3266 list_for_each_entry(q, &qpd->queues_list, list) { 3267 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 3268 deallocate_sdma_queue(dqm, q); 3269 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) 3270 deallocate_sdma_queue(dqm, q); 3271 3272 if (q->properties.is_active) { 3273 decrement_queue_count(dqm, qpd, q); 3274 3275 if (dqm->dev->kfd->shared_resources.enable_mes) { 3276 retval = remove_queue_mes(dqm, q, qpd); 3277 if (retval) 3278 dev_err(dev, "Failed to remove queue %d\n", 3279 q->properties.queue_id); 3280 } 3281 } 3282 3283 dqm->total_queue_count--; 3284 } 3285 3286 /* Unregister process */ 3287 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { 3288 if (qpd == cur->qpd) { 3289 list_del(&cur->list); 3290 kfree(cur); 3291 dqm->processes_count--; 3292 found = true; 3293 break; 3294 } 3295 } 3296 3297 if (!dqm->dev->kfd->shared_resources.enable_mes) 3298 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD); 3299 3300 if ((retval || qpd->reset_wavefronts) && 3301 down_read_trylock(&dqm->dev->adev->reset_domain->sem)) { 3302 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); 3303 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); 3304 qpd->reset_wavefronts = false; 3305 up_read(&dqm->dev->adev->reset_domain->sem); 3306 } 3307 3308 /* Lastly, free mqd resources. 3309 * Do free_mqd() after dqm_unlock to avoid circular locking. 3310 */ 3311 while (!list_empty(&qpd->queues_list)) { 3312 q = list_first_entry(&qpd->queues_list, struct queue, list); 3313 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( 3314 q->properties.type)]; 3315 list_del(&q->list); 3316 qpd->queue_count--; 3317 dqm_unlock(dqm); 3318 /* Repin the MQD BO if still evicted for hibernation, before free. */ 3319 dqm_repin_mqd_bo(dqm, q); 3320 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); 3321 dqm_lock(dqm); 3322 } 3323 dqm_unlock(dqm); 3324 3325 /* Outside the DQM lock because under the DQM lock we can't do 3326 * reclaim or take other locks that others hold while reclaiming. 3327 */ 3328 if (found) 3329 kfd_dec_compute_active(dqm->dev); 3330 3331 return retval; 3332 } 3333 3334 static int init_mqd_managers(struct device_queue_manager *dqm) 3335 { 3336 int i, j; 3337 struct device *dev = dqm->dev->adev->dev; 3338 struct mqd_manager *mqd_mgr; 3339 3340 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) { 3341 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); 3342 if (!mqd_mgr) { 3343 dev_err(dev, "mqd manager [%d] initialization failed\n", i); 3344 goto out_free; 3345 } 3346 dqm->mqd_mgrs[i] = mqd_mgr; 3347 } 3348 3349 return 0; 3350 3351 out_free: 3352 for (j = 0; j < i; j++) { 3353 kfree(dqm->mqd_mgrs[j]); 3354 dqm->mqd_mgrs[j] = NULL; 3355 } 3356 3357 return -ENOMEM; 3358 } 3359 3360 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/ 3361 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) 3362 { 3363 int retval; 3364 struct kfd_node *dev = dqm->dev; 3365 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; 3366 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * 3367 get_num_all_sdma_engines(dqm) * 3368 dev->kfd->device_info.num_sdma_queues_per_engine + 3369 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * 3370 NUM_XCC(dqm->dev->xcc_mask)); 3371 3372 retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, size, 3373 AMDGPU_GEM_DOMAIN_GTT, 3374 &(mem_obj->mem), &(mem_obj->gpu_addr), 3375 (void *)&(mem_obj->cpu_ptr), false); 3376 3377 return retval; 3378 } 3379 3380 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, 3381 struct kfd_mem_obj *mqd) 3382 { 3383 WARN(!mqd, "No hiq sdma mqd trunk to free"); 3384 3385 amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem); 3386 } 3387 3388 static void mes_notify_unmap_work_handler(struct work_struct *work) 3389 { 3390 struct device_queue_manager *dqm = 3391 container_of(work, struct device_queue_manager, 3392 notify_unmap_work.work); 3393 3394 amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev); 3395 3396 /* Re-arm if still oversubscribed */ 3397 if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm)) 3398 queue_delayed_work(system_wq, &dqm->notify_unmap_work, 3399 usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US)); 3400 } 3401 3402 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev) 3403 { 3404 struct device_queue_manager *dqm; 3405 int i; 3406 3407 pr_debug("Loading device queue manager\n"); 3408 3409 dqm = kzalloc_obj(*dqm); 3410 if (!dqm) 3411 return NULL; 3412 3413 switch (dev->adev->asic_type) { 3414 /* HWS is not available on Hawaii. */ 3415 case CHIP_HAWAII: 3416 /* HWS depends on CWSR for timely dequeue. CWSR is not 3417 * available on Tonga. 3418 * 3419 * FIXME: This argument also applies to Kaveri. 3420 */ 3421 case CHIP_TONGA: 3422 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; 3423 break; 3424 default: 3425 dqm->sched_policy = sched_policy; 3426 break; 3427 } 3428 3429 dqm->dev = dev; 3430 switch (dqm->sched_policy) { 3431 case KFD_SCHED_POLICY_HWS: 3432 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 3433 /* initialize dqm for cp scheduling */ 3434 dqm->ops.create_queue = create_queue_cpsch; 3435 dqm->ops.initialize = initialize_cpsch; 3436 dqm->ops.start = start_cpsch; 3437 dqm->ops.stop = stop_cpsch; 3438 dqm->ops.halt = halt_cpsch; 3439 dqm->ops.unhalt = unhalt_cpsch; 3440 dqm->ops.destroy_queue = destroy_queue_cpsch; 3441 dqm->ops.update_queue = update_queue; 3442 dqm->ops.register_process = register_process; 3443 dqm->ops.unregister_process = unregister_process; 3444 dqm->ops.uninitialize = uninitialize; 3445 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 3446 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 3447 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 3448 dqm->ops.process_termination = process_termination_cpsch; 3449 dqm->ops.evict_process_queues = evict_process_queues_cpsch; 3450 dqm->ops.restore_process_queues = restore_process_queues_cpsch; 3451 dqm->ops.get_wave_state = get_wave_state; 3452 dqm->ops.reset_queues = reset_queues_cpsch; 3453 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 3454 dqm->ops.checkpoint_mqd = checkpoint_mqd; 3455 dqm->ops.set_perfcount = set_perfcount; 3456 dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch; 3457 break; 3458 case KFD_SCHED_POLICY_NO_HWS: 3459 /* initialize dqm for no cp scheduling */ 3460 dqm->ops.start = start_nocpsch; 3461 dqm->ops.stop = stop_nocpsch; 3462 dqm->ops.create_queue = create_queue_nocpsch; 3463 dqm->ops.destroy_queue = destroy_queue_nocpsch; 3464 dqm->ops.update_queue = update_queue; 3465 dqm->ops.register_process = register_process; 3466 dqm->ops.unregister_process = unregister_process; 3467 dqm->ops.initialize = initialize_nocpsch; 3468 dqm->ops.uninitialize = uninitialize; 3469 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 3470 dqm->ops.process_termination = process_termination_nocpsch; 3471 dqm->ops.evict_process_queues = evict_process_queues_nocpsch; 3472 dqm->ops.restore_process_queues = 3473 restore_process_queues_nocpsch; 3474 dqm->ops.get_wave_state = get_wave_state; 3475 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info; 3476 dqm->ops.checkpoint_mqd = checkpoint_mqd; 3477 dqm->ops.set_perfcount = set_perfcount; 3478 dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch; 3479 break; 3480 default: 3481 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy); 3482 goto out_free; 3483 } 3484 3485 switch (dev->adev->asic_type) { 3486 case CHIP_KAVERI: 3487 case CHIP_HAWAII: 3488 device_queue_manager_init_cik(&dqm->asic_ops); 3489 break; 3490 3491 case CHIP_CARRIZO: 3492 case CHIP_TONGA: 3493 case CHIP_FIJI: 3494 case CHIP_POLARIS10: 3495 case CHIP_POLARIS11: 3496 case CHIP_POLARIS12: 3497 case CHIP_VEGAM: 3498 device_queue_manager_init_vi(&dqm->asic_ops); 3499 break; 3500 3501 default: 3502 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0)) 3503 device_queue_manager_init_v12_1(&dqm->asic_ops); 3504 else if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0)) 3505 device_queue_manager_init_v12(&dqm->asic_ops); 3506 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0)) 3507 device_queue_manager_init_v11(&dqm->asic_ops); 3508 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1)) 3509 device_queue_manager_init_v10(&dqm->asic_ops); 3510 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) 3511 device_queue_manager_init_v9(&dqm->asic_ops); 3512 else { 3513 WARN(1, "Unexpected ASIC family %u", 3514 dev->adev->asic_type); 3515 goto out_free; 3516 } 3517 } 3518 3519 if (init_mqd_managers(dqm)) 3520 goto out_free; 3521 3522 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) { 3523 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n"); 3524 goto out_free; 3525 } 3526 3527 if (!dqm->ops.initialize(dqm)) { 3528 init_waitqueue_head(&dqm->destroy_wait); 3529 INIT_DELAYED_WORK(&dqm->notify_unmap_work, 3530 mes_notify_unmap_work_handler); 3531 return dqm; 3532 } 3533 3534 if (!dev->kfd->shared_resources.enable_mes) 3535 deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd); 3536 3537 out_free: 3538 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) 3539 kfree(dqm->mqd_mgrs[i]); 3540 3541 kfree(dqm); 3542 return NULL; 3543 } 3544 3545 void device_queue_manager_uninit(struct device_queue_manager *dqm) 3546 { 3547 cancel_delayed_work_sync(&dqm->notify_unmap_work); 3548 dqm->ops.stop(dqm); 3549 dqm->ops.uninitialize(dqm); 3550 if (!dqm->dev->kfd->shared_resources.enable_mes) 3551 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); 3552 kfree(dqm); 3553 } 3554 3555 /* bad queue notified by interrupt from CP */ 3556 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) 3557 { 3558 struct kfd_process_device *pdd = NULL; 3559 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); 3560 struct device_queue_manager *dqm = knode->dqm; 3561 struct qcm_process_device *qpd; 3562 struct queue *q = NULL; 3563 int ret = 0; 3564 3565 if (!pdd) 3566 return -EINVAL; 3567 3568 dqm_lock(dqm); 3569 3570 if (pdd) { 3571 qpd = &pdd->qpd; 3572 3573 list_for_each_entry(q, &qpd->queues_list, list) { 3574 if (q->doorbell_id == doorbell_id && q->properties.is_active) { 3575 reset_queues_mes(dqm, q); 3576 q->properties.is_evicted = true; 3577 q->properties.is_active = false; 3578 decrement_queue_count(dqm, qpd, q); 3579 break; 3580 } 3581 } 3582 } 3583 3584 dqm_unlock(dqm); 3585 kfd_unref_process(p); 3586 return ret; 3587 } 3588 3589 int kfd_evict_process_device(struct kfd_process_device *pdd) 3590 { 3591 struct device_queue_manager *dqm; 3592 struct kfd_process *p; 3593 3594 p = pdd->process; 3595 dqm = pdd->dev->dqm; 3596 3597 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 3598 3599 return dqm->ops.evict_process_queues(dqm, &pdd->qpd); 3600 } 3601 3602 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 3603 struct qcm_process_device *qpd) 3604 { 3605 int r; 3606 struct device *dev = dqm->dev->adev->dev; 3607 int updated_vmid_mask; 3608 3609 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3610 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3611 return -EINVAL; 3612 } 3613 3614 dqm_lock(dqm); 3615 3616 if (dqm->trap_debug_vmid != 0) { 3617 dev_err(dev, "Trap debug id already reserved\n"); 3618 r = -EBUSY; 3619 goto out_unlock; 3620 } 3621 3622 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3623 USE_DEFAULT_GRACE_PERIOD, false); 3624 if (r) 3625 goto out_unlock; 3626 3627 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3628 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd); 3629 3630 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3631 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd; 3632 r = set_sched_resources(dqm); 3633 if (r) 3634 goto out_unlock; 3635 3636 r = map_queues_cpsch(dqm); 3637 if (r) 3638 goto out_unlock; 3639 3640 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid); 3641 3642 out_unlock: 3643 dqm_unlock(dqm); 3644 return r; 3645 } 3646 3647 /* 3648 * Releases vmid for the trap debugger 3649 */ 3650 int release_debug_trap_vmid(struct device_queue_manager *dqm, 3651 struct qcm_process_device *qpd) 3652 { 3653 struct device *dev = dqm->dev->adev->dev; 3654 int r; 3655 int updated_vmid_mask; 3656 uint32_t trap_debug_vmid; 3657 3658 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 3659 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 3660 return -EINVAL; 3661 } 3662 3663 dqm_lock(dqm); 3664 trap_debug_vmid = dqm->trap_debug_vmid; 3665 if (dqm->trap_debug_vmid == 0) { 3666 dev_err(dev, "Trap debug id is not reserved\n"); 3667 r = -EINVAL; 3668 goto out_unlock; 3669 } 3670 3671 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 3672 USE_DEFAULT_GRACE_PERIOD, false); 3673 if (r) 3674 goto out_unlock; 3675 3676 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap; 3677 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd); 3678 3679 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask; 3680 dqm->trap_debug_vmid = 0; 3681 r = set_sched_resources(dqm); 3682 if (r) 3683 goto out_unlock; 3684 3685 r = map_queues_cpsch(dqm); 3686 if (r) 3687 goto out_unlock; 3688 3689 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid); 3690 3691 out_unlock: 3692 dqm_unlock(dqm); 3693 return r; 3694 } 3695 3696 #define QUEUE_NOT_FOUND -1 3697 /* invalidate queue operation in array */ 3698 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids) 3699 { 3700 int i; 3701 3702 for (i = 0; i < num_queues; i++) 3703 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK; 3704 } 3705 3706 /* find queue index in array */ 3707 static int q_array_get_index(unsigned int queue_id, 3708 uint32_t num_queues, 3709 uint32_t *queue_ids) 3710 { 3711 int i; 3712 3713 for (i = 0; i < num_queues; i++) 3714 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK)) 3715 return i; 3716 3717 return QUEUE_NOT_FOUND; 3718 } 3719 3720 struct copy_context_work_handler_workarea { 3721 struct work_struct copy_context_work; 3722 struct kfd_process *p; 3723 }; 3724 3725 static void copy_context_work_handler(struct work_struct *work) 3726 { 3727 struct copy_context_work_handler_workarea *workarea; 3728 struct mqd_manager *mqd_mgr; 3729 struct queue *q; 3730 struct mm_struct *mm; 3731 struct kfd_process *p; 3732 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size; 3733 int i; 3734 3735 workarea = container_of(work, 3736 struct copy_context_work_handler_workarea, 3737 copy_context_work); 3738 3739 p = workarea->p; 3740 mm = get_task_mm(p->lead_thread); 3741 3742 if (!mm) 3743 return; 3744 3745 kthread_use_mm(mm); 3746 for (i = 0; i < p->n_pdds; i++) { 3747 struct kfd_process_device *pdd = p->pdds[i]; 3748 struct device_queue_manager *dqm = pdd->dev->dqm; 3749 struct qcm_process_device *qpd = &pdd->qpd; 3750 3751 list_for_each_entry(q, &qpd->queues_list, list) { 3752 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) 3753 continue; 3754 3755 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP]; 3756 3757 /* We ignore the return value from get_wave_state 3758 * because 3759 * i) right now, it always returns 0, and 3760 * ii) if we hit an error, we would continue to the 3761 * next queue anyway. 3762 */ 3763 mqd_mgr->get_wave_state(mqd_mgr, 3764 q->mqd, 3765 &q->properties, 3766 (void __user *) q->properties.ctx_save_restore_area_address, 3767 &tmp_ctl_stack_used_size, 3768 &tmp_save_area_used_size); 3769 } 3770 } 3771 kthread_unuse_mm(mm); 3772 mmput(mm); 3773 } 3774 3775 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array) 3776 { 3777 if (!usr_queue_id_array) 3778 return num_queues ? ERR_PTR(-EINVAL) : NULL; 3779 3780 if (num_queues > KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) 3781 return ERR_PTR(-EINVAL); 3782 3783 return memdup_user(usr_queue_id_array, 3784 array_size(num_queues, sizeof(uint32_t))); 3785 } 3786 3787 int resume_queues(struct kfd_process *p, 3788 uint32_t num_queues, 3789 uint32_t *usr_queue_id_array) 3790 { 3791 uint32_t *queue_ids = NULL; 3792 int total_resumed = 0; 3793 int i; 3794 3795 if (usr_queue_id_array) { 3796 queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3797 3798 if (IS_ERR(queue_ids)) 3799 return PTR_ERR(queue_ids); 3800 3801 /* mask all queues as invalid. unmask per successful request */ 3802 q_array_invalidate(num_queues, queue_ids); 3803 } 3804 3805 for (i = 0; i < p->n_pdds; i++) { 3806 struct kfd_process_device *pdd = p->pdds[i]; 3807 struct device_queue_manager *dqm = pdd->dev->dqm; 3808 struct device *dev = dqm->dev->adev->dev; 3809 struct qcm_process_device *qpd = &pdd->qpd; 3810 struct queue *q; 3811 int r, per_device_resumed = 0; 3812 3813 dqm_lock(dqm); 3814 3815 /* unmask queues that resume or already resumed as valid */ 3816 list_for_each_entry(q, &qpd->queues_list, list) { 3817 int q_idx = QUEUE_NOT_FOUND; 3818 3819 if (queue_ids) 3820 q_idx = q_array_get_index( 3821 q->properties.queue_id, 3822 num_queues, 3823 queue_ids); 3824 3825 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) { 3826 int err = resume_single_queue(dqm, &pdd->qpd, q); 3827 3828 if (queue_ids) { 3829 if (!err) { 3830 queue_ids[q_idx] &= 3831 ~KFD_DBG_QUEUE_INVALID_MASK; 3832 } else { 3833 queue_ids[q_idx] |= 3834 KFD_DBG_QUEUE_ERROR_MASK; 3835 break; 3836 } 3837 } 3838 3839 if (dqm->dev->kfd->shared_resources.enable_mes) { 3840 wake_up_all(&dqm->destroy_wait); 3841 if (!err) 3842 total_resumed++; 3843 } else { 3844 per_device_resumed++; 3845 } 3846 } 3847 } 3848 3849 if (!per_device_resumed) { 3850 dqm_unlock(dqm); 3851 continue; 3852 } 3853 3854 r = execute_queues_cpsch(dqm, 3855 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 3856 0, 3857 USE_DEFAULT_GRACE_PERIOD); 3858 if (r) { 3859 dev_err(dev, "Failed to resume process queues\n"); 3860 if (queue_ids) { 3861 list_for_each_entry(q, &qpd->queues_list, list) { 3862 int q_idx = q_array_get_index( 3863 q->properties.queue_id, 3864 num_queues, 3865 queue_ids); 3866 3867 /* mask queue as error on resume fail */ 3868 if (q_idx != QUEUE_NOT_FOUND) 3869 queue_ids[q_idx] |= 3870 KFD_DBG_QUEUE_ERROR_MASK; 3871 } 3872 } 3873 } else { 3874 wake_up_all(&dqm->destroy_wait); 3875 total_resumed += per_device_resumed; 3876 } 3877 3878 dqm_unlock(dqm); 3879 } 3880 3881 if (queue_ids) { 3882 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 3883 num_queues * sizeof(uint32_t))) 3884 pr_err("copy_to_user failed on queue resume\n"); 3885 3886 kfree(queue_ids); 3887 } 3888 3889 return total_resumed; 3890 } 3891 3892 int suspend_queues(struct kfd_process *p, 3893 uint32_t num_queues, 3894 uint32_t grace_period, 3895 uint64_t exception_clear_mask, 3896 uint32_t *usr_queue_id_array) 3897 { 3898 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array); 3899 int total_suspended = 0; 3900 int i; 3901 3902 if (IS_ERR(queue_ids)) 3903 return PTR_ERR(queue_ids); 3904 3905 /* mask all queues as invalid. umask on successful request */ 3906 q_array_invalidate(num_queues, queue_ids); 3907 3908 for (i = 0; i < p->n_pdds; i++) { 3909 struct kfd_process_device *pdd = p->pdds[i]; 3910 struct device_queue_manager *dqm = pdd->dev->dqm; 3911 struct device *dev = dqm->dev->adev->dev; 3912 struct qcm_process_device *qpd = &pdd->qpd; 3913 struct queue *q; 3914 int r, per_device_suspended = 0; 3915 3916 mutex_lock(&p->event_mutex); 3917 dqm_lock(dqm); 3918 3919 /* unmask queues that suspend or already suspended */ 3920 list_for_each_entry(q, &qpd->queues_list, list) { 3921 int q_idx = q_array_get_index(q->properties.queue_id, 3922 num_queues, 3923 queue_ids); 3924 3925 if (q_idx != QUEUE_NOT_FOUND) { 3926 int err = suspend_single_queue(dqm, pdd, q); 3927 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes; 3928 3929 if (!err) { 3930 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK; 3931 if (exception_clear_mask && is_mes) 3932 q->properties.exception_status &= 3933 ~exception_clear_mask; 3934 3935 if (is_mes) 3936 total_suspended++; 3937 else 3938 per_device_suspended++; 3939 } else if (err != -EBUSY) { 3940 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3941 break; 3942 } 3943 } 3944 } 3945 3946 if (!per_device_suspended) { 3947 dqm_unlock(dqm); 3948 mutex_unlock(&p->event_mutex); 3949 if (total_suspended) { 3950 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); 3951 /* Heavy-weight TLB flush after MES suspends queues */ 3952 kfd_flush_tlb(pdd); 3953 } 3954 continue; 3955 } 3956 3957 r = execute_queues_cpsch(dqm, 3958 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, 3959 grace_period); 3960 3961 if (r) 3962 dev_err(dev, "Failed to suspend process queues.\n"); 3963 else 3964 total_suspended += per_device_suspended; 3965 3966 list_for_each_entry(q, &qpd->queues_list, list) { 3967 int q_idx = q_array_get_index(q->properties.queue_id, 3968 num_queues, queue_ids); 3969 3970 if (q_idx == QUEUE_NOT_FOUND) 3971 continue; 3972 3973 /* mask queue as error on suspend fail */ 3974 if (r) 3975 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK; 3976 else if (exception_clear_mask) 3977 q->properties.exception_status &= 3978 ~exception_clear_mask; 3979 } 3980 3981 dqm_unlock(dqm); 3982 mutex_unlock(&p->event_mutex); 3983 amdgpu_device_flush_hdp(dqm->dev->adev, NULL); 3984 } 3985 3986 if (total_suspended) { 3987 struct copy_context_work_handler_workarea copy_context_worker; 3988 3989 INIT_WORK_ONSTACK( 3990 ©_context_worker.copy_context_work, 3991 copy_context_work_handler); 3992 3993 copy_context_worker.p = p; 3994 3995 schedule_work(©_context_worker.copy_context_work); 3996 3997 3998 flush_work(©_context_worker.copy_context_work); 3999 destroy_work_on_stack(©_context_worker.copy_context_work); 4000 } 4001 4002 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids, 4003 num_queues * sizeof(uint32_t))) 4004 pr_err("copy_to_user failed on queue suspend\n"); 4005 4006 kfree(queue_ids); 4007 4008 return total_suspended; 4009 } 4010 4011 static uint32_t set_queue_type_for_user(struct queue_properties *q_props) 4012 { 4013 switch (q_props->type) { 4014 case KFD_QUEUE_TYPE_COMPUTE: 4015 return q_props->format == KFD_QUEUE_FORMAT_PM4 4016 ? KFD_IOC_QUEUE_TYPE_COMPUTE 4017 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; 4018 case KFD_QUEUE_TYPE_SDMA: 4019 return KFD_IOC_QUEUE_TYPE_SDMA; 4020 case KFD_QUEUE_TYPE_SDMA_XGMI: 4021 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI; 4022 default: 4023 WARN_ONCE(true, "queue type not recognized!"); 4024 return 0xffffffff; 4025 }; 4026 } 4027 4028 void set_queue_snapshot_entry(struct queue *q, 4029 uint64_t exception_clear_mask, 4030 struct kfd_queue_snapshot_entry *qss_entry) 4031 { 4032 qss_entry->ring_base_address = q->properties.queue_address; 4033 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr; 4034 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr; 4035 qss_entry->ctx_save_restore_address = 4036 q->properties.ctx_save_restore_area_address; 4037 qss_entry->ctx_save_restore_area_size = 4038 q->properties.ctx_save_restore_area_size; 4039 qss_entry->exception_status = q->properties.exception_status; 4040 qss_entry->queue_id = q->properties.queue_id; 4041 qss_entry->gpu_id = q->device->id; 4042 qss_entry->ring_size = (uint32_t)q->properties.queue_size; 4043 qss_entry->queue_type = set_queue_type_for_user(&q->properties); 4044 q->properties.exception_status &= ~exception_clear_mask; 4045 } 4046 4047 int debug_lock_and_unmap(struct device_queue_manager *dqm) 4048 { 4049 struct device *dev = dqm->dev->adev->dev; 4050 int r; 4051 4052 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 4053 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 4054 return -EINVAL; 4055 } 4056 4057 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 4058 return 0; 4059 4060 dqm_lock(dqm); 4061 4062 r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false); 4063 if (r) 4064 dqm_unlock(dqm); 4065 4066 return r; 4067 } 4068 4069 int debug_map_and_unlock(struct device_queue_manager *dqm) 4070 { 4071 struct device *dev = dqm->dev->adev->dev; 4072 int r; 4073 4074 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 4075 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy); 4076 return -EINVAL; 4077 } 4078 4079 if (!kfd_dbg_is_per_vmid_supported(dqm->dev)) 4080 return 0; 4081 4082 r = map_queues_cpsch(dqm); 4083 4084 dqm_unlock(dqm); 4085 4086 return r; 4087 } 4088 4089 int debug_refresh_runlist(struct device_queue_manager *dqm) 4090 { 4091 int r = debug_lock_and_unmap(dqm); 4092 4093 if (r) 4094 return r; 4095 4096 return debug_map_and_unlock(dqm); 4097 } 4098 4099 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, 4100 struct qcm_process_device *qpd, 4101 int doorbell_off, u32 *queue_format) 4102 { 4103 struct queue *q; 4104 bool r = false; 4105 4106 if (!queue_format) 4107 return r; 4108 4109 dqm_lock(dqm); 4110 4111 list_for_each_entry(q, &qpd->queues_list, list) { 4112 if (q->properties.doorbell_off == doorbell_off) { 4113 *queue_format = q->properties.format; 4114 r = true; 4115 goto out; 4116 } 4117 } 4118 4119 out: 4120 dqm_unlock(dqm); 4121 return r; 4122 } 4123 4124 size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, enum kfd_queue_type type) 4125 { 4126 return dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)]->mqd_size; 4127 } 4128 4129 #if defined(CONFIG_DEBUG_FS) 4130 4131 static void seq_reg_dump(struct seq_file *m, 4132 uint32_t (*dump)[2], uint32_t n_regs) 4133 { 4134 uint32_t i, count; 4135 4136 for (i = 0, count = 0; i < n_regs; i++) { 4137 if (count == 0 || 4138 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) { 4139 seq_printf(m, "%s %08x: %08x", 4140 i ? "\n" : "", 4141 dump[i][0], dump[i][1]); 4142 count = 7; 4143 } else { 4144 seq_printf(m, " %08x", dump[i][1]); 4145 count--; 4146 } 4147 } 4148 4149 seq_puts(m, "\n"); 4150 } 4151 4152 int dqm_debugfs_hqds(struct seq_file *m, void *data) 4153 { 4154 struct device_queue_manager *dqm = data; 4155 uint32_t xcc_mask = dqm->dev->xcc_mask; 4156 uint32_t (*dump)[2], n_regs; 4157 int pipe, queue; 4158 int r = 0, xcc_id; 4159 uint32_t sdma_engine_start; 4160 4161 if (!dqm->sched_running) { 4162 seq_puts(m, " Device is stopped\n"); 4163 return 0; 4164 } 4165 4166 for_each_inst(xcc_id, xcc_mask) { 4167 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 4168 KFD_CIK_HIQ_PIPE, 4169 KFD_CIK_HIQ_QUEUE, &dump, 4170 &n_regs, xcc_id); 4171 if (!r) { 4172 seq_printf( 4173 m, 4174 " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n", 4175 xcc_id, 4176 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1, 4177 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm), 4178 KFD_CIK_HIQ_QUEUE); 4179 seq_reg_dump(m, dump, n_regs); 4180 4181 kfree(dump); 4182 } 4183 4184 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { 4185 int pipe_offset = pipe * get_queues_per_pipe(dqm); 4186 4187 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { 4188 if (!test_bit(pipe_offset + queue, 4189 dqm->dev->kfd->shared_resources.cp_queue_bitmap)) 4190 continue; 4191 4192 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev, 4193 pipe, queue, 4194 &dump, &n_regs, 4195 xcc_id); 4196 if (r) 4197 break; 4198 4199 seq_printf(m, 4200 " Inst %d, CP Pipe %d, Queue %d\n", 4201 xcc_id, pipe, queue); 4202 seq_reg_dump(m, dump, n_regs); 4203 4204 kfree(dump); 4205 } 4206 } 4207 } 4208 4209 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm); 4210 for (pipe = sdma_engine_start; 4211 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm)); 4212 pipe++) { 4213 for (queue = 0; 4214 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine; 4215 queue++) { 4216 r = dqm->dev->kfd2kgd->hqd_sdma_dump( 4217 dqm->dev->adev, pipe, queue, &dump, &n_regs); 4218 if (r == -EINVAL) { 4219 /* 4220 * SDMA engine not present/mapped on this 4221 * device; skip it instead of aborting the 4222 * whole dump. 4223 */ 4224 r = 0; 4225 break; 4226 } 4227 if (r) 4228 break; 4229 4230 seq_printf(m, " SDMA Engine %d, RLC %d\n", 4231 pipe, queue); 4232 seq_reg_dump(m, dump, n_regs); 4233 4234 kfree(dump); 4235 } 4236 } 4237 4238 return r; 4239 } 4240 4241 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm) 4242 { 4243 int r = 0; 4244 4245 dqm_lock(dqm); 4246 r = pm_debugfs_hang_hws(&dqm->packet_mgr); 4247 if (r) { 4248 dqm_unlock(dqm); 4249 return r; 4250 } 4251 dqm->active_runlist = true; 4252 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 4253 0, USE_DEFAULT_GRACE_PERIOD); 4254 dqm_unlock(dqm); 4255 4256 return r; 4257 } 4258 4259 #endif 4260