1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2025 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/printk.h> 26 #include <linux/slab.h> 27 #include <linux/uaccess.h> 28 #include "kfd_priv.h" 29 #include "kfd_mqd_manager.h" 30 #include "v12_structs.h" 31 #include "gc/gc_12_1_0_sh_mask.h" 32 #include "amdgpu_amdkfd.h" 33 #include "kfd_device_queue_manager.h" 34 35 static void update_mqd(struct mqd_manager *mm, void *mqd, 36 struct queue_properties *q, 37 struct mqd_update_info *minfo); 38 39 static inline struct v12_1_compute_mqd *get_mqd(void *mqd) 40 { 41 return (struct v12_1_compute_mqd *)mqd; 42 } 43 44 static inline struct v12_sdma_mqd *get_sdma_mqd(void *mqd) 45 { 46 return (struct v12_sdma_mqd *)mqd; 47 } 48 49 static void mqd_symmetrically_map_cu_mask_v12_1(struct mqd_manager *mm, 50 const uint32_t *cu_mask, uint32_t cu_mask_count, 51 uint32_t *se_mask, uint32_t inst) 52 { 53 struct amdgpu_cu_info *cu_info = &mm->dev->adev->gfx.cu_info; 54 struct amdgpu_gfx_config *gfx_info = &mm->dev->adev->gfx.config; 55 uint32_t cu_per_sh[2][2] = {0}; 56 uint32_t en_mask = 0x3; 57 int i, se, sh, cu, cu_inc = 0; 58 uint32_t cu_active_per_node; 59 int inc = NUM_XCC(mm->dev->xcc_mask); 60 int xcc_inst = inst + ffs(mm->dev->xcc_mask) - 1; 61 62 cu_active_per_node = cu_info->number / mm->dev->kfd->num_nodes; 63 if (cu_mask_count > cu_active_per_node) 64 cu_mask_count = cu_active_per_node; 65 66 /* 67 * Count active CUs per SE/SH. 68 */ 69 for (se = 0; se < gfx_info->max_shader_engines; se++) 70 for (sh = 0; sh < gfx_info->max_sh_per_se; sh++) 71 cu_per_sh[se][sh] = hweight32( 72 cu_info->bitmap[xcc_inst][se][sh]); 73 74 /* Symmetrically map cu_mask to all SEs & SHs: 75 * For GFX 12.1.0, the following code only looks at a 76 * subset of the cu_mask corresponding to the inst parameter. 77 * If we have n XCCs under one GPU node 78 * cu_mask[0] bit0 -> XCC0 se_mask[0] bit0 (XCC0,SE0,SH0,CU0) 79 * cu_mask[0] bit1 -> XCC1 se_mask[0] bit0 (XCC1,SE0,SH0,CU0) 80 * .. 81 * cu_mask[0] bitn -> XCCn se_mask[0] bit0 (XCCn,SE0,SH0,CU0) 82 * cu_mask[0] bit n+1 -> XCC0 se_mask[1] bit0 (XCC0,SE1,SH0,CU0) 83 * 84 * For example, if there are 6 XCCs under 1 KFD node, this code 85 * running for each inst, will look at the bits as: 86 * inst, inst + 6, inst + 12... 87 * 88 * First ensure all CUs are disabled, then enable user specified CUs. 89 */ 90 for (i = 0; i < gfx_info->max_shader_engines; i++) 91 se_mask[i] = 0; 92 93 i = inst; 94 for (cu = 0; cu < 16; cu++) { 95 for (sh = 0; sh < gfx_info->max_sh_per_se; sh++) { 96 for (se = 0; se < gfx_info->max_shader_engines; se++) { 97 if (cu_per_sh[se][sh] > cu) { 98 if (cu_mask[i / 32] & (1U << (i % 32))) { 99 if (cu == 8 && sh == 0) 100 se_mask[se] |= en_mask << 30; 101 else 102 se_mask[se] |= en_mask << (cu_inc + sh * 16); 103 } 104 i += inc; 105 if (i >= cu_mask_count) 106 return; 107 } 108 } 109 } 110 cu_inc += 2; 111 } 112 } 113 114 static void update_cu_mask(struct mqd_manager *mm, void *mqd, 115 struct mqd_update_info *minfo, uint32_t inst) 116 { 117 struct v12_1_compute_mqd *m; 118 uint32_t se_mask[2] = {0}; 119 120 if (!minfo || !minfo->cu_mask.ptr) 121 return; 122 123 mqd_symmetrically_map_cu_mask_v12_1(mm, 124 minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, inst); 125 126 m = get_mqd(mqd); 127 m->compute_static_thread_mgmt_se0 = se_mask[0]; 128 m->compute_static_thread_mgmt_se1 = se_mask[1]; 129 130 pr_debug("update cu mask to %#x %#x\n", 131 m->compute_static_thread_mgmt_se0, 132 m->compute_static_thread_mgmt_se1); 133 } 134 135 static void set_priority(struct v12_1_compute_mqd *m, struct queue_properties *q) 136 { 137 m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; 138 } 139 140 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, 141 struct queue_properties *q) 142 { 143 u32 mqd_size = AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size); 144 struct kfd_node *node = mm->dev; 145 struct kfd_mem_obj *mqd_mem_obj; 146 147 if (q->type == KFD_QUEUE_TYPE_COMPUTE) 148 mqd_size *= NUM_XCC(node->xcc_mask); 149 150 if (kfd_gtt_sa_allocate(node, mqd_size, &mqd_mem_obj)) 151 return NULL; 152 153 return mqd_mem_obj; 154 } 155 156 static void init_mqd(struct mqd_manager *mm, void **mqd, 157 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 158 struct queue_properties *q) 159 { 160 uint64_t addr; 161 struct v12_1_compute_mqd *m; 162 u32 mqd_size = AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size); 163 164 m = (struct v12_1_compute_mqd *) mqd_mem_obj->cpu_ptr; 165 addr = mqd_mem_obj->gpu_addr; 166 167 memset(m, 0, mqd_size); 168 169 m->header = 0xC0310800; 170 m->compute_pipelinestat_enable = 1; 171 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 172 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 173 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 174 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 175 m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF; 176 m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF; 177 m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF; 178 m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF; 179 m->compute_static_thread_mgmt_se8 = 0xFFFFFFFF; 180 181 m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | 182 0x63 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; 183 184 m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT; 185 186 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 187 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 188 189 m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT | 190 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT | 191 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT; 192 193 /* Set cp_hqd_hq_status0.c_queue_debug_en to 1 to have the CP set up the 194 * DISPATCH_PTR. This is required for the kfd debugger 195 */ 196 m->cp_hqd_hq_status0 = 1 << 14; 197 198 if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev)) 199 m->cp_hqd_hq_status0 |= 1 << 29; 200 201 if (q->format == KFD_QUEUE_FORMAT_AQL) { 202 m->cp_hqd_aql_control = 203 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT; 204 } 205 206 if (mm->dev->kfd->cwsr_enabled) { 207 m->cp_hqd_persistent_state |= 208 (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT); 209 m->cp_hqd_ctx_save_base_addr_lo = 210 lower_32_bits(q->ctx_save_restore_area_address); 211 m->cp_hqd_ctx_save_base_addr_hi = 212 upper_32_bits(q->ctx_save_restore_area_address); 213 m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size; 214 m->cp_hqd_cntl_stack_size = q->ctl_stack_size; 215 m->cp_hqd_cntl_stack_offset = q->ctl_stack_size; 216 m->cp_hqd_wg_state_offset = q->ctl_stack_size; 217 } 218 219 *mqd = m; 220 if (gart_addr) 221 *gart_addr = addr; 222 update_mqd(mm, m, q, NULL); 223 } 224 225 static int load_mqd(struct mqd_manager *mm, void *mqd, 226 uint32_t pipe_id, uint32_t queue_id, 227 struct queue_properties *p, struct mm_struct *mms) 228 { 229 int r = 0; 230 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ 231 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); 232 233 r = mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id, 234 (uint32_t __user *)p->write_ptr, 235 wptr_shift, 0, mms, 0); 236 return r; 237 } 238 239 static void update_mqd(struct mqd_manager *mm, void *mqd, 240 struct queue_properties *q, 241 struct mqd_update_info *minfo) 242 { 243 struct v12_1_compute_mqd *m; 244 245 m = get_mqd(mqd); 246 247 m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT; 248 m->cp_hqd_pq_control |= 249 ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1; 250 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK; 251 pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control); 252 253 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 254 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 255 256 if (q->metadata_queue_size) { 257 /* On GC 12.1 is 64 DWs which is 4 times size of AQL packet */ 258 if (q->metadata_queue_size == q->queue_size * 4) { 259 /* 260 * User application allocates main queue ring and metadata queue ring 261 * with a single allocation. metadata queue ring starts after main 262 * queue ring. 263 */ 264 m->cp_hqd_kd_base = 265 lower_32_bits((q->queue_address + q->queue_size) >> 8); 266 m->cp_hqd_kd_base_hi = 267 upper_32_bits((q->queue_address + q->queue_size) >> 8); 268 269 m->cp_hqd_kd_cntl |= CP_HQD_KD_CNTL__KD_FETCHER_ENABLE_MASK; 270 /* KD_SIZE = 2 for metadata packet = 64 DWs */ 271 m->cp_hqd_kd_cntl |= 2 << CP_HQD_KD_CNTL__KD_SIZE__SHIFT; 272 } else { 273 pr_warn("Invalid metadata ring size, metadata queue will be ignored\n"); 274 } 275 } 276 277 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 278 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 279 m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); 280 m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); 281 282 m->cp_hqd_pq_doorbell_control = 283 q->doorbell_off << 284 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; 285 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", 286 m->cp_hqd_pq_doorbell_control); 287 288 m->cp_hqd_ib_control = 1 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT; 289 290 /* 291 * HW does not clamp this field correctly. Maximum EOP queue size 292 * is constrained by per-SE EOP done signal count, which is 8-bit. 293 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit 294 * more than (EOP entry count - 1) so a queue size of 0x800 dwords 295 * is safe, giving a maximum field value of 0xA. 296 */ 297 m->cp_hqd_eop_control = q->eop_ring_buffer_size ? min(0xA, 298 ffs(q->eop_ring_buffer_size / sizeof(unsigned int) / 4)) : 0; 299 m->cp_hqd_eop_base_addr_lo = 300 lower_32_bits(q->eop_ring_buffer_address >> 8); 301 m->cp_hqd_eop_base_addr_hi = 302 upper_32_bits(q->eop_ring_buffer_address >> 8); 303 304 m->cp_hqd_iq_timer = 0; 305 306 m->cp_hqd_vmid = q->vmid; 307 308 if (q->format == KFD_QUEUE_FORMAT_AQL) { 309 /* GC 10 removed WPP_CLAMP from PQ Control */ 310 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK | 311 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT | 312 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT; 313 m->cp_hqd_pq_doorbell_control |= 314 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT; 315 } 316 if (mm->dev->kfd->cwsr_enabled) 317 m->cp_hqd_ctx_save_control = 0; 318 319 set_priority(m, q); 320 321 q->is_active = QUEUE_IS_ACTIVE(*q); 322 } 323 324 static bool check_preemption_failed(struct mqd_manager *mm, void *mqd) 325 { 326 return false; 327 } 328 329 static int get_wave_state(struct mqd_manager *mm, void *mqd, 330 struct queue_properties *q, 331 void __user *ctl_stack, 332 u32 *ctl_stack_used_size, 333 u32 *save_area_used_size) 334 { 335 struct v12_1_compute_mqd *m; 336 struct mqd_user_context_save_area_header header; 337 338 m = get_mqd(mqd); 339 340 /* Control stack is written backwards, while workgroup context data 341 * is written forwards. Both starts from m->cp_hqd_cntl_stack_size. 342 * Current position is at m->cp_hqd_cntl_stack_offset and 343 * m->cp_hqd_wg_state_offset, respectively. 344 */ 345 *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - 346 m->cp_hqd_cntl_stack_offset; 347 *save_area_used_size = m->cp_hqd_wg_state_offset - 348 m->cp_hqd_cntl_stack_size; 349 350 /* Control stack is not copied to user mode for GFXv12 because 351 * it's part of the context save area that is already 352 * accessible to user mode 353 */ 354 header.control_stack_size = *ctl_stack_used_size; 355 header.wave_state_size = *save_area_used_size; 356 357 header.wave_state_offset = m->cp_hqd_wg_state_offset; 358 header.control_stack_offset = m->cp_hqd_cntl_stack_offset; 359 360 if (copy_to_user(ctl_stack, &header, sizeof(header))) 361 return -EFAULT; 362 363 return 0; 364 } 365 366 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, 367 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 368 struct queue_properties *q) 369 { 370 struct v12_1_compute_mqd *m; 371 372 init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); 373 374 m = get_mqd(*mqd); 375 376 m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT | 377 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT; 378 } 379 380 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, 381 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 382 struct queue_properties *q) 383 { 384 struct v12_sdma_mqd *m; 385 386 m = (struct v12_sdma_mqd *) mqd_mem_obj->cpu_ptr; 387 388 memset(m, 0, PAGE_SIZE); 389 390 *mqd = m; 391 if (gart_addr) 392 *gart_addr = mqd_mem_obj->gpu_addr; 393 394 mm->update_mqd(mm, m, q, NULL); 395 } 396 397 #define SDMA_RLC_DUMMY_DEFAULT 0xf 398 399 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, 400 struct queue_properties *q, 401 struct mqd_update_info *minfo) 402 { 403 struct v12_sdma_mqd *m; 404 405 m = get_sdma_mqd(mqd); 406 m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1) 407 << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT | 408 q->vmid << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_VMID__SHIFT | 409 1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | 410 6 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | 411 1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT; 412 413 m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8); 414 m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); 415 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 416 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 417 m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); 418 m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); 419 m->sdmax_rlcx_doorbell_offset = 420 q->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT; 421 422 m->sdmax_rlcx_sched_cntl = (amdgpu_sdma_phase_quantum 423 << SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT) 424 & SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK; 425 426 m->sdma_engine_id = q->sdma_engine_id; 427 m->sdma_queue_id = q->sdma_queue_id; 428 429 m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT; 430 431 /* Allow context switch so we don't cross-process starve with a massive 432 * command buffer of long-running SDMA commands 433 * sdmax_rlcx_ib_cntl represent SDMA_QUEUE0_IB_CNTL register 434 */ 435 m->sdmax_rlcx_ib_cntl |= SDMA0_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK; 436 437 q->is_active = QUEUE_IS_ACTIVE(*q); 438 } 439 440 static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj, 441 struct kfd_mem_obj *xcc_mqd_mem_obj, 442 uint64_t offset) 443 { 444 xcc_mqd_mem_obj->mem = (offset == 0) ? 445 mqd_mem_obj->mem : NULL; 446 xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset; 447 xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr 448 + offset); 449 } 450 451 static void init_mqd_v12_1(struct mqd_manager *mm, void **mqd, 452 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 453 struct queue_properties *q) 454 { 455 struct v12_1_compute_mqd *m; 456 int xcc = 0; 457 struct kfd_mem_obj xcc_mqd_mem_obj; 458 uint64_t xcc_gart_addr = 0; 459 uint64_t xcc_ctx_save_restore_area_address; 460 uint64_t offset = mm->mqd_stride(mm, q); 461 uint32_t local_xcc_start = mm->dev->dqm->current_logical_xcc_start++; 462 463 memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj)); 464 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { 465 get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset*xcc); 466 467 init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q); 468 469 m->cp_mqd_stride_size = offset; 470 471 /* 472 * Update the CWSR address for each XCC if CWSR is enabled 473 * and CWSR area is allocated in thunk 474 */ 475 if (mm->dev->kfd->cwsr_enabled && 476 q->ctx_save_restore_area_address) { 477 xcc_ctx_save_restore_area_address = 478 q->ctx_save_restore_area_address + 479 (xcc * q->ctx_save_restore_area_size); 480 481 m->cp_hqd_ctx_save_base_addr_lo = 482 lower_32_bits(xcc_ctx_save_restore_area_address); 483 m->cp_hqd_ctx_save_base_addr_hi = 484 upper_32_bits(xcc_ctx_save_restore_area_address); 485 } 486 487 if (q->format == KFD_QUEUE_FORMAT_AQL) { 488 m->compute_tg_chunk_size = 1; 489 m->compute_current_logical_xcc_id = 490 (local_xcc_start + xcc) % 491 NUM_XCC(mm->dev->xcc_mask); 492 } else { 493 /* PM4 Queue */ 494 m->compute_current_logical_xcc_id = 0; 495 m->compute_tg_chunk_size = 0; 496 m->pm4_target_xcc_in_xcp = q->pm4_target_xcc; 497 } 498 499 if (xcc == 0) { 500 /* Set the MQD pointer and gart address to XCC0 MQD */ 501 *mqd = m; 502 *gart_addr = xcc_gart_addr; 503 } 504 } 505 } 506 507 static void update_mqd_v12_1(struct mqd_manager *mm, void *mqd, 508 struct queue_properties *q, struct mqd_update_info *minfo) 509 { 510 struct v12_1_compute_mqd *m; 511 int xcc = 0; 512 uint64_t size = mm->mqd_stride(mm, q); 513 514 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { 515 m = get_mqd(mqd + size * xcc); 516 update_mqd(mm, m, q, minfo); 517 518 update_cu_mask(mm, m, minfo, xcc); 519 520 if (q->format == KFD_QUEUE_FORMAT_AQL) { 521 m->compute_tg_chunk_size = 1; 522 } else { 523 /* PM4 Queue */ 524 m->compute_current_logical_xcc_id = 0; 525 m->compute_tg_chunk_size = 0; 526 m->pm4_target_xcc_in_xcp = q->pm4_target_xcc; 527 } 528 } 529 } 530 531 static int destroy_mqd_v12_1(struct mqd_manager *mm, void *mqd, 532 enum kfd_preempt_type type, unsigned int timeout, 533 uint32_t pipe_id, uint32_t queue_id) 534 { 535 uint32_t xcc_mask = mm->dev->xcc_mask; 536 int xcc_id, err, inst = 0; 537 void *xcc_mqd; 538 struct v12_1_compute_mqd *m; 539 uint64_t mqd_offset; 540 541 m = get_mqd(mqd); 542 mqd_offset = m->cp_mqd_stride_size; 543 544 for_each_inst(xcc_id, xcc_mask) { 545 xcc_mqd = mqd + mqd_offset * inst; 546 err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd, 547 type, timeout, pipe_id, 548 queue_id, xcc_id); 549 if (err) { 550 pr_debug("Destroy MQD failed for xcc: %d\n", inst); 551 break; 552 } 553 ++inst; 554 } 555 556 return err; 557 } 558 559 static int load_mqd_v12_1(struct mqd_manager *mm, void *mqd, 560 uint32_t pipe_id, uint32_t queue_id, 561 struct queue_properties *p, struct mm_struct *mms) 562 { 563 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ 564 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); 565 uint32_t xcc_mask = mm->dev->xcc_mask; 566 int xcc_id, err, inst = 0; 567 void *xcc_mqd; 568 uint64_t mqd_stride_size = mm->mqd_stride(mm, p); 569 570 for_each_inst(xcc_id, xcc_mask) { 571 xcc_mqd = mqd + mqd_stride_size * inst; 572 err = mm->dev->kfd2kgd->hqd_load( 573 mm->dev->adev, xcc_mqd, pipe_id, queue_id, 574 (uint32_t __user *)p->write_ptr, wptr_shift, 0, mms, 575 xcc_id); 576 if (err) { 577 pr_debug("Load MQD failed for xcc: %d\n", inst); 578 break; 579 } 580 ++inst; 581 } 582 583 return err; 584 } 585 586 static int get_wave_state_v12_1(struct mqd_manager *mm, void *mqd, 587 struct queue_properties *q, 588 void __user *ctl_stack, 589 u32 *ctl_stack_used_size, 590 u32 *save_area_used_size) 591 { 592 int xcc, err = 0; 593 void *xcc_mqd; 594 void __user *xcc_ctl_stack; 595 uint64_t mqd_stride_size = mm->mqd_stride(mm, q); 596 u32 tmp_ctl_stack_used_size = 0, tmp_save_area_used_size = 0; 597 598 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { 599 xcc_mqd = mqd + mqd_stride_size * xcc; 600 xcc_ctl_stack = (void __user *)((uintptr_t)ctl_stack + 601 q->ctx_save_restore_area_size * xcc); 602 603 err = get_wave_state(mm, xcc_mqd, q, xcc_ctl_stack, 604 &tmp_ctl_stack_used_size, 605 &tmp_save_area_used_size); 606 if (err) 607 break; 608 609 /* 610 * Set the ctl_stack_used_size and save_area_used_size to 611 * ctl_stack_used_size and save_area_used_size of XCC 0 when 612 * passing the info to user-space. 613 * For multi XCC, user-space would have to look at the header 614 * info of each Control stack area to determine the control 615 * stack size and save area used. 616 */ 617 if (xcc == 0) { 618 *ctl_stack_used_size = tmp_ctl_stack_used_size; 619 *save_area_used_size = tmp_save_area_used_size; 620 } 621 } 622 623 return err; 624 } 625 626 #if defined(CONFIG_DEBUG_FS) 627 628 static int debugfs_show_mqd(struct seq_file *m, void *data) 629 { 630 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, 631 data, sizeof(struct v12_1_compute_mqd), false); 632 return 0; 633 } 634 635 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) 636 { 637 seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, 638 data, sizeof(struct v12_sdma_mqd), false); 639 return 0; 640 } 641 642 #endif 643 644 static void restore_mqd_v12_1(struct mqd_manager *mm, void **mqd, 645 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 646 struct queue_properties *qp, const void *mqd_src, 647 const void *ctl_stack_src, const u32 ctl_stack_size) 648 { 649 u64 addr; 650 struct v12_1_compute_mqd *m; 651 652 /* 653 * GFX12.1 is multi-XCC capable but this restore handles XCC0 only. 654 * Multi-XCC CRIU restore is currently unreachable because 655 * kfd_criu_restore_queue() validates against unscaled mqd_size. 656 */ 657 if (NUM_XCC(mm->dev->xcc_mask) > 1) 658 pr_warn_once("GFX12.1 multi-XCC CRIU restore not fully supported\n"); 659 660 m = (struct v12_1_compute_mqd *)mqd_mem_obj->cpu_ptr; 661 addr = mqd_mem_obj->gpu_addr; 662 663 memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size) * 664 NUM_XCC(mm->dev->xcc_mask)); 665 memcpy(m, mqd_src, sizeof(*m)); 666 667 /* Update MQD base address to the newly allocated location */ 668 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 669 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 670 671 m->cp_hqd_pq_doorbell_control &= 672 ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK; 673 m->cp_hqd_pq_doorbell_control |= 674 qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; 675 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control); 676 677 *mqd = m; 678 if (gart_addr) 679 *gart_addr = addr; 680 681 qp->is_active = 0; 682 } 683 684 static void restore_mqd_sdma_v12_1(struct mqd_manager *mm, void **mqd, 685 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, 686 struct queue_properties *qp, 687 const void *mqd_src, 688 const void *ctl_stack_src, 689 const u32 ctl_stack_size) 690 { 691 u64 addr; 692 struct v12_sdma_mqd *m; 693 694 m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr; 695 addr = mqd_mem_obj->gpu_addr; 696 697 memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size)); 698 memcpy(m, mqd_src, sizeof(*m)); 699 700 m->sdmax_rlcx_doorbell_offset = 701 qp->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT; 702 703 *mqd = m; 704 if (gart_addr) 705 *gart_addr = addr; 706 707 qp->is_active = 0; 708 } 709 710 struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type, 711 struct kfd_node *dev) 712 { 713 struct mqd_manager *mqd; 714 715 if (WARN_ON(type >= KFD_MQD_TYPE_MAX)) 716 return NULL; 717 718 mqd = kzalloc_obj(*mqd); 719 if (!mqd) 720 return NULL; 721 722 mqd->dev = dev; 723 724 switch (type) { 725 case KFD_MQD_TYPE_CP: 726 pr_debug("%s@%i\n", __func__, __LINE__); 727 mqd->allocate_mqd = allocate_mqd; 728 mqd->init_mqd = init_mqd_v12_1; 729 mqd->free_mqd = kfd_free_mqd_cp; 730 mqd->load_mqd = load_mqd_v12_1; 731 mqd->update_mqd = update_mqd_v12_1; 732 mqd->destroy_mqd = destroy_mqd_v12_1; 733 mqd->is_occupied = kfd_is_occupied_cp; 734 mqd->mqd_size = sizeof(struct v12_1_compute_mqd); 735 mqd->get_wave_state = get_wave_state_v12_1; 736 mqd->mqd_stride = kfd_mqd_stride; 737 mqd->restore_mqd = restore_mqd_v12_1; 738 #if defined(CONFIG_DEBUG_FS) 739 mqd->debugfs_show_mqd = debugfs_show_mqd; 740 #endif 741 pr_debug("%s@%i\n", __func__, __LINE__); 742 break; 743 case KFD_MQD_TYPE_HIQ: 744 pr_debug("%s@%i\n", __func__, __LINE__); 745 mqd->allocate_mqd = allocate_hiq_mqd; 746 mqd->init_mqd = init_mqd_hiq; 747 mqd->free_mqd = free_mqd_hiq_sdma; 748 mqd->load_mqd = kfd_hiq_load_mqd_kiq; 749 mqd->update_mqd = update_mqd; 750 mqd->destroy_mqd = kfd_destroy_mqd_cp; 751 mqd->is_occupied = kfd_is_occupied_cp; 752 mqd->mqd_size = sizeof(struct v12_1_compute_mqd); 753 mqd->mqd_stride = kfd_mqd_stride; 754 #if defined(CONFIG_DEBUG_FS) 755 mqd->debugfs_show_mqd = debugfs_show_mqd; 756 #endif 757 mqd->check_preemption_failed = check_preemption_failed; 758 pr_debug("%s@%i\n", __func__, __LINE__); 759 break; 760 case KFD_MQD_TYPE_DIQ: 761 mqd->allocate_mqd = allocate_mqd; 762 mqd->init_mqd = init_mqd_hiq; 763 mqd->free_mqd = kfd_free_mqd_cp; 764 mqd->load_mqd = load_mqd; 765 mqd->update_mqd = update_mqd; 766 mqd->destroy_mqd = kfd_destroy_mqd_cp; 767 mqd->is_occupied = kfd_is_occupied_cp; 768 mqd->mqd_size = sizeof(struct v12_1_compute_mqd); 769 #if defined(CONFIG_DEBUG_FS) 770 mqd->debugfs_show_mqd = debugfs_show_mqd; 771 #endif 772 break; 773 case KFD_MQD_TYPE_SDMA: 774 pr_debug("%s@%i\n", __func__, __LINE__); 775 mqd->allocate_mqd = allocate_mqd; 776 mqd->init_mqd = init_mqd_sdma; 777 mqd->free_mqd = kfd_free_mqd_cp; 778 mqd->load_mqd = kfd_load_mqd_sdma; 779 mqd->update_mqd = update_mqd_sdma; 780 mqd->destroy_mqd = kfd_destroy_mqd_sdma; 781 mqd->is_occupied = kfd_is_occupied_sdma; 782 mqd->mqd_size = sizeof(struct v12_sdma_mqd); 783 mqd->mqd_stride = kfd_mqd_stride; 784 mqd->restore_mqd = restore_mqd_sdma_v12_1; 785 #if defined(CONFIG_DEBUG_FS) 786 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; 787 #endif 788 pr_debug("%s@%i\n", __func__, __LINE__); 789 break; 790 default: 791 kfree(mqd); 792 return NULL; 793 } 794 795 return mqd; 796 } 797