1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include <linux/mmu_context.h> 23 #include "amdgpu.h" 24 #include "amdgpu_amdkfd.h" 25 #include "gc/gc_11_0_0_offset.h" 26 #include "gc/gc_11_0_0_sh_mask.h" 27 #include "oss/osssys_6_0_0_offset.h" 28 #include "oss/osssys_6_0_0_sh_mask.h" 29 #include "soc15_common.h" 30 #include "soc15d.h" 31 #include "v11_structs.h" 32 #include "soc21.h" 33 #include <uapi/linux/kfd_ioctl.h> 34 35 enum hqd_dequeue_request_type { 36 NO_ACTION = 0, 37 DRAIN_PIPE, 38 RESET_WAVES, 39 SAVE_WAVES 40 }; 41 42 static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe, 43 uint32_t queue, uint32_t vmid) 44 { 45 mutex_lock(&adev->srbm_mutex); 46 soc21_grbm_select(adev, mec, pipe, queue, vmid); 47 } 48 49 static void unlock_srbm(struct amdgpu_device *adev) 50 { 51 soc21_grbm_select(adev, 0, 0, 0, 0); 52 mutex_unlock(&adev->srbm_mutex); 53 } 54 55 static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id, 56 uint32_t queue_id) 57 { 58 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 59 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 60 61 lock_srbm(adev, mec, pipe, queue_id, 0); 62 } 63 64 static uint64_t get_queue_mask(struct amdgpu_device *adev, 65 uint32_t pipe_id, uint32_t queue_id) 66 { 67 unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe + 68 queue_id; 69 70 return 1ull << bit; 71 } 72 73 static void release_queue(struct amdgpu_device *adev) 74 { 75 unlock_srbm(adev); 76 } 77 78 static void program_sh_mem_settings_v11(struct amdgpu_device *adev, uint32_t vmid, 79 uint32_t sh_mem_config, 80 uint32_t sh_mem_ape1_base, 81 uint32_t sh_mem_ape1_limit, 82 uint32_t sh_mem_bases, uint32_t inst) 83 { 84 lock_srbm(adev, 0, 0, 0, vmid); 85 86 WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_CONFIG), sh_mem_config); 87 WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_BASES), sh_mem_bases); 88 89 unlock_srbm(adev); 90 } 91 92 static int set_pasid_vmid_mapping_v11(struct amdgpu_device *adev, unsigned int pasid, 93 unsigned int vmid, uint32_t inst) 94 { 95 uint32_t value = pasid << IH_VMID_0_LUT__PASID__SHIFT; 96 97 /* Mapping vmid to pasid also for IH block */ 98 pr_debug("mapping vmid %d -> pasid %d in IH block for GFX client\n", 99 vmid, pasid); 100 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid, value); 101 102 return 0; 103 } 104 105 static int init_interrupts_v11(struct amdgpu_device *adev, uint32_t pipe_id, 106 uint32_t inst) 107 { 108 uint32_t mec; 109 uint32_t pipe; 110 111 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 112 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 113 114 lock_srbm(adev, mec, pipe, 0, 0); 115 116 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, 117 CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | 118 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); 119 120 unlock_srbm(adev); 121 122 return 0; 123 } 124 125 static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, 126 unsigned int engine_id, 127 unsigned int queue_id) 128 { 129 uint32_t sdma_engine_reg_base = 0; 130 uint32_t sdma_rlc_reg_offset; 131 132 switch (engine_id) { 133 case 0: 134 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0, 135 regSDMA0_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL; 136 break; 137 case 1: 138 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, 0, 139 regSDMA1_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL; 140 break; 141 default: 142 WARN(1, "Invalid SDMA engine id %d\n", engine_id); 143 break; 144 } 145 146 sdma_rlc_reg_offset = sdma_engine_reg_base 147 + queue_id * (regSDMA0_QUEUE1_RB_CNTL - regSDMA0_QUEUE0_RB_CNTL); 148 149 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id, 150 queue_id, sdma_rlc_reg_offset); 151 152 return sdma_rlc_reg_offset; 153 } 154 155 static inline struct v11_compute_mqd *get_mqd(void *mqd) 156 { 157 return (struct v11_compute_mqd *)mqd; 158 } 159 160 static inline struct v11_sdma_mqd *get_sdma_mqd(void *mqd) 161 { 162 return (struct v11_sdma_mqd *)mqd; 163 } 164 165 static int hqd_load_v11(struct amdgpu_device *adev, void *mqd, uint32_t pipe_id, 166 uint32_t queue_id, uint32_t __user *wptr, 167 uint32_t wptr_shift, uint32_t wptr_mask, 168 struct mm_struct *mm, uint32_t inst) 169 { 170 struct v11_compute_mqd *m; 171 uint32_t *mqd_hqd; 172 uint32_t reg, hqd_base, data; 173 174 m = get_mqd(mqd); 175 176 pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id); 177 acquire_queue(adev, pipe_id, queue_id); 178 179 /* HIQ is set during driver init period with vmid set to 0*/ 180 if (m->cp_hqd_vmid == 0) { 181 uint32_t value, mec, pipe; 182 183 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 184 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 185 186 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", 187 mec, pipe, queue_id); 188 value = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS)); 189 value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1, 190 ((mec << 5) | (pipe << 3) | queue_id | 0x80)); 191 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS), value); 192 } 193 194 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ 195 mqd_hqd = &m->cp_mqd_base_addr_lo; 196 hqd_base = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR); 197 198 for (reg = hqd_base; 199 reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++) 200 WREG32(reg, mqd_hqd[reg - hqd_base]); 201 202 203 /* Activate doorbell logic before triggering WPTR poll. */ 204 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, 205 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 206 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), data); 207 208 if (wptr) { 209 /* Don't read wptr with get_user because the user 210 * context may not be accessible (if this function 211 * runs in a work queue). Instead trigger a one-shot 212 * polling read from memory in the CP. This assumes 213 * that wptr is GPU-accessible in the queue's VMID via 214 * ATC or SVM. WPTR==RPTR before starting the poll so 215 * the CP starts fetching new commands from the right 216 * place. 217 * 218 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit 219 * tricky. Assume that the queue didn't overflow. The 220 * number of valid bits in the 32-bit RPTR depends on 221 * the queue size. The remaining bits are taken from 222 * the saved 64-bit WPTR. If the WPTR wrapped, add the 223 * queue size. 224 */ 225 uint32_t queue_size = 226 2 << REG_GET_FIELD(m->cp_hqd_pq_control, 227 CP_HQD_PQ_CONTROL, QUEUE_SIZE); 228 uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); 229 230 if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) 231 guessed_wptr += queue_size; 232 guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); 233 guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; 234 235 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_LO), 236 lower_32_bits(guessed_wptr)); 237 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI), 238 upper_32_bits(guessed_wptr)); 239 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 240 lower_32_bits((uint64_t)wptr)); 241 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 242 upper_32_bits((uint64_t)wptr)); 243 pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__, 244 (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); 245 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_PQ_WPTR_POLL_CNTL1), 246 (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); 247 } 248 249 /* Start the EOP fetcher */ 250 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_EOP_RPTR), 251 REG_SET_FIELD(m->cp_hqd_eop_rptr, 252 CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); 253 254 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 255 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE), data); 256 257 release_queue(adev); 258 259 return 0; 260 } 261 262 static int hiq_mqd_load_v11(struct amdgpu_device *adev, void *mqd, 263 uint32_t pipe_id, uint32_t queue_id, 264 uint32_t doorbell_off, uint32_t inst) 265 { 266 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 267 struct v11_compute_mqd *m; 268 uint32_t mec, pipe; 269 int r; 270 271 m = get_mqd(mqd); 272 273 acquire_queue(adev, pipe_id, queue_id); 274 275 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 276 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 277 278 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", 279 mec, pipe, queue_id); 280 281 spin_lock(&adev->gfx.kiq[0].ring_lock); 282 r = amdgpu_ring_alloc(kiq_ring, 7); 283 if (r) { 284 pr_err("Failed to alloc KIQ (%d).\n", r); 285 goto out_unlock; 286 } 287 288 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 289 amdgpu_ring_write(kiq_ring, 290 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 291 PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */ 292 PACKET3_MAP_QUEUES_QUEUE(queue_id) | 293 PACKET3_MAP_QUEUES_PIPE(pipe) | 294 PACKET3_MAP_QUEUES_ME((mec - 1)) | 295 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 296 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 297 PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */ 298 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 299 amdgpu_ring_write(kiq_ring, 300 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off)); 301 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_lo); 302 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_hi); 303 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_lo); 304 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_hi); 305 amdgpu_ring_commit(kiq_ring); 306 307 out_unlock: 308 spin_unlock(&adev->gfx.kiq[0].ring_lock); 309 release_queue(adev); 310 311 return r; 312 } 313 314 static int hqd_dump_v11(struct amdgpu_device *adev, 315 uint32_t pipe_id, uint32_t queue_id, 316 uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst) 317 { 318 uint32_t i = 0, reg; 319 #define HQD_N_REGS 56 320 #define DUMP_REG(addr) do { \ 321 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ 322 break; \ 323 (*dump)[i][0] = (addr) << 2; \ 324 (*dump)[i++][1] = RREG32(addr); \ 325 } while (0) 326 327 *dump = kmalloc_objs(**dump, HQD_N_REGS); 328 if (*dump == NULL) 329 return -ENOMEM; 330 331 acquire_queue(adev, pipe_id, queue_id); 332 333 for (reg = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR); 334 reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++) 335 DUMP_REG(reg); 336 337 release_queue(adev); 338 339 WARN_ON_ONCE(i != HQD_N_REGS); 340 *n_regs = i; 341 342 return 0; 343 } 344 345 static int hqd_sdma_load_v11(struct amdgpu_device *adev, void *mqd, 346 uint32_t __user *wptr, struct mm_struct *mm) 347 { 348 struct v11_sdma_mqd *m; 349 uint32_t sdma_rlc_reg_offset; 350 unsigned long end_jiffies; 351 uint32_t data; 352 uint64_t data64; 353 uint64_t __user *wptr64 = (uint64_t __user *)wptr; 354 355 m = get_sdma_mqd(mqd); 356 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 357 m->sdma_queue_id); 358 359 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, 360 m->sdmax_rlcx_rb_cntl & (~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK)); 361 362 end_jiffies = msecs_to_jiffies(2000) + jiffies; 363 while (true) { 364 data = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS); 365 if (data & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK) 366 break; 367 if (time_after(jiffies, end_jiffies)) { 368 pr_err("SDMA RLC not idle in %s\n", __func__); 369 return -ETIME; 370 } 371 usleep_range(500, 1000); 372 } 373 374 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL_OFFSET, 375 m->sdmax_rlcx_doorbell_offset); 376 377 data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_QUEUE0_DOORBELL, 378 ENABLE, 1); 379 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, data); 380 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR, 381 m->sdmax_rlcx_rb_rptr); 382 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI, 383 m->sdmax_rlcx_rb_rptr_hi); 384 385 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 1); 386 if (read_user_wptr(mm, wptr64, data64)) { 387 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR, 388 lower_32_bits(data64)); 389 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI, 390 upper_32_bits(data64)); 391 } else { 392 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR, 393 m->sdmax_rlcx_rb_rptr); 394 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI, 395 m->sdmax_rlcx_rb_rptr_hi); 396 } 397 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 0); 398 399 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE, m->sdmax_rlcx_rb_base); 400 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE_HI, 401 m->sdmax_rlcx_rb_base_hi); 402 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_LO, 403 m->sdmax_rlcx_rb_rptr_addr_lo); 404 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_HI, 405 m->sdmax_rlcx_rb_rptr_addr_hi); 406 407 data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_QUEUE0_RB_CNTL, 408 RB_ENABLE, 1); 409 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, data); 410 411 return 0; 412 } 413 414 static int hqd_sdma_dump_v11(struct amdgpu_device *adev, 415 uint32_t engine_id, uint32_t queue_id, 416 uint32_t (**dump)[2], uint32_t *n_regs) 417 { 418 uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, 419 engine_id, queue_id); 420 uint32_t i = 0, reg; 421 #undef HQD_N_REGS 422 #define HQD_N_REGS (7+11+1+12+12) 423 424 *dump = kmalloc_objs(**dump, HQD_N_REGS); 425 if (*dump == NULL) 426 return -ENOMEM; 427 428 for (reg = regSDMA0_QUEUE0_RB_CNTL; 429 reg <= regSDMA0_QUEUE0_RB_WPTR_HI; reg++) 430 DUMP_REG(sdma_rlc_reg_offset + reg); 431 for (reg = regSDMA0_QUEUE0_RB_RPTR_ADDR_HI; 432 reg <= regSDMA0_QUEUE0_DOORBELL; reg++) 433 DUMP_REG(sdma_rlc_reg_offset + reg); 434 for (reg = regSDMA0_QUEUE0_DOORBELL_LOG; 435 reg <= regSDMA0_QUEUE0_DOORBELL_LOG; reg++) 436 DUMP_REG(sdma_rlc_reg_offset + reg); 437 for (reg = regSDMA0_QUEUE0_DOORBELL_OFFSET; 438 reg <= regSDMA0_QUEUE0_RB_PREEMPT; reg++) 439 DUMP_REG(sdma_rlc_reg_offset + reg); 440 for (reg = regSDMA0_QUEUE0_MIDCMD_DATA0; 441 reg <= regSDMA0_QUEUE0_MIDCMD_CNTL; reg++) 442 DUMP_REG(sdma_rlc_reg_offset + reg); 443 444 WARN_ON_ONCE(i != HQD_N_REGS); 445 *n_regs = i; 446 447 return 0; 448 } 449 450 static bool hqd_is_occupied_v11(struct amdgpu_device *adev, uint64_t queue_address, 451 uint32_t pipe_id, uint32_t queue_id, uint32_t inst) 452 { 453 uint32_t act; 454 bool retval = false; 455 uint32_t low, high; 456 457 acquire_queue(adev, pipe_id, queue_id); 458 act = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE)); 459 if (act) { 460 low = lower_32_bits(queue_address >> 8); 461 high = upper_32_bits(queue_address >> 8); 462 463 if (low == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE)) && 464 high == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE_HI))) 465 retval = true; 466 } 467 release_queue(adev); 468 return retval; 469 } 470 471 static bool hqd_sdma_is_occupied_v11(struct amdgpu_device *adev, void *mqd) 472 { 473 struct v11_sdma_mqd *m; 474 uint32_t sdma_rlc_reg_offset; 475 uint32_t sdma_rlc_rb_cntl; 476 477 m = get_sdma_mqd(mqd); 478 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 479 m->sdma_queue_id); 480 481 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL); 482 483 if (sdma_rlc_rb_cntl & SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK) 484 return true; 485 486 return false; 487 } 488 489 static int hqd_destroy_v11(struct amdgpu_device *adev, void *mqd, 490 enum kfd_preempt_type reset_type, 491 unsigned int utimeout, uint32_t pipe_id, 492 uint32_t queue_id, uint32_t inst) 493 { 494 enum hqd_dequeue_request_type type; 495 unsigned long end_jiffies; 496 uint32_t temp; 497 struct v11_compute_mqd *m = get_mqd(mqd); 498 499 acquire_queue(adev, pipe_id, queue_id); 500 501 if (m->cp_hqd_vmid == 0) 502 WREG32_FIELD15_PREREG(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); 503 504 switch (reset_type) { 505 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: 506 type = DRAIN_PIPE; 507 break; 508 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: 509 type = RESET_WAVES; 510 break; 511 default: 512 type = DRAIN_PIPE; 513 break; 514 } 515 516 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_DEQUEUE_REQUEST), type); 517 518 end_jiffies = (utimeout * HZ / 1000) + jiffies; 519 while (true) { 520 temp = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE)); 521 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) 522 break; 523 if (time_after(jiffies, end_jiffies)) { 524 pr_err("cp queue pipe %d queue %d preemption failed\n", 525 pipe_id, queue_id); 526 release_queue(adev); 527 return -ETIME; 528 } 529 usleep_range(500, 1000); 530 } 531 532 release_queue(adev); 533 return 0; 534 } 535 536 static int hqd_sdma_destroy_v11(struct amdgpu_device *adev, void *mqd, 537 unsigned int utimeout) 538 { 539 struct v11_sdma_mqd *m; 540 uint32_t sdma_rlc_reg_offset; 541 uint32_t temp; 542 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; 543 544 m = get_sdma_mqd(mqd); 545 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 546 m->sdma_queue_id); 547 548 temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL); 549 temp = temp & ~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK; 550 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, temp); 551 552 while (true) { 553 temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS); 554 if (temp & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK) 555 break; 556 if (time_after(jiffies, end_jiffies)) { 557 pr_err("SDMA RLC not idle in %s\n", __func__); 558 return -ETIME; 559 } 560 usleep_range(500, 1000); 561 } 562 563 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, 0); 564 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, 565 RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL) | 566 SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK); 567 568 m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR); 569 m->sdmax_rlcx_rb_rptr_hi = 570 RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI); 571 572 return 0; 573 } 574 575 static int wave_control_execute_v11(struct amdgpu_device *adev, 576 uint32_t gfx_index_val, 577 uint32_t sq_cmd, uint32_t inst) 578 { 579 uint32_t data = 0; 580 581 mutex_lock(&adev->grbm_idx_mutex); 582 583 WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), gfx_index_val); 584 WREG32(SOC15_REG_OFFSET(GC, 0, regSQ_CMD), sq_cmd); 585 586 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 587 INSTANCE_BROADCAST_WRITES, 1); 588 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 589 SA_BROADCAST_WRITES, 1); 590 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 591 SE_BROADCAST_WRITES, 1); 592 593 WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), data); 594 mutex_unlock(&adev->grbm_idx_mutex); 595 596 return 0; 597 } 598 599 static void set_vm_context_page_table_base_v11(struct amdgpu_device *adev, 600 uint32_t vmid, uint64_t page_table_base) 601 { 602 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { 603 pr_err("trying to set page table base for wrong VMID %u\n", 604 vmid); 605 return; 606 } 607 608 /* SDMA is on gfxhub as well for gfx11 adapters */ 609 adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); 610 } 611 612 /* 613 * Returns TRAP_EN, EXCP_EN and EXCP_REPLACE. 614 * 615 * restore_dbg_registers is ignored here but is a general interface requirement 616 * for devices that support GFXOFF and where the RLC save/restore list 617 * does not support hw registers for debugging i.e. the driver has to manually 618 * initialize the debug mode registers after it has disabled GFX off during the 619 * debug session. 620 */ 621 static uint32_t kgd_gfx_v11_enable_debug_trap(struct amdgpu_device *adev, 622 bool restore_dbg_registers, 623 uint32_t vmid) 624 { 625 uint32_t data = 0; 626 627 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 628 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, 0); 629 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, 0); 630 631 return data; 632 } 633 634 /* Returns TRAP_EN, EXCP_EN and EXCP_REPLACE. */ 635 static uint32_t kgd_gfx_v11_disable_debug_trap(struct amdgpu_device *adev, 636 bool keep_trap_enabled, 637 uint32_t vmid) 638 { 639 uint32_t data = 0; 640 641 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 642 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, 0); 643 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, 0); 644 645 return data; 646 } 647 648 static int kgd_gfx_v11_validate_trap_override_request(struct amdgpu_device *adev, 649 uint32_t trap_override, 650 uint32_t *trap_mask_supported) 651 { 652 *trap_mask_supported &= KFD_DBG_TRAP_MASK_FP_INVALID | 653 KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL | 654 KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO | 655 KFD_DBG_TRAP_MASK_FP_OVERFLOW | 656 KFD_DBG_TRAP_MASK_FP_UNDERFLOW | 657 KFD_DBG_TRAP_MASK_FP_INEXACT | 658 KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO | 659 KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH | 660 KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION; 661 662 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 4)) 663 *trap_mask_supported |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START | 664 KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END; 665 666 if (trap_override != KFD_DBG_TRAP_OVERRIDE_OR && 667 trap_override != KFD_DBG_TRAP_OVERRIDE_REPLACE) 668 return -EPERM; 669 670 return 0; 671 } 672 673 static uint32_t trap_mask_map_sw_to_hw(uint32_t mask) 674 { 675 uint32_t trap_on_start = (mask & KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START) ? 1 : 0; 676 uint32_t trap_on_end = (mask & KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END) ? 1 : 0; 677 uint32_t excp_en = mask & (KFD_DBG_TRAP_MASK_FP_INVALID | 678 KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL | 679 KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO | 680 KFD_DBG_TRAP_MASK_FP_OVERFLOW | 681 KFD_DBG_TRAP_MASK_FP_UNDERFLOW | 682 KFD_DBG_TRAP_MASK_FP_INEXACT | 683 KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO | 684 KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH | 685 KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION); 686 uint32_t ret; 687 688 ret = REG_SET_FIELD(0, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, excp_en); 689 ret = REG_SET_FIELD(ret, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_START, trap_on_start); 690 ret = REG_SET_FIELD(ret, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_END, trap_on_end); 691 692 return ret; 693 } 694 695 static uint32_t trap_mask_map_hw_to_sw(uint32_t mask) 696 { 697 uint32_t ret = REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, EXCP_EN); 698 699 if (REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_START)) 700 ret |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START; 701 702 if (REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_END)) 703 ret |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END; 704 705 return ret; 706 } 707 708 /* Returns TRAP_EN, EXCP_EN and EXCP_REPLACE. */ 709 static uint32_t kgd_gfx_v11_set_wave_launch_trap_override(struct amdgpu_device *adev, 710 uint32_t vmid, 711 uint32_t trap_override, 712 uint32_t trap_mask_bits, 713 uint32_t trap_mask_request, 714 uint32_t *trap_mask_prev, 715 uint32_t kfd_dbg_trap_cntl_prev) 716 { 717 uint32_t data = 0; 718 719 *trap_mask_prev = trap_mask_map_hw_to_sw(kfd_dbg_trap_cntl_prev); 720 721 data = (trap_mask_bits & trap_mask_request) | (*trap_mask_prev & ~trap_mask_request); 722 data = trap_mask_map_sw_to_hw(data); 723 724 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 725 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, trap_override); 726 727 return data; 728 } 729 730 static uint32_t kgd_gfx_v11_set_wave_launch_mode(struct amdgpu_device *adev, 731 uint8_t wave_launch_mode, 732 uint32_t vmid) 733 { 734 uint32_t data = 0; 735 736 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, LAUNCH_MODE, wave_launch_mode); 737 738 return data; 739 } 740 741 #define TCP_WATCH_STRIDE (regTCP_WATCH1_ADDR_H - regTCP_WATCH0_ADDR_H) 742 static uint32_t kgd_gfx_v11_set_address_watch(struct amdgpu_device *adev, 743 uint64_t watch_address, 744 uint32_t watch_address_mask, 745 uint32_t watch_id, 746 uint32_t watch_mode, 747 uint32_t debug_vmid, 748 uint32_t inst) 749 { 750 uint32_t watch_address_high; 751 uint32_t watch_address_low; 752 uint32_t watch_address_cntl; 753 754 watch_address_cntl = 0; 755 watch_address_low = lower_32_bits(watch_address); 756 watch_address_high = upper_32_bits(watch_address) & 0xffff; 757 758 watch_address_cntl = REG_SET_FIELD(watch_address_cntl, 759 TCP_WATCH0_CNTL, 760 MODE, 761 watch_mode); 762 763 watch_address_cntl = REG_SET_FIELD(watch_address_cntl, 764 TCP_WATCH0_CNTL, 765 MASK, 766 watch_address_mask >> 7); 767 768 watch_address_cntl = REG_SET_FIELD(watch_address_cntl, 769 TCP_WATCH0_CNTL, 770 VALID, 771 1); 772 773 WREG32_RLC((SOC15_REG_OFFSET(GC, 0, regTCP_WATCH0_ADDR_H) + 774 (watch_id * TCP_WATCH_STRIDE)), 775 watch_address_high); 776 777 WREG32_RLC((SOC15_REG_OFFSET(GC, 0, regTCP_WATCH0_ADDR_L) + 778 (watch_id * TCP_WATCH_STRIDE)), 779 watch_address_low); 780 781 return watch_address_cntl; 782 } 783 784 static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev, 785 uint32_t watch_id) 786 { 787 return 0; 788 } 789 790 static uint64_t kgd_gfx_v11_hqd_get_pq_addr(struct amdgpu_device *adev, 791 uint32_t pipe_id, uint32_t queue_id, 792 uint32_t inst) 793 { 794 return 0; 795 } 796 797 static uint64_t kgd_gfx_v11_hqd_reset(struct amdgpu_device *adev, 798 uint32_t pipe_id, uint32_t queue_id, 799 uint32_t inst, unsigned int utimeout) 800 { 801 return 0; 802 } 803 804 static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev, 805 int engine, int queue) 806 { 807 return 0; 808 } 809 810 const struct kfd2kgd_calls gfx_v11_kfd2kgd = { 811 .program_sh_mem_settings = program_sh_mem_settings_v11, 812 .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11, 813 .init_interrupts = init_interrupts_v11, 814 .hqd_load = hqd_load_v11, 815 .hiq_mqd_load = hiq_mqd_load_v11, 816 .hqd_sdma_load = hqd_sdma_load_v11, 817 .hqd_dump = hqd_dump_v11, 818 .hqd_sdma_dump = hqd_sdma_dump_v11, 819 .hqd_is_occupied = hqd_is_occupied_v11, 820 .hqd_sdma_is_occupied = hqd_sdma_is_occupied_v11, 821 .hqd_destroy = hqd_destroy_v11, 822 .hqd_sdma_destroy = hqd_sdma_destroy_v11, 823 .wave_control_execute = wave_control_execute_v11, 824 .get_atc_vmid_pasid_mapping_info = NULL, 825 .set_vm_context_page_table_base = set_vm_context_page_table_base_v11, 826 .enable_debug_trap = kgd_gfx_v11_enable_debug_trap, 827 .disable_debug_trap = kgd_gfx_v11_disable_debug_trap, 828 .validate_trap_override_request = kgd_gfx_v11_validate_trap_override_request, 829 .set_wave_launch_trap_override = kgd_gfx_v11_set_wave_launch_trap_override, 830 .set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode, 831 .set_address_watch = kgd_gfx_v11_set_address_watch, 832 .clear_address_watch = kgd_gfx_v11_clear_address_watch, 833 .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, 834 .hqd_reset = kgd_gfx_v11_hqd_reset, 835 .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell 836 }; 837