1 /*
2 * Copyright 2023 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include <linux/module.h>
26 #include "amdgpu.h"
27 #include "gfx_v12_0.h"
28 #include "soc15_common.h"
29 #include "soc24.h"
30 #include "gc/gc_12_0_0_offset.h"
31 #include "gc/gc_12_0_0_sh_mask.h"
32 #include "gc/gc_11_0_0_default.h"
33 #include "v12_structs.h"
34 #include "mes_v12_api_def.h"
35
36 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes.bin");
37 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes1.bin");
38 MODULE_FIRMWARE("amdgpu/gc_12_0_0_uni_mes.bin");
39 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes.bin");
40 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes1.bin");
41 MODULE_FIRMWARE("amdgpu/gc_12_0_1_uni_mes.bin");
42
43 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block);
44 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block);
45 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
46 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
47
48 #define MES_EOP_SIZE 2048
49
50 #define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */
51 #define MES12_HUNG_HQD_INFO_OFFSET 4
52
mes_v12_0_ring_set_wptr(struct amdgpu_ring * ring)53 static void mes_v12_0_ring_set_wptr(struct amdgpu_ring *ring)
54 {
55 struct amdgpu_device *adev = ring->adev;
56
57 if (ring->use_doorbell) {
58 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
59 ring->wptr);
60 WDOORBELL64(ring->doorbell_index, ring->wptr);
61 } else {
62 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
63 }
64 }
65
mes_v12_0_ring_get_rptr(struct amdgpu_ring * ring)66 static u64 mes_v12_0_ring_get_rptr(struct amdgpu_ring *ring)
67 {
68 return *ring->rptr_cpu_addr;
69 }
70
mes_v12_0_ring_get_wptr(struct amdgpu_ring * ring)71 static u64 mes_v12_0_ring_get_wptr(struct amdgpu_ring *ring)
72 {
73 struct amdgpu_device *adev = ring->adev;
74 u64 wptr;
75
76 if (ring->use_doorbell) {
77 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
78 } else {
79 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
80 wptr = 0;
81 }
82 return wptr;
83 }
84
85 static const struct amdgpu_ring_funcs mes_v12_0_ring_funcs = {
86 .type = AMDGPU_RING_TYPE_MES,
87 .align_mask = 1,
88 .nop = 0,
89 .support_64bit_ptrs = true,
90 .get_rptr = mes_v12_0_ring_get_rptr,
91 .get_wptr = mes_v12_0_ring_get_wptr,
92 .set_wptr = mes_v12_0_ring_set_wptr,
93 .insert_nop = amdgpu_ring_insert_nop,
94 };
95
96 static const char *mes_v12_0_opcodes[] = {
97 "SET_HW_RSRC",
98 "SET_SCHEDULING_CONFIG",
99 "ADD_QUEUE",
100 "REMOVE_QUEUE",
101 "PERFORM_YIELD",
102 "SET_GANG_PRIORITY_LEVEL",
103 "SUSPEND",
104 "RESUME",
105 "RESET",
106 "SET_LOG_BUFFER",
107 "CHANGE_GANG_PRORITY",
108 "QUERY_SCHEDULER_STATUS",
109 "unused",
110 "SET_DEBUG_VMID",
111 "MISC",
112 "UPDATE_ROOT_PAGE_TABLE",
113 "AMD_LOG",
114 "SET_SE_MODE",
115 "SET_GANG_SUBMIT",
116 "SET_HW_RSRC_1",
117 "INVALIDATE_TLBS",
118 };
119
120 static const char *mes_v12_0_misc_opcodes[] = {
121 "WRITE_REG",
122 "INV_GART",
123 "QUERY_STATUS",
124 "READ_REG",
125 "WAIT_REG_MEM",
126 "SET_SHADER_DEBUGGER",
127 "NOTIFY_WORK_ON_UNMAPPED_QUEUE",
128 "NOTIFY_TO_UNMAP_PROCESSES",
129 };
130
mes_v12_0_get_op_string(union MESAPI__MISC * x_pkt)131 static const char *mes_v12_0_get_op_string(union MESAPI__MISC *x_pkt)
132 {
133 const char *op_str = NULL;
134
135 if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_0_opcodes))
136 op_str = mes_v12_0_opcodes[x_pkt->header.opcode];
137
138 return op_str;
139 }
140
mes_v12_0_get_misc_op_string(union MESAPI__MISC * x_pkt)141 static const char *mes_v12_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
142 {
143 const char *op_str = NULL;
144
145 if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
146 (x_pkt->opcode < ARRAY_SIZE(mes_v12_0_misc_opcodes)))
147 op_str = mes_v12_0_misc_opcodes[x_pkt->opcode];
148
149 return op_str;
150 }
151
mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes * mes,int pipe,void * pkt,int size,int api_status_off)152 static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
153 int pipe, void *pkt, int size,
154 int api_status_off)
155 {
156 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
157 signed long timeout = 2100000; /* 2100 ms */
158 struct amdgpu_device *adev = mes->adev;
159 struct amdgpu_ring *ring = &mes->ring[pipe];
160 spinlock_t *ring_lock = &mes->ring_lock[pipe];
161 struct MES_API_STATUS *api_status;
162 union MESAPI__MISC *x_pkt = pkt;
163 const char *op_str, *misc_op_str;
164 unsigned long flags;
165 u64 status_gpu_addr;
166 u32 seq, status_offset;
167 u64 *status_ptr;
168 signed long r;
169 int ret;
170
171 if (x_pkt->header.opcode >= MES_SCH_API_MAX)
172 return -EINVAL;
173
174 if (amdgpu_emu_mode) {
175 timeout *= 100;
176 } else if (amdgpu_sriov_vf(adev)) {
177 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
178 timeout = 15 * 600 * 1000;
179 }
180
181 ret = amdgpu_wb_get(adev, &status_offset);
182 if (ret)
183 return ret;
184
185 status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
186 status_ptr = (u64 *)&adev->wb.wb[status_offset];
187 *status_ptr = 0;
188
189 spin_lock_irqsave(ring_lock, flags);
190 r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
191 if (r)
192 goto error_unlock_free;
193
194 seq = ++ring->fence_drv.sync_seq;
195 r = amdgpu_fence_wait_polling(ring,
196 seq - ring->fence_drv.num_fences_mask,
197 timeout);
198 if (r < 1)
199 goto error_undo;
200
201 api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
202 api_status->api_completion_fence_addr = status_gpu_addr;
203 api_status->api_completion_fence_value = 1;
204
205 amdgpu_ring_write_multiple(ring, pkt, size / 4);
206
207 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
208 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
209 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
210 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
211 mes_status_pkt.api_status.api_completion_fence_addr =
212 ring->fence_drv.gpu_addr;
213 mes_status_pkt.api_status.api_completion_fence_value = seq;
214
215 amdgpu_ring_write_multiple(ring, &mes_status_pkt,
216 sizeof(mes_status_pkt) / 4);
217
218 amdgpu_ring_commit(ring);
219 spin_unlock_irqrestore(ring_lock, flags);
220
221 op_str = mes_v12_0_get_op_string(x_pkt);
222 misc_op_str = mes_v12_0_get_misc_op_string(x_pkt);
223
224 if (misc_op_str)
225 dev_dbg(adev->dev, "MES(%d) msg=%s (%s) was emitted\n",
226 pipe, op_str, misc_op_str);
227 else if (op_str)
228 dev_dbg(adev->dev, "MES(%d) msg=%s was emitted\n",
229 pipe, op_str);
230 else
231 dev_dbg(adev->dev, "MES(%d) msg=%d was emitted\n",
232 pipe, x_pkt->header.opcode);
233
234 r = amdgpu_fence_wait_polling(ring, seq, timeout);
235
236 /*
237 * status_ptr[31:0] == 0 (fail) or status_ptr[63:0] == 1 (success).
238 * If status_ptr[31:0] == 0 then status_ptr[63:32] will have debug error information.
239 */
240 if (r < 1 || !(lower_32_bits(*status_ptr))) {
241
242 if (misc_op_str)
243 dev_err(adev->dev, "MES(%d) failed to respond to msg=%s (%s)\n",
244 pipe, op_str, misc_op_str);
245 else if (op_str)
246 dev_err(adev->dev, "MES(%d) failed to respond to msg=%s\n",
247 pipe, op_str);
248 else
249 dev_err(adev->dev, "MES(%d) failed to respond to msg=%d\n",
250 pipe, x_pkt->header.opcode);
251
252 while (halt_if_hws_hang)
253 schedule();
254
255 r = -ETIMEDOUT;
256 goto error_wb_free;
257 }
258
259 amdgpu_wb_free(adev, status_offset);
260 return 0;
261
262 error_undo:
263 dev_err(adev->dev, "MES ring buffer is full.\n");
264 amdgpu_ring_undo(ring);
265
266 error_unlock_free:
267 spin_unlock_irqrestore(ring_lock, flags);
268
269 error_wb_free:
270 amdgpu_wb_free(adev, status_offset);
271 return r;
272 }
273
convert_to_mes_queue_type(int queue_type)274 static int convert_to_mes_queue_type(int queue_type)
275 {
276 if (queue_type == AMDGPU_RING_TYPE_GFX)
277 return MES_QUEUE_TYPE_GFX;
278 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
279 return MES_QUEUE_TYPE_COMPUTE;
280 else if (queue_type == AMDGPU_RING_TYPE_SDMA)
281 return MES_QUEUE_TYPE_SDMA;
282 else if (queue_type == AMDGPU_RING_TYPE_MES)
283 return MES_QUEUE_TYPE_SCHQ;
284 else
285 WARN(1, "Invalid queue type %d\n", queue_type);
286 return MES_QUEUE_TYPE_GFX;
287 }
288
convert_to_mes_priority_level(int priority_level)289 static int convert_to_mes_priority_level(int priority_level)
290 {
291 switch (priority_level) {
292 case AMDGPU_MES_PRIORITY_LEVEL_LOW:
293 return AMD_PRIORITY_LEVEL_LOW;
294 case AMDGPU_MES_PRIORITY_LEVEL_NORMAL:
295 default:
296 return AMD_PRIORITY_LEVEL_NORMAL;
297 case AMDGPU_MES_PRIORITY_LEVEL_MEDIUM:
298 return AMD_PRIORITY_LEVEL_MEDIUM;
299 case AMDGPU_MES_PRIORITY_LEVEL_HIGH:
300 return AMD_PRIORITY_LEVEL_HIGH;
301 case AMDGPU_MES_PRIORITY_LEVEL_REALTIME:
302 return AMD_PRIORITY_LEVEL_REALTIME;
303 }
304 }
305
mes_v12_0_add_hw_queue(struct amdgpu_mes * mes,struct mes_add_queue_input * input)306 static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,
307 struct mes_add_queue_input *input)
308 {
309 struct amdgpu_device *adev = mes->adev;
310 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
311 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
312 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
313
314 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
315
316 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
317 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
318 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
319
320 mes_add_queue_pkt.process_id = input->process_id;
321 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
322 mes_add_queue_pkt.process_va_start = input->process_va_start;
323 mes_add_queue_pkt.process_va_end = input->process_va_end;
324 mes_add_queue_pkt.process_quantum = input->process_quantum;
325 mes_add_queue_pkt.process_context_addr = input->process_context_addr;
326 mes_add_queue_pkt.gang_quantum = input->gang_quantum;
327 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
328 mes_add_queue_pkt.inprocess_gang_priority =
329 convert_to_mes_priority_level(input->inprocess_gang_priority);
330 mes_add_queue_pkt.gang_global_priority_level =
331 convert_to_mes_priority_level(input->gang_global_priority_level);
332 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
333 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
334
335 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
336
337 mes_add_queue_pkt.queue_type =
338 convert_to_mes_queue_type(input->queue_type);
339 mes_add_queue_pkt.paging = input->paging;
340 mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
341 mes_add_queue_pkt.gws_base = input->gws_base;
342 mes_add_queue_pkt.gws_size = input->gws_size;
343 mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
344 mes_add_queue_pkt.tma_addr = input->tma_addr;
345 mes_add_queue_pkt.trap_en = input->trap_en;
346 mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
347 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
348
349 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
350 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
351 mes_add_queue_pkt.gds_size = input->queue_size;
352
353 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
354 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
355 mes_add_queue_pkt.gds_size = input->queue_size;
356
357 return mes_v12_0_submit_pkt_and_poll_completion(mes,
358 AMDGPU_MES_SCHED_PIPE,
359 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
360 offsetof(union MESAPI__ADD_QUEUE, api_status));
361 }
362
mes_v12_0_remove_hw_queue(struct amdgpu_mes * mes,struct mes_remove_queue_input * input)363 static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,
364 struct mes_remove_queue_input *input)
365 {
366 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
367 uint32_t mes_rev = mes->sched_version & AMDGPU_MES_VERSION_MASK;
368
369 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
370
371 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
372 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
373 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
374
375 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
376 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
377 mes_remove_queue_pkt.queue_type =
378 convert_to_mes_queue_type(input->queue_type);
379
380 if (mes_rev >= 0x5a)
381 mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset;
382
383 return mes_v12_0_submit_pkt_and_poll_completion(mes,
384 AMDGPU_MES_SCHED_PIPE,
385 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
386 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
387 }
388
gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device * adev,bool req)389 int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev,
390 bool req)
391 {
392 u32 i, tmp, val;
393
394 for (i = 0; i < adev->usec_timeout; i++) {
395 /* Request with MeId=2, PipeId=0 */
396 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
397 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
398 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
399
400 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
401 if (req) {
402 if (val == tmp)
403 break;
404 } else {
405 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
406 REQUEST, 1);
407
408 /* unlocked or locked by firmware */
409 if (val != tmp)
410 break;
411 }
412 udelay(1);
413 }
414
415 if (i >= adev->usec_timeout)
416 return -EINVAL;
417
418 return 0;
419 }
420
mes_v12_0_pipe_reset_support(struct amdgpu_device * adev)421 static bool mes_v12_0_pipe_reset_support(struct amdgpu_device *adev)
422 {
423 /* Disable the pipe reset until the CPFW fully support it.*/
424 dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
425 return false;
426 }
427
mes_v12_0_reset_gfx_pipe_mmio(struct amdgpu_device * adev,u32 me,u32 pipe,u32 queue)428 static int mes_v12_0_reset_gfx_pipe_mmio(struct amdgpu_device *adev,
429 u32 me, u32 pipe, u32 queue)
430 {
431 uint32_t reset_pipe = 0, clean_pipe = 0;
432 int r;
433
434 if (!mes_v12_0_pipe_reset_support(adev))
435 return -EOPNOTSUPP;
436
437 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
438 mutex_lock(&adev->srbm_mutex);
439 soc24_grbm_select(adev, me, pipe, queue, 0);
440
441 switch (pipe) {
442 case 0:
443 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
444 PFP_PIPE0_RESET, 1);
445 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
446 ME_PIPE0_RESET, 1);
447 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
448 PFP_PIPE0_RESET, 0);
449 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
450 ME_PIPE0_RESET, 0);
451 break;
452 case 1:
453 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
454 PFP_PIPE1_RESET, 1);
455 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
456 ME_PIPE1_RESET, 1);
457 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
458 PFP_PIPE1_RESET, 0);
459 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
460 ME_PIPE1_RESET, 0);
461 break;
462 default:
463 break;
464 }
465
466 WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
467 WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
468
469 r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
470 RS64_FW_UC_START_ADDR_LO;
471 soc24_grbm_select(adev, 0, 0, 0, 0);
472 mutex_unlock(&adev->srbm_mutex);
473 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
474
475 dev_info(adev->dev, "The gfx pipe reset: %s\n",
476 r == 0 ? "successfully" : "failed");
477 /* Sometimes the ME start pc counter can't cache correctly, so the
478 * PC check only as a reference and pipe reset result rely on the
479 * later ring test.
480 */
481 return 0;
482 }
483
484 /*
485 * With MEC pipe reset asserted, clear CP_HQD_ACTIVE / CP_HQD_DEQUEUE_REQUEST for
486 * every queue on (me, pipe). HQDs must be torn down while pipe reset stays
487 * asserted; only then clear the pipe reset bit.
488 * Caller must hold adev->srbm_mutex.
489 */
mes_v12_0_clear_hqds_on_mec_pipe(struct amdgpu_device * adev,u32 me,u32 pipe)490 static void mes_v12_0_clear_hqds_on_mec_pipe(struct amdgpu_device *adev, u32 me,
491 u32 pipe)
492 {
493 unsigned int q;
494
495 for (q = 0; q < adev->gfx.mec.num_queue_per_pipe; q++) {
496 soc24_grbm_select(adev, me, pipe, q, 0);
497 /* Start from a clean HQD dequeue state before forcing HQD inactive. */
498 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
499 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0);
500 }
501 }
502
mes_v12_0_reset_compute_pipe_mmio(struct amdgpu_device * adev,u32 me,u32 pipe,u32 queue)503 static int mes_v12_0_reset_compute_pipe_mmio(struct amdgpu_device *adev,
504 u32 me, u32 pipe, u32 queue)
505 {
506 uint32_t reset_val, clean_val;
507 int r = 0;
508
509 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
510 mutex_lock(&adev->srbm_mutex);
511 soc24_grbm_select(adev, me, pipe, queue, 0);
512 if (adev->gfx.rs64_enable) {
513 reset_val = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
514 clean_val = reset_val;
515
516 switch (pipe) {
517 case 0:
518 reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
519 MEC_PIPE0_RESET, 1);
520 clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
521 MEC_PIPE0_RESET, 0);
522 break;
523 case 1:
524 reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
525 MEC_PIPE1_RESET, 1);
526 clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
527 MEC_PIPE1_RESET, 0);
528 break;
529 case 2:
530 reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
531 MEC_PIPE2_RESET, 1);
532 clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
533 MEC_PIPE2_RESET, 0);
534 break;
535 case 3:
536 reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
537 MEC_PIPE3_RESET, 1);
538 clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
539 MEC_PIPE3_RESET, 0);
540 break;
541 default:
542 break;
543 }
544 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_val);
545 mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe);
546 soc24_grbm_select(adev, me, pipe, queue, 0);
547 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_val);
548 r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
549 RS64_FW_UC_START_ADDR_LO;
550 } else {
551 reset_val = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
552 clean_val = reset_val;
553
554 switch (pipe) {
555 case 0:
556 reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
557 MEC_ME1_PIPE0_RESET, 1);
558 clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
559 MEC_ME1_PIPE0_RESET, 0);
560 break;
561 case 1:
562 reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
563 MEC_ME1_PIPE1_RESET, 1);
564 clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
565 MEC_ME1_PIPE1_RESET, 0);
566 break;
567 default:
568 break;
569 }
570
571 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_val);
572 mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe);
573 soc24_grbm_select(adev, me, pipe, queue, 0);
574 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_val);
575 }
576
577 soc24_grbm_select(adev, 0, 0, 0, 0);
578 mutex_unlock(&adev->srbm_mutex);
579 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
580
581 dev_dbg(adev->dev, "MEC pipe me%u pipe%u queue%u resets to MEC FW start PC: %s\n",
582 me, pipe, queue, r == 0 ? "successfully" : "failed");
583 return 0;
584 }
585
mes_v12_0_reset_queue_mmio(struct amdgpu_mes * mes,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t vmid)586 static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
587 uint32_t me_id, uint32_t pipe_id,
588 uint32_t queue_id, uint32_t vmid)
589 {
590 struct amdgpu_device *adev = mes->adev;
591 uint32_t value, reg;
592 int i, r = 0;
593
594 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
595
596 if (queue_type == AMDGPU_RING_TYPE_GFX) {
597 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",
598 me_id, pipe_id, queue_id, vmid);
599
600 mutex_lock(&adev->gfx.reset_sem_mutex);
601 gfx_v12_0_request_gfx_index_mutex(adev, true);
602 /* all se allow writes */
603 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,
604 (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));
605 value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
606 if (pipe_id == 0)
607 value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);
608 else
609 value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);
610 WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);
611 gfx_v12_0_request_gfx_index_mutex(adev, false);
612 mutex_unlock(&adev->gfx.reset_sem_mutex);
613
614 mutex_lock(&adev->srbm_mutex);
615 soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0);
616 /* wait till dequeue take effects */
617 for (i = 0; i < adev->usec_timeout; i++) {
618 if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
619 break;
620 udelay(1);
621 }
622 if (i >= adev->usec_timeout) {
623 dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");
624 r = -ETIMEDOUT;
625 }
626
627 soc24_grbm_select(adev, 0, 0, 0, 0);
628 mutex_unlock(&adev->srbm_mutex);
629 } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
630 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
631 me_id, pipe_id, queue_id);
632 mutex_lock(&adev->srbm_mutex);
633 soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0);
634 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
635 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
636
637 /* wait till dequeue take effects */
638 for (i = 0; i < adev->usec_timeout; i++) {
639 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
640 break;
641 udelay(1);
642 }
643 if (i >= adev->usec_timeout) {
644 dev_err(adev->dev, "failed to wait on hqd deactivate\n");
645 r = -ETIMEDOUT;
646 }
647 soc24_grbm_select(adev, 0, 0, 0, 0);
648 mutex_unlock(&adev->srbm_mutex);
649 } else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
650 dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n",
651 me_id, pipe_id, queue_id);
652 switch (me_id) {
653 case 1:
654 reg = SOC15_REG_OFFSET(GC, 0, regSDMA1_QUEUE_RESET_REQ);
655 break;
656 case 0:
657 default:
658 reg = SOC15_REG_OFFSET(GC, 0, regSDMA0_QUEUE_RESET_REQ);
659 break;
660 }
661
662 value = 1 << queue_id;
663 WREG32(reg, value);
664 /* wait for queue reset done */
665 for (i = 0; i < adev->usec_timeout; i++) {
666 if (!(RREG32(reg) & value))
667 break;
668 udelay(1);
669 }
670 if (i >= adev->usec_timeout) {
671 dev_err(adev->dev, "failed to wait on sdma queue reset done\n");
672 r = -ETIMEDOUT;
673 }
674 }
675
676 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
677 return r;
678 }
679
mes_v12_0_reset_pipe_mmio(struct amdgpu_mes * mes,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t vmid)680 static int mes_v12_0_reset_pipe_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
681 uint32_t me_id, uint32_t pipe_id,
682 uint32_t queue_id, uint32_t vmid)
683 {
684 struct amdgpu_device *adev = mes->adev;
685
686 if (queue_type == AMDGPU_RING_TYPE_GFX)
687 return mes_v12_0_reset_gfx_pipe_mmio(adev, me_id, pipe_id, queue_id);
688 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
689 return mes_v12_0_reset_compute_pipe_mmio(adev, me_id, pipe_id, queue_id);
690 else
691 return -EOPNOTSUPP;
692 }
693
mes_v12_0_map_legacy_queue(struct amdgpu_mes * mes,struct mes_map_legacy_queue_input * input)694 static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
695 struct mes_map_legacy_queue_input *input)
696 {
697 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
698 int pipe;
699
700 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
701
702 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
703 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
704 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
705
706 mes_add_queue_pkt.pipe_id = input->pipe_id;
707 mes_add_queue_pkt.queue_id = input->queue_id;
708 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
709 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
710 mes_add_queue_pkt.wptr_addr = input->wptr_addr;
711 mes_add_queue_pkt.queue_type =
712 convert_to_mes_queue_type(input->queue_type);
713 mes_add_queue_pkt.map_legacy_kq = 1;
714
715 if (mes->adev->enable_uni_mes) {
716 /* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
717 if (input->queue_type == AMDGPU_RING_TYPE_MES)
718 pipe = AMDGPU_MES_KIQ_PIPE;
719 else
720 pipe = AMDGPU_MES_SCHED_PIPE;
721 } else {
722 pipe = AMDGPU_MES_SCHED_PIPE;
723 }
724
725 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
726 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
727 offsetof(union MESAPI__ADD_QUEUE, api_status));
728 }
729
mes_v12_0_unmap_legacy_queue(struct amdgpu_mes * mes,struct mes_unmap_legacy_queue_input * input)730 static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
731 struct mes_unmap_legacy_queue_input *input)
732 {
733 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
734 int pipe;
735
736 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
737
738 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
739 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
740 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
741
742 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
743 mes_remove_queue_pkt.gang_context_addr = 0;
744
745 mes_remove_queue_pkt.pipe_id = input->pipe_id;
746 mes_remove_queue_pkt.queue_id = input->queue_id;
747
748 if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
749 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
750 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
751 mes_remove_queue_pkt.tf_data =
752 lower_32_bits(input->trail_fence_data);
753 } else {
754 mes_remove_queue_pkt.unmap_legacy_queue = 1;
755 mes_remove_queue_pkt.queue_type =
756 convert_to_mes_queue_type(input->queue_type);
757 /*
758 * A reset-time unmap: the queue was already reset via MMIO while
759 * gangs are suspended and it is on the MES hung/fail list. Tell
760 * MES to just drop its internal state for it. Without this flag
761 * MES asks CP to unmap the already-reset (still wedged) queue
762 * again, which times out and forces a GPU reset.
763 */
764 if (input->action == RESET_QUEUES &&
765 (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x5a)
766 mes_remove_queue_pkt.remove_queue_after_reset = 1;
767
768 }
769
770 if (mes->adev->enable_uni_mes) {
771 /* Keep scheduler queue on KIQ pipe; unmap all other kernel queues on sched pipe. */
772 if (input->queue_type == AMDGPU_RING_TYPE_MES)
773 pipe = AMDGPU_MES_KIQ_PIPE;
774 else
775 pipe = AMDGPU_MES_SCHED_PIPE;
776 } else {
777 pipe = AMDGPU_MES_SCHED_PIPE;
778 }
779
780 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
781 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
782 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
783 }
784
mes_v12_0_suspend_gang(struct amdgpu_mes * mes,struct mes_suspend_gang_input * input)785 static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes,
786 struct mes_suspend_gang_input *input)
787 {
788 union MESAPI__SUSPEND mes_suspend_gang_pkt;
789
790 memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
791
792 mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
793 mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
794 mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
795
796 mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
797 mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
798 mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
799 mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
800 mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset;
801
802 return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
803 &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
804 offsetof(union MESAPI__SUSPEND, api_status));
805 }
806
mes_v12_0_resume_gang(struct amdgpu_mes * mes,struct mes_resume_gang_input * input)807 static int mes_v12_0_resume_gang(struct amdgpu_mes *mes,
808 struct mes_resume_gang_input *input)
809 {
810 union MESAPI__RESUME mes_resume_gang_pkt;
811
812 memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
813
814 mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
815 mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
816 mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
817
818 mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
819 mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
820 mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset;
821
822 return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
823 &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
824 offsetof(union MESAPI__RESUME, api_status));
825 }
826
mes_v12_0_query_sched_status(struct amdgpu_mes * mes,int pipe)827 static int mes_v12_0_query_sched_status(struct amdgpu_mes *mes, int pipe)
828 {
829 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
830
831 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
832
833 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
834 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
835 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
836
837 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
838 &mes_status_pkt, sizeof(mes_status_pkt),
839 offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
840 }
841
mes_v12_0_misc_op(struct amdgpu_mes * mes,struct mes_misc_op_input * input)842 static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
843 struct mes_misc_op_input *input)
844 {
845 union MESAPI__MISC misc_pkt;
846 int pipe;
847
848 if (mes->adev->enable_uni_mes)
849 pipe = AMDGPU_MES_KIQ_PIPE;
850 else
851 pipe = AMDGPU_MES_SCHED_PIPE;
852
853 memset(&misc_pkt, 0, sizeof(misc_pkt));
854
855 misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
856 misc_pkt.header.opcode = MES_SCH_API_MISC;
857 misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
858
859 switch (input->op) {
860 case MES_MISC_OP_READ_REG:
861 misc_pkt.opcode = MESAPI_MISC__READ_REG;
862 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
863 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
864 break;
865 case MES_MISC_OP_WRITE_REG:
866 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
867 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
868 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
869 break;
870 case MES_MISC_OP_WRM_REG_WAIT:
871 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
872 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
873 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
874 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
875 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
876 misc_pkt.wait_reg_mem.reg_offset2 = 0;
877 break;
878 case MES_MISC_OP_WRM_REG_WR_WAIT:
879 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
880 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
881 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
882 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
883 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
884 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
885 break;
886 case MES_MISC_OP_SET_SHADER_DEBUGGER:
887 pipe = AMDGPU_MES_SCHED_PIPE;
888 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
889 misc_pkt.set_shader_debugger.process_context_addr =
890 input->set_shader_debugger.process_context_addr;
891 misc_pkt.set_shader_debugger.flags.u32all =
892 input->set_shader_debugger.flags.u32all;
893 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
894 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
895 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
896 input->set_shader_debugger.tcp_watch_cntl,
897 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
898 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
899 break;
900 case MES_MISC_OP_CHANGE_CONFIG:
901 misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
902 misc_pkt.change_config.opcode =
903 MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
904 misc_pkt.change_config.option.bits.limit_single_process =
905 input->change_config.option.limit_single_process;
906 break;
907
908 default:
909 DRM_ERROR("unsupported misc op (%d)\n", input->op);
910 return -EINVAL;
911 }
912
913 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
914 &misc_pkt, sizeof(misc_pkt),
915 offsetof(union MESAPI__MISC, api_status));
916 }
917
mes_v12_0_set_hw_resources_1(struct amdgpu_mes * mes,int pipe)918 static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
919 {
920 union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
921
922 memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
923
924 mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
925 mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
926 mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
927 mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 0xa;
928 mes_set_hw_res_1_pkt.cleaner_shader_fence_mc_addr =
929 mes->resource_1_gpu_addr[pipe];
930
931 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
932 &mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
933 offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
934 }
935
mes_v12_0_set_hw_resources(struct amdgpu_mes * mes,int pipe)936 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
937 {
938 int i;
939 struct amdgpu_device *adev = mes->adev;
940 union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
941 uint32_t mes_rev = (pipe == AMDGPU_MES_SCHED_PIPE) ?
942 (mes->sched_version & AMDGPU_MES_VERSION_MASK) :
943 (mes->kiq_version & AMDGPU_MES_VERSION_MASK);
944
945 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
946
947 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
948 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
949 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
950
951 if (pipe == AMDGPU_MES_SCHED_PIPE) {
952 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
953 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
954 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
955 mes_set_hw_res_pkt.paging_vmid = 0;
956
957 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
958 mes_set_hw_res_pkt.compute_hqd_mask[i] =
959 mes->compute_hqd_mask[i];
960
961 for (i = 0; i < MAX_GFX_PIPES; i++)
962 mes_set_hw_res_pkt.gfx_hqd_mask[i] =
963 mes->gfx_hqd_mask[i];
964
965 for (i = 0; i < MAX_SDMA_PIPES; i++)
966 mes_set_hw_res_pkt.sdma_hqd_mask[i] =
967 mes->sdma_hqd_mask[i];
968
969 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
970 mes_set_hw_res_pkt.aggregated_doorbells[i] =
971 mes->aggregated_doorbells[i];
972 }
973
974 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
975 mes->sch_ctx_gpu_addr[pipe];
976 mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
977 mes->query_status_fence_gpu_addr[pipe];
978
979 for (i = 0; i < 5; i++) {
980 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
981 mes_set_hw_res_pkt.mmhub_base[i] =
982 adev->reg_offset[MMHUB_HWIP][0][i];
983 mes_set_hw_res_pkt.osssys_base[i] =
984 adev->reg_offset[OSSSYS_HWIP][0][i];
985 }
986
987 mes_set_hw_res_pkt.disable_reset = 1;
988 mes_set_hw_res_pkt.disable_mes_log = 1;
989 mes_set_hw_res_pkt.use_different_vmid_compute = 1;
990 mes_set_hw_res_pkt.enable_reg_active_poll = 1;
991 mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
992
993 /*
994 * Keep oversubscribe timer for sdma . When we have unmapped doorbell
995 * handling support, other queue will not use the oversubscribe timer.
996 * handling mode - 0: disabled; 1: basic; 2: basic+; 3: basic++
997 *
998 * Use basic+ (2): on an unmapped-queue doorbell ring MES reads the
999 * per-queue CP_UNMAPPED_QUEUE bitmap and clears the ringing queue's
1000 * work_done so it is rescheduled. basic (1) only sets a coarse
1001 * level-wide ready flag without clearing work_done, so a queue that
1002 * rings after work_done was set is skipped and its work is lost.
1003 */
1004 mes_set_hw_res_pkt.oversubscription_timer = mes_rev < 0x8b ? 0 : 50;
1005 mes_set_hw_res_pkt.unmapped_doorbell_handling = 2;
1006
1007 if (amdgpu_mes_log_enable) {
1008 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
1009 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr +
1010 pipe * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
1011 }
1012
1013 if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
1014 mes_set_hw_res_pkt.limit_single_process = 1;
1015
1016 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
1017 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
1018 offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
1019 }
1020
mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes * mes)1021 static void mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
1022 {
1023 struct amdgpu_device *adev = mes->adev;
1024 uint32_t data;
1025
1026 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
1027 data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
1028 CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
1029 CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
1030 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
1031 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
1032 data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
1033 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
1034
1035 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
1036 data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
1037 CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
1038 CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
1039 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
1040 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
1041 data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
1042 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
1043
1044 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
1045 data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
1046 CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
1047 CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
1048 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
1049 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
1050 data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
1051 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
1052
1053 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
1054 data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
1055 CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
1056 CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
1057 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
1058 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
1059 data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
1060 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
1061
1062 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
1063 data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
1064 CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
1065 CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
1066 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
1067 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
1068 data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
1069 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
1070
1071 data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
1072 WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
1073 }
1074
1075
mes_v12_0_enable_unmapped_doorbell_handling(struct amdgpu_mes * mes,bool enable)1076 static void mes_v12_0_enable_unmapped_doorbell_handling(
1077 struct amdgpu_mes *mes, bool enable)
1078 {
1079 struct amdgpu_device *adev = mes->adev;
1080 uint32_t data = RREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL);
1081
1082 /*
1083 * The default PROC_LSB settng is 0xc which means doorbell
1084 * addr[16:12] gives the doorbell page number. For kfd, each
1085 * process will use 2 pages of doorbell, we need to change the
1086 * setting to 0xd
1087 */
1088 data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
1089 data |= 0xd << CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
1090
1091 data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
1092
1093 WREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL, data);
1094 }
1095
mes_v12_0_reset_hw_queue(struct amdgpu_mes * mes,struct mes_reset_queue_input * input)1096 static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
1097 struct mes_reset_queue_input *input)
1098 {
1099 union MESAPI__RESET mes_reset_queue_pkt;
1100 int pipe;
1101
1102 if (input->use_mmio) {
1103 int r = mes_v12_0_reset_queue_mmio(mes, input->queue_type,
1104 input->me_id, input->pipe_id,
1105 input->queue_id, input->vmid);
1106 if (r)
1107 return mes_v12_0_reset_pipe_mmio(mes, input->queue_type,
1108 input->me_id, input->pipe_id,
1109 input->queue_id, input->vmid);
1110 return 0;
1111 }
1112
1113 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
1114
1115 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
1116 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
1117 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
1118
1119 mes_reset_queue_pkt.queue_type =
1120 convert_to_mes_queue_type(input->queue_type);
1121
1122 if (input->legacy_gfx) {
1123 mes_reset_queue_pkt.reset_legacy_gfx = 1;
1124 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
1125 mes_reset_queue_pkt.queue_id_lp = input->queue_id;
1126 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
1127 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
1128 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
1129 mes_reset_queue_pkt.vmid_id_lp = input->vmid;
1130 } else {
1131 mes_reset_queue_pkt.reset_queue_only = 1;
1132 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
1133 }
1134
1135 pipe = AMDGPU_MES_SCHED_PIPE;
1136
1137 return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
1138 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
1139 offsetof(union MESAPI__RESET, api_status));
1140 }
1141
mes_v12_0_detect_and_reset_hung_queues(struct amdgpu_mes * mes,struct mes_detect_and_reset_queue_input * input)1142 static int mes_v12_0_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
1143 struct mes_detect_and_reset_queue_input *input)
1144 {
1145 union MESAPI__RESET mes_reset_queue_pkt;
1146
1147 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
1148
1149 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
1150 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
1151 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
1152
1153 mes_reset_queue_pkt.queue_type =
1154 convert_to_mes_queue_type(input->queue_type);
1155 mes_reset_queue_pkt.doorbell_offset_addr =
1156 mes->hung_queue_db_array_gpu_addr[0];
1157
1158 if (input->detect_only)
1159 mes_reset_queue_pkt.hang_detect_only = 1;
1160 else
1161 mes_reset_queue_pkt.hang_detect_then_reset = 1;
1162
1163 return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
1164 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
1165 offsetof(union MESAPI__RESET, api_status));
1166 }
1167
mes_v12_inv_tlb_convert_hub_id(uint8_t id)1168 static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
1169 {
1170 /*
1171 * MES doesn't support invalidate gc_hub on slave xcc individually
1172 * master xcc will invalidate all gc_hub for the partition
1173 */
1174 if (AMDGPU_IS_GFXHUB(id))
1175 return 0;
1176 else if (AMDGPU_IS_MMHUB0(id))
1177 return 1;
1178 else
1179 return -EINVAL;
1180
1181 }
1182
mes_v12_0_inv_tlbs_pasid(struct amdgpu_mes * mes,struct mes_inv_tlbs_pasid_input * input)1183 static int mes_v12_0_inv_tlbs_pasid(struct amdgpu_mes *mes,
1184 struct mes_inv_tlbs_pasid_input *input)
1185 {
1186 union MESAPI__INV_TLBS mes_inv_tlbs;
1187 int ret;
1188
1189 memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
1190
1191 mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
1192 mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
1193 mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
1194
1195 mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
1196 mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
1197 mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
1198
1199 /*convert amdgpu_mes_hub_id to mes expected hub_id */
1200 ret = mes_v12_inv_tlb_convert_hub_id(input->hub_id);
1201 if (ret < 0)
1202 return -EINVAL;
1203 mes_inv_tlbs.invalidate_tlbs.hub_id = ret;
1204 return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_KIQ_PIPE,
1205 &mes_inv_tlbs, sizeof(mes_inv_tlbs),
1206 offsetof(union MESAPI__INV_TLBS, api_status));
1207
1208 }
1209
1210 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
1211 .add_hw_queue = mes_v12_0_add_hw_queue,
1212 .remove_hw_queue = mes_v12_0_remove_hw_queue,
1213 .map_legacy_queue = mes_v12_0_map_legacy_queue,
1214 .unmap_legacy_queue = mes_v12_0_unmap_legacy_queue,
1215 .suspend_gang = mes_v12_0_suspend_gang,
1216 .resume_gang = mes_v12_0_resume_gang,
1217 .misc_op = mes_v12_0_misc_op,
1218 .reset_hw_queue = mes_v12_0_reset_hw_queue,
1219 .invalidate_tlbs_pasid = mes_v12_0_inv_tlbs_pasid,
1220 .detect_and_reset_hung_queues = mes_v12_0_detect_and_reset_hung_queues,
1221 };
1222
mes_v12_0_allocate_ucode_buffer(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1223 static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
1224 enum amdgpu_mes_pipe pipe)
1225 {
1226 int r;
1227 const struct mes_firmware_header_v1_0 *mes_hdr;
1228 const __le32 *fw_data;
1229 unsigned fw_size;
1230
1231 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1232 adev->mes.fw[pipe]->data;
1233
1234 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1235 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1236 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1237
1238 r = amdgpu_bo_create_reserved(adev, fw_size,
1239 PAGE_SIZE,
1240 AMDGPU_GEM_DOMAIN_VRAM,
1241 &adev->mes.ucode_fw_obj[pipe],
1242 &adev->mes.ucode_fw_gpu_addr[pipe],
1243 (void **)&adev->mes.ucode_fw_ptr[pipe]);
1244 if (r) {
1245 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
1246 return r;
1247 }
1248
1249 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
1250
1251 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
1252 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
1253
1254 return 0;
1255 }
1256
mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1257 static int mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
1258 enum amdgpu_mes_pipe pipe)
1259 {
1260 int r;
1261 const struct mes_firmware_header_v1_0 *mes_hdr;
1262 const __le32 *fw_data;
1263 unsigned fw_size;
1264
1265 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1266 adev->mes.fw[pipe]->data;
1267
1268 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1269 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1270 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1271
1272 r = amdgpu_bo_create_reserved(adev, fw_size,
1273 64 * 1024,
1274 AMDGPU_GEM_DOMAIN_VRAM,
1275 &adev->mes.data_fw_obj[pipe],
1276 &adev->mes.data_fw_gpu_addr[pipe],
1277 (void **)&adev->mes.data_fw_ptr[pipe]);
1278 if (r) {
1279 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
1280 return r;
1281 }
1282
1283 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
1284
1285 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
1286 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
1287
1288 return 0;
1289 }
1290
mes_v12_0_free_ucode_buffers(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1291 static void mes_v12_0_free_ucode_buffers(struct amdgpu_device *adev,
1292 enum amdgpu_mes_pipe pipe)
1293 {
1294 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
1295 &adev->mes.data_fw_gpu_addr[pipe],
1296 (void **)&adev->mes.data_fw_ptr[pipe]);
1297
1298 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
1299 &adev->mes.ucode_fw_gpu_addr[pipe],
1300 (void **)&adev->mes.ucode_fw_ptr[pipe]);
1301 }
1302
mes_v12_0_enable(struct amdgpu_device * adev,bool enable)1303 static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
1304 {
1305 uint64_t ucode_addr;
1306 uint32_t pipe, data = 0;
1307
1308 if (enable) {
1309 mutex_lock(&adev->srbm_mutex);
1310 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1311 soc24_grbm_select(adev, 3, pipe, 0, 0);
1312 if (amdgpu_mes_log_enable) {
1313 u32 log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE;
1314 /* In case uni mes is not enabled, only program for pipe 0 */
1315 if (adev->mes.event_log_size >= (pipe + 1) * log_size) {
1316 WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO,
1317 lower_32_bits(adev->mes.event_log_gpu_addr +
1318 pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
1319 WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI,
1320 upper_32_bits(adev->mes.event_log_gpu_addr +
1321 pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
1322 dev_info(adev->dev, "Setup CP MES MSCRATCH address : 0x%x. 0x%x\n",
1323 RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI),
1324 RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO));
1325 }
1326 }
1327
1328 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1329 if (pipe == 0)
1330 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1331 else
1332 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1333 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1334
1335 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1336 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1337 lower_32_bits(ucode_addr));
1338 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1339 upper_32_bits(ucode_addr));
1340
1341 /* unhalt MES and activate one pipe each loop */
1342 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
1343 if (pipe)
1344 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
1345 dev_info(adev->dev, "program CP_MES_CNTL : 0x%x\n", data);
1346
1347 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1348
1349 }
1350 soc24_grbm_select(adev, 0, 0, 0, 0);
1351 mutex_unlock(&adev->srbm_mutex);
1352
1353 if (amdgpu_emu_mode)
1354 msleep(100);
1355 else if (adev->enable_uni_mes)
1356 udelay(500);
1357 else
1358 udelay(50);
1359 } else {
1360 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1361 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
1362 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
1363 data = REG_SET_FIELD(data, CP_MES_CNTL,
1364 MES_INVALIDATE_ICACHE, 1);
1365 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1366 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1367 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
1368 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1369 }
1370 }
1371
mes_v12_0_set_ucode_start_addr(struct amdgpu_device * adev)1372 static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
1373 {
1374 uint64_t ucode_addr;
1375 int pipe;
1376
1377 mes_v12_0_enable(adev, false);
1378
1379 mutex_lock(&adev->srbm_mutex);
1380 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1381 /* me=3, queue=0 */
1382 soc24_grbm_select(adev, 3, pipe, 0, 0);
1383
1384 /* set ucode start address */
1385 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1386 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1387 lower_32_bits(ucode_addr));
1388 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1389 upper_32_bits(ucode_addr));
1390
1391 soc24_grbm_select(adev, 0, 0, 0, 0);
1392 }
1393 mutex_unlock(&adev->srbm_mutex);
1394 }
1395
1396 /* This function is for backdoor MES firmware */
mes_v12_0_load_microcode(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,bool prime_icache)1397 static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
1398 enum amdgpu_mes_pipe pipe, bool prime_icache)
1399 {
1400 int r;
1401 uint32_t data;
1402
1403 mes_v12_0_enable(adev, false);
1404
1405 if (!adev->mes.fw[pipe])
1406 return -EINVAL;
1407
1408 r = mes_v12_0_allocate_ucode_buffer(adev, pipe);
1409 if (r)
1410 return r;
1411
1412 r = mes_v12_0_allocate_ucode_data_buffer(adev, pipe);
1413 if (r) {
1414 mes_v12_0_free_ucode_buffers(adev, pipe);
1415 return r;
1416 }
1417
1418 mutex_lock(&adev->srbm_mutex);
1419 /* me=3, pipe=0, queue=0 */
1420 soc24_grbm_select(adev, 3, pipe, 0, 0);
1421
1422 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
1423
1424 /* set ucode fimrware address */
1425 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
1426 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1427 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
1428 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1429
1430 /* set ucode instruction cache boundary to 2M-1 */
1431 WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
1432
1433 /* set ucode data firmware address */
1434 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
1435 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1436 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
1437 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1438
1439 /* Set data cache boundary CP_MES_MDBOUND_LO */
1440 WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);
1441
1442 if (prime_icache) {
1443 /* invalidate ICACHE */
1444 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1445 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
1446 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1447 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1448
1449 /* prime the ICACHE. */
1450 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1451 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
1452 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1453 }
1454
1455 soc24_grbm_select(adev, 0, 0, 0, 0);
1456 mutex_unlock(&adev->srbm_mutex);
1457
1458 return 0;
1459 }
1460
mes_v12_0_allocate_eop_buf(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1461 static int mes_v12_0_allocate_eop_buf(struct amdgpu_device *adev,
1462 enum amdgpu_mes_pipe pipe)
1463 {
1464 int r;
1465 u32 *eop;
1466
1467 r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1468 AMDGPU_GEM_DOMAIN_GTT,
1469 &adev->mes.eop_gpu_obj[pipe],
1470 &adev->mes.eop_gpu_addr[pipe],
1471 (void **)&eop);
1472 if (r) {
1473 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1474 return r;
1475 }
1476
1477 memset(eop, 0,
1478 adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
1479
1480 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
1481 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
1482
1483 return 0;
1484 }
1485
mes_v12_0_mqd_init(struct amdgpu_ring * ring)1486 static int mes_v12_0_mqd_init(struct amdgpu_ring *ring)
1487 {
1488 struct v12_compute_mqd *mqd = ring->mqd_ptr;
1489 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1490 uint32_t tmp;
1491
1492 mqd->header = 0xC0310800;
1493 mqd->compute_pipelinestat_enable = 0x00000001;
1494 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1495 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1496 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1497 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1498 mqd->compute_misc_reserved = 0x00000007;
1499
1500 eop_base_addr = ring->eop_gpu_addr >> 8;
1501
1502 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1503 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1504 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1505 (order_base_2(MES_EOP_SIZE / 4) - 1));
1506
1507 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1508 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1509 mqd->cp_hqd_eop_control = tmp;
1510
1511 /* disable the queue if it's active */
1512 ring->wptr = 0;
1513 mqd->cp_hqd_pq_rptr = 0;
1514 mqd->cp_hqd_pq_wptr_lo = 0;
1515 mqd->cp_hqd_pq_wptr_hi = 0;
1516
1517 /* set the pointer to the MQD */
1518 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1519 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1520
1521 /* set MQD vmid to 0 */
1522 tmp = regCP_MQD_CONTROL_DEFAULT;
1523 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1524 mqd->cp_mqd_control = tmp;
1525
1526 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1527 hqd_gpu_addr = ring->gpu_addr >> 8;
1528 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1529 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1530
1531 /* set the wb address whether it's enabled or not */
1532 wb_gpu_addr = ring->rptr_gpu_addr;
1533 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1534 mqd->cp_hqd_pq_rptr_report_addr_hi =
1535 upper_32_bits(wb_gpu_addr) & 0xffff;
1536
1537 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1538 wb_gpu_addr = ring->wptr_gpu_addr;
1539 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1540 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1541
1542 /* set up the HQD, this is similar to CP_RB0_CNTL */
1543 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1544 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1545 (order_base_2(ring->ring_size / 4) - 1));
1546 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1547 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1548 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1549 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1550 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1551 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1552 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1553 mqd->cp_hqd_pq_control = tmp;
1554
1555 /* enable doorbell */
1556 tmp = 0;
1557 if (ring->use_doorbell) {
1558 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1559 DOORBELL_OFFSET, ring->doorbell_index);
1560 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1561 DOORBELL_EN, 1);
1562 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1563 DOORBELL_SOURCE, 0);
1564 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1565 DOORBELL_HIT, 0);
1566 } else {
1567 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1568 DOORBELL_EN, 0);
1569 }
1570 mqd->cp_hqd_pq_doorbell_control = tmp;
1571
1572 mqd->cp_hqd_vmid = 0;
1573 /* activate the queue */
1574 mqd->cp_hqd_active = 1;
1575
1576 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1577 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1578 PRELOAD_SIZE, 0x55);
1579 mqd->cp_hqd_persistent_state = tmp;
1580
1581 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
1582 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1583 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1584
1585 /*
1586 * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
1587 * doorbell handling. This is a reserved CP internal register can
1588 * not be accesss by others
1589 */
1590 mqd->reserved_184 = BIT(15);
1591
1592 return 0;
1593 }
1594
mes_v12_0_queue_init_register(struct amdgpu_ring * ring)1595 static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
1596 {
1597 struct v12_compute_mqd *mqd = ring->mqd_ptr;
1598 struct amdgpu_device *adev = ring->adev;
1599 uint32_t data = 0;
1600
1601 mutex_lock(&adev->srbm_mutex);
1602 soc24_grbm_select(adev, 3, ring->pipe, 0, 0);
1603
1604 /* set CP_HQD_VMID.VMID = 0. */
1605 data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
1606 data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1607 WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
1608
1609 /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1610 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1611 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1612 DOORBELL_EN, 0);
1613 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1614
1615 /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1616 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1617 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1618
1619 /* set CP_MQD_CONTROL.VMID=0 */
1620 data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
1621 data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1622 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
1623
1624 /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1625 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1626 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1627
1628 /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1629 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
1630 mqd->cp_hqd_pq_rptr_report_addr_lo);
1631 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1632 mqd->cp_hqd_pq_rptr_report_addr_hi);
1633
1634 /* set CP_HQD_PQ_CONTROL */
1635 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1636
1637 /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1638 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
1639 mqd->cp_hqd_pq_wptr_poll_addr_lo);
1640 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1641 mqd->cp_hqd_pq_wptr_poll_addr_hi);
1642
1643 /* set CP_HQD_PQ_DOORBELL_CONTROL */
1644 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
1645 mqd->cp_hqd_pq_doorbell_control);
1646
1647 /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1648 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1649
1650 /* set CP_HQD_ACTIVE.ACTIVE=1 */
1651 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1652
1653 soc24_grbm_select(adev, 0, 0, 0, 0);
1654 mutex_unlock(&adev->srbm_mutex);
1655 }
1656
mes_v12_0_kiq_enable_queue(struct amdgpu_device * adev)1657 static int mes_v12_0_kiq_enable_queue(struct amdgpu_device *adev)
1658 {
1659 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1660 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
1661 int r;
1662
1663 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1664 return -EINVAL;
1665
1666 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1667 if (r) {
1668 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1669 return r;
1670 }
1671
1672 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);
1673
1674 r = amdgpu_ring_test_ring(kiq_ring);
1675 if (r) {
1676 DRM_ERROR("kfq enable failed\n");
1677 kiq_ring->sched.ready = false;
1678 }
1679 return r;
1680 }
1681
mes_v12_0_queue_init(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1682 static int mes_v12_0_queue_init(struct amdgpu_device *adev,
1683 enum amdgpu_mes_pipe pipe)
1684 {
1685 struct amdgpu_ring *ring;
1686 int r;
1687
1688 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1689 ring = &adev->gfx.kiq[0].ring;
1690 else
1691 ring = &adev->mes.ring[pipe];
1692
1693 if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
1694 (amdgpu_in_reset(adev) || adev->in_suspend)) {
1695 *(ring->wptr_cpu_addr) = 0;
1696 *(ring->rptr_cpu_addr) = 0;
1697 amdgpu_ring_clear_ring(ring);
1698 }
1699
1700 r = mes_v12_0_mqd_init(ring);
1701 if (r)
1702 return r;
1703
1704 if (pipe == AMDGPU_MES_SCHED_PIPE) {
1705 if (adev->enable_uni_mes)
1706 r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
1707 else
1708 r = mes_v12_0_kiq_enable_queue(adev);
1709 if (r)
1710 return r;
1711 } else {
1712 mes_v12_0_queue_init_register(ring);
1713 }
1714
1715 if (((pipe == AMDGPU_MES_SCHED_PIPE) && !adev->mes.sched_version) ||
1716 ((pipe == AMDGPU_MES_KIQ_PIPE) && !adev->mes.kiq_version)) {
1717 /* get MES scheduler/KIQ versions */
1718 mutex_lock(&adev->srbm_mutex);
1719 soc24_grbm_select(adev, 3, pipe, 0, 0);
1720
1721 if (pipe == AMDGPU_MES_SCHED_PIPE)
1722 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1723 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
1724 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1725
1726 soc24_grbm_select(adev, 0, 0, 0, 0);
1727 mutex_unlock(&adev->srbm_mutex);
1728 }
1729
1730 return 0;
1731 }
1732
mes_v12_0_ring_init(struct amdgpu_device * adev,int pipe)1733 static int mes_v12_0_ring_init(struct amdgpu_device *adev, int pipe)
1734 {
1735 struct amdgpu_ring *ring;
1736
1737 ring = &adev->mes.ring[pipe];
1738
1739 ring->funcs = &mes_v12_0_ring_funcs;
1740
1741 ring->me = 3;
1742 ring->pipe = pipe;
1743 ring->queue = 0;
1744
1745 ring->ring_obj = NULL;
1746 ring->use_doorbell = true;
1747 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[pipe];
1748 ring->no_scheduler = true;
1749 sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1750
1751 if (pipe == AMDGPU_MES_SCHED_PIPE)
1752 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
1753 else
1754 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1755
1756 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1757 AMDGPU_RING_PRIO_DEFAULT, NULL);
1758 }
1759
mes_v12_0_kiq_ring_init(struct amdgpu_device * adev)1760 static int mes_v12_0_kiq_ring_init(struct amdgpu_device *adev)
1761 {
1762 struct amdgpu_ring *ring;
1763
1764 spin_lock_init(&adev->gfx.kiq[0].ring_lock);
1765
1766 ring = &adev->gfx.kiq[0].ring;
1767
1768 ring->me = 3;
1769 ring->pipe = 1;
1770 ring->queue = 0;
1771
1772 ring->adev = NULL;
1773 ring->ring_obj = NULL;
1774 ring->use_doorbell = true;
1775 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1776 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1777 ring->no_scheduler = true;
1778 sprintf(ring->name, "mes_kiq_%d.%d.%d",
1779 ring->me, ring->pipe, ring->queue);
1780
1781 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1782 AMDGPU_RING_PRIO_DEFAULT, NULL);
1783 }
1784
mes_v12_0_mqd_sw_init(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe)1785 static int mes_v12_0_mqd_sw_init(struct amdgpu_device *adev,
1786 enum amdgpu_mes_pipe pipe)
1787 {
1788 int r, mqd_size = sizeof(struct v12_compute_mqd);
1789 struct amdgpu_ring *ring;
1790
1791 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1792 ring = &adev->gfx.kiq[0].ring;
1793 else
1794 ring = &adev->mes.ring[pipe];
1795
1796 if (ring->mqd_obj)
1797 return 0;
1798
1799 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1800 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1801 &ring->mqd_gpu_addr, &ring->mqd_ptr);
1802 if (r) {
1803 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1804 return r;
1805 }
1806
1807 memset(ring->mqd_ptr, 0, mqd_size);
1808
1809 /* prepare MQD backup */
1810 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1811 if (!adev->mes.mqd_backup[pipe])
1812 dev_warn(adev->dev,
1813 "no memory to create MQD backup for ring %s\n",
1814 ring->name);
1815
1816 return 0;
1817 }
1818
mes_v12_0_sw_init(struct amdgpu_ip_block * ip_block)1819 static int mes_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
1820 {
1821 struct amdgpu_device *adev = ip_block->adev;
1822 int pipe, r;
1823
1824 adev->mes.funcs = &mes_v12_0_funcs;
1825 adev->mes.kiq_hw_init = &mes_v12_0_kiq_hw_init;
1826 adev->mes.kiq_hw_fini = &mes_v12_0_kiq_hw_fini;
1827 adev->mes.enable_legacy_queue_map = true;
1828
1829 adev->mes.event_log_size = adev->enable_uni_mes ?
1830 (AMDGPU_MAX_MES_PIPES * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE)) :
1831 (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
1832 r = amdgpu_mes_init(adev);
1833 if (r)
1834 return r;
1835
1836 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1837 r = mes_v12_0_allocate_eop_buf(adev, pipe);
1838 if (r)
1839 return r;
1840
1841 r = mes_v12_0_mqd_sw_init(adev, pipe);
1842 if (r)
1843 return r;
1844
1845 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE) {
1846 r = mes_v12_0_kiq_ring_init(adev);
1847 }
1848 else {
1849 r = mes_v12_0_ring_init(adev, pipe);
1850 if (r)
1851 return r;
1852 r = amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1853 AMDGPU_GEM_DOMAIN_VRAM,
1854 &adev->mes.resource_1[pipe],
1855 &adev->mes.resource_1_gpu_addr[pipe],
1856 &adev->mes.resource_1_addr[pipe]);
1857 if (r) {
1858 dev_err(adev->dev, "(%d) failed to create mes resource_1 bo pipe[%d]\n", r, pipe);
1859 return r;
1860 }
1861 }
1862 }
1863
1864 return 0;
1865 }
1866
mes_v12_0_sw_fini(struct amdgpu_ip_block * ip_block)1867 static int mes_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
1868 {
1869 struct amdgpu_device *adev = ip_block->adev;
1870 int pipe;
1871
1872 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1873 amdgpu_bo_free_kernel(&adev->mes.resource_1[pipe],
1874 &adev->mes.resource_1_gpu_addr[pipe],
1875 &adev->mes.resource_1_addr[pipe]);
1876
1877 kfree(adev->mes.mqd_backup[pipe]);
1878
1879 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1880 &adev->mes.eop_gpu_addr[pipe],
1881 NULL);
1882 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1883
1884 if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
1885 amdgpu_bo_free_kernel(&adev->mes.ring[pipe].mqd_obj,
1886 &adev->mes.ring[pipe].mqd_gpu_addr,
1887 &adev->mes.ring[pipe].mqd_ptr);
1888 amdgpu_ring_fini(&adev->mes.ring[pipe]);
1889 }
1890 }
1891
1892 if (!adev->enable_uni_mes) {
1893 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,
1894 &adev->gfx.kiq[0].ring.mqd_gpu_addr,
1895 &adev->gfx.kiq[0].ring.mqd_ptr);
1896 amdgpu_ring_fini(&adev->gfx.kiq[0].ring);
1897 }
1898
1899 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1900 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1901 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1902 }
1903
1904 amdgpu_mes_fini(adev);
1905 return 0;
1906 }
1907
mes_v12_0_kiq_dequeue_sched(struct amdgpu_device * adev)1908 static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1909 {
1910 uint32_t data;
1911 int i;
1912
1913 mutex_lock(&adev->srbm_mutex);
1914 soc24_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1915
1916 /* disable the queue if it's active */
1917 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1918 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1919 for (i = 0; i < adev->usec_timeout; i++) {
1920 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1921 break;
1922 udelay(1);
1923 }
1924 }
1925 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1926 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1927 DOORBELL_EN, 0);
1928 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1929 DOORBELL_HIT, 1);
1930 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1931
1932 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1933
1934 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1935 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1936 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1937
1938 soc24_grbm_select(adev, 0, 0, 0, 0);
1939 mutex_unlock(&adev->srbm_mutex);
1940
1941 adev->mes.ring[0].sched.ready = false;
1942 }
1943
mes_v12_0_kiq_setting(struct amdgpu_ring * ring)1944 static void mes_v12_0_kiq_setting(struct amdgpu_ring *ring)
1945 {
1946 uint32_t tmp;
1947 struct amdgpu_device *adev = ring->adev;
1948
1949 /* tell RLC which is KIQ queue */
1950 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1951 tmp &= 0xffffff00;
1952 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1953 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
1954 }
1955
mes_v12_0_kiq_hw_init(struct amdgpu_device * adev,uint32_t xcc_id)1956 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
1957 {
1958 int r = 0;
1959 struct amdgpu_ip_block *ip_block;
1960
1961 if (adev->enable_uni_mes)
1962 mes_v12_0_kiq_setting(&adev->mes.ring[AMDGPU_MES_KIQ_PIPE]);
1963 else
1964 mes_v12_0_kiq_setting(&adev->gfx.kiq[0].ring);
1965
1966 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1967
1968 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1969 if (r) {
1970 DRM_ERROR("failed to load MES fw, r=%d\n", r);
1971 return r;
1972 }
1973
1974 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1975 if (r) {
1976 DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1977 return r;
1978 }
1979
1980 mes_v12_0_set_ucode_start_addr(adev);
1981
1982 } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1983 mes_v12_0_set_ucode_start_addr(adev);
1984
1985 mes_v12_0_enable(adev, true);
1986
1987 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
1988 if (unlikely(!ip_block)) {
1989 dev_err(adev->dev, "Failed to get MES handle\n");
1990 return -EINVAL;
1991 }
1992
1993 r = mes_v12_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1994 if (r)
1995 goto failure;
1996
1997 if (adev->enable_uni_mes) {
1998 r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1999 if (r)
2000 goto failure;
2001
2002 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_KIQ_PIPE);
2003 }
2004
2005 if (adev->mes.enable_legacy_queue_map) {
2006 r = mes_v12_0_hw_init(ip_block);
2007 if (r)
2008 goto failure;
2009 }
2010
2011 return r;
2012
2013 failure:
2014 mes_v12_0_hw_fini(ip_block);
2015 return r;
2016 }
2017
mes_v12_0_kiq_hw_fini(struct amdgpu_device * adev,uint32_t xcc_id)2018 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
2019 {
2020 if (adev->mes.ring[0].sched.ready) {
2021 if (adev->enable_uni_mes)
2022 amdgpu_mes_unmap_legacy_queue(adev,
2023 &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],
2024 RESET_QUEUES, 0, 0, 0);
2025 else
2026 mes_v12_0_kiq_dequeue_sched(adev);
2027
2028 adev->mes.ring[0].sched.ready = false;
2029 }
2030
2031 mes_v12_0_enable(adev, false);
2032
2033 return 0;
2034 }
2035
mes_v12_0_hw_init(struct amdgpu_ip_block * ip_block)2036 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
2037 {
2038 int r;
2039 struct amdgpu_device *adev = ip_block->adev;
2040
2041 if (adev->mes.ring[0].sched.ready)
2042 goto out;
2043
2044 if (!adev->enable_mes_kiq) {
2045 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2046 r = mes_v12_0_load_microcode(adev,
2047 AMDGPU_MES_SCHED_PIPE, true);
2048 if (r) {
2049 DRM_ERROR("failed to MES fw, r=%d\n", r);
2050 return r;
2051 }
2052
2053 mes_v12_0_set_ucode_start_addr(adev);
2054
2055 } else if (adev->firmware.load_type ==
2056 AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2057
2058 mes_v12_0_set_ucode_start_addr(adev);
2059 }
2060
2061 mes_v12_0_enable(adev, true);
2062 }
2063
2064 /* Enable the MES to handle doorbell ring on unmapped queue */
2065 mes_v12_0_enable_unmapped_doorbell_handling(&adev->mes, true);
2066
2067 r = mes_v12_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
2068 if (r)
2069 goto failure;
2070
2071 r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_SCHED_PIPE);
2072 if (r)
2073 goto failure;
2074
2075 if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x4b)
2076 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE);
2077
2078 mes_v12_0_init_aggregated_doorbell(&adev->mes);
2079
2080 r = mes_v12_0_query_sched_status(&adev->mes, AMDGPU_MES_SCHED_PIPE);
2081 if (r) {
2082 DRM_ERROR("MES is busy\n");
2083 goto failure;
2084 }
2085
2086 r = amdgpu_mes_update_enforce_isolation(adev);
2087 if (r)
2088 goto failure;
2089
2090 amdgpu_mes_validate_fw_version(adev);
2091 out:
2092 /*
2093 * Disable KIQ ring usage from the driver once MES is enabled.
2094 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
2095 * with MES enabled.
2096 */
2097 adev->gfx.kiq[0].ring.sched.ready = false;
2098 adev->mes.ring[0].sched.ready = true;
2099
2100 return 0;
2101
2102 failure:
2103 mes_v12_0_hw_fini(ip_block);
2104 return r;
2105 }
2106
mes_v12_0_hw_fini(struct amdgpu_ip_block * ip_block)2107 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
2108 {
2109 return 0;
2110 }
2111
mes_v12_0_suspend(struct amdgpu_ip_block * ip_block)2112 static int mes_v12_0_suspend(struct amdgpu_ip_block *ip_block)
2113 {
2114 return mes_v12_0_hw_fini(ip_block);
2115 }
2116
mes_v12_0_resume(struct amdgpu_ip_block * ip_block)2117 static int mes_v12_0_resume(struct amdgpu_ip_block *ip_block)
2118 {
2119 return mes_v12_0_hw_init(ip_block);
2120 }
2121
mes_v12_0_early_init(struct amdgpu_ip_block * ip_block)2122 static int mes_v12_0_early_init(struct amdgpu_ip_block *ip_block)
2123 {
2124 struct amdgpu_device *adev = ip_block->adev;
2125 int pipe, r;
2126
2127 adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE;
2128 adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET;
2129
2130 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
2131 r = amdgpu_mes_init_microcode(adev, pipe);
2132 if (r)
2133 return r;
2134 }
2135
2136 return 0;
2137 }
2138
2139 static const struct amd_ip_funcs mes_v12_0_ip_funcs = {
2140 .name = "mes_v12_0",
2141 .early_init = mes_v12_0_early_init,
2142 .late_init = NULL,
2143 .sw_init = mes_v12_0_sw_init,
2144 .sw_fini = mes_v12_0_sw_fini,
2145 .hw_init = mes_v12_0_hw_init,
2146 .hw_fini = mes_v12_0_hw_fini,
2147 .suspend = mes_v12_0_suspend,
2148 .resume = mes_v12_0_resume,
2149 };
2150
2151 const struct amdgpu_ip_block_version mes_v12_0_ip_block = {
2152 .type = AMD_IP_BLOCK_TYPE_MES,
2153 .major = 12,
2154 .minor = 0,
2155 .rev = 0,
2156 .funcs = &mes_v12_0_ip_funcs,
2157 };
2158