1 /*
2 * Copyright 2025 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include <linux/module.h>
26 #include "amdgpu.h"
27 #include "soc15_common.h"
28 #include "soc_v1_0.h"
29 #include "gc/gc_12_1_0_offset.h"
30 #include "gc/gc_12_1_0_sh_mask.h"
31 #include "gc/gc_11_0_0_default.h"
32 #include "v12_structs.h"
33 #include "mes_v12_api_def.h"
34 #include "gfx_v12_1_pkt.h"
35 #include "sdma_v7_1_0_pkt_open.h"
36
37 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes.bin");
38 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes1.bin");
39 MODULE_FIRMWARE("amdgpu/gc_12_1_0_uni_mes.bin");
40
41 static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block);
42 static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id);
43 static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
44 static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
45 static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
46 static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id);
47 static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id);
48
49 #define MES_EOP_SIZE 2048
50 #define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */
51 #define MES12_HUNG_HQD_INFO_OFFSET 4
52
53 #define regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT 0x100000
54 #define XCC_MID_MASK 0x41000000
55
mes_v12_1_ring_set_wptr(struct amdgpu_ring * ring)56 static void mes_v12_1_ring_set_wptr(struct amdgpu_ring *ring)
57 {
58 struct amdgpu_device *adev = ring->adev;
59
60 if (ring->use_doorbell) {
61 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
62 ring->wptr);
63 WDOORBELL64(ring->doorbell_index, ring->wptr);
64 } else {
65 BUG();
66 }
67 }
68
mes_v12_1_ring_get_rptr(struct amdgpu_ring * ring)69 static u64 mes_v12_1_ring_get_rptr(struct amdgpu_ring *ring)
70 {
71 return *ring->rptr_cpu_addr;
72 }
73
mes_v12_1_ring_get_wptr(struct amdgpu_ring * ring)74 static u64 mes_v12_1_ring_get_wptr(struct amdgpu_ring *ring)
75 {
76 u64 wptr;
77
78 if (ring->use_doorbell)
79 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
80 else
81 BUG();
82 return wptr;
83 }
84
85 static const struct amdgpu_ring_funcs mes_v12_1_ring_funcs = {
86 .type = AMDGPU_RING_TYPE_MES,
87 .align_mask = 1,
88 .nop = 0,
89 .support_64bit_ptrs = true,
90 .get_rptr = mes_v12_1_ring_get_rptr,
91 .get_wptr = mes_v12_1_ring_get_wptr,
92 .set_wptr = mes_v12_1_ring_set_wptr,
93 .insert_nop = amdgpu_ring_insert_nop,
94 };
95
96 static const char *mes_v12_1_opcodes[] = {
97 "SET_HW_RSRC",
98 "SET_SCHEDULING_CONFIG",
99 "ADD_QUEUE",
100 "REMOVE_QUEUE",
101 "PERFORM_YIELD",
102 "SET_GANG_PRIORITY_LEVEL",
103 "SUSPEND",
104 "RESUME",
105 "RESET",
106 "SET_LOG_BUFFER",
107 "CHANGE_GANG_PRORITY",
108 "QUERY_SCHEDULER_STATUS",
109 "unused",
110 "SET_DEBUG_VMID",
111 "MISC",
112 "UPDATE_ROOT_PAGE_TABLE",
113 "AMD_LOG",
114 "SET_SE_MODE",
115 "SET_GANG_SUBMIT",
116 "SET_HW_RSRC_1",
117 "INVALIDATE_TLBS",
118 };
119
120 static const char *mes_v12_1_misc_opcodes[] = {
121 "WRITE_REG",
122 "INV_GART",
123 "QUERY_STATUS",
124 "READ_REG",
125 "WAIT_REG_MEM",
126 "SET_SHADER_DEBUGGER",
127 "NOTIFY_WORK_ON_UNMAPPED_QUEUE",
128 "NOTIFY_TO_UNMAP_PROCESSES",
129 };
130
mes_v12_1_get_op_string(union MESAPI__MISC * x_pkt)131 static const char *mes_v12_1_get_op_string(union MESAPI__MISC *x_pkt)
132 {
133 const char *op_str = NULL;
134
135 if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_1_opcodes))
136 op_str = mes_v12_1_opcodes[x_pkt->header.opcode];
137
138 return op_str;
139 }
140
mes_v12_1_get_misc_op_string(union MESAPI__MISC * x_pkt)141 static const char *mes_v12_1_get_misc_op_string(union MESAPI__MISC *x_pkt)
142 {
143 const char *op_str = NULL;
144
145 if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
146 (x_pkt->opcode < ARRAY_SIZE(mes_v12_1_misc_opcodes)))
147 op_str = mes_v12_1_misc_opcodes[x_pkt->opcode];
148
149 return op_str;
150 }
151
mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes * mes,int xcc_id,int pipe,void * pkt,int size,int api_status_off)152 static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
153 int xcc_id, int pipe, void *pkt,
154 int size, int api_status_off)
155 {
156 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
157 signed long timeout = 2100000; /* 2100 ms */
158 struct amdgpu_device *adev = mes->adev;
159 struct amdgpu_ring *ring = &mes->ring[MES_PIPE_INST(xcc_id, pipe)];
160 spinlock_t *ring_lock = &mes->ring_lock[MES_PIPE_INST(xcc_id, pipe)];
161 struct MES_API_STATUS *api_status;
162 union MESAPI__MISC *x_pkt = pkt;
163 const char *op_str, *misc_op_str;
164 unsigned long flags;
165 u64 status_gpu_addr;
166 u32 seq, status_offset;
167 u64 *status_ptr;
168 signed long r;
169 int ret;
170
171 if (x_pkt->header.opcode >= MES_SCH_API_MAX)
172 return -EINVAL;
173
174 if (amdgpu_emu_mode) {
175 timeout *= 1000;
176 } else if (amdgpu_sriov_vf(adev)) {
177 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
178 timeout = 15 * 600 * 1000;
179 }
180
181 ret = amdgpu_device_wb_get(adev, &status_offset);
182 if (ret)
183 return ret;
184
185 status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
186 status_ptr = (u64 *)&adev->wb.wb[status_offset];
187 *status_ptr = 0;
188
189 spin_lock_irqsave(ring_lock, flags);
190 r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
191 if (r)
192 goto error_unlock_free;
193
194 seq = ++ring->fence_drv.sync_seq;
195 r = amdgpu_fence_wait_polling(ring,
196 seq - ring->fence_drv.num_fences_mask,
197 timeout);
198 if (r < 1)
199 goto error_undo;
200
201 api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
202 api_status->api_completion_fence_addr = status_gpu_addr;
203 api_status->api_completion_fence_value = 1;
204
205 amdgpu_ring_write_multiple(ring, pkt, size / 4);
206
207 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
208 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
209 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
210 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
211 mes_status_pkt.api_status.api_completion_fence_addr =
212 ring->fence_drv.gpu_addr;
213 mes_status_pkt.api_status.api_completion_fence_value = seq;
214
215 amdgpu_ring_write_multiple(ring, &mes_status_pkt,
216 sizeof(mes_status_pkt) / 4);
217
218 amdgpu_ring_commit(ring);
219 spin_unlock_irqrestore(ring_lock, flags);
220
221 op_str = mes_v12_1_get_op_string(x_pkt);
222 misc_op_str = mes_v12_1_get_misc_op_string(x_pkt);
223
224 if (misc_op_str)
225 dev_dbg(adev->dev, "MES(%d, %d) msg=%s (%s) was emitted\n",
226 xcc_id, pipe, op_str, misc_op_str);
227 else if (op_str)
228 dev_dbg(adev->dev, "MES(%d, %d) msg=%s was emitted\n",
229 xcc_id, pipe, op_str);
230 else
231 dev_dbg(adev->dev, "MES(%d, %d) msg=%d was emitted\n",
232 xcc_id, pipe, x_pkt->header.opcode);
233
234 r = amdgpu_fence_wait_polling(ring, seq, timeout);
235 if (r < 1 || !lower_32_bits(*status_ptr)) {
236 if (misc_op_str)
237 dev_err(adev->dev,
238 "MES(%d, %d) failed to respond to msg=%s (%s)\n",
239 xcc_id, pipe, op_str, misc_op_str);
240 else if (op_str)
241 dev_err(adev->dev,
242 "MES(%d, %d) failed to respond to msg=%s\n",
243 xcc_id, pipe, op_str);
244 else
245 dev_err(adev->dev,
246 "MES(%d, %d) failed to respond to msg=%d\n",
247 xcc_id, pipe, x_pkt->header.opcode);
248
249 while (halt_if_hws_hang)
250 schedule();
251
252 r = -ETIMEDOUT;
253 goto error_wb_free;
254 }
255
256 amdgpu_device_wb_free(adev, status_offset);
257 return 0;
258
259 error_undo:
260 dev_err(adev->dev, "MES(%d, %d) ring buffer is full.\n", xcc_id, pipe);
261 amdgpu_ring_undo(ring);
262
263 error_unlock_free:
264 spin_unlock_irqrestore(ring_lock, flags);
265
266 error_wb_free:
267 amdgpu_device_wb_free(adev, status_offset);
268 return r;
269 }
270
convert_to_mes_queue_type(int queue_type)271 static int convert_to_mes_queue_type(int queue_type)
272 {
273 if (queue_type == AMDGPU_RING_TYPE_GFX)
274 return MES_QUEUE_TYPE_GFX;
275 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
276 return MES_QUEUE_TYPE_COMPUTE;
277 else if (queue_type == AMDGPU_RING_TYPE_SDMA)
278 return MES_QUEUE_TYPE_SDMA;
279 else if (queue_type == AMDGPU_RING_TYPE_MES)
280 return MES_QUEUE_TYPE_SCHQ;
281 else
282 BUG();
283 return -1;
284 }
285
mes_v12_1_add_hw_queue(struct amdgpu_mes * mes,struct mes_add_queue_input * input)286 static int mes_v12_1_add_hw_queue(struct amdgpu_mes *mes,
287 struct mes_add_queue_input *input)
288 {
289 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
290 int xcc_id = input->xcc_id;
291 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
292
293 if (mes->enable_coop_mode)
294 xcc_id = mes->master_xcc_ids[inst];
295
296 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
297
298 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
299 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
300 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
301
302 mes_add_queue_pkt.process_id = input->process_id;
303 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
304 mes_add_queue_pkt.process_va_start = input->process_va_start;
305 mes_add_queue_pkt.process_va_end = input->process_va_end;
306 mes_add_queue_pkt.process_quantum = input->process_quantum;
307 mes_add_queue_pkt.process_context_addr = input->process_context_addr;
308 mes_add_queue_pkt.gang_quantum = input->gang_quantum;
309 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
310 mes_add_queue_pkt.inprocess_gang_priority =
311 input->inprocess_gang_priority;
312 mes_add_queue_pkt.gang_global_priority_level =
313 input->gang_global_priority_level;
314 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
315 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
316
317 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
318
319 mes_add_queue_pkt.queue_type =
320 convert_to_mes_queue_type(input->queue_type);
321 mes_add_queue_pkt.paging = input->paging;
322 mes_add_queue_pkt.vm_context_cntl = input->vm_cntx_cntl;
323 mes_add_queue_pkt.gws_base = input->gws_base;
324 mes_add_queue_pkt.gws_size = input->gws_size;
325 mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
326 mes_add_queue_pkt.tma_addr = input->tma_addr;
327 mes_add_queue_pkt.trap_en = input->trap_en;
328 mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
329 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
330
331 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
332 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
333 mes_add_queue_pkt.gds_size = input->queue_size;
334
335 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
336 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
337 mes_add_queue_pkt.gds_size = input->queue_size;
338
339 mes_add_queue_pkt.full_sh_mem_config_data = input->sh_mem_config_data;
340
341 return mes_v12_1_submit_pkt_and_poll_completion(mes,
342 xcc_id, AMDGPU_MES_SCHED_PIPE,
343 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
344 offsetof(union MESAPI__ADD_QUEUE, api_status));
345 }
346
mes_v12_1_remove_hw_queue(struct amdgpu_mes * mes,struct mes_remove_queue_input * input)347 static int mes_v12_1_remove_hw_queue(struct amdgpu_mes *mes,
348 struct mes_remove_queue_input *input)
349 {
350 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
351 int xcc_id = input->xcc_id;
352 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
353
354 if (mes->enable_coop_mode)
355 xcc_id = mes->master_xcc_ids[inst];
356
357 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
358
359 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
360 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
361 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
362
363 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
364 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
365
366 return mes_v12_1_submit_pkt_and_poll_completion(mes,
367 xcc_id, AMDGPU_MES_SCHED_PIPE,
368 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
369 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
370 }
371
mes_v12_1_reset_hw_queue(struct amdgpu_mes * mes,struct mes_reset_queue_input * input)372 static int mes_v12_1_reset_hw_queue(struct amdgpu_mes *mes,
373 struct mes_reset_queue_input *input)
374 {
375 union MESAPI__RESET mes_reset_queue_pkt;
376 int pipe;
377
378 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
379
380 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
381 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
382 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
383
384 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
385 /* mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr; */
386 /*mes_reset_queue_pkt.reset_queue_only = 1;*/
387
388 if (mes->adev->enable_uni_mes)
389 pipe = AMDGPU_MES_KIQ_PIPE;
390 else
391 pipe = AMDGPU_MES_SCHED_PIPE;
392
393 return mes_v12_1_submit_pkt_and_poll_completion(mes,
394 input->xcc_id, pipe,
395 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
396 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
397 }
398
mes_v12_1_map_legacy_queue(struct amdgpu_mes * mes,struct mes_map_legacy_queue_input * input)399 static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes,
400 struct mes_map_legacy_queue_input *input)
401 {
402 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
403 int pipe;
404
405 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
406
407 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
408 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
409 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
410
411 mes_add_queue_pkt.pipe_id = input->pipe_id;
412 mes_add_queue_pkt.queue_id = input->queue_id;
413 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
414 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
415 mes_add_queue_pkt.wptr_addr = input->wptr_addr;
416 mes_add_queue_pkt.queue_type =
417 convert_to_mes_queue_type(input->queue_type);
418 mes_add_queue_pkt.map_legacy_kq = 1;
419
420 if (mes->adev->enable_uni_mes)
421 pipe = AMDGPU_MES_KIQ_PIPE;
422 else
423 pipe = AMDGPU_MES_SCHED_PIPE;
424
425 return mes_v12_1_submit_pkt_and_poll_completion(mes,
426 input->xcc_id, pipe,
427 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
428 offsetof(union MESAPI__ADD_QUEUE, api_status));
429 }
430
mes_v12_1_unmap_legacy_queue(struct amdgpu_mes * mes,struct mes_unmap_legacy_queue_input * input)431 static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
432 struct mes_unmap_legacy_queue_input *input)
433 {
434 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
435 int pipe;
436
437 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
438
439 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
440 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
441 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
442
443 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
444 mes_remove_queue_pkt.gang_context_addr = 0;
445
446 mes_remove_queue_pkt.pipe_id = input->pipe_id;
447 mes_remove_queue_pkt.queue_id = input->queue_id;
448
449 if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
450 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
451 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
452 mes_remove_queue_pkt.tf_data =
453 lower_32_bits(input->trail_fence_data);
454 } else {
455 mes_remove_queue_pkt.unmap_legacy_queue = 1;
456 mes_remove_queue_pkt.queue_type =
457 convert_to_mes_queue_type(input->queue_type);
458 }
459
460 if (mes->adev->enable_uni_mes)
461 pipe = AMDGPU_MES_KIQ_PIPE;
462 else
463 pipe = AMDGPU_MES_SCHED_PIPE;
464
465 return mes_v12_1_submit_pkt_and_poll_completion(mes,
466 input->xcc_id, pipe,
467 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
468 offsetof(union MESAPI__REMOVE_QUEUE, api_status));
469 }
470
mes_v12_1_suspend_gang(struct amdgpu_mes * mes,struct mes_suspend_gang_input * input)471 static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes,
472 struct mes_suspend_gang_input *input)
473 {
474 union MESAPI__SUSPEND mes_suspend_gang_pkt;
475
476 memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
477
478 mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
479 mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
480 mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
481
482 mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
483 mes_suspend_gang_pkt.suspend_all_sdma_gangs = input->suspend_all_sdma_gangs;
484 mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
485 mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
486 mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
487 mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset;
488
489 /* Suspend gang is handled by master MES */
490 return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
491 &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
492 offsetof(union MESAPI__SUSPEND, api_status));
493 }
494
mes_v12_1_resume_gang(struct amdgpu_mes * mes,struct mes_resume_gang_input * input)495 static int mes_v12_1_resume_gang(struct amdgpu_mes *mes,
496 struct mes_resume_gang_input *input)
497 {
498 union MESAPI__RESUME mes_resume_gang_pkt;
499
500 memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
501
502 mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
503 mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
504 mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
505
506 mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
507 mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
508 mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset;
509
510 /* Resume gang is handled by master MES */
511 return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
512 &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
513 offsetof(union MESAPI__RESUME, api_status));
514 }
515
mes_v12_1_query_sched_status(struct amdgpu_mes * mes,int pipe,int xcc_id)516 static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes,
517 int pipe, int xcc_id)
518 {
519 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
520
521 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
522
523 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
524 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
525 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
526
527 return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
528 &mes_status_pkt, sizeof(mes_status_pkt),
529 offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
530 }
mes_v12_1_get_xcc_from_reg(uint32_t reg_offset)531 static uint32_t mes_v12_1_get_xcc_from_reg(uint32_t reg_offset)
532 {
533 return ((reg_offset >> 16) & 0x7);
534 }
535
mes_v12_1_get_rrmt(uint32_t reg,uint32_t xcc_id,struct RRMT_OPTION * rrmt_opt,uint32_t * out_reg)536 static void mes_v12_1_get_rrmt(uint32_t reg, uint32_t xcc_id,
537 struct RRMT_OPTION *rrmt_opt,
538 uint32_t *out_reg)
539 {
540 uint32_t normalized_reg = soc_v1_0_normalize_xcc_reg_offset(reg);
541
542 if (soc_v1_0_normalize_xcc_reg_range(normalized_reg)) {
543 rrmt_opt->xcd_die_id = mes_v12_1_get_xcc_from_reg(reg);
544 rrmt_opt->mode = (xcc_id == rrmt_opt->xcd_die_id) ?
545 MES_RRMT_MODE_LOCAL_XCD : MES_RRMT_MODE_REMOTE_XCD;
546 } else {
547 rrmt_opt->mode = MES_RRMT_MODE_REMOTE_MID;
548 if (soc_v1_0_mid1_reg_range(reg))
549 rrmt_opt->mid_die_id = 1;
550 }
551
552 *out_reg = soc_v1_0_normalize_reg_offset(reg);
553 }
554
mes_v12_1_misc_op(struct amdgpu_mes * mes,struct mes_misc_op_input * input)555 static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
556 struct mes_misc_op_input *input)
557 {
558 struct amdgpu_device *adev = mes->adev;
559 union MESAPI__MISC misc_pkt;
560 int pipe;
561
562 if (mes->adev->enable_uni_mes)
563 pipe = AMDGPU_MES_KIQ_PIPE;
564 else
565 pipe = AMDGPU_MES_SCHED_PIPE;
566
567 memset(&misc_pkt, 0, sizeof(misc_pkt));
568
569 misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
570 misc_pkt.header.opcode = MES_SCH_API_MISC;
571 misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
572
573 switch (input->op) {
574 case MES_MISC_OP_READ_REG:
575 misc_pkt.opcode = MESAPI_MISC__READ_REG;
576 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
577 mes_v12_1_get_rrmt(input->read_reg.reg_offset,
578 GET_INST(GC, input->xcc_id),
579 &misc_pkt.read_reg.rrmt_opt,
580 &misc_pkt.read_reg.reg_offset);
581 break;
582 case MES_MISC_OP_WRITE_REG:
583 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
584 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
585 mes_v12_1_get_rrmt(input->write_reg.reg_offset,
586 GET_INST(GC, input->xcc_id),
587 &misc_pkt.write_reg.rrmt_opt,
588 &misc_pkt.write_reg.reg_offset);
589 break;
590 case MES_MISC_OP_WRM_REG_WAIT:
591 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
592 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
593 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
594 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
595 misc_pkt.wait_reg_mem.reg_offset2 = 0;
596 mes_v12_1_get_rrmt(input->wrm_reg.reg0,
597 GET_INST(GC, input->xcc_id),
598 &misc_pkt.wait_reg_mem.rrmt_opt1,
599 &misc_pkt.wait_reg_mem.reg_offset1);
600 break;
601 case MES_MISC_OP_WRM_REG_WR_WAIT:
602 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
603 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
604 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
605 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
606 mes_v12_1_get_rrmt(input->wrm_reg.reg0,
607 GET_INST(GC, input->xcc_id),
608 &misc_pkt.wait_reg_mem.rrmt_opt1,
609 &misc_pkt.wait_reg_mem.reg_offset1);
610 mes_v12_1_get_rrmt(input->wrm_reg.reg1,
611 GET_INST(GC, input->xcc_id),
612 &misc_pkt.wait_reg_mem.rrmt_opt2,
613 &misc_pkt.wait_reg_mem.reg_offset2);
614 break;
615 case MES_MISC_OP_SET_SHADER_DEBUGGER:
616 pipe = AMDGPU_MES_SCHED_PIPE;
617 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
618 misc_pkt.set_shader_debugger.process_context_addr =
619 input->set_shader_debugger.process_context_addr;
620 misc_pkt.set_shader_debugger.flags.u32all =
621 input->set_shader_debugger.flags.u32all;
622 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
623 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
624 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
625 input->set_shader_debugger.tcp_watch_cntl,
626 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
627 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
628 break;
629 case MES_MISC_OP_CHANGE_CONFIG:
630 misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
631 misc_pkt.change_config.opcode =
632 MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
633 misc_pkt.change_config.option.bits.limit_single_process =
634 input->change_config.option.limit_single_process;
635 break;
636 default:
637 DRM_ERROR("unsupported misc op (%d) \n", input->op);
638 return -EINVAL;
639 }
640
641 return mes_v12_1_submit_pkt_and_poll_completion(mes,
642 input->xcc_id, pipe,
643 &misc_pkt, sizeof(misc_pkt),
644 offsetof(union MESAPI__MISC, api_status));
645 }
646
mes_v12_1_set_hw_resources_1(struct amdgpu_mes * mes,int pipe,int xcc_id)647 static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
648 int pipe, int xcc_id)
649 {
650 union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
651 int master_xcc_id, inst = MES_PIPE_INST(xcc_id, pipe);
652
653 memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
654
655 mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
656 mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
657 mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
658 mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100;
659
660 /* From version 0x74 above, pipe1 support use shared command buffer
661 to distribute some tasks on individual XCCs*/
662 if (mes->enable_coop_mode &&
663 ((pipe == AMDGPU_MES_SCHED_PIPE) ||
664 ((mes->kiq_version & AMDGPU_MES_VERSION_MASK) >= 0x74))) {
665 master_xcc_id = mes->master_xcc_ids[inst];
666 mes_set_hw_res_1_pkt.mes_coop_mode = 1;
667 mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr =
668 mes->shared_cmd_buf_gpu_addr[master_xcc_id + pipe];
669 }
670
671 return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
672 &mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
673 offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
674 }
675
mes_v12_1_set_hw_resources(struct amdgpu_mes * mes,int pipe,int xcc_id)676 static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
677 int pipe, int xcc_id)
678 {
679 int i, status;
680 struct amdgpu_device *adev = mes->adev;
681 union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
682
683 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
684
685 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
686 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
687 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
688
689 if (pipe == AMDGPU_MES_SCHED_PIPE) {
690 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
691 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
692 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
693 mes_set_hw_res_pkt.paging_vmid = 0;
694
695 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
696 mes_set_hw_res_pkt.compute_hqd_mask[i] =
697 mes->compute_hqd_mask[i];
698
699 for (i = 0; i < MAX_GFX_PIPES; i++)
700 mes_set_hw_res_pkt.gfx_hqd_mask[i] =
701 mes->gfx_hqd_mask[i];
702
703 for (i = 0; i < MAX_SDMA_PIPES; i++)
704 mes_set_hw_res_pkt.sdma_hqd_mask[i] =
705 mes->sdma_hqd_mask[i];
706
707 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
708 mes_set_hw_res_pkt.aggregated_doorbells[i] =
709 mes->aggregated_doorbells[i];
710 }
711
712 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
713 mes->sch_ctx_gpu_addr[pipe];
714 mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
715 mes->query_status_fence_gpu_addr[pipe];
716
717 for (i = 0; i < 5; i++) {
718 mes_set_hw_res_pkt.gc_base[i] =
719 adev->reg_offset[GC_HWIP][0][i];
720 mes_set_hw_res_pkt.mmhub_base[i] =
721 adev->reg_offset[MMHUB_HWIP][0][i];
722 mes_set_hw_res_pkt.osssys_base[i] =
723 adev->reg_offset[OSSSYS_HWIP][0][i];
724 }
725
726 mes_set_hw_res_pkt.disable_reset = 1;
727 mes_set_hw_res_pkt.disable_mes_log = 1;
728 mes_set_hw_res_pkt.use_different_vmid_compute = 1;
729 mes_set_hw_res_pkt.enable_reg_active_poll = 1;
730 mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
731
732 /*
733 * Keep oversubscribe timer for sdma . When we have unmapped doorbell
734 * handling support, other queue will not use the oversubscribe timer.
735 * handling mode - 0: disabled; 1: basic version; 2: basic+ version
736 */
737 mes_set_hw_res_pkt.oversubscription_timer = 50;
738 mes_set_hw_res_pkt.unmapped_doorbell_handling = 1;
739
740 if (amdgpu_mes_log_enable) {
741 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
742 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr =
743 mes->event_log_gpu_addr + MES_PIPE_INST(xcc_id, pipe) * AMDGPU_MES_LOG_BUFFER_SIZE;
744 }
745
746 if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
747 mes_set_hw_res_pkt.limit_single_process = 1;
748
749 status = mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
750 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
751 offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
752
753 /* get MES scheduler versions */
754 mutex_lock(&adev->srbm_mutex);
755 soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
756
757 if (pipe == AMDGPU_MES_SCHED_PIPE)
758 adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
759 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
760 adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
761
762 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
763 mutex_unlock(&adev->srbm_mutex);
764
765 return status;
766 }
767
mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes * mes,int xcc_id)768 static void mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes *mes,
769 int xcc_id)
770 {
771 struct amdgpu_device *adev = mes->adev;
772 uint32_t data;
773
774 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL1);
775 data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
776 CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
777 CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
778 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
779 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
780 data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
781 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL1, data);
782
783 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL2);
784 data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
785 CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
786 CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
787 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
788 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
789 data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
790 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL2, data);
791
792 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL3);
793 data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
794 CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
795 CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
796 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
797 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
798 data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
799 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL3, data);
800
801 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL4);
802 data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
803 CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
804 CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
805 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
806 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
807 data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
808 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL4, data);
809
810 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL5);
811 data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
812 CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
813 CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
814 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
815 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
816 data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
817 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL5, data);
818
819 data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
820 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_GFX_CONTROL, data);
821 }
822
823
mes_v12_1_enable_unmapped_doorbell_handling(struct amdgpu_mes * mes,bool enable,int xcc_id)824 static void mes_v12_1_enable_unmapped_doorbell_handling(
825 struct amdgpu_mes *mes, bool enable, int xcc_id)
826 {
827 struct amdgpu_device *adev = mes->adev;
828 uint32_t data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_UNMAPPED_DOORBELL);
829
830 /*
831 * The default PROC_LSB settng is 0xc which means doorbell
832 * addr[16:12] gives the doorbell page number. For kfd, each
833 * process will use 2 pages of doorbell, we need to change the
834 * setting to 0xd
835 */
836 data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
837 data |= 0xd << CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
838
839 data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
840
841 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_UNMAPPED_DOORBELL, data);
842 }
843
844 #if 0
845 static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
846 struct mes_reset_legacy_queue_input *input)
847 {
848 union MESAPI__RESET mes_reset_queue_pkt;
849 int pipe;
850
851 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
852
853 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
854 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
855 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
856
857 mes_reset_queue_pkt.queue_type =
858 convert_to_mes_queue_type(input->queue_type);
859
860 if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {
861 mes_reset_queue_pkt.reset_legacy_gfx = 1;
862 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
863 mes_reset_queue_pkt.queue_id_lp = input->queue_id;
864 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
865 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
866 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
867 mes_reset_queue_pkt.vmid_id_lp = input->vmid;
868 } else {
869 mes_reset_queue_pkt.reset_queue_only = 1;
870 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
871 }
872
873 if (mes->adev->enable_uni_mes)
874 pipe = AMDGPU_MES_KIQ_PIPE;
875 else
876 pipe = AMDGPU_MES_SCHED_PIPE;
877
878 return mes_v12_1_submit_pkt_and_poll_completion(mes,
879 input->xcc_id, pipe,
880 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
881 offsetof(union MESAPI__RESET, api_status));
882 }
883 #endif
884
mes_v12_1_detect_and_reset_hung_queues(struct amdgpu_mes * mes,struct mes_detect_and_reset_queue_input * input)885 static int mes_v12_1_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
886 struct mes_detect_and_reset_queue_input *input)
887 {
888 union MESAPI__RESET mes_reset_queue_pkt;
889
890 memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
891
892 mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
893 mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
894 mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
895
896 mes_reset_queue_pkt.queue_type =
897 convert_to_mes_queue_type(input->queue_type);
898 mes_reset_queue_pkt.doorbell_offset_addr =
899 mes->hung_queue_db_array_gpu_addr[0];
900
901 if (input->detect_only)
902 mes_reset_queue_pkt.hang_detect_only = 1;
903 else
904 mes_reset_queue_pkt.hang_detect_then_reset = 1;
905
906 return mes_v12_1_submit_pkt_and_poll_completion(mes,
907 input->xcc_id, AMDGPU_MES_SCHED_PIPE,
908 &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
909 offsetof(union MESAPI__RESET, api_status));
910 }
911
mes_v12_inv_tlb_convert_hub_id(uint8_t id)912 static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
913 {
914 /*
915 * MES doesn't support invalidate gc_hub on slave xcc individually
916 * master xcc will invalidate all gc_hub for the partition
917 */
918 if (AMDGPU_IS_GFXHUB(id))
919 return 0;
920 else if (AMDGPU_IS_MMHUB0(id))
921 return 1;
922 else if (AMDGPU_IS_MMHUB1(id))
923 return 2;
924 return -EINVAL;
925
926 }
927
mes_v12_1_inv_tlbs_pasid(struct amdgpu_mes * mes,struct mes_inv_tlbs_pasid_input * input)928 static int mes_v12_1_inv_tlbs_pasid(struct amdgpu_mes *mes,
929 struct mes_inv_tlbs_pasid_input *input)
930 {
931 union MESAPI__INV_TLBS mes_inv_tlbs;
932 int xcc_id = input->xcc_id;
933 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
934 int ret;
935
936 if (mes->enable_coop_mode)
937 xcc_id = mes->master_xcc_ids[inst];
938
939 memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
940
941 mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
942 mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
943 mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
944
945 mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
946 mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
947 mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
948
949 /*convert amdgpu_mes_hub_id to mes expected hub_id */
950 ret = mes_v12_inv_tlb_convert_hub_id(input->hub_id);
951 if (ret < 0)
952 return -EINVAL;
953 mes_inv_tlbs.invalidate_tlbs.hub_id = ret;
954 return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, AMDGPU_MES_KIQ_PIPE,
955 &mes_inv_tlbs, sizeof(mes_inv_tlbs),
956 offsetof(union MESAPI__INV_TLBS, api_status));
957
958 }
959
960 static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
961 .add_hw_queue = mes_v12_1_add_hw_queue,
962 .remove_hw_queue = mes_v12_1_remove_hw_queue,
963 .map_legacy_queue = mes_v12_1_map_legacy_queue,
964 .unmap_legacy_queue = mes_v12_1_unmap_legacy_queue,
965 .suspend_gang = mes_v12_1_suspend_gang,
966 .resume_gang = mes_v12_1_resume_gang,
967 .misc_op = mes_v12_1_misc_op,
968 .reset_hw_queue = mes_v12_1_reset_hw_queue,
969 .detect_and_reset_hung_queues = mes_v12_1_detect_and_reset_hung_queues,
970 .invalidate_tlbs_pasid = mes_v12_1_inv_tlbs_pasid,
971 };
972
mes_v12_1_allocate_ucode_buffer(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)973 static int mes_v12_1_allocate_ucode_buffer(struct amdgpu_device *adev,
974 enum amdgpu_mes_pipe pipe,
975 int xcc_id)
976 {
977 int r, inst = MES_PIPE_INST(xcc_id, pipe);
978 const struct mes_firmware_header_v1_0 *mes_hdr;
979 const __le32 *fw_data;
980 unsigned fw_size;
981
982 mes_hdr = (const struct mes_firmware_header_v1_0 *)
983 adev->mes.fw[pipe]->data;
984
985 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
986 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
987 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
988
989 r = amdgpu_bo_create_reserved(adev, fw_size,
990 PAGE_SIZE,
991 AMDGPU_GEM_DOMAIN_VRAM,
992 &adev->mes.ucode_fw_obj[inst],
993 &adev->mes.ucode_fw_gpu_addr[inst],
994 (void **)&adev->mes.ucode_fw_ptr[inst]);
995 if (r) {
996 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
997 return r;
998 }
999
1000 memcpy(adev->mes.ucode_fw_ptr[inst], fw_data, fw_size);
1001
1002 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[inst]);
1003 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[inst]);
1004
1005 return 0;
1006 }
1007
mes_v12_1_allocate_ucode_data_buffer(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1008 static int mes_v12_1_allocate_ucode_data_buffer(struct amdgpu_device *adev,
1009 enum amdgpu_mes_pipe pipe,
1010 int xcc_id)
1011 {
1012 int r, inst = MES_PIPE_INST(xcc_id, pipe);
1013 const struct mes_firmware_header_v1_0 *mes_hdr;
1014 const __le32 *fw_data;
1015 unsigned fw_size;
1016
1017 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1018 adev->mes.fw[pipe]->data;
1019
1020 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1021 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1022 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1023
1024 r = amdgpu_bo_create_reserved(adev, fw_size,
1025 64 * 1024,
1026 AMDGPU_GEM_DOMAIN_VRAM,
1027 &adev->mes.data_fw_obj[inst],
1028 &adev->mes.data_fw_gpu_addr[inst],
1029 (void **)&adev->mes.data_fw_ptr[inst]);
1030 if (r) {
1031 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
1032 return r;
1033 }
1034
1035 memcpy(adev->mes.data_fw_ptr[inst], fw_data, fw_size);
1036
1037 amdgpu_bo_kunmap(adev->mes.data_fw_obj[inst]);
1038 amdgpu_bo_unreserve(adev->mes.data_fw_obj[inst]);
1039
1040 return 0;
1041 }
1042
mes_v12_1_free_ucode_buffers(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1043 static void mes_v12_1_free_ucode_buffers(struct amdgpu_device *adev,
1044 enum amdgpu_mes_pipe pipe,
1045 int xcc_id)
1046 {
1047 int inst = MES_PIPE_INST(xcc_id, pipe);
1048
1049 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[inst],
1050 &adev->mes.data_fw_gpu_addr[inst],
1051 (void **)&adev->mes.data_fw_ptr[inst]);
1052
1053 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[inst],
1054 &adev->mes.ucode_fw_gpu_addr[inst],
1055 (void **)&adev->mes.ucode_fw_ptr[inst]);
1056 }
1057
mes_v12_1_enable(struct amdgpu_device * adev,bool enable,int xcc_id)1058 static void mes_v12_1_enable(struct amdgpu_device *adev,
1059 bool enable, int xcc_id)
1060 {
1061 uint64_t ucode_addr;
1062 uint32_t pipe, data = 0;
1063
1064 if (enable) {
1065 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL);
1066 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1067 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1068 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
1069
1070 mutex_lock(&adev->srbm_mutex);
1071 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1072 soc_v1_0_grbm_select(adev, 3, pipe, 0, 0,
1073 GET_INST(GC, xcc_id));
1074
1075 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1076 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1077 regCP_MES_PRGRM_CNTR_START,
1078 lower_32_bits(ucode_addr));
1079 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1080 regCP_MES_PRGRM_CNTR_START_HI,
1081 upper_32_bits(ucode_addr));
1082 }
1083 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1084 mutex_unlock(&adev->srbm_mutex);
1085
1086 /* unhalt MES and activate pipe0 */
1087 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
1088 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
1089 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
1090
1091 if (amdgpu_emu_mode)
1092 msleep(500);
1093 else if (adev->enable_uni_mes)
1094 udelay(500);
1095 else
1096 udelay(50);
1097 } else {
1098 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL);
1099 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
1100 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
1101 data = REG_SET_FIELD(data, CP_MES_CNTL,
1102 MES_INVALIDATE_ICACHE, 1);
1103 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1104 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1105 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
1106 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
1107 }
1108 }
1109
mes_v12_1_set_ucode_start_addr(struct amdgpu_device * adev,int xcc_id)1110 static void mes_v12_1_set_ucode_start_addr(struct amdgpu_device *adev,
1111 int xcc_id)
1112 {
1113 uint64_t ucode_addr;
1114 int pipe;
1115
1116 mes_v12_1_enable(adev, false, xcc_id);
1117
1118 mutex_lock(&adev->srbm_mutex);
1119 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1120 /* me=3, queue=0 */
1121 soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
1122
1123 /* set ucode start address */
1124 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1125 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_PRGRM_CNTR_START,
1126 lower_32_bits(ucode_addr));
1127 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_PRGRM_CNTR_START_HI,
1128 upper_32_bits(ucode_addr));
1129
1130 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1131 }
1132 mutex_unlock(&adev->srbm_mutex);
1133 }
1134
1135 /* This function is for backdoor MES firmware */
mes_v12_1_load_microcode(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,bool prime_icache,int xcc_id)1136 static int mes_v12_1_load_microcode(struct amdgpu_device *adev,
1137 enum amdgpu_mes_pipe pipe,
1138 bool prime_icache, int xcc_id)
1139 {
1140 int r, inst = MES_PIPE_INST(xcc_id, pipe);
1141 uint32_t data;
1142
1143 mes_v12_1_enable(adev, false, xcc_id);
1144
1145 if (!adev->mes.fw[pipe])
1146 return -EINVAL;
1147
1148 r = mes_v12_1_allocate_ucode_buffer(adev, pipe, xcc_id);
1149 if (r)
1150 return r;
1151
1152 r = mes_v12_1_allocate_ucode_data_buffer(adev, pipe, xcc_id);
1153 if (r) {
1154 mes_v12_1_free_ucode_buffers(adev, pipe, xcc_id);
1155 return r;
1156 }
1157
1158 mutex_lock(&adev->srbm_mutex);
1159 /* me=3, pipe=0, queue=0 */
1160 soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
1161
1162 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_CNTL, 0);
1163
1164 /* set ucode fimrware address */
1165 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_LO,
1166 lower_32_bits(adev->mes.ucode_fw_gpu_addr[inst]));
1167 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_HI,
1168 upper_32_bits(adev->mes.ucode_fw_gpu_addr[inst]));
1169
1170 /* set ucode instruction cache boundary to 2M-1 */
1171 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MIBOUND_LO, 0x1FFFFF);
1172
1173 /* set ucode data firmware address */
1174 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBASE_LO,
1175 lower_32_bits(adev->mes.data_fw_gpu_addr[inst]));
1176 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBASE_HI,
1177 upper_32_bits(adev->mes.data_fw_gpu_addr[inst]));
1178
1179 /* Set data cache boundary CP_MES_MDBOUND_LO */
1180 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBOUND_LO, 0x7FFFF);
1181
1182 if (prime_icache) {
1183 /* invalidate ICACHE */
1184 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL);
1185 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
1186 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1187 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL, data);
1188
1189 /* prime the ICACHE. */
1190 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL);
1191 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
1192 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL, data);
1193 }
1194
1195 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1196 mutex_unlock(&adev->srbm_mutex);
1197
1198 return 0;
1199 }
1200
mes_v12_1_allocate_eop_buf(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1201 static int mes_v12_1_allocate_eop_buf(struct amdgpu_device *adev,
1202 enum amdgpu_mes_pipe pipe,
1203 int xcc_id)
1204 {
1205 int r, inst = MES_PIPE_INST(xcc_id, pipe);
1206 u32 *eop;
1207
1208 r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1209 AMDGPU_GEM_DOMAIN_GTT,
1210 &adev->mes.eop_gpu_obj[inst],
1211 &adev->mes.eop_gpu_addr[inst],
1212 (void **)&eop);
1213 if (r) {
1214 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1215 return r;
1216 }
1217
1218 memset(eop, 0,
1219 adev->mes.eop_gpu_obj[inst]->tbo.base.size);
1220
1221 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[inst]);
1222 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[inst]);
1223
1224 return 0;
1225 }
1226
mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1227 static int mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device *adev,
1228 enum amdgpu_mes_pipe pipe,
1229 int xcc_id)
1230 {
1231 int r, inst = MES_PIPE_INST(xcc_id, pipe);
1232
1233 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
1234 AMDGPU_GEM_DOMAIN_VRAM,
1235 &adev->mes.shared_cmd_buf_obj[inst],
1236 &adev->mes.shared_cmd_buf_gpu_addr[inst],
1237 NULL);
1238 if (r) {
1239 dev_err(adev->dev,
1240 "(%d) failed to create shared cmd buf bo\n", r);
1241 return r;
1242 }
1243
1244 return 0;
1245 }
1246
mes_v12_1_mqd_init(struct amdgpu_ring * ring)1247 static int mes_v12_1_mqd_init(struct amdgpu_ring *ring)
1248 {
1249 struct v12_1_mes_mqd *mqd = ring->mqd_ptr;
1250 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1251 uint32_t tmp;
1252
1253 mqd->header = 0xC0310800;
1254 mqd->compute_pipelinestat_enable = 0x00000001;
1255 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1256 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1257 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1258 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1259 mqd->compute_misc_reserved = 0x00000007;
1260
1261 eop_base_addr = ring->eop_gpu_addr >> 8;
1262
1263 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1264 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1265 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1266 (order_base_2(MES_EOP_SIZE / 4) - 1));
1267
1268 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1269 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1270 mqd->cp_hqd_eop_control = tmp;
1271
1272 /* disable the queue if it's active */
1273 ring->wptr = 0;
1274 mqd->cp_hqd_pq_rptr = 0;
1275 mqd->cp_hqd_pq_wptr_lo = 0;
1276 mqd->cp_hqd_pq_wptr_hi = 0;
1277
1278 /* set the pointer to the MQD */
1279 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1280 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1281
1282 /* set MQD vmid to 0 */
1283 tmp = regCP_MQD_CONTROL_DEFAULT;
1284 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1285 mqd->cp_mqd_control = tmp;
1286
1287 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1288 hqd_gpu_addr = ring->gpu_addr >> 8;
1289 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1290 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1291
1292 /* set the wb address whether it's enabled or not */
1293 wb_gpu_addr = ring->rptr_gpu_addr;
1294 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1295 mqd->cp_hqd_pq_rptr_report_addr_hi =
1296 upper_32_bits(wb_gpu_addr) & 0xffff;
1297
1298 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1299 wb_gpu_addr = ring->wptr_gpu_addr;
1300 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1301 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1302
1303 /* set up the HQD, this is similar to CP_RB0_CNTL */
1304 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1305 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1306 (order_base_2(ring->ring_size / 4) - 1));
1307 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1308 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1309 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1310 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1311 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1312 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1313 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1314 mqd->cp_hqd_pq_control = tmp;
1315
1316 /* enable doorbell */
1317 tmp = 0;
1318 if (ring->use_doorbell) {
1319 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1320 DOORBELL_OFFSET, ring->doorbell_index);
1321 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1322 DOORBELL_EN, 1);
1323 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1324 DOORBELL_SOURCE, 0);
1325 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1326 DOORBELL_HIT, 0);
1327 } else {
1328 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1329 DOORBELL_EN, 0);
1330 }
1331 mqd->cp_hqd_pq_doorbell_control = tmp;
1332
1333 mqd->cp_hqd_vmid = 0;
1334 /* activate the queue */
1335 mqd->cp_hqd_active = 1;
1336
1337 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1338 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1339 PRELOAD_SIZE, 0x63);
1340 mqd->cp_hqd_persistent_state = tmp;
1341
1342 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT;
1343 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1344 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1345
1346 /*
1347 * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
1348 * doorbell handling. This is a reserved CP internal register can
1349 * not be accesss by others
1350 */
1351 mqd->cp_hqd_gfx_control = BIT(15);
1352
1353 return 0;
1354 }
1355
mes_v12_1_queue_init_register(struct amdgpu_ring * ring,int xcc_id)1356 static void mes_v12_1_queue_init_register(struct amdgpu_ring *ring,
1357 int xcc_id)
1358 {
1359 struct v12_1_mes_mqd *mqd = ring->mqd_ptr;
1360 struct amdgpu_device *adev = ring->adev;
1361 uint32_t data = 0;
1362
1363 mutex_lock(&adev->srbm_mutex);
1364 soc_v1_0_grbm_select(adev, 3, ring->pipe, 0, 0, GET_INST(GC, xcc_id));
1365
1366 /* set CP_HQD_VMID.VMID = 0. */
1367 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID);
1368 data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1369 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, data);
1370
1371 /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1372 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
1373 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1374 DOORBELL_EN, 0);
1375 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, data);
1376
1377 /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1378 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1379 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1380
1381 /* set CP_MQD_CONTROL.VMID=0 */
1382 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL);
1383 data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1384 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL, 0);
1385
1386 /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1387 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1388 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1389
1390 /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1391 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
1392 mqd->cp_hqd_pq_rptr_report_addr_lo);
1393 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1394 mqd->cp_hqd_pq_rptr_report_addr_hi);
1395
1396 /* set CP_HQD_PQ_CONTROL */
1397 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1398
1399 /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1400 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
1401 mqd->cp_hqd_pq_wptr_poll_addr_lo);
1402 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1403 mqd->cp_hqd_pq_wptr_poll_addr_hi);
1404
1405 /* set CP_HQD_PQ_DOORBELL_CONTROL */
1406 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
1407 mqd->cp_hqd_pq_doorbell_control);
1408
1409 /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1410 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1411
1412 /* set CP_HQD_ACTIVE.ACTIVE=1 */
1413 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1414
1415 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1416 mutex_unlock(&adev->srbm_mutex);
1417 }
1418
mes_v12_1_kiq_enable_queue(struct amdgpu_device * adev,int xcc_id)1419 static int mes_v12_1_kiq_enable_queue(struct amdgpu_device *adev, int xcc_id)
1420 {
1421 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1422 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[xcc_id].ring;
1423 int r, inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
1424
1425 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1426 return -EINVAL;
1427
1428 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1429 if (r) {
1430 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1431 return r;
1432 }
1433
1434 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[inst]);
1435
1436 r = amdgpu_ring_test_ring(kiq_ring);
1437 if (r) {
1438 DRM_ERROR("kfq enable failed\n");
1439 kiq_ring->sched.ready = false;
1440 }
1441 return r;
1442 }
1443
mes_v12_1_queue_init(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1444 static int mes_v12_1_queue_init(struct amdgpu_device *adev,
1445 enum amdgpu_mes_pipe pipe,
1446 int xcc_id)
1447 {
1448 struct amdgpu_ring *ring;
1449 int r;
1450
1451 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1452 ring = &adev->gfx.kiq[xcc_id].ring;
1453 else
1454 ring = &adev->mes.ring[MES_PIPE_INST(xcc_id, pipe)];
1455
1456 if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
1457 (amdgpu_in_reset(adev) || adev->in_suspend)) {
1458 *(ring->wptr_cpu_addr) = 0;
1459 *(ring->rptr_cpu_addr) = 0;
1460 amdgpu_ring_clear_ring(ring);
1461 }
1462
1463 r = mes_v12_1_mqd_init(ring);
1464 if (r)
1465 return r;
1466
1467 if (pipe == AMDGPU_MES_SCHED_PIPE) {
1468 if (adev->enable_uni_mes)
1469 r = amdgpu_mes_map_legacy_queue(adev, ring, xcc_id);
1470 else
1471 r = mes_v12_1_kiq_enable_queue(adev, xcc_id);
1472 if (r)
1473 return r;
1474 } else {
1475 mes_v12_1_queue_init_register(ring, xcc_id);
1476 }
1477
1478 return 0;
1479 }
1480
mes_v12_1_ring_init(struct amdgpu_device * adev,int xcc_id,int pipe)1481 static int mes_v12_1_ring_init(struct amdgpu_device *adev,
1482 int xcc_id, int pipe)
1483 {
1484 struct amdgpu_ring *ring;
1485 int inst = MES_PIPE_INST(xcc_id, pipe);
1486
1487 ring = &adev->mes.ring[inst];
1488
1489 ring->funcs = &mes_v12_1_ring_funcs;
1490
1491 ring->me = 3;
1492 ring->pipe = pipe;
1493 ring->queue = 0;
1494 ring->xcc_id = xcc_id;
1495 ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
1496
1497 ring->ring_obj = NULL;
1498 ring->use_doorbell = true;
1499 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[inst];
1500 ring->no_scheduler = true;
1501 snprintf(ring->name, sizeof(ring->name), "mes_%hhu.%hhu.%hhu.%hhu",
1502 (unsigned char)xcc_id, (unsigned char)ring->me,
1503 (unsigned char)ring->pipe, (unsigned char)ring->queue);
1504
1505 if (pipe == AMDGPU_MES_SCHED_PIPE)
1506 ring->doorbell_index =
1507 (adev->doorbell_index.mes_ring0 +
1508 xcc_id * adev->doorbell_index.xcc_doorbell_range)
1509 << 1;
1510 else
1511 ring->doorbell_index =
1512 (adev->doorbell_index.mes_ring1 +
1513 xcc_id * adev->doorbell_index.xcc_doorbell_range)
1514 << 1;
1515
1516 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1517 AMDGPU_RING_PRIO_DEFAULT, NULL);
1518 }
1519
mes_v12_1_kiq_ring_init(struct amdgpu_device * adev,int xcc_id)1520 static int mes_v12_1_kiq_ring_init(struct amdgpu_device *adev, int xcc_id)
1521 {
1522 struct amdgpu_ring *ring;
1523 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_KIQ_PIPE);
1524
1525 spin_lock_init(&adev->gfx.kiq[xcc_id].ring_lock);
1526
1527 ring = &adev->gfx.kiq[xcc_id].ring;
1528
1529 ring->me = 3;
1530 ring->pipe = 1;
1531 ring->queue = 0;
1532 ring->xcc_id = xcc_id;
1533 ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
1534
1535 ring->adev = NULL;
1536 ring->ring_obj = NULL;
1537 ring->use_doorbell = true;
1538 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[inst];
1539 ring->no_scheduler = true;
1540 ring->doorbell_index =
1541 (adev->doorbell_index.mes_ring1 +
1542 xcc_id * adev->doorbell_index.xcc_doorbell_range)
1543 << 1;
1544
1545 snprintf(ring->name, sizeof(ring->name), "mes_kiq_%hhu.%hhu.%hhu.%hhu",
1546 (unsigned char)xcc_id, (unsigned char)ring->me,
1547 (unsigned char)ring->pipe, (unsigned char)ring->queue);
1548
1549 return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1550 AMDGPU_RING_PRIO_DEFAULT, NULL);
1551 }
1552
mes_v12_1_mqd_sw_init(struct amdgpu_device * adev,enum amdgpu_mes_pipe pipe,int xcc_id)1553 static int mes_v12_1_mqd_sw_init(struct amdgpu_device *adev,
1554 enum amdgpu_mes_pipe pipe,
1555 int xcc_id)
1556 {
1557 int r, mqd_size = sizeof(struct v12_1_mes_mqd);
1558 struct amdgpu_ring *ring;
1559 int inst = MES_PIPE_INST(xcc_id, pipe);
1560
1561 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1562 ring = &adev->gfx.kiq[xcc_id].ring;
1563 else
1564 ring = &adev->mes.ring[inst];
1565
1566 if (ring->mqd_obj)
1567 return 0;
1568
1569 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1570 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1571 &ring->mqd_gpu_addr, &ring->mqd_ptr);
1572 if (r) {
1573 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1574 return r;
1575 }
1576
1577 memset(ring->mqd_ptr, 0, mqd_size);
1578
1579 /* prepare MQD backup */
1580 adev->mes.mqd_backup[inst] = kmalloc(mqd_size, GFP_KERNEL);
1581 if (!adev->mes.mqd_backup[inst])
1582 dev_warn(adev->dev,
1583 "no memory to create MQD backup for ring %s\n",
1584 ring->name);
1585
1586 return 0;
1587 }
1588
mes_v12_1_sw_init(struct amdgpu_ip_block * ip_block)1589 static int mes_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
1590 {
1591 struct amdgpu_device *adev = ip_block->adev;
1592 int pipe, r, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1593
1594 adev->mes.funcs = &mes_v12_1_funcs;
1595 adev->mes.kiq_hw_init = &mes_v12_1_kiq_hw_init;
1596 adev->mes.kiq_hw_fini = &mes_v12_1_kiq_hw_fini;
1597 adev->mes.enable_legacy_queue_map = true;
1598
1599 adev->mes.event_log_size =
1600 adev->enable_uni_mes ? (AMDGPU_MAX_MES_PIPES * AMDGPU_MES_LOG_BUFFER_SIZE * num_xcc) : AMDGPU_MES_LOG_BUFFER_SIZE;
1601
1602 r = amdgpu_mes_init(adev);
1603 if (r)
1604 return r;
1605
1606 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1607 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1608 r = mes_v12_1_allocate_eop_buf(adev, pipe, xcc_id);
1609 if (r)
1610 return r;
1611
1612 r = mes_v12_1_mqd_sw_init(adev, pipe, xcc_id);
1613 if (r)
1614 return r;
1615
1616 if (!adev->enable_uni_mes && pipe ==
1617 AMDGPU_MES_KIQ_PIPE)
1618 r = mes_v12_1_kiq_ring_init(adev, xcc_id);
1619 else
1620 r = mes_v12_1_ring_init(adev, xcc_id, pipe);
1621 if (r)
1622 return r;
1623
1624 if (adev->enable_uni_mes && num_xcc > 1) {
1625 r = mes_v12_1_allocate_shared_cmd_buf(adev,
1626 pipe, xcc_id);
1627 if (r)
1628 return r;
1629 }
1630 }
1631 }
1632
1633 return 0;
1634 }
1635
mes_v12_1_sw_fini(struct amdgpu_ip_block * ip_block)1636 static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
1637 {
1638 struct amdgpu_device *adev = ip_block->adev;
1639 int pipe, inst, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1640
1641 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1642 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1643 inst = MES_PIPE_INST(xcc_id, pipe);
1644
1645 amdgpu_bo_free_kernel(&adev->mes.shared_cmd_buf_obj[inst],
1646 &adev->mes.shared_cmd_buf_gpu_addr[inst],
1647 NULL);
1648
1649 kfree(adev->mes.mqd_backup[inst]);
1650
1651 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[inst],
1652 &adev->mes.eop_gpu_addr[inst],
1653 NULL);
1654
1655 if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
1656 amdgpu_bo_free_kernel(&adev->mes.ring[inst].mqd_obj,
1657 &adev->mes.ring[inst].mqd_gpu_addr,
1658 &adev->mes.ring[inst].mqd_ptr);
1659 amdgpu_ring_fini(&adev->mes.ring[inst]);
1660 }
1661 }
1662 }
1663
1664 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++)
1665 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1666
1667 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1668 if (!adev->enable_uni_mes) {
1669 amdgpu_bo_free_kernel(&adev->gfx.kiq[xcc_id].ring.mqd_obj,
1670 &adev->gfx.kiq[xcc_id].ring.mqd_gpu_addr,
1671 &adev->gfx.kiq[xcc_id].ring.mqd_ptr);
1672 amdgpu_ring_fini(&adev->gfx.kiq[xcc_id].ring);
1673 }
1674
1675 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1676 mes_v12_1_free_ucode_buffers(adev,
1677 AMDGPU_MES_KIQ_PIPE, xcc_id);
1678 mes_v12_1_free_ucode_buffers(adev,
1679 AMDGPU_MES_SCHED_PIPE, xcc_id);
1680 }
1681 }
1682
1683 amdgpu_mes_fini(adev);
1684 return 0;
1685 }
1686
mes_v12_1_kiq_dequeue_sched(struct amdgpu_device * adev,int xcc_id)1687 static void mes_v12_1_kiq_dequeue_sched(struct amdgpu_device *adev,
1688 int xcc_id)
1689 {
1690 uint32_t data;
1691 int i;
1692
1693 mutex_lock(&adev->srbm_mutex);
1694 soc_v1_0_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0,
1695 GET_INST(GC, xcc_id));
1696
1697 /* disable the queue if it's active */
1698 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
1699 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
1700 for (i = 0; i < adev->usec_timeout; i++) {
1701 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
1702 break;
1703 udelay(1);
1704 }
1705 }
1706 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
1707 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1708 DOORBELL_EN, 0);
1709 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1710 DOORBELL_HIT, 1);
1711 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, data);
1712
1713 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1714
1715 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 0);
1716 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 0);
1717 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 0);
1718
1719 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1720 mutex_unlock(&adev->srbm_mutex);
1721
1722 adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = false;
1723 }
1724
mes_v12_1_kiq_setting(struct amdgpu_ring * ring,int xcc_id)1725 static void mes_v12_1_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
1726 {
1727 uint32_t tmp;
1728 struct amdgpu_device *adev = ring->adev;
1729
1730 /* tell RLC which is KIQ queue */
1731 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
1732 tmp &= 0xffffff00;
1733 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1734 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
1735 tmp |= 0x80;
1736 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
1737 }
1738
mes_v12_1_kiq_hw_init(struct amdgpu_device * adev,uint32_t xcc_id)1739 static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
1740 {
1741 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_KIQ_PIPE);
1742 int r = 0;
1743 struct amdgpu_ip_block *ip_block;
1744
1745 if (adev->enable_uni_mes)
1746 mes_v12_1_kiq_setting(&adev->mes.ring[inst], xcc_id);
1747 else
1748 mes_v12_1_kiq_setting(&adev->gfx.kiq[xcc_id].ring, xcc_id);
1749
1750 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1751
1752 r = mes_v12_1_load_microcode(adev, AMDGPU_MES_SCHED_PIPE,
1753 false, xcc_id);
1754 if (r) {
1755 DRM_ERROR("failed to load MES fw, r=%d\n", r);
1756 return r;
1757 }
1758
1759 r = mes_v12_1_load_microcode(adev, AMDGPU_MES_KIQ_PIPE,
1760 true, xcc_id);
1761 if (r) {
1762 DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1763 return r;
1764 }
1765
1766 mes_v12_1_set_ucode_start_addr(adev, xcc_id);
1767
1768 } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1769 mes_v12_1_set_ucode_start_addr(adev, xcc_id);
1770
1771 mes_v12_1_enable(adev, true, xcc_id);
1772
1773 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
1774 if (unlikely(!ip_block)) {
1775 dev_err(adev->dev, "Failed to get MES handle\n");
1776 return -EINVAL;
1777 }
1778
1779 r = mes_v12_1_queue_init(adev, AMDGPU_MES_KIQ_PIPE, xcc_id);
1780 if (r)
1781 goto failure;
1782
1783 if (adev->enable_uni_mes) {
1784 r = mes_v12_1_setup_coop_mode(adev, xcc_id);
1785 if (r)
1786 goto failure;
1787
1788 r = mes_v12_1_set_hw_resources(&adev->mes,
1789 AMDGPU_MES_KIQ_PIPE, xcc_id);
1790 if (r)
1791 goto failure;
1792
1793 mes_v12_1_set_hw_resources_1(&adev->mes,
1794 AMDGPU_MES_KIQ_PIPE, xcc_id);
1795 }
1796
1797 if (adev->mes.enable_legacy_queue_map) {
1798 r = mes_v12_1_xcc_hw_init(ip_block, xcc_id);
1799 if (r)
1800 goto failure;
1801 }
1802
1803 return r;
1804
1805 failure:
1806 mes_v12_1_hw_fini(ip_block);
1807 return r;
1808 }
1809
mes_v12_1_kiq_hw_fini(struct amdgpu_device * adev,uint32_t xcc_id)1810 static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
1811 {
1812 int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
1813
1814 if (adev->mes.ring[inst].sched.ready) {
1815 if (adev->enable_uni_mes)
1816 amdgpu_mes_unmap_legacy_queue(adev,
1817 &adev->mes.ring[inst],
1818 RESET_QUEUES, 0, 0, xcc_id);
1819 else
1820 mes_v12_1_kiq_dequeue_sched(adev, xcc_id);
1821
1822 adev->mes.ring[inst].sched.ready = false;
1823 }
1824
1825 mes_v12_1_enable(adev, false, xcc_id);
1826
1827 return 0;
1828 }
1829
mes_v12_1_setup_coop_mode(struct amdgpu_device * adev,int xcc_id)1830 static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id)
1831 {
1832 u32 num_xcc_per_xcp, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1833 int r = 0;
1834
1835 if (num_xcc == 1)
1836 return r;
1837
1838 if (adev->gfx.funcs &&
1839 adev->gfx.funcs->get_xccs_per_xcp)
1840 num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev);
1841 else
1842 return -EINVAL;
1843
1844 switch (adev->xcp_mgr->mode) {
1845 case AMDGPU_SPX_PARTITION_MODE:
1846 adev->mes.enable_coop_mode = 1;
1847 adev->mes.master_xcc_ids[xcc_id] = 0;
1848 break;
1849 case AMDGPU_DPX_PARTITION_MODE:
1850 adev->mes.enable_coop_mode = 1;
1851 adev->mes.master_xcc_ids[xcc_id] =
1852 (xcc_id/num_xcc_per_xcp) * (num_xcc / 2);
1853 break;
1854 case AMDGPU_QPX_PARTITION_MODE:
1855 adev->mes.enable_coop_mode = 1;
1856 adev->mes.master_xcc_ids[xcc_id] =
1857 (xcc_id/num_xcc_per_xcp) * (num_xcc / 4);
1858 break;
1859 case AMDGPU_CPX_PARTITION_MODE:
1860 adev->mes.enable_coop_mode = 0;
1861 break;
1862 default:
1863 r = -EINVAL;
1864 break;
1865 }
1866 return r;
1867 }
1868
mes_v12_1_xcc_hw_init(struct amdgpu_ip_block * ip_block,int xcc_id)1869 static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
1870 {
1871 int r;
1872 struct amdgpu_device *adev = ip_block->adev;
1873
1874 if (adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready)
1875 goto out;
1876
1877 if (!adev->enable_mes_kiq) {
1878 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1879 r = mes_v12_1_load_microcode(adev,
1880 AMDGPU_MES_SCHED_PIPE, true, xcc_id);
1881 if (r) {
1882 DRM_ERROR("failed to MES fw, r=%d\n", r);
1883 return r;
1884 }
1885
1886 mes_v12_1_set_ucode_start_addr(adev, xcc_id);
1887
1888 } else if (adev->firmware.load_type ==
1889 AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1890
1891 mes_v12_1_set_ucode_start_addr(adev, xcc_id);
1892 }
1893
1894 mes_v12_1_enable(adev, true, xcc_id);
1895 }
1896
1897 /* Enable the MES to handle doorbell ring on unmapped queue */
1898 mes_v12_1_enable_unmapped_doorbell_handling(&adev->mes, true, xcc_id);
1899
1900 r = mes_v12_1_queue_init(adev, AMDGPU_MES_SCHED_PIPE, xcc_id);
1901 if (r)
1902 goto failure;
1903
1904 r = mes_v12_1_set_hw_resources(&adev->mes,
1905 AMDGPU_MES_SCHED_PIPE, xcc_id);
1906 if (r)
1907 goto failure;
1908
1909 if (adev->enable_uni_mes) {
1910 mes_v12_1_set_hw_resources_1(&adev->mes,
1911 AMDGPU_MES_SCHED_PIPE, xcc_id);
1912 }
1913 mes_v12_1_init_aggregated_doorbell(&adev->mes, xcc_id);
1914
1915 r = mes_v12_1_query_sched_status(&adev->mes,
1916 AMDGPU_MES_SCHED_PIPE, xcc_id);
1917 if (r) {
1918 DRM_ERROR("MES is busy\n");
1919 goto failure;
1920 }
1921
1922 amdgpu_mes_validate_fw_version(adev);
1923 out:
1924 /*
1925 * Disable KIQ ring usage from the driver once MES is enabled.
1926 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1927 * with MES enabled.
1928 */
1929 adev->gfx.kiq[xcc_id].ring.sched.ready = false;
1930 adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = true;
1931
1932 return 0;
1933
1934 failure:
1935 mes_v12_1_hw_fini(ip_block);
1936 return r;
1937 }
1938
mes_v12_1_hw_init(struct amdgpu_ip_block * ip_block)1939 static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
1940 {
1941 struct amdgpu_device *adev = ip_block->adev;
1942 int r, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1943
1944 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1945 r = mes_v12_1_xcc_hw_init(ip_block, xcc_id);
1946 if (r)
1947 return r;
1948 }
1949
1950 return 0;
1951 }
1952
mes_v12_1_hw_fini(struct amdgpu_ip_block * ip_block)1953 static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
1954 {
1955 return 0;
1956 }
1957
mes_v12_1_suspend(struct amdgpu_ip_block * ip_block)1958 static int mes_v12_1_suspend(struct amdgpu_ip_block *ip_block)
1959 {
1960 return mes_v12_1_hw_fini(ip_block);
1961 }
1962
mes_v12_1_resume(struct amdgpu_ip_block * ip_block)1963 static int mes_v12_1_resume(struct amdgpu_ip_block *ip_block)
1964 {
1965 return mes_v12_1_hw_init(ip_block);
1966 }
1967
mes_v12_1_early_init(struct amdgpu_ip_block * ip_block)1968 static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
1969 {
1970 struct amdgpu_device *adev = ip_block->adev;
1971 int pipe, r;
1972
1973 adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE;
1974 adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET;
1975
1976 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1977 r = amdgpu_mes_init_microcode(adev, pipe);
1978 if (r)
1979 return r;
1980 }
1981
1982 return 0;
1983 }
1984
mes_v12_1_late_init(struct amdgpu_ip_block * ip_block)1985 static int mes_v12_1_late_init(struct amdgpu_ip_block *ip_block)
1986 {
1987 struct amdgpu_device *adev = ip_block->adev;
1988 int xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1989
1990 /* TODO: remove it if issue fixed. */
1991 if (adev->mes.enable_coop_mode)
1992 return 0;
1993
1994 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1995 /* for COOP mode, only test master xcc. */
1996 if (adev->mes.enable_coop_mode &&
1997 adev->mes.master_xcc_ids[xcc_id] != xcc_id)
1998 continue;
1999
2000 mes_v12_1_self_test(adev, xcc_id);
2001 }
2002
2003 return 0;
2004 }
2005
2006 static const struct amd_ip_funcs mes_v12_1_ip_funcs = {
2007 .name = "mes_v12_1",
2008 .early_init = mes_v12_1_early_init,
2009 .late_init = mes_v12_1_late_init,
2010 .sw_init = mes_v12_1_sw_init,
2011 .sw_fini = mes_v12_1_sw_fini,
2012 .hw_init = mes_v12_1_hw_init,
2013 .hw_fini = mes_v12_1_hw_fini,
2014 .suspend = mes_v12_1_suspend,
2015 .resume = mes_v12_1_resume,
2016 };
2017
2018 const struct amdgpu_ip_block_version mes_v12_1_ip_block = {
2019 .type = AMD_IP_BLOCK_TYPE_MES,
2020 .major = 12,
2021 .minor = 1,
2022 .rev = 0,
2023 .funcs = &mes_v12_1_ip_funcs,
2024 };
2025
mes_v12_1_alloc_test_buf(struct amdgpu_device * adev,struct amdgpu_bo ** bo,uint64_t * addr,void ** ptr,int size)2026 static int mes_v12_1_alloc_test_buf(struct amdgpu_device *adev,
2027 struct amdgpu_bo **bo, uint64_t *addr,
2028 void **ptr, int size)
2029 {
2030 amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2031 bo, addr, ptr);
2032 if (!*bo) {
2033 dev_err(adev->dev, "failed to allocate test buffer bo\n");
2034 return -ENOMEM;
2035 }
2036 memset(*ptr, 0, size);
2037 return 0;
2038 }
2039
mes_v12_1_map_test_bo(struct amdgpu_device * adev,struct amdgpu_bo * bo,struct amdgpu_vm * vm,struct amdgpu_bo_va ** bo_va,u64 va,int size)2040 static int mes_v12_1_map_test_bo(struct amdgpu_device *adev,
2041 struct amdgpu_bo *bo, struct amdgpu_vm *vm,
2042 struct amdgpu_bo_va **bo_va, u64 va, int size)
2043 {
2044 struct amdgpu_sync sync;
2045 int r;
2046
2047 r = amdgpu_map_static_csa(adev, vm, bo, bo_va, va, size);
2048 if (r)
2049 return r;
2050
2051 amdgpu_sync_create(&sync);
2052
2053 r = amdgpu_vm_bo_update(adev, *bo_va, false);
2054 if (r) {
2055 dev_err(adev->dev, "failed to do vm_bo_update on meta data\n");
2056 goto error;
2057 }
2058 amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update, GFP_KERNEL);
2059
2060 r = amdgpu_vm_update_pdes(adev, vm, false);
2061 if (r) {
2062 dev_err(adev->dev, "failed to update pdes on meta data\n");
2063 goto error;
2064 }
2065 amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);
2066 amdgpu_sync_wait(&sync, false);
2067
2068 error:
2069 amdgpu_sync_free(&sync);
2070 return r;
2071 }
2072
mes_v12_1_test_ring(struct amdgpu_device * adev,int xcc_id,u32 * queue_ptr,u64 fence_gpu_addr,void * fence_cpu_ptr,void * wptr_cpu_addr,u64 doorbell_idx,int queue_type)2073 static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id,
2074 u32 *queue_ptr, u64 fence_gpu_addr,
2075 void *fence_cpu_ptr, void *wptr_cpu_addr,
2076 u64 doorbell_idx, int queue_type)
2077 {
2078 volatile uint32_t *cpu_ptr = fence_cpu_ptr;
2079 int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2080 int sdma_ring_align = 0x10, compute_ring_align = 0x100;
2081 uint32_t tmp, xcc_offset;
2082 int r = 0, i, j, wptr = 0;
2083
2084 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
2085 if (!adev->mes.enable_coop_mode) {
2086 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2087 regSCRATCH_REG0, 0xCAFEDEAD);
2088 } else {
2089 for (i = 0; i < num_xcc; i++) {
2090 if (adev->mes.master_xcc_ids[i] == xcc_id)
2091 WREG32_SOC15(GC, GET_INST(GC, i),
2092 regSCRATCH_REG0, 0xCAFEDEAD);
2093 }
2094 }
2095
2096 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
2097 queue_ptr[wptr++] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2098 queue_ptr[wptr++] = xcc_offset - PACKET3_SET_UCONFIG_REG_START;
2099 queue_ptr[wptr++] = 0xDEADBEEF;
2100
2101 for (i = wptr; i < compute_ring_align; i++)
2102 queue_ptr[wptr++] = PACKET3(PACKET3_NOP, 0x3FFF);
2103
2104 } else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
2105 *cpu_ptr = 0xCAFEDEAD;
2106
2107 queue_ptr[wptr++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
2108 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
2109 queue_ptr[wptr++] = lower_32_bits(fence_gpu_addr);
2110 queue_ptr[wptr++] = upper_32_bits(fence_gpu_addr);
2111 queue_ptr[wptr++] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
2112 queue_ptr[wptr++] = 0xDEADBEEF;
2113
2114 for (i = wptr; i < sdma_ring_align; i++)
2115 queue_ptr[wptr++] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
2116
2117 wptr <<= 2;
2118 }
2119
2120 atomic64_set((atomic64_t *)wptr_cpu_addr, wptr);
2121 WDOORBELL64(doorbell_idx, wptr);
2122
2123 for (i = 0; i < adev->usec_timeout; i++) {
2124 if (queue_type == AMDGPU_RING_TYPE_SDMA) {
2125 tmp = le32_to_cpu(*cpu_ptr);
2126 } else {
2127 if (!adev->mes.enable_coop_mode) {
2128 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2129 regSCRATCH_REG0);
2130 } else {
2131 for (j = 0; j < num_xcc; j++) {
2132 if (xcc_id != adev->mes.master_xcc_ids[j])
2133 continue;
2134
2135 tmp = RREG32_SOC15(GC, GET_INST(GC, j),
2136 regSCRATCH_REG0);
2137 if (tmp != 0xDEADBEEF)
2138 break;
2139 }
2140 }
2141 }
2142
2143 if (tmp == 0xDEADBEEF)
2144 break;
2145
2146 if (amdgpu_emu_mode == 1)
2147 msleep(1);
2148 else
2149 udelay(1);
2150 }
2151
2152 if (i >= adev->usec_timeout) {
2153 dev_err(adev->dev, "xcc%d: mes self test (%s) failed\n", xcc_id,
2154 queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
2155
2156 while (halt_if_hws_hang)
2157 schedule();
2158
2159 r = -ETIMEDOUT;
2160 } else {
2161 dev_info(adev->dev, "xcc%d: mes self test (%s) pass\n", xcc_id,
2162 queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
2163 }
2164
2165 return r;
2166 }
2167
2168 #define USER_CTX_SIZE (PAGE_SIZE * 2)
2169 #define USER_CTX_VA AMDGPU_VA_RESERVED_BOTTOM
2170 #define RING_OFFSET(addr) ((addr))
2171 #define EOP_OFFSET(addr) ((addr) + PAGE_SIZE)
2172 #define WPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64))
2173 #define RPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 2)
2174 #define FENCE_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 3)
2175
mes_v12_1_test_queue(struct amdgpu_device * adev,int xcc_id,int pasid,struct amdgpu_vm * vm,u64 meta_gpu_addr,u64 queue_gpu_addr,void * ctx_ptr,int queue_type)2176 static int mes_v12_1_test_queue(struct amdgpu_device *adev, int xcc_id,
2177 int pasid, struct amdgpu_vm *vm, u64 meta_gpu_addr,
2178 u64 queue_gpu_addr, void *ctx_ptr, int queue_type)
2179 {
2180 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
2181 struct amdgpu_mqd *mqd_mgr = &adev->mqds[queue_type];
2182 struct amdgpu_mqd_prop mqd_prop = {0};
2183 struct mes_add_queue_input add_queue = {0};
2184 struct mes_remove_queue_input remove_queue = {0};
2185 struct amdgpu_bo *mqd_bo = NULL;
2186 int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2187 int i, r, off, mqd_size, mqd_count = 1;
2188 void *mqd_ptr = NULL;
2189 u64 mqd_gpu_addr, doorbell_idx;
2190
2191 /* extra one page size padding for mes fw */
2192 mqd_size = mqd_mgr->mqd_size + PAGE_SIZE;
2193
2194 if (queue_type == AMDGPU_RING_TYPE_SDMA) {
2195 doorbell_idx = adev->mes.db_start_dw_offset + \
2196 adev->doorbell_index.sdma_engine[0];
2197 } else {
2198 doorbell_idx = adev->mes.db_start_dw_offset + \
2199 adev->doorbell_index.userqueue_start;
2200 }
2201
2202 if (adev->mes.enable_coop_mode &&
2203 queue_type == AMDGPU_RING_TYPE_COMPUTE) {
2204 for (i = 0, mqd_count = 0; i < num_xcc; i++) {
2205 if (adev->mes.master_xcc_ids[i] == xcc_id)
2206 mqd_count++;
2207 }
2208 mqd_size *= mqd_count;
2209 }
2210
2211 r = mes_v12_1_alloc_test_buf(adev, &mqd_bo, &mqd_gpu_addr,
2212 &mqd_ptr, mqd_size * mqd_count);
2213 if (r < 0)
2214 return r;
2215
2216 mqd_prop.mqd_gpu_addr = mqd_gpu_addr;
2217 mqd_prop.hqd_base_gpu_addr = RING_OFFSET(USER_CTX_VA);
2218 mqd_prop.eop_gpu_addr = EOP_OFFSET(USER_CTX_VA);
2219 mqd_prop.wptr_gpu_addr = WPTR_OFFSET(USER_CTX_VA);
2220 mqd_prop.rptr_gpu_addr = RPTR_OFFSET(USER_CTX_VA);
2221 mqd_prop.doorbell_index = doorbell_idx;
2222 mqd_prop.queue_size = PAGE_SIZE;
2223 mqd_prop.mqd_stride_size = mqd_size;
2224 mqd_prop.use_doorbell = true;
2225 mqd_prop.hqd_active = false;
2226
2227 mqd_mgr->init_mqd(adev, mqd_ptr, &mqd_prop);
2228 if (mqd_count > 1) {
2229 for (i = 1; i < mqd_count; i++) {
2230 off = mqd_size * i;
2231 mqd_prop.mqd_gpu_addr = mqd_gpu_addr + off;
2232 mqd_mgr->init_mqd(adev, (char *)mqd_ptr + off,
2233 &mqd_prop);
2234 }
2235 }
2236
2237 add_queue.xcc_id = xcc_id;
2238 add_queue.process_id = pasid;
2239 add_queue.page_table_base_addr = adev->vm_manager.vram_base_offset +
2240 amdgpu_bo_gpu_offset(vm->root.bo) - adev->gmc.vram_start;
2241 add_queue.process_va_start = 0;
2242 add_queue.process_va_end = adev->vm_manager.max_pfn - 1;
2243 add_queue.process_context_addr = meta_gpu_addr;
2244 add_queue.gang_context_addr = meta_gpu_addr + AMDGPU_MES_PROC_CTX_SIZE;
2245 add_queue.doorbell_offset = doorbell_idx;
2246 add_queue.mqd_addr = mqd_gpu_addr;
2247 add_queue.wptr_addr = mqd_prop.wptr_gpu_addr;
2248 add_queue.wptr_mc_addr = WPTR_OFFSET(queue_gpu_addr);
2249 add_queue.queue_type = queue_type;
2250 add_queue.vm_cntx_cntl = hub->vm_cntx_cntl;
2251
2252 r = mes_v12_1_add_hw_queue(&adev->mes, &add_queue);
2253 if (r)
2254 goto error;
2255
2256 mes_v12_1_test_ring(adev, xcc_id, (u32 *)RING_OFFSET((char *)ctx_ptr),
2257 FENCE_OFFSET(USER_CTX_VA),
2258 FENCE_OFFSET((char *)ctx_ptr),
2259 WPTR_OFFSET((char *)ctx_ptr),
2260 doorbell_idx, queue_type);
2261
2262 remove_queue.xcc_id = xcc_id;
2263 remove_queue.doorbell_offset = doorbell_idx;
2264 remove_queue.gang_context_addr = add_queue.gang_context_addr;
2265 r = mes_v12_1_remove_hw_queue(&adev->mes, &remove_queue);
2266
2267 error:
2268 amdgpu_bo_free_kernel(&mqd_bo, &mqd_gpu_addr, &mqd_ptr);
2269 return r;
2270 }
2271
mes_v12_1_self_test(struct amdgpu_device * adev,int xcc_id)2272 static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id)
2273 {
2274 int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE,
2275 /* AMDGPU_RING_TYPE_SDMA */ };
2276 struct amdgpu_bo_va *bo_va = NULL;
2277 struct amdgpu_vm *vm = NULL;
2278 struct amdgpu_bo *meta_bo = NULL, *ctx_bo = NULL;
2279 void *meta_ptr = NULL, *ctx_ptr = NULL;
2280 u64 meta_gpu_addr, ctx_gpu_addr;
2281 int size, i, r, pasid;
2282
2283 pasid = amdgpu_pasid_alloc(16);
2284 if (pasid < 0)
2285 pasid = 0;
2286
2287 size = AMDGPU_MES_PROC_CTX_SIZE + AMDGPU_MES_GANG_CTX_SIZE;
2288 r = mes_v12_1_alloc_test_buf(adev, &meta_bo, &meta_gpu_addr,
2289 &meta_ptr, size);
2290 if (r < 0)
2291 goto err2;
2292
2293 r = mes_v12_1_alloc_test_buf(adev, &ctx_bo, &ctx_gpu_addr,
2294 &ctx_ptr, USER_CTX_SIZE);
2295 if (r < 0)
2296 goto err2;
2297
2298 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2299 if (!vm) {
2300 r = -ENOMEM;
2301 goto err2;
2302 }
2303
2304 r = amdgpu_vm_init(adev, vm, -1, pasid);
2305 if (r)
2306 goto err1;
2307
2308 r = mes_v12_1_map_test_bo(adev, ctx_bo, vm, &bo_va,
2309 USER_CTX_VA, USER_CTX_SIZE);
2310 if (r)
2311 goto err0;
2312
2313 for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
2314 memset(ctx_ptr, 0, USER_CTX_SIZE);
2315
2316 r = mes_v12_1_test_queue(adev, xcc_id, pasid, vm, meta_gpu_addr,
2317 ctx_gpu_addr, ctx_ptr, queue_types[i]);
2318 if (r)
2319 break;
2320 }
2321
2322 amdgpu_unmap_static_csa(adev, vm, ctx_bo, bo_va, USER_CTX_VA);
2323 err0:
2324 amdgpu_vm_fini(adev, vm);
2325 err1:
2326 kfree(vm);
2327 err2:
2328 amdgpu_bo_free_kernel(&meta_bo, &meta_gpu_addr, &meta_ptr);
2329 amdgpu_bo_free_kernel(&ctx_bo, &ctx_gpu_addr, &ctx_ptr);
2330 amdgpu_pasid_free(pasid);
2331 return r;
2332 }
2333
2334